诡异的防火墙故障,能PING通,但访问ORACLE不稳定

上周四我们的IDC核心数据区到DMZ区的防火墙经历了十分诡异的故障。

虽然我从一开始就断定非ORACLE问题,因为连接核心数据区的多个ORACLE数据库的应用都无一例外的出现了相同的问题,如果三个以上ORACLE能同时出想这么严重的问题,那么我想ORACLE也该“停业整顿”了。但是应用的同事看得出来还是非常怀疑ORACLE出现了问题,再加上链路可以PING通,最后通过替换法确定是防火墙问题,总算告一段落。处理从当天下午4:00一直持续到第二天上午8:00,可谓惊心动魄。

故障现象:应用报如下错误:

SQL state [null]; error code [17410]; No more data to read from socket; nested exception is java.sql.SQLException: No more data to read from socket

java.sql.SQLException: No more data to read from socket

at oracle.jdbc.driver.DatabaseError.throwSqlException(DatabaseError.java:112)

at oracle.jdbc.driver.DatabaseError.throwSqlException(DatabaseError.java:146)

at oracle.jdbc.driver.DatabaseError.throwSqlException(DatabaseError.java:208)

at oracle.jdbc.driver.T4CMAREngine.unmarshalUB1(T4CMAREngine.java:1118)

at oracle.jdbc.driver.T4CMAREngine.unmarshalSB1(T4CMAREngine.java:1070)

at oracle.jdbc.driver.T4C8Oall.receive(T4C8Oall.java:478)

at oracle.jdbc.driver.T4CPreparedStatement.doOall8(T4CPreparedStatement.java:216)

at oracle.jdbc.driver.T4CPreparedStatement.executeForDescribe(T4CPreparedStatement.java:799)

at oracle.jdbc.driver.OracleStatement.executeMaybeDescribe(OracleStatement.java:1037)

at oracle.jdbc.driver.T4CPreparedStatement.executeMaybeDescribe(T4CPreparedStatement.java:839)

at oracle.jdbc.driver.OracleStatement.doExecuteWithTimeout(OracleStatement.java:1132)

at oracle.jdbc.driver.OraclePreparedStatement.executeInternal(OraclePreparedStatement.java:3316)

at oracle.jdbc.driver.OraclePreparedStatement.executeQuery(OraclePreparedStatement.java:3361)

at weblogic.jdbc.wrapper.PreparedStatement.executeQuery(PreparedStatement.java:100)

at org.hibernate.jdbc.AbstractBatcher.getResultSet(AbstractBatcher.java:120)

at org.hibernate.loader.Loader.getResultSet(Loader.java:1272)

at org.hibernate.loader.Loader.doQuery(Loader.java:391)

at org.hibernate.loader.Loader.doQueryAndInitializeNonLazyCollections(Loader.java:218)

at org.hibernate.loader.Loader.doList(Loader.java:1593)

at org.hibernate.loader.Loader.list(Loader.java:1577)

at org.hibernate.loader.hql.QueryLoader.list(QueryLoader.java:395)

at org.hibernate.hql.ast.QueryTranslatorImpl.list(QueryTranslatorImpl.java:271)

at org.hibernate.impl.SessionImpl.list(SessionImpl.java:844)

at org.hibernate.impl.QueryImpl.list(QueryImpl.java:74)

at cn.gov.zjport.persistence.core.hibernate.PersistenceHibernateImpl$1.doInHibernate(PersistenceHibernateImpl.java:96)

at org.springframework.orm.hibernate3.HibernateTemplate.execute(HibernateTemplate.java:366)

at org.springframework.orm.hibernate3.HibernateTemplate.execute(HibernateTemplate.java:334)

at cn.gov.zjport.persistence.core.hibernate.PersistenceHibernateImpl.query(PersistenceHibernateImpl.java:93)

at cn.gov.zjport.normandy.dao.persistence.base.impl.BaseDAOImpl.getListByFilter(BaseDAOImpl.java:109)

at sun.reflect.GeneratedMethodAccessor466.invoke(Unknown Source)

at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

at java.lang.reflect.Method.invoke(Method.java:618)

at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:287)

at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:181)

at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:148)

at cn.gov.zjport.houston.service.base.cache.interceptor.ParaCacheInterceptor.invoke(ParaCacheInterceptor.java:68)

at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:170)

at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:96)

at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:170)

at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:176)

at $Proxy595.getListByFilter(Unknown Source)

at cn.gov.zjport.normandy.service.exchange.impl.ImbackDeclareServiceImpl.queryImbackDeclare(ImbackDeclareServiceImpl.java:65)

at sun.reflect.GeneratedMethodAccessor465.invoke(Unknown Source)

at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

at java.lang.reflect.Method.invoke(Method.java:618)

at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:287)

at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:181)

at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:148)

at cn.gov.zjport.houston.service.base.cache.interceptor.ParaCacheInterceptor.invoke(ParaCacheInterceptor.java:68)

at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:170)

at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:96)

at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:170)

at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:176)

at $Proxy602.queryImbackDeclare(Unknown Source)

at cn.gov.zjport.normandy.service.webservice.ImbackDeclareServiceImpl.queryImbackDeclare(ImbackDeclareServiceImpl.java:57)

at sun.reflect.GeneratedMethodAccessor464.invoke(Unknown Source)

at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

at java.lang.reflect.Method.invoke(Method.java:618)

at org.apache.axis.providers.java.RPCProvider.invokeMethod(RPCProvider.java:410)

at org.apache.axis.providers.java.RPCProvider.processMessage(RPCProvider.java:186)

at org.apache.axis.providers.java.JavaProvider.invoke(JavaProvider.java:332)

at org.apache.axis.strategies.InvocationStrategy.visit(InvocationStrategy.java:32)

at org.apache.axis.SimpleChain.doVisiting(SimpleChain.java:118)

at org.apache.axis.SimpleChain.invoke(SimpleChain.java:83)

at org.apache.axis.handlers.soap.SOAPService.invoke(SOAPService.java:454)

at org.apache.axis.server.AxisServer.invokeService(AxisServer.java:234)

at org.apache.axis.server.AxisServer.invoke(AxisServer.java:375)

at org.apache.axis.transport.http.AxisServlet.doPost(AxisServlet.java:689)

at javax.servlet.http.HttpServlet.service(HttpServlet.java:763)

at org.apache.axis.transport.http.AxisServletBase.service(AxisServletBase.java:327)

at javax.servlet.http.HttpServlet.service(HttpServlet.java:856)

at weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:227)

at weblogic.servlet.internal.StubSecurityHelper.invokeServlet(StubSecurityHelper.java:125)

at weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:283)

at weblogic.servlet.internal.TailFilter.doFilter(TailFilter.java:26)

at weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:42)

at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:75)

at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:77)

at weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:42)

at com.opensymphony.webwork.dispatcher.ActionContextCleanUp.doFilter(ActionContextCleanUp.java:88)

at weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:42)

at weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.run(WebAppServletContext.java:3242)

at weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:321)

at weblogic.security.service.SecurityManager.runAs(SecurityManager.java:121)

at weblogic.servlet.internal.WebAppServletContext.securedExecute(WebAppServletContext.java:2010)

at weblogic.servlet.internal.WebAppServletContext.execute(WebAppServletContext.java:1916)

at weblogic.servlet.internal.ServletRequestImpl.run(ServletRequestImpl.java:1366)

at weblogic.work.ExecuteThread.execute(ExecuteThread.java:209)

at weblogic.work.ExecuteThread.run(ExecuteThread.java:181)

究竟是什么问题,还在调查中

本文转自zylhsy 51CTO博客,原文链接:http://blog.51cto.com/yunlongzheng/509680,如需转载请自行联系原作者

上一篇:没有了
下一篇:Linux软件路由防火墙ipcop1.4.5总结