-
Notifications
You must be signed in to change notification settings - Fork 15
Shorten error message when request is forward to dead node #28
Description
I suggest special case handling "connect refused" and reducing the error to something like:
11:32:13.805 INFO - Error while forwarding request to cluster:cluster1 http://localhost:8082: Connection refused.
To reduce log file pollution since the stack trace is not useful. We can add other common exceptions to the list as we find them.
Should this be an INFO instead of an ERROR?
From the forwarding code's point of view, this is normal behavior.
I expect there would be other special monitoring for dead nodes.
11:32:13.805 ERROR - Error while forwarding request to cluster:cluster1 http://localhost:8082
org.apache.http.conn.HttpHostConnectException: Connect to localhost:8082 [localhost/127.0.0.1] failed: Connection refused
at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:151) ~[httpclient-4.5.1.jar:4.5.1]
at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353) ~[httpclient-4.5.1.jar:4.5.1]
at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380) ~[httpclient-4.5.1.jar:4.5.1]
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236) ~[httpclient-4.5.1.jar:4.5.1]
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184) ~[httpclient-4.5.1.jar:4.5.1]
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88) ~[httpclient-4.5.1.jar:4.5.1]
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) ~[httpclient-4.5.1.jar:4.5.1]
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184) ~[httpclient-4.5.1.jar:4.5.1]
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82) ~[httpclient-4.5.1.jar:4.5.1]
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107) ~[httpclient-4.5.1.jar:4.5.1]
at com.yahoo.gondola.container.client.ApacheHttpComponentProxyClient.proxyRequest(ApacheHttpComponentProxyClient.java:60) ~[classes/:na]
at com.yahoo.gondola.container.RoutingFilter.proxyRequestToLeader(RoutingFilter.java:293) [classes/:na]
at com.yahoo.gondola.container.RoutingFilter.filter(RoutingFilter.java:196) [classes/:na]
at org.glassfish.jersey.server.ContainerFilteringStage.apply(ContainerFilteringStage.java:132) [jersey-server-2.22.jar:na]
at org.glassfish.jersey.server.ContainerFilteringStage.apply(ContainerFilteringStage.java:68) [jersey-server-2.22.jar:na]
at org.glassfish.jersey.process.internal.Stages.process(Stages.java:197) [jersey-common-2.22.jar:na]
at org.glassfish.jersey.server.ServerRuntime$2.run(ServerRuntime.java:309) [jersey-server-2.22.jar:na]
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:271) [jersey-common-2.22.jar:na]
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:267) [jersey-common-2.22.jar:na]
at org.glassfish.jersey.internal.Errors.process(Errors.java:315) [jersey-common-2.22.jar:na]
at org.glassfish.jersey.internal.Errors.process(Errors.java:297) [jersey-common-2.22.jar:na]
at org.glassfish.jersey.internal.Errors.process(Errors.java:267) [jersey-common-2.22.jar:na]
at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:317) [jersey-common-2.22.jar:na]
at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:298) [jersey-server-2.22.jar:na]
at org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:1154) [jersey-server-2.22.jar:na]
at org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.java:471) [jersey-container-servlet-core-2.22.jar:na]
at org.glassfish.jersey.servlet.WebComponent.service(WebComponent.java:425) [jersey-container-servlet-core-2.22.jar:na]
at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:383) [jersey-container-servlet-core-2.22.jar:na]
at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:336) [jersey-container-servlet-core-2.22.jar:na]
at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:223) [jersey-container-servlet-core-2.22.jar:na]
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:806) [jetty-servlet-9.3.0.M2.jar:9.3.0.M2]
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:587) [jetty-servlet-9.3.0.M2.jar:9.3.0.M2]
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143) [jetty-server-9.3.0.M2.jar:9.3.0.M2]
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:550) [jetty-security-9.3.0.M2.jar:9.3.0.M2]
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:223) [jetty-server-9.3.0.M2.jar:9.3.0.M2]
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1128) [jetty-server-9.3.0.M2.jar:9.3.0.M2]
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515) [jetty-servlet-9.3.0.M2.jar:9.3.0.M2]
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) [jetty-server-9.3.0.M2.jar:9.3.0.M2]
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1062) [jetty-server-9.3.0.M2.jar:9.3.0.M2]
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) [jetty-server-9.3.0.M2.jar:9.3.0.M2]
at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:215) [jetty-server-9.3.0.M2.jar:9.3.0.M2]
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:110) [jetty-server-9.3.0.M2.jar:9.3.0.M2]
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:113) [jetty-server-9.3.0.M2.jar:9.3.0.M2]
at org.eclipse.jetty.server.Server.handle(Server.java:507) [jetty-server-9.3.0.M2.jar:9.3.0.M2]
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:284) [jetty-server-9.3.0.M2.jar:9.3.0.M2]
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:237) [jetty-server-9.3.0.M2.jar:9.3.0.M2]
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:240) [jetty-io-9.3.0.M2.jar:9.3.0.M2]
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:93) [jetty-io-9.3.0.M2.jar:9.3.0.M2]
at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:53) [jetty-io-9.3.0.M2.jar:9.3.0.M2]
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceRun.produceAndRun(ExecuteProduceRun.java:191) [jetty-util-9.3.0.M2.jar:9.3.0.M2]
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceRun.run(ExecuteProduceRun.java:126) [jetty-util-9.3.0.M2.jar:9.3.0.M2]
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:641) [jetty-util-9.3.0.M2.jar:9.3.0.M2]
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:559) [jetty-util-9.3.0.M2.jar:9.3.0.M2]
at java.lang.Thread.run(Thread.java:745) [na:1.8.0_60]
Caused by: java.net.ConnectException: Connection refused
at java.net.PlainSocketImpl.socketConnect(Native Method) ~[na:1.8.0_60]
at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) ~[na:1.8.0_60]
at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) ~[na:1.8.0_60]
at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) ~[na:1.8.0_60]
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) ~[na:1.8.0_60]
at java.net.Socket.connect(Socket.java:589) ~[na:1.8.0_60]
at org.apache.http.conn.socket.PlainConnectionSocketFactory.connectSocket(PlainConnectionSocketFactory.java:74) ~[httpclient-4.5.1.jar:4.5.1]
at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:134) ~[httpclient-4.5.1.jar:4.5.1]
... 53 common frames omitted