Periodically solr down issue.

Alex Feng sweden.feng at gmail.com
Sat Feb 4 22:48:28 EST 2017


Hi Luke,

Here is some error log from solr.log.


2017-02-04 13:15:15,420 [ERROR] <qtp775533998-16>@SolrException.java:109
org.apache.solr.common.SolrException:
org.apache.solr.client.solrj.SolrServerException: IOException occured when
talking to server at:
http://nosql-2.dsdb:8093/internal_solr/production_scheduling
at
org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:308)
at
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:1916)
at
org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:780)
at
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:427)
at
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:217)
at
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
at
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
at
org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:557)
at
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:231)
at
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1075)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
at
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
at
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
at
org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:255)
at
org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:154)
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
at org.eclipse.jetty.server.Server.handle(Server.java:368)
at
org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
at
org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
at
org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:953)
at
org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1014)
at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:861)
at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240)
at
org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
at
org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException
occured when talking to server at:
http://nosql-2.dsdb:8093/internal_solr/production_scheduling
at
org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:507)
at
org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199)
at
org.apache.solr.handler.component.HttpShardHandler$1.call(HttpShardHandler.java:156)
at
org.apache.solr.handler.component.HttpShardHandler$1.call(HttpShardHandler.java:118)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
... 1 more
Caused by: org.apache.http.NoHttpResponseException: The target server
failed to respond
at
org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
at
org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
at
org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
at
org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
at
org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
at
org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
at
org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
at
org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
at
org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
at
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
at
org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
at
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
at
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
at
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
at
org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:395)
... 9 more

2017-02-04 13:15:15,431 [ERROR] <qtp775533998-16>@SolrException.java:120
null:org.apache.solr.common.SolrException:
org.apache.solr.client.solrj.SolrServerException: IOException occured when
talking to server at:
http://nosql-2.dsdb:8093/internal_solr/production_scheduling
at
org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:308)
at
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:1916)
at
org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:780)
at
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:427)
at
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:217)
at
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
at
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
at
org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:557)
at
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:231)
at
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1075)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
at
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
at
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
at
org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:255)
at
org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:154)
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
at org.eclipse.jetty.server.Server.handle(Server.java:368)
at
org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
at
org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
at
org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:953)
at
org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1014)
at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:861)
at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240)
at
org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
at
org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException
occured when talking to server at:
http://nosql-2.dsdb:8093/internal_solr/production_scheduling
at
org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:507)
at
org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199)
at
org.apache.solr.handler.component.HttpShardHandler$1.call(HttpShardHandler.java:156)
at
org.apache.solr.handler.component.HttpShardHandler$1.call(HttpShardHandler.java:118)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
... 1 more
Caused by: org.apache.http.NoHttpResponseException: The target server
failed to respond
at
org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
at
org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
at
org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
at
org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
at
org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
at
org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
at
org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
at
org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
at
org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
at
org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
at
org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
at
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
at
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
at
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
at
org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:395)
... 9 more


Br,
Alex

2017-02-04 23:30 GMT+08:00 Luke Bakken <lbakken at basho.com>:

> Hi Alex -
>
> What is in solr.log ?
> --
> Luke Bakken
> Engineer
> lbakken at basho.com
>
>
> On Sat, Feb 4, 2017 at 3:37 AM, Alex Feng <sweden.feng at gmail.com> wrote:
> > Hello Riak users,
> >
> > I recently found our solr system got down after running some time, it got
> > recovered after a restart. But then it happened again.
> >
> > Below is the error output, do you guys have any clue about this ?
> >
> >
> > 2017-02-04 18:12:22.329 [error] <0.830.0>@yz_kv:index_internal:237
> failed to
> > index object
> > {{<<"process_test_result">>,<<83,104,105,76,111,110,103,90,105,9
> > 5,77,66,80,65,67,75,45,231,148,159,228,186,167,232,189,
> 166,233,151,180,50,48,49,55,50,52>>},<<"9H5AlII0OZm8xwnGbI">>}
> > with error {"Failed to index docs",o
> > ther,{error,{conn_failed,{error,econnrefused}}}} because
> > [{yz_solr,index,3,[{file,"src/yz_solr.erl"},{line,205}]},{
> yz_kv,index,7,[{file,"src/yz_kv.erl"},{
> > line,293}]},{yz_kv,index_internal,5,[{file,"src/yz_kv.
> erl"},{line,224}]},{riak_kv_vnode,actual_put,6,[{file,"
> src/riak_kv_vnode.erl"},{line,1619}]},{riak_k
> > v_vnode,perform_put,3,[{file,"src/riak_kv_vnode.erl"},{line,
> 1607}]},{riak_kv_vnode,do_put,7,[{file,"src/riak_kv_vnode.
> erl"},{line,1398}]},{riak_kv_vnode,h
> > andle_command,3,[{file,"src/riak_kv_vnode.erl"},{line,558}
> ]},{riak_core_vnode,vnode_command,3,[{file,"src/riak_
> core_vnode.erl"},{line,346}]}]
> >
> >
> >
> > restart node, got this>
> >
> >
> > 2017-02-04 18:13:47.026 [error] <0.4340.0>@yz_pb_search:maybe_
> process:111
> > {solr_error,{500,"http://localhost:8093/internal_solr/
> process_test_result/select
> > ",<<"{\"error\":{\"msg\":\"org.apache.solr.client.solrj.
> SolrServerException:
> > Server refused connection at:
> > http://nosql-3.dsdb:8093/internal_solr/process_
> > test_result\",\"trace\":\"org.apache.solr.common.SolrException:
> > org.apache.solr.client.solrj.SolrServerException: Server refused
> connection
> > at: http://nos
> > ql-3.dsdb:8093/internal_solr/process_test_result\\n\\tat
> > org.apache.solr.handler.component.SearchHandler.handleRequestBody(
> SearchHandler.java:308)\\n\\tat
> >
> > org.apache.solr.handler.RequestHandlerBase.handleRequest(
> RequestHandlerBase.java:135)\\n\\tat
> > org.apache.solr.core.SolrCore.execute(SolrCore.java:1916)\\
> > n\\tat
> > org.apache.solr.servlet.SolrDispatchFilter.execute(
> SolrDispatchFilter.java:780)\\n\\tat
> > org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDis
> > patchFilter.java:427)\\n\\tat
> > org.apache.solr.servlet.SolrDispatchFilter.doFilter(
> SolrDispatchFilter.java:217)\\n\\tat
> > org.eclipse.jetty.servlet.ServletHa
> > ndler$CachedChain.doFilter(ServletHandler.java:1419)\\n\\tat
> > org.eclipse.jetty.servlet.ServletHandler.doHandle(
> ServletHandler.java:455)\\n\\tat
> > org.eclipse.jetty.server.handler.ScopedHandler.handle(
> ScopedHandler.java:137)\\n\\tat
> > org.eclipse.jetty.security.SecurityHandler.handle(
> SecurityHandler.java:557)\\n\\tat
> > org.eclipse.jetty.server.session.SessionHandler.
> doHandle(SessionHandler.java:231)\\n\\tat
> > org.eclipse.jetty.server.handler.ContextHandler.
> doHandle(ContextHandler.java:1075)\\n\\tat
> > org.eclipse.jetty.servlet.ServletHandler.doScope(
> ServletHandler.java:384)\\n\\tat
> > org.eclipse.jetty.server.session.SessionHandler.
> doScope(SessionHandler.java:193)\\n\\tat
> > org.eclipse.jetty.server.handler.ContextHandler.
> doScope(ContextHandler.java:1009)\\n\\tat
> > org.eclipse.jetty.server.handler.ScopedHandler.handle(
> ScopedHandler.java:135)\\n\\tat
> > org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(
> ContextHandlerCollection.java:255)\\n\\tat
> > org.eclipse.jetty.server.handler.HandlerCollection.
> handle(HandlerCollection.java:154)\\n\\tat
> > org.eclipse.jetty.server.handler.HandlerWrap...">>}}
> > [{yz_solr,search,3,[{file,"src/yz_solr.erl"},{line,292}]}
> ,{yz_pb_search,maybe_process,3,[{file,"src/yz_pb_search.
> erl"},{line,76}]},{riak_api_pb_server,process_message,4,[{
> file,"src/riak_api_pb_server.erl"},{line,388}]},{riak_api_
> pb_server,connected,2,[{file,"src/riak_api_pb_server.erl"},{
> line,226}]},{riak_api_pb_server,decode_buffer,2,[{file,
> "src/riak_api_pb_server.erl"},{line,364}]},{gen_fsm,handle_
> msg,7,[{file,"gen_fsm.erl"},{line,505}]},{proc_lib,init_p_
> do_apply,3,[{file,"proc_lib.erl"},{line,239}]}]
> > 2017-02-04 18:17:02.756 [error]
> > <0.6871.0>@riak_core_handoff_receiver:handle_info:97 Handoff receiver
> for
> > partition 0 exited abnormally after processing 0 objects from
> > {"192.168.9.247",39483}:
> > {error,{vnode_timeout,60000,844,<<131,104,2,100,0,10,101,
> 110,99,111,100,101,95,114,97,119,104,3,104,2,109,0,0,0,19,
> 112,114,111,99,101,115,115,95,116,101,115,116,95,114,101,
> 115,117,108,116,109,0,0,0,35,83,104,105,76,111,110,103,90,
> 105,95,77,66,80,65,67,75,45,231,148,159,228,186,167,232,
> 189,166,233,151,180,50,48,49,55,50,52,109,0,0,0,18,57,72,
> 53,65,109,84,122,54,79,109,99,108,87,107,87,68,80,85,109,0,
> 0,2,220,53,1,0,0,0,38,131,108,0,0,0,1,104,2,109,0>>}}
> >
> >
> > Br,
> >
> > Alex
> >
> >
> > _______________________________________________
> > riak-users mailing list
> > riak-users at lists.basho.com
> > http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com
> >
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.basho.com/pipermail/riak-users_lists.basho.com/attachments/20170205/e54fe6d4/attachment-0002.html>


More information about the riak-users mailing list