1. Riak Crashing on Data Load from CSV and 2. How to get Ripple to load js functions

Gustav Gennrich ggennrich at gmail.com
Tue May 22 12:22:50 EDT 2012


Hi Dave,

This is some output from the crash.log file on the nodes. app.config file
is at the end:

*Node 1 (Hosting Riak Control):*

2012-05-18 04:58:25 =ERROR REPORT====
{mochiweb_socket_server,310,{acceptor_error,{error,accept_failed}}}2012-05-18
04:58:25 =ERROR REPORT====
{mochiweb_socket_server,310,{acceptor_error,{error,accept_failed}}}2012-05-18
04:58:25 =ERROR REPORT====
{mochiweb_socket_server,310,{acceptor_error,{error,accept_failed}}}2012-05-18
04:58:25 =ERROR REPORT====
{mochiweb_socket_server,310,{acceptor_error,{error,accept_failed}}}2012-05-18
04:58:26 =ERROR REPORT====
{mochiweb_socket_server,310,{acceptor_error,{error,accept_failed}}}2012-05-18
04:58:26 =ERROR REPORT====
{mochiweb_socket_server,310,{acceptor_error,{error,accept_failed}}}2012-05-18
04:58:26 =ERROR REPORT====
{mochiweb_socket_server,310,{acceptor_error,{error,accept_failed}}}2012-05-18
04:58:26 =ERROR REPORT====
{mochiweb_socket_server,310,{acceptor_error,{error,accept_failed}}}2012-05-18
04:58:26 =ERROR REPORT====
{mochiweb_socket_server,310,{acceptor_error,{error,accept_failed}}}2012-05-18
04:58:26 =ERROR REPORT====
{mochiweb_socket_server,310,{acceptor_error,{error,accept_failed}}}2012-05-18
04:58:26 =ERROR REPORT====
{mochiweb_socket_server,310,{acceptor_error,{error,accept_failed}}}2012-05-18
04:58:26 =ERROR REPORT====
{mochiweb_socket_server,310,{acceptor_error,{error,accept_failed}}}2012-05-18
04:58:26 =ERROR REPORT====
{mochiweb_socket_server,310,{acceptor_error,{error,accept_failed}}}2012-05-18
04:58:26 =ERROR REPORT====
{mochiweb_socket_server,310,{acceptor_error,{error,accept_failed}}}2012-05-18
04:58:27 =ERROR REPORT====
{mochiweb_socket_server,310,{acceptor_error,{error,accept_failed}}}2012-05-18
04:59:00 =ERROR REPORT====
** Generic server riak_kv_stat terminating
** Last message in was
{'$gen_cast',{update,{get_fsm,undefined,1158,1,4559},63504536340}}
** When Server state ==
{state,{spiral,63504536340,[1,0,0,0,0,0,0,0,0,0,0,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,8,0,0,11,0,0,0,0,0,14,0,0,0,17,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},{spiral,63504536326,[7,0,0,0,0,0,0,0,0,0,0,0,4,0,0,7,0,0,0,0,0,9,0,0,0,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},65,38,{spiral,63504536326,[84,0,0,0,0,0,0,0,0,0,0,0,84,0,0,84,0,0,0,0,0,126,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},378,{spiral,63504536326,[7,0,0,0,0,0,0,0,0,0,0,0,4,0,0,7,0,0,0,0,0,9,0,0,0,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},38,{spiral,63504536326,[1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},7,{spiral,63504536301,[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},0,{spiral,63504536326,[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},1,29,18,{slide,63504536301,60,60,"/var/lib/riak/slide-data/3053/1337.331501.51152",{file_descriptor,prim_file,{#Port<0.2186418>,128}},63504536326},{slide,63504536301,60,60,"/var/lib/riak/slide-data/3053/1337.331501.51714",{file_descriptor,prim_file,{#Port<0.2186419>,177}},63504536326},{slide,63504536301,60,60,"/var/lib/riak/slide-data/3053/1337.331501.52049",{file_descriptor,prim_file,{#Port<0.2186417>,69}},63504536326},{slide,63504536301,60,60,"/var/lib/riak/slide-data/3053/1337.331501.52247",{file_descriptor,prim_file,{#Port<0.2186420>,40}},63504536326},{spiral,63504536301,[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},0,0,{spiral,63504536301,[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},0,undefined,0,0,undefined,undefined,0,0,true}
** Reason for termination ==
** {emfile,[{erlang,open_port,[{spawn,"/bin/sh -s unix:cmd
2>&1"},[stream]]},{os,start_port_srv_handle,1},{os,start_port_srv_loop,0}]}
2012-05-18 04:59:00 =CRASH REPORT====
  crasher:
    initial call: gen:init_it/7
    pid: <0.31225.398>
    registered_name: riak_kv_stat
    exception exit: {emfile,[{erlang,open_port,[{spawn,"/bin/sh -s unix:cmd
2>&1"},[stream]]},{os,start_port_srv_handle,1},{os,start_port_srv_loop,0}]}
      in function  gen_server2:terminate/6
      in call from proc_lib:init_p_do_apply/3
    ancestors: [riak_kv_sup,<0.323.0>]
    messages: []
    links: [#Port<0.2186419>,#Port<0.2186420>,<0.420.0>,#Port<0.2186418>]
    dictionary: []
    trap_exit: true
    status: running
    heap_size: 377
    stack_size: 24
    reductions: 41467
  neighbours:
2012-05-18 04:59:00 =SUPERVISOR REPORT====
     Supervisor: {local,riak_kv_sup}
     Context:    child_terminated
     Reason:     {emfile,[{erlang,open_port,[{spawn,"/bin/sh -s unix:cmd
2>&1"},[stream]]},{os,start_port_srv_handle,1},{os,start_port_srv_loop,0}]}
     Offender:
[{pid,<0.31225.398>},{name,riak_kv_stat},{mfargs,{riak_kv_stat,start_link,[]}},{restart_type,permanent},{shutdown,5000},{child_type,worker}]

2012-05-18 04:59:00 =CRASH REPORT====
  crasher:
    initial call: riak_core_vnode:init/1
    pid: <0.22390.463>
    registered_name: []
    exception exit:
{{badmatch,{error,emfile}},[{riak_kv_vnode,init,1},{riak_core_vnode,init,1},{gen_fsm,init_it,6},{proc_lib,init_p_do_apply,3}]}
      in function  gen_fsm:init_it/6
      in call from proc_lib:init_p_do_apply/3
    ancestors: [riak_core_vnode_sup,riak_core_sup,<0.91.0>]
    messages: []
    links: [<0.95.0>]
    dictionary: []
    trap_exit: true
    status: running
    heap_size: 987
    stack_size: 24
    reductions: 2734
  neighbours:
2012-05-18 04:59:00 =SUPERVISOR REPORT====
     Supervisor: {local,riak_core_vnode_sup}
     Context:    child_terminated
     Reason:
{{badmatch,{error,emfile}},[{riak_kv_vnode,init,1},{riak_core_vnode,init,1},{gen_fsm,init_it,6},{proc_lib,init_p_do_apply,3}]}
     Offender:
[{pid,<0.22390.463>},{name,undefined},{mfargs,{riak_core_vnode,start_link,undefined}},{restart_type,temporary},{shutdown,300000},{child_type,worker}]

2012-05-18 04:59:00 =ERROR REPORT====
** Generic server riak_core_vnode_manager terminating
** Last message in was
{137015778499772148581595453067151533092743675904,riak_kv_vnode,get_vnode}
** When Server state ==
{state,40798511124,[{{riak_kv_vnode,0},undefined},{{riak_kv_vnode,11417981541647679048466287755595961091061972992},undefined},{{riak_kv_vnode,22835963083295358096932575511191922182123945984},undefined},{{riak_kv_vnode,34253944624943037145398863266787883273185918976},undefined},{{riak_kv_vnode,45671926166590716193865151022383844364247891968},undefined},{{riak_kv_vnode,57089907708238395242331438777979805455309864960},undefined},{{riak_kv_vnode,68507889249886074290797726533575766546371837952},undefined},{{riak_kv_vnode,79925870791533753339264014289171727637433810944},undefined},{{riak_kv_vnode,91343852333181432387730302044767688728495783936},undefined},{{riak_kv_vnode,102761833874829111436196589800363649819557756928},undefined},{{riak_kv_vnode,114179815416476790484662877555959610910619729920},undefined},{{riak_kv_vnode,125597796958124469533129165311555572001681702912},undefined},{{riak_kv_vnode,137015778499772148581595453067151533092743675904},undefined},{{riak_kv_vnode,148433760041419827630061740822747494183805648896},undefined},{{riak_kv_vnode,159851741583067506678528028578343455274867621888},undefined},{{riak_kv_vnode,171269723124715185726994316333939416365929594880},undefined},{{riak_kv_vnode,182687704666362864775460604089535377456991567872},undefined},{{riak_kv_vnode,194105686208010543823926891845131338548053540864},undefined},{{riak_kv_vnode,205523667749658222872393179600727299639115513856},undefined},{{riak_kv_vnode,216941649291305901920859467356323260730177486848},undefined},{{riak_kv_vnode,228359630832953580969325755111919221821239459840},undefined},{{riak_kv_vnode,239777612374601260017792042867515182912301432832},undefined},{{riak_kv_vnode,251195593916248939066258330623111144003363405824},undefined},{{riak_kv_vnode,262613575457896618114724618378707105094425378816},undefined},{{riak_kv_vnode,274031556999544297163190906134303066185487351808},undefined},{{riak_kv_vnode,285449538541191976211657193889899027276549324800},undefined},{{riak_kv_vnode,296867520082839655260123481645494988367611297792},undefined},{{riak_kv_vnode,308285501624487334308589769401090949458673270784},undefined},{{riak_kv_vnode,319703483166135013357056057156686910549735243776},undefined},{{riak_kv_vnode,331121464707782692405522344912282871640797216768},undefined},{{riak_kv_vnode,342539446249430371453988632667878832731859189760},undefined},{{riak_kv_vnode,353957427791078050502454920423474793822921162752},undefined},{{riak_kv_vnode,365375409332725729550921208179070754913983135744},undefined},{{riak_kv_vnode,376793390874373408599387495934666716005045108736},undefined},{{riak_kv_vnode,388211372416021087647853783690262677096107081728},undefined},{{riak_kv_vnode,399629353957668766696320071445858638187169054720},undefined},{{riak_kv_vnode,411047335499316445744786359201454599278231027712},undefined},{{riak_kv_vnode,422465317040964124793252646957050560369293000704},undefined},{{riak_kv_vnode,433883298582611803841718934712646521460354973696},undefined},{{riak_kv_vnode,445301280124259482890185222468242482551416946688},undefined},{{riak_kv_vnode,456719261665907161938651510223838443642478919680},undefined},{{riak_kv_vnode,468137243207554840987117797979434404733540892672},undefined},{{riak_kv_vnode,479555224749202520035584085735030365824602865664},undefined},{{riak_kv_vnode,490973206290850199084050373490626326915664838656},undefined},{{riak_kv_vnode,502391187832497878132516661246222288006726811648},undefined},{{riak_kv_vnode,513809169374145557180982949001818249097788784640},undefined},{{riak_kv_vnode,525227150915793236229449236757414210188850757632},undefined},{{riak_kv_vnode,536645132457440915277915524513010171279912730624},undefined},{{riak_kv_vnode,548063113999088594326381812268606132370974703616},undefined},{{riak_kv_vnode,559481095540736273374848100024202093462036676608},undefined},{{riak_kv_vnode,570899077082383952423314387779798054553098649600},undefined},{{riak_kv_vnode,582317058624031631471780675535394015644160622592},undefined},{{riak_kv_vnode,593735040165679310520246963290989976735222595584},undefined},{{riak_kv_vnode,605153021707326989568713251046585937826284568576},undefined},{{riak_kv_vnode,616571003248974668617179538802181898917346541568},undefined},{{riak_kv_vnode,627988984790622347665645826557777860008408514560},undefined},{{riak_kv_vnode,639406966332270026714112114313373821099470487552},undefined},{{riak_kv_vnode,650824947873917705762578402068969782190532460544},undefined},{{riak_kv_vnode,662242929415565384811044689824565743281594433536},undefined},{{riak_kv_vnode,673660910957213063859510977580161704372656406528},undefined},{{riak_kv_vnode,685078892498860742907977265335757665463718379520},undefined},{{riak_kv_vnode,696496874040508421956443553091353626554780352512},undefined},{{riak_kv_vnode,707914855582156101004909840846949587645842325504},undefined},{{riak_kv_vnode,719332837123803780053376128602545548736904298496},undefined},{{riak_kv_vnode,730750818665451459101842416358141509827966271488},undefined},{{riak_kv_vnode,742168800207099138150308704113737470919028244480},undefined},{{riak_kv_vnode,753586781748746817198774991869333432010090217472},undefined},{{riak_kv_vnode,765004763290394496247241279624929393101152190464},undefined},{{riak_kv_vnode,776422744832042175295707567380525354192214163456},undefined},{{riak_kv_vnode,787840726373689854344173855136121315283276136448},undefined},{{riak_kv_vnode,799258707915337533392640142891717276374338109440},undefined},{{riak_kv_vnode,810676689456985212441106430647313237465400082432},undefined},{{riak_kv_vnode,822094670998632891489572718402909198556462055424},undefined},{{riak_kv_vnode,833512652540280570538039006158505159647524028416},undefined},{{riak_kv_vnode,844930634081928249586505293914101120738586001408},undefined},{{riak_kv_vnode,856348615623575928634971581669697081829647974400},undefined},{{riak_kv_vnode,867766597165223607683437869425293042920709947392},undefined},{{riak_kv_vnode,879184578706871286731904157180889004011771920384},undefined},{{riak_kv_vnode,890602560248518965780370444936484965102833893376},undefined},{{riak_kv_vnode,902020541790166644828836732692080926193895866368},undefined},{{riak_kv_vnode,913438523331814323877303020447676887284957839360},undefined},{{riak_kv_vnode,924856504873462002925769308203272848376019812352},undefined},{{riak_kv_vnode,936274486415109681974235595958868809467081785344},undefined},{{riak_kv_vnode,947692467956757361022701883714464770558143758336},undefined},{{riak_kv_vnode,959110449498405040071168171470060731649205731328},undefined},{{riak_kv_vnode,970528431040052719119634459225656692740267704320},undefined},{{riak_kv_vnode,981946412581700398168100746981252653831329677312},undefined},{{riak_kv_vnode,993364394123348077216567034736848614922391650304},undefined},{{riak_kv_vnode,1004782375664995756265033322492444576013453623296},undefined},{{riak_kv_vnode,1016200357206643435313499610248040537104515596288},undefined},{{riak_kv_vnode,1027618338748291114361965898003636498195577569280},undefined},{{riak_kv_vnode,1039036320289938793410432185759232459286639542272},undefined},{{riak_kv_vnode,1050454301831586472458898473514828420377701515264},undefined},{{riak_kv_vnode,1061872283373234151507364761270424381468763488256},undefined},{{riak_kv_vnode,1073290264914881830555831049026020342559825461248},undefined},{{riak_kv_vnode,1084708246456529509604297336781616303650887434240},undefined},{{riak_kv_vnode,1096126227998177188652763624537212264741949407232},undefined},{{riak_kv_vnode,1107544209539824867701229912292808225833011380224},undefined},{{riak_kv_vnode,1118962191081472546749696200048404186924073353216},undefined},{{riak_kv_vnode,1130380172623120225798162487804000148015135326208},undefined},{{riak_kv_vnode,1141798154164767904846628775559596109106197299200},undefined},{{riak_kv_vnode,1153216135706415583895095063315192070197259272192},undefined},{{riak_kv_vnode,1164634117248063262943561351070788031288321245184},undefined},{{riak_kv_vnode,1176052098789710941992027638826383992379383218176},undefined},{{riak_kv_vnode,1187470080331358621040493926581979953470445191168},undefined},{{riak_kv_vnode,1198888061873006300088960214337575914561507164160},undefined},{{riak_kv_vnode,1210306043414653979137426502093171875652569137152},undefined},{{riak_kv_vnode,1221724024956301658185892789848767836743631110144},undefined},{{riak_kv_vnode,1233142006497949337234359077604363797834693083136},undefined},{{riak_kv_vnode,1244559988039597016282825365359959758925755056128},undefined},{{riak_kv_vnode,1255977969581244695331291653115555720016817029120},undefined},{{riak_kv_vnode,1267395951122892374379757940871151681107879002112},undefined},{{riak_kv_vnode,1278813932664540053428224228626747642198940975104},undefined},{{riak_kv_vnode,1290231914206187732476690516382343603290002948096},undefined},{{riak_kv_vnode,1301649895747835411525156804137939564381064921088},undefined},{{riak_kv_vnode,1313067877289483090573623091893535525472126894080},undefined},{{riak_kv_vnode,1324485858831130769622089379649131486563188867072},undefined},{{riak_kv_vnode,1335903840372778448670555667404727447654250840064},undefined},{{riak_kv_vnode,1347321821914426127719021955160323408745312813056},undefined},{{riak_kv_vnode,1358739803456073806767488242915919369836374786048},undefined},{{riak_kv_vnode,1370157784997721485815954530671515330927436759040},undefined},{{riak_kv_vnode,1381575766539369164864420818427111292018498732032},undefined},{{riak_kv_vnode,1392993748081016843912887106182707253109560705024},undefined},{{riak_kv_vnode,1404411729622664522961353393938303214200622678016},undefined},{{riak_kv_vnode,1415829711164312202009819681693899175291684651008},undefined},{{riak_kv_vnode,1427247692705959881058285969449495136382746624000},undefined},{{riak_kv_vnode,1438665674247607560106752257205091097473808596992},undefined},{{riak_kv_vnode,1450083655789255239155218544960687058564870569984},undefined},{{riak_pipe_vnode,0},undefined},{{riak_pipe_vnode,11417981541647679048466287755595961091061972992},undefined},{{riak_pipe_vnode,22835963083295358096932575511191922182123945984},undefined},{{riak_pipe_vnode,34253944624943037145398863266787883273185918976},undefined},{{riak_pipe_vnode,45671926166590716193865151022383844364247891968},undefined},{{riak_pipe_vnode,57089907708238395242331438777979805455309864960},undefined},{{riak_pipe_vnode,68507889249886074290797726533575766546371837952},undefined},{{riak_pipe_vnode,79925870791533753339264014289171727637433810944},undefined},{{riak_pipe_vnode,91343852333181432387730302044767688728495783936},undefined},{{riak_pipe_vnode,102761833874829111436196589800363649819557756928},undefined},{{riak_pipe_vnode,114179815416476790484662877555959610910619729920},undefined},{{riak_pipe_vnode,125597796958124469533129165311555572001681702912},undefined},{{riak_pipe_vnode,137015778499772148581595453067151533092743675904},undefined},{{riak_pipe_vnode,148433760041419827630061740822747494183805648896},undefined},{{riak_pipe_vnode,159851741583067506678528028578343455274867621888},undefined},{{riak_pipe_vnode,171269723124715185726994316333939416365929594880},undefined},{{riak_pipe_vnode,182687704666362864775460604089535377456991567872},undefined},{{riak_pipe_vnode,194105686208010543823926891845131338548053540864},undefined},{{riak_pipe_vnode,205523667749658222872393179600727299639115513856},undefined},{{riak_pipe_vnode,216941649291305901920859467356323260730177486848},undefined},{{riak_pipe_vnode,228359630832953580969325755111919221821239459840},undefined},{{riak_pipe_vnode,239777612374601260017792042867515182912301432832},undefined},{{riak_pipe_vnode,251195593916248939066258330623111144003363405824},undefined},{{riak_pipe_vnode,262613575457896618114724618378707105094425378816},undefined},{{riak_pipe_vnode,274031556999544297163190906134303066185487351808},undefined},{{riak_pipe_vnode,285449538541191976211657193889899027276549324800},undefined},{{riak_pipe_vnode,296867520082839655260123481645494988367611297792},undefined},{{riak_pipe_vnode,308285501624487334308589769401090949458673270784},undefined},{{riak_pipe_vnode,319703483166135013357056057156686910549735243776},undefined},{{riak_pipe_vnode,331121464707782692405522344912282871640797216768},undefined},{{riak_pipe_vnode,342539446249430371453988632667878832731859189760},undefined},{{riak_pipe_vnode,353957427791078050502454920423474793822921162752},undefined},{{riak_pipe_vnode,365375409332725729550921208179070754913983135744},undefined},{{riak_pipe_vnode,376793390874373408599387495934666716005045108736},undefined},{{riak_pipe_vnode,388211372416021087647853783690262677096107081728},undefined},{{riak_pipe_vnode,399629353957668766696320071445858638187169054720},undefined},{{riak_pipe_vnode,411047335499316445744786359201454599278231027712},undefined},{{riak_pipe_vnode,422465317040964124793252646957050560369293000704},undefined},{{riak_pipe_vnode,433883298582611803841718934712646521460354973696},undefined},{{riak_pipe_vnode,445301280124259482890185222468242482551416946688},undefined},{{riak_pipe_vnode,456719261665907161938651510223838443642478919680},undefined},{{riak_pipe_vnode,468137243207554840987117797979434404733540892672},undefined},{{riak_pipe_vnode,479555224749202520035584085735030365824602865664},undefined},{{riak_pipe_vnode,490973206290850199084050373490626326915664838656},undefined},{{riak_pipe_vnode,502391187832497878132516661246222288006726811648},undefined},{{riak_pipe_vnode,513809169374145557180982949001818249097788784640},undefined},{{riak_pipe_vnode,525227150915793236229449236757414210188850757632},undefined},{{riak_pipe_vnode,536645132457440915277915524513010171279912730624},undefined},{{riak_pipe_vnode,548063113999088594326381812268606132370974703616},undefined},{{riak_pipe_vnode,559481095540736273374848100024202093462036676608},undefined},{{riak_pipe_vnode,570899077082383952423314387779798054553098649600},undefined},{{riak_pipe_vnode,582317058624031631471780675535394015644160622592},undefined},{{riak_pipe_vnode,593735040165679310520246963290989976735222595584},undefined},{{riak_pipe_vnode,605153021707326989568713251046585937826284568576},undefined},{{riak_pipe_vnode,616571003248974668617179538802181898917346541568},undefined},{{riak_pipe_vnode,627988984790622347665645826557777860008408514560},undefined},{{riak_pipe_vnode,639406966332270026714112114313373821099470487552},undefined},{{riak_pipe_vnode,650824947873917705762578402068969782190532460544},undefined},{{riak_pipe_vnode,662242929415565384811044689824565743281594433536},undefined},{{riak_pipe_vnode,673660910957213063859510977580161704372656406528},undefined},{{riak_pipe_vnode,685078892498860742907977265335757665463718379520},undefined},{{riak_pipe_vnode,696496874040508421956443553091353626554780352512},undefined},{{riak_pipe_vnode,707914855582156101004909840846949587645842325504},undefined},{{riak_pipe_vnode,719332837123803780053376128602545548736904298496},undefined},{{riak_pipe_vnode,730750818665451459101842416358141509827966271488},undefined},{{riak_pipe_vnode,742168800207099138150308704113737470919028244480},undefined},{{riak_pipe_vnode,753586781748746817198774991869333432010090217472},undefined},{{riak_pipe_vnode,765004763290394496247241279624929393101152190464},undefined},{{riak_pipe_vnode,776422744832042175295707567380525354192214163456},undefined},{{riak_pipe_vnode,787840726373689854344173855136121315283276136448},undefined},{{riak_pipe_vnode,799258707915337533392640142891717276374338109440},undefined},{{riak_pipe_vnode,810676689456985212441106430647313237465400082432},undefined},{{riak_pipe_vnode,822094670998632891489572718402909198556462055424},undefined},{{riak_pipe_vnode,833512652540280570538039006158505159647524028416},undefined},{{riak_pipe_vnode,844930634081928249586505293914101120738586001408},undefined},{{riak_pipe_vnode,856348615623575928634971581669697081829647974400},undefined},{{riak_pipe_vnode,867766597165223607683437869425293042920709947392},undefined},{{riak_pipe_vnode,879184578706871286731904157180889004011771920384},undefined},{{riak_pipe_vnode,890602560248518965780370444936484965102833893376},undefined},{{riak_pipe_vnode,902020541790166644828836732692080926193895866368},undefined},{{riak_pipe_vnode,913438523331814323877303020447676887284957839360},undefined},{{riak_pipe_vnode,924856504873462002925769308203272848376019812352},undefined},{{riak_pipe_vnode,936274486415109681974235595958868809467081785344},undefined},{{riak_pipe_vnode,947692467956757361022701883714464770558143758336},undefined},{{riak_pipe_vnode,959110449498405040071168171470060731649205731328},undefined},{{riak_pipe_vnode,970528431040052719119634459225656692740267704320},undefined},{{riak_pipe_vnode,981946412581700398168100746981252653831329677312},undefined},{{riak_pipe_vnode,993364394123348077216567034736848614922391650304},undefined},{{riak_pipe_vnode,1004782375664995756265033322492444576013453623296},undefined},{{riak_pipe_vnode,1016200357206643435313499610248040537104515596288},undefined},{{riak_pipe_vnode,1027618338748291114361965898003636498195577569280},undefined},{{riak_pipe_vnode,1039036320289938793410432185759232459286639542272},undefined},{{riak_pipe_vnode,1050454301831586472458898473514828420377701515264},undefined},{{riak_pipe_vnode,1061872283373234151507364761270424381468763488256},undefined},{{riak_pipe_vnode,1073290264914881830555831049026020342559825461248},undefined},{{riak_pipe_vnode,1084708246456529509604297336781616303650887434240},undefined},{{riak_pipe_vnode,1096126227998177188652763624537212264741949407232},undefined},{{riak_pipe_vnode,1107544209539824867701229912292808225833011380224},undefined},{{riak_pipe_vnode,1118962191081472546749696200048404186924073353216},undefined},{{riak_pipe_vnode,1130380172623120225798162487804000148015135326208},undefined},{{riak_pipe_vnode,1141798154164767904846628775559596109106197299200},undefined},{{riak_pipe_vnode,1153216135706415583895095063315192070197259272192},undefined},{{riak_pipe_vnode,1164634117248063262943561351070788031288321245184},undefined},{{riak_pipe_vnode,1176052098789710941992027638826383992379383218176},undefined},{{riak_pipe_vnode,1187470080331358621040493926581979953470445191168},undefined},{{riak_pipe_vnode,1198888061873006300088960214337575914561507164160},undefined},{{riak_pipe_vnode,1210306043414653979137426502093171875652569137152},undefined},{{riak_pipe_vnode,1221724024956301658185892789848767836743631110144},undefined},{{riak_pipe_vnode,1233142006497949337234359077604363797834693083136},undefined},{{riak_pipe_vnode,1244559988039597016282825365359959758925755056128},undefined},{{riak_pipe_vnode,1255977969581244695331291653115555720016817029120},undefined},{{riak_pipe_vnode,1267395951122892374379757940871151681107879002112},undefined},{{riak_pipe_vnode,1278813932664540053428224228626747642198940975104},undefined},{{riak_pipe_vnode,1290231914206187732476690516382343603290002948096},undefined},{{riak_pipe_vnode,1301649895747835411525156804137939564381064921088},undefined},{{riak_pipe_vnode,1313067877289483090573623091893535525472126894080},undefined},{{riak_pipe_vnode,1324485858831130769622089379649131486563188867072},undefined},{{riak_pipe_vnode,1335903840372778448670555667404727447654250840064},undefined},{{riak_pipe_vnode,1347321821914426127719021955160323408745312813056},undefined},{{riak_pipe_vnode,1358739803456073806767488242915919369836374786048},undefined},{{riak_pipe_vnode,1370157784997721485815954530671515330927436759040},undefined},{{riak_pipe_vnode,1381575766539369164864420818427111292018498732032},undefined},{{riak_pipe_vnode,1392993748081016843912887106182707253109560705024},undefined},{{riak_pipe_vnode,1404411729622664522961353393938303214200622678016},undefined},{{riak_pipe_vnode,1415829711164312202009819681693899175291684651008},undefined},{{riak_pipe_vnode,1427247692705959881058285969449495136382746624000},undefined},{{riak_pipe_vnode,1438665674247607560106752257205091097473808596992},undefined},{{riak_pipe_vnode,1450083655789255239155218544960687058564870569984},undefined},{{riak_search_vnode,0},undefined},{{riak_search_vnode,11417981541647679048466287755595961091061972992},undefined},{{riak_search_vnode,22835963083295358096932575511191922182123945984},undefined},{{riak_search_vnode,34253944624943037145398863266787883273185918976},undefined},{{riak_search_vnode,45671926166590716193865151022383844364247891968},undefined},{{riak_search_vnode,57089907708238395242331438777979805455309864960},undefined},{{riak_search_vnode,68507889249886074290797726533575766546371837952},undefined},{{riak_search_vnode,79925870791533753339264014289171727637433810944},undefined},{{riak_search_vnode,91343852333181432387730302044767688728495783936},undefined},{{riak_search_vnode,102761833874829111436196589800363649819557756928},undefined},{{riak_search_vnode,114179815416476790484662877555959610910619729920},undefined},{{riak_search_vnode,125597796958124469533129165311555572001681702912},undefined},{{riak_search_vnode,137015778499772148581595453067151533092743675904},undefined},{{riak_search_vnode,148433760041419827630061740822747494183805648896},undefined},{{riak_search_vnode,159851741583067506678528028578343455274867621888},undefined},{{riak_search_vnode,171269723124715185726994316333939416365929594880},undefined},{{riak_search_vnode,182687704666362864775460604089535377456991567872},undefined},{{riak_search_vnode,194105686208010543823926891845131338548053540864},undefined},{{riak_search_vnode,205523667749658222872393179600727299639115513856},undefined},{{riak_search_vnode,216941649291305901920859467356323260730177486848},undefined},{{riak_search_vnode,228359630832953580969325755111919221821239459840},undefined},{{riak_search_vnode,239777612374601260017792042867515182912301432832},undefined},{{riak_search_vnode,251195593916248939066258330623111144003363405824},undefined},{{riak_search_vnode,262613575457896618114724618378707105094425378816},undefined},{{riak_search_vnode,274031556999544297163190906134303066185487351808},undefined},{{riak_search_vnode,285449538541191976211657193889899027276549324800},undefined},{{riak_search_vnode,296867520082839655260123481645494988367611297792},undefined},{{riak_search_vnode,308285501624487334308589769401090949458673270784},undefined},{{riak_search_vnode,319703483166135013357056057156686910549735243776},undefined},{{riak_search_vnode,331121464707782692405522344912282871640797216768},undefined},{{riak_search_vnode,342539446249430371453988632667878832731859189760},undefined},{{riak_search_vnode,353957427791078050502454920423474793822921162752},undefined},{{riak_search_vnode,365375409332725729550921208179070754913983135744},undefined},{{riak_search_vnode,376793390874373408599387495934666716005045108736},undefined},{{riak_search_vnode,388211372416021087647853783690262677096107081728},undefined},{{riak_search_vnode,399629353957668766696320071445858638187169054720},undefined},{{riak_search_vnode,411047335499316445744786359201454599278231027712},undefined},{{riak_search_vnode,422465317040964124793252646957050560369293000704},undefined},{{riak_search_vnode,433883298582611803841718934712646521460354973696},undefined},{{riak_search_vnode,445301280124259482890185222468242482551416946688},undefined},{{riak_search_vnode,456719261665907161938651510223838443642478919680},undefined},{{riak_search_vnode,468137243207554840987117797979434404733540892672},undefined},{{riak_search_vnode,479555224749202520035584085735030365824602865664},undefined},{{riak_search_vnode,490973206290850199084050373490626326915664838656},undefined},{{riak_search_vnode,502391187832497878132516661246222288006726811648},undefined},{{riak_search_vnode,513809169374145557180982949001818249097788784640},undefined},{{riak_search_vnode,525227150915793236229449236757414210188850757632},undefined},{{riak_search_vnode,536645132457440915277915524513010171279912730624},undefined},{{riak_search_vnode,548063113999088594326381812268606132370974703616},undefined},{{riak_search_vnode,559481095540736273374848100024202093462036676608},undefined},{{riak_search_vnode,570899077082383952423314387779798054553098649600},undefined},{{riak_search_vnode,582317058624031631471780675535394015644160622592},undefined},{{riak_search_vnode,593735040165679310520246963290989976735222595584},undefined},{{riak_search_vnode,605153021707326989568713251046585937826284568576},undefined},{{riak_search_vnode,616571003248974668617179538802181898917346541568},undefined},{{riak_search_vnode,627988984790622347665645826557777860008408514560},undefined},{{riak_search_vnode,639406966332270026714112114313373821099470487552},undefined},{{riak_search_vnode,650824947873917705762578402068969782190532460544},undefined},{{riak_search_vnode,662242929415565384811044689824565743281594433536},undefined},{{riak_search_vnode,673660910957213063859510977580161704372656406528},undefined},{{riak_search_vnode,685078892498860742907977265335757665463718379520},undefined},{{riak_search_vnode,696496874040508421956443553091353626554780352512},undefined},{{riak_search_vnode,707914855582156101004909840846949587645842325504},undefined},{{riak_search_vnode,719332837123803780053376128602545548736904298496},undefined},{{riak_search_vnode,730750818665451459101842416358141509827966271488},undefined},{{riak_search_vnode,742168800207099138150308704113737470919028244480},undefined},{{riak_search_vnode,753586781748746817198774991869333432010090217472},undefined},{{riak_search_vnode,765004763290394496247241279624929393101152190464},undefined},{{riak_search_vnode,776422744832042175295707567380525354192214163456},undefined},{{riak_search_vnode,787840726373689854344173855136121315283276136448},undefined},{{riak_search_vnode,799258707915337533392640142891717276374338109440},undefined},{{riak_search_vnode,810676689456985212441106430647313237465400082432},undefined},{{riak_search_vnode,822094670998632891489572718402909198556462055424},undefined},{{riak_search_vnode,833512652540280570538039006158505159647524028416},undefined},{{riak_search_vnode,844930634081928249586505293914101120738586001408},undefined},{{riak_search_vnode,856348615623575928634971581669697081829647974400},undefined},{{riak_search_vnode,867766597165223607683437869425293042920709947392},undefined},{{riak_search_vnode,879184578706871286731904157180889004011771920384},undefined},{{riak_search_vnode,890602560248518965780370444936484965102833893376},undefined},{{riak_search_vnode,902020541790166644828836732692080926193895866368},undefined},{{riak_search_vnode,913438523331814323877303020447676887284957839360},undefined},{{riak_search_vnode,924856504873462002925769308203272848376019812352},undefined},{{riak_search_vnode,936274486415109681974235595958868809467081785344},undefined},{{riak_search_vnode,947692467956757361022701883714464770558143758336},undefined},{{riak_search_vnode,959110449498405040071168171470060731649205731328},undefined},{{riak_search_vnode,970528431040052719119634459225656692740267704320},undefined},{{riak_search_vnode,981946412581700398168100746981252653831329677312},undefined},{{riak_search_vnode,993364394123348077216567034736848614922391650304},undefined},{{riak_search_vnode,1004782375664995756265033322492444576013453623296},undefined},{{riak_search_vnode,1016200357206643435313499610248040537104515596288},undefined},{{riak_search_vnode,1027618338748291114361965898003636498195577569280},undefined},{{riak_search_vnode,1039036320289938793410432185759232459286639542272},undefined},{{riak_search_vnode,1050454301831586472458898473514828420377701515264},undefined},{{riak_search_vnode,1061872283373234151507364761270424381468763488256},undefined},{{riak_search_vnode,1073290264914881830555831049026020342559825461248},undefined},{{riak_search_vnode,1084708246456529509604297336781616303650887434240},undefined},{{riak_search_vnode,1096126227998177188652763624537212264741949407232},undefined},{{riak_search_vnode,1107544209539824867701229912292808225833011380224},undefined},{{riak_search_vnode,1118962191081472546749696200048404186924073353216},undefined},{{riak_search_vnode,1130380172623120225798162487804000148015135326208},undefined},{{riak_search_vnode,1141798154164767904846628775559596109106197299200},undefined},{{riak_search_vnode,1153216135706415583895095063315192070197259272192},undefined},{{riak_search_vnode,1164634117248063262943561351070788031288321245184},undefined},{{riak_search_vnode,1176052098789710941992027638826383992379383218176},undefined},{{riak_search_vnode,1187470080331358621040493926581979953470445191168},undefined},{{riak_search_vnode,1198888061873006300088960214337575914561507164160},undefined},{{riak_search_vnode,1210306043414653979137426502093171875652569137152},undefined},{{riak_search_vnode,1221724024956301658185892789848767836743631110144},undefined},{{riak_search_vnode,1233142006497949337234359077604363797834693083136},undefined},{{riak_search_vnode,1244559988039597016282825365359959758925755056128},undefined},{{riak_search_vnode,1255977969581244695331291653115555720016817029120},undefined},{{riak_search_vnode,1267395951122892374379757940871151681107879002112},undefined},{{riak_search_vnode,1278813932664540053428224228626747642198940975104},undefined},{{riak_search_vnode,1290231914206187732476690516382343603290002948096},undefined},{{riak_search_vnode,1301649895747835411525156804137939564381064921088},undefined},{{riak_search_vnode,1313067877289483090573623091893535525472126894080},undefined},{{riak_search_vnode,1324485858831130769622089379649131486563188867072},undefined},{{riak_search_vnode,1335903840372778448670555667404727447654250840064},undefined},{{riak_search_vnode,1347321821914426127719021955160323408745312813056},undefined},{{riak_search_vnode,1358739803456073806767488242915919369836374786048},undefined},{{riak_search_vnode,1370157784997721485815954530671515330927436759040},undefined},{{riak_search_vnode,1381575766539369164864420818427111292018498732032},undefined},{{riak_search_vnode,1392993748081016843912887106182707253109560705024},undefined},{{riak_search_vnode,1404411729622664522961353393938303214200622678016},undefined},{{riak_search_vnode,1415829711164312202009819681693899175291684651008},undefined},{{riak_search_vnode,1427247692705959881058285969449495136382746624000},undefined},{{riak_search_vnode,1438665674247607560106752257205091097473808596992},undefined},{{riak_search_vnode,1450083655789255239155218544960687058564870569984},undefined}],[]}
** Reason for termination ==
**
{{badmatch,{error,{{badmatch,{error,emfile}},[{riak_kv_vnode,init,1},{riak_core_vnode,init,1},{gen_fsm,init_it,6},{proc_lib,init_p_do_apply,3}]}}},[{riak_core_vnode_manager,get_vnode,3},{riak_core_vnode_manager,handle_call,3},{gen_server,handle_msg,5},{proc_lib,init_p_do_apply,3}]}
2012-05-18 04:59:00 =CRASH REPORT====
  crasher:
    initial call: riak_core_vnode_manager:init/1
    pid: <0.18742.398>
    registered_name: riak_core_vnode_manager
    exception exit:
{{badmatch,{error,{{badmatch,{error,emfile}},[{riak_kv_vnode,init,1},{riak_core_vnode,init,1},{gen_fsm,init_it,6},{proc_lib,init_p_do_apply,3}]}}},[{riak_core_vnode_manager,get_vnode,3},{riak_core_vnode_manager,handle_call,3},{gen_server,handle_msg,5},{proc_lib,init_p_do_apply,3}]}
      in function  gen_server:terminate/6
      in call from proc_lib:init_p_do_apply/3
    ancestors: [riak_core_sup,<0.91.0>]
    messages: []
    links: [<0.92.0>]
    dictionary: []
    trap_exit: false
    status: running
    heap_size: 17711
    stack_size: 24
    reductions: 546975
  neighbours:
2012-05-18 04:59:00 =SUPERVISOR REPORT====
     Supervisor: {local,riak_core_sup}
     Context:    child_terminated
     Reason:
{{badmatch,{error,{{badmatch,{error,emfile}},[{riak_kv_vnode,init,1},{riak_core_vnode,init,1},{gen_fsm,init_it,6},{proc_lib,init_p_do_apply,3}]}}},[{riak_core_vnode_manager,get_vnode,3},{riak_core_vnode_manager,handle_call,3},{gen_server,handle_msg,5},{proc_lib,init_p_do_apply,3}]}
     Offender:
[{pid,<0.18742.398>},{name,riak_core_vnode_manager},{mfargs,{riak_core_vnode_manager,start_link,[]}},{restart_type,permanent},{shutdown,5000},{child_type,worker}]

2012-05-18 04:59:00 =CRASH REPORT====
  crasher:
    initial call: riak_core_vnode_proxy:init/1
    pid: <0.21306.398>
    registered_name:
proxy_riak_kv_vnode_137015778499772148581595453067151533092743675904
    exception exit:
{{{badmatch,{error,{{badmatch,{error,emfile}},[{riak_kv_vnode,init,1},{riak_core_vnode,init,1},{gen_fsm,init_it,6},{proc_lib,init_p_do_apply,3}]}}},[{riak_core_vnode_manager,get_vnode,3},{riak_core_vnode_manager,handle_call,3},{gen_server,handle_msg,5},{proc_lib,init_p_do_apply,3}]},{gen_server,call,[riak_core_vnode_manager,{137015778499772148581595453067151533092743675904,riak_kv_vnode,get_vnode},infinity]}}
      in function  gen_server:call/3
      in call from riak_core_vnode_proxy:get_vnode_pid/1
      in call from riak_core_vnode_proxy:handle_proxy/2
      in call from riak_core_vnode_proxy:loop/2
      in call from proc_lib:init_p_do_apply/3
    ancestors: [riak_core_vnode_proxy_sup,riak_core_sup,<0.91.0>]
    messages: []
    links: [<0.20960.398>]
    dictionary: []
    trap_exit: false
    status: running
    heap_size: 377
    stack_size: 24
    reductions: 89
  neighbours:

*Node 2:*

2012-05-18 04:49:22 =ERROR REPORT====
webmachine error: path="/buckets/contacts/index/email_bin/
event at panchabuta.com"
{error,{error,{case_clause,{error,timeout,[[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[<<"N8XpooDmyVDCLPqGiAhjDVsXNqC">>],[],[],[],[],[],[]]}},[{riak_kv_wm_index,produce_index_results,2},{webmachine_resource,resource_call,3},{webmachine_resource,do,3},{webmachine_decision_core,resource_call,1},{webmachine_decision_core,decision,1},{webmachine_decision_core,handle_request,2},{webmachine_mochiweb,loop,1},{mochiweb_http,parse_headers,5}]}}
2012-05-18 04:49:34 =ERROR REPORT====
** Node 'riak at 10.38.24.196' not responding **
** Removing (timedout) connection **
2012-05-18 04:49:34 =SUPERVISOR REPORT====
     Supervisor: {local,riak_kv_put_fsm_sup}
     Context:    child_terminated
     Reason:     {{nodedown,'riak at 10.38.24.196
'},{gen_server,call,[{riak_search_vnode_master,'riak at 10.38.24.196
'},{riak_vnode_req_v1,1004782375664995756265033322492444576013453623296,{server,undefined,undefined},{delete_v1,[{<<"contacts">>,<<"page_keys">>,<<"njxqbdh1xgelrafoxahmlxjsuy7">>,<<"KDad4R2BVEdZXf0msX5OoTCr3lq">>,[{p,[0]}],1337330961353632}]}},infinity]}}
     Offender:
[{pid,<0.2228.397>},{name,undefined},{mfargs,{riak_kv_put_fsm,start_link,undefined}},{restart_type,temporary},{shutdown,5000},{child_type,worker}]

2012-05-18 04:49:41 =ERROR REPORT====
webmachine error: path="/buckets/contacts/index/email_bin/
inspector at bchouseinspections.ca"
{error,{error,{case_clause,{error,timeout,[[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]]}},[{riak_kv_wm_index,produce_index_results,2},{webmachine_resource,resource_call,3},{webmachine_resource,do,3},{webmachine_decision_core,resource_call,1},{webmachine_decision_core,decision,1},{webmachine_decision_core,handle_request,2},{webmachine_mochiweb,loop,1},{mochiweb_http,parse_headers,5}]}}
2012-05-18 04:50:23 =ERROR REPORT====
webmachine error: path="/buckets/contacts/index/email_bin/
event at panchabuta.com"
{error,{error,{case_clause,{error,timeout,[[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[<<"N8XpooDmyVDCLPqGiAhjDVsXNqC">>],[],[],[]]}},[{riak_kv_wm_index,produce_index_results,2},{webmachine_resource,resource_call,3},{webmachine_resource,do,3},{webmachine_decision_core,resource_call,1},{webmachine_decision_core,decision,1},{webmachine_decision_core,handle_request,2},{webmachine_mochiweb,loop,1},{mochiweb_http,parse_headers,5}]}}
2012-05-18 04:50:49 =ERROR REPORT====
** Node 'riak at 10.38.24.196' not responding **
** Removing (timedout) connection **
2012-05-18 04:52:49 =ERROR REPORT====
** Node 'riak at 10.38.24.196' not responding **
** Removing (timedout) connection **
2012-05-18 04:52:49 =SUPERVISOR REPORT====
     Supervisor: {local,riak_kv_put_fsm_sup}
     Context:    child_terminated
     Reason:     {{nodedown,'riak at 10.38.24.196
'},{gen_server,call,[{riak_search_vnode_master,'riak at 10.38.24.196
'},{riak_vnode_req_v1,1050454301831586472458898473514828420377701515264,{server,undefined,undefined},{delete_v1,[{<<"sites">>,<<"updated_at">>,<<"2012-05-18">>,<<"fa3FeLb9UYhf0K1VdmR7CMdGCV">>,[{p,[0]}],1337331089082754}]}},infinity]}}
     Offender:
[{pid,<0.3077.397>},{name,undefined},{mfargs,{riak_kv_put_fsm,start_link,undefined}},{restart_type,temporary},{shutdown,5000},{child_type,worker}]

2012-05-18 04:52:49 =SUPERVISOR REPORT====
     Supervisor: {local,riak_kv_put_fsm_sup}
     Context:    child_terminated
     Reason:     {{nodedown,'riak at 10.38.24.196
'},{gen_server,call,[{riak_search_vnode_master,'riak at 10.38.24.196
'},{riak_vnode_req_v1,45671926166590716193865151022383844364247891968,{server,undefined,undefined},{delete_v1,[{<<"sites">>,<<"page_keys">>,<<"m9ksizkajltxgqeck2dix0y6vzs">>,<<"5Cd4M1awqLwGsRKJi2QkVB79h3g">>,[{p,[23]}],1337331105707883}]}},infinity]}}
     Offender:
[{pid,<0.3102.397>},{name,undefined},{mfargs,{riak_kv_put_fsm,start_link,undefined}},{restart_type,temporary},{shutdown,5000},{child_type,worker}]

2012-05-18 04:52:49 =SUPERVISOR REPORT====
     Supervisor: {local,riak_kv_put_fsm_sup}
     Context:    child_terminated
     Reason:     {{nodedown,'riak at 10.38.24.196
'},{gen_server,call,[{riak_search_vnode_master,'riak at 10.38.24.196
'},{riak_vnode_req_v1,1084708246456529509604297336781616303650887434240,{server,undefined,undefined},{index_v1,[{<<"sites">>,<<"num_contacts">>,<<"0000000000">>,<<"Z6bUs5DmmdRggHv1XjXZe30a1Zz">>,[{p,[0]}],1337331091385105}]}},infinity]}}
     Offender:
[{pid,<0.3083.397>},{name,undefined},{mfargs,{riak_kv_put_fsm,start_link,undefined}},{restart_type,temporary},{shutdown,5000},{child_type,worker}]

....

2012-05-18 04:59:01 =ERROR REPORT====
** Generic server riak_core_vnode_manager terminating
** Last message in was
{947692467956757361022701883714464770558143758336,riak_kv_vnode,get_vnode}
** When Server state ==
{state,40799215793,[{{riak_kv_vnode,0},undefined},{{riak_kv_vnode,11417981541647679048466287755595961091061972992},undefined},{{riak_kv_vnode,22835963083295358096932575511191922182123945984},undefined},{{riak_kv_vnode,34253944624943037145398863266787883273185918976},undefined},{{riak_kv_vnode,45671926166590716193865151022383844364247891968},undefined},{{riak_kv_vnode,57089907708238395242331438777979805455309864960},undefined},{{riak_kv_vnode,68507889249886074290797726533575766546371837952},undefined},{{riak_kv_vnode,79925870791533753339264014289171727637433810944},undefined},{{riak_kv_vnode,91343852333181432387730302044767688728495783936},undefined},{{riak_kv_vnode,102761833874829111436196589800363649819557756928},undefined},{{riak_kv_vnode,114179815416476790484662877555959610910619729920},undefined},{{riak_kv_vnode,125597796958124469533129165311555572001681702912},undefined},{{riak_kv_vnode,137015778499772148581595453067151533092743675904},undefined},{{riak_kv_vnode,148433760041419827630061740822747494183805648896},undefined},{{riak_kv_vnode,159851741583067506678528028578343455274867621888},undefined},{{riak_kv_vnode,171269723124715185726994316333939416365929594880},undefined},{{riak_kv_vnode,182687704666362864775460604089535377456991567872},undefined},{{riak_kv_vnode,194105686208010543823926891845131338548053540864},undefined},{{riak_kv_vnode,205523667749658222872393179600727299639115513856},undefined},{{riak_kv_vnode,216941649291305901920859467356323260730177486848},undefined},{{riak_kv_vnode,228359630832953580969325755111919221821239459840},undefined},{{riak_kv_vnode,239777612374601260017792042867515182912301432832},undefined},{{riak_kv_vnode,251195593916248939066258330623111144003363405824},undefined},{{riak_kv_vnode,262613575457896618114724618378707105094425378816},undefined},{{riak_kv_vnode,274031556999544297163190906134303066185487351808},undefined},{{riak_kv_vnode,285449538541191976211657193889899027276549324800},undefined},{{riak_kv_vnode,296867520082839655260123481645494988367611297792},undefined},{{riak_kv_vnode,308285501624487334308589769401090949458673270784},undefined},{{riak_kv_vnode,319703483166135013357056057156686910549735243776},undefined},{{riak_kv_vnode,331121464707782692405522344912282871640797216768},undefined},{{riak_kv_vnode,342539446249430371453988632667878832731859189760},undefined},{{riak_kv_vnode,353957427791078050502454920423474793822921162752},undefined},{{riak_kv_vnode,365375409332725729550921208179070754913983135744},undefined},{{riak_kv_vnode,376793390874373408599387495934666716005045108736},undefined},{{riak_kv_vnode,388211372416021087647853783690262677096107081728},undefined},{{riak_kv_vnode,399629353957668766696320071445858638187169054720},undefined},{{riak_kv_vnode,411047335499316445744786359201454599278231027712},undefined},{{riak_kv_vnode,422465317040964124793252646957050560369293000704},undefined},{{riak_kv_vnode,433883298582611803841718934712646521460354973696},undefined},{{riak_kv_vnode,445301280124259482890185222468242482551416946688},undefined},{{riak_kv_vnode,456719261665907161938651510223838443642478919680},undefined},{{riak_kv_vnode,468137243207554840987117797979434404733540892672},undefined},{{riak_kv_vnode,479555224749202520035584085735030365824602865664},undefined},{{riak_kv_vnode,490973206290850199084050373490626326915664838656},undefined},{{riak_kv_vnode,502391187832497878132516661246222288006726811648},undefined},{{riak_kv_vnode,513809169374145557180982949001818249097788784640},undefined},{{riak_kv_vnode,525227150915793236229449236757414210188850757632},undefined},{{riak_kv_vnode,536645132457440915277915524513010171279912730624},undefined},{{riak_kv_vnode,548063113999088594326381812268606132370974703616},undefined},{{riak_kv_vnode,559481095540736273374848100024202093462036676608},undefined},{{riak_kv_vnode,570899077082383952423314387779798054553098649600},undefined},{{riak_kv_vnode,582317058624031631471780675535394015644160622592},undefined},{{riak_kv_vnode,593735040165679310520246963290989976735222595584},undefined},{{riak_kv_vnode,605153021707326989568713251046585937826284568576},undefined},{{riak_kv_vnode,616571003248974668617179538802181898917346541568},undefined},{{riak_kv_vnode,627988984790622347665645826557777860008408514560},undefined},{{riak_kv_vnode,639406966332270026714112114313373821099470487552},undefined},{{riak_kv_vnode,650824947873917705762578402068969782190532460544},undefined},{{riak_kv_vnode,662242929415565384811044689824565743281594433536},undefined},{{riak_kv_vnode,673660910957213063859510977580161704372656406528},undefined},{{riak_kv_vnode,685078892498860742907977265335757665463718379520},undefined},{{riak_kv_vnode,696496874040508421956443553091353626554780352512},undefined},{{riak_kv_vnode,707914855582156101004909840846949587645842325504},undefined},{{riak_kv_vnode,719332837123803780053376128602545548736904298496},undefined},{{riak_kv_vnode,730750818665451459101842416358141509827966271488},undefined},{{riak_kv_vnode,742168800207099138150308704113737470919028244480},undefined},{{riak_kv_vnode,753586781748746817198774991869333432010090217472},undefined},{{riak_kv_vnode,765004763290394496247241279624929393101152190464},undefined},{{riak_kv_vnode,776422744832042175295707567380525354192214163456},undefined},{{riak_kv_vnode,787840726373689854344173855136121315283276136448},undefined},{{riak_kv_vnode,799258707915337533392640142891717276374338109440},undefined},{{riak_kv_vnode,810676689456985212441106430647313237465400082432},undefined},{{riak_kv_vnode,822094670998632891489572718402909198556462055424},undefined},{{riak_kv_vnode,833512652540280570538039006158505159647524028416},undefined},{{riak_kv_vnode,844930634081928249586505293914101120738586001408},undefined},{{riak_kv_vnode,856348615623575928634971581669697081829647974400},undefined},{{riak_kv_vnode,867766597165223607683437869425293042920709947392},undefined},{{riak_kv_vnode,879184578706871286731904157180889004011771920384},undefined},{{riak_kv_vnode,890602560248518965780370444936484965102833893376},undefined},{{riak_kv_vnode,902020541790166644828836732692080926193895866368},undefined},{{riak_kv_vnode,913438523331814323877303020447676887284957839360},undefined},{{riak_kv_vnode,924856504873462002925769308203272848376019812352},undefined},{{riak_kv_vnode,936274486415109681974235595958868809467081785344},undefined},{{riak_kv_vnode,947692467956757361022701883714464770558143758336},undefined},{{riak_kv_vnode,959110449498405040071168171470060731649205731328},undefined},{{riak_kv_vnode,970528431040052719119634459225656692740267704320},undefined},{{riak_kv_vnode,981946412581700398168100746981252653831329677312},undefined},{{riak_kv_vnode,993364394123348077216567034736848614922391650304},undefined},{{riak_kv_vnode,1004782375664995756265033322492444576013453623296},undefined},{{riak_kv_vnode,1016200357206643435313499610248040537104515596288},undefined},{{riak_kv_vnode,1027618338748291114361965898003636498195577569280},undefined},{{riak_kv_vnode,1039036320289938793410432185759232459286639542272},undefined},{{riak_kv_vnode,1050454301831586472458898473514828420377701515264},undefined},{{riak_kv_vnode,1061872283373234151507364761270424381468763488256},undefined},{{riak_kv_vnode,1073290264914881830555831049026020342559825461248},undefined},{{riak_kv_vnode,1084708246456529509604297336781616303650887434240},undefined},{{riak_kv_vnode,1096126227998177188652763624537212264741949407232},undefined},{{riak_kv_vnode,1107544209539824867701229912292808225833011380224},undefined},{{riak_kv_vnode,1118962191081472546749696200048404186924073353216},undefined},{{riak_kv_vnode,1130380172623120225798162487804000148015135326208},undefined},{{riak_kv_vnode,1141798154164767904846628775559596109106197299200},undefined},{{riak_kv_vnode,1153216135706415583895095063315192070197259272192},undefined},{{riak_kv_vnode,1164634117248063262943561351070788031288321245184},undefined},{{riak_kv_vnode,1176052098789710941992027638826383992379383218176},undefined},{{riak_kv_vnode,1187470080331358621040493926581979953470445191168},undefined},{{riak_kv_vnode,1198888061873006300088960214337575914561507164160},undefined},{{riak_kv_vnode,1210306043414653979137426502093171875652569137152},undefined},{{riak_kv_vnode,1221724024956301658185892789848767836743631110144},undefined},{{riak_kv_vnode,1233142006497949337234359077604363797834693083136},undefined},{{riak_kv_vnode,1244559988039597016282825365359959758925755056128},undefined},{{riak_kv_vnode,1255977969581244695331291653115555720016817029120},undefined},{{riak_kv_vnode,1267395951122892374379757940871151681107879002112},undefined},{{riak_kv_vnode,1278813932664540053428224228626747642198940975104},undefined},{{riak_kv_vnode,1290231914206187732476690516382343603290002948096},undefined},{{riak_kv_vnode,1301649895747835411525156804137939564381064921088},undefined},{{riak_kv_vnode,1313067877289483090573623091893535525472126894080},undefined},{{riak_kv_vnode,1324485858831130769622089379649131486563188867072},undefined},{{riak_kv_vnode,1335903840372778448670555667404727447654250840064},undefined},{{riak_kv_vnode,1347321821914426127719021955160323408745312813056},undefined},{{riak_kv_vnode,1358739803456073806767488242915919369836374786048},undefined},{{riak_kv_vnode,1370157784997721485815954530671515330927436759040},undefined},{{riak_kv_vnode,1381575766539369164864420818427111292018498732032},undefined},{{riak_kv_vnode,1392993748081016843912887106182707253109560705024},undefined},{{riak_kv_vnode,1404411729622664522961353393938303214200622678016},undefined},{{riak_kv_vnode,1415829711164312202009819681693899175291684651008},undefined},{{riak_kv_vnode,1427247692705959881058285969449495136382746624000},undefined},{{riak_kv_vnode,1438665674247607560106752257205091097473808596992},undefined},{{riak_kv_vnode,1450083655789255239155218544960687058564870569984},undefined},{{riak_pipe_vnode,0},undefined},{{riak_pipe_vnode,11417981541647679048466287755595961091061972992},undefined},{{riak_pipe_vnode,22835963083295358096932575511191922182123945984},undefined},{{riak_pipe_vnode,34253944624943037145398863266787883273185918976},undefined},{{riak_pipe_vnode,45671926166590716193865151022383844364247891968},undefined},{{riak_pipe_vnode,57089907708238395242331438777979805455309864960},undefined},{{riak_pipe_vnode,68507889249886074290797726533575766546371837952},undefined},{{riak_pipe_vnode,79925870791533753339264014289171727637433810944},undefined},{{riak_pipe_vnode,91343852333181432387730302044767688728495783936},undefined},{{riak_pipe_vnode,102761833874829111436196589800363649819557756928},undefined},{{riak_pipe_vnode,114179815416476790484662877555959610910619729920},undefined},{{riak_pipe_vnode,125597796958124469533129165311555572001681702912},undefined},{{riak_pipe_vnode,137015778499772148581595453067151533092743675904},undefined},{{riak_pipe_vnode,148433760041419827630061740822747494183805648896},undefined},{{riak_pipe_vnode,159851741583067506678528028578343455274867621888},undefined},{{riak_pipe_vnode,171269723124715185726994316333939416365929594880},undefined},{{riak_pipe_vnode,182687704666362864775460604089535377456991567872},undefined},{{riak_pipe_vnode,194105686208010543823926891845131338548053540864},undefined},{{riak_pipe_vnode,205523667749658222872393179600727299639115513856},undefined},{{riak_pipe_vnode,216941649291305901920859467356323260730177486848},undefined},{{riak_pipe_vnode,228359630832953580969325755111919221821239459840},undefined},{{riak_pipe_vnode,239777612374601260017792042867515182912301432832},undefined},{{riak_pipe_vnode,251195593916248939066258330623111144003363405824},undefined},{{riak_pipe_vnode,262613575457896618114724618378707105094425378816},undefined},{{riak_pipe_vnode,274031556999544297163190906134303066185487351808},undefined},{{riak_pipe_vnode,285449538541191976211657193889899027276549324800},undefined},{{riak_pipe_vnode,296867520082839655260123481645494988367611297792},undefined},{{riak_pipe_vnode,308285501624487334308589769401090949458673270784},undefined},{{riak_pipe_vnode,319703483166135013357056057156686910549735243776},undefined},{{riak_pipe_vnode,331121464707782692405522344912282871640797216768},undefined},{{riak_pipe_vnode,342539446249430371453988632667878832731859189760},undefined},{{riak_pipe_vnode,353957427791078050502454920423474793822921162752},undefined},{{riak_pipe_vnode,365375409332725729550921208179070754913983135744},undefined},{{riak_pipe_vnode,376793390874373408599387495934666716005045108736},undefined},{{riak_pipe_vnode,388211372416021087647853783690262677096107081728},undefined},{{riak_pipe_vnode,399629353957668766696320071445858638187169054720},undefined},{{riak_pipe_vnode,411047335499316445744786359201454599278231027712},undefined},{{riak_pipe_vnode,422465317040964124793252646957050560369293000704},undefined},{{riak_pipe_vnode,433883298582611803841718934712646521460354973696},undefined},{{riak_pipe_vnode,445301280124259482890185222468242482551416946688},undefined},{{riak_pipe_vnode,456719261665907161938651510223838443642478919680},undefined},{{riak_pipe_vnode,468137243207554840987117797979434404733540892672},undefined},{{riak_pipe_vnode,479555224749202520035584085735030365824602865664},undefined},{{riak_pipe_vnode,490973206290850199084050373490626326915664838656},undefined},{{riak_pipe_vnode,502391187832497878132516661246222288006726811648},undefined},{{riak_pipe_vnode,513809169374145557180982949001818249097788784640},undefined},{{riak_pipe_vnode,525227150915793236229449236757414210188850757632},undefined},{{riak_pipe_vnode,536645132457440915277915524513010171279912730624},undefined},{{riak_pipe_vnode,548063113999088594326381812268606132370974703616},undefined},{{riak_pipe_vnode,559481095540736273374848100024202093462036676608},undefined},{{riak_pipe_vnode,570899077082383952423314387779798054553098649600},undefined},{{riak_pipe_vnode,582317058624031631471780675535394015644160622592},undefined},{{riak_pipe_vnode,593735040165679310520246963290989976735222595584},undefined},{{riak_pipe_vnode,605153021707326989568713251046585937826284568576},undefined},{{riak_pipe_vnode,616571003248974668617179538802181898917346541568},undefined},{{riak_pipe_vnode,627988984790622347665645826557777860008408514560},undefined},{{riak_pipe_vnode,639406966332270026714112114313373821099470487552},undefined},{{riak_pipe_vnode,650824947873917705762578402068969782190532460544},undefined},{{riak_pipe_vnode,662242929415565384811044689824565743281594433536},undefined},{{riak_pipe_vnode,673660910957213063859510977580161704372656406528},undefined},{{riak_pipe_vnode,685078892498860742907977265335757665463718379520},undefined},{{riak_pipe_vnode,696496874040508421956443553091353626554780352512},undefined},{{riak_pipe_vnode,707914855582156101004909840846949587645842325504},undefined},{{riak_pipe_vnode,719332837123803780053376128602545548736904298496},undefined},{{riak_pipe_vnode,730750818665451459101842416358141509827966271488},undefined},{{riak_pipe_vnode,742168800207099138150308704113737470919028244480},undefined},{{riak_pipe_vnode,753586781748746817198774991869333432010090217472},undefined},{{riak_pipe_vnode,765004763290394496247241279624929393101152190464},undefined},{{riak_pipe_vnode,776422744832042175295707567380525354192214163456},undefined},{{riak_pipe_vnode,787840726373689854344173855136121315283276136448},undefined},{{riak_pipe_vnode,799258707915337533392640142891717276374338109440},undefined},{{riak_pipe_vnode,810676689456985212441106430647313237465400082432},undefined},{{riak_pipe_vnode,822094670998632891489572718402909198556462055424},undefined},{{riak_pipe_vnode,833512652540280570538039006158505159647524028416},undefined},{{riak_pipe_vnode,844930634081928249586505293914101120738586001408},undefined},{{riak_pipe_vnode,856348615623575928634971581669697081829647974400},undefined},{{riak_pipe_vnode,867766597165223607683437869425293042920709947392},undefined},{{riak_pipe_vnode,879184578706871286731904157180889004011771920384},undefined},{{riak_pipe_vnode,890602560248518965780370444936484965102833893376},undefined},{{riak_pipe_vnode,902020541790166644828836732692080926193895866368},undefined},{{riak_pipe_vnode,913438523331814323877303020447676887284957839360},undefined},{{riak_pipe_vnode,924856504873462002925769308203272848376019812352},undefined},{{riak_pipe_vnode,936274486415109681974235595958868809467081785344},undefined},{{riak_pipe_vnode,947692467956757361022701883714464770558143758336},undefined},{{riak_pipe_vnode,959110449498405040071168171470060731649205731328},undefined},{{riak_pipe_vnode,970528431040052719119634459225656692740267704320},undefined},{{riak_pipe_vnode,981946412581700398168100746981252653831329677312},undefined},{{riak_pipe_vnode,993364394123348077216567034736848614922391650304},undefined},{{riak_pipe_vnode,1004782375664995756265033322492444576013453623296},undefined},{{riak_pipe_vnode,1016200357206643435313499610248040537104515596288},undefined},{{riak_pipe_vnode,1027618338748291114361965898003636498195577569280},undefined},{{riak_pipe_vnode,1039036320289938793410432185759232459286639542272},undefined},{{riak_pipe_vnode,1050454301831586472458898473514828420377701515264},undefined},{{riak_pipe_vnode,1061872283373234151507364761270424381468763488256},undefined},{{riak_pipe_vnode,1073290264914881830555831049026020342559825461248},undefined},{{riak_pipe_vnode,1084708246456529509604297336781616303650887434240},undefined},{{riak_pipe_vnode,1096126227998177188652763624537212264741949407232},undefined},{{riak_pipe_vnode,1107544209539824867701229912292808225833011380224},undefined},{{riak_pipe_vnode,1118962191081472546749696200048404186924073353216},undefined},{{riak_pipe_vnode,1130380172623120225798162487804000148015135326208},undefined},{{riak_pipe_vnode,1141798154164767904846628775559596109106197299200},undefined},{{riak_pipe_vnode,1153216135706415583895095063315192070197259272192},undefined},{{riak_pipe_vnode,1164634117248063262943561351070788031288321245184},undefined},{{riak_pipe_vnode,1176052098789710941992027638826383992379383218176},undefined},{{riak_pipe_vnode,1187470080331358621040493926581979953470445191168},undefined},{{riak_pipe_vnode,1198888061873006300088960214337575914561507164160},undefined},{{riak_pipe_vnode,1210306043414653979137426502093171875652569137152},undefined},{{riak_pipe_vnode,1221724024956301658185892789848767836743631110144},undefined},{{riak_pipe_vnode,1233142006497949337234359077604363797834693083136},undefined},{{riak_pipe_vnode,1244559988039597016282825365359959758925755056128},undefined},{{riak_pipe_vnode,1255977969581244695331291653115555720016817029120},undefined},{{riak_pipe_vnode,1267395951122892374379757940871151681107879002112},undefined},{{riak_pipe_vnode,1278813932664540053428224228626747642198940975104},undefined},{{riak_pipe_vnode,1290231914206187732476690516382343603290002948096},undefined},{{riak_pipe_vnode,1301649895747835411525156804137939564381064921088},undefined},{{riak_pipe_vnode,1313067877289483090573623091893535525472126894080},undefined},{{riak_pipe_vnode,1324485858831130769622089379649131486563188867072},undefined},{{riak_pipe_vnode,1335903840372778448670555667404727447654250840064},undefined},{{riak_pipe_vnode,1347321821914426127719021955160323408745312813056},undefined},{{riak_pipe_vnode,1358739803456073806767488242915919369836374786048},undefined},{{riak_pipe_vnode,1370157784997721485815954530671515330927436759040},undefined},{{riak_pipe_vnode,1381575766539369164864420818427111292018498732032},undefined},{{riak_pipe_vnode,1392993748081016843912887106182707253109560705024},undefined},{{riak_pipe_vnode,1404411729622664522961353393938303214200622678016},undefined},{{riak_pipe_vnode,1415829711164312202009819681693899175291684651008},undefined},{{riak_pipe_vnode,1427247692705959881058285969449495136382746624000},undefined},{{riak_pipe_vnode,1438665674247607560106752257205091097473808596992},undefined},{{riak_pipe_vnode,1450083655789255239155218544960687058564870569984},undefined},{{riak_search_vnode,0},undefined},{{riak_search_vnode,11417981541647679048466287755595961091061972992},undefined},{{riak_search_vnode,22835963083295358096932575511191922182123945984},undefined},{{riak_search_vnode,34253944624943037145398863266787883273185918976},undefined},{{riak_search_vnode,45671926166590716193865151022383844364247891968},undefined},{{riak_search_vnode,57089907708238395242331438777979805455309864960},undefined},{{riak_search_vnode,68507889249886074290797726533575766546371837952},undefined},{{riak_search_vnode,79925870791533753339264014289171727637433810944},undefined},{{riak_search_vnode,91343852333181432387730302044767688728495783936},undefined},{{riak_search_vnode,102761833874829111436196589800363649819557756928},undefined},{{riak_search_vnode,114179815416476790484662877555959610910619729920},undefined},{{riak_search_vnode,125597796958124469533129165311555572001681702912},undefined},{{riak_search_vnode,137015778499772148581595453067151533092743675904},undefined},{{riak_search_vnode,148433760041419827630061740822747494183805648896},undefined},{{riak_search_vnode,159851741583067506678528028578343455274867621888},undefined},{{riak_search_vnode,171269723124715185726994316333939416365929594880},undefined},{{riak_search_vnode,182687704666362864775460604089535377456991567872},undefined},{{riak_search_vnode,194105686208010543823926891845131338548053540864},undefined},{{riak_search_vnode,205523667749658222872393179600727299639115513856},undefined},{{riak_search_vnode,216941649291305901920859467356323260730177486848},undefined},{{riak_search_vnode,228359630832953580969325755111919221821239459840},undefined},{{riak_search_vnode,239777612374601260017792042867515182912301432832},undefined},{{riak_search_vnode,251195593916248939066258330623111144003363405824},undefined},{{riak_search_vnode,262613575457896618114724618378707105094425378816},undefined},{{riak_search_vnode,274031556999544297163190906134303066185487351808},undefined},{{riak_search_vnode,285449538541191976211657193889899027276549324800},undefined},{{riak_search_vnode,296867520082839655260123481645494988367611297792},undefined},{{riak_search_vnode,308285501624487334308589769401090949458673270784},undefined},{{riak_search_vnode,319703483166135013357056057156686910549735243776},undefined},{{riak_search_vnode,331121464707782692405522344912282871640797216768},undefined},{{riak_search_vnode,342539446249430371453988632667878832731859189760},undefined},{{riak_search_vnode,353957427791078050502454920423474793822921162752},undefined},{{riak_search_vnode,365375409332725729550921208179070754913983135744},undefined},{{riak_search_vnode,376793390874373408599387495934666716005045108736},undefined},{{riak_search_vnode,388211372416021087647853783690262677096107081728},undefined},{{riak_search_vnode,399629353957668766696320071445858638187169054720},undefined},{{riak_search_vnode,411047335499316445744786359201454599278231027712},undefined},{{riak_search_vnode,422465317040964124793252646957050560369293000704},undefined},{{riak_search_vnode,433883298582611803841718934712646521460354973696},undefined},{{riak_search_vnode,445301280124259482890185222468242482551416946688},undefined},{{riak_search_vnode,456719261665907161938651510223838443642478919680},undefined},{{riak_search_vnode,468137243207554840987117797979434404733540892672},undefined},{{riak_search_vnode,479555224749202520035584085735030365824602865664},undefined},{{riak_search_vnode,490973206290850199084050373490626326915664838656},undefined},{{riak_search_vnode,502391187832497878132516661246222288006726811648},undefined},{{riak_search_vnode,513809169374145557180982949001818249097788784640},undefined},{{riak_search_vnode,525227150915793236229449236757414210188850757632},undefined},{{riak_search_vnode,536645132457440915277915524513010171279912730624},undefined},{{riak_search_vnode,548063113999088594326381812268606132370974703616},undefined},{{riak_search_vnode,559481095540736273374848100024202093462036676608},undefined},{{riak_search_vnode,570899077082383952423314387779798054553098649600},undefined},{{riak_search_vnode,582317058624031631471780675535394015644160622592},undefined},{{riak_search_vnode,593735040165679310520246963290989976735222595584},undefined},{{riak_search_vnode,605153021707326989568713251046585937826284568576},undefined},{{riak_search_vnode,616571003248974668617179538802181898917346541568},undefined},{{riak_search_vnode,627988984790622347665645826557777860008408514560},undefined},{{riak_search_vnode,639406966332270026714112114313373821099470487552},undefined},{{riak_search_vnode,650824947873917705762578402068969782190532460544},undefined},{{riak_search_vnode,662242929415565384811044689824565743281594433536},undefined},{{riak_search_vnode,673660910957213063859510977580161704372656406528},undefined},{{riak_search_vnode,685078892498860742907977265335757665463718379520},undefined},{{riak_search_vnode,696496874040508421956443553091353626554780352512},undefined},{{riak_search_vnode,707914855582156101004909840846949587645842325504},undefined},{{riak_search_vnode,719332837123803780053376128602545548736904298496},undefined},{{riak_search_vnode,730750818665451459101842416358141509827966271488},undefined},{{riak_search_vnode,742168800207099138150308704113737470919028244480},undefined},{{riak_search_vnode,753586781748746817198774991869333432010090217472},undefined},{{riak_search_vnode,765004763290394496247241279624929393101152190464},undefined},{{riak_search_vnode,776422744832042175295707567380525354192214163456},undefined},{{riak_search_vnode,787840726373689854344173855136121315283276136448},undefined},{{riak_search_vnode,799258707915337533392640142891717276374338109440},undefined},{{riak_search_vnode,810676689456985212441106430647313237465400082432},undefined},{{riak_search_vnode,822094670998632891489572718402909198556462055424},undefined},{{riak_search_vnode,833512652540280570538039006158505159647524028416},undefined},{{riak_search_vnode,844930634081928249586505293914101120738586001408},undefined},{{riak_search_vnode,856348615623575928634971581669697081829647974400},undefined},{{riak_search_vnode,867766597165223607683437869425293042920709947392},undefined},{{riak_search_vnode,879184578706871286731904157180889004011771920384},undefined},{{riak_search_vnode,890602560248518965780370444936484965102833893376},undefined},{{riak_search_vnode,902020541790166644828836732692080926193895866368},undefined},{{riak_search_vnode,913438523331814323877303020447676887284957839360},undefined},{{riak_search_vnode,924856504873462002925769308203272848376019812352},undefined},{{riak_search_vnode,936274486415109681974235595958868809467081785344},undefined},{{riak_search_vnode,947692467956757361022701883714464770558143758336},undefined},{{riak_search_vnode,959110449498405040071168171470060731649205731328},undefined},{{riak_search_vnode,970528431040052719119634459225656692740267704320},undefined},{{riak_search_vnode,981946412581700398168100746981252653831329677312},undefined},{{riak_search_vnode,993364394123348077216567034736848614922391650304},undefined},{{riak_search_vnode,1004782375664995756265033322492444576013453623296},undefined},{{riak_search_vnode,1016200357206643435313499610248040537104515596288},undefined},{{riak_search_vnode,1027618338748291114361965898003636498195577569280},undefined},{{riak_search_vnode,1039036320289938793410432185759232459286639542272},undefined},{{riak_search_vnode,1050454301831586472458898473514828420377701515264},undefined},{{riak_search_vnode,1061872283373234151507364761270424381468763488256},undefined},{{riak_search_vnode,1073290264914881830555831049026020342559825461248},undefined},{{riak_search_vnode,1084708246456529509604297336781616303650887434240},undefined},{{riak_search_vnode,1096126227998177188652763624537212264741949407232},undefined},{{riak_search_vnode,1107544209539824867701229912292808225833011380224},undefined},{{riak_search_vnode,1118962191081472546749696200048404186924073353216},undefined},{{riak_search_vnode,1130380172623120225798162487804000148015135326208},undefined},{{riak_search_vnode,1141798154164767904846628775559596109106197299200},undefined},{{riak_search_vnode,1153216135706415583895095063315192070197259272192},undefined},{{riak_search_vnode,1164634117248063262943561351070788031288321245184},undefined},{{riak_search_vnode,1176052098789710941992027638826383992379383218176},undefined},{{riak_search_vnode,1187470080331358621040493926581979953470445191168},undefined},{{riak_search_vnode,1198888061873006300088960214337575914561507164160},undefined},{{riak_search_vnode,1210306043414653979137426502093171875652569137152},undefined},{{riak_search_vnode,1221724024956301658185892789848767836743631110144},undefined},{{riak_search_vnode,1233142006497949337234359077604363797834693083136},undefined},{{riak_search_vnode,1244559988039597016282825365359959758925755056128},undefined},{{riak_search_vnode,1255977969581244695331291653115555720016817029120},undefined},{{riak_search_vnode,1267395951122892374379757940871151681107879002112},undefined},{{riak_search_vnode,1278813932664540053428224228626747642198940975104},undefined},{{riak_search_vnode,1290231914206187732476690516382343603290002948096},undefined},{{riak_search_vnode,1301649895747835411525156804137939564381064921088},undefined},{{riak_search_vnode,1313067877289483090573623091893535525472126894080},undefined},{{riak_search_vnode,1324485858831130769622089379649131486563188867072},undefined},{{riak_search_vnode,1335903840372778448670555667404727447654250840064},undefined},{{riak_search_vnode,1347321821914426127719021955160323408745312813056},undefined},{{riak_search_vnode,1358739803456073806767488242915919369836374786048},undefined},{{riak_search_vnode,1370157784997721485815954530671515330927436759040},undefined},{{riak_search_vnode,1381575766539369164864420818427111292018498732032},undefined},{{riak_search_vnode,1392993748081016843912887106182707253109560705024},undefined},{{riak_search_vnode,1404411729622664522961353393938303214200622678016},undefined},{{riak_search_vnode,1415829711164312202009819681693899175291684651008},undefined},{{riak_search_vnode,1427247692705959881058285969449495136382746624000},undefined},{{riak_search_vnode,1438665674247607560106752257205091097473808596992},undefined},{{riak_search_vnode,1450083655789255239155218544960687058564870569984},undefined}],[]}
** Reason for termination ==
**
{{badmatch,{error,{{badmatch,{error,emfile}},[{riak_kv_vnode,init,1},{riak_core_vnode,init,1},{gen_fsm,init_it,6},{proc_lib,init_p_do_apply,3}]}}},[{riak_core_vnode_manager,get_vnode,3},{riak_core_vnode_manager,handle_call,3},{gen_server,handle_msg,5},{proc_lib,init_p_do_apply,3}]}
2012-05-18 04:59:01 =CRASH REPORT====
  crasher:
    initial call: riak_core_vnode_manager:init/1
    pid: <0.22403.463>
    registered_name: riak_core_vnode_manager
    exception exit:
{{badmatch,{error,{{badmatch,{error,emfile}},[{riak_kv_vnode,init,1},{riak_core_vnode,init,1},{gen_fsm,init_it,6},{proc_lib,init_p_do_apply,3}]}}},[{riak_core_vnode_manager,get_vnode,3},{riak_core_vnode_manager,handle_call,3},{gen_server,handle_msg,5},{proc_lib,init_p_do_apply,3}]}
      in function  gen_server:terminate/6
      in call from proc_lib:init_p_do_apply/3
    ancestors: [riak_core_sup,<0.91.0>]
    messages: []
    links: [<0.92.0>]
    dictionary: []
    trap_exit: false
    status: running
    heap_size: 17711
    stack_size: 24
    reductions: 170622
  neighbours:
2012-05-18 04:59:01 =SUPERVISOR REPORT====
     Supervisor: {local,riak_core_sup}
     Context:    child_terminated
     Reason:
{{badmatch,{error,{{badmatch,{error,emfile}},[{riak_kv_vnode,init,1},{riak_core_vnode,init,1},{gen_fsm,init_it,6},{proc_lib,init_p_do_apply,3}]}}},[{riak_core_vnode_manager,get_vnode,3},{riak_core_vnode_manager,handle_call,3},{gen_server,handle_msg,5},{proc_lib,init_p_do_apply,3}]}
     Offender:
[{pid,<0.22403.463>},{name,riak_core_vnode_manager},{mfargs,{riak_core_vnode_manager,start_link,[]}},{restart_type,permanent},{shutdown,5000},{child_type,worker}]

2012-05-18 04:59:01 =CRASH REPORT====
  crasher:
    initial call: riak_core_vnode_proxy:init/1
    pid: <0.21469.398>
    registered_name:
proxy_riak_kv_vnode_947692467956757361022701883714464770558143758336
    exception exit:
{{{badmatch,{error,{{badmatch,{error,emfile}},[{riak_kv_vnode,init,1},{riak_core_vnode,init,1},{gen_fsm,init_it,6},{proc_lib,init_p_do_apply,3}]}}},[{riak_core_vnode_manager,get_vnode,3},{riak_core_vnode_manager,handle_call,3},{gen_server,handle_msg,5},{proc_lib,init_p_do_apply,3}]},{gen_server,call,[riak_core_vnode_manager,{947692467956757361022701883714464770558143758336,riak_kv_vnode,get_vnode},infinity]}}
      in function  gen_server:call/3
      in call from riak_core_vnode_proxy:get_vnode_pid/1
      in call from riak_core_vnode_proxy:handle_proxy/2
      in call from riak_core_vnode_proxy:loop/2
      in call from proc_lib:init_p_do_apply/3
    ancestors: [riak_core_vnode_proxy_sup,riak_core_sup,<0.91.0>]
    messages: []
    links: [<0.20960.398>]
    dictionary: []
    trap_exit: false
    status: running
    heap_size: 377
    stack_size: 24
    reductions: 89
  neighbours:
2012-05-18 04:59:01 =SUPERVISOR REPORT====
     Supervisor: {local,riak_core_vnode_proxy_sup}
     Context:    child_terminated
     Reason:
{{{badmatch,{error,{{badmatch,{error,emfile}},[{riak_kv_vnode,init,1},{riak_core_vnode,init,1},{gen_fsm,init_it,6},{proc_lib,init_p_do_apply,3}]}}},[{riak_core_vnode_manager,get_vnode,3},{riak_core_vnode_manager,handle_call,3},{gen_server,handle_msg,5},{proc_lib,init_p_do_apply,3}]},{gen_server,call,[riak_core_vnode_manager,{947692467956757361022701883714464770558143758336,riak_kv_vnode,get_vnode},infinity]}}
     Offender:
[{pid,<0.21469.398>},{name,{riak_kv_vnode,947692467956757361022701883714464770558143758336}},{mfargs,{riak_core_vnode_proxy,start_link,[riak_kv_vnode,947692467956757361022701883714464770558143758336]}},{restart_type,permanent},{shutdown,5000},{child_type,worker}]

2012-05-18 04:59:06 =ERROR REPORT====
** Generic server riak_kv_stat terminating
** Last message in was {'$gen_cast',{update,{put_fsm_time,732},63504536346}}
** When Server state ==
{state,{spiral,63504536346,[6,0,0,0,0,35,20,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},{spiral,63504536346,[2,0,0,0,0,21,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},61,37,{spiral,63504536341,[126,84,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},210,{spiral,63504536346,[2,0,0,0,0,21,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},37,{spiral,63504536346,[0,0,0,0,0,4,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},7,{spiral,63504536340,[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},0,{spiral,63504536346,[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},0,12,8,{slide,63504536340,60,60,"/var/lib/riak/slide-data/3053/1337.331540.24720",{file_descriptor,prim_file,{#Port<0.2186439>,69}},63504536341},{slide,63504536340,60,60,"/var/lib/riak/slide-data/3053/1337.331540.25076",{file_descriptor,prim_file,{#Port<0.2186440>,128}},63504536341},{slide,63504536340,60,60,"/var/lib/riak/slide-data/3053/1337.331540.25785",{file_descriptor,prim_file,{#Port<0.2186444>,40}},63504536346},{slide,63504536340,60,60,"/var/lib/riak/slide-data/3053/1337.331540.26002",{file_descriptor,prim_file,{#Port<0.2186441>,177}},63504536341},{spiral,63504536340,[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},0,0,{spiral,63504536340,[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},0,undefined,0,0,undefined,undefined,0,0,true}
** Reason for termination ==
** {emfile,[{erlang,open_port,[{spawn,"/bin/sh -s unix:cmd
2>&1"},[stream]]},{os,start_port_srv_handle,1},{os,start_port_srv_loop,0}]}
2012-05-18 04:59:06 =CRASH REPORT====
  crasher:
    initial call: gen:init_it/7
    pid: <0.22372.463>
    registered_name: riak_kv_stat
    exception exit: {emfile,[{erlang,open_port,[{spawn,"/bin/sh -s unix:cmd
2>&1"},[stream]]},{os,start_port_srv_handle,1},{os,start_port_srv_loop,0}]}
      in function  gen_server2:terminate/6
      in call from proc_lib:init_p_do_apply/3
    ancestors: [riak_kv_sup,<0.323.0>]
    messages:
[{'$gen_cast',{update,{vnode_index_write,0,0},63504536346}},{'$gen_cast',{update,vnode_put,63504536346}},{'$gen_cast',{update,{vnode_index_write,1,0},63504536346}},{'$gen_cast',{update,vnode_put,63504536346}},{'$gen_cast',{update,{vnode_index_write,1,0},63504536346}},{'$gen_cast',{update,vnode_put,63504536346}},{'$gen_cast',{update,{vnode_index_write,1,0},63504536346}},{'$gen_cast',{update,vnode_put,63504536346}},{'$gen_cast',{update,{put_fsm_time,5241},63504536346}}]
    links: [#Port<0.2186440>,#Port<0.2186444>,<0.420.0>,#Port<0.2186439>]
    dictionary: []
    trap_exit: true
    status: running
    heap_size: 6765
    stack_size: 24
    reductions: 23738
  neighbours:
2012-05-18 04:59:06 =SUPERVISOR REPORT====
     Supervisor: {local,riak_kv_sup}
     Context:    child_terminated
     Reason:     {emfile,[{erlang,open_port,[{spawn,"/bin/sh -s unix:cmd
2>&1"},[stream]]},{os,start_port_srv_handle,1},{os,start_port_srv_loop,0}]}
     Offender:
[{pid,<0.22372.463>},{name,riak_kv_stat},{mfargs,{riak_kv_stat,start_link,[]}},{restart_type,permanent},{shutdown,5000},{child_type,worker}]

2012-05-18 04:59:06 =SUPERVISOR REPORT====
     Supervisor: {local,riak_kv_sup}
     Context:    start_error
     Reason:     {emfile,[{erlang,open_port,[{spawn,"/bin/sh -s unix:cmd
2>&1"},[stream]]},{os,start_port_srv_handle,1},{os,start_port_srv_loop,0}]}
     Offender:
[{pid,<0.22372.463>},{name,riak_kv_stat},{mfargs,{riak_kv_stat,start_link,[]}},{restart_type,permanent},{shutdown,5000},{child_type,worker}]

2012-05-18 04:59:06 =CRASH REPORT====
  crasher:
    initial call: gen:init_it/7
    pid: <0.22460.463>
    registered_name: []
    exception exit: {emfile,[{erlang,open_port,[{spawn,"/bin/sh -s unix:cmd
2>&1"},[stream]]},{os,start_port_srv_handle,1},{os,start_port_srv_loop,0}]}
      in function  gen_server2:init_it/6
      in call from proc_lib:init_p_do_apply/3
    ancestors: [riak_kv_sup,<0.323.0>]
    messages: []
    links: [<0.420.0>]
    dictionary: []
    trap_exit: true
    status: running
    heap_size: 610
    stack_size: 24
    reductions: 363
  neighbours:
2012-05-18 04:59:06 =CRASH REPORT====
  crasher:
    initial call: gen:init_it/7
    pid: <0.22462.463>
    registered_name: []
    exception exit: {emfile,[{erlang,open_port,[{spawn,"/bin/sh -s unix:cmd
2>&1"},[stream]]},{os,start_port_srv_handle,1},{os,start_port_srv_loop,0}]}
      in function  gen_server2:init_it/6
      in call from proc_lib:init_p_do_apply/3
    ancestors: [riak_kv_sup,<0.323.0>]
    messages: []
    links: [<0.420.0>]
    dictionary: []
    trap_exit: true
    status: running
    heap_size: 610
    stack_size: 24
    reductions: 367
  neighbours:
2012-05-18 04:59:06 =SUPERVISOR REPORT====
     Supervisor: {local,riak_kv_sup}
     Context:    start_error
     Reason:     {emfile,[{erlang,open_port,[{spawn,"/bin/sh -s unix:cmd
2>&1"},[stream]]},{os,start_port_srv_handle,1},{os,start_port_srv_loop,0}]}
     Offender:
[{pid,<0.22372.463>},{name,riak_kv_stat},{mfargs,{riak_kv_stat,start_link,[]}},{restart_type,permanent},{shutdown,5000},{child_type,worker}]

2012-05-18 04:59:06 =SUPERVISOR REPORT====
     Supervisor: {local,riak_kv_sup}
     Context:    start_error
     Reason:     {emfile,[{erlang,open_port,[{spawn,"/bin/sh -s unix:cmd
2>&1"},[stream]]},{os,start_port_srv_handle,1},{os,start_port_srv_loop,0}]}
     Offender:
[{pid,<0.22372.463>},{name,riak_kv_stat},{mfargs,{riak_kv_stat,start_link,[]}},{restart_type,permanent},{shutdown,5000},{child_type,worker}]


*Node 3:*
*
*
2012-05-18 04:58:20 =ERROR REPORT====
** Generic server riak_core_vnode_manager terminating
** Last message in was
{947692467956757361022701883714464770558143758336,riak_kv_vnode,get_vnode}
** When Server state ==
{state,40798507028,[{{riak_kv_vnode,0},undefined},{{riak_kv_vnode,11417981541647679048466287755595961091061972992},undefined},{{riak_kv_vnode,22835963083295358096932575511191922182123945984},undefined},{{riak_kv_vnode,34253944624943037145398863266787883273185918976},undefined},{{riak_kv_vnode,45671926166590716193865151022383844364247891968},undefined},{{riak_kv_vnode,57089907708238395242331438777979805455309864960},undefined},{{riak_kv_vnode,68507889249886074290797726533575766546371837952},undefined},{{riak_kv_vnode,79925870791533753339264014289171727637433810944},undefined},{{riak_kv_vnode,91343852333181432387730302044767688728495783936},undefined},{{riak_kv_vnode,102761833874829111436196589800363649819557756928},undefined},{{riak_kv_vnode,114179815416476790484662877555959610910619729920},undefined},{{riak_kv_vnode,125597796958124469533129165311555572001681702912},undefined},{{riak_kv_vnode,137015778499772148581595453067151533092743675904},undefined},{{riak_kv_vnode,148433760041419827630061740822747494183805648896},undefined},{{riak_kv_vnode,159851741583067506678528028578343455274867621888},undefined},{{riak_kv_vnode,171269723124715185726994316333939416365929594880},undefined},{{riak_kv_vnode,182687704666362864775460604089535377456991567872},undefined},{{riak_kv_vnode,194105686208010543823926891845131338548053540864},undefined},{{riak_kv_vnode,205523667749658222872393179600727299639115513856},undefined},{{riak_kv_vnode,216941649291305901920859467356323260730177486848},undefined},{{riak_kv_vnode,228359630832953580969325755111919221821239459840},undefined},{{riak_kv_vnode,239777612374601260017792042867515182912301432832},undefined},{{riak_kv_vnode,251195593916248939066258330623111144003363405824},undefined},{{riak_kv_vnode,262613575457896618114724618378707105094425378816},undefined},{{riak_kv_vnode,274031556999544297163190906134303066185487351808},undefined},{{riak_kv_vnode,285449538541191976211657193889899027276549324800},undefined},{{riak_kv_vnode,296867520082839655260123481645494988367611297792},undefined},{{riak_kv_vnode,308285501624487334308589769401090949458673270784},undefined},{{riak_kv_vnode,319703483166135013357056057156686910549735243776},undefined},{{riak_kv_vnode,331121464707782692405522344912282871640797216768},undefined},{{riak_kv_vnode,342539446249430371453988632667878832731859189760},undefined},{{riak_kv_vnode,353957427791078050502454920423474793822921162752},undefined},{{riak_kv_vnode,365375409332725729550921208179070754913983135744},undefined},{{riak_kv_vnode,376793390874373408599387495934666716005045108736},undefined},{{riak_kv_vnode,388211372416021087647853783690262677096107081728},undefined},{{riak_kv_vnode,399629353957668766696320071445858638187169054720},undefined},{{riak_kv_vnode,411047335499316445744786359201454599278231027712},undefined},{{riak_kv_vnode,422465317040964124793252646957050560369293000704},undefined},{{riak_kv_vnode,433883298582611803841718934712646521460354973696},undefined},{{riak_kv_vnode,445301280124259482890185222468242482551416946688},undefined},{{riak_kv_vnode,456719261665907161938651510223838443642478919680},undefined},{{riak_kv_vnode,468137243207554840987117797979434404733540892672},undefined},{{riak_kv_vnode,479555224749202520035584085735030365824602865664},undefined},{{riak_kv_vnode,490973206290850199084050373490626326915664838656},undefined},{{riak_kv_vnode,502391187832497878132516661246222288006726811648},undefined},{{riak_kv_vnode,513809169374145557180982949001818249097788784640},undefined},{{riak_kv_vnode,525227150915793236229449236757414210188850757632},undefined},{{riak_kv_vnode,536645132457440915277915524513010171279912730624},undefined},{{riak_kv_vnode,548063113999088594326381812268606132370974703616},undefined},{{riak_kv_vnode,559481095540736273374848100024202093462036676608},undefined},{{riak_kv_vnode,570899077082383952423314387779798054553098649600},undefined},{{riak_kv_vnode,582317058624031631471780675535394015644160622592},undefined},{{riak_kv_vnode,593735040165679310520246963290989976735222595584},undefined},{{riak_kv_vnode,605153021707326989568713251046585937826284568576},undefined},{{riak_kv_vnode,616571003248974668617179538802181898917346541568},undefined},{{riak_kv_vnode,627988984790622347665645826557777860008408514560},undefined},{{riak_kv_vnode,639406966332270026714112114313373821099470487552},undefined},{{riak_kv_vnode,650824947873917705762578402068969782190532460544},undefined},{{riak_kv_vnode,662242929415565384811044689824565743281594433536},undefined},{{riak_kv_vnode,673660910957213063859510977580161704372656406528},undefined},{{riak_kv_vnode,685078892498860742907977265335757665463718379520},undefined},{{riak_kv_vnode,696496874040508421956443553091353626554780352512},undefined},{{riak_kv_vnode,707914855582156101004909840846949587645842325504},undefined},{{riak_kv_vnode,719332837123803780053376128602545548736904298496},undefined},{{riak_kv_vnode,730750818665451459101842416358141509827966271488},undefined},{{riak_kv_vnode,742168800207099138150308704113737470919028244480},undefined},{{riak_kv_vnode,753586781748746817198774991869333432010090217472},undefined},{{riak_kv_vnode,765004763290394496247241279624929393101152190464},undefined},{{riak_kv_vnode,776422744832042175295707567380525354192214163456},undefined},{{riak_kv_vnode,787840726373689854344173855136121315283276136448},undefined},{{riak_kv_vnode,799258707915337533392640142891717276374338109440},undefined},{{riak_kv_vnode,810676689456985212441106430647313237465400082432},undefined},{{riak_kv_vnode,822094670998632891489572718402909198556462055424},undefined},{{riak_kv_vnode,833512652540280570538039006158505159647524028416},undefined},{{riak_kv_vnode,844930634081928249586505293914101120738586001408},undefined},{{riak_kv_vnode,856348615623575928634971581669697081829647974400},undefined},{{riak_kv_vnode,867766597165223607683437869425293042920709947392},undefined},{{riak_kv_vnode,879184578706871286731904157180889004011771920384},undefined},{{riak_kv_vnode,890602560248518965780370444936484965102833893376},undefined},{{riak_kv_vnode,902020541790166644828836732692080926193895866368},undefined},{{riak_kv_vnode,913438523331814323877303020447676887284957839360},undefined},{{riak_kv_vnode,924856504873462002925769308203272848376019812352},undefined},{{riak_kv_vnode,936274486415109681974235595958868809467081785344},undefined},{{riak_kv_vnode,947692467956757361022701883714464770558143758336},undefined},{{riak_kv_vnode,959110449498405040071168171470060731649205731328},undefined},{{riak_kv_vnode,970528431040052719119634459225656692740267704320},undefined},{{riak_kv_vnode,981946412581700398168100746981252653831329677312},undefined},{{riak_kv_vnode,993364394123348077216567034736848614922391650304},undefined},{{riak_kv_vnode,1004782375664995756265033322492444576013453623296},undefined},{{riak_kv_vnode,1016200357206643435313499610248040537104515596288},undefined},{{riak_kv_vnode,1027618338748291114361965898003636498195577569280},undefined},{{riak_kv_vnode,1039036320289938793410432185759232459286639542272},undefined},{{riak_kv_vnode,1050454301831586472458898473514828420377701515264},undefined},{{riak_kv_vnode,1061872283373234151507364761270424381468763488256},undefined},{{riak_kv_vnode,1073290264914881830555831049026020342559825461248},undefined},{{riak_kv_vnode,1084708246456529509604297336781616303650887434240},undefined},{{riak_kv_vnode,1096126227998177188652763624537212264741949407232},undefined},{{riak_kv_vnode,1107544209539824867701229912292808225833011380224},undefined},{{riak_kv_vnode,1118962191081472546749696200048404186924073353216},undefined},{{riak_kv_vnode,1130380172623120225798162487804000148015135326208},undefined},{{riak_kv_vnode,1141798154164767904846628775559596109106197299200},undefined},{{riak_kv_vnode,1153216135706415583895095063315192070197259272192},undefined},{{riak_kv_vnode,1164634117248063262943561351070788031288321245184},undefined},{{riak_kv_vnode,1176052098789710941992027638826383992379383218176},undefined},{{riak_kv_vnode,1187470080331358621040493926581979953470445191168},undefined},{{riak_kv_vnode,1198888061873006300088960214337575914561507164160},undefined},{{riak_kv_vnode,1210306043414653979137426502093171875652569137152},undefined},{{riak_kv_vnode,1221724024956301658185892789848767836743631110144},undefined},{{riak_kv_vnode,1233142006497949337234359077604363797834693083136},undefined},{{riak_kv_vnode,1244559988039597016282825365359959758925755056128},undefined},{{riak_kv_vnode,1255977969581244695331291653115555720016817029120},undefined},{{riak_kv_vnode,1267395951122892374379757940871151681107879002112},undefined},{{riak_kv_vnode,1278813932664540053428224228626747642198940975104},undefined},{{riak_kv_vnode,1290231914206187732476690516382343603290002948096},undefined},{{riak_kv_vnode,1301649895747835411525156804137939564381064921088},undefined},{{riak_kv_vnode,1313067877289483090573623091893535525472126894080},undefined},{{riak_kv_vnode,1324485858831130769622089379649131486563188867072},undefined},{{riak_kv_vnode,1335903840372778448670555667404727447654250840064},undefined},{{riak_kv_vnode,1347321821914426127719021955160323408745312813056},undefined},{{riak_kv_vnode,1358739803456073806767488242915919369836374786048},undefined},{{riak_kv_vnode,1370157784997721485815954530671515330927436759040},undefined},{{riak_kv_vnode,1381575766539369164864420818427111292018498732032},undefined},{{riak_kv_vnode,1392993748081016843912887106182707253109560705024},undefined},{{riak_kv_vnode,1404411729622664522961353393938303214200622678016},undefined},{{riak_kv_vnode,1415829711164312202009819681693899175291684651008},undefined},{{riak_kv_vnode,1427247692705959881058285969449495136382746624000},undefined},{{riak_kv_vnode,1438665674247607560106752257205091097473808596992},undefined},{{riak_kv_vnode,1450083655789255239155218544960687058564870569984},undefined},{{riak_pipe_vnode,0},undefined},{{riak_pipe_vnode,11417981541647679048466287755595961091061972992},undefined},{{riak_pipe_vnode,22835963083295358096932575511191922182123945984},undefined},{{riak_pipe_vnode,34253944624943037145398863266787883273185918976},undefined},{{riak_pipe_vnode,45671926166590716193865151022383844364247891968},undefined},{{riak_pipe_vnode,57089907708238395242331438777979805455309864960},undefined},{{riak_pipe_vnode,68507889249886074290797726533575766546371837952},undefined},{{riak_pipe_vnode,79925870791533753339264014289171727637433810944},undefined},{{riak_pipe_vnode,91343852333181432387730302044767688728495783936},undefined},{{riak_pipe_vnode,102761833874829111436196589800363649819557756928},undefined},{{riak_pipe_vnode,114179815416476790484662877555959610910619729920},undefined},{{riak_pipe_vnode,125597796958124469533129165311555572001681702912},undefined},{{riak_pipe_vnode,137015778499772148581595453067151533092743675904},undefined},{{riak_pipe_vnode,148433760041419827630061740822747494183805648896},undefined},{{riak_pipe_vnode,159851741583067506678528028578343455274867621888},undefined},{{riak_pipe_vnode,171269723124715185726994316333939416365929594880},undefined},{{riak_pipe_vnode,182687704666362864775460604089535377456991567872},undefined},{{riak_pipe_vnode,194105686208010543823926891845131338548053540864},undefined},{{riak_pipe_vnode,205523667749658222872393179600727299639115513856},undefined},{{riak_pipe_vnode,216941649291305901920859467356323260730177486848},undefined},{{riak_pipe_vnode,228359630832953580969325755111919221821239459840},undefined},{{riak_pipe_vnode,239777612374601260017792042867515182912301432832},undefined},{{riak_pipe_vnode,251195593916248939066258330623111144003363405824},undefined},{{riak_pipe_vnode,262613575457896618114724618378707105094425378816},undefined},{{riak_pipe_vnode,274031556999544297163190906134303066185487351808},undefined},{{riak_pipe_vnode,285449538541191976211657193889899027276549324800},undefined},{{riak_pipe_vnode,296867520082839655260123481645494988367611297792},undefined},{{riak_pipe_vnode,308285501624487334308589769401090949458673270784},undefined},{{riak_pipe_vnode,319703483166135013357056057156686910549735243776},undefined},{{riak_pipe_vnode,331121464707782692405522344912282871640797216768},undefined},{{riak_pipe_vnode,342539446249430371453988632667878832731859189760},undefined},{{riak_pipe_vnode,353957427791078050502454920423474793822921162752},undefined},{{riak_pipe_vnode,365375409332725729550921208179070754913983135744},undefined},{{riak_pipe_vnode,376793390874373408599387495934666716005045108736},undefined},{{riak_pipe_vnode,388211372416021087647853783690262677096107081728},undefined},{{riak_pipe_vnode,399629353957668766696320071445858638187169054720},undefined},{{riak_pipe_vnode,411047335499316445744786359201454599278231027712},undefined},{{riak_pipe_vnode,422465317040964124793252646957050560369293000704},undefined},{{riak_pipe_vnode,433883298582611803841718934712646521460354973696},undefined},{{riak_pipe_vnode,445301280124259482890185222468242482551416946688},undefined},{{riak_pipe_vnode,456719261665907161938651510223838443642478919680},undefined},{{riak_pipe_vnode,468137243207554840987117797979434404733540892672},undefined},{{riak_pipe_vnode,479555224749202520035584085735030365824602865664},undefined},{{riak_pipe_vnode,490973206290850199084050373490626326915664838656},undefined},{{riak_pipe_vnode,502391187832497878132516661246222288006726811648},undefined},{{riak_pipe_vnode,513809169374145557180982949001818249097788784640},undefined},{{riak_pipe_vnode,525227150915793236229449236757414210188850757632},undefined},{{riak_pipe_vnode,536645132457440915277915524513010171279912730624},undefined},{{riak_pipe_vnode,548063113999088594326381812268606132370974703616},undefined},{{riak_pipe_vnode,559481095540736273374848100024202093462036676608},undefined},{{riak_pipe_vnode,570899077082383952423314387779798054553098649600},undefined},{{riak_pipe_vnode,582317058624031631471780675535394015644160622592},undefined},{{riak_pipe_vnode,593735040165679310520246963290989976735222595584},undefined},{{riak_pipe_vnode,605153021707326989568713251046585937826284568576},undefined},{{riak_pipe_vnode,616571003248974668617179538802181898917346541568},undefined},{{riak_pipe_vnode,627988984790622347665645826557777860008408514560},undefined},{{riak_pipe_vnode,639406966332270026714112114313373821099470487552},undefined},{{riak_pipe_vnode,650824947873917705762578402068969782190532460544},undefined},{{riak_pipe_vnode,662242929415565384811044689824565743281594433536},undefined},{{riak_pipe_vnode,673660910957213063859510977580161704372656406528},undefined},{{riak_pipe_vnode,685078892498860742907977265335757665463718379520},undefined},{{riak_pipe_vnode,696496874040508421956443553091353626554780352512},undefined},{{riak_pipe_vnode,707914855582156101004909840846949587645842325504},undefined},{{riak_pipe_vnode,719332837123803780053376128602545548736904298496},undefined},{{riak_pipe_vnode,730750818665451459101842416358141509827966271488},undefined},{{riak_pipe_vnode,742168800207099138150308704113737470919028244480},undefined},{{riak_pipe_vnode,753586781748746817198774991869333432010090217472},undefined},{{riak_pipe_vnode,765004763290394496247241279624929393101152190464},undefined},{{riak_pipe_vnode,776422744832042175295707567380525354192214163456},undefined},{{riak_pipe_vnode,787840726373689854344173855136121315283276136448},undefined},{{riak_pipe_vnode,799258707915337533392640142891717276374338109440},undefined},{{riak_pipe_vnode,810676689456985212441106430647313237465400082432},undefined},{{riak_pipe_vnode,822094670998632891489572718402909198556462055424},undefined},{{riak_pipe_vnode,833512652540280570538039006158505159647524028416},undefined},{{riak_pipe_vnode,844930634081928249586505293914101120738586001408},undefined},{{riak_pipe_vnode,856348615623575928634971581669697081829647974400},undefined},{{riak_pipe_vnode,867766597165223607683437869425293042920709947392},undefined},{{riak_pipe_vnode,879184578706871286731904157180889004011771920384},undefined},{{riak_pipe_vnode,890602560248518965780370444936484965102833893376},undefined},{{riak_pipe_vnode,902020541790166644828836732692080926193895866368},undefined},{{riak_pipe_vnode,913438523331814323877303020447676887284957839360},undefined},{{riak_pipe_vnode,924856504873462002925769308203272848376019812352},undefined},{{riak_pipe_vnode,936274486415109681974235595958868809467081785344},undefined},{{riak_pipe_vnode,947692467956757361022701883714464770558143758336},undefined},{{riak_pipe_vnode,959110449498405040071168171470060731649205731328},undefined},{{riak_pipe_vnode,970528431040052719119634459225656692740267704320},undefined},{{riak_pipe_vnode,981946412581700398168100746981252653831329677312},undefined},{{riak_pipe_vnode,993364394123348077216567034736848614922391650304},undefined},{{riak_pipe_vnode,1004782375664995756265033322492444576013453623296},undefined},{{riak_pipe_vnode,1016200357206643435313499610248040537104515596288},undefined},{{riak_pipe_vnode,1027618338748291114361965898003636498195577569280},undefined},{{riak_pipe_vnode,1039036320289938793410432185759232459286639542272},undefined},{{riak_pipe_vnode,1050454301831586472458898473514828420377701515264},undefined},{{riak_pipe_vnode,1061872283373234151507364761270424381468763488256},undefined},{{riak_pipe_vnode,1073290264914881830555831049026020342559825461248},undefined},{{riak_pipe_vnode,1084708246456529509604297336781616303650887434240},undefined},{{riak_pipe_vnode,1096126227998177188652763624537212264741949407232},undefined},{{riak_pipe_vnode,1107544209539824867701229912292808225833011380224},undefined},{{riak_pipe_vnode,1118962191081472546749696200048404186924073353216},undefined},{{riak_pipe_vnode,1130380172623120225798162487804000148015135326208},undefined},{{riak_pipe_vnode,1141798154164767904846628775559596109106197299200},undefined},{{riak_pipe_vnode,1153216135706415583895095063315192070197259272192},undefined},{{riak_pipe_vnode,1164634117248063262943561351070788031288321245184},undefined},{{riak_pipe_vnode,1176052098789710941992027638826383992379383218176},undefined},{{riak_pipe_vnode,1187470080331358621040493926581979953470445191168},undefined},{{riak_pipe_vnode,1198888061873006300088960214337575914561507164160},undefined},{{riak_pipe_vnode,1210306043414653979137426502093171875652569137152},undefined},{{riak_pipe_vnode,1221724024956301658185892789848767836743631110144},undefined},{{riak_pipe_vnode,1233142006497949337234359077604363797834693083136},undefined},{{riak_pipe_vnode,1244559988039597016282825365359959758925755056128},undefined},{{riak_pipe_vnode,1255977969581244695331291653115555720016817029120},undefined},{{riak_pipe_vnode,1267395951122892374379757940871151681107879002112},undefined},{{riak_pipe_vnode,1278813932664540053428224228626747642198940975104},undefined},{{riak_pipe_vnode,1290231914206187732476690516382343603290002948096},undefined},{{riak_pipe_vnode,1301649895747835411525156804137939564381064921088},undefined},{{riak_pipe_vnode,1313067877289483090573623091893535525472126894080},undefined},{{riak_pipe_vnode,1324485858831130769622089379649131486563188867072},undefined},{{riak_pipe_vnode,1335903840372778448670555667404727447654250840064},undefined},{{riak_pipe_vnode,1347321821914426127719021955160323408745312813056},undefined},{{riak_pipe_vnode,1358739803456073806767488242915919369836374786048},undefined},{{riak_pipe_vnode,1370157784997721485815954530671515330927436759040},undefined},{{riak_pipe_vnode,1381575766539369164864420818427111292018498732032},undefined},{{riak_pipe_vnode,1392993748081016843912887106182707253109560705024},undefined},{{riak_pipe_vnode,1404411729622664522961353393938303214200622678016},undefined},{{riak_pipe_vnode,1415829711164312202009819681693899175291684651008},undefined},{{riak_pipe_vnode,1427247692705959881058285969449495136382746624000},undefined},{{riak_pipe_vnode,1438665674247607560106752257205091097473808596992},undefined},{{riak_pipe_vnode,1450083655789255239155218544960687058564870569984},undefined},{{riak_search_vnode,0},undefined},{{riak_search_vnode,11417981541647679048466287755595961091061972992},undefined},{{riak_search_vnode,22835963083295358096932575511191922182123945984},undefined},{{riak_search_vnode,34253944624943037145398863266787883273185918976},undefined},{{riak_search_vnode,45671926166590716193865151022383844364247891968},undefined},{{riak_search_vnode,57089907708238395242331438777979805455309864960},undefined},{{riak_search_vnode,68507889249886074290797726533575766546371837952},undefined},{{riak_search_vnode,79925870791533753339264014289171727637433810944},undefined},{{riak_search_vnode,91343852333181432387730302044767688728495783936},undefined},{{riak_search_vnode,102761833874829111436196589800363649819557756928},undefined},{{riak_search_vnode,114179815416476790484662877555959610910619729920},undefined},{{riak_search_vnode,125597796958124469533129165311555572001681702912},undefined},{{riak_search_vnode,137015778499772148581595453067151533092743675904},undefined},{{riak_search_vnode,148433760041419827630061740822747494183805648896},undefined},{{riak_search_vnode,159851741583067506678528028578343455274867621888},undefined},{{riak_search_vnode,171269723124715185726994316333939416365929594880},undefined},{{riak_search_vnode,182687704666362864775460604089535377456991567872},undefined},{{riak_search_vnode,194105686208010543823926891845131338548053540864},undefined},{{riak_search_vnode,205523667749658222872393179600727299639115513856},undefined},{{riak_search_vnode,216941649291305901920859467356323260730177486848},undefined},{{riak_search_vnode,228359630832953580969325755111919221821239459840},undefined},{{riak_search_vnode,239777612374601260017792042867515182912301432832},undefined},{{riak_search_vnode,251195593916248939066258330623111144003363405824},undefined},{{riak_search_vnode,262613575457896618114724618378707105094425378816},undefined},{{riak_search_vnode,274031556999544297163190906134303066185487351808},undefined},{{riak_search_vnode,285449538541191976211657193889899027276549324800},undefined},{{riak_search_vnode,296867520082839655260123481645494988367611297792},undefined},{{riak_search_vnode,308285501624487334308589769401090949458673270784},undefined},{{riak_search_vnode,319703483166135013357056057156686910549735243776},undefined},{{riak_search_vnode,331121464707782692405522344912282871640797216768},undefined},{{riak_search_vnode,342539446249430371453988632667878832731859189760},undefined},{{riak_search_vnode,353957427791078050502454920423474793822921162752},undefined},{{riak_search_vnode,365375409332725729550921208179070754913983135744},undefined},{{riak_search_vnode,376793390874373408599387495934666716005045108736},undefined},{{riak_search_vnode,388211372416021087647853783690262677096107081728},undefined},{{riak_search_vnode,399629353957668766696320071445858638187169054720},undefined},{{riak_search_vnode,411047335499316445744786359201454599278231027712},undefined},{{riak_search_vnode,422465317040964124793252646957050560369293000704},undefined},{{riak_search_vnode,433883298582611803841718934712646521460354973696},undefined},{{riak_search_vnode,445301280124259482890185222468242482551416946688},undefined},{{riak_search_vnode,456719261665907161938651510223838443642478919680},undefined},{{riak_search_vnode,468137243207554840987117797979434404733540892672},undefined},{{riak_search_vnode,479555224749202520035584085735030365824602865664},undefined},{{riak_search_vnode,490973206290850199084050373490626326915664838656},undefined},{{riak_search_vnode,502391187832497878132516661246222288006726811648},undefined},{{riak_search_vnode,513809169374145557180982949001818249097788784640},undefined},{{riak_search_vnode,525227150915793236229449236757414210188850757632},undefined},{{riak_search_vnode,536645132457440915277915524513010171279912730624},undefined},{{riak_search_vnode,548063113999088594326381812268606132370974703616},undefined},{{riak_search_vnode,559481095540736273374848100024202093462036676608},undefined},{{riak_search_vnode,570899077082383952423314387779798054553098649600},undefined},{{riak_search_vnode,582317058624031631471780675535394015644160622592},undefined},{{riak_search_vnode,593735040165679310520246963290989976735222595584},undefined},{{riak_search_vnode,605153021707326989568713251046585937826284568576},undefined},{{riak_search_vnode,616571003248974668617179538802181898917346541568},undefined},{{riak_search_vnode,627988984790622347665645826557777860008408514560},undefined},{{riak_search_vnode,639406966332270026714112114313373821099470487552},undefined},{{riak_search_vnode,650824947873917705762578402068969782190532460544},undefined},{{riak_search_vnode,662242929415565384811044689824565743281594433536},undefined},{{riak_search_vnode,673660910957213063859510977580161704372656406528},undefined},{{riak_search_vnode,685078892498860742907977265335757665463718379520},undefined},{{riak_search_vnode,696496874040508421956443553091353626554780352512},undefined},{{riak_search_vnode,707914855582156101004909840846949587645842325504},undefined},{{riak_search_vnode,719332837123803780053376128602545548736904298496},undefined},{{riak_search_vnode,730750818665451459101842416358141509827966271488},undefined},{{riak_search_vnode,742168800207099138150308704113737470919028244480},undefined},{{riak_search_vnode,753586781748746817198774991869333432010090217472},undefined},{{riak_search_vnode,765004763290394496247241279624929393101152190464},undefined},{{riak_search_vnode,776422744832042175295707567380525354192214163456},undefined},{{riak_search_vnode,787840726373689854344173855136121315283276136448},undefined},{{riak_search_vnode,799258707915337533392640142891717276374338109440},undefined},{{riak_search_vnode,810676689456985212441106430647313237465400082432},undefined},{{riak_search_vnode,822094670998632891489572718402909198556462055424},undefined},{{riak_search_vnode,833512652540280570538039006158505159647524028416},undefined},{{riak_search_vnode,844930634081928249586505293914101120738586001408},undefined},{{riak_search_vnode,856348615623575928634971581669697081829647974400},undefined},{{riak_search_vnode,867766597165223607683437869425293042920709947392},undefined},{{riak_search_vnode,879184578706871286731904157180889004011771920384},undefined},{{riak_search_vnode,890602560248518965780370444936484965102833893376},undefined},{{riak_search_vnode,902020541790166644828836732692080926193895866368},undefined},{{riak_search_vnode,913438523331814323877303020447676887284957839360},undefined},{{riak_search_vnode,924856504873462002925769308203272848376019812352},undefined},{{riak_search_vnode,936274486415109681974235595958868809467081785344},undefined},{{riak_search_vnode,947692467956757361022701883714464770558143758336},undefined},{{riak_search_vnode,959110449498405040071168171470060731649205731328},undefined},{{riak_search_vnode,970528431040052719119634459225656692740267704320},undefined},{{riak_search_vnode,981946412581700398168100746981252653831329677312},undefined},{{riak_search_vnode,993364394123348077216567034736848614922391650304},undefined},{{riak_search_vnode,1004782375664995756265033322492444576013453623296},undefined},{{riak_search_vnode,1016200357206643435313499610248040537104515596288},undefined},{{riak_search_vnode,1027618338748291114361965898003636498195577569280},undefined},{{riak_search_vnode,1039036320289938793410432185759232459286639542272},undefined},{{riak_search_vnode,1050454301831586472458898473514828420377701515264},undefined},{{riak_search_vnode,1061872283373234151507364761270424381468763488256},undefined},{{riak_search_vnode,1073290264914881830555831049026020342559825461248},undefined},{{riak_search_vnode,1084708246456529509604297336781616303650887434240},undefined},{{riak_search_vnode,1096126227998177188652763624537212264741949407232},undefined},{{riak_search_vnode,1107544209539824867701229912292808225833011380224},undefined},{{riak_search_vnode,1118962191081472546749696200048404186924073353216},undefined},{{riak_search_vnode,1130380172623120225798162487804000148015135326208},undefined},{{riak_search_vnode,1141798154164767904846628775559596109106197299200},undefined},{{riak_search_vnode,1153216135706415583895095063315192070197259272192},undefined},{{riak_search_vnode,1164634117248063262943561351070788031288321245184},undefined},{{riak_search_vnode,1176052098789710941992027638826383992379383218176},undefined},{{riak_search_vnode,1187470080331358621040493926581979953470445191168},undefined},{{riak_search_vnode,1198888061873006300088960214337575914561507164160},undefined},{{riak_search_vnode,1210306043414653979137426502093171875652569137152},undefined},{{riak_search_vnode,1221724024956301658185892789848767836743631110144},undefined},{{riak_search_vnode,1233142006497949337234359077604363797834693083136},undefined},{{riak_search_vnode,1244559988039597016282825365359959758925755056128},undefined},{{riak_search_vnode,1255977969581244695331291653115555720016817029120},undefined},{{riak_search_vnode,1267395951122892374379757940871151681107879002112},undefined},{{riak_search_vnode,1278813932664540053428224228626747642198940975104},undefined},{{riak_search_vnode,1290231914206187732476690516382343603290002948096},undefined},{{riak_search_vnode,1301649895747835411525156804137939564381064921088},undefined},{{riak_search_vnode,1313067877289483090573623091893535525472126894080},undefined},{{riak_search_vnode,1324485858831130769622089379649131486563188867072},undefined},{{riak_search_vnode,1335903840372778448670555667404727447654250840064},undefined},{{riak_search_vnode,1347321821914426127719021955160323408745312813056},undefined},{{riak_search_vnode,1358739803456073806767488242915919369836374786048},undefined},{{riak_search_vnode,1370157784997721485815954530671515330927436759040},undefined},{{riak_search_vnode,1381575766539369164864420818427111292018498732032},undefined},{{riak_search_vnode,1392993748081016843912887106182707253109560705024},undefined},{{riak_search_vnode,1404411729622664522961353393938303214200622678016},undefined},{{riak_search_vnode,1415829711164312202009819681693899175291684651008},undefined},{{riak_search_vnode,1427247692705959881058285969449495136382746624000},undefined},{{riak_search_vnode,1438665674247607560106752257205091097473808596992},undefined},{{riak_search_vnode,1450083655789255239155218544960687058564870569984},undefined}],[]}
** Reason for termination ==
**
{{badmatch,{error,{{badmatch,{error,emfile}},[{riak_kv_vnode,init,1},{riak_core_vnode,init,1},{gen_fsm,init_it,6},{proc_lib,init_p_do_apply,3}]}}},[{riak_core_vnode_manager,get_vnode,3},{riak_core_vnode_manager,handle_call,3},{gen_server,handle_msg,5},{proc_lib,init_p_do_apply,3}]}
2012-05-18 04:58:20 =CRASH REPORT====
  crasher:
    initial call: riak_core_vnode_manager:init/1
    pid: <0.15696.398>
    registered_name: riak_core_vnode_manager
    exception exit:
{{badmatch,{error,{{badmatch,{error,emfile}},[{riak_kv_vnode,init,1},{riak_core_vnode,init,1},{gen_fsm,init_it,6},{proc_lib,init_p_do_apply,3}]}}},[{riak_core_vnode_manager,get_vnode,3},{riak_core_vnode_manager,handle_call,3},{gen_server,handle_msg,5},{proc_lib,init_p_do_apply,3}]}
      in function  gen_server:terminate/6
      in call from proc_lib:init_p_do_apply/3
    ancestors: [riak_core_sup,<0.91.0>]
    messages: []
    links: [<0.92.0>]
    dictionary: []
    trap_exit: false
    status: running
    heap_size: 17711
    stack_size: 24
    reductions: 170595
  neighbours:
2012-05-18 04:58:20 =CRASH REPORT====
  crasher:
    initial call: riak_core_vnode_proxy:init/1
    pid: <0.15703.398>
    registered_name:
proxy_riak_kv_vnode_947692467956757361022701883714464770558143758336
    exception exit:
{{{badmatch,{error,{{badmatch,{error,emfile}},[{riak_kv_vnode,init,1},{riak_core_vnode,init,1},{gen_fsm,init_it,6},{proc_lib,init_p_do_apply,3}]}}},[{riak_core_vnode_manager,get_vnode,3},{riak_core_vnode_manager,handle_call,3},{gen_server,handle_msg,5},{proc_lib,init_p_do_apply,3}]},{gen_server,call,[riak_core_vnode_manager,{947692467956757361022701883714464770558143758336,riak_kv_vnode,get_vnode},infinity]}}
      in function  gen_server:call/3
      in call from riak_core_vnode_proxy:get_vnode_pid/1
      in call from riak_core_vnode_proxy:handle_proxy/2
      in call from riak_core_vnode_proxy:loop/2
      in call from proc_lib:init_p_do_apply/3
    ancestors: [riak_core_vnode_proxy_sup,riak_core_sup,<0.91.0>]
    messages: []
    links: [<0.107.0>]
    dictionary: []
    trap_exit: false
    status: running
    heap_size: 377
    stack_size: 24
    reductions: 89
  neighbours:
2012-05-18 04:58:20 =SUPERVISOR REPORT====
     Supervisor: {local,riak_core_sup}
     Context:    child_terminated
     Reason:
{{badmatch,{error,{{badmatch,{error,emfile}},[{riak_kv_vnode,init,1},{riak_core_vnode,init,1},{gen_fsm,init_it,6},{proc_lib,init_p_do_apply,3}]}}},[{riak_core_vnode_manager,get_vnode,3},{riak_core_vnode_manager,handle_call,3},{gen_server,handle_msg,5},{proc_lib,init_p_do_apply,3}]}
     Offender:
[{pid,<0.15696.398>},{name,riak_core_vnode_manager},{mfargs,{riak_core_vnode_manager,start_link,[]}},{restart_type,permanent},{shutdown,5000},{child_type,worker}]

2012-05-18 04:58:20 =SUPERVISOR REPORT====
     Supervisor: {local,riak_core_vnode_proxy_sup}
     Context:    child_terminated
     Reason:
{{{badmatch,{error,{{badmatch,{error,emfile}},[{riak_kv_vnode,init,1},{riak_core_vnode,init,1},{gen_fsm,init_it,6},{proc_lib,init_p_do_apply,3}]}}},[{riak_core_vnode_manager,get_vnode,3},{riak_core_vnode_manager,handle_call,3},{gen_server,handle_msg,5},{proc_lib,init_p_do_apply,3}]},{gen_server,call,[riak_core_vnode_manager,{947692467956757361022701883714464770558143758336,riak_kv_vnode,get_vnode},infinity]}}
     Offender:
[{pid,<0.15703.398>},{name,{riak_kv_vnode,947692467956757361022701883714464770558143758336}},{mfargs,{riak_core_vnode_proxy,start_link,[riak_kv_vnode,947692467956757361022701883714464770558143758336]}},{restart_type,permanent},{shutdown,5000},{child_type,worker}]

2012-05-18 04:58:20 =SUPERVISOR REPORT====
     Supervisor: {local,riak_core_vnode_proxy_sup}
     Context:    shutdown
     Reason:     reached_max_restart_intensity
     Offender:
[{pid,<0.15703.398>},{name,{riak_kv_vnode,947692467956757361022701883714464770558143758336}},{mfargs,{riak_core_vnode_proxy,start_link,[riak_kv_vnode,947692467956757361022701883714464770558143758336]}},{restart_type,permanent},{shutdown,5000},{child_type,worker}]

2012-05-18 04:58:20 =ERROR REPORT====
** Generic server riak_kv_vnode_master terminating
** Last message in was
{'$gen_cast',{riak_vnode_req_v1,947692467956757361022701883714464770558143758336,{fsm,undefined,<11602.10975.494>},{riak_kv_put_req_v1,{<<"_rsid_pages">>,<<"K4fUuIQqNsUHdH2rNqMzk6AKSot">>},{r_object,<<"_rsid_pages">>,<<"K4fUuIQqNsUHdH2rNqMzk6AKSot">>,[{r_content,{dict,3,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},{{[],[],[],[],[],[],[],[],[],[],[[<<"X-Riak-VTag">>,53,49,103,55,99,86,83,98,79,82,83,73,110,68,56,84,109,118,55,89,71,72]],[[<<"index">>]],[],[[<<"X-Riak-Last-Modified">>|{1337,331480,246971}]],[],[]}}},{riak_idx_doc,<<"pages">>,<<"K4fUuIQqNsUHdH2rNqMzk6AKSot">>,[{<<"_type">>,<<"Page">>,[{<<"Page">>,[0]}]},{<<"classification">>,<<"0">>,[{<<"0000000000">>,[0]}]},{<<"contact_email">>,<<"
david at serieaticketsonline.com">>,[{<<"david">>,[0]},{<<"
serieaticketsonline.com
">>,[1]}]},{<<"created_at">>,<<"2012-05-18">>,[{<<"2012-05-18">>,[0]}]},{<<"domain">>,<<"
serieaticketsonline.com">>,[{<<"serieaticketsonline.com">>,[0]}]},{<<"outbound_links">>,<<"16">>,[{<<"0000000016">>,[0]}]},{<<"revaluate">>,<<"false">>,[]},{<<"search_terms">>,<<"44
999 This Day and Age Safetysuit Civil Twilight Reign of Kindo As Tall as
Lions Copeland Alpha Rev Secondhand Serenade 38th Parallel Sent By Ravens
Boys Like Girls Manic Drive P.O.D Ivoryline We Shot the Moon Deas Vail Neon
Horse Grey Holiday Jon Mclaughlin Jonezetta Pillar Barcelona Mutemath Lydia
Jonsi Matt Nathanson Rocket Summer Rapture Ruckus My Morning Jacket
Doubledge Hey Monday Newworldson Above the Golden State Matt Wertz James
Morrison VersaEmerge Thousand Foot Krutch The Drawing Room Manafest Jimmy
Needham Waking Ashland Flyleaf Falling Up A Dream to the Moon Since October
Fireflight Stereophonics The Fray Arcade Fire Mat Kearney Turn off the
Stars Skillet Snow Patrol Shiny Toy Guns Lifehouse Children 18:3
Celldweller Article One HB Abandon Lost Ocean Staple Spoken Ever Stays Red
This Beautiful Republic The Send Oceans Above Needtobreathe The Beatles
Breaking Benjamin I See Stars Radiohead Muse AFI Alkaline Trio Pink Floyd
U2 REM Blues Traveler Creedence Clearwater Revival Poets of the Fall
Disciple Emarosa Evanescence Fall Out Boy Underoath The Killers Linkin Park
My Chemical Romance Red Hot Chili Peppers Simon and Garfunkel Sixpence
(None The Richer) System of a Down Weezer The Who Tokio Hotel Paramore
Green Day The Cure Depeche Mode Black Veil Brides Korn Oomph! 3 Days Grace
Escape the Fate The All-American Rejects All Time Low 30 Seconds to Mars
Plain White T's Kings of Leon Silversun Pickups Nirvana Kerli David Bowie
ACDC LED ZEPLIN BEATLES FOO FIGHTERS GUNS N ROSES PINK FLYOD THE JIMI
HENDRIX EXPERIANCE BLACK SABBATH IRON MAIDEN THE ROLLING STONES K-ROCK
AEROSMITH THE BLACK KEYS THE CLASH PEARL JAM THE ANIMALS THE DOORS BON JOVI
IGGY AND THE STOOGES VAMPIRE WEEKEND DRAGON FORCE WARRANT QUEEN DEEP PURPLE
THE RAMONES KISS THE POLICE COLDPLAY MOTORHEARDS black veiled brides
misfits ten years trapt kid cudi neon trees silverstein velvet revolver
hoobastank adema seether shinedown 3 doors down default whitesnake motley
crue panic at the disco disterbed anberlin lovehatehero slipknot rise
against ateyu Cream Tokyo Police Club MGMT Passion Pit Animal Collective
Cocoon Death Cab for Cutie Def Leopard Dirty Projectors Elton John Keith
Urban Black Eyed Peas Florence and the Machine The Postal Service Chromeo
Freelance Whales Blind Pilot The Man Beach House Ghostland Observatory The
Temper Trap Hall & Oates Led Zeppelin Santana Van Morrison Journey The Pet
Shop Boys Lynayrd Skynard ZZ Top Sufjan Stevens The Welcome Wagon Lionel
Richie Pablo Cruise Discovery Tears for Fears El Ten Eleven Prince Stevie
Wonder Fleetwood Mac Bjork Tom Petty and the Heartbreakers Wolf Parade
Shout Out Louds Yeah Yeah Yeahs Keane Matt & Kim She and Him Frans
Ferdinand">>,[{<<"moon">>,[30,81]},{<<"journey">>,[354]},{<<"who">>,[173]},{<<"motley">>,[289]},{<<"club">>,[304]},{<<"needtobreathe">>,"z"},{<<"rolling">>,[231]},{<<"him">>,[401]},{<<"beatles">>,[124,215]},{<<"barcelona">>,")"},{<<"jimi">>,[223]},{<<"iron">>,[228]},{<<"hoobastank">>,[281]},{<<"fray">>,"W"},{<<"eyed">>,[324]},{<<"dirty">>,[317]},{<<"petty">>,[384]},{<<"staple">>,"n"},{<<"kerli">>,[209]},{<<"abandon">>,"k"},{<<"republic">>,"u"},{<<"mode">>,[182]},{<<"doubledge">>,"5"},{<<"john">>,[320]},{<<"spoken">>,"o"},{<<"bon">>,[246]},{<<"pilot">>,[337]},{<<"999">>,[0]},{<<"day">>,[2,178]},{<<"parallel">>,[18]},{<<"pillar">>,"("},{<<"santana">>,[351]},{<<"versaemerge">>,"A"},{<<"passion">>,[306]},{<<"skillet">>,"`"},{<<"whales">>,[335]},{<<"animal">>,[308]},{<<"house">>,[341]},{<<"observatory">>,[343]},{<<"ruckus">>,"2"},{<<"manafest">>,"H"},{<<"falling">>,"N"},{<<"beautiful">>,"t"},{<<"weezer">>,[171]},{<<"collective">>,[309]},{<<"cruise">>,[370]},{<<"sufjan">>,[362]},{<<"force">>,[255]},{<<"roses">>,[219]},{<<"ten">>,[271,375]},{<<"ever">>,"p"},{<<"tom">>,[383]},{<<"krutch">>,"D"},{<<"kings">>,[204]},{<<"blind">>,[336]},{<<"rem">>,[136]},{<<"clearwater">>,[140]},{<<"one">>,"j"},{<<"man">>,[339]},{<<"silverstein">>,[278]},{<<"dream">>,"O"},{<<"kearney">>,"["},{<<"jonezetta">>,"'"},{<<"cure">>,[180]},{<<"clash">>,[239]},{<<"shop">>,[357]},{<<"kid">>,[274]},{<<"stars">>,[95,128]},{<<"warrant">>,[256]},{<<"copeland">>,[12]},{<<"experiance">>,[225]},{<<"boys">>,[21,358]},{<<"stereophonics">>,"U"},{<<"trio">>,[133]},{<<"louds">>,[392]},{<<"weekend">>,[253]},{<<"animals">>,[243]},{<<"peas">>,[325]},{<<"tears">>,[372]},{<<"manic">>,[24]},{<<"down">>,[170,286]},{<<"maiden">>,[229]},{<<"misfits">>,[270]},{<<"wonder">>,[379]},{<<"summer">>,"0"},{<<"patrol">>,"b"},{<<"jonsi">>,","},{<<"stooges">>,[251]},{<<"shot">>,[28]},{<<"vail">>,"
"},{<<"out">>,[149,391]},{<<"simon">>,[162]},{<<"beach">>,[340]},{<<"shiny">>,"c"},{<<"default">>,[287]},{<<"oceans">>,"x"},{<<"low">>,[199]},{<<"police">>,[264,303]},{<<"sent">>,[19]},{<<"sabbath">>,[227]},{<<"cab">>,[312]},{<<"rejects">>,[196]},{<<"discovery">>,[371]},{<<"breaking">>,"}"},{<<"golden">>,";"},{<<"since">>,"R"},{<<"parade">>,[389]},{<<"pink">>,[134,220]},{<<"welcome">>,[365]},{<<"kindo">>,"\t"},{<<"death">>,[311]},{<<"postal">>,[331]},{<<"time">>,[198]},{<<"matt">>,[45,61,397]},{<<"snow">>,"a"},{<<"linkin">>,[154]},{<<"pickups">>,[207]},{<<"hendrix">>,[224]},{<<"disterbed">>,[294]},{<<"prince">>,[377]},{<<"mac">>,[381]},{<<"sixpence">>,[165]},{<<"garfunkel">>,[164]},{<<"green">>,[177]},{<<"queen">>,[257]},{<<"pablo">>,[369]},{<<"trapt">>,[273]},{<<"mutemath">>,"*"},{<<"rocket">>,"/"},{<<"underoath">>,[151]},{<<"ghostland">>,[342]},{<<"system">>,[169]},{<<"cudi">>,[275]},{<<"rise">>,[298]},{<<"led">>,[213,349]},{<<"state">>,"<"},{<<"room">>,"G"},{<<"romance">>,[157]},{<<"richer">>,[168]},{<<"rev">>,[14]},{<<"flyleaf">>,"M"},{<<"years">>,[272]},{<<"slipknot">>,[297]},{<<"cream">>,[301]},{<<"needham">>,"J"},{<<"twilight">>,[7]},{<<"article">>,"i"},{<<"evanescence">>,[147]},{<<"flyod">>,[221]},{<<"trees">>,[277]},{<<"anberlin">>,[295]},{<<"benjamin">>,"~"},{<<"service">>,[332]},{<<"thousand">>,"B"},{<<"shout">>,[390]},{<<"nathanson">>,"."},{<<"ashland">>,"L"},{<<"age">>,[4]},{<<"ocean">>,"m"},{<<"mat">>,"Z"},{<<"velvet">>,[279]},{<<"iggy">>,[248]},{<<"celldweller">>,"h"},{<<"leon">>,[205]},{<<"peppers">>,[161]},{<<"heartbreakers">>,[387]},{<<"paramore">>,[176]},{<<"frans">>,[402]},{<<"motorheards">>,[266]},{<<"ferdinand">>,[403]},{<<"monday">>,"7"},{<<"wertz">>,">"},{<<"escape">>,[190]},{<<"secondhand">>,[15]},{<<"jimmy">>,"I"},{<<"veil">>,[184]},{<<"toy">>,"d"},{<<"chromeo">>,[333]},{<<"wolf">>,[388]},{<<"james">>,"?"},{<<"top">>,[361]},{<<"turn">>,"\"},{<<"panic">>,[291]},{<<"cocoon">>,[310]},{<<"lifehouse">>,"f"},{<<"tokio">>,[174]},{<<"yeah">>,[393,394]},{<<"lovehatehero">>,[296]},{<<"kim">>,[398]},{<<"all">>,[194,197]},{<<"lydia">>,"+"},{<<"foo">>,[216]},{<<"disciple">>,[145]},{<<"vampire">>,[252]},{<<"lost">>,"l"},{<<"silversun">>,[206]},{<<"stevie">>,[378]},{<<"38th">>,[17]},{<<"horse">>,"""},{<<"deep">>,[258]},{<<"florence">>,[326]},{<<"nirvana">>,[208]},{<<"eleven">>,[376]},{<<"drive">>,[25]},{<<"zeppelin">>,[350]},{<<"emarosa">>,[146]},{<<"hall">>,[347]},{<<"bowie">>,[211]},{<<"above">>,"9y"},{<<"seconds">>,[200]},{<<"deas">>,[31]},{<<"arcade">>,"X"},{<<"plain">>,[202]},{<<"projectors">>,[318]},{<<"acdc">>,[212]},{<<"jovi">>,[247]},{<<"shinedown">>,[284]},{<<"traveler">>,[138]},{<<"alpha">>,"\r"},{<<"korn">>,[186]},{<<"alkaline">>,[132]},{<<"creedence">>,[139]},{<<"mars">>,[201]},{<<"like">>,[22]},{<<"temper">>,[345]},{<<"neon">>,[33,276]},{<<"fire">>,"Y"},{<<"cutie">>,[314]},{<<"park">>,[155]},{<<"against">>,[299]},{<<"she">>,[399]},{<<"boy">>,[150]},{<<"elton">>,[319]},{<<"wagon">>,[366]},{<<"purple">>,[259]},{<<"serenade">>,[16]},{<<"children">>,"g"},{<<"hotel">>,[175]},{<<"civil">>,[6]},{<<"jacket">>,"4"},{<<"ramones">>,[261]},{<<"oates">>,[348]},{<<"red">>,[114,158]},{<<"fears">>,[374]},{<<"revolver">>,[280]},{<<"david">>,[210]},{<<"jam">>,[241]},{<<"pit">>,[307]},{<<"keith">>,[321]},{<<"ivoryline">>,[27]},{<<"waking">>,"K"},{<<"richie">>,[368]},{<<"fall">>,[144,148]},{<<"killers">>,[153]},{<<"grace">>,[189]},{<<"skynard">>,[360]},{<<"p.o.d">>,[26]},{<<"chili">>,[160]},{<<"pearl">>,[240]},{<<"machine">>,[329]},{<<"adema">>,[282]},{<<"safetysuit">>,[5]},{<<"none">>,[166]},{<<"oomph">>,[187]},{<<"reign">>,[8]},{<<"october">>,"S"},{<<"def">>,[315]},{<<"girls">>,[23]},{<<"crue">>,[290]},{<<"grey">>,"#"},{<<"floyd">>,[135]},{<<"ravens">>,[20]},{<<"stones">>,[232]},{<<"tall">>,"\n"},{<<"van">>,[352]},{<<"drawing">>,"F"},{<<"urban">>,[322]},{<<"foot">>,"C"},{<<"veiled">>,[268]},{<<"days">>,[188]},{<<"kiss">>,[262]},{<<"stays">>,"q"},{<<"seether">>,[283]},{<<"hey">>,"6"},{<<"dragon">>,[254]},{<<"radiohead">>,[129]},{<<"lynayrd">>,[359]},{<<"brides">>,[185,269]},{<<"tokyo">>,[302]},{<<"morning">>,"3"},{<<"muse">>,[130]},{<<"pet">>,[356]},{<<"bjork">>,[382]},{<<"holiday">>,"$"},{<<"lionel">>,[367]},{<<"morrison">>,[64,353]},{<<"whitesnake">>,[288]},{<<"rock">>,[233]},{<<"disco">>,[293]},{<<"depeche">>,[181]},{<<"keane">>,[396]},{<<"fate">>,[192]},{<<"off">>,"]"},{<<"mgmt">>,[305]},{<<"blues">>,[137]},{<<"yeahs">>,[395]},{<<"newworldson">>,"8"},{<<"see">>,[127]},{<<"fleetwood">>,[380]},{<<"lions">>,[11]},{<<"white">>,[203]},{<<"coldplay">>,[265]},{<<"trap">>,[346]},{<<"revival">>,[141]},{<<"aerosmith">>,[234]},{<<"guns">>,[101,218]},{<<"american">>,[195]},{<<"ateyu">>,[300]},{<<"leopard">>,[316]},{<<"stevens">>,[363]},{<<"fighters">>,[217]},{<<"chemical">>,[156]},{<<"doors">>,[245,285]},{<<"jon">>,"%"},{<<"rapture">>,"1"},{<<"keys">>,[237]},{<<"send">>,"w"},{<<"afi">>,[131]},{<<"mclaughlin">>,"&"},{<<"freelance">>,[334]},{<<"poets">>,[142]},{<<"fireflight">>,"T"},{<<"hot">>,[159]},{<<"black">>,[183,226,236,267,323]},{<<"zeplin">>,[214]}]},{<<"updated_at">>,<<"2012-05-18">>,[{<<"2012-05-18">>,[0]}]},{<<"url">>,<<"
http://www.serieaticketsonline.com/content-milan-players-looking-forward-for-second-leg-against-barcelona/
">>,[{<<"barcelona">>,[11]},{<<"leg">>,"\t"},{<<"http">>,[0]},{<<"players">>,[4]},{<<"
www.serieaticketsonline.com
">>,[1]},{<<"second">>,[8]},{<<"content">>,[2]},{<<"looking">>,[5]},{<<"against">>,"\n"},{<<"milan">>,[3]},{<<"forward">>,[6]}]}],[],[],true}}],[{<<15,27,99,155,79,178,239,200>>,{3,63504550680}}],{dict,1,16,16,8,80,48,{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[]},{{[],[],[],[],[],[],[],[],[],[],[],[],[],[],[[clean|true]],[]}}},undefined},13537270,63504550680,[]}}}
** When Server state ==
{state,undefined,undefined,riak_kv_vnode,riak_kv_legacy_vnode}
** Reason for termination ==
**
{badarg,[{gen_fsm,send_event,2},{riak_core_vnode_master,handle_cast,2},{gen_server,handle_msg,5},{proc_lib,init_p_do_apply,3}]}
2012-05-18 04:58:20 =CRASH REPORT====
  crasher:
    initial call: riak_core_vnode_master:init/1
    pid: <0.421.0>
    registered_name: riak_kv_vnode_master
    exception exit:
{badarg,[{gen_fsm,send_event,2},{riak_core_vnode_master,handle_cast,2},{gen_server,handle_msg,5},{proc_lib,init_p_do_apply,3}]}
      in function  gen_server:terminate/6
      in call from proc_lib:init_p_do_apply/3
    ancestors: [riak_kv_sup,<0.323.0>]
    messages: []
    links: [<0.420.0>]
    dictionary: []
    trap_exit: false
    status: running
    heap_size: 6765
    stack_size: 24
    reductions: 547963388
  neighbours:
2012-05-18 04:58:20 =SUPERVISOR REPORT====
     Supervisor: {local,riak_kv_sup}
     Context:    child_terminated
     Reason:
{badarg,[{gen_fsm,send_event,2},{riak_core_vnode_master,handle_cast,2},{gen_server,handle_msg,5},{proc_lib,init_p_do_apply,3}]}
     Offender:
[{pid,<0.421.0>},{name,riak_kv_vnode_master},{mfargs,{riak_core_vnode_master,start_link,[riak_kv_vnode,riak_kv_legacy_vnode]}},{restart_type,permanent},{shutdown,5000},{child_type,worker}]

2012-05-18 04:58:20 =SUPERVISOR REPORT====
     Supervisor: {local,riak_core_sup}
     Context:    child_terminated
     Reason:     shutdown
     Offender:
[{pid,<0.107.0>},{name,riak_core_vnode_proxy_sup},{mfargs,{riak_core_vnode_proxy_sup,start_link,[]}},{restart_type,permanent},{shutdown,5000},{child_type,supervisor}]

2012-05-18 04:58:20 =ERROR REPORT====
** Generic server riak_kv_stat terminating
** Last message in was
{'$gen_cast',{update,{get_fsm,undefined,279,1,1350},63504536300}}
** When Server state ==
{state,{spiral,63504536300,[11,0,0,0,0,0,0,0,0,0,0,18,0,0,0,3,17,0,0,0,0,0,0,0,0,0,23,0,0,0,9,12,0,0,0,14,0,0,0,0,0,3,5,0,0,0,2,6,0,0,0,0,0,0,0,0,0,30,0,0,16,27,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,6,0,0,14,0,0,0,0,0,7,22]},{spiral,63504536300,[6,0,0,0,0,0,0,0,0,0,0,12,0,0,0,3,10,0,0,0,0,0,0,0,0,0,16,0,0,0,6,7,0,0,0,7,0,0,0,0,0,3,2,0,0,0,2,2,0,0,0,0,0,0,0,0,0,18,0,0,11,18,22,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,6,0,0,7,0,0,0,0,0,7,12]},5887366,3601568,{spiral,63504536300,[42,0,0,0,0,0,0,0,0,0,0,42,0,0,0,42,42,0,0,0,0,0,0,0,0,0,42,0,0,0,42,84,0,0,0,84,0,0,0,0,0,42,42,0,0,0,42,42,0,0,0,0,0,0,0,0,0,42,0,0,0,84,42,0,0,0,0,0,0,0,0,42,0,0,0,0,0,84,0,0,0,84,0,0,0,0,0,42,168]},10079908,{spiral,63504536300,[6,0,0,0,0,0,0,0,0,0,0,12,0,0,0,3,10,0,0,0,0,0,0,0,0,0,16,0,0,0,6,7,0,0,0,7,0,0,0,0,0,3,2,0,0,0,2,2,0,0,0,0,0,0,0,0,0,18,0,0,11,18,22,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,6,0,0,7,0,0,0,0,0,7,12]},3601568,{spiral,63504536300,[3,0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,1,0,0,0,0,0,0,0,0,0,3,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,0,0,0,3,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,7]},788755,{spiral,63504477129,[1,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,0]},32,{spiral,63504536300,[0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},17236,2394509,1571512,{slide,63504471451,60,60,"/var/lib/riak/slide-data/3053/1337.266551.794838",{file_descriptor,prim_file,{#Port<0.2186351>,40}},63504536289},{slide,63504471451,60,60,"/var/lib/riak/slide-data/3053/1337.266551.794979",{file_descriptor,prim_file,{#Port<0.2186352>,128}},63504536289},{slide,63504471451,60,60,"/var/lib/riak/slide-data/3053/1337.266551.794984",{file_descriptor,prim_file,{#Port<0.2186349>,69}},63504536289},{slide,63504475275,60,60,"/var/lib/riak/slide-data/3053/1337.266551.794989",{file_descriptor,prim_file,{#Port<0.2186353>,177}},63504536289},{spiral,63504471351,[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},0,0,{spiral,63504536285,[2,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1]},132,undefined,259259,0,undefined,undefined,0,0,true}
** Reason for termination ==
** {emfile,[{erlang,open_port,[{spawn,"/bin/sh -s unix:cmd
2>&1"},[stream]]},{os,start_port_srv_handle,1},{os,start_port_srv_loop,0}]}
2012-05-18 04:58:20 =CRASH REPORT====
  crasher:
    initial call: gen:init_it/7
    pid: <0.426.0>
    registered_name: riak_kv_stat
    exception exit: {emfile,[{erlang,open_port,[{spawn,"/bin/sh -s unix:cmd
2>&1"},[stream]]},{os,start_port_srv_handle,1},{os,start_port_srv_loop,0}]}
      in function  gen_server2:terminate/6
      in call from proc_lib:init_p_do_apply/3
    ancestors: [riak_kv_sup,<0.323.0>]
    messages:
[{'$gen_cast',{update,vnode_get,63504536300}},{'$gen_cast',{update,vnode_get,63504536300}},{'$gen_cast',{update,vnode_get,63504536300}},{'$gen_cast',{update,vnode_get,63504536300}},{'$gen_cast',{update,{get_fsm,undefined,337,1,1350},63504536300}},{'$gen_cast',{update,{vnode_index_write,0,0},63504536300}},{'$gen_cast',{update,vnode_put,63504536300}},{'$gen_cast',{update,{vnode_index_write,0,0},63504536300}},{'$gen_cast',{update,vnode_put,63504536300}},{'$gen_cast',{update,{put_fsm_time,603},63504536300}},{'$gen_cast',{update,{vnode_index_write,0,0},63504536300}},{'$gen_cast',{update,vnode_put,63504536300}},{'$gen_cast',{update,{vnode_index_write,0,0},63504536300}},{'$gen_cast',{update,vnode_put,63504536300}},{'$gen_cast',{update,{vnode_index_write,0,0},63504536300}},{'$gen_cast',{update,vnode_put,63504536300}},{'$gen_cast',{update,{put_fsm_time,13883},63504536300}},{'$gen_cast',{update,{vnode_index_write,0,0},63504536300}},{'$gen_cast',{update,vnode_put,63504536300}}]
    links: [#Port<0.2186353>,<0.420.0>,#Port<0.2186351>,#Port<0.2186352>]
    dictionary: []
    trap_exit: true
    status: running
    heap_size: 6765
    stack_size: 24
    reductions: 1873973117
  neighbours:
2012-05-18 04:58:20 =SUPERVISOR REPORT====
     Supervisor: {local,riak_kv_sup}
     Context:    child_terminated
     Reason:     {emfile,[{erlang,open_port,[{spawn,"/bin/sh -s unix:cmd
2>&1"},[stream]]},{os,start_port_srv_handle,1},{os,start_port_srv_loop,0}]}
     Offender:
[{pid,<0.426.0>},{name,riak_kv_stat},{mfargs,{riak_kv_stat,start_link,[]}},{restart_type,permanent},{shutdown,5000},{child_type,worker}]

*
*
*App Config:*
*
*
%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
%% ex: ft=erlang ts=4 sw=4 et
[
 %% Riak Core config
 {riak_core, [
              %% Default location of ringstate
              {ring_state_dir, "/var/lib/riak/ring"},

      %% The number of partitions to divide the hash space into
      %% The default is 64. Set to a size to allow for growth as
      %% this cannot be changed easily after a cluster is set up.
      %% Each node should have at least 10 partitions assigned to it.
      {ring_creation_size, 128},

              %% http is a list of IP addresses and TCP ports that the Riak
              %% HTTP interface will bind.
              {http, [ {"10.38.24.196", 8098 } ]},

              %% https is a list of IP addresses and TCP ports that the Riak
              %% HTTPS interface will bind.
              {https, [{"50.28.61.141", 8069 }]},

              %% Default cert and key locations for https can be overridden
              %% with the ssl config variable, for example:
              {ssl, [
                     {certfile, "/etc/riak/poseidon_cert.pem"},
                     {keyfile, "/etc/riak/poseidon_key.pem"}
                    ]},

              %% riak_handoff_port is the TCP port that Riak uses for
              %% intra-cluster data handoff.
              {handoff_port, 8099 },

              %% To encrypt riak_core intra-cluster data handoff traffic,
              %% uncomment the following line and edit its path to an
              %% appropriate certfile and keyfile.  (This example uses a
              %% single file with both items concatenated together.)
              %{handoff_ssl_options, [{certfile, "/tmp/erlserver.pem"}]},

              %% Platform-specific installation paths (substituted by rebar)
              {platform_bin_dir, "/usr/sbin"},
              {platform_data_dir, "/var/lib/riak"},
              {platform_etc_dir, "/etc/riak"},
              {platform_lib_dir, "/usr/lib/riak"},
              {platform_log_dir, "/var/log/riak"}
             ]},

 %% Riak KV config
 {riak_kv, [
            %% Storage_backend specifies the Erlang module defining the
storage
            %% mechanism that will be used on this node.
            {storage_backend, riak_kv_eleveldb_backend},

            %% pb_ip is the IP address that the Riak Protocol Buffers
interface
            %% will bind to.  If this is undefined, the interface will not
run.
            {pb_ip,   "127.0.0.1" },

            %% pb_port is the TCP port that the Riak Protocol Buffers
interface
            %% will bind to
            {pb_port, 8087 },

            %% pb_backlog is the maximum length to which the queue of
pending
            %% connections may grow. If set, it must be an integer >= 0.
            %% By default the value is 5. If you anticipate a huge number of
            %% connections being initialised *simultaneously*, set this
number
            %% higher.
            %% {pb_backlog, 64},

            %% raw_name is the first part of all URLS used by the Riak raw
HTTP
            %% interface.  See riak_web.erl and raw_http_resource.erl for
            %% details.
            %{raw_name, "riak"},

            %% mapred_name is URL used to submit map/reduce requests to
Riak.
            {mapred_name, "mapred"},

            %% mapred_system indicates which version of the MapReduce
            %% system should be used: 'pipe' means riak_pipe will
            %% power MapReduce queries, while 'legacy' means that luke
            %% will be used
            {mapred_system, pipe},

            %% mapred_2i_pipe indicates whether secondary-index
            %% MapReduce inputs are queued in parallel via their own
            %% pipe ('true'), or serially via a helper process
            %% ('false' or undefined).  Set to 'false' or leave
            %% undefined during a rolling upgrade from 1.0.
            {mapred_2i_pipe, true},

            %% directory used to store a transient queue for pending
            %% map tasks
            %% Only valid when mapred_system == legacy
            %% {mapred_queue_dir, "/var/lib/riak/mr_queue" },

            %% Each of the following entries control how many Javascript
            %% virtual machines are available for executing map, reduce,
            %% pre- and post-commit hook functions.
            {map_js_vm_count, 8 },
            {reduce_js_vm_count, 6 },
            {hook_js_vm_count, 2 },

            %% Number of items the mapper will fetch in one request.
            %% Larger values can impact read/write performance for
            %% non-MapReduce requests.
            %% Only valid when mapred_system == legacy
            %% {mapper_batch_size, 5},

            %% js_max_vm_mem is the maximum amount of memory, in megabytes,
            %% allocated to the Javascript VMs. If unset, the default is
            %% 8MB.
            {js_max_vm_mem, 8},

            %% js_thread_stack is the maximum amount of thread stack, in
megabyes,
            %% allocate to the Javascript VMs. If unset, the default is
16MB.
            %% NOTE: This is not the same as the C thread stack.
            {js_thread_stack, 16},

            %% Number of objects held in the MapReduce cache. These will be
            %% ejected when the cache runs out of room or the bucket/key
            %% pair for that entry changes
            %% Only valid when mapred_system == legacy
            %% {map_cache_size, 10000},

            %% js_source_dir should point to a directory containing
Javascript
            %% source files which will be loaded by Riak when it initializes
            %% Javascript VMs.
            {js_source_dir, "/tmp/js_source"},

            %% http_url_encoding determines how Riak treats URL encoded
            %% buckets, keys, and links over the REST API. When set to 'on'
            %% Riak always decodes encoded values sent as URLs and Headers.
            %% Otherwise, Riak defaults to compatibility mode where links
            %% are decoded, but buckets and keys are not. The compatibility
            %% mode will be removed in a future release.
            {http_url_encoding, on},

            %% riak_stat enables the use of the "riak-admin status" command
to
            %% retrieve information the Riak node for performance and
debugging needs
            {riak_kv_stat, true},

            %% When using riak_kv_stat, use the legacy routines for tracking
            {legacy_stats, true},

            %% Switch to vnode-based vclocks rather than client ids.  This
            %% significantly reduces the number of vclock entries.
            %% Only set true if *all* nodes in the cluster are upgraded to
1.0
            {vnode_vclocks, true},

            %% This option enables compatability of bucket and key listing
            %% with 0.14 and earlier versions. Once a rolling upgrade to
            %% a version > 0.14 is completed for a cluster, this should be
            %% set to false for improved performance for bucket and key
            %% listing operations.
            {legacy_keylisting, false},

            %% This option toggles compatibility of keylisting with 1.0
            %% and earlier versions.  Once a rolling upgrade to a version
            %% > 1.0 is completed for a cluster, this should be set to
            %% true for better control of memory usage during key listing
            %% operations
            {listkeys_backpressure, true}
           ]},

 %% Riak Search Config
 {riak_search, [
                %% To enable Search functionality set this 'true'.
                {enabled, true}
               ]},

 %% Merge Index Config
 {merge_index, [
                %% The root dir to store search merge_index data
                {data_root, "/var/lib/riak/merge_index"},

                %% The root dir to store secondary index merge_index data
                {data_root_2i, "/var/lib/riak/merge_index_2i"},

                %% Size, in bytes, of the in-memory buffer.  When this
                %% threshold has been reached the data is transformed
                %% into a segment file which resides on disk.
                {buffer_rollover_size, 1048576},

                %% Overtime the segment files need to be compacted.
                %% This is the maximum number of segments that will be
                %% compacted at once.  A lower value will lead to
                %% quicker but more frequent compactions.
                {max_compact_segments, 20}
               ]},

 %% Bitcask Config
 {bitcask, [
             {data_root, "/var/lib/riak/bitcask"}
           ]},

 %% eLevelDB Config
 {eleveldb, [
             {data_root, "/var/lib/riak/leveldb"}
            ]},

 %% Lager Config
 {lager, [
            %% What handlers to install with what arguments
            %% The defaults for the logfiles are to rotate the files when
            %% they reach 10Mb or at midnight, whichever comes first, and
keep
            %% the last 5 rotations. See the lager README for a description
of
            %% the time rotation format:
            %% https://github.com/basho/lager/blob/master/README.org
            %%
            %% If you wish to disable rotation, you can either set the size
to 0
            %% and the rotation time to "", or instead specify a 2-tuple
that only
            %% consists of {Logfile, Level}.
            {handlers, [
                {lager_console_backend, info},
                {lager_file_backend, [
                    {"/var/log/riak/error.log", error, 10485760, "$D0", 5},
                    {"/var/log/riak/console.log", info, 10485760, "$D0", 5}
                ]}
            ]},

            %% Whether to write a crash log, and where.
            %% Commented/omitted/undefined means no crash logger.
            {crash_log, "/var/log/riak/crash.log"},

            %% Maximum size in bytes of events in the crash log - defaults
to 65536
            {crash_log_msg_size, 65536},

            %% Maximum size of the crash log in bytes, before its rotated,
set
            %% to 0 to disable rotation - default is 0
            {crash_log_size, 10485760},

            %% What time to rotate the crash log - default is no time
            %% rotation. See the lager README for a description of this
format:
            %% https://github.com/basho/lager/blob/master/README.org
            {crash_log_date, "$D0"},

            %% Number of rotated crash logs to keep, 0 means keep only the
            %% current one - default is 0
            {crash_log_count, 5},

            %% Whether to redirect error_logger messages into lager -
defaults to true
            {error_logger_redirect, true}
        ]},

 %% riak_sysmon config
 {riak_sysmon, [
         %% To disable forwarding events of a particular type, use a
         %% limit of 0.
         {process_limit, 30},
         {port_limit, 2},

         %% Finding reasonable limits for a given workload is a matter
         %% of experimentation.
         {gc_ms_limit, 100},
         {heap_word_limit, 40111000},

         %% Configure the following items to 'false' to disable logging
         %% of that event type.
         {busy_port, true},
         {busy_dist_port, true}
        ]},

 %% SASL config
 {sasl, [
         {sasl_error_logger, false}
        ]},

 %% riak_control config
 {riak_control, [
                %% Set to false to disable the admin panel.
                {enabled, true},

                %% Authentication style used for access to the admin
                %% panel. Valid styles are 'userlist' <TODO>.
                {auth, userlist},

                %% If auth is set to 'userlist' then this is the
                %% list of usernames and passwords for access to the
                %% admin panel.
                {userlist, [{"Aquaman", "At1ant1s"}
                           ]},

                %% The admin panel is broken up into multiple
                %% components, each of which is enabled or disabled
                %% by one of these settings.
                {admin, true}
                ]}
].

Hopefully this is helpful. Thanks Dave.

-Gustav

On Tue, May 22, 2012 at 6:35 AM, Dave Smith <dizzyd at basho.com> wrote:

> Hi Gustav,
>
> Can you share the logs or any other information from the server that
> crashes _first_? It might be good to also include app.config, just so we
> can double-check that no other settings have changed.
>
> Thanks,
>
> D.
>
> --
> Dave Smith
> VP, Engineering
> Basho Technologies, Inc.
> dizzyd at basho.com
>
>
>
> On Friday, May 18, 2012 at 10:03 AM, Gustav Gennrich wrote:
>
> > Hi,
> >
> > Below is our setup to better diagnose:
> >
> > Ring creation size: 128
> > Backend: LevelDB
> >
> > All other settings in app.config are using the default values
> >
> > We have 3 servers (nodes) each of which have 8GB of ram.
> >
> > Question 1: We are attempting to load data into Riak using CSVs that we
> generated from data previously stored is our MySQL database. We are using
> Ripple to create the documents and associations (links). The problem we are
> having is Riak continue to grow in memory consumption until it finally
> crashed while loading this data. We are only able to add around a couple
> hundred thousand rows from the CSV's before this happens. Riak then
> responds with 500 errors and eventually all of the nodes crash and the
> nodes need to be restarted.
> >
> > Question 2 (unrelated): I am attempting to use some of the already
> defined js functions in ripple.js in a mapreduce, however, I keep getting
> the following error:
> >
> > Riak::MapReduceError:
> {"phase":0,"error":"[{<<\"lineno\">>,1},{<<\"message\">>,<<\"ReferenceError:
> Ripple is not defined\">>},{<<\"source\">>,<<\"unknown\">>}]"
> >
> > In ripple.yml I have the following (***** represent hidden values):
> >
> > # Configure Riak connections for the Ripple library.
> > development:
> > # http_port: 8091
> > # pb_port: 8087
> > # host: 127.0.0.1
> > basic_auth: *****
> > http_port: 80
> > pb_port: 8087
> > host: *****
> > http_backend: Excon
> > solr: "/solr"
> > js_source_dir: <%= Rails.root + "app/mapreduce" %>
> >
> > # The test environment has additional keys for configuring the
> > # Riak::TestServer for your test/spec suite:
> > #
> > # * bin_dir specifies the path to the "riak" script that you use to
> > # start Riak (just the directory)
> > # * js_source_dir specifies where your custom Javascript functions for
> > # MapReduce should be loaded from. Usually app/mapreduce.
> > test:
> > http_port: 9000
> > pb_port: 9002
> > host: localhost
> >
> > source: /usr/local/bin # Default for Homebrew.
> > js_source_dir: <%= Rails.root + "/app/mapreduce" %>
> >
> > production:
> > nodes:
> > - host: *****
> > http_port: 8098
> > pb_port: 8087
> > - host: *****
> > http_port: 8098
> > pb_port: 8087
> > - host: *****
> > http_port: 8098
> > pb_port: 8087
> > http_backend: Excon
> > solr: "/solr"
> > js_source_dir: <%= Rails.root + "app/mapreduce" %>
> >
> >
> > I don't see why I am getting this error when I have defined where the
> js_source_dir is.
> >
> > Any help would be greatly appreciated!
> >
> > Thanks,
> >
> > Gustav
> > _______________________________________________
> > riak-users mailing list
> > riak-users at lists.basho.com (mailto:riak-users at lists.basho.com)
> > http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com
>
>
>
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.basho.com/pipermail/riak-users_lists.basho.com/attachments/20120522/ca5978cd/attachment.html>


More information about the riak-users mailing list