What is max_open_files set to in the eleveldb section of app.config? If
unspecified I think the limit is 20. Remember that this number is per
vnode. The process limit specified by ulimit -n must be greater than
max_open_files * num_vnodes / num_nodes allowing room for vnode
multiplexing and fallback when losing a physical node.
On Wed, Aug 1, 2012 at 1:42 PM, John Roy <j...@thoughtleadr.com> wrote:
> I'm seeing significant downtime on Riak now. Much like the "Riak Crashing
> Constantly" thread. However in this case we get a "Too many open files"
> error, and also "contains pointer that is greater than the total data
> size." See the error messages below for more details.
>
> If others have an idea on this I'd appreciate your help.
>
> Thanks!
>
> John
>
> 2012-08-01 14:53:28 =ERROR REPORT====
> Hintfile
> '/disk1/riak/bitcask/1347321821914426127719021955160323408745312813056/12.bitcask.hint'
> contains pointer 119561351 4415 that is greater than total data size
> 119562240
>
> 2012-08-01 14:54:26 =CRASH REPORT====
> crasher:
> initial call: riak_core_vnode:init/1
> pid: <0.29239.0>
> registered_name: []
> exception exit: [{riak_kv_eleveldb_backend,{db_open,"IO error:
> /disk1/riak/leveldb/753586781748746817198774991869333432010090217472/CURRENT:
> Too many open files"}}]
> in function gen_fsm:init_it/6
> in call from proc_lib:init_p_do_apply/3
> ancestors: [riak_core_vnode_sup,riak_core_sup,<0.88.0>]
> messages: []
> links: [<0.92.0>]
> dictionary:
> [{#Ref<0.0.0.37922>,{bc_state,"/disk1/riak/bitcask/753586781748746817198774991869333432010090217472",fresh,undefined,[{filestate,read_only,"/disk1/riak/bitcask/753586781748746817198774991869333432010090217472/1.bitcask.data",1,<<>>,undefined,0,0},{filestate,read_only,"/disk1/riak/bitcask/753586781748746817198774991869333432010090217472/2.bitcask.data",2,<<>>,undefined,0,0}],2147483648,[{expiry_secs,-1},{read_write,true}],<<>>}},{random_seed,{17770,26756,17419}}]
> trap_exit: true
> status: running
> heap_size: 4181
> stack_size: 24
> reductions: 13734
> neighbours:
> 2012-08-01 14:54:26 =SUPERVISOR REPORT====
> Supervisor: {local,riak_core_vnode_sup}
> Context: child_terminated
> Reason: [{riak_kv_eleveldb_backend,{db_open,"IO error:
> /disk1/riak/leveldb/753586781748746817198774991869333432010090217472/CURRENT:
> Too many open files"}}]
> Offender:
> [{pid,<0.29239.0>},{name,undefined},{mfargs,{riak_core_vnode,start_link,undefined}},{restart_type,temporary},{shutdown,300000},{child_type,worker}]
>
> 2012-08-01 14:54:26 =ERROR REPORT====
> ** Generic server riak_core_vnode_manager terminating
> ** Last message in was
> {753586781748746817198774991869333432010090217472,riak_kv_vnode,get_vnode}
> ** When Server state ==
> {state,12308,[{{riak_kv_vnode,0},undefined},{{riak_kv_vnode,22835963083295358096932575511191922182123945984},undefined},{{riak_kv_vnode,45671926166590716193865151022383844364247891968},undefined},{{riak_kv_vnode,68507889249886074290797726533575766546371837952},undefined},{{riak_kv_vnode,91343852333181432387730302044767688728495783936},undefined},{{riak_kv_vnode,114179815416476790484662877555959610910619729920},undefined},{{riak_kv_vnode,137015778499772148581595453067151533092743675904},undefined},{{riak_kv_vnode,159851741583067506678528028578343455274867621888},undefined},{{riak_kv_vnode,182687704666362864775460604089535377456991567872},undefined},{{riak_kv_vnode,205523667749658222872393179600727299639115513856},undefined},{{riak_kv_vnode,228359630832953580969325755111919221821239459840},undefined},{{riak_kv_vnode,251195593916248939066258330623111144003363405824},undefined},{{riak_kv_vnode,274031556999544297163190906134303066185487351808},undefined},{{riak_kv_vnode,296867520082839655260123481645494988367611297792},undefined},{{riak_kv_vnode,319703483166135013357056057156686910549735243776},undefined},{{riak_kv_vnode,342539446249430371453988632667878832731859189760},undefined},{{riak_kv_vnode,365375409332725729550921208179070754913983135744},undefined},{{riak_kv_vnode,388211372416021087647853783690262677096107081728},undefined},{{riak_kv_vnode,411047335499316445744786359201454599278231027712},undefined},{{riak_kv_vnode,433883298582611803841718934712646521460354973696},undefined},{{riak_kv_vnode,456719261665907161938651510223838443642478919680},undefined},{{riak_kv_vnode,479555224749202520035584085735030365824602865664},undefined},{{riak_kv_vnode,502391187832497878132516661246222288006726811648},undefined},{{riak_kv_vnode,525227150915793236229449236757414210188850757632},undefined},{{riak_kv_vnode,548063113999088594326381812268606132370974703616},undefined},{{riak_kv_vnode,570899077082383952423314387779798054553098649600},undefined},{{riak_kv_vnode,593735040165679310520246963290989976735222595584},undefined},{{riak_kv_vnode,616571003248974668617179538802181898917346541568},undefined},{{riak_kv_vnode,639406966332270026714112114313373821099470487552},undefined},{{riak_kv_vnode,662242929415565384811044689824565743281594433536},undefined},{{riak_kv_vnode,685078892498860742907977265335757665463718379520},undefined},{{riak_kv_vnode,707914855582156101004909840846949587645842325504},undefined},{{riak_kv_vnode,730750818665451459101842416358141509827966271488},undefined},{{riak_kv_vnode,753586781748746817198774991869333432010090217472},undefined},{{riak_kv_vnode,776422744832042175295707567380525354192214163456},undefined},{{riak_kv_vnode,799258707915337533392640142891717276374338109440},undefined},{{riak_kv_vnode,822094670998632891489572718402909198556462055424},undefined},{{riak_kv_vnode,844930634081928249586505293914101120738586001408},undefined},{{riak_kv_vnode,867766597165223607683437869425293042920709947392},undefined},{{riak_kv_vnode,890602560248518965780370444936484965102833893376},undefined},{{riak_kv_vnode,913438523331814323877303020447676887284957839360},undefined},{{riak_kv_vnode,936274486415109681974235595958868809467081785344},undefined},{{riak_kv_vnode,959110449498405040071168171470060731649205731328},undefined},{{riak_kv_vnode,981946412581700398168100746981252653831329677312},undefined},{{riak_kv_vnode,1004782375664995756265033322492444576013453623296},undefined},{{riak_kv_vnode,1027618338748291114361965898003636498195577569280},undefined},{{riak_kv_vnode,1050454301831586472458898473514828420377701515264},undefined},{{riak_kv_vnode,1073290264914881830555831049026020342559825461248},undefined},{{riak_kv_vnode,1096126227998177188652763624537212264741949407232},undefined},{{riak_kv_vnode,1118962191081472546749696200048404186924073353216},undefined},{{riak_kv_vnode,1141798154164767904846628775559596109106197299200},undefined},{{riak_kv_vnode,1164634117248063262943561351070788031288321245184},undefined},{{riak_kv_vnode,1187470080331358621040493926581979953470445191168},undefined},{{riak_kv_vnode,1210306043414653979137426502093171875652569137152},undefined},{{riak_kv_vnode,1233142006497949337234359077604363797834693083136},undefined},{{riak_kv_vnode,1255977969581244695331291653115555720016817029120},undefined},{{riak_kv_vnode,1278813932664540053428224228626747642198940975104},undefined},{{riak_kv_vnode,1301649895747835411525156804137939564381064921088},undefined},{{riak_kv_vnode,1324485858831130769622089379649131486563188867072},undefined},{{riak_kv_vnode,1347321821914426127719021955160323408745312813056},undefined},{{riak_kv_vnode,1370157784997721485815954530671515330927436759040},undefined},{{riak_kv_vnode,1392993748081016843912887106182707253109560705024},undefined},{{riak_kv_vnode,1415829711164312202009819681693899175291684651008},undefined},{{riak_kv_vnode,1438665674247607560106752257205091097473808596992},undefined},{{riak_pipe_vnode,0},undefined},{{riak_pipe_vnode,22835963083295358096932575511191922182123945984},undefined},{{riak_pipe_vnode,45671926166590716193865151022383844364247891968},undefined},{{riak_pipe_vnode,68507889249886074290797726533575766546371837952},undefined},{{riak_pipe_vnode,91343852333181432387730302044767688728495783936},undefined},{{riak_pipe_vnode,114179815416476790484662877555959610910619729920},undefined},{{riak_pipe_vnode,137015778499772148581595453067151533092743675904},undefined},{{riak_pipe_vnode,159851741583067506678528028578343455274867621888},undefined},{{riak_pipe_vnode,182687704666362864775460604089535377456991567872},undefined},{{riak_pipe_vnode,205523667749658222872393179600727299639115513856},undefined},{{riak_pipe_vnode,228359630832953580969325755111919221821239459840},undefined},{{riak_pipe_vnode,251195593916248939066258330623111144003363405824},undefined},{{riak_pipe_vnode,274031556999544297163190906134303066185487351808},undefined},{{riak_pipe_vnode,296867520082839655260123481645494988367611297792},undefined},{{riak_pipe_vnode,319703483166135013357056057156686910549735243776},undefined},{{riak_pipe_vnode,342539446249430371453988632667878832731859189760},undefined},{{riak_pipe_vnode,365375409332725729550921208179070754913983135744},undefined},{{riak_pipe_vnode,388211372416021087647853783690262677096107081728},undefined},{{riak_pipe_vnode,411047335499316445744786359201454599278231027712},undefined},{{riak_pipe_vnode,433883298582611803841718934712646521460354973696},undefined},{{riak_pipe_vnode,456719261665907161938651510223838443642478919680},undefined},{{riak_pipe_vnode,479555224749202520035584085735030365824602865664},undefined},{{riak_pipe_vnode,502391187832497878132516661246222288006726811648},undefined},{{riak_pipe_vnode,525227150915793236229449236757414210188850757632},undefined},{{riak_pipe_vnode,548063113999088594326381812268606132370974703616},undefined},{{riak_pipe_vnode,570899077082383952423314387779798054553098649600},undefined},{{riak_pipe_vnode,593735040165679310520246963290989976735222595584},undefined},{{riak_pipe_vnode,616571003248974668617179538802181898917346541568},undefined},{{riak_pipe_vnode,639406966332270026714112114313373821099470487552},undefined},{{riak_pipe_vnode,662242929415565384811044689824565743281594433536},undefined},{{riak_pipe_vnode,685078892498860742907977265335757665463718379520},undefined},{{riak_pipe_vnode,707914855582156101004909840846949587645842325504},undefined},{{riak_pipe_vnode,730750818665451459101842416358141509827966271488},undefined},{{riak_pipe_vnode,753586781748746817198774991869333432010090217472},undefined},{{riak_pipe_vnode,776422744832042175295707567380525354192214163456},undefined},{{riak_pipe_vnode,799258707915337533392640142891717276374338109440},undefined},{{riak_pipe_vnode,822094670998632891489572718402909198556462055424},undefined},{{riak_pipe_vnode,844930634081928249586505293914101120738586001408},undefined},{{riak_pipe_vnode,867766597165223607683437869425293042920709947392},undefined},{{riak_pipe_vnode,890602560248518965780370444936484965102833893376},undefined},{{riak_pipe_vnode,913438523331814323877303020447676887284957839360},undefined},{{riak_pipe_vnode,936274486415109681974235595958868809467081785344},undefined},{{riak_pipe_vnode,959110449498405040071168171470060731649205731328},undefined},{{riak_pipe_vnode,981946412581700398168100746981252653831329677312},undefined},{{riak_pipe_vnode,1004782375664995756265033322492444576013453623296},undefined},{{riak_pipe_vnode,1027618338748291114361965898003636498195577569280},undefined},{{riak_pipe_vnode,1050454301831586472458898473514828420377701515264},undefined},{{riak_pipe_vnode,1073290264914881830555831049026020342559825461248},undefined},{{riak_pipe_vnode,1096126227998177188652763624537212264741949407232},undefined},{{riak_pipe_vnode,1118962191081472546749696200048404186924073353216},undefined},{{riak_pipe_vnode,1141798154164767904846628775559596109106197299200},undefined},{{riak_pipe_vnode,1164634117248063262943561351070788031288321245184},undefined},{{riak_pipe_vnode,1187470080331358621040493926581979953470445191168},undefined},{{riak_pipe_vnode,1210306043414653979137426502093171875652569137152},undefined},{{riak_pipe_vnode,1233142006497949337234359077604363797834693083136},undefined},{{riak_pipe_vnode,1255977969581244695331291653115555720016817029120},undefined},{{riak_pipe_vnode,1278813932664540053428224228626747642198940975104},undefined},{{riak_pipe_vnode,1301649895747835411525156804137939564381064921088},undefined},{{riak_pipe_vnode,1324485858831130769622089379649131486563188867072},undefined},{{riak_pipe_vnode,1347321821914426127719021955160323408745312813056},undefined},{{riak_pipe_vnode,1370157784997721485815954530671515330927436759040},undefined},{{riak_pipe_vnode,1392993748081016843912887106182707253109560705024},undefined},{{riak_pipe_vnode,1415829711164312202009819681693899175291684651008},undefined},{{riak_pipe_vnode,1438665674247607560106752257205091097473808596992},undefined}],[{{riak_kv_vnode,1324485858831130769622089379649131486563188867072},'
> riak@10.54.80.151'}]}
>
>
> _______________________________________________
> riak-users mailing list
> riak-users@lists.basho.com
> http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com
>
_______________________________________________
riak-users mailing list
riak-users@lists.basho.com
http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com
