> Can you attach the eleveldb portion of your app.config file?
> Configuration problems, especially max_open_files being too low, can
> often cause issues like this.
>
> If it isn't sensitive, the whole app.config and vm.args files are also
> often helpful.
Hello Evan,
thanks for responding.
I originally had default LevelDB settings. When the node stalled, I changed it
to
{eleveldb, [
{data_root, "/home/riak/leveldb"},
{max_open_files, 132},
{cache_size, 377487360}
]},
on all nodes and I restarted them all. The application started to run with
about 1000 requests/second, after about 1 minute it dropped to <500
requests/second, and the node stalled again after 41 minutes. BTW according to
lsof(1) it had 267 open LevelDB files which is more than the 132 files limit
(??).
Here is the configuration (with comments removed to work around 40 KB message
limit on the Riak users list).
/etc/riak/vm.args:
-name riak@172.16.0.2
-setcookie riak_0627f0bc6c9
+K true
+A 64
+W w
-env ERL_MAX_PORTS 4096
-env ERL_FULLSWEEP_AFTER 0
-env ERL_CRASH_DUMP /var/log/riak/erl_crash.dump
-env ERL_MAX_ETS_TABLES 8192
/etc/riak/app.config:
[
{riak_api, [
{pb_ip, "172.16.0.2" },
{pb_port, 8087 }
]},
{riak_core, [
{ring_state_dir, "/var/lib/riak/ring"},
%{ring_creation_size, 64},
{http, [ {"127.0.0.1", 8098 } ]},
%{https, [{ "127.0.0.1", 8098 }]},
%{ssl, [
% {certfile, "/etc/riak/cert.pem"},
% {keyfile, "/etc/riak/key.pem"}
% ]},
{handoff_port, 8099 },
%{handoff_ssl_options, [{certfile, "/tmp/erlserver.pem"}]},
{dtrace_support, false},
{platform_bin_dir, "/usr/sbin"},
{platform_data_dir, "/var/lib/riak"},
{platform_etc_dir, "/etc/riak"},
{platform_lib_dir, "/usr/lib/riak/lib"},
{platform_log_dir, "/var/log/riak"}
]},
{riak_kv, [
{add_paths,
["/usr/local/share/riak-mapred/riak-1.2.0/deps/greylisting/ebin/"]},
{storage_backend, riak_kv_eleveldb_backend},
%{raw_name, "riak"},
{mapred_name, "mapred"},
{mapred_system, pipe},
{mapred_2i_pipe, true},
{map_js_vm_count, 8 },
{reduce_js_vm_count, 6 },
{hook_js_vm_count, 2 },
{js_max_vm_mem, 8},
{js_thread_stack, 16},
%{js_source_dir, "/tmp/js_source"},
{http_url_encoding, on},
{vnode_vclocks, true},
{legacy_keylisting, false},
{listkeys_backpressure, true}
]},
{riak_search, [
{enabled, false}
]},
{merge_index, [
{data_root, "/var/lib/riak/merge_index"},
{buffer_rollover_size, 1048576},
{max_compact_segments, 20}
]},
{bitcask, [
{data_root, "/var/lib/riak/bitcask"}
]},
{eleveldb, [
{data_root, "/home/riak/leveldb"},
{max_open_files, 132},
{cache_size, 377487360}
]},
{lager, [
{handlers, [
{lager_console_backend, info},
{lager_file_backend, [
{"/var/log/riak/error.log", error, 10485760, "$D0", 5},
{"/var/log/riak/console.log", info, 10485760, "$D0", 5}
]}
]},
{crash_log, "/var/log/riak/crash.log"},
{crash_log_msg_size, 65536},
{crash_log_size, 10485760},
{crash_log_date, "$D0"},
{crash_log_count, 5},
{error_logger_redirect, true}
]},
{riak_sysmon, [
{process_limit, 30},
{port_limit, 2},
{gc_ms_limit, 100},
{heap_word_limit, 40111000},
{busy_port, true},
{busy_dist_port, true}
]},
{sasl, [
{sasl_error_logger, false}
]},
{riak_control, [
{enabled, false},
{auth, userlist},
{userlist, [{"user", "pass"}
]},
{admin, true}
]}
].
Thanks.
--
Jan
_______________________________________________
riak-users mailing list
riak-users@lists.basho.com
http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com
