It looks like this issues
https://github.com/basho/yokozuna/issues/320
I try to set
* |maxThreads| to |150|
* |Acceptors| to |10|
* |lowResourcesMaxIdleTime| to |50000|
in /usr/lib/riak/lib/yokozuna/priv/solr/etc/jetty.xml as recommended in
https://github.com/basho/yokozuna/issues/330
but it has no effect
On 10/01/2015 11:53 PM, Fred Dushin wrote:
Is there any more information in these logs that you can share? For example,
is this the only entry with this exception? Or are there more? Are there any
associated stack traces? An EOF exception can come from many different
scenarios.
Is there anything in the Riak console.log that looks suspicious?
Finally, you might want to take a look at what is going on inside of riak when
you get into this state (slow writes to Solr), by looking at Riak stats.
You get get to Riak stats via curl, e.g.,
curlhttp://localhost:8098/stats | python -m json.tool
ok, output is attached
Stats you might want to pay special attention to:
riak_kv_vnodeq (min, max, median, etc) -- the aggregate length of the vnode
queues. Long vnode queues may mean your vnode is locked waiting on Solr
vnode_put_fsm_time (mean, media, percentile, etc) The amount of time spent on
average waiting for a vnode put to complete. Long times may also be indicative
of waits writing into Solr.
"riak_kv_vnodeq_max": 0,
"riak_kv_vnodeq_mean": 0.0,
"riak_kv_vnodeq_median": 0,
"riak_kv_vnodeq_min": 0,
"riak_kv_vnodeq_total": 0,
{
"asn1_version": "2.0.3",
"basho_stats_version": "1.0.3",
"bitcask_version": "1.7.0",
"clique_version": "0.2.6-0-g40072d2",
"cluster_info_version": "2.0.2-0-ge231144",
"compiler_version": "4.9.3",
"connected_nodes": [
"riak@10.31.0.68",
"riak@10.31.0.67",
"riak@10.31.0.66",
"riak@10.31.0.65"
],
"consistent_get_objsize_100": 0,
"consistent_get_objsize_95": 0,
"consistent_get_objsize_99": 0,
"consistent_get_objsize_mean": 0,
"consistent_get_objsize_median": 0,
"consistent_get_time_100": 0,
"consistent_get_time_95": 0,
"consistent_get_time_99": 0,
"consistent_get_time_mean": 0,
"consistent_get_time_median": 0,
"consistent_gets": 0,
"consistent_gets_total": 0,
"consistent_put_objsize_100": 0,
"consistent_put_objsize_95": 0,
"consistent_put_objsize_99": 0,
"consistent_put_objsize_mean": 0,
"consistent_put_objsize_median": 0,
"consistent_put_time_100": 0,
"consistent_put_time_95": 0,
"consistent_put_time_99": 0,
"consistent_put_time_mean": 0,
"consistent_put_time_median": 0,
"consistent_puts": 0,
"consistent_puts_total": 0,
"converge_delay_last": 0,
"converge_delay_max": 0,
"converge_delay_mean": 0,
"converge_delay_min": 0,
"coord_redirs_total": 5066,
"counter_actor_counts_100": 0,
"counter_actor_counts_95": 0,
"counter_actor_counts_99": 0,
"counter_actor_counts_mean": 0,
"counter_actor_counts_median": 0,
"cpu_avg1": 82,
"cpu_avg15": 123,
"cpu_avg5": 108,
"cpu_nprocs": 552,
"crypto_version": "3.1",
"disk": [
{
"id": "/",
"size": 474203304,
"used": 2
},
{
"id": "/dev",
"size": 10240,
"used": 0
},
{
"id": "/run",
"size": 3285280,
"used": 10
},
{
"id": "/dev/shm",
"size": 8213192,
"used": 0
},
{
"id": "/run/lock",
"size": 5120,
"used": 0
},
{
"id": "/sys/fs/cgroup",
"size": 8213192,
"used": 0
},
{
"id": "/tmp",
"size": 1933648,
"used": 1
},
{
"id": "/boot",
"size": 457760,
"used": 12
},
{
"id": "/var/lib/riak",
"size": 468361760,
"used": 1
},
{
"id": "/run/user/1004",
"size": 1642640,
"used": 0
},
{
"id": "/run/user/1001",
"size": 1642640,
"used": 0
}
],
"dropped_vnode_requests_total": 0,
"eleveldb_version": "2.1.0-0-ga36dbd6",
"erlang_js_version": "1.3.0-0-g07467d8",
"erlydtl_version": "0.7.0",
"executing_mappers": 0,
"exometer_core_version": "1.0.0-basho2-0-gb47a5d6",
"goldrush_version": "0.1.6",
"gossip_received": 7,
"handoff_timeouts": 0,
"ibrowse_version": "4.0.2",
"ignored_gossip_total": 0,
"index_fsm_active": 0,
"index_fsm_create": 0,
"index_fsm_create_error": 0,
"inets_version": "5.9.6",
"kernel_version": "2.16.3",
"lager_version": "2.0.3",
"late_put_fsm_coordinator_ack": 0,
"leveldb_read_block_error": 0,
"list_fsm_active": 0,
"list_fsm_create": 0,
"list_fsm_create_error": 0,
"list_fsm_create_error_total": 0,
"list_fsm_create_total": 0,
"map_actor_counts_100": 0,
"map_actor_counts_95": 0,
"map_actor_counts_99": 0,
"map_actor_counts_mean": 0,
"map_actor_counts_median": 0,
"mem_allocated": 7542112256,
"mem_total": 16820621312,
"memory_atom": 752537,
"memory_atom_used": 745783,
"memory_binary": 68929880,
"memory_code": 16809458,
"memory_ets": 8286048,
"memory_processes": 114315184,
"memory_processes_used": 114277960,
"memory_system": 115784256,
"memory_total": 230099440,
"merge_index_version": "2.0.0-0-gb701dde",
"mochiweb_version": "2.9.0",
"node_get_fsm_active": 0,
"node_get_fsm_active_60s": 0,
"node_get_fsm_counter_objsize_100": 0,
"node_get_fsm_counter_objsize_95": 0,
"node_get_fsm_counter_objsize_99": 0,
"node_get_fsm_counter_objsize_mean": 0,
"node_get_fsm_counter_objsize_median": 0,
"node_get_fsm_counter_siblings_100": 0,
"node_get_fsm_counter_siblings_95": 0,
"node_get_fsm_counter_siblings_99": 0,
"node_get_fsm_counter_siblings_mean": 0,
"node_get_fsm_counter_siblings_median": 0,
"node_get_fsm_counter_time_100": 0,
"node_get_fsm_counter_time_95": 0,
"node_get_fsm_counter_time_99": 0,
"node_get_fsm_counter_time_mean": 0,
"node_get_fsm_counter_time_median": 0,
"node_get_fsm_errors": 0,
"node_get_fsm_errors_total": 0,
"node_get_fsm_in_rate": 0,
"node_get_fsm_map_objsize_100": 0,
"node_get_fsm_map_objsize_95": 0,
"node_get_fsm_map_objsize_99": 0,
"node_get_fsm_map_objsize_mean": 0,
"node_get_fsm_map_objsize_median": 0,
"node_get_fsm_map_siblings_100": 0,
"node_get_fsm_map_siblings_95": 0,
"node_get_fsm_map_siblings_99": 0,
"node_get_fsm_map_siblings_mean": 0,
"node_get_fsm_map_siblings_median": 0,
"node_get_fsm_map_time_100": 0,
"node_get_fsm_map_time_95": 0,
"node_get_fsm_map_time_99": 0,
"node_get_fsm_map_time_mean": 0,
"node_get_fsm_map_time_median": 0,
"node_get_fsm_objsize_100": 0,
"node_get_fsm_objsize_95": 0,
"node_get_fsm_objsize_99": 0,
"node_get_fsm_objsize_mean": 0,
"node_get_fsm_objsize_median": 0,
"node_get_fsm_out_rate": 0,
"node_get_fsm_rejected": 0,
"node_get_fsm_rejected_60s": 0,
"node_get_fsm_rejected_total": 0,
"node_get_fsm_set_objsize_100": 0,
"node_get_fsm_set_objsize_95": 0,
"node_get_fsm_set_objsize_99": 0,
"node_get_fsm_set_objsize_mean": 0,
"node_get_fsm_set_objsize_median": 0,
"node_get_fsm_set_siblings_100": 0,
"node_get_fsm_set_siblings_95": 0,
"node_get_fsm_set_siblings_99": 0,
"node_get_fsm_set_siblings_mean": 0,
"node_get_fsm_set_siblings_median": 0,
"node_get_fsm_set_time_100": 0,
"node_get_fsm_set_time_95": 0,
"node_get_fsm_set_time_99": 0,
"node_get_fsm_set_time_mean": 0,
"node_get_fsm_set_time_median": 0,
"node_get_fsm_siblings_100": 0,
"node_get_fsm_siblings_95": 0,
"node_get_fsm_siblings_99": 0,
"node_get_fsm_siblings_mean": 0,
"node_get_fsm_siblings_median": 0,
"node_get_fsm_time_100": 0,
"node_get_fsm_time_95": 0,
"node_get_fsm_time_99": 0,
"node_get_fsm_time_mean": 0,
"node_get_fsm_time_median": 0,
"node_gets": 0,
"node_gets_counter": 0,
"node_gets_counter_total": 0,
"node_gets_map": 0,
"node_gets_map_total": 0,
"node_gets_set": 0,
"node_gets_set_total": 0,
"node_gets_total": 500,
"node_put_fsm_active": 1,
"node_put_fsm_active_60s": 27,
"node_put_fsm_counter_time_100": 0,
"node_put_fsm_counter_time_95": 0,
"node_put_fsm_counter_time_99": 0,
"node_put_fsm_counter_time_mean": 0,
"node_put_fsm_counter_time_median": 0,
"node_put_fsm_in_rate": 0,
"node_put_fsm_map_time_100": 0,
"node_put_fsm_map_time_95": 0,
"node_put_fsm_map_time_99": 0,
"node_put_fsm_map_time_mean": 0,
"node_put_fsm_map_time_median": 0,
"node_put_fsm_out_rate": 0,
"node_put_fsm_rejected": 0,
"node_put_fsm_rejected_60s": 0,
"node_put_fsm_rejected_total": 0,
"node_put_fsm_set_time_100": 0,
"node_put_fsm_set_time_95": 0,
"node_put_fsm_set_time_99": 0,
"node_put_fsm_set_time_mean": 0,
"node_put_fsm_set_time_median": 0,
"node_put_fsm_time_100": 26949207,
"node_put_fsm_time_95": 23320347,
"node_put_fsm_time_99": 26949207,
"node_put_fsm_time_mean": 6500451,
"node_put_fsm_time_median": 960480,
"node_puts": 30,
"node_puts_counter": 0,
"node_puts_counter_total": 0,
"node_puts_map": 0,
"node_puts_map_total": 0,
"node_puts_set": 0,
"node_puts_set_total": 0,
"node_puts_total": 35224,
"nodename": "riak@10.31.0.69",
"object_counter_merge": 0,
"object_counter_merge_time_100": 0,
"object_counter_merge_time_95": 0,
"object_counter_merge_time_99": 0,
"object_counter_merge_time_mean": 0,
"object_counter_merge_time_median": 0,
"object_counter_merge_total": 0,
"object_map_merge": 0,
"object_map_merge_time_100": 0,
"object_map_merge_time_95": 0,
"object_map_merge_time_99": 0,
"object_map_merge_time_mean": 0,
"object_map_merge_time_median": 0,
"object_map_merge_total": 0,
"object_merge": 0,
"object_merge_time_100": 0,
"object_merge_time_95": 0,
"object_merge_time_99": 0,
"object_merge_time_mean": 0,
"object_merge_time_median": 0,
"object_merge_total": 9926,
"object_set_merge": 0,
"object_set_merge_time_100": 0,
"object_set_merge_time_95": 0,
"object_set_merge_time_99": 0,
"object_set_merge_time_mean": 0,
"object_set_merge_time_median": 0,
"object_set_merge_total": 0,
"os_mon_version": "2.2.13",
"pbc_active": 0,
"pbc_connects": 0,
"pbc_connects_total": 8,
"pbkdf2_version": "2.0.0-0-g7076584",
"pipeline_active": 0,
"pipeline_create_count": 0,
"pipeline_create_error_count": 0,
"pipeline_create_error_one": 0,
"pipeline_create_one": 0,
"poolboy_version": "0.8.1p3-0-g8bb45fb",
"postcommit_fail": 0,
"precommit_fail": 0,
"protobuffs_version": "0.8.1p5-0-gf88fc3c",
"public_key_version": "0.20",
"read_repairs": 0,
"read_repairs_counter": 0,
"read_repairs_counter_total": 0,
"read_repairs_fallback_notfound_count": "undefined",
"read_repairs_fallback_notfound_one": "undefined",
"read_repairs_fallback_outofdate_count": "undefined",
"read_repairs_fallback_outofdate_one": "undefined",
"read_repairs_map": 0,
"read_repairs_map_total": 0,
"read_repairs_primary_notfound_count": 146,
"read_repairs_primary_notfound_one": 0,
"read_repairs_primary_outofdate_count": 32,
"read_repairs_primary_outofdate_one": 0,
"read_repairs_set": 0,
"read_repairs_set_total": 0,
"read_repairs_total": 123,
"rebalance_delay_last": 0,
"rebalance_delay_max": 0,
"rebalance_delay_mean": 0,
"rebalance_delay_min": 0,
"rejected_handoffs": 32,
"riak_api_version": "2.1.1-2-g94a9485",
"riak_auth_mods_version": "2.0.1-0-g31b8b30",
"riak_control_version": "2.1.1-0-g5898c40",
"riak_core_version": "2.1.1-0-g429c22d",
"riak_dt_version": "2.1.0-2-ga2986bc",
"riak_kv_version": "2.1.0-0-g6e88b24",
"riak_kv_vnodeq_max": 0,
"riak_kv_vnodeq_mean": 0.0,
"riak_kv_vnodeq_median": 0,
"riak_kv_vnodeq_min": 0,
"riak_kv_vnodeq_total": 0,
"riak_kv_vnodes_running": 16,
"riak_pb_version": "2.1.0.2-0-g620bc70",
"riak_pipe_version": "2.1.0-2-gc2d7d28",
"riak_pipe_vnodeq_max": 0,
"riak_pipe_vnodeq_mean": 0.0,
"riak_pipe_vnodeq_median": 0,
"riak_pipe_vnodeq_min": 0,
"riak_pipe_vnodeq_total": 0,
"riak_pipe_vnodes_running": 16,
"riak_search_version": "2.0.2-0-g8fe4a8c",
"riak_sysmon_version": "2.0.0",
"ring_creation_size": 64,
"ring_members": [
"riak@10.31.0.65",
"riak@10.31.0.66",
"riak@10.31.0.67",
"riak@10.31.0.68",
"riak@10.31.0.69"
],
"ring_num_partitions": 64,
"ring_ownership": "[{'riak@10.31.0.65',12},\n {'riak@10.31.0.66',12},\n
{'riak@10.31.0.67',12},\n {'riak@10.31.0.68',12},\n {'riak@10.31.0.69',16}]",
"rings_reconciled": 0,
"rings_reconciled_total": 96,
"runtime_tools_version": "1.8.12",
"sasl_version": "2.3.3",
"search_index_fail_count": 0,
"search_index_fail_one": 0,
"search_index_latency_95": 20666433,
"search_index_latency_99": 24443579,
"search_index_latency_999": 26953345,
"search_index_latency_max": 26953345,
"search_index_latency_mean": 3000132,
"search_index_latency_median": 63194,
"search_index_latency_min": 797,
"search_index_throughput_count": 139189,
"search_index_throughput_one": 180,
"search_query_fail_count": 0,
"search_query_fail_one": 0,
"search_query_latency_95": 0,
"search_query_latency_99": 0,
"search_query_latency_999": 0,
"search_query_latency_max": 0,
"search_query_latency_mean": 0,
"search_query_latency_median": 0,
"search_query_latency_min": 0,
"search_query_throughput_count": 0,
"search_query_throughput_one": 0,
"set_actor_counts_100": 0,
"set_actor_counts_95": 0,
"set_actor_counts_99": 0,
"set_actor_counts_mean": 0,
"set_actor_counts_median": 0,
"sidejob_version": "2.0.0-0-gc5aabba",
"skipped_read_repairs": 0,
"skipped_read_repairs_total": 0,
"ssl_version": "5.3.1",
"stdlib_version": "1.19.3",
"storage_backend": "riak_kv_multi_backend",
"syntax_tools_version": "1.6.11",
"sys_driver_version": "2.2",
"sys_global_heaps_size": "deprecated",
"sys_heap_type": "private",
"sys_logical_processors": 8,
"sys_monitor_count": 666,
"sys_otp_release": "R16B02_basho8",
"sys_port_count": 170,
"sys_process_count": 1347,
"sys_smp_support": true,
"sys_system_architecture": "x86_64-unknown-linux-gnu",
"sys_system_version": "Erlang R16B02_basho8 (erts-5.10.3) [source] [64-bit]
[smp:8:8] [async-threads:64] [kernel-poll:true] [frame-pointer]",
"sys_thread_pool_size": 64,
"sys_threads_enabled": true,
"sys_wordsize": 8,
"vnode_counter_update": 0,
"vnode_counter_update_time_100": 0,
"vnode_counter_update_time_95": 0,
"vnode_counter_update_time_99": 0,
"vnode_counter_update_time_mean": 0,
"vnode_counter_update_time_median": 0,
"vnode_counter_update_total": 0,
"vnode_get_fsm_time_100": 238,
"vnode_get_fsm_time_95": 238,
"vnode_get_fsm_time_99": 238,
"vnode_get_fsm_time_mean": 220,
"vnode_get_fsm_time_median": 216,
"vnode_gets": 3,
"vnode_gets_total": 3710,
"vnode_index_deletes": 0,
"vnode_index_deletes_postings": 0,
"vnode_index_deletes_postings_total": 0,
"vnode_index_deletes_total": 0,
"vnode_index_reads": 0,
"vnode_index_reads_total": 0,
"vnode_index_refreshes": 0,
"vnode_index_refreshes_total": 0,
"vnode_index_writes": 180,
"vnode_index_writes_postings": 180,
"vnode_index_writes_postings_total": 120120,
"vnode_index_writes_total": 139258,
"vnode_map_update": 0,
"vnode_map_update_time_100": 0,
"vnode_map_update_time_95": 0,
"vnode_map_update_time_99": 0,
"vnode_map_update_time_mean": 0,
"vnode_map_update_time_median": 0,
"vnode_map_update_total": 0,
"vnode_put_fsm_time_100": 26953961,
"vnode_put_fsm_time_95": 20667408,
"vnode_put_fsm_time_99": 24444125,
"vnode_put_fsm_time_mean": 3016580,
"vnode_put_fsm_time_median": 63517,
"vnode_puts": 180,
"vnode_puts_total": 139403,
"vnode_set_update": 0,
"vnode_set_update_time_100": 0,
"vnode_set_update_time_95": 0,
"vnode_set_update_time_99": 0,
"vnode_set_update_time_mean": 0,
"vnode_set_update_time_median": 0,
"vnode_set_update_total": 0,
"webmachine_version": "1.10.8-0-g7677c24",
"write_once_merge": 0,
"write_once_put_objsize_100": 0,
"write_once_put_objsize_95": 0,
"write_once_put_objsize_99": 0,
"write_once_put_objsize_mean": 0,
"write_once_put_objsize_median": 0,
"write_once_put_time_100": 0,
"write_once_put_time_95": 0,
"write_once_put_time_99": 0,
"write_once_put_time_mean": 0,
"write_once_put_time_median": 0,
"write_once_puts": 0,
"write_once_puts_total": 0,
"xmerl_version": "1.3.4",
"yokozuna_version": "2.1.0-0-gcb41c27"
}
_______________________________________________
riak-users mailing list
riak-users@lists.basho.com
http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com
