Yokozuna indexes too slow

ilyas i.sergeev at keepsolid.com
Fri Oct 2 02:23:26 EDT 2015


It looks like this issues

https://github.com/basho/yokozuna/issues/320

I try to set

  * |maxThreads| to |150|
  * |Acceptors| to |10|
  * |lowResourcesMaxIdleTime| to |50000|

in /usr/lib/riak/lib/yokozuna/priv/solr/etc/jetty.xml as recommended in 
https://github.com/basho/yokozuna/issues/330

but it has no effect

On 10/01/2015 11:53 PM, Fred Dushin wrote:
> Is there any more information in these logs that you can share?  For example, is this the only entry with this exception?  Or are there more?  Are there any associated stack traces?  An EOF exception can come from many different scenarios.
>
> Is there anything in the Riak console.log that looks suspicious?
>
> Finally, you might want to take a look at what is going on inside of riak when you get into this state (slow writes to Solr), by looking at Riak stats.
>
> You get get to Riak stats via curl, e.g.,
>
> 	curlhttp://localhost:8098/stats  | python -m json.tool
ok, output is attached

> Stats you might want to pay special attention to:
>
> riak_kv_vnodeq (min, max, median, etc)  -- the aggregate length of the vnode queues.  Long vnode queues may mean your vnode is locked waiting on Solr
> vnode_put_fsm_time (mean, media, percentile, etc) The amount of time spent on average waiting for a vnode put to complete.  Long times may also be indicative of waits writing into Solr.
"riak_kv_vnodeq_max": 0,
"riak_kv_vnodeq_mean": 0.0,
"riak_kv_vnodeq_median": 0,
"riak_kv_vnodeq_min": 0,
"riak_kv_vnodeq_total": 0,


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.basho.com/pipermail/riak-users_lists.basho.com/attachments/20151002/f36868f8/attachment-0002.html>
-------------- next part --------------
{
    "asn1_version": "2.0.3",
    "basho_stats_version": "1.0.3",
    "bitcask_version": "1.7.0",
    "clique_version": "0.2.6-0-g40072d2",
    "cluster_info_version": "2.0.2-0-ge231144",
    "compiler_version": "4.9.3",
    "connected_nodes": [
        "riak at 10.31.0.68",
        "riak at 10.31.0.67",
        "riak at 10.31.0.66",
        "riak at 10.31.0.65"
    ],
    "consistent_get_objsize_100": 0,
    "consistent_get_objsize_95": 0,
    "consistent_get_objsize_99": 0,
    "consistent_get_objsize_mean": 0,
    "consistent_get_objsize_median": 0,
    "consistent_get_time_100": 0,
    "consistent_get_time_95": 0,
    "consistent_get_time_99": 0,
    "consistent_get_time_mean": 0,
    "consistent_get_time_median": 0,
    "consistent_gets": 0,
    "consistent_gets_total": 0,
    "consistent_put_objsize_100": 0,
    "consistent_put_objsize_95": 0,
    "consistent_put_objsize_99": 0,
    "consistent_put_objsize_mean": 0,
    "consistent_put_objsize_median": 0,
    "consistent_put_time_100": 0,
    "consistent_put_time_95": 0,
    "consistent_put_time_99": 0,
    "consistent_put_time_mean": 0,
    "consistent_put_time_median": 0,
    "consistent_puts": 0,
    "consistent_puts_total": 0,
    "converge_delay_last": 0,
    "converge_delay_max": 0,
    "converge_delay_mean": 0,
    "converge_delay_min": 0,
    "coord_redirs_total": 5066,
    "counter_actor_counts_100": 0,
    "counter_actor_counts_95": 0,
    "counter_actor_counts_99": 0,
    "counter_actor_counts_mean": 0,
    "counter_actor_counts_median": 0,
    "cpu_avg1": 82,
    "cpu_avg15": 123,
    "cpu_avg5": 108,
    "cpu_nprocs": 552,
    "crypto_version": "3.1",
    "disk": [
        {
            "id": "/",
            "size": 474203304,
            "used": 2
        },
        {
            "id": "/dev",
            "size": 10240,
            "used": 0
        },
        {
            "id": "/run",
            "size": 3285280,
            "used": 10
        },
        {
            "id": "/dev/shm",
            "size": 8213192,
            "used": 0
        },
        {
            "id": "/run/lock",
            "size": 5120,
            "used": 0
        },
        {
            "id": "/sys/fs/cgroup",
            "size": 8213192,
            "used": 0
        },
        {
            "id": "/tmp",
            "size": 1933648,
            "used": 1
        },
        {
            "id": "/boot",
            "size": 457760,
            "used": 12
        },
        {
            "id": "/var/lib/riak",
            "size": 468361760,
            "used": 1
        },
        {
            "id": "/run/user/1004",
            "size": 1642640,
            "used": 0
        },
        {
            "id": "/run/user/1001",
            "size": 1642640,
            "used": 0
        }
    ],
    "dropped_vnode_requests_total": 0,
    "eleveldb_version": "2.1.0-0-ga36dbd6",
    "erlang_js_version": "1.3.0-0-g07467d8",
    "erlydtl_version": "0.7.0",
    "executing_mappers": 0,
    "exometer_core_version": "1.0.0-basho2-0-gb47a5d6",
    "goldrush_version": "0.1.6",
    "gossip_received": 7,
    "handoff_timeouts": 0,
    "ibrowse_version": "4.0.2",
    "ignored_gossip_total": 0,
    "index_fsm_active": 0,
    "index_fsm_create": 0,
    "index_fsm_create_error": 0,
    "inets_version": "5.9.6",
    "kernel_version": "2.16.3",
    "lager_version": "2.0.3",
    "late_put_fsm_coordinator_ack": 0,
    "leveldb_read_block_error": 0,
    "list_fsm_active": 0,
    "list_fsm_create": 0,
    "list_fsm_create_error": 0,
    "list_fsm_create_error_total": 0,
    "list_fsm_create_total": 0,
    "map_actor_counts_100": 0,
    "map_actor_counts_95": 0,
    "map_actor_counts_99": 0,
    "map_actor_counts_mean": 0,
    "map_actor_counts_median": 0,
    "mem_allocated": 7542112256,
    "mem_total": 16820621312,
    "memory_atom": 752537,
    "memory_atom_used": 745783,
    "memory_binary": 68929880,
    "memory_code": 16809458,
    "memory_ets": 8286048,
    "memory_processes": 114315184,
    "memory_processes_used": 114277960,
    "memory_system": 115784256,
    "memory_total": 230099440,
    "merge_index_version": "2.0.0-0-gb701dde",
    "mochiweb_version": "2.9.0",
    "node_get_fsm_active": 0,
    "node_get_fsm_active_60s": 0,
    "node_get_fsm_counter_objsize_100": 0,
    "node_get_fsm_counter_objsize_95": 0,
    "node_get_fsm_counter_objsize_99": 0,
    "node_get_fsm_counter_objsize_mean": 0,
    "node_get_fsm_counter_objsize_median": 0,
    "node_get_fsm_counter_siblings_100": 0,
    "node_get_fsm_counter_siblings_95": 0,
    "node_get_fsm_counter_siblings_99": 0,
    "node_get_fsm_counter_siblings_mean": 0,
    "node_get_fsm_counter_siblings_median": 0,
    "node_get_fsm_counter_time_100": 0,
    "node_get_fsm_counter_time_95": 0,
    "node_get_fsm_counter_time_99": 0,
    "node_get_fsm_counter_time_mean": 0,
    "node_get_fsm_counter_time_median": 0,
    "node_get_fsm_errors": 0,
    "node_get_fsm_errors_total": 0,
    "node_get_fsm_in_rate": 0,
    "node_get_fsm_map_objsize_100": 0,
    "node_get_fsm_map_objsize_95": 0,
    "node_get_fsm_map_objsize_99": 0,
    "node_get_fsm_map_objsize_mean": 0,
    "node_get_fsm_map_objsize_median": 0,
    "node_get_fsm_map_siblings_100": 0,
    "node_get_fsm_map_siblings_95": 0,
    "node_get_fsm_map_siblings_99": 0,
    "node_get_fsm_map_siblings_mean": 0,
    "node_get_fsm_map_siblings_median": 0,
    "node_get_fsm_map_time_100": 0,
    "node_get_fsm_map_time_95": 0,
    "node_get_fsm_map_time_99": 0,
    "node_get_fsm_map_time_mean": 0,
    "node_get_fsm_map_time_median": 0,
    "node_get_fsm_objsize_100": 0,
    "node_get_fsm_objsize_95": 0,
    "node_get_fsm_objsize_99": 0,
    "node_get_fsm_objsize_mean": 0,
    "node_get_fsm_objsize_median": 0,
    "node_get_fsm_out_rate": 0,
    "node_get_fsm_rejected": 0,
    "node_get_fsm_rejected_60s": 0,
    "node_get_fsm_rejected_total": 0,
    "node_get_fsm_set_objsize_100": 0,
    "node_get_fsm_set_objsize_95": 0,
    "node_get_fsm_set_objsize_99": 0,
    "node_get_fsm_set_objsize_mean": 0,
    "node_get_fsm_set_objsize_median": 0,
    "node_get_fsm_set_siblings_100": 0,
    "node_get_fsm_set_siblings_95": 0,
    "node_get_fsm_set_siblings_99": 0,
    "node_get_fsm_set_siblings_mean": 0,
    "node_get_fsm_set_siblings_median": 0,
    "node_get_fsm_set_time_100": 0,
    "node_get_fsm_set_time_95": 0,
    "node_get_fsm_set_time_99": 0,
    "node_get_fsm_set_time_mean": 0,
    "node_get_fsm_set_time_median": 0,
    "node_get_fsm_siblings_100": 0,
    "node_get_fsm_siblings_95": 0,
    "node_get_fsm_siblings_99": 0,
    "node_get_fsm_siblings_mean": 0,
    "node_get_fsm_siblings_median": 0,
    "node_get_fsm_time_100": 0,
    "node_get_fsm_time_95": 0,
    "node_get_fsm_time_99": 0,
    "node_get_fsm_time_mean": 0,
    "node_get_fsm_time_median": 0,
    "node_gets": 0,
    "node_gets_counter": 0,
    "node_gets_counter_total": 0,
    "node_gets_map": 0,
    "node_gets_map_total": 0,
    "node_gets_set": 0,
    "node_gets_set_total": 0,
    "node_gets_total": 500,
    "node_put_fsm_active": 1,
    "node_put_fsm_active_60s": 27,
    "node_put_fsm_counter_time_100": 0,
    "node_put_fsm_counter_time_95": 0,
    "node_put_fsm_counter_time_99": 0,
    "node_put_fsm_counter_time_mean": 0,
    "node_put_fsm_counter_time_median": 0,
    "node_put_fsm_in_rate": 0,
    "node_put_fsm_map_time_100": 0,
    "node_put_fsm_map_time_95": 0,
    "node_put_fsm_map_time_99": 0,
    "node_put_fsm_map_time_mean": 0,
    "node_put_fsm_map_time_median": 0,
    "node_put_fsm_out_rate": 0,
    "node_put_fsm_rejected": 0,
    "node_put_fsm_rejected_60s": 0,
    "node_put_fsm_rejected_total": 0,
    "node_put_fsm_set_time_100": 0,
    "node_put_fsm_set_time_95": 0,
    "node_put_fsm_set_time_99": 0,
    "node_put_fsm_set_time_mean": 0,
    "node_put_fsm_set_time_median": 0,
    "node_put_fsm_time_100": 26949207,
    "node_put_fsm_time_95": 23320347,
    "node_put_fsm_time_99": 26949207,
    "node_put_fsm_time_mean": 6500451,
    "node_put_fsm_time_median": 960480,
    "node_puts": 30,
    "node_puts_counter": 0,
    "node_puts_counter_total": 0,
    "node_puts_map": 0,
    "node_puts_map_total": 0,
    "node_puts_set": 0,
    "node_puts_set_total": 0,
    "node_puts_total": 35224,
    "nodename": "riak at 10.31.0.69",
    "object_counter_merge": 0,
    "object_counter_merge_time_100": 0,
    "object_counter_merge_time_95": 0,
    "object_counter_merge_time_99": 0,
    "object_counter_merge_time_mean": 0,
    "object_counter_merge_time_median": 0,
    "object_counter_merge_total": 0,
    "object_map_merge": 0,
    "object_map_merge_time_100": 0,
    "object_map_merge_time_95": 0,
    "object_map_merge_time_99": 0,
    "object_map_merge_time_mean": 0,
    "object_map_merge_time_median": 0,
    "object_map_merge_total": 0,
    "object_merge": 0,
    "object_merge_time_100": 0,
    "object_merge_time_95": 0,
    "object_merge_time_99": 0,
    "object_merge_time_mean": 0,
    "object_merge_time_median": 0,
    "object_merge_total": 9926,
    "object_set_merge": 0,
    "object_set_merge_time_100": 0,
    "object_set_merge_time_95": 0,
    "object_set_merge_time_99": 0,
    "object_set_merge_time_mean": 0,
    "object_set_merge_time_median": 0,
    "object_set_merge_total": 0,
    "os_mon_version": "2.2.13",
    "pbc_active": 0,
    "pbc_connects": 0,
    "pbc_connects_total": 8,
    "pbkdf2_version": "2.0.0-0-g7076584",
    "pipeline_active": 0,
    "pipeline_create_count": 0,
    "pipeline_create_error_count": 0,
    "pipeline_create_error_one": 0,
    "pipeline_create_one": 0,
    "poolboy_version": "0.8.1p3-0-g8bb45fb",
    "postcommit_fail": 0,
    "precommit_fail": 0,
    "protobuffs_version": "0.8.1p5-0-gf88fc3c",
    "public_key_version": "0.20",
    "read_repairs": 0,
    "read_repairs_counter": 0,
    "read_repairs_counter_total": 0,
    "read_repairs_fallback_notfound_count": "undefined",
    "read_repairs_fallback_notfound_one": "undefined",
    "read_repairs_fallback_outofdate_count": "undefined",
    "read_repairs_fallback_outofdate_one": "undefined",
    "read_repairs_map": 0,
    "read_repairs_map_total": 0,
    "read_repairs_primary_notfound_count": 146,
    "read_repairs_primary_notfound_one": 0,
    "read_repairs_primary_outofdate_count": 32,
    "read_repairs_primary_outofdate_one": 0,
    "read_repairs_set": 0,
    "read_repairs_set_total": 0,
    "read_repairs_total": 123,
    "rebalance_delay_last": 0,
    "rebalance_delay_max": 0,
    "rebalance_delay_mean": 0,
    "rebalance_delay_min": 0,
    "rejected_handoffs": 32,
    "riak_api_version": "2.1.1-2-g94a9485",
    "riak_auth_mods_version": "2.0.1-0-g31b8b30",
    "riak_control_version": "2.1.1-0-g5898c40",
    "riak_core_version": "2.1.1-0-g429c22d",
    "riak_dt_version": "2.1.0-2-ga2986bc",
    "riak_kv_version": "2.1.0-0-g6e88b24",
    "riak_kv_vnodeq_max": 0,
    "riak_kv_vnodeq_mean": 0.0,
    "riak_kv_vnodeq_median": 0,
    "riak_kv_vnodeq_min": 0,
    "riak_kv_vnodeq_total": 0,
    "riak_kv_vnodes_running": 16,
    "riak_pb_version": "2.1.0.2-0-g620bc70",
    "riak_pipe_version": "2.1.0-2-gc2d7d28",
    "riak_pipe_vnodeq_max": 0,
    "riak_pipe_vnodeq_mean": 0.0,
    "riak_pipe_vnodeq_median": 0,
    "riak_pipe_vnodeq_min": 0,
    "riak_pipe_vnodeq_total": 0,
    "riak_pipe_vnodes_running": 16,
    "riak_search_version": "2.0.2-0-g8fe4a8c",
    "riak_sysmon_version": "2.0.0",
    "ring_creation_size": 64,
    "ring_members": [
        "riak at 10.31.0.65",
        "riak at 10.31.0.66",
        "riak at 10.31.0.67",
        "riak at 10.31.0.68",
        "riak at 10.31.0.69"
    ],
    "ring_num_partitions": 64,
    "ring_ownership": "[{'riak at 10.31.0.65',12},\n {'riak at 10.31.0.66',12},\n {'riak at 10.31.0.67',12},\n {'riak at 10.31.0.68',12},\n {'riak at 10.31.0.69',16}]",
    "rings_reconciled": 0,
    "rings_reconciled_total": 96,
    "runtime_tools_version": "1.8.12",
    "sasl_version": "2.3.3",
    "search_index_fail_count": 0,
    "search_index_fail_one": 0,
    "search_index_latency_95": 20666433,
    "search_index_latency_99": 24443579,
    "search_index_latency_999": 26953345,
    "search_index_latency_max": 26953345,
    "search_index_latency_mean": 3000132,
    "search_index_latency_median": 63194,
    "search_index_latency_min": 797,
    "search_index_throughput_count": 139189,
    "search_index_throughput_one": 180,
    "search_query_fail_count": 0,
    "search_query_fail_one": 0,
    "search_query_latency_95": 0,
    "search_query_latency_99": 0,
    "search_query_latency_999": 0,
    "search_query_latency_max": 0,
    "search_query_latency_mean": 0,
    "search_query_latency_median": 0,
    "search_query_latency_min": 0,
    "search_query_throughput_count": 0,
    "search_query_throughput_one": 0,
    "set_actor_counts_100": 0,
    "set_actor_counts_95": 0,
    "set_actor_counts_99": 0,
    "set_actor_counts_mean": 0,
    "set_actor_counts_median": 0,
    "sidejob_version": "2.0.0-0-gc5aabba",
    "skipped_read_repairs": 0,
    "skipped_read_repairs_total": 0,
    "ssl_version": "5.3.1",
    "stdlib_version": "1.19.3",
    "storage_backend": "riak_kv_multi_backend",
    "syntax_tools_version": "1.6.11",
    "sys_driver_version": "2.2",
    "sys_global_heaps_size": "deprecated",
    "sys_heap_type": "private",
    "sys_logical_processors": 8,
    "sys_monitor_count": 666,
    "sys_otp_release": "R16B02_basho8",
    "sys_port_count": 170,
    "sys_process_count": 1347,
    "sys_smp_support": true,
    "sys_system_architecture": "x86_64-unknown-linux-gnu",
    "sys_system_version": "Erlang R16B02_basho8 (erts-5.10.3) [source] [64-bit] [smp:8:8] [async-threads:64] [kernel-poll:true] [frame-pointer]",
    "sys_thread_pool_size": 64,
    "sys_threads_enabled": true,
    "sys_wordsize": 8,
    "vnode_counter_update": 0,
    "vnode_counter_update_time_100": 0,
    "vnode_counter_update_time_95": 0,
    "vnode_counter_update_time_99": 0,
    "vnode_counter_update_time_mean": 0,
    "vnode_counter_update_time_median": 0,
    "vnode_counter_update_total": 0,
    "vnode_get_fsm_time_100": 238,
    "vnode_get_fsm_time_95": 238,
    "vnode_get_fsm_time_99": 238,
    "vnode_get_fsm_time_mean": 220,
    "vnode_get_fsm_time_median": 216,
    "vnode_gets": 3,
    "vnode_gets_total": 3710,
    "vnode_index_deletes": 0,
    "vnode_index_deletes_postings": 0,
    "vnode_index_deletes_postings_total": 0,
    "vnode_index_deletes_total": 0,
    "vnode_index_reads": 0,
    "vnode_index_reads_total": 0,
    "vnode_index_refreshes": 0,
    "vnode_index_refreshes_total": 0,
    "vnode_index_writes": 180,
    "vnode_index_writes_postings": 180,
    "vnode_index_writes_postings_total": 120120,
    "vnode_index_writes_total": 139258,
    "vnode_map_update": 0,
    "vnode_map_update_time_100": 0,
    "vnode_map_update_time_95": 0,
    "vnode_map_update_time_99": 0,
    "vnode_map_update_time_mean": 0,
    "vnode_map_update_time_median": 0,
    "vnode_map_update_total": 0,
    "vnode_put_fsm_time_100": 26953961,
    "vnode_put_fsm_time_95": 20667408,
    "vnode_put_fsm_time_99": 24444125,
    "vnode_put_fsm_time_mean": 3016580,
    "vnode_put_fsm_time_median": 63517,
    "vnode_puts": 180,
    "vnode_puts_total": 139403,
    "vnode_set_update": 0,
    "vnode_set_update_time_100": 0,
    "vnode_set_update_time_95": 0,
    "vnode_set_update_time_99": 0,
    "vnode_set_update_time_mean": 0,
    "vnode_set_update_time_median": 0,
    "vnode_set_update_total": 0,
    "webmachine_version": "1.10.8-0-g7677c24",
    "write_once_merge": 0,
    "write_once_put_objsize_100": 0,
    "write_once_put_objsize_95": 0,
    "write_once_put_objsize_99": 0,
    "write_once_put_objsize_mean": 0,
    "write_once_put_objsize_median": 0,
    "write_once_put_time_100": 0,
    "write_once_put_time_95": 0,
    "write_once_put_time_99": 0,
    "write_once_put_time_mean": 0,
    "write_once_put_time_median": 0,
    "write_once_puts": 0,
    "write_once_puts_total": 0,
    "xmerl_version": "1.3.4",
    "yokozuna_version": "2.1.0-0-gcb41c27"
}




More information about the riak-users mailing list