Re: Re: Riak performance problems when LevelDB database grows beyond 16GB

Jan.Evangelista at seznam.cz Jan.Evangelista at seznam.cz
Fri Oct 12 08:02:36 EDT 2012


> Can you attach the eleveldb portion of your app.config file?
> Configuration problems, especially max_open_files being too low, can
> often cause issues like this.
>
> If it isn't sensitive, the whole app.config and vm.args files are also
> often helpful.

Hello Evan,

thanks for responding.

I originally had default LevelDB settings. When the node stalled, I changed it
 to

 {eleveldb, [
             {data_root, "/home/riak/leveldb"},
             {max_open_files, 132},
             {cache_size, 377487360}
            ]},

on all nodes and I restarted them all. The application started to run with 
about 1000 requests/second, after about 1 minute it dropped to <500 
requests/second, and the node stalled again after 41 minutes. BTW according to
 lsof(1) it had 267 open LevelDB files which is more than the 132 files limit 
(??).

Here is the configuration (with comments removed to work around 40 KB message 
limit on the Riak users list).

/etc/riak/vm.args:

-name riak at 172.16.0.2
-setcookie riak_0627f0bc6c9
+K true
+A 64
+W w
-env ERL_MAX_PORTS 4096
-env ERL_FULLSWEEP_AFTER 0
-env ERL_CRASH_DUMP /var/log/riak/erl_crash.dump
-env ERL_MAX_ETS_TABLES 8192

/etc/riak/app.config:

[
 {riak_api, [
            {pb_ip,   "172.16.0.2" },
            {pb_port, 8087 }
            ]},
 {riak_core, [
              {ring_state_dir, "/var/lib/riak/ring"},
              %{ring_creation_size, 64},
              {http, [ {"127.0.0.1", 8098 } ]},
              %{https, [{ "127.0.0.1", 8098 }]},
              %{ssl, [
              %       {certfile, "/etc/riak/cert.pem"},
              %       {keyfile, "/etc/riak/key.pem"}
              %      ]},
              {handoff_port, 8099 },
              %{handoff_ssl_options, [{certfile, "/tmp/erlserver.pem"}]},
              {dtrace_support, false},
              {platform_bin_dir, "/usr/sbin"},
              {platform_data_dir, "/var/lib/riak"},
              {platform_etc_dir, "/etc/riak"},
              {platform_lib_dir, "/usr/lib/riak/lib"},
              {platform_log_dir, "/var/log/riak"}
             ]},
 {riak_kv, [
            {add_paths, 
["/usr/local/share/riak-mapred/riak-1.2.0/deps/greylisting/ebin/"]},
            {storage_backend, riak_kv_eleveldb_backend},
            %{raw_name, "riak"},
            {mapred_name, "mapred"},
            {mapred_system, pipe},
            {mapred_2i_pipe, true},
            {map_js_vm_count, 8 },
            {reduce_js_vm_count, 6 },
            {hook_js_vm_count, 2 },
            {js_max_vm_mem, 8},
            {js_thread_stack, 16},
            %{js_source_dir, "/tmp/js_source"},
            {http_url_encoding, on},
            {vnode_vclocks, true},
            {legacy_keylisting, false},
            {listkeys_backpressure, true}
           ]},
 {riak_search, [
                {enabled, false}
               ]},
 {merge_index, [
                {data_root, "/var/lib/riak/merge_index"},
                {buffer_rollover_size, 1048576},
                {max_compact_segments, 20}
               ]},
 {bitcask, [
             {data_root, "/var/lib/riak/bitcask"}
           ]},
 {eleveldb, [
             {data_root, "/home/riak/leveldb"},
             {max_open_files, 132},
             {cache_size, 377487360}
            ]},
 {lager, [
            {handlers, [
                {lager_console_backend, info},
                {lager_file_backend, [
                    {"/var/log/riak/error.log", error, 10485760, "$D0", 5},
                    {"/var/log/riak/console.log", info, 10485760, "$D0", 5}
                ]}
            ]},
            {crash_log, "/var/log/riak/crash.log"},
            {crash_log_msg_size, 65536},
            {crash_log_size, 10485760},
            {crash_log_date, "$D0"},
            {crash_log_count, 5},
            {error_logger_redirect, true}
        ]},
 {riak_sysmon, [
         {process_limit, 30},
         {port_limit, 2},
         {gc_ms_limit, 100},
         {heap_word_limit, 40111000},
         {busy_port, true},
         {busy_dist_port, true}
        ]},
 {sasl, [
         {sasl_error_logger, false}
        ]},
 {riak_control, [
                {enabled, false},
                {auth, userlist},
                {userlist, [{"user", "pass"}
                           ]},
                {admin, true}
                ]}
].

Thanks.
--
Jan




More information about the riak-users mailing list