Riak pre-commit hook crashing unexpectedly

David Rogers predictivestatmech at gmail.com
Sun Dec 6 22:34:43 EST 2015


Hello,

I'm using riak {release,"riak","2.1.0","5.10.3",
           [{kernel,"2.16.3",
"$HOME/src/riak-2.1.1/rel/riak/lib/kernel-2.16.3"} ...}
on a cluster with 1 Linux 3.16.0-38 and 4 OSX machines.

I have installed the following pre-commit hook (see end of email) to 
validate protobuf-formatted values whose key should be the sha1-hash of 
part of their payload.

     props = 
{"props":{"allow_mult":false,"basic_quorum":false,"big_vclock":50,"chash_keyfun":{"mod":"riak_core_util","fun":"chash_std_keyfun"},"dvv_enabled":false,"dw":"quorum","last_write_wins":false,"linkfun":{"mod":"riak_kv_wm_link_walker","fun":"mapreduce_linkfun"},"n_val":3,"name":"sil/code","notfound_ok":true,"old_vclock":86400,"postcommit":[],"pr":0,"precommit":[{"mod":"validate_hash","fun":"validate"}],"pw":0,"r":"quorum","rw":"quorum","small_vclock":50,"w":"quorum","write_once":false,"young_vclock":20}})

I attached the failed data element.  I'm using the riak-c-client to 
interface.

   Although almost everything I've added to the bucket this way has 
worked so far, a few (like the sample I sent) fail, with riak_put 
returning only the unhelpful ERIAK_SERVER_ERROR ("An error was returned 
from the server").  This happens even though manually running the 
validation works fine.  It is consistently reproducible on this input.

   In my attempt to track down the error, I added a test at the end of 
riak_sync_request (below) that will log all server errors.  The only 
problem is that cfg->log_fn is always NULL when riak_log_error is 
called, rather than what I set it to initially with 
riak_config_set_logging!  Gdb can't seem to catch any writes there, so 
maybe cfg gets incompletely copied somewhere?

   Anyway, I can get out the error message from the debugger.

riak_sync_request (rop_target=rop_target at entry=0x7fffffffe0b0, 
response=response at entry=0x7fffffffe130)
     at src/riak.c:81
81            riak_log_error(cxn, "%.*s\n", (int)msg->len, msg->data);
(gdb) print msg->data
$13 = (riak_uint8_t *) 0x670a40 
"{precommit_fail,{hook_crashed,{validate_hash,validate,error,badarg}}}"

so the server error shows only that the precommit hook 
(validate_hash:validate/1) crashes.  Again, manually running doesn't 
crash it...

Questions:

   First, I don't understand why riak-c-client doesn't call the function 
I supplied to riak_config_set_logging.  Second, I don't know where to 
look for more details on why the pre-commit hook crashes only when I 
actually try to add this particular key/value. Can anyone spot a problem 
in its error handling or suggest a way to debug?

Note: I get slightly more info. when posting from curl,
$ curl -XPOST 
http://127.0.0.1:8098/types/default/buckets/sil%2Fcode/keys/$hash -H 
'Content-Type: application/octet-stream' --data-binary @"$hash"
<html><head><title>500 Internal Server 
Error</title></head><body><h1>Internal Server Error</h1>The server 
encountered an error while processing this request:<br><pre>{error,
     {error,badarg,
         [{erlang,iolist_to_binary,
              [{hook_crashed,{validate_hash,validate,error,badarg}}],
              []},
{wrq,append_to_response_body,2,[{file,"src/wrq.erl"},{line,215}]},
          {riak_kv_wm_object,handle_common_error,3,
              [{file,"src/riak_kv_wm_object.erl"},{line,1178}]},
          {webmachine_resource,resource_call,3,
              [{file,"src/webmachine_resource.erl"},{line,186}]},
          {webmachine_resource,do,3,
              [{file,"src/webmachine_resource.erl"},{line,142}]},
          {webmachine_decision_core,resource_call,1,
              [{file,"src/webmachine_decision_core.erl"},{line,48}]},
          {webmachine_decision_core,decision,1,
[{file,"src/webmachine_decision_core.erl"},{line,490}]},
          {webmachine_decision_core,handle_request,2,
[{file,"src/webmachine_decision_core.erl"},{line,33}]}]}}</pre><P><HR><ADDRESS>mochiweb+webmachine 
web server</ADDRESS></body></html>

Sincerely,
~ David M. Rogers

signed.proto:
```
message sign {
     required bytes  signer  = 1;
     required uint32 dig_alg = 2;
     required bytes  sign    = 3;
     required uint64 ctime   = 4;
     optional uint32 flags   = 5;

     optional bytes  obj     = 10;
}
```


validate_hash.erl:
```
-module(validate_hash).
-export([validate/1, ck_hash/2, start/0]).
%-on_load(load_proto/0).

-author("David M. Rogers <predictivestatmech at gmail.com>").

%load_proto() ->
%    protobuffs_compile:scan_file("signed.proto").

validate(Object) ->
   try
     dict:is_key(<<"X-Riak-Deleted">>, riak_object:get_metadata(Object)) of
         true -> Object;
         false -> correct_hash(Object)
   catch
     error:Error ->
       {fail, "Invalid Commit: " ++ 
binary_to_list(list_to_binary(io_lib:format("~p", [Error])))}
   end.

correct_hash(Object) ->
     Msg = term_to_binary(riak_object:get_value(Object)),
     Hash = binary_to_list(riak_object:key(Object)),
     case ck_hash(Hash, Msg) of
         true  -> Object;
         false -> {fail, "Invalid Commit: Bad hash value."}
     end.

%% Hash : string of 40 hex chars
%% Msg : protocol buffer sign message (signed.proto)
%%  returns bool
ck_hash(Hash, Msg) ->
     Obj = signed_pb:decode_sign(Msg), %% signed and sobject come from 
file naming
     Digest = crypto:hash(sha, element(7,Obj)),
     bin_to_hexstr(Digest) == Hash
   .

bin_to_hexstr(Bin) ->
       lists:flatten([io_lib:format("~2.16.0b", [X]) ||
                          X <- binary_to_list(Bin)]).

hexstr_to_bin(S) ->
       hexstr_to_bin(S, []).
hexstr_to_bin([], Acc) ->
       list_to_binary(lists:reverse(Acc));
hexstr_to_bin([X,Y|T], Acc) ->
       {ok, [V], []} = io_lib:fread("~16u", [X,Y]),
         hexstr_to_bin(T, [V | Acc]).

start() ->
     Hash = "fc144f6728c994a10309a922d0fd9758b9999cc4",
     {ok, Msg} = file:read_file(Hash),
     Ret = ck_hash(Hash, Msg),
     io:fwrite( "Returned: ~p.~n", [Ret] ).
```


-------------- next part --------------
A non-text attachment was scrubbed...
Name: fc144f6728c994a10309a922d0fd9758b9999cc4
Type: application/octet-stream
Size: 747 bytes
Desc: not available
URL: <http://lists.basho.com/pipermail/riak-users_lists.basho.com/attachments/20151206/8c3ce183/attachment.obj>


More information about the riak-users mailing list