diff --git a/.github/workflows/erlang.yml b/.github/workflows/erlang.yml index c142f59c9..6b174f3dc 100644 --- a/.github/workflows/erlang.yml +++ b/.github/workflows/erlang.yml @@ -34,3 +34,5 @@ jobs: run: ./rebar3 do xref, dialyzer - name: Run eunit run: ./rebar3 as gha do eunit + - name: Check format + run: ./rebar3 fmt --check diff --git a/eqc/kv_crdt_eqc.erl b/eqc/kv_crdt_eqc.erl deleted file mode 100644 index b791bc5a1..000000000 --- a/eqc/kv_crdt_eqc.erl +++ /dev/null @@ -1,295 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% kv_counter_eqc: Quickcheck test for riak_kv_counter -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - --module(kv_crdt_eqc). - --ifdef(EQC). --include_lib("eqc/include/eqc.hrl"). --include_lib("eunit/include/eunit.hrl"). --include("include/riak_kv_types.hrl"). --include("src/riak_kv_wm_raw.hrl"). - --compile([export_all, nowarn_export_all]). - --define(TAG, 69). --define(V1_VERS, 1). - --define(BUCKET, <<"b">>). --define(KEY, <<"k">>). --define(NUMTESTS, 500). --define(QC_OUT(P), - eqc:on_output(fun(Str, Args) -> - io:format(user, Str, Args) end, P)). - -%%==================================================================== -%% properties -%%==================================================================== - -prop_value() -> - ?FORALL(RObj, riak_object(), - begin - Expected = sumthem(RObj), - {{_, Cnt},_} = riak_kv_crdt:value(RObj, ?V1_COUNTER_TYPE), - ?WHENFAIL( - begin - io:format(user, "Gen ~p\n", [RObj]) - end, - equals(Expected, Cnt)) - end). - -prop_merge() -> - ?FORALL(RObj, riak_object(), - begin - Merged = riak_kv_crdt:merge(RObj), - FExpectedCounters = fun(NumGeneratedCounters) -> - case NumGeneratedCounters of - 0 -> 0; - _ -> 1 - end - end, - MergeSeed = undefined, - {{_, Cnt},_} = riak_kv_crdt:value(Merged, ?V1_COUNTER_TYPE), - - ?WHENFAIL( - begin - io:format("Gen ~p\n", [RObj]), - io:format("Merged ~p\n", [Merged]) - end, - - conjunction([ - {value, equals(sumthem(RObj), Cnt)}, - {verify_merge, verify_merge(RObj, Merged, FExpectedCounters, MergeSeed)} - ])) - end). - -prop_update() -> - ?FORALL({RObj, Actor, Amt}, - {riak_object(), noshrink(binary(4)), int()}, - begin - CounterOp = counter_op(Amt), - Op = ?CRDT_OP{mod=?V1_COUNTER_TYPE, op=CounterOp}, - Updated = riak_kv_crdt:update(RObj, Actor, Op), - FExpectedCounters = fun(_NumGeneratedCounters) -> - 1 - end, - MergeSeed = case Amt of - 0 -> ?V1_COUNTER_TYPE:new(Actor, Amt); - _ -> {ok, Cnter} = ?V1_COUNTER_TYPE:new(Actor, Amt), - Cnter - end, - {{_, Cnt},_} = riak_kv_crdt:value(Updated, ?V1_COUNTER_TYPE), - ?WHENFAIL( - begin - io:format("Gen ~p~n", [RObj]), - io:format("Updated ~p~n", [Updated]), - io:format("Amt ~p~n", [Amt]) - end, - conjunction([ - {counter_value, equals(sumthem(RObj) + Amt, - Cnt)}, - {verify_merge, verify_merge(RObj, Updated, FExpectedCounters, MergeSeed)} - ])) - end). - -%%==================================================================== -%% Helpers -%%==================================================================== -counter_op(Amt) when Amt < 0 -> - {decrement, Amt*-1}; -counter_op(Amt) -> - {increment, Amt}. - -%% Update and Merge are the same, except for the -%% end value of the counter. Reuse the common properties. -verify_merge(Generated, PostAction, FExpectedCounters, MergeSeed) -> - NumGeneratedCounters = num_counters(Generated), - ExpectedCounters = FExpectedCounters(NumGeneratedCounters), - ExpectedCounter = merge_object(Generated, MergeSeed), - NumMergedCounters = num_counters(PostAction), - {MergedMeta, MergedCounter} = single_counter(PostAction), - ExpectedSiblings = non_counter_siblings(Generated), - ActualSiblings = non_counter_siblings(PostAction), - conjunction([{number_of_counters, - equals(ExpectedCounters, NumMergedCounters)}, - {counter_structure, - counters_equal(ExpectedCounter, MergedCounter)}, - {siblings, equals(ExpectedSiblings, ActualSiblings)}, - {meta, equals(latest_meta(Generated, MergedMeta), MergedMeta)}]). - -latest_meta(RObj, MergedMeta) -> - %% Get the largest last modified containing meta data - latest_counter_meta(riak_object:get_contents(RObj), MergedMeta). - -latest_counter_meta([], Latest) -> - Latest; -latest_counter_meta([{MD, Val}|Rest], Latest) -> - case is_counter(Val) of - true -> - latest_counter_meta(Rest, get_latest_meta(MD, Latest)); - false -> - latest_counter_meta(Rest, Latest) - end; -latest_counter_meta([_|Rest], Latest) -> - latest_counter_meta(Rest, Latest). - -get_latest_meta(MD, undefined) -> - MD; -get_latest_meta(MD1, MD2) -> - TS1 = dict:fetch(?MD_LASTMOD, MD1), - TS2 = dict:fetch(?MD_LASTMOD, MD2), - case timer:now_diff(TS1, TS2) of - N when N < 0 -> - MD2; - _ -> - MD1 - end. - -%% safe wrap of ?V1_COUNTER_TYPE:equal/2 -counters_equal(undefined, undefined) -> - true; -counters_equal(_C1, undefined) -> - false; -counters_equal(undefined, _C2) -> - false; -counters_equal(C1B, C2B) when is_binary(C1B), is_binary(C2B) -> - {ok, ?CRDT{value=C1}} = riak_kv_crdt:from_binary(C1B), - {ok, ?CRDT{value=C2}} = riak_kv_crdt:from_binary(C2B), - counters_equal(C1, C2); -counters_equal(C1B, C2) when is_binary(C1B) -> - {ok, ?CRDT{value=C1}} = riak_kv_crdt:from_binary(C1B), - counters_equal(C1, C2); -counters_equal(C1, C2B) when is_binary(C2B) -> - {ok, ?CRDT{value=C2}} = riak_kv_crdt:from_binary(C2B), - counters_equal(C1, C2); -counters_equal(C1, C2) -> - ?V1_COUNTER_TYPE:equal(C1, C2). - - -%% Extract a single {meta, counter} value -single_counter(Merged) -> - Contents = riak_object:get_contents(Merged), - case [begin - {ok, ?CRDT{value=Counter}} = riak_kv_crdt:from_binary(Val), - {Meta, Counter} - end || {Meta, Val} <- Contents, - is_counter(Val)] of - [Single] -> - Single; - _Many -> {undefined, undefined} - end. - - -is_counter(Val) -> - case riak_kv_crdt:from_binary(Val) of - {ok, ?CRDT{mod=?V1_COUNTER_TYPE}} -> - true; - _ -> - false - end. - -non_counter_siblings(RObj) -> - Contents = riak_object:get_contents(RObj), - {_Counters, NonCounters} = lists:partition(fun({_Md, Val}) -> - is_counter(Val) end, - Contents), - lists:sort(NonCounters). - - -num_counters(RObj) -> - Values = riak_object:get_values(RObj), - length([ok || Val <- Values, - is_counter(Val)]). - -merge_object(RObj, Seed) -> - Values = riak_object:get_values(RObj), - lists:foldl(fun(<>, undefined) -> - try ?V1_COUNTER_TYPE:from_binary(CounterBin) of - Counter -> Counter - catch _:_ -> undefined - end; - (<>, Mergedest) -> - try ?V1_COUNTER_TYPE:from_binary(CounterBin) of - Counter -> ?V1_COUNTER_TYPE:merge(Counter, Mergedest) - catch _:_ -> Mergedest - end; - (_Bin, Mergedest) -> - Mergedest end, - Seed, - Values). - -%% Somewhat duplicates the logic under test -%% but is a different implementation, at least -sumthem(RObj) -> - Merged = merge_object(RObj, ?V1_COUNTER_TYPE:new()), - ?V1_COUNTER_TYPE:value(Merged). - -%%==================================================================== -%% Generators -%%==================================================================== -riak_object() -> - ?LET({Contents, VClock}, - {contents(), fsm_eqc_util:vclock()}, - riak_object:set_contents( - riak_object:set_vclock( - riak_object:new(?BUCKET, ?KEY, <<>>), - VClock), - Contents)). - -contents() -> - list(content()). - -content() -> - oneof([{metadata(), binary()}, {counter_meta(), pncounter()}]). - -counter_meta() -> - %% generate a dict of metadata - ?LET({_Mega, _Sec, _Micro}=Now, {nat(), nat(), nat()}, - dict:store(?MD_CTYPE, "application/riak_counter", dict:store(?MD_VTAG, riak_kv_util:make_vtag(Now), - dict:store(?MD_LASTMOD, Now, dict:new())))). - -metadata() -> - %% generate a dict of metadata - ?LET(Meta, metadatas(), dict:from_list(Meta)). - -metadatas() -> - list(metadatum()). - -metadatum() -> - %% doesn't need to be realistic, - %% just present - {binary(), binary()}. - -gcounter() -> - list(clock()). - -pncounterds() -> - {gcounter(), gcounter()}. - -pncounter() -> - ?LET(PNCounter, pncounterds(), - riak_kv_crdt:to_binary(?CRDT{mod=?V1_COUNTER_TYPE, value=PNCounter}, ?V1_VERS)). - -clock() -> - {int(), nat()}. - --endif. % EQC - diff --git a/eqc/put_fsm_eqc.erl b/eqc/put_fsm_eqc.erl index 279b13e8b..8e4541d40 100644 --- a/eqc/put_fsm_eqc.erl +++ b/eqc/put_fsm_eqc.erl @@ -44,7 +44,7 @@ -include_lib("eqc/include/eqc.hrl"). -include_lib("eunit/include/eunit.hrl"). -include("include/riak_kv_vnode.hrl"). --include("../src/riak_kv_wm_raw.hrl"). +-include("include/riak_object.hrl"). -define(REQ_ID, 1234). -define(DEFAULT_BUCKET_PROPS, diff --git a/include/riak_kv_web.hrl b/include/riak_kv_web.hrl new file mode 100644 index 000000000..a982854aa --- /dev/null +++ b/include/riak_kv_web.hrl @@ -0,0 +1,68 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2026 Martin Sumner +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +%% @doc common definitions for web handlers + +-define(TXT_HEADER, {'Content-Type', <<"text/plain">>}). +-define(JSN_HEADER, {'Content-Type', <<"application/json">>}). +-define(BIN_HEADER, {'Content-Type', <<"application/octet-stream">>}). + +-define(HEAD_VCLOCK, <<"X-Riak-Vclock">>). +-define(HEAD_USERMETA_PREFIX, <<"X-Riak-Meta-">>). +-define(HEAD_INDEX_PREFIX, <<"X-Riak-Index-">>). +-define(HEAD_DELETED, <<"X-Riak-Deleted">>). +-define(HEAD_CONTINUATION, <<"X-Riak-Continuation">>). + +%% Case-folded headers to be used in lookups +-define(HEAD_VCLOCK_CASEFOLD, <<"x-riak-vclock">>). +-define(HEAD_IFNOTMOD_CASEFOLD, <<"x-riak-if-not-modified">>). +-define(HEAD_USERMETA_CASEFOLD, <<"x-riak-meta-">>). +-define(HEAD_INDEX_CASEFOLD, <<"x-riak-index-">>). +-define(HEAD_CLIENTID_CASEFOLD, <<"x-riak-clientid">>). + +-define(Q_2I_CONTINUATION_BIN, <<"continuation">>). +-define(Q_RESULTS_BIN, <<"results">>). +-define(Q_KEYS_BIN, <<"keys">>). + +%% Names of JSON fields in bucket properties +-define(JSON_PROPS, <<"props">>). +-define(JSON_BUCKETS, <<"buckets">>). +-define(JSON_KEYS, <<"keys">>). +-define(JSON_MOD, <<"mod">>). +-define(JSON_FUN, <<"fun">>). +-define(JSON_NAME, <<"name">>). +-define(JSON_ARG, <<"arg">>). +-define(JSON_CHASH, <<"chash_keyfun">>). +-define(JSON_JSFUN, <<"jsfun">>). +-define(JSON_JSANON, <<"jsanon">>). +-define(JSON_JSBUCKET, <<"bucket">>). +-define(JSON_JSKEY, <<"key">>). +-define(JSON_ALLOW_MULT, <<"allow_mult">>). +-define(JSON_DATATYPE, <<"datatype">>). +-define(JSON_POSTC, <<"postcommit">>). +-define(JSON_PREC, <<"precommit">>). + +%% erlfmt:ignore-begin +-type stream_fun() :: + fun(() -> + {binary(), stream_fun()} + | done + | error + ). +%% erlfmt:ignore-end diff --git a/include/riak_object.hrl b/include/riak_object.hrl index 8658d8fff..4bc4a9d0e 100644 --- a/include/riak_object.hrl +++ b/include/riak_object.hrl @@ -1,3 +1,13 @@ -define(DOT, <<"dot">>). %% The event at which a value was written, stored in metadata - +%% Names of riak_object metadata fields +-define(MD_CTYPE, <<"content-type">>). +-define(MD_CHARSET, <<"charset">>). +-define(MD_ENCODING, <<"content-encoding">>). +-define(MD_VTAG, <<"X-Riak-VTag">>). +-define(MD_LINKS, <<"Links">>). +-define(MD_LASTMOD, <<"X-Riak-Last-Modified">>). +-define(MD_USERMETA, <<"X-Riak-Meta">>). +-define(MD_INDEX, <<"index">>). +-define(MD_DELETED, <<"X-Riak-Deleted">>). +-define(MD_VAL_ENCODING, <<"X-Riak-Val-Encoding">>). \ No newline at end of file diff --git a/rebar.config b/rebar.config index 1e54d81e4..c0756fa25 100644 --- a/rebar.config +++ b/rebar.config @@ -19,14 +19,14 @@ %% under the License. %% %% ------------------------------------------------------------------- -{minimum_otp_vsn, "24.0"}. +{minimum_otp_vsn, "26.0"}. {cover_enabled, false}. {erl_opts, [ debug_info, warnings_as_errors, - {src_dirs, ["src", "priv/tracers"]}, + {src_dirs, ["src"]}, {i, "_build/default/plugins/gpb/include"}, %% ToDo: Clean these out {d, namespaced_types}, @@ -50,6 +50,30 @@ {report_missing_types, true} ]}. +{erlfmt, [ + write, + {print_width, 80}, + {files, [ + "include/riak_kv_web.hrl", + "src/riak_kv_ag_aaefold.erl", + "src/riak_kv_web_common.erl", + "src/riak_kv_ag_object_delete.erl", + "src/riak_kv_ag_crdt.erl", + "src/riak_kv_ag_ping.erl", + "src/riak_kv_ag_stats.erl", + "src/riak_kv_ag_bprops.erl", + "src/riak_kv_ag_index.erl", + "src/riak_kv_ag_object_read.erl", + "src/riak_kv_ag_query.erl", + "src/riak_kv_ag_bucketlist.erl", + "src/riak_kv_ag_keylist.erl", + "src/riak_kv_ag_object_store.erl", + "src/riak_kv_ag_queue.erl", + "rebar.config" + ]}, + {exclude_files, []} +]}. + {eunit_opts, [verbose, {report, {eunit_progress, [colored, profile]}}]}. {xref_checks, [ @@ -61,45 +85,88 @@ ]}. {xref_ignores, [ - riak_core_pb, % Generated file - gen_fsm % Yeah, some day ... + % Generated file + riak_core_pb, + % Yeah, some day ... + gen_fsm +]}. + +{project_plugins, [ + {erlfmt, {git, "https://github.com/OpenRiak/erlfmt.git", {branch, "main"}}} ]}. -{plugins, [{rebar3_gpb_plugin, {git, "https://github.com/OpenRiak/rebar3_gpb_plugin", {branch, "openriak-3.4"}}}, - {eqc_rebar, {git, "https://github.com/Quviq/eqc-rebar", {branch, "master"}}}]}. +{erl_first_files, ["src/riak_kv_backend.erl"]}. + +{plugins, [ + {rebar3_gpb_plugin, + {git, "https://github.com/OpenRiak/rebar3_gpb_plugin", + {branch, "openriak-3.4"}}}, + {eqc_rebar, {git, "https://github.com/Quviq/eqc-rebar", {branch, "master"}}} +]}. -{gpb_opts, [{module_name_suffix, "_pb"}, - {i, "src"}]}. +{gpb_opts, [{module_name_suffix, "_pb"}, {i, "src"}]}. {dialyzer, [ {plt_apps, all_deps}, {warnings, [ - % error_handling, - % unknown, + % error_handling + % unknown % unmatched_returns + % missing_return + % overspecs ]} ]}. -{provider_hooks, [ - {pre, [{compile, {protobuf, compile}}]} - ]}. +{provider_hooks, [{pre, [{compile, {protobuf, compile}}]}]}. {profiles, [ - {test, [{deps, [{meck, {git, "https://github.com/OpenRiak/meck.git", {branch, "openriak-3.4"}}}]}]}, - {eqc, [{deps, [{meck, {git, "https://github.com/OpenRiak/meck.git", {branch, "openriak-3.4"}}}]}, {extra_src_dirs, ["eqc"]}]}, + {test, [ + {deps, [ + {meck, + {git, "https://github.com/OpenRiak/meck.git", + {branch, "openriak-3.4"}}} + ]} + ]}, + {eqc, [ + {deps, [ + {meck, + {git, "https://github.com/OpenRiak/meck.git", + {branch, "openriak-3.4"}}} + ]}, + {extra_src_dirs, ["eqc"]} + ]}, {gha, [{erl_opts, [{d, 'GITHUBEXCLUDE'}]}]} ]}. {deps, [ - {riak_core, {git, "https://github.com/OpenRiak/riak_core.git", {branch, "openriak-4.0"}}}, - {sidejob, {git, "https://github.com/OpenRiak/sidejob.git", {branch, "openriak-3.4"}}}, - {bitcask, {git, "https://github.com/OpenRiak/bitcask.git", {branch, "openriak-3.4"}}}, - {redbug, {git, "https://github.com/OpenRiak/redbug", {branch, "openriak-3.4"}}}, - {recon, {git, "https://github.com/OpenRiak/recon", {branch, "openriak-3.4"}}}, - {sext, {git, "https://github.com/OpenRiak/sext.git", {branch, "openriak-3.4"}}}, - {riak_dt, {git, "https://github.com/OpenRiak/riak_dt.git", {branch, "openriak-3.4"}}}, - {riak_pb, {git, "https://github.com/OpenRiak/riak_pb.git", {branch, "openriak-3.4"}}}, - {riak_api, {git, "https://github.com/OpenRiak/riak_api.git", {branch, "openriak-4.0"}}}, - {kv_index_tictactree, {git, "https://github.com/OpenRiak/kv_index_tictactree.git", {branch, "openriak-4.0"}}}, - {rhc, {git, "https://github.com/OpenRiak/riak-erlang-http-client", {branch, "openriak-3.4"}}} + {riak_core, + {git, "https://github.com/OpenRiak/riak_core.git", + {branch, "openriak-4.0"}}}, + {sidejob, + {git, "https://github.com/OpenRiak/sidejob.git", + {branch, "openriak-3.4"}}}, + {bitcask, + {git, "https://github.com/OpenRiak/bitcask.git", + {branch, "openriak-3.4"}}}, + {redbug, + {git, "https://github.com/OpenRiak/redbug", {branch, "openriak-3.4"}}}, + {recon, + {git, "https://github.com/OpenRiak/recon", {branch, "openriak-3.4"}}}, + {sext, + {git, "https://github.com/OpenRiak/sext.git", {branch, "openriak-3.4"}}}, + {riak_dt, + {git, "https://github.com/OpenRiak/riak_dt.git", + {branch, "openriak-3.4"}}}, + {riak_pb, + {git, "https://github.com/OpenRiak/riak_pb.git", + {branch, "openriak-3.4"}}}, + {riak_api, + {git, "https://github.com/OpenRiak/riak_api.git", + {branch, "nhse-o40-orkv.i141-silvermachine"}}}, + {kv_index_tictactree, + {git, "https://github.com/OpenRiak/kv_index_tictactree.git", + {branch, "openriak-4.0"}}}, + {rhc, + {git, "https://github.com/OpenRiak/riak-erlang-http-client", + {branch, "openriak-4.0"}}} ]}. diff --git a/src/json_pp.erl b/src/json_pp.erl deleted file mode 100644 index 96cb12650..000000000 --- a/src/json_pp.erl +++ /dev/null @@ -1,90 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% json_pp: pretty print serialized JSON strings -%% -%% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- --module(json_pp). - --define(SPACE, 32). --define(is_quote(C), (C == $\") orelse (C == $\')). --define(is_indent(C), (C == 91) orelse (C == 123)). % [, { --define(is_undent(C), (C == 93) orelse (C == 125)). % ], } --export([print/1, - test/0]). - -print(Str) when is_list(Str) -> json_pp(Str, 0, undefined, []). - -json_pp([$\\, C| Rest], I, C, Acc) -> % in quote - json_pp(Rest, I, C, [C, $\\| Acc]); -json_pp([C| Rest], I, undefined, Acc) when ?is_quote(C) -> - json_pp(Rest, I, C, [C| Acc]); -json_pp([C| Rest], I, C, Acc) -> % in quote - json_pp(Rest, I, undefined, [C| Acc]); -json_pp([C| Rest], I, undefined, Acc) when ?is_indent(C) -> - json_pp(Rest, I+1, undefined, [pp_indent(I+1), $\n, C| Acc]); -json_pp([C| Rest], I, undefined, Acc) when ?is_undent(C) -> - json_pp(Rest, I-1, undefined, [C, pp_indent(I-1), $\n| Acc]); -json_pp([$,| Rest], I, undefined, Acc) -> - json_pp(Rest, I, undefined, [pp_indent(I), $\n, $,| Acc]); -json_pp([$:| Rest], I, undefined, Acc) -> - json_pp(Rest, I, undefined, [?SPACE, $:| Acc]); -json_pp([C|Rest], I, Q, Acc) -> - json_pp(Rest, I, Q, [C| Acc]); -json_pp([], _I, _Q, Acc) -> % done - lists:reverse(Acc). - -pp_indent(I) -> lists:duplicate(I*4, ?SPACE). - -%% testing - -test_data() -> - {struct, [{foo, true}, - {bar, false}, - {baz, {array, [1, 2, 3, 4]}}, - {'fiz:f', null}, - {"fozzer\"", 5}]}. - -listify(IoList) -> binary_to_list(list_to_binary(IoList)). - -test() -> - J1 = listify(mochijson:encode(test_data())), - io:format("~s~n", [listify(print(J1))]). - --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). - -basic_test() -> - J1 = listify(mochijson:encode(test_data())), - L1 = - "{\n" - " \"foo\": true,\n" - " \"bar\": false,\n" - " \"baz\": [\n" - " 1,\n" - " 2,\n" - " 3,\n" - " 4\n" - " ],\n" - " \"fiz:f\": null,\n" - " \"fozzer\\\"\": 5\n" - "}", - ?assertEqual(L1, listify(print(J1))), - ok. - --endif. diff --git a/src/mochihex.erl b/src/mochihex.erl new file mode 100644 index 000000000..91b2789da --- /dev/null +++ b/src/mochihex.erl @@ -0,0 +1,106 @@ +%% @author Bob Ippolito +%% @copyright 2006 Mochi Media, Inc. +%% +%% Permission is hereby granted, free of charge, to any person obtaining a +%% copy of this software and associated documentation files (the "Software"), +%% to deal in the Software without restriction, including without limitation +%% the rights to use, copy, modify, merge, publish, distribute, sublicense, +%% and/or sell copies of the Software, and to permit persons to whom the +%% Software is furnished to do so, subject to the following conditions: +%% +%% The above copyright notice and this permission notice shall be included in +%% all copies or substantial portions of the Software. +%% +%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +%% THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +%% FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +%% DEALINGS IN THE SOFTWARE. + +%% @doc Utilities for working with hexadecimal strings. + +-module(mochihex). +-author('bob@mochimedia.com'). + +-export([to_hex/1, to_bin/1, to_int/1, dehex/1, hexdigit/1]). + +%% @spec to_hex(integer | iolist()) -> string() +%% @doc Convert an iolist to a hexadecimal string. +to_hex(0) -> + "0"; +to_hex(I) when is_integer(I), I > 0 -> + to_hex_int(I, []); +to_hex(B) -> + to_hex(iolist_to_binary(B), []). + +%% @spec to_bin(string()) -> binary() +%% @doc Convert a hexadecimal string to a binary. +to_bin(L) -> + to_bin(L, []). + +%% @spec to_int(string()) -> integer() +%% @doc Convert a hexadecimal string to an integer. +to_int(L) -> + erlang:list_to_integer(L, 16). + +%% @spec dehex(char()) -> integer() +%% @doc Convert a hex digit to its integer value. +dehex(C) when C >= $0, C =< $9 -> + C - $0; +dehex(C) when C >= $a, C =< $f -> + C - $a + 10; +dehex(C) when C >= $A, C =< $F -> + C - $A + 10. + +%% @spec hexdigit(integer()) -> char() +%% @doc Convert an integer less than 16 to a hex digit. +hexdigit(C) when C >= 0, C =< 9 -> + C + $0; +hexdigit(C) when C =< 15 -> + C + $a - 10. + +%% Internal API + +to_hex(<<>>, Acc) -> + lists:reverse(Acc); +to_hex(<>, Acc) -> + to_hex(Rest, [hexdigit(C2), hexdigit(C1) | Acc]). + +to_hex_int(0, Acc) -> + Acc; +to_hex_int(I, Acc) -> + to_hex_int(I bsr 4, [hexdigit(I band 15) | Acc]). + +to_bin([], Acc) -> + iolist_to_binary(lists:reverse(Acc)); +to_bin([C1, C2 | Rest], Acc) -> + to_bin(Rest, [(dehex(C1) bsl 4) bor dehex(C2) | Acc]). + + + +%% +%% Tests +%% +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +to_hex_test() -> + "ff000ff1" = to_hex([255, 0, 15, 241]), + "ff000ff1" = to_hex(16#ff000ff1), + "0" = to_hex(16#0), + ok. + +to_bin_test() -> + <<255, 0, 15, 241>> = to_bin("ff000ff1"), + <<255, 0, 10, 161>> = to_bin("Ff000aA1"), + ok. + +to_int_test() -> + 16#ff000ff1 = to_int("ff000ff1"), + 16#ff000aa1 = to_int("FF000Aa1"), + 16#0 = to_int("0"), + ok. + +-endif. diff --git a/src/riak.erl b/src/riak.erl index 580860778..5dd2ad925 100644 --- a/src/riak.erl +++ b/src/riak.erl @@ -26,7 +26,8 @@ -export([client_connect/1,client_connect/2, client_test/1, local_client/0,local_client/1, - join/1]). + join/1, + deadmanshand_restart/0]). -export([code_hash/0]). -include_lib("kernel/include/logger.hrl"). @@ -164,6 +165,25 @@ code_hash() -> riak_core_util:integer_to_list(MD5Sum, 62). +%% Helper function called from riak_admin_api_wm_ctl_cluster, as an +%% action to initiate riak restart. Actual restart (strictly, `riak +%% stop` followed by `riak start`) is performed by an external script, +%% run as a systemd service alongside riak. See +%% rel/files/riak-deadmanshand. +-spec deadmanshand_restart() -> ok. +deadmanshand_restart() -> + P = + case lists:keyfind("RELEASE_PROG", 1, os:env()) of + {_, "/usr" ++ _} -> + "/run/riak/"; + _ -> + "" + end, + _ = file:write_file(P ++ "RESTART_RIAK", <<>>), + ok. + + + %% %% Internal functions for testing a Riak node through single read/write cycle %% diff --git a/src/riak_client.erl b/src/riak_client.erl index d0cdc74d0..f56792b21 100644 --- a/src/riak_client.erl +++ b/src/riak_client.erl @@ -477,8 +477,10 @@ delete(Bucket,Key,RW,Timeout,THIS) -> normal_delete(Bucket, Key, Options, Timeout, {?MODULE, [Node, ClientId]}) -> Me = self(), ReqId = mk_reqid(), - riak_kv_delete_sup:start_delete(Node, [ReqId, Bucket, Key, Options, Timeout, - Me, ClientId]), + riak_kv_delete_sup:start_delete( + Node, + [ReqId, Bucket, Key, Options, Timeout, Me, ClientId] + ), RTimeout = recv_timeout(Options), wait_for_reqid(ReqId, erlang:min(Timeout, RTimeout)). @@ -554,8 +556,10 @@ delete_vclock(Bucket,Key,VClock,RW,Timeout,THIS) -> normal_delete_vclock(Bucket, Key, VClock, Options, Timeout, {?MODULE, [Node, ClientId]}) -> Me = self(), ReqId = mk_reqid(), - riak_kv_delete_sup:start_delete(Node, [ReqId, Bucket, Key, Options, Timeout, - Me, ClientId, VClock]), + riak_kv_delete_sup:start_delete( + Node, + [ReqId, Bucket, Key, Options, Timeout, Me, ClientId, VClock] + ), RTimeout = recv_timeout(Options), wait_for_reqid(ReqId, erlang:min(Timeout, RTimeout)). @@ -954,16 +958,25 @@ get_index(Bucket, Query, Opts, {?MODULE, [Node, _ClientId]}) -> wait_for_query_results(ReqId, Timeout). %% @doc Run the provided index query, return a stream handle. --spec stream_get_index(Bucket :: binary(), Query :: riak_index:query_def(), - riak_client()) -> - {ok, ReqId :: term(), FSMPid :: pid()} | {error, Reason :: term()}. +-spec stream_get_index( + Bucket :: riak_object:bucket(), + Query :: riak_index:query_def(), + riak_client() +) -> + {ok, ReqId :: non_neg_integer(), FSMPid :: pid()} + | {error, Reason :: term()}. stream_get_index(Bucket, Query, {?MODULE, [_Node, _ClientId]}=THIS) -> stream_get_index(Bucket, Query, [{timeout, ?DEFAULT_TIMEOUT}], THIS). %% @doc Run the provided index query, return a stream handle. --spec stream_get_index(Bucket :: binary(), Query :: riak_index:query_def(), - Opts :: proplists:proplist(), riak_client()) -> - {ok, ReqId :: term(), FSMPid :: pid()} | {error, Reason :: term()}. +-spec stream_get_index( + Bucket :: riak_object:bucket(), + Query :: riak_index:query_def(), + Opts :: proplists:proplist(), + riak_client() +) -> + {ok, ReqId :: non_neg_integer(), FSMPid :: pid()} + | {error, Reason :: term()}. stream_get_index(Bucket, Query, Opts, {?MODULE, [Node, _ClientId]}) -> Timeout = proplists:get_value(timeout, Opts, ?DEFAULT_TIMEOUT), MaxResults = proplists:get_value(max_results, Opts, all), diff --git a/src/riak_index.erl b/src/riak_index.erl index a2913d73c..8391e8dc7 100644 --- a/src/riak_index.erl +++ b/src/riak_index.erl @@ -38,7 +38,8 @@ upgrade_query/1, object_key_in_range/3, index_key_in_range/3, - add_timeout_opt/2 + add_timeout_opt/2, + decode_continuation/1 ]). -include_lib("kernel/include/logger.hrl"). @@ -47,7 +48,7 @@ -include_lib("eunit/include/eunit.hrl"). -endif. --include("riak_kv_wm_raw.hrl"). +-include("riak_object.hrl"). -include("riak_kv_index.hrl"). -include("riak_kv_capability.hrl"). -define(TIMEOUT, 30000). diff --git a/src/riak_kv.app.src b/src/riak_kv.app.src index 9f535a49f..419887256 100644 --- a/src/riak_kv.app.src +++ b/src/riak_kv.app.src @@ -12,8 +12,6 @@ riak_api, riak_core, sidejob, - mochiweb, - webmachine, os_mon, riak_dt, riak_pb, diff --git a/src/riak_kv_2i_aae.erl b/src/riak_kv_2i_aae.erl index 3e7f87af3..99e67d722 100644 --- a/src/riak_kv_2i_aae.erl +++ b/src/riak_kv_2i_aae.erl @@ -40,8 +40,6 @@ ] ). --include("riak_kv_wm_raw.hrl"). - -export([start/2, stop/1, get_status/0, to_report/1]). -export([first_partition/2, wait_for_aae_pid/2, wait_for_repair/3, diff --git a/src/riak_kv_ag_aaefold.erl b/src/riak_kv_ag_aaefold.erl new file mode 100644 index 000000000..3f119079a --- /dev/null +++ b/src/riak_kv_ag_aaefold.erl @@ -0,0 +1,1258 @@ +%% -------------------------------------------------------- +%% +%% Copyright (c) 2026 Martin Sumner +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +%% @doc Handler for HTTP API requests for (legacy) 2i queries + +-module(riak_kv_ag_aaefold). + +-if(?OTP_RELEASE == 26). +-feature(maybe_expr, enable). +-endif. + +-include_lib("kernel/include/logger.hrl"). +-include("riak_kv_web.hrl"). + +-behaviour(riak_api_web_handler). + +-export( + [ + match_route/3, + check_permissions/5, + parse_query_params/2, + parse_request_headers/2, + process_request/2, + record_request/3 + ] +). + +-record(filter, { + key_range = all :: {binary(), binary()} | all, + date_range = all :: {date, non_neg_integer(), non_neg_integer()} | all, + hash_method = pre_hash :: {rehash, non_neg_integer()} | pre_hash, + segment_filter = all :: segment_list() | all, + change_method = count :: {job, pos_integer()} | local | count +}). + +-record(context, { + fold_type = {undefined} :: fold_type(), + filter_expected = true :: boolean(), + tree_size :: leveled_tictac:tree_size() | undefined, + repl_queue :: atom() | undefined, + filter = #filter{} :: filter() +}). + +-type segment_list() :: + {segments, list(pos_integer()), leveled_tictac:tree_size() | n_val}. +-type filter() :: #filter{}. +-type context() :: #context{}. +-type fold_type() :: + {n_val, root | branch | clocks, pos_integer()} + | { + range_action, + merge_tree | clocks | repl_keys | repair_keys | erase_keys | reap_tombs, + riak_object:bucket() + } + | { + range_query, + object_stats | find_tombs, + riak_object:bucket() + } + | { + find_keys, + riak_object:bucket(), + {sibling_count, pos_integer()} | {object_size, pos_integer()} + } + | {bucket_list, pos_integer() | undefined} + | {undefined}. + +%% =================================================================== +%% Callback functions +%% =================================================================== + +%% Available operations (NOTE: within square brackets means optional): +%% GET /cachedtrees/nvals/NVal/root +%% GET /cachedtrees/nvals/NVal/branch?filter +%% GET /cachedtrees/nvals/NVal/keysclocks?filter +%% GET /rangetrees/[types/Type/]buckets/Bucket/trees/Size?filter +%% GET /rangetrees/[types/Type/]buckets/Bucket/keysclocks?filter +%% GET /rangerepl/[types/Type/]buckets/Bucket/queuename/Queue?filter +%% GET /rangerepair/[types/Type/]buckets/Bucket?filter +%% GET /siblings/[types/Type/]buckets/Bucket/counts/Cnt?filter +%% GET /objectsizes/[types/Type/]buckets/Bucket/sizes/Size?filter +%% GET /objectstats/[types/Type/]buckets/Bucket?filter +%% GET /tombs/[types/Type/]buckets/Bucket?filter +%% GET /reap/[types/Type/]buckets/Bucket?filter +%% GET /erase/[types/Type/]buckets/Bucket?filter +%% GET /aaebucketlist?nval + +-spec match_route( + riak_api_web_acceptor:method(), + unicode:chardata(), + list(unicode:chardata()) +) -> + nomatch + | {method_not_allowed, list(riak_api_web_acceptor:method())} + | {ok, riak_api_web_handler:limits(), context()}. +match_route(Method, _Path, [<<"cachedtrees">>, <<"nvals">>, NVal, Type]) -> + case check_integer(NVal) of + NValInt when is_integer(NValInt), NValInt > 0 -> + case Type of + <<"root">> -> + only_get( + #context{ + fold_type = {n_val, root, NValInt}, + filter_expected = false + }, + Method + ); + <<"branch">> -> + only_get( + #context{fold_type = {n_val, branch, NValInt}}, + Method + ); + <<"keysclocks">> -> + only_get( + #context{fold_type = {n_val, clocks, NValInt}}, + Method + ); + _ -> + nomatch + end; + _NotValidInt -> + nomatch + end; +match_route(Method, Path, [<<"rangetrees">> | Rest]) -> + case Rest of + [<<"buckets">> | _] -> + match_route( + Method, + Path, + [<<"rangetrees">>, <<"types">>, <<"default">>] ++ Rest + ); + [<<"types">>, Type, <<"buckets">>, Bucket, <<"trees">>, Size] -> + case check_size(Size) of + invalid -> + nomatch; + {valid, ValidSize} -> + FoldType = + { + range_action, + merge_tree, + riak_kv_web_common:set_bucket(Type, Bucket) + }, + only_get( + #context{fold_type = FoldType, tree_size = ValidSize}, + Method + ) + end; + [<<"types">>, Type, <<"buckets">>, Bucket, <<"keysclocks">>] -> + FoldType = + { + range_action, + clocks, + riak_kv_web_common:set_bucket(Type, Bucket) + }, + only_get( + #context{fold_type = FoldType}, + Method + ); + _ -> + nomatch + end; +match_route(Method, Path, [<<"rangerepl">> | Rest]) -> + case Rest of + [<<"buckets">> | _] -> + match_route( + Method, + Path, + [<<"rangerepl">>, <<"types">>, <<"default">>] ++ Rest + ); + [<<"types">>, Type, <<"buckets">>, Bucket, <<"queuename">>, Queue] -> + BT = riak_kv_web_common:set_bucket(Type, Bucket), + only_get( + #context{ + fold_type = {range_action, repl_keys, BT}, + repl_queue = riak_kv_web_common:check_queuename(Queue) + }, + Method + ); + _ -> + nomatch + end; +match_route(Method, Path, [<<"rangerepair">> | Rest]) -> + case Rest of + [<<"buckets">> | _] -> + match_route( + Method, + Path, + [<<"rangerepair">>, <<"types">>, <<"default">>] ++ Rest + ); + [<<"types">>, Type, <<"buckets">>, Bucket] -> + BT = riak_kv_web_common:set_bucket(Type, Bucket), + only_get( + #context{fold_type = {range_action, repair_keys, BT}}, + Method + ); + _ -> + nomatch + end; +match_route(Method, Path, [<<"siblings">> | Rest]) -> + case Rest of + [<<"buckets">> | _] -> + match_route( + Method, + Path, + [<<"siblings">>, <<"types">>, <<"default">>] ++ Rest + ); + [<<"types">>, Type, <<"buckets">>, Bucket, <<"counts">>, Count] -> + BT = riak_kv_web_common:set_bucket(Type, Bucket), + case check_integer(Count) of + Int when is_integer(Int), Int > 0 -> + only_get( + #context{ + fold_type = {find_keys, BT, {sibling_count, Int}} + }, + Method + ); + _ -> + nomatch + end; + _ -> + nomatch + end; +match_route(Method, Path, [<<"objectstats">> | Rest]) -> + case Rest of + [<<"buckets">> | _] -> + match_route( + Method, + Path, + [<<"objectstats">>, <<"types">>, <<"default">>] ++ Rest + ); + [<<"types">>, Type, <<"buckets">>, Bucket] -> + BT = riak_kv_web_common:set_bucket(Type, Bucket), + only_get( + #context{fold_type = {range_query, object_stats, BT}}, + Method + ); + _ -> + nomatch + end; +match_route(Method, Path, [<<"objectsizes">> | Rest]) -> + case Rest of + [<<"buckets">> | _] -> + match_route( + Method, + Path, + [<<"objectsizes">>, <<"types">>, <<"default">>] ++ Rest + ); + [<<"types">>, Type, <<"buckets">>, Bucket, <<"sizes">>, Size] -> + BT = riak_kv_web_common:set_bucket(Type, Bucket), + case check_integer(Size) of + Int when is_integer(Int), Int > 0 -> + only_get( + #context{ + fold_type = {find_keys, BT, {object_size, Int}} + }, + Method + ); + _ -> + nomatch + end; + _ -> + nomatch + end; +match_route(Method, Path, [<<"tombs">> | Rest]) -> + case Rest of + [<<"buckets">> | _] -> + match_route( + Method, + Path, + [<<"tombs">>, <<"types">>, <<"default">>] ++ Rest + ); + [<<"types">>, Type, <<"buckets">>, Bucket] -> + BT = riak_kv_web_common:set_bucket(Type, Bucket), + only_get( + #context{fold_type = {range_query, find_tombs, BT}}, + Method + ); + _ -> + nomatch + end; +match_route(Method, Path, [<<"reap">> | Rest]) -> + case Rest of + [<<"buckets">> | _] -> + match_route( + Method, + Path, + [<<"reap">>, <<"types">>, <<"default">>] ++ Rest + ); + [<<"types">>, Type, <<"buckets">>, Bucket] -> + BT = riak_kv_web_common:set_bucket(Type, Bucket), + only_get( + #context{fold_type = {range_action, reap_tombs, BT}}, + Method + ); + _ -> + nomatch + end; +match_route(Method, Path, [<<"erase">> | Rest]) -> + case Rest of + [<<"buckets">> | _] -> + match_route( + Method, + Path, + [<<"erase">>, <<"types">>, <<"default">>] ++ Rest + ); + [<<"types">>, Type, <<"buckets">>, Bucket] -> + BT = riak_kv_web_common:set_bucket(Type, Bucket), + only_get( + #context{fold_type = {range_action, erase_keys, BT}}, + Method + ); + _ -> + nomatch + end; +match_route(Method, _Path, [<<"aaebucketlist">>]) -> + only_get( + #context{ + fold_type = {bucket_list, undefined}, + filter_expected = false + }, + Method + ); +match_route(_Method, _Path, _SplitPath) -> + nomatch. + +%% @doc check_permissions for using this module or route +-spec check_permissions( + riak_api_web_headers:headers(), + riak_api_web_socket:scheme(), + riak_api_web_handler:peer_ip(), + riak_api_web_handler:peer_cert(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +check_permissions(ReqHeaders, Scheme, Peer, _Cert, Ctx) -> + case application:get_env(riak_kv, permit_insecure_http_ops, false) of + true -> + {ok, Ctx}; + false -> + Check = + riak_kv_web_common:check_permissions( + ReqHeaders, + Scheme, + Peer, + undefined, + undefined + ), + case Check of + true -> + {ok, Ctx}; + HaltResponse -> + HaltResponse + end + end. + +%% @doc parse and validate query params, passed as a map +-spec parse_query_params( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +parse_query_params(Params, Ctx = #context{filter_expected = true}) -> + case lists:keyfind(<<"filter">>, 1, Params) of + {<<"filter">>, B64FilterJson} when is_binary(B64FilterJson) -> + MaybeSegList = element(1, Ctx#context.fold_type) == n_val, + case validate_range_filter(B64FilterJson, MaybeSegList) of + {valid, Filter} -> + {ok, Ctx#context{filter = Filter}}; + {invalid, ErrMsg} -> + {halt, 400, [?TXT_HEADER], ErrMsg, []} + end; + _UseDefault -> + {ok, Ctx} + end; +parse_query_params(Params, Ctx) -> + case {Ctx#context.fold_type, lists:keyfind(<<"nval">>, 1, Params)} of + {{bucket_list, undefined}, {<<"nval">>, NVal}} -> + case check_integer(NVal) of + Int when is_integer(Int), Int > 0 -> + {ok, Ctx#context{fold_type = {bucket_list, Int}}}; + _NotInt -> + ErrMsg = <<"Invalid nval in query params ~0p">>, + {halt, 400, [?TXT_HEADER], ErrMsg, [NVal]} + end; + {{bucket_list, undefined}, false} -> + {ok, Ctx#context{fold_type = {bucket_list, 1}}}; + _ -> + {ok, Ctx} + end. + +%% @doc parse and validate the request headers +-spec parse_request_headers( + riak_api_web_headers:headers(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +parse_request_headers(_ReqHeaders, Ctx) -> + {ok, Ctx}. + +%% @doc Process the request and produce a response +-spec process_request( + none, + context() +) -> + { + ok, + { + riak_api_web_acceptor:response_code(), + riak_api_web_headers:header_list(), + riak_api_web_handler:response_body(), + boolean(), + none + }, + context() + } + | riak_api_web_acceptor:halt_response(). +process_request(none, Ctx) -> + Query = convert_to_query(Ctx), + case Query of + invalid -> + {halt, 400, [?TXT_HEADER], <<"Invalid query definition">>, []}; + _Valid -> + case riak_client:aae_fold(Query) of + {ok, Results} -> + QueryName = element(1, Query), + JsonResults = + iolist_to_binary( + riak_kv_clusteraae_fsm:json_encode_results( + QueryName, + Results + ) + ), + {ok, {200, [?JSN_HEADER], JsonResults, true, none}, Ctx}; + {error, timeout} -> + {halt, 503, [?TXT_HEADER], <<"Request timed out">>, []}; + {error, Reason} -> + ErrMsg = <<"Fold failure due to ~0p">>, + {halt, 500, [?TXT_HEADER], ErrMsg, [Reason]} + end + end. + +%% @doc Record the output of the interaction +-spec record_request( + riak_api_web_handler:timings(), + riak_api_web_handler:completion(), + context() +) -> + ok. +record_request(_Timings, _Completion, _Ctx) -> + ok. + +%% =================================================================== +%% Validation functions +%% =================================================================== + +-type filter_field() :: + segment_filter | key_range | date_range | hash_iv | change_method. + +-spec validate_range_filter( + binary(), + boolean() +) -> + {valid, filter()} | {invalid, binary()}. +validate_range_filter(B64JsonB, MaybeSegList) when is_binary(B64JsonB) -> + try + case riak_kv_wm_json:decode(base64:decode(B64JsonB)) of + Filter when is_map(Filter) -> + validate_range_filter( + Filter, + [ + segment_filter, + key_range, + date_range, + hash_iv, + change_method + ], + #filter{} + ); + Filter when is_list(Filter), MaybeSegList -> + true = check_seglist(Filter), + {valid, #filter{segment_filter = {segments, Filter, n_val}}} + end + catch + _:Error -> + ?LOG_WARNING("Issure decoding filter ~0p", [Error]), + {invalid, <<"Exception decoding filter">>} + end. + +-spec validate_range_filter( + map(), list(filter_field()), filter() +) -> + {valid, filter()} | {invalid, binary()}. +validate_range_filter(_FilterJson, [] = _Fields, Filter) -> + {valid, Filter}; +validate_range_filter(FilterJson, [Field | Fields], Filter0) -> + FieldVal = maps:get(atom_to_binary(Field), FilterJson, undefined), + case validate_field(Field, FieldVal, Filter0) of + {valid, Filter} -> + validate_range_filter(FilterJson, Fields, Filter); + {invalid, Reason} when is_binary(Reason) -> + {invalid, Reason} + end. + +-spec validate_field( + filter_field(), any(), filter() +) -> + {valid, filter()} | {invalid, binary()}. + +validate_field(segment_filter, <<"all">>, Filter) -> + {valid, Filter}; +validate_field(segment_filter, undefined, Filter) -> + {valid, Filter}; +validate_field(segment_filter, SegF, Filter) when is_map(SegF) -> + ValidTreeSize = + case maps:get(<<"tree_size">>, SegF, undefined) of + undefined -> + {invalid, <<"Segment filter has no tree_size">>}; + TreeSize -> + case check_size(TreeSize) of + invalid -> + {invalid, <<"Segment filter has invalid tree_size">>}; + {valid, ValidSize} -> + {valid, ValidSize} + end + end, + ValidSegList = + case maps:get(<<"segments">>, SegF, undefined) of + SegList when is_list(SegList) -> + case check_seglist(SegList) of + true -> + {valid, SegList}; + _ -> + {invalid, <<"Segment filter non-integer segment">>} + end; + undefined -> + {invalid, <<"Segment filter has no segment list">>}; + _ -> + {invalid, <<"Segment filter has invalid segment list">>} + end, + case {ValidTreeSize, ValidSegList} of + {{valid, VTS}, {valid, VSL}} -> + {valid, Filter#filter{segment_filter = {segments, VSL, VTS}}}; + {{invalid, ITS}, _} -> + {invalid, ITS}; + {_, {invalid, ISL}} -> + {invalid, ISL} + end; +validate_field(segment_filter, _Other, _Filter) -> + {invalid, <<"Segment filter is badly formed">>}; +validate_field(key_range, <<"all">>, Filter) -> + {valid, Filter}; +validate_field(key_range, undefined, Filter) -> + {valid, Filter}; +validate_field(key_range, KeyRange, Filter) when is_map(KeyRange) -> + Start = maps:get(<<"start">>, KeyRange, undefined), + End = maps:get(<<"end">>, KeyRange, undefined), + case {Start, End} of + {Start, End} when is_binary(Start), is_binary(End), End >= Start -> + {valid, Filter#filter{key_range = {Start, End}}}; + _Other -> + { + invalid, + <<"Key range does not contain both a valid start and end">> + } + end; +validate_field(key_range, _Other, _Filter) -> + {invalid, <<"Key range is badly formed">>}; +validate_field(date_range, <<"all">>, Filter) -> + {valid, Filter}; +validate_field(date_range, undefined, Filter) -> + {valid, Filter}; +validate_field(date_range, DateRange, Filter) when is_map(DateRange) -> + Start = maps:get(<<"start">>, DateRange, undefined), + End = maps:get(<<"end">>, DateRange, undefined), + case {Start, End} of + {Start, End} when + is_integer(Start), + Start >= 0, + is_integer(End), + End >= 0, + End >= Start + -> + {valid, Filter#filter{date_range = {date, Start, End}}}; + _Other -> + { + invalid, + <<"Date range does not contain both a valid start and end">> + } + end; +validate_field(date_range, _Other, _Filter) -> + {invalid, <<"Date range is badly formed">>}; +validate_field(hash_iv, undefined, Filter) -> + {valid, Filter}; +validate_field(hash_iv, <<"pre_hash">>, Filter) -> + {valid, Filter}; +validate_field(hash_iv, IV, Filter) when is_integer(IV) andalso IV > -1 -> + {valid, Filter#filter{hash_method = {rehash, IV}}}; +validate_field(hash_iv, _Other, _Filter) -> + {invalid, <<"Hash initialisation vector not an integer">>}; +validate_field(change_method, undefined, Filter) -> + {valid, Filter}; +validate_field(change_method, <<"count">>, Filter) -> + {valid, Filter#filter{change_method = count}}; +validate_field(change_method, <<"local">>, Filter) -> + {valid, Filter#filter{change_method = local}}; +validate_field(change_method, #{<<"job_id">> := JobID}, Filter) when + is_integer(JobID) +-> + {valid, Filter#filter{change_method = {job, JobID}}}; +validate_field(change_method, _Other, _Filter) -> + {invalid, <<"Change method is badly formed">>}. + +-spec check_integer(binary()) -> integer() | false. +check_integer(Bin) -> + try + binary_to_integer(Bin) + catch + _:_ -> + false + end. + +-spec check_size( + binary() | string() +) -> + {valid, leveled_tictac:tree_size()} | invalid. +check_size(TreeSize) -> + try + TS = + case is_binary(TreeSize) of + true -> + binary_to_existing_atom(TreeSize); + _ when is_list(TreeSize) -> + list_to_existing_atom(TreeSize) + end, + true = leveled_tictac:valid_size(TS), + {valid, TS} + catch + _:_ -> + ?LOG_WARNING("Invalid Tree Size ~0p", [TreeSize]), + invalid + end. + +-spec check_seglist(list()) -> boolean(). +check_seglist(SegList) -> + IntMembers = + lists:filter( + fun(I) -> is_integer(I) andalso I >= 0 end, + SegList + ), + length(IntMembers) == length(SegList). + +%% =================================================================== +%% Internal Functions +%% =================================================================== + +-spec only_get( + context(), + riak_api_web_acceptor:method() +) -> + {method_not_allowed, list(riak_api_web_acceptor:method())} + | {ok, riak_api_web_handler:limits(), context()}. +only_get(Context, 'GET') -> + {ok, size_limits(), Context}; +only_get(_Context, _Method) -> + {method_not_allowed, ['GET']}. + +size_limits() -> + { + 16, + 1024, + 0 + }. + +%% =================================================================== +%% Compose query +%% =================================================================== + +-spec convert_to_query( + context() +) -> + invalid | riak_kv_clusteraae_fsm:query_definition(). +convert_to_query(Ctx) -> + case Ctx#context.fold_type of + {n_val, root, N} -> + {merge_root_nval, N}; + {n_val, branch, N} -> + case (Ctx#context.filter)#filter.segment_filter of + {segments, SL, n_val} when length(SL) > 0 -> + {merge_branch_nval, N, SL}; + _ -> + invalid + end; + {n_val, clocks, N} -> + case (Ctx#context.filter)#filter.segment_filter of + {segments, SL, _} when length(SL) > 0 -> + case (Ctx#context.filter)#filter.date_range of + {date, Start, End} -> + {fetch_clocks_nval, N, SL, {date, Start, End}}; + _ -> + {fetch_clocks_nval, N, SL} + end; + _ -> + invalid + end; + {range_action, merge_tree, B} -> + { + merge_tree_range, + B, + (Ctx#context.filter)#filter.key_range, + Ctx#context.tree_size, + (Ctx#context.filter)#filter.segment_filter, + (Ctx#context.filter)#filter.date_range, + (Ctx#context.filter)#filter.hash_method + }; + {range_action, clocks, B} -> + { + fetch_clocks_range, + B, + (Ctx#context.filter)#filter.key_range, + (Ctx#context.filter)#filter.segment_filter, + (Ctx#context.filter)#filter.date_range + }; + {range_action, repl_keys, B} -> + { + repl_keys_range, + B, + (Ctx#context.filter)#filter.key_range, + (Ctx#context.filter)#filter.date_range, + Ctx#context.repl_queue + }; + {range_action, repair_keys, B} -> + { + repair_keys_range, + B, + (Ctx#context.filter)#filter.key_range, + (Ctx#context.filter)#filter.date_range, + all + }; + {range_action, DelAction, B} when + DelAction == erase_keys; DelAction == reap_tombs + -> + { + DelAction, + B, + (Ctx#context.filter)#filter.key_range, + (Ctx#context.filter)#filter.segment_filter, + (Ctx#context.filter)#filter.date_range, + (Ctx#context.filter)#filter.change_method + }; + {range_query, object_stats, B} -> + { + object_stats, + B, + (Ctx#context.filter)#filter.key_range, + (Ctx#context.filter)#filter.date_range + }; + {range_query, find_tombs, B} -> + { + find_tombs, + B, + (Ctx#context.filter)#filter.key_range, + (Ctx#context.filter)#filter.segment_filter, + (Ctx#context.filter)#filter.date_range + }; + {find_keys, B, FindType} -> + { + find_keys, + B, + (Ctx#context.filter)#filter.key_range, + (Ctx#context.filter)#filter.date_range, + FindType + }; + {bucket_list, N} when is_integer(N), N > 0 -> + {list_buckets, N}; + _ -> + invalid + end. + +%% =================================================================== +%% EUnit tests +%% =================================================================== + +-ifdef(TEST). + +-include_lib("eunit/include/eunit.hrl"). + +check_valid_routes_test() -> + R1 = [<<"cachedtrees">>, <<"nvals">>, <<"3">>, <<"root">>], + {ok, _, Ctx1} = match_route('GET', <<>>, R1), + ?assertMatch({n_val, root, 3}, Ctx1#context.fold_type), + R2 = [<<"cachedtrees">>, <<"nvals">>, <<"5">>, <<"branch">>], + {ok, _, Ctx2} = match_route('GET', <<>>, R2), + ?assertMatch({n_val, branch, 5}, Ctx2#context.fold_type), + R3 = [<<"cachedtrees">>, <<"nvals">>, <<"3">>, <<"keysclocks">>], + {ok, _, Ctx3} = match_route('GET', <<>>, R3), + ?assertMatch({n_val, clocks, 3}, Ctx3#context.fold_type), + R4 = [<<"rangetrees">>, <<"buckets">>, <<"B">>, <<"trees">>, <<"large">>], + {ok, _, Ctx4} = match_route('GET', <<>>, R4), + ?assertMatch({range_action, merge_tree, <<"B">>}, Ctx4#context.fold_type), + ?assertMatch(large, Ctx4#context.tree_size), + R5 = + [ + <<"rangetrees">>, + <<"types">>, + <<"T">>, + <<"buckets">>, + <<"B">>, + <<"trees">>, + <<"small">> + ], + {ok, _, Ctx5} = match_route('GET', <<>>, R5), + ?assertMatch( + {range_action, merge_tree, {<<"T">>, <<"B">>}}, + Ctx5#context.fold_type + ), + ?assertMatch(small, Ctx5#context.tree_size), + R6 = [<<"rangetrees">>, <<"buckets">>, <<"B">>, <<"keysclocks">>], + {ok, _, Ctx6} = match_route('GET', <<>>, R6), + ?assertMatch({range_action, clocks, <<"B">>}, Ctx6#context.fold_type), + R7 = + [<<"rangerepl">>, <<"buckets">>, <<"B">>, <<"queuename">>, <<"test">>], + {ok, _, Ctx7} = match_route('GET', <<>>, R7), + ?assertMatch({range_action, repl_keys, <<"B">>}, Ctx7#context.fold_type), + R8 = [<<"rangerepair">>, <<"buckets">>, <<"B">>], + {ok, _, Ctx8} = match_route('GET', <<>>, R8), + ?assertMatch({range_action, repair_keys, <<"B">>}, Ctx8#context.fold_type), + R9 = [<<"siblings">>, <<"buckets">>, <<"B">>, <<"counts">>, <<"2">>], + {ok, _, Ctx9} = match_route('GET', <<>>, R9), + ?assertMatch( + {find_keys, <<"B">>, {sibling_count, 2}}, + Ctx9#context.fold_type + ), + R10 = + [<<"objectsizes">>, <<"buckets">>, <<"B">>, <<"sizes">>, <<"10000">>], + {ok, _, Ctx10} = match_route('GET', <<>>, R10), + ?assertMatch( + {find_keys, <<"B">>, {object_size, 10000}}, + Ctx10#context.fold_type + ), + R11 = [<<"tombs">>, <<"buckets">>, <<"B">>], + {ok, _, Ctx11} = match_route('GET', <<>>, R11), + ?assertMatch({range_query, find_tombs, <<"B">>}, Ctx11#context.fold_type), + R12 = [<<"reap">>, <<"buckets">>, <<"B">>], + {ok, _, Ctx12} = match_route('GET', <<>>, R12), + ?assertMatch({range_action, reap_tombs, <<"B">>}, Ctx12#context.fold_type), + R13 = [<<"erase">>, <<"buckets">>, <<"B">>], + {ok, _, Ctx13} = match_route('GET', <<>>, R13), + ?assertMatch({range_action, erase_keys, <<"B">>}, Ctx13#context.fold_type), + R14 = [<<"aaebucketlist">>], + {ok, _, Ctx14} = match_route('GET', <<>>, R14), + ?assertMatch({bucket_list, undefined}, Ctx14#context.fold_type). + +nomatch_test() -> + nomatch([<<"bucket_list">>]), + nomatch([<<"cachedtrees">>, <<"nvals">>, <<"0">>, <<"root">>]), + nomatch([<<"cachedtrees">>, <<"nvals">>, <<"A">>, <<"branch">>]), + nomatch([<<"cachedtrees">>, <<"nvals">>, <<"3">>, <<"tree">>]), + nomatch([<<"siblings">>, <<"buckets">>, <<"B">>, <<"counts">>, <<"0">>]), + nomatch([<<"siblings">>, <<"buckets">>, <<"B">>, <<"counts">>, <<"A">>]), + nomatch([<<"objectstats">>, <<"counts">>]), + nomatch([<<"objectsizes">>, <<"buckets">>, <<"B">>, <<"sizes">>, <<"0">>]), + nomatch([<<"objectsizes">>, <<"buckets">>, <<"B">>, <<"sizes">>, <<"A">>]), + nomatch( + [<<"objectsizes">>, <<"buckets">>, <<"B">>, <<"counts">>, <<"1000">>] + ), + nomatch([<<"rangetrees">>, <<"buckets">>, <<"B">>, <<"trees">>, <<"xl">>]), + nomatch([<<"rangetrees">>, <<"buckets">>, <<"B">>, <<"t">>, <<"large">>]), + nomatch([<<"siblings">>, <<"buckets">>, <<"B">>, <<"sizes">>, <<"1">>]), + nomatch([<<"reap">>, <<"tipes">>, <<"T">>, <<"buckets">>, <<"B">>]), + nomatch([<<"tombs">>, <<"tipes">>, <<"T">>, <<"buckets">>, <<"B">>]), + nomatch([<<"erase">>, <<"types">>, <<"T">>, <<"bickets">>, <<"B">>]), + nomatch([<<"rangerepair">>]), + nomatch([<<"rangerepl">>, <<"B">>]), + nomatch([]). + +notallowed_test() -> + notallowed([<<"aaebucketlist">>]), + notallowed([<<"tombs">>, <<"types">>, <<"T">>, <<"buckets">>, <<"B">>]). + +nomatch(SP) -> + ?assertMatch(nomatch, match_route('GET', <<>>, SP)). + +notallowed(SP) -> + ?assertMatch({method_not_allowed, ['GET']}, match_route('PUT', <<>>, SP)). + +valid_filter() -> + #{ + <<"segment_filter">> => + #{ + <<"tree_size">> => <<"large">>, + <<"segments">> => [1, 2, 3, 5] + }, + <<"key_range">> => + #{ + <<"start">> => <<"Key00001">>, + <<"end">> => <<"Key00099">> + }, + <<"date_range">> => + #{ + <<"start">> => 300000, + <<"end">> => 400000 + }, + <<"hash_iv">> => <<"pre_hash">>, + <<"change_method">> => <<"count">> + }. + +check_valid_filter_test() -> + AssertionFun = + fun(Ctx) -> + ?assertMatch( + {segments, [1, 2, 3, 5], large}, + (Ctx#context.filter)#filter.segment_filter + ), + ?assertMatch( + {date, 300000, 400000}, + (Ctx#context.filter)#filter.date_range + ), + ?assertMatch( + {<<"Key00001">>, <<"Key00099">>}, + (Ctx#context.filter)#filter.key_range + ), + ?assertMatch( + pre_hash, + (Ctx#context.filter)#filter.hash_method + ), + ?assertMatch( + count, + (Ctx#context.filter)#filter.change_method + ) + end, + check_valid_filter_tester(valid_filter(), AssertionFun), + Filter1 = maps:remove(change_method, valid_filter()), + %% Change method is default so erasing it should change nothing + check_valid_filter_tester(Filter1, AssertionFun), + Filter2 = maps:put(change_method, <<"local">>, Filter1), + check_valid_filter_tester( + Filter2, + fun(Ctx) -> + ?assertMatch( + local, + (Ctx#context.filter)#filter.change_method + ) + end + ), + Filter3 = maps:put(change_method, #{<<"job_id">> => 1}, Filter1), + check_valid_filter_tester( + Filter3, + fun(Ctx) -> + ?assertMatch( + {job, 1}, + (Ctx#context.filter)#filter.change_method + ) + end + ), + Filter4 = maps:put(hash_iv, 99999, Filter1), + check_valid_filter_tester( + Filter4, + fun(Ctx) -> + ?assertMatch( + {rehash, 99999}, + (Ctx#context.filter)#filter.hash_method + ) + end + ). + +empty_filter_test() -> + check_valid_filter_tester( + #{}, + fun(Ctx) -> + ?assertMatch(#filter{}, Ctx#context.filter) + end + ). + +filter_all_test() -> + AllFilter = + #{ + <<"segment_filter">> => <<"all">>, + <<"key_range">> => <<"all">>, + <<"date_range">> => all + }, + AssertionFun = + fun(Ctx) -> + ?assertMatch(all, (Ctx#context.filter)#filter.segment_filter), + ?assertMatch(all, (Ctx#context.filter)#filter.date_range), + ?assertMatch(all, (Ctx#context.filter)#filter.key_range) + end, + check_valid_filter_tester(AllFilter, AssertionFun). + +invalid_segment_filter_test() -> + InvalidTreeSize = + maps:put( + <<"segment_filter">>, + maps:put( + <<"tree_size">>, + <<"supersize">>, + maps:get(<<"segment_filter">>, valid_filter()) + ), + valid_filter() + ), + check_invalid_filter_tester( + InvalidTreeSize, + <<"Segment filter has invalid tree_size">> + ), + InvalidSegmentList = <<"all">>, + InvalidSegment = [1, 2, 3, 5, <<"a">>], + SetISFun = + fun(ISL) -> + maps:put( + <<"segment_filter">>, + maps:put( + <<"segments">>, + ISL, + maps:get(<<"segment_filter">>, valid_filter()) + ), + valid_filter() + ) + end, + check_invalid_filter_tester( + SetISFun(InvalidSegmentList), + <<"Segment filter has invalid segment list">> + ), + check_invalid_filter_tester( + SetISFun(InvalidSegment), + <<"Segment filter non-integer segment">> + ), + DelISFun = + fun(Key) -> + maps:put( + <<"segment_filter">>, + maps:remove( + Key, + maps:get(<<"segment_filter">>, valid_filter()) + ), + valid_filter() + ) + end, + check_invalid_filter_tester( + DelISFun(<<"segments">>), + <<"Segment filter has no segment list">> + ), + check_invalid_filter_tester( + DelISFun(<<"tree_size">>), + <<"Segment filter has no tree_size">> + ). + +invalid_date_range_test() -> + InvalidStartDate = + maps:put( + <<"date_range">>, + maps:put( + <<"start">>, + <<"today">>, + maps:get(<<"date_range">>, valid_filter()) + ), + valid_filter() + ), + NoStartDate = + maps:put( + <<"date_range">>, + maps:remove( + <<"start">>, + maps:get(<<"date_range">>, valid_filter()) + ), + valid_filter() + ), + check_invalid_filter_tester( + InvalidStartDate, + <<"Date range does not contain both a valid start and end">> + ), + check_invalid_filter_tester( + NoStartDate, + <<"Date range does not contain both a valid start and end">> + ). + +invalid_key_range_test() -> + InvalidEndKey = + maps:put( + <<"key_range">>, + #{<<"start">> => <<"K00001">>, <<"end">> => <<"K00000">>}, + valid_filter() + ), + check_invalid_filter_tester( + InvalidEndKey, + <<"Key range does not contain both a valid start and end">> + ). + +check_valid_filter_tester(Filter, AssertionFun) -> + B64Json = base64:encode(iolist_to_binary(riak_kv_wm_json:encode(Filter))), + QPS = iolist_to_binary(io_lib:format(<<"filter=~s">>, [B64Json])), + QPD = uri_string:dissect_query(QPS), + {ok, Ctx} = parse_query_params(QPD, #context{filter_expected = true}), + AssertionFun(Ctx). + +check_invalid_filter_tester(Filter, ExpectedErrMsg) -> + B64Json = base64:encode(iolist_to_binary(riak_kv_wm_json:encode(Filter))), + QPS = iolist_to_binary(io_lib:format(<<"filter=~s">>, [B64Json])), + QPD = uri_string:dissect_query(QPS), + {halt, 400, [?TXT_HEADER], HaltMsg, []} = + parse_query_params(QPD, #context{filter_expected = true}), + ?assertMatch(ExpectedErrMsg, HaltMsg). + +decode_error_test() -> + B64Json = + base64:encode( + iolist_to_binary(riak_kv_wm_json:encode(valid_filter())), + #{mode => urlsafe, padding => false} + ), + QPS1 = iolist_to_binary(io_lib:format(<<"filter=~s">>, [B64Json])), + QPD1 = uri_string:dissect_query(QPS1), + {halt, 400, [?TXT_HEADER], HaltMsg1, []} = + parse_query_params(QPD1, #context{filter_expected = true}), + ?assertMatch( + <<"Exception decoding filter">>, + HaltMsg1 + ), + BadJson = base64:encode(<<"[Key, Value]">>), + QPS2 = iolist_to_binary(io_lib:format(<<"filter=~s">>, [BadJson])), + QPD2 = uri_string:dissect_query(QPS2), + {halt, 400, [?TXT_HEADER], HaltMsg2, []} = + parse_query_params(QPD2, #context{filter_expected = true}), + io:format("~0p~n", [HaltMsg2]), + ?assertMatch( + <<"Exception decoding filter">>, + HaltMsg2 + ). + +ignore_filter_if_unexpected_test() -> + Filter = valid_filter(), + EmptyFilter = #filter{}, + AssertionFun = + fun(C) -> ?assertNotMatch(EmptyFilter, C#context.filter) end, + check_valid_filter_tester(Filter, AssertionFun), + B64Json = base64:encode(iolist_to_binary(riak_kv_wm_json:encode(Filter))), + QPS = iolist_to_binary(io_lib:format(<<"filter=~s">>, [B64Json])), + QPD = uri_string:dissect_query(QPS), + {ok, Ctx} = parse_query_params(QPD, #context{filter_expected = false}), + ?assertMatch(EmptyFilter, Ctx#context.filter), + QPE = uri_string:dissect_query(<<>>), + {ok, CtxE} = parse_query_params(QPE, #context{filter_expected = true}), + ?assertMatch(EmptyFilter, CtxE#context.filter). + +bucket_list_qparam_test() -> + {ok, _, Ctx} = match_route('GET', <<>>, [<<"aaebucketlist">>]), + {halt, 400, _, <<"Invalid nval in query params ~0p">>, [<<"A">>]} = + parse_query_params(uri_string:dissect_query(<<"nval=A">>), Ctx), + {ok, Ctx0} = + parse_query_params(uri_string:dissect_query(<<"filter=A">>), Ctx), + ?assertMatch({bucket_list, 1}, Ctx0#context.fold_type), + {ok, Ctx1} = + parse_query_params(uri_string:dissect_query(<<"nval=3">>), Ctx), + ?assertMatch({bucket_list, 3}, Ctx1#context.fold_type). + +segment_list_on_branch_test() -> + {ok, _, Ctx} = + match_route( + 'GET', + <<>>, + [<<"cachedtrees">>, <<"nvals">>, <<"3">>, <<"branch">>] + ), + F = <<<<"filter=">>/binary, (base64:encode(<<"[1, 2, 3, 5]">>))/binary>>, + {ok, Ctx1} = parse_query_params(uri_string:dissect_query(F), Ctx), + ?assertMatch( + {segments, [1, 2, 3, 5], n_val}, + (Ctx1#context.filter)#filter.segment_filter + ), + BadF = <<<<"filter=">>/binary, (base64:encode(<<"[1, 2, A]">>))/binary>>, + ?assertMatch( + {halt, 400, _, <<"Exception decoding filter">>, _}, + parse_query_params(uri_string:dissect_query(BadF), Ctx) + ). + +valid_query_conversion_test() -> + VF = + base64:encode( + iolist_to_binary( + riak_kv_wm_json:encode(valid_filter()) + ) + ), + VSL1 = base64:encode(<<"[1, 2, 3, 5]">>), + SegDateFilter = + #{ + <<"segment_filter">> => + #{ + <<"tree_size">> => <<"large">>, + <<"segments">> => [1, 2, 3, 5] + }, + <<"date_range">> => + #{ + <<"start">> => 300000, + <<"end">> => 400000 + } + }, + VSL2 = + base64:encode( + iolist_to_binary( + riak_kv_wm_json:encode(SegDateFilter) + ) + ), + check_uri(<<"/aaebucketlist">>, 3), + check_uri(<<"/erase/types/Type/buckets/Bucket">>, VF), + check_uri(<<"/reap/types/Type/buckets/Bucket">>, VF), + check_uri(<<"/tombs/types/Type/buckets/Bucket">>, VF), + check_uri(<<"/objectstats/types/Type/buckets/Bucket">>, VF), + check_uri(<<"/objectstats/buckets/Bucket">>, VF), + check_uri(<<"/objectsizes/buckets/Bucket/sizes/10000">>, VF), + check_uri(<<"/siblings/buckets/Bucket/counts/2">>, VF), + check_uri(<<"/rangerepair/types/Type/buckets/Bucket">>, VF), + check_uri(<<"/rangerepl/buckets/Bucket/queuename/queue">>, VF), + check_uri(<<"/rangetrees/types/Type/buckets/Bucket/keysclocks">>, VF), + check_uri(<<"/rangetrees/types/Type/buckets/Bucket/trees/large">>, VF), + check_uri(<<"/cachedtrees/nvals/3/root">>, none), + check_uri(<<"/cachedtrees/nvals/3/branch">>, VSL1), + check_uri(<<"/cachedtrees/nvals/3/keysclocks">>, VSL1), + check_uri(<<"/cachedtrees/nvals/3/keysclocks">>, VSL2). + +create_uri(Path, none) -> + Path; +create_uri(Path, NVal) when is_integer(NVal) -> + iolist_to_binary([Path, <<"?nval=">>, integer_to_binary(NVal)]); +create_uri(Path, Filter) when is_binary(Filter) -> + iolist_to_binary([Path, <<"?filter=">>, Filter]). + +check_uri(Path, Filter) -> + URI = create_uri(Path, Filter), + {Route, QS} = + case binary:split(URI, <<"?">>) of + [R, Q] -> + {R, Q}; + [R] -> + {R, <<"">>} + end, + {ok, _, Ctx} = + match_route( + 'GET', + Route, + binary:split(Route, <<"/">>, [global, trim_all]) + ), + QP = uri_string:dissect_query(QS), + {ok, Ctx1} = parse_query_params(QP, Ctx), + ?assertNotMatch(invalid, convert_to_query(Ctx1)). + +-endif. diff --git a/src/riak_kv_ag_bprops.erl b/src/riak_kv_ag_bprops.erl new file mode 100644 index 000000000..9ad846473 --- /dev/null +++ b/src/riak_kv_ag_bprops.erl @@ -0,0 +1,452 @@ +%% -------------------------------------------------------- +%% +%% Copyright (c) 2026 Martin Sumner +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +%% @doc Handler for HTTP bucket properties fetch/store + +-module(riak_kv_ag_bprops). + +-include("riak_kv_web.hrl"). +-include_lib("kernel/include/logger.hrl"). + +-behaviour(riak_api_web_handler). + +-export( + [ + match_route/3, + check_permissions/5, + parse_query_params/2, + parse_request_headers/2, + process_request/2, + record_request/3 + ] +). + +-record(context, { + client = riak_client:new(node(), undefined) :: riak_client:riak_client(), + bucket :: riak_object:bucket() | binary(), + request = bucket :: bucket | type_only, + op :: get | set | reset +}). + +-type context() :: #context{}. + +%% =================================================================== +%% Callback functions +%% =================================================================== + +-spec match_route( + riak_api_web_acceptor:method(), + unicode:chardata(), + list(unicode:chardata()) +) -> + nomatch + | {method_not_allowed, list(riak_api_web_acceptor:method())} + | {ok, riak_api_web_handler:limits(), context()}. +match_route('PUT', _, [<<"types">>, T, <<"buckets">>, B, <<"props">>]) -> + { + ok, + {32, 2048, 64 * 1024}, + #context{bucket = riak_kv_web_common:set_bucket(T, B), op = set} + }; +match_route('GET', _, [<<"types">>, T, <<"buckets">>, B, <<"props">>]) -> + { + ok, + {32, 2048, 0}, + #context{bucket = riak_kv_web_common:set_bucket(T, B), op = get} + }; +match_route('DELETE', _, [<<"types">>, T, <<"buckets">>, B, <<"props">>]) -> + { + ok, + {32, 2048, 0}, + #context{bucket = riak_kv_web_common:set_bucket(T, B), op = reset} + }; +match_route(_, _, [<<"types">>, _T, <<"buckets">>, _B, <<"props">>]) -> + {method_not_allowed, ['GET', 'PUT', 'DELETE']}; +match_route(Method, Path, [<<"buckets">>, B, <<"props">>]) -> + match_route( + Method, + Path, + [<<"types">>, <<"default">>, <<"buckets">>, B, <<"props">>] + ); +match_route('PUT', _, [<<"types">>, T, <<"props">>]) -> + { + ok, + {32, 2048, 64 * 1024}, + #context{bucket = T, request = type_only, op = set} + }; +match_route('GET', _, [<<"types">>, T, <<"props">>]) -> + { + ok, + {32, 2048, 0}, + #context{bucket = T, request = type_only, op = get} + }; +match_route(_, _, [<<"types">>, _T, <<"props">>]) -> + {method_not_allowed, ['GET', 'PUT']}; +match_route(_, _, _) -> + nomatch. + +%% @doc check_permissions for using this module or route +-spec check_permissions( + riak_api_web_headers:headers(), + riak_api_web_socket:scheme(), + riak_api_web_handler:peer_ip(), + riak_api_web_handler:peer_cert(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +check_permissions(ReqHeaders, Scheme, Peer, _Cert, Ctx) -> + Grant = + case Ctx#context.op of + get -> + "riak_core.get_bucket"; + Op when Op == set; Op == reset -> + "riak_core.set_bucket" + end, + Check = + riak_kv_web_common:check_permissions( + ReqHeaders, + Scheme, + Peer, + Ctx#context.bucket, + Grant + ), + case Check of + true -> + {ok, Ctx}; + HaltResponse -> + HaltResponse + end. + +%% @doc parse and validate query params, passed as a map +-spec parse_query_params( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +parse_query_params(_Params, Ctx) -> + {ok, Ctx}. + +%% @doc parse and validate the request headers +-spec parse_request_headers( + riak_api_web_headers:headers(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +parse_request_headers(ReqHeaders, #context{op = get} = Ctx) -> + case riak_kv_web_common:accept_json_only(ReqHeaders) of + ok -> + {ok, Ctx}; + HaltResponse -> + HaltResponse + end; +parse_request_headers(_ReqHeaders, Ctx) -> + {ok, Ctx}. + +%% @doc Process the request and produce a response +-spec process_request( + riak_api_web_body:req_body() | none, + context() +) -> + { + ok, + { + riak_api_web_acceptor:response_code(), + riak_api_web_headers:header_list(), + riak_api_web_handler:response_body(), + boolean(), + riak_api_web_body:req_body() | none + }, + context() + } + | riak_api_web_acceptor:halt_response(). +process_request(none, #context{op = get, request = bucket} = Ctx) -> + Props1 = riak_client:get_bucket(Ctx#context.bucket, Ctx#context.client), + {ok, {200, [?JSN_HEADER], encode_properties(Props1), true, none}, Ctx}; +process_request(none, #context{op = reset, request = bucket} = Ctx) -> + riak_client:reset_bucket(Ctx#context.bucket, Ctx#context.client), + {ok, {204, [], <<>>, true, none}, Ctx}; +process_request(RqBdy, #context{op = set, request = bucket} = Ctx) when + RqBdy =/= none +-> + case riak_api_web_body:get_body(RqBdy, all, 10000) of + {error, content_too_large} -> + {halt, 413, [], <<>>, []}; + {ObjBody, UpdRqBody} when is_binary(ObjBody) -> + case safe_apply(ObjBody, Ctx) of + ok -> + {ok, {204, [], <<>>, true, UpdRqBody}, Ctx}; + {error, Details} -> + JSON = iolist_to_binary(riak_kv_wm_json:encode(Details)), + {halt, 400, [?JSN_HEADER], JSON, []} + end + end; +process_request(none, #context{op = get, request = type_only} = Ctx) -> + Props = riak_core_bucket_type:get(Ctx#context.bucket), + {ok, {200, [?JSN_HEADER], encode_properties(Props), true, none}, Ctx}; +process_request(RqBdy, #context{op = set, request = type_only} = Ctx) when + RqBdy =/= none +-> + case riak_api_web_body:get_body(RqBdy, all, 10000) of + {error, content_too_large} -> + {halt, 413, [], <<>>, []}; + {ObjBody, UpdRqBody} when is_binary(ObjBody) -> + case safe_apply(ObjBody, Ctx) of + ok -> + {ok, {204, [], <<>>, true, UpdRqBody}, Ctx}; + {error, Details} -> + JSON = iolist_to_binary(riak_kv_wm_json:encode(Details)), + {halt, 400, [?JSN_HEADER], JSON, []} + end + end. + +%% @doc Record the output of the interaction +-spec record_request( + riak_api_web_handler:timings(), + riak_api_web_handler:completion(), + context() +) -> + ok. +record_request(_Timings, _Completion, _Ctx) -> + ok. + +%% =================================================================== +%% Internal Functions +%% =================================================================== + +-spec safe_apply(binary(), context()) -> ok | {error, map()}. +safe_apply(ObjBody, #context{request = bucket} = Ctx) -> + try + ErlPropList = decode_properties(ObjBody), + riak_client:set_bucket( + Ctx#context.bucket, + ErlPropList, + Ctx#context.client + ) + catch + _:Error -> + ?LOG_WARNING( + "Decode failure applying bucket properties ~0p", + [Error] + ), + {error, #{error => <<"decode failure">>}} + end; +safe_apply(ObjBody, #context{request = type_only} = Ctx) -> + try + ErlPropList = decode_properties(ObjBody), + riak_core_bucket_type:update(Ctx#context.bucket, ErlPropList) + catch + _:Error -> + ?LOG_WARNING( + "Decode failure applying bucket type properties ~0p", + [Error] + ), + {error, #{error => <<"decode failure">>}} + end. + +-spec encode_properties(list({atom(), any() | {atom(), atom()}})) -> binary(). +encode_properties(BucketProps) -> + JsonReadyProps = + lists:filter( + fun(T) -> T =/= none end, + lists:map(fun jsonify_bucket_prop/1, BucketProps) + ), + iolist_to_binary( + riak_kv_wm_json:encode( + #{<<"props">> => maps:from_list(JsonReadyProps)} + ) + ). + +-spec decode_properties(binary()) -> list({atom(), any() | {atom(), atom()}}). +decode_properties(Json) -> + lists:map( + fun erlify_bucket_prop/1, + maps:to_list( + maps:get(<<"props">>, riak_kv_wm_json:decode(Json)) + ) + ). + +-spec jsonify_bucket_prop( + {atom(), any() | {atom(), atom()}} +) -> + {binary(), any()} | none. +jsonify_bucket_prop({linkfun, _}) -> + none; +jsonify_bucket_prop({chash_keyfun, {Mod, Fun}}) when + is_atom(Mod), is_atom(Fun) +-> + { + ?JSON_CHASH, + #{ + ?JSON_MOD => atom_to_binary(Mod, utf8), + ?JSON_FUN => atom_to_binary(Fun, utf8) + } + }; +jsonify_bucket_prop({postcommit, FunList}) -> + {?JSON_POSTC, jsonify_commit_hooks(FunList)}; +jsonify_bucket_prop({precommit, FunList}) -> + {?JSON_PREC, jsonify_commit_hooks(FunList)}; +jsonify_bucket_prop({rs_extractfun, _}) -> + none; +jsonify_bucket_prop({search_extractor, _}) -> + none; +jsonify_bucket_prop({name, {_T, B}}) when is_binary(B) -> + {<<"name">>, B}; +jsonify_bucket_prop({Prop, Value}) -> + {atom_to_binary(Prop, utf8), Value}. + +-spec erlify_bucket_prop( + {binary(), any()} +) -> + {atom(), any() | {atom(), atom()}}. +erlify_bucket_prop({?JSON_DATATYPE, Type}) when is_binary(Type) -> + {datatype, binary_to_existing_atom(Type, utf8)}; +erlify_bucket_prop({?JSON_CHASH, Props}) -> + { + chash_keyfun, + { + binary_to_existing_atom(maps:get(?JSON_MOD, Props)), + binary_to_existing_atom(maps:get(?JSON_FUN, Props)) + } + }; +erlify_bucket_prop({?JSON_POSTC, FunList}) -> + {postcommit, erlify_commit_hooks(FunList)}; +erlify_bucket_prop({?JSON_PREC, FunList}) -> + {precommit, erlify_commit_hooks(FunList)}; +erlify_bucket_prop({Prop, Value}) when is_binary(Value) -> + { + binary_to_existing_atom(Prop), + binary_to_existing_atom(Value) + }; +erlify_bucket_prop({Prop, Value}) when is_integer(Value); is_boolean(Value) -> + { + binary_to_existing_atom(Prop), + Value + }. + +erlify_commit_hooks(FunList) -> + lists:map( + fun(P) -> + case maps:get(?JSON_NAME, P, undefined) of + undefined -> + { + struct, + [ + {?JSON_MOD, maps:get(?JSON_MOD, P)}, + {?JSON_FUN, maps:get(?JSON_FUN, P)} + ] + }; + HookName -> + {struct, [{?JSON_NAME, HookName}]} + end + end, + FunList + ). + +jsonify_commit_hooks(FunList) -> + lists:map( + fun + ({struct, [{?JSON_NAME, Name}]}) -> + #{?JSON_NAME => Name}; + ({struct, ModFun}) -> + {?JSON_MOD, Mod} = lists:keyfind(?JSON_MOD, 1, ModFun), + {?JSON_FUN, Fun} = lists:keyfind(?JSON_FUN, 1, ModFun), + #{ + ?JSON_MOD => Mod, + ?JSON_FUN => Fun + } + end, + FunList + ). + +%% =================================================================== +%% EUnit tests +%% =================================================================== + +-ifdef(TEST). + +-include_lib("eunit/include/eunit.hrl"). + +circle_default_props_test() -> + BucketProps = + [ + {linkfun, {modfun, riak_kv_wm_link_walker, mapreduce_linkfun}}, + {old_vclock, 86400}, + {young_vclock, 20}, + {big_vclock, 50}, + {small_vclock, 50}, + {pr, 0}, + {r, quorum}, + {w, quorum}, + {pw, 0}, + {node_confirms, 0}, + {dw, quorum}, + {rw, quorum}, + {basic_quorum, false}, + {notfound_ok, true} + ], + Json = encode_properties(BucketProps), + SupportedProps = lists:sort(lists:keydelete(linkfun, 1, BucketProps)), + CircleProps = decode_properties(Json), + ?assertMatch(SupportedProps, lists:sort(CircleProps)). + +circle_special_props_test() -> + BucketProps = + [ + {rs_extractfun, modfun}, + {search_extractor, modfun}, + {chash_keyfun, {keymod, keymodfun}}, + { + postcommit, + [ + { + struct, + [ + {<<"mod">>, <<"commitmod">>}, + {<<"fun">>, <<"commitfun">>} + ] + } + ] + }, + {precommit, [{struct, [{<<"name">>, <<"name">>}]}]}, + {datatype, counter} + ], + SupportedProps = + lists:sort( + lists:keydelete( + rs_extractfun, + 1, + lists:keydelete(search_extractor, 1, BucketProps) + ) + ), + Json = encode_properties(BucketProps), + CircleProps = decode_properties(Json), + ?assertMatch(SupportedProps, lists:sort(CircleProps)). + +bad_special_props_test() -> + LinkJson = + << + "{\"props\":" + "[{\"linkfun\":{\"fun\":\"linkmodfun\",\"mod\":\"linkmod\"}}]}" + >>, + ?assertMatch( + {error, #{error := <<"decode failure">>}}, + safe_apply(LinkJson, #context{bucket = <<"B">>, op = set}) + ). + +-endif. diff --git a/src/riak_kv_ag_bucketlist.erl b/src/riak_kv_ag_bucketlist.erl new file mode 100644 index 000000000..68598dc48 --- /dev/null +++ b/src/riak_kv_ag_bucketlist.erl @@ -0,0 +1,316 @@ +%% -------------------------------------------------------- +%% +%% Copyright (c) 2026 Martin Sumner +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +%% @doc Handler for HTTP bucketlist API (v2 or higher) + +-module(riak_kv_ag_bucketlist). + +-include("riak_kv_web.hrl"). + +-behaviour(riak_api_web_handler). + +-export( + [ + match_route/3, + check_permissions/5, + parse_query_params/2, + parse_request_headers/2, + process_request/2, + record_request/3 + ] +). + +-record(context, { + client = riak_client:new(node(), undefined) :: riak_client:riak_client(), + bucket_type = <<"default">> :: binary(), + stream = false :: boolean(), + timeout :: pos_integer() | undefined +}). + +-type context() :: #context{}. + +%% =================================================================== +%% Callback functions +%% =================================================================== + +-spec match_route( + riak_api_web_acceptor:method(), + unicode:chardata(), + list(unicode:chardata()) +) -> + nomatch + | {method_not_allowed, list(riak_api_web_acceptor:method())} + | {ok, riak_api_web_handler:limits(), context()}. +match_route('GET', _, [<<"types">>, T, <<"buckets">>]) -> + { + ok, + {32, 2048, 0}, + #context{bucket_type = T} + }; +match_route(Method, _, [<<"types">>, _T, <<"buckets">>]) when + Method =/= 'GET' +-> + {method_not_allowed, ['GET']}; +match_route(Method, Path, [<<"buckets">>]) -> + match_route( + Method, + Path, + [<<"types">>, <<"default">>, <<"buckets">>] + ); +match_route(_, _, _) -> + nomatch. + +%% @doc check_permissions for using this module or route +-spec check_permissions( + riak_api_web_headers:headers(), + riak_api_web_socket:scheme(), + riak_api_web_handler:peer_ip(), + riak_api_web_handler:peer_cert(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +check_permissions(ReqHeaders, Scheme, Peer, _Cert, Ctx) -> + Check = + riak_kv_web_common:check_permissions( + ReqHeaders, + Scheme, + Peer, + Ctx#context.bucket_type, + "riak_kv.list_buckets" + ), + case Check of + true -> + {ok, Ctx}; + HaltResponse -> + HaltResponse + end. + +%% @doc parse and validate query params, passed as a map +-spec parse_query_params( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +parse_query_params(Params, Ctx) -> + Ctx1 = + case lists:keyfind(<<"buckets">>, 1, Params) of + {<<"buckets">>, <<"stream">>} -> + Ctx#context{stream = true}; + _ -> + Ctx + end, + validate_timeout(Params, Ctx1). + +%% @doc parse and validate the request headers +-spec parse_request_headers( + riak_api_web_headers:headers(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +parse_request_headers(ReqHeaders, Ctx) -> + case riak_kv_web_common:accept_json_only(ReqHeaders) of + ok -> + {ok, Ctx}; + HaltResponse -> + HaltResponse + end. + +%% @doc Process the request and produce a response +-spec process_request( + none, + context() +) -> + { + ok, + { + riak_api_web_acceptor:response_code(), + riak_api_web_headers:header_list(), + riak_api_web_handler:response_body(), + boolean(), + none + }, + context() + } + | riak_api_web_acceptor:halt_response(). +process_request(none, #context{bucket_type = BT, client = C} = Ctx) -> + case Ctx#context.stream of + true -> + {ok, ReqId} = + riak_client:stream_list_buckets( + none, + Ctx#context.timeout, + BT, + C + ), + + { + ok, + { + 200, + [?JSN_HEADER], + {stream, bucket_stream_fun(ReqId)}, + true, + none + }, + Ctx + }; + false -> + case riak_client:list_buckets(none, Ctx#context.timeout, BT, C) of + {ok, Buckets} -> + JsonResults = + riak_kv_wm_json:encode( + #{<<"buckets">> => handle_unicode_buckets(Buckets)} + ), + { + ok, + { + 200, + [?JSN_HEADER], + iolist_to_binary(JsonResults), + true, + none + }, + Ctx + }; + {error, timeout} -> + ErrMsg = + riak_kv_wm_json:encode( + #{<<"error">> => <<"Request timed out">>} + ), + {halt, 503, [?JSN_HEADER], iolist_to_binary(ErrMsg), []}; + {error, Reason} -> + ErrMsg = + riak_kv_wm_json:encode( + #{ + <<"error">> => + iolist_to_binary( + io_lib:format("~0p", [Reason]) + ) + } + ), + {halt, 503, [?JSN_HEADER], iolist_to_binary(ErrMsg), []} + end + end. + +%% @doc Record the output of the interaction +-spec record_request( + riak_api_web_handler:timings(), + riak_api_web_handler:completion(), + context() +) -> + ok. +record_request(_Timings, _Completion, _Ctx) -> + ok. + +%% =================================================================== +%% Internal Functions +%% =================================================================== + +-spec bucket_stream_fun(non_neg_integer()) -> stream_fun(). +bucket_stream_fun(ReqId) -> + fun() -> + receive + {ReqId, _From, {buckets_stream, Buckets}} -> + JsonResults = + riak_kv_wm_json:encode( + #{<<"buckets">> => handle_unicode_buckets(Buckets)} + ), + { + iolist_to_binary(JsonResults), + bucket_stream_fun(ReqId) + }; + {ReqId, {buckets_stream, Buckets}} -> + JsonResults = + riak_kv_wm_json:encode( + #{<<"buckets">> => handle_unicode_buckets(Buckets)} + ), + { + iolist_to_binary(JsonResults), + bucket_stream_fun(ReqId) + }; + {ReqId, done} -> + {<<>>, fun() -> done end}; + {ReqId, {error, timeout}} -> + JsonError = + riak_kv_wm_json:encode( + #{<<"error">> => <<"Request timed out">>} + ), + { + iolist_to_binary(JsonError), + fun() -> done end + } + end + end. + +-spec validate_timeout( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +validate_timeout(Params, Ctx) -> + case riak_kv_web_common:get_timeout(Params) of + {ok, none} -> + {ok, Ctx}; + {ok, Timeout} -> + {ok, Ctx#context{timeout = Timeout}}; + HaltResponse -> + HaltResponse + end. + +-spec handle_unicode_buckets(list(binary())) -> list(binary()). +handle_unicode_buckets(Buckets) -> + % lists:map(fun uri_string:quote/1, Buckets). + Buckets. + +%% =================================================================== +%% EUnit tests +%% =================================================================== + +-ifdef(TEST). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("kernel/include/logger.hrl"). + +parse_test() -> + InitCtx = #context{bucket_type = <<"T">>}, + {ok, Ctx1} = + parse_query_params( + [{<<"buckets">>, <<"stream">>}, {<<"timeout">>, <<"1000">>}], + InitCtx + ), + ?assertMatch(true, Ctx1#context.stream), + ?assertMatch(1000, Ctx1#context.timeout), + ?assertMatch( + {halt, 400, _, _, _}, + parse_query_params( + [{<<"keys">>, <<"stream">>}, {<<"timeout">>, <<"A">>}], + InitCtx + ) + ), + ReqHeaders = riak_api_web_headers:make([{'Accept', <<"*/*">>}]), + ?assertMatch({ok, _}, parse_request_headers(ReqHeaders, Ctx1)), + ReqHeadersNoAccept = riak_api_web_headers:make([]), + ?assertMatch({ok, _}, parse_request_headers(ReqHeadersNoAccept, Ctx1)), + ReqHeadersPlain = riak_api_web_headers:make([{'Accept', <<"text/plain">>}]), + ?assertMatch( + {halt, 406, _, _, _}, + parse_request_headers(ReqHeadersPlain, Ctx1) + ). + +-endif. diff --git a/src/riak_kv_ag_crdt.erl b/src/riak_kv_ag_crdt.erl new file mode 100644 index 000000000..823df2f11 --- /dev/null +++ b/src/riak_kv_ag_crdt.erl @@ -0,0 +1,595 @@ +%% -------------------------------------------------------- +%% +%% Copyright (c) 2026 Martin Sumner +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +%% @doc Handler for HTTP CRDT requests (legacy) + +-module(riak_kv_ag_crdt). + +-if(?OTP_RELEASE == 26). +-feature(maybe_expr, enable). +-endif. + +-include("riak_kv_web.hrl"). +-include("riak_kv_types.hrl"). + +-behaviour(riak_api_web_handler). + +-export( + [ + match_route/3, + check_permissions/5, + parse_query_params/2, + parse_request_headers/2, + process_request/2, + record_request/3 + ] +). + +-define(OPTION_DEFAULTS, #{ + r => default, + w => default, + dw => default, + pw => default, + pr => default, + node_confirms => default, + n_val => default, + basic_quorum => default, + sloppy_quorum => default, + returnbody => false, + notfound_ok => default, + include_context => true, + timeout => undefined +}). + +-record(context, { + client = riak_client:new(node(), undefined) :: riak_client:riak_client(), + method :: 'GET' | 'HEAD' | 'POST', + bucket :: riak_object:bucket(), + key :: riak_object:key() | {generated, riak_object:key()}, + crdt_mod :: module() | undefined, + crdt_type :: riak_kv_crdt_json:toplevel_type() | undefined, + crdt_options = ?OPTION_DEFAULTS :: crdt_options() +}). + +-type crdt_option_key() :: + r + | w + | dw + | pw + | pr + | node_confirms + | n_val + | basic_quorum + | sloppy_quorum + | notfound_ok + | returnbody + | include_context + | timeout. + +-type crdt_options() :: + #{ + r => pos_integer() | default | quorum | all, + w => pos_integer() | default | quorum | all, + dw => non_neg_integer() | default | quorum | all, + pw => non_neg_integer() | default | quorum | all, + pr => non_neg_integer() | default | quorum | all, + n_val => pos_integer() | default, + node_confirms => non_neg_integer() | default | quorum | all, + basic_quorum => boolean() | default, + sloppy_quorum => boolean() | default, + returnbody => boolean(), + include_context => true, + notfound_ok => boolean() | default, + timeout => pos_integer() | undefined + }. + +-type context() :: #context{}. + +%% =================================================================== +%% Callback functions +%% =================================================================== + +-spec match_route( + riak_api_web_acceptor:method(), + unicode:chardata(), + list(unicode:chardata() | {generated, binary()}) +) -> + nomatch + | {method_not_allowed, list(riak_api_web_acceptor:method())} + | {ok, riak_api_web_handler:limits(), context()}. +match_route( + Method, + _, + [<<"types">>, BucketType, <<"buckets">>, Bucket, <<"datatypes">>, Key] +) when + BucketType =/= <<"default">> +-> + case Method of + Method when Method == 'POST' -> + MaxUpdateSize = + application:get_env( + riak_kv, + max_crdt_update_size, + 4 * 1024 * 1024 + ), + { + ok, + {32, 2048, MaxUpdateSize}, + #context{ + bucket = riak_kv_web_common:set_bucket(BucketType, Bucket), + method = 'POST', + key = Key + } + }; + Method when Method == 'GET'; Method == 'HEAD' -> + { + ok, + {32, 2048, 0}, + #context{ + bucket = riak_kv_web_common:set_bucket(BucketType, Bucket), + method = Method, + key = Key + } + }; + _ -> + {method_not_allowed, ['GET', 'HEAD', 'POST']} + end; +match_route( + 'POST', + Path, + [<<"types">>, BucketType, <<"buckets">>, Bucket, <<"datatypes">>] +) -> + K = {generated, iolist_to_binary(riak_core_util:unique_id_62())}, + match_route( + 'POST', + Path, + [<<"types">>, BucketType, <<"buckets">>, Bucket, <<"datatypes">>, K] + ); +match_route(_, _, _) -> + nomatch. + +%% @doc check_permissions for using this module or route +-spec check_permissions( + riak_api_web_headers:headers(), + riak_api_web_socket:scheme(), + riak_api_web_handler:peer_ip(), + riak_api_web_handler:peer_cert(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +check_permissions(ReqHeaders, Scheme, Peer, _Cert, Ctx) -> + GrantSought = + case Ctx#context.method of + 'POST' -> + "riak_kv.put"; + _ -> + "riak_kv.get" + end, + Check = + riak_kv_web_common:check_permissions( + ReqHeaders, + Scheme, + Peer, + Ctx#context.bucket, + GrantSought + ), + case Check of + true -> + check_crdt_type(Ctx); + HaltResponse -> + HaltResponse + end. + +%% @doc parse and validate query params, passed as a map +-spec parse_query_params( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +parse_query_params([], Ctx) -> + % Typically we expect no options - so shortcut the validation in this case + {ok, Ctx}; +parse_query_params(Params, Ctx) -> + maybe + {ok, Ctx0} ?= validate_timeout(Params, Ctx), + {ok, Ctx1} ?= validate_counts(Params, Ctx0), + {ok, Ctx2} ?= validate_booleans(Params, Ctx1), + {ok, Ctx2} + else + HaltResponse -> + HaltResponse + end. + +%% @doc parse and validate the request headers +-spec parse_request_headers( + riak_api_web_headers:headers(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +parse_request_headers(ReqHeaders, Ctx) -> + case riak_kv_web_common:accept_json_only(ReqHeaders) of + ok -> + {ok, Ctx}; + HaltResponse -> + HaltResponse + end. + +%% @doc Process the request and produce a response +-spec process_request( + riak_api_web_body:req_body() | none, + context() +) -> + { + ok, + { + riak_api_web_acceptor:response_code(), + riak_api_web_headers:header_list(), + riak_api_web_handler:response_body(), + boolean(), + riak_api_web_body:req_body() | none + }, + context() + } + | riak_api_web_acceptor:halt_response(). +process_request(none, #context{method = Method, crdt_mod = Mod} = Ctx) when + Method == 'GET'; Method == 'HEAD' +-> + GetOptions = + riak_kv_web_common:filter_options( + Ctx#context.crdt_options + ), + GetResult = + riak_client:get( + Ctx#context.bucket, + Ctx#context.key, + [{crdt_op, Mod} | GetOptions], + Ctx#context.client + ), + case GetResult of + {ok, RObj} -> + JsonBody = iolist_to_binary(produce_json(RObj, Ctx, Mod)), + case Method of + 'GET' -> + {ok, {200, [?JSN_HEADER], JsonBody, true, none}, Ctx}; + 'HEAD' -> + RspHdrs = + [?JSN_HEADER, {'Content-Length', byte_size(JsonBody)}], + {ok, {200, RspHdrs, <<>>, true, none}, Ctx} + end; + {error, Reason} -> + handle_common_error(Reason, Ctx) + end; +process_request(RqBdy, #context{method = 'POST', crdt_mod = Mod} = Ctx) -> + case riak_api_web_body:get_body(RqBdy, all, 60000) of + {error, content_too_large} -> + {halt, 413, [], <<>>, []}; + {ObjBody, UpdRqBdy} when is_binary(ObjBody) -> + case check_post_body(ObjBody, Ctx) of + {ok, {_UpdType, UpdOp, UpdOpCtx}} -> + {Type, Bucket} = Ctx#context.bucket, + {Key, LocHdr} = + case Ctx#context.key of + {generated, K} -> + Location = set_location(Type, Bucket, K), + {K, [{'Location', Location}]}; + K when is_binary(K) -> + {K, []} + end, + O = riak_kv_crdt:new({Type, Bucket}, Key, Mod), + PutOptions = + riak_kv_web_common:filter_options( + Ctx#context.crdt_options + ), + CrdtOp = #crdt_op{mod = Mod, op = UpdOp, ctx = UpdOpCtx}, + Options = + [ + {crdt_op, CrdtOp}, + {retry_put_coordinator_failure, false} + ] ++ + PutOptions, + case riak_client:put(O, Options, Ctx#context.client) of + ok -> + {ok, {204, LocHdr, <<>>, true, UpdRqBdy}, Ctx}; + {ok, RObj} -> + JsonBody = + iolist_to_binary(produce_json(RObj, Ctx, Mod)), + { + ok, + { + 200, + [?JSN_HEADER | LocHdr], + JsonBody, + true, + UpdRqBdy + }, + Ctx + }; + {error, Reason} -> + handle_common_error(Reason, Ctx) + end; + HaltResponse -> + HaltResponse + end + end. + +%% @doc Record the output of the interaction +-spec record_request( + riak_api_web_handler:timings(), + riak_api_web_handler:completion(), + context() +) -> + ok. +record_request(_Timings, _Completion, _Ctx) -> + ok. + +%% =================================================================== +%% Validation Functions +%% =================================================================== + +-spec validate_timeout( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +validate_timeout(Params, Ctx) -> + case riak_kv_web_common:get_timeout(Params) of + {ok, none} -> + {ok, Ctx}; + {ok, Timeout} -> + {ok, set_option(timeout, Timeout, Ctx)}; + HaltResponse -> + HaltResponse + end. + +-spec validate_counts( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +validate_counts(Params, Context) -> + % List of allowed integer parameters based on riak_kv_pb_crdt + ParamList = + case Context#context.method of + 'POST' -> + [ + <<"w">>, + <<"dw">>, + <<"pw">>, + <<"n_val">>, + <<"node_confirms">> + ]; + _ -> + [<<"r">>, <<"pr">>, <<"n_Val">>] + end, + FoldResult = + riak_kv_web_common:count_fold( + Params, + ParamList, + Context#context.crdt_options + ), + case FoldResult of + UpdOpts when is_map(UpdOpts) -> + {ok, Context#context{crdt_options = UpdOpts}}; + HaltResponse -> + HaltResponse + end. + +-spec validate_booleans( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +validate_booleans(Params, Context) -> + % List of allowed boolean parameters based on riak_kv_pb_crdt + ParamList = + case Context#context.method of + 'POST' -> + [<<"sloppy_quorum">>, <<"returnbody">>]; + _ -> + [<<"sloppy_quorum">>, <<"basic_quorum">>, <<"notfound_ok">>] + end, + FoldResult = + riak_kv_web_common:boolean_fold( + Params, + ParamList, + Context#context.crdt_options + ), + case FoldResult of + UpdOpts when is_map(UpdOpts) -> + {ok, Context#context{crdt_options = UpdOpts}}; + HaltResponse -> + HaltResponse + end. + +%% =================================================================== +%% Internal Functions +%% =================================================================== + +handle_common_error(Reason, Ctx) -> + case Reason of + too_many_fails -> + ErrMsg = <<"Too many write failures to satisfy W/DW">>, + {halt, 503, [?TXT_HEADER], ErrMsg, []}; + timeout -> + {halt, 503, [?TXT_HEADER], <<"request timed out">>, []}; + notfound -> + NotFndMsg = + #{ + <<"type">> => + atom_to_binary( + riak_kv_crdt:from_mod(Ctx#context.crdt_mod), + utf8 + ), + <<"error">> => <<"notfound">> + }, + {halt, 404, [?JSN_HEADER], riak_kv_wm_json:encode(NotFndMsg), []}; + {deleted, _VClock} -> + RspHdrs = [{?HEAD_DELETED, <<"true">>}, ?JSN_HEADER], + NotFndMsg = + #{ + <<"type">> => + atom_to_binary( + riak_kv_crdt:from_mod(Ctx#context.crdt_mod), + utf8 + ), + <<"error">> => <<"notfound">> + }, + {halt, 404, RspHdrs, riak_kv_wm_json:encode(NotFndMsg), []}; + {n_val_violation, N} -> + ErrMsg = + << + "Specified w/dw/pw/node_confirms values invalid for" + " bucket n value of ~0p" + >>, + {halt, 400, [?TXT_HEADER], ErrMsg, [N]}; + {r_val_unsatisfied, Requested, Returned} -> + ErrMsg = "R-value unsatisfied: ~p/~p", + {halt, 503, [?TXT_HEADER], ErrMsg, [Returned, Requested]}; + {dw_val_unsatisfied, Requested, Returned} -> + ErrMsg = <<"DW-value unsatisfied: ~p/~p">>, + {halt, 503, [?TXT_HEADER], ErrMsg, [Returned, Requested]}; + {pr_val_unsatisfied, Requested, Returned} -> + ErrMsg = <<"PR-value unsatisfied: ~p/~p">>, + {halt, 503, [?TXT_HEADER], ErrMsg, [Returned, Requested]}; + {pw_val_unsatisfied, Requested, Returned} -> + ErrMsg = <<"PW-value unsatisfied: ~p/~p">>, + {halt, 503, [?TXT_HEADER], ErrMsg, [Returned, Requested]}; + {node_confirms_val_unsatisfied, Requested, Returned} -> + ErrMsg = <<"node_confirms-value unsatisfied: ~p/~p">>, + {halt, 503, [?TXT_HEADER], ErrMsg, [Returned, Requested]}; + Err -> + {halt, 500, [?TXT_HEADER], <<"Error:~n~0p~n">>, [Err]} + end. + +-spec produce_json(riak_object:riak_object(), context(), module()) -> binary(). +produce_json(RObj, Ctx, Mod) -> + IncludeContext = + maps:get(include_context, (Ctx#context.crdt_options), true), + Type = riak_kv_crdt:from_mod(Mod), + {{RespCtx, Value}, Stats} = riak_kv_crdt:value(RObj, Mod), + _ = [ok = riak_kv_stat:update(S) || S <- Stats], + Body = + riak_kv_crdt_json:fetch_response_to_json( + Type, + Value, + case IncludeContext of + true -> + RespCtx; + _ -> + undefined + end, + riak_kv_crdt:mod_map(Type) + ), + mochijson2:encode(Body). + +-spec check_post_body( + binary(), + context() +) -> + {ok, riak_kv_crdt_json:update()} | riak_api_web_acceptor:halt_response(). +check_post_body(ReqBody, #context{crdt_type = CRDTType}) -> + try + JSON = mochijson2:decode(ReqBody), + Update = + {CRDTType, _Op, _Context} = + riak_kv_crdt_json:update_request_from_json( + CRDTType, + JSON, + riak_kv_crdt:mod_map(CRDTType) + ), + {ok, Update} + catch + throw:{invalid_operation, {BadType, BadOp}} -> + { + halt, + 400, + [?TXT_HEADER], + <<"Invalid operation on datatype '~s': ~s">>, + [BadType, mochijson2:encode(BadOp)] + }; + throw:{invalid_field_name, Field} -> + { + halt, + 400, + [?TXT_HEADER], + <<"Invalid map field name '~s'">>, + [Field] + }; + throw:invalid_utf8 -> + ErrMsg = <<"Malformed JSON submitted, invalid UTF-8">>, + {halt, 400, [?TXT_HEADER], ErrMsg, []}; + _Other:Reason -> + ErrMsg = <<"Couldn't decode JSON: ~p">>, + {halt, 400, [?TXT_HEADER], ErrMsg, [Reason]} + end. + +-spec check_crdt_type( + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +check_crdt_type(Context) -> + {Type, Bucket} = Context#context.bucket, + case riak_core_bucket:get_bucket({Type, Bucket}) of + BProps when is_list(BProps) -> + DataType = proplists:get_value(datatype, BProps), + AllowMult = proplists:get_value(allow_mult, BProps), + Mod = riak_kv_crdt:to_mod(DataType), + case {AllowMult, riak_kv_crdt:supported(Mod)} of + {false, _} -> + ErrMsg = <<"Bucket must be allow_mult=true">>, + {halt, 400, [?TXT_HEADER], ErrMsg, []}; + {_, false} -> + ErrMsg = + <<"Bucket datatype '~s' is not a supported type">>, + {halt, 400, [?TXT_HEADER], ErrMsg, [DataType]}; + _ -> + {ok, Context#context{crdt_type = DataType, crdt_mod = Mod}} + end; + {error, no_type} -> + {halt, 404, [?TXT_HEADER], <<"Unknown bucket type: ~s">>, [Type]} + end. + +-spec set_option( + crdt_option_key(), + non_neg_integer() | quorum | all | backend | one | boolean(), + context() +) -> + context(). +set_option(Option, Value, Context) -> + Context#context{ + crdt_options = maps:put(Option, Value, Context#context.crdt_options) + }. + +set_location(Type, Bucket, Key) -> + iolist_to_binary( + io_lib:format( + "/types/~s/buckets/~s/datatypes/~s", + [Type, Bucket, Key] + ) + ). + +%% =================================================================== +%% EUnit tests +%% =================================================================== + +-ifdef(TEST). + +-include_lib("eunit/include/eunit.hrl"). + +-endif. diff --git a/src/riak_kv_ag_index.erl b/src/riak_kv_ag_index.erl new file mode 100644 index 000000000..687a33478 --- /dev/null +++ b/src/riak_kv_ag_index.erl @@ -0,0 +1,1152 @@ +%% -------------------------------------------------------- +%% +%% Copyright (c) 2026 Martin Sumner +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +%% @doc Handler for HTTP API requests for (legacy) 2i queries + +-module(riak_kv_ag_index). + +-if(?OTP_RELEASE == 26). +-feature(maybe_expr, enable). +-endif. + +-include_lib("kernel/include/logger.hrl"). +-include("riak_kv_web.hrl"). + +-behaviour(riak_api_web_handler). + +-export( + [ + match_route/3, + check_permissions/5, + parse_query_params/2, + parse_request_headers/2, + process_request/2, + record_request/3 + ] +). + +-export( + [ + encode_results/3 + ] +). + +-record(context, { + client = riak_client:new(node(), undefined) :: riak_client:riak_client(), + bucket :: riak_object:bucket(), + field :: binary(), + field_type = bin :: bin | int | dollar, + start_term :: binary(), + end_term :: binary(), + max_results = all :: pos_integer() | all, + return_terms_client = false :: boolean(), + pagination_sort = false :: boolean(), + stream = false :: boolean(), + term_regex :: binary() | undefined, + continuation :: binary() | undefined, + timeout :: pos_integer() | undefined +}). + +-type context() :: #context{}. + +%% =================================================================== +%% Callback functions +%% =================================================================== + +-spec match_route( + riak_api_web_acceptor:method(), + unicode:chardata(), + list(unicode:chardata()) +) -> + nomatch + | {method_not_allowed, list(riak_api_web_acceptor:method())} + | {ok, riak_api_web_handler:limits(), context()}. +match_route( + Method, + _Path, + [<<"types">>, BucketType, <<"buckets">>, Bucket, <<"index">>, Idx, ST, ET] +) -> + case Method of + 'GET' -> + Context = + #context{ + bucket = riak_kv_web_common:set_bucket(BucketType, Bucket), + field = Idx, + start_term = ST, + end_term = ET + }, + {ok, size_limits(), Context}; + _ -> + {method_not_allowed, ['GET']} + end; +match_route( + Method, + Path, + [<<"types">>, BType, <<"buckets">>, Bucket, <<"index">>, Idx, T] +) -> + match_route( + Method, + Path, + [<<"types">>, BType, <<"buckets">>, Bucket, <<"index">>, Idx, T, T] + ); +match_route( + Method, + Path, + [<<"buckets">>, _Bucket, <<"index">>, _Idx, _ST, _ET] = SplitPath +) -> + match_route( + Method, + Path, + [<<"types">>, <<"default">>] ++ SplitPath + ); +match_route( + Method, + Path, + [<<"buckets">>, _Bucket, <<"index">>, _Idx, T] = SplitPath +) -> + match_route( + Method, + Path, + [<<"types">>, <<"default">>] ++ SplitPath ++ [T] + ); +match_route(_Method, _Path, _SP) -> + nomatch. + +%% @doc check_permissions for using this module or route +-spec check_permissions( + riak_api_web_headers:headers(), + riak_api_web_socket:scheme(), + riak_api_web_handler:peer_ip(), + riak_api_web_handler:peer_cert(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +check_permissions(ReqHeaders, Scheme, Peer, _Cert, Ctx) -> + Check = + riak_kv_web_common:check_permissions( + ReqHeaders, + Scheme, + Peer, + Ctx#context.bucket, + "riak_kv.index" + ), + case Check of + true -> + B = Ctx#context.bucket, + case riak_kv_web_common:check_type_exists(B) of + ok -> + {ok, Ctx}; + HaltResponse -> + HaltResponse + end; + HaltResponse -> + HaltResponse + end. + +%% @doc parse and validate query params, passed as a map +-spec parse_query_params( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +parse_query_params(Params, Ctx) -> + maybe + {ok, Ctx0} ?= validate_query_type(Ctx), + {ok, Ctx1} ?= validate_timeout(Params, Ctx0), + {ok, Ctx2} ?= validate_max_results(Params, Ctx1), + {ok, Ctx3} ?= validate_maybe_true(return_terms, Params, Ctx2), + {ok, Ctx4} ?= validate_maybe_true(pagination_sort, Params, Ctx3), + {ok, Ctx5} ?= validate_maybe_true(stream, Params, Ctx4), + {ok, Ctx6} ?= validate_term_regex(Params, Ctx5), + {ok, Ctx7} ?= validate_continuation(Params, Ctx6), + {ok, Ctx7} + else + HaltResponse -> + HaltResponse + end. + +%% @doc parse and validate the request headers +-spec parse_request_headers( + riak_api_web_headers:headers(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +parse_request_headers(ReqHeaders, Ctx) -> + case riak_kv_web_common:accept_json_only(ReqHeaders) of + ok -> + {ok, Ctx}; + HaltResponse -> + HaltResponse + end. + +%% @doc Process the request and produce a response +-spec process_request( + none, + context() +) -> + { + ok, + { + riak_api_web_acceptor:response_code(), + riak_api_web_headers:header_list(), + riak_api_web_handler:response_body(), + boolean(), + none + }, + context() + } + | riak_api_web_acceptor:halt_response(). +process_request(none, Ctx) -> + KeyOnly = + Ctx#context.field_type == dollar orelse + Ctx#context.start_term == Ctx#context.end_term, + IndexQuery = + riak_index:to_index_query( + [ + {field, Ctx#context.field}, + {start_term, Ctx#context.start_term}, + {end_term, Ctx#context.end_term}, + {return_terms, not KeyOnly}, + {continuation, Ctx#context.continuation}, + {term_regex, Ctx#context.term_regex} + ] + ), + case {IndexQuery, Ctx#context.stream} of + {{ok, Q}, true} -> + process_stream_query(Q, Ctx); + {{ok, Q}, false} -> + process_memory_query(Q, Ctx); + {{error, Error}, _} -> + ErrMsg = <<"Error parsing query ~0p">>, + {halt, 400, [?TXT_HEADER], ErrMsg, [Error]} + end. + +%% @doc Record the output of the interaction +-spec record_request( + riak_api_web_handler:timings(), + riak_api_web_handler:completion(), + context() +) -> + ok. +record_request(_Timings, _Completion, _Ctx) -> + ok. + +%% =================================================================== +%% Validation functions +%% =================================================================== + +-spec validate_timeout( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +validate_timeout(Params, Ctx) -> + case riak_kv_web_common:get_timeout(Params) of + {ok, none} -> + {ok, Ctx}; + {ok, Timeout} -> + {ok, Ctx#context{timeout = Timeout}}; + HaltResponse -> + HaltResponse + end. + +-spec validate_max_results( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +validate_max_results(Params, Ctx) -> + case lists:keyfind(<<"max_results">>, 1, Params) of + false -> + {ok, Ctx}; + {<<"max_results">>, MR} when is_binary(MR) -> + try + IntMR = binary_to_integer(MR), + true = IntMR > 0, + {ok, Ctx#context{max_results = IntMR}} + catch + _:_ -> + ErrMsg = + << + "Invalid max_results ~0p " + "is not a positive integer" + >>, + {halt, 400, [?TXT_HEADER], ErrMsg, [MR]} + end + end. + +-spec validate_maybe_true( + return_terms | pagination_sort | stream, + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +validate_maybe_true(Key, Params, Ctx) -> + case lists:keyfind(atom_to_binary(Key), 1, Params) of + false -> + {ok, Ctx}; + {_, MT} -> + case riak_kv_web_common:normalise_boolean_param(MT) of + true -> + case Key of + return_terms -> + KeyOnly = + Ctx#context.field_type == dollar orelse + Ctx#context.start_term == + Ctx#context.end_term, + case KeyOnly of + true -> + { + ok, + Ctx#context{ + return_terms_client = false + } + }; + false -> + { + ok, + Ctx#context{ + return_terms_client = true + } + } + end; + pagination_sort -> + {ok, Ctx#context{pagination_sort = true}}; + stream -> + {ok, Ctx#context{stream = true}} + end; + bad_param -> + ErrMsg = <<"Invalid ~0p. ~0p is not a boolean">>, + {halt, 400, [?TXT_HEADER], ErrMsg, [Key, MT]}; + _ -> + {ok, Ctx} + end + end. + +-spec validate_query_type( + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +validate_query_type(Ctx = #context{field = DollarI}) when + DollarI == <<"$key">>; DollarI == <<"$bucket">> +-> + {ok, Ctx#context{field_type = dollar}}; +validate_query_type(Ctx = #context{field = Index}) when is_binary(Index) -> + case byte_size(Index) of + L when L > 4 -> + <<_Idx:(L - 4)/binary, Suffix:4/binary>> = Index, + case string:casefold(Suffix) of + <<"_bin">> -> + {ok, Ctx#context{field_type = bin}}; + <<"_int">> -> + {ok, Ctx#context{field_type = int}}; + _ -> + ErrMsg = <<"Invalid IndexName ~0p">>, + {halt, 400, [?TXT_HEADER], ErrMsg, [Index]} + end; + _ -> + ErrMsg = <<"Invalid IndexName ~0p">>, + {halt, 400, [?TXT_HEADER], ErrMsg, [Index]} + end. + +-spec validate_term_regex( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +validate_term_regex(Params, Ctx) -> + case lists:keyfind(<<"term_regex">>, 1, Params) of + false -> + {ok, Ctx}; + {<<"term_regex">>, Re} when is_binary(Re) -> + case {re:compile(Re), Ctx#context.field_type} of + {{ok, _CompiledRe}, FT} when FT =/= int -> + {ok, Ctx#context{term_regex = Re}}; + {_, int} -> + ErrMsg = + << + "Can not use term regular expressions" + " on integer queries" + >>, + {halt, 400, [?TXT_HEADER], ErrMsg, []}; + {{error, ErrorSpec}, _} -> + ErrMsg = + <<"Invalid term regular expression ~p : ~p">>, + {halt, 400, [?TXT_HEADER], ErrMsg, [Re, ErrorSpec]} + end + end. + +-spec validate_continuation( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +validate_continuation(Params, Ctx) -> + case lists:keyfind(<<"continuation">>, 1, Params) of + false -> + {ok, Ctx}; + {<<"continuation">>, C} when is_binary(C) -> + try + _ = riak_index:decode_continuation(C), + {ok, Ctx#context{continuation = C, pagination_sort = true}} + catch + _:_ -> + ErrMsg = + <<"Invalid continuation ~p - cannot be decoded">>, + {halt, 400, [?TXT_HEADER], ErrMsg, [C]} + end + end. + +%% =================================================================== +%% Query Handling +%% =================================================================== + +-spec process_memory_query( + any(), + context() +) -> + { + ok, + { + riak_api_web_acceptor:response_code(), + riak_api_web_headers:header_list(), + riak_api_web_handler:response_body(), + boolean(), + none + }, + context() + } + | riak_api_web_acceptor:halt_response(). +process_memory_query(Query, Ctx) -> + Client = Ctx#context.client, + Bucket = Ctx#context.bucket, + InitOpts = + case Ctx#context.pagination_sort of + true -> + [ + {max_results, Ctx#context.max_results}, + {pagination_sort, true} + ]; + false -> + [ + {max_results, Ctx#context.max_results} + ] + end, + Opts = riak_index:add_timeout_opt(Ctx#context.timeout, InitOpts), + + %% Do the index lookup... + case riak_client:get_index(Bucket, Query, Opts, Client) of + {ok, Results} -> + Continuation = + make_continuation( + Ctx#context.max_results, + Results + ), + JsonResults = + encode_results( + Ctx#context.return_terms_client, + Results, + Continuation + ), + { + ok, + { + 200, + [?JSN_HEADER], + iolist_to_binary(JsonResults), + true, + none + }, + Ctx + }; + {error, timeout} -> + ErrMsg = <<"Request timed out">>, + {halt, 503, [?TXT_HEADER], ErrMsg, []}; + {error, Reason} -> + ErrMsg = <<"Query failed due to ~0p">>, + {halt, 503, [?TXT_HEADER], ErrMsg, [Reason]} + end. + +-spec process_stream_query( + any(), + context() +) -> + { + ok, + { + riak_api_web_acceptor:response_code(), + riak_api_web_headers:header_list(), + riak_api_web_handler:response_body(), + boolean(), + none + }, + context() + }. +process_stream_query(Query, Ctx) -> + Client = Ctx#context.client, + Bucket = Ctx#context.bucket, + + %% Create a new multipart/mixed boundary + Boundary = riak_core_util:unique_id_62(), + CTypeHdr = + { + 'Content-Type', + iolist_to_binary( + [<<"multipart/mixed;boundary=">>, list_to_binary(Boundary)] + ) + }, + InitOpts = + case Ctx#context.pagination_sort of + true -> + [ + {max_results, Ctx#context.max_results}, + {pagination_sort, true} + ]; + false -> + [ + {max_results, Ctx#context.max_results} + ] + end, + Opts = riak_index:add_timeout_opt(Ctx#context.timeout, InitOpts), + {ok, ReqID, FSMPid} = + riak_client:stream_get_index(Bucket, Query, Opts, Client), + StreamFun = + index_stream_fun( + {ReqID, FSMPid}, + { + Boundary, + Ctx#context.return_terms_client, + Ctx#context.max_results + }, + {undefined, 0}, + proplists:get_value(timeout, Opts) + % Need to use same timeout as query, which may be different + % to any client timeout + ), + {ok, {200, [CTypeHdr], {stream, StreamFun}, true, none}, Ctx}. + +-spec index_stream_fun( + {non_neg_integer(), pid()}, + {unicode:chardata(), boolean(), pos_integer() | all}, + {{binary(), riak_object:key()} | undefined, non_neg_integer()}, + non_neg_integer() +) -> + stream_fun(). +index_stream_fun( + {ReqID, FSMPid}, + {Boundary, ReturnTerms, MaxResults}, + {LastResult, Count}, + Timeout +) -> + fun() -> + receive + {ReqID, done} -> + ToComeLessLast = + case MaxResults of + MR when is_integer(MR) -> + (MR - Count) + 1; + MR -> + MR + end, + Final = + case make_continuation(ToComeLessLast, [LastResult]) of + undefined -> + ["\r\n--", Boundary, "--\r\n"]; + Continuation -> + Json = + riak_kv_wm_json:encode( + #{?Q_2I_CONTINUATION_BIN => Continuation} + ), + [ + "\r\n--", + Boundary, + "\r\n", + "Content-Type: application/json\r\n\r\n", + Json, + "\r\n--", + Boundary, + "--\r\n" + ] + end, + { + iolist_to_binary(Final), + fun() -> done end + }; + {ReqID, {results, []}} -> + { + <<>>, + index_stream_fun( + {ReqID, FSMPid}, + {Boundary, ReturnTerms, MaxResults}, + {LastResult, Count}, + Timeout + ) + }; + {ReqID, {results, Results}} -> + JsonResults = + encode_results(ReturnTerms, Results, undefined), + Body = + [ + "\r\n--", + Boundary, + "\r\n", + "Content-Type: application/json\r\n\r\n", + JsonResults + ], + { + iolist_to_binary(Body), + index_stream_fun( + {ReqID, FSMPid}, + {Boundary, ReturnTerms, MaxResults}, + {lists:last(Results), Count + length(Results)}, + Timeout + ) + }; + {ReqID, Error} -> + stream_error(Error, Boundary) + after Timeout -> + whack_index_fsm(ReqID, FSMPid), + stream_error({error, timeout}, Boundary) + end + end. + +%% @doc When a streaming index query ends due to timeout, the web acceptor +%% process should not receive messages left over from the query +-spec whack_index_fsm(non_neg_integer(), pid()) -> ok. +whack_index_fsm(ReqID, Pid) -> + wait_for_death(Pid), + clear_index_fsm_msgs(ReqID). + +wait_for_death(Pid) -> + Ref = erlang:monitor(process, Pid), + exit(Pid, kill), + receive + {'DOWN', Ref, process, Pid, _Info} -> + ok + end. + +clear_index_fsm_msgs(ReqID) -> + receive + {ReqID, _} -> + clear_index_fsm_msgs(ReqID) + after 0 -> + ok + end. + +stream_error(Error, Boundary) -> + ?LOG_ERROR("Error in index wm: ~p", [Error]), + ErrorJson = encode_error(Error), + Body = [ + "\r\n--", + Boundary, + "\r\n", + "Content-Type: application/json\r\n\r\n", + ErrorJson, + "\r\n--", + Boundary, + "--\r\n" + ], + {iolist_to_binary(Body), fun() -> done end}. + +encode_error({error, E}) -> + encode_error(E); +encode_error(Error) when is_atom(Error); is_binary(Error) -> + riak_kv_wm_json:encode(#{error => Error}); +encode_error(Error) -> + E = io_lib:format("~0p", [Error]), + riak_kv_wm_json:encode(#{error => iolist_to_binary(E)}). + +%% =================================================================== +%% Internal Functions +%% =================================================================== + +-spec make_continuation( + all | non_neg_integer(), + list() +) -> + binary() | undefined. +make_continuation(MR, Results) when is_integer(MR), length(Results) == MR -> + riak_index:make_continuation(Results); +make_continuation(_, _) -> + undefined. + +size_limits() -> + { + 32, + 1024, + 0 + }. + +%% =================================================================== +%% JSON Encoding implementations +%% =================================================================== + +otp_encode_results(true, Results, undefined) -> + riak_kv_wm_json:encode( + #{?Q_RESULTS_BIN => Results}, + fun results_encode/2 + ); +otp_encode_results(true, Results, Continuation) -> + riak_kv_wm_json:encode( + #{ + ?Q_RESULTS_BIN => Results, + ?Q_2I_CONTINUATION_BIN => Continuation + }, + fun results_encode/2 + ); +otp_encode_results(false, Results, undefined) -> + riak_kv_wm_json:encode( + #{?Q_KEYS_BIN => Results}, + fun keys_encode/2 + ); +otp_encode_results(false, Results, Continuation) -> + riak_kv_wm_json:encode( + #{ + ?Q_KEYS_BIN => Results, + ?Q_2I_CONTINUATION_BIN => Continuation + }, + fun keys_encode/2 + ). + +results_encode({Term, Key}, Encode) when is_binary(Term), is_binary(Key) -> + [${, [Encode(Term, Encode), $: | Encode(Key, Encode)], $}]; +results_encode({Term, Key}, Encode) when is_integer(Term), is_binary(Key) -> + [ + ${, + [Encode(integer_to_binary(Term), Encode), $: | Encode(Key, Encode)], + $} + ]; +results_encode(Result, Encode) -> + riak_kv_wm_json:encode_value(Result, Encode). + +keys_encode({_Term, Key}, Encode) when is_binary(Key) -> + riak_kv_wm_json:encode_value(Key, Encode); +keys_encode(Object, Encode) -> + riak_kv_wm_json:encode_value(Object, Encode). + +encode_results(ReturnTerms, Results, Continuation) -> + otp_encode_results(ReturnTerms, Results, Continuation). + +%% =================================================================== +%% EUnit tests +%% =================================================================== + +-ifdef(TEST). + +-include_lib("eunit/include/eunit.hrl"). + +otp_encode_results(ReturnTerms, Results) -> + otp_encode_results(ReturnTerms, Results, undefined). + +encoder_test_() -> + {timeout, 600, fun encode_tester/0}. + +encode_tester() -> + % awkward silence to tidy screen output + timer:sleep(100), + encode_implementation_tester(otp). + +encode_implementation_tester(_Otp) -> + garbage_collect(), + + io:format(user, "~n~nTesting Implementation ~w~n", [otp]), + ResultSetsTiny = + [ + {<<"1K">>, large_results(1000)}, + {<<"2K">>, large_results(2000)}, + {<<"3K">>, large_results(3000)}, + {<<"5K">>, large_results(5000)}, + {<<"8K">>, large_results(8000)} + ], + encode_tester(ResultSetsTiny, microseconds), + + garbage_collect(), + + ResultSetsSmall = + [ + {<<"13K">>, large_results(13000)}, + {<<"21K">>, large_results(21000)}, + {<<"34K">>, large_results(34000)}, + {<<"55K">>, large_results(55000)} + ], + encode_tester(ResultSetsSmall, milliseconds), + + garbage_collect(), + + ResultSetsMid = + [ + {<<"100K">>, large_results(100000)}, + {<<"200K">>, large_results(200000)}, + {<<"300K">>, large_results(300000)}, + {<<"500K">>, large_results(500000)} + ], + encode_tester(ResultSetsMid, milliseconds), + + ok. + +encode_tester(ResultSets, Unit) -> + Divisor = + case Unit of + microseconds -> + 1; + milliseconds -> + 1000 + end, + Fun = fun otp_encode_results/2, + + TotalTime = + lists:sum( + lists:map( + fun({Tag, RS}) -> + garbage_collect(), + {TC, _Json} = + timer:tc(fun() -> Fun(true, RS) end), + io:format( + user, + "Result set of ~s in ~w ~p ", + [Tag, TC div Divisor, Unit] + ), + TC + end, + ResultSets + ) + ), + io:format(user, "Total time ~w ~p~n", [TotalTime div Divisor, Unit]). + +large_results(N) -> + lists:map( + fun(I) -> {generate_term(I), generate_key(I)} end, + lists:seq(1, N) + ). + +generate_term(I) -> + iolist_to_binary(io_lib:format("q~9..0B", [I])). + +generate_key(K) -> + iolist_to_binary(io_lib:format("k~9..0B", [rand:uniform(K)])). + +extract_params(URI) -> + uri_string:dissect_query( + maps:get( + query, + uri_string:normalize(URI, [return_map]) + ) + ). + +test_uri(URI) -> + URIBase = <<"types/T/buckets/B/index/index_bin/aStart/zEnd">>, + <>. + +valid_dollarkey_test() -> + InitCtx = + #context{ + bucket = {<<"T">>, <<"B">>}, + field = <<"$key">>, + start_term = <<"aStart">>, + end_term = <<"zEnd">> + }, + {ok, Ctx1} = parse_query_params([], InitCtx), + ?assertMatch(dollar, Ctx1#context.field_type), + {ok, Ctx2} = parse_query_params([], InitCtx#context{field = <<"$bucket">>}), + ?assertMatch(dollar, Ctx2#context.field_type). + +validate_return_terms_test() -> + % If $bucket, $key or equality query - return_terms should be ignored + InitCtx = + #context{ + bucket = {<<"T">>, <<"B">>}, + field = <<"$key">>, + start_term = <<"aStart">>, + end_term = <<"zEnd">> + }, + {ok, Ctx1} = parse_query_params([{<<"return_terms">>, true}], InitCtx), + ?assertMatch(false, Ctx1#context.return_terms_client), + {ok, Ctx2} = + parse_query_params( + [{<<"return_terms">>, true}], + InitCtx#context{field = <<"$bucket">>} + ), + ?assertMatch(false, Ctx2#context.return_terms_client), + {ok, Ctx3} = + parse_query_params( + [{<<"return_terms">>, true}], + InitCtx#context{ + field = <<"field_bin">>, + start_term = <<"term">>, + end_term = <<"term">> + } + ), + ?assertMatch(false, Ctx3#context.return_terms_client), + ValidCtx = + #context{ + bucket = {<<"T">>, <<"B">>}, + field = <<"field_bin">>, + start_term = <<"aStart">>, + end_term = <<"zEnd">> + }, + {ok, Ctx4} = parse_query_params([{<<"return_terms">>, true}], ValidCtx), + ?assertMatch(true, Ctx4#context.return_terms_client). + +validation_test() -> + InitCtx = + #context{ + bucket = {<<"T">>, <<"B">>}, + field = <<"index_bin">>, + start_term = <<"aStart">>, + end_term = <<"zEnd">> + }, + C1 = + base64:encode( + term_to_binary( + {<<"mTerm">>, <<"K">>} + ) + ), + URI1 = test_uri(<<"?timeout=10">>), + QP1 = extract_params(URI1), + {ok, Ctx1} = parse_query_params(QP1, InitCtx), + ?assertMatch(10, Ctx1#context.timeout), + ?assertMatch(bin, Ctx1#context.field_type), + URI2 = + test_uri( + iolist_to_binary( + [ + <<"?max_results=10&continuation=">>, + C1, + <<"&return_terms">>, + <<"&pagination_sort=true">>, + <<"&stream=true">>, + <<"&term_regex=">>, + uri_string:quote(<<".*[A-Z]{1}">>) + ] + ) + ), + {ok, Ctx2} = parse_query_params(extract_params(URI2), InitCtx), + ?assertMatch(10, Ctx2#context.max_results), + ?assertMatch(C1, Ctx2#context.continuation), + ?assertMatch(true, Ctx2#context.return_terms_client), + ?assertMatch(true, Ctx2#context.pagination_sort), + ?assertMatch(true, Ctx2#context.stream), + ?assertMatch(<<".*[A-Z]{1}">>, Ctx2#context.term_regex), + + % Try and regex an integer query + ?assertMatch( + halt, + element( + 1, + parse_query_params( + extract_params(URI2), + InitCtx#context{field = <<"index_int">>} + ) + ) + ), + ?assertMatch( + halt, + element( + 1, + parse_query_params( + extract_params(URI2), + InitCtx#context{field = <<"indexbin">>} + ) + ) + ), + ?assertMatch( + halt, + element( + 1, + parse_query_params( + extract_params(URI2), + InitCtx#context{field = <<"idx">>} + ) + ) + ), + + URI3 = test_uri(<<"?max_results=A">>), + ?assertMatch( + halt, + element(1, parse_query_params(extract_params(URI3), InitCtx)) + ), + URI4 = test_uri(<<"?timeout=A">>), + ?assertMatch( + halt, + element(1, parse_query_params(extract_params(URI4), InitCtx)) + ), + URI5 = test_uri(<<"?continuation=unencodedboundary">>), + ?assertMatch( + halt, + element(1, parse_query_params(extract_params(URI5), InitCtx)) + ), + URI6 = test_uri(<<"?return_terms=keys">>), + ?assertMatch( + halt, + element(1, parse_query_params(extract_params(URI6), InitCtx)) + ), + URI7 = + test_uri( + iolist_to_binary( + [ + <<"?term_regex=">>, + uri_string:quote(<<"(*invalid)">>) + ] + ) + ), + ?assertMatch( + halt, + element(1, parse_query_params(extract_params(URI7), InitCtx)) + ), + + URI8 = test_uri(<<"?return_terms=false">>), + {ok, Ctx8} = parse_query_params(extract_params(URI8), InitCtx), + ?assertMatch(false, Ctx8#context.return_terms_client). + +accept_header_test() -> + InitCtx = + #context{ + bucket = {<<"T">>, <<"B">>}, + field = <<"index_bin">>, + start_term = <<"aStart">>, + end_term = <<"zEnd">> + }, + Hdr1 = riak_api_web_headers:make([{'Accept', <<"application/json">>}]), + ?assertMatch(ok, element(1, parse_request_headers(Hdr1, InitCtx))), + Hdr2 = riak_api_web_headers:make([{'Accept', <<"application/*">>}]), + ?assertMatch(ok, element(1, parse_request_headers(Hdr2, InitCtx))), + Hdr3 = + riak_api_web_headers:make( + [ + {'Accept', <<"text/plain, application/*">>} + ] + ), + ?assertMatch(ok, element(1, parse_request_headers(Hdr3, InitCtx))), + Hdr4 = + riak_api_web_headers:make( + [ + {'Accept', <<"text/*, application/octet-stream">>} + ] + ), + ?assertMatch(halt, element(1, parse_request_headers(Hdr4, InitCtx))), + Hdr5 = + riak_api_web_headers:make( + [ + {'Accept', <<"*/*, application/octet-stream">>} + ] + ), + ?assertMatch(ok, element(1, parse_request_headers(Hdr5, InitCtx))), + Hdr6 = + riak_api_web_headers:make([ + {'Accept', <<"application/octet-stream">>} + ]), + ?assertMatch(halt, element(1, parse_request_headers(Hdr6, InitCtx))), + Hdr7 = + riak_api_web_headers:make([ + {'Accept', <<"*/*">>} + ]), + ?assertMatch(ok, element(1, parse_request_headers(Hdr7, InitCtx))), + Hdr8 = riak_api_web_headers:make([]), + ?assertMatch(ok, element(1, parse_request_headers(Hdr8, InitCtx))). + +simple_stream_test() -> + ReqID = rand:uniform(1000) + 1, + Boundary = riak_core_util:unique_id_62(), + GenFun = + fun(I) -> + { + list_to_binary(io_lib:format(<<"T~8..0B">>, [I])), + list_to_binary(io_lib:format(<<"K~8..0B">>, [I])) + } + end, + Me = self(), + FSMPid = + spawn( + fun() -> + Me ! {ReqID, {results, lists:map(GenFun, lists:seq(1, 100))}}, + Me ! {ReqID, {results, lists:map(GenFun, lists:seq(101, 200))}}, + Me ! {ReqID, {results, []}}, + Me ! {ReqID, {results, lists:map(GenFun, lists:seq(201, 300))}}, + Me ! {ReqID, done} + end + ), + StreamFun = + index_stream_fun( + {ReqID, FSMPid}, + {Boundary, true, 1000}, + {undefined, 0}, + 10000 + ), + {RBin1, StreamFun1} = StreamFun(), + {RBin2, StreamFun2} = StreamFun1(), + {RBin3, StreamFun3} = StreamFun2(), + {RBin4, StreamFun4} = StreamFun3(), + {RBin5, StreamFun5} = StreamFun4(), + ?assertMatch(done, StreamFun5()), + ?assertMatch(<<>>, RBin3), + Bin = iolist_to_binary([RBin1, RBin2, RBin3, RBin4, RBin5]), + R = + decode_results( + Bin, + iolist_to_binary(["\r\n--", Boundary]), + <<"\r\nContent-Type: application/json\r\n\r\n">>, + <<"--\r\n">>, + [] + ), + ?assertMatch(300, length(R)), + ok. + +decode_results(Footer, _Boundary, _Header, Footer, Acc) -> + Acc; +decode_results(Bin, Boundary, Header, Footer, Acc) -> + HS = byte_size(Header), + BS = byte_size(Boundary), + case Bin of + <> -> + [JsonBin, Remainder] = + string:split(Rest, Boundary, leading), + case JsonBin of + <<>> -> + decode_results( + Remainder, + Boundary, + Header, + Footer, + Acc + ); + JsonBin -> + #{<<"results">> := RL} = + riak_kv_wm_json:decode(JsonBin), + decode_results( + Remainder, + Boundary, + Header, + Footer, + Acc ++ RL + ) + end; + <> -> + decode_results(Rest, Boundary, Header, Footer, Acc) + end. + +stream_error_test() -> + Boundary = riak_core_util:unique_id_62(), + {Err1, ErrFun1} = stream_error({error, timeout}, Boundary), + ?assert(is_binary(Err1)), + ?assertMatch(done, ErrFun1()), + {Err2, ErrFun2} = stream_error({'EXIT', timeout}, Boundary), + ?assert(is_binary(Err2)), + ?assertMatch(done, ErrFun2()). + +-endif. diff --git a/src/riak_kv_ag_keylist.erl b/src/riak_kv_ag_keylist.erl new file mode 100644 index 000000000..4a8e34fce --- /dev/null +++ b/src/riak_kv_ag_keylist.erl @@ -0,0 +1,312 @@ +%% -------------------------------------------------------- +%% +%% Copyright (c) 2026 Martin Sumner +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +%% @doc Handler for HTTP keylist API (v2 or higher) + +-module(riak_kv_ag_keylist). + +-include("riak_kv_web.hrl"). + +-behaviour(riak_api_web_handler). + +-export( + [ + match_route/3, + check_permissions/5, + parse_query_params/2, + parse_request_headers/2, + process_request/2, + record_request/3 + ] +). + +-record(context, { + client = riak_client:new(node(), undefined) :: riak_client:riak_client(), + bucket :: riak_object:bucket(), + stream = false :: boolean(), + timeout :: pos_integer() | undefined +}). + +-type context() :: #context{}. + +%% =================================================================== +%% Callback functions +%% =================================================================== + +-spec match_route( + riak_api_web_acceptor:method(), + unicode:chardata(), + list(unicode:chardata()) +) -> + nomatch + | {method_not_allowed, list(riak_api_web_acceptor:method())} + | {ok, riak_api_web_handler:limits(), context()}. +match_route('GET', _, [<<"types">>, T, <<"buckets">>, B, <<"keys">>]) -> + { + ok, + {32, 2048, 0}, + #context{bucket = riak_kv_web_common:set_bucket(T, B)} + }; +match_route(Method, _, [<<"types">>, _T, <<"buckets">>, _B, <<"keys">>]) when + Method =/= 'GET', Method =/= 'POST' +-> + {method_not_allowed, ['GET', 'POST']}; +match_route(Method, Path, [<<"buckets">>, B, <<"keys">>]) -> + match_route( + Method, + Path, + [<<"types">>, <<"default">>, <<"buckets">>, B, <<"keys">>] + ); +match_route(_, _, _) -> + nomatch. + +%% @doc check_permissions for using this module or route +-spec check_permissions( + riak_api_web_headers:headers(), + riak_api_web_socket:scheme(), + riak_api_web_handler:peer_ip(), + riak_api_web_handler:peer_cert(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +check_permissions(ReqHeaders, Scheme, Peer, _Cert, Ctx) -> + Check = + riak_kv_web_common:check_permissions( + ReqHeaders, + Scheme, + Peer, + Ctx#context.bucket, + "riak_kv.list_keys" + ), + case Check of + true -> + {ok, Ctx}; + HaltResponse -> + HaltResponse + end. + +%% @doc parse and validate query params, passed as a map +-spec parse_query_params( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +parse_query_params(Params, Ctx) -> + Ctx1 = + case lists:keyfind(<<"keys">>, 1, Params) of + {<<"keys">>, <<"stream">>} -> + Ctx#context{stream = true}; + _ -> + Ctx + end, + validate_timeout(Params, Ctx1). + +%% @doc parse and validate the request headers +-spec parse_request_headers( + riak_api_web_headers:headers(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +parse_request_headers(ReqHeaders, Ctx) -> + case riak_kv_web_common:accept_json_only(ReqHeaders) of + ok -> + {ok, Ctx}; + HaltResponse -> + HaltResponse + end. + +%% @doc Process the request and produce a response +-spec process_request( + none, + context() +) -> + { + ok, + { + riak_api_web_acceptor:response_code(), + riak_api_web_headers:header_list(), + riak_api_web_handler:response_body(), + boolean(), + none + }, + context() + } + | riak_api_web_acceptor:halt_response(). +process_request(none, #context{bucket = B, client = C} = Ctx) -> + case Ctx#context.stream of + true -> + {ok, ReqId} = + riak_client:stream_list_keys(B, Ctx#context.timeout, C), + + { + ok, + { + 200, + [?JSN_HEADER], + {stream, key_stream_fun(ReqId)}, + true, + none + }, + Ctx + }; + false -> + case riak_client:list_keys(B, Ctx#context.timeout, C) of + {ok, KeyList} -> + JsonResults = + riak_kv_ag_index:encode_results( + false, + KeyList, + undefined + ), + { + ok, + { + 200, + [?JSN_HEADER], + iolist_to_binary(JsonResults), + true, + none + }, + Ctx + }; + {error, timeout} -> + ErrMsg = + riak_kv_wm_json:encode( + #{<<"error">> => <<"Request timed out">>} + ), + {halt, 503, [?JSN_HEADER], iolist_to_binary(ErrMsg), []}; + {error, Reason} -> + ErrMsg = + riak_kv_wm_json:encode( + #{ + <<"error">> => + iolist_to_binary( + io_lib:format("~0p", [Reason]) + ) + } + ), + {halt, 503, [?JSN_HEADER], iolist_to_binary(ErrMsg), []} + end + end. + +%% @doc Record the output of the interaction +-spec record_request( + riak_api_web_handler:timings(), + riak_api_web_handler:completion(), + context() +) -> + ok. +record_request(_Timings, _Completion, _Ctx) -> + ok. + +%% =================================================================== +%% Internal Functions +%% =================================================================== + +-spec key_stream_fun(non_neg_integer()) -> stream_fun(). +key_stream_fun(ReqId) -> + fun() -> + receive + {ReqId, From, {keys, Keys}} -> + _ = riak_kv_keys_fsm:ack_keys(From), + JsonResults = + riak_kv_ag_index:encode_results( + false, + Keys, + undefined + ), + { + iolist_to_binary(JsonResults), + key_stream_fun(ReqId) + }; + {ReqId, {keys, Keys}} -> + JsonResults = + riak_kv_ag_index:encode_results( + false, + Keys, + undefined + ), + { + iolist_to_binary(JsonResults), + key_stream_fun(ReqId) + }; + {ReqId, done} -> + {<<>>, fun() -> done end}; + {ReqId, {error, timeout}} -> + JsonError = + riak_kv_wm_json:encode( + #{<<"error">> => <<"Request timed out">>} + ), + { + iolist_to_binary(JsonError), + fun() -> done end + } + end + end. + +-spec validate_timeout( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +validate_timeout(Params, Ctx) -> + case riak_kv_web_common:get_timeout(Params) of + {ok, none} -> + {ok, Ctx}; + {ok, Timeout} -> + {ok, Ctx#context{timeout = Timeout}}; + HaltResponse -> + HaltResponse + end. + +%% =================================================================== +%% EUnit tests +%% =================================================================== + +-ifdef(TEST). + +-include_lib("eunit/include/eunit.hrl"). + +parse_test() -> + InitCtx = #context{bucket = {<<"T">>, <<"B">>}}, + {ok, Ctx1} = + parse_query_params( + [{<<"keys">>, <<"stream">>}, {<<"timeout">>, <<"1000">>}], + InitCtx + ), + ?assertMatch(true, Ctx1#context.stream), + ?assertMatch(1000, Ctx1#context.timeout), + ?assertMatch( + {halt, 400, _, _, _}, + parse_query_params( + [{<<"keys">>, <<"stream">>}, {<<"timeout">>, <<"A">>}], + InitCtx + ) + ), + ReqHeaders = riak_api_web_headers:make([{'Accept', <<"*/*">>}]), + ?assertMatch({ok, _}, parse_request_headers(ReqHeaders, Ctx1)), + ReqHeadersNoAccept = riak_api_web_headers:make([]), + ?assertMatch({ok, _}, parse_request_headers(ReqHeadersNoAccept, Ctx1)), + ReqHeadersPlain = riak_api_web_headers:make([{'Accept', <<"text/plain">>}]), + ?assertMatch( + {halt, 406, _, _, _}, + parse_request_headers(ReqHeadersPlain, Ctx1) + ). + +-endif. diff --git a/src/riak_kv_ag_object_delete.erl b/src/riak_kv_ag_object_delete.erl new file mode 100644 index 000000000..569d68cc7 --- /dev/null +++ b/src/riak_kv_ag_object_delete.erl @@ -0,0 +1,487 @@ +%% -------------------------------------------------------- +%% +%% Copyright (c) 2026 Martin Sumner +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +%% @doc Handler for HTTP API requests to 'DELETE' an object + +-module(riak_kv_ag_object_delete). + +-if(?OTP_RELEASE == 26). +-feature(maybe_expr, enable). +-endif. + +-include("riak_kv_web.hrl"). + +-behaviour(riak_api_web_handler). + +-export( + [ + match_route/3, + check_permissions/5, + parse_query_params/2, + parse_request_headers/2, + process_request/2, + record_request/3 + ] +). + +-define(DEL_DEFAULTS, #{ + w => default, + dw => default, + pw => default, + r => default, + pr => default, + rw => default, + n_val => default, + sloppy_quorum => default, + timeout => undefined +}). + +-type del_option_key() :: + w + | dw + | pw + | r + | pr + | rw + | n_val + | sloppy_quorum + | timeout. + +-type del_options() :: + #{ + w => pos_integer() | default | quorum | all, + dw => non_neg_integer() | default | quorum | all, + pw => non_neg_integer() | default | quorum | all, + r => non_neg_integer() | default | quorum | all, + pr => non_neg_integer() | default | quorum | all, + rw => non_neg_integer() | default | quorum | all, + n_val => non_neg_integer() | default | quorum | all, + sloppy_quorum => boolean() | default, + timeout => pos_integer() | undefined + }. + +-record(context, { + client = riak_client:new(node(), undefined) :: riak_client:riak_client(), + bucket :: riak_object:bucket(), + key :: riak_object:key(), + del_options = ?DEL_DEFAULTS :: del_options(), + vclock :: vclock:vclock() | undefined +}). + +-type context() :: #context{}. + +%% =================================================================== +%% Callback functions +%% =================================================================== + +-spec match_route( + riak_api_web_acceptor:method(), + unicode:chardata(), + list(unicode:chardata()) +) -> + nomatch + | {method_not_allowed, list(riak_api_web_acceptor:method())} + | {ok, riak_api_web_handler:limits(), context()}. +match_route( + Method, + _Path, + [<<"types">>, BucketType, <<"buckets">>, Bucket, <<"keys">>, Key] +) when is_binary(Key) -> + case Method of + Method when Method == 'DELETE' -> + Context = + #context{ + bucket = riak_kv_web_common:set_bucket(BucketType, Bucket), + key = Key + }, + {ok, size_limits(), Context}; + _OtherMethod -> + {method_not_allowed, ['DELETE']} + end; +match_route( + Method, + Path, + [<<"buckets">>, Bucket, <<"keys">>, Key] +) when is_binary(Key) -> + match_route( + Method, + Path, + [<<"types">>, <<"default">>, <<"buckets">>, Bucket, <<"keys">>, Key] + ); +match_route( + Method, + Path, + [<<"types">>, BucketType, <<"buckets">>, Bucket, <<"keys">>] +) when Method == 'POST' -> + K = iolist_to_binary(riak_core_util:unique_id_62()), + match_route( + Method, + Path, + [<<"types">>, BucketType, <<"buckets">>, Bucket, <<"keys">>, K] + ); +match_route(_Method, _Path, _SplitPath) -> + nomatch. + +%% @doc check_permissions for using this module or route +-spec check_permissions( + riak_api_web_headers:headers(), + riak_api_web_socket:scheme(), + riak_api_web_handler:peer_ip(), + riak_api_web_handler:peer_cert(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +check_permissions(ReqHeaders, Scheme, Peer, _Cert, Ctx) -> + Check = + riak_kv_web_common:check_permissions( + ReqHeaders, + Scheme, + Peer, + Ctx#context.bucket, + "riak_kv.delete" + ), + case Check of + true -> + % The PB API doesn't check type exists, however, the FSM will crash + % if it does not exist - so better to give a sensible error here. + % Note this requires the fetching (and discarding) of the type + % properties. + case riak_kv_web_common:check_type_exists(Ctx#context.bucket) of + ok -> + {ok, Ctx}; + HaltResponse -> + HaltResponse + end; + HaltResponse -> + HaltResponse + end. + +%% @doc parse and validate query params, passed as a map +-spec parse_query_params( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +parse_query_params([], Ctx) -> + % Typically we expect no options - so shortcut the validation in this case + {ok, Ctx}; +parse_query_params(Params, Ctx) -> + maybe + {ok, Ctx0} ?= validate_timeout(Params, Ctx), + {ok, Ctx1} ?= validate_counts(Params, Ctx0), + {ok, Ctx2} ?= validate_booleans(Params, Ctx1), + {ok, Ctx2} + else + HaltResponse -> + HaltResponse + end. + +%% @doc parse and validate the request headers +-spec parse_request_headers( + riak_api_web_headers:headers(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +parse_request_headers(ReqHeaders, Ctx) -> + maybe + {ok, Ctx0} ?= set_version_vector(ReqHeaders, Ctx), + {ok, Ctx0} + else + HaltResponse -> + HaltResponse + end. + +%% @doc Process the request and produce a response +-spec process_request( + none, + context() +) -> + { + ok, + { + riak_api_web_acceptor:response_code(), + riak_api_web_headers:header_list(), + riak_api_web_handler:response_body(), + boolean(), + none + }, + context() + } + | riak_api_web_acceptor:halt_response(). +process_request(none, Context) -> + DelOptions = + riak_kv_web_common:filter_options(Context#context.del_options), + Result = + case Context#context.vclock of + undefined -> + riak_client:delete( + Context#context.bucket, + Context#context.key, + DelOptions, + Context#context.client + ); + DecodedClock -> + riak_client:delete_vclock( + Context#context.bucket, + Context#context.key, + DecodedClock, + DelOptions, + Context#context.client + ) + end, + case Result of + ok -> + {ok, {204, [], <<>>, true, none}, Context}; + {error, notfound} -> + {ok, {404, [], <<>>, true, none}, Context}; + {error, Reason} -> + handle_error(Reason, Context) + end. + +%% @doc Record the output of the interaction +-spec record_request( + riak_api_web_handler:timings(), + riak_api_web_handler:completion(), + context() +) -> + ok. +record_request(_Timings, _Completion, _Ctx) -> + ok. + +%% =================================================================== +%% Validation functions +%% =================================================================== + +-spec validate_timeout( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +validate_timeout(Params, Ctx) -> + case riak_kv_web_common:get_timeout(Params) of + {ok, none} -> + {ok, Ctx}; + {ok, Timeout} -> + {ok, set_option(timeout, Timeout, Ctx)}; + HaltResponse -> + HaltResponse + end. + +-spec validate_counts( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +validate_counts(Params, Context) -> + FoldResult = + riak_kv_web_common:count_fold( + Params, + [ + <<"w">>, + <<"dw">>, + <<"pw">>, + <<"r">>, + <<"pr">>, + <<"rw">>, + <<"n_val">> + ], + Context#context.del_options + ), + case FoldResult of + UpdOpts when is_map(UpdOpts) -> + {ok, Context#context{del_options = UpdOpts}}; + HaltResponse -> + HaltResponse + end. + +-spec validate_booleans( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +validate_booleans(Params, Context) -> + FoldResult = + riak_kv_web_common:boolean_fold( + Params, + [<<"sloppy_quorum">>], + Context#context.del_options + ), + case FoldResult of + UpdOpts when is_map(UpdOpts) -> + {ok, Context#context{del_options = UpdOpts}}; + HaltResponse -> + HaltResponse + end. + +-spec set_version_vector( + riak_api_web_headers:headers(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +set_version_vector(ReqHeaders, Ctx) -> + case riak_kv_web_common:get_version_vector(ReqHeaders) of + {ok, none} -> + {ok, Ctx}; + {ok, DecodedClock} -> + {ok, Ctx#context{vclock = DecodedClock}}; + HaltResponse -> + HaltResponse + end. + +-spec handle_error(term(), context()) -> riak_api_web_acceptor:halt_response(). +handle_error(too_many_fails, _Ctx) -> + Msg = <<"Too Many write failures to satisfy W/DW">>, + {halt, 503, [?TXT_HEADER], Msg, []}; +handle_error(timeout, _Ctx) -> + {halt, 503, [?TXT_HEADER], <<"request timed out">>, []}; +handle_error({n_val_violation, N}, _Ctx) -> + Msg = + << + "Specified w/dw/pw/node_confirms" + " values invalid for bucket n value of ~p" + >>, + {halt, 400, [?TXT_HEADER], Msg, [N]}; +handle_error({dw_val_unsatisfied, DW, NumDW}, _Ctx) -> + {halt, 503, [?TXT_HEADER], <<"DW-value unsatisfied: ~p/~p">>, [NumDW, DW]}; +handle_error({pw_val_unsatisfied, PW, NumPW}, _Ctx) -> + {halt, 503, [?TXT_HEADER], <<"PW-value unsatisfied: ~p/~p">>, [NumPW, PW]}; +handle_error({pr_val_unsatisfied, PR, NumPR}, _Ctx) -> + Msg = <<"PR-value unsatisfied: ~p/~p">>, + {halt, 503, [?TXT_HEADER], Msg, [NumPR, PR]}; +handle_error({r_val_unsatisfied, R, NumR}, _Ctx) -> + Msg = <<"R-value unsatisfied: ~p/~p">>, + {halt, 503, [?TXT_HEADER], Msg, [NumR, R]}; +handle_error(OtherError, _Ctx) -> + {halt, 500, [?TXT_HEADER], <<"Error:~n~p">>, [OtherError]}. + +%% =================================================================== +%% Internal Functions +%% =================================================================== + +-spec set_option( + del_option_key(), + non_neg_integer() | quorum | all | boolean(), + context() +) -> + context(). +set_option(Option, Value, Context) -> + Context#context{ + del_options = maps:put(Option, Value, Context#context.del_options) + }. + +size_limits() -> + { + 1024, + 2048, + % A DELETE can never have a request body + 0 + }. + +%% =================================================================== +%% EUnit tests +%% =================================================================== + +-ifdef(TEST). + +-include_lib("eunit/include/eunit.hrl"). + +extract_params(URI) -> + uri_string:dissect_query( + maps:get( + query, + uri_string:normalize(URI, [return_map]) + ) + ). + +parameter_validation_test() -> + InitCtx = #context{bucket = {<<"T">>, <<"B">>}, key = <<"K">>}, + {ok, Ctx1} = + parse_query_params( + extract_params( + <<"/types/T/buckets/B/keys/K?timeout=10">> + ), + InitCtx + ), + ?assertMatch(10, maps:get(timeout, Ctx1#context.del_options)), + ?assertMatch( + {halt, 400, _, _, _}, + parse_query_params( + extract_params(<<"/types/T/buckets/B/keys/K?timeout=*">>), + InitCtx + ) + ), + {ok, Ctx2} = + parse_query_params( + extract_params( + <<"/types/T/buckets/B/keys/K?w=1&rw=1&sloppy_quorum=true">> + ), + InitCtx + ), + ?assertMatch(1, maps:get(w, Ctx2#context.del_options)), + ?assertMatch(1, maps:get(rw, Ctx2#context.del_options)), + ?assertMatch(true, maps:get(sloppy_quorum, Ctx2#context.del_options)), + + ?assertMatch( + {halt, 400, _, _, _}, + parse_query_params( + extract_params( + <<"/types/T/buckets/B/keys/K?w=A&rw=1&sloppy_quorum=true">> + ), + InitCtx + ) + ), + ?assertMatch( + {halt, 400, _, _, _}, + parse_query_params( + extract_params( + <<"/types/T/buckets/B/keys/K?w=1&rw=1&sloppy_quorum=1">> + ), + InitCtx + ) + ). + +header_validation_test() -> + Vc0 = vclock:increment('node1@127.0.0.1', vclock:fresh()), + Vc1 = vclock:increment('node1@127.0.0.1', Vc0), + Hdr0 = + { + <<"X-Riak-vclock">>, + base64:encode(riak_object:encode_vclock(Vc0)) + }, + Hdr1 = + { + <<"X-Riak-vclock">>, + base64:encode(riak_object:encode_vclock(Vc1)) + }, + InitCtx = #context{bucket = {<<"T">>, <<"B">>}, key = <<"K">>}, + {ok, Ctx1} = + parse_request_headers( + riak_api_web_headers:make([Hdr1]), + InitCtx + ), + ?assertMatch(Vc1, Ctx1#context.vclock), + ?assertMatch( + {halt, 400, _, _, _}, + parse_request_headers( + riak_api_web_headers:make([Hdr0, Hdr1]), + InitCtx + ) + ). + +-endif. diff --git a/src/riak_kv_ag_object_read.erl b/src/riak_kv_ag_object_read.erl new file mode 100644 index 000000000..780436b0f --- /dev/null +++ b/src/riak_kv_ag_object_read.erl @@ -0,0 +1,1468 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2026 Martin Sumner +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +%% @doc Handler for HTTP API requests to read an object ('GET' or 'HEAD' +%% requests) + +-module(riak_kv_ag_object_read). + +-if(?OTP_RELEASE == 26). +-feature(maybe_expr, enable). +-endif. + +-include("riak_object.hrl"). +-include("riak_kv_web.hrl"). + +-behaviour(riak_api_web_handler). + +-export( + [ + match_route/3, + check_permissions/5, + parse_query_params/2, + parse_request_headers/2, + process_request/2, + record_request/3 + ] +). + +-export( + [ + produce_response/1 + ] +). + +-define(GET_DEFAULTS, #{ + r => default, + pr => default, + node_confirms => default, + n_val => default, + notfound_ok => default, + deletedvclock => true, + basic_quorum => default, + sloppy_quorum => default, + return_body => true, + timeout => undefined +}). + +-define(NOT_FOUND(Headers, Ctx), { + ok, + {404, Headers, <<"not found">>, true, none}, + Context +}). + +-type get_options() :: + #{ + r => pos_integer() | default | quorum | all, + pr => non_neg_integer() | default | quorum | all, + node_confirms => non_neg_integer() | default | quorum | all, + n_val => pos_integer() | default, + notfound_ok => boolean() | default, + deletedvclock => true, + basic_quorum => boolean() | default, + sloppy_quorum => boolean() | default, + return_body => boolean(), + timeout => pos_integer() | undefined + }. + +-type get_option_key() :: + r + | pr + | node_confirms + | n_val + | notfound_ok + | basic_quorum + | sloppy_quorum + | timeout. + +-record(context, { + client = riak_client:new(node(), undefined) :: riak_client:riak_client(), + method :: 'GET' | 'HEAD', + bucket :: riak_object:bucket(), + key :: riak_object:key(), + get_options = ?GET_DEFAULTS :: get_options(), + vtag :: binary() | undefined, + all_types_accepted = true :: boolean(), + preferred_types = [] :: list(binary()) +}). + +-type context() :: #context{}. + +-export_type([get_options/0]). + +%% =================================================================== +%% Callback functions +%% =================================================================== + +-spec match_route( + riak_api_web_acceptor:method(), + unicode:chardata(), + list(unicode:chardata()) +) -> + nomatch + | {method_not_allowed, list(riak_api_web_acceptor:method())} + | {ok, riak_api_web_handler:limits(), context()}. +match_route( + Method, + _Path, + [<<"types">>, BucketType, <<"buckets">>, Bucket, <<"keys">>, Key] +) when is_binary(Key) -> + case Method of + Method when Method == 'GET'; Method == 'HEAD' -> + Context = + #context{ + method = Method, + bucket = riak_kv_web_common:set_bucket(BucketType, Bucket), + key = Key + }, + {ok, size_limits(), Context}; + _OtherMethod -> + {method_not_allowed, ['GET', 'HEAD']} + end; +match_route( + Method, + Path, + [<<"buckets">>, Bucket, <<"keys">>, Key] +) when is_binary(Key) -> + match_route( + Method, + Path, + [<<"types">>, <<"default">>, <<"buckets">>, Bucket, <<"keys">>, Key] + ); +match_route(_Method, _Path, _SplitPath) -> + nomatch. + +%% @doc check_permissions for using this module or route +-spec check_permissions( + riak_api_web_headers:headers(), + riak_api_web_socket:scheme(), + riak_api_web_handler:peer_ip(), + riak_api_web_handler:peer_cert(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +check_permissions(ReqHeaders, Scheme, Peer, _Cert, Ctx) -> + Check = + riak_kv_web_common:check_permissions( + ReqHeaders, + Scheme, + Peer, + Ctx#context.bucket, + "riak_kv.get" + ), + case Check of + true -> + % The PB API doesn't check type exists, however, the FSM will crash + % if it does not exist - so better to give a sensible error here. + % Note this requires the fetching (and discarding) of the type + % properties. + case riak_kv_web_common:check_type_exists(Ctx#context.bucket) of + ok -> + {ok, Ctx}; + HaltResponse -> + HaltResponse + end; + HaltResponse -> + HaltResponse + end. + +%% @doc parse and validate query params, passed as a map +-spec parse_query_params( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +parse_query_params([], Ctx) -> + % Typically we expect no options - so shortcut the validation in this case + {ok, Ctx}; +parse_query_params(Params, Ctx) -> + maybe + {ok, Ctx0} ?= validate_timeout(Params, Ctx), + {ok, Ctx1} ?= validate_counts(Params, Ctx0), + {ok, Ctx2} ?= validate_booleans(Params, Ctx1), + maybe_set_vtag(Params, Ctx2) + else + HaltResponse -> + HaltResponse + end. + +%% @doc parse and validate the request headers +-spec parse_request_headers( + riak_api_web_headers:headers(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +parse_request_headers(ReqHeaders, Ctx) -> + %% Normally the content-type is either accepted or not, and this needs to + %% be quickest if all content is accepted. + %% Preference may be required in sibling state. + case riak_api_web_headers:get_value('Accept', ReqHeaders) of + CTL when is_list(CTL) -> + { + ok, + Ctx#context{ + all_types_accepted = lists:any(fun maybe_all/1, CTL), + preferred_types = CTL + } + }; + CT when is_binary(CT) -> + { + ok, + Ctx#context{ + all_types_accepted = maybe_all(CT), + preferred_types = [CT] + } + }; + undefined -> + {ok, Ctx#context{all_types_accepted = true}} + end. + +%% @doc Process the request and produce a response +-spec process_request( + none, + context() +) -> + { + ok, + { + riak_api_web_acceptor:response_code(), + riak_api_web_headers:header_list(), + riak_api_web_handler:response_body(), + boolean(), + none + }, + context() + } + | riak_api_web_acceptor:halt_response(). +process_request(none, Context) -> + GetResponse = + riak_client:get( + Context#context.bucket, + Context#context.key, + riak_kv_web_common:filter_options( + Context#context.get_options + ), + Context#context.client + ), + case GetResponse of + {ok, RObj} -> + {Code, Headers, Body} = + produce_response(RObj, Context), + {ok, {Code, Headers, Body, true, none}, Context}; + {error, notfound} -> + %% Not a halt response - as connection may keepalive + ?NOT_FOUND([?TXT_HEADER], Context); + {error, {deleted, VClock}} -> + Headers = + [ + ?TXT_HEADER, + {?HEAD_DELETED, <<"true">>}, + { + ?HEAD_VCLOCK, + base64:encode( + riak_object:encode_vclock(VClock) + ) + } + ], + ?NOT_FOUND(Headers, Context); + Error -> + halt_error(Error) + end. + +%% @doc Record the output of the interaction +-spec record_request( + riak_api_web_handler:timings(), + riak_api_web_handler:completion(), + context() +) -> + ok. +record_request(_Timings, _Completion, _Ctx) -> + ok. + +%% =================================================================== +%% Validation functions +%% =================================================================== + +-spec maybe_set_vtag( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()}. +maybe_set_vtag(QueryParams, Context) -> + case lists:keyfind(<<"vtag">>, 1, QueryParams) of + false -> + {ok, Context}; + {<<"vtag">>, VTag} when is_binary(VTag) -> + {ok, Context#context{vtag = VTag}} + end. + +-spec validate_timeout( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +validate_timeout(Params, Ctx) -> + case riak_kv_web_common:get_timeout(Params) of + {ok, none} -> + {ok, Ctx}; + {ok, Timeout} -> + {ok, set_option(timeout, Timeout, Ctx)}; + HaltResponse -> + HaltResponse + end. + +-spec validate_counts( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +validate_counts(Params, Context) -> + FoldResult = + riak_kv_web_common:count_fold( + Params, + [<<"r">>, <<"pr">>, <<"n_val">>, <<"node_confirms">>], + Context#context.get_options + ), + case FoldResult of + UpdOpts when is_map(UpdOpts) -> + {ok, Context#context{get_options = UpdOpts}}; + HaltResponse -> + HaltResponse + end. + +-spec validate_booleans( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +validate_booleans(Params, Context) -> + FoldResult = + riak_kv_web_common:boolean_fold( + Params, + [<<"basic_quorum">>, <<"notfound_ok">>], + Context#context.get_options + ), + case FoldResult of + UpdOpts when is_map(UpdOpts) -> + {ok, Context#context{get_options = UpdOpts}}; + HaltResponse -> + HaltResponse + end. + +%% =================================================================== +%% Produce Response +%% =================================================================== + +-spec halt_error({error, term()}) -> riak_api_web_acceptor:halt_response(). +halt_error({error, timeout}) -> + {halt, 503, [?TXT_HEADER], <<"request timed out">>, []}; +halt_error({error, {n_val_violation, N}}) -> + Msg = + << + "Specified w/dw/pw/node_confirms values invalid" + " for bucket n value of ~0p" + >>, + {halt, 400, [?TXT_HEADER], Msg, [N]}; +halt_error({error, {r_val_unsatisfied, Requested, Returned}}) -> + Msg = <<"R-value unsatisfied: ~p/~p">>, + {halt, 503, [?TXT_HEADER], Msg, [Returned, Requested]}; +halt_error({error, {pr_val_unsatisfied, Requested, Returned}}) -> + Msg = <<"PR-value unsatisfied: ~p/~p">>, + {halt, 503, [?TXT_HEADER], Msg, [Returned, Requested]}; +halt_error({error, UnexpectedError}) -> + {halt, 500, [?TXT_HEADER], <<"Error:~n~p~n">>, [UnexpectedError]}. + +-spec produce_response( + riak_object:riak_object() +) -> + {200 | 300 | 400 | 406, riak_api_web_headers:header_list(), binary()}. +produce_response(Object) -> + produce_response( + Object, + #context{ + method = 'GET', + bucket = riak_object:bucket(Object), + key = riak_object:key(Object) + } + ). + +-spec produce_response( + riak_object:riak_object(), + context() +) -> + {200 | 300 | 400 | 406, riak_api_web_headers:header_list(), binary()}. +produce_response(RObj, Ctx) -> + Vclock = riak_object:vclock(RObj), + VcHdr = + { + ?HEAD_VCLOCK, + base64:encode(riak_object:encode_vclock(Vclock)) + }, + case riak_object:get_contents(RObj) of + [SingletonObject] -> + handle_singleton_object(SingletonObject, VcHdr, Ctx); + Siblings -> + EtHdr = + { + 'Etag', + riak_kv_web_common:make_clock_etag(Vclock) + }, + handle_multiple_objects(Siblings, VcHdr, EtHdr, Ctx) + end. + +-spec handle_singleton_object( + {riak_object:riak_object_meta(), riak_object:value()}, + {binary(), binary()}, + context() +) -> + {200 | 400 | 406, riak_api_web_headers:header_list(), binary()}. +handle_singleton_object({MD0, Value0}, VcHdr, Ctx) -> + {MD, Value} = + encode_value(MD0, Value0, Ctx#context.method == 'GET'), + EtHdr = + { + 'Etag', + list_to_binary(riak_object:metadata_fetch(?MD_VTAG, MD)) + }, + LmdHdr = + { + 'Last-Modified', + riak_api_web:rfc1123_date( + riak_object:metadata_fetch(?MD_LASTMOD, MD) + ) + }, + InitHdrs = [LmdHdr, VcHdr, EtHdr], + case produce_response_headers(MD, InitHdrs, type, Ctx) of + Hdrs when is_list(Hdrs) -> + {200, Hdrs, Value}; + {RspCode, Hdrs, Body} -> + {RspCode, Hdrs, Body} + end. + +-spec handle_multiple_objects( + list({riak_object:riak_object_meta(), riak_object:value()}), + {binary(), binary()}, + {'Etag', binary()}, + context() +) -> + {200 | 400 | 406, riak_api_web_headers:header_list(), binary()}. +handle_multiple_objects( + Siblings, + VcHdr, + _EtHdr, + Ctx = #context{vtag = VTag} +) when is_binary(VTag) -> + ChosenSibs = + lists:filter( + fun({MD, _V}) -> + iolist_to_binary(riak_object:metadata_fetch(?MD_VTAG, MD)) == + VTag + end, + Siblings + ), + case ChosenSibs of + [{MD0, Value0}] -> + {MD, Value} = + encode_value(MD0, Value0, Ctx#context.method == 'GET'), + EtHdr = {'Etag', VTag}, + LmHdr = + { + 'Last-Modified', + riak_api_web:rfc1123_date( + riak_object:metadata_fetch( + ?MD_LASTMOD, + MD + ) + ) + }, + SibHeaders = + produce_response_headers(MD, [VcHdr, EtHdr, LmHdr], type, Ctx), + case SibHeaders of + Hdrs when is_list(Hdrs) -> + {200, Hdrs, Value}; + {RspCode, Hdrs, Body} -> + {RspCode, Hdrs, Body} + end; + _ -> + { + 400, + [?TXT_HEADER], + iolist_to_binary( + io_lib:format( + <<"VTag ~s failed to match an individual sibling">>, + [VTag] + ) + ) + } + end; +handle_multiple_objects(Siblings, VcHdr, EtHdr, Ctx) -> + case multipart_preferred(Ctx) of + true -> + Boundary = produce_boundary(), + EncodedSibs = + lists:map( + fun({MD0, V0}) -> + {MD, V} = + encode_value(MD0, V0, Ctx#context.method == 'GET'), + multipart_body_part(Boundary, MD, V, Ctx) + end, + Siblings + ), + Terminator = + iolist_to_binary([<<"\r\n--">>, Boundary, <<"--\r\n">>]), + SibValue = iolist_to_binary(EncodedSibs), + ObjHeaders = + [ + VcHdr, + EtHdr, + last_modified_header(Siblings), + { + 'Content-Type', + iolist_to_binary( + [<<"multipart/mixed; boundary=">>, Boundary] + ) + } + ], + {200, ObjHeaders, <>}; + false -> + VTags = + lists:map( + fun({M, _V}) -> + riak_object:metadata_fetch(?MD_VTAG, M) + end, + Siblings + ), + LmdHdr = last_modified_header(Siblings), + Body = [<<"Siblings:\n">>, [[V, <<"\n">>] || V <- VTags]], + {300, [VcHdr, EtHdr, LmdHdr, ?TXT_HEADER], iolist_to_binary(Body)}; + error -> + {406, [], <<>>} + end. + +-spec multipart_preferred(context()) -> boolean() | error. +multipart_preferred(Context) -> + {MultiPreference, MultiScore} = + riak_kv_web_common:type_preference( + <<"multipart/mixed">>, + Context#context.preferred_types + ), + {TextPreference, TextScore} = + riak_kv_web_common:type_preference( + <<"text/plain">>, + Context#context.preferred_types + ), + TextAccepted = TextPreference orelse Context#context.all_types_accepted, + case {MultiPreference, MultiScore > TextScore, TextAccepted} of + {true, true, _} -> + true; + {_, _, true} -> + false; + _ -> + error + end. + +-spec last_modified_header( + list({riak_object:riak_object_meta(), riak_object:value()}) +) -> + {'Last-Modified', binary()}. +last_modified_header(Siblings) when is_list(Siblings) -> + LMDs = + lists:map( + fun({M, _V}) -> + riak_object:metadata_fetch(?MD_LASTMOD, M) + end, + Siblings + ), + LastLMD = lists:last(lists:sort(LMDs)), + {'Last-Modified', riak_api_web:rfc1123_date(LastLMD)}. + +-spec multipart_body_part( + binary(), + riak_object:riak_object_meta(), + riak_object:value(), + context() +) -> + binary(). +multipart_body_part(Boundary, MD, Val, Ctx) -> + EtHdr = + { + 'Etag', + uri_string:quote(riak_object:metadata_fetch(?MD_VTAG, MD)) + }, + LmdHdr = + { + 'Last-Modified', + riak_api_web:rfc1123_date( + riak_object:metadata_fetch(?MD_LASTMOD, MD) + ) + }, + Hdrs = + produce_response_headers( + MD, + [EtHdr, LmdHdr], + type, + Ctx#context{all_types_accepted = true} + ), + case Hdrs of + Hdrs when is_list(Hdrs) -> + BodyL = + [ + <<"\r\n--">>, + Boundary, + <<"\r\n">>, + riak_api_web_headers:output_response_block( + riak_api_web_headers:make_rsp_header(Hdrs) + ), + <<"\r\n">>, + Val + ], + iolist_to_binary(BodyL); + _ -> + <<>> + end. + +produce_boundary() -> + <> = + crypto:hash(sha, term_to_binary({self(), os:timestamp()})), + integer_to_binary(I, 36). + +-spec produce_response_headers( + riak_object:riak_object_meta(), + riak_api_web_headers:header_list(), + type | encoding | meta | index, + context() +) -> + riak_api_web_headers:header_list() + | {200 | 300 | 406, riak_api_web_headers:header_list(), binary()}. +produce_response_headers(MD, Hdrs, type, Context) -> + ContentType = get_ctype(MD), + TypeMatch = + case Context#context.all_types_accepted of + true -> + true; + false -> + riak_kv_web_common:type_match( + ContentType, + Context#context.preferred_types + ) + end, + case TypeMatch of + true -> + ExtendedCType = + case riak_object:metadata_find(?MD_CHARSET, MD) of + {ok, CS} when is_binary(CS); is_list(CS); is_atom(CS) -> + iolist_to_binary( + [ContentType, <<"; charset=">>, ensure_binary(CS)] + ); + error -> + ContentType + end, + produce_response_headers( + MD, + [{'Content-Type', ExtendedCType} | Hdrs], + encoding, + Context + ); + false -> + ErrMsg = + io_lib:format( + <<"Content-Type ~0p not in accepted types of ~0p">>, + [ContentType, Context#context.preferred_types] + ), + {406, [?TXT_HEADER], iolist_to_binary(ErrMsg)}; + error -> + DefaultCType = <<"application/octet-stream">>, + MatchDefault = + riak_kv_web_common:type_match( + DefaultCType, + Context#context.preferred_types + ), + case MatchDefault of + true -> + produce_response_headers( + MD, + [{'Content-Type', DefaultCType} | Hdrs], + encoding, + Context + ); + _ -> + ErrMsg = + io_lib:format( + << + "Content-Type ~0p invalid and " + "default of ~0p not accepted" + >>, + [ContentType, DefaultCType] + ), + {406, [?TXT_HEADER], iolist_to_binary(ErrMsg)} + end + end; +produce_response_headers(MD, Hdrs, encoding, Context) -> + case riak_object:metadata_find(?MD_ENCODING, MD) of + {ok, Enc} when is_binary(Enc); is_list(Enc); is_atom(Enc) -> + produce_response_headers( + MD, + [{'Content-Encoding', ensure_binary(Enc)} | Hdrs], + meta, + Context + ); + error -> + produce_response_headers(MD, Hdrs, meta, Context) + end; +produce_response_headers(MD, Hdrs, meta, Context) -> + UpdHdrs = + binary_header_fold(MD, Hdrs, ?MD_USERMETA, ?HEAD_USERMETA_PREFIX), + produce_response_headers(MD, UpdHdrs, index, Context); +produce_response_headers(MD, Hdrs, index, _Context) -> + binary_header_fold(MD, Hdrs, ?MD_INDEX, ?HEAD_INDEX_PREFIX). + +binary_header_fold(MD, Hdrs, MetaKey, Prefix) -> + UserMeta = + case riak_object:metadata_find(MetaKey, MD) of + {ok, KVL} when is_list(KVL) -> + KVL; + _ -> + [] + end, + lists:foldl( + fun({K, V}, Acc) -> + [ + { + << + Prefix/binary, + (ensure_binary(K))/binary + >>, + ensure_binary(V) + } + | Acc + ] + end, + Hdrs, + UserMeta + ). + +-spec get_ctype(riak_object:riak_object_meta()) -> binary(). +%% @doc Work out the content type for this object - use the metadata if provided +get_ctype(MD) -> + case riak_object:metadata_find(?MD_CTYPE, MD) of + {ok, SingleType} when is_binary(SingleType) -> + SingleType; + {ok, TypeAsCharData} when is_list(TypeAsCharData) -> + ensure_binary(TypeAsCharData); + error -> + <<"application/octet-stream">> + end. + +-spec ensure_binary(atom() | binary() | list()) -> binary(). +ensure_binary(B) when is_binary(B) -> + B; +ensure_binary(A) when is_atom(A) -> + atom_to_binary(A); +ensure_binary(List) when is_list(List) -> + iolist_to_binary(List). + +%% @doc +%% Need to handle values stored not as binaries but as Erlang terms through +%% the direct Erlang API. If it is not a binary, we assume it is a term and +%% set the content type as such regardless. +%% Also the context may not be interested in the body, so we cna replace with +%% an empty binary in this case (e.g. 'HEAD' request). +-spec encode_value( + riak_object:riak_object_meta(), + binary() | term(), + boolean() +) -> + {riak_object:riak_object_meta(), binary()}. +encode_value(MD, Value, true) when is_binary(Value) -> + {MD, Value}; +encode_value(MD, Value, false) when is_binary(Value) -> + {MD, <<>>}; +encode_value(MD, Value, IsGet) -> + MD0 = + riak_object:metadata_store( + ?MD_CTYPE, + <<"application/x-erlang-binary">>, + MD + ), + case IsGet of + true -> + {MD0, term_to_binary(Value)}; + false -> + {MD0, <<>>} + end. + +%% =================================================================== +%% Internal Functions +%% =================================================================== + +-spec set_option( + get_option_key(), + non_neg_integer() | quorum | all | boolean(), + context() +) -> + context(). +set_option(Option, Value, Context) -> + Context#context{ + get_options = maps:put(Option, Value, Context#context.get_options) + }. + +size_limits() -> + { + 1024, + 2048, + % A GET/HEAD can never have a request body + 0 + }. + +-spec maybe_all(binary()) -> boolean(). +maybe_all(CType) -> + case hd(binary:split(CType, <<";">>, [])) of + <<"*/*">> -> + true; + _ -> + false + end. + +%% =================================================================== +%% EUnit tests +%% =================================================================== + +-ifdef(TEST). + +-include_lib("eunit/include/eunit.hrl"). + +type_match(Type, AcceptedTypes) -> + riak_kv_web_common:type_match(Type, AcceptedTypes). + +accept_multipart_test() -> + Accept1 = + [ + { + 'Accept', + <<"*/*">> + } + ], + % Accept anything, and so that includes multipart + Headers1 = riak_api_web_headers:make(Accept1), + DummyCtx = + #context{ + bucket = {<<"Type">>, <<"B">>}, + key = <<"K">>, + method = 'GET' + }, + {ok, Ctx1} = parse_request_headers(Headers1, DummyCtx), + ?assertMatch(false, multipart_preferred(Ctx1)), + ?assertMatch(true, Ctx1#context.all_types_accepted), + Accept2 = + [ + { + 'Accept', + [<<"multipart/mixed">>, <<"*/*;q=0.9">>] + } + ], + Headers2 = riak_api_web_headers:make(Accept2), + {ok, Ctx2} = parse_request_headers(Headers2, DummyCtx), + ?assertMatch(true, multipart_preferred(Ctx2)), + ?assertMatch(true, Ctx2#context.all_types_accepted), + Accept3 = + [ + { + 'Accept', + [<<"application/json">>, <<"multipart/*;q=0.9">>] + } + ], + Headers3 = riak_api_web_headers:make(Accept3), + {ok, Ctx3} = parse_request_headers(Headers3, DummyCtx), + ?assertMatch(true, multipart_preferred(Ctx3)), + ?assertMatch( + [<<"application/json">>, <<"multipart/*;q=0.9">>], + Ctx3#context.preferred_types + ), + ?assertMatch(false, Ctx3#context.all_types_accepted), + Accept4 = + [ + { + 'Accept', + <<"multipart/mixed">> + } + ], + Headers4 = riak_api_web_headers:make(Accept4), + {ok, Ctx4} = parse_request_headers(Headers4, DummyCtx), + ?assertMatch(true, multipart_preferred(Ctx4)), + ?assertMatch( + [<<"multipart/mixed">>], + Ctx4#context.preferred_types + ). + +accept_filter_test() -> + Accept1 = + [ + { + 'Accept', + [ + <<"text/html">>, + <<"application/xhtml+xml">>, + <<"application/xml;q=0.9">>, + <<"image/*">> + ] + } + ], + Headers1 = riak_api_web_headers:make(Accept1), + DummyCtx = + #context{ + bucket = {<<"Type">>, <<"B">>}, + key = <<"K">>, + method = 'GET' + }, + {ok, UpdCtx} = parse_request_headers(Headers1, DummyCtx), + AcceptedTypes1 = UpdCtx#context.preferred_types, + ?assert(is_list(AcceptedTypes1)), + ?assertNot(UpdCtx#context.all_types_accepted), + ?assert(type_match(ensure_binary("text/html"), AcceptedTypes1)), + ?assertNot(type_match(ensure_binary("application/json"), AcceptedTypes1)), + ?assert(type_match(ensure_binary("image/jpeg"), AcceptedTypes1)), + ?assertNot(type_match(ensure_binary("application/xhtml"), AcceptedTypes1)), + ?assert(type_match(ensure_binary("application/xhtml+xml"), AcceptedTypes1)), + ?assert(type_match(ensure_binary("application/xml"), AcceptedTypes1)), + + HdrList1 = + produce_response_headers( + #{<<"content-type">> => "application/xml"}, [], type, UpdCtx + ), + ?assertMatch([{'Content-Type', <<"application/xml">>}], HdrList1), + ?assertMatch( + {406, [?TXT_HEADER], _}, + produce_response_headers( + #{<<"content-type">> => "application+badly-formatted"}, + [], + type, + UpdCtx + ) + ), + ?assertMatch( + {406, [?TXT_HEADER], _}, + produce_response_headers( + #{<<"content-type">> => "application/json"}, [], type, UpdCtx + ) + ). + +metadata_format_test() -> + UserMeta = + [ + {<<"postCode">>, <<"LS1 4BT">>}, + {<<"postCode">>, <<"LS11_0ES">>}, + {<<"familyName">>, <<"ROBERTS">>} + ], + IndexSpecs = + [ + {<<"pc_bin">>, <<"LS1_4BT|ROBERTS">>}, + {<<"pc_bin">>, <<"LS11_0ES|ROBERTS">>}, + {<<"family_bin">>, <<"ROBERTS|LS1_4BT.LS11_0ES">>} + ], + VTag = + riak_core_util:integer_to_list( + erlang:phash2(term_to_binary({'node1', os:timestamp()})), + 62 + ), + LMD = os:timestamp(), + MetaData = + maps:from_list( + [ + {?MD_USERMETA, UserMeta}, + {?MD_INDEX, IndexSpecs}, + {?MD_VTAG, VTag}, + {?MD_LASTMOD, LMD}, + {?MD_CTYPE, "application/json"} + ] + ), + O = riak_object:new({<<"BT">>, <<"B">>}, <<"K">>, <<"ObjVal">>), + O1 = riak_object:update_metadata(O, MetaData), + O2 = riak_object:apply_updates(O1), + Ctx = + #context{ + method = 'GET', + bucket = {<<"BT">>, <<"B">>}, + key = <<"K">> + }, + {200, HdrList, <<"ObjVal">>} = produce_response(O2, Ctx), + ?assertMatch(10, length(HdrList)), + HeaderMap = + riak_api_web_headers:enter_from_list( + HdrList, + riak_api_web_headers:make_rsp_header( + [{'Server', <<"RiakEUnit/4.0 SilverMachine">>}] + ) + ), + HeaderBin = riak_api_web_headers:output_response_block(HeaderMap), + % confirm multiple index entries folded into one line + ?assertNotMatch( + nomatch, + string:find( + HeaderBin, + <<"X-Riak-Index-pc_bin: LS1_4BT\|ROBERTS, LS11_0ES|ROBERTS">> + ) + ), + % likewise, multiple metadata with the same key folded into list + ?assertNotMatch( + nomatch, + string:find( + HeaderBin, + <<"X-Riak-Meta-postCode: LS1 4BT, LS11_0ES">> + ) + ), + ErlLMD = + iolist_to_binary( + io_lib:format( + <<"Last-Modified: ~s">>, + [httpd_util:rfc1123_date(calendar:now_to_local_time(LMD))] + ) + ), + ?assertNotMatch( + nomatch, + string:find( + HeaderBin, + ErlLMD + ) + ), + SWs = os:system_time(microsecond), + lists:map( + fun(_I) -> + {200, HdrList, <<"ObjVal">>} = produce_response(O2, Ctx), + HeaderMap = + riak_api_web_headers:enter_from_list( + HdrList, + riak_api_web_headers:make_rsp_header( + [{'Server', <<"RiakEUnit/4.0 SilverMachine">>}] + ) + ) + end, + lists:seq(1, 1000) + ), + SWe = os:system_time(microsecond), + io:format( + user, + "1000 response header blocks in ~w microseconds~n", + [SWe - SWs] + ). + +validate_timeout_test() -> + Ctx = + #context{ + method = 'GET', + bucket = {<<"T">>, <<"B">>}, + key = <<"K">> + }, + QP1 = extract_params(<<"types/T/buckets/B/keys/K?timeout=10">>), + {ok, Ctx1} = validate_timeout(QP1, Ctx), + ?assertMatch(10, maps:get(timeout, Ctx1#context.get_options)), + QP2 = extract_params(<<"types/T/buckets/B/keys/K?timeout=-2">>), + ?assertMatch( + {halt, 400, [?TXT_HEADER], <<"Bad timeout value ~0p">>, [<<"-2">>]}, + validate_timeout(QP2, Ctx) + ), + QP3 = extract_params(<<"types/T/buckets/B/keys/K?timeout=XC">>), + ?assertMatch( + {halt, 400, [?TXT_HEADER], <<"Bad timeout value ~0p">>, [<<"XC">>]}, + validate_timeout(QP3, Ctx) + ), + QP4 = extract_params(<<"types/T/buckets/B/keys/K?timoeut=100">>), + ?assertMatch( + % Not timeout extracted - misspelling + {ok, Ctx}, + validate_timeout(QP4, Ctx) + ). + +validate_counts_test() -> + Ctx = + #context{ + method = 'GET', + bucket = {<<"T">>, <<"B">>}, + key = <<"K">> + }, + QP1 = + extract_params( + <<"types/T/buckets/B/keys/K?r=all&pr=2&node_confirms=1&n_val=default">> + ), + {ok, Ctx1} = validate_counts(QP1, Ctx), + ?assertMatch(all, maps:get(r, Ctx1#context.get_options)), + ?assertMatch(2, maps:get(pr, Ctx1#context.get_options)), + ?assertMatch(1, maps:get(node_confirms, Ctx1#context.get_options)), + ?assertMatch(default, maps:get(n_val, Ctx1#context.get_options)), + QP2 = + extract_params( + <<"types/T/buckets/B/keys/K?r=all&pr=2&node_confirms=1&nval=-1">> + ), + {ok, Ctx2} = validate_counts(QP2, Ctx), + % n_val still default due to misspelling + ?assertMatch(Ctx1, Ctx2), + QP3 = + extract_params( + <<"types/T/buckets/B/keys/K?r=-1&pr=2&node_confirms=1">> + ), + {halt, 400, _, _, [<<"r">>]} = validate_counts(QP3, Ctx), + {ok, Ctx3} = parse_query_params(QP2, Ctx), + ?assertMatch(Ctx2, Ctx3). + +validate_bools_test() -> + Ctx = + #context{ + method = 'GET', + bucket = {<<"T">>, <<"B">>}, + key = <<"K">> + }, + QP1 = + extract_params( + <<"types/T/buckets/B/keys/K?basic_quorum=true¬found_ok=false">> + ), + {ok, Ctx1} = validate_booleans(QP1, Ctx), + ?assertMatch(false, maps:get(notfound_ok, Ctx1#context.get_options)), + ?assertMatch(true, maps:get(basic_quorum, Ctx1#context.get_options)), + QP2 = + extract_params( + <<"types/T/buckets/B/keys/K?basic_quorum=true¬found_ok=flase">> + ), + {halt, 400, _, _, [<<"notfound_ok">>]} = validate_booleans(QP2, Ctx), + QP3 = + extract_params( + <<"types/T/buckets/B/keys/K?basic_quorum=true¬foundok=false">> + ), + {ok, Ctx3} = validate_booleans(QP3, Ctx), + ?assertMatch(default, maps:get(notfound_ok, Ctx3#context.get_options)), + % Misspell notfound_ok + ?assertMatch(true, maps:get(basic_quorum, Ctx3#context.get_options)). + +extract_params(URI) -> + uri_string:dissect_query( + maps:get( + query, + uri_string:normalize(URI, [return_map]) + ) + ). + +with_bucket_prop_test_() -> + { + setup, + fun() -> + meck:new(riak_core_bucket), + meck:expect(riak_core_bucket, get_bucket, fun(_) -> [] end) + end, + fun(_) -> meck:unload(riak_core_bucket) end, + [ + {"Singleton response", fun singleton_response/0}, + {"Sibling response", fun sibling_response/0} + ] + }. + +singleton_response() -> + LastMod = os:timestamp(), + ET1 = "a123456zz", + ET1B = list_to_binary(ET1), + O0 = riak_object:new({<<"T">>, <<"B">>}, <<"K">>, <<"willy">>), + MD0 = get_metadata(LastMod, ET1), + O1 = + riak_object:increment_vclock( + riak_object:update_metadata( + riak_object:update_value( + O0, + <<"gnonto">> + ), + MD0 + ), + x + ), + OM1 = riak_object:syntactic_merge(O0, O1), + Ctx = + #context{ + method = 'GET', + bucket = {<<"T">>, <<"B">>}, + key = <<"K">> + }, + {200, HdrList1, <<"gnonto">>} = produce_response(OM1, Ctx), + ?assertMatch({'Etag', ET1B}, lists:keyfind('Etag', 1, HdrList1)), + ErlTerm = maps:put(name, <<"gnonto">>, maps:new()), + O2 = + riak_object:increment_vclock( + riak_object:update_value( + O1, + ErlTerm + ), + y + ), + OM2 = riak_object:syntactic_merge(OM1, O2), + {200, HdrList2, ConvertedVal2} = + produce_response(OM2, Ctx), + ?assertMatch(ErlTerm, binary_to_term(ConvertedVal2)), + ?assertMatch( + {'Content-Type', <<"application/x-erlang-binary">>}, + lists:keyfind('Content-Type', 1, HdrList2) + ), + ?assertMatch({'Etag', ET1B}, lists:keyfind('Etag', 1, HdrList2)), + PlainTextValue = "gnonto", + O3 = + riak_object:increment_vclock( + riak_object:update_value( + O2, + PlainTextValue + ), + z + ), + OM3 = riak_object:syntactic_merge(OM2, O3), + {200, HdrList3, ConvertedVal3} = + produce_response(OM3, Ctx), + ?assertMatch("gnonto", binary_to_term(ConvertedVal3)), + ?assertMatch({'Etag', ET1B}, lists:keyfind('Etag', 1, HdrList3)), + CtxHead = + #context{ + method = 'HEAD', + bucket = {<<"T">>, <<"B">>}, + key = <<"K">> + }, + {200, HdrList4, <<>>} = + produce_response(OM3, CtxHead), + ?assertMatch({'Etag', ET1B}, lists:keyfind('Etag', 1, HdrList4)). + +sibling_response() -> + LastMod1 = os:timestamp(), + LastMod2 = setelement(2, LastMod1, element(2, LastMod1) + 1), + ET1 = <<"a123456zz">>, + ET2 = <<"b123456zz">>, + O0 = riak_object:new({<<"T">>, <<"B">>}, <<"K">>, <<"willy">>), + O1 = + riak_object:increment_vclock( + riak_object:update_metadata( + riak_object:update_value( + O0, + <<"gnonto">> + ), + get_metadata(LastMod1, ET1) + ), + x + ), + O2 = + riak_object:increment_vclock( + riak_object:update_metadata( + riak_object:update_value( + O0, + <<"gnonto mk2">> + ), + get_metadata(LastMod2, ET2) + ), + y + ), + + OM1 = riak_object:reconcile([O1, O2], true), + Ctx1 = + #context{ + method = 'GET', + bucket = {<<"T">>, <<"B">>}, + key = <<"K">>, + vtag = ET1 + }, + {200, HdrList1, <<"gnonto">>} = produce_response(OM1, Ctx1), + ?assertMatch({'Etag', ET1}, lists:keyfind('Etag', 1, HdrList1)), + {'Last-Modified', FormattedDate1} = + lists:keyfind('Last-Modified', 1, HdrList1), + ?assertMatch( + FormattedDate1, + iolist_to_binary( + httpd_util:rfc1123_date( + calendar:now_to_local_time(LastMod1) + ) + ) + ), + ?assertMatch( + {200, HdrList1, <<>>}, + produce_response(OM1, Ctx1#context{method = 'HEAD'}) + ), + Ctx2 = + Ctx1#context{ + preferred_types = [], + all_types_accepted = true, + vtag = undefined + }, + {300, HdrList2, SibBody2} = produce_response(OM1, Ctx2), + ?assert( + lists:keyfind(?HEAD_VCLOCK, 1, HdrList2) == + lists:keyfind(?HEAD_VCLOCK, 1, HdrList1) + ), + ?assertNotMatch( + nomatch, + string:find(SibBody2, ET1) + ), + ?assertNotMatch( + nomatch, + string:find(SibBody2, ET2) + ), + %% Last Modfied Date should be later of two dates (from sibling 2) + {'Last-Modified', FormattedDate2} = + lists:keyfind('Last-Modified', 1, HdrList2), + ?assertMatch( + FormattedDate2, + iolist_to_binary( + httpd_util:rfc1123_date( + calendar:now_to_local_time(LastMod2) + ) + ) + ), + + %% Return multipart body + Ctx3 = + Ctx1#context{ + preferred_types = [<<"multipart/mixed">>], + vtag = undefined + }, + {200, HdrList3, MultiBody3} = produce_response(OM1, Ctx3), + {'Last-Modified', FormattedDate3} = + lists:keyfind('Last-Modified', 1, HdrList3), + ?assertMatch( + FormattedDate3, + iolist_to_binary( + httpd_util:rfc1123_date( + calendar:now_to_local_time(LastMod2) + ) + ) + ), + ?assert( + lists:keyfind(?HEAD_VCLOCK, 1, HdrList3) == + lists:keyfind(?HEAD_VCLOCK, 1, HdrList1) + ), + {'Content-Type', MultipartHeader} = + lists:keyfind('Content-Type', 1, HdrList3), + << + "multipart/mixed; boundary=", + Boundary/binary + >> = MultipartHeader, + [<<"\r\n--">>, SS1] = string:split(MultiBody3, Boundary), + [B1, SS2] = string:split(SS1, Boundary), + [B2, <<"--\r\n">>] = string:split(SS2, Boundary), + ?assertNotMatch( + nomatch, + string:find( + B1, + <<<<"Etag: ">>/binary, ET1/binary>> + ) + ), + ?assertMatch( + nomatch, + string:find( + B1, + <<<<"Etag: ">>/binary, ET2/binary>> + ) + ), + ?assertNotMatch( + nomatch, + string:find( + B2, + <<<<"Etag: ">>/binary, ET2/binary>> + ) + ), + ?assertMatch( + nomatch, + string:find( + B2, + <<<<"Etag: ">>/binary, ET1/binary>> + ) + ), + ?assertNotMatch( + nomatch, + string:find( + B1, + <<"Content-Type: application/octet-stream">> + ) + ), + ?assertNotMatch( + nomatch, + string:find( + B2, + <<"Content-Type: application/octet-stream">> + ) + ), + ?assertNotMatch( + nomatch, + string:find( + B1, + <<"\r\ngnonto\r\n">> + ) + ), + ?assertNotMatch( + nomatch, + string:find( + B2, + <<"\r\ngnonto mk2\r\n">> + ) + ), + + % Non-unique vtags + O3 = + riak_object:increment_vclock( + riak_object:update_metadata( + riak_object:update_value( + O0, + <<"gnonto mk3">> + ), + get_metadata(LastMod1, ET1) + ), + z + ), + OM2 = riak_object:reconcile([O1, O2, O3], true), + {400, _, <<"VTag a123456zz failed to match an individual sibling">>} = + produce_response(OM2, Ctx1). + +get_metadata(LastMod, ETag) -> + riak_object:metadata_fromlist( + [ + {?MD_CTYPE, <<"application/octet-stream">>}, + { + ?MD_INDEX, + [ + {<<"familyname_bin">>, <<"gnonto">>}, + {<<"dob_bin">>, <<"20031105">>} + ] + }, + {?MD_VTAG, ETag}, + {?MD_LASTMOD, LastMod} + ] + ). + +hidden_all_accepted_test() -> + ReqHeaders1 = + riak_api_web_headers:make( + [{'Accept', [<<"multipart/mixed">>, <<"*/*;q=0.9">>]}] + ), + InitCtx = + #context{method = 'GET', bucket = {<<"T">>, <<"B">>}, key = <<"K">>}, + {ok, Ctx1} = parse_request_headers(ReqHeaders1, InitCtx), + CType = <<"application/octet-stream">>, + ?assert(riak_kv_web_common:type_match(CType, Ctx1#context.preferred_types)), + + ReqHeaders2 = + riak_api_web_headers:make( + [{'Accept', [<<"multipart/mixed">>, <<"application/*;q=0.9">>]}] + ), + {ok, Ctx2} = parse_request_headers(ReqHeaders2, InitCtx), + ?assert(riak_kv_web_common:type_match(CType, Ctx2#context.preferred_types)), + + ReqHeaders3 = + riak_api_web_headers:make( + [{'Accept', <<"*/*;q=0.9">>}] + ), + {ok, Ctx3} = parse_request_headers(ReqHeaders3, InitCtx), + ?assert(riak_kv_web_common:type_match(CType, Ctx3#context.preferred_types)). + +-endif. diff --git a/src/riak_kv_ag_object_store.erl b/src/riak_kv_ag_object_store.erl new file mode 100644 index 000000000..ac266b458 --- /dev/null +++ b/src/riak_kv_ag_object_store.erl @@ -0,0 +1,1065 @@ +%% -------------------------------------------------------- +%% +%% Copyright (c) 2026 Martin Sumner +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +%% @doc Handler for HTTP API requests to store an object ('PUT' or 'POST' +%% requests) + +-module(riak_kv_ag_object_store). + +-if(?OTP_RELEASE == 26). +-feature(maybe_expr, enable). +-endif. + +-include("riak_object.hrl"). +-include("riak_kv_web.hrl"). +-include_lib("kernel/include/logger.hrl"). + +-behaviour(riak_api_web_handler). + +-export( + [ + match_route/3, + check_permissions/5, + parse_query_params/2, + parse_request_headers/2, + process_request/2, + record_request/3 + ] +). + +-define(PUT_DEFAULTS, #{ + w => default, + dw => default, + pw => default, + node_confirms => default, + sync_on_write => default, + n_val => default, + asis => default, + returnbody => default, + timeout => undefined +}). + +-type put_option_key() :: + w + | dw + | pw + | node_confirms + | sync_on_write + | n_val + | asis + | returnbody + | timeout. + +-type put_options() :: + #{ + w => pos_integer() | default | quorum | all, + dw => non_neg_integer() | default | quorum | all, + pw => non_neg_integer() | default | quorum | all, + node_confirms => non_neg_integer() | default | quorum | all, + sync_on_write => default | backend | one | all, + n_val => pos_integer() | default, + asis => boolean() | default, + returnbody => boolean() | default, + timeout => pos_integer() | undefined + }. + +-record(context, { + client = riak_client:new(node(), undefined) :: riak_client:riak_client(), + method :: 'PUT' | 'POST', + bucket :: riak_object:bucket(), + key :: riak_object:key(), + put_options = ?PUT_DEFAULTS :: put_options(), + object :: riak_object:riak_object() | undefined, + if_not_modified :: true | undefined, + if_not_modified_clock :: vclock:vclock() | undefined, + if_none_match :: true | undefined, + if_match :: list(binary()) | undefined +}). + +-type context() :: #context{}. + +%% =================================================================== +%% Callback functions +%% =================================================================== + +-spec match_route( + riak_api_web_acceptor:method(), + unicode:chardata(), + list(unicode:chardata()) +) -> + nomatch + | {method_not_allowed, list(riak_api_web_acceptor:method())} + | {ok, riak_api_web_handler:limits(), context()}. +match_route( + Method, + _Path, + [<<"types">>, BucketType, <<"buckets">>, Bucket, <<"keys">>, Key] +) when is_binary(Key) -> + case Method of + Method when Method == 'PUT'; Method == 'POST' -> + Context = + #context{ + method = Method, + bucket = riak_kv_web_common:set_bucket(BucketType, Bucket), + key = Key + }, + {ok, size_limits(), Context}; + _OtherMethod -> + {method_not_allowed, ['PUT', 'POST']} + end; +match_route( + Method, + Path, + [<<"buckets">>, Bucket, <<"keys">>, Key] +) when is_binary(Key) -> + match_route( + Method, + Path, + [<<"types">>, <<"default">>, <<"buckets">>, Bucket, <<"keys">>, Key] + ); +match_route( + 'POST', + Path, + [<<"types">>, BucketType, <<"buckets">>, Bucket, <<"keys">>] +) -> + % Note - overlap with riak_kv_ag_keylist + % Hence no method_not_allowed + K = iolist_to_binary(riak_core_util:unique_id_62()), + match_route( + 'POST', + Path, + [<<"types">>, BucketType, <<"buckets">>, Bucket, <<"keys">>, K] + ); +match_route( + 'POST', + Path, + [<<"buckets">>, Bucket, <<"keys">>] +) -> + match_route( + 'POST', + Path, + [<<"types">>, <<"default">>, <<"buckets">>, Bucket, <<"keys">>] + ); +match_route(_Method, _Path, _SplitPath) -> + nomatch. + +%% @doc check_permissions for using this module or route +-spec check_permissions( + riak_api_web_headers:headers(), + riak_api_web_socket:scheme(), + riak_api_web_handler:peer_ip(), + riak_api_web_handler:peer_cert(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +check_permissions(ReqHeaders, Scheme, Peer, _Cert, Ctx) -> + Check = + riak_kv_web_common:check_permissions( + ReqHeaders, + Scheme, + Peer, + Ctx#context.bucket, + "riak_kv.put" + ), + case Check of + true -> + % The PB API doesn't check type exists, however, the FSM will crash + % if it does not exist - so better to give a sensible error here. + % Note this requires the fetching (and discarding) of the type + % properties. + case riak_kv_web_common:check_type_exists(Ctx#context.bucket) of + ok -> + {ok, Ctx}; + HaltResponse -> + HaltResponse + end; + HaltResponse -> + HaltResponse + end. + +%% @doc parse and validate query params, passed as a map +-spec parse_query_params( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +parse_query_params([], Ctx) -> + % Typically we expect no options - so shortcut the validation in this case + {ok, Ctx}; +parse_query_params(Params, Ctx) -> + maybe + {ok, Ctx0} ?= validate_timeout(Params, Ctx), + {ok, Ctx1} ?= validate_counts(Params, Ctx0), + {ok, Ctx2} ?= validate_booleans(Params, Ctx1), + {ok, Ctx3} ?= validate_synconwrite(Params, Ctx2), + {ok, Ctx3} + else + HaltResponse -> + HaltResponse + end. + +%% @doc parse and validate the request headers +-spec parse_request_headers( + riak_api_web_headers:headers(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +parse_request_headers(ReqHeaders, Ctx) -> + maybe + {ok, Ctx0} ?= validate_conditional_request(ReqHeaders, Ctx), + Obj = riak_object:new(Ctx#context.bucket, Ctx#context.key, <<>>), + {ok, Obj1} ?= set_version_vector(ReqHeaders, Obj), + {ok, MD0} ?= set_index_specs(ReqHeaders, riak_object:metadata_new()), + {ok, MD1} ?= set_user_metadata(ReqHeaders, MD0), + {ok, MD2} ?= set_content_type_and_encoding(ReqHeaders, MD1), + {ok, Ctx0#context{object = riak_object:update_metadata(Obj1, MD2)}} + else + HaltResponse -> + HaltResponse + end. + +%% @doc Process the request and produce a response +-spec process_request( + riak_api_web_body:req_body(), + context() +) -> + { + ok, + { + riak_api_web_acceptor:response_code(), + riak_api_web_headers:header_list(), + riak_api_web_handler:response_body(), + boolean(), + riak_api_web_body:req_body() + }, + context() + } + | riak_api_web_acceptor:halt_response(). +process_request(RqBdy, Context) -> + case riak_api_web_body:get_body(RqBdy, all, 60000) of + {error, content_too_large} -> + {halt, 413, [], <<>>, []}; + {ObjBody, UpdRqBody} when is_binary(ObjBody) -> + PutRsp = + do_put( + riak_object:update_value(Context#context.object, ObjBody), + Context + ), + case PutRsp of + {error, Reason} -> + case handle_error(Reason, Context) of + {complete, RspCode, Ctx0} -> + {ok, {RspCode, [], <<>>, true, UpdRqBody}, Ctx0}; + HaltResponse -> + ?LOG_WARNING("Halt response: ~0p", [HaltResponse]), + HaltResponse + end; + ok -> + {ok, {204, [], <<>>, true, UpdRqBody}, Context}; + {ok, Obj} -> + {RspCode, RspHdrs, RspBody} = + riak_kv_ag_object_read:produce_response(Obj), + {ok, {RspCode, RspHdrs, RspBody, true, UpdRqBody}, Context} + end + end. + +%% @doc Record the output of the interaction +-spec record_request( + riak_api_web_handler:timings(), + riak_api_web_handler:completion(), + context() +) -> + ok. +record_request(_Timings, _Completion, _Ctx) -> + ok. + +%% =================================================================== +%% Validation functions +%% =================================================================== + +-spec validate_timeout( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +validate_timeout(Params, Ctx) -> + case riak_kv_web_common:get_timeout(Params) of + {ok, none} -> + {ok, Ctx}; + {ok, Timeout} -> + {ok, set_option(timeout, Timeout, Ctx)}; + HaltResponse -> + HaltResponse + end. + +-spec validate_counts( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +validate_counts(Params, Context) -> + FoldResult = + riak_kv_web_common:count_fold( + Params, + [<<"w">>, <<"dw">>, <<"pw">>, <<"n_val">>, <<"node_confirms">>], + Context#context.put_options + ), + case FoldResult of + UpdOpts when is_map(UpdOpts) -> + {ok, Context#context{put_options = UpdOpts}}; + HaltResponse -> + HaltResponse + end. + +-spec validate_booleans( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +validate_booleans(Params, Context) -> + FoldResult = + riak_kv_web_common:boolean_fold( + Params, + [<<"returnbody">>, <<"basic_quorum">>, <<"asis">>], + Context#context.put_options + ), + case FoldResult of + UpdOpts when is_map(UpdOpts) -> + {ok, Context#context{put_options = UpdOpts}}; + HaltResponse -> + HaltResponse + end. + +-spec validate_synconwrite( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +validate_synconwrite(QueryParams, Context) -> + case lists:keyfind(<<"sync_on_write">>, 1, QueryParams) of + false -> + {ok, Context}; + {<<"sync_on_write">>, Valid} when + Valid == <<"default">>; + Valid == <<"backend">>; + Valid == <<"one">>; + Valid == <<"all">> + -> + {ok, set_option(sync_on_write, binary_to_atom(Valid), Context)}; + _Invalid -> + ErrorText = + <<"~w query parameter must be one of the following words: ~0p">>, + { + halt, + 400, + [?TXT_HEADER], + ErrorText, + [sync_on_write, [default, backend, one, all]] + } + end. + +-spec validate_conditional_request( + riak_api_web_headers:headers(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +validate_conditional_request(ReqHeaders, Ctx) -> + Ctx0 = + case riak_api_web_headers:get_value('If-None-Match', ReqHeaders) of + undefined -> + Ctx; + _ -> + Ctx#context{if_none_match = true} + end, + Ctx1 = + case riak_api_web_headers:get_value('If-Match', ReqHeaders) of + undefined -> + Ctx0; + V when is_binary(V) -> + Ctx0#context{if_match = [V]}; + V when is_list(V) -> + Ctx0#context{if_match = V} + end, + IfNotModClock = + riak_api_web_headers:lookup(?HEAD_IFNOTMOD_CASEFOLD, ReqHeaders, true), + case IfNotModClock of + undefined -> + {ok, Ctx1}; + {_OrigKey, [EncodedClock]} -> + case riak_kv_web_common:decode_clock(EncodedClock) of + error -> + ErrorRsp = + << + "Error decoding vector clock in " + "x-riak-if-not-modified header" + >>, + {halt, 400, [?TXT_HEADER], ErrorRsp, []}; + DecodedClock -> + { + ok, + Ctx1#context{ + if_not_modified = true, + if_not_modified_clock = DecodedClock + } + } + end; + {_OrigKey, _MultipleClocks} -> + ErrorRsp = + << + "Only one value may be set " + "for x-riak-if-not-modified header" + >>, + {halt, 400, [?TXT_HEADER], ErrorRsp, []} + end. + +%% =================================================================== +%% Build object +%% =================================================================== + +-spec set_version_vector( + riak_api_web_headers:headers(), + riak_object:riak_object() +) -> + {ok, riak_object:riak_object()} | riak_api_web_acceptor:halt_response(). +set_version_vector(ReqHeaders, Obj) -> + case riak_kv_web_common:get_version_vector(ReqHeaders) of + {ok, none} -> + {ok, Obj}; + {ok, DecodedClock} -> + {ok, riak_object:set_vclock(Obj, DecodedClock)}; + HaltResponse -> + HaltResponse + end. + +-spec set_index_specs( + riak_api_web_headers:headers(), + riak_object:riak_object_meta() +) -> + {ok, riak_object:riak_object_meta()}. +set_index_specs(ReqHeaders, MD) -> + IndexHeaders = + riak_api_web_headers:prefix_fold( + ?HEAD_INDEX_CASEFOLD, + ReqHeaders, + true + ), + IndexSpecs = + lists:foldl( + fun({Fld, Terms}, IdxAcc) -> + lists:map(fun(T) -> {Fld, T} end, Terms) ++ IdxAcc + end, + [], + IndexHeaders + ), + {ok, riak_object:metadata_store(?MD_INDEX, IndexSpecs, MD)}. + +-spec set_user_metadata( + riak_api_web_headers:headers(), + riak_object:riak_object_meta() +) -> + {ok, riak_object:riak_object_meta()}. +set_user_metadata(ReqHeaders, MD) -> + MetaHeaders = + riak_api_web_headers:prefix_fold( + ?HEAD_USERMETA_CASEFOLD, + ReqHeaders, + true + ), + MetaSpecs = + lists:foldl( + fun({K, VL}, MetaAcc) -> + lists:map(fun(V) -> {K, V} end, VL) ++ MetaAcc + end, + [], + MetaHeaders + ), + {ok, riak_object:metadata_store(?MD_USERMETA, MetaSpecs, MD)}. + +-spec set_content_type_and_encoding( + riak_api_web_headers:headers(), + riak_object:riak_object_meta() +) -> + {ok, riak_object:riak_object_meta()} + | riak_api_web_acceptor:halt_response(). +set_content_type_and_encoding(ReqHeaders, MD) -> + ContentEncodingHeader = + riak_api_web_headers:get_value('Content-Encoding', ReqHeaders), + MD1 = + case ContentEncodingHeader of + undefined -> + MD; + EncodingList when is_list(EncodingList) -> + Encoding = + lists:flatten( + lists:join( + ", ", + lists:map(fun binary_to_list/1, EncodingList) + ) + ), + riak_object:metadata_store( + ?MD_ENCODING, + Encoding, + MD + ); + Encoding -> + riak_object:metadata_store( + ?MD_ENCODING, + binary_to_list(Encoding), + MD + ) + end, + ContentTypeHeader = + riak_api_web_headers:get_unique_value('Content-Type', ReqHeaders), + case ContentTypeHeader of + undefined -> + { + halt, + 415, + [?TXT_HEADER, {<<"Accept-Post">>, <<"*/*">>}], + <<"Missing Content-Type request header">>, + [] + }; + {error, multiple_values} -> + { + halt, + 415, + [?TXT_HEADER, {<<"Accept-Post">>, <<"*/*">>}], + <<"Only one Content-Type may be specified">>, + [] + }; + ContentType when is_binary(ContentType) -> + [CType | RawParams] = string:lexemes(ContentType, "; "), + case take_first_encoding(RawParams) of + undefined -> + {ok, riak_object:metadata_store(?MD_CTYPE, CType, MD1)}; + Charset -> + { + ok, + riak_object:metadata_store( + ?MD_CTYPE, + binary_to_list(CType), + % list for backwards compatibility + riak_object:metadata_store( + ?MD_CHARSET, + binary_to_list(Charset), + MD1 + ) + ) + } + end + end. + +take_first_encoding([]) -> + undefined; +take_first_encoding([Param | Rest]) -> + case binary:split(Param, <<"=">>, []) of + [<<"charset">>, Charset] -> + Charset; + _ -> + take_first_encoding(Rest) + end. + +-spec do_put( + riak_object:riak_object(), + context() +) -> + ok | {ok, riak_object:riak_object()} | {error, term()}. +do_put(Object, Ctx) -> + {CheckResult, CondPutOptions, SessionToken} = + riak_kv_put_core:ready_conditional_check( + Ctx#context.if_not_modified, + Ctx#context.if_none_match, + Ctx#context.if_match, + fun() -> Ctx#context.if_not_modified_clock end, + Ctx#context.bucket, + Ctx#context.key, + Ctx#context.client + ), + StandardPutOptions = + riak_kv_web_common:filter_options(Ctx#context.put_options), + ?LOG_DEBUG( + "Put with options ~0p", + [StandardPutOptions] + ), + ?LOG_DEBUG( + "Put with conditions ~0p ~0p ~0p", + [CheckResult, CondPutOptions, SessionToken] + ), + case CheckResult of + ok -> + PutRsp = + case SessionToken of + none -> + riak_client:put( + Object, + CondPutOptions ++ StandardPutOptions, + Ctx#context.client + ); + _ -> + riak_kv_token_session:session_use( + SessionToken, + put, + [ + Object, + CondPutOptions ++ StandardPutOptions + ] + ) + end, + riak_kv_token_session:session_release(SessionToken), + PutRsp; + {error, Reason} -> + riak_kv_token_session:session_release(SessionToken), + {error, Reason} + end. + +%% =================================================================== +%% Internal Functions +%% =================================================================== + +-spec handle_error( + term(), + context() +) -> + {complete, riak_api_web_acceptor:response_code(), context()} + | riak_api_web_acceptor:halt_response(). +handle_error(precommit_fail, Ctx) -> + Msg = + iolist_to_binary( + io_lib:format( + <<"~w aborted by pre-commit hook.">>, + [Ctx#context.method] + ) + ), + handle_error({precommit_fail, Msg}, Ctx); +handle_error({precommit_fail, Msg}, _Ctx) -> + % There are specific error codes tested by riak_test verify_commit_hooks + case is_binary(Msg) of + true -> + {halt, 403, [?TXT_HEADER], Msg, []}; + false when is_list(Msg) -> + {halt, 403, [?TXT_HEADER], iolist_to_binary(Msg), []}; + false -> + BinMsg = iolist_to_binary(io_lib:format("~0p", [Msg])), + {halt, 500, [?TXT_HEADER], BinMsg, []} + end; +handle_error(too_many_fails, _Ctx) -> + Msg = <<"Too Many write failures to satisfy W/DW">>, + {halt, 503, [?TXT_HEADER], Msg, []}; +handle_error(timeout, _Ctx) -> + {halt, 503, [?TXT_HEADER], <<"request timed out">>, []}; +handle_error({n_val_violation, N}, _Ctx) -> + Msg = + << + "Specified w/dw/pw/node_confirms" + " values invalid for bucket n value of ~p" + >>, + {halt, 400, [?TXT_HEADER], Msg, [N]}; +handle_error({dw_val_unsatisfied, DW, NumDW}, _Ctx) -> + {halt, 503, [?TXT_HEADER], <<"DW-value unsatisfied: ~p/~p">>, [NumDW, DW]}; +handle_error({pw_val_unsatisfied, PW, NumPW}, _Ctx) -> + {halt, 503, [?TXT_HEADER], <<"PW-value unsatisfied: ~p/~p">>, [NumPW, PW]}; +handle_error({node_confirms_val_unsatisfied, NC, NumNC}, _Ctx) -> + Msg = <<"node_confirms-value unsatisfied: ~p/~p">>, + {halt, 503, [?TXT_HEADER], Msg, [NumNC, NC]}; +handle_error(failed, Ctx) -> + {complete, 412, Ctx}; +handle_error("match_found", Ctx) -> + {complete, 412, Ctx}; +handle_error("modified", Ctx) -> + {complete, 409, Ctx}; +handle_error("notfound", Ctx) -> + {complete, 409, Ctx}; +handle_error(not_matched, Ctx) -> + {complete, 412, Ctx}; +handle_error(OtherError, _Ctx) -> + {halt, 500, [?TXT_HEADER], <<"Error:~n~p">>, [OtherError]}. + +-spec set_option( + put_option_key(), + non_neg_integer() | quorum | all | backend | one | boolean(), + context() +) -> + context(). +set_option(Option, Value, Context) -> + Context#context{ + put_options = maps:put(Option, Value, Context#context.put_options) + }. + +size_limits() -> + { + application:get_env(riak_kv, max_header_count, 1024), + application:get_env(riak_kv, max_header_size, 16384), + application:get_env(riak_kv, max_object_size, 52428800) + }. + +%% =================================================================== +%% EUnit tests +%% =================================================================== + +-ifdef(TEST). + +-include_lib("eunit/include/eunit.hrl"). + +request_headers_test() -> + Vc = + base64:encode( + riak_object:encode_vclock( + vclock:increment('node1@127.0.0.1', vclock:fresh()) + ) + ), + TestHeaders = + [ + {<<"X-riak-Index-date_bin">>, <<"date1, date2">>}, + {<<"x-riak-index-date_bin">>, <<"date3">>}, + {<<"x-riak-index-name_bin">>, <<"name1">>}, + {<<"X-Riak-Meta-postcode">>, <<"postcode1">>}, + {<<"X-Riak-Meta-postcode">>, <<"postcode2">>}, + {<<"X-Riak-ClientId">>, <<"LocalID0001">>}, + {'If-None-Match', <<"*">>}, + {'Content-Type', <<"application/json; charset=utf8">>}, + {'Content-Encoding', <<"gzip, deflate">>}, + {<<"X-Riak-vclock">>, Vc} + ], + InitCtx = + #context{ + method = 'PUT', + bucket = {<<"T">>, <<"B">>}, + key = <<"K">> + }, + TestHeaderObj = riak_api_web_headers:make(TestHeaders), + {ok, CtxOut} = parse_request_headers(TestHeaderObj, InitCtx), + MD = + riak_object:get_metadata( + riak_object:apply_updates(CtxOut#context.object) + ), + ?assertMatch( + [ + {<<"date_bin">>, <<"date1">>}, + {<<"date_bin">>, <<"date2">>}, + {<<"date_bin">>, <<"date3">>}, + {<<"name_bin">>, <<"name1">>} + ], + lists:sort(riak_object:metadata_fetch(?MD_INDEX, MD)) + ), + ?assertMatch( + [ + {<<"postcode">>, <<"postcode1">>}, + {<<"postcode">>, <<"postcode2">>} + ], + lists:sort(riak_object:metadata_fetch(?MD_USERMETA, MD)) + ), + ?assertMatch( + "application/json", + riak_object:metadata_fetch(?MD_CTYPE, MD) + ), + ?assertMatch( + "utf8", + riak_object:metadata_fetch(?MD_CHARSET, MD) + ), + ?assertMatch( + "gzip, deflate", + riak_object:metadata_fetch(?MD_ENCODING, MD) + ), + ?assert(CtxOut#context.if_none_match), + ?assertMatch( + [_, undefined], + % The vnode_vclock capability present since 1.0 + % So ignore any passed in client ID + % The code no longer sets the ID in the client, and then removes it + % after a capability check + element(2, CtxOut#context.client) + ). + +headers_clock_error1_test() -> + Vc = + base64:encode( + riak_object:encode_vclock( + vclock:increment('node1@127.0.0.1', vclock:fresh()) + ), + #{mode => urlsafe} + ), + TestHeaders = + [ + {<<"X-riak-Index-date_bin">>, <<"date1, date2">>}, + {'Content-Type', <<"application/json; charset=utf8">>}, + {'Content-Encoding', <<"gzip, deflate">>}, + {<<"X-Riak-vclock">>, Vc} + ], + InitCtx = + #context{ + method = 'PUT', + bucket = {<<"T">>, <<"B">>}, + key = <<"K">> + }, + TestHeaderObj = riak_api_web_headers:make(TestHeaders), + {halt, 400, [?TXT_HEADER], Msg, _Subs} = + parse_request_headers(TestHeaderObj, InitCtx), + ?assertMatch( + <<"Error decoding vector clock in x-riak-vclock header">>, + Msg + ). + +headers_clock_error2_test() -> + VcB = + base64:encode( + riak_object:encode_vclock( + vclock:increment('node1@127.0.0.1', vclock:fresh()) + ), + #{mode => urlsafe} + ), + VcG = + base64:encode( + riak_object:encode_vclock( + vclock:increment('node1@127.0.0.1', vclock:fresh()) + ) + ), + TestHeaders = + [ + {<<"X-riak-if-not-modified">>, VcB}, + {'Content-Type', <<"application/json; charset=utf8">>}, + {'Content-Encoding', <<"gzip, deflate">>}, + {<<"X-Riak-vclock">>, VcG} + ], + InitCtx = + #context{ + method = 'PUT', + bucket = {<<"T">>, <<"B">>}, + key = <<"K">> + }, + TestHeaderObj = riak_api_web_headers:make(TestHeaders), + {halt, 400, [?TXT_HEADER], Msg, _Subs} = + parse_request_headers(TestHeaderObj, InitCtx), + ?assertMatch( + <<"Error decoding vector clock in x-riak-if-not-modified header">>, + Msg + ). + +headers_single_encoding_test() -> + VcE = vclock:increment('node1@127.0.0.1', vclock:fresh()), + Vc = base64:encode(riak_object:encode_vclock(VcE)), + TestHeaders = + [ + {'Content-Type', <<"application/json; charset=utf8">>}, + {'Content-Encoding', <<"gzip">>}, + {<<"X-riak-if-not-modified">>, Vc}, + {<<"X-Riak-vclock">>, Vc} + ], + InitCtx = + #context{ + method = 'PUT', + bucket = {<<"T">>, <<"B">>}, + key = <<"K">> + }, + TestHeaderObj = riak_api_web_headers:make(TestHeaders), + {ok, CtxOut} = parse_request_headers(TestHeaderObj, InitCtx), + MD = + riak_object:get_metadata( + riak_object:apply_updates(CtxOut#context.object) + ), + ?assertMatch("gzip", riak_object:metadata_fetch(?MD_ENCODING, MD)), + ?assertMatch(VcE, riak_object:vclock(CtxOut#context.object)). + +headers_multiple_value_error1_test() -> + Vc0 = vclock:increment('node1@127.0.0.1', vclock:fresh()), + Vc1 = vclock:increment('node1@127.0.0.1', Vc0), + TestHeaders = + [ + {'Content-Type', <<"application/json; charset=utf8">>}, + {'Content-Encoding', <<"gzip">>}, + { + <<"X-riak-if-not-modified">>, + base64:encode(riak_object:encode_vclock(Vc0)) + }, + { + <<"X-riak-if-not-modified">>, + base64:encode(riak_object:encode_vclock(Vc1)) + }, + { + <<"X-Riak-vclock">>, + base64:encode(riak_object:encode_vclock(Vc0)) + } + ], + InitCtx = + #context{ + method = 'PUT', + bucket = {<<"T">>, <<"B">>}, + key = <<"K">> + }, + TestHeaderObj = riak_api_web_headers:make(TestHeaders), + {halt, 400, [?TXT_HEADER], Msg, []} = + parse_request_headers(TestHeaderObj, InitCtx), + ?assertMatch( + << + "Only one value may be set " + "for x-riak-if-not-modified header" + >>, + Msg + ), + MultiClockHeaders = + [ + {'Content-Type', <<"application/json">>}, + { + <<"X-Riak-vclock">>, + base64:encode(riak_object:encode_vclock(Vc0)) + }, + { + <<"X-Riak-vclock">>, + base64:encode(riak_object:encode_vclock(Vc1)) + } + ], + TestHeaderObj2 = riak_api_web_headers:make(MultiClockHeaders), + {halt, 400, [?TXT_HEADER], Msg2, []} = + parse_request_headers(TestHeaderObj2, InitCtx), + ?assertMatch( + << + "Only one x-riak-vclock may be specified" + >>, + Msg2 + ). + +headers_multiple_value_error2_test() -> + Vc = vclock:increment('node1@127.0.0.1', vclock:fresh()), + TestHeaders = + [ + { + 'Content-Type', + <<"application/json; charset=utf8, application/octet-stream">> + }, + {'Content-Encoding', <<"gzip">>}, + { + <<"X-Riak-vclock">>, + base64:encode(riak_object:encode_vclock(Vc)) + } + ], + InitCtx = + #context{ + method = 'PUT', + bucket = {<<"T">>, <<"B">>}, + key = <<"K">> + }, + TestHeaderObj = riak_api_web_headers:make(TestHeaders), + {halt, 415, Hdrs, Msg, []} = + parse_request_headers(TestHeaderObj, InitCtx), + ?assertMatch( + <<"Only one Content-Type may be specified">>, + Msg + ), + ?assertMatch( + [{'Content-Type', <<"text/plain">>}, {<<"Accept-Post">>, <<"*/*">>}], + lists:sort(Hdrs) + ), + MissingHeader = + [ + {'Content-Encoding', <<"gzip">>}, + { + <<"X-Riak-vclock">>, + base64:encode(riak_object:encode_vclock(Vc)) + } + ], + {halt, 415, Hdrs, MissingMsg, []} = + parse_request_headers( + riak_api_web_headers:make(MissingHeader), + InitCtx + ), + ?assertMatch(<<"Missing Content-Type request header">>, MissingMsg). + +query_params_positive_test() -> + Ctx = + #context{ + method = 'POST', + bucket = {<<"T">>, <<"B">>}, + key = <<"K">> + }, + URI1 = + << + "types/T/buckets/B/keys/K?timeout=10" + "&asis&returnbody=true&basic_quorum=true" + "&dw=0&pw=1" + "&sync_on_write=one" + >>, + QP1 = extract_params(URI1), + {ok, CtxUpd} = parse_query_params(QP1, Ctx), + PutOpts = CtxUpd#context.put_options, + ?assertMatch(0, maps:get(dw, PutOpts)), + ?assertMatch(1, maps:get(pw, PutOpts)), + ?assertMatch(one, maps:get(sync_on_write, PutOpts)), + ?assertMatch(true, maps:get(returnbody, PutOpts)), + ?assertMatch(true, maps:get(basic_quorum, PutOpts)), + ?assertMatch(10, maps:get(timeout, PutOpts)). + +extract_params(URI) -> + uri_string:dissect_query( + maps:get( + query, + uri_string:normalize(URI, [return_map]) + ) + ). + +query_params_error_test() -> + InitCtx = + #context{ + method = 'POST', + bucket = {<<"T">>, <<"B">>}, + key = <<"K">> + }, + URI1 = + << + "types/T/buckets/B/keys/K?timeout=A" + "&asis&returnbody=true&basic_quorum=true" + "&dw=0&pw=1" + "&sync_on_write=one" + >>, + ?assertMatch( + {halt, 400, _, _, _}, + parse_query_params(extract_params(URI1), InitCtx) + ), + URI2 = + << + "types/T/buckets/B/keys/K?timeout=10" + "&asis&returnbody=true&basic_quorum=1" + "&dw=0&pw=1" + >>, + ?assertMatch( + {halt, 400, _, _, _}, + parse_query_params(extract_params(URI2), InitCtx) + ), + URI3 = + << + "types/T/buckets/B/keys/K?timeout=10" + "&asis&returnbody=true&basic_quorum=true" + "&dw=0&pw=true" + "&sync_on_write=one" + >>, + ?assertMatch( + {halt, 400, _, _, _}, + parse_query_params(extract_params(URI3), InitCtx) + ), + URI4 = + << + "types/T/buckets/B/keys/K?" + "&asis&returnbody=true&basic_quorum=true" + "&dw=0&pw=1" + "&sync_on_write=false" + >>, + ?assertMatch( + {halt, 400, _, _, _}, + parse_query_params(extract_params(URI4), InitCtx) + ). + +-endif. diff --git a/src/riak_kv_ag_ping.erl b/src/riak_kv_ag_ping.erl new file mode 100644 index 000000000..b6743075c --- /dev/null +++ b/src/riak_kv_ag_ping.erl @@ -0,0 +1,149 @@ +%% -------------------------------------------------------- +%% +%% Copyright (c) 2026 Martin Sumner +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +%% @doc Handler for HTTP ping API + +-module(riak_kv_ag_ping). + +-include("riak_kv_web.hrl"). + +-behaviour(riak_api_web_handler). + +-export( + [ + match_route/3, + check_permissions/5, + parse_query_params/2, + parse_request_headers/2, + process_request/2, + record_request/3 + ] +). + +-record(context, { + ping = true :: true +}). + +-type context() :: #context{}. + +%% =================================================================== +%% Callback functions +%% =================================================================== + +-spec match_route( + riak_api_web_acceptor:method(), + unicode:chardata(), + list(unicode:chardata()) +) -> + nomatch + | {method_not_allowed, list(riak_api_web_acceptor:method())} + | {ok, riak_api_web_handler:limits(), context()}. +match_route('GET', _, [<<"ping">>]) -> + {ok, {32, 2048, 0}, #context{}}; +match_route(_, _, [<<"ping">>]) -> + {method_not_allowed, ['GET']}; +match_route(_, _, _) -> + nomatch. + +%% @doc check_permissions for using this module or route +-spec check_permissions( + riak_api_web_headers:headers(), + riak_api_web_socket:scheme(), + riak_api_web_handler:peer_ip(), + riak_api_web_handler:peer_cert(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +check_permissions(ReqHeaders, Scheme, Peer, _Cert, Ctx) -> + Check = + riak_kv_web_common:check_permissions( + ReqHeaders, + Scheme, + Peer, + undefined, + undefined + ), + case Check of + true -> + {ok, Ctx}; + HaltResponse -> + HaltResponse + end. + +%% @doc parse and validate query params, passed as a map +-spec parse_query_params( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +parse_query_params(_Params, Ctx) -> + {ok, Ctx}. + +%% @doc parse and validate the request headers +-spec parse_request_headers( + riak_api_web_headers:headers(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +parse_request_headers(_ReqHeaders, Ctx) -> + {ok, Ctx}. + +%% @doc Process the request and produce a response +-spec process_request( + none, + context() +) -> + { + ok, + { + riak_api_web_acceptor:response_code(), + riak_api_web_headers:header_list(), + riak_api_web_handler:response_body(), + boolean(), + none + }, + context() + } + | riak_api_web_acceptor:halt_response(). +process_request(none, Ctx) -> + {ok, {200, [?TXT_HEADER], <<"OK">>, true, none}, Ctx}. + +%% @doc Record the output of the interaction +-spec record_request( + riak_api_web_handler:timings(), + riak_api_web_handler:completion(), + context() +) -> + ok. +record_request(_Timings, _Completion, _Ctx) -> + ok. + +%% =================================================================== +%% Internal Functions +%% ====== + +%% =================================================================== +%% EUnit tests +%% =================================================================== + +-ifdef(TEST). + +-include_lib("eunit/include/eunit.hrl"). + +-endif. diff --git a/src/riak_kv_ag_query.erl b/src/riak_kv_ag_query.erl new file mode 100644 index 000000000..3ed434564 --- /dev/null +++ b/src/riak_kv_ag_query.erl @@ -0,0 +1,1412 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2026 Martin Sumner. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- + +-module(riak_kv_ag_query). + +-if(?OTP_RELEASE == 26). +-feature(maybe_expr, enable). +-endif. + +-include_lib("riak_kv/include/riak_kv_web.hrl"). + +-export( + [ + match_route/3, + check_permissions/5, + parse_query_params/2, + parse_request_headers/2, + process_request/2, + record_request/3 + ] +). + +-export([get_result_key/1, encode_key/2, encode_key_withterm/2]). + +-record(context, { + client = riak_client:new(node(), undefined) :: riak_client:riak_client(), + bucket :: riak_object:bucket(), + request_type :: submit_query | fetch_results, + queue_request :: undefined | #{atom() => term()} +}). + +-type context() :: #context{}. + +-define(ACCKEY_KEYS, <<"keys">>). +-define(ACCKEY_TERMS, <<"terms">>). +-define(ACCKEY_COUNT, <<"count">>). +-define(ACCKEY_TERMCOUNT, <<"term_with_count">>). +-define(ACCKEY_RAWKEYS, <<"raw_keys">>). +-define(ACCKEY_RAWTERMS, <<"raw_terms">>). +-define(ACCKEY_RAWCOUNT, <<"raw_count">>). +-define(ACCKEY_TERMRAWCOUNT, <<"term_with_rawcount">>). + +-define(AGGREGATION_EXPRESSION, <<"aggregation_expression">>). +-define(ACCUMULATION_OPTION, <<"accumulation_option">>). +-define(ACCUMULATION_TERM, <<"accumulation_term">>). +-define(SUBSTITUTIONS, <<"substitutions">>). +-define(TIMEOUT, <<"timeout">>). +-define(INACTIVITY_TIMEOUT, <<"inactivity_timeout">>). +-define(MAX_RESULTS, <<"max_results">>). +-define(CONTINUATION, <<"continuation">>). +-define(QUERY_LIST, <<"query_list">>). +-define(QL_AGGREGATION_TAG, <<"aggregation_tag">>). +-define(QL_INDEX_NAME, <<"index_name">>). +-define(QL_START_TERM, <<"start_term">>). +-define(QL_END_TERM, <<"end_term">>). +-define(QL_REGULAR_EXPRESSION, <<"regular_expression">>). +-define(QL_EVALUATION_EXPRESSION, <<"evaluation_expression">>). +-define(QL_FILTER_EXPRESSION, <<"filter_expression">>). + +-define(REQUIRED_KEYS, [?QUERY_LIST]). +-define(POSSIBLE_KEYS, [ + ?AGGREGATION_EXPRESSION, + ?ACCUMULATION_OPTION, + ?ACCUMULATION_TERM, + ?SUBSTITUTIONS, + ?TIMEOUT, + ?INACTIVITY_TIMEOUT, + ?QUERY_LIST, + ?MAX_RESULTS, + ?CONTINUATION +]). +-define(REQUIRED_QL_KEYS, [ + ?QL_INDEX_NAME, + ?QL_START_TERM, + ?QL_END_TERM +]). +-define(POSSIBLE_QL_KEYS, [ + ?QL_AGGREGATION_TAG, + ?QL_INDEX_NAME, + ?QL_START_TERM, + ?QL_END_TERM, + ?QL_REGULAR_EXPRESSION, + ?QL_EVALUATION_EXPRESSION, + ?QL_FILTER_EXPRESSION +]). + +-define(QUERY_TIMEOUT, 60). +-define(QUEUE_INACTIVITY_TIMEOUT, 120). +-define(MAX_RESULTS_FROM_QUEUE, 1000). + +-type query_map() :: + #{binary() => binary() | non_neg_integer() | list(map())}. + +-type stage() :: + key_check + | query_key_check + | init + | riak_kv_query:validation_stage(). + +%% =================================================================== +%% Callback functions +%% =================================================================== + +-spec match_route( + riak_api_web_acceptor:method(), + unicode:chardata(), + list(unicode:chardata()) +) -> + nomatch + | {method_not_allowed, list(riak_api_web_acceptor:method())} + | {ok, riak_api_web_handler:limits(), context()}. +match_route(Method, _, [<<"types">>, T, <<"buckets">>, B, <<"query">>]) -> + case Method of + 'GET' -> + Context = + #context{ + bucket = riak_kv_web_common:set_bucket(T, B), + request_type = fetch_results + }, + {ok, {32, 2048, 0}, Context}; + 'POST' -> + Context = + #context{ + bucket = riak_kv_web_common:set_bucket(T, B), + request_type = submit_query + }, + {ok, {32, 2048, 64 * 1024}, Context}; + _Other -> + {method_not_allowed, ['GET', 'POST']} + end; +match_route(Method, Path, [<<"buckets">>, B, <<"query">>]) -> + match_route( + Method, + Path, + [<<"types">>, <<"default">>, <<"buckets">>, B, <<"query">>] + ); +match_route(_Method, _Path, _SplitPath) -> + nomatch. + +%% @doc check_permissions for using this module or route +-spec check_permissions( + riak_api_web_headers:headers(), + riak_api_web_socket:scheme(), + riak_api_web_handler:peer_ip(), + riak_api_web_handler:peer_cert(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +check_permissions(ReqHeaders, Scheme, Peer, _Cert, Ctx) -> + Check = + riak_kv_web_common:check_permissions( + ReqHeaders, + Scheme, + Peer, + Ctx#context.bucket, + "riak_kv.index" + ), + case Check of + true -> + % The PB API doesn't check type exists, however, the FSM will crash + % if it does not exist - so better to give a sensible error here. + % Note this requires the fetching (and discarding) of the type + % properties. + case riak_kv_web_common:check_type_exists(Ctx#context.bucket) of + ok -> + {ok, Ctx}; + HaltResponse -> + HaltResponse + end; + HaltResponse -> + HaltResponse + end. + +%% @doc parse and validate query params, passed as a map +-spec parse_query_params( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +parse_query_params(_Params, #context{request_type = submit_query} = Ctx) -> + {ok, Ctx}; +parse_query_params(Params, #context{request_type = fetch_results} = Ctx) -> + case lists:keyfind(<<"result_queue">>, 1, Params) of + {<<"result_queue">>, Queue} when is_binary(Queue) -> + MaxResults = + case lists:keyfind(<<"max_results">>, 1, Params) of + {<<"max_results">>, MR} -> + MR; + _ -> + application:get_env( + riak_kv, + queue_raw_max_results, + ?MAX_RESULTS_FROM_QUEUE + ) + end, + try + MRI = + case is_integer(MaxResults) of + true -> + MaxResults; + false -> + binary_to_integer(MaxResults) + end, + true = is_integer(MRI), + true = MRI >= 0, + { + ok, + Ctx#context{ + queue_request = + make_queue_request( + Ctx#context.bucket, + Queue, + MRI + ) + } + } + catch + _:_ -> + json_validation_error( + <<"Invalid max_results query parameter">> + ) + end; + _ -> + json_validation_error( + <<"No valid result_queue reference passed as query parameter">> + ) + end. + +%% @doc parse and validate the request headers +-spec parse_request_headers( + riak_api_web_headers:headers(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +parse_request_headers(ReqHeaders, Ctx) -> + case riak_kv_web_common:accept_json_only(ReqHeaders) of + ok -> + {ok, Ctx}; + HaltResponse -> + HaltResponse + end. + +%% @doc Process the request and produce a response +-spec process_request( + riak_api_web_body:req_body() | none, + context() +) -> + { + ok, + { + riak_api_web_acceptor:response_code(), + riak_api_web_headers:header_list(), + riak_api_web_handler:response_body(), + boolean(), + riak_api_web_body:req_body() | none + }, + context() + } + | riak_api_web_acceptor:halt_response(). +process_request(none, #context{request_type = fetch_results} = Ctx) -> + QRR = + riak_client:query_result_request( + Ctx#context.queue_request, + Ctx#context.client + ), + case QRR of + {ok, ResultMap} -> + ResultBin = encode_queued_results(ResultMap), + { + ok, + { + 200, + [?JSN_HEADER], + ResultBin, + true, + none + }, + Ctx + }; + {error, result_server_terminated} -> + { + halt, + 410, + % Response code for Gone, and likely to be permanent. + % This may be as a result of an error on the server, but + % is probably as a result of an error on the client - and + % so to help with load-balancers tracking server errors, + % err on the side of blaming the client + [?JSN_HEADER], + iolist_to_binary( + riak_kv_wm_json:encode( + #{ + error => + << + "queue no longer present or" + " not currently reachable" + >> + } + ) + ), + [] + }; + {error, unexpected_reference_format} -> + json_validation_error( + <<"queue reference passed had an invalid format">> + ); + {error, Reason} -> + { + halt, + 500, + [?JSN_HEADER], + iolist_to_binary( + riak_kv_wm_json:encode( + #{error => <<"~0p">>} + ) + ), + [Reason] + } + end; +process_request(RqBdy, #context{request_type = submit_query} = Ctx) -> + case riak_api_web_body:get_body(RqBdy, all, 60000) of + {error, content_too_large} -> + {halt, 413, [], <<>>, []}; + {ObjBody, UpdRqBody} when is_binary(ObjBody) -> + case decode_json_body(ObjBody) of + {ok, QueryMap} -> + case validate_query(QueryMap, Ctx#context.bucket) of + {ok, Query} -> + case process_query(Query, Ctx) of + {ok, RspHdrs, RspBdy} -> + { + ok, + { + 200, + RspHdrs, + RspBdy, + true, + UpdRqBody + }, + Ctx + }; + HaltResponse -> + HaltResponse + end; + {error, Stage, Reason} -> + { + halt, + 400, + [?JSN_HEADER], + expand_query_reason(Stage, Reason), + [] + } + end; + {error, Reason} when is_binary(Reason) -> + { + halt, + 400, + [?JSN_HEADER], + expand_query_reason(init, Reason), + [] + } + end + end. + +%% @doc Record the output of the interaction +-spec record_request( + riak_api_web_handler:timings(), + riak_api_web_handler:completion(), + context() +) -> + ok. +record_request(_Timings, _Completion, _Ctx) -> + ok. + +%% =================================================================== +%% External Helper Functions +%% =================================================================== + +-spec get_result_key(riak_kv_query:accumulation_option()) -> binary(). +get_result_key(keys) -> ?ACCKEY_KEYS; +get_result_key(raw_keys) -> ?ACCKEY_RAWKEYS; +get_result_key(terms) -> ?ACCKEY_TERMS; +get_result_key(raw_terms) -> ?ACCKEY_RAWTERMS; +get_result_key(count) -> ?ACCKEY_COUNT; +get_result_key(raw_count) -> ?ACCKEY_RAWCOUNT; +get_result_key(term_with_count) -> ?ACCKEY_TERMCOUNT; +get_result_key(term_with_rawcount) -> ?ACCKEY_TERMRAWCOUNT. + +%% =================================================================== +%% Internal Functions +%% =================================================================== + +-spec validate_query( + query_map(), + riak_object:bucket() +) -> + {ok, riak_kv_query:complex_query_definition()} + | {error, stage(), binary()}. +validate_query(QueryMap, Bucket) -> + case check_keys(maps:keys(QueryMap), request) of + ok -> + QueryList = maps:get(?QUERY_LIST, QueryMap), + case check_querylist(QueryList, false) of + ok -> + make_query_request(Bucket, QueryMap); + {error, Reason} -> + {error, query_key_check, Reason} + end; + {error, Reason} -> + {error, key_check, Reason} + end. + +-spec check_querylist(list(binary()), boolean()) -> ok | {error, binary()}. +check_querylist([], true) -> + ok; +check_querylist([], false) -> + {error, <<"No valid query provided">>}; +check_querylist([HdQuery | Rest], _AtLeastOne) -> + case check_keys(maps:keys(HdQuery), query) of + ok -> + check_querylist(Rest, true); + Error -> + Error + end. + +-spec check_keys( + list(binary()), + request | query +) -> + ok | {error, binary()}. +check_keys(Keys, request) -> + check_keys(Keys, ?REQUIRED_KEYS, ?POSSIBLE_KEYS); +check_keys(Keys, query) -> + check_keys(Keys, ?REQUIRED_QL_KEYS, ?POSSIBLE_QL_KEYS). + +-spec check_keys( + list(binary()), + list(binary()), + list(binary()) +) -> + ok | {error, binary()}. +check_keys(Keys, RequiredKeys, PossibleKeys) -> + RequiredKeyList = + lists:filter( + fun(K) -> lists:member(K, Keys) end, + RequiredKeys + ), + PossibleKeyList = + lists:filter( + fun(K) -> lists:member(K, PossibleKeys) end, + Keys + ), + case RequiredKeyList of + RequiredKeys -> + case PossibleKeyList of + Keys -> + ok; + NotAllKeys -> + ExtraKeys = lists:subtract(Keys, NotAllKeys), + { + error, + iolist_to_binary( + io_lib:format( + <<"Unexpected keys in request ~0p">>, + [ExtraKeys] + ) + ) + } + end; + NotAllRequiredKeys -> + MissingKeys = lists:subtract(RequiredKeys, NotAllRequiredKeys), + { + error, + iolist_to_binary( + io_lib:format( + <<"Missing required keys in request ~0p">>, + [MissingKeys] + ) + ) + } + end. + +-spec make_query_request( + riak_object:bucket(), query_map() +) -> + {ok, riak_kv_query:complex_query_definition()} + | riak_kv_query:validation_error(). +make_query_request(BucketType, QueryMap) -> + maybe + {ok, Timeout, InactivityTimeout} ?= fetch_timeouts(QueryMap), + QueryType = + case maps:get(?QUERY_LIST, QueryMap) of + QueryList when length(QueryList) == 1 -> + single_query; + QueryList when length(QueryList) > 1 -> + combo_query + end, + InitQuery = + riak_kv_query:new( + BucketType, + QueryType, + Timeout, + InactivityTimeout + ), + {ok, Q1} ?= add_accumulation(QueryMap, InitQuery), + {ok, Q2} ?= add_queries(QueryMap, Q1, QueryList), + case maps:get(?CONTINUATION, QueryMap, none) of + none -> + {ok, Q2}; + Continuation -> + riak_kv_query:add_continuation(Q2, Continuation) + end + else + {error, Stage, Reason} -> + {error, Stage, Reason} + end. + +-spec decode_json_body(binary()) -> {ok, map()} | {error, binary()}. +decode_json_body(JsonBody) -> + try + DecodedBody = riak_kv_wm_json:decode(JsonBody), + {ok, DecodedBody} + catch + error:Reason -> + ExpandedReason = + iolist_to_binary( + io_lib:format( + <<"Malformed json request - ~0p">>, + [Reason] + ) + ), + {error, ExpandedReason} + end. + +json_validation_error(Text) when is_binary(Text) -> + { + halt, + 400, + [?JSN_HEADER], + iolist_to_binary(riak_kv_wm_json:encode(#{error => Text})), + [] + }. + +-spec make_queue_request( + riak_object:bucket(), binary(), non_neg_integer() +) -> + #{atom() => term()}. +make_queue_request(Bucket, EncodedQueueRef, MaxResults) -> + #{ + bucket => Bucket, + encoded_queue_reference => EncodedQueueRef, + max_results => MaxResults + }. + +-spec encode_queued_results( + riak_kv_query_server:partial_result_map() +) -> binary(). +encode_queued_results(ResultMap) -> + case maps:is_key(get_result_key(raw_keys), ResultMap) of + true -> + iolist_to_binary( + riak_kv_wm_json:encode( + ResultMap, + fun riak_kv_ag_query:encode_key/2 + ) + ); + false -> + case maps:is_key(get_result_key(raw_terms), ResultMap) of + true -> + iolist_to_binary( + riak_kv_wm_json:encode( + ResultMap, + fun riak_kv_ag_query:encode_key_withterm/2 + ) + ) + end + end. + +-spec add_accumulation( + query_map(), + riak_kv_query:complex_query_definition() +) -> + {ok, riak_kv_query:complex_query_definition()} + | riak_kv_query:validation_error(). +add_accumulation(QueryMap, InitQuery) -> + AccOpt = maps:get(?ACCUMULATION_OPTION, QueryMap, undefined), + AccTerm = maps:get(?ACCUMULATION_TERM, QueryMap, undefined), + MaxResults = maps:get(?MAX_RESULTS, QueryMap, undefined), + case riak_kv_query:add_accumulation_option(InitQuery, AccOpt) of + {ok, UpdQuery0} -> + case riak_kv_query:add_accumulation_term(UpdQuery0, AccTerm) of + {ok, UpdQuery1} -> + case MaxResults of + undefined -> + {ok, UpdQuery1}; + MR -> + riak_kv_query:add_maxresults(UpdQuery1, MR) + end; + Error -> + Error + end; + Error -> + Error + end. + +-spec add_queries( + query_map(), + riak_kv_query:complex_query_definition(), + list(#{binary() => binary()}) +) -> + {ok, riak_kv_query:complex_query_definition()} + | riak_kv_query:validation_error(). +add_queries(QueryMap, Query, QueryList) -> + AggExpr = + maps:get(?AGGREGATION_EXPRESSION, QueryMap, undefined), + case riak_kv_query:add_aggregation_expression(Query, AggExpr) of + {ok, Q2} -> + Subs = + maps:get(?SUBSTITUTIONS, QueryMap, maps:new()), + riak_kv_query:add_queries( + Q2, + lists:map(fun convert_query/1, QueryList), + Subs + ); + Error -> + Error + end. + +-spec expand_query_reason(stage(), binary()) -> binary(). +expand_query_reason(Stage, Reason) -> + ErrMsg = + iolist_to_binary( + io_lib:format( + <<"Validation failure at stage ~w due to ~s">>, + [Stage, Reason] + ) + ), + iolist_to_binary(riak_kv_wm_json:encode(#{error => ErrMsg})). + +-spec convert_query(map()) -> riak_kv_query:query_user_input(). +convert_query(QM) -> + { + maps:get(<<"aggregation_tag">>, QM, undefined), + maps:get(<<"index_name">>, QM), + maps:get(<<"start_term">>, QM), + maps:get(<<"end_term">>, QM), + maps:get(<<"regular_expression">>, QM, undefined), + maps:get(<<"evaluation_expression">>, QM, undefined), + maps:get(<<"filter_expression">>, QM, undefined) + }. + +-spec fetch_timeouts( + query_map() +) -> + {ok, pos_integer(), pos_integer()} | {error, init, binary()}. +fetch_timeouts(QueryMap) -> + Timeout = + maps:get( + ?TIMEOUT, + QueryMap, + application:get_env(riak_kv, query_timeout_secs, ?QUERY_TIMEOUT) + ), + InactivityTimeout = + maps:get( + ?INACTIVITY_TIMEOUT, + QueryMap, + application:get_env( + riak_kv, + queue_inactivity_timeout_secs, + ?QUEUE_INACTIVITY_TIMEOUT + ) + ), + case Timeout of + T when is_integer(T), T > 0 -> + case InactivityTimeout of + IT when is_integer(IT), IT > 0 -> + {ok, T, IT}; + _ -> + {error, init, <<"Bad inactivity timeout">>} + end; + _ -> + {error, init, <<"Bad timeout">>} + end. + +%% =================================================================== +%% Internal Functions +%% =================================================================== + +-spec process_query( + riak_kv_query:complex_query_definition(), + context() +) -> + {ok, riak_api_web_headers:header_list(), binary()} + | riak_api_web_acceptor:halt_response(). +process_query(InitQuery, Ctx) -> + Client = Ctx#context.client, + AccOpt = riak_kv_query:get_accumulator(InitQuery), + {ok, Query} = + riak_kv_query:add_result_encodingfun( + InitQuery, + encoding_function(AccOpt) + ), + case riak_client:query(Query, Client) of + {error, timeout} -> + {halt, 503, [?JSN_HEADER], <<"timeout">>, []}; + {error, Reason} -> + Error = <<"Query with option ~w failed - ~0p">>, + {halt, 500, [?JSN_HEADER], Error, [AccOpt, Reason]}; + {result_queue, ResultReference} when is_binary(ResultReference) -> + { + ok, + [?JSN_HEADER], + iolist_to_binary( + riak_kv_wm_json:encode( + #{result_queue => ResultReference} + ) + ) + }; + {JsonEncodedResults, none} when is_binary(JsonEncodedResults) -> + {ok, [?JSN_HEADER], JsonEncodedResults}; + {JsonEncodedResults, {{LT, LK}}} when + is_binary(JsonEncodedResults), + is_binary(LT), + is_binary(LK) + -> + Continuation = riak_kv_query:make_continuation(LT, LK), + { + ok, + [ + ?JSN_HEADER, + {?HEAD_CONTINUATION, Continuation} + ], + JsonEncodedResults + } + end. + +-spec encoding_function(riak_kv_query:accumulation_option()) -> + fun((riak_kv_query_server:results()) -> binary()). +encoding_function(AccOpt) -> + fun(Results) -> encode_results(AccOpt, Results) end. + +-spec encode_results( + riak_kv_query:accumulation_option(), riak_kv_query_server:results() +) -> binary(). +encode_results(AccOpt, Results) when AccOpt == keys; AccOpt == raw_keys -> + iolist_to_binary( + riak_kv_wm_json:encode( + #{get_result_key(AccOpt) => Results}, + fun riak_kv_ag_query:encode_key/2 + ) + ); +encode_results(AccOpt, Results) when AccOpt == terms; AccOpt == raw_terms -> + iolist_to_binary( + riak_kv_wm_json:encode( + #{get_result_key(AccOpt) => Results}, + fun riak_kv_ag_query:encode_key_withterm/2 + ) + ); +encode_results(AccOpt, Count) when AccOpt == count; AccOpt == raw_count -> + iolist_to_binary( + riak_kv_wm_json:encode(#{get_result_key(AccOpt) => Count}) + ); +encode_results(AccOpt, CountMap) when + AccOpt == term_with_count; AccOpt == term_with_rawcount +-> + iolist_to_binary( + riak_kv_wm_json:encode(#{get_result_key(AccOpt) => CountMap}) + ). + +encode_key({{_Term, Key}}, Encode) when is_binary(Key) -> + encode_key(Key, Encode); +encode_key({Key}, Encode) when is_binary(Key) -> + encode_key(Key, Encode); +encode_key(Key, Encode) -> + riak_kv_wm_json:encode_value(Key, Encode). + +encode_key_withterm({TermKeyTuple}, Encode) when is_tuple(TermKeyTuple) -> + encode_key_withterm(TermKeyTuple, Encode); +encode_key_withterm({Term, Key}, Encode) when is_binary(Term), is_binary(Key) -> + [123, [Encode(Term, Encode), $: | Encode(Key, Encode)], 125]; +encode_key_withterm(Result, Encode) -> + riak_kv_wm_json:encode_value(Result, Encode). + +%% =================================================================== +%% EUnit tests +%% =================================================================== + +-ifdef(TEST). + +-include_lib("eunit/include/eunit.hrl"). + +invalid_json_test() -> + InvalidJson = + % Missing comma after example_bin + << + "\n" + " {\n" + " \"accumulation_option\" : \"keys\",\n" + " \"timeout\" : 60,\n" + " \"query_list\" :\n" + " [\n" + " {\n" + " \"index_name\" : \"example_bin\"\n" + " \"start_term\" : \"A\",\n" + " \"end_term\" : \"B\"\n" + " }\n" + " ]\n" + " }\n" + " " + >>, + R = decode_json_body(InvalidJson), + io:format("~p~n", [R]), + ?assertMatch( + {error, <<"Malformed json request - {invalid_byte,34}">>}, + R + ). + +simple_query_test() -> + SimpleQueryJson = + << + "\n" + " {\n" + " \"timeout\" : 60,\n" + " \"query_list\" :\n" + " [\n" + " {\n" + " \"index_name\" : \"example_bin\",\n" + " \"start_term\" : \"A\",\n" + " \"end_term\" : \"B\"\n" + " }\n" + " ]\n" + " }\n" + " " + >>, + {ok, M} = decode_json_body(SimpleQueryJson), + {ok, Q} = make_query_request({<<"BT">>, <<"B">>}, M), + ?assert(riak_kv_query:is_query(Q)). + +invalid_query_ae1_test() -> + IQJson = + << + "\n" + " {\n" + " \"aggregation_expression\" : \"$1 INTERSECT $2\",\n" + " \"timeout\" : 60,\n" + " \"query_list\" :\n" + " [\n" + " {\n" + " \"index_name\" : \"example_bin\",\n" + " \"start_term\" : \"A\",\n" + " \"end_term\" : \"B\"\n" + " }\n" + " ]\n" + " }\n" + " " + >>, + {ok, M} = decode_json_body(IQJson), + {error, S, _E} = make_query_request({<<"BT">>, <<"B">>}, M), + ?assertMatch(aggregation_expression, S). + +invalid_query_ae2_test() -> + IQJson = + << + "\n" + " {\n" + " \"timeout\" : 60,\n" + " \"query_list\" :\n" + " [\n" + " {\n" + " \"aggregation_tag\" : 1,\n" + " \"index_name\" : \"example_bin\",\n" + " \"start_term\" : \"A\",\n" + " \"end_term\" : \"B\"\n" + " },\n" + " {\n" + " \"aggregation_tag\" : 2,\n" + " \"index_name\" : \"example_bin\",\n" + " \"start_term\" : \"A\",\n" + " \"end_term\" : \"B\"\n" + " }\n" + "\n" + " ]\n" + " }\n" + " " + >>, + {ok, M} = decode_json_body(IQJson), + {error, S, _E} = make_query_request({<<"BT">>, <<"B">>}, M), + ?assertMatch(aggregation_expression, S). + +invalid_query_ae3_test() -> + IQJson = + << + "\n" + " {\n" + " \"aggregation_expression\" : \"$1 INTERSECT $2\",\n" + " \"timeout\" : 60,\n" + " \"query_list\" :\n" + " [\n" + " {\n" + " \"index_name\" : \"example_bin\",\n" + " \"start_term\" : \"A\",\n" + " \"end_term\" : \"B\"\n" + " },\n" + " {\n" + " \"aggregation_tag\" : 2,\n" + " \"index_name\" : \"example_bin\",\n" + " \"start_term\" : \"A\",\n" + " \"end_term\" : \"B\"\n" + " }\n" + "\n" + " ]\n" + " }\n" + " " + >>, + {ok, M} = decode_json_body(IQJson), + {error, S, E} = make_query_request({<<"BT">>, <<"B">>}, M), + ?assertMatch(query_evaluation, S), + ?assertMatch(<<"Untagged query in combination request">>, E). + +valid_query_ae4_test() -> + IQJson = + << + "\n" + " {\n" + " \"aggregation_expression\" : \"$1 INTERSECT $2\",\n" + " \"timeout\" : 60,\n" + " \"inactivity_timeout\" : 180,\n" + " \"query_list\" :\n" + " [\n" + " {\n" + " \"aggregation_tag\" : 1,\n" + " \"index_name\" : \"example_bin\",\n" + " \"start_term\" : \"A\",\n" + " \"end_term\" : \"B\"\n" + " },\n" + " {\n" + " \"aggregation_tag\" : 2,\n" + " \"index_name\" : \"example_bin\",\n" + " \"start_term\" : \"A\",\n" + " \"end_term\" : \"B\"\n" + " }\n" + "\n" + " ]\n" + " }\n" + " " + >>, + {ok, M} = decode_json_body(IQJson), + {ok, Q} = make_query_request({<<"BT">>, <<"B">>}, M), + ?assert(riak_kv_query:is_query(Q)), + QueryList = maps:get(<<"query_list">>, M), + ?assertMatch(ok, check_querylist(QueryList, false)). + +valid_query_ae5_test() -> + IQJson = + << + "\n" + " {\n" + " \"aggregation_expression\" : \"$1 INTERSECT $2\",\n" + " \"timeout\" : 60,\n" + " \"accumulation_option\" : \"keys\",\n" + " \"substitutions\" :\n" + " {\"low_dob\" : \"20210804\", \"high_dob\" : \"20223101\", \"gnsc\" : \"Ma\"},\n" + " \"query_list\" :\n" + " [\n" + " {\n" + " \"aggregation_tag\" : 1,\n" + " \"index_name\" : \"example1_bin\",\n" + " \"start_term\" : \"A\",\n" + " \"end_term\" : \"B\",\n" + " \"evaluation_expression\" :\n" + " \"delim($term, \\\"|\\\", ($fn, $dob, $dod, $gns, $pcs)) | slice($gns, 2, $gns)\",\n" + " \"filter_expression\" : \"($dob BETWEEN :low_dob AND :high_dob\) AND contains($gns, :gnsc)\"\n" + " },\n" + " {\n" + " \"aggregation_tag\" : 2,\n" + " \"index_name\" : \"example2_bin\",\n" + " \"start_term\" : \"C\",\n" + " \"end_term\" : \"D\"\n" + " }\n" + "\n" + " ]\n" + " }\n" + " " + >>, + {ok, M} = decode_json_body(IQJson), + {ok, Q} = make_query_request({<<"BT">>, <<"B">>}, M), + ?assert(riak_kv_query:is_query(Q)), + QueryList = maps:get(<<"query_list">>, M), + ?assertMatch(ok, check_querylist(QueryList, false)). + +invalid_query_ae6_test() -> + % unescaped "|" in eval expression + IQJson = + << + "\n" + " {\n" + " \"aggregation_expression\" : \"$1 INTERSECT $2\",\n" + " \"timeout\" : 60,\n" + " \"accumulation_option\" : \"keys\",\n" + " \"substitutions\" :\n" + " {\"low_dob\" : \"20210804\", \"high_dob\" : \"20223101\", \"gnsc\" : \"Ma\"},\n" + " \"query_list\" :\n" + " [\n" + " {\n" + " \"aggregation_tag\" : 1,\n" + " \"index_name\" : \"example1_bin\",\n" + " \"start_term\" : \"A\",\n" + " \"end_term\" : \"B\",\n" + " \"evaluation_expression\" :\n" + " \"delim($term, |, ($fn, $dob, $dod, $gns, $pcs)) | slice($gns, 2, $gns)\",\n" + " \"filter_expression\" : \"($dob BETWEEN :low_dob AND :high_dob\) AND contains($gns, :gnsc)\"\n" + " },\n" + " {\n" + " \"aggregation_tag\" : 2,\n" + " \"index_name\" : \"example2_bin\",\n" + " \"start_term\" : \"C\",\n" + " \"end_term\" : \"D\"\n" + " }\n" + "\n" + " ]\n" + " }\n" + " " + >>, + {ok, M} = decode_json_body(IQJson), + ?assertMatch( + {error, query_evaluation, <<"Invalid eval function">>}, + make_query_request({<<"BT">>, <<"B">>}, M) + ). + +invalid_query_ae7_test() -> + % BETWEN not BETWEEN + IQJson = + << + "\n" + " {\n" + " \"aggregation_expression\" : \"$1 INTERSECT $2\",\n" + " \"timeout\" : 60,\n" + " \"accumulation_option\" : \"keys\",\n" + " \"substitutions\" :\n" + " {\"low_dob\" : \"20210804\", \"high_dob\" : \"20223101\", \"gnsc\" : \"Ma\"},\n" + " \"query_list\" :\n" + " [\n" + " {\n" + " \"aggregation_tag\" : 1,\n" + " \"index_name\" : \"example1_bin\",\n" + " \"start_term\" : \"A\",\n" + " \"end_term\" : \"B\",\n" + " \"evaluation_expression\" :\n" + " \"delim($term, \\\"|\\\", ($fn, $dob, $dod, $gns, $pcs)) | slice($gns, 2, $gns)\",\n" + " \"filter_expression\" : \"($dob BETWEN :low_dob AND :high_dob\) AND contains($gns, :gnsc)\"\n" + " },\n" + " {\n" + " \"aggregation_tag\" : 2,\n" + " \"index_name\" : \"example2_bin\",\n" + " \"start_term\" : \"C\",\n" + " \"end_term\" : \"D\"\n" + " }\n" + "\n" + " ]\n" + " }\n" + " " + >>, + {ok, M} = decode_json_body(IQJson), + ?assertMatch( + {error, query_evaluation, <<"Invalid filter function">>}, + make_query_request({<<"BT">>, <<"B">>}, M) + ). + +invalid_query_ae8_test() -> + % missing substitution + IQJson = + << + "\n" + " {\n" + " \"aggregation_expression\" : \"$1 INTERSECT $2\",\n" + " \"timeout\" : 60,\n" + " \"accumulation_option\" : \"keys\",\n" + " \"substitutions\" :\n" + " {\"low_dob\" : \"20210804\", \"gnsc\" : \"Ma\"},\n" + " \"query_list\" :\n" + " [\n" + " {\n" + " \"aggregation_tag\" : 1,\n" + " \"index_name\" : \"example1_bin\",\n" + " \"start_term\" : \"A\",\n" + " \"end_term\" : \"B\",\n" + " \"evaluation_expression\" :\n" + " \"delim($term, \\\"|\\\", ($fn, $dob, $dod, $gns, $pcs)) | slice($gns, 2, $gns)\",\n" + " \"filter_expression\" : \"($dob BETWEEN :low_dob AND :high_dob\) AND contains($gns, :gnsc)\"\n" + " },\n" + " {\n" + " \"aggregation_tag\" : 2,\n" + " \"index_name\" : \"example2_bin\",\n" + " \"start_term\" : \"C\",\n" + " \"end_term\" : \"D\"\n" + " }\n" + "\n" + " ]\n" + " }\n" + " " + >>, + {ok, M} = decode_json_body(IQJson), + ?assertMatch( + {error, query_evaluation, <<"Invalid filter function">>}, + make_query_request({<<"BT">>, <<"B">>}, M) + ). + +invalid_query_to_test() -> + IQJson = + << + "\n" + " {\n" + " \"aggregation_expression\" : \"$1 INTERSECT $2\",\n" + " \"timeout\" : 0,\n" + " \"query_list\" :\n" + " [\n" + " {\n" + " \"aggregation_tag\" : 1,\n" + " \"index_name\" : \"example_bin\",\n" + " \"start_term\" : \"A\",\n" + " \"end_term\" : \"B\"\n" + " },\n" + " {\n" + " \"aggregation_tag\" : 2,\n" + " \"index_name\" : \"example_bin\",\n" + " \"start_term\" : \"A\",\n" + " \"end_term\" : \"B\"\n" + " }\n" + "\n" + " ]\n" + " }\n" + " " + >>, + {ok, M} = decode_json_body(IQJson), + {error, S, E} = make_query_request({<<"BT">>, <<"B">>}, M), + ?assertMatch(init, S), + ?assertMatch(<<"Bad timeout">>, E). + +invalid_query_extratag_test() -> + IQJson = + << + "\n" + " {\n" + " \"aggregation_expression\" : \"$1 INTERSECT $2\",\n" + " \"timeout\" : 60,\n" + " \"subs\" : {\"dob\" : \"19260812\"},\n" + " \"query_list\" :\n" + " [\n" + " {\n" + " \"aggregation_tag\" : 1,\n" + " \"index_name\" : \"example_bin\",\n" + " \"start_term\" : \"A\",\n" + " \"end_term\" : \"B\"\n" + " },\n" + " {\n" + " \"aggregation_tag\" : 2,\n" + " \"index_name\" : \"example_bin\",\n" + " \"start_term\" : \"A\",\n" + " \"end_term\" : \"B\",\n" + " \"end_key\" : \"B\"\n" + " }\n" + "\n" + " ]\n" + " }\n" + " " + >>, + {ok, M} = decode_json_body(IQJson), + ?assertMatch( + {error, <<"Unexpected keys in request [<<\"subs\">>]">>}, + check_keys(maps:keys(M), request) + ), + ?assertMatch( + {error, <<"Unexpected keys in request [<<\"end_key\">>]">>}, + check_querylist(maps:get(<<"query_list">>, M), false) + ). + +invalid_query_missingtag1_test() -> + IQJson = + << + "\n" + " {\n" + " \"aggregation_expression\" : \"$1 INTERSECT $2\",\n" + " \"timeout\" : 60,\n" + " \"subs\" : {\"dob\" : \"19260812\"}\n" + " }\n" + " " + >>, + {ok, M} = decode_json_body(IQJson), + ?assertMatch( + {error, <<"Missing required keys in request [<<\"query_list\">>]">>}, + check_keys(maps:keys(M), request) + ). + +invalid_query_missingtag2_test() -> + IQJson = + << + "\n" + " {\n" + " \"aggregation_expression\" : \"$1 INTERSECT $2\",\n" + " \"timeout\" : 60,\n" + " \"query_list\" :\n" + " [\n" + " {\n" + " \"aggregation_tag\" : 1,\n" + " \"index_name\" : \"example_bin\",\n" + " \"start_term\" : \"A\",\n" + " \"end_term\" : \"B\"\n" + " },\n" + " {\n" + " \"aggregation_tag\" : 2,\n" + " \"index_name\" : \"example_bin\",\n" + " \"end_term\" : \"B\"\n" + " }\n" + "\n" + " ]\n" + " }\n" + " " + >>, + {ok, M} = decode_json_body(IQJson), + ?assertMatch( + {error, <<"Missing required keys in request [<<\"start_term\">>]">>}, + check_querylist(maps:get(<<"query_list">>, M), false) + ). + +encode_results_test() -> + BinMC = encode_results(raw_count, 500), + ?assertMatch( + 500, + maps:get(?ACCKEY_RAWCOUNT, riak_kv_wm_json:decode(BinMC)) + ), + BinKC = encode_results(count, 600), + ?assertMatch( + 600, + maps:get(?ACCKEY_COUNT, riak_kv_wm_json:decode(BinKC)) + ), + KeyList = [<<"K00001">>, <<"K00002">>, <<"K0003">>], + BinKL = encode_results(keys, KeyList), + ?assertMatch( + KeyList, + maps:get(?ACCKEY_KEYS, riak_kv_wm_json:decode(BinKL)) + ), + KeyListT = [{<<"K00001">>}, {<<"K00002">>}, {<<"K0003">>}], + BinKLT = encode_results(keys, KeyListT), + ?assertMatch( + KeyList, + maps:get(?ACCKEY_KEYS, riak_kv_wm_json:decode(BinKLT)) + ), + TermKeyList = [{<<"T0001">>, <<"K0002">>}, {<<"T0002">>, <<"K0001">>}], + BinTKL = encode_results(terms, TermKeyList), + ?assertMatch( + TermKeyList, + lists:sort( + lists:map( + fun(M) -> + [{T, K}] = maps:to_list(M), + {T, K} + end, + maps:get(?ACCKEY_TERMS, riak_kv_wm_json:decode(BinTKL)) + ) + ) + ), + TermKeyListT = + [{{<<"T0001">>, <<"K0002">>}}, {{<<"T0002">>, <<"K0001">>}}], + BinTKLT = encode_results(terms, TermKeyListT), + ?assertMatch( + TermKeyList, + lists:sort( + lists:map( + fun(M) -> + [{T, K}] = maps:to_list(M), + {T, K} + end, + maps:get(?ACCKEY_TERMS, riak_kv_wm_json:decode(BinTKLT)) + ) + ) + ), + TermCount = #{<<"T0001">> => 12, <<"T0002">> => 10}, + BinTKC = encode_results(term_with_count, TermCount), + ?assertMatch( + 10, + maps:get( + <<"T0002">>, + maps:get(?ACCKEY_TERMCOUNT, riak_kv_wm_json:decode(BinTKC)) + ) + ), + BinTMC = encode_results(term_with_rawcount, TermCount), + ?assertMatch( + 12, + maps:get( + <<"T0001">>, + maps:get(?ACCKEY_TERMRAWCOUNT, riak_kv_wm_json:decode(BinTMC)) + ) + ). + +validate_parameter_test_() -> + { + foreach, + setup(), + cleanup(), + [ + fun validate_parameters/0, + fun validate_headers/0 + ] + }. + +validate_parameters() -> + InitCtx = + #context{bucket = {<<"T">>, <<"B">>}, request_type = fetch_results}, + DummyQRef = base64:encode(term_to_binary({node(), self(), make_ref()})), + {ok, Ctx1} = + parse_query_params( + [{<<"max_results">>, <<"1000">>}, {<<"result_queue">>, DummyQRef}], + InitCtx + ), + ?assert(is_map(Ctx1#context.queue_request)), + ?assertMatch(1000, maps:get(max_results, Ctx1#context.queue_request)), + + {ok, Ctx2} = + parse_query_params( + [{<<"result_queue">>, DummyQRef}], + InitCtx + ), + ?assert(is_map(Ctx2#context.queue_request)), + ?assertMatch( + ?MAX_RESULTS_FROM_QUEUE, + maps:get(max_results, Ctx2#context.queue_request) + ), + + {halt, 400, [?JSN_HEADER], Error3, []} = + parse_query_params( + [{<<"max_results">>, <<"-1">>}, {<<"result_queue">>, DummyQRef}], + InitCtx + ), + ?assert(is_binary(Error3)), + {halt, 400, [?JSN_HEADER], Error4, []} = + parse_query_params( + [{<<"max_results">>, <<"A">>}, {<<"result_queue">>, DummyQRef}], + InitCtx + ), + ?assert(is_binary(Error4)), + {halt, 400, [?JSN_HEADER], Error5, []} = + parse_query_params( + [{<<"max_results">>, <<"100">>}], + InitCtx + ), + ?assert(is_binary(Error5)), + InitCtxQ = + #context{bucket = {<<"T">>, <<"B">>}, request_type = submit_query}, + {ok, _Ctx3} = parse_query_params([], InitCtxQ). + +validate_headers() -> + InitCtx1 = + #context{bucket = {<<"T">>, <<"B">>}, request_type = fetch_results}, + InitCtx2 = + #context{bucket = {<<"T">>, <<"B">>}, request_type = submit_query}, + Header1 = riak_api_web_headers:make([{'Accept', <<"application/json">>}]), + Header2 = + riak_api_web_headers:make( + [{'Accept', [<<"application/octet-stream">>, <<"*/*;q=0.9">>]}] + ), + Header3 = riak_api_web_headers:make([]), + lists:foreach( + fun({Ctx, Hdrs}) -> + ?assertMatch( + {ok, _}, + parse_request_headers(Hdrs, Ctx) + ) + end, + [ + {InitCtx1, Header1}, + {InitCtx1, Header2}, + {InitCtx1, Header3}, + {InitCtx2, Header1} + ] + ), + Header4 = riak_api_web_headers:make([{'Accept', <<"text/xml">>}]), + {halt, 406, [TXT_HEADER], Error, []} = + parse_request_headers(Header4, InitCtx1), + {halt, 406, [TXT_HEADER], Error, []} = + parse_request_headers(Header4, InitCtx2), + ?assert(is_binary(Error)). + +setup() -> + riak_kv_test_util:common_setup(?MODULE, fun configure/1). + +cleanup() -> + riak_kv_test_util:common_cleanup(?MODULE, fun configure/1). + +configure(load) -> + application:set_env(riak_core, default_bucket_props, []), + application:set_env(riak_kv, storage_backend, riak_kv_memory_backend); +configure(_) -> + ok. + +-endif. diff --git a/src/riak_kv_ag_queue.erl b/src/riak_kv_ag_queue.erl new file mode 100644 index 000000000..414b624f5 --- /dev/null +++ b/src/riak_kv_ag_queue.erl @@ -0,0 +1,543 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2026 Martin Sumner. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- + +-module(riak_kv_ag_queue). + +-if(?OTP_RELEASE == 26). +-feature(maybe_expr, enable). +-endif. + +-include_lib("riak_kv/include/riak_kv_web.hrl"). +-include_lib("kernel/include/logger.hrl"). + +-export( + [ + match_route/3, + check_permissions/5, + parse_query_params/2, + parse_request_headers/2, + process_request/2, + record_request/3 + ] +). + +-record(context, { + client = riak_client:new(node(), undefined) :: riak_client:riak_client(), + request_type :: fetch_request | repl_request | membership_request, + queue_name :: atom() | undefined, + object_format = internal :: internal | internal_aaehash +}). + +-type context() :: #context{}. + +%% =================================================================== +%% Callback functions +%% =================================================================== + +-spec match_route( + riak_api_web_acceptor:method(), + unicode:chardata(), + list(unicode:chardata()) +) -> + nomatch + | {method_not_allowed, list(riak_api_web_acceptor:method())} + | {ok, riak_api_web_handler:limits(), context()}. +match_route('GET', _Path, [<<"queuename">>, Queue]) -> + case riak_kv_web_common:check_queuename(Queue) of + undefined -> + nomatch; + QueueA -> + { + ok, + {16, 2048, 0}, + #context{request_type = fetch_request, queue_name = QueueA} + } + end; +match_route('POST', _Path, [<<"queuename">>, Queue]) -> + case riak_kv_web_common:check_queuename(Queue) of + undefined -> + nomatch; + QueueA -> + { + ok, + {16, 2048, 8 * 1024 * 1024}, + #context{request_type = repl_request, queue_name = QueueA} + } + end; +match_route(_Method, _Path, [<<"queuename">>, _Queue]) -> + {method_not_allowed, ['GET', 'POST']}; +match_route('GET', _Path, [<<"membership_request">>]) -> + {ok, {16, 2048, 0}, #context{request_type = membership_request}}; +match_route(_Method, _Path, [<<"membership_request">>]) -> + {method_not_allowed, ['GET']}; +match_route(_Method, _Path, _SplitPath) -> + nomatch. + +%% @doc check_permissions for using this module or route +-spec check_permissions( + riak_api_web_headers:headers(), + riak_api_web_socket:scheme(), + riak_api_web_handler:peer_ip(), + riak_api_web_handler:peer_cert(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +check_permissions(ReqHeaders, Scheme, Peer, _Cert, Ctx) -> + case application:get_env(riak_kv, permit_insecure_http_ops, false) of + true -> + {ok, Ctx}; + false -> + Check = + riak_kv_web_common:check_permissions( + ReqHeaders, + Scheme, + Peer, + undefined, + undefined + ), + case Check of + true -> + {ok, Ctx}; + HaltResponse -> + HaltResponse + end + end. + +%% @doc parse and validate query params, passed as a map +-spec parse_query_params( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +parse_query_params(Params, #context{request_type = fetch_request} = Ctx) -> + case lists:keyfind(<<"object_format">>, 1, Params) of + {<<"object_format">>, <<"internal">>} -> + {ok, Ctx#context{object_format = internal}}; + {<<"object_format">>, <<"internal_aaehash">>} -> + {ok, Ctx#context{object_format = internal_aaehash}}; + {<<"object_format">>, Unexpected} -> + { + halt, + 400, + [?TXT_HEADER], + <<"Format ~0p no defined">>, + [Unexpected] + }; + _ -> + {ok, Ctx#context{object_format = internal}} + end; +parse_query_params(_Params, Ctx) -> + {ok, Ctx}. + +%% @doc parse and validate the request headers +-spec parse_request_headers( + riak_api_web_headers:headers(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +parse_request_headers(ReqHeaders, Ctx) -> + AcceptedTypes = + case riak_api_web_headers:get_value('Accept', ReqHeaders) of + undefined -> + all; + SingleType when is_binary(SingleType) -> + [SingleType]; + MultipleTypes when is_list(MultipleTypes) -> + MultipleTypes + end, + case AcceptedTypes of + all -> + {ok, Ctx}; + L when is_list(L) -> + CTypeRequired = + case Ctx#context.request_type of + fetch_request -> + <<"application/octet-stream">>; + membership_request -> + <<"application/json">>; + repl_request -> + <<"text/plain">> + end, + case riak_kv_web_common:type_match(CTypeRequired, AcceptedTypes) of + true -> + {ok, Ctx}; + false -> + { + halt, + 406, + [?TXT_HEADER], + <<"~s must be accepted">>, + [CTypeRequired] + } + end + end. + +%% @doc Process the request and produce a response +-spec process_request( + riak_api_web_body:req_body() | none, + context() +) -> + { + ok, + { + riak_api_web_acceptor:response_code(), + riak_api_web_headers:header_list(), + riak_api_web_handler:response_body(), + boolean(), + riak_api_web_body:req_body() | none + }, + context() + } + | riak_api_web_acceptor:halt_response(). +process_request(none, #context{request_type = fetch_request} = Ctx) -> + R = riak_client:fetch(Ctx#context.queue_name, Ctx#context.client), + case format_response(Ctx#context.object_format, R) of + {ok, Bin} -> + {ok, {200, [?BIN_HEADER], Bin, true, none}, Ctx}; + {error, Bin} -> + {halt, 500, [?TXT_HEADER], Bin, []} + end; +process_request(none, #context{request_type = membership_request} = Ctx) -> + case riak_client:membership_request(http) of + AddrL when is_list(AddrL) -> + RMap = + #{ + <<"up_nodes">> => + lists:map( + fun({IP, Port}) -> + #{ + <<"ip">> => list_to_binary(IP), + <<"port">> => integer_to_binary(Port) + } + end, + AddrL + ) + }, + Body = iolist_to_binary(riak_kv_wm_json:encode(RMap)), + {ok, {200, [?BIN_HEADER], Body, true, none}, Ctx} + end; +process_request(RqBdy, #context{request_type = repl_request} = Ctx) -> + case riak_api_web_body:get_body(RqBdy, all, 60000) of + {EncodedKeyList, UpdRqBdy} when is_binary(EncodedKeyList) -> + case decode_keylist(EncodedKeyList) of + false -> + ErrMsg = <<"Malformed Keyclock list">>, + {halt, 400, [?TXT_HEADER], ErrMsg, []}; + KCL -> + QN = Ctx#context.queue_name, + ok = riak_kv_replrtq_src:replrtq_ttaaefs(QN, KCL), + RBin = + case riak_kv_replrtq_src:length_rtq(QN) of + {_, {FL, FSL, RTL}} -> + iolist_to_binary( + io_lib:format( + "Queue ~w: ~w ~w ~w", + [QN, FL, FSL, RTL] + ) + ); + _ -> + iolist_to_binary( + io_lib:format( + "No queue ~w", + [QN] + ) + ) + end, + {ok, {200, [?TXT_HEADER], RBin, true, UpdRqBdy}, Ctx} + end; + {error, content_too_large} -> + {halt, 413, [], <<>>, []} + end. + +%% @doc Record the output of the interaction +-spec record_request( + riak_api_web_handler:timings(), + riak_api_web_handler:completion(), + context() +) -> + ok. +record_request(_Timings, _Completion, _Ctx) -> + ok. + +%% =================================================================== +%% Internal Functions +%% =================================================================== + +decode_keylist(EncodedKeyList) -> + case riak_kv_wm_json:decode(EncodedKeyList) of + #{<<"keys-clocks">> := KCL} when is_list(KCL) -> + KeyClockList = + lists:foldl(fun decode_bucketkeyclock/2, [], KCL), + case {length(KeyClockList), length(KCL)} of + {N, N} -> + lists:reverse(KeyClockList); + {N, M} -> + ?LOG_INFO( + "Malformed requests ~w within push of ~w", + [M - N, M] + ), + false + end; + _ -> + false + end. + +decode_bucketkeyclock( + #{ + <<"bucket-type">> := T, + <<"bucket">> := B, + <<"key">> := K, + <<"clock">> := C + }, + Acc +) -> + case decode_clock(C) of + false -> + Acc; + DecodedClock -> + [{{T, B}, K, DecodedClock, to_fetch} | Acc] + end; +decode_bucketkeyclock( + #{ + <<"bucket">> := B, + <<"key">> := K, + <<"clock">> := C + }, + Acc +) -> + case decode_clock(C) of + false -> + Acc; + DecodedClock -> + [{B, K, DecodedClock, to_fetch} | Acc] + end; +decode_bucketkeyclock(_, Acc) -> + Acc. + +-spec decode_clock(list()) -> vclock:vclock() | false. +decode_clock(EncodedClock) -> + try + riak_object:decode_vclock(base64:decode(EncodedClock)) + catch + _:_ -> + false + end. + +-type fetch_result() :: + riak_object:riak_object() + | queue_empty + | {deleted, vclock:vclock(), riak_object:riak_object()} + | { + reap, + { + riak_object:bucket(), + riak_object:key(), + vclock:vclock(), + erlang:timestamp() + } + }. + +-spec format_response( + internal | internal_aaehash, + {ok, fetch_result()} | {error, Err :: term()} +) -> + {ok, binary()} | {error, binary()}. +format_response(_, {ok, queue_empty}) -> + {ok, <<0:8/integer>>}; +format_response(_, {error, Reason}) -> + ?LOG_WARNING("Fetch error ~w", [Reason]), + {error, iolist_to_binary(io_lib:format("~0p", [Reason]))}; +format_response(internal_aaehash, {ok, {reap, {B, K, TC, LMD}}}) -> + BK = make_binarykey(B, K), + {SegmentID, SegmentHash} = + leveled_tictac:tictac_hash(BK, lists:sort(TC)), + SuccessMark = <<1:8/integer>>, + IsTombstone = <<0:8/integer>>, + ObjBin = encode_riakobject({reap, {B, K, TC, LMD}}), + { + ok, + << + SuccessMark/binary, + IsTombstone/binary, + SegmentID:32/integer, + SegmentHash:32/integer, + ObjBin/binary + >> + }; +format_response(internal_aaehash, {ok, {deleted, TombClock, RObj}}) -> + BK = make_binarykey(riak_object:bucket(RObj), riak_object:key(RObj)), + {SegmentID, SegmentHash} = + leveled_tictac:tictac_hash(BK, lists:sort(TombClock)), + SuccessMark = <<1:8/integer>>, + IsTombstone = <<1:8/integer>>, + ObjBin = encode_riakobject(RObj), + TombClockBin = term_to_binary(TombClock), + TCL = byte_size(TombClockBin), + { + ok, + << + SuccessMark/binary, + IsTombstone/binary, + SegmentID:32/integer, + SegmentHash:32/integer, + TCL:32/integer, + TombClockBin/binary, + ObjBin/binary + >> + }; +format_response(internal_aaehash, {ok, RObj}) -> + BK = make_binarykey(riak_object:bucket(RObj), riak_object:key(RObj)), + {SegmentID, SegmentHash} = + leveled_tictac:tictac_hash(BK, lists:sort(riak_object:vclock(RObj))), + SuccessMark = <<1:8/integer>>, + IsTombstone = <<0:8/integer>>, + ObjBin = encode_riakobject(RObj), + { + ok, + << + SuccessMark/binary, + IsTombstone/binary, + SegmentID:32/integer, + SegmentHash:32/integer, + ObjBin/binary + >> + }; +format_response(internal, {ok, {reap, {B, K, TC, LMD}}}) -> + SuccessMark = <<1:8/integer>>, + IsTombstone = <<0:8/integer>>, + ObjBin = encode_riakobject({reap, {B, K, TC, LMD}}), + { + ok, + << + SuccessMark/binary, + IsTombstone/binary, + ObjBin/binary + >> + }; +format_response(internal, {ok, {deleted, TombClock, RObj}}) -> + SuccessMark = <<1:8/integer>>, + IsTombstone = <<1:8/integer>>, + ObjBin = encode_riakobject(RObj), + TombClockBin = term_to_binary(TombClock), + TCL = byte_size(TombClockBin), + { + ok, + << + SuccessMark/binary, + IsTombstone/binary, + TCL:32/integer, + TombClockBin/binary, + ObjBin/binary + >> + }; +format_response(internal, {ok, RObj}) -> + SuccessMark = <<1:8/integer>>, + IsTombstone = <<0:8/integer>>, + ObjBin = encode_riakobject(RObj), + { + ok, + << + SuccessMark/binary, + IsTombstone/binary, + ObjBin/binary + >> + }. + +-spec encode_riakobject( + riak_object:riak_object() | riak_object:repl_ref() +) -> + binary(). +encode_riakobject(RObj) -> + ToCompress = app_helper:get_env(riak_kv, replrtq_compressonwire, false), + FullObjBin = riak_object:nextgenrepl_encode(repl_v1, RObj, ToCompress), + CRC = erlang:crc32(FullObjBin), + <>. + +-spec make_binarykey(riak_object:bucket(), riak_object:key()) -> binary(). +%% @doc +%% Convert Bucket and Key into a single binary +make_binarykey( + {Type, Bucket}, Key +) when + is_binary(Type), is_binary(Bucket), is_binary(Key) +-> + <>; +make_binarykey(Bucket, Key) when is_binary(Bucket), is_binary(Key) -> + <>. + +%% =================================================================== +%% Eunit tests +%% =================================================================== + +-ifdef(TEST). + +-include_lib("eunit/include/eunit.hrl"). + +test_kcl() -> + A = vclock:fresh(), + B = vclock:fresh(), + A1 = vclock:increment(a, A), + B1 = vclock:increment(b, B), + E1 = {<<"B1">>, <<"K1">>, A1}, + E2 = {{<<"T">>, <<"B2">>}, <<"K2">>, B1}, + [E1, E2]. + +encode_keys_and_clocks(KeysNClocks) -> + Keys = + { + struct, + [ + { + <<"keys-clocks">>, + [ + {struct, encode_key_and_clock(Bucket, Key, Clock)} + || {Bucket, Key, Clock} <- KeysNClocks + ] + } + ] + }, + mochijson2:encode(Keys). + +encode_key_and_clock({Type, Bucket}, Key, C) -> + [ + {<<"bucket-type">>, Type}, + {<<"bucket">>, Bucket}, + {<<"key">>, Key}, + {<<"clock">>, base64:encode_to_string(riak_object:encode_vclock(C))} + ]; +encode_key_and_clock(Bucket, Key, C) -> + [ + {<<"bucket">>, Bucket}, + {<<"key">>, Key}, + {<<"clock">>, base64:encode_to_string(riak_object:encode_vclock(C))} + ]. + +keylist_decode_test() -> + KCL = test_kcl(), + KeyJson = + iolist_to_binary( + encode_keys_and_clocks(KCL) + ), + ActualBKCL = decode_keylist(KeyJson), + ExpectedBKCL = + lists:map(fun({B, K, C}) -> {B, K, C, to_fetch} end, KCL), + ?assertMatch(ExpectedBKCL, ActualBKCL). + +-endif. diff --git a/src/riak_kv_ag_stats.erl b/src/riak_kv_ag_stats.erl new file mode 100644 index 000000000..26cc2dd2a --- /dev/null +++ b/src/riak_kv_ag_stats.erl @@ -0,0 +1,348 @@ +%% -------------------------------------------------------- +%% +%% Copyright (c) 2026 Martin Sumner +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +%% @doc Handler for HTTP API requests for node statistics + +-module(riak_kv_ag_stats). + +-include("riak_kv_web.hrl"). + +-behaviour(riak_api_web_handler). + +-export( + [ + match_route/3, + check_permissions/5, + parse_query_params/2, + parse_request_headers/2, + process_request/2, + record_request/3 + ] +). + +-record(context, { + timeout = 30000 :: pos_integer(), + content_type = json :: content_type() +}). + +-type content_type() :: json | plain. +-type context() :: #context{}. + +%% =================================================================== +%% Callback functions +%% =================================================================== + +-spec match_route( + riak_api_web_acceptor:method(), + unicode:chardata(), + list(unicode:chardata()) +) -> + nomatch + | {method_not_allowed, list(riak_api_web_acceptor:method())} + | {ok, riak_api_web_handler:limits(), context()}. +match_route(Method, Path, _) -> + ExpectedStatsPath = + iolist_to_binary( + application:get_env(riak_kv, stats_urlpath, "stats") + ), + case {string:trim(Path, both, "/"), Method} of + {ExpectedStatsPath, 'GET'} -> + {ok, size_limits(), #context{}}; + {ExpectedStatsPath, _OtherMethod} -> + {method_not_allowed, ['GET']}; + {_OtherPath, _} -> + nomatch + end. + +%% @doc check_permissions for using this module or route +-spec check_permissions( + riak_api_web_headers:headers(), + riak_api_web_socket:scheme(), + riak_api_web_handler:peer_ip(), + riak_api_web_handler:peer_cert(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +check_permissions(ReqHeaders, Scheme, Peer, _Cert, Ctx) -> + case application:get_env(riak_kv, permit_insecure_http_ops, false) of + true -> + {ok, Ctx}; + false -> + Check = + riak_kv_web_common:check_permissions( + ReqHeaders, + Scheme, + Peer, + undefined, + undefined + ), + case Check of + true -> + {ok, Ctx}; + HaltResponse -> + HaltResponse + end + end. + +%% @doc parse and validate query params, passed as a map +-spec parse_query_params( + riak_api_web_handler:query_params(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +parse_query_params([], Ctx) -> + % Typically we expect no options - so shortcut the validation in this case + {ok, Ctx}; +parse_query_params(Params, Ctx) -> + case riak_kv_web_common:get_timeout(Params) of + {ok, none} -> + {ok, Ctx}; + {ok, Timeout} -> + {ok, Ctx#context{timeout = Timeout}}; + HaltResponse -> + HaltResponse + end. + +%% @doc parse and validate the request headers +-spec parse_request_headers( + riak_api_web_headers:headers(), + context() +) -> + {ok, context()} | riak_api_web_acceptor:halt_response(). +parse_request_headers(ReqHeaders, Ctx) -> + case riak_api_web_headers:get_value('Accept', ReqHeaders) of + undefined -> + {ok, Ctx#context{content_type = json}}; + AcceptType -> + {JsonOK, JsonScore} = + riak_kv_web_common:type_preference( + <<"application/json">>, + AcceptType + ), + {TextOK, TextScore} = + riak_kv_web_common:type_preference( + <<"text/plain">>, + AcceptType + ), + case {JsonOK, JsonScore >= TextScore, TextOK} of + {true, true, _} -> + {ok, Ctx#context{content_type = json}}; + {false, false, true} -> + {ok, Ctx#context{content_type = plain}}; + _ -> + {halt, 406, [], <<>>, []} + end + end. + +%% @doc Process the request and produce a response +-spec process_request( + none, + context() +) -> + { + ok, + { + riak_api_web_acceptor:response_code(), + riak_api_web_headers:header_list(), + riak_api_web_handler:response_body(), + boolean(), + none + }, + context() + } + | riak_api_web_acceptor:halt_response(). +process_request(none, Ctx) -> + try + Stats = riak_kv_http_cache:get_stats(Ctx#context.timeout), + {Rsp, CType} = + produce_response(Stats, Ctx#context.content_type), + {ok, {200, [{'Content-Type', CType}], Rsp, true, none}, Ctx} + catch + exit:{timeout, _} -> + ErrMsg = <<"Request timed out after ~w ms">>, + {halt, 503, [?TXT_HEADER], ErrMsg, [Ctx#context.timeout]} + end. + +%% @doc Record the output of the interaction +-spec record_request( + riak_api_web_handler:timings(), + riak_api_web_handler:completion(), + context() +) -> + ok. +record_request(_Timings, _Completion, _Ctx) -> + ok. + +%% =================================================================== +%% Internal Functions +%% =================================================================== + +-if(?OTP_RELEASE >= 28). + +produce_response(Stats, json) -> + { + iolist_to_binary(json:encode(maps:from_list(Stats))), + <<"application/json">> + }; +produce_response(Stats, plain) -> + % No pretty-print available + { + iolist_to_binary(json:format(maps:from_list(Stats))), + <<"text/plain">> + }. + +-else. +produce_response(Stats, json) -> + { + iolist_to_binary(riak_kv_wm_json:encode(maps:from_list(Stats))), + <<"application/json">> + }; +produce_response(Stats, plain) -> + % No pretty-print available in riak_kv_wm_json + { + iolist_to_binary(riak_kv_wm_json:encode(maps:from_list(Stats))), + <<"text/plain">> + }. +-endif. + +size_limits() -> + { + 32, + 1024, + 0 + }. + +%% =================================================================== +%% EUnit tests +%% =================================================================== + +-ifdef(TEST). + +-include_lib("eunit/include/eunit.hrl"). + +extract_params(URI) -> + uri_string:dissect_query( + maps:get( + query, + uri_string:normalize(URI, [return_map]) + ) + ). + +parameter_validation_test() -> + {ok, Ctx1} = + parse_query_params( + extract_params(<<"/stats?timeout=10">>), + #context{} + ), + ?assertMatch(10, Ctx1#context.timeout), + {ok, Ctx2} = + parse_query_params( + extract_params(<<"/stats?undefined_param">>), + #context{} + ), + ?assertMatch(30000, Ctx2#context.timeout), + ?assertMatch( + {halt, 400, _, _, _}, + parse_query_params( + extract_params(<<"/stats?timeout=B">>), + #context{} + ) + ). + +stats_test_() -> + { + foreach, + setup(), + cleanup(), + [ + fun check_stats_not_crash/0 + ] + }. + +check_stats_not_crash() -> + Stats = riak_kv_http_cache:get_stats(30000), + _DummyRun = timer:tc(fun() -> produce_response(Stats, json) end), + % Timings for first run are slow ?? + {T0, {JR, <<"application/json">>}} = + timer:tc(fun() -> produce_response(Stats, json) end), + {T1, {JP, <<"text/plain">>}} = + timer:tc(fun() -> produce_response(Stats, plain) end), + io:format(user, "Stats took ~w json ~w plain~n", [T0, T1]), + ?assert(is_binary(JR)), + ?assert(is_binary(JP)). + +setup() -> + riak_kv_test_util:common_setup(?MODULE, fun configure/1). + +cleanup() -> + riak_kv_test_util:common_cleanup(?MODULE, fun configure/1). + +configure(load) -> + application:set_env(riak_core, default_bucket_props, []), + application:set_env(riak_kv, storage_backend, riak_kv_memory_backend); +configure(_) -> + ok. + +accept_header_test() -> + InitCtx = #context{}, + Hdr1 = riak_api_web_headers:make([{'Accept', <<"application/json">>}]), + {ok, Ctx1} = parse_request_headers(Hdr1, InitCtx), + ?assertMatch(json, Ctx1#context.content_type), + Hdr2 = riak_api_web_headers:make([{'Accept', <<"application/*">>}]), + {ok, Ctx2} = parse_request_headers(Hdr2, InitCtx), + ?assertMatch(json, Ctx2#context.content_type), + Hdr3 = + riak_api_web_headers:make( + [ + {'Accept', <<"text/plain, application/*">>} + ] + ), + {ok, Ctx3} = parse_request_headers(Hdr3, InitCtx), + ?assertMatch(json, Ctx3#context.content_type), + Hdr4 = + riak_api_web_headers:make( + [ + {'Accept', <<"text/*, application/octet-stream">>} + ] + ), + {ok, Ctx4} = parse_request_headers(Hdr4, InitCtx), + ?assertMatch(plain, Ctx4#context.content_type), + Hdr5 = + riak_api_web_headers:make( + [ + {'Accept', <<"*/*, application/octet-stream">>} + ] + ), + {ok, Ctx5} = parse_request_headers(Hdr5, InitCtx), + ?assertMatch(json, Ctx5#context.content_type), + Hdr6 = + riak_api_web_headers:make([ + {'Accept', <<"application/octet-stream">>} + ]), + ?assertMatch(halt, element(1, parse_request_headers(Hdr6, InitCtx))), + Hdr7 = + riak_api_web_headers:make([ + {'Accept', <<"application/octet-stream, text/html">>} + ]), + ?assertMatch(halt, element(1, parse_request_headers(Hdr7, InitCtx))), + Hdr8 = riak_api_web_headers:make([]), + {ok, Ctx8} = parse_request_headers(Hdr8, InitCtx), + ?assertMatch(json, Ctx8#context.content_type). + +-endif. diff --git a/src/riak_kv_app.erl b/src/riak_kv_app.erl index 669ca4085..df63d4b5d 100644 --- a/src/riak_kv_app.erl +++ b/src/riak_kv_app.erl @@ -240,9 +240,10 @@ start(_Type, _StartArgs) -> ok = riak_kv_cli_registry:register_cli(), - %% Add routes to webmachine - [ webmachine_router:add_route(R) - || R <- lists:reverse(riak_kv_web:dispatch_table()) ], + %% Add routes for web API + ok = riak_kv_web_common:add_routes(), + ok = riak_kv_web_common:compile_splitters(), + {ok, Pid}; {error, Reason} -> {error, Reason} @@ -259,10 +260,10 @@ prep_stop(_State) -> ok = riak_api_pb_service:deregister(?SERVICES), ?LOG_INFO("Unregistered pb services"), - %% Gracefully unregister riak_kv webmachine endpoints. - [ webmachine_router:remove_route(R) || R <- - riak_kv_web:dispatch_table() ], - ?LOG_INFO("unregistered webmachine routes"), + %% Gracefully unregister riak_kv web endpoints. + %% TODO + + wait_for_put_fsms(), ?LOG_INFO("all active put FSMs completed"), ok diff --git a/src/riak_kv_console.erl b/src/riak_kv_console.erl index 9315da0af..34ebc61df 100644 --- a/src/riak_kv_console.erl +++ b/src/riak_kv_console.erl @@ -47,6 +47,22 @@ bucket_type_list/1 ]). +%% Names of JSON fields in bucket properties +-define(JSON_PROPS, <<"props">>). +-define(JSON_BUCKETS, <<"buckets">>). +-define(JSON_KEYS, <<"keys">>). +-define(JSON_LINKFUN, <<"linkfun">>). +-define(JSON_MOD, <<"mod">>). +-define(JSON_FUN, <<"fun">>). +-define(JSON_ARG, <<"arg">>). +-define(JSON_CHASH, <<"chash_keyfun">>). +-define(JSON_JSFUN, <<"jsfun">>). +-define(JSON_JSANON, <<"jsanon">>). +-define(JSON_JSBUCKET, <<"bucket">>). +-define(JSON_JSKEY, <<"key">>). +-define(JSON_ALLOW_MULT, <<"allow_mult">>). +-define(JSON_DATATYPE, <<"datatype">>). + -export([command/1]). %% new callbacks go here, and are to be implemented using clique %% Reused by Yokozuna for printing AAE status. @@ -56,6 +72,9 @@ -include_lib("kernel/include/logger.hrl"). +-type jsonpropvalue() :: integer()|string()|boolean()|{struct,[jsonmodfun()]}. +-type jsonmodfun() :: {ModBinary :: term(), binary()}|{FunBinary :: term(), binary()}. +-type erlpropvalue() :: integer()|string()|boolean(). -spec command([string()]) -> ok. command(Cmd) -> @@ -533,7 +552,7 @@ bucket_type_create([TypeStr, PropsStr]) -> bucket_type_create(Type, {struct, Fields}) -> case proplists:get_value(<<"props">>, Fields) of {struct, Props} -> - ErlProps = [riak_kv_wm_utils:erlify_bucket_prop(P) || P <- Props], + ErlProps = [erlify_bucket_prop(P) || P <- Props], bucket_type_print_create_result(Type, riak_core_bucket_type:create(Type, ErlProps)); _ -> io:format("Cannot create bucket type ~ts: no props field found in json~n", [Type]), @@ -567,7 +586,7 @@ bucket_type_update([TypeStr, PropsStr]) -> bucket_type_update(Type, {struct, Fields}) -> case proplists:get_value(<<"props">>, Fields) of {struct, Props} -> - ErlProps = [riak_kv_wm_utils:erlify_bucket_prop(P) || P <- Props], + ErlProps = [erlify_bucket_prop(P) || P <- Props], bucket_type_print_update_result(Type, riak_core_bucket_type:update(Type, ErlProps)); _ -> io:format("Cannot create bucket type ~ts: no props field found in json~n", [Type]), @@ -721,6 +740,46 @@ repair_2i(Args) -> error end. +-spec erlify_bucket_prop({Property::binary(), jsonpropvalue()}) -> + {Property::atom(), erlpropvalue()}. +%% @doc The reverse of jsonify_bucket_prop/1. Converts JSON representation +%% of bucket properties to their Erlang form. +erlify_bucket_prop({?JSON_DATATYPE, Type}) when is_binary(Type) -> + {datatype, binary_to_existing_atom(Type, utf8)}; +erlify_bucket_prop({?JSON_LINKFUN, {struct, Props}}) -> + case {proplists:get_value(?JSON_MOD, Props), + proplists:get_value(?JSON_FUN, Props)} of + {Mod, Fun} when is_binary(Mod), is_binary(Fun) -> + {linkfun, {modfun, + list_to_existing_atom(binary_to_list(Mod)), + list_to_existing_atom(binary_to_list(Fun))}}; + {undefined, undefined} -> + case proplists:get_value(?JSON_JSFUN, Props) of + Name when is_binary(Name) -> + {linkfun, {jsfun, Name}}; + undefined -> + case proplists:get_value(?JSON_JSANON, Props) of + {struct, Bkey} -> + Bucket = proplists:get_value(?JSON_JSBUCKET, Bkey), + Key = proplists:get_value(?JSON_JSKEY, Bkey), + %% bomb if malformed + true = is_binary(Bucket) andalso is_binary(Key), + {linkfun, {jsanon, {Bucket, Key}}}; + Source when is_binary(Source) -> + {linkfun, {jsanon, Source}} + end + end + end; +erlify_bucket_prop({?JSON_CHASH, {struct, Props}}) -> + {chash_keyfun, {list_to_existing_atom( + binary_to_list( + proplists:get_value(?JSON_MOD, Props))), + list_to_existing_atom( + binary_to_list( + proplists:get_value(?JSON_FUN, Props)))}}; +erlify_bucket_prop({Prop, Value}) -> + {list_to_existing_atom(binary_to_list(Prop)), Value}. + %%%=================================================================== %%% Private diff --git a/src/riak_kv_crdt.erl b/src/riak_kv_crdt.erl index 4126d3118..200b4e8d3 100644 --- a/src/riak_kv_crdt.erl +++ b/src/riak_kv_crdt.erl @@ -36,7 +36,6 @@ -include_lib("kernel/include/logger.hrl"). --include("riak_kv_wm_raw.hrl"). -include("riak_object.hrl"). -include_lib("riak_kv_types.hrl"). -include("riak_kv_capability.hrl"). diff --git a/src/riak_kv_crdt_json.erl b/src/riak_kv_crdt_json.erl index ab8ff2996..c53a37313 100644 --- a/src/riak_kv_crdt_json.erl +++ b/src/riak_kv_crdt_json.erl @@ -32,7 +32,7 @@ -include("riak_kv_types.hrl"). -endif. --export_type([context/0, all_type/0, all_type_op/0]). +-export_type([context/0, all_type/0, all_type_op/0, update/0, toplevel_type/0]). %% Mostly copied from riak_pb_dt_codec %% Value types diff --git a/src/riak_kv_delete.erl b/src/riak_kv_delete.erl index 0477bd369..3ad01a235 100644 --- a/src/riak_kv_delete.erl +++ b/src/riak_kv_delete.erl @@ -27,7 +27,8 @@ -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). -endif. --include("riak_kv_wm_raw.hrl"). + +-include("riak_object.hrl"). -export([start_link/6, start_link/7, start_link/8, delete/8, generate_tombstone/2]). diff --git a/src/riak_kv_get_fsm.erl b/src/riak_kv_get_fsm.erl index 91b5fe3ab..7ce709a0d 100644 --- a/src/riak_kv_get_fsm.erl +++ b/src/riak_kv_get_fsm.erl @@ -52,19 +52,24 @@ vnodes. -type details() :: [detail()]. --type option() :: {r, pos_integer()} | %% Minimum number of successful responses - {pr, non_neg_integer()} | %% Minimum number of primary vnodes participating - {basic_quorum, boolean()} | %% Whether to use basic quorum (return early - %% in some failure cases. - {notfound_ok, boolean()} | %% Count notfound responses as successful. - {timeout, pos_integer() | infinity} | %% Timeout for vnode responses - {details, details()} | %% Return extra details as a 3rd element - {details, true} | - details | - {sloppy_quorum, boolean()} | %% default = true - {n_val, pos_integer()} | %% default = bucket props - {crdt_op, true | undefined}. %% default = undefined - +-type option() :: + {r, pos_integer()} | %% Minimum number of successful responses + {pr, non_neg_integer()} | %% Minimum number of primary vnodes participating + {basic_quorum, boolean()} | %% Whether to use basic quorum (return early + %% in some failure cases. + {notfound_ok, boolean()} | %% Count notfound responses as successful. + {timeout, pos_integer() | infinity} | %% Timeout for vnode responses + {details, details()} | %% Return extra details as a 3rd element + {details, true} | + details | + {sloppy_quorum, boolean()} | %% default = true + {n_val, pos_integer()} | %% default = bucket props + {crdt_op, true | undefined} | %% default = undefined + {node_confirms, non_neg_integer()} | + {deletedvclock, boolean()} | + {basic_quorum, boolean()} | + {return_body, boolean()}. + -type options() :: [option()]. -type req_id() :: non_neg_integer(). -type request_type() :: head | get | update. diff --git a/src/riak_kv_pb_object.erl b/src/riak_kv_pb_object.erl index 5ebda25f7..cb12aaebc 100644 --- a/src/riak_kv_pb_object.erl +++ b/src/riak_kv_pb_object.erl @@ -52,10 +52,8 @@ -module(riak_kv_pb_object). -include_lib("riak_pb/include/riak_kv_pb.hrl"). --include_lib("kernel/include/logger.hrl"). -include("riak_kv_capability.hrl"). - -ifdef(TEST). -compile([export_all, nowarn_export_all]). -include_lib("eunit/include/eunit.hrl"). @@ -325,62 +323,18 @@ process( %% Don't return the key since we're not generating one ReturnKey = undefined end, - CondPutMode = - application:get_env(riak_kv, conditional_put_mode, api_only), - MakeTokenRequest = CondPutMode =/= api_only, {CheckResult, CondPutOpts, SessionToken} = - case {IfNotModified, IfNoneMatch, MakeTokenRequest} of - {undefined, undefined, _} -> - {ok, [], none}; - {NotMod, NoneMatch, true} -> - GetOpts = - make_options( - [ - {basic_quorum, true}, - {return_body, false}, - {deleted_vclock, true} - ]), - TokenResult = - riak_kv_token_session:session_request_retry({B, K}), - case TokenResult of - {true, Token} -> - Condition = - case NotMod of - undefined -> - {undefined, true, GetOpts}; - _ -> - InClock = erlify_rpbvc(PbVC), - {{true, InClock}, undefined, GetOpts} - end, - {ok, [{condition_check, Condition}], Token}; - _ -> - ?LOG_WARNING( - "Fallback to weak check as no token available " - "for ~p ~p", - [B, K] - ), - {CheckR, PutOpts} = - riak_kv_put_fsm:conditional_check( - riak_client:get(B, K, GetOpts, C), - {NotMod, erlify_rpbvc(PbVC)}, - NoneMatch - ), - {CheckR, PutOpts, none} - end; - {NotMod, NoneMatch, false} -> - GetOpts = - make_option(n_val, N_val) ++ - make_option(sloppy_quorum, SloppyQuorum) ++ - make_option(timeout, Timeout), - {CheckR, PutOpts} = - riak_kv_put_fsm:conditional_check( - riak_client:get(B, K, GetOpts, C), - {NotMod, erlify_rpbvc(PbVC)}, - NoneMatch - ), - {CheckR, PutOpts, none} - end, + riak_kv_put_core:ready_conditional_check( + IfNotModified, + IfNoneMatch, + undefined, + fun() -> erlify_rpbvc(PbVC) end, + B, + K, + C + ), + case CheckResult of {error, Reason} -> riak_kv_token_session:session_release(SessionToken), diff --git a/src/riak_kv_put_core.erl b/src/riak_kv_put_core.erl index 6c3802147..fdba1ce0c 100644 --- a/src/riak_kv_put_core.erl +++ b/src/riak_kv_put_core.erl @@ -20,10 +20,13 @@ %% %% ------------------------------------------------------------------- -module(riak_kv_put_core). +-export([ready_conditional_check/7]). -export([init/8, add_result/2, enough/1, response/1, final/1, result_shortcode/1, result_idx/1]). -export_type([putcore/0, result/0, reply/0]). +-include_lib("kernel/include/logger.hrl"). + -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). -endif. @@ -62,6 +65,81 @@ }). -opaque putcore() :: #putcore{}. + +%% ==================================================================== +%% Ready Conditional PUT +%% ==================================================================== + +-type get_clock_fun() :: fun(() -> vclock:vclock()). + +-spec ready_conditional_check( + true | undefined, + true | undefined, + list(binary()) | undefined, + get_clock_fun(), + riak_object:bucket(), + riak_object:key(), + riak_client:riak_client() +) -> + { + ok | {error, term()}, + riak_kv_put_fsm:options(), + riak_kv_token_session:session_ref()|none + }. +ready_conditional_check(IfNotModified, IfNoneMatch, IfMatch, ModClockFun, B, K, C) -> + CondPutMode = + application:get_env(riak_kv, conditional_put_mode, api_only), + MakeTokenRequest = CondPutMode =/= api_only, + GetOpts = + [ + {basic_quorum, true}, + {return_body, false}, + {deleted_vclock, true} + ], + case {IfNotModified, IfNoneMatch, IfMatch, MakeTokenRequest} of + {undefined, undefined, undefined, _} -> + {ok, [], none}; + {NotMod, NoneMatch, Match, true} -> + TokenResult = + riak_kv_token_session:session_request_retry({B, K}), + case TokenResult of + {true, Token} -> + Condition = + case {NotMod, IfNoneMatch, Match} of + {undefined, true, undefined} -> + {undefined, true, undefined, GetOpts}; + {true, undefined, undefined} -> + {{true, ModClockFun()}, undefined, undefined, GetOpts}; + {undefined, undefined, MatchL} when MatchL =/= undefined -> + {undefined, undefined, MatchL, GetOpts} + end, + {ok, [{condition_check, Condition}], Token}; + _ -> + ?LOG_WARNING( + "Fallback to weak check as no token available " + "for ~p ~p", + [B, K] + ), + {CheckR, PutOpts} = + riak_kv_put_fsm:conditional_check( + riak_client:get(B, K, GetOpts, C), + {NotMod, ModClockFun()}, + NoneMatch, + IfMatch + ), + {CheckR, PutOpts, none} + end; + {NotMod, NoneMatch, Match, false} -> + {CheckR, PutOpts} = + riak_kv_put_fsm:conditional_check( + riak_client:get(B, K, GetOpts, C), + {NotMod, ModClockFun()}, + NoneMatch, + Match + ), + {CheckR, PutOpts, none} + end. + %% ==================================================================== %% Public API %% ==================================================================== diff --git a/src/riak_kv_put_fsm.erl b/src/riak_kv_put_fsm.erl index 952a45d19..582b02258 100644 --- a/src/riak_kv_put_fsm.erl +++ b/src/riak_kv_put_fsm.erl @@ -50,7 +50,7 @@ waiting_local_vnode/2, waiting_remote_vnode/2, postcommit/2, finish/2]). --export([conditional_check/3]). +-export([conditional_check/4]). -ifdef(TEST). -export([test_link/4]). @@ -69,6 +69,7 @@ -include_lib("kernel/include/logger.hrl"). -include("riak_kv_types.hrl"). -include("riak_kv_capability.hrl"). +-include_lib("riak_kv/include/riak_object.hrl"). -type detail_info() :: timing. -type detail() :: true | @@ -312,7 +313,7 @@ prepare(timeout, State = #state{robj = RObj, options=Options}) -> case ConditionCheck of false -> ok; - {NotMod, NoneMatch, GetOpts} -> + {NotMod, NoneMatch, IfMatch, GetOpts} -> Key = riak_object:key(RObj), {GetCheck, _PutOpts} = riak_kv_put_fsm:conditional_check( @@ -323,7 +324,8 @@ prepare(timeout, State = #state{robj = RObj, options=Options}) -> riak_client:new(node(), condition_check) ), NotMod, - NoneMatch + NoneMatch, + IfMatch ), GetCheck end, @@ -1173,11 +1175,16 @@ get_soft_limit_option(Options) -> -spec conditional_check( {ok, riak_object:riak_object()}|{error, term()}, - {boolean()|undefined, vclock:vclock()}, - boolean()|undefined) -> {ok|{error, term()}, list()}. -conditional_check({ok, _}, _NotMod, NoneMatch) when NoneMatch -> + {true|undefined, vclock:vclock()}, + true|undefined, + list(binary())|undefined +) -> + {ok|{error, term()}, list()}. +conditional_check({ok, _}, _NotMod, true, _IfMatch) -> {{error, "match_found"}, []}; -conditional_check({ok, PreFetchO}, {NotMod, InClock}, _NoneMatch) when NotMod -> +conditional_check({error, notfound}, _NotMod, true, _IfMatch) -> + {ok, [{if_none_match, true}]}; +conditional_check({ok, PreFetchO}, {true, InClock}, _NoneMatch, _IfMatch) -> CurrClock = riak_object:vclock(PreFetchO), case vclock:equal(InClock, CurrClock) of true -> @@ -1185,13 +1192,32 @@ conditional_check({ok, PreFetchO}, {NotMod, InClock}, _NoneMatch) when NotMod -> _ -> {{error, "modified"}, []} end; -conditional_check({error, notfound}, _NotMod, NoneMatch) when NoneMatch -> - {ok, [{if_none_match, true}]}; -conditional_check({error, notfound}, {NotMod, _}, _NoneMatch) when NotMod -> +conditional_check({error, notfound}, {true, _}, _NoneMatch, _IfMatch) -> {{error, "notfound"}, []}; -conditional_check({error, PreFetchError}, _NotMod, _NoneMatch) -> +conditional_check({ok, PreFetch0}, _NotMod, _NoneMatch, IfMatch) when IfMatch =/= undefined -> + VTag = + case riak_object:get_metadatas(PreFetch0) of + [MD] -> + list_to_binary(riak_object:metadata_fetch(?MD_VTAG, MD)); + Sibs when length(Sibs) > 1 -> + riak_kv_web_common:make_clock_etag(riak_object:vclock(PreFetch0)) + end, + case + lists:member( + VTag, + lists:map(fun(IfM) -> string:trim(IfM, both, [$"]) end, IfMatch) + ) of + true -> + {ok, []}; + false -> + {{error, not_matched}, []} + end; +conditional_check({error, notfound}, _NotMod, _NoneMatch, IfMatch) when IfMatch =/= undefined -> + {{error, "modified"}, []}; +conditional_check({error, PreFetchError}, _NotMod, _NoneMatch, _IfMatch) -> {{error, {format, PreFetchError}}, []}. + %% @private the local node is not in the preflist, or is overloaded, %% forward to another node forward(CoordNode, State) -> diff --git a/src/riak_kv_query.erl b/src/riak_kv_query.erl index 7c14d387d..a6f41c609 100644 --- a/src/riak_kv_query.erl +++ b/src/riak_kv_query.erl @@ -185,7 +185,8 @@ query_definition/0, complex_query_definition/0, query_user_input/0, - encoding_fun/0 + encoding_fun/0, + validation_stage/0 ] ). @@ -479,7 +480,7 @@ add_queries(Query, Queries, Subs) -> case Query#riak_kv_query.type of single_query when length(Queries) == 1 -> [SingleQuery] = Queries, - case evaluate_query(single, SingleQuery, Subs) of + case evaluate_single_query(SingleQuery, Subs) of {ok, EvaluatedQuery} -> { ok, @@ -509,10 +510,10 @@ add_queries(Query, Queries, Subs) -> Error end. --spec make_continuation(index_limiter(), riak_object:key()) -> string(). +-spec make_continuation(index_limiter(), riak_object:key()) -> binary(). make_continuation(StartTerm, StartKeyExclusive) -> M = #{?CONT_ST => StartTerm, ?CONT_SK => StartKeyExclusive}, - base64:encode_to_string(iolist_to_binary(riak_kv_wm_json:encode(M))). + base64:encode(iolist_to_binary(riak_kv_wm_json:encode(M))). -spec validate_substitutions(substitutions()) -> ok|validation_error(). validate_substitutions(Subs) -> @@ -547,17 +548,17 @@ evaluate_combo_queries(Queries, Subs) -> evaluate_combo_queries([], _Subs, Acc) -> {ok, lists:reverse(Acc)}; evaluate_combo_queries([Q|Rest], Subs, Acc) -> - case evaluate_query(multi, Q, Subs) of + case evaluate_multi_query(Q, Subs) of {AT, {ok, EvaluatedQuery}} -> evaluate_combo_queries(Rest, Subs, [{AT, EvaluatedQuery}|Acc]); {_AT, Error} -> Error end. -evaluate_query(multi, {AT, IN, ST, ET, RE, EE, FE}, Subs) -> +evaluate_multi_query({AT, IN, ST, ET, RE, EE, FE}, Subs) -> case AT of PI when is_integer(PI), PI >= 0 -> - {AT, evaluate_query(single, {AT, IN, ST, ET, RE, EE, FE}, Subs)}; + {AT, evaluate_single_query({AT, IN, ST, ET, RE, EE, FE}, Subs)}; _ -> {AT, { @@ -566,8 +567,9 @@ evaluate_query(multi, {AT, IN, ST, ET, RE, EE, FE}, Subs) -> <<"Untagged query in combination request">> } } - end; -evaluate_query(single, {_AT, IN, ST, ET, RE, EE, FE}, Subs) -> + end. + +evaluate_single_query({_AT, IN, ST, ET, RE, EE, FE}, Subs) -> IndexL = string:length(IN), case string:slice(IN, IndexL - 4) of <<"_bin">> -> diff --git a/src/riak_kv_query_filebuffer.erl b/src/riak_kv_query_filebuffer.erl index 962589b52..c1c20ba58 100644 --- a/src/riak_kv_query_filebuffer.erl +++ b/src/riak_kv_query_filebuffer.erl @@ -425,7 +425,7 @@ safe_decode(B64Ref) -> riak_kv_query_server:partial_result_map(). encode_results(Results, RspCount, RcvCount, Complete, AccOpt) -> #{ - riak_kv_wm_query:get_result_key(AccOpt) => Results, + riak_kv_ag_query:get_result_key(AccOpt) => Results, ?RSPCOUNT_KEY => RspCount, ?RCVCOUNT_KEY => RcvCount, ?QCOMPLETE_KEY => Complete @@ -497,7 +497,7 @@ fetch_all_tester(MaxCount, AccOpt) -> {ok, QFB, QFR} = unlink_new(RootPath, 2000, Bucket, AccOpt), ExpectedInitResult = #{ - riak_kv_wm_query:get_result_key(AccOpt) => [], + riak_kv_ag_query:get_result_key(AccOpt) => [], ?RSPCOUNT_KEY => 0, ?RCVCOUNT_KEY => 0, ?QCOMPLETE_KEY => false @@ -513,7 +513,7 @@ fetch_all_tester(MaxCount, AccOpt) -> send_keys_in_batches(BatchSize, InitialKeys, QFB), ExpectedEmptyResult = #{ - riak_kv_wm_query:get_result_key(AccOpt) => [], + riak_kv_ag_query:get_result_key(AccOpt) => [], ?RSPCOUNT_KEY => 0, ?RCVCOUNT_KEY => MaxCount div 2, ?QCOMPLETE_KEY => false @@ -525,7 +525,7 @@ fetch_all_tester(MaxCount, AccOpt) -> ?assertMatch(InitialKeys, InitialRsp), ExpectedHWResult = #{ - riak_kv_wm_query:get_result_key(AccOpt) => [], + riak_kv_ag_query:get_result_key(AccOpt) => [], ?RSPCOUNT_KEY => MaxCount div 2, ?RCVCOUNT_KEY => MaxCount div 2, ?QCOMPLETE_KEY => false @@ -548,7 +548,7 @@ fetch_all_tester(MaxCount, AccOpt) -> ?assertMatch(AllKeys, TotalRsp), ExpectedFinalResult = #{ - riak_kv_wm_query:get_result_key(AccOpt) => [], + riak_kv_ag_query:get_result_key(AccOpt) => [], ?RSPCOUNT_KEY => MaxCount, ?RCVCOUNT_KEY => MaxCount, ?QCOMPLETE_KEY => true @@ -585,7 +585,7 @@ fetch_keys_in_batches(MaxResults, Acc, QFR, Bucket, AccOpt) -> UpdAcc = lists:reverse( maps:get( - riak_kv_wm_query:get_result_key(AccOpt), + riak_kv_ag_query:get_result_key(AccOpt), ResultMap ) ) ++ Acc, diff --git a/src/riak_kv_status.erl b/src/riak_kv_status.erl index c26e405e7..04efc901b 100644 --- a/src/riak_kv_status.erl +++ b/src/riak_kv_status.erl @@ -151,6 +151,18 @@ get_exometer_values(Entry, DPmap) -> end. expand_disk_stats([{disk, Stats}]) -> - [{disk, [{struct, [{id, list_to_binary(Id)}, {size, Size}, {used, Used}]} - || {Id, Size, Used} <- Stats]}]. - + [ + { + disk, + lists:map( + fun({Id, Size, Used}) -> + #{ + id => list_to_binary(Id), + size => Size, + used => Used + } + end, + Stats + ) + } + ]. diff --git a/src/riak_kv_test_util.erl b/src/riak_kv_test_util.erl index 82b670054..3004754b9 100644 --- a/src/riak_kv_test_util.erl +++ b/src/riak_kv_test_util.erl @@ -296,10 +296,10 @@ dep_apps(Test, Extra) -> ok end, [ - Silencer, exometer_core, runtime_tools, - mochiweb, webmachine, sidejob, poolboy, basho_stats, bitcask, - eleveldb, riak_core, riak_api, riak_dt, riak_pb, - riak_kv, DefaultSetupFun, Extra + sasl, Silencer, exometer_core, runtime_tools, + sidejob, poolboy, basho_stats, bitcask, + eleveldb, riak_core, riak_api, riak_dt, riak_pb, riak_kv, + DefaultSetupFun, Extra ]. diff --git a/src/riak_kv_tictacaae_cli.erl b/src/riak_kv_tictacaae_cli.erl index d9cb24204..516530ff4 100644 --- a/src/riak_kv_tictacaae_cli.erl +++ b/src/riak_kv_tictacaae_cli.erl @@ -493,47 +493,9 @@ treestatus_usage() -> ]. treestatus_cmd([_, _, _], _, Options) -> - Report = get_aae_progress_report(), + Report = riak_kv_tictacaae_report:produce(), print_aae_progress_report(Report, Options). -get_aae_progress_report() -> - VVSS = - lists:append( - [case sys:get_state(P) of - {active, _CoreVnodeState = - {state, Idx, riak_kv_vnode, VSx, _, _, _, _, _, _, _, _}} -> - [{Idx, VSx}]; - _ -> - [] - end || {_, P, _, _} <- supervisor:which_children(riak_core_vnode_sup)]), - [begin - AAECntrl = riak_kv_vnode:aae_controller(VNState), - TictacRebuilding = riak_kv_vnode:aae_rebuilding(VNState), - InProgress = TictacRebuilding /= false, - AAEReport = aae_controller:aae_produce_progress_report(AAECntrl), - IsEmpty = proplists:get_value(is_empty, AAEReport), - LastRebuild = proplists:get_value(last_rebuild, AAEReport), - NextRebuild = proplists:get_value(next_rebuild, AAEReport), - Status = - case {IsEmpty, LastRebuild, InProgress, NextRebuild} of - {true, _, _, _} -> - empty; - {_, never, false, Scheduled} when Scheduled /= undefined -> - partial; - {_, Built, false, _} when Built /= never -> - built; - {_, Built, true, _} when Built /= never -> - rebuilding; - {_, never, true, _} -> - building - end, - Extra = [{status, Status}, - {partition, Idx}, - {controller_pid, list_to_binary(pid_to_list(AAECntrl))} - ], - AAEReport ++ Extra - end || {Idx, VNState} <- VVSS]. - print_aae_progress_report(Report, Options) -> Show_ = case proplists:get_all_values(show, Options) of diff --git a/src/riak_kv_tictacaae_report.erl b/src/riak_kv_tictacaae_report.erl new file mode 100644 index 000000000..f4330d959 --- /dev/null +++ b/src/riak_kv_tictacaae_report.erl @@ -0,0 +1,65 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2026 TI Tokyo. All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- + +-module(riak_kv_tictacaae_report). + +-export([produce/0]). + +%% The report is collected in both cli and ag (Ag, argentum, silver, +%% SilverMachine, aha!) handlers, so let's isolate it into a module of +%% its own +-spec produce() -> proplists:proplist(). +produce() -> + VVSS = + lists:append( + [case sys:get_state(P) of + {active, _CoreVnodeState = + {state, Idx, riak_kv_vnode, VSx, _, _, _, _, _, _, _, _}} -> + [{Idx, VSx}]; + _ -> + [] + end || {_, P, _, _} <- supervisor:which_children(riak_core_vnode_sup)]), + [begin + AAECntrl = riak_kv_vnode:aae_controller(VNState), + TictacRebuilding = riak_kv_vnode:aae_rebuilding(VNState), + InProgress = TictacRebuilding /= false, + AAEReport = aae_controller:aae_produce_progress_report(AAECntrl), + IsEmpty = proplists:get_value(is_empty, AAEReport), + LastRebuild = proplists:get_value(last_rebuild, AAEReport), + NextRebuild = proplists:get_value(next_rebuild, AAEReport), + Status = + case {IsEmpty, LastRebuild, InProgress, NextRebuild} of + {true, _, _, _} -> + empty; + {_, never, false, Scheduled} when Scheduled /= undefined -> + partial; + {_, Built, false, _} when Built /= never -> + built; + {_, Built, true, _} when Built /= never -> + rebuilding; + {_, never, true, _} -> + building + end, + Extra = [{status, Status}, + {partition, Idx}, + {controller_pid, list_to_binary(pid_to_list(AAECntrl))} + ], + lists:keydelete(is_empty, 1, AAEReport ++ Extra) + end || {Idx, VNState} <- VVSS]. diff --git a/src/riak_kv_util.erl b/src/riak_kv_util.erl index 3d6d48a43..6d25ddd67 100644 --- a/src/riak_kv_util.erl +++ b/src/riak_kv_util.erl @@ -52,7 +52,13 @@ shuffle_list/1, kv_ready/0, ngr_initial_timeout/0, - sys_monitor_count/0 + sys_monitor_count/0, + node_info_for_riak_control/0, + system_info/0, + collect_all_app_env/0, + apply_app_env/1, + get_advanced_config/0, + write_advanced_config/1 ]). -export([report_hashtree_tokens/0, reset_hashtree_tokens/2]). -export([reset_aae_key_filter/0]). @@ -752,6 +758,124 @@ sys_monitor_count() -> ). + + +%% @doc Return current nodes information, to be sent to riak_control +%% over http (see riak_kv_wm_cluster) +-spec node_info_for_riak_control() -> proplists:proplist(). +node_info_for_riak_control() -> + {Total, Used} = node_memory_usage(), + Handoffs = node_handoff_status(), + VNodes = riak_core_vnode_manager:all_vnodes(), + ErlangMemory = proplists:get_value(total,erlang:memory()), + [{reachable, true}, + {mem_total, Total}, + {mem_used, Used}, + {mem_erlang, ErlangMemory}, + {vnodes, VNodes}, + {handoffs, Handoffs}, + {system_info, system_info()} + ]. + +node_memory_usage() -> + Mem = memsup:get_system_memory_data(), + Total = proplists:get_value(total_memory, Mem), + Free = proplists:get_value(free_memory, Mem), + Buffered = + case lists:keyfind(buffered_memory, 1, Mem) of + {_, BufferedMem} -> BufferedMem; + false -> 0 + end, + Cached = + case lists:keyfind(cached_memory, 1, Mem) of + {_, CachedMem} -> CachedMem; + false -> 0 + end, + {Total, Total - (Free + Cached + Buffered)}. + +format_transfer({status_v2, Handoff}) -> + Mod = proplists:get_value(mod, Handoff), + SrcPartition = proplists:get_value(src_partition, Handoff), + SrcNode = proplists:get_value(src_node, Handoff), + {Mod, SrcPartition, SrcNode}. + +node_handoff_status() -> + Transfers = riak_core_handoff_manager:status({direction, outbound}), + [format_transfer(T) || T <- lists:flatten(Transfers)]. + +system_info() -> + {MS, _} = erlang:statistics(wall_clock), + St = MS div 1000, + S = St rem 60, + Mt = St div 60, + M = Mt rem 60, + Ht = Mt div 60, + H = Ht rem 24, + Dt = Ht div 24, + D = Dt, + Str = case {D, H, M} of + {A, _, _} when A > 0 -> io_lib:format("~b day~s, ~b hour~s, ~b minute~s, ~b sec", [D, s(D), H, s(H), M, s(M), S]); + {_, A, _} when A > 0 -> io_lib:format("~b hour~s, ~b minute~s, ~b sec", [H, s(H), M, s(M), S]); + {_, _, A} when A > 0 -> io_lib:format("~b minute~s, ~b sec", [M, s(M), S]); + _ -> io_lib:format("~b sec", [S]) + end, + #{riak_version => list_to_binary(riak_version()), + system_version => list_to_binary(lists:droplast(erlang:system_info(system_version))), + nodename => node(), + uptime => St, + uptime_str => iolist_to_binary(Str) + }. + +s(1) -> ""; +s(_) -> "s". + +riak_version() -> + element(2, lists:keyfind("riak", 1, release_handler:which_releases())). + +-spec collect_all_app_env() -> proplists:proplist(). +collect_all_app_env() -> + Apps = [A || {A, _, _} <- application:which_applications()], + lists:filter( + fun({_, E}) -> E /= [] end, + [{App, application:get_all_env(App)} || App <- Apps] + ). + +-spec apply_app_env(proplists:proplist()) -> ok. +apply_app_env(AppEE) -> + lists:map( + fun({App, EE}) -> + [application:set_env(App, K, V) || {K, V} <- EE] + end, + AppEE), + ok. + +-spec get_advanced_config() -> + {ok, proplists:proplist()} | {error, bad_config | file:posix() | badarg | terminated | system_limit}. +get_advanced_config() -> + case file:consult(which_advanced_config()) of + {error, {_Line, _Mod, _Term} = FE} -> + ?LOG_WARNING("advanced.config is not consultable: ~s", [file:format_error(FE)]), + {error, bad_config}; + {ok, [Config]} -> + {ok, Config}; + Other -> + Other + end. + +-spec write_advanced_config(iolist()|binary()) -> + ok | {error, file:posix() | badarg | terminated | system_limit}. +write_advanced_config(Blob) -> + file:write_file(which_advanced_config(), Blob). + +which_advanced_config() -> + case os:getenv("USER") of + "riak" -> + "/etc/riak/advanced.config"; + _ -> + "etc/advanced.config" + end. + + %% =================================================================== %% EUnit tests %% =================================================================== diff --git a/src/riak_kv_web.erl b/src/riak_kv_web.erl deleted file mode 100644 index 08dcc24db..000000000 --- a/src/riak_kv_web.erl +++ /dev/null @@ -1,190 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% riak_kv_web: setup Riak's KV HTTP interface -%% -%% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - -%% @doc Convenience functions for setting up the HTTP interface -%% of Riak. This module loads parameters from the application -%% environment: -%% -%%
raw_name -%%
the base path under which the riak_kv_wm_raw -%% should be exposed; defaulted to "raw" -%%
--module(riak_kv_web). - --export([dispatch_table/0]). --include("riak_kv_wm_raw.hrl"). --include("riak_kv_types.hrl"). - -dispatch_table() -> - StatsProps = stats_props(), - lists:append( - raw_dispatch(), - [ - { - [proplists:get_value(prefix, StatsProps)], - riak_kv_wm_stats, StatsProps - }, - {["ping"], riak_kv_wm_ping, []} - ] -). - -raw_dispatch() -> - case app_helper:get_env(riak_kv, raw_name) of - undefined -> raw_dispatch("riak"); - Name -> lists:append(raw_dispatch(Name), raw_dispatch("riak")) - end. - -raw_dispatch(Name) -> - APIv2Props = [{bucket_type, <<"default">>}, {api_version, 2}|raw_props(Name)], - Props1 = [{bucket_type, <<"default">>}, {api_version, 1}|raw_props(Name)], - Props2 = [ - {["types", bucket_type], [{api_version, 3} | raw_props(Name)]}, - {[], APIv2Props} - ], - - [ - %% OLD API, remove in v2.2 - {[Name], - riak_kv_wm_buckets, Props1}, - - {[Name, bucket], fun is_post/1, - riak_kv_wm_object, Props1}, - - {[Name, bucket], fun is_props/1, - riak_kv_wm_props, Props1}, - - {[Name, bucket], fun is_keylist/1, - riak_kv_wm_keylist, [{allow_props_param, true}|Props1]}, - - {[Name, bucket, key], - riak_kv_wm_object, Props1}, - - {[Name, bucket, key, '*'], - riak_kv_wm_link_walker, Props1} - - ] ++ - - [ {["types", bucket_type, "props"], riak_kv_wm_bucket_type, - [{api_version, 3}|raw_props(Name)]}, - {["types", bucket_type, "buckets", bucket, "datatypes"], fun is_post/1, - riak_kv_wm_crdt, [{api_version, 3}]}, - {["types", bucket_type, "buckets", bucket, "datatypes", key], - riak_kv_wm_crdt, [{api_version, 3}]}] - - ++ - - lists:flatten([ - [ - %% NEW API - {Prefix ++ ["buckets"], - riak_kv_wm_buckets, Props}, - - {Prefix ++ ["buckets", bucket, "props"], - riak_kv_wm_props, Props}, - - {Prefix ++ ["buckets", bucket, "keys"], fun is_post/1, - riak_kv_wm_object, Props}, - - {Prefix ++ ["buckets", bucket, "keys"], - riak_kv_wm_keylist, Props}, - - {Prefix ++ ["buckets", bucket, "keys", key], - riak_kv_wm_object, Props}, - - {Prefix ++ ["buckets", bucket, "keys", key, '*'], - riak_kv_wm_link_walker, Props}, - - {Prefix ++ ["buckets", bucket, "index", field, '*'], - riak_kv_wm_index, Props}, - - {Prefix ++ ["buckets", bucket, "query"], - riak_kv_wm_query, Props}, - - %% AAE fold URLs - {["cachedtrees", "nvals", nval, "root"], - riak_kv_wm_aaefold, Props}, - - {["cachedtrees", "nvals", nval, "branch"], - riak_kv_wm_aaefold, Props}, - - {["cachedtrees", "nvals", nval, "keysclocks"], - riak_kv_wm_aaefold, Props}, - - {["rangetrees"] ++ Prefix ++ ["buckets", bucket, "trees", size], - riak_kv_wm_aaefold, Props}, - - {["rangetrees"] ++ Prefix ++ ["buckets", bucket, "keysclocks"], - riak_kv_wm_aaefold, Props}, - - {["rangerepl"] ++ Prefix ++ ["buckets", bucket, "queuename", queuename], - riak_kv_wm_aaefold, Props}, - - {["rangerepair"] ++ Prefix ++ ["buckets", bucket], - riak_kv_wm_aaefold, Props}, - - {["siblings"] ++ Prefix ++ ["buckets", bucket, "counts", count], - riak_kv_wm_aaefold, Props}, - - {["objectsizes"] ++ Prefix ++ ["buckets", bucket, "sizes", size], - riak_kv_wm_aaefold, Props}, - - {["objectstats"] ++ Prefix ++ ["buckets", bucket], - riak_kv_wm_aaefold, Props}, - - {["tombs"] ++ Prefix ++ ["buckets", bucket], - riak_kv_wm_aaefold, Props}, - - {["reap"] ++ Prefix ++ ["buckets", bucket], - riak_kv_wm_aaefold, Props}, - - {["erase"] ++ Prefix ++ ["buckets", bucket], - riak_kv_wm_aaefold, Props}, - - {["aaebucketlist"], - riak_kv_wm_aaefold, Props}, - - - %% Repl queue fetch URL - - {["queuename", queuename], - riak_kv_wm_queue, Props}, - - {["membership_request"], - riak_kv_wm_queue, Props} - - ] || {Prefix, Props} <- Props2 ]). - -is_post(Req) -> - wrq:method(Req) == 'POST'. - -is_props(Req) -> - (wrq:get_qs_value(?Q_PROPS, Req) /= ?Q_FALSE) andalso (not is_keylist(Req)). - -is_keylist(Req) -> - X = wrq:get_qs_value(?Q_KEYS, Req), - (X == ?Q_STREAM) orelse (X == ?Q_TRUE). - -raw_props(Prefix) -> - [{prefix, Prefix}, {riak, local}]. - -stats_props() -> - [{prefix, app_helper:get_env(riak_kv, stats_urlpath, "stats")}]. diff --git a/src/riak_kv_web_common.erl b/src/riak_kv_web_common.erl new file mode 100644 index 000000000..303221527 --- /dev/null +++ b/src/riak_kv_web_common.erl @@ -0,0 +1,697 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2026 Martin Sumner +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +%% @doc common functions used by web API handlers + +-module(riak_kv_web_common). + +-include_lib("kernel/include/logger.hrl"). +-include("riak_kv_web.hrl"). + +-export( + [ + check_permissions/5, + set_bucket/2, + check_type_exists/1, + count_fold/3, + boolean_fold/3, + decode_clock/1, + normalise_boolean_param/1, + type_preference/2, + type_match/2, + add_routes/0, + get_version_vector/1, + get_timeout/1, + filter_options/1, + make_clock_etag/1, + compile_splitters/0, + check_queuename/1, + accept_json_only/1 + ] +). + +-define(BAD_COUNT_PARAM_TEXT, << + "~s query parameter must be an integer or " + "one of the following words: 'one', 'quorum' or 'all'" +>>). + +-define(BAD_BOOLEAN_PARAM_TEXT, << + "~s query parameter must be true or false" +>>). + +-spec filter_options(#{atom() => term()}) -> list({atom(), term()}). +filter_options(Options) -> + maps:to_list( + maps:filter( + fun(_K, V) -> V =/= undefined andalso V =/= default end, + Options + ) + ). + +-spec add_routes() -> ok. +add_routes() -> + Routes = + [ + {5, riak_kv_ag_object_read}, + {10, riak_kv_ag_object_store}, + {12, riak_kv_ag_queue}, + {15, riak_kv_ag_object_delete}, + {20, riak_kv_ag_query}, + {30, riak_kv_ag_index}, + {40, riak_kv_ag_crdt}, + {60, riak_kv_ag_stats}, + {80, riak_kv_ag_aaefold}, + {85, riak_kv_ag_keylist}, + {86, riak_kv_ag_bucketlist}, + {95, riak_kv_ag_ping}, + {98, riak_kv_ag_bprops} + ], + riak_api_web:add_routes(Routes). + +-spec check_permissions( + riak_api_web_headers:headers(), + riak_api_web_socket:scheme(), + riak_api_web_handler:peer_ip(), + riak_object:bucket() | undefined, + string() | undefined +) -> + true | riak_api_web_acceptor:halt_response(). +check_permissions(ReqHeaders, Scheme, Peer, Bucket, GrantSought) when + is_binary(Bucket) +-> + check_permissions( + ReqHeaders, Scheme, Peer, {<<"default">>, Bucket}, GrantSought + ); +check_permissions(ReqHeaders, Scheme, Peer, Bucket, GrantSought) -> + Authorised = + riak_api_web_security:is_authorised( + riak_core_security:is_enabled(), + Scheme, + ReqHeaders, + Peer + ), + case Authorised of + {ok, undefined} -> + true; + {ok, _SecContext} when GrantSought == undefined -> + true; + {ok, SecContext} -> + PermissionGranted = + riak_core_security:check_permission( + {GrantSought, Bucket}, + SecContext + ), + case PermissionGranted of + {true, _SecContext} -> + true; + {false, Error, _SecContext} when is_binary(Error) -> + {halt, 403, [?TXT_HEADER], Error, []} + end; + {halt, RC, RH, RB, RS} -> + {halt, RC, RH, RB, RS} + end. + +-spec accept_json_only( + riak_api_web_headers:headers() +) -> + ok | riak_api_web_acceptor:halt_response(). +accept_json_only(ReqHeaders) -> + AcceptedTypes = + case riak_api_web_headers:get_value('Accept', ReqHeaders) of + undefined -> + [<<"application/json">>]; + SingleType when is_binary(SingleType) -> + [SingleType]; + MultipleTypes when is_list(MultipleTypes) -> + MultipleTypes + end, + case type_match(<<"application/json">>, AcceptedTypes) of + true -> + ok; + false -> + { + halt, + 406, + [?TXT_HEADER], + <<"application/json must be accepted">>, + [] + } + end. + +-spec set_bucket(binary(), binary()) -> riak_object:bucket(). +set_bucket(<<"default">>, Bucket) -> + Bucket; +set_bucket(BucketType, Bucket) -> + {BucketType, Bucket}. + +-spec check_type_exists( + riak_object:bucket() +) -> + ok | riak_api_web_acceptor:halt_response(). +check_type_exists({Type, _Bucket}) -> + case riak_core_bucket_type:get(Type) of + undefined -> + % Not that this fetches the properties for the type from + % the metadata - which may be an unnecessary cost. + {halt, 404, [], <<"Unknown bucket type: ~0p">>, [Type]}; + _ -> + ok + end; +check_type_exists(Bucket) when is_binary(Bucket) -> + ok. + +-spec count_fold( + riak_api_web_handler:query_params(), + list(binary()), + #{atom() => term()} +) -> + #{atom() => term()} | riak_api_web_acceptor:halt_response(). +count_fold([], _CountKeys, Opts) -> + Opts; +count_fold([{PK, PV} | Rest], CountKeys, Opts) when is_map(Opts) -> + case lists:member(PK, CountKeys) of + false -> + count_fold(Rest, CountKeys, Opts); + true -> + case normalise_rw_param(PV) of + bad_param -> + {halt, 400, [?TXT_HEADER], ?BAD_COUNT_PARAM_TEXT, [PK]}; + V -> + count_fold( + Rest, + CountKeys, + maps:put(binary_to_existing_atom(PK), V, Opts) + ) + end + end. + +-spec boolean_fold( + riak_api_web_handler:query_params(), + list(binary()), + #{atom() => term()} +) -> + #{atom() => term()} | riak_api_web_acceptor:halt_response(). +boolean_fold([], _BoolKeys, Opts) -> + Opts; +boolean_fold([{PK, PV} | Rest], BoolKeys, Opts) when is_map(Opts) -> + case lists:member(PK, BoolKeys) of + false -> + boolean_fold(Rest, BoolKeys, Opts); + true -> + case normalise_boolean_param(PV) of + bad_param -> + {halt, 400, [?TXT_HEADER], ?BAD_BOOLEAN_PARAM_TEXT, [PK]}; + V -> + boolean_fold( + Rest, + BoolKeys, + maps:put(binary_to_existing_atom(PK), V, Opts) + ) + end + end. + +normalise_rw_param(<<"default">>) -> + default; +normalise_rw_param(<<"one">>) -> + 1; +normalise_rw_param(<<"quorum">>) -> + quorum; +normalise_rw_param(<<"all">>) -> + all; +normalise_rw_param(V) when is_binary(V) -> + try + case binary_to_integer(V) of + I when I >= 0 -> + I + end + catch + _:_ -> + bad_param + end; +normalise_rw_param(_) -> + bad_param. + +normalise_boolean_param(true) -> + true; +normalise_boolean_param(V) when is_binary(V) -> + case string:casefold(V) of + <<"true">> -> + true; + <<"false">> -> + false; + <<"default">> -> + default; + _ -> + bad_param + end. + +-spec decode_clock(unicode:chardata()) -> vclock:vclock() | error. +decode_clock(EncodedClock) -> + try + riak_object:decode_vclock(base64:decode(EncodedClock)) + catch + _:Error -> + ?LOG_WARNING( + "Unexpected error decoding clock ~0p", + [Error] + ), + error + end. + +-spec type_preference( + binary(), + list(binary()) | binary() +) -> + {boolean(), float()}. +type_preference(ContentType, AcceptedTypes) -> + AcceptedTypeList = + case is_list(AcceptedTypes) of + true -> + AcceptedTypes; + false -> + [AcceptedTypes] + end, + type_preference(split_type(ContentType), AcceptedTypeList, false, 0.0). + +type_preference(error, _AcceptedTypes, Match, BestQ) -> + {Match, BestQ}; +type_preference(_, [], Match, BestQ) -> + {Match, BestQ}; +type_preference({PMT, SMT}, [ThisType | Rest], Match, BestQ) -> + {TypeInfo, MaybeQ} = + case binary:split(ThisType, get_subsplitter()) of + [PlainType] when is_binary(PlainType) -> + {PlainType, <<>>}; + [PlainType, Suffix] when is_binary(Suffix) -> + {PlainType, Suffix} + end, + case match_type(TypeInfo, {PMT, SMT}) of + true -> + QV = + case string:trim(MaybeQ, both) of + <<"q=", BF/binary>> -> + try + binary_to_float(BF) + catch + _:_ -> + +0.0 + end; + _ -> + 1.0 + end, + type_preference({PMT, SMT}, Rest, true, max(QV, BestQ)); + false -> + type_preference({PMT, SMT}, Rest, Match, BestQ) + end. + +-spec type_match( + binary() | {binary(), binary()} | error, + list(binary()) | binary() +) -> + boolean() | error. +type_match(CType, AcceptedTypes) when is_binary(AcceptedTypes) -> + type_match(CType, [AcceptedTypes]); +type_match(CType, AcceptedTypes) when is_binary(CType) -> + type_match(split_type(CType), AcceptedTypes); +type_match({_PMT, _SMT}, []) -> + false; +type_match(error, _AcceptedTypes) -> + error; +type_match({PMT, SMT}, [AcceptedType | Rest]) -> + case match_type(AcceptedType, {PMT, SMT}) of + true -> + true; + false -> + type_match({PMT, SMT}, Rest) + end. + +match_type(AcceptedType, {PMT, SMT}) -> + case {split_type(AcceptedType), {PMT, SMT}} of + {{PMT, SMT}, {PMT, SMT}} -> + true; + {{PMT, <<"*">>}, {PMT, _}} -> + true; + {{<<"*">>, <<"*">>}, _} -> + true; + _ -> + false + end. + +%% @doc Call this function when initialising API +-spec compile_splitters() -> ok. +compile_splitters() -> + CP = binary:compile_pattern([<<";">>, <<" ;">>]), + persistent_term:put({?MODULE, compile_patterns}, CP). + +-spec get_subsplitter() -> list(binary()) | binary:cp(). +get_subsplitter() -> + persistent_term:get({?MODULE, compile_patterns}, [<<";">>, <<" ;">>]). + +-spec split_type(binary()) -> {binary(), binary()} | error. +split_type(BinType) -> + [PrimaryTypeInfo | _Rest] = binary:split(BinType, get_subsplitter()), + case binary:split(PrimaryTypeInfo, <<"/">>) of + [Type, SubType] when is_binary(Type), is_binary(SubType) -> + {Type, SubType}; + _NotSplitAsExpected -> + error + end. + +-spec get_version_vector( + riak_api_web_headers:headers() +) -> + {ok, vclock:vclock()} | {ok, none} | riak_api_web_acceptor:halt_response(). +get_version_vector(ReqHeaders) -> + ClockHeader = + riak_api_web_headers:lookup(?HEAD_VCLOCK_CASEFOLD, ReqHeaders, true), + case ClockHeader of + undefined -> + {ok, none}; + {_OrigKey, [EncodedClock]} -> + case riak_kv_web_common:decode_clock(EncodedClock) of + error -> + ErrorRsp = + << + "Error decoding vector clock in " + "x-riak-vclock header" + >>, + {halt, 400, [?TXT_HEADER], ErrorRsp, []}; + DecodedClock -> + {ok, DecodedClock} + end; + {_OrigKey, _MultipleClocks} -> + ErrorRsp = + << + "Only one x-riak-vclock may be specified" + >>, + {halt, 400, [?TXT_HEADER], ErrorRsp, []} + end. + +-spec get_timeout( + riak_api_web_handler:query_params() +) -> + {ok, non_neg_integer()} + | {ok, none} + | riak_api_web_acceptor:halt_response(). +get_timeout(Params) -> + case lists:keyfind(<<"timeout">>, 1, Params) of + false -> + {ok, none}; + {<<"timeout">>, TO} when is_binary(TO) -> + try + IntTO = binary_to_integer(TO), + true = IntTO > 0, + {ok, IntTO} + catch + _:_ -> + ErrMsg = <<"Bad timeout value ~0p">>, + {halt, 400, [?TXT_HEADER], ErrMsg, [TO]} + end + end. + +-spec make_clock_etag(vclock:vclock()) -> binary(). +make_clock_etag(Vclock) -> + <> = crypto:hash(md5, term_to_binary(Vclock)), + list_to_binary(riak_core_util:integer_to_list(ETag, 62)). + +-spec check_queuename(binary()) -> atom(). +check_queuename(Queue) -> + try + binary_to_existing_atom(Queue) + catch + _:_ -> + undefined + end. + +%% =================================================================== +%% EUnit tests +%% =================================================================== + +-ifdef(TEST). + +-include_lib("eunit/include/eunit.hrl"). + +split_path(RequestLine) -> + {ok, {http_request, Method, {abs_path, Path}, _Version}, _Rest} = + erlang:decode_packet(http_bin, RequestLine, []), + URIMap = uri_string:normalize(Path, [return_map]), + NormalisedPath = maps:get(path, URIMap, <<"">>), + case string:split(NormalisedPath, <<"/">>, all) of + [<<>> | Rest] -> + {ok, Method, Rest, NormalisedPath}; + PathList when is_list(PathList) -> + {ok, Method, PathList, NormalisedPath} + end. + +check_path(RequestLine) -> + {ok, Method, SplitPath, AbsPath} = split_path(RequestLine), + riak_api_web:get_route(8000, Method, AbsPath, SplitPath). + +routing_test() -> + add_routes(), + ?assertMatch( + {ok, riak_kv_ag_object_read, _, _}, + check_path(<<"GET /types/T/buckets/B/keys/K HTTP/1.1\r\n">>) + ), + ?assertMatch( + {ok, riak_kv_ag_object_read, _, _}, + check_path(<<"HEAD /types/T/buckets/B/keys/K HTTP/1.1\r\n">>) + ), + ?assertMatch( + {ok, riak_kv_ag_object_read, _, _}, + check_path(<<"GET /buckets/B/keys/K HTTP/1.1\r\n">>) + ), + ?assertMatch( + {halt, 405, [{'Allow', <<"DELETE, GET, HEAD, POST, PUT">>}], <<>>, []}, + check_path(<<"OPTIONS /types/T/buckets/B/keys/K HTTP/1.1\r\n">>) + ), + ?assertMatch( + {ok, riak_kv_ag_object_store, _, _}, + check_path(<<"POST /types/T/buckets/B/keys HTTP/1.1\r\n">>) + ), + ?assertMatch( + {halt, 405, [{'Allow', _}], <<>>, []}, + check_path(<<"PUT /types/T/buckets/B/keys HTTP/1.1\r\n">>) + ), + ?assertMatch( + {ok, riak_kv_ag_object_store, _, _}, + check_path(<<"POST /buckets/B/keys HTTP/1.1\r\n">>) + ), + ?assertMatch( + {ok, riak_kv_ag_object_delete, _, _}, + check_path(<<"DELETE /types/T/buckets/B/keys/K HTTP/1.1\r\n">>) + ), + ?assertMatch( + {ok, riak_kv_ag_object_delete, _, _}, + check_path(<<"DELETE /buckets/B/keys/K HTTP/1.1\r\n">>) + ), + ?assertMatch( + {ok, riak_kv_ag_index, _, _}, + check_path( + <<"GET /types/T/buckets/B/index/field1_bin/exact HTTP/1.1\r\n">> + ) + ), + ?assertMatch( + {ok, riak_kv_ag_index, _, _}, + check_path( + <<"GET /types/T/buckets/B/index/field1_bin/s/e HTTP/1.1\r\n">> + ) + ), + ?assertMatch( + {ok, riak_kv_ag_index, _, _}, + check_path( + <<"GET /buckets/B/index/field1_bin/exact HTTP/1.1\r\n">> + ) + ), + ?assertMatch( + {ok, riak_kv_ag_index, _, _}, + check_path( + <<"GET /buckets/B/index/field1_bin/s/e HTTP/1.1\r\n">> + ) + ), + ?assertMatch( + {ok, riak_kv_ag_index, _, _}, + check_path( + <<"GET /buckets/B/index/field1_int/1/10 HTTP/1.1\r\n">> + ) + ), + ?assertMatch( + {halt, 405, [{'Allow', <<"GET">>}], <<>>, []}, + check_path( + <<"HEAD /types/T/buckets/B/index/field1_bin/s/e HTTP/1.1\r\n">> + ) + ), + ?assertMatch( + {ok, riak_kv_ag_stats, _, _}, + check_path( + <<"GET /stats HTTP/1.1\r\n">> + ) + ), + ?assertMatch( + {halt, 404, [], <<>>, []}, + check_path( + <<"GET /other HTTP/1.1\r\n">> + ) + ), + ?assertMatch( + {ok, riak_kv_ag_query, _, _}, + check_path(<<"GET /buckets/B/query HTTP/1.1\r\n">>) + ), + ?assertMatch( + {ok, riak_kv_ag_query, _, _}, + check_path(<<"POST /buckets/B/query HTTP/1.1\r\n">>) + ), + ?assertMatch( + {ok, riak_kv_ag_query, _, _}, + check_path(<<"GET /types/T/buckets/B/query HTTP/1.1\r\n">>) + ), + ?assertMatch( + {halt, 405, [{'Allow', <<"GET, POST">>}], <<>>, []}, + check_path(<<"PUT /buckets/B/query HTTP/1.1\r\n">>) + ), + ?assertMatch( + {ok, riak_kv_ag_queue, _, _}, + check_path((<<"GET /queuename/queue HTTP/1.1\r\n">>)) + ), + ?assertMatch( + {ok, riak_kv_ag_queue, _, _}, + check_path((<<"POST /queuename/queue HTTP/1.1\r\n">>)) + ), + ?assertMatch( + {halt, 404, [], <<>>, []}, + check_path((<<"POST /queuename/notanexistingatom HTTP/1.1\r\n">>)) + ), + ?assertMatch( + {ok, riak_kv_ag_queue, _, _}, + check_path((<<"GET /membership_request HTTP/1.1\r\n">>)) + ), + ?assertMatch( + {halt, 405, [{'Allow', <<"GET">>}], <<>>, []}, + check_path(<<"POST /membership_request HTTP/1.1\r\n">>) + ), + ?assertMatch( + {halt, 405, [{'Allow', <<"GET, POST">>}], <<>>, []}, + check_path((<<"PUT /queuename/queue HTTP/1.1\r\n">>)) + ), + ?assertMatch( + {ok, riak_kv_ag_keylist, _, _}, + check_path((<<"GET /types/T/buckets/B/keys?keys=true HTTP/1.1\r\n">>)) + ), + ?assertMatch( + {ok, riak_kv_ag_keylist, _, _}, + check_path((<<"GET /buckets/B/keys?keys=true HTTP/1.1\r\n">>)) + ), + ?assertMatch( + {halt, 405, [{'Allow', <<"GET, POST">>}], <<>>, []}, + check_path(<<"PUT /buckets/B/keys?keys=true HTTP/1.1\r\n">>) + ), + ?assertMatch( + {ok, riak_kv_ag_bucketlist, _, _}, + check_path((<<"GET /types/T/buckets?buckets=true HTTP/1.1\r\n">>)) + ), + ?assertMatch( + {ok, riak_kv_ag_bucketlist, _, _}, + check_path((<<"GET /buckets?buckets=true HTTP/1.1\r\n">>)) + ), + ?assertMatch( + {halt, 405, [{'Allow', <<"GET">>}], <<>>, []}, + check_path(<<"PUT /buckets?buckets=true HTTP/1.1\r\n">>) + ), + ?assertMatch( + {ok, riak_kv_ag_bprops, _, _}, + check_path((<<"GET /types/T/buckets/B/props HTTP/1.1\r\n">>)) + ), + ?assertMatch( + {ok, riak_kv_ag_bprops, _, _}, + check_path((<<"GET /buckets/B/props HTTP/1.1\r\n">>)) + ), + ?assertMatch( + {ok, riak_kv_ag_bprops, _, _}, + check_path((<<"DELETE /buckets/B/props HTTP/1.1\r\n">>)) + ), + ?assertMatch( + {ok, riak_kv_ag_bprops, _, _}, + check_path((<<"PUT /types/T/props HTTP/1.1\r\n">>)) + ), + ?assertMatch( + {ok, riak_kv_ag_bprops, _, _}, + check_path((<<"GET /types/T/props HTTP/1.1\r\n">>)) + ), + ?assertMatch( + {halt, 405, [{'Allow', <<"DELETE, GET, PUT">>}], <<>>, []}, + check_path(<<"POST /buckets/B/props HTTP/1.1\r\n">>) + ), + ?assertMatch( + {halt, 405, [{'Allow', <<"GET, PUT">>}], <<>>, []}, + check_path(<<"POST /types/T/props HTTP/1.1\r\n">>) + ). + +type_preference_test() -> + Accept1 = [<<"multipart/mixed">>, <<"*/*;q=0.9">>], + Accept2 = [<<"*/*;q=0.9">>, <<"multipart/mixed">>], + Accept3 = + [ + <<"text/plain;q=0.9 ">>, + <<"multipart_mixed;q=0.8">>, + <<"*/*; q=0.7 ">> + ], + ?assertMatch({true, 1.0}, type_preference(<<"multipart/mixed">>, Accept1)), + ?assertMatch({true, 0.9}, type_preference(<<"text/plain">>, Accept1)), + ?assertMatch({true, 1.0}, type_preference(<<"multipart/mixed">>, Accept2)), + ?assertMatch({true, 0.9}, type_preference(<<"text/plain">>, Accept2)), + ?assertMatch({true, 0.7}, type_preference(<<"multipart/mixed">>, Accept3)), + ?assertMatch({true, 0.9}, type_preference(<<"text/plain">>, Accept3)), + ?assertMatch( + {true, 0.7}, + type_preference(<<"application/json">>, Accept3) + ), + Accept4 = + [ + <<"text/plain; q=0.9 ">>, + <<"application/json">>, + <<"multipart/* ; q=0.7 ">> + ], + ?assertMatch({true, 0.7}, type_preference(<<"multipart/mixed">>, Accept4)), + ?assertMatch({true, 0.9}, type_preference(<<"text/plain">>, Accept4)), + ?assertMatch( + {true, 1.0}, + type_preference(<<"application/json">>, Accept4) + ), + ?assertMatch({false, +0.0}, type_preference(<<"text/xml">>, Accept4)), + + ?assertMatch(true, type_match(<<"multipart/mixed">>, Accept1)), + ?assertMatch(true, type_match(<<"text/plain">>, Accept1)), + ?assertMatch(true, type_match(<<"multipart/mixed">>, Accept4)), + ?assertMatch(true, type_match(<<"text/plain">>, Accept4)), + ?assertMatch(false, type_match(<<"text/xml">>, Accept4)), + + Accept5 = + [ + <<"text/plain; q=0.9 ">>, + <<"application-json">>, + <<"multipart/* ; q=A ">> + ], + ?assertMatch({true, 0.9}, type_preference(<<"text/plain">>, Accept5)), + ?assertMatch({true, +0.0}, type_preference(<<"multipart/mixed">>, Accept5)), + ?assertMatch( + {false, +0.0}, + type_preference(<<"application/json">>, Accept5) + ). + +-endif. diff --git a/src/riak_kv_wm_aaefold.erl b/src/riak_kv_wm_aaefold.erl deleted file mode 100644 index 563608732..000000000 --- a/src/riak_kv_wm_aaefold.erl +++ /dev/null @@ -1,1235 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - -%% @doc Webmachine resource for running aae fold queries. -%% -%% Available operations (NOTE: within square brackets means optional) -%% -%% ``` -%% GET /cachedtrees/nvals/NVal/root -%% GET /cachedtrees/nvals/NVal/branch?filter -%% GET /cachedtrees/nvals/NVal/keysclocks?filter -%% GET /rangetrees/[types/Type/]buckets/Bucket/trees/Size?filter -%% GET /rangetrees/[types/Type/]buckets/Bucket/keysclocks?filter -%% GET /rangerepl/[types/Type/]buckets/Bucket?filter -%% GET /rangerepair/[types/Type/]buckets/Bucket?filter -%% GET /siblings/[types/Type/]buckets/Bucket/counts/Cnt?filter -%% GET /objectsizes/[types/Type/]buckets/Bucket/sizes/Size?filter -%% GET /objectstats/[types/Type/]buckets/Bucket?filter -%% GET /tombs/[types/Type/]buckets/Bucket?filter -%% GET /reap/[types/Type/]buckets/Bucket?filter -%% GET /erase/[types/Type/]buckets/Bucket?filter -%% GET /aaebucketlist?nval -%% ''' -%% @TODO Filter contains key ranges, date ranges, has_fun, segment -%% filter now (doc below) -%% -%% Run an AAE Fold on the underlying parallel or native AAE store -%% -%% The contents of the `filter' URL parameter varies depending on -%% URL. The `filter' is a set either a list of integers (for -%% cachedtrees branch and keysclocks) meaninb branches for -%% cachedtrees/branch and segments for cachedtrees/keysclocks OR -%% filter is set of Key->Value pairs encoded as JSON with the -%% following possible values: - - -%%
    -%%
  • key_range=Range
    -%% an object with two keys start=Binary, and end=Binary. Where -%% each is a key. Used to limit all of rangetrees, siblings, -%% objectsizes and objectstats queries. If this element is -%% absent from the filter then ALL keys will be queried -%%
    • -%%
  • date_range=Range
    an object with two keys -%% start=Integer, and end=Integer. Where each is a 32bit unix -%% seconds since the epoch timestamp. Used to limit all of -%% rangetrees, siblings, objectsizes and objectstats -%% queries. If this element is absent from the filter then ALL -%% lastmodified will be queried -%%
    • -%%
  • segment_filter=Object
    -%% rangetrees only -%% An object with two keys, segments=array(Integer) an array -%% of segment IDs to query. And tree_size= [xxsmall | xsmall | -%% small | medium | large | xlarge] not the tree size for this -%% query, but the tree size from which the segments in -%% `segments' where requested. This means that in the case of -%% rangetrees queries you may request a tree of size X but -%% with a filter on segments that were originally fetched with -%% a tree of size Y. If absent then `all' segments will be queried. -%%
    • -%%
  • hash_iv=Integer
    -%% rangetrees trees query only -%% an integer that is an initialisation vector for the hash -%% method for the trees. Useful for avoiding hash collision, -%% this IV will be used to initialise the hash function that -%% will be used to hash version vectors that go into creating -%% the merkle tree. If absent then the default hash fun is used. -%%
    • -%%
  • change_method=count|local|{job, Integer}
    -%% Used only on reap and erase queries. If the change_method is set -%% to count, then no reaps or erases will be performed - a count will -%% simply be taken. local will mean that on each node participating -%% in the query, that node's eraser/reaper queue will be used for that -%% nodes's share of the query. {job, Integer} will set up a specific -%% reap/erase worker for this job, identified in logs by Integer, and -%% running on the node coordinating the query -%%
    • -%%
- --module(riak_kv_wm_aaefold). - -%% webmachine resource exports --export([ - init/1, - service_available/2, - is_authorized/2, - malformed_request/2, - content_types_provided/2, - encodings_provided/2, - resource_exists/2, - produce_fold_results/2 - ]). - - --ifdef(TEST). --ifdef(EQC). --include_lib("eqc/include/eqc.hrl"). --export([prop_not_malformed/0]). --define(NUMTESTS, 1000). --define(QC_OUT(P), - eqc:on_output(fun(Str, Args) -> io:format(user, Str, Args) end, P)). --endif. --include_lib("eunit/include/eunit.hrl"). --endif. - --include_lib("webmachine/include/webmachine.hrl"). --include("riak_kv_wm_raw.hrl"). - --record(ctx, { - client, %% riak_client() - the store client - riak, %% local | {node(), atom()} - params for riak client - bucket_type, %% Bucket type (from uri) - bucket, %% The bucket to query (if relevant) - security, %% AAE Fold not currently subject to grant check - %% so security context will be ignored. - query %% The query.. - }). - -%% used for all the different query types that take a filter, only -%% some values used for some queries. --record(filter, { - key_range = all :: {binary(), binary()} | all, - date_range = all :: {date, pos_integer(), pos_integer()} | all, - hash_method = pre_hash :: {rehash, non_neg_integer()} | pre_hash, - segment_filter = all :: {segments, list(pos_integer()), leveled_tictac:tree_size()} | all, - change_method = count :: {job, pos_integer()}|local|count - }). - --type context() :: #ctx{}. --type filter() :: #filter{}. - --define(SEG_FILT, <<"segment_filter">>). --define(KEY_RANGE, <<"key_range">>). --define(DATE_RANGE, <<"date_range">>). --define(HASH_IV, <<"hash_iv">>). --define(CHANGE_METHOD, <<"change_method">>). - --define(FILTER_FIELDS, - [?SEG_FILT, ?KEY_RANGE, ?DATE_RANGE, ?HASH_IV, ?CHANGE_METHOD]). - - -%% @doc Initialize this resource. --spec init(proplists:proplist()) -> {ok, context()}. -init(Props) -> - {ok, #ctx{ - riak=proplists:get_value(riak, Props), - bucket_type=proplists:get_value(bucket_type, Props) - }}. - -%% @doc Determine whether or not a connection to Riak -%% can be established. Also, extract query params. --spec service_available(#wm_reqdata{}, context()) -> - {boolean(), #wm_reqdata{}, context()}. -service_available(RD, Ctx0=#ctx{riak=RiakProps}) -> - Ctx = riak_kv_wm_utils:ensure_bucket_type(RD, Ctx0, #ctx.bucket_type), - case riak_kv_wm_utils:get_riak_client(RiakProps, riak_kv_wm_utils:get_client_id(RD)) of - {ok, C} -> - {true, RD, Ctx#ctx { client=C }}; - Error -> - {false, - wrq:set_resp_body( - io_lib:format("Unable to connect to Riak: ~p~n", [Error]), - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD)), - Ctx} - end. - -is_authorized(ReqData, Ctx) -> - case application:get_env(riak_kv, permit_insecure_http_ops, false) of - true -> - {true, ReqData, Ctx}; - false -> - case riak_api_web_security:is_authorized(ReqData) of - false -> - {"Basic realm=\"Riak\"", ReqData, Ctx}; - {true, SecContext} -> - {true, ReqData, Ctx#ctx{security=SecContext}}; - insecure -> - { - {halt, 426}, - wrq:append_to_resp_body( - << - "Security is enabled and " - "Riak does not accept credentials over HTTP. Try HTTPS " - "instead. Or configure `permit_insecure_http_ops`" - >>, - ReqData - ), - Ctx - } - end - end. - -%% @doc Determine whether query parameters are badly-formed. -%% Specifically, we check that the aaefold operation is of -%% a known type. --spec malformed_request(#wm_reqdata{}, context()) -> - {boolean(), #wm_reqdata{}, context()}. -malformed_request(RD, Ctx) -> - %% determine type of query, and check for each type - PathTokens = path_tokens(RD), - - FoldType = riak_kv_wm_utils:maybe_decode_uri(RD, hd(PathTokens)), - - case FoldType of - "cachedtrees" -> malformed_cached_tree_request(RD, Ctx); - "rangetrees" -> malformed_range_tree_request(RD, Ctx); - "rangerepl" -> malformed_range_repl_request(RD, Ctx); - "rangerepair" -> malformed_range_repair_request(RD, Ctx); - "siblings" -> malformed_find_keys_request({sibling_count, count}, RD, Ctx); - "objectsizes" -> malformed_find_keys_request({object_size, size}, RD, Ctx); - "objectstats" -> malformed_object_stats_request(RD, Ctx); - "tombs" -> malformed_find_tombs_request(RD, Ctx); - "reap" -> malformed_reap_tombs_request(RD, Ctx); - "erase" -> malformed_erase_keys_request(RD, Ctx); - "aaebucketlist" -> malformed_list_buckets_request(RD, Ctx) - end. - -%% @private check that we can parse out a valid cached tree aae fold -%% query, if so, store it in the ctx for execution --spec malformed_cached_tree_request(#wm_reqdata{}, context()) -> - {boolean(), #wm_reqdata{}, context()}. -malformed_cached_tree_request(RD, Ctx) -> - case validate_nval(wrq:path_info(nval, RD)) of - {invalid, Reason} -> - malformed_response("Cached tree request invalid nval ~p", - [Reason], - RD, Ctx); - {valid, NVal} -> - malformed_cached_tree_request(NVal, RD, Ctx) - end. - -%% @private check that we have what we need in the request for a -%% cached tree query --spec malformed_cached_tree_request(pos_integer(), #wm_reqdata{}, context()) -> - {boolean(), #wm_reqdata{}, context()}. -malformed_cached_tree_request(NVal, RD, Ctx) -> - QueryType = riak_kv_wm_utils:maybe_decode_uri(RD, lists:last(path_tokens(RD))), - case QueryType of - "root" -> - %% root query, has an nval, all good - {false, RD, - Ctx#ctx{ - query = {merge_root_nval, NVal} - }}; - Q when Q == "branch"; - Q == "keysclocks" -> - Filter0 = wrq:get_qs_value(?Q_AAEFOLD_FILTER, RD), - malformed_cached_tree_request_filter(list_to_existing_atom(Q), Filter0, NVal, RD, Ctx); - Other -> - malformed_response("unkown cached aae tree query ~p", - [Other], - RD, Ctx) - end. - -%% @private check that the request provides a valid filter for cached -%% tree queries that need it --spec malformed_cached_tree_request_filter(branch | keysclocks, - Filter::any(), - pos_integer(), - #wm_reqdata{}, - context()) -> - {boolean(), #wm_reqdata{}, context()}. -malformed_cached_tree_request_filter(QType, undefined, _NVal, RD, Ctx) -> - malformed_response("Filter query param required for ~p aae fold", - [QType], - RD, - Ctx); -malformed_cached_tree_request_filter(QType, Filter0, NVal, RD, Ctx) -> - case validate_cached_tree_filter(Filter0) of - {invalid, Reason} -> - malformed_response("Invalid branch | segment filter ~p", - [Reason], - RD, - Ctx); - {valid, Filter} when is_list(Filter) -> - Query = - case QType of - branch -> - {merge_branch_nval, NVal, Filter}; - keysclocks -> - {fetch_clocks_nval, NVal, Filter} - end, - {false, RD, Ctx#ctx{query = Query}}; - {valid, Filter} when is_record(Filter, filter) -> - case QType of - keysclocks -> - Query = - {fetch_clocks_nval, - NVal, - element(2, Filter#filter.segment_filter), - Filter#filter.date_range}, - {false, RD, Ctx#ctx{query = Query}}; - InvalidQType -> - malformed_response("Invalid filter for ~p", - [InvalidQType], - RD, - Ctx) - end - end. - -%% @private validate the request for range tree query and populate -%% context with query (if valid.) --spec malformed_range_tree_request(#wm_reqdata{}, context()) -> - {boolean(), #wm_reqdata{}, context()}. -malformed_range_tree_request(RD, Ctx) -> - case wrq:path_info(bucket, RD) of - undefined -> - malformed_response("Bucket required", [], RD, Ctx); - Bucket0 -> - Bucket = erlang:list_to_binary( - riak_kv_wm_utils:maybe_decode_uri(RD, Bucket0) - ), - TreeSize = wrq:path_info(size, RD), - malformed_range_tree_request(TreeSize, RD, Ctx#ctx{bucket=Bucket}) - end. - -%% @private decide on query type and parse out filter --spec malformed_range_tree_request(undefined | string(), #wm_reqdata{}, context()) -> - {boolean(), #wm_reqdata{}, context()}. -malformed_range_tree_request(undefined=_TreeSize, RD, Ctx) -> - %% no tree size, last token _MUST_ be "keysclocks" or it's invalid - QueryType = riak_kv_wm_utils:maybe_decode_uri(RD, lists:last(path_tokens(RD))), - case QueryType of - "keysclocks" -> - Filter0 = wrq:get_qs_value(?Q_AAEFOLD_FILTER, RD), - malformed_range_tree_keysclocks_request(Filter0, RD, Ctx); - Other -> - malformed_response("Invalid rangetree aae query ~p", - [Other], - RD, - Ctx) - end; -malformed_range_tree_request(TreeSize0, RD, Ctx) -> - case validate_treesize(TreeSize0) of - {invalid, Reason} -> - malformed_response("Invalid treesize ~p", [Reason], RD, Ctx); - {valid, TreeSize} -> - Filter0 = wrq:get_qs_value(?Q_AAEFOLD_FILTER, RD), - malformed_range_tree_request(Filter0, TreeSize, RD, Ctx) - end. - -%% @private finally parse out the query filter and validate it --spec malformed_range_tree_request(undefined | string(), - leveled_tictac:tree_size(), - #wm_reqdata{}, - context()) -> - {boolean(), #wm_reqdata{}, context()}. -malformed_range_tree_request(Filter0, TreeSize, RD, Ctx) -> - case validate_range_filter(Filter0) of - {invalid, Reason} -> - malformed_response("Invalid range filter ~p", - [Reason], - RD, - Ctx); - {valid, Filter} -> - QBucket = query_bucket(Ctx), - Query = {merge_tree_range, - QBucket, - Filter#filter.key_range, - TreeSize, - Filter#filter.segment_filter, - Filter#filter.date_range, - Filter#filter.hash_method}, - {false, RD, - Ctx#ctx{query= Query}} - end. - -%% @private finally, parse out query filter and add query to context --spec malformed_range_tree_keysclocks_request(undefined | string(), - #wm_reqdata{}, - context()) -> - {boolean(), #wm_reqdata{}, context()}. -malformed_range_tree_keysclocks_request(Filter0, RD, Ctx) -> - case validate_range_filter(Filter0) of - {invalid, Reason} -> - malformed_response("Invalid range filter ~p", - [Reason], - RD, - Ctx); - {valid, Filter} -> - QBucket = query_bucket(Ctx), - Query = {fetch_clocks_range, - QBucket, - Filter#filter.key_range, - Filter#filter.segment_filter, - Filter#filter.date_range}, - {false, RD, - Ctx#ctx{query= Query}} - end. - -%% @private validate and parse the find keys queries --spec malformed_find_keys_request({sibling_count, count} | {object_size, size}, - #wm_reqdata{}, - context()) -> - {boolean(), #wm_reqdata{}, context()}. -malformed_find_keys_request({QType, UrlArg}, RD, Ctx) -> - case wrq:path_info(bucket, RD) of - undefined -> - malformed_response("Bucket required", [], RD, Ctx); - Bucket0 -> - Bucket = erlang:list_to_binary( - riak_kv_wm_utils:maybe_decode_uri(RD, Bucket0) - ), - QArg = validate_int(wrq:path_info(UrlArg, RD)), - malformed_find_keys_request({QType, UrlArg}, QArg, RD, Ctx#ctx{bucket=Bucket}) - end. - --spec malformed_find_keys_request({sibling_count, count} | {object_size, size}, - {valid, pos_integer()} | {invalid, any()}, - #wm_reqdata{}, - context()) -> - {boolean(), #wm_reqdata{}, context()}. -malformed_find_keys_request({QType, UrlArg}, {invalid, Reason}, RD, Ctx) -> - malformed_response("~p requires integer for ~p. Value ~p invalid", - [QType, UrlArg, Reason], - RD, - Ctx); -malformed_find_keys_request({QType, _UrlArg}, {valid, QArg}, RD, Ctx) -> - Filter0 = wrq:get_qs_value(?Q_AAEFOLD_FILTER, RD), - case validate_range_filter(Filter0) of - {invalid, Reason} -> - malformed_response("Invalid range filter ~p", - [Reason], - RD, - Ctx); - {valid, Filter} -> - QBucket = query_bucket(Ctx), - Query = { - find_keys, - QBucket, - Filter#filter.key_range, - Filter#filter.date_range, - {QType, QArg} - }, - {false, RD, - Ctx#ctx{query = Query}} - end. - - --spec malformed_find_tombs_request(#wm_reqdata{}, context()) -> - {boolean(), #wm_reqdata{}, context()}. -malformed_find_tombs_request(RD, Ctx) -> - case wrq:path_info(bucket, RD) of - undefined -> - malformed_response("Bucket required", [], RD, Ctx); - Bucket0 -> - Bucket = erlang:list_to_binary( - riak_kv_wm_utils:maybe_decode_uri(RD, Bucket0) - ), - Ctx2 = Ctx#ctx{bucket=Bucket}, - Filter0 = wrq:get_qs_value(?Q_AAEFOLD_FILTER, RD), - case validate_range_filter(Filter0) of - {invalid, Reason} -> - malformed_response("Invalid range filter ~p", - [Reason], - RD, - Ctx2); - {valid, Filter} -> - QBucket = query_bucket(Ctx2), - Query = - {find_tombs, - QBucket, - Filter#filter.key_range, - Filter#filter.segment_filter, - Filter#filter.date_range}, - {false, RD, Ctx2#ctx{query= Query}} - end - end. - --spec malformed_reap_tombs_request(#wm_reqdata{}, context()) -> - {boolean(), #wm_reqdata{}, context()}. -malformed_reap_tombs_request(RD, Ctx) -> - case wrq:path_info(bucket, RD) of - undefined -> - malformed_response("Bucket required", [], RD, Ctx); - Bucket0 -> - Bucket = erlang:list_to_binary( - riak_kv_wm_utils:maybe_decode_uri(RD, Bucket0) - ), - Ctx2 = Ctx#ctx{bucket=Bucket}, - Filter0 = wrq:get_qs_value(?Q_AAEFOLD_FILTER, RD), - case validate_range_filter(Filter0) of - {invalid, Reason} -> - malformed_response("Invalid range filter ~p", - [Reason], - RD, - Ctx2); - {valid, Filter} -> - QBucket = query_bucket(Ctx2), - Query = - {reap_tombs, - QBucket, - Filter#filter.key_range, - Filter#filter.segment_filter, - Filter#filter.date_range, - Filter#filter.change_method}, - {false, RD, Ctx2#ctx{query= Query}} - end - end. - --spec malformed_erase_keys_request(#wm_reqdata{}, context()) -> - {boolean(), #wm_reqdata{}, context()}. -malformed_erase_keys_request(RD, Ctx) -> - case wrq:path_info(bucket, RD) of - undefined -> - malformed_response("Bucket required", [], RD, Ctx); - Bucket0 -> - Bucket = erlang:list_to_binary( - riak_kv_wm_utils:maybe_decode_uri(RD, Bucket0) - ), - Ctx2 = Ctx#ctx{bucket=Bucket}, - Filter0 = wrq:get_qs_value(?Q_AAEFOLD_FILTER, RD), - case validate_range_filter(Filter0) of - {invalid, Reason} -> - malformed_response("Invalid range filter ~p", - [Reason], - RD, - Ctx2); - {valid, Filter} -> - QBucket = query_bucket(Ctx2), - Query = - {erase_keys, - QBucket, - Filter#filter.key_range, - Filter#filter.segment_filter, - Filter#filter.date_range, - Filter#filter.change_method}, - {false, RD, Ctx2#ctx{query= Query}} - end - end. - -%% @private validate and parse a range repl request -%% Needs a bucket, queue_name as minimum - plus also maybe a key range and a -%% date range --spec malformed_range_repl_request(#wm_reqdata{}, - context()) -> - {boolean(), #wm_reqdata{}, context()}. -malformed_range_repl_request(RD, Ctx) -> - case wrq:path_info(bucket, RD) of - undefined -> - malformed_response("Bucket required", [], RD, Ctx); - Bucket0 -> - Bucket = erlang:list_to_binary( - riak_kv_wm_utils:maybe_decode_uri(RD, Bucket0) - ), - case wrq:path_info(queuename, RD) of - undefined -> - malformed_response("Queue Name required", [], RD, Ctx); - QN0 -> - Filter0 = - wrq:get_qs_value(?Q_AAEFOLD_FILTER, RD), - malformed_range_repl_request(Filter0, QN0, - RD, Ctx#ctx{bucket=Bucket}) - end - end. - --spec malformed_range_repl_request(undefined | string(), - undefined | string(), - #wm_reqdata{}, - context()) -> - {boolean(), #wm_reqdata{}, context()}. -malformed_range_repl_request(Filter0, QN0, RD, Ctx) -> - case validate_range_filter(Filter0) of - {invalid, Reason} -> - malformed_response("Invalid range filter ~p", - [Reason], - RD, - Ctx); - {valid, Filter} -> - QN = list_to_atom(QN0), - QBucket = query_bucket(Ctx), - Query = {repl_keys_range, - QBucket, - Filter#filter.key_range, - Filter#filter.date_range, - QN}, - {false, RD, Ctx#ctx{query= Query}} - end. - -%% @private validate and parse a range repair request -%% Needs a bucket as minimum - plus also maybe a key range and a -%% date range --spec malformed_range_repair_request(#wm_reqdata{}, - context()) -> - {boolean(), #wm_reqdata{}, context()}. -malformed_range_repair_request(RD, Ctx) -> - case wrq:path_info(bucket, RD) of - undefined -> - malformed_response("Bucket required", [], RD, Ctx); - Bucket0 -> - Bucket = - erlang:list_to_binary( - riak_kv_wm_utils:maybe_decode_uri(RD, Bucket0)), - Filter0 = wrq:get_qs_value(?Q_AAEFOLD_FILTER, RD), - malformed_range_repair_request(Filter0, - RD, - Ctx#ctx{bucket=Bucket}) - end. - --spec malformed_range_repair_request(undefined | string(), - #wm_reqdata{}, - context()) -> - {boolean(), #wm_reqdata{}, context()}. -malformed_range_repair_request(Filter0, RD, Ctx) -> - case validate_range_filter(Filter0) of - {invalid, Reason} -> - malformed_response("Invalid range filter ~p", - [Reason], - RD, - Ctx); - {valid, Filter} -> - QBucket = query_bucket(Ctx), - Query = {repair_keys_range, - QBucket, - Filter#filter.key_range, - Filter#filter.date_range, - all}, - {false, RD, Ctx#ctx{query= Query}} - end. - - -%% @private validate and populate the object stats query --spec malformed_object_stats_request(#wm_reqdata{}, context()) -> - {boolean(), #wm_reqdata{}, context()}. -malformed_object_stats_request(RD, Ctx) -> - case wrq:path_info(bucket, RD) of - undefined -> - malformed_response("Bucket required", [], RD, Ctx); - Bucket0 -> - Bucket = erlang:list_to_binary( - riak_kv_wm_utils:maybe_decode_uri(RD, Bucket0) - ), - Ctx2 = Ctx#ctx{bucket=Bucket}, - Filter0 = wrq:get_qs_value(?Q_AAEFOLD_FILTER, RD), - case validate_range_filter(Filter0) of - {invalid, Reason} -> - malformed_response("Invalid range filter ~p", - [Reason], - RD, - Ctx2); - {valid, Filter} -> - QBucket = query_bucket(Ctx2), - Query = { - object_stats, - QBucket, - Filter#filter.key_range, - Filter#filter.date_range - }, - {false, RD, - Ctx2#ctx{query= Query}} - end - end. - --spec malformed_list_buckets_request(#wm_reqdata{}, context()) -> - {boolean(), #wm_reqdata{}, context()}. -malformed_list_buckets_request(RD, Ctx) -> - NVal = wrq:get_qs_value(?Q_NVAL, integer_to_list(1), RD), - case validate_integer(NVal) of - {valid, N} -> - Query = {list_buckets, N}, - {false, RD, Ctx#ctx{query = Query}}; - {invalid, Reason} -> - malformed_response("Invalid n_val ~p", [Reason], RD, Ctx) - end. - - -%% @private since we use it so often, wrap it up --spec malformed_response(string(), list(any()), #wm_reqdata{}, context()) -> - {true, #wm_reqdata{}, context()}. -malformed_response(MessageFmt, FmtArgs, RD, Ctx) -> - {true, - wrq:set_resp_body(io_lib:format(MessageFmt, - FmtArgs), - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD)), - Ctx}. - -%% @private buckets, are they typed, are they not? --spec query_bucket(context()) -> riak_object:bucket(). -query_bucket(Ctx) -> - riak_kv_wm_utils:maybe_bucket_type(Ctx#ctx.bucket_type, Ctx#ctx.bucket). - --spec validate_nval(undefined | string()) -> - {invalid, Reason::any()} | - {valid, pos_integer()}. -validate_nval(NVal) -> - validate_int(NVal). - --spec validate_int(undefined | string()) -> - {invalid, Reason::any()} | - {valid, pos_integer()}. -validate_int(undefined) -> - {invalid, undefined}; -validate_int(String) -> - try - list_to_integer(String) of - N when N > 0 -> - {valid, N}; - Neg -> - {invalid, Neg} - catch _:_ -> - {invalid, String} - end. - --spec validate_cached_tree_filter(string()) -> - {valid, list(non_neg_integer())} | - {valid, filter()} | - {invalid, Reason::any()}. -validate_cached_tree_filter(String) -> - try mochijson2:decode(base64:decode(String)) of - Filter when is_list(Filter) -> - {valid, Filter}; - {struct, Filter} -> - validate_range_filter(Filter, [?SEG_FILT, ?DATE_RANGE], #filter{}); - Other -> - {invalid, Other} - catch _:_ -> - {invalid, String} - end. - - --spec validate_range_filter(undefined | string()) -> - {valid, filter()} | - {invalid, Reason::any()}. -validate_range_filter(undefined) -> - {valid, #filter{}}; -validate_range_filter(String) -> - try mochijson2:decode(base64:decode(String)) of - {struct, Filter} -> - validate_range_filter(Filter, ?FILTER_FIELDS, #filter{}); - Other -> - {invalid, Other} - catch _:_ -> - {invalid, String} - end. - --spec validate_integer(string()) -> {valid, pos_integer()}|{invalid, any()}. -validate_integer(String) -> - try list_to_integer(String) of - Int when Int > 0 -> - {valid, Int}; - _ -> - {invalid, String} - catch _:_ -> - {invalid, String} - end. - --spec validate_range_filter(list(), list(), filter()) -> - {invalid, Reason::term()} | - {valid, filter()}. -validate_range_filter(_FilterJson, []=_Fields, Filter) -> - {valid, Filter}; -validate_range_filter(FilterJson, [Field | Fields], Filter0) -> - FieldVal = proplists:get_value(Field, FilterJson), - case validate_filter_field(Field, FieldVal, Filter0) of - {valid, Filter} -> - validate_range_filter(FilterJson, Fields, Filter); - Other -> - Other - end. - --spec validate_filter_field(binary(), any(), filter()) -> - {valid, filter()} | - {invalid, Reason::any()}. -validate_filter_field(?SEG_FILT, {struct, SegFiltJson}, Filter) -> - case validate_treesize(proplists:get_value(<<"tree_size">>, SegFiltJson)) of - {valid, TreeSize} -> - case validate_segment_list(proplists:get_value(<<"segments">>, SegFiltJson)) of - {valid, Segments} -> - {valid, Filter#filter{segment_filter= {segments, Segments, TreeSize}}}; - {invalid, Reason} -> - {invalid, Reason} - end; - {invalid, ITS} -> - {invalid, ITS} - end; -validate_filter_field(?SEG_FILT, <<"all">>, Filter) -> - {valid, Filter}; -validate_filter_field(?SEG_FILT, undefined, Filter) -> - {valid, Filter}; -validate_filter_field(?SEG_FILT, Other, _Filter) -> - {invalid, {?SEG_FILT, Other}}; -validate_filter_field(?KEY_RANGE, {struct, KeyRangeJson}, Filter) -> - case {proplists:get_value(<<"start">>, KeyRangeJson), - proplists:get_value(<<"end">>, KeyRangeJson)} of - {Start, End} when is_binary(Start), is_binary(End) -> - {valid, Filter#filter{key_range= {Start, End}}}; - Other -> - {invalid, {?KEY_RANGE, Other}} - end; -validate_filter_field(?KEY_RANGE, <<"all">>, Filter) -> - {valid, Filter}; -validate_filter_field(?KEY_RANGE, undefined, Filter) -> - {valid, Filter}; -validate_filter_field(?KEY_RANGE, Other, _Filter) -> - {invalid, {?KEY_RANGE, Other}}; -validate_filter_field(?DATE_RANGE, {struct, DateRangeJson}, Filter) -> - case {proplists:get_value(<<"start">>, DateRangeJson), - proplists:get_value(<<"end">>, DateRangeJson)} of - {Start, End} when is_integer(Start), - is_integer(End), - Start >= 0, - End >= 0 -> - {valid, Filter#filter{date_range= {date, Start, End}}}; - Other -> - {invalid, {?DATE_RANGE, Other}} - end; -validate_filter_field(?DATE_RANGE, <<"all">>, Filter) -> - {valid, Filter}; -validate_filter_field(?DATE_RANGE, undefined, Filter) -> - {valid, Filter}; -validate_filter_field(?DATE_RANGE, Other, _Filter) -> - {invalid, {?DATE_RANGE, Other}}; -validate_filter_field(?HASH_IV, IV, Filter) when is_integer(IV) andalso IV > -1 -> - {valid, Filter#filter{hash_method={rehash, IV}}}; -validate_filter_field(?HASH_IV, undefined, Filter) -> - {valid, Filter}; -validate_filter_field(?HASH_IV, Other, _Filter) -> - {invalid, {?HASH_IV, Other}}; -validate_filter_field(?CHANGE_METHOD, <<"count">>, Filter) -> - {valid, Filter#filter{change_method=count}}; -validate_filter_field(?CHANGE_METHOD, <<"local">>, Filter) -> - {valid, Filter#filter{change_method=local}}; -validate_filter_field(?CHANGE_METHOD, {struct, JobJson}, Filter) -> - case proplists:get_value(<<"job_id">>, JobJson) of - JobID when is_integer(JobID), JobID > 0 -> - {valid, Filter#filter{change_method={job, JobID}}}; - Other -> - {invalid, {?CHANGE_METHOD, Other}} - end; -validate_filter_field(?CHANGE_METHOD, undefined, Filter) -> - {valid, Filter}; -validate_filter_field(?CHANGE_METHOD, Other, _Filter) -> - {invalid, {?CHANGE_METHOD, Other}}. - --spec validate_segment_list(any()) -> - {valid, list()} | - {invalid, Reason::any()}. -validate_segment_list(undefined) -> - {invalid, undefined}; -validate_segment_list(SegList) when is_list(SegList) -> - %% @TODO should check contents?? - {valid, SegList}; -validate_segment_list(Other) -> - {invalid, Other}. - --spec validate_treesize(undefined | binary() | string()) -> - {valid, leveled_tictac:tree_size()} | - {invalid, Reason::any()}. -validate_treesize(undefined) -> - {invalid, undefined}; -validate_treesize(TreeSizeBin) when is_binary(TreeSizeBin) -> - validate_treesize(binary_to_list(TreeSizeBin)); -validate_treesize(TreeSizeStr) -> - %% if it's a valid leveled_tictac:tree_size/0 there will be an - %% existing atom! - try list_to_existing_atom(TreeSizeStr) of - TreeSize -> - case leveled_tictac:valid_size(TreeSize) of - true -> - {valid, TreeSize}; - false -> - {invalid, TreeSize} - end - catch _:_ -> - {invalid, TreeSizeStr} - end. - --spec content_types_provided(#wm_reqdata{}, context()) -> - {[{ContentType::string(), Producer::atom()}], #wm_reqdata{}, context()}. -%% @doc List the content types available for representing this resource. -%% "application/json" is the content-type for bucket lists. -content_types_provided(RD, Ctx) -> - {[{"application/json", produce_fold_results}], RD, Ctx}. - - --spec encodings_provided(#wm_reqdata{}, context()) -> - {[{Encoding::string(), Producer::function()}], #wm_reqdata{}, context()}. -%% @doc List the encodings available for representing this resource. -%% "identity" and "gzip" are available for bucket lists. -encodings_provided(RD, Ctx) -> - {riak_kv_wm_utils:default_encodings(), RD, Ctx}. - - -resource_exists(RD, #ctx{bucket_type=BType}=Ctx) -> - {riak_kv_wm_utils:bucket_type_exists(BType), RD, Ctx}. - --spec produce_fold_results(#wm_reqdata{}, context()) -> - {any(), #wm_reqdata{}, context()}. -%% @doc Produce the JSON response to an aae fold -produce_fold_results(RD, Ctx) -> - Query = Ctx#ctx.query, - case riak_client:aae_fold(Query, Ctx#ctx.client) of - {ok, Results} -> - QueryName = element(1, Query), - JsonResults = riak_kv_clusteraae_fsm:json_encode_results(QueryName, Results), - {JsonResults, RD, Ctx}; - {error, timeout} -> - {{halt, 503}, - wrq:set_resp_header("Content-Type", "text/plain", - wrq:append_to_response_body( - io_lib:format("request timed out~n", - []), - RD)), - Ctx}; - {error, Reason} -> - {{error, Reason}, RD, Ctx} - end. - --spec path_tokens(#wm_reqdata{}) -> list(string()). -path_tokens(RD) -> - string:tokens(wrq:path(RD), "/"). - --ifdef(TEST). --ifdef(EQC). - -%% run the eqc property as an eunit test -malformed_request_test_() -> - {setup, - fun setup_mocks/0, - fun teardown_mocks/1, - {timeout, 120, ?_assertEqual(true, eqc:quickcheck(eqc:testing_time(50, ?QC_OUT(prop_not_malformed()))))} - }. - -setup_mocks() -> - meck:new(wrq). - -teardown_mocks(_) -> - (catch meck:unload(wrq)). - -%% @private happy-path testing for the query types. NOTE: only happy -%% path -prop_not_malformed() -> - ?FORALL({Query, Filter}, gen_query(), - begin - MockReqData = setup_reqdata(Query, Filter), - ExpectedQuery = element(1, Query), - QueryName = element(1, ExpectedQuery), - Ctx = #ctx{bucket_type= <<"default">>}, - {Res, ResRD, ResCtx} = ?MODULE:malformed_request(MockReqData, Ctx), - aggregate([QueryName], - conjunction([{non_malformed, equals(Res, false)}, - {query_equal, equals(ResCtx#ctx.query, ExpectedQuery)}, - {no_resp, equals([], ResRD)}])) - end). - -%%% -%% query generetors -%%% - -%% Pick one of the query types supported by aaefold. NOTE a `query' -%% here is two tuple. The first element is a 3-tuple of -%% expected-query, the tuple we expect to be passed to aae-fold. The -%% second is the path to return from the wrq mock, the 3rd the -%% path-info to return from the wrq mock. We return these from the -%% generator as they depend on generated parameters. The second -%% element of the returned 2-tuple is a proplist that can be -%% mochijson2 encoded. This JSON is what the WM resource works on to -%% generate the query. It also depends on the generated params. The -%% property then only checks that given the params in JSON the -%% resource produces a query that matches the generated query. -gen_query() -> - oneof([gen_cached(), - gen_find_keys(), - gen_object_stats(), - gen_merge_tree_range(), - gen_fetch_clocks_range()]). - -%% generate a fetch_clocks_range query -gen_fetch_clocks_range() -> - ?LET(Bucket, gen_bucket(), gen_fetch_clocks_range(Bucket)). - -gen_fetch_clocks_range(Bucket) -> - ?LET({KeyRange, DateRange, SegmentFilter}, - {gen_keyrange(), - gen_daterange(), - gen_segment_filter()}, - begin - {BucketPath, BucketPathInfo} = bucket_path(Bucket), - {{{fetch_clocks_range, Bucket, - KeyRange, SegmentFilter, DateRange}, - ["rangetrees"] ++ BucketPath ++ ["keysclocks"], - BucketPathInfo}, - gen_range_filter([ - {?KEY_RANGE, KeyRange}, - {?DATE_RANGE, DateRange}, - {?SEG_FILT, SegmentFilter} - ]) - } - end). - -gen_merge_tree_range() -> - ?LET(Bucket, gen_bucket(), gen_merge_tree_range(Bucket)). - -gen_merge_tree_range(Bucket) -> - ?LET({TreeSize, KeyRange, DateRange, SegmentFilter, HashMethod}, - {gen_treesize(), - gen_keyrange(), - gen_daterange(), - gen_segment_filter(), - gen_hash_method()}, - begin - {BucketPath, BucketPathInfo} = bucket_path(Bucket), - {{{merge_tree_range, Bucket, - KeyRange, TreeSize, SegmentFilter, DateRange, HashMethod}, - ["rangetrees"] ++ BucketPath ++ ["trees", TreeSize], - [{size, atom_to_list(TreeSize)}] ++ BucketPathInfo}, - gen_range_filter([ - {?KEY_RANGE, KeyRange}, - {?DATE_RANGE, DateRange}, - {?SEG_FILT, SegmentFilter}, - {?HASH_IV, HashMethod} - ]) - } - end). - -gen_object_stats() -> - ?LET(Bucket, gen_bucket(), gen_object_stats(Bucket)). - -gen_object_stats(Bucket) -> - ?LET({KeyRange, DateRange}, - {gen_keyrange(), - gen_daterange()}, - begin - {BucketPath, BucketPathInfo} = bucket_path(Bucket), - {{{object_stats, Bucket, KeyRange, DateRange}, - ["objectstats"] ++ BucketPath, - BucketPathInfo - }, - gen_range_filter([ - {?KEY_RANGE, KeyRange}, - {?DATE_RANGE, DateRange} - ]) - } - end). - -gen_find_keys() -> - ?LET({CntOrSize, Bucket}, {choose(1, 100), gen_bucket()}, - oneof([gen_find_siblings(CntOrSize, Bucket), - gen_find_objsize(CntOrSize, Bucket)])). - -gen_find_objsize(Size, Bucket) -> - gen_find_keys(Size, Bucket, object_size, "objectsizes", "sizes", size). - -gen_find_siblings(Cnt, Bucket) -> - gen_find_keys(Cnt, Bucket, sibling_count, "siblings", "counts", count). - -gen_find_keys(CntOrSize, Bucket, QTag, PathPrefix, PathElem, PathInfoTag) -> - CntOrSizeStr = integer_to_list(CntOrSize), - ?LET({KeyRange, DateRange}, - {gen_keyrange(), - gen_daterange()}, - begin - {BucketPath, BucketPathInfo} = bucket_path(Bucket), - {{{find_keys, Bucket, KeyRange, DateRange, {QTag, CntOrSize}}, - [PathPrefix] ++ BucketPath ++ [PathElem, CntOrSizeStr], - [{PathInfoTag, CntOrSizeStr} | BucketPathInfo] - }, - gen_range_filter([ - {?KEY_RANGE, KeyRange}, - {?DATE_RANGE, DateRange} - ]) - } - end). - -gen_cached() -> - ?LET(NVal, choose(2, 5), gen_cached(NVal)). - -gen_cached(NVal) -> - oneof([gen_cached_root(NVal), - gen_cached_branch(NVal), - gen_cached_keysclocks(NVal) - ]). - -gen_cached_root(NVal) -> - NValStr = integer_to_list(NVal), - { - {{merge_root_nval, NVal}, - ["cachedtrees", "nvals", NValStr, "root"], %% path tokens - [{nval, NValStr}] %% path info mappings - }, - undefined}. - -gen_cached_branch(NVal) -> - NValStr = integer_to_list(NVal), - ?LET(Branches, gen_branch_filter(), - { - {{merge_branch_nval, NVal, Branches}, - ["cachedtrees", "nvals", NValStr, "branch"], %% path tokens - [{nval, NValStr}] %% path info mappings - }, - Branches}). - -gen_cached_keysclocks(NVal) -> - NValStr = integer_to_list(NVal), - %% NOTE: branches and segments are just lists of ints - ?LET(Segs, gen_seg_filter(), - { - {{fetch_clocks_nval, NVal, Segs}, - ["cachedtrees", "nvals", NValStr, "keysclocks"], %% path tokens - [{nval, NValStr}] %% path info mappings - }, - Segs}). - -%%% -%% param generetors -%%% - -gen_segment_filter() -> - ?LET({Segments, TreeSize}, {gen_seg_filter(), gen_treesize()}, - oneof([{segments, Segments, TreeSize}, all])). - -gen_treesize() -> - oneof([xxsmall, xsmall, small, medium, large, xlarge]). - -gen_hash_method() -> - ?LET(IV, nat(), - oneof([pre_hash, {rehash, IV}])). - -gen_range_filter(Props) -> - Filt = lists:foldl(fun gen_filter_element/2, [], Props), - {struct, Filt}. - -gen_filter_element({Range, all}, Acc) when Range == ?KEY_RANGE; - Range == ?DATE_RANGE; - Range == ?SEG_FILT -> - Acc; -gen_filter_element({Range, {Start, End}}, Acc) when Range == ?KEY_RANGE -> - [{Range, {struct, [{<<"start">>, Start}, - {<<"end">>, End}]}} | Acc]; -gen_filter_element({Range, {date, Start, End}}, Acc) when Range == ?DATE_RANGE -> - [{Range, {struct, [{<<"start">>, Start}, - {<<"end">>, End}]}} | Acc]; -gen_filter_element({?SEG_FILT, {segments, Segs, TreeSize}}, Acc) -> - [{?SEG_FILT, {struct, [{<<"tree_size">>, atom_to_list(TreeSize)}, - {<<"segments">>, Segs}]}} | Acc]; -gen_filter_element({?HASH_IV, pre_hash}, Acc) -> - Acc; -gen_filter_element({?HASH_IV, {rehash, IV}}, Acc) when is_integer(IV) -> - [{?HASH_IV, IV} | Acc]. - -bucket_path(Bucket) -> - BuckStr = binary_to_list(Bucket), - {["buckets", BuckStr], - [{bucket, BuckStr}]}. - -gen_bucket() -> - oneof([<<"bucket1">>, <<"bucket2">>]). - -gen_keyrange() -> - oneof([all, {<<"key1">>, <<"key99">>}]). - - -gen_daterange() -> - oneof([{date, 1543357393, 1543417393}, - all]). - -gen_seg_filter() -> - non_empty(list(nat())). - -gen_branch_filter() -> - non_empty(list(nat())). - -%%%% -%% helpers -%%%% - -encode_filter(undefined) -> - undefined; -encode_filter(all) -> - undefined; -encode_filter({struct, []}) -> - undefined; -encode_filter(Filter) -> - base64:encode(list_to_binary( - lists:flatten(mochijson2:encode(Filter) - ))). - -to_string(Atom) when is_atom(Atom) -> - atom_to_list(Atom); -to_string(Int) when is_integer(Int) -> - integer_to_list(Int); -to_string(Bin) when is_binary(Bin) -> - binary_to_list(Bin); -to_string(Str) when is_list(Str) -> - Str. - -setup_reqdata({_Q, PathTokens0, PathInfo}, Filter) -> - %% we're changing the expectations per-run - (catch teardown_mocks(ok)), - setup_mocks(), - PathTokens = [to_string(PathToken) || PathToken <- PathTokens0], - Path = "/" ++ string:join(PathTokens, "/"), - meck:expect(wrq, path, fun(_) -> - Path - end), - meck:expect(wrq, path_info, fun(Name, _) -> - proplists:get_value(Name, PathInfo) - end), - EncodedFilter = encode_filter(Filter), - meck:expect(wrq, get_qs_value, fun(?Q_AAEFOLD_FILTER, _) -> - EncodedFilter - end), - %% for riak_kv_web_utils - meck:expect(wrq, get_req_header, fun("X-Riak-URL-Encoding", _RD) -> - "off" - end), - %% for malformed, just return message - meck:expect(wrq, set_resp_header, fun(Header, Val, RD) -> - [{Header, Val} | RD] - end), - meck:expect(wrq, set_resp_body, fun(Body, RD) -> - [{body, lists:flatten(Body)} | RD] - end), - %% the mock req data is a list to accumulate resp-headers - []. - --endif. - --endif. diff --git a/src/riak_kv_wm_bucket_type.erl b/src/riak_kv_wm_bucket_type.erl deleted file mode 100644 index 6c09ca2cb..000000000 --- a/src/riak_kv_wm_bucket_type.erl +++ /dev/null @@ -1,272 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% riak_kv_wm_bucket_type: Webmachine resource for bucket type properties -%% -%% Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - -%% @doc Resource for serving Riak bucket properties over HTTP. -%% -%% Available operations: -%% -%% `GET /types/Type/props' -%% -%% Get information about the named Type, in JSON form: -%% `{"props":{Prop1:Val1,Prop2:Val2,...}}' -%% -%% Each bucket-type property will be included in the "props" object. -%% -%% `linkfun' and `chash_keyfun' properties will be encoded as -%% JSON objects of the form: -%% -%% ``` -%% {"mod":ModuleName, -%% "fun":FunctionName}''' -%% -%% Where ModuleName and FunctionName are each strings representing -%% a module and function. -%% -%% `POST|PUT /types/Type/props' -%% -%% Modify bucket-type properties. -%% -%% Content-type must be `application/json', and the body must have -%% the form: -%% -%% `{"props":{Prop:Val}}' -%% -%% Where the "props" object takes the same form as returned from -%% a GET of the same resource. - --module(riak_kv_wm_bucket_type). - -%% webmachine resource exports --export([ - init/1, - service_available/2, - is_authorized/2, - forbidden/2, - allowed_methods/2, - malformed_request/2, - content_types_provided/2, - encodings_provided/2, - content_types_accepted/2, - resource_exists/2, - produce_bucket_type_body/2, - accept_bucket_type_body/2 - ]). - --record(ctx, {bucket_type, %% binary() - Bucket type (from uri) - client, %% riak_client() - the store client - prefix, %% string() - prefix for resource uris - riak, %% local | {node(), atom()} - params for riak client - bucketprops, %% proplist() - properties of the bucket - method, %% atom() - HTTP method for the request - api_version, %% non_neg_integer() - old or new http api - security %% security context - }). --type context() :: #ctx{}. - --include_lib("webmachine/include/webmachine.hrl"). --include("riak_kv_wm_raw.hrl"). - --spec init(proplists:proplist()) -> {ok, context()}. -%% @doc Initialize this resource. This function extracts the -%% 'prefix' and 'riak' properties from the dispatch args. -init(Props) -> - {ok, #ctx{ - prefix=proplists:get_value(prefix, Props), - riak=proplists:get_value(riak, Props), - api_version=proplists:get_value(api_version,Props), - bucket_type=proplists:get_value(bucket_type, Props) - }}. - --spec service_available(#wm_reqdata{}, context()) -> - {boolean(), #wm_reqdata{}, context()}. -%% @doc Determine whether or not a connection to Riak can be -%% established. This function also takes this opportunity to extract -%% the 'bucket' bindings from the dispatch. -service_available(RD, Ctx0=#ctx{riak=RiakProps}) -> - Ctx = riak_kv_wm_utils:ensure_bucket_type(RD, Ctx0, #ctx.bucket_type), - case riak_kv_wm_utils:get_riak_client(RiakProps, riak_kv_wm_utils:get_client_id(RD)) of - {ok, C} -> - {true, RD, Ctx#ctx{method=wrq:method(RD), client=C}}; - Error -> - {false, - wrq:set_resp_body( - io_lib:format("Unable to connect to Riak: ~p~n", [Error]), - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD)), - Ctx} - end. - -is_authorized(ReqData, Ctx) -> - case riak_api_web_security:is_authorized(ReqData) of - false -> - {"Basic realm=\"Riak\"", ReqData, Ctx}; - {true, SecContext} -> - {true, ReqData, Ctx#ctx{security=SecContext}}; - insecure -> - %% XXX 301 may be more appropriate here, but since the http and - %% https port are different and configurable, it is hard to figure - %% out the redirect URL to serve. - {{halt, 426}, wrq:append_to_resp_body(<<"Security is enabled and " - "Riak does not accept credentials over HTTP. Try HTTPS " - "instead.">>, ReqData), Ctx} - end. - -forbidden(RD, Ctx = #ctx{security=undefined}) -> - {riak_kv_wm_utils:is_forbidden(RD), RD, Ctx}; -forbidden(RD, Ctx=#ctx{security=Security}) -> - case riak_kv_wm_utils:is_forbidden(RD) of - true -> - {true, RD, Ctx}; - false -> - Perm = case Ctx#ctx.method of - 'PUT' -> - "riak_core.set_bucket_type"; - 'GET' -> - "riak_core.get_bucket_type"; - 'HEAD' -> - "riak_core.get_bucket_type" - end, - - Res = riak_core_security:check_permission({Perm, Ctx#ctx.bucket_type}, Security), - case Res of - {false, Error, _} -> - RD1 = wrq:set_resp_header("Content-Type", "text/plain", RD), - {true, wrq:append_to_resp_body(unicode:characters_to_binary(Error, utf8, utf8), RD1), Ctx}; - {true, _} -> - forbidden_check_bucket_type(RD, Ctx) - end - end. - -%% @doc Detects whether the requested bucket-type exists. -forbidden_check_bucket_type(RD, #ctx{method=M}=Ctx) when M =:= 'PUT' -> - %% If the bucket type doesn't exist, we want to bail early so that - %% users cannot PUT bucket types that don't exist. - case riak_kv_wm_utils:bucket_type_exists(Ctx#ctx.bucket_type) of - true -> - {false, RD, Ctx}; - false -> - {{halt, 404}, - wrq:set_resp_body( - io_lib:format("Cannot modify unknown bucket type: ~p", [Ctx#ctx.bucket_type]), - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD)), - Ctx} - end; -forbidden_check_bucket_type(RD, Ctx) -> - {false, RD, Ctx}. - - --spec allowed_methods(#wm_reqdata{}, context()) -> - {[atom()], #wm_reqdata{}, context()}. -%% @doc Get the list of methods this resource supports. -%% Properties allows HEAD, GET, and PUT. -allowed_methods(RD, Ctx) when Ctx#ctx.api_version =:= 3 -> - {['HEAD', 'GET', 'PUT'], RD, Ctx}. - --spec malformed_request(#wm_reqdata{}, context()) -> - {boolean(), #wm_reqdata{}, context()}. -%% @doc Determine whether query parameters, request headers, -%% and request body are badly-formed. -%% Body format is checked to be valid JSON, including -%% a "props" object for a bucket-PUT. -malformed_request(RD, Ctx) when Ctx#ctx.method =:= 'PUT' -> - malformed_props(RD, Ctx); -malformed_request(RD, Ctx) -> - {false, RD, Ctx}. - --spec malformed_props(#wm_reqdata{}, context()) -> - {boolean(), #wm_reqdata{}, context()}. -%% @doc Check the JSON format of a bucket-level PUT. -%% Must be a valid JSON object, containing a "props" object. -malformed_props(RD, Ctx) -> - case catch mochijson2:decode(wrq:req_body(RD)) of - {struct, Fields} -> - case proplists:get_value(?JSON_PROPS, Fields) of - {struct, Props} -> - {false, RD, Ctx#ctx{bucketprops=Props}}; - _ -> - {true, props_format_message(RD), Ctx} - end; - _ -> - {true, props_format_message(RD), Ctx} - end. - --spec props_format_message(#wm_reqdata{}) -> #wm_reqdata{}. -%% @doc Put an error about the format of the bucket-PUT body -%% in the response body of the #wm_reqdata{}. -props_format_message(RD) -> - wrq:append_to_resp_body( - ["bucket type PUT must be a JSON object of the form:\n", - "{\"",?JSON_PROPS,"\":{...bucket properties...}}"], - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD)). - - -resource_exists(RD, Ctx) -> - {riak_kv_wm_utils:bucket_type_exists(Ctx#ctx.bucket_type), RD, Ctx}. - --spec content_types_provided(#wm_reqdata{}, context()) -> - {[{ContentType::string(), Producer::atom()}], #wm_reqdata{}, context()}. -%% @doc List the content types available for representing this resource. -%% "application/json" is the content-type for props requests. -content_types_provided(RD, Ctx) -> - {[{"application/json", produce_bucket_type_body}], RD, Ctx}. - --spec encodings_provided(#wm_reqdata{}, context()) -> - {[{Encoding::string(), Producer::function()}], #wm_reqdata{}, context()}. -%% @doc List the encodings available for representing this resource. -%% "identity" and "gzip" are available for props requests. -encodings_provided(RD, Ctx) -> - {riak_kv_wm_utils:default_encodings(), RD, Ctx}. - --spec content_types_accepted(#wm_reqdata{}, context()) -> - {[{ContentType::string(), Acceptor::atom()}], - #wm_reqdata{}, context()}. -%% @doc Get the list of content types this resource will accept. -%% "application/json" is the only type accepted for props PUT. -content_types_accepted(RD, Ctx) -> - {[{"application/json", accept_bucket_type_body}], RD, Ctx}. - --spec produce_bucket_type_body(#wm_reqdata{}, context()) -> - {binary(), #wm_reqdata{}, context()}. -%% @doc Produce the bucket properties as JSON. -produce_bucket_type_body(RD, Ctx) -> - Props = riak_core_bucket_type:get(Ctx#ctx.bucket_type), - JsonProps = mochijson2:encode( - {struct, - [ - {?JSON_PROPS, - [ riak_kv_wm_utils:jsonify_bucket_prop(P) || P <- Props ]} - ]}), - {JsonProps, RD, Ctx}. - --spec accept_bucket_type_body(#wm_reqdata{}, context()) -> - {true, #wm_reqdata{}, context()}. -%% @doc Modify the bucket properties according to the body of the -%% bucket-level PUT request. -accept_bucket_type_body(RD, Ctx=#ctx{bucket_type=T, bucketprops=Props}) -> - ErlProps = lists:map(fun riak_kv_wm_utils:erlify_bucket_prop/1, Props), - case riak_core_bucket_type:update(T, ErlProps) of - ok -> - {true, RD, Ctx}; - {error, Details} -> - JSON = mochijson2:encode(Details), - RD2 = wrq:append_to_resp_body(JSON, RD), - {{halt, 400}, RD2, Ctx} - end. diff --git a/src/riak_kv_wm_buckets.erl b/src/riak_kv_wm_buckets.erl deleted file mode 100644 index e5771b7bd..000000000 --- a/src/riak_kv_wm_buckets.erl +++ /dev/null @@ -1,256 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2016 Basho Technologies, Inc. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - -%% @doc Webmachine resource for listing Riak buckets over HTTP. -%% -%% URLs that begin with `/types' are necessary for the new bucket -%% types implementation in Riak 2.0, those that begin with `/buckets' -%% are for the default bucket type, and `/riak' is an old URL style, -%% also only works for the default bucket type. -%% -%% It is possible to reconfigure the `/riak' prefix but that seems to -%% be rarely if ever used. -%% -%% ``` -%% GET /types/Type/buckets?buckets=true -%% GET /types/Type/buckets?buckets=stream -%% GET /buckets?buckets=true -%% GET /buckets?buckets=stream -%% GET /riak?buckets=true -%% ''' -%% Get information about available buckets. Note that generating the -%% bucket list is expensive, so we require the `buckets=true' or -%% `buckets=stream' argument. -%% - --module(riak_kv_wm_buckets). - -%% webmachine resource exports --export([ - init/1, - service_available/2, - is_authorized/2, - forbidden/2, - resource_exists/2, - content_types_provided/2, - encodings_provided/2, - produce_bucket_list/2, - malformed_request/2 - ]). - --record(ctx, { - bucket_type, %% binary() - bucket type (from uri) - api_version, %% integer() - Determine which version of the API to use. - client, %% riak_client() - the store client - prefix, %% string() - prefix for resource uris - riak, %% local | {node(), atom()} - params for riak client - method, %% atom() - HTTP method for the request - timeout, %% integer() - list buckets timeout - security %% security context - }). --type context() :: #ctx{}. - --include_lib("webmachine/include/webmachine.hrl"). --include("riak_kv_wm_raw.hrl"). - --define(DEFAULT_TIMEOUT, 5 * 60000). - --spec init(proplists:proplist()) -> {ok, context()}. -%% @doc Initialize this resource. This function extracts the -%% 'prefix' and 'riak' properties from the dispatch args. -init(Props) -> - {ok, #ctx{ - api_version=proplists:get_value(api_version, Props), - prefix=proplists:get_value(prefix, Props), - riak=proplists:get_value(riak, Props), - bucket_type=proplists:get_value(bucket_type, Props) - }}. - - --spec service_available(#wm_reqdata{}, context()) -> - {boolean(), #wm_reqdata{}, context()}. -%% @doc Determine whether or not a connection to Riak -%% can be established. This function also takes this -%% opportunity to extract the 'bucket' and 'key' path -%% bindings from the dispatch, as well as any vtag -%% query parameter. -service_available(RD, Ctx0=#ctx{riak=RiakProps}) -> - Ctx = riak_kv_wm_utils:ensure_bucket_type(RD, Ctx0, #ctx.bucket_type), - case riak_kv_wm_utils:get_riak_client(RiakProps, riak_kv_wm_utils:get_client_id(RD)) of - {ok, C} -> - {true, - RD, - Ctx#ctx{ - method=wrq:method(RD), - client=C - }}; - Error -> - {false, - wrq:set_resp_body( - io_lib:format("Unable to connect to Riak: ~p~n", [Error]), - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD)), - Ctx} - end. - -is_authorized(ReqData, Ctx) -> - case riak_api_web_security:is_authorized(ReqData) of - false -> - {"Basic realm=\"Riak\"", ReqData, Ctx}; - {true, SecContext} -> - {true, ReqData, Ctx#ctx{security=SecContext}}; - insecure -> - %% XXX 301 may be more appropriate here, but since the http and - %% https port are different and configurable, it is hard to figure - %% out the redirect URL to serve. - {{halt, 426}, wrq:append_to_resp_body(<<"Security is enabled and " - "Riak does not accept credentials over HTTP. Try HTTPS " - "instead.">>, ReqData), Ctx} - end. - --spec forbidden(#wm_reqdata{}, context()) - -> {boolean(), #wm_reqdata{}, context()}. -forbidden(ReqDataIn, #ctx{security = undefined} = Context) -> - Class = request_class(ReqDataIn), - riak_kv_wm_utils:is_forbidden(ReqDataIn, Class, Context); -forbidden(ReqDataIn, #ctx{bucket_type = BT, security = Sec} = Context) -> - Class = request_class(ReqDataIn), - {Answer, ReqData, _} = Result = - riak_kv_wm_utils:is_forbidden(ReqDataIn, Class, Context), - case Answer of - false -> - case riak_core_security:check_permission( - {"riak_kv.list_buckets", BT}, Sec) of - {false, Error, _} -> - {true, - wrq:append_to_resp_body( - unicode:characters_to_binary(Error, utf8, utf8), - wrq:set_resp_header( - "Content-Type", "text/plain", ReqData)), - Context}; - {true, _} -> - {false, ReqData, Context} - end; - _ -> - Result - end. - --spec request_class(#wm_reqdata{}) -> term(). -request_class(ReqData) -> - case wrq:get_qs_value(?Q_BUCKETS, ReqData) of - ?Q_STREAM -> - {riak_kv, stream_list_buckets}; - _ -> - {riak_kv, list_buckets} - end. - --spec content_types_provided(#wm_reqdata{}, context()) -> - {[{ContentType::string(), Producer::atom()}], #wm_reqdata{}, context()}. -%% @doc List the content types available for representing this resource. -%% "application/json" is the content-type for bucket lists. -content_types_provided(RD, Ctx) -> - {[{"application/json", produce_bucket_list}], RD, Ctx}. - - --spec encodings_provided(#wm_reqdata{}, context()) -> - {[{Encoding::string(), Producer::function()}], #wm_reqdata{}, context()}. -%% @doc List the encodings available for representing this resource. -%% "identity" and "gzip" are available for bucket lists. -encodings_provided(RD, Ctx) -> - {riak_kv_wm_utils:default_encodings(), RD, Ctx}. - -malformed_request(RD, Ctx) -> - malformed_timeout_param(RD, Ctx). - --spec malformed_timeout_param(#wm_reqdata{}, context()) -> - {boolean(), #wm_reqdata{}, context()}. -%% @doc Check that the timeout parameter is are a -%% string-encoded integer. Store the integer value -%% in context() if so. -malformed_timeout_param(RD, Ctx) -> - case wrq:get_qs_value("timeout", RD) of - undefined -> - {false, RD, Ctx}; - TimeoutStr -> - try - Timeout = list_to_integer(TimeoutStr), - {false, RD, Ctx#ctx{timeout=Timeout}} - catch - _:_ -> - {true, - wrq:append_to_resp_body(io_lib:format("Bad timeout " - "value ~p~n", - [TimeoutStr]), - wrq:set_resp_header(?HEAD_CTYPE, - "text/plain", RD)), - Ctx} - end - end. - -resource_exists(RD, #ctx{bucket_type=BType}=Ctx) -> - {riak_kv_wm_utils:bucket_type_exists(BType), RD, Ctx}. - --spec produce_bucket_list(#wm_reqdata{}, context()) -> - {binary(), #wm_reqdata{}, context()}. -%% @doc Produce the JSON response to a bucket-level GET. -%% Includes a list of known buckets if the "buckets=true" query -%% param is specified. -produce_bucket_list(RD, #ctx{client=Client, - timeout=Timeout0, - bucket_type=BType}=Ctx) -> - Timeout = - case Timeout0 of - undefined -> ?DEFAULT_TIMEOUT; - Set -> Set - end, - case wrq:get_qs_value(?Q_BUCKETS, RD) of - ?Q_TRUE -> - %% Get the buckets. - {ok, Buckets} = - riak_client:list_buckets(none, Timeout, BType, Client), - {mochijson2:encode({struct, [{?JSON_BUCKETS, Buckets}]}), - RD, Ctx}; - ?Q_STREAM -> - F = - fun() -> - {ok, ReqId} = - riak_client:stream_list_buckets(none, - Timeout, - BType, - Client), - stream_buckets(ReqId) - end, - {{stream, {[], F}}, RD, Ctx}; - _ -> - {mochijson2:encode({struct, [{?JSON_BUCKETS, []}]})} - end. - -stream_buckets(ReqId) -> - receive - {ReqId, done} -> - {mochijson2:encode({struct, - [{<<"buckets">>, []}]}), done}; - {ReqId, _From, {buckets_stream, Buckets}} -> - {mochijson2:encode({struct, [{<<"buckets">>, Buckets}]}), - fun() -> stream_buckets(ReqId) end}; - {ReqId, {buckets_stream, Buckets}} -> - {mochijson2:encode({struct, [{<<"buckets">>, Buckets}]}), - fun() -> stream_buckets(ReqId) end}; - {ReqId, {error, timeout}} -> {mochijson2:encode({struct, [{error, timeout}]}), done} - end. diff --git a/src/riak_kv_wm_crdt.erl b/src/riak_kv_wm_crdt.erl deleted file mode 100644 index 21e6054c6..000000000 --- a/src/riak_kv_wm_crdt.erl +++ /dev/null @@ -1,546 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% riak_kv_wm_crdt: Webmachine resource for convergent data types -%% -%% Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- -%% @doc Resource for serving data-types over HTTP. -%% -%% Available operations: -%% -%% `GET /types/BucketType/buckets/Bucket/datatypes/Key' -%% -%% Get the current value of the data-type at `BucketType', `Bucket', `Key'. -%% -%% Result is a JSON body with a structured value, or `404 Not Found' if no -%% datatype exists at that resource location. -%% -%% The format of the JSON response will be roughly -%% `{"type":..., "value":..., "context":...}', where the -%% `type' is a string designating which data-type is presented, the -%% `value' is a representation of the data-type's value (see below), -%% and the `context' is the opaque context, if needed or requested. -%% -%% The type and structure of the `value' field in the response -%% depends on the `type' field. -%%
-%%
counter
an integer
-%%
set
an array of strings
-%%
map
an object where the fields are as described below.
-%%
-%% -%% The format of a field name in the map value determines both the -%% name of the entry and the type, joined with an underscore. For -%% example, a `register' with name `firstname' would be -%% `"firstname_register"'. Valid types embeddable in a map are -%% `counter', `flag', `register', `set', and `map'. -%% -%% The following query params are accepted: -%% -%%
-%%
r
Read quorum. See below for defaults and values.
-%%
pr
Primary read quorum. See below for defaults and values.
-%%
basic_quorum
Boolean. Return as soon as a quorum of responses are received -%% if true. Default is the bucket default, if absent.
-%%
notfound_ok
Boolean. A `not_found` response from a vnode counts toward -%% `r' quorum if true. Default is the bucket default, if absent.
-%%
include_context
Boolean. If the datatype requires the opaque "context" for -%% safe removal, include it in the response. Defaults to `true'.
-%%
-%% -%% ``` -%% POST /types/BucketType/buckets/Bucket/datatypes -%% POST /types/BucketType/buckets/Bucket/datatypes/Key''' -%% -%% Mutate the data-type at `BucketType', `Bucket', `Key' by applying -%% the submitted operation contained in a JSON payload. If `Key' is -%% not specified, one will be generated for the client and included -%% in the returned `Location' header. -%% -%% The format of the operation payload depends on the data-type. -%%
-%%
counter
An integer, or an object containing a single field, either -%% `"increment"' or `"decrement"', and an associated integer value.
-%%
set
An object containing any combination of `"add"', `"add_all"', -%% `"remove"', `"remove_all"' fields. `"add"' and `"remove"' should refer to -%% single string values, while `"add_all"' and `"remove_all"' should be arrays -%% of strings. The `"context"' field may be included.
-%%
map
An object containing any of the fields `"remove"', or `"update"'. -%% `"remove"' should be a list of field names as described above. -%% `"update"` should be an object containing fields and the operation to apply -%% to the type associated with the field.
-%%
register (embedded in map only)
`{"assign":Value}' where `Value' is the new string -%% value of the register.
-%%
flag (embedded in map only)
The string "enable" or "disable".
-%%
-%% -%% The following query params are accepted (see {@link riak_kv_wm_object} docs, too): -%% -%%
-%%
w
The write quorum. See below for defaults and values.
-%%
pw
The primary write quorum. See below for defaults and values.
-%%
dw
The durable write quorum. See below for default and values.
-%%
returnbody
Boolean. Default is `false' if not provided. When `true' -%% the response body will be the value of the datatype.
-%%
include_context
Boolean. Default is `true' if not provided. When `true' -%% and `returnbody' is `true', the opaque context will be included.
-%%
-%% -%% == Quorum values (r/pr/w/pw/dw) == -%%
-%%
default
Whatever the bucket default is. This is the value used -%% for any absent value.
-%%
quorum
(Bucket N val / 2) + 1
-%%
all
All replicas must respond
-%%
one
Any one response is enough
-%%
Integer
That specific number of vnodes must respond. Must be `=<' N
-%%
-%% -%% Please see [http://docs.basho.com] for details of all the quorum values and their effects. - --module(riak_kv_wm_crdt). --record(ctx, { - api_version, - client, %% riak:local_client() - bucket_type, - bucket, - key, - crdt_type, - data, - module, - r, - w, - dw, - rw, - pr, - pw, - node_confirms, - basic_quorum, - notfound_ok, - include_context, - returnbody, - method, - timeout, - security - }). --include("riak_kv_wm_raw.hrl"). --include("riak_kv_types.hrl"). - --export([ - init/1, - service_available/2, - malformed_request/2, - is_authorized/2, - forbidden/2, - allowed_methods/2, - content_types_provided/2, - encodings_provided/2, - resource_exists/2, - process_post/2, %% POST handler - produce_json/2 %% GET/HEAD handler - ]). - --include_lib("webmachine/include/webmachine.hrl"). - -init(Props) -> - {ok, #ctx{api_version=proplists:get_value(api_version, Props)}}. - -service_available(RD, Ctx0) -> - Ctx = riak_kv_wm_utils:ensure_bucket_type(RD, Ctx0, #ctx.bucket_type), - {ok, Client} = riak_kv_wm_utils:get_riak_client( - local, riak_kv_wm_utils:get_client_id(RD)), - {true, RD, - Ctx#ctx{client=Client, - bucket=path_segment_to_bin(bucket, RD), - key=path_segment_to_bin(key, RD), - method=wrq:method(RD)}}. - -allowed_methods(RD, Ctx) -> - {['GET', 'HEAD', 'POST'], RD, Ctx}. - -is_authorized(ReqData, Ctx) -> - case riak_api_web_security:is_authorized(ReqData) of - false -> - {"Basic realm=\"Riak\"", ReqData, Ctx}; - {true, SecContext} -> - {true, ReqData, Ctx#ctx{security=SecContext}}; - insecure -> - %% XXX 301 may be more appropriate here, but since the http and - %% https port are different and configurable, it is hard to figure - %% out the redirect URL to serve. - halt_with_message(426, - <<"Security is enabled and Riak does not accept " - "credentials over HTTP. Try HTTPS instead.">>, - ReqData, Ctx) - end. - -malformed_request(RD, Ctx=#ctx{method='POST'}) -> - malformed_check_post_ctype(RD, Ctx); -malformed_request(RD, Ctx) -> - malformed_rw_params(RD, Ctx). - -malformed_check_post_ctype(RD, Ctx) -> - CType = wrq:get_req_header(?HEAD_CTYPE, RD), - case mochiweb_util:parse_header(CType) of - {"application/json",_} -> - malformed_rw_params(RD, Ctx); - _Other -> - {{halt, 406}, RD, Ctx} - end. - -malformed_rw_params(RD, Ctx) -> - Res = lists:foldl(fun malformed_rw_param/2, - {false, RD, Ctx}, - [{#ctx.r, "r", "default"}, - {#ctx.w, "w", "default"}, - {#ctx.dw, "dw", "default"}, - {#ctx.pw, "pw", "default"}, - {#ctx.node_confirms, "node_confirms", "default"}, - {#ctx.pr, "pr", "default"}]), - Res1 = lists:foldl(fun malformed_boolean_param/2, - Res, - [{#ctx.basic_quorum, "basic_quorum", "default"}, - {#ctx.notfound_ok, "notfound_ok", "default"}, - {#ctx.include_context, "include_context", "true"}, - {#ctx.returnbody, "returnbody", "false"}]), - malformed_timeout_param(Res1). - -malformed_rw_param({Idx, Name, Default}, {Result, RD, Ctx}) -> - case catch normalize_rw_param(wrq:get_qs_value(Name, Default, RD)) of - P when (is_atom(P) orelse is_integer(P)) -> - {Result, RD, setelement(Idx, Ctx, P)}; - _ -> - {true, - error_response("~s query parameter must be an integer or " - "one of the following words: 'one', 'quorum' or 'all'~n", - [Name], RD), - Ctx} - end. - -malformed_boolean_param({Idx, Name, Default}, {Result, RD, Ctx}) -> - case string:to_lower(wrq:get_qs_value(Name, Default, RD)) of - "true" -> - {Result, RD, setelement(Idx, Ctx, true)}; - "false" -> - {Result, RD, setelement(Idx, Ctx, false)}; - "default" -> - {Result, RD, setelement(Idx, Ctx, default)}; - _ -> - {true, - error_response("~s query parameter must be true or false~n", - [Name], RD), - Ctx} - end. - -malformed_timeout_param({Result, RD, Ctx}) -> - case wrq:get_qs_value("timeout", RD) of - undefined -> - {Result, RD, Ctx}; - TimeoutStr when is_list(TimeoutStr) -> - try list_to_integer(TimeoutStr) of - 0 -> - {Result, RD, Ctx#ctx{timeout=infinity}}; - Timeout when is_integer(Timeout), Timeout > 0 -> - {Result, RD, Ctx#ctx{timeout=Timeout}}; - _Other -> - {true, - error_response("timeout query parameter must be an " - "integer greater than 0 (or 0 for disabled), " - "~s is invalid~n", - [TimeoutStr], RD), - Ctx} - catch - error:badarg -> - {true, - error_response("timeout query parameter must be an " - "integer greater than 0, ~s is invalid~n", - [TimeoutStr], RD), - Ctx} - end - end. - -forbidden(RD, Ctx) -> - case riak_kv_wm_utils:is_forbidden(RD) of - true -> - {true, RD, Ctx}; - false -> - forbidden_check_security(RD, Ctx) - end. - -forbidden_check_security(RD, Ctx=#ctx{security=undefined}) -> - forbidden_check_bucket_type(RD, Ctx); -forbidden_check_security(RD, Ctx=#ctx{bucket_type=BType, bucket=Bucket, - security=SecContext, method=Method}) -> - Perm = permission(Method), - case riak_core_security:check_permission({Perm, {BType, Bucket}}, - SecContext) of - {false, Error, _} -> - {true, error_response(Error, RD), Ctx}; - {true, _} -> - forbidden_check_bucket_type(RD, Ctx) - end. - -%% @doc Detects whether the requested object's bucket-type exists. -forbidden_check_bucket_type(RD, Ctx) -> - case riak_kv_wm_utils:bucket_type_exists(Ctx#ctx.bucket_type) of - true -> - forbidden_check_crdt_type(RD, Ctx); - false -> - handle_common_error(bucket_type_unknown, RD, Ctx) - end. - -forbidden_check_crdt_type(RD, Ctx=#ctx{bucket_type = <<"default">>, - bucket=B0, - key=K0}) -> - %% Only legacy/1.4 counters are supported in the default/undefined - %% bucket type. Since we don't want to confuse semantics of the - %% new types or duplicate code, we redirect to the old resource - %% instead. - B = mochiweb_util:quote_plus(B0), - K = mochiweb_util:quote_plus(K0), - CountersUrl = lists:flatten( - io_lib:format("/buckets/~s/counters/~s",[B, K])), - halt_with_message(301, - "Counters in the default bucket-type should use the " - "legacy URL\n", - wrq:set_resp_header("Location", CountersUrl, RD), - Ctx); -forbidden_check_crdt_type(RD, Ctx=#ctx{bucket_type=T, bucket=B}) -> - case riak_core_bucket:get_bucket({T, B}) of - BProps when is_list(BProps) -> - DataType = proplists:get_value(datatype, BProps), - AllowMult = proplists:get_value(allow_mult, BProps), - Mod = riak_kv_crdt:to_mod(DataType), - case {AllowMult, riak_kv_crdt:supported(Mod)} of - {false, _} -> - {true, error_response("Bucket must be allow_mult=true~n", - [], RD), Ctx}; - {_, false} -> - {true, error_response("Bucket datatype '~s' is not a " - "supported type.~n", [DataType], RD), Ctx}; - _ -> - {false, RD, Ctx#ctx{module=Mod, crdt_type=DataType}} - end; - {error, no_type} -> - %% This should be handled by forbidden_check_bucket_type/2 - handle_common_error(bucket_type_unknown, RD, Ctx) - end. - -content_types_provided(RD, Ctx) -> - {[{"application/json", produce_json}], RD, Ctx}. - -encodings_provided(RD, Ctx) -> - {riak_kv_wm_utils:default_encodings(), RD, Ctx}. - -resource_exists(RD, Ctx=#ctx{method='POST'}) -> - %% When submitting an operation, the resource always exists, even - %% if key is unspecified. - {true, RD, Ctx}; -resource_exists(RD, Ctx=#ctx{key=undefined}) -> - %% When fetching, if the key does not exist, we should give a not - %% found. - handle_common_error(notfound, RD, Ctx); -resource_exists(RD, Ctx=#ctx{client=C, bucket_type=T, bucket=B, key=K, module=Mod}) -> - Options = [{crdt_op, Mod}|make_options(Ctx)], - case riak_client:get({T, B}, K, Options, C) of - {ok, O} -> - {true, RD, Ctx#ctx{data=O}}; - {error, Reason} -> - handle_common_error(Reason, RD, Ctx) - end. - -process_post(RD0, Ctx0=#ctx{client=C, bucket_type=T, bucket=B, module=Mod}) -> - case check_post_body(RD0, Ctx0) of - {error, RD} -> - {{halt, 400}, RD, Ctx0}; - {ok, {_Type, Op, OpCtx}} -> - {RD, Ctx} = maybe_generate_key(RD0, Ctx0), - O = riak_kv_crdt:new({T, B}, Ctx#ctx.key, Mod), - Options0 = make_options(Ctx), - CrdtOp = make_operation(Mod, Op, OpCtx), - Options = [{crdt_op, CrdtOp}, - {retry_put_coordinator_failure,false}|Options0], - case riak_client:put(O, Options, C) of - ok -> - {true, RD, Ctx}; - {ok, RObj} -> - {Body, RD1, Ctx1} = produce_json(RD, Ctx#ctx{data=RObj}), - {true, - wrq:set_resp_body(Body, wrq:set_resp_header( - ?HEAD_CTYPE, "application/json", - RD1)), - Ctx1}; - {error, Reason} -> - handle_common_error(Reason, RD, Ctx) - end - end. - -produce_json(RD, Ctx=#ctx{module=Mod, data=RObj, include_context=I}) -> - Type = riak_kv_crdt:from_mod(Mod), - {{RespCtx, Value}, Stats} = riak_kv_crdt:value(RObj, Mod), - _ = [ ok = riak_kv_stat:update(S) || S <- Stats ], - ModMap = riak_kv_crdt:mod_map(Type), - Body = riak_kv_crdt_json:fetch_response_to_json( - Type, Value, get_context(RespCtx,I), ModMap), - {mochijson2:encode(Body), RD, Ctx}. - -%% Internal functions - -check_post_body(RD, #ctx{crdt_type=CRDTType}) -> - try - JSON = mochijson2:decode(wrq:req_body(RD)), - ModMap = riak_kv_crdt:mod_map(CRDTType), - Data = {CRDTType, _Op, _Context} = - riak_kv_crdt_json:update_request_from_json(CRDTType, JSON, - ModMap), - {ok, Data} - catch - throw:{invalid_operation, {BadType, BadOp}} -> - {error, - error_response("Invalid operation on datatype '~s': ~s~n", - [BadType, mochijson2:encode(BadOp)], RD)}; - throw:{invalid_field_name, Field} -> - {error, - error_response("Invalid map field name '~s'~n", [Field], RD)}; - throw:invalid_utf8 -> - {error, - error_response("Malformed JSON submitted, invalid UTF-8", RD)}; - _Other:Reason -> - {error, - error_response("Couldn't decode JSON: ~p~n", [Reason], RD)} - end. - - -%% @doc Converts a query string value into a quorum value. -normalize_rw_param("default") -> default; -normalize_rw_param("one") -> one; -normalize_rw_param("quorum") -> quorum; -normalize_rw_param("all") -> all; -normalize_rw_param(V) -> list_to_integer(V). - -%% @doc Returns the appropriate permission for a given request method. -permission('POST') -> "riak_kv.put"; -permission('GET') -> "riak_kv.get"; -permission('HEAD') -> "riak_kv.get". - -%% @doc Halts the resource with the given formatted response message. -halt_with_message(Status, Format, Args, RD, Ctx) -> - halt_with_message(Status, io_lib:format(Format, Args), RD, Ctx). - -%% @doc Halts the resource with the given response message. -halt_with_message(Status, Message, RD, Ctx) -> - {{halt, Status}, error_response(Message,RD), Ctx}. - -%% @doc Outputs a formatted error response with the text/plain content type. -error_response(Fmt, Args, RD) -> - error_response(io_lib:format(Fmt, Args), RD). - -%% @doc Outputs an error response with the text/plain content type. -error_response(Msg, RD) -> - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", - wrq:append_to_response_body(Msg, RD)). - -%% @doc Converts an error into the appropriate resource halt and message. -handle_common_error(Reason, RD, Ctx) -> - case Reason of - too_many_fails -> - halt_with_message(503, "Too many write failures to satisfy W/DW\n", - RD, Ctx); - timeout -> - halt_with_message(503, "request timed out\n", RD, Ctx); - notfound -> - {{halt, 404}, notfound_body(RD, Ctx), Ctx}; - bucket_type_unknown -> - halt_with_message(404, "Unknown bucket type: ~s~n", - [Ctx#ctx.bucket_type], RD, Ctx); - {deleted, _VClock} -> - {{halt,404}, - wrq:set_resp_header(?HEAD_DELETED, "true", notfound_body(RD, Ctx)), - Ctx}; - {n_val_violation, N} -> - halt_with_message(400, - "Specified w/dw/pw/node_confirms values invalid for bucket n " - "value of ~p~n",[N], RD, Ctx); - {r_val_unsatisfied, Requested, Returned} -> - halt_with_message(503, "R-value unsatisfied: ~p/~p~n", - [Returned, Requested], RD, Ctx); - {dw_val_unsatisfied, DW, NumDW} -> - halt_with_message(503, "DW-value unsatisfied: ~p/~p~n", [NumDW, DW], - RD, Ctx); - {pr_val_unsatisfied, Requested, Returned} -> - halt_with_message(503, "PR-value unsatisfied: ~p/~p~n", - [Returned, Requested], RD, Ctx); - {pw_val_unsatisfied, Requested, Returned} -> - halt_with_message(503, "PW-value unsatisfied: ~p/~p~n", - [Returned, Requested], RD, Ctx); - {node_confirms_val_unsatisfied, Requested, Returned} -> - halt_with_message(503, "node_confirms-value unsatisfied: ~p/~p~n", - [Returned, Requested], RD, Ctx); - failed -> - halt_with_message(412, "", RD, Ctx); - Err -> - halt_with_message(500, "Error:~n~p~n", [Err], RD, Ctx) - end. - -%% @doc Converts a path segment into a binary by key. -path_segment_to_bin(Key, RD) -> - Segment = proplists:get_value(Key, wrq:path_info(RD)), - case Segment of - undefined -> undefined; - _ -> - list_to_binary(riak_kv_wm_utils:maybe_decode_uri(RD, Segment)) - end. - -%% @doc If the key is not submitted on POST, generate a key and set -%% the appropriate redirect location. -maybe_generate_key(RD, Ctx=#ctx{api_version=V, bucket_type=T, bucket=B, - key=undefined}) -> - K = riak_core_util:unique_id_62(), - {wrq:set_resp_header("Location", - riak_kv_wm_utils:format_uri(T, B, K, undefined, V), RD), - Ctx#ctx{key=list_to_binary(K)}}; -maybe_generate_key(RD, Ctx) -> - {RD, Ctx}. - -make_operation(Mod, Op, Ctx) -> - #crdt_op{mod=Mod, op=Op, ctx=Ctx}. - -get_context(_Ctx, false) -> - undefined; -get_context(Ctx, true) -> - Ctx. - -make_options(Ctx) -> - OptList = [{r, Ctx#ctx.r}, - {w, Ctx#ctx.w}, - {dw, Ctx#ctx.dw}, - {rw, Ctx#ctx.rw}, - {pr, Ctx#ctx.pr}, - {pw, Ctx#ctx.pw}, - {node_confirms, Ctx#ctx.node_confirms}, - {basic_quorum, Ctx#ctx.basic_quorum}, - {notfound_ok, Ctx#ctx.notfound_ok}, - {timeout, Ctx#ctx.timeout}, - {returnbody, Ctx#ctx.returnbody}], - [ {K,V} || {K,V} <- OptList, V /= default, V /= undefined ]. - -notfound_body(RD, #ctx{module=Mod}) -> - JSON = {struct, [{<<"type">>, atom_to_binary(riak_kv_crdt:from_mod(Mod), utf8)}, - {<<"error">>, <<"notfound">>}]}, - wrq:set_resp_header(?HEAD_CTYPE, "application/json", - wrq:set_resp_body(mochijson2:encode(JSON), RD)). diff --git a/src/riak_kv_wm_index.erl b/src/riak_kv_wm_index.erl deleted file mode 100644 index b8f27c088..000000000 --- a/src/riak_kv_wm_index.erl +++ /dev/null @@ -1,764 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2016 Basho Technologies, Inc. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - -%% @doc Webmachine resource for running queries on secondary indexes. -%% -%% Available operations: -%% -%% ``` -%% GET /buckets/Bucket/index/IndexName/Key -%% GET /buckets/Bucket/index/IndexName/Start/End''' -%% -%% Run an index lookup, return the results as JSON. -%% -%% ``` -%% GET /types/Type/buckets/Bucket/index/IndexName/Key -%% GET /types/Type/buckets/Bucket/index/IndexName/Start/End''' -%% -%% Run an index lookup over the Bucket in BucketType, returning the -%% results in JSON. -%% -%% Both URL formats support the following query-string options: -%%
    -%%
  • max_results=Integer
    -%% limits the number of results returned
    • -%%
  • stream=true
    -%% streams the results back in multipart/mixed chunks
    • -%%
  • continuation=C
    -%% the continuation returned from the last query, used to -%% fetch the next "page" of results.
    • -%%
  • return_terms=true
    -%% when querying with a range, returns the value of the index -%% along with the key.
    • -%%
  • pagination_sort=true|false
    -%% whether the results will be sorted. Ignored when max_results -%% is set, as pagination requires sorted results.
    • -%%
- --module(riak_kv_wm_index). - -%% webmachine resource exports --export([ - init/1, - service_available/2, - is_authorized/2, - forbidden/2, - malformed_request/2, - content_types_provided/2, - encodings_provided/2, - resource_exists/2, - produce_index_results/2 -]). --export([ - mochijson_encode_results/2, - mochijson_encode_results/3, - otp_encode_results/2, - otp_encode_results/3, - results_encode/2, - keys_encode/2 -]). - --record(ctx, { - client, %% riak_client() - the store client - riak, %% local | {node(), atom()} - params for riak client - bucket_type, %% Bucket type (from uri) - bucket, %% The bucket to query. - index_query, %% The query.. - streamed = false :: boolean(), %% stream results to client - max_results :: all | pos_integer(), %% maximum number of 2i results to return, the page size. - return_terms = false :: boolean(), %% should the index values be returned - timeout :: non_neg_integer() | undefined | infinity, - pagination_sort :: boolean() | undefined, - security %% security context - }). --type context() :: #ctx{}. - --define(DEFAULT_JSON_ENCODING, otp). - --include_lib("kernel/include/logger.hrl"). - --include_lib("webmachine/include/webmachine.hrl"). --include("riak_kv_wm_raw.hrl"). --include("riak_kv_index.hrl"). - --spec init(proplists:proplist()) -> {ok, context()}. -%% @doc Initialize this resource. -init(Props) -> - {ok, #ctx{ - riak=proplists:get_value(riak, Props), - bucket_type=proplists:get_value(bucket_type, Props), - max_results=all % may be refined once params evaluated - }}. - - --spec service_available(#wm_reqdata{}, context()) -> - {boolean(), #wm_reqdata{}, context()}. -%% @doc Determine whether or not a connection to Riak -%% can be established. Also, extract query params. -service_available(RD, Ctx0=#ctx{riak=RiakProps}) -> - Ctx = riak_kv_wm_utils:ensure_bucket_type(RD, Ctx0, #ctx.bucket_type), - case riak_kv_wm_utils:get_riak_client(RiakProps, riak_kv_wm_utils:get_client_id(RD)) of - {ok, C} -> - {true, RD, Ctx#ctx { client=C }}; - Error -> - {false, - wrq:set_resp_body( - io_lib:format("Unable to connect to Riak: ~p~n", [Error]), - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD)), - Ctx} - end. - -is_authorized(ReqData, Ctx) -> - case riak_api_web_security:is_authorized(ReqData) of - false -> - {"Basic realm=\"Riak\"", ReqData, Ctx}; - {true, SecContext} -> - {true, ReqData, Ctx#ctx{security=SecContext}}; - insecure -> - %% XXX 301 may be more appropriate here, but since the http and - %% https port are different and configurable, it is hard to figure - %% out the redirect URL to serve. - {{halt, 426}, wrq:append_to_resp_body(<<"Security is enabled and " - "Riak does not accept credentials over HTTP. Try HTTPS " - "instead.">>, ReqData), Ctx} - end. - --spec forbidden(#wm_reqdata{}, context()) - -> {boolean(), #wm_reqdata{}, context()}. -forbidden(ReqDataIn, #ctx{security = undefined} = Context) -> - Class = request_class(ReqDataIn), - riak_kv_wm_utils:is_forbidden(ReqDataIn, Class, Context); -forbidden(ReqDataIn, #ctx{bucket_type = BT, security = Sec} = Context) -> - Class = request_class(ReqDataIn), - {Answer, ReqData, _} = Result = - riak_kv_wm_utils:is_forbidden(ReqDataIn, Class, Context), - case Answer of - false -> - Bucket = erlang:list_to_binary( - riak_kv_wm_utils:maybe_decode_uri( - ReqData, wrq:path_info(bucket, ReqData))), - case riak_core_security:check_permission( - {"riak_kv.index", {BT, Bucket}}, Sec) of - {false, Error, _} -> - {true, - wrq:append_to_resp_body( - unicode:characters_to_binary(Error, utf8, utf8), - wrq:set_resp_header( - "Content-Type", "text/plain", ReqData)), - Context}; - {true, _} -> - {false, ReqData, Context} - end; - _ -> - Result - end. - --spec request_class(#wm_reqdata{}) -> term(). -request_class(ReqData) -> - case wrq:get_qs_value(?Q_STREAM, ?Q_FALSE, ReqData) of - ?Q_TRUE -> - {riak_kv, stream_secondary_index}; - _ -> - {riak_kv, secondary_index} - end. - --spec malformed_request(#wm_reqdata{}, context()) -> - {boolean(), #wm_reqdata{}, context()}. -%% @doc Determine whether query parameters are badly-formed. -%% Specifically, we check that the index operation is of -%% a known type. -malformed_request(RD, Ctx) -> - %% Pull the params... - Bucket = list_to_binary(riak_kv_wm_utils:maybe_decode_uri(RD, wrq:path_info(bucket, RD))), - IndexField = list_to_binary(riak_kv_wm_utils:maybe_decode_uri(RD, wrq:path_info(field, RD))), - Args1 = wrq:path_tokens(RD), - Args2 = [list_to_binary(riak_kv_wm_utils:maybe_decode_uri(RD, X)) || X <- Args1], - ReturnTerms0 = wrq:get_qs_value(?Q_2I_RETURNTERMS, "false", RD), - ReturnTerms1 = normalize_boolean(string:to_lower(ReturnTerms0)), - Continuation = wrq:get_qs_value(?Q_2I_CONTINUATION, RD), - PgSort0 = wrq:get_qs_value(?Q_2I_PAGINATION_SORT, RD), - PgSort = case PgSort0 of - undefined -> undefined; - _ -> normalize_boolean(string:to_lower(PgSort0)) - end, - MaxVal = validate_max(wrq:get_qs_value(?Q_2I_MAX_RESULTS, RD)), - TermRegex = wrq:get_qs_value(?Q_2I_TERM_REGEX, RD), - Timeout0 = wrq:get_qs_value("timeout", RD), - {Start, End} = case {IndexField, Args2} of - {<<"$bucket">>, _} -> {undefined, undefined}; - {_, []} -> {undefined, undefined}; - {_, [S]} -> {S, S}; - {_, [S, E]} -> {S, E} - end, - IsEqualOp = length(Args1) == 1, - InternalReturnTerms = not( IsEqualOp orelse IndexField == <<"$field">> ), - QRes = riak_index:to_index_query( - [ - {field, IndexField}, {start_term, Start}, {end_term, End}, - {return_terms, InternalReturnTerms}, - {continuation, Continuation}, - {term_regex, TermRegex} - ] - ++ [{max_results, MaxResults} || {true, MaxResults} <- [MaxVal]] - ), - ValRe = case TermRegex of - undefined -> - ok; - _ -> - re:compile(TermRegex) - end, - - Stream0 = wrq:get_qs_value("stream", "false", RD), - Stream = normalize_boolean(string:to_lower(Stream0)), - - case {PgSort, ReturnTerms1, validate_timeout(Timeout0), MaxVal, - QRes, - ValRe, Stream} of - {malformed, _, _, _, - _, - _, _} -> - {true, - wrq:set_resp_body(io_lib:format("Invalid ~p. ~p is not a boolean", - [?Q_2I_PAGINATION_SORT, PgSort0]), - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD)), - Ctx}; - {_, malformed, _, _, - _, - _, _} -> - {true, - wrq:set_resp_body(io_lib:format("Invalid ~p. ~p is not a boolean", - [?Q_2I_RETURNTERMS, ReturnTerms0]), - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD)), - Ctx}; - {_, _, _, _, - {ok, ?KV_INDEX_Q{start_term=NormStart}}, - {ok, _CompiledRe}, _} - when is_integer(NormStart) -> - {true, - wrq:set_resp_body("Can not use term regular expressions" - " on integer queries", - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD)), - Ctx}; - {_, _, _, _, - _, - {error, ReError}, _} -> - {true, - wrq:set_resp_body( - io_lib:format("Invalid term regular expression ~p : ~p", - [TermRegex, ReError]), - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD)), - Ctx}; - {_, _, {true, Timeout}, {true, MaxResults}, - {ok, Query}, - _, _} -> - %% Request is valid. - ReturnTerms2 = riak_index:return_terms(ReturnTerms1, Query), - %% Special case: a continuation implies pagination sort - %% even if no max_results was given. - PgSortFinal = case Continuation of - undefined -> PgSort; - _ -> true - end, - NewCtx = Ctx#ctx{ - bucket = Bucket, - index_query = Query, - max_results = MaxResults, - return_terms = ReturnTerms2, - timeout=Timeout, - pagination_sort = PgSortFinal, - streamed = Stream - }, - {false, RD, NewCtx}; - {_, _, _, _, - {error, Reason}, - _, _} -> - {true, - wrq:set_resp_body( - io_lib:format("Invalid query: ~p~n", [Reason]), - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD)), - Ctx}; - {_, _, _, {false, BadVal}, - _, - _, _} -> - {true, - wrq:set_resp_body(io_lib:format("Invalid ~p. ~p is not a positive integer", - [?Q_2I_MAX_RESULTS, BadVal]), - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD)), - Ctx}; - {_, _, {error, Input}, _, - _, - _, _} -> - {true, wrq:append_to_resp_body(io_lib:format("Bad timeout " - "value ~p. Must be a non-negative integer~n", - [Input]), - wrq:set_resp_header(?HEAD_CTYPE, - "text/plain", RD)), Ctx} - end. - -validate_timeout(undefined) -> - {true, undefined}; -validate_timeout(Str) -> - try - list_to_integer(Str) of - Int when Int >= 0 -> - {true, Int}; - Neg -> - {error, Neg} - catch - _:_ -> - {error, Str} - end. - -validate_max(undefined) -> - {true, all}; -validate_max(N) when is_list(N) -> - try - list_to_integer(N) of - Max when Max > 0 -> - {true, Max}; - ZeroOrLess -> - {false, ZeroOrLess} - catch _:_ -> - {false, N} - end. - -normalize_boolean("false") -> - false; -normalize_boolean("true") -> - true; -normalize_boolean(_) -> - malformed. - --spec content_types_provided(#wm_reqdata{}, context()) -> - {[{ContentType::string(), Producer::atom()}], #wm_reqdata{}, context()}. -%% @doc List the content types available for representing this resource. -%% "application/json" is the content-type for bucket lists. -content_types_provided(RD, Ctx) -> - {[{"application/json", produce_index_results}], RD, Ctx}. - - --spec encodings_provided(#wm_reqdata{}, context()) -> - {[{Encoding::string(), Producer::function()}], #wm_reqdata{}, context()}. -%% @doc List the encodings available for representing this resource. -%% "identity" and "gzip" are available for bucket lists. -encodings_provided(RD, Ctx) -> - {riak_kv_wm_utils:default_encodings(), RD, Ctx}. - - -resource_exists(RD, #ctx{bucket_type=BType}=Ctx) -> - {riak_kv_wm_utils:bucket_type_exists(BType), RD, Ctx}. - --spec produce_index_results(#wm_reqdata{}, context()) -> - {binary(), #wm_reqdata{}, context()}. -%% @doc Produce the JSON response to an index lookup. -produce_index_results(RD, Ctx) -> - case wrq:get_qs_value("stream", "false", RD) of - "true" -> - handle_streaming_index_query(RD, Ctx); - _ -> - handle_all_in_memory_index_query(RD, Ctx) - end. - -handle_streaming_index_query(RD, Ctx) -> - Client = Ctx#ctx.client, - Bucket = riak_kv_wm_utils:maybe_bucket_type(Ctx#ctx.bucket_type, Ctx#ctx.bucket), - Query = Ctx#ctx.index_query, - MaxResults = Ctx#ctx.max_results, - ReturnTerms = Ctx#ctx.return_terms, - Timeout = Ctx#ctx.timeout, - PgSort = Ctx#ctx.pagination_sort, - - %% Create a new multipart/mixed boundary - Boundary = riak_core_util:unique_id_62(), - CTypeRD = wrq:set_resp_header( - "Content-Type", - "multipart/mixed;boundary="++Boundary, - RD), - - Opts0 = [{max_results, MaxResults}] ++ [{pagination_sort, PgSort} || PgSort /= undefined], - Opts = riak_index:add_timeout_opt(Timeout, Opts0), - - {ok, ReqID, FSMPid} = - riak_client:stream_get_index(Bucket, Query, Opts, Client), - StreamFun = index_stream_helper(ReqID, FSMPid, Boundary, ReturnTerms, MaxResults, proplists:get_value(timeout, Opts), undefined, 0), - {{stream, {<<>>, StreamFun}}, CTypeRD, Ctx}. - -index_stream_helper(ReqID, FSMPid, Boundary, ReturnTerms, MaxResults, Timeout, LastResult, Count) -> - fun() -> - receive - {ReqID, done} -> - Final = case make_continuation(MaxResults, [LastResult], Count) of - undefined -> ["\r\n--", Boundary, "--\r\n"]; - Continuation -> - Json = mochijson2:encode(mochify_continuation(Continuation)), - ["\r\n--", Boundary, "\r\n", - "Content-Type: application/json\r\n\r\n", - Json, - "\r\n--", Boundary, "--\r\n"] - end, - {iolist_to_binary(Final), done}; - {ReqID, {results, []}} -> - {<<>>, index_stream_helper(ReqID, FSMPid, Boundary, ReturnTerms, MaxResults, Timeout, LastResult, Count)}; - {ReqID, {results, Results}} -> - %% JSONify the results - JsonResults = encode_results(ReturnTerms, Results), - Body = ["\r\n--", Boundary, "\r\n", - "Content-Type: application/json\r\n\r\n", - JsonResults], - LastResult1 = last_result(Results), - Count1 = Count + length(Results), - {iolist_to_binary(Body), - index_stream_helper(ReqID, FSMPid, Boundary, ReturnTerms, MaxResults, Timeout, LastResult1, Count1)}; - {ReqID, Error} -> - stream_error(Error, Boundary) - after Timeout -> - whack_index_fsm(ReqID, FSMPid), - stream_error({error, timeout}, Boundary) - end - end. - -whack_index_fsm(ReqID, Pid) -> - wait_for_death(Pid), - clear_index_fsm_msgs(ReqID). - -wait_for_death(Pid) -> - Ref = erlang:monitor(process, Pid), - exit(Pid, kill), - receive - {'DOWN', Ref, process, Pid, _Info} -> - ok - end. - -clear_index_fsm_msgs(ReqID) -> - receive - {ReqID, _} -> - clear_index_fsm_msgs(ReqID) - after - 0 -> - ok - end. - -stream_error(Error, Boundary) -> - ?LOG_ERROR("Error in index wm: ~p", [Error]), - ErrorJson = encode_error(Error), - Body = ["\r\n--", Boundary, "\r\n", - "Content-Type: application/json\r\n\r\n", - ErrorJson, - "\r\n--", Boundary, "--\r\n"], - {iolist_to_binary(Body), done}. - -encode_error({error, E}) -> - encode_error(E); -encode_error(Error) when is_atom(Error); is_binary(Error) -> - mochijson2:encode({struct, [{error, Error}]}); -encode_error(Error) -> - E = io_lib:format("~p",[Error]), - mochijson2:encode({struct, [{error, erlang:iolist_to_binary(E)}]}). - -handle_all_in_memory_index_query(RD, Ctx) -> - Client = Ctx#ctx.client, - Bucket = riak_kv_wm_utils:maybe_bucket_type(Ctx#ctx.bucket_type, Ctx#ctx.bucket), - Query = Ctx#ctx.index_query, - Timeout = Ctx#ctx.timeout, - - % Standard secondary index query - MaxResults = Ctx#ctx.max_results, - ReturnTerms = Ctx#ctx.return_terms, - PgSort = Ctx#ctx.pagination_sort, - - Opts0 = - [{max_results, MaxResults}] ++ - [{pagination_sort, PgSort} || PgSort /= undefined], - Opts = - riak_index:add_timeout_opt(Timeout, Opts0), - - %% Do the index lookup... - case riak_client:get_index(Bucket, Query, Opts, Client) of - {ok, Results} -> - Continuation = - make_continuation( - MaxResults, Results, length(Results)), - JsonResults = - encode_results( - ReturnTerms, Results, Continuation), - {JsonResults, RD, Ctx}; - {error, timeout} -> - {{halt, 503}, - wrq:set_resp_header("Content-Type", "text/plain", - wrq:append_to_response_body( - io_lib:format("request timed out~n", - []), - RD)), - Ctx}; - {error, Reason} -> - {{error, Reason}, RD, Ctx} - end. - - -%% @doc Like `lists:last/1' but doesn't choke on an empty list --spec last_result([] | list()) -> term() | undefined. -last_result([]) -> - undefined; -last_result(L) -> - lists:last(L). - -%% @doc if this is a paginated query make a continuation --spec make_continuation(Max::non_neg_integer() | undefined, - list(), - ResultCount::non_neg_integer()) -> binary() | undefined. -make_continuation(MaxResults, Results, MaxResults) -> - riak_index:make_continuation(Results); -make_continuation(_, _, _) -> - undefined. - -%% =================================================================== -%% JSON Encoding implementations -%% =================================================================== - -mochijson_encode_results(ReturnTerms, Results) -> - mochijson_encode_results(ReturnTerms, Results, undefined). - -mochijson_encode_results(true, Results, Continuation) -> - JsonKeys2 = {struct, [{?Q_RESULTS, [{struct, [{Val, Key}]} || {Val, Key} <- Results]}] ++ - mochify_continuation(Continuation)}, - mochijson2:encode(JsonKeys2); -mochijson_encode_results(false, Results, Continuation) -> - JustTheKeys = filter_values(Results), - JsonKeys1 = {struct, [{?Q_KEYS, JustTheKeys}] ++ mochify_continuation(Continuation)}, - mochijson2:encode(JsonKeys1). - -mochify_continuation(undefined) -> - []; -mochify_continuation(Continuation) -> - [{?Q_2I_CONTINUATION, Continuation}]. - -filter_values([]) -> - []; -filter_values([{_, _} | _T]=Results) -> - [K || {_V, K} <- Results]; -filter_values(Results) -> - Results. - - -otp_encode_results(ReturnTerms, Results) -> - otp_encode_results(ReturnTerms, Results, undefined). - -otp_encode_results(true, Results, undefined) -> - riak_kv_wm_json:encode( - #{?Q_RESULTS_BIN => Results}, - fun results_encode/2 - ); -otp_encode_results(true, Results, Continuation) -> - riak_kv_wm_json:encode( - #{?Q_RESULTS_BIN => Results, - ?Q_2I_CONTINUATION_BIN => Continuation}, - fun results_encode/2 - ); -otp_encode_results(false, Results, undefined) -> - riak_kv_wm_json:encode( - #{?Q_KEYS_BIN => Results}, - fun keys_encode/2 - ); -otp_encode_results(false, Results, Continuation) -> - riak_kv_wm_json:encode( - #{?Q_KEYS_BIN => Results, - ?Q_2I_CONTINUATION_BIN => Continuation}, - fun keys_encode/2 - ). - -results_encode({Term, Key}, Encode) when is_binary(Term), is_binary(Key) -> - [${, [Encode(Term, Encode), $: | Encode(Key, Encode)], $}]; -results_encode({Term, Key}, Encode) when is_integer(Term), is_binary(Key) -> - [ - ${, - [Encode(integer_to_binary(Term), Encode), $: | Encode(Key, Encode)], - $} - ]; -results_encode(Result, Encode) -> - riak_kv_wm_json:encode_value(Result, Encode). - -keys_encode({_Term, Key}, Encode) when is_binary(Key) -> - riak_kv_wm_json:encode_value(Key, Encode); -keys_encode(Object, Encode) -> - riak_kv_wm_json:encode_value(Object, Encode). - -encode_results(ReturnTerms, Results) -> - encode_results(ReturnTerms, Results, undefined). - -encode_results(ReturnTerms, Results, Continuation) -> - Library = - app_helper:get_env( - riak_kv, secondary_index_json, ?DEFAULT_JSON_ENCODING), - case Library of - mochijson -> - mochijson_encode_results(ReturnTerms, Results, Continuation); - otp -> - otp_encode_results(ReturnTerms, Results, Continuation) - end. - - - -%% =================================================================== -%% EUnit tests -%% =================================================================== - --ifdef(TEST). - --include_lib("eunit/include/eunit.hrl"). - - -otp_decode(JsonIOL) -> - riak_kv_wm_json:decode(iolist_to_binary(JsonIOL)). - -compare_encode_test() -> - Results = large_results(10), - OTPjsonA = otp_encode_results(true, Results), - MjsonA = mochijson_encode_results(true, Results), - ?assert(mochijson2:decode(MjsonA) == mochijson2:decode(OTPjsonA)), - ?assert(otp_decode(MjsonA) == otp_decode(OTPjsonA)), - ?assert(iolist_to_binary(MjsonA) == iolist_to_binary(OTPjsonA)), - Continuation = make_continuation(10, Results, 10), - OTPjsonB = otp_encode_results(true, Results, Continuation), - MjsonB = mochijson_encode_results(true, Results, Continuation), - {struct, MDecodeOTPjsonB} = mochijson2:decode(OTPjsonB), - {struct, MDecodeMjsonB} = mochijson2:decode(MjsonB), - ?assert(lists:sort(MDecodeOTPjsonB) == lists:sort(MDecodeMjsonB)), - OTPjsonC = otp_encode_results(false, Results), - MjsonC = mochijson_encode_results(false, Results), - ?assert(mochijson2:decode(MjsonC) == mochijson2:decode(OTPjsonC)), - OTPjsonD = otp_encode_results(false, Results, Continuation), - MjsonD = mochijson_encode_results(false, Results, Continuation), - {struct, MDecodeOTPjsonD} = mochijson2:decode(OTPjsonD), - {struct, MDecodeMjsonD} = mochijson2:decode(MjsonD), - ?assert(lists:sort(MDecodeOTPjsonD) == lists:sort(MDecodeMjsonD)), - IntIdxResults = [{1, <<"K1">>}, {2, <<"K2">>}], - OTPjsonE = otp_encode_results(true, IntIdxResults), - MjsonE = mochijson_encode_results(true, IntIdxResults), - ?assert(mochijson2:decode(MjsonE) == mochijson2:decode(OTPjsonE)), - ?assert(otp_decode(MjsonE) == otp_decode(OTPjsonE)), - ?assert(iolist_to_binary(MjsonE) == iolist_to_binary(OTPjsonE)) - - . - -encoder_test_() -> - {timeout, 600, fun encode_tester/0}. - -encode_tester() -> - timer:sleep(100), % awkward silence to tidy screen output - encode_implementation_tester(mochijson2), - encode_implementation_tester(otp). - -encode_implementation_tester(Imp) -> - garbage_collect(), - - io:format(user, "~n~nTesting Implementation ~w~n", [Imp]), - ResultSetsTiny = - [{<<"1K">>, large_results(1000)}, - {<<"2K">>, large_results(2000)}, - {<<"3K">>, large_results(3000)}, - {<<"5K">>, large_results(5000)}, - {<<"8K">>, large_results(8000)}], - encode_tester(Imp, ResultSetsTiny, microseconds), - - garbage_collect(), - - ResultSetsSmall = - [{<<"13K">>, large_results(13000)}, - {<<"21K">>, large_results(21000)}, - {<<"34K">>, large_results(34000)}, - {<<"55K">>, large_results(55000)}], - encode_tester(Imp, ResultSetsSmall, milliseconds), - - garbage_collect(), - - ResultSetsMid = - [{<<"100K">>, large_results(100000)}, - {<<"200K">>, large_results(200000)}, - {<<"300K">>, large_results(300000)}, - {<<"500K">>, large_results(500000)}], - encode_tester(Imp, ResultSetsMid, milliseconds), - - % garbage_collect(), - - % ResultSetsLarge = - % [{<<"1M">>, large_results(1000000)}, - % {<<"2M">>, large_results(2000000)}, - % {<<"3M">>, large_results(3000000)}, - % {<<"5M">>, large_results(5000000)}], - % encode_tester(Imp, ResultSetsLarge, milliseconds), - - % garbage_collect(), - % ResultSetsHuge = - % [{<<"8M">>, large_results(8000000)}], - % encode_tester(Imp, ResultSetsHuge, milliseconds), - - % garbage_collect(), - % ResultSetsSuperSize = - % [{<<"13M">>, large_results(13000000)}], - % encode_tester(Imp, ResultSetsSuperSize, milliseconds), - - ok. - -encode_tester(Lib, ResultSets, Unit) -> - Divisor = - case Unit of - microseconds -> - 1; - milliseconds -> - 1000 - end, - Fun = - case Lib of - mochijson2 -> - fun mochijson_encode_results/2; - otp -> - fun otp_encode_results/2 - end, - - TotalTime = - lists:sum( - lists:map( - fun({Tag, RS}) -> - garbage_collect(), - {TC, _Json} = - timer:tc(fun() -> Fun(true, RS) end), - io:format( - user, - "Result set of ~s in ~w ~p ", - [Tag, TC div Divisor, Unit]), - TC - end, - ResultSets - ) - ), - io:format(user, "Total time ~w ~p~n", [TotalTime div Divisor, Unit]). - -large_results(N) -> - lists:map( - fun(I) -> {generate_term(I), generate_key(I)} end, - lists:seq(1, N)). - -generate_term(I) -> - iolist_to_binary(io_lib:format("q~9..0B", [I])). - -generate_key(K) -> - iolist_to_binary(io_lib:format("k~9..0B", [rand:uniform(K)])). - --endif. \ No newline at end of file diff --git a/src/riak_kv_wm_keylist.erl b/src/riak_kv_wm_keylist.erl deleted file mode 100644 index 95625402a..000000000 --- a/src/riak_kv_wm_keylist.erl +++ /dev/null @@ -1,288 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2016 Basho Technologies, Inc. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - -%% @doc Webmachine resource for listing bucket keys over HTTP. -%% -%% URLs that begin with `/types' are necessary for the new bucket -%% types implementation in Riak 2.0, those that begin with `/buckets' -%% are for the default bucket type, and `/riak' is an old URL style, -%% also only works for the default bucket type. -%% -%% It is possible to reconfigure the `/riak' prefix but that seems to -%% be rarely if ever used. -%% -%% ``` -%% GET /types/Type/buckets/Bucket/keys?keys=true|stream -%% GET /buckets/Bucket/keys?keys=true|stream -%% GET /riak/Bucket?keys=true|stream''' -%% -%% Get the keys for a bucket. This is an expensive operation. -%% -%% Keys are returned in JSON form: `{"keys":[Key1,Key2,...]}' -%% -%% If the `keys' param is set to `true', then keys are sent back in -%% a single JSON structure. If set to "stream" then keys are -%% streamed in multiple JSON snippets. Otherwise, no keys are sent. -%% -%% If the `allow_props_param' context setting is `true', then -%% the user can also specify a `props=true' to include props in the -%% JSON response. This provides backward compatibility with the -%% old HTTP API. -%% - --module(riak_kv_wm_keylist). - -%% webmachine resource exports --export([ - init/1, - service_available/2, - is_authorized/2, - forbidden/2, - content_types_provided/2, - encodings_provided/2, - resource_exists/2, - produce_bucket_body/2, - malformed_request/2 - ]). - --record(ctx, {api_version, %% integer() - Determine which version of the API to use. - bucket_type, %% binary() - Bucket type (from uri) - bucket, %% binary() - Bucket name (from uri) - client, %% riak_client() - the store client - prefix, %% string() - prefix for resource uris - riak, %% local | {node(), atom()} - params for riak client - allow_props_param, %% true if the user can also list props. (legacy API) - timeout, %% integer() - list keys timeout - security %% security context - }). --type context() :: #ctx{}. - --include_lib("webmachine/include/webmachine.hrl"). --include("riak_kv_wm_raw.hrl"). - --spec init(proplists:proplist()) -> {ok, context()}. -%% @doc Initialize this resource. This function extracts the -%% 'prefix' and 'riak' properties from the dispatch args. -init(Props) -> - {ok, #ctx{api_version=proplists:get_value(api_version, Props), - prefix=proplists:get_value(prefix, Props), - riak=proplists:get_value(riak, Props), - allow_props_param=proplists:get_value(allow_props_param, Props), - bucket_type=proplists:get_value(bucket_type, Props) - }}. - --spec service_available(#wm_reqdata{}, context()) -> - {boolean(), #wm_reqdata{}, context()}. -%% @doc Determine whether or not a connection to Riak -%% can be established. This function also takes this -%% opportunity to extract the 'bucket' and 'key' path -%% bindings from the dispatch, as well as any vtag -%% query parameter. -service_available(RD, Ctx0=#ctx{riak=RiakProps}) -> - Ctx = riak_kv_wm_utils:ensure_bucket_type(RD, Ctx0, #ctx.bucket_type), - case riak_kv_wm_utils:get_riak_client(RiakProps, riak_kv_wm_utils:get_client_id(RD)) of - {ok, C} -> - {true, - RD, - Ctx#ctx{ - client=C, - bucket=case wrq:path_info(bucket, RD) of - undefined -> undefined; - B -> list_to_binary(riak_kv_wm_utils:maybe_decode_uri(RD, B)) - end - }}; - Error -> - {false, - wrq:set_resp_body( - io_lib:format("Unable to connect to Riak: ~p~n", [Error]), - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD)), - Ctx} - end. - -is_authorized(ReqData, Ctx) -> - case riak_api_web_security:is_authorized(ReqData) of - false -> - {"Basic realm=\"Riak\"", ReqData, Ctx}; - {true, SecContext} -> - {true, ReqData, Ctx#ctx{security=SecContext}}; - insecure -> - %% XXX 301 may be more appropriate here, but since the http and - %% https port are different and configurable, it is hard to figure - %% out the redirect URL to serve. - {{halt, 426}, wrq:append_to_resp_body(<<"Security is enabled and " - "Riak does not accept credentials over HTTP. Try HTTPS " - "instead.">>, ReqData), Ctx} - end. - --spec forbidden(#wm_reqdata{}, context()) - -> {boolean(), #wm_reqdata{}, context()}. -forbidden(ReqDataIn, #ctx{security = undefined} = Context) -> - Class = request_class(ReqDataIn), - riak_kv_wm_utils:is_forbidden(ReqDataIn, Class, Context); -forbidden(ReqDataIn, - #ctx{bucket_type = BT, bucket = B, security = Sec} = Context) -> - Class = request_class(ReqDataIn), - {Answer, ReqData, _} = Result = - riak_kv_wm_utils:is_forbidden(ReqDataIn, Class, Context), - case Answer of - false -> - case riak_core_security:check_permission( - {"riak_kv.list_keys", {BT, B}}, Sec) of - {false, Error, _} -> - {true, - wrq:append_to_resp_body( - unicode:characters_to_binary(Error, utf8, utf8), - wrq:set_resp_header( - "Content-Type", "text/plain", ReqData)), - Context}; - {true, _} -> - {false, ReqData, Context} - end; - _ -> - Result - end. - --spec request_class(#wm_reqdata{}) -> term(). -request_class(ReqData) -> - case wrq:get_qs_value(?Q_KEYS, ReqData) of - ?Q_STREAM -> - {riak_kv, stream_list_keys}; - _ -> - {riak_kv, list_keys} - end. - --spec content_types_provided(#wm_reqdata{}, context()) -> - {[{ContentType::string(), Producer::atom()}], #wm_reqdata{}, context()}. -%% @doc List the content types available for representing this resource. -%% "application/json" is the content-type for listing keys. -content_types_provided(RD, Ctx) -> - %% bucket-level: JSON description only - {[{"application/json", produce_bucket_body}], RD, Ctx}. - --spec encodings_provided(#wm_reqdata{}, context()) -> - {[{Encoding::string(), Producer::function()}], #wm_reqdata{}, context()}. -%% @doc List the encodings available for representing this resource. -%% "identity" and "gzip" are available for listing keys. -encodings_provided(RD, Ctx) -> - %% identity and gzip for top-level and bucket-level requests - {riak_kv_wm_utils:default_encodings(), RD, Ctx}. - -malformed_request(RD, Ctx) -> - malformed_timeout_param(RD, Ctx). - --spec malformed_timeout_param(#wm_reqdata{}, context()) -> - {boolean(), #wm_reqdata{}, context()}. -%% @doc Check that the timeout parameter is are a -%% string-encoded integer. Store the integer value -%% in context() if so. -malformed_timeout_param(RD, Ctx) -> - case wrq:get_qs_value("timeout", RD) of - undefined -> - {false, RD, Ctx}; - TimeoutStr -> - try - Timeout = list_to_integer(TimeoutStr), - {false, RD, Ctx#ctx{timeout=Timeout}} - catch - _:_ -> - {true, - wrq:append_to_resp_body(io_lib:format("Bad timeout " - "value ~p~n", - [TimeoutStr]), - wrq:set_resp_header(?HEAD_CTYPE, - "text/plain", RD)), - Ctx} - end - end. - -resource_exists(RD, Ctx) -> - {riak_kv_wm_utils:bucket_type_exists(Ctx#ctx.bucket_type), RD, Ctx}. - --spec produce_bucket_body(#wm_reqdata{}, context()) -> - {binary(), #wm_reqdata{}, context()}. -%% @doc Produce the JSON response to a bucket-level GET. -%% Includes the keys of the documents in the bucket unless the -%% "keys=false" query param is specified. If "keys=stream" query param -%% is specified, keys will be streamed back to the client in JSON chunks -%% like so: {"keys":[Key1, Key2,...]}. -produce_bucket_body(RD, #ctx{client=Client, - bucket=Bucket0, - bucket_type=Type, - timeout=Timeout, - allow_props_param=AllowProps}=Ctx) -> - Bucket = riak_kv_wm_utils:maybe_bucket_type(Type, Bucket0), - IncludeBucketProps = (AllowProps == true) - andalso (wrq:get_qs_value(?Q_PROPS, RD) /= ?Q_FALSE), - - BucketPropsJson = - case IncludeBucketProps of - true -> - [riak_kv_wm_props:get_bucket_props_json(Client, Bucket)]; - false -> - [] - end, - - case wrq:get_qs_value(?Q_KEYS, RD) of - ?Q_STREAM -> - %% Start streaming the keys... - F = - fun() -> - {ok, ReqId} = - riak_client:stream_list_keys(Bucket, Timeout, Client), - stream_keys(ReqId) - end, - - %% For API Version 1, send back the BucketPropsJson first - %% (if defined) or an empty resultset. For API Version 2, - %% use an empty list, which doesn't send an resultset. - FirstResult = - case Ctx#ctx.api_version of - 1 -> - mochijson2:encode({struct, BucketPropsJson}); - Two when Two >= 2 -> - [] - end, - {{stream, {FirstResult, F}}, RD, Ctx}; - - ?Q_TRUE -> - %% Get the JSON response... - case riak_client:list_keys(Bucket, Timeout, Client) of - {ok, KeyList} -> - JsonKeys = mochijson2:encode({struct, BucketPropsJson ++ - [{?Q_KEYS, KeyList}]}), - {JsonKeys, RD, Ctx}; - {error, Reason} -> - {mochijson2:encode({struct, [{error, Reason}]}), RD, Ctx} - end; - _ -> - JsonProps = mochijson2:encode({struct, BucketPropsJson}), - {JsonProps, RD, Ctx} - end. - -stream_keys(ReqId) -> - receive - {ReqId, From, {keys, Keys}} -> - _ = riak_kv_keys_fsm:ack_keys(From), - {mochijson2:encode({struct, [{<<"keys">>, Keys}]}), fun() -> stream_keys(ReqId) end}; - {ReqId, {keys, Keys}} -> - {mochijson2:encode({struct, [{<<"keys">>, Keys}]}), fun() -> stream_keys(ReqId) end}; - {ReqId, done} -> {mochijson2:encode({struct, [{<<"keys">>, []}]}), done}; - {ReqId, {error, timeout}} -> {mochijson2:encode({struct, [{error, timeout}]}), done} - end. diff --git a/src/riak_kv_wm_object.erl b/src/riak_kv_wm_object.erl deleted file mode 100644 index 67a7bab15..000000000 --- a/src/riak_kv_wm_object.erl +++ /dev/null @@ -1,1789 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% riak_kv_wm_object: Webmachine resource for KV object level operations. -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - -%% @doc Resource for serving Riak objects over HTTP. -%% -%% URLs that begin with `/types' are necessary for the new bucket -%% types implementation in Riak 2.0, those that begin with `/buckets' -%% are for the default bucket type, and `/riak' is an old URL style, -%% also only works for the default bucket type. -%% -%% It is possible to reconfigure the `/riak' prefix but that seems to -%% be rarely if ever used. -%% -%% ``` -%% POST /types/Type/buckets/Bucket/keys -%% POST /buckets/Bucket/keys -%% POST /riak/Bucket''' -%% -%% Allow the server to choose a key for the data. -%% -%% ``` -%% GET /types/Type/buckets/Bucket/keys/Key -%% GET /buckets/Bucket/keys/Key -%% GET /riak/Bucket/Key''' -%% -%% Get the data stored in the named Bucket under the named Key. -%% -%% Content-type of the response will be taken from the -%% Content-type was used in the request that stored the data. -%% -%% Additional headers will include: -%%
    -%%
  • `X-Riak-Vclock': The vclock of the object
    • -%%
  • `Link': The links the object has
    • -%%
  • `Etag': The Riak "vtag" metadata of the object
    • -%%
  • `Last-Modified': The last-modified time of the object
    • -%%
  • `Encoding': The value of the incoming Encoding header from -%% the request that stored the data.
    • -%%
  • `X-Riak-Meta-': Any headers prefixed by X-Riak-Meta- supplied -%% on PUT are returned verbatim
    • -%%
-%% -%% Specifying the query param `r=R', where `R' is an integer will -%% cause Riak to use `R' as the r-value for the read request. A -%% default r-value of 2 will be used if none is specified. -%% -%% If the object is found to have siblings (only possible if the -%% bucket property `allow_mult' is true), then -%% Content-type will be `text/plain'; `Link', `Etag', and `Last-Modified' -%% headers will be omitted; and the body of the response will -%% be a list of the vtags of each sibling. To request a specific -%% sibling, include the query param `vtag=V', where `V' is the vtag -%% of the sibling you want. -%% -%% ``` -%% PUT /types/Type/buckets/Bucket/keys/Key -%% PUT /buckets/Bucket/keys/Key -%% PUT /riak/Bucket/Key''' -%% -%% Store new data in the named Bucket under the named Key. -%% -%% A Content-type header *must* be included in the request. The -%% value of this header will be used in the response to subsequent -%% GET requests. -%% -%% The body of the request will be stored literally as the value -%% of the riak_object, and will be served literally as the body of -%% the response to subsequent GET requests. -%% -%% Include an X-Riak-Vclock header to modify data without creating -%% siblings. -%% -%% Include a Link header to set the links of the object. -%% -%% Include an Encoding header if you would like an Encoding header -%% to be included in the response to subsequent GET requests. -%% -%% Include custom metadata using headers prefixed with X-Riak-Meta-. -%% They will be returned verbatim on subsequent GET requests. -%% -%% Specifying the query param `w=W', where W is an integer will -%% cause Riak to use W as the w-value for the write request. A -%% default w-value of 2 will be used if none is specified. -%% -%% Specifying the query param `dw=DW', where DW is an integer will -%% cause Riak to use DW as the dw-value for the write request. A -%% default dw-value of 2 will be used if none is specified. -%% -%% Specifying the query param `r=R', where R is an integer will -%% cause Riak to use R as the r-value for the read request (used -%% to determine whether or not the resource exists). A default -%% r-value of 2 will be used if none is specified. -%% -%% ``` -%% POST /types/Type/buckets/Bucket/keys/Key -%% POST /buckets/Bucket/keys/Key -%% POST /riak/Bucket/Key''' -%% -%% Equivalent to `PUT /riak/Bucket/Key' (useful for clients that -%% do not support the PUT method). -%% -%% ``` -%% DELETE /types/Type/buckets/Bucket/keys/Key (with bucket-type) -%% DELETE /buckets/Bucket/keys/Key (NEW) -%% DELETE /riak/Bucket/Key (OLD)''' -%% -%% Delete the data stored in the named Bucket under the named Key. - --module(riak_kv_wm_object). - -%% webmachine resource exports --export([ - init/1, - service_available/2, - is_authorized/2, - forbidden/2, - allowed_methods/2, - allow_missing_post/2, - malformed_request/2, - resource_exists/2, - is_conflict/2, - last_modified/2, - generate_etag/2, - content_types_provided/2, - charsets_provided/2, - encodings_provided/2, - content_types_accepted/2, - post_is_create/2, - create_path/2, - process_post/2, - produce_doc_body/2, - accept_doc_body/2, - produce_sibling_message_body/2, - produce_multipart_body/2, - multiple_choices/2, - delete_resource/2 - ]). - --record(ctx, - { - api_version, %% integer() - Determine which version of the API to use. - bucket_type, %% binary() - Bucket type (from uri) - type_exists, %% bool() - Type exists as riak_core_bucket_type - bucket, %% binary() - Bucket name (from uri) - key, %% binary() - Key (from uri) - client, %% riak_client() - the store client - r, %% integer() - r-value for reads - w, %% integer() - w-value for writes - dw, %% integer() - dw-value for writes - rw, %% integer() - rw-value for deletes - pr, %% integer() - number of primary nodes required in preflist on read - pw, %% integer() - number of primary nodes required in preflist on write - node_confirms,%% integer() - number of physically diverse nodes required in preflist on write - basic_quorum, %% boolean() - whether to use basic_quorum - notfound_ok, %% boolean() - whether to treat notfounds as successes - asis, %% boolean() - whether to send the put without modifying the vclock - sync_on_write,%% string() - sync on write behaviour to pass to backend - prefix, %% string() - prefix for resource uris - riak, %% local | {node(), atom()} - params for riak client - doc, %% {ok, riak_object()}|{error, term()} - the object found - vtag, %% string() - vtag the user asked for - links, %% [link()] - links of the object - index_fields, %% [index_field()] - method, %% atom() - HTTP method for the request - ctype, %% string() - extracted content-type provided - charset, %% string() | undefined - extracted character set provided - timeout, %% integer() - passed-in timeout value in ms - security, %% security context - not_modified, %% decoded vector clock to be used in not_modified check - header_map %% processed request headers, with lowercase keys in a map - } -). - --include_lib("webmachine/include/webmachine.hrl"). --include("riak_kv_wm_raw.hrl"). - --type context() :: #ctx{}. --type request_data() :: #wm_reqdata{}. - --type validation_function() :: - fun((request_data(), context()) -> - {boolean()|{halt, pos_integer()}, request_data(), context()}). - --type link() :: {{Bucket::binary(), Key::binary()}, Tag::binary()}. - --define(BINHEAD_CTYPE, <<"content-type">>). --define(BINHEAD_IF_NOT_MODIFIED, <<"x-riak-if-not-modified">>). --define(BINHEAD_NONE_MATCH, <<"if-none-match">>). --define(BINHEAD_MATCH, <<"if-match">>). --define(BINHEAD_UNMODIFIED_SINCE, <<"if-unmodified-since">>). --define(BINHEAD_ENCODING, <<"content-encoding">>). --define(BINHEAD_ACCEPT, <<"accept">>). --define(BINHEAD_LINK, <<"link">>). --define(BINHEAD_VCLOCK, <<"x-riak-vclock">>). --define(PREFIX_USERMETA, "x-riak-meta-"). --define(PREFIX_INDEX, "x-riak-index-"). - --define(V1_BUCKET_REGEX, "/([^/]+)>; ?rel=\"([^\"]+)\""). --define(V1_KEY_REGEX, "/([^/]+)/([^/]+)>; ?riaktag=\"([^\"]+)\""). --define(V2_BUCKET_REGEX, "; ?rel=\"([^\"]+)\""). --define(V2_KEY_REGEX, - "; ?riaktag=\"([^\"]+)\""). - - --spec make_reqheader_map(request_data()) -> #{binary() => any()}. -make_reqheader_map(RD) -> - lists:foldl( - fun({HeadKey, HeadVal}, AccMap) -> - BinHeadKey = - case HeadKey of - HeadKey when is_binary(HeadKey) -> - HeadKey; - HeadKey when is_list(HeadKey) -> - list_to_binary(HeadKey); - HeadKey when is_atom(HeadKey) -> - atom_to_binary(HeadKey) - end, - accumulate_header_info( - string:lowercase(BinHeadKey), - BinHeadKey, - HeadVal, - AccMap - ) - end, - maps:new(), - mochiweb_headers:to_list(wrq:req_headers(RD)) - ). - --if(?OTP_RELEASE >= 25). --if(byte_size(<>) /= 12). --error("length ?PREFIX_USERMETA differs from 12"). --endif. --if(byte_size(<>) /= 13). --error("length ?PREFIX_INDEX differs from 13"). --endif. --endif. - -%% @doc -%% The usermeta header may have been added by a previous version of the -%% riak_kv_wm_object, in which case the key would have a x-riak-meta- prefix. -%% If this prefix exists, no need to add it, otherwise add it. -%% Note that the PB API has never added the prefix --spec generate_usermeta_header( - {binary()|list(), any()}, list({any(), any()})) -> list({any(), any()}). -generate_usermeta_header({<> = K, V}, Acc) -> - case string:lowercase(Prefix) of - <> -> - [{<>, V} | Acc]; - _NotMyPrefix -> - [{<>, V} | Acc] - end; -generate_usermeta_header({Key, V}, Acc) when is_binary(Key) -> - [{<>, V} | Acc]; -generate_usermeta_header({KeyAsList, V}, Acc) when length(KeyAsList) > 12 -> - case string:lowercase(lists:sublist(KeyAsList, 12)) of - ?PREFIX_USERMETA -> - [ - { - ?PREFIX_USERMETA ++ - lists:sublist(KeyAsList, 13, length(KeyAsList) - 12), - V - } - | Acc - ]; - _ -> - [{?PREFIX_USERMETA ++ KeyAsList, V} | Acc] - end; -generate_usermeta_header({KeyAsList, V}, Acc) -> - [{?PREFIX_USERMETA ++ KeyAsList, V} | Acc]. - -accumulate_header_info(<>, OriginalKey, T, MapAcc) -> - <<_Prefix:13/binary, Field/binary>> = OriginalKey, - maps:update_with( - <>, - fun(Indices) -> [{Field, T}|Indices] end, - [{Field, T}], - MapAcc - ); -accumulate_header_info(<>, OriginalKey, V, MapAcc) -> - <<_Prefix:12/binary, MetaKey/binary>> = OriginalKey, - maps:update_with( - <>, - fun(Indices) -> [{MetaKey, V}|Indices] end, - [{MetaKey, V}], - MapAcc - ); -accumulate_header_info(?BINHEAD_ACCEPT, _OK, V, MapAcc) -> - maps:put(?BINHEAD_ACCEPT, V, MapAcc); -accumulate_header_info(?BINHEAD_CTYPE, _OK, V, MapAcc) -> - maps:put(?BINHEAD_CTYPE, V, MapAcc); -accumulate_header_info(?BINHEAD_ENCODING, _OK, V, MapAcc) -> - maps:put(?BINHEAD_ENCODING, V, MapAcc); -accumulate_header_info(?BINHEAD_VCLOCK, _OK, VC, MapAcc) -> - maps:put( - ?BINHEAD_VCLOCK, - riak_object:decode_vclock(base64:decode(VC)), - MapAcc - ); -accumulate_header_info(?BINHEAD_LINK, _OK, V, MapAcc) -> - maps:put(?BINHEAD_LINK, V, MapAcc); -accumulate_header_info(?BINHEAD_IF_NOT_MODIFIED, _OK, V, MapAcc) -> - maps:put(?BINHEAD_IF_NOT_MODIFIED, V, MapAcc); -accumulate_header_info(?BINHEAD_UNMODIFIED_SINCE, _OK, V, MapAcc) -> - maps:put(?BINHEAD_UNMODIFIED_SINCE, V, MapAcc); -accumulate_header_info(?BINHEAD_MATCH, _OK, V, MapAcc) -> - maps:put(?BINHEAD_MATCH, V, MapAcc); -accumulate_header_info(?BINHEAD_NONE_MATCH, _OK, V, MapAcc) -> - maps:put(?BINHEAD_NONE_MATCH, V, MapAcc); -accumulate_header_info(_DiscardIdx, _OK, _Value, MapAcc) -> - MapAcc. - --spec init(proplists:proplist()) -> {ok, context()}. -%% @doc Initialize this resource. This function extracts the -%% 'prefix' and 'riak' properties from the dispatch args. -init(Props) -> - {ok, #ctx{api_version=proplists:get_value(api_version, Props), - prefix=proplists:get_value(prefix, Props), - riak=proplists:get_value(riak, Props), - bucket_type=proplists:get_value(bucket_type, Props)}}. - --spec service_available(request_data(), context()) -> - {boolean(), request_data(), context()}. -%% @doc Determine whether or not a connection to Riak -%% can be established. This function also takes this -%% opportunity to extract the 'bucket' and 'key' path -%% bindings from the dispatch, as well as any vtag -%% query parameter. -service_available(RD, Ctx0=#ctx{riak=RiakProps}) -> - Ctx = ensure_bucket_type(RD, Ctx0), - ClientID = riak_kv_wm_utils:get_client_id(RD), - case riak_kv_wm_utils:get_riak_client(RiakProps, ClientID) of - {ok, C} -> - Bucket = - case wrq:path_info(bucket, RD) of - undefined -> - undefined; - B -> - list_to_binary( - riak_kv_wm_utils:maybe_decode_uri(RD, B)) - end, - Key = - case wrq:path_info(key, RD) of - undefined -> - undefined; - K -> - list_to_binary( - riak_kv_wm_utils:maybe_decode_uri(RD, K)) - end, - { - true, - RD, - Ctx#ctx{ - method = wrq:method(RD), - client = C, - bucket = Bucket, - key = Key, - vtag = wrq:get_qs_value(?Q_VTAG, RD), - header_map = make_reqheader_map(RD) - } - }; - Error -> - {false, - wrq:set_resp_body( - io_lib:format("Unable to connect to Riak: ~p~n", [Error]), - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD)), - Ctx} - end. - -is_authorized(ReqData, Ctx) -> - case riak_api_web_security:is_authorized(ReqData) of - false -> - {"Basic realm=\"Riak\"", ReqData, Ctx}; - {true, SecContext} -> - {true, ReqData, Ctx#ctx{security=SecContext}}; - insecure -> - %% XXX 301 may be more appropriate here, but since the http and - %% https port are different and configurable, it is hard to figure - %% out the redirect URL to serve. - {{halt, 426}, wrq:append_to_resp_body(<<"Security is enabled and " - "Riak does not accept credentials over HTTP. Try HTTPS " - "instead.">>, ReqData), Ctx} - end. - --spec forbidden(request_data(), context()) -> term(). -forbidden(RD, Ctx) -> - case riak_kv_wm_utils:is_forbidden(RD) of - true -> - {true, RD, Ctx}; - false -> - validate(RD, Ctx) - end. - --spec validate(request_data(), context()) -> term(). -validate(RD, Ctx=#ctx{security=undefined}) -> - validate_resource( - RD, Ctx, riak_kv_wm_utils:method_to_perm(Ctx#ctx.method)); -validate(RD, Ctx=#ctx{security=Security}) -> - Perm = riak_kv_wm_utils:method_to_perm(Ctx#ctx.method), - Res = riak_core_security:check_permission({Perm, - {Ctx#ctx.bucket_type, - Ctx#ctx.bucket}}, - Security), - maybe_validate_resource(Res, RD, Ctx, Perm). - --spec maybe_validate_resource( - term(), request_data(), context(), string()) -> term(). -maybe_validate_resource({false, Error, _}, RD, Ctx, _Perm) -> - RD1 = wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD), - { - true, - wrq:append_to_resp_body( - unicode:characters_to_binary(Error, utf8, utf8), - RD1 - ), - Ctx - }; -maybe_validate_resource({true, _}, RD, Ctx, Perm) -> - validate_resource(RD, Ctx, Perm). - --spec validate_resource(request_data(), context(), string()) -> term(). -validate_resource(RD, Ctx, Perm) when Perm == "riak_kv.get" -> - %% Ensure the key is here, otherwise 404 - %% we do this early as it used to be done in the - %% malformed check, so the rest of the resource - %% assumes that the key is present. - validate_doc(RD, Ctx); -validate_resource(RD, Ctx, _Perm) -> - %% Ensure the bucket type exists, otherwise 404 early. - validate_bucket_type(RD, Ctx). - -%% @doc Detects whether fetching the requested object results in an -%% error. -validate_doc(RD, Ctx) -> - DocCtx = ensure_doc(Ctx), - case DocCtx#ctx.doc of - {error, Reason} -> - handle_common_error(Reason, RD, DocCtx); - _ -> - {false, RD, DocCtx} - end. - -%% @doc Detects whether the requested object's bucket-type exists. -validate_bucket_type(RD, Ctx) -> - case Ctx#ctx.type_exists of - true -> - {false, RD, Ctx}; - false -> - handle_common_error(bucket_type_unknown, RD, Ctx) - end. - --spec allowed_methods(request_data(), context()) -> - {[atom()], request_data(), context()}. -%% @doc Get the list of methods this resource supports. -allowed_methods(RD, Ctx) -> - {['HEAD', 'GET', 'POST', 'PUT', 'DELETE'], RD, Ctx}. - --spec allow_missing_post(request_data(), context()) -> - {true, request_data(), context()}. -%% @doc Makes POST and PUT equivalent for creating new -%% bucket entries. -allow_missing_post(RD, Ctx) -> - {true, RD, Ctx}. - --spec malformed_request(request_data(), context()) -> - {boolean(), request_data(), context()}. -%% @doc Determine whether query parameters, request headers, -%% and request body are badly-formed. -%% Body format is checked to be valid JSON, including -%% a "props" object for a bucket-PUT. Body format -%% is not tested for a key-level request (since the -%% body may be any content the client desires). -%% Query parameters r, w, dw, and rw are checked to -%% be valid integers. Their values are stored in -%% the context() at this time. -%% Link headers are checked for the form: -%% </Prefix/Bucket/Key>; riaktag="Tag",... -%% The parsed links are stored in the context() -%% at this time. -malformed_request(RD, Ctx) when Ctx#ctx.method =:= 'POST' - orelse Ctx#ctx.method =:= 'PUT' -> - malformed_request( - [ - fun malformed_content_type/2, - fun malformed_timeout_param/2, - fun malformed_rw_params/2, - fun malformed_link_headers/2, - fun malformed_index_headers/2 - ], - RD, - Ctx - ); -malformed_request(RD, Ctx) -> - malformed_request( - [ - fun malformed_timeout_param/2, - fun malformed_rw_params/2 - ], - RD, - Ctx - ). - -%% @doc Given a list of 2-arity funs, threads through the request data -%% and context, returning as soon as a single fun discovers a -%% malformed request or halts. --spec malformed_request( - list(validation_function()), request_data(), context()) -> - {boolean() | {halt, pos_integer()}, request_data(), context()}. -malformed_request([], RD, Ctx) -> - {false, RD, Ctx}; -malformed_request([H|T], RD, Ctx) -> - case H(RD, Ctx) of - {true, _, _} = Result -> - Result; - {{halt,_}, _, _} = Halt -> - Halt; - {false, RD1, Ctx1} -> - malformed_request(T, RD1, Ctx1) - end. - -%% @doc Detects whether the Content-Type header is missing on -%% PUT/POST. -%% This should probably result in a 415 using the known_content_type callback -malformed_content_type(RD, Ctx) -> - case maps:get(?BINHEAD_CTYPE, Ctx#ctx.header_map, undefined) of - undefined -> - {true, missing_content_type(RD), Ctx}; - RawCType -> - [ContentType|RawParams] = string:lexemes(RawCType, "; "), - Params = [ list_to_tuple(string:lexemes(P, "=")) || P <- RawParams], - Charset = proplists:get_value("charset", Params), - {false, RD, Ctx#ctx{ctype = ContentType, charset = Charset}} - end. - --spec malformed_timeout_param(request_data(), context()) -> - {boolean(), request_data(), context()}. -%% @doc Check that the timeout parameter is are a -%% string-encoded integer. Store the integer value -%% in context() if so. -malformed_timeout_param(RD, Ctx) -> - case wrq:get_qs_value("timeout", RD) of - undefined -> - {false, RD, Ctx}; - TimeoutStr -> - try - Timeout = list_to_integer(TimeoutStr), - {false, RD, Ctx#ctx{timeout=Timeout}} - catch - _:_ -> - { - true, - wrq:append_to_resp_body( - io_lib:format( - "Bad timeout value ~p~n", - [TimeoutStr] - ), - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD) - ), - Ctx - } - end - end. - --spec malformed_rw_params(request_data(), context()) -> - {boolean(), request_data(), context()}. -%% @doc Check that r, w, dw, and rw query parameters are -%% string-encoded integers. Store the integer values -%% in context() if so. -malformed_rw_params(RD, Ctx) -> - Res = - lists:foldl( - fun malformed_rw_param/2, - {false, RD, Ctx}, - [ - {#ctx.r, "r", default}, - {#ctx.w, "w", default}, - {#ctx.dw, "dw", default}, - {#ctx.rw, "rw", default}, - {#ctx.pw, "pw", default}, - {#ctx.node_confirms, "node_confirms", default}, - {#ctx.pr, "pr", default} - ] - ), - Res2 = - lists:foldl( - fun malformed_custom_param/2, - Res, - [ - { - #ctx.sync_on_write, - "sync_on_write", - default, - [default, backend, one, all] - } - ] - ), - lists:foldl( - fun malformed_boolean_param/2, - Res2, - [ - {#ctx.basic_quorum, "basic_quorum", default}, - {#ctx.notfound_ok, "notfound_ok", default}, - {#ctx.asis, "asis", false} - ] - ). - --spec malformed_rw_param({Idx::integer(), Name::string(), Default::string()}, - {boolean(), #wm_reqdata{}, context()}) -> - {boolean(), #wm_reqdata{}, context()}. -%% @doc Check that a specific r, w, dw, or rw query param is a -%% string-encoded integer. Store its result in context() if it -%% is, or print an error message in #wm_reqdata{} if it is not. -malformed_rw_param({Idx, Name, Default}, {Result, RD, Ctx}) -> - case wrq:get_qs_value(Name, RD) of - undefined -> - {Result, RD, setelement(Idx, Ctx, Default)}; - ExtractedString -> - case catch normalize_rw_param(ExtractedString) of - P when (is_atom(P) orelse is_integer(P)) -> - {Result, RD, setelement(Idx, Ctx, P)}; - _ -> - {true, - wrq:append_to_resp_body( - io_lib:format( - "~s query parameter must be an integer or " - "one of the following words: 'one', 'quorum' or 'all'~n", - [Name] - ), - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD)), - Ctx} - end - end. - --spec malformed_custom_param( - { - Idx::integer(), - Name::string(), - Default::atom(), - AllowedValues::[atom()] - }, - {boolean(), request_data(), context()}) -> - {boolean(), request_data(), context()}. -%% @doc Check that a custom parameter is one of the AllowedValues -%% Store its result in context() if it is, or print an error message -%% in #wm_reqdata{} if it is not. -malformed_custom_param({Idx, Name, Default, AllowedValues}, {Result, RD, Ctx}) -> - case wrq:get_qs_value(Name, RD) of - undefined -> - {Result, RD, setelement(Idx, Ctx, Default)}; - ExtractedString -> - UsableValue = list_to_atom(string:lowercase(ExtractedString)), - case lists:member(UsableValue, AllowedValues) of - true -> - {Result, RD, setelement(Idx, Ctx, UsableValue)}; - false -> - ErrorText = - "~s query parameter must be one of the following words: ~p~n", - {true, - wrq:append_to_resp_body( - io_lib:format(ErrorText, [Name, AllowedValues]), - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD)), - Ctx} - end - end. - -%% @doc Check that a specific query param is a -%% string-encoded boolean. Store its result in context() if it -%% is, or print an error message in #wm_reqdata{} if it is not. -malformed_boolean_param({Idx, Name, Default}, {Result, RD, Ctx}) -> - case wrq:get_qs_value(Name, RD) of - undefined -> - {Result, RD, setelement(Idx, Ctx, Default)}; - ExtractedString -> - case string:lowercase(ExtractedString) of - "true" -> - {Result, RD, setelement(Idx, Ctx, true)}; - "false" -> - {Result, RD, setelement(Idx, Ctx, false)}; - "default" -> - {Result, RD, setelement(Idx, Ctx, default)}; - _ -> - {true, - wrq:append_to_resp_body( - io_lib:format("~s query parameter must be true or false~n", - [Name]), - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD)), - Ctx} - end - end. - -normalize_rw_param("backend") -> backend; -normalize_rw_param("default") -> default; -normalize_rw_param("one") -> one; -normalize_rw_param("quorum") -> quorum; -normalize_rw_param("all") -> all; -normalize_rw_param(V) -> list_to_integer(V). - --spec malformed_link_headers(request_data(), context()) -> - {boolean(), request_data(), context()}. -%% @doc Check that the Link header in the request() is valid. -%% Store the parsed links in context() if the header is valid, -%% or print an error in #wm_reqdata{} if it is not. -%% A link header should be of the form: -%% </Prefix/Bucket/Key>; riaktag="Tag",... -malformed_link_headers(RD, Ctx) -> - case catch get_link_heads(Ctx) of - Links when is_list(Links) -> - {false, RD, Ctx#ctx{links=Links}}; - _Error when Ctx#ctx.api_version == 1-> - { - true, - wrq:append_to_resp_body( - io_lib:format( - "Invalid Link header. Links must be of the form~n" - "; riaktag=\"TAG\"~n", - [Ctx#ctx.prefix] - ), - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD) - ), - Ctx - }; - _Error when Ctx#ctx.api_version == 2 -> - { - true, - wrq:append_to_resp_body( - io_lib:format( - "Invalid Link header. Links must be of the form~n" - "; riaktag=\"TAG\"~n", - [] - ), - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD) - ), - Ctx - } - - end. - --spec malformed_index_headers(request_data(), context()) -> - {boolean(), request_data(), context()}. -%% @doc Check that the Index headers (HTTP headers prefixed with index_") -%% are valid. Store the parsed headers in context() if valid, -%% or print an error in #wm_reqdata{} if not. -%% An index field should be of the form "index_fieldname_type" -malformed_index_headers(RD, Ctx) -> - %% Get a list of index_headers... - IndexFields1 = extract_index_fields(Ctx), - - %% Validate the fields. If validation passes, then the index - %% headers are correctly formed. - case riak_index:parse_fields(IndexFields1) of - {ok, IndexFields2} -> - {false, RD, Ctx#ctx { index_fields=IndexFields2 }}; - {error, Reasons} -> - { - true, - wrq:append_to_resp_body( - [riak_index:format_failure_reason(X) || X <- Reasons], - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD) - ), - Ctx - } - end. - --spec extract_index_fields(context()) -> proplists:proplist(). -%% @doc Extract fields from headers prefixed by "x-riak-index-" in the -%% client's PUT request, to be indexed at write time. -extract_index_fields(Ctx) -> - RE = get_compiled_index_regex(), - lists:flatten( - lists:map( - fun({Field, Term}) -> - Values = re:split(Term, RE, [{return, binary}]), - [{Field, X} || X <- Values] - end, - maps:get(<>, Ctx#ctx.header_map, []) - ) - ). - --spec content_types_provided(request_data(), context()) -> - {[{ContentType::string(), Producer::atom()}], request_data(), context()}. -%% @doc List the content types available for representing this resource. -%% The content-type for a key-level request is the content-type that -%% was used in the PUT request that stored the document in Riak. -content_types_provided(RD, Ctx=#ctx{method=Method, ctype=ContentType}) - when Method =:= 'PUT'; Method =:= 'POST' -> - {[{ContentType, produce_doc_body}], RD, Ctx}; -content_types_provided(RD, Ctx=#ctx{method=Method}) - when Method =:= 'DELETE' -> - {[{"text/html", to_html}], RD, Ctx}; -content_types_provided(RD, Ctx0) -> - DocCtx = ensure_doc(Ctx0), - %% we can assume DocCtx#ctx.doc is {ok,Doc} because of malformed_request - case select_doc(DocCtx) of - {MD, V} -> - {[{get_ctype(MD,V), produce_doc_body}], RD, DocCtx}; - multiple_choices -> - { - [ - {"text/plain", produce_sibling_message_body}, - {"multipart/mixed", produce_multipart_body} - ], - RD, - DocCtx - } - end. - --spec charsets_provided(request_data(), context()) -> - { - no_charset|[{Charset::string(), Producer::function()}], - request_data(), - context() - }. -%% @doc List the charsets available for representing this resource. -%% The charset for a key-level request is the charset that was used -%% in the PUT request that stored the document in Riak (none if -%% no charset was specified at PUT-time). -charsets_provided(RD, Ctx=#ctx{method=Method}) - when Method =:= 'PUT'; Method =:= 'POST' -> - case Ctx#ctx.charset of - undefined -> - {no_charset, RD, Ctx}; - Charset -> - {[{Charset, fun(X) -> X end}], RD, Ctx} - end; -charsets_provided(RD, Ctx=#ctx{method=Method}) - when Method =:= 'DELETE' -> - {no_charset, RD, Ctx}; -charsets_provided(RD, Ctx0) -> - DocCtx = ensure_doc(Ctx0), - case DocCtx#ctx.doc of - {ok, _} -> - case select_doc(DocCtx) of - {MD, _} -> - case riak_object:metadata_find(?MD_CHARSET, MD) of - {ok, CS} -> - {[{CS, fun(X) -> X end}], RD, DocCtx}; - error -> - {no_charset, RD, DocCtx} - end; - multiple_choices -> - {no_charset, RD, DocCtx} - end; - {error, _} -> - {no_charset, RD, DocCtx} - end. - --spec encodings_provided(request_data(), context()) -> - {[{Encoding::string(), Producer::function()}], request_data(), context()}. -%% @doc List the encodings available for representing this resource. -%% The encoding for a key-level request is the encoding that was -%% used in the PUT request that stored the document in Riak, or -%% "identity" and "gzip" if no encoding was specified at PUT-time. -encodings_provided(RD, Ctx0) -> - DocCtx = - case Ctx0#ctx.method of - UpdM when UpdM =:= 'PUT'; UpdM =:= 'POST'; UpdM =:= 'DELETE' -> - Ctx0; - _ -> - ensure_doc(Ctx0) - end, - case DocCtx#ctx.doc of - {ok, _} -> - case select_doc(DocCtx) of - {MD, _} -> - case riak_object:metadata_find(?MD_ENCODING, MD) of - {ok, Enc} -> - {[{Enc, fun(X) -> X end}], RD, DocCtx}; - error -> - {riak_kv_wm_utils:default_encodings(), RD, DocCtx} - end; - multiple_choices -> - {riak_kv_wm_utils:default_encodings(), RD, DocCtx} - end; - _ -> - {riak_kv_wm_utils:default_encodings(), RD, DocCtx} - end. - --spec content_types_accepted(request_data(), context()) -> - {[{ContentType::string(), Acceptor::atom()}], - request_data(), context()}. -%% @doc Get the list of content types this resource will accept. -%% Whatever content type is specified by the Content-Type header -%% of a key-level PUT request will be accepted by this resource. -%% (A key-level put *must* include a Content-Type header.) -content_types_accepted(RD, Ctx) -> - case maps:get(?BINHEAD_CTYPE, Ctx#ctx.header_map, undefined) of - undefined -> - %% user must specify content type of the data - {[], RD, Ctx}; - CType -> - Media = hd(string:tokens(CType, ";")), - case string:tokens(Media, "/") of - [_Type, _Subtype] -> - %% accept whatever the user says - {[{Media, accept_doc_body}], RD, Ctx}; - _ -> - { - [], - wrq:set_resp_header( - ?HEAD_CTYPE, - "text/plain", - wrq:set_resp_body( - [ - "\"", Media, "\"" - " is not a valid media type" - " for the Content-type header.\n" - ], - RD - ) - ), - Ctx - } - end - end. - --spec resource_exists(request_data(), context()) -> - {boolean(), request_data(), context()}. -%% @doc Determine whether or not the requested item exists. -%% Documents exists if a read request to Riak returns {ok, riak_object()}, -%% and either no vtag query parameter was specified, or the value of the -%% vtag param matches the vtag of some value of the Riak object. -resource_exists(RD, Ctx0) -> - Method = Ctx0#ctx.method, - ToFetch = - case Method of - UpdM when UpdM =:= 'PUT'; UpdM =:= 'POST'; UpdM =:= 'DELETE' -> - conditional_headers_present(Ctx0) == true; - _ -> - true - end, - case ToFetch of - true -> - DocCtx = ensure_doc(Ctx0), - case DocCtx#ctx.doc of - {ok, Doc} -> - case DocCtx#ctx.vtag of - undefined -> - {true, RD, DocCtx}; - VTag -> - VtagMatchesASibling = - lists:any( - fun(M) -> - SibTag = - riak_object:metadata_fetch( - ?MD_VTAG, - M - ), - VTag == SibTag - end, - riak_object:get_metadatas(Doc) - ), - {VtagMatchesASibling, RD, DocCtx} - end; - {error, _} -> - %% This should never actually be reached because all the - %% error conditions from ensure_doc are handled up in - %% malformed_request. - {false, RD, DocCtx} - end; - false -> - % Fake it - rather than fetch to see. If we're deleting we assume - % it does exist, and if PUT/POST, assume it doesn't - case Ctx0#ctx.method of - 'DELETE' -> - {true, RD, Ctx0}; - _ -> - {false, RD, Ctx0} - end - end. - --spec doc_required(context()) -> {boolean(), boolean()}. -doc_required(Context) -> - case Context#ctx.method of - UpdM when UpdM =:= 'PUT'; UpdM =:= 'POST'; UpdM =:= 'DELETE' -> - {conditional_headers_present(Context) == true, false}; - _ -> - {true, true} - end. - --spec is_conflict(request_data(), context()) -> - {boolean(), request_data(), context()}. -is_conflict(RD, Ctx) -> - NotModified = - maps:get(?BINHEAD_IF_NOT_MODIFIED, Ctx#ctx.header_map, undefined), - case {Ctx#ctx.method, NotModified} of - {_ , undefined} -> - {false, RD, Ctx}; - {UpdM, NotModifiedClock} when UpdM =:= 'PUT'; UpdM =:= 'POST' -> - case Ctx#ctx.doc of - {ok, Obj} -> - InClock = - riak_object:decode_vclock( - base64:decode(NotModifiedClock)), - CurrentClock = - riak_object:vclock(Obj), - {not vclock:equal(InClock, CurrentClock), - RD, - Ctx#ctx{not_modified = InClock} - }; - _ -> - {true, RD, Ctx} - end; - _ -> - {false, RD, Ctx} - end. - --spec conditional_headers_present(context()) -> boolean(). -conditional_headers_present(Ctx) -> - NoneMatch = maps:is_key(?BINHEAD_NONE_MATCH, Ctx#ctx.header_map), - Match = maps:is_key(?BINHEAD_MATCH, Ctx#ctx.header_map), - UnModifiedSince = maps:is_key(?BINHEAD_UNMODIFIED_SINCE, Ctx#ctx.header_map), - NotModified = maps:is_key(?BINHEAD_IF_NOT_MODIFIED, Ctx#ctx.header_map), - (NoneMatch orelse Match orelse UnModifiedSince orelse NotModified). - --spec post_is_create(request_data(), context()) -> - {boolean(), request_data(), context()}. -%% @doc POST is considered a document-creation operation for bucket-level -%% requests (this makes webmachine call create_path/2, where the key -%% for the created document will be chosen). -post_is_create(RD, Ctx=#ctx{key=undefined}) -> - %% bucket-POST is create - {true, RD, Ctx}; -post_is_create(RD, Ctx) -> - %% key-POST is not create - {false, RD, Ctx}. - --spec create_path(request_data(), context()) -> - {string(), request_data(), context()}. -%% @doc Choose the Key for the document created during a bucket-level POST. -%% This function also sets the Location header to generate a -%% 201 Created response. -create_path(RD, Ctx=#ctx{prefix=P, bucket_type=T, bucket=B, api_version=V}) -> - K = riak_core_util:unique_id_62(), - { - K, - wrq:set_resp_header( - "Location", - riak_kv_wm_utils:format_uri(T, B, K, P, V), - RD - ), - Ctx#ctx{key=list_to_binary(K)}}. - --spec process_post(request_data(), context()) -> - {true, request_data(), context()}. -%% @doc Pass-through for key-level requests to allow POST to function -%% as PUT for clients that do not support PUT. -process_post(RD, Ctx) -> accept_doc_body(RD, Ctx). - --spec accept_doc_body(#wm_reqdata{}, context()) -> - {true, #wm_reqdata{}, context()}. -%% @doc Store the data the client is PUTing in the document. -%% This function translates the headers and body of the HTTP request -%% into their final riak_object() form, and executes the Riak put. -accept_doc_body( - RD, - Ctx=#ctx{ - bucket_type=T, bucket=B, key=K, client=C, - links=L, ctype=CType, charset=Charset, - index_fields=IF, - not_modified = IfNotModified - }) -> - Doc0 = riak_object:new(riak_kv_wm_utils:maybe_bucket_type(T,B), K, <<>>), - VclockDoc = - riak_object:set_vclock( - Doc0, - maps:get(?BINHEAD_VCLOCK, Ctx#ctx.header_map, vclock:fresh()) - ), - UserMeta = maps:get(<>, Ctx#ctx.header_map, []), - CTypeMD = - riak_object:metadata_store( - ?MD_CTYPE, - CType, - riak_object:metadata_new() - ), - CharsetMD = - if Charset /= undefined -> - riak_object:metadata_store(?MD_CHARSET, Charset, CTypeMD); - true -> - CTypeMD - end, - EncMD = - case maps:get(?BINHEAD_ENCODING, Ctx#ctx.header_map, undefined) of - undefined -> - CharsetMD; - E -> - riak_object:metadata_store(?MD_ENCODING, E, CharsetMD) - end, - LinkMD = riak_object:metadata_store(?MD_LINKS, L, EncMD), - UserMetaMD = riak_object:metadata_store(?MD_USERMETA, UserMeta, LinkMD), - IndexMD = riak_object:metadata_store(?MD_INDEX, IF, UserMetaMD), - MDDoc = riak_object:update_metadata(VclockDoc, IndexMD), - Doc = - riak_object:update_value( - MDDoc, riak_kv_wm_utils:accept_value(CType, wrq:req_body(RD))), - Options0 = - case wrq:get_qs_value(?Q_RETURNBODY, RD) of - ?Q_TRUE -> [returnbody]; - _ -> [] - end, - Options = make_options(Options0, Ctx), - IfNoneMatch = maps:is_key(?BINHEAD_NONE_MATCH, Ctx#ctx.header_map), - CondPutMode = - application:get_env(riak_kv, conditional_put_mode, api_only), - MakeTokenRequest = CondPutMode =/= api_only, - - {CondPutOptions, SessionToken} = - case {IfNotModified, IfNoneMatch, MakeTokenRequest} of - {undefined, false, _} -> - {[], none}; - {NotMod, NoneMatch, true} -> - TokenResult = - riak_kv_token_session:session_request_retry({B, K}), - case TokenResult of - {true, Token} -> - GetOpts = - [ - {basic_quorum, true}, - {return_body, false}, - {deleted_vclock, true} - ], - Condition = - case NotMod of - undefined -> - {undefined, true, GetOpts}; - InClock -> - {{true, InClock}, undefined, GetOpts} - end, - {[{condition_check, Condition}], Token}; - _ -> - %% Pass the condition downstream, but currently that - %% condition is ignored - case {NotMod, NoneMatch} of - {_, true} -> - {[{if_none_match, true}], none}; - {InClock, _} -> - {[{if_not_modified, InClock}], none} - end - end; - {NotMod, NoneMatch, false} -> - %% Pass the condition downstream, but currently that - %% condition is ignored - case {NotMod, NoneMatch} of - {_, true} -> - {[{if_none_match, true}], none}; - {InClock, _} -> - {[{if_not_modified, InClock}], none} - end - end, - PutRsp = - case SessionToken of - none -> - riak_client:put(Doc, CondPutOptions ++ Options, C); - _ -> - riak_kv_token_session:session_use( - SessionToken, put, [Doc, CondPutOptions ++ Options] - ) - end, - riak_kv_token_session:session_release(SessionToken), - case PutRsp of - {error, Reason} -> - handle_common_error(Reason, RD, Ctx); - ok -> - {true, RD, Ctx#ctx{doc={ok, Doc}}}; - {ok, RObj} -> - DocCtx = Ctx#ctx{doc={ok, RObj}}, - HasSiblings = (select_doc(DocCtx) == multiple_choices), - send_returnbody(RD, DocCtx, HasSiblings) - end. - -%% Handle the no-sibling case. Just send the object. -send_returnbody(RD, DocCtx, _HasSiblings = false) -> - {Body, DocRD, DocCtx2} = produce_doc_body(RD, DocCtx), - {DocRD2, DocCtx3} = add_conditional_headers(DocRD, DocCtx2), - {true, wrq:append_to_response_body(Body, DocRD2), DocCtx3}; - -%% Handle the sibling case. Send either the sibling message body, or a -%% multipart body, depending on what the client accepts. -send_returnbody(RD, DocCtx, _HasSiblings = true) -> - AcceptHdr = maps:get(?BINHEAD_ACCEPT, DocCtx#ctx.header_map, undefined), - PossibleTypes = ["multipart/mixed", "text/plain"], - case webmachine_util:choose_media_type(PossibleTypes, AcceptHdr) of - "multipart/mixed" -> - {Body, DocRD, DocCtx2} = produce_multipart_body(RD, DocCtx), - {DocRD2, DocCtx3} = add_conditional_headers(DocRD, DocCtx2), - {true, wrq:append_to_response_body(Body, DocRD2), DocCtx3}; - _ -> - {Body, DocRD, DocCtx2} = produce_sibling_message_body(RD, DocCtx), - {DocRD2, DocCtx3} = add_conditional_headers(DocRD, DocCtx2), - {true, wrq:append_to_response_body(Body, DocRD2), DocCtx3} - end. - -%% Add ETag and Last-Modified headers to responses that might not -%% necessarily include them, specifically when the client requests -%% returnbody on a PUT or POST. -add_conditional_headers(RD, Ctx) -> - {ETag, RD2, Ctx2} = generate_etag(RD, Ctx), - {LM, RD3, Ctx3} = last_modified(RD2, Ctx2), - RD4 = - wrq:set_resp_header( - "ETag", webmachine_util:quoted_string(ETag), RD3), - RD5 = - wrq:set_resp_header( - "Last-Modified", - httpd_util:rfc1123_date( - calendar:universal_time_to_local_time(LM)), RD4), - {RD5,Ctx3}. - --spec multiple_choices(request_data(), context()) -> - {boolean(), request_data(), context()}. -%% @doc Determine whether a document has siblings. If the user has -%% specified a specific vtag, the document is considered not to -%% have sibling versions. This is a safe assumption, because -%% resource_exists will have filtered out requests earlier for -%% vtags that are invalid for this version of the document. -multiple_choices(RD, Ctx=#ctx{vtag=undefined, doc={ok, Doc}}) -> - %% user didn't specify a vtag, so there better not be siblings - case riak_object:get_update_value(Doc) of - undefined -> - case riak_object:value_count(Doc) of - 1 -> - {false, RD, Ctx}; - _ -> - {true, RD, Ctx} - end; - _ -> - %% just updated can't have multiple - {false, RD, Ctx} - end; -multiple_choices(RD, Ctx) -> - %% specific vtag was specified - %% if it's a tombstone add the X-Riak-Deleted header - case select_doc(Ctx) of - {M, _} -> - case riak_object:metadata_find(?MD_DELETED, M) of - {ok, "true"} -> - {false, - wrq:set_resp_header(?HEAD_DELETED, "true", RD), - Ctx}; - error -> - {false, RD, Ctx} - end; - multiple_choices -> - throw( - {unexpected_code_path, - ?MODULE, - multiple_choices, - multiple_choices}) - end. - --spec produce_doc_body(request_data(), context()) -> - {binary(), request_data(), context()}. -%% @doc Extract the value of the document, and place it in the -%% response body of the request. This function also adds the -%% Link, X-Riak-Meta- headers, and X-Riak-Index- headers to the -%% response. One link will point to the bucket, with the -%% property "rel=container". The rest of the links will be -%% constructed from the links of the document. -produce_doc_body(RD, Ctx) -> - Prefix = Ctx#ctx.prefix, - Bucket = Ctx#ctx.bucket, - APIVersion = Ctx#ctx.api_version, - case select_doc(Ctx) of - {MD, Doc} -> - %% Add links to response... - Links1 = - case riak_object:metadata_find(?MD_LINKS, MD) of - {ok, L} -> L; - error -> [] - end, - Links2 = - riak_kv_wm_utils:format_links( - [{Bucket, "up"}|Links1], Prefix, APIVersion), - - %% Add user metadata to response... - UserMetaRD = - case riak_object:metadata_find(?MD_USERMETA, MD) of - {ok, UserMeta} -> - lists:foldl( - fun generate_usermeta_header/2, - Links2, - UserMeta - ); - error -> - Links2 - end, - - %% Add index metadata to response... - IndexRD = - case riak_object:metadata_find(?MD_INDEX, MD) of - {ok, IndexMeta} -> - lists:foldl( - fun({K,V}, Acc) -> - K1 = riak_kv_wm_utils:any_to_list(K), - V1 = riak_kv_wm_utils:any_to_list(V), - [{?HEAD_INDEX_PREFIX ++ K1, V1} | Acc] - end, - UserMetaRD, - IndexMeta - ); - error -> - UserMetaRD - end, - { - riak_kv_wm_utils:encode_value(Doc), - encode_vclock_header(wrq:merge_resp_headers(IndexRD, RD), Ctx), Ctx - }; - multiple_choices -> - throw( - {unexpected_code_path, - ?MODULE, - produce_doc_body, - multiple_choices}) - end. - --spec produce_sibling_message_body(request_data(), context()) -> - {iolist(), request_data(), context()}. -%% @doc Produce the text message informing the user that there are multiple -%% values for this document, and giving that user the vtags of those -%% values so they can get to them with the vtag query param. -produce_sibling_message_body(RD, Ctx=#ctx{doc={ok, Doc}}) -> - Vtags = - [ - riak_object:metadata_fetch(?MD_VTAG, M) - || M <- riak_object:get_metadatas(Doc) - ], - { - [<<"Siblings:\n">>, [ [V,<<"\n">>] || V <- Vtags]], - wrq:set_resp_header( - ?HEAD_CTYPE, - "text/plain", - encode_vclock_header(RD, Ctx) - ), - Ctx - }. - --spec produce_multipart_body(request_data(), context()) -> - {iolist(), request_data(), context()}. -%% @doc Produce a multipart body representation of an object with multiple -%% values (siblings), each sibling being one part of the larger -%% document. -produce_multipart_body(RD, Ctx=#ctx{doc={ok, Doc}, bucket=B, prefix=P}) -> - APIVersion = Ctx#ctx.api_version, - Boundary = riak_core_util:unique_id_62(), - {[[["\r\n--",Boundary,"\r\n", - riak_kv_wm_utils:multipart_encode_body(P, B, Content, APIVersion)] - || Content <- riak_object:get_contents(Doc)], - "\r\n--",Boundary,"--\r\n"], - wrq:set_resp_header(?HEAD_CTYPE, - "multipart/mixed; boundary="++Boundary, - encode_vclock_header(RD, Ctx)), - Ctx}. - - --spec select_doc(context()) -> - {Metadata :: term(), Value :: term()}|multiple_choices. -%% @doc Selects the "proper" document: -%% - chooses update-value/metadata if update-value is set -%% - chooses only val/md if only one exists -%% - chooses val/md matching given Vtag if multiple contents exist -%% (assumes a vtag has been specified) -select_doc(#ctx{doc={ok, Doc}, vtag=Vtag}) -> - case riak_object:get_update_value(Doc) of - undefined -> - case riak_object:get_contents(Doc) of - [Single] -> Single; - Mult -> - case lists:dropwhile( - fun({M,_}) -> - riak_object:metadata_fetch(?MD_VTAG, M) /= Vtag - end, - Mult) of - [Match|_] -> Match; - [] -> multiple_choices - end - end; - UpdateValue -> - {riak_object:get_update_metadata(Doc), UpdateValue} - end. - --spec encode_vclock_header(request_data(), context()) -> request_data(). -%% @doc Add the X-Riak-Vclock header to the response. -encode_vclock_header(RD, #ctx{doc={ok, Doc}}) -> - {Head, Val} = riak_object:vclock_header(Doc), - wrq:set_resp_header(Head, Val, RD); -encode_vclock_header(RD, #ctx{doc={error, {deleted, VClock}}}) -> - BinVClock = riak_object:encode_vclock(VClock), - wrq:set_resp_header( - ?HEAD_VCLOCK, binary_to_list(base64:encode(BinVClock)), RD). - --spec ensure_doc(context()) -> context(). -%% @doc Ensure that the 'doc' field of the context() has been filled -%% with the result of a riak_client:get request. This is a -%% convenience for memoizing the result of a get so it can be -%% used in multiple places in this resource, without having to -%% worry about the order of executing of those places. -ensure_doc(Ctx=#ctx{doc=undefined, key=undefined}) -> - Ctx#ctx{doc={error, notfound}}; -ensure_doc(Ctx=#ctx{doc=undefined, bucket_type=T, bucket=B, key=K, client=C, - basic_quorum=Quorum, notfound_ok=NotFoundOK}) -> - case Ctx#ctx.type_exists of - true -> - case doc_required(Ctx) of - {true, BodyRequired} -> - Options0 = - [ - deletedvclock, - {basic_quorum, Quorum}, - {return_body, BodyRequired}, - {notfound_ok, NotFoundOK} - ], - Options = make_options(Options0, Ctx), - BT = riak_kv_wm_utils:maybe_bucket_type(T, B), - Ctx#ctx{doc=riak_client:get(BT, K, Options, C)}; - _ -> - Ctx - end; - false -> - Ctx#ctx{doc={error, bucket_type_unknown}} - end; -ensure_doc(Ctx) -> - Ctx. - --spec delete_resource(request_data(), context()) -> - {true, request_data(), context()}. -%% @doc Delete the document specified. -delete_resource(RD, Ctx=#ctx{bucket_type=T, bucket=B, key=K, client=C}) -> - Options = make_options([], Ctx), - BT = riak_kv_wm_utils:maybe_bucket_type(T,B), - Result = - case maps:get(?BINHEAD_VCLOCK, Ctx#ctx.header_map, undefined) of - undefined -> - riak_client:delete(BT, K, Options, C); - VC -> - riak_client:delete_vclock(BT, K, VC, Options, C) - end, - case Result of - ok -> - {true, RD, Ctx}; - {error, Reason} -> - handle_common_error(Reason, RD, Ctx) - end. - -md5(Bin) -> - crypto:hash(md5, Bin). - --spec generate_etag(request_data(), context()) -> - {undefined|string(), request_data(), context()}. -%% @doc Get the etag for this resource. -%% Documents will have an etag equal to their vtag. For documents with -%% siblings when no vtag is specified, this will be an etag derived from -%% the vector clock. -generate_etag(RD, Ctx) -> - case select_doc(Ctx) of - {MD, _} -> - {riak_object:metadata_fetch(?MD_VTAG, MD), RD, Ctx}; - multiple_choices -> - {ok, Doc} = Ctx#ctx.doc, - <> = - md5(term_to_binary(riak_object:vclock(Doc))), - {riak_core_util:integer_to_list(ETag, 62), RD, Ctx} - end. - --spec last_modified(request_data(), context()) -> - {undefined|calendar:datetime(), request_data(), context()}. -%% @doc Get the last-modified time for this resource. -%% Documents will have the last-modified time specified by the -%% riak_object. -%% For documents with siblings, this is the last-modified time of the -%% latest sibling. -last_modified(RD, Ctx) -> - case select_doc(Ctx) of - {MD, _} -> - {normalize_last_modified(MD),RD, Ctx}; - multiple_choices -> - {ok, Doc} = Ctx#ctx.doc, - LMDates = - [ - normalize_last_modified(MD) || - MD <- riak_object:get_metadatas(Doc) - ], - {lists:max(LMDates), RD, Ctx} - end. - --spec normalize_last_modified( - riak_object:riak_object_meta()) -> - calendar:datetime(). -%% @doc Extract and convert the Last-Modified metadata into a normalized form -%% for use in the last_modified/2 callback. -normalize_last_modified(MD) -> - case riak_object:metadata_fetch(?MD_LASTMOD, MD) of - Now={_,_,_} -> - calendar:now_to_universal_time(Now); - Rfc1123 when is_list(Rfc1123) -> - httpd_util:convert_request_date(Rfc1123) - end. - --spec get_link_heads(context()) -> [link()]. -%% @doc Extract the list of links from the Link request header. -%% This function will die if an invalid link header format -%% is found. -get_link_heads(Ctx) -> - APIVersion = Ctx#ctx.api_version, - Prefix = Ctx#ctx.prefix, - Bucket = Ctx#ctx.bucket, - - %% Get a list of link headers... - LinkHeaders = - case maps:get(?BINHEAD_LINK, Ctx#ctx.header_map, undefined) of - undefined -> []; - Heads -> string:tokens(Heads, ",") - end, - - %% Decode the link headers. Throw an exception if we can't - %% properly parse any of the headers... - {BucketLinks, KeyLinks} = - case LinkHeaders of - [] -> - {[], []}; - LinkHeaders -> - {KeyRegex, BucketRegex} = - get_compiled_link_regex(APIVersion, Prefix), - extract_links(LinkHeaders, BucketRegex, KeyRegex) - end, - - %% Validate that the only bucket header is pointing to the parent - %% bucket... - IsValid = (BucketLinks == []) orelse (BucketLinks == [{Bucket, <<"up">>}]), - case IsValid of - true -> - KeyLinks; - false -> - throw({invalid_link_headers, LinkHeaders}) - end. - -%% Run each LinkHeader string() through the BucketRegex and -%% KeyRegex. Return {BucketLinks, KeyLinks}. -extract_links(LinkHeaders, BucketRegex, KeyRegex) -> - %% Run each regex against each string... - extract_links_1(LinkHeaders, BucketRegex, KeyRegex, [], []). -extract_links_1([LinkHeader|Rest], BucketRegex, KeyRegex, BucketAcc, KeyAcc) -> - case re:run(LinkHeader, BucketRegex, [{capture, all_but_first, list}]) of - {match, [Bucket, Tag]} -> - Bucket1 = list_to_binary(mochiweb_util:unquote(Bucket)), - Tag1 = list_to_binary(mochiweb_util:unquote(Tag)), - NewBucketAcc = [{Bucket1, Tag1}|BucketAcc], - extract_links_1(Rest, BucketRegex, KeyRegex, NewBucketAcc, KeyAcc); - nomatch -> - case re:run(LinkHeader, KeyRegex, [{capture, all_but_first, list}]) of - {match, [Bucket, Key, Tag]} -> - Bucket1 = list_to_binary(mochiweb_util:unquote(Bucket)), - Key1 = list_to_binary(mochiweb_util:unquote(Key)), - Tag1 = list_to_binary(mochiweb_util:unquote(Tag)), - NewKeyAcc = [{{Bucket1, Key1}, Tag1}|KeyAcc], - extract_links_1(Rest, BucketRegex, KeyRegex, BucketAcc, NewKeyAcc); - nomatch -> - throw({invalid_link_header, LinkHeader}) - end - end; -extract_links_1([], _BucketRegex, _KeyRegex, BucketAcc, KeyAcc) -> - {BucketAcc, KeyAcc}. - --type mp() :: {re_pattern, _, _, _, _}. - --spec get_compiled_link_regex(non_neg_integer(), string()) -> {mp(), mp()}. -get_compiled_link_regex(1, Prefix) -> - case persistent_term:get({?MODULE, compiled_link_regex_v1}, undefined) of - undefined -> - {ok, KeyRegex} = re:compile(" - PreCompiledExpressions - end; -get_compiled_link_regex(Two, _Prefix) when Two >= 2 -> - case persistent_term:get({?MODULE, compiled_link_regex_v2}, undefined) of - undefined -> - {ok, KeyRegex} = re:compile(?V2_KEY_REGEX), - {ok, BucketRegex} = re:compile(?V2_BUCKET_REGEX), - persistent_term:put( - {?MODULE, compiled_link_regex_v2}, - {KeyRegex, BucketRegex} - ), - {KeyRegex, BucketRegex}; - PreCompiledExpressions -> - PreCompiledExpressions - end. - --spec get_compiled_index_regex() -> mp(). -get_compiled_index_regex() -> - case persistent_term:get({?MODULE, compiled_index_regex}, undefined) of - undefined -> - {ok, IndexRegex} = re:compile(",\\s"), - persistent_term:put( - {?MODULE, compiled_index_regex}, - IndexRegex - ), - IndexRegex; - PreCompiledIndexRegex -> - PreCompiledIndexRegex - end. - --spec get_ctype(riak_object:riak_object_meta(), term()) -> string(). -%% @doc Work out the content type for this object - use the metadata if provided -get_ctype(MD,V) -> - case riak_object:metadata_find(?MD_CTYPE, MD) of - {ok, Ctype} -> - Ctype; - error when is_binary(V) -> - "application/octet-stream"; - error -> - "application/x-erlang-binary" - end. - -missing_content_type(RD) -> - RD1 = wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD), - wrq:append_to_response_body(<<"Missing Content-Type request header">>, RD1). - -send_precommit_error(RD, Reason) -> - RD1 = wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD), - Error = if - Reason =:= undefined -> - list_to_binary([atom_to_binary(wrq:method(RD1), utf8), - <<" aborted by pre-commit hook.">>]); - true -> - Reason - end, - wrq:append_to_response_body(Error, RD1). - -handle_common_error(Reason, RD, Ctx) -> - case {error, Reason} of - {error, precommit_fail} -> - {{halt, 403}, send_precommit_error(RD, undefined), Ctx}; - {error, {precommit_fail, Message}} -> - {{halt, 403}, send_precommit_error(RD, Message), Ctx}; - {error, too_many_fails} -> - {{halt, 503}, wrq:append_to_response_body("Too Many write failures" - " to satisfy W/DW\n", RD), Ctx}; - {error, timeout} -> - {{halt, 503}, - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", - wrq:append_to_response_body( - io_lib:format("request timed out~n",[]), - RD)), - Ctx}; - {error, notfound} -> - {{halt, 404}, - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", - wrq:append_to_response_body( - io_lib:format("not found~n",[]), - RD)), - Ctx}; - {error, bucket_type_unknown} -> - {{halt, 404}, - wrq:set_resp_body( - io_lib:format("Unknown bucket type: ~s", [Ctx#ctx.bucket_type]), - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD)), - Ctx}; - {error, {deleted, _VClock}} -> - {{halt, 404}, - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", - wrq:set_resp_header(?HEAD_DELETED, "true", - wrq:append_to_response_body( - io_lib:format("not found~n",[]), - encode_vclock_header(RD, Ctx)))), - Ctx}; - {error, {n_val_violation, N}} -> - Msg = io_lib:format("Specified w/dw/pw/node_confirms values invalid for bucket" - " n value of ~p~n", [N]), - {{halt, 400}, wrq:append_to_response_body(Msg, RD), Ctx}; - {error, {r_val_unsatisfied, Requested, Returned}} -> - {{halt, 503}, - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", - wrq:append_to_response_body( - io_lib:format("R-value unsatisfied: ~p/~p~n", - [Returned, Requested]), - RD)), - Ctx}; - {error, {dw_val_unsatisfied, DW, NumDW}} -> - {{halt, 503}, - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", - wrq:append_to_response_body( - io_lib:format("DW-value unsatisfied: ~p/~p~n", - [NumDW, DW]), - RD)), - Ctx}; - {error, {pr_val_unsatisfied, Requested, Returned}} -> - {{halt, 503}, - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", - wrq:append_to_response_body( - io_lib:format("PR-value unsatisfied: ~p/~p~n", - [Returned, Requested]), - RD)), - Ctx}; - {error, {pw_val_unsatisfied, Requested, Returned}} -> - Msg = io_lib:format("PW-value unsatisfied: ~p/~p~n", [Returned, - Requested]), - {{halt, 503}, wrq:append_to_response_body(Msg, RD), Ctx}; - {error, {node_confirms_val_unsatisfied, Requested, Returned}} -> - Msg = io_lib:format("node_confirms-value unsatisfied: ~p/~p~n", [Returned, - Requested]), - {{halt, 503}, wrq:append_to_response_body(Msg, RD), Ctx}; - {error, failed} -> - {{halt, 412}, RD, Ctx}; - {error, "match_found"} -> - {{halt, 412}, RD, Ctx}; - {error, "modified"} -> - {{halt, 409}, RD, Ctx}; - - {error, Err} -> - {{halt, 500}, - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", - wrq:append_to_response_body( - io_lib:format("Error:~n~p~n",[Err]), - RD)), - Ctx} - end. - -make_options(Prev, Ctx) -> - NewOpts0 = [{rw, Ctx#ctx.rw}, {r, Ctx#ctx.r}, {w, Ctx#ctx.w}, - {pr, Ctx#ctx.pr}, {pw, Ctx#ctx.pw}, {dw, Ctx#ctx.dw}, - {node_confirms, Ctx#ctx.node_confirms}, - {sync_on_write, Ctx#ctx.sync_on_write}, - {timeout, Ctx#ctx.timeout}, - {asis, Ctx#ctx.asis}], - NewOpts = [ {Opt, Val} || {Opt, Val} <- NewOpts0, - Val /= undefined, Val /= default ], - Prev ++ NewOpts. - -ensure_bucket_type(RD, Ctx) -> - Ctx0 = riak_kv_wm_utils:ensure_bucket_type(RD, Ctx, #ctx.bucket_type), - Ctx0#ctx{type_exists = - riak_kv_wm_utils:bucket_type_exists(Ctx0#ctx.bucket_type)}. - - --ifdef(TEST). - --include_lib("eunit/include/eunit.hrl"). - -generate_usermeta_header_test() -> - KV1 = {<<"X-Riak-Meta-Key">>, <<"Value1">>}, - KV2 = {<<"x-riak-meta-key">>, <<"Value2">>}, - KV3 = {<<"SomeKey">>, <<"Value3">>}, - KV4 = {<<"AMuchLongerKeyWithoutPrefix">>, <<"Value4">>}, - KV5 = {"X-Riak-Meta-Key", <<"Value5">>}, - KV6 = {"x-riak-meta-key", <<"Value6">>}, - KV7 = {"SomeKey", <<"Value7">>}, - KV8 = {"AMuchLongerKeyWithoutPrefix", <<"Value8">>}, - UserMetaHeaders = - lists:foldr( - fun generate_usermeta_header/2, - [], - [KV1, KV2, KV3, KV4, KV5, KV6, KV7, KV8] - ), - ExpResult = - [ - {<<"x-riak-meta-Key">>, <<"Value1">>}, - {<<"x-riak-meta-key">>, <<"Value2">>}, - {<<"x-riak-meta-SomeKey">>, <<"Value3">>}, - {<<"x-riak-meta-AMuchLongerKeyWithoutPrefix">>, <<"Value4">>}, - {"x-riak-meta-Key", <<"Value5">>}, - {"x-riak-meta-key", <<"Value6">>}, - {"x-riak-meta-SomeKey", <<"Value7">>}, - {"x-riak-meta-AMuchLongerKeyWithoutPrefix", <<"Value8">>} - ], - ?assertMatch(ExpResult, UserMetaHeaders). - --endif. \ No newline at end of file diff --git a/src/riak_kv_wm_ping.erl b/src/riak_kv_wm_ping.erl deleted file mode 100644 index 6d400b4db..000000000 --- a/src/riak_kv_wm_ping.erl +++ /dev/null @@ -1,55 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% riak_kv_wm_ping: simple Webmachine resource for availability test -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - -%% @doc simple Webmachine resource for availability test - --module(riak_kv_wm_ping). - -%% webmachine resource exports --export([ - init/1, - is_authorized/2, - to_html/2 - ]). - --include_lib("webmachine/include/webmachine.hrl"). - -init([]) -> - {ok, undefined}. - -is_authorized(ReqData, Ctx) -> - case riak_api_web_security:is_authorized(ReqData) of - false -> - {"Basic realm=\"Riak\"", ReqData, Ctx}; - {true, _SecContext} -> - {true, ReqData, Ctx}; - insecure -> - %% XXX 301 may be more appropriate here, but since the http and - %% https port are different and configurable, it is hard to figure - %% out the redirect URL to serve. - {{halt, 426}, wrq:append_to_resp_body(<<"Security is enabled and " - "Riak does not accept credentials over HTTP. Try HTTPS " - "instead.">>, ReqData), Ctx} - end. - -to_html(ReqData, Ctx) -> - {"OK", ReqData, Ctx}. diff --git a/src/riak_kv_wm_props.erl b/src/riak_kv_wm_props.erl deleted file mode 100644 index df6c0336b..000000000 --- a/src/riak_kv_wm_props.erl +++ /dev/null @@ -1,298 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% riak_kv_wm_props: Webmachine resource for listing bucket properties. -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - -%% @doc Resource for serving Riak bucket properties over HTTP. -%% -%% URLs that begin with `/types' are necessary for the new bucket -%% types implementation in Riak 2.0, those that begin with `/buckets' -%% are for the default bucket type, and `/riak' is an old URL style, -%% also only works for the default bucket type. -%% -%% It is possible to reconfigure the `/riak' prefix but that seems to -%% be rarely if ever used. -%% -%% ``` -%% GET /types/Type/buckets/Bucket/props -%% GET /buckets/Bucket/props -%% GET /riak/Bucket''' -%% -%% Get information about the named Bucket, in JSON form: -%% `{"props":{Prop1:Val1,Prop2:Val2,...}}' -%% -%% Each bucket property will be included in the `props' object. -%% `linkfun' and `chash_keyfun' properties will be encoded as -%% JSON objects of the form: -%% ``` -%% {"mod":ModuleName, -%% "fun":FunctionName}''' -%% -%% Where ModuleName and FunctionName are each strings representing -%% a module and function. -%% -%% ``` -%% PUT /types/Type/buckets/Bucket/props -%% PUT /buckets/Bucket/props -%% PUT /riak/Bucket''' -%% -%% Modify bucket properties. -%% -%% Content-type must be `application/json', and the body must have -%% the form: -%% `{"props":{Prop:Val}}' -%% -%% Where the `props' object takes the same form as returned from -%% a GET of the same resource. -%% -%% ``` -%% DELETE /types/Type/buckets/Bucket/props -%% DELETE /buckets/Bucket/props''' -%% -%% Reset bucket properties back to the default settings - --module(riak_kv_wm_props). - -%% webmachine resource exports --export([ - init/1, - service_available/2, - is_authorized/2, - forbidden/2, - allowed_methods/2, - malformed_request/2, - content_types_provided/2, - encodings_provided/2, - content_types_accepted/2, - resource_exists/2, - produce_bucket_body/2, - accept_bucket_body/2, - get_bucket_props_json/2, - delete_resource/2 - ]). - --record(ctx, {bucket_type, %% binary() - Bucket type (from uri) - bucket, %% binary() - Bucket name (from uri) - client, %% riak_client() - the store client - prefix, %% string() - prefix for resource uris - riak, %% local | {node(), atom()} - params for riak client - bucketprops, %% proplist() - properties of the bucket - method, %% atom() - HTTP method for the request - api_version, %% non_neg_integer() - old or new http api - security %% security context - }). --type context() :: #ctx{}. - --include_lib("webmachine/include/webmachine.hrl"). --include("riak_kv_wm_raw.hrl"). - --spec init(proplists:proplist()) -> {ok, context()}. -%% @doc Initialize this resource. This function extracts the -%% 'prefix' and 'riak' properties from the dispatch args. -init(Props) -> - {ok, #ctx{ - prefix=proplists:get_value(prefix, Props), - riak=proplists:get_value(riak, Props), - api_version=proplists:get_value(api_version,Props), - bucket_type=proplists:get_value(bucket_type, Props) - }}. - --spec service_available(#wm_reqdata{}, context()) -> - {boolean(), #wm_reqdata{}, context()}. -%% @doc Determine whether or not a connection to Riak can be -%% established. This function also takes this opportunity to extract -%% the 'bucket' bindings from the dispatch. -service_available(RD, Ctx0=#ctx{riak=RiakProps}) -> - Ctx = riak_kv_wm_utils:ensure_bucket_type(RD, Ctx0, #ctx.bucket_type), - case riak_kv_wm_utils:get_riak_client(RiakProps, riak_kv_wm_utils:get_client_id(RD)) of - {ok, C} -> - {true, - RD, - Ctx#ctx{ - method=wrq:method(RD), - client=C, - bucket=case wrq:path_info(bucket, RD) of - undefined -> undefined; - B -> list_to_binary(riak_kv_wm_utils:maybe_decode_uri(RD, B)) - end - }}; - Error -> - {false, - wrq:set_resp_body( - io_lib:format("Unable to connect to Riak: ~p~n", [Error]), - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD)), - Ctx} - end. - -is_authorized(ReqData, Ctx) -> - case riak_api_web_security:is_authorized(ReqData) of - false -> - {"Basic realm=\"Riak\"", ReqData, Ctx}; - {true, SecContext} -> - {true, ReqData, Ctx#ctx{security=SecContext}}; - insecure -> - %% XXX 301 may be more appropriate here, but since the http and - %% https port are different and configurable, it is hard to figure - %% out the redirect URL to serve. - {{halt, 426}, wrq:append_to_resp_body(<<"Security is enabled and " - "Riak does not accept credentials over HTTP. Try HTTPS " - "instead.">>, ReqData), Ctx} - end. - -forbidden(RD, Ctx = #ctx{security=undefined}) -> - {riak_kv_wm_utils:is_forbidden(RD), RD, Ctx}; -forbidden(RD, Ctx=#ctx{security=Security}) -> - case riak_kv_wm_utils:is_forbidden(RD) of - true -> - {true, RD, Ctx}; - false -> - Perm = case Ctx#ctx.method of - 'PUT' -> - "riak_core.set_bucket"; - 'GET' -> - "riak_core.get_bucket"; - 'HEAD' -> - "riak_core.get_bucket"; - 'DELETE' -> - "riak_core.set_bucket" - end, - - Res = riak_core_security:check_permission({Perm, - {Ctx#ctx.bucket_type, - Ctx#ctx.bucket}}, - Security), - case Res of - {false, Error, _} -> - RD1 = wrq:set_resp_header("Content-Type", "text/plain", RD), - {true, wrq:append_to_resp_body(unicode:characters_to_binary(Error, utf8, utf8), RD1), Ctx}; - {true, _} -> - {false, RD, Ctx} - end - end. - --spec allowed_methods(#wm_reqdata{}, context()) -> - {[atom()], #wm_reqdata{}, context()}. -%% @doc Get the list of methods this resource supports. -%% Properties allows HEAD, GET, and PUT. -allowed_methods(RD, Ctx) when Ctx#ctx.api_version =:= 1 -> - {['HEAD', 'GET', 'PUT'], RD, Ctx}; -allowed_methods(RD, Ctx) when Ctx#ctx.api_version =:= 2; - Ctx#ctx.api_version =:= 3 -> - {['HEAD', 'GET', 'PUT', 'DELETE'], RD, Ctx}. - --spec malformed_request(#wm_reqdata{}, context()) -> - {boolean(), #wm_reqdata{}, context()}. -%% @doc Determine whether query parameters, request headers, -%% and request body are badly-formed. -%% Body format is checked to be valid JSON, including -%% a "props" object for a bucket-PUT. -malformed_request(RD, Ctx) when Ctx#ctx.method =:= 'PUT' -> - malformed_bucket_put(RD, Ctx); -malformed_request(RD, Ctx) -> - {false, RD, Ctx}. - --spec malformed_bucket_put(#wm_reqdata{}, context()) -> - {boolean(), #wm_reqdata{}, context()}. -%% @doc Check the JSON format of a bucket-level PUT. -%% Must be a valid JSON object, containing a "props" object. -malformed_bucket_put(RD, Ctx) -> - case catch mochijson2:decode(wrq:req_body(RD)) of - {struct, Fields} -> - case proplists:get_value(?JSON_PROPS, Fields) of - {struct, Props} -> - {false, RD, Ctx#ctx{bucketprops=Props}}; - _ -> - {true, bucket_format_message(RD), Ctx} - end; - _ -> - {true, bucket_format_message(RD), Ctx} - end. - --spec bucket_format_message(#wm_reqdata{}) -> #wm_reqdata{}. -%% @doc Put an error about the format of the bucket-PUT body -%% in the response body of the #wm_reqdata{}. -bucket_format_message(RD) -> - wrq:append_to_resp_body( - ["bucket PUT must be a JSON object of the form:\n", - "{\"",?JSON_PROPS,"\":{...bucket properties...}}"], - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD)). - - -resource_exists(RD, Ctx) -> - {riak_kv_wm_utils:bucket_type_exists(Ctx#ctx.bucket_type), RD, Ctx}. - --spec content_types_provided(#wm_reqdata{}, context()) -> - {[{ContentType::string(), Producer::atom()}], #wm_reqdata{}, context()}. -%% @doc List the content types available for representing this resource. -%% "application/json" is the content-type for props requests. -content_types_provided(RD, Ctx) -> - {[{"application/json", produce_bucket_body}], RD, Ctx}. - --spec encodings_provided(#wm_reqdata{}, context()) -> - {[{Encoding::string(), Producer::function()}], #wm_reqdata{}, context()}. -%% @doc List the encodings available for representing this resource. -%% "identity" and "gzip" are available for props requests. -encodings_provided(RD, Ctx) -> - {riak_kv_wm_utils:default_encodings(), RD, Ctx}. - --spec content_types_accepted(#wm_reqdata{}, context()) -> - {[{ContentType::string(), Acceptor::atom()}], - #wm_reqdata{}, context()}. -%% @doc Get the list of content types this resource will accept. -%% "application/json" is the only type accepted for props PUT. -content_types_accepted(RD, Ctx) -> - {[{"application/json", accept_bucket_body}], RD, Ctx}. - --spec produce_bucket_body(#wm_reqdata{}, context()) -> - {binary(), #wm_reqdata{}, context()}. -%% @doc Produce the bucket properties as JSON. -produce_bucket_body(RD, Ctx) -> - Client = Ctx#ctx.client, - Bucket = riak_kv_wm_utils:maybe_bucket_type(Ctx#ctx.bucket_type, Ctx#ctx.bucket), - JsonProps1 = get_bucket_props_json(Client, Bucket), - JsonProps2 = {struct, [JsonProps1]}, - JsonProps3 = mochijson2:encode(JsonProps2), - {JsonProps3, RD, Ctx}. - -get_bucket_props_json(Client, Bucket) -> - Props1 = riak_client:get_bucket(Bucket, Client), - Props2 = lists:map(fun riak_kv_wm_utils:jsonify_bucket_prop/1, Props1), - {?JSON_PROPS, {struct, Props2}}. - --spec accept_bucket_body(#wm_reqdata{}, context()) -> {true, #wm_reqdata{}, context()}. -%% @doc Modify the bucket properties according to the body of the -%% bucket-level PUT request. -accept_bucket_body(RD, Ctx=#ctx{bucket_type=T, bucket=B, client=C, bucketprops=Props}) -> - ErlProps = lists:map(fun riak_kv_wm_utils:erlify_bucket_prop/1, Props), - case riak_client:set_bucket({T,B}, ErlProps, C) of - ok -> - {true, RD, Ctx}; - {error, Details} -> - JSON = mochijson2:encode(Details), - RD2 = wrq:append_to_resp_body(JSON, RD), - {{halt, 400}, RD2, Ctx} - end. - --spec delete_resource(#wm_reqdata{}, context()) -> - {boolean(), #wm_reqdata{}, context()}. -%% @doc Reset the bucket properties back to the default values -delete_resource(RD, Ctx=#ctx{bucket_type=T, bucket=B, client=C}) -> - riak_client:reset_bucket({T,B}, C), - {true, RD, Ctx}. diff --git a/src/riak_kv_wm_query.erl b/src/riak_kv_wm_query.erl deleted file mode 100644 index bfcdab7f1..000000000 --- a/src/riak_kv_wm_query.erl +++ /dev/null @@ -1,1287 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2016 Basho Technologies, Inc. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - -%% @doc Webmachine resource for running queries on secondary indexes. -%% -%% Available operations: -%% -%% ``` -%% POST types//buckets//query -%% POST buckets//query (legacy support for untyped buckets) -%% ``` -%% -%% The query should be posted as the HTTP body, where there are the following -%% JSON keys at the root of the document -%% -%% ?AGGREGATION_EXPRESSION -%% ?ACCUMULATION_OPTION -%% ?ACCUMULATION_TERM -%% ?SUBSTITUTIONS -%% ?TIMEOUT -%% ?MAX_RESULTS -%% ?CONTINUATION -%% ?QUERY_LIST -%% -%% Each query in the query list must be a JSON document supporting the -%% following keys: -%% -%% ?QL_AGGREGATION_TAG -%% ?QL_INDEX_NAME -%% ?QL_START_TERM -%% ?QL_END_TERM -%% ?QL_REGULAR_EXPRESSION -%% ?QL_EVALUATION_EXPRESSION -%% ?QL_FILTER_EXPRESSION -%% -%% For details on usage see https://openriak.github.io/riak_kv/QueryAPI.html. -%% -%% Queries POST'd will return a JSON object with the result format determined -%% by the ?ACCUMULATION_OPTION passed in the query -%% -%% ``` -%% GET types//bucket//query -%% GET buckets//query (legacy support for untyped buckets) -%% ``` -%% -%% The GET operation is used to extract results queued using a query with the -%% ?ACCUMULATION_OPTION of `queue_raw_keys` or `queue_raw_terms` -%% -%% Requests support two query parameters: -%% -%% ?result_queue=%max_results= -%% -%% The result_queue is a mandatory parameter, and is returned as the -%% ?RSP_RESULT_QUEUE key in the JSON object received from POSTing a query with -%% a queue-based ?ACCUMULATION_OPTION. -%% -%% Multiple client-side processes may request results from the queue -%% concurrently, from any connected node within the cluster. Each result will -%% be returned once only. - --module(riak_kv_wm_query). - --include_lib("webmachine/include/webmachine.hrl"). --include("riak_kv_wm_raw.hrl"). - -%% webmachine resource exports --export([ - init/1, - service_available/2, - is_authorized/2, - forbidden/2, - allowed_methods/2, - malformed_request/2, - resource_exists/2, - process_post/2, - encode_key/2, - encode_key_withterm/2, - content_types_provided/2, - return_queued_results/2 -]). - --export( - [ - get_result_key/1 - ] -). - --record(ctx, - { - client, %% riak_client() - the store client - riak, %% local | {node(), atom()} - params for riak client - bucket_type, %% Bucket type (from uri) - query_request, %% The query.. - queue_request, - security, - method - } -). - --type context() :: #ctx{}. --type request_data() :: #wm_reqdata{}. - --define(AGGREGATION_EXPRESSION, <<"aggregation_expression">>). --define(ACCUMULATION_OPTION, <<"accumulation_option">>). --define(ACCUMULATION_TERM, <<"accumulation_term">>). --define(SUBSTITUTIONS, <<"substitutions">>). --define(TIMEOUT, <<"timeout">>). --define(INACTIVITY_TIMEOUT, <<"inactivity_timeout">>). --define(MAX_RESULTS, <<"max_results">>). --define(CONTINUATION, <<"continuation">>). --define(QUERY_LIST, <<"query_list">>). --define(QL_AGGREGATION_TAG, <<"aggregation_tag">>). --define(QL_INDEX_NAME, <<"index_name">>). --define(QL_START_TERM, <<"start_term">>). --define(QL_END_TERM, <<"end_term">>). --define(QL_REGULAR_EXPRESSION, <<"regular_expression">>). --define(QL_EVALUATION_EXPRESSION, <<"evaluation_expression">>). --define(QL_FILTER_EXPRESSION, <<"filter_expression">>). - --define(ACCKEY_KEYS, <<"keys">>). --define(ACCKEY_TERMS, <<"terms">>). --define(ACCKEY_COUNT, <<"count">>). --define(ACCKEY_TERMCOUNT, <<"term_with_count">>). --define(ACCKEY_RAWKEYS, <<"raw_keys">>). --define(ACCKEY_RAWTERMS, <<"raw_terms">>). --define(ACCKEY_RAWCOUNT, <<"raw_count">>). --define(ACCKEY_TERMRAWCOUNT, <<"term_with_rawcount">>). - --define(REQUIRED_KEYS, [?QUERY_LIST]). --define(POSSIBLE_KEYS, - [ - ?AGGREGATION_EXPRESSION, - ?ACCUMULATION_OPTION, - ?ACCUMULATION_TERM, - ?SUBSTITUTIONS, - ?TIMEOUT, - ?INACTIVITY_TIMEOUT, - ?QUERY_LIST, - ?MAX_RESULTS, - ?CONTINUATION - ] -). --define(REQUIRED_QL_KEYS, - [ - ?QL_INDEX_NAME, - ?QL_START_TERM, - ?QL_END_TERM - ] -). --define(POSSIBLE_QL_KEYS, - [ - ?QL_AGGREGATION_TAG, - ?QL_INDEX_NAME, - ?QL_START_TERM, - ?QL_END_TERM, - ?QL_REGULAR_EXPRESSION, - ?QL_EVALUATION_EXPRESSION, - ?QL_FILTER_EXPRESSION - ] -). - --define(REQUEST_CLASS, {riak_kv, secondary_index}). - --define(QUERY_TIMEOUT, 60). --define(QUEUE_INACTIVITY_TIMEOUT, 120). --define(MAX_RESULTS_FROM_QUEUE, 1000). - --define(HEAD_CONTINUATION, "X-Riak-Continuation"). - --type query_map() :: - #{binary() => binary()|non_neg_integer()|list(map())}. - --spec init(proplists:proplist()) -> {ok, context()}. -%% @doc Initialize this resource. -init(Props) -> - {ok, #ctx{ - riak=proplists:get_value(riak, Props), - bucket_type=proplists:get_value(bucket_type, Props) - }}. - --spec service_available(request_data(), context()) -> - {boolean(), request_data(), context()}. -%% @doc Determine whether or not a connection to Riak -%% can be established. Also, extract query params. -service_available(RD, Ctx0=#ctx{riak=RiakProps}) -> - Ctx = riak_kv_wm_utils:ensure_bucket_type(RD, Ctx0, #ctx.bucket_type), - ClientID = riak_kv_wm_utils:get_client_id(RD), - case riak_kv_wm_utils:get_riak_client(RiakProps, ClientID) of - {ok, C} -> - { - true, - RD, - Ctx#ctx{client = C, method = wrq:method(RD)} - }; - Error -> - { - false, - wrq:set_resp_body( - io_lib:format("Unable to connect to Riak: ~p~n", [Error]), - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD)), - Ctx - } - end. - -resource_exists(RD, #ctx{bucket_type=BType}=Ctx) -> - {riak_kv_wm_utils:bucket_type_exists(BType), RD, Ctx}. - --spec is_authorized(request_data(), context()) -> - {true | string() | {halt, 426}, request_data(), context()}. -is_authorized(ReqData, Ctx) -> - case riak_api_web_security:is_authorized(ReqData) of - false -> - {"Basic realm=\"Riak\"", ReqData, Ctx}; - {true, SecContext} -> - {true, ReqData, Ctx#ctx{security=SecContext}}; - insecure -> - %% XXX 301 may be more appropriate here, but since the http and - %% https port are different and configurable, it is hard to figure - %% out the redirect URL to serve. - { - {halt, 426}, - wrq:append_to_resp_body( - <<"Security is enabled and " - "Riak does not accept credentials over HTTP. Try HTTPS " - "instead.">>, - ReqData - ), - Ctx - } - end. - --spec forbidden(request_data(), context()) - -> {boolean(), request_data(), context()}. -forbidden(ReqDataIn, #ctx{security = undefined} = Context) -> - riak_kv_wm_utils:is_forbidden(ReqDataIn, ?REQUEST_CLASS, Context); -forbidden(ReqDataIn, #ctx{bucket_type = BT, security = Sec} = Context) -> - {Answer, ReqData, _} = Result = - riak_kv_wm_utils:is_forbidden(ReqDataIn, ?REQUEST_CLASS, Context), - case Answer of - false -> - Bucket = erlang:list_to_binary( - riak_kv_wm_utils:maybe_decode_uri( - ReqData, wrq:path_info(bucket, ReqData))), - case riak_core_security:check_permission( - {"riak_kv.index", {BT, Bucket}}, Sec) of - {false, Error, _} -> - {true, - wrq:append_to_resp_body( - unicode:characters_to_binary(Error, utf8, utf8), - wrq:set_resp_header( - "Content-Type", "text/plain", ReqData)), - Context}; - {true, _} -> - {false, ReqData, Context} - end; - _ -> - Result - end. - --spec allowed_methods( - request_data(), context()) -> {list(atom()), request_data(), context()}. -allowed_methods(RD, Ctx) -> - {['POST', 'GET'], RD, Ctx}. - --spec malformed_request( - request_data(), context()) -> - {boolean(), request_data(), context()}. -malformed_request(RD, Ctx) when Ctx#ctx.method =:= 'POST' -> - Bucket = get_bucket(RD), - BT = riak_kv_wm_utils:maybe_bucket_type(Ctx#ctx.bucket_type, Bucket), - Body = riak_kv_wm_utils:accept_value("application/json", wrq:req_body(RD)), - case decode_json_body(Body) of - {ok, QueryMap} -> - case check_keys(maps:keys(QueryMap), request) of - ok -> - QueryList = maps:get(?QUERY_LIST, QueryMap), - case check_querylist(QueryList, false) of - ok -> - case make_query_request(BT, QueryMap) of - {ok, Query} -> - {false, RD, Ctx#ctx{query_request = Query}}; - {error, Stage, Reason} -> - { - true, - return_json_error( - expand_query_reason(Stage, Reason), - RD - ), - Ctx - } - end; - {error, Reason} -> - {true, return_json_error(Reason, RD), Ctx} - end; - {error, Reason} -> - {true, return_json_error(Reason, RD), Ctx} - end; - {error, Reason} -> - {true, return_json_error(Reason, RD), Ctx} - end; -malformed_request(RD, Ctx) when Ctx#ctx.method =:= 'GET' -> - Bucket = get_bucket(RD), - BT = riak_kv_wm_utils:maybe_bucket_type(Ctx#ctx.bucket_type, Bucket), - case wrq:get_qs_value("result_queue", RD) of - QueueString when is_list(QueueString) -> - case wrq:get_qs_value("max_results", RD) of - undefined -> - application:get_env( - riak_kv, - queue_raw_max_results, - ?MAX_RESULTS_FROM_QUEUE - ); - MaxResultsString -> - try - case list_to_integer(MaxResultsString) of - MR when is_integer(MR), MR >= 0 -> - { - false, - RD, - Ctx#ctx{ - queue_request = - make_queue_request( - BT, - list_to_binary(QueueString), - MR - ) - } - } - - end - catch - _CP:_EP -> - { - true, - return_json_error( - "Invalid max_results parameter", - RD - ), - Ctx - } - end - end; - _ -> - { - true, - return_json_error( - "No valid result_queue reference" - "passed as query parameter", - RD - ), - Ctx - } - end. - --spec content_types_provided(request_data(), context()) -> - {[{ContentType::string(), Producer::atom()}], request_data(), context()}. -%% @doc List the content types available for representing this resource. -%% "application/json" is the content-type for bucket lists. -content_types_provided(RD, Ctx) when Ctx#ctx.method =:= 'POST' -> - {[{"application/json", nop}], RD, Ctx}; -content_types_provided(RD, Ctx) when Ctx#ctx.method =:= 'GET' -> - {[{"application/json", return_queued_results}], RD, Ctx}. - --spec return_queued_results( - request_data(), context() -) -> - {binary(), request_data(), context()}. -return_queued_results(RD, Ctx = #ctx{queue_request = QR, client = C}) -> - case riak_client:query_result_request(QR, C) of - {ok, ResultMap} -> - {encode_queued_results(ResultMap), RD, Ctx}; - {error, result_server_terminated} -> - { - {halt, 410}, - % Response code for Gone, and likely to be permanent. - % This may be as a result of an error on the server, but - % is probably as a result of an error on the client - and - % so to help with load-balancers tracking server errors, - % err on the side of blaming the client - return_json_error( - "queue no longer present or not currently reachable\n", - RD - ), - Ctx - }; - {error, unexpected_reference_format} -> - { - {halt, 400}, - return_json_error( - "queue reference passed had an invalid format\n", - RD - ), - Ctx - }; - {error, Reason} -> - {{error, Reason}, RD, Ctx} - end. - -%% The bucket is available in the dispatch properties, however it may need to -%% URL quoted, and so it needs to be unquoted. -%% -%% Note that it is possible to disable quoting, and force it per request using -%% the "X-Riak-URL-Encoding" header - hence why the full RD is required to -%% decide on the unquoting or not of one part. -get_bucket(RD) -> - list_to_binary( - riak_kv_wm_utils:maybe_decode_uri(RD, wrq:path_info(bucket, RD) - ) - ). - -expand_query_reason(Stage, Reason) -> - lists:flatten( - io_lib:format( - << "Validation failure at stage ~0p due to ~s">>, - [Stage, Reason] - ) - ). - --spec return_json_error(string(), request_data()) -> request_data(). -return_json_error(Reason, RD) -> - wrq:append_to_resp_body( - riak_kv_wm_json:encode(#{error => Reason}), - wrq:set_resp_header( - ?HEAD_CTYPE, "application/json", RD - ) - ). - --spec decode_json_body(binary()) -> {ok, map()}| {error, term()}. -decode_json_body(JsonBody) -> - try - DecodedBody = riak_kv_wm_json:decode(JsonBody), - {ok, DecodedBody} - catch - error:Reason -> - ExpandedReason = - lists:flatten( - io_lib:format( - <<"Malformed json request - ~0p">>, - [Reason] - ) - ), - {error, ExpandedReason} - end. - -check_querylist([], true) -> - ok; -check_querylist([], false) -> - {error, <<"No valid query provided">>}; -check_querylist([HdQuery|Rest], _AtLeastOne) -> - case check_keys(maps:keys(HdQuery), query) of - ok -> - check_querylist(Rest, true); - Error -> - Error - end. - -check_keys(Keys, request) -> - check_keys(Keys, ?REQUIRED_KEYS, ?POSSIBLE_KEYS); -check_keys(Keys, query) -> - check_keys(Keys, ?REQUIRED_QL_KEYS, ?POSSIBLE_QL_KEYS). - --spec check_keys( - list(binary()), list(binary()), list(binary())) -> ok|{error, string()}. -check_keys(Keys, RequiredKeys, PossibleKeys) -> - RequiredKeyList = - lists:filter( - fun(K) -> lists:member(K, Keys) end, - RequiredKeys - ), - PossibleKeyList = - lists:filter( - fun(K) -> lists:member(K, PossibleKeys) end, - Keys - ), - case RequiredKeyList of - RequiredKeys -> - case PossibleKeyList of - Keys -> - ok; - NotAllKeys -> - ExtraKeys = lists:subtract(Keys, NotAllKeys), - { - error, - lists:flatten( - io_lib:format( - <<"Unexpected keys in request ~0p">>, - [ExtraKeys] - ) - ) - } - end; - NotAllRequiredKeys -> - MissingKeys = lists:subtract(RequiredKeys, NotAllRequiredKeys), - { - error, - lists:flatten( - io_lib:format( - <<"Missing required keys in request ~0p">>, - [MissingKeys] - ) - ) - } - end. - --spec make_queue_request( - riak_object:bucket(), binary(), non_neg_integer() -) -> - #{atom() => term()}. -make_queue_request(Bucket, EncodedQueueRef, MaxResults) -> - #{ - bucket => Bucket, - encoded_queue_reference => EncodedQueueRef, - max_results => MaxResults - }. - --spec make_query_request( - riak_object:bucket(), query_map()) -> - {ok, riak_kv_query:complex_query_definition()}|riak_kv_query:validation_error(). -make_query_request(BucketType, QueryMap) -> - case fetch_timeouts(QueryMap) of - {ok, Timeout, InactivityTimeout} -> - QueryType = - case maps:get(?QUERY_LIST, QueryMap) of - QueryList when length(QueryList) == 1 -> - single_query; - QueryList when length(QueryList) > 1 -> - combo_query - end, - InitQuery = - riak_kv_query:new( - BucketType, - QueryType, - Timeout, - InactivityTimeout - ), - case add_accumulation(QueryMap, InitQuery) of - {ok, Q1} -> - case add_queries(QueryMap, Q1, QueryList) of - {ok, Q2} -> - case maps:get(?CONTINUATION, QueryMap, none) of - none -> - {ok, Q2}; - Continuation -> - riak_kv_query:add_continuation(Q2, Continuation) - end; - Error -> - Error - end; - Error -> - Error - end; - Error -> - Error - end. - --spec fetch_timeouts( - query_map() -) -> - {ok, pos_integer(), pos_integer()} | riak_kv_query:validation_error(). -fetch_timeouts(QueryMap) -> - Timeout = - maps:get( - ?TIMEOUT, - QueryMap, - application:get_env(riak_kv, query_timeout_secs, ?QUERY_TIMEOUT) - ), - InactivityTimeout = - maps:get( - ?INACTIVITY_TIMEOUT, - QueryMap, - application:get_env( - riak_kv, - queue_inactivity_timeout_secs, - ?QUEUE_INACTIVITY_TIMEOUT - ) - ), - case Timeout of - T when is_integer(T), T > 0 -> - case InactivityTimeout of - IT when is_integer(IT), IT > 0 -> - {ok, T, IT}; - _ -> - {error, init, <<"Bad inactivity timeout">>} - end; - _ -> - {error, init, <<"Bad timeout">>} - end. - --spec add_accumulation( - query_map(), riak_kv_query:complex_query_definition()) - -> - {ok, riak_kv_query:complex_query_definition()} | - riak_kv_query:validation_error(). -add_accumulation(QueryMap, InitQuery) -> - AccOpt = maps:get(?ACCUMULATION_OPTION, QueryMap, undefined), - AccTerm = maps:get(?ACCUMULATION_TERM, QueryMap, undefined), - MaxResults = maps:get(?MAX_RESULTS, QueryMap, undefined), - case riak_kv_query:add_accumulation_option(InitQuery, AccOpt) of - {ok, UpdQuery0} -> - case riak_kv_query:add_accumulation_term(UpdQuery0, AccTerm) of - {ok, UpdQuery1} -> - case MaxResults of - undefined -> - {ok, UpdQuery1}; - MR -> - riak_kv_query:add_maxresults(UpdQuery1, MR) - end; - Error -> - Error - end; - Error -> - Error - end. - --spec add_queries( - query_map(), - riak_kv_query:complex_query_definition(), - list(#{binary() => binary()})) -> - {ok, riak_kv_query:complex_query_definition()}| - riak_kv_query:validation_error(). -add_queries(QueryMap, Query, QueryList) -> - AggExpr = - maps:get(?AGGREGATION_EXPRESSION, QueryMap, undefined), - case riak_kv_query:add_aggregation_expression(Query, AggExpr) of - {ok, Q2} -> - Subs = - maps:get(?SUBSTITUTIONS, QueryMap, maps:new()), - riak_kv_query:add_queries( - Q2, - lists:map(fun convert_query/1, QueryList), - Subs - ); - Error -> - Error - end. - --spec convert_query(map()) -> riak_kv_query:query_user_input(). -convert_query(QM) -> - { - maps:get(<<"aggregation_tag">>, QM, undefined), - maps:get(<<"index_name">>, QM), - maps:get(<<"start_term">>, QM), - maps:get(<<"end_term">>, QM), - maps:get(<<"regular_expression">>, QM, undefined), - maps:get(<<"evaluation_expression">>, QM, undefined), - maps:get(<<"filter_expression">>, QM, undefined) - }. - --spec process_post(request_data(), context()) -> - {boolean()|{halt, pos_integer()}, request_data(), context()}. -%% @doc Produce the JSON response to an index lookup. -process_post(RD, Ctx) -> - Client = Ctx#ctx.client, - AccOpt = riak_kv_query:get_accumulator(Ctx#ctx.query_request), - {ok, Query} = - riak_kv_query:add_result_encodingfun( - Ctx#ctx.query_request, - encoding_function(AccOpt) - ), - case riak_client:query(Query, Client) of - {error, timeout} -> - {{halt, 503}, return_json_error("timeout", RD), Ctx}; - {error, Reason} -> - Error = - lists:flatten( - io_lib:format( - <<"Query with option ~w failed - ~0p">>, - [AccOpt, Reason] - ) - ), - {{halt, 500}, return_json_error(Error, RD), Ctx}; - {result_queue, ResultReference} when is_binary(ResultReference) -> - { - true, - wrq:append_to_resp_body( - riak_kv_wm_json:encode( - #{result_queue => ResultReference} - ), - wrq:set_resp_header(?HEAD_CTYPE, "application/json", RD) - ), - Ctx - }; - {JsonEncodedResults, none} when is_binary(JsonEncodedResults) -> - { - true, - wrq:append_to_resp_body( - JsonEncodedResults, - wrq:set_resp_header(?HEAD_CTYPE, "application/json", RD) - ), - Ctx - }; - {JsonEncodedResults, {{LT, LK}}} - when - is_binary(JsonEncodedResults), - is_binary(LT), - is_binary(LK) -> - Continuation = riak_kv_query:make_continuation(LT, LK), - { - true, - wrq:append_to_resp_body( - JsonEncodedResults, - wrq:set_resp_header( - ?HEAD_CONTINUATION, - Continuation, - wrq:set_resp_header( - ?HEAD_CTYPE, - "application/json", - RD - ) - ) - ), - Ctx - } - end. - --spec encoding_function(riak_kv_query:accumulation_option()) -> - fun((riak_kv_query_server:results()) -> binary()). -encoding_function(AccOpt) -> - fun(Results) -> encode_results(AccOpt, Results) end. - --spec encode_queued_results( - riak_kv_query_server:partial_result_map()) -> binary(). -encode_queued_results(ResultMap) -> - case maps:is_key(get_result_key(raw_keys), ResultMap) of - true -> - iolist_to_binary( - riak_kv_wm_json:encode( - ResultMap, - fun riak_kv_wm_query:encode_key/2 - ) - ); - false -> - case maps:is_key(get_result_key(raw_terms), ResultMap) of - true -> - iolist_to_binary( - riak_kv_wm_json:encode( - ResultMap, - fun riak_kv_wm_query:encode_key_withterm/2 - ) - ) - end - end. - --spec encode_results( - riak_kv_query:accumulation_option(), riak_kv_query_server:results()) -> binary(). -encode_results(AccOpt, Results) when AccOpt == keys; AccOpt == raw_keys -> - iolist_to_binary( - riak_kv_wm_json:encode( - #{get_result_key(AccOpt) => Results}, - fun riak_kv_wm_query:encode_key/2 - ) - ); -encode_results(AccOpt, Results) when AccOpt == terms; AccOpt == raw_terms -> - iolist_to_binary( - riak_kv_wm_json:encode( - #{get_result_key(AccOpt) => Results}, - fun riak_kv_wm_query:encode_key_withterm/2 - ) - ); -encode_results(AccOpt, Count) when AccOpt == count; AccOpt == raw_count -> - iolist_to_binary( - riak_kv_wm_json:encode(#{get_result_key(AccOpt) => Count}) - ); -encode_results(AccOpt, CountMap) - when AccOpt == term_with_count; AccOpt == term_with_rawcount -> - iolist_to_binary( - riak_kv_wm_json:encode(#{get_result_key(AccOpt) => CountMap}) - ). - -encode_key({{_Term, Key}}, Encode) when is_binary(Key) -> - encode_key(Key, Encode); -encode_key({Key}, Encode) when is_binary(Key) -> - encode_key(Key, Encode); -encode_key(Key, Encode) -> - riak_kv_wm_json:encode_value(Key, Encode). - -encode_key_withterm({TermKeyTuple}, Encode) when is_tuple(TermKeyTuple) -> - encode_key_withterm(TermKeyTuple, Encode); -encode_key_withterm({Term, Key}, Encode) when is_binary(Term), is_binary(Key) -> - [123, [Encode(Term, Encode), $: | Encode(Key, Encode)], 125]; -encode_key_withterm(Result, Encode) -> - riak_kv_wm_json:encode_value(Result, Encode). - --spec get_result_key(riak_kv_query:accumulation_option()) -> binary(). -get_result_key(keys) -> ?ACCKEY_KEYS; -get_result_key(raw_keys) -> ?ACCKEY_RAWKEYS; -get_result_key(terms) -> ?ACCKEY_TERMS; -get_result_key(raw_terms) -> ?ACCKEY_RAWTERMS; -get_result_key(count) -> ?ACCKEY_COUNT; -get_result_key(raw_count) -> ?ACCKEY_RAWCOUNT; -get_result_key(term_with_count) -> ?ACCKEY_TERMCOUNT; -get_result_key(term_with_rawcount) -> ?ACCKEY_TERMRAWCOUNT. - - - -%% =================================================================== -%% EUnit tests -%% =================================================================== - --ifdef(TEST). - --include_lib("eunit/include/eunit.hrl"). - -invalid_json_test() -> - InvalidJson = - <<" - { - \"accumulation_option\" : \"keys\", - \"timeout\" : 60, - \"query_list\" : - [ - { - \"index_name\" : \"example_bin\" - \"start_term\" : \"A\", - \"end_term\" : \"B\" - } - ] - } - ">>, % Missing comma after example_bin - R = decode_json_body(InvalidJson), - io:format("~p~n", [R]), - ?assertMatch( - {error, "Malformed json request - {invalid_byte,34}"}, - R - ). - -simple_query_test() -> - SimpleQueryJson = - <<" - { - \"timeout\" : 60, - \"query_list\" : - [ - { - \"index_name\" : \"example_bin\", - \"start_term\" : \"A\", - \"end_term\" : \"B\" - } - ] - } - ">>, - {ok, M} = decode_json_body(SimpleQueryJson), - {ok, Q} = make_query_request({<<"BT">>, <<"B">>}, M), - ?assert(riak_kv_query:is_query(Q)). - -invalid_query_ae1_test() -> - IQJson = - <<" - { - \"aggregation_expression\" : \"$1 INTERSECT $2\", - \"timeout\" : 60, - \"query_list\" : - [ - { - \"index_name\" : \"example_bin\", - \"start_term\" : \"A\", - \"end_term\" : \"B\" - } - ] - } - ">>, - {ok, M} = decode_json_body(IQJson), - {error, S, _E} = make_query_request({<<"BT">>, <<"B">>}, M), - ?assertMatch(aggregation_expression, S). - -invalid_query_ae2_test() -> - IQJson = - <<" - { - \"timeout\" : 60, - \"query_list\" : - [ - { - \"aggregation_tag\" : 1, - \"index_name\" : \"example_bin\", - \"start_term\" : \"A\", - \"end_term\" : \"B\" - }, - { - \"aggregation_tag\" : 2, - \"index_name\" : \"example_bin\", - \"start_term\" : \"A\", - \"end_term\" : \"B\" - } - - ] - } - ">>, - {ok, M} = decode_json_body(IQJson), - {error, S, _E} = make_query_request({<<"BT">>, <<"B">>}, M), - ?assertMatch(aggregation_expression, S). - -invalid_query_ae3_test() -> - IQJson = - <<" - { - \"aggregation_expression\" : \"$1 INTERSECT $2\", - \"timeout\" : 60, - \"query_list\" : - [ - { - \"index_name\" : \"example_bin\", - \"start_term\" : \"A\", - \"end_term\" : \"B\" - }, - { - \"aggregation_tag\" : 2, - \"index_name\" : \"example_bin\", - \"start_term\" : \"A\", - \"end_term\" : \"B\" - } - - ] - } - ">>, - {ok, M} = decode_json_body(IQJson), - {error, S, E} = make_query_request({<<"BT">>, <<"B">>}, M), - ?assertMatch(query_evaluation, S), - ?assertMatch(<<"Untagged query in combination request">>, E). - -valid_query_ae4_test() -> - IQJson = - <<" - { - \"aggregation_expression\" : \"$1 INTERSECT $2\", - \"timeout\" : 60, - \"inactivity_timeout\" : 180, - \"query_list\" : - [ - { - \"aggregation_tag\" : 1, - \"index_name\" : \"example_bin\", - \"start_term\" : \"A\", - \"end_term\" : \"B\" - }, - { - \"aggregation_tag\" : 2, - \"index_name\" : \"example_bin\", - \"start_term\" : \"A\", - \"end_term\" : \"B\" - } - - ] - } - ">>, - {ok, M} = decode_json_body(IQJson), - {ok, Q} = make_query_request({<<"BT">>, <<"B">>}, M), - ?assert(riak_kv_query:is_query(Q)), - QueryList = maps:get(<<"query_list">>, M), - ?assertMatch(ok, check_querylist(QueryList, false)). - -valid_query_ae5_test() -> - IQJson = - <<" - { - \"aggregation_expression\" : \"$1 INTERSECT $2\", - \"timeout\" : 60, - \"accumulation_option\" : \"keys\", - \"substitutions\" : - {\"low_dob\" : \"20210804\", \"high_dob\" : \"20223101\", \"gnsc\" : \"Ma\"}, - \"query_list\" : - [ - { - \"aggregation_tag\" : 1, - \"index_name\" : \"example1_bin\", - \"start_term\" : \"A\", - \"end_term\" : \"B\", - \"evaluation_expression\" : - \"delim($term, \\\"|\\\", ($fn, $dob, $dod, $gns, $pcs)) | slice($gns, 2, $gns)\", - \"filter_expression\" : \"($dob BETWEEN :low_dob AND :high_dob\) AND contains($gns, :gnsc)\" - }, - { - \"aggregation_tag\" : 2, - \"index_name\" : \"example2_bin\", - \"start_term\" : \"C\", - \"end_term\" : \"D\" - } - - ] - } - ">>, - {ok, M} = decode_json_body(IQJson), - {ok, Q} = make_query_request({<<"BT">>, <<"B">>}, M), - ?assert(riak_kv_query:is_query(Q)), - QueryList = maps:get(<<"query_list">>, M), - ?assertMatch(ok, check_querylist(QueryList, false)). - -invalid_query_ae6_test() -> - IQJson = % unescaped "|" in eval expression - <<" - { - \"aggregation_expression\" : \"$1 INTERSECT $2\", - \"timeout\" : 60, - \"accumulation_option\" : \"keys\", - \"substitutions\" : - {\"low_dob\" : \"20210804\", \"high_dob\" : \"20223101\", \"gnsc\" : \"Ma\"}, - \"query_list\" : - [ - { - \"aggregation_tag\" : 1, - \"index_name\" : \"example1_bin\", - \"start_term\" : \"A\", - \"end_term\" : \"B\", - \"evaluation_expression\" : - \"delim($term, |, ($fn, $dob, $dod, $gns, $pcs)) | slice($gns, 2, $gns)\", - \"filter_expression\" : \"($dob BETWEEN :low_dob AND :high_dob\) AND contains($gns, :gnsc)\" - }, - { - \"aggregation_tag\" : 2, - \"index_name\" : \"example2_bin\", - \"start_term\" : \"C\", - \"end_term\" : \"D\" - } - - ] - } - ">>, - {ok, M} = decode_json_body(IQJson), - ?assertMatch( - {error, query_evaluation, <<"Invalid eval function">>}, - make_query_request({<<"BT">>, <<"B">>}, M) - ). - -invalid_query_ae7_test() -> - IQJson = % BETWEN not BETWEEN - <<" - { - \"aggregation_expression\" : \"$1 INTERSECT $2\", - \"timeout\" : 60, - \"accumulation_option\" : \"keys\", - \"substitutions\" : - {\"low_dob\" : \"20210804\", \"high_dob\" : \"20223101\", \"gnsc\" : \"Ma\"}, - \"query_list\" : - [ - { - \"aggregation_tag\" : 1, - \"index_name\" : \"example1_bin\", - \"start_term\" : \"A\", - \"end_term\" : \"B\", - \"evaluation_expression\" : - \"delim($term, \\\"|\\\", ($fn, $dob, $dod, $gns, $pcs)) | slice($gns, 2, $gns)\", - \"filter_expression\" : \"($dob BETWEN :low_dob AND :high_dob\) AND contains($gns, :gnsc)\" - }, - { - \"aggregation_tag\" : 2, - \"index_name\" : \"example2_bin\", - \"start_term\" : \"C\", - \"end_term\" : \"D\" - } - - ] - } - ">>, - {ok, M} = decode_json_body(IQJson), - ?assertMatch( - {error, query_evaluation, <<"Invalid filter function">>}, - make_query_request({<<"BT">>, <<"B">>}, M) - ). - -invalid_query_ae8_test() -> - IQJson = % missing substitution - <<" - { - \"aggregation_expression\" : \"$1 INTERSECT $2\", - \"timeout\" : 60, - \"accumulation_option\" : \"keys\", - \"substitutions\" : - {\"low_dob\" : \"20210804\", \"gnsc\" : \"Ma\"}, - \"query_list\" : - [ - { - \"aggregation_tag\" : 1, - \"index_name\" : \"example1_bin\", - \"start_term\" : \"A\", - \"end_term\" : \"B\", - \"evaluation_expression\" : - \"delim($term, \\\"|\\\", ($fn, $dob, $dod, $gns, $pcs)) | slice($gns, 2, $gns)\", - \"filter_expression\" : \"($dob BETWEEN :low_dob AND :high_dob\) AND contains($gns, :gnsc)\" - }, - { - \"aggregation_tag\" : 2, - \"index_name\" : \"example2_bin\", - \"start_term\" : \"C\", - \"end_term\" : \"D\" - } - - ] - } - ">>, - {ok, M} = decode_json_body(IQJson), - ?assertMatch( - {error, query_evaluation, <<"Invalid filter function">>}, - make_query_request({<<"BT">>, <<"B">>}, M) - ). - -invalid_query_to_test() -> - IQJson = - <<" - { - \"aggregation_expression\" : \"$1 INTERSECT $2\", - \"timeout\" : 0, - \"query_list\" : - [ - { - \"aggregation_tag\" : 1, - \"index_name\" : \"example_bin\", - \"start_term\" : \"A\", - \"end_term\" : \"B\" - }, - { - \"aggregation_tag\" : 2, - \"index_name\" : \"example_bin\", - \"start_term\" : \"A\", - \"end_term\" : \"B\" - } - - ] - } - ">>, - {ok, M} = decode_json_body(IQJson), - {error, S, E} = make_query_request({<<"BT">>, <<"B">>}, M), - ?assertMatch(init, S), - ?assertMatch(<<"Bad timeout">>, E). - -invalid_query_extratag_test() -> - IQJson = - <<" - { - \"aggregation_expression\" : \"$1 INTERSECT $2\", - \"timeout\" : 60, - \"subs\" : {\"dob\" : \"19260812\"}, - \"query_list\" : - [ - { - \"aggregation_tag\" : 1, - \"index_name\" : \"example_bin\", - \"start_term\" : \"A\", - \"end_term\" : \"B\" - }, - { - \"aggregation_tag\" : 2, - \"index_name\" : \"example_bin\", - \"start_term\" : \"A\", - \"end_term\" : \"B\", - \"end_key\" : \"B\" - } - - ] - } - ">>, - {ok, M} = decode_json_body(IQJson), - ?assertMatch( - {error, "Unexpected keys in request [<<\"subs\">>]"}, - check_keys(maps:keys(M), request) - ), - ?assertMatch( - {error, "Unexpected keys in request [<<\"end_key\">>]"}, - check_querylist(maps:get(<<"query_list">>, M), false) - ). - -invalid_query_missingtag1_test() -> - IQJson = - <<" - { - \"aggregation_expression\" : \"$1 INTERSECT $2\", - \"timeout\" : 60, - \"subs\" : {\"dob\" : \"19260812\"} - } - ">>, - {ok, M} = decode_json_body(IQJson), - ?assertMatch( - {error, "Missing required keys in request [<<\"query_list\">>]"}, - check_keys(maps:keys(M), request) - ). - -invalid_query_missingtag2_test() -> - IQJson = - <<" - { - \"aggregation_expression\" : \"$1 INTERSECT $2\", - \"timeout\" : 60, - \"query_list\" : - [ - { - \"aggregation_tag\" : 1, - \"index_name\" : \"example_bin\", - \"start_term\" : \"A\", - \"end_term\" : \"B\" - }, - { - \"aggregation_tag\" : 2, - \"index_name\" : \"example_bin\", - \"end_term\" : \"B\" - } - - ] - } - ">>, - {ok, M} = decode_json_body(IQJson), - ?assertMatch( - {error, "Missing required keys in request [<<\"start_term\">>]"}, - check_querylist(maps:get(<<"query_list">>, M), false) - ). - -encode_results_test() -> - BinMC = encode_results(raw_count, 500), - ?assertMatch( - 500, - maps:get(?ACCKEY_RAWCOUNT, riak_kv_wm_json:decode(BinMC)) - ), - BinKC = encode_results(count, 600), - ?assertMatch( - 600, - maps:get(?ACCKEY_COUNT, riak_kv_wm_json:decode(BinKC)) - ), - KeyList = [<<"K00001">>, <<"K00002">>, <<"K0003">>], - BinKL = encode_results(keys, KeyList), - ?assertMatch( - KeyList, - maps:get(?ACCKEY_KEYS, riak_kv_wm_json:decode(BinKL)) - ), - KeyListT = [{<<"K00001">>}, {<<"K00002">>}, {<<"K0003">>}], - BinKLT = encode_results(keys, KeyListT), - ?assertMatch( - KeyList, - maps:get(?ACCKEY_KEYS, riak_kv_wm_json:decode(BinKLT)) - ), - TermKeyList = [{<<"T0001">>, <<"K0002">>}, {<<"T0002">>, <<"K0001">>}], - BinTKL = encode_results(terms, TermKeyList), - ?assertMatch( - TermKeyList, - lists:sort( - lists:map( - fun(M) -> [{T, K}] = maps:to_list(M), {T, K} end, - maps:get(?ACCKEY_TERMS, riak_kv_wm_json:decode(BinTKL)) - ) - ) - ), - TermKeyListT = - [{{<<"T0001">>, <<"K0002">>}}, {{<<"T0002">>, <<"K0001">>}}], - BinTKLT = encode_results(terms, TermKeyListT), - ?assertMatch( - TermKeyList, - lists:sort( - lists:map( - fun(M) -> [{T, K}] = maps:to_list(M), {T, K} end, - maps:get(?ACCKEY_TERMS, riak_kv_wm_json:decode(BinTKLT)) - ) - ) - ), - TermCount = #{<<"T0001">> => 12, <<"T0002">> => 10}, - BinTKC = encode_results(term_with_count, TermCount), - ?assertMatch( - 10, - maps:get( - <<"T0002">>, - maps:get(?ACCKEY_TERMCOUNT, riak_kv_wm_json:decode(BinTKC)) - ) - ), - BinTMC = encode_results(term_with_rawcount, TermCount), - ?assertMatch( - 12, - maps:get( - <<"T0001">>, - maps:get(?ACCKEY_TERMRAWCOUNT, riak_kv_wm_json:decode(BinTMC)) - ) - ) - . - --endif. \ No newline at end of file diff --git a/src/riak_kv_wm_queue.erl b/src/riak_kv_wm_queue.erl deleted file mode 100644 index 8536663eb..000000000 --- a/src/riak_kv_wm_queue.erl +++ /dev/null @@ -1,450 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2016 Basho Technologies, Inc. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - -%% @doc Webmachine resource for fetching for real-time replication -%% -%% Available operations: -%% -%% ``` -%% GET /queuename/QueueName -%% -%% Will return an object if an object is present in the queue -%% -%% Parameters to pass: -%% object_format - internal (return object in internal repl format) -%% - internal_aaehash (also return segment hash and vc hash) -%% -%% GET /membership_request -%% -%% Will return a list of IP addresses and Ports in a JSON body -%% -%% POST /queuename/QueueName -%% -%% Body should be a JSON in the format returned from -%% riak_kv_clusteraae_fsm:json_encode_results(fetch_clocks_range, KeysNClocks). -%% -%% ``` - --module(riak_kv_wm_queue). - -%% webmachine resource exports --export([ - init/1, - service_available/2, - is_authorized/2, - allowed_methods/2, - malformed_request/2, - content_types_provided/2, - process_post/2, - produce_queue_fetch/2, - produce_membership_request/2 - ]). - --include_lib("kernel/include/logger.hrl"). - --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). --endif. - --record(ctx, { - client, %% riak_client() - the store client - riak, %% local | {node(), atom()} - params for riak client - queuename, %% Queue Name (from uri) - keyclocklist = [] :: list(riak_kv_replrtq_src:repl_entry()), - %% List of Bucket, Key, Clock tuples - object_format = internal :: internal|internal_aaehash, - %% object format to be used in response - method :: 'GET'|'PUT'|'POST'|undefined, - get_type :: fetch|membership|post|undefined, - security %% AAE Fold not currently subject to grant check - %% so security context will be ignored. - - }). --type context() :: #ctx{}. - --include_lib("webmachine/include/webmachine.hrl"). --include("riak_kv_wm_raw.hrl"). - --define(MEMBER_REQ, "/membership_request"). - -%% @doc Initialize this resource. --spec init(proplists:proplist()) -> {ok, context()}. -init(Props) -> - {ok, #ctx{riak=proplists:get_value(riak, Props)}}. - --spec service_available(#wm_reqdata{}, context()) -> - {boolean(), #wm_reqdata{}, context()}. -%% @doc Determine whether or not a connection to Riak -%% can be established. -service_available(RD, Ctx=#ctx{riak=RiakProps}) -> - ClientID = riak_kv_wm_utils:get_client_id(RD), - case riak_kv_wm_utils:get_riak_client(RiakProps, ClientID) of - {ok, C} -> - {true, RD, Ctx#ctx{client = C, method=wrq:method(RD)}}; - Error -> - {false, - wrq:set_resp_body( - io_lib:format("Unable to connect to Riak: ~p~n", [Error]), - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD)), - Ctx} - end. - -is_authorized(ReqData, Ctx) -> - case application:get_env(riak_kv, permit_insecure_http_ops, false) of - true -> - {true, ReqData, Ctx}; - false -> - case riak_api_web_security:is_authorized(ReqData) of - false -> - {"Basic realm=\"Riak\"", ReqData, Ctx}; - {true, SecContext} -> - {true, ReqData, Ctx#ctx{security=SecContext}}; - insecure -> - { - {halt, 426}, - wrq:append_to_resp_body( - << - "Security is enabled and " - "Riak does not accept credentials over HTTP. Try HTTPS " - "instead. Or configure `permit_insecure_http_ops`" - >>, - ReqData - ), - Ctx - } - end - end. - -allowed_methods(RD, Ctx) -> - {['GET', 'POST'], RD, Ctx}. - --spec malformed_request(#wm_reqdata{}, context()) -> - {boolean(), #wm_reqdata{}, context()}. -%% @doc Determine whether request is well-formed -malformed_request(RD, Ctx) -> - RDForError = wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD), - case wrq:method(RD) of - 'GET' -> - case wrq:path(RD) of - ?MEMBER_REQ -> - {false, RD, Ctx#ctx{get_type = membership}}; - _FetchUrl -> - QueueName = malformed_queuename(RD), - ObjectFormatRaw = - wrq:get_qs_value(?Q_OBJECT_FORMAT, "internal", RD), - case existing_atom(ObjectFormatRaw) of - false -> - ErrorBody = - io_lib:format("Format ~w not defined~n", - [ObjectFormatRaw]), - {true, - wrq:append_to_resp_body(ErrorBody, RDForError), - Ctx}; - ObjectFormat -> - {false, - RD, - Ctx#ctx{queuename = QueueName, - get_type = fetch, - object_format = ObjectFormat}} - end - end; - 'POST' -> - case malformed_queuename(RD) of - false -> - ErrorBody = - io_lib:format("Queue name ~w not defined~n", - [wrq:path_info(queuename, RD)]), - {true, - wrq:append_to_resp_body(ErrorBody, RDForError), - Ctx}; - QueueName -> - case malformed_keyclocklist(wrq:req_body(RD)) of - false -> - ErrorBody = "Malformed Keyclock list~n", - {true, - wrq:append_to_resp_body(ErrorBody, RDForError), - Ctx}; - KeyClockList -> - {false, - RD, - Ctx#ctx{queuename = QueueName, - get_type = post, - keyclocklist = KeyClockList}} - end - end - end. - --spec malformed_queuename(#wm_reqdata{}) -> atom()|false. -malformed_queuename(RD) -> - QueueNameRaw = wrq:path_info(queuename, RD), - existing_atom(riak_kv_wm_utils:maybe_decode_uri(RD, QueueNameRaw)). - --spec malformed_keyclocklist(iolist()) -> - list(riak_kv_replrtq_src:repl_entry())|false. -malformed_keyclocklist(ReqBody) -> - %% If individual elements of the Key Clock list are malformed then - %% the whole request is considered malformed - case mochijson2:decode(ReqBody) of - {struct, [{<<"keys-clocks">>, KCL}]} -> - KeyClockList = - lists:foldl(fun decode_bucketkeyclock/2, [], KCL), - case {length(KeyClockList), length(KCL)} of - {N, N} -> - KeyClockList; - {N, M} -> - ?LOG_INFO( - "Malformed requests ~w within push of ~w", - [M - N, M]), - false - end; - _ -> - false - end. - - --spec content_types_provided(#wm_reqdata{}, context()) -> - {[{ContentType::string(), Producer::atom()}], #wm_reqdata{}, context()}. -%% @doc List the content types available for representing this resource. -content_types_provided(RD, Ctx) when Ctx#ctx.get_type =:= fetch -> - {[{"application/octet-stream", produce_queue_fetch}], RD, Ctx}; -content_types_provided(RD, Ctx) when Ctx#ctx.get_type =:= membership -> - {[{"application/json", produce_membership_request}], RD, Ctx}; -content_types_provided(RD, Ctx) when Ctx#ctx.get_type =:= post -> - {[{"text/plain", nop}], RD, Ctx}. - - --spec process_post(#wm_reqdata{}, context()) -> - {true, #wm_reqdata{}, context()}. -%% @doc Pass-through for key-level requests to allow POST to function -%% as PUT for clients that do not support PUT. -process_post(RD, Ctx) -> - QueueName = Ctx#ctx.queuename, - KeyClockList = - lists:map(fun({B, K, C}) -> {B, K, C, to_fetch} end, - lists:reverse(Ctx#ctx.keyclocklist)), - ok = riak_kv_replrtq_src:replrtq_ttaaefs(QueueName, KeyClockList), - R = - case riak_kv_replrtq_src:length_rtq(QueueName) of - {QueueName, {FL, FSL, RTL}} -> - io_lib:format("Queue ~w: ~w ~w ~w", [QueueName, FL, FSL, RTL]); - _ -> - io_lib:format("No queue ~w", [QueueName]) - end, - {true, - wrq:set_resp_body( - iolist_to_binary(R), - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD)), - Ctx}. - --spec produce_queue_fetch(#wm_reqdata{}, context()) -> - {binary()|{error, any()}, #wm_reqdata{}, context()}. -%% @doc Produce the binary response to a queue fetch request -produce_queue_fetch(RD, Ctx) -> - Client = Ctx#ctx.client, - QueueName = Ctx#ctx.queuename, - format_response(Ctx#ctx.object_format, - riak_client:fetch(QueueName, Client), - RD, - Ctx). - --spec produce_membership_request(#wm_reqdata{}, context()) -> - {iolist(), #wm_reqdata{}, context()}. -produce_membership_request(RD, Ctx) -> - R = riak_client:membership_request(http), - MapToJsonFun = - fun({IP, Port}) -> - {struct, - [{<<"ip">>, list_to_binary(IP)}, - {<<"port">>, integer_to_binary(Port)}]} - end, - {mochijson2:encode( - {struct, [{"up_nodes", lists:map(MapToJsonFun, R)}]}), - RD, Ctx}. - - -decode_bucketkeyclock({struct, BKC}, Acc) -> - B = - case lists:keyfind(<<"bucket">>, 1, BKC) of - {<<"bucket">>, Bucket} when is_binary(Bucket) -> - case lists:keyfind(<<"bucket-type">>, 1, BKC) of - {<<"bucket-type">>, BucketType} -> - {BucketType, Bucket}; - false -> - Bucket - end; - false -> - false - end, - case B of - false -> - Acc; - _ -> - case lists:keyfind(<<"key">>, 1, BKC) of - {<<"key">>, K} when is_binary(K) -> - case lists:keyfind(<<"clock">>, 1, BKC) of - {<<"clock">>, EncodedClock} -> - case decode_clock(EncodedClock) of - false -> - Acc; - VC -> - [{B, K, VC}|Acc] - end; - false -> - Acc - end; - false -> - Acc - end - end; -decode_bucketkeyclock(_, Acc) -> - Acc. - --spec existing_atom(list()) -> atom()|false. -existing_atom(ListToConvert)-> - try - list_to_existing_atom(ListToConvert) - catch - _:_ -> - false - end. - --spec decode_clock(list()) -> vclock:vclock()|false. -decode_clock(EncodedClock) -> - try - riak_object:decode_vclock(base64:decode(EncodedClock)) - catch - _:_ -> - false - end. - -format_response(_, {ok, queue_empty}, RD, Ctx) -> - {<<0:8/integer>>, RD, Ctx}; -format_response(_, {error, Reason}, RD, Ctx) -> - ?LOG_WARNING("Fetch error ~w", [Reason]), - {{error, Reason}, RD, Ctx}; -format_response(internal_aaehash, {ok, {reap, {B, K, TC, LMD}}}, RD, Ctx) -> - BK = make_binarykey(B, K), - {SegmentID, SegmentHash} = - leveled_tictac:tictac_hash(BK, lists:sort(TC)), - SuccessMark = <<1:8/integer>>, - IsTombstone = <<0:8/integer>>, - ObjBin = encode_riakobject({reap, {B, K, TC, LMD}}), - {<>, RD, Ctx}; -format_response(internal_aaehash, {ok, {deleted, TombClock, RObj}}, RD, Ctx) -> - BK = make_binarykey(riak_object:bucket(RObj), riak_object:key(RObj)), - {SegmentID, SegmentHash} = - leveled_tictac:tictac_hash(BK, lists:sort(TombClock)), - SuccessMark = <<1:8/integer>>, - IsTombstone = <<1:8/integer>>, - ObjBin = encode_riakobject(RObj), - TombClockBin = term_to_binary(TombClock), - TCL = byte_size(TombClockBin), - {<>, RD, Ctx}; -format_response(internal_aaehash, {ok, RObj}, RD, Ctx) -> - BK = make_binarykey(riak_object:bucket(RObj), riak_object:key(RObj)), - {SegmentID, SegmentHash} = - leveled_tictac:tictac_hash(BK, lists:sort(riak_object:vclock(RObj))), - SuccessMark = <<1:8/integer>>, - IsTombstone = <<0:8/integer>>, - ObjBin = encode_riakobject(RObj), - {<>, RD, Ctx}; -format_response(internal, {ok, {reap, {B, K, TC, LMD}}}, RD, Ctx) -> - SuccessMark = <<1:8/integer>>, - IsTombstone = <<0:8/integer>>, - ObjBin = encode_riakobject({reap, {B, K, TC, LMD}}), - {<>, RD, Ctx}; -format_response(internal, {ok, {deleted, TombClock, RObj}}, RD, Ctx) -> - SuccessMark = <<1:8/integer>>, - IsTombstone = <<1:8/integer>>, - ObjBin = encode_riakobject(RObj), - TombClockBin = term_to_binary(TombClock), - TCL = byte_size(TombClockBin), - {<>, RD, Ctx}; -format_response(internal, {ok, RObj}, RD, Ctx) -> - SuccessMark = <<1:8/integer>>, - IsTombstone = <<0:8/integer>>, - ObjBin = encode_riakobject(RObj), - {<>, RD, Ctx}. - -encode_riakobject(RObj) -> - ToCompress = app_helper:get_env(riak_kv, replrtq_compressonwire, false), - FullObjBin = riak_object:nextgenrepl_encode(repl_v1, RObj, ToCompress), - CRC = erlang:crc32(FullObjBin), - <>. - --spec make_binarykey(riak_object:bucket(), riak_object:key()) -> binary(). -%% @doc -%% Convert Bucket and Key into a single binary -make_binarykey({Type, Bucket}, Key) - when is_binary(Type), is_binary(Bucket), is_binary(Key) -> - <>; -make_binarykey(Bucket, Key) when is_binary(Bucket), is_binary(Key) -> - <>. - --ifdef(TEST). - -test_kcl() -> - A = vclock:fresh(), - B = vclock:fresh(), - A1 = vclock:increment(a, A), - B1 = vclock:increment(b, B), - E1 = {<<"B1">>, <<"K1">>, A1}, - E2 = {{<<"T">>, <<"B2">>}, <<"K2">>, B1}, - [E1, E2]. - -json_encode_keys_test() -> - KCL = lists:sort(test_kcl()), - KCEncoded = - riak_kv_clusteraae_fsm:json_encode_results(fetch_clocks_range, KCL), - ?assertMatch(KCL, lists:sort(malformed_keyclocklist(KCEncoded))). - -malformed_clock_test() -> - KCL = lists:sort(test_kcl()), - KCEncoded = - riak_kv_clusteraae_fsm:json_encode_results(fetch_clocks_range, KCL), - {struct, [{<<"keys-clocks">>, EncKCL}]} = - mochijson2:decode(KCEncoded), - BadEncoded = - mochijson2:encode({struct, [{<<"keys-xxx-clocks">>, EncKCL}]}), - ?assertMatch(false, malformed_keyclocklist(BadEncoded)), - Q = q1_ttaaefs, - ?assertMatch(Q, existing_atom("q1_ttaaefs")), - ?assertMatch(false, existing_atom("q1_xxx_ttaaefs")), - VC = - base64:encode( - riak_object:encode_vclock( - vclock:increment(a, vclock:fresh()))), - Garbage = - <<"zzz">>, - ?assertMatch(false, decode_clock(<>)). - - --endif. diff --git a/src/riak_kv_wm_raw.hrl b/src/riak_kv_wm_raw.hrl deleted file mode 100644 index 6d4cce0aa..000000000 --- a/src/riak_kv_wm_raw.hrl +++ /dev/null @@ -1,85 +0,0 @@ -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at - -%% http://www.apache.org/licenses/LICENSE-2.0 - -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. - -%% Constants used by the raw_http resources - -%% Names of riak_object metadata fields --define(MD_CTYPE, <<"content-type">>). --define(MD_CHARSET, <<"charset">>). --define(MD_ENCODING, <<"content-encoding">>). --define(MD_VTAG, <<"X-Riak-VTag">>). --define(MD_LINKS, <<"Links">>). --define(MD_LASTMOD, <<"X-Riak-Last-Modified">>). --define(MD_USERMETA, <<"X-Riak-Meta">>). --define(MD_INDEX, <<"index">>). --define(MD_DELETED, <<"X-Riak-Deleted">>). --define(MD_VAL_ENCODING, <<"X-Riak-Val-Encoding">>). - -%% Names of HTTP header fields --define(HEAD_CTYPE, "Content-Type"). --define(HEAD_VCLOCK, "X-Riak-Vclock"). --define(HEAD_LINK, "Link"). --define(HEAD_ENCODING, "Content-Encoding"). --define(HEAD_CLIENT, "X-Riak-ClientId"). --define(HEAD_USERMETA_PREFIX, "x-riak-meta-"). --define(HEAD_INDEX_PREFIX, "x-riak-index-"). --define(HEAD_DELETED, "X-Riak-Deleted"). --define(HEAD_TIMEOUT, "X-Riak-Timeout"). --define(HEAD_CRDT_CONTEXT, "X-Riak-CRDT-Ctx"). --define(HEAD_IF_NOT_MODIFIED, "X-Riak-If-Not-Modified"). - -%% Names of JSON fields in bucket properties --define(JSON_PROPS, <<"props">>). --define(JSON_BUCKETS, <<"buckets">>). --define(JSON_KEYS, <<"keys">>). --define(JSON_LINKFUN, <<"linkfun">>). --define(JSON_MOD, <<"mod">>). --define(JSON_FUN, <<"fun">>). --define(JSON_ARG, <<"arg">>). --define(JSON_CHASH, <<"chash_keyfun">>). --define(JSON_JSFUN, <<"jsfun">>). --define(JSON_JSANON, <<"jsanon">>). --define(JSON_JSBUCKET, <<"bucket">>). --define(JSON_JSKEY, <<"key">>). --define(JSON_ALLOW_MULT, <<"allow_mult">>). --define(JSON_EXTRACT, <<"search_extractor">>). --define(JSON_EXTRACT_LEGACY, <<"rs_extractfun">>). --define(JSON_DATATYPE, <<"datatype">>). --define(JSON_HLL_PRECISION, <<"hll_precision">>). - -%% Names of HTTP query parameters --define(Q_PROPS, "props"). --define(Q_BUCKETS, "buckets"). --define(Q_KEYS, "keys"). --define(Q_KEYS_BIN, <<"keys">>). --define(Q_FALSE, "false"). --define(Q_TRUE, "true"). --define(Q_STREAM, "stream"). --define(Q_VTAG, "vtag"). --define(Q_RETURNBODY, "returnbody"). --define(Q_2I_RETURNTERMS, "return_terms"). --define(Q_2I_MAX_RESULTS, "max_results"). --define(Q_2I_TERM_REGEX, "term_regex"). --define(Q_2I_CONTINUATION, "continuation"). --define(Q_2I_CONTINUATION_BIN, <<"continuation">>). --define(Q_2I_PAGINATION_SORT, "pagination_sort"). --define(Q_RESULTS, "results"). --define(Q_RESULTS_BIN, <<"results">>). --define(Q_RETURNVALUE, "returnvalue"). --define(Q_2I_MAPFOLD, "mapfold"). --define(Q_MF_MAPFOLDMOD, "mapfoldmod"). --define(Q_MF_MAPFOLDOPTS, "mapfoldoptions"). --define(Q_AAEFOLD_FILTER, "filter"). --define(Q_OBJECT_FORMAT, "object_format"). --define(Q_NVAL, "nval"). diff --git a/src/riak_kv_wm_stats.erl b/src/riak_kv_wm_stats.erl deleted file mode 100644 index 867cdb83f..000000000 --- a/src/riak_kv_wm_stats.erl +++ /dev/null @@ -1,174 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% riak_kv_wm_stats: publishing Riak runtime stats via HTTP -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - --module(riak_kv_wm_stats). - -%% webmachine resource exports --export([ - init/1, - service_available/2, - is_authorized/2, - encodings_provided/2, - content_types_provided/2, - forbidden/2, - malformed_request/2, - produce_body/2, - pretty_print/2 - ]). - --define(TIMEOUT, 30000). %% In milliseconds - --include_lib("webmachine/include/webmachine.hrl"). --include("riak_kv_wm_raw.hrl"). - --record(ctx, - { - timeout = ?TIMEOUT :: non_neg_integer(), - security - %% Stats not currently subject to grant check so security context - %% will be ignored. - } -). - -init(_) -> - {ok, #ctx{}}. - -service_available(ReqData, Ctx) -> - {true, ReqData, Ctx}. - -is_authorized(ReqData, Ctx) -> - case application:get_env(riak_kv, permit_insecure_http_ops, false) of - true -> - {true, ReqData, Ctx}; - false -> - case riak_api_web_security:is_authorized(ReqData) of - false -> - {"Basic realm=\"Riak\"", ReqData, Ctx}; - {true, SecContext} -> - {true, ReqData, Ctx#ctx{security=SecContext}}; - insecure -> - { - {halt, 426}, - wrq:append_to_resp_body( - << - "Security is enabled and " - "Riak does not accept credentials over HTTP. Try HTTPS " - "instead. Or configure `permit_insecure_http_ops`" - >>, - ReqData - ), - Ctx - } - end - end. - -%% @spec encodings_provided(webmachine:wrq(), context()) -> -%% {[encoding()], webmachine:wrq(), context()} -%% @doc Get the list of encodings this resource provides. -%% "identity" is provided for all methods, and "gzip" is -%% provided for GET as well -encodings_provided(ReqData, Context) -> - case wrq:method(ReqData) of - 'GET' -> - {[{"identity", fun(X) -> X end}, - {"gzip", fun(X) -> zlib:gzip(X) end}], ReqData, Context}; - _ -> - {[{"identity", fun(X) -> X end}], ReqData, Context} - end. - -%% @spec content_types_provided(webmachine:wrq(), context()) -> -%% {[ctype()], webmachine:wrq(), context()} -%% @doc Get the list of content types this resource provides. -%% "application/json" and "text/plain" are both provided -%% for all requests. "text/plain" is a "pretty-printed" -%% version of the "application/json" content. -content_types_provided(ReqData, Context) -> - {[{"application/json", produce_body}, - {"text/plain", pretty_print}], - ReqData, Context}. - -malformed_request(RD, Ctx) -> - case wrq:get_qs_value("timeout", RD) of - undefined -> - {false, RD, Ctx}; - TimeoutStr -> - try - case list_to_integer(TimeoutStr) of - Timeout when Timeout > 0 -> - {false, RD, Ctx#ctx{timeout=Timeout}} - end - catch - _:_ -> - {true, - wrq:append_to_resp_body( - io_lib:format( - "Bad timeout value ~0p " - "expected milliseconds > 0", - [TimeoutStr] - ), - wrq:set_resp_header(?HEAD_CTYPE, "text/plain", RD)), - Ctx} - end - end. - -forbidden(RD, Ctx) -> - {riak_kv_wm_utils:is_forbidden(RD), RD, Ctx}. - -produce_body(RD, Ctx) -> - try - Stats = riak_kv_http_cache:get_stats(Ctx#ctx.timeout), - Body = mochijson2:encode({struct, Stats}), - {Body, RD, Ctx} - catch - exit:{timeout, _} -> - { - {halt, 503}, - wrq:set_resp_header( - ?HEAD_CTYPE, - "text/plain", - wrq:append_to_response_body( - io_lib:format( - "Request timed out after ~w ms", - [Ctx#ctx.timeout] - ), - RD - ) - ), - Ctx - } - end. - -%% @spec pretty_print(webmachine:wrq(), context()) -> -%% {string(), webmachine:wrq(), context()} -%% @doc Format the respons JSON object is a "pretty-printed" style. -pretty_print(RD, Ctx) -> - case produce_body(RD, Ctx) of - {{halt, RepsonseCode}, UpdRD, UpdCtx} -> - {{halt, RepsonseCode}, UpdRD, UpdCtx}; - {Json, UpdRD, UpdCtx} -> - { - json_pp:print(binary_to_list(list_to_binary(Json))), - UpdRD, - UpdCtx - } - end. - diff --git a/src/riak_kv_wm_utils.erl b/src/riak_kv_wm_utils.erl deleted file mode 100644 index 923c61471..000000000 --- a/src/riak_kv_wm_utils.erl +++ /dev/null @@ -1,503 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2016 Basho Technologies, Inc. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - -%% @doc Common functions used by riak_kv_wm_* modules. --module(riak_kv_wm_utils). - -%% webmachine resource exports --export([ - maybe_decode_uri/2, - get_riak_client/2, - get_client_id/1, - default_encodings/0, - multipart_encode_body/4, - format_links/3, - format_uri/4, - format_uri/5, - encode_value/1, - accept_value/2, - any_to_list/1, - any_to_bool/1, - is_forbidden/1, - is_forbidden/3, - jsonify_bucket_prop/1, - erlify_bucket_prop/1, - ensure_bucket_type/3, - bucket_type_exists/1, - maybe_bucket_type/2, - method_to_perm/1 - ]). - --include_lib("kernel/include/logger.hrl"). - --include_lib("webmachine/include/webmachine.hrl"). --include("riak_kv_wm_raw.hrl"). - --type jsonpropvalue() :: integer()|string()|boolean()|{struct,[jsonmodfun()]}. --type jsonmodfun() :: {ModBinary :: term(), binary()}|{FunBinary :: term(), binary()}. --type erlpropvalue() :: integer()|string()|boolean(). - -maybe_decode_uri(RD, Val) -> - case application:get_env(riak_kv, http_url_encoding) of - {ok, on} -> - mochiweb_util:unquote(Val); - _ -> - case wrq:get_req_header("X-Riak-URL-Encoding", RD) of - "on" -> - mochiweb_util:unquote(Val); - _ -> - Val - end - end. - --spec get_riak_client(local|{node(),Cookie::atom()}, term()) -> - {ok, RiakClient :: term()} | {error, term()}. -%% @doc Get a riak_client. -get_riak_client(local, ClientId) -> - riak:local_client(ClientId); -get_riak_client({Node, Cookie}, ClientId) -> - erlang:set_cookie(node(), Cookie), - riak:client_connect(Node, ClientId). - --spec get_client_id(#wm_reqdata{}) -> term(). -%% @doc Extract the request's preferred client id from the -%% X-Riak-ClientId header. Return value will be: -%% 'undefined' if no header was found -%% 32-bit binary() if the header could be base64-decoded -%% into a 32-bit binary -%% string() if the header could not be base64-decoded -%% into a 32-bit binary -get_client_id(RD) -> - case wrq:get_req_header(?HEAD_CLIENT, RD) of - undefined -> undefined; - RawId -> - case catch base64:decode(RawId) of - ClientId= <<_:32>> -> ClientId; - _ -> RawId - end - end. - - --spec default_encodings() -> [{Encoding::string(), Producer::function()}]. -%% @doc The default encodings available: identity and gzip. -default_encodings() -> - [{"identity", fun(X) -> X end}, - {"gzip", fun(X) -> zlib:gzip(X) end}]. - --spec multipart_encode_body( - string(), - binary(), - {riak_object:riak_object_meta(), binary()}, - term()) -> - iolist(). -%% @doc Produce one part of a multipart body, representing one sibling -%% of a multi-valued document. -multipart_encode_body(Prefix, Bucket, {MD, V}, APIVersion) -> - Links1 = - case riak_object:metadata_find(?MD_LINKS, MD) of - {ok, Ls} -> Ls; - error -> [] - end, - Links2 = format_links([{Bucket, "up"}|Links1], Prefix, APIVersion), - Links3 = mochiweb_headers:make(Links2), - [{?HEAD_LINK, Links4}] = mochiweb_headers:to_list(Links3), - - [ - ?HEAD_CTYPE, ": ",get_ctype(MD,V), - case riak_object:metadata_find(?MD_CHARSET, MD) of - {ok, CS} -> ["; charset=",CS]; - error -> [] - end, - "\r\n", - case riak_object:metadata_find(?MD_ENCODING, MD) of - {ok, Enc} -> [?HEAD_ENCODING,": ",Enc,"\r\n"]; - error -> [] - end, - ?HEAD_LINK,": ",Links4,"\r\n", - "Etag: ", riak_object:metadata_fetch(?MD_VTAG, MD),"\r\n", - "Last-Modified: ", - case riak_object:metadata_fetch(?MD_LASTMOD, MD) of - Now={_,_,_} -> - httpd_util:rfc1123_date( - calendar:now_to_local_time(Now)); - Rfc1123 when is_list(Rfc1123) -> - Rfc1123 - end, - "\r\n", - case riak_object:metadata_find(?MD_DELETED, MD) of - {ok, "true"} -> - [?HEAD_DELETED, ": true\r\n"]; - error -> - [] - end, - case riak_object:metadata_find(?MD_USERMETA, MD) of - {ok, M} -> - lists:foldl(fun({Hdr,Val},Acc) -> - [Acc|[Hdr,": ",Val,"\r\n"]] - end, - [], M); - error -> [] - end, - case riak_object:metadata_find(?MD_INDEX, MD) of - {ok, IF} -> - [ - [?HEAD_INDEX_PREFIX,Key,": ",any_to_list(Val),"\r\n"] - || {Key,Val} <- IF - ]; - error -> [] - end, - "\r\n", - encode_value(V) - ]. - -format_links(Links, Prefix, APIVersion) -> - format_links(Links, Prefix, APIVersion, []). -format_links([{{Bucket,Key}, Tag}|Rest], Prefix, APIVersion, Acc) -> - format_links([{Bucket, Key, Tag}|Rest], Prefix, APIVersion, Acc); -format_links([{Bucket, Tag}|Rest], Prefix, APIVersion, Acc) -> - Bucket1 = mochiweb_util:quote_plus(Bucket), - Tag1 = mochiweb_util:quote_plus(Tag), - Val = - case APIVersion of - 1 -> - io_lib:format("; rel=\"~s\"", - [Prefix, Bucket1, Tag1]); - Two when Two >= 2 -> - io_lib:format("; rel=\"~s\"", - [Bucket1, Tag1]) - end, - format_links(Rest, Prefix, APIVersion, [{?HEAD_LINK, Val}|Acc]); -format_links([{Bucket, Key, Tag}|Rest], Prefix, APIVersion, Acc) -> - Bucket1 = mochiweb_util:quote_plus(Bucket), - Key1 = mochiweb_util:quote_plus(Key), - Tag1 = mochiweb_util:quote_plus(Tag), - Val = io_lib:format("<~s>; riaktag=\"~s\"", - [format_uri(Bucket1, Key1, Prefix, APIVersion), - Tag1]), - format_links(Rest, Prefix, APIVersion, [{?HEAD_LINK, Val}|Acc]); -format_links([], _Prefix, _APIVersion, Acc) -> - Acc. - -%% @doc Format the URI for a bucket/key correctly for the api version -%% used. (APIVersion is the final parameter.) -format_uri(Bucket, Key, Prefix, 1) -> - io_lib:format("/~s/~s/~s", [Prefix, Bucket, Key]); -format_uri(Bucket, Key, _Prefix, 2) -> - io_lib:format("/buckets/~s/keys/~s", [Bucket, Key]). - - -%% @doc Format the URI for a type/bucket/key correctly for the api version -%% used. (APIVersion is the final parameter.) -format_uri(_Type, Bucket, Key, Prefix, 1) -> - format_uri(Bucket, Key, Prefix, 1); -format_uri(_Type, Bucket, Key, Prefix, 2) -> - format_uri(Bucket, Key, Prefix, 2); -format_uri(Type, Bucket, Key, _Prefix, 3) -> - io_lib:format("/types/~s/buckets/~s/keys/~s", [Type, Bucket, Key]). - --spec get_ctype(riak_object:riak_object_meta(), term()) -> string(). -%% @doc Work out the content type for this object - use the metadata if provided -get_ctype(MD,V) -> - case riak_object:metadata_find(?MD_CTYPE, MD) of - {ok, Ctype} -> - Ctype; - error when is_binary(V) -> - "application/octet-stream"; - error -> - "application/x-erlang-binary" - end. - --spec encode_value(term()) -> binary(). -%% @doc Encode the object value as a binary - content type can be used -%% to decode -encode_value(V) when is_binary(V) -> - V; -encode_value(V) -> - term_to_binary(V). - --spec accept_value(string(), binary()) -> term(). -%% @doc Accept the object value as a binary - content type can be used -%% to decode -accept_value("application/x-erlang-binary",V) -> - binary_to_term(V); -accept_value(_Ctype, V) -> - V. - -any_to_list(V) when is_list(V) -> - V; -any_to_list(V) when is_atom(V) -> - atom_to_list(V); -any_to_list(V) when is_binary(V) -> - binary_to_list(V); -any_to_list(V) when is_integer(V) -> - integer_to_list(V). - -any_to_bool(V) when is_list(V) -> - (V == "1") orelse (V == "true") orelse (V == "TRUE"); -any_to_bool(V) when is_binary(V) -> - any_to_bool(binary_to_list(V)); -any_to_bool(V) when is_integer(V) -> - V /= 0; -any_to_bool(V) when is_boolean(V) -> - V. - --spec is_forbidden(#wm_reqdata{}) -> boolean(). -%% @doc Determine whether the request is forbidden. -is_forbidden(RD) -> - % TODO: what log level(s) here? - case is_null_origin(RD) of - true -> - ?LOG_INFO( - "Request from ~p denied due to null origin", - [{wrq:scheme(RD), wrq:peer(RD)}]), - true; - _ -> - case app_helper:get_env(riak_kv, secure_referer_check, true) - andalso not is_valid_referer(RD) of - true -> - ?LOG_INFO( - "Request from ~p denied due to invalid referrer", - [{wrq:scheme(RD), wrq:peer(RD)}]), - true; - _ -> - false - end - end. - --spec is_forbidden(#wm_reqdata{}, term(), term()) - -> {boolean(), #wm_reqdata{}, term()}. -%% @doc Like is_forbidden/1, but also checks if the job class is enabled. -%% May modify RequestData, if we choose to include why it's forbidden. -%% ReqContext is passed through untouched, so that the result tuple can be -%% returned directly by WM forbidden callbacks. -is_forbidden(ReqData, JobClass, ReqContext) -> - % Logging for non-job criteria is performed by is_forbidden/1. - case is_forbidden(ReqData) of - true -> - {true, ReqData, ReqContext}; - _ -> - Accept = riak_core_util:job_class_enabled(JobClass), - _ = riak_core_util:report_job_request_disposition( - Accept, JobClass, ?MODULE, is_forbidden, ?LINE, - {wrq:scheme(ReqData), wrq:peer(ReqData)}), - case Accept of - true -> - {false, ReqData, ReqContext}; - _ -> - {true, - wrq:append_to_resp_body( - unicode:characters_to_binary( - riak_core_util:job_class_disabled_message( - text, JobClass), utf8, utf8), - wrq:set_resp_header( - "Content-Type", "text/plain", ReqData)), - ReqContext} - end - end. - -%% @doc Check if the Origin header is "null". This is useful to look for attempts -%% at CSRF, but is not a complete answer to the problem. -is_null_origin(RD) -> - case wrq:get_req_header("Origin", RD) of - "null" -> - true; - _ -> - false - end. - -%% @doc Validate that the Referer matches up with scheme, host and port of the -%% machine that received the request. -is_valid_referer(RD) -> - OriginTuple = {wrq:scheme(RD), string:join(wrq:host_tokens(RD), "."), wrq:port(RD)}, - case referer_tuple(RD) of - undefined -> - true; - {invalid, Url} -> - ?LOG_DEBUG("WM unparsable referer: ~s\n", [Url]), - false; - OriginTuple -> - true; - RefererTuple -> - ?LOG_DEBUG("WM referrer not origin. Origin ~p != Referer ~p\n", [OriginTuple, RefererTuple]), - false - end. - -referer_tuple(RD) -> - case wrq:get_req_header("Referer", RD) of - undefined -> - undefined; - Url -> - case uri_string:parse(Url) of - #{scheme := Scheme, host := Host, port := Port} -> - Scheme0 = - case Scheme of - "http" -> http; - "HTTP" -> http; - "https" -> https; - "HTTPS" -> https; - Scheme -> Scheme - end, - {Scheme0, Host, Port}; - {error, _, _} -> - {invalid, Url} - end - end. - --spec jsonify_bucket_prop({Property::atom(), erlpropvalue()}) -> - {Property::binary(), jsonpropvalue()}. -%% {modfun, atom(), atom()}|{atom(), atom()} -%% @doc Convert erlang bucket properties to JSON bucket properties. -%% Property names are converted from atoms to binaries. -%% Integer, string, and boolean property values are left as integer, -%% string, or boolean JSON values. -%% {modfun, Module, Function} or {Module, Function} values of the -%% linkfun and chash_keyfun properties are converted to JSON objects -%% of the form: -%% {"mod":ModuleNameAsString, -%% "fun":FunctionNameAsString} -jsonify_bucket_prop({linkfun, {modfun, Module, Function}}) -> - {?JSON_LINKFUN, {struct, [{?JSON_MOD, - atom_to_binary(Module, utf8)}, - {?JSON_FUN, - atom_to_binary(Function, utf8)}]}}; -jsonify_bucket_prop({linkfun, {qfun, _}}) -> - {?JSON_LINKFUN, <<"qfun">>}; -jsonify_bucket_prop({linkfun, {jsfun, Name}}) -> - {?JSON_LINKFUN, {struct, [{?JSON_JSFUN, Name}]}}; -jsonify_bucket_prop({linkfun, {jsanon, {Bucket, Key}}}) -> - {?JSON_LINKFUN, {struct, [{?JSON_JSANON, - {struct, [{?JSON_JSBUCKET, Bucket}, - {?JSON_JSKEY, Key}]}}]}}; -jsonify_bucket_prop({linkfun, {jsanon, Source}}) -> - {?JSON_LINKFUN, {struct, [{?JSON_JSANON, Source}]}}; -jsonify_bucket_prop({chash_keyfun, {Module, Function}}) -> - {?JSON_CHASH, {struct, [{?JSON_MOD, - atom_to_binary(Module, utf8)}, - {?JSON_FUN, - atom_to_binary(Function, utf8)}]}}; -%% TODO Remove Legacy extractor prop in future version -jsonify_bucket_prop({rs_extractfun, {modfun, M, F}}) -> - {?JSON_EXTRACT_LEGACY, {struct, [{?JSON_MOD, - atom_to_binary(M, utf8)}, - {?JSON_FUN, - atom_to_binary(F, utf8)}]}}; -jsonify_bucket_prop({rs_extractfun, {{modfun, M, F}, Arg}}) -> - {?JSON_EXTRACT_LEGACY, {struct, [{?JSON_MOD, - atom_to_binary(M, utf8)}, - {?JSON_FUN, - atom_to_binary(F, utf8)}, - {?JSON_ARG, Arg}]}}; -jsonify_bucket_prop({search_extractor, {struct, _}=S}) -> - {?JSON_EXTRACT, S}; -jsonify_bucket_prop({search_extractor, {M, F}}) -> - {?JSON_EXTRACT, {struct, [{?JSON_MOD, - atom_to_binary(M, utf8)}, - {?JSON_FUN, - atom_to_binary(F, utf8)}]}}; -jsonify_bucket_prop({search_extractor, {M, F, Arg}}) -> - {?JSON_EXTRACT, {struct, [{?JSON_MOD, - atom_to_binary(M, utf8)}, - {?JSON_FUN, - atom_to_binary(F, utf8)}, - {?JSON_ARG, Arg}]}}; -jsonify_bucket_prop({name, {_T, B}}) -> - {<<"name">>, B}; -jsonify_bucket_prop({Prop, Value}) -> - {atom_to_binary(Prop, utf8), Value}. - --spec erlify_bucket_prop({Property::binary(), jsonpropvalue()}) -> - {Property::atom(), erlpropvalue()}. -%% @doc The reverse of jsonify_bucket_prop/1. Converts JSON representation -%% of bucket properties to their Erlang form. -erlify_bucket_prop({?JSON_DATATYPE, Type}) when is_binary(Type) -> - {datatype, binary_to_existing_atom(Type, utf8)}; -erlify_bucket_prop({?JSON_LINKFUN, {struct, Props}}) -> - case {proplists:get_value(?JSON_MOD, Props), - proplists:get_value(?JSON_FUN, Props)} of - {Mod, Fun} when is_binary(Mod), is_binary(Fun) -> - {linkfun, {modfun, - list_to_existing_atom(binary_to_list(Mod)), - list_to_existing_atom(binary_to_list(Fun))}}; - {undefined, undefined} -> - case proplists:get_value(?JSON_JSFUN, Props) of - Name when is_binary(Name) -> - {linkfun, {jsfun, Name}}; - undefined -> - case proplists:get_value(?JSON_JSANON, Props) of - {struct, Bkey} -> - Bucket = proplists:get_value(?JSON_JSBUCKET, Bkey), - Key = proplists:get_value(?JSON_JSKEY, Bkey), - %% bomb if malformed - true = is_binary(Bucket) andalso is_binary(Key), - {linkfun, {jsanon, {Bucket, Key}}}; - Source when is_binary(Source) -> - {linkfun, {jsanon, Source}} - end - end - end; -erlify_bucket_prop({?JSON_CHASH, {struct, Props}}) -> - {chash_keyfun, {list_to_existing_atom( - binary_to_list( - proplists:get_value(?JSON_MOD, Props))), - list_to_existing_atom( - binary_to_list( - proplists:get_value(?JSON_FUN, Props)))}}; -erlify_bucket_prop({Prop, Value}) -> - {list_to_existing_atom(binary_to_list(Prop)), Value}. - -%% @doc Populates the resource's context/state with the bucket type -%% from the path info, if not already set. -ensure_bucket_type(RD, Ctx, FNum) -> - case {element(FNum, Ctx), wrq:path_info(bucket_type, RD)} of - {undefined, undefined} -> - setelement(FNum, Ctx, <<"default">>); - {_BType, undefined} -> - Ctx; - {_, B} -> - BType = list_to_binary(riak_kv_wm_utils:maybe_decode_uri(RD, B)), - setelement(FNum, Ctx, BType) - end. - -%% @doc Checks whether the given bucket type "exists" (for use in -%% resource_exists callback). -bucket_type_exists(<<"default">>) -> true; -bucket_type_exists(Type) -> - riak_core_bucket_type:get(Type) /= undefined. - -%% Construct a {Type, Bucket} tuple, if not working with the default bucket -maybe_bucket_type(undefined, B) -> - B; -maybe_bucket_type(<<"default">>, B) -> - B; -maybe_bucket_type(T, B) -> - {T, B}. - -%% @doc Maps an HTTP method to an internal permission -%%-spec method_to_perm(atom()) -> string(). -method_to_perm('POST') -> - "riak_kv.put"; -method_to_perm('PUT') -> - "riak_kv.put"; -method_to_perm('HEAD') -> - "riak_kv.get"; -method_to_perm('GET') -> - "riak_kv.get"; -method_to_perm('DELETE') -> - "riak_kv.delete". diff --git a/src/riak_object.erl b/src/riak_object.erl index 0fccaf166..4c9707bf4 100644 --- a/src/riak_object.erl +++ b/src/riak_object.erl @@ -26,7 +26,6 @@ -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). -endif. --include("riak_kv_wm_raw.hrl"). -include("riak_object.hrl"). -include("riak_kv_capability.hrl"). @@ -42,7 +41,8 @@ index_spec/0, riak_object_meta/0, old_object/0, - hook_old_object/0 + hook_old_object/0, + repl_ref/0 ] ). @@ -106,6 +106,8 @@ -type index_value() :: integer() | binary(). -type index_spec() :: {index_op(), binary(), index_value()}. -type binary_version() :: v0 | v1. +-type repl_ref() :: + {reap, {bucket(), key(), vclock:vclock(), erlang:timestamp()}}. -define(MAX_KEY_SIZE, 65536). @@ -119,12 +121,11 @@ -export([actor_counter/2]). -export([key/1, get_metadata/1, get_metadatas/1, get_values/1, get_value/1, get_dotted_values/1]). -export([hash/1, hash/2, hash/4, approximate_size/2, proxy_size/1]). --export([vclock_encoding_method/0, vclock/1, vclock_header/1, encode_vclock/1, decode_vclock/1]). +-export([vclock_encoding_method/0, vclock/1, encode_vclock/1, decode_vclock/1]). -export([encode_vclock/2, decode_vclock/2]). -export([update/5, update_value/2, update_metadata/2, bucket/1, bucket_only/1, type/1, value_count/1]). -export([get_update_metadata/1, get_update_value/1, get_contents/1]). -export([merge/2, apply_updates/1, syntactic_merge/2]). --export([to_json/1, from_json/1]). -export([index_data/1, diff_index_data/2]). -export([index_specs/1, diff_index_specs/2]). -export([to_binary/2, from_binary/3, to_binary_version/4, binary_version/1]). @@ -298,7 +299,11 @@ metadata_fetch(Key, MetaAsMap) when is_map(MetaAsMap) -> metadata_fetch(Key, Meta) -> dict:fetch(Key, Meta). --spec metadata_find(metadata_key(), riak_object_meta()) -> metadata_value(). +-spec metadata_find( + metadata_key(), + riak_object_meta() +) -> + {ok, metadata_value()}|error. metadata_find(Key, MetaAsMap) when is_map(MetaAsMap) -> maps:find(Key, MetaAsMap); metadata_find(Key, Meta) -> @@ -1228,26 +1233,6 @@ assemble_index_specs(Indexes, IndexOp) -> set_contents(Object=#r_object{}, MVs) when is_list(MVs) -> Object#r_object{contents=[#r_content{metadata=M,value=V} || {M, V} <- MVs]}. --spec vclock_header(riak_object()) -> {Name::string(), Value::string()}. -%% @doc Transform the Erlang representation of the document's vclock -%% into something suitable for an HTTP header -vclock_header(Doc) -> - VClock = riak_object:vclock(Doc), - EncodedVClock = binary_to_list(base64:encode(encode_vclock(VClock))), - {?HEAD_VCLOCK, EncodedVClock}. - -%% @doc Converts a riak_object into its JSON equivalent -%% @deprecated use `riak_object_json:encode' directly --spec to_json(riak_object()) -> {struct, list(any())}. -to_json(Obj) -> - ?LOG_WARNING("Change uses of riak_object:to_json/1 to riak_object_json:encode/1"), - riak_object_json:encode(Obj). - -%% @deprecated Use `riak_object_json:decode' now. -from_json(JsonObj) -> - ?LOG_WARNING("Change uses of riak_object:from_json/1 to riak_object_json:decode/1"), - riak_object_json:decode(JsonObj). - is_updated(_Object=#r_object{updatemetadata=M,updatevalue=V}) -> case metadata_find(clean, M) of error -> true; @@ -1372,11 +1357,6 @@ to_binary_version(Vsn, _B, _K, Obj = #r_object{}) -> binary_version(<<131,_/binary>>) -> v0; binary_version(<>) -> v1. --type repl_ref() :: - {reap, - {riak_object:bucket(), riak_object:key(), - vclock:vclock(), erlang:timestamp()}}. - %% @doc Encode for nextgen_repl -spec nextgenrepl_encode( repl_v1, riak_object()|repl_ref(), boolean()) -> binary(). diff --git a/src/riak_object_json.erl b/src/riak_object_json.erl deleted file mode 100644 index 5004fd3be..000000000 --- a/src/riak_object_json.erl +++ /dev/null @@ -1,225 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - -%% @doc JSON encoding/decoding utilities for riak_object - - --module(riak_object_json). --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). --endif. --include("riak_kv_wm_raw.hrl"). --include("riak_object.hrl"). - --export([encode/1,decode/1]). - -%% @doc Converts a riak_object into its JSON equivalent --spec encode(riak_object:riak_object()) -> {struct, list(any())}. -encode(Obj) -> - {_,Vclock} = riak_object:vclock_header(Obj), - {struct, [{<<"bucket_type">>, riak_object:type(Obj)}, - {<<"bucket">>, riak_object:bucket_only(Obj)}, - {<<"key">>, riak_object:key(Obj)}, - {<<"vclock">>, list_to_binary(Vclock)}, - {<<"values">>, - [{struct, - [{<<"metadata">>, jsonify_metadata(MD)}, - {<<"data">>, V}]} - || {MD, V} <- riak_object:get_contents(Obj) - ]}]}. - --spec decode(any()) -> riak_object:riak_object(). -decode({struct, Obj}) -> - decode(Obj); -decode(Obj) -> - BucketType = proplists:get_value(<<"bucket_type">>, Obj), - Bucket0 = proplists:get_value(<<"bucket">>, Obj), - Bucket = case BucketType of - null -> - Bucket0; - undefined -> - Bucket0; - <<"default">> -> - Bucket0; - _ -> - {BucketType, Bucket0} - end, - Key = proplists:get_value(<<"key">>, Obj), - VClock0 = proplists:get_value(<<"vclock">>, Obj), - VClock = riak_object:decode_vclock(base64:decode(VClock0)), - [{struct, Values}] = proplists:get_value(<<"values">>, Obj), - RObj0 = riak_object:new(Bucket, Key, <<"">>), - RObj1 = riak_object:set_vclock(RObj0, VClock), - riak_object:set_contents(RObj1, dejsonify_values(Values, [])). - -jsonify_metadata(MD) -> - L = - [ - jsonify_pair(Pair) - || {Key,_}=Pair <- riak_object:metadata_tolist(MD), Key /= ?DOT - ], - {struct, L}. - --spec jsonify_pair({term(), term()}) -> {term(), term()}. -jsonify_pair({LastMod, Now={_,_,_}}) -> - %% convert Now to JS-readable time string - {LastMod, list_to_binary( - httpd_util:rfc1123_date( - calendar:now_to_local_time(Now)))}; -jsonify_pair({?MD_USERMETA, []}) -> - %% When the user metadata is empty, it should still be a struct - {?MD_USERMETA, {struct, []}}; -jsonify_pair({?MD_LINKS, Links}) -> - {?MD_LINKS, [ [B, K, T] || {{B, K}, T} <- Links ]}; -jsonify_pair({Name, List=[_|_]}) -> - {Name, jsonify_metadata_list(List)}; -jsonify_pair({Name, Value}) -> - {Name, Value}. - -%% @doc convert strings to binaries, and proplists to JSON objects -jsonify_metadata_list([]) -> []; -jsonify_metadata_list(List) -> - Classifier = fun({Key,_}, Type) when (is_binary(Key) orelse is_list(Key)), - Type /= array, Type /= string -> - struct; - (C, Type) when is_integer(C), C >= 0, C < 256, - Type /= array, Type /= struct -> - string; - (_, _) -> - array - end, - case lists:foldl(Classifier, undefined, List) of - struct -> {struct, jsonify_proplist(List)}; - string -> list_to_binary(List); - array -> List - end. - -%% @doc converts a proplist with potentially multiple values for the -%% same key into a JSON object with single or multi-valued keys. -jsonify_proplist([]) -> []; -jsonify_proplist(List) -> - dict:to_list(lists:foldl(fun({Key, Value}, Dict) -> - JSONKey = if is_list(Key) -> list_to_binary(Key); - true -> Key - end, - JSONVal = if is_list(Value) -> jsonify_metadata_list(Value); - true -> Value - end, - case dict:find(JSONKey, Dict) of - {ok, ListVal} when is_list(ListVal) -> - dict:append(JSONKey, JSONVal, Dict); - {ok, Other} -> - dict:store(JSONKey, [Other,JSONVal], Dict); - _ -> - dict:store(JSONKey, JSONVal, Dict) - end - end, dict:new(), List)). - -dejsonify_values([], Accum) -> - lists:reverse(Accum); -dejsonify_values([{<<"metadata">>, {struct, MD0}}, - {<<"data">>, D}|T], Accum) -> - Converter = - fun({Key, Val}) -> - case Key of - ?MD_LINKS -> - {Key, [{{B, K}, Tag} || [B, K, Tag] <- Val]}; - ?MD_LASTMOD -> - {Key, os:timestamp()}; - _ -> - { - Key, - if - is_binary(Val) -> - binary_to_list(Val); - true -> - dejsonify_meta_value(Val) - end - } - end - end, - MD = riak_object:metadata_fromlist([Converter(KV) || KV <- MD0]), - dejsonify_values(T, [{MD, D}|Accum]). - -%% @doc convert structs back into proplists -dejsonify_meta_value({struct, PList}) -> - lists:foldl(fun({Key, List}, Acc) when is_list(List) -> - %% This reverses the {k,v},{k,v2} pattern that - %% is possible in multi-valued indexes. - [{Key, dejsonify_meta_value(L)} || L <- List] ++ Acc; - ({Key, V}, Acc) -> - [{Key, dejsonify_meta_value(V)}|Acc] - end, [], PList); -dejsonify_meta_value(Value) -> Value. - --ifdef(TEST). - -jsonify_multivalued_indexes_test() -> - Indexes = [{<<"test_bin">>, <<"one">>}, - {<<"test_bin">>, <<"two">>}, - {<<"test2_int">>, 4}], - ?assertEqual({struct, [{<<"test2_int">>,4},{<<"test_bin">>,[<<"one">>,<<"two">>]}]}, - jsonify_metadata_list(Indexes)). - - -jsonify_round_trip_test() -> - Links = [{{<<"b">>,<<"k2">>},<<"tag">>}, - {{<<"b2">>,<<"k2">>},<<"tag2">>}], - Indexes = [{<<"test_bin">>, <<"one">>}, - {<<"test_bin">>, <<"two">>}, - {<<"test2_int">>, 4}], - Meta = [{<<"foo">>, <<"bar">>}, {<<"baz">>, <<"quux">>}], - MD = - riak_object:metadata_fromlist( - [{?MD_USERMETA, Meta}, - {?MD_CTYPE, "application/json"}, - {?MD_INDEX, Indexes}, - {?MD_LINKS, Links}] - ), - [ - begin - O = riak_object:new(B, K, V, MD), - O2 = decode(encode(O)), - ?assertEqual(riak_object:bucket(O), riak_object:bucket(O2)), - ?assertEqual(riak_object:key(O), riak_object:key(O2)), - ?assert(vclock:equal(riak_object:vclock(O), riak_object:vclock(O2))), - ?assertEqual( - lists:sort(Meta), - lists:sort( - riak_object:metadata_fetch( - ?MD_USERMETA, - riak_object:get_metadata(O2)) - ) - ), - ?assertEqual( - Links, - riak_object:metadata_fetch(?MD_LINKS, riak_object:get_metadata(O2)) - ), - ?assertEqual(lists:sort(Indexes), lists:sort(riak_object:index_data(O2))), - ?assertEqual(riak_object:get_contents(O), riak_object:get_contents(O2)) - end || - {B, K, V} <- - [ - {<<"b">>, <<"k">>, <<"{\"a\":1}">>}, - {{<<"t">>, <<"b">>}, <<"k2">>, <<"{\"a\":2}">>} - ] - ]. - --endif.