Skip to content

Commit 48285d5

Browse files
hmmrpjaclarkAndriy Zavada
authored
CLI support for riak_client:repair_node() (#156)
* Added CLI support for riak_client:repair_node() * node_cli: repair start/status/stop * node_cli: update docs * node_cli: get all nodes in a riak_test friendly manner * node_cli: tweak printed messages * node_cli: don't report vnode pid --------- Co-authored-by: Peter Clark <peter.clark@tiot.jp> Co-authored-by: Andriy Zavada <andriy.zavada@tiot.jp>
1 parent 5e046d2 commit 48285d5

4 files changed

Lines changed: 256 additions & 6 deletions

File tree

‎docs/OperationsAndTroubleshootingGuide.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,14 +104,16 @@ riak eval "riak_client:remove_node_from_coverage()."
104104

105105
#### Completing a Repair
106106

107-
The data can then be recovered from the other nodes in the cluster issuing the `riak_client:repair_node()` command from the `remote_console` of the replacement node. This will prompt all vnodes which partially overlap the data held in the vnodes on the replacement node to race to play a role in repairing the node. Each vnode will only repair the data which overlaps, filtering out any data that another vnode has already repaired (or is in the process of repairing).
107+
The data can then be recovered from the other nodes in the cluster issuing the `riak admin node repair start [-n NODE]` command. This will prompt all vnodes which partially overlap the data held in the vnodes on the replacement node to race to play a role in repairing the node. Each vnode will only repair the data which overlaps, filtering out any data that another vnode has already repaired (or is in the process of repairing).
108108

109109
<span>Available from Riak 3.4.0</span>{: .label .label-purple }To improve the performance of repair, the `repair_span` configuration in the [riak_core schema section of riak.conf](https://github.com/OpenRiak/riak_core/blob/openriak-3.4/priv/riak_core.schema) can be changed to `double_pair`, and this has been proven to be more effective when used with the leveled backend together with the enablement of the `repair_deferred` option in the [riak_kv schema section of riak.conf](https://github.com/OpenRiak/riak_kv/blob/openriak-3.4/priv/riak_kv.schema).
110110

111111
The combination of `repair_span = double_pair, repair_deferred = enabled` is significantly more effective when repairing under load. With these configuration options, it should be noted that repairs will happen in key order, not in reverse order of receipt (the default). With these changes, using the leveled backend, non-functional testing demonstrates that repairs can complete efficiently even when nodes are persistently at 100% CPU utilisation due to the handling of application requests.
112112

113113
Repair uses handoffs, and so can be tracked as with other cluster change operations. Once handoffs are complete, Tictac AAE should be re-enabled, e.g. by using `riak_client:tictacaae_resume_node().`. Once Tictac AAE confirms all vnodes are in-sync - then [`participate_in_coverage` can be re-enabled](#riak_client-remote_console-commands).
114114

115+
The progress of repairs can be inspected with `riak admin node repair status`, and stopped with `riak admin node repair stop`.
116+
115117
### Rolling Replacement
116118

117119
A rolling replacement is an extension of the [proactive replacement](#proactive-replacement) process. In a rolling replacement, a group of new nodes are installed. There is then a rolling process where some nodes are proactively replaced by the new nodes; and once those replaced nodes are free - they are use to proactively replace other nodes in the cluster.

‎src/riak_kv_cli_registry.erl‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@
2121
-module(riak_kv_cli_registry).
2222

2323
-define(CLI_MODULES, [riak_kv_tictacaae_cli,
24-
riak_kv_vnode_status_cli
24+
riak_kv_vnode_status_cli,
25+
riak_kv_node_cli
2526
]).
2627

2728
-export([register_cli/0

‎src/riak_kv_node_cli.erl‎

Lines changed: 247 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,247 @@
1+
%% -------------------------------------------------------------------
2+
%%
3+
%% Copyright (c) 2026 TI Tokyo. All Rights Reserved.
4+
%%
5+
%% This file is provided to you under the Apache License,
6+
%% Version 2.0 (the "License"); you may not use this file
7+
%% except in compliance with the License. You may obtain
8+
%% a copy of the License at
9+
%%
10+
%% http://www.apache.org/licenses/LICENSE-2.0
11+
%%
12+
%% Unless required by applicable law or agreed to in writing,
13+
%% software distributed under the License is distributed on an
14+
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
%% KIND, either express or implied. See the License for the
16+
%% specific language governing permissions and limitations
17+
%% under the License.
18+
%%
19+
%% -------------------------------------------------------------------
20+
21+
-module(riak_kv_node_cli).
22+
23+
-behaviour(clique_handler).
24+
25+
-include_lib("kernel/include/logger.hrl").
26+
27+
-export([register_cli/0]).
28+
29+
register_cli() ->
30+
register_all_usage(),
31+
register_all_commands().
32+
33+
register_all_usage() ->
34+
clique:register_usage(["riak-admin", "node"], node_usage()),
35+
clique:register_usage(["riak-admin", "node", "repair"], node_repair_usage()),
36+
clique:register_usage(["riak-admin", "node", "repair", "start"], node_repair_start_usage()),
37+
clique:register_usage(["riak-admin", "node", "repair", "status"], node_repair_status_usage()),
38+
clique:register_usage(["riak-admin", "node", "repair", "stop", '*'], node_repair_stop_usage()).
39+
40+
register_all_commands() ->
41+
lists:foreach(
42+
fun(Args) -> apply(clique, register_command, Args) end,
43+
[node_repair_status_specs(),
44+
node_repair_start_specs(),
45+
node_repair_stop_specs()
46+
]).
47+
48+
node_usage() ->
49+
["riak admin node repair { start | status | stop REASON } [OPTIONS]\n",
50+
"See individual subcommand usage for options and arguments.\n"
51+
].
52+
node_repair_usage() ->
53+
node_usage().
54+
55+
node_repair_status_usage() ->
56+
["riak admin node repair status [-n|--node NODE|all] [-f|--format table|json]\n",
57+
"Print status of any ongoing partition repairs on NODE.\n"
58+
].
59+
60+
node_repair_start_usage() ->
61+
["riak admin node repair start [-n|--node NODE]\n",
62+
"Start partition repair on NODE.\n"
63+
].
64+
65+
node_repair_stop_usage() ->
66+
["riak admin node repair stop [-n|--node NODE|all] REASON\n",
67+
"Kill all ongoing partition repair on NODE, with REASON.\n"
68+
].
69+
70+
-define(NODEOPT, {node, [{shortname, "n"},
71+
{longname, "node"},
72+
{typecast, fun to_node/1}]}).
73+
-define(FMTOPTION, {format, [{shortname, "f"},
74+
{longname, "format"},
75+
{typecast, fun to_fmt/1}]}).
76+
77+
node_repair_status_specs() ->
78+
[["riak-admin", "node", "repair", "status"],
79+
[], [?NODEOPT, ?FMTOPTION],
80+
fun(A, B, C) -> main(fun node_repair_status_cmd/3, A, B, C) end
81+
].
82+
node_repair_start_specs() ->
83+
[["riak-admin", "node", "repair", "start"],
84+
[], [?NODEOPT],
85+
fun(A, B, C) -> main(fun node_repair_start_cmd/3, A, B, C) end
86+
].
87+
node_repair_stop_specs() ->
88+
[["riak-admin", "node", "repair", "stop", '*'],
89+
[], [?NODEOPT],
90+
fun(A, B, C) -> main(fun node_repair_stop_cmd/3, A, B, C) end
91+
].
92+
93+
94+
main(Fun, A, B, C) ->
95+
try
96+
Fun(A, B, C)
97+
catch
98+
Class:Reason:Stack ->
99+
logger:error("node repair: handler failed: ~p:~p stack=~p",
100+
[Class, Reason, Stack]),
101+
[alert("Error: ~p:~p", [Class, Reason])]
102+
end.
103+
104+
node_repair_status_cmd(_Cmd, _Args, Opts) ->
105+
Nodes =
106+
case [A || {node, A} <- Opts] of
107+
[all] ->
108+
get_nodes();
109+
[] ->
110+
[node()];
111+
NN ->
112+
NN
113+
end,
114+
Fmt = extract_fmt_option(Opts),
115+
116+
Res = get_node_repair_status(Nodes),
117+
118+
case Fmt of
119+
table ->
120+
Table =
121+
[begin
122+
Rows =
123+
[[{mod, Mod}, {idx, integer_to_binary(Idx)}]
124+
|| {Mod, Idx} <- NRes],
125+
case Rows of
126+
[] ->
127+
text("No active node repairs on ~s\n", [Node]);
128+
_ ->
129+
[text("Vnode repairs triggered by node repair on ~s", [Node]), table(Rows)]
130+
end
131+
end || {Node, NRes} <- Res],
132+
lists:flatten(Table);
133+
json ->
134+
[text("~s", [riak_kv_wm_json:encode(
135+
[#{node => Node,
136+
status => [jsonify_status(S) || S <- Statuses]}
137+
|| {Node, Statuses} <- Res])])]
138+
end.
139+
140+
get_node_repair_status(Nodes) ->
141+
[begin
142+
Vnodes = erpc:call(Node, riak_core_vnode_manager, all_vnodes, []),
143+
Statuses = [{Mod, Idx,
144+
erpc:call(Node, riak_core_vnode_manager, repair_status, [{Mod, Idx}])}
145+
|| {Mod, Idx, _Pid} <- Vnodes],
146+
{Node, [{Mod, Idx} || {Mod, Idx, Status} <- Statuses, Status /= not_found]}
147+
end || Node <- Nodes].
148+
149+
nodes_running_repair() ->
150+
AllSS = get_node_repair_status(get_nodes()),
151+
[N || {N, SS} <- AllSS, SS /= []].
152+
153+
jsonify_status({Mod, Idx}) ->
154+
#{mod => Mod,
155+
idx => Idx}.
156+
157+
node_repair_start_cmd(_Cmd, _Args, Opts) ->
158+
Node =
159+
case [A || {node, A} <- Opts] of
160+
[all] -> invalid;
161+
[] -> node();
162+
[N] -> N;
163+
_ -> invalid
164+
end,
165+
case Node of
166+
invalid ->
167+
[alert("Error: node repair can be started on one node at a time")];
168+
Node ->
169+
case nodes_running_repair() of
170+
[] ->
171+
case erpc:call(Node, riak_client, repair_node, []) of
172+
ok ->
173+
[text("Node repair started on ~s.", [Node])];
174+
{error, BadRpcReason} ->
175+
[alert("Error: failed to start node repair on ~s: ~p", [Node, BadRpcReason])]
176+
end;
177+
NwAA ->
178+
[alert("There are repairs currently ongoing on node~s ~s.\n"
179+
"Wait until these are completed before starting a new node repair.",
180+
[ending(NwAA), string:join([atom_to_list(N) || N <- NwAA], ",")])]
181+
end
182+
end.
183+
184+
node_repair_stop_cmd([_, _, _, _, Reason], _, Opts) ->
185+
Nodes =
186+
case [A || {node, A} <- Opts] of
187+
[all] ->
188+
get_nodes();
189+
[] ->
190+
[node()];
191+
NN ->
192+
NN
193+
end,
194+
AllNodesWithRepairs = nodes_running_repair(),
195+
Items =
196+
[begin
197+
case lists:member(Node, AllNodesWithRepairs) of
198+
false ->
199+
io_lib:format("\n* ~s: No active repairs", [Node]);
200+
true ->
201+
case erpc:call(Node, riak_core_vnode_manager, kill_repairs, [Reason]) of
202+
ok ->
203+
io_lib:format("\n* ~s: Node repair stopped", [Node]);
204+
{error, BadRpcReason} ->
205+
io_lib:format("\n* ~s: Failed to stop node repair on: ~p", [Node, BadRpcReason])
206+
end
207+
end
208+
end || Node <- Nodes],
209+
[clique_status:list("Stopping repairs", Items)].
210+
211+
get_nodes() ->
212+
{ok, Ring} = riak_core_ring_manager:get_my_ring(),
213+
Members = riak_core_ring:all_member_status(Ring),
214+
[N || {N, Valid} <- Members, is_really(Valid)].
215+
is_really(A) when A == joining;
216+
A == valid;
217+
A == leaving -> true;
218+
is_really(_) -> false.
219+
220+
221+
extract_fmt_option(Opts) ->
222+
case [A || {format, A} <- Opts] of
223+
[] -> table;
224+
["table"] -> table;
225+
["json"] -> json;
226+
_ -> invalid
227+
end.
228+
229+
to_node("all") ->
230+
all;
231+
to_node(A) ->
232+
clique_typecast:to_node(A).
233+
234+
to_fmt(A) ->
235+
A.
236+
237+
text(F, A) ->
238+
clique_status:text(lists:flatten(io_lib:format(F, A))).
239+
alert(S) ->
240+
alert(S, []).
241+
alert(F, A) ->
242+
clique_status:alert([text(F, A)]).
243+
table(A) ->
244+
clique_status:table(A).
245+
246+
ending([_]) -> "";
247+
ending(_) -> "s".

‎src/riak_kv_tictacaae_cli.erl‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -679,7 +679,7 @@ fold_cmd([_, _, _ | Items], Keys, Options) ->
679679
},
680680
{ok, BB} = riak_client:aae_fold(Query),
681681
Printable = [printable_bin(B) || B <- BB],
682-
io:format(FD, "~s\n", [mochijson2:encode(Printable)])
682+
io:format(FD, "~s\n", [riak_kv_wm_json:encode(Printable)])
683683
end);
684684

685685
["find-keys"] ->
@@ -697,7 +697,7 @@ fold_cmd([_, _, _ | Items], Keys, Options) ->
697697
Printable = [#{<<"key">> => printable_bin(K),
698698
<<"sibling_count">> => SibCnt
699699
} || {_B, K, SibCnt} <- KK],
700-
io:format(FD, "~s\n", [mochijson2:encode(Printable)])
700+
io:format(FD, "~s\n", [riak_kv_wm_json:encode(Printable)])
701701
end);
702702

703703
["count-keys"] ->
@@ -730,7 +730,7 @@ fold_cmd([_, _, _ | Items], Keys, Options) ->
730730
Printable = [#{bucket => printable_bin(B),
731731
key => printable_bin(K),
732732
vclock => printable_vclock(VC)} || {B, K, VC} <- TT],
733-
io:format(FD, "~s\n", [mochijson2:encode(Printable)])
733+
io:format(FD, "~s\n", [riak_kv_wm_json:encode(Printable)])
734734
end);
735735

736736
["count-tombstones"] ->
@@ -779,7 +779,7 @@ fold_cmd([_, _, _ | Items], Keys, Options) ->
779779
TS = proplists:get_value(total_size, SS),
780780
Sizes = proplists:get_value(sizes, SS),
781781
Siblings = proplists:get_value(siblings, SS),
782-
io:format(FD, "~s\n", [mochijson2:encode(
782+
io:format(FD, "~s\n", [riak_kv_wm_json:encode(
783783
#{total_count => TC,
784784
total_size => TS,
785785
sizes => [#{min => Min,

0 commit comments

Comments
 (0)