Skip to content

updowngrade: better prepare riak.conf & caps, make it work for 1.4-1.5 and 1.5-1.6 cycles #1283

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Mar 3, 2017
Merged

updowngrade: better prepare riak.conf & caps, make it work for 1.4-1.5 and 1.5-1.6 cycles #1283

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6a23c82
updowngrade: better prepare riak.conf & caps to wait_until
hmmr Mar 3, 2017
626c4d5
Fix regular expression backslashes
macintux Mar 3, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions tests/ts_cluster_updowngrade_group_by_SUITE.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
%% -------------------------------------------------------------------
%%
%% Copyright (c) 2016 Basho Technologies, Inc.
%% Copyright (c) 2016, 2017 Basho Technologies, Inc.
%%
%% This file is provided to you under the Apache License,
%% Version 2.0 (the "License"); you may not use this file
Expand Down Expand Up @@ -40,8 +40,8 @@ make_scenarios() ->
need_query_node_transition = NeedQueryNodeTransition,
need_pre_cluster_mixed = NeedPreClusterMixed,
need_post_cluster_mixed = NeedPostClusterMixed,
ensure_full_caps = [{{riak_kv, sql_select_version}, v3}, {{riak_kv, riak_ql_ddl_rec_version}, v2}, {{riak_kv, decode_query_results_at_vnode}, true}],
ensure_degraded_caps = [{{riak_kv, sql_select_version}, v2}, {{riak_kv, riak_ql_ddl_rec_version}, v1}, {{riak_kv, decode_query_results_at_vnode}, false}],
ensure_full_caps = ts_updown_util:caps_to_ensure(full),
ensure_degraded_caps = ts_updown_util:caps_to_ensure(degraded),
convert_config_to_previous = fun ts_updown_util:convert_riak_conf_to_previous/1}
|| TableNodeVsn <- [previous, current],
QueryNodeVsn <- [previous, current],
Expand All @@ -51,6 +51,7 @@ make_scenarios() ->
NeedPostClusterMixed <- [true, false]],
[add_tests(X) || X <- BaseScenarios].


%% This test will not use config invariants
%% see ts_cluster_updowngrade_select_aggregation_SUITE.erl for an example
%% of how to use them
Expand Down
6 changes: 3 additions & 3 deletions tests/ts_cluster_updowngrade_order_by_SUITE.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
%% -------------------------------------------------------------------
%%
%% Copyright (c) 2016 Basho Technologies, Inc.
%% Copyright (c) 2016, 2017 Basho Technologies, Inc.
%%
%% This file is provided to you under the Apache License,
%% Version 2.0 (the "License"); you may not use this file
Expand Down Expand Up @@ -60,8 +60,8 @@ make_scenarios() ->
need_query_node_transition = NeedQueryNodeTransition,
need_pre_cluster_mixed = NeedPreClusterMixed,
need_post_cluster_mixed = NeedPostClusterMixed,
ensure_full_caps = [{{riak_kv, sql_select_version}, v3}, {{riak_kv, riak_ql_ddl_rec_version}, v2}, {{riak_kv, decode_query_results_at_vnode}, true}],
ensure_degraded_caps = [{{riak_kv, sql_select_version}, v2}, {{riak_kv, riak_ql_ddl_rec_version}, v1}, {{riak_kv, decode_query_results_at_vnode}, false}],
ensure_full_caps = ts_updown_util:caps_to_ensure(full),
ensure_degraded_caps = ts_updown_util:caps_to_ensure(degraded),
convert_config_to_previous = fun ts_updown_util:convert_riak_conf_to_previous/1}
|| TableNodeVsn <- [previous, current],
QueryNodeVsn <- [current],
Expand Down
6 changes: 3 additions & 3 deletions tests/ts_cluster_updowngrade_select_aggregation_SUITE.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
%% -------------------------------------------------------------------
%%
%% Copyright (c) 2016 Basho Technologies, Inc.
%% Copyright (c) 2016, 2017 Basho Technologies, Inc.
%%
%% This file is provided to you under the Apache License,
%% Version 2.0 (the "License"); you may not use this file
Expand Down Expand Up @@ -35,8 +35,8 @@ make_scenarios() ->
need_query_node_transition = NeedQueryNodeTransition,
need_pre_cluster_mixed = NeedPreClusterMixed,
need_post_cluster_mixed = NeedPostClusterMixed,
ensure_full_caps = [{{riak_kv, sql_select_version}, v3}, {{riak_kv, riak_ql_ddl_rec_version}, v2}, {{riak_kv, decode_query_results_at_vnode}, true}],
ensure_degraded_caps = [{{riak_kv, sql_select_version}, v2}, {{riak_kv, riak_ql_ddl_rec_version}, v1}, {{riak_kv, decode_query_results_at_vnode}, false}],
ensure_full_caps = ts_updown_util:caps_to_ensure(full),
ensure_degraded_caps = ts_updown_util:caps_to_ensure(degraded),
convert_config_to_previous = fun ts_updown_util:convert_riak_conf_to_previous/1}
|| TableNodeVsn <- [previous, current],
QueryNodeVsn <- [previous, current],
Expand Down
93 changes: 75 additions & 18 deletions tests/ts_updown_util.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
%% -------------------------------------------------------------------
%%
%% Copyright (c) 2016 Basho Technologies, Inc.
%% Copyright (c) 2016, 2017 Basho Technologies, Inc.
%%
%% This file is provided to you under the Apache License,
%% Version 2.0 (the "License"); you may not use this file
Expand Down Expand Up @@ -50,6 +50,7 @@
-module(ts_updown_util).

-export([
caps_to_ensure/1,
convert_riak_conf_to_previous/1,
maybe_shutdown_client_node/1,
setup/1,
Expand Down Expand Up @@ -581,31 +582,87 @@ wait_until_active_table(TargetNode, PrevClientNode, Table, N) ->
make_msg(Format, Payload) ->
list_to_binary(fmt(Format, Payload)).


get_riak_release_in_slot(VsnSlot) ->
case rtdev:get_version(VsnSlot) of
unknown ->
ct:fail("Failed to determine riak version in '~s' slot", [VsnSlot]);
Known ->
case re:run(Known, "riak_ts-(\\d+)\.(\\d+)\.(\\d+)", [{capture, all_but_first, list}]) of
Copy link
Contributor

@macintux macintux Mar 3, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The \. should be \\.

{match, [V1, V2, V3]} ->
{list_to_integer(V1),
list_to_integer(V2),
list_to_integer(V3)};
nomatch ->
ct:fail("Failed to parse riak version in '~s' slot", [VsnSlot])
end
end.

%% ---------------------------------

%% Dealing with case-by-case upgrade particulars, such as:
%%
%% * newly introduced keys in riak.conf that need to be deleted on
%% downgrade;
%%
%% * capabilities that need to be ensured before running the tests
%% (arguments to `wait_until_capability`).

%% We need to comment out those settings which appear in version
%% 1.x. For version 1.x-1 to work with riak.conf initially created in
%% 1.x, the offending settings need to be deleted. We do it here, by
%% commenting them out.

%% riak.conf created under 1.6 cannot be read by 1.5 because of new keys:
%% riak_kv.query.timeseries.qbuf_inmem_max,
%%
%% riak.conf created under 1.5 cannot be read by 1.4 because of new keys:
%% riak_kv.query.timeseries.max_returned_data_size,
%% riak_kv.query.timeseries.qbuf_soft_watermark,
%% riak_kv.query.timeseries.qbuf_hard_watermark,
%% riak_kv.query.timeseries.qbuf_expire_ms,
%% riak_kv.query.timeseries.qbuf_incomplete_release_ms
%%
convert_riak_conf_to_previous(Config) ->
DatafPath = ?CFG(new_data_dir, Config),
RiakConfPath = filename:join(DatafPath, "../etc/riak.conf"),
{ok, Content0} = file:read_file(RiakConfPath),
Content9 =
re:replace(
Content0,
<<"^riak_kv.query.timeseries.qbuf_inmem_max">>,
<<"#\\1">>, [global, multiline, {return, binary}]),
ok = file:write_file(RiakConfPath, Content9).
{ok, Contents0} = file:read_file(RiakConfPath),
Contents9 =
lists:foldl(
fun(KeyToDelete, Contents) ->
re:replace(Contents, ["^", KeyToDelete], "#\\1",
[global, multiline, {return, list}])
end,
Contents0,
get_riak_conf_new_keys()),
ok = file:write_file(RiakConfPath, Contents9).

%% When a new release is cut, register newly introduced keys here:
get_riak_conf_new_keys() ->
%% the current version may have not been tagged yet, so look at
%% previous version
case get_riak_release_in_slot(previous) of
{1, 5, _} ->
["riak_kv.query.timeseries.qbuf_inmem_max"];
{1, 4, _} ->
["riak_kv.query.timeseries.max_returned_data_size",
"riak_kv.query.timeseries.qbuf_soft_watermark",
"riak_kv.query.timeseries.qbuf_hard_watermark",
"riak_kv.query.timeseries.qbuf_expire_ms",
"riak_kv.query.timeseries.qbuf_incomplete_release_ms"]
end.

%% Wait for these capabilities to settle at these versions at the end
%% of upgrade/downgrade:
caps_to_ensure(full) ->
case get_riak_release_in_slot(previous) of
{1, 5, _} ->
[]; %% no new caps in 1.6 since 1.5
{1, 4, _} ->
[{{riak_kv, sql_select_version}, v3},
{{riak_kv, riak_ql_ddl_rec_version}, v2},
{{riak_kv, decode_query_results_at_vnode}, true}]
end;
caps_to_ensure(degraded) ->
case get_riak_release_in_slot(previous) of
{1, 5, _} ->
[]; %% no new caps in 1.6 since 1.5
{1, 4, _} ->
[{{riak_kv, sql_select_version}, v2},
{{riak_kv, riak_ql_ddl_rec_version}, v1},
{{riak_kv, decode_query_results_at_vnode}, false}]
end.


%% Keep the old convert_riak_conf_to_previous functions around, for
%% reference and occasional test rerun
Expand Down