From a5e0c8f92718e4a6e87878c19d58ab8bd4431069 Mon Sep 17 00:00:00 2001 From: Martin Cox Date: Fri, 13 Mar 2020 08:46:07 +0000 Subject: [PATCH 01/47] Add tools application to include profiling modules. --- apps/riak/src/riak.app.src | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/riak/src/riak.app.src b/apps/riak/src/riak.app.src index 23ae71d48..921cadee0 100644 --- a/apps/riak/src/riak.app.src +++ b/apps/riak/src/riak.app.src @@ -7,6 +7,7 @@ kernel, stdlib, lager, + tools, sasl, public_key, ssl, From 7fbdd4b774cc78e082f79a9f256a634917b81a0d Mon Sep 17 00:00:00 2001 From: Nicholas Adams Date: Fri, 21 Aug 2020 17:17:24 +0900 Subject: [PATCH 02/47] Edit Makefile so package names include OTP version Added OTP_VER to initial variables so that it can be included in the PKG_ID variable and thus the binary file name --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 02585f0e1..da9ea4c5e 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ PKG_REVISION ?= $(shell git describe --tags 2>/dev/null) PKG_BUILD = 1 BASE_DIR = $(shell pwd) ERLANG_BIN = $(shell dirname $(shell which erl 2>/dev/null) 2>/dev/null) +OTP_VER = $(shell echo $(ERLANG_BIN) | rev | cut -d "/" -f 2 | rev) REBAR ?= $(BASE_DIR)/rebar3 OVERLAY_VARS ?= TEST_IGNORE ?= lager riak basho_bench @@ -219,7 +220,7 @@ MAJOR_VERSION ?= $(shell echo $(REVISION) | sed -e 's/\([0-9.]*\)-.*/\1/') # This enables the toplevel repository package to change names # when underlying dependencies change. NAME_HASH = $(shell git hash-object distdir/$(CLONEDIR)/$(MANIFEST_FILE) 2>/dev/null | cut -c 1-8) -PKG_ID := $(REPO_TAG) +PKG_ID := "$(REPO_TAG)_OTP_$(OTP_VER)" ## ## Packaging targets From a1bf88165c19e1bed6d239cebf16afad9beccd9f Mon Sep 17 00:00:00 2001 From: Nicholas Adams Date: Fri, 21 Aug 2020 22:07:33 +0900 Subject: [PATCH 03/47] Updated PKG_ID to Debian and RHEL friendly name Swapped underscores (not allowed by Ubuntu) for a hyphen. Removed hyphen from after OTP as the meaning is still clear without it. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index da9ea4c5e..12699f026 100644 --- a/Makefile +++ b/Makefile @@ -220,7 +220,7 @@ MAJOR_VERSION ?= $(shell echo $(REVISION) | sed -e 's/\([0-9.]*\)-.*/\1/') # This enables the toplevel repository package to change names # when underlying dependencies change. NAME_HASH = $(shell git hash-object distdir/$(CLONEDIR)/$(MANIFEST_FILE) 2>/dev/null | cut -c 1-8) -PKG_ID := "$(REPO_TAG)_OTP_$(OTP_VER)" +PKG_ID := "$(REPO_TAG)-OTP$(OTP_VER)" ## ## Packaging targets From f1f997e7a970e347246daab877ec5577a3dfa034 Mon Sep 17 00:00:00 2001 From: Martin Cox Date: Fri, 20 Nov 2020 05:42:28 +0000 Subject: [PATCH 04/47] escape props arg to preserve quotes --- rel/files/riak | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rel/files/riak b/rel/files/riak index b41b97c2f..aecf7da63 100755 --- a/rel/files/riak +++ b/rel/files/riak @@ -41,7 +41,8 @@ else su - riak -c "RELX_CONFIG_PATH=${RELX_CONFIG_PATH} VMARGS_PATH=${VMARGS_PATH} RUNNER_LOG_DIR=${RUNNER_LOG_DIR} PIPE_DIR=${PIPE_DIR} ${COMMAND} ${*} -pa {{platform_lib_dir}}/patches" ;; *) - su - riak -c "RELX_CONFIG_PATH=${RELX_CONFIG_PATH} VMARGS_PATH=${VMARGS_PATH} RUNNER_LOG_DIR=${RUNNER_LOG_DIR} PIPE_DIR=${PIPE_DIR} ${COMMAND} ${*}" + ESCAPED_ARGS=`echo "$@" | sed -e 's/\([\\\(\\\){}"\x27]\)/\\\\\1/g'`"]'` + su - riak -c "RELX_CONFIG_PATH=${RELX_CONFIG_PATH} VMARGS_PATH=${VMARGS_PATH} RUNNER_LOG_DIR=${RUNNER_LOG_DIR} PIPE_DIR=${PIPE_DIR} ${COMMAND} ${ESCAPED_ARGS}" ;; esac fi From a4cb61c749c3ed0f80b4098f08fe9dddb4476438 Mon Sep 17 00:00:00 2001 From: Martin Cox Date: Fri, 20 Nov 2020 14:40:29 +0000 Subject: [PATCH 05/47] correct sed - weird char matching from vim. --- rel/files/riak | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rel/files/riak b/rel/files/riak index aecf7da63..06f6fc29f 100755 --- a/rel/files/riak +++ b/rel/files/riak @@ -41,7 +41,7 @@ else su - riak -c "RELX_CONFIG_PATH=${RELX_CONFIG_PATH} VMARGS_PATH=${VMARGS_PATH} RUNNER_LOG_DIR=${RUNNER_LOG_DIR} PIPE_DIR=${PIPE_DIR} ${COMMAND} ${*} -pa {{platform_lib_dir}}/patches" ;; *) - ESCAPED_ARGS=`echo "$@" | sed -e 's/\([\\\(\\\){}"\x27]\)/\\\\\1/g'`"]'` + ESCAPED_ARGS=`echo "$@" | sed -e 's/\([\\\(\\\){}"\x27]\)/\\\\\1/g'` su - riak -c "RELX_CONFIG_PATH=${RELX_CONFIG_PATH} VMARGS_PATH=${VMARGS_PATH} RUNNER_LOG_DIR=${RUNNER_LOG_DIR} PIPE_DIR=${PIPE_DIR} ${COMMAND} ${ESCAPED_ARGS}" ;; esac From c78874dc2857dd1e99f812138c40922bff141742 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Mon, 14 Dec 2020 17:05:31 +0000 Subject: [PATCH 06/47] Branch updates for release --- rebar.config | 6 +++--- rebar.lock | 24 ++++++++++++------------ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/rebar.config b/rebar.config index a0600d558..5fd8fe11d 100644 --- a/rebar.config +++ b/rebar.config @@ -11,12 +11,12 @@ {deps, [ {lager_syslog, {git, "git://github.com/basho/lager_syslog.git", {tag, "3.1.0"}}}, {cluster_info, {git, "git://github.com/basho/cluster_info.git", {tag, "2.1.0"}}}, - {riak_kv, {git, "git://github.com/basho/riak_kv.git", {tag, "riak_kv-3.0.1"}}}, + {riak_kv, {git, "git://github.com/basho/riak_kv.git", {branch, "develop-3.0"}}}, {riak_auth_mods, {git, "git://github.com/basho/riak_auth_mods.git", {tag, "riak_kv-3.0.0"}}}, % {yokozuna, {git, "git://github.com/basho/yokozuna.git", {branch, "develop-3.0"}}}, - {riak_repl, {git, "git://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.1"}}} + {riak_repl, {git, "git://github.com/basho/riak_repl.git", {branch, "develop-3.0"}}} ]}. {project_plugins, [ @@ -45,7 +45,7 @@ ]} ]}. -{relx, [{release, {riak, "3.0"}, +{relx, [{release, {riak, "3.0.2"}, [kernel, stdlib, lager, diff --git a/rebar.lock b/rebar.lock index 624c0fcd3..27b1b25e4 100644 --- a/rebar.lock +++ b/rebar.lock @@ -53,7 +53,7 @@ 2}, {<<"kv_index_tictactree">>, {git,"https://github.com/martinsumner/kv_index_tictactree.git", - {ref,"fe623784cc3e3185238924bc6b1786f2d79edb36"}}, + {ref,"81ecb0c5082e2246d14592c7649513fb89c39d82"}}, 1}, {<<"lager">>, {git,"git://github.com/erlang-lager/lager.git", @@ -65,7 +65,7 @@ 0}, {<<"leveled">>, {git,"https://github.com/martinsumner/leveled.git", - {ref,"bf591a5aa988ef4c3699152d129fc81ca0896030"}}, + {ref,"182395ee00665bc8869bd9f3c6b6ebb3b65fbce6"}}, 1}, {<<"lz4">>, {git,"https://github.com/martinsumner/erlang-lz4", @@ -74,7 +74,7 @@ {<<"meck">>, {git,"https://github.com/eproxus/meck.git", {ref,"4ecc1ae9089edc6977e8c8c4cd41081513cc5590"}}, - 3}, + 4}, {<<"mochiweb">>, {git,"git://github.com/basho/mochiweb.git", {ref,"fb7bc696357b77fd8075de796d3687dfb5ed0d0b"}}, @@ -91,7 +91,7 @@ {<<"protobuffs">>, {git,"https://github.com/basho/erlang_protobuffs.git", {ref,"098efad8f85dfe556d64e2cf6ce31f2075808f67"}}, - 2}, + 3}, {<<"ranch">>, {git,"git://github.com/ninenines/ranch.git", {ref,"d0fad24657711cf584da61b5163c29fdafbd9201"}}, @@ -106,7 +106,7 @@ 1}, {<<"riak_api">>, {git,"git://github.com/basho/riak_api.git", - {ref,"0aec475c38ea17232b2e80d9c6c684a6ad9cd226"}}, + {ref,"4f68783d7669fc165dbc98edfc97d8ece7fcf356"}}, 1}, {<<"riak_auth_mods">>, {git,"git://github.com/basho/riak_auth_mods.git", @@ -114,7 +114,7 @@ 0}, {<<"riak_core">>, {git,"git://github.com/basho/riak_core.git", - {ref,"3903efc8c975c47125babe06f2a4cc674616b658"}}, + {ref,"e817ee7555dc4a327be0843ef8e6c1c0974226f2"}}, 1}, {<<"riak_dt">>, {git,"git://github.com/basho/riak_dt.git", @@ -126,19 +126,19 @@ 2}, {<<"riak_kv">>, {git,"git://github.com/basho/riak_kv.git", - {ref,"d17495741ed001dcb54fa9e75e0dff026baaefde"}}, + {ref,"75e142ef5a9d02cbcd8f84d409817e86197b2a64"}}, 0}, {<<"riak_pb">>, {git,"git://github.com/basho/riak_pb.git", - {ref,"64cabe25eec68acd6cc38035cb91814dff3c5f14"}}, - 1}, + {ref,"b8a946426e0254abba2015f9b5c1a9db4e2405bc"}}, + 2}, {<<"riak_pipe">>, {git,"git://github.com/basho/riak_pipe.git", {ref,"811ae70fc258e4c3e8d65ff0593ea06989b6dec1"}}, 1}, {<<"riak_repl">>, {git,"git://github.com/basho/riak_repl.git", - {ref,"90c317286fdc8cff3709e8dcd6d99633d89201ce"}}, + {ref,"64cbbcaca3c97929d8f543d8f104a935600d433c"}}, 0}, {<<"riak_sysmon">>, {git,"git://github.com/basho/riak_sysmon.git", @@ -146,11 +146,11 @@ 2}, {<<"riakc">>, {git,"git://github.com/basho/riak-erlang-client", - {ref,"3c6c0d28507b87026f99b7de583eedda851338f7"}}, + {ref,"a17ad08019ce7564fce8017a8d758aa486c97b25"}}, 2}, {<<"riakhttpc">>, {git,"git://github.com/basho/riak-erlang-http-client", - {ref,"762c7cb5a1ca440207817dd452840431697430ab"}}, + {ref,"15174fa6631d66f11c0a31050df5e6ab34b9b69b"}}, 1}, {<<"setup">>,{pkg,<<"setup">>,<<"2.0.2">>},3}, {<<"sext">>, From 4bfbe3b717c15cd0284b9719143adf0841224335 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Thu, 17 Dec 2020 19:35:54 +0000 Subject: [PATCH 07/47] Mas i1037 riakdiag (#1047) Re-introduce capability to run `sudo riak debug` (not riak-debug). This is considered as a special case, and bypasses the standard relx generated riak start script. There were a number of issues with individual debug scripts related to permissions and the relative start location when running via he relx generated script. This adds back into riak the riaknostic library, allowing for `riak admin diag` to be run. The level of diagnostic checks supplied via this library are minimal at present though. The diagnostic checks will be run by default as part of `riak debug` Co-authored-by: Bob-The-Marauder --- rebar.config | 21 ++-- rebar.lock | 6 +- rel/files/app_epath.sh | 173 +++++++++++++++++++++++++++++++++ rel/files/riak | 7 ++ rel/files/riak-debug | 216 +++++++++++++++++++++++++++++------------ rel/pkg/rpm/specfile | 1 - 6 files changed, 348 insertions(+), 76 deletions(-) create mode 100755 rel/files/app_epath.sh diff --git a/rebar.config b/rebar.config index 5fd8fe11d..9304efd8a 100644 --- a/rebar.config +++ b/rebar.config @@ -13,6 +13,7 @@ {cluster_info, {git, "git://github.com/basho/cluster_info.git", {tag, "2.1.0"}}}, {riak_kv, {git, "git://github.com/basho/riak_kv.git", {branch, "develop-3.0"}}}, {riak_auth_mods, {git, "git://github.com/basho/riak_auth_mods.git", {tag, "riak_kv-3.0.0"}}}, + {riaknostic, {git, "git://github.com/basho/riaknostic.git", {branch, "develop-3.0"}}}, % {yokozuna, {git, "git://github.com/basho/yokozuna.git", {branch, "develop-3.0"}}}, @@ -71,6 +72,7 @@ riak_repl, cluster_info, % yokozuna, + riaknostic, riak_auth_mods]}, {dev_mode, false}, @@ -83,16 +85,16 @@ {template, "rel/files/advanced.config", "etc/advanced.config"}, %% Copy additional bin scripts - {template, "rel/files/riak-admin", "bin/riak-admin"}, - {template, "rel/files/riak-debug", "bin/riak-debug"}, - {template, "rel/files/riak-chkconfig", "bin/riak-chkconfig"}, - {template, "rel/files/riak-repl", "bin/riak-repl"}, + {template, "rel/files/riak-admin", "bin/riak-admin"}, + {template, "rel/files/riak-debug", "bin/riak-debug"}, + {template, "rel/files/riak-chkconfig", "bin/riak-chkconfig"}, + {template, "rel/files/riak-repl", "bin/riak-repl"}, + {template, "rel/files/riak", "usr/bin/riak"}, - {template, "rel/files/riak", "usr/bin/riak"}, - - {copy, "rel/files/check_ulimit", "bin/hooks/check_ulimit"}, - {copy, "rel/files/erl_maxlogsize", "bin/hooks/erl_maxlogsize"}, - {copy, "rel/files/riak_not_running", "bin/hooks/riak_not_running"} + {copy, "rel/files/check_ulimit", "bin/hooks/check_ulimit"}, + {copy, "rel/files/erl_maxlogsize", "bin/hooks/erl_maxlogsize"}, + {copy, "rel/files/riak_not_running", "bin/hooks/riak_not_running"}, + {copy, "rel/files/app_epath.sh", "lib/app_epath.sh"} ]}, @@ -100,7 +102,6 @@ {extended_start_script, true}, {extended_start_script_extensions, [ {admin, "riak-admin"}, - {debug, "riak-debug"}, {repl, "riak-repl"}, {chkconfig, "riak-chkconfig"} ]} diff --git a/rebar.lock b/rebar.lock index 27b1b25e4..cf8c11a05 100644 --- a/rebar.lock +++ b/rebar.lock @@ -40,7 +40,7 @@ {<<"getopt">>, {git,"git://github.com/jcomellas/getopt.git", {ref,"fbd210550677114a6c96cdf66e340eb248908d93"}}, - 4}, + 1}, {<<"goldrush">>,{pkg,<<"goldrush">>,<<"0.1.9">>},2}, {<<"hut">>,{pkg,<<"hut">>,<<"1.2.1">>},3}, {<<"hyper">>, @@ -152,6 +152,10 @@ {git,"git://github.com/basho/riak-erlang-http-client", {ref,"15174fa6631d66f11c0a31050df5e6ab34b9b69b"}}, 1}, + {<<"riaknostic">>, + {git,"git://github.com/basho/riaknostic.git", + {ref,"8aef63a89d719b22754683cce187eaeeecdd789c"}}, + 0}, {<<"setup">>,{pkg,<<"setup">>,<<"2.0.2">>},3}, {<<"sext">>, {git,"git://github.com/uwiger/sext.git", diff --git a/rel/files/app_epath.sh b/rel/files/app_epath.sh new file mode 100755 index 000000000..a3de910b0 --- /dev/null +++ b/rel/files/app_epath.sh @@ -0,0 +1,173 @@ +#!/bin/sh + +## --------------------------------------------------------------------- +## +## app_epath.sh: Parse Erlang app.config with POSIX tools for automation +## +## Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. +## +## This file is provided to you under the Apache License, +## Version 2.0 (the "License"); you may not use this file +## except in compliance with the License. You may obtain +## a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, +## software distributed under the License is distributed on an +## "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +## KIND, either express or implied. See the License for the +## specific language governing permissions and limitations +## under the License. +## +## --------------------------------------------------------------------- + +## Example usage: +# +# #!/bin/sh +# +# # Load the functions +# . path/to/app_epath.sh +# +# # Build the path info +# epaths=`make_app_epaths path/to/app.config` +# +# # View all of the settings. Quotes are important. +# echo "$epaths" +# +# # Grep around in the paths for various items. +# echo "$epaths" | grep 'riak_core ring_creation_size' +# echo "$epaths" | grep "lager handlers lager_file_backend" | grep info +# +# # Use the epath function to get directly at settings +# epath 'riak_core ring_creation_size' "$epaths" +# epath 'riak_core platform_bin_dir' "$epaths" +# epath 'riak_kv storage_backend' "$epaths" +# +# # Use epath to view all of the riak_core settings +# epath riak_core +# +## End example + +## Here is a command you can put in your path for general cli use. +# +# #!/bin/sh +# +# # $1 quoted eterm path for search: 'riak_core ring_creation_size' +# # $2 path/to/app.config +# +# . path/to/app_epath.sh +# +# epaths=`make_app_epaths "$2"` +# epath "$1" "$epaths" +# +## End epath command + +# make_app_epaths takes a path to an app.config file as its argument and +# and returns (prints) a flattened text structure of configuration settings. + +make_app_epaths () { + # Remove all lines containing comments + # Remove all blank lines + # Remove the first [ + # Remove the last ]. + # Remove all blank lines again (just in case) + appconfig=`cat $1 \ + | sed '/^[ \t]*%/d' \ + | sed '/^[ \t]*$/d' \ + | sed -e '/\[/{s///;:a' -e '$!N;$!ba' -e '}' \ + | sed '$ s/\]\.//g' \ + | sed '/^[ \t]*$/d'` + + STACK= + INQUOTE=0 + # The awk puts each char, spaces included, on their own line for parsing. + echo "$appconfig" | awk '{gsub(//,"\n");print}' | while read i; do + case "x${i}x" in + "x\"x") + # Flip the INQUOTE state and echo the quote + if [ 1 -eq $INQUOTE ]; then + INQUOTE=0 + else + INQUOTE=1 + fi + STACK="${STACK}${i}" + ;; + "xx") + if [ 1 -eq $INQUOTE ]; then + # If in quotes, keep this space + STACK="${STACK} " + fi + ;; + "x,x") + if [ 1 -eq $INQUOTE ]; then + # If in quotes, keep this comma + STACK="${STACK}," + else + # Commas outside quotes means a new item is next + STACK="${STACK} " + fi + ;; + "x{x") + if [ 1 -eq $INQUOTE ]; then + # If in quotes, keep this bracket + STACK="${STACK}{" + else + # Add brace to the stack; will pop off from here on next } + STACK="${STACK} __EBRACE__ " + fi + ;; + "x}x") + if [ 1 -eq $INQUOTE ]; then + # If in quotes, keep this bracket + STACK="${STACK}}" + else + # We're only interested in printing leaves, not + # intermediates like 'riak_core http', which contain lists. + # See if the current stack ends with ] (end of list). + echo $STACK | grep -E "__EBRACKET__$" >/dev/null 2>&1 + if [ 1 -eq $? ]; then + # If not, print the stack without all of the mess. + echo "$STACK" | \ + sed 's/ *__EBRACE__//g;s/ *__EBRACKET__//g;s/^ *//' + fi + # Pop off everything from the last brace on. + STACK=`echo "$STACK" | sed 's/\(.*\) __EBRACE__.*/\1/g'` + fi + ;; + "x[x") + if [ 1 -eq $INQUOTE ]; then + # If in quotes, keep this bracket + STACK="${STACK}[" + else + # Add a placeholder to aid in determining whether or not to + # print. That is, we don't want to print 'riak_core http'. + STACK="${STACK} __EBRACKET__ " + fi + ;; + "x]x") + if [ 1 -eq $INQUOTE ]; then + # If in quotes, keep this bracket + STACK="${STACK}]" + fi + # Don't actually do anything with ], as the starting brackets + # are instead removed with }. + ;; + *) + # Anything else is just pushed. + STACK="${STACK}${i}" + ;; + esac + done +} + +epath () { + # arg1 - a pattern to search for + # arg2 - output of make_app_epaths, passed in quoted + # output - search of arg2 for arg1, trimming arg1 from the beginning + # Note: there may be multiple lines of output. + pat=$1 + shift + echo "$*" | grep "$pat " | sed "s/^${pat} *//" +} + diff --git a/rel/files/riak b/rel/files/riak index 06f6fc29f..70fef3db6 100755 --- a/rel/files/riak +++ b/rel/files/riak @@ -4,6 +4,7 @@ RUNNER_GEN_DIR="${RUNNER_GEN_DIR:-{{ platform_gen_dir }}}" RELEASE_ROOT_DIR="${RELEASE_ROOT_DIR:-{{ runner_base_dir }}}" PID_DIR={{pid_dir}} COMMAND={{platform_bin_dir}}/riak +DEBUG_COMMAND={{platform_bin_dir}}/riak-debug RUNNER_LOG_DIR={{platform_log_dir}} RELX_CONFIG_PATH=${RUNNER_GEN_DIR}/sys.config VMARGS_PATH=${RUNNER_GEN_DIR}/vm.args @@ -40,6 +41,12 @@ else start|console|foreground) su - riak -c "RELX_CONFIG_PATH=${RELX_CONFIG_PATH} VMARGS_PATH=${VMARGS_PATH} RUNNER_LOG_DIR=${RUNNER_LOG_DIR} PIPE_DIR=${PIPE_DIR} ${COMMAND} ${*} -pa {{platform_lib_dir}}/patches" ;; + debug) + # Drop the "debug" from the args as we're going to directly call riak-debug now + shift + # Debug may fail if run via relx script due to use of relative start location, and also need for root access + RELX_CONFIG_PATH=${RELX_CONFIG_PATH} VMARGS_PATH=${VMARGS_PATH} RUNNER_LOG_DIR=${RUNNER_LOG_DIR} PIPE_DIR=${PIPE_DIR} ${DEBUG_COMMAND} ${*} + ;; *) ESCAPED_ARGS=`echo "$@" | sed -e 's/\([\\\(\\\){}"\x27]\)/\\\\\1/g'` su - riak -c "RELX_CONFIG_PATH=${RELX_CONFIG_PATH} VMARGS_PATH=${VMARGS_PATH} RUNNER_LOG_DIR=${RUNNER_LOG_DIR} PIPE_DIR=${PIPE_DIR} ${COMMAND} ${ESCAPED_ARGS}" diff --git a/rel/files/riak-debug b/rel/files/riak-debug index b64bfe4d4..24b3e4ce9 100755 --- a/rel/files/riak-debug +++ b/rel/files/riak-debug @@ -23,7 +23,7 @@ ## ------------------------------------------------------------------- # If you start to think "We should execute some Erlang in here", then go work -# on Riaknostic, which is called with `riak-admin diag` below. +# on Riaknostic, which is called with `riak admin diag` below. # /bin/sh on Solaris is not a POSIX compatible shell, but /usr/bin/ksh is. if [ `uname -s` = 'SunOS' -a "${POSIX_SHELL}" != "true" ]; then @@ -140,6 +140,7 @@ RUNNER_BASE_DIR={{runner_base_dir}} riak_base_dir={{runner_base_dir}} riak_bin_dir={{runner_script_dir}} riak_etc_dir={{runner_etc_dir}} +riak_sbin_dir=/usr/sbin get_cfgs=0 get_ssl_certs=0 @@ -153,7 +154,7 @@ verbose_output=0 ## Use riak config generate to set $riak_app_config and ## $riak_vm_args -gen_result=`"$riak_bin_dir"/riak config generate | cut -d' ' -f 3,5` +gen_result=`"$riak_bin_dir"/riak chkconfig | grep -v "OK" | cut -d' ' -f 2,4` riak_app_config=`echo $gen_result | cut -d' ' -f 1` generated_config_dir=`dirname "$riak_app_config"` riak_vm_args=`echo $gen_result | cut -d' ' -f 2` @@ -467,27 +468,32 @@ if [ 1 -eq $get_riakcmds ]; then mkdir_or_die "${start_dir}"/"${debug_dir}"/commands/.info cd "${start_dir}"/"${debug_dir}"/commands - # Note that 'riak-admin status' and 'riak-admin transfers' are heavy + # Note that 'riak admin status' and 'riak admin transfers' are heavy # commands on Riak<=1.2.0 and should not be executed in a loop against all # nodes in a cluster. - dump riak_ping "$riak_bin_dir"/riak ping - dump riak_version "$riak_bin_dir"/riak version - dump riak_member_status "$riak_bin_dir"/riak-admin member-status - dump riak_ring_status "$riak_bin_dir"/riak-admin ring-status - dump riak_status "$riak_bin_dir"/riak-admin status - dump riak_transfers "$riak_bin_dir"/riak-admin transfers - dump riak_aae_status "$riak_bin_dir"/riak-admin aae-status - dump riak_search_aae_status "$riak_bin_dir"/riak-admin search aae-status - dump riak_diag "$riak_bin_dir"/riak-admin diag - dump riak_repl_status "$riak_bin_dir"/riak-repl status - dump riak_repl_connections "$riak_bin_dir"/riak-repl connections - dump riak_repl_clusterstats "$riak_bin_dir"/riak-repl clusterstats - dump riak_repl_modes "$riak_bin_dir"/riak-repl modes + dump riak_ping "$riak_sbin_dir"/riak ping + dump riak_version "$riak_sbin_dir"/riak versions + dump riak_member_status "$riak_sbin_dir"/riak admin member-status + dump riak_ring_status "$riak_sbin_dir"/riak admin ring-status + dump riak_status "$riak_sbin_dir"/riak admin status + dump riak_transfers "$riak_sbin_dir"/riak admin transfers + dump riak_diag "$riak_sbin_dir"/riak admin diag + dump riak_repl_status "$riak_sbin_dir"/riak repl status + dump riak_repl_connections "$riak_sbin_dir"/riak repl connections + dump riak_repl_clusterstats "$riak_sbin_dir"/riak repl clusterstats + dump riak_repl_modes "$riak_sbin_dir"/riak repl modes # Make a flat, easily searchable version of the app.config settings riak_epaths=`make_app_epaths "${riak_app_config}"` + # If Yokozuna is not installed, $yz_test will be null. + yz_test="`epath 'yokozuna root_dir' "$riak_epaths" | sed -e 's/^\"//' -e 's/\".*$//'`" + # Only execute the search aae_statis command if $yz_test is not null. + if [ -n "$yz_test" ]; then + dump riak_search_aae_status "$riak_sbin_dir"/riak admin search aae-status + fi + # Get one http listener (epath might output a newline-separated list). riak_api_http="`epath 'riak_api http' "$riak_epaths" | sed -e 's/^\"//' -e 's/\" /:/' | head -n1`" @@ -532,55 +538,59 @@ if [ 1 -eq $get_yzcmds ]; then [ -z "$riak_epaths" ] && riak_epaths=`make_app_epaths "${riak_app_config}"` yz_dir="`epath 'yokozuna root_dir' "$riak_epaths" | sed -e 's/^\"//' -e 's/\".*$//'`" - if [ '/' != `echo "$yz_dir" | cut -c1` ]; then - # relative path. prepend base dir - yz_dir="$riak_base_dir"/"$yz_dir" - fi + # If Yokozuna is not installed, $yz_dir will be null and cause errors. + # Only execute the Yokozuna commands if $yz_dir is not null. + if [ -n "$yz_dir" ]; then + if [ '/' != `echo "$yz_dir" | cut -c1` ]; then + # relative path. prepend base dir + yz_dir="$riak_base_dir"/"$yz_dir" + fi - yz_aae_dir="`epath 'yokozuna anti_entropy_data_dir' "$riak_epaths" | sed -e 's/^\"//' -e 's/\".*$//'`" - if [ '/' != `echo "$yz_aae_dir" | cut -c1` ]; then - # relative path. prepend base dir - yz_aae_dir="$riak_base_dir"/"$yz_aae_dir" - fi + yz_aae_dir="`epath 'yokozuna anti_entropy_data_dir' "$riak_epaths" | sed -e 's/^\"//' -e 's/\".*$//'`" + if [ '/' != `echo "$yz_aae_dir" | cut -c1` ]; then + # relative path. prepend base dir + yz_aae_dir="$riak_base_dir"/"$yz_aae_dir" + fi - if [ -d "$yz_dir" ]; then - mkdir_or_die "${start_dir}"/"${debug_dir}"/yokozuna/.info - cd "${start_dir}"/"${debug_dir}"/yokozuna + if [ -d "$yz_dir" ]; then + mkdir_or_die "${start_dir}"/"${debug_dir}"/yokozuna/.info + cd "${start_dir}"/"${debug_dir}"/yokozuna - # grab a listing of the yokozuna directory - dump ls_yokozuna_dir ls -lhR "$yz_dir" + # grab a listing of the yokozuna directory + dump ls_yokozuna_dir ls -lhR "$yz_dir" - # Take a du listing of the yokozuna directory for size checking. - # This info is included in the ls, but parsing ls is not recommended. - dump du_yokozuna_dir du "$yz_dir"/* - - # tar up every /conf directory. This will assure we capture the - # schema (regardless of name/extension). - cd "$yz_dir" - for f in `ls -1 .`; do - if [ -d "$f" -a -d "$f"/conf ]; then - tar -czf ""${start_dir}"/"${debug_dir}"/yokozuna/${f}_conf.tar.gz" "$f"/conf - fi - done - fi + # Take a du listing of the yokozuna directory for size checking. + # This info is included in the ls, but parsing ls is not recommended. + dump du_yokozuna_dir du "$yz_dir"/* - if [ -d "$yz_aae_dir" ]; then - mkdir_or_die "${start_dir}"/"${debug_dir}"/yokozuna/anti_entropy/.info - cd "${start_dir}"/"${debug_dir}"/yokozuna/anti_entropy + # tar up every /conf directory. This will assure we capture the + # schema (regardless of name/extension). + cd "$yz_dir" + for f in `ls -1 .`; do + if [ -d "$f" -a -d "$f"/conf ]; then + tar -czf ""${start_dir}"/"${debug_dir}"/yokozuna/${f}_conf.tar.gz" "$f"/conf + fi + done + fi - # Take a listing of the yz_aae_dir directory for reference - dump ls_yokozuna_anti_entropy_dir ls -lhR "$yz_aae_dir" + if [ -d "$yz_aae_dir" ]; then + mkdir_or_die "${start_dir}"/"${debug_dir}"/yokozuna/anti_entropy/.info + cd "${start_dir}"/"${debug_dir}"/yokozuna/anti_entropy - # Take a du listing of the yz_aae_dir directory for size checking. - # This info is included in the ls, but parsing ls is not recommended. - dump du_yokozuna_anti_entropy_dir du "$yz_aae_dir"/* + # Take a listing of the yz_aae_dir directory for reference + dump ls_yokozuna_anti_entropy_dir ls -lhR "$yz_aae_dir" - # Mirror the directory, only copying files that match the pattern - cd "$yz_aae_dir" - find . -type f -name 'LOG*' -exec sh -c ' - mkdir -p "$0/${1%/*}"; - cp "$1" "$0/$1" - ' "${start_dir}"/"${debug_dir}"/yokozuna/anti_entropy {} \; + # Take a du listing of the yz_aae_dir directory for size checking. + # This info is included in the ls, but parsing ls is not recommended. + dump du_yokozuna_anti_entropy_dir du "$yz_aae_dir"/* + + # Mirror the directory, only copying files that match the pattern + cd "$yz_aae_dir" + find . -type f -name 'LOG*' -exec sh -c ' + mkdir -p "$0/${1%/*}"; + cp "$1" "$0/$1" + ' "${start_dir}"/"${debug_dir}"/yokozuna/anti_entropy {} \; + fi fi fi @@ -591,7 +601,7 @@ fi if [ 1 -eq $get_extracmds ]; then mkdir_or_die "${start_dir}"/"${debug_dir}"/commands/.info cd "${start_dir}"/"${debug_dir}"/commands - dump riak_vnode_status "$riak_bin_dir"/riak-admin vnode-status + dump riak_vnode_status "$riak_sbin_dir"/riak admin vnode-status fi ### @@ -716,6 +726,30 @@ if [ 1 -eq $get_logs ]; then # This info is included in the ls, but parsing ls is not recommended. dump du_bitcask_dir du "$bitcask_dir"/* + elif [ 'riak_kv_leveled_backend' = "$backend" ]; then + leveled_dir="`epath 'leveled data_root' "$riak_epaths" | sed -e 's/^\"//' -e 's/\".*$//'`" + + if [ '/' != `echo "$bitcask_dir" | cut -c1` ]; then + # relative path. prepend base dir + leveled_dir="$riak_base_dir"/"$leveled_dir" + fi + + if [ ! -d "$bitcask_dir" ]; then + echoerr "Unable to locate Leveled data directory. Aborting." + echoerr "Using leveled data_root: $leveled_dir" + exit 1 + fi + + mkdir_or_die "${start_dir}"/"${debug_dir}"/logs/leveled/.info + cd "${start_dir}"/"${debug_dir}"/logs/leveled + + # Take a listing of the bitcask directory for reference + dump ls_leveled_dir ls -lhR "$leveled_dir" + + # Take a du listing of the bitcask directory for size checking. + # This info is included in the ls, but parsing ls is not recommended. + dump du_leveled_dir du "$leveled_dir"/* + # Walk multi-backends and collect whatever information is there to collect. elif [ 'riak_kv_multi_backend' = "$backend" ] || [ 'riak_cs_kv_multi_backend' = "$backend" ] ; then @@ -777,6 +811,30 @@ if [ 1 -eq $get_logs ]; then # Take a du listing of the bitcask directory for size checking. # This info is included in the ls, but parsing ls is not recommended. dump du_bitcask_dir du "$dr"/* + elif [ 'riak_kv_leveled_backend' = "$backend" ]; then + dr="`epath "riak_kv multi_backend $b riak_kv_leveled_backend data_root" "$riak_epaths" | + sed -e 's/^\"//' -e 's/\".*$//'`" + + if [ '/' != `echo "$dr" | cut -c1` ]; then + # relative path. prepend base dir + dr="$riak_base_dir"/"$dr" + fi + + if [ ! -d "${dr}" ]; then + echoerr "Unable to locate $b Leveled data directory. Aborting." + echoerr "Using riak_kv_leveled_backend data_root: $dr" + exit 1 + fi + + mkdir_or_die "${start_dir}"/"${debug_dir}"/logs/leveled-${b}/.info + cd "${start_dir}"/"${debug_dir}"/logs/leveled-${b} + + # Take a listing of the bitcask directory for reference + dump ls_leveled_dir ls -lhR "$dr" + + # Take a du listing of the bitcask directory for size checking. + # This info is included in the ls, but parsing ls is not recommended. + dump du_leveled_dir du "$dr"/* fi done fi @@ -811,27 +869,57 @@ if [ 1 -eq $get_logs ]; then cp "$1" "$0/$1" ' "${start_dir}"/"${debug_dir}"/logs/anti_entropy {} \; fi + # Gather AAE logs, listing, sizing information, etc.. + # Calling `epath 'riak_kv tictacaae' "$riak_epaths. . .` will result in + # an empty variable regardless of the value set in the configuration + # settings. We're going to grab the `tictacaae_dataroot` instead, and + # assume it's presence on disk indicates activity. + tictacaae_dir="`epath 'riak_kv tictacaae_dataroot' "$riak_epaths" | sed -e 's/^\"//' -e 's/\".*$//'`" + + if [ '/' != `echo "$tictacaae_dir" | cut -c1` ]; then + # relative path. prepend base dir + tictacaae_dir="$riak_base_dir"/"$tictacaae_dir" + fi + + if [ -d "$tictacaae_dir" ]; then + mkdir_or_die "${start_dir}"/"${debug_dir}"/logs/tictacaae/.info + cd "${start_dir}"/"${debug_dir}"/logs/tictacaae + + # Take a listing of the anti_entropy_dir directory for reference + dump ls_tictacaae_dir ls -lhR "$tictacaae_dir" + + # Take a du listing of the anti_entropy_dir directory for size checking. + # This info is included in the ls, but parsing ls is not recommended. + dump du_tictacaae_dir du "$tictacaae_dir"/* + + # Mirror the directory, only copying files that match the pattern + cd "$tictacaae_dir" + find . -type f -name 'LOG*' -exec sh -c ' + mkdir -p "$0/${1%/*}"; + cp "$1" "$0/$1" + ' "${start_dir}"/"${debug_dir}"/logs/tictacaae {} \; + fi fi ### -### Gather Riak's basho-patches directory +### Gather Riak's patches directory ### if [ 1 -eq $get_patches ]; then - mkdir_or_die "${start_dir}"/"${debug_dir}"/basho-patches/.info - cd "${start_dir}"/"${debug_dir}"/basho-patches + mkdir_or_die "${start_dir}"/"${debug_dir}"/patches/.info + cd "${start_dir}"/"${debug_dir}"/patches # As per the below link, this patch should be based off the base_dir. # http://docs.basho.com/riakee/latest/cookbooks/Rolling-Upgrade-to-Enterprise/#Basho-Patches - riak_lib_dir="${riak_base_dir}/lib/basho-patches" + riak_lib_dir="${riak_base_dir}/lib/patches" # Use dump to execute the copy. This will provide a progress dot and # capture any error messages. - dump cp_basho_patches cp -R "$riak_lib_dir"/* . + dump cp_patches cp -R "$riak_lib_dir"/* . # If the copy succeeded, then the output will be empty and it is unneeded. if [ 0 -eq $? ]; then - rm -f cp_basho_patches + rm -f cp_patches fi fi diff --git a/rel/pkg/rpm/specfile b/rel/pkg/rpm/specfile index 05ab83604..58a3a3d1f 100644 --- a/rel/pkg/rpm/specfile +++ b/rel/pkg/rpm/specfile @@ -122,7 +122,6 @@ fi %post # Post Installation Script - # For distros with SELinux (RHEL/Fedora) if [ `which selinuxenabled > /dev/null 2>&1` ] ; then # Fixup perms for SELinux (if it is enabled) From 08f8bef792adf5e0201b960a2b7f6389479a6e82 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Tue, 22 Dec 2020 16:54:40 +0000 Subject: [PATCH 08/47] Update deps for release --- rebar.config | 6 +++--- rebar.lock | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/rebar.config b/rebar.config index 9304efd8a..c1d5b2657 100644 --- a/rebar.config +++ b/rebar.config @@ -11,13 +11,13 @@ {deps, [ {lager_syslog, {git, "git://github.com/basho/lager_syslog.git", {tag, "3.1.0"}}}, {cluster_info, {git, "git://github.com/basho/cluster_info.git", {tag, "2.1.0"}}}, - {riak_kv, {git, "git://github.com/basho/riak_kv.git", {branch, "develop-3.0"}}}, + {riak_kv, {git, "git://github.com/basho/riak_kv.git", {tag, "riak_kv-3.0.2"}}}, {riak_auth_mods, {git, "git://github.com/basho/riak_auth_mods.git", {tag, "riak_kv-3.0.0"}}}, - {riaknostic, {git, "git://github.com/basho/riaknostic.git", {branch, "develop-3.0"}}}, + {riaknostic, {git, "git://github.com/basho/riaknostic.git", {tag, "riak_kv-3.0.2"}}}, % {yokozuna, {git, "git://github.com/basho/yokozuna.git", {branch, "develop-3.0"}}}, - {riak_repl, {git, "git://github.com/basho/riak_repl.git", {branch, "develop-3.0"}}} + {riak_repl, {git, "git://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.2"}}} ]}. {project_plugins, [ diff --git a/rebar.lock b/rebar.lock index cf8c11a05..6fc2f5f40 100644 --- a/rebar.lock +++ b/rebar.lock @@ -53,7 +53,7 @@ 2}, {<<"kv_index_tictactree">>, {git,"https://github.com/martinsumner/kv_index_tictactree.git", - {ref,"81ecb0c5082e2246d14592c7649513fb89c39d82"}}, + {ref,"e1f580a908014bdc2461198dd8f5e8012ec152c1"}}, 1}, {<<"lager">>, {git,"git://github.com/erlang-lager/lager.git", @@ -65,7 +65,7 @@ 0}, {<<"leveled">>, {git,"https://github.com/martinsumner/leveled.git", - {ref,"182395ee00665bc8869bd9f3c6b6ebb3b65fbce6"}}, + {ref,"142efbcee69ddaf3b70d837b15936e13c8332767"}}, 1}, {<<"lz4">>, {git,"https://github.com/martinsumner/erlang-lz4", @@ -106,7 +106,7 @@ 1}, {<<"riak_api">>, {git,"git://github.com/basho/riak_api.git", - {ref,"4f68783d7669fc165dbc98edfc97d8ece7fcf356"}}, + {ref,"ad40275a6a90e4f3d3c59331bdedb5e9860e1ca5"}}, 1}, {<<"riak_auth_mods">>, {git,"git://github.com/basho/riak_auth_mods.git", @@ -126,7 +126,7 @@ 2}, {<<"riak_kv">>, {git,"git://github.com/basho/riak_kv.git", - {ref,"75e142ef5a9d02cbcd8f84d409817e86197b2a64"}}, + {ref,"3c144e57f60e37b91af48bf6598f11056de65319"}}, 0}, {<<"riak_pb">>, {git,"git://github.com/basho/riak_pb.git", @@ -134,11 +134,11 @@ 2}, {<<"riak_pipe">>, {git,"git://github.com/basho/riak_pipe.git", - {ref,"811ae70fc258e4c3e8d65ff0593ea06989b6dec1"}}, + {ref,"d340aee6c5a9e4a84138275ae07e15938ff75653"}}, 1}, {<<"riak_repl">>, {git,"git://github.com/basho/riak_repl.git", - {ref,"64cbbcaca3c97929d8f543d8f104a935600d433c"}}, + {ref,"4ab6084fd58a514baa021e4e9e82af097ba8f746"}}, 0}, {<<"riak_sysmon">>, {git,"git://github.com/basho/riak_sysmon.git", @@ -146,11 +146,11 @@ 2}, {<<"riakc">>, {git,"git://github.com/basho/riak-erlang-client", - {ref,"a17ad08019ce7564fce8017a8d758aa486c97b25"}}, + {ref,"e51b233a43cfe606bbab9b0e2881609f6d846f0e"}}, 2}, {<<"riakhttpc">>, {git,"git://github.com/basho/riak-erlang-http-client", - {ref,"15174fa6631d66f11c0a31050df5e6ab34b9b69b"}}, + {ref,"c97355baab13c495b75e8d9d8666cde597c28d92"}}, 1}, {<<"riaknostic">>, {git,"git://github.com/basho/riaknostic.git", From 298b4ba19589eac22ccb524973f3aad12e7996cc Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Mon, 4 Jan 2021 10:54:18 +0000 Subject: [PATCH 09/47] Release notes for 3.0.2 added --- RELEASE-NOTES.md | 52 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 9b5c52493..23e58149e 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,4 +1,19 @@ -# Riak KV 3.0 Release Notes +# Riak KV 3.0.2 Release Notes + +There are four broad changes made in Release 3.0.2: + +- Inclusion of backend fixes released within 2.9.8. + +- The addition of the [`range_check` in the Tictac AAE based full-sync replication](https://github.com/basho/riak_kv/blob/develop-3.0/docs/NextGenREPL-GettingStarted.md#configure-full-sync-replication), when replicating between entire clusters. This, along with the backend performance improvements delivered in 2.9.8, significantly improve the stability of Riak clusters when resolving large deltas. + +- A number of issues with command-line functions and packaging related to the switch from `node_package` to `relx` have now been resolved. + +- Riak tombstones, [empty objects used by Riak to replace deleted objects](https://riak.com/posts/technical/riaks-config-behaviors-part-3/index.html), will now have a last_modified_date added to the metadata, although this will only be visible internally within Riak. + +This release is tested with OTP 20, OTP 21 and OTP 22; but optimal performance is likely to be achieved when using OTP 22. + + +# Riak KV 3.0.1 Release Notes This major release allows Riak to run on OTP versions 20, 21 and 22 - but is not fully backwards-compatible with previous releases. Some limitations and key changes should be noted: @@ -16,6 +31,29 @@ This major release allows Riak to run on OTP versions 20, 21 and 22 - but is not Other than the limitations listed above, the release should be functionally identical to Riak KV 2.9.7. Throughput improvements may be seen as a result of the OTP 20 upgrade on some CPU-bound workloads. For disk-bound workloads, additional benefit may be achieved by upgrading further to OTP 22. +# Riak KV 2.9.8 Release Notes + +This release improves the performance and stability of the leveled backend and of AAE folds. These performance improvements are based on feedback from deployments with > 1bn keys per cluster. + +The particular improvements are: + +- In leveled, caching of individual file scores so not all files are required to be scored each journal compaction run. + +- In leveled, a change to the default journal compaction scoring percentages to make longer runs more likely (i.e. achieve more compaction per scoring run). + +- In leveled, a change to the caching of SST file block-index in the ledger, that makes repeated folds with a last modified date range an order of magnitude faster through improved computational efficiency. + +- In leveled, a fix to prevent very long list-buckets queries when buckets have just been deleted (by erasing all keys). + +- In kv_index_tictcatree, improved logging and exchange controls to make exchanges easier to monitor and less likely to prompt unnecessary work. + +- In kv_index_tictcatree, a change to speed-up the necessary rebuilds of aae tree-caches following a node crash, by only testing journal presence in scheduled rebuilds. + +- In riak_kv_ttaaefs_manager, some essential fixes to prevent excessive CPU load when comparing large volumes of keys and clocks, due to a failure to decode clocks correctly before passing to the exchange. + +Further significant improvements have been made to Tictac AAE full-sync, to greatly improve the efficiency of operation when there exists relatively large deltas between relatively large clusters (in terms of key counts). Those changes, which introduce the use of 'day_check', 'hour_check' and 'range_check' options to nval-based full-sync will be available in a future 3.0.2 release of Riak. For those wishing to use Tictac AAE full-sync at a non-trivial scale, it is recommended moving straight to 3.0.2 when it is available. + + # Riak KV 2.9.7 Release Notes This release improves the stability of Riak when running with Tictac AAE in parallel mode: @@ -34,20 +72,24 @@ This release improves the stability of Riak when running with Tictac AAE in para The system is now stable under specific load tests designed to trigger AAE failure. However, parallel mode should still not be used in production systems unless it has been subject to environment-specific load testing. + # Riak KV 2.9.6 Release Notes Withdrawn. + # Riak KV 2.9.5 Release Notes Withdrawn. + # Riak KV 2.9.4 Release Notes This release replaces the Riak KV 2.9.3 release, extending the issue resolution in kv_index_tictactree to detect other files where file truncation means the CRC is not present. This release has a key [outstanding issue](https://github.com/basho/riak_kv/issues/1765) when Tictac AAE is used in parallel mode. On larger clusters, this has been seen to cause significant issues, and so this feature should not be used other than in native mode. + # Riak KV 2.9.3 Release Notes - NOT TO BE RELEASED This release is focused on fixing a number of non-critical issues: @@ -75,6 +117,7 @@ This release includes: - An improvement to the efficiency of [compaction in the leveled LSM-tree based ledger](https://github.com/martinsumner/leveled/issues/311) with large numbers of tombstones (or modified index entries), by using a `grooming` selection strategy 50% of the time when selecting files to merge rather than selecting files at random each time. The `grooming` selection, will take a sample of files and merge the one with the most tombstones. The use of the grooming strategy is not configurable, and will have no impact until the vast majority of SST files have been re-written under this release. + # Riak KV 2.9.1 Release Notes This release adds a number of features built on top of the Tictac AAE feature made available in 2.9.0. The new features depend on Tictac AAE being enabled, but are backend independent. The primary features of the release are: @@ -89,6 +132,7 @@ This release adds a number of features built on top of the Tictac AAE feature ma Detail of volume testing related to the replication uplift can be found [here](https://github.com/martinsumner/riak_testing_notes/blob/master/Release%202.9.1%20-%20Volume%20Tests.md). + # Riak KV 2.9.0 Release Notes - Patch 5 This patch release is primarily required to improve two scenarios where leveled's handling of file-system corruption was unsatisfactory. One scenario related to [corrupted blocks that fail CRC checks](https://github.com/martinsumner/leveled/issues/298), another related to failing to consistently check that [a file write had been flushed to disk](https://github.com/martinsumner/leveled/issues/301). @@ -103,6 +147,7 @@ The patch changes the capability request for `HEAD` to be bucket-dependent, so t Finally, the aae_fold features of 2.9.0 were previously only available via the HTTP API. Such folds can now also be [requested via the PB API](https://github.com/basho/riak_kv/pull/1732). + # Riak KV 2.9.0 Release Notes - Patch 4 There are a number of fixes in this patch: @@ -119,6 +164,7 @@ There are a number of fixes in this patch: It is recommended that any 2.9.0 installations be upgraded to include this path, although if Tictac AAE is not used there is no immediate urgency to making the change. + # Riak KV 2.9.0 Release Notes - Patch 3 An [issue](https://github.com/martinsumner/leveled/issues/287) was discovered in leveled, whereby following a restart of Riak and a workload of fetch requests, the backend demanded excess amounts of binary heap references. Underlying was an issue with the use of sub-binary references during the lazy load of slot header information after a SST file process restart. This has been resolved, and with [greater control added](https://github.com/martinsumner/leveled/blob/0.9.18/priv/leveled.schema#L86-L93) to force the ledger contents into the page cache at startup. @@ -127,18 +173,22 @@ A further [issue](https://github.com/martinsumner/leveled/issues/289) was discov The issues resolved in this patch impact only the use of leveled backend, either directly or via the use of Tictac AAE. + # Riak KV 2.9.0 Release Notes - Patch 2 An [issue](https://github.com/martinsumner/leveled/issues/285) with leveled holding references to binaries what could cause severe memory depletion, when a consecutive series of very large objects are received by a vnode. + # Riak KV 2.9.0 Release Notes - Patch 1 An [issue](https://github.com/basho/riak_kv/issues/1699) was discovered whereby leveled would leak file descriptors under heavy write pressure (e.g. handoffs). + # Riak KV 2.9.0 Release Notes See [here for notes on 2.9.0](doc/Release%202.9%20Series%20-%20Overview.md) + # Riak KV 2.2.5 Release Notes > This release is dedicated to the memory of Andy Gross. Thank you and RIP. From 5b0114f24818b38cd45ecbd53651bb01bf9e2e83 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Mon, 4 Jan 2021 11:24:58 +0000 Subject: [PATCH 10/47] Update RELEASE-NOTES.md --- RELEASE-NOTES.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 23e58149e..52f28ae2e 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,14 +1,14 @@ # Riak KV 3.0.2 Release Notes -There are four broad changes made in Release 3.0.2: +There are four changes made in Release 3.0.2: -- Inclusion of backend fixes released within 2.9.8. +- Inclusion of backend fixes introduced in [2.9.8](#riak-kv-298-release-notes). -- The addition of the [`range_check` in the Tictac AAE based full-sync replication](https://github.com/basho/riak_kv/blob/develop-3.0/docs/NextGenREPL-GettingStarted.md#configure-full-sync-replication), when replicating between entire clusters. This, along with the backend performance improvements delivered in 2.9.8, significantly improve the stability of Riak clusters when resolving large deltas. +- The addition of the [`range_check` in the Tictac AAE based full-sync replication]https://github.com/basho/riak_kv/blob/riak_kv-3.0.2/docs/NextGenREPL-GettingStarted.md#configure-full-sync-replication), when replicating between entire clusters. This, along with the backend performance improvements delivered in 2.9.8, can significantly improve the stability of Riak clusters when resolving large deltas. - A number of issues with command-line functions and packaging related to the switch from `node_package` to `relx` have now been resolved. -- Riak tombstones, [empty objects used by Riak to replace deleted objects](https://riak.com/posts/technical/riaks-config-behaviors-part-3/index.html), will now have a last_modified_date added to the metadata, although this will only be visible internally within Riak. +- Riak tombstones, [empty objects used by Riak to replace deleted objects](https://riak.com/posts/technical/riaks-config-behaviors-part-3/index.html), will now have a last_modified_date added to the metadata, although this will not be visible externally via the API. This release is tested with OTP 20, OTP 21 and OTP 22; but optimal performance is likely to be achieved when using OTP 22. From a1bcc0cc5f8c89e618f8052d9f529810a9657edb Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Mon, 4 Jan 2021 11:25:46 +0000 Subject: [PATCH 11/47] Fix broken link --- RELEASE-NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 52f28ae2e..9abe52ed8 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -4,7 +4,7 @@ There are four changes made in Release 3.0.2: - Inclusion of backend fixes introduced in [2.9.8](#riak-kv-298-release-notes). -- The addition of the [`range_check` in the Tictac AAE based full-sync replication]https://github.com/basho/riak_kv/blob/riak_kv-3.0.2/docs/NextGenREPL-GettingStarted.md#configure-full-sync-replication), when replicating between entire clusters. This, along with the backend performance improvements delivered in 2.9.8, can significantly improve the stability of Riak clusters when resolving large deltas. +- The addition of the [`range_check` in the Tictac AAE based full-sync replication](https://github.com/basho/riak_kv/blob/riak_kv-3.0.2/docs/NextGenREPL-GettingStarted.md#configure-full-sync-replication), when replicating between entire clusters. This, along with the backend performance improvements delivered in 2.9.8, can significantly improve the stability of Riak clusters when resolving large deltas. - A number of issues with command-line functions and packaging related to the switch from `node_package` to `relx` have now been resolved. From 95bb2479180c5cd9b8aabb57dfd1b40cc9662709 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Tue, 5 Jan 2021 08:40:00 +0000 Subject: [PATCH 12/47] erl scripts (#1049) * erl scripts Some riak admin scripts require an erlang node to be started, and then connected to the riak node (rather than using rpc). These scripts don't work in 3.0.1. $ERTS_PATH and $NAME_PARAM are not defined (as was previously defined in node_package); boot file - cannot find start.boot only start_clean.boot (for reasons unknown); observer not an installed application These changes work with OTP 20.3, OTP 21.3 and OTP 22.3. Tested using: `make rel` `rel/riak/bin/riak start` `rel/riak/bin/riak admin top` * Remove bespoke top formatter There was a special formatter for etop packaged within riak top. This may have been visually more appealing than the format for etop in OTP16? However, going forward want to take advantage of mainstream OTP improvements not rely on bespoke formatter. The bespoke formatter may not have been working anyway. --- apps/riak/src/etop_txt.erl | 135 ------------------------------------- apps/riak/src/riak.app.src | 1 - rebar.config | 1 + rel/files/riak-admin | 10 +++ 4 files changed, 11 insertions(+), 136 deletions(-) delete mode 100644 apps/riak/src/etop_txt.erl diff --git a/apps/riak/src/etop_txt.erl b/apps/riak/src/etop_txt.erl deleted file mode 100644 index aad011677..000000000 --- a/apps/riak/src/etop_txt.erl +++ /dev/null @@ -1,135 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2002-2009. All Rights Reserved. -%% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. -%% -%% %CopyrightEnd% -%% --module(etop_txt). --author('siri@erix.ericsson.se'). --author('scott@basho.com'). - -%%-compile(export_all). --export([init/1,stop/1]). --export([do_update/3]). - -%%-include("etop.hrl"). --record(etop_proc_info, - {pid, - mem=0, - reds=0, - name, - runtime=0, - cf, - mq=0}). --record(etop_info, - {now = {0, 0, 0}, - n_procs = 0, - wall_clock = {0, 0}, - runtime = {0, 0}, - run_queue = 0, - alloc_areas = [], - memi = [{total, 0}, - {processes, 0}, - {ets, 0}, - {atom, 0}, - {code, 0}, - {binary, 0}], - procinfo = [] - }). -%%-include("etop_defs.hrl"). --define(SYSFORM, - " ~-72w~10s~n" - " Load: cpu ~8w Memory: total ~8w binary ~8w~n" - " procs~8w processes~8w code ~8w~n" - " runq ~8w atom ~8w ets ~8w~n"). --record(opts, {node=node(), port = 8415, accum = false, intv = 5000, lines = 10, - width = 700, height = 340, sort = runtime, tracing = on, - %% Other state information - out_mod=etop_gui, out_proc, server, host, tracer, store, - accum_tab, remote}). - --import(etop,[loadinfo/1,meminfo/2]). --import(etop_gui,[formatmfa/1,to_list/1]). - --define(PROCFORM,"~-20w~-25s~8w~11w~11w~11w ~-40s~n"). - -stop(Pid) -> Pid ! stop. - -init(Config) -> - loop(Config). - -loop(Config) -> - Info = do_update(Config), - receive - stop -> stopped; - {dump,Fd} -> do_update(Fd,Info,Config), loop(Config); - {config,_,Config1} -> loop(Config1) - after Config#opts.intv-500 -> loop(Config) - end. - -do_update(Config) -> - Info = etop:update(Config), - do_update(standard_io,Info,Config). - -do_update(Fd,Info,Config) -> - {Cpu,NProcs,RQ,Clock} = loadinfo(Info), - io:nl(Fd), - writedoubleline(Fd), - case Info#etop_info.memi of - undefined -> - io:fwrite(Fd, " ~-72w~10s~n" - " Load: cpu ~8w~n" - " procs~8w~n" - " runq ~8w~n", - [Config#opts.node,Clock, - Cpu,NProcs,RQ]); - Memi -> - [Tot,Procs,Atom,Bin,Code,Ets] = - meminfo(Memi, [total,processes,atom,binary,code,ets]), - io:fwrite(Fd, ?SYSFORM, - [Config#opts.node,Clock, - Cpu,Tot,Bin, - NProcs,Procs,Code, - RQ,Atom,Ets]) - end, - io:nl(Fd), - writepinfo_header(Fd), - writesingleline(Fd), - writepinfo(Fd,Info#etop_info.procinfo), - %%writedoubleline(Fd), - %%io:nl(Fd), - Info. - -writepinfo_header(Fd) -> - io:fwrite(Fd,"Pid Name or Initial Func Time Reds Memory MsgQ Current Function~n",[]). - -writesingleline(Fd) -> - io:fwrite(Fd,"-------------------------------------------------------------------------------------------------------------------------------~n",[]). -writedoubleline(Fd) -> - io:fwrite(Fd,"===============================================================================================================================~n",[]). - -writepinfo(Fd,[#etop_proc_info{pid=Pid, - mem=Mem, - reds=Reds, - name=Name, - runtime=Time, - cf=MFA, - mq=MQ} - |T]) -> - io:fwrite(Fd,?PROCFORM,[Pid,to_list(Name),Time,Reds,Mem,MQ,formatmfa(MFA)]), - writepinfo(Fd,T); -writepinfo(_Fd,[]) -> - ok. - diff --git a/apps/riak/src/riak.app.src b/apps/riak/src/riak.app.src index 47c3843c6..541e742f1 100644 --- a/apps/riak/src/riak.app.src +++ b/apps/riak/src/riak.app.src @@ -2,7 +2,6 @@ [{description, "The Riak Database"}, {vsn, "3.0"}, {registered, []}, - {modules, [etop_txt]}, {applications, [ kernel, stdlib, diff --git a/rebar.config b/rebar.config index c1d5b2657..8d96c07c0 100644 --- a/rebar.config +++ b/rebar.config @@ -58,6 +58,7 @@ riak_sysmon, os_mon, crypto, + observer, runtime_tools, xmerl, mochiweb, diff --git a/rel/files/riak-admin b/rel/files/riak-admin index 6f4cdc2ce..e82514e51 100755 --- a/rel/files/riak-admin +++ b/rel/files/riak-admin @@ -11,6 +11,11 @@ cd $RUNNER_BASE_DIR # Identify the script name SCRIPT="riak-admin" +# Setup required due to relx not node-package +ERTS_PATH=$BINDIR +NAME_PARAM="-name" +BOOT_FILE="start_clean" + usage() { echo "Usage: $SCRIPT { cluster | join | leave | backup | restore | test | " echo " reip | js-reload | erl-reload | wait-for-service | " @@ -908,6 +913,7 @@ case "$1" in NEWNODE=$2 $ERTS_PATH/erl -noshell \ -pa $RUNNER_LIB_DIR/basho-patches \ + -boot $BOOT_FILE \ $CONFIG_ARGS \ -eval "riak_kv_console:$ACTION(['$OLDNODE', '$NEWNODE'])" \ -s init stop @@ -928,6 +934,7 @@ case "$1" in $ERTS_PATH/erl -noshell $NAME_PARAM riak_kv_backup$NAME_HOST -setcookie $COOKIE \ -pa $RUNNER_LIB_DIR/basho-patches \ + -boot $BOOT_FILE \ -eval "riak_kv_backup:$ACTION('$NODE', \"$FILENAME\")" -s init stop ;; @@ -947,6 +954,7 @@ case "$1" in $ERTS_PATH/erl -noshell $NAME_PARAM riak_kv_backup$NAME_HOST -setcookie $COOKIE \ -pa $RUNNER_LIB_DIR/basho-patches \ + -boot $BOOT_FILE \ -eval "riak_kv_backup:$ACTION('$NODE', \"$FILENAME\", \"$TYPE\")" -s init stop ;; @@ -960,6 +968,7 @@ case "$1" in $ERTS_PATH/erl -noshell $NAME_PARAM riak_test$NAME_HOST $COOKIE_ARG \ -pa $RUNNER_LIB_DIR/basho-patches \ + -boot $BOOT_FILE \ -eval "case catch(riak:client_test(\"$NODE_NAME\")) of \ ok -> init:stop(); \ _ -> init:stop(1) \ @@ -995,6 +1004,7 @@ case "$1" in # Using np_etop instead of riak_etop to follow node_package convention $ERTS_PATH/erl -noshell -noinput \ -pa $RUNNER_LIB_DIR/basho-patches \ + -boot $BOOT_FILE \ -hidden $NAME_PARAM np_etop$RAND$NAME_HOST $COOKIE_ARG \ -s etop -s erlang halt -output text \ -node $NODE_NAME \ From 1ed8f1279b850d24ba63b27e89f2a42d1d103e74 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Wed, 13 Jan 2021 13:43:13 +0000 Subject: [PATCH 13/47] 3.0.3 - ready for release --- eunit.log | 1 + rebar.config | 6 +++--- rebar.lock | 6 +++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/eunit.log b/eunit.log index 89887af3f..2665bc6ca 100644 --- a/eunit.log +++ b/eunit.log @@ -39,6 +39,7 @@ Eunit: riak_repl PASSED Eunit: riak_sysmon PASSED Eunit: riakc PASSED Eunit: riakhttpc PASSED +Eunit: riaknostic PASSED Eunit: setup PASSED Eunit: sext PASSED Eunit: sidejob PASSED diff --git a/rebar.config b/rebar.config index 8d96c07c0..67bc71660 100644 --- a/rebar.config +++ b/rebar.config @@ -11,13 +11,13 @@ {deps, [ {lager_syslog, {git, "git://github.com/basho/lager_syslog.git", {tag, "3.1.0"}}}, {cluster_info, {git, "git://github.com/basho/cluster_info.git", {tag, "2.1.0"}}}, - {riak_kv, {git, "git://github.com/basho/riak_kv.git", {tag, "riak_kv-3.0.2"}}}, + {riak_kv, {git, "git://github.com/basho/riak_kv.git", {tag, "riak_kv-3.0.3"}}}, {riak_auth_mods, {git, "git://github.com/basho/riak_auth_mods.git", {tag, "riak_kv-3.0.0"}}}, {riaknostic, {git, "git://github.com/basho/riaknostic.git", {tag, "riak_kv-3.0.2"}}}, % {yokozuna, {git, "git://github.com/basho/yokozuna.git", {branch, "develop-3.0"}}}, - {riak_repl, {git, "git://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.2"}}} + {riak_repl, {git, "git://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.3"}}} ]}. {project_plugins, [ @@ -46,7 +46,7 @@ ]} ]}. -{relx, [{release, {riak, "3.0.2"}, +{relx, [{release, {riak, "3.0.3"}, [kernel, stdlib, lager, diff --git a/rebar.lock b/rebar.lock index 6fc2f5f40..ec34ef69b 100644 --- a/rebar.lock +++ b/rebar.lock @@ -65,7 +65,7 @@ 0}, {<<"leveled">>, {git,"https://github.com/martinsumner/leveled.git", - {ref,"142efbcee69ddaf3b70d837b15936e13c8332767"}}, + {ref,"71ba64ea53d1a3c410c61838b13d57a82f8b07a3"}}, 1}, {<<"lz4">>, {git,"https://github.com/martinsumner/erlang-lz4", @@ -126,7 +126,7 @@ 2}, {<<"riak_kv">>, {git,"git://github.com/basho/riak_kv.git", - {ref,"3c144e57f60e37b91af48bf6598f11056de65319"}}, + {ref,"5eeec5171ecc44e8cfc94c293facde41b27ddcd9"}}, 0}, {<<"riak_pb">>, {git,"git://github.com/basho/riak_pb.git", @@ -138,7 +138,7 @@ 1}, {<<"riak_repl">>, {git,"git://github.com/basho/riak_repl.git", - {ref,"4ab6084fd58a514baa021e4e9e82af097ba8f746"}}, + {ref,"a664735b09e67c8436db56f1436c697152b0fe43"}}, 0}, {<<"riak_sysmon">>, {git,"git://github.com/basho/riak_sysmon.git", From 18a4b3c65edcf49493a698c4dc1de49620128bf2 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Thu, 14 Jan 2021 12:34:15 +0000 Subject: [PATCH 14/47] 3.0.3 - release notes --- RELEASE-NOTES.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 9abe52ed8..6a059af5c 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,3 +1,14 @@ +# Riak KV 3.0.3 Release Notes + +There are two fixes provided in Release 3.0.3: + +- A [performance issue with OTP 22 and leveled](https://github.com/martinsumner/leveled/issues/326) has been corrected. This generally did not have a significant impact when running Riak, but there were some potential cases with Tictac AAE and AAE Folds where there could have been a noticeable slowdown. + +- An [issue with console commands for bucket types](https://github.com/basho/riak/issues/1043) has now been fully corrected, having been partially mitigated in 3.0.2. + +This release is tested with OTP 20, OTP 21 and OTP 22; but optimal performance is likely to be achieved when using OTP 22. + + # Riak KV 3.0.2 Release Notes There are four changes made in Release 3.0.2: From 583f236da8442946b90312df8225d10543b236af Mon Sep 17 00:00:00 2001 From: Martin Cox Date: Wed, 27 Jan 2021 11:25:15 +0000 Subject: [PATCH 15/47] Pipe json props string to tr to truncate whitespace --- rel/files/riak-admin | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/rel/files/riak-admin b/rel/files/riak-admin index e82514e51..118ab201a 100755 --- a/rel/files/riak-admin +++ b/rel/files/riak-admin @@ -583,12 +583,12 @@ btype_admin() # bucket-type create {"props": { ... }}" # if [ $# -le 3 ]; then - json=${3:-} + json=`echo ${3:-}|tr -d " "` relx_nodetool rpc riak_kv_console bucket_type_create "$2" "$json" else # greater than 3 parameters two=$2 shift 2 - three=$@ + three=`echo $@|tr -d " "` relx_nodetool rpc riak_kv_console bucket_type_create "$two" "$three" fi @@ -604,11 +604,12 @@ btype_admin() # bucket-type update {"props": { ... }}" # if [ $# -eq 3 ]; then - relx_nodetool rpc riak_kv_console bucket_type_update "$2" "$3" + json=`echo $3|tr -d " "` + relx_nodetool rpc riak_kv_console bucket_type_update "$2" "$json" else # greater than 3 parameters two=$2 shift 2 - three=$@ + three=`echo $@|tr -d " "` relx_nodetool rpc riak_kv_console bucket_type_update "$two" "$three" fi ;; From 4db1da79fddc2deaea1276bbdf645bb81f5c215a Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Tue, 23 Mar 2021 15:18:18 +0000 Subject: [PATCH 16/47] Update for 3.0.4 release --- rebar.config | 6 +++--- rebar.lock | 18 +++++++++--------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/rebar.config b/rebar.config index 67bc71660..17c91d229 100644 --- a/rebar.config +++ b/rebar.config @@ -11,13 +11,13 @@ {deps, [ {lager_syslog, {git, "git://github.com/basho/lager_syslog.git", {tag, "3.1.0"}}}, {cluster_info, {git, "git://github.com/basho/cluster_info.git", {tag, "2.1.0"}}}, - {riak_kv, {git, "git://github.com/basho/riak_kv.git", {tag, "riak_kv-3.0.3"}}}, + {riak_kv, {git, "git://github.com/basho/riak_kv.git", {tag, "riak_kv-3.0.4"}}}, {riak_auth_mods, {git, "git://github.com/basho/riak_auth_mods.git", {tag, "riak_kv-3.0.0"}}}, {riaknostic, {git, "git://github.com/basho/riaknostic.git", {tag, "riak_kv-3.0.2"}}}, % {yokozuna, {git, "git://github.com/basho/yokozuna.git", {branch, "develop-3.0"}}}, - {riak_repl, {git, "git://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.3"}}} + {riak_repl, {git, "git://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.4"}}} ]}. {project_plugins, [ @@ -46,7 +46,7 @@ ]} ]}. -{relx, [{release, {riak, "3.0.3"}, +{relx, [{release, {riak, "3.0.4"}, [kernel, stdlib, lager, diff --git a/rebar.lock b/rebar.lock index ec34ef69b..ced458df4 100644 --- a/rebar.lock +++ b/rebar.lock @@ -49,7 +49,7 @@ 1}, {<<"ibrowse">>, {git,"git://github.com/basho/ibrowse.git", - {ref,"aee5b93411b0a853494025005661623b13bf43f4"}}, + {ref,"cdde6fe5e80ce71e7b5570e40a45d9f862721499"}}, 2}, {<<"kv_index_tictactree">>, {git,"https://github.com/martinsumner/kv_index_tictactree.git", @@ -65,7 +65,7 @@ 0}, {<<"leveled">>, {git,"https://github.com/martinsumner/leveled.git", - {ref,"71ba64ea53d1a3c410c61838b13d57a82f8b07a3"}}, + {ref,"489d89fb16af987f0b74fb42dfa6e4f9e6c2f7f1"}}, 1}, {<<"lz4">>, {git,"https://github.com/martinsumner/erlang-lz4", @@ -77,7 +77,7 @@ 4}, {<<"mochiweb">>, {git,"git://github.com/basho/mochiweb.git", - {ref,"fb7bc696357b77fd8075de796d3687dfb5ed0d0b"}}, + {ref,"8471d2197afd412fcec239dd9ad5cbe90e64eefa"}}, 2}, {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.3.0">>},3}, {<<"pbkdf2">>, @@ -106,7 +106,7 @@ 1}, {<<"riak_api">>, {git,"git://github.com/basho/riak_api.git", - {ref,"ad40275a6a90e4f3d3c59331bdedb5e9860e1ca5"}}, + {ref,"41f4b72d6a946c710e1523fd69b5b8deb5b57153"}}, 1}, {<<"riak_auth_mods">>, {git,"git://github.com/basho/riak_auth_mods.git", @@ -126,11 +126,11 @@ 2}, {<<"riak_kv">>, {git,"git://github.com/basho/riak_kv.git", - {ref,"5eeec5171ecc44e8cfc94c293facde41b27ddcd9"}}, + {ref,"aeef1591704d32230b773d952a2f1543cbfa1889"}}, 0}, {<<"riak_pb">>, {git,"git://github.com/basho/riak_pb.git", - {ref,"b8a946426e0254abba2015f9b5c1a9db4e2405bc"}}, + {ref,"db5c228385f9125185a310bacd9d51bcb0fdab5f"}}, 2}, {<<"riak_pipe">>, {git,"git://github.com/basho/riak_pipe.git", @@ -138,7 +138,7 @@ 1}, {<<"riak_repl">>, {git,"git://github.com/basho/riak_repl.git", - {ref,"a664735b09e67c8436db56f1436c697152b0fe43"}}, + {ref,"0e0fd356c99d1074c7fca7dc9b040882ce05f1e0"}}, 0}, {<<"riak_sysmon">>, {git,"git://github.com/basho/riak_sysmon.git", @@ -146,11 +146,11 @@ 2}, {<<"riakc">>, {git,"git://github.com/basho/riak-erlang-client", - {ref,"e51b233a43cfe606bbab9b0e2881609f6d846f0e"}}, + {ref,"ea4fcd062f55a67e31c7294a1b8b8ccd957526c8"}}, 2}, {<<"riakhttpc">>, {git,"git://github.com/basho/riak-erlang-http-client", - {ref,"c97355baab13c495b75e8d9d8666cde597c28d92"}}, + {ref,"51b7a8781db1c9644f4464aeec92621415f8a2e6"}}, 1}, {<<"riaknostic">>, {git,"git://github.com/basho/riaknostic.git", From ca792b027ffe40ebb85e6fd4c0df78beb2cab081 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Wed, 24 Mar 2021 11:02:18 +0000 Subject: [PATCH 17/47] Add 3.0.4 release notes --- RELEASE-NOTES.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 6a059af5c..82b220bf3 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,3 +1,14 @@ +# Riak KV 3.0.4 Release Notes + +There are two fixes provided in Release 3.0.4: + +- An issue with [leveled application dependencies](https://github.com/basho/riak/issues/1059) has been resolved, and so lz4 can now again be used as the compression method. + +- The [riak clients are now compatible with systems that require semantic versioning](https://github.com/basho/riak-erlang-client/issues/400). + +This release is tested with OTP 20, OTP 21 and OTP 22; but optimal performance is likely to be achieved when using OTP 22. + + # Riak KV 3.0.3 Release Notes There are two fixes provided in Release 3.0.3: From 97652c12b365711e6da1f4fc92a12c411d48dc2f Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Fri, 7 May 2021 12:14:34 +0100 Subject: [PATCH 18/47] Mas pr967 location (#1064) * Changes to support location * Tag and release notes for 3.0.5 * Update links and THANKS * Add help to location command Also, update tagging * Typo fix --- RELEASE-NOTES.md | 14 ++++++++++++++ THANKS | 1 + rebar.config | 4 ++-- rebar.lock | 8 ++++---- rel/files/riak-admin | 6 ++++++ 5 files changed, 27 insertions(+), 6 deletions(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 82b220bf3..2e8eab841 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,3 +1,17 @@ +# Riak KV 3.0.6 Release Notes + +Release 3.0.5 adds [location-awareness to Riak cluster management](https://github.com/basho/riak_core/blob/riak_kv-3.0.5/docs/rack-awareness.md). The broad aim is to improve data diversity across locations (e.g. racks) to reduce the probability of data-loss should a set of nodes fail concurrently within a location. The location-awareness does not provide firm guarantees of data diversity that will always be maintained across all cluster changes - but [testing](https://github.com/basho/riak_test/pull/1353) has indicated it will generally find a cluster arrangement which is close to optimal in terms of data protection. + +If location information is not added to the cluster, there will be no change in behaviour from previous releases. + +There may be some performance advantages when using the location-awareness feature in conjunction with the leveled backend when handling GET requests. Location awareness will mean that the cluster will be more likely to be able return an object from within the location in response to the request, without having to fetch that object from another location (only HEAD request/response will commonly be required inter-location). + +This release is tested with OTP 20 and OTP 22; but optimal performance is likely to be achieved when using OTP 22. + +# Riak KV 3.0.5 Release Notes + +Release version skipped due to tagging error + # Riak KV 3.0.4 Release Notes There are two fixes provided in Release 3.0.4: diff --git a/THANKS b/THANKS index 440c68fff..b9a05c502 100644 --- a/THANKS +++ b/THANKS @@ -473,6 +473,7 @@ Peter Garbers Peter Gribanov Peter Lemenkov Peter Teichman +Peter Tihanyi Petr A. Korobeinikov Petr Gotthard Phil Pirozhkov diff --git a/rebar.config b/rebar.config index 17c91d229..f1b1de96a 100644 --- a/rebar.config +++ b/rebar.config @@ -11,7 +11,7 @@ {deps, [ {lager_syslog, {git, "git://github.com/basho/lager_syslog.git", {tag, "3.1.0"}}}, {cluster_info, {git, "git://github.com/basho/cluster_info.git", {tag, "2.1.0"}}}, - {riak_kv, {git, "git://github.com/basho/riak_kv.git", {tag, "riak_kv-3.0.4"}}}, + {riak_kv, {git, "git://github.com/basho/riak_kv.git", {tag, "riak_kv-3.0.6"}}}, {riak_auth_mods, {git, "git://github.com/basho/riak_auth_mods.git", {tag, "riak_kv-3.0.0"}}}, {riaknostic, {git, "git://github.com/basho/riaknostic.git", {tag, "riak_kv-3.0.2"}}}, @@ -46,7 +46,7 @@ ]} ]}. -{relx, [{release, {riak, "3.0.4"}, +{relx, [{release, {riak, "3.0.6"}, [kernel, stdlib, lager, diff --git a/rebar.lock b/rebar.lock index ced458df4..8d819c297 100644 --- a/rebar.lock +++ b/rebar.lock @@ -106,7 +106,7 @@ 1}, {<<"riak_api">>, {git,"git://github.com/basho/riak_api.git", - {ref,"41f4b72d6a946c710e1523fd69b5b8deb5b57153"}}, + {ref,"94489df53296c20a8f5efb2ab3f50b248086f6e4"}}, 1}, {<<"riak_auth_mods">>, {git,"git://github.com/basho/riak_auth_mods.git", @@ -114,7 +114,7 @@ 0}, {<<"riak_core">>, {git,"git://github.com/basho/riak_core.git", - {ref,"e817ee7555dc4a327be0843ef8e6c1c0974226f2"}}, + {ref,"1f7e204548148f7dda9727113838d40b559310e3"}}, 1}, {<<"riak_dt">>, {git,"git://github.com/basho/riak_dt.git", @@ -126,7 +126,7 @@ 2}, {<<"riak_kv">>, {git,"git://github.com/basho/riak_kv.git", - {ref,"aeef1591704d32230b773d952a2f1543cbfa1889"}}, + {ref,"9066933ccdbec7a3f636cb4ef48631f5b7cd6b4b"}}, 0}, {<<"riak_pb">>, {git,"git://github.com/basho/riak_pb.git", @@ -134,7 +134,7 @@ 2}, {<<"riak_pipe">>, {git,"git://github.com/basho/riak_pipe.git", - {ref,"d340aee6c5a9e4a84138275ae07e15938ff75653"}}, + {ref,"8dd2a6352e58a7d8215ba5afad4cf7b9e4807714"}}, 1}, {<<"riak_repl">>, {git,"git://github.com/basho/riak_repl.git", diff --git a/rel/files/riak-admin b/rel/files/riak-admin index 118ab201a..d9e94b18c 100755 --- a/rel/files/riak-admin +++ b/rel/files/riak-admin @@ -295,6 +295,10 @@ cluster_admin() shift relx_nodetool rpc riak_core_console command $SCRIPT cluster partition-count $@ ;; + location) + shift + relx_nodetool rpc riak_core_console command $SCRIPT cluster location $@ + ;; *) echo "\ Usage: $SCRIPT cluster @@ -317,6 +321,8 @@ plan must be committed to take effect: force-replace Reassign all partitions owned by to without first handing off data, and remove from the cluster. + + location Assign a location to the current node Staging commands: plan Display the staged changes to the cluster From ff65b0a2ac63dcb5fca5add95cb1b72bb7e8676a Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Fri, 7 May 2021 14:40:11 +0100 Subject: [PATCH 19/47] Improve readability --- RELEASE-NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 2e8eab841..af2969d42 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -4,7 +4,7 @@ Release 3.0.5 adds [location-awareness to Riak cluster management](https://githu If location information is not added to the cluster, there will be no change in behaviour from previous releases. -There may be some performance advantages when using the location-awareness feature in conjunction with the leveled backend when handling GET requests. Location awareness will mean that the cluster will be more likely to be able return an object from within the location in response to the request, without having to fetch that object from another location (only HEAD request/response will commonly be required inter-location). +There may be some performance advantages when using the location-awareness feature in conjunction with the leveled backend when handling GET requests. With location awareness, when responding to a GET request, a cluster is more likely to be able to fetch an object from a vnode within the location that received the GET request, without having to fetch that object from another location (only HEAD request/response will commonly travel between locations). This release is tested with OTP 20 and OTP 22; but optimal performance is likely to be achieved when using OTP 22. From 96c38699b45155061f0c0224c193a5cad467e7fa Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Thu, 15 Jul 2021 17:53:00 +0100 Subject: [PATCH 20/47] Add pre_start to set as interactive (#1077) Returns riak to pre-3.0 default of interactive mode, by setting a pre-start script. Should be possible to override by setting the environment variable before starting (if one wishes to run embedded). Required for user-added code (e.g. to find user-defined map/reduce functions) --- rebar.config | 13 +++++++++---- rel/files/erl_codeloadingmode | 3 +++ 2 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 rel/files/erl_codeloadingmode diff --git a/rebar.config b/rebar.config index f1b1de96a..fcc29d704 100644 --- a/rebar.config +++ b/rebar.config @@ -94,6 +94,7 @@ {copy, "rel/files/check_ulimit", "bin/hooks/check_ulimit"}, {copy, "rel/files/erl_maxlogsize", "bin/hooks/erl_maxlogsize"}, + {copy, "rel/files/erl_codeloadingmode","bin/hooks/erl_codeloadingmode"}, {copy, "rel/files/riak_not_running", "bin/hooks/riak_not_running"}, {copy, "rel/files/app_epath.sh", "lib/app_epath.sh"} @@ -119,7 +120,8 @@ [{pre_start, [{custom, "hooks/riak_not_running"}, {custom, "hooks/check_ulimit"}, - {custom, "hooks/erl_maxlogsize"}]}, + {custom, "hooks/erl_maxlogsize"}, + {custom, "hooks/erl_codeloadingmode"}]}, {post_start, [{wait_for_process, riak_core_node_watcher}]} ]} @@ -131,7 +133,8 @@ [{pre_start, [{custom, "hooks/riak_not_running"}, {custom, "hooks/check_ulimit"}, - {custom, "hooks/erl_maxlogsize"}]}, + {custom, "hooks/erl_maxlogsize"}, + {custom, "hooks/erl_codeloadingmode"}]}, {post_start, [{wait_for_process, riak_core_node_watcher}]} ]} @@ -148,7 +151,8 @@ [{pre_start, [{custom, "hooks/riak_not_running"}, {custom, "hooks/check_ulimit"}, - {custom, "hooks/erl_maxlogsize"}]}, + {custom, "hooks/erl_maxlogsize"}, + {custom, "hooks/erl_codeloadingmode"}]}, {post_start, [{pid, "/var/run/riak/riak.pid"}, {wait_for_process, riak_core_node_watcher}]} @@ -165,7 +169,8 @@ [{pre_start, [{custom, "hooks/riak_not_running"}, {custom, "hooks/check_ulimit"}, - {custom, "hooks/erl_maxlogsize"}]}, + {custom, "hooks/erl_maxlogsize"}, + {custom, "hooks/erl_codeloadingmode"}]}, {post_start, [wait_for_vm_start, {pid, "/run/riak/riak.pid"}, diff --git a/rel/files/erl_codeloadingmode b/rel/files/erl_codeloadingmode new file mode 100644 index 000000000..b6c8c6716 --- /dev/null +++ b/rel/files/erl_codeloadingmode @@ -0,0 +1,3 @@ +#!/bin/sh +CODE_LOADING_MODE="${CODE_LOADING_MODE:-interactive}" +export CODE_LOADING_MODE From 6062a00cbe7a799761041f527327c5aa09d6498f Mon Sep 17 00:00:00 2001 From: Nicholas Adams Date: Fri, 16 Jul 2021 01:53:19 +0900 Subject: [PATCH 21/47] Update Riak rpm naming to match global standards (#1076) Currently, when building rpm packages, riak-2.x.x names correctly e.g. `riak-2.9.8-1.el8.x86_64.rpm` but the riak-3.x.x rpms are missing a dot before the basearch i.e. `riak-3.0.6-1.el8x86_64.rpm`. This incredibly tiny fix re-adds the missing dot in the package name. I have already tested this successfully on one of the regular package builders we use to publish the binaries by creating a phony tag of riak-3.0.61 and building it accordingly: ``` [root@localhost riak]# ls rel/pkg/out/packages/ riak-3.0.61.OTP20.3-1.el8.src.rpm riak-3.0.61.OTP20.3-1.el8.x86_64.rpm riak-3.0.61.OTP20.3-1.el8.src.rpm.sha riak-3.0.61.OTP20.3-1.el8.x86_64.rpm.sha ``` --- rel/pkg/rpm/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rel/pkg/rpm/Makefile b/rel/pkg/rpm/Makefile index fed36bf7a..da19ab270 100644 --- a/rel/pkg/rpm/Makefile +++ b/rel/pkg/rpm/Makefile @@ -6,7 +6,7 @@ PWD = $(shell pwd) PKG_VERSION_NO_H ?= $(shell echo $(PKG_VERSION) | tr - .) default: - rpmbuild --define "_rpmfilename %%{NAME}-%%{VERSION}-%%{RELEASE}$(DISTRO)%%{ARCH}.rpm" \ + rpmbuild --define "_rpmfilename %%{NAME}-%%{VERSION}-%%{RELEASE}$(DISTRO).%%{ARCH}.rpm" \ --define '_topdir $(BASE_DIR)/rel/pkg/out/' \ --define '_sourcedir $(BASE_DIR)/rel/pkg/out/' \ --define '_specdir $(BASE_DIR)/rel/pkg/out/' \ From 7d786661498e84788a36bf8228d633230243a66c Mon Sep 17 00:00:00 2001 From: James M <35449344+JMercerGit@users.noreply.github.com> Date: Thu, 15 Jul 2021 17:54:21 +0100 Subject: [PATCH 22/47] Update riak admin usage and examples to reflect changes (#1069) * Update riak admin usage and examples to reflect changes Since the format of `riak-admin` was changed to `riak admin`, examples such as the usage prompt from running `riak admin` need to be corrected. * Update riak-admin --- rel/files/riak-admin | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/rel/files/riak-admin b/rel/files/riak-admin index d9e94b18c..7a4188f9d 100755 --- a/rel/files/riak-admin +++ b/rel/files/riak-admin @@ -83,7 +83,7 @@ Usage: $SCRIPT stat The following commands display, enable/disable and reset statistics. A statistics entry is given either as a 'dotted' exometer name - Identifiers separated by periods, '.', e.g. riak.riak_kv.node.gets, -or as a 'legacy' name (same as in riak-admin status) - e.g. node_gets. +or as a 'legacy' name (same as in riak admin status) - e.g. node_gets. When a legacy name is listed, the corresponding exometer name is shown as well. Two kinds of wildcard are suppored: @@ -122,7 +122,7 @@ will match all stats entries. All Riak stat entry names start with 'riak', so \`stat show riak.**\` will match all riak stat entries. Example: -\$ bin/riak-admin stat show riak.riak_kv.node.gets +\$ bin/riak admin stat show riak.riak_kv.node.gets [riak,riak_kv,node,gets]: [{count,0},{one,0}] - 'Legacy name': The stat names used e.g. in \`$SCRIPT status\` can be used @@ -130,7 +130,7 @@ here, but also with wildcard support. The corresponding Exometer name and datapoint will be shown as well. Example: -\$ bin/riak-admin stat show node_gets +\$ bin/riak admin stat show node_gets == node_gets (Legacy pattern): == node_gets: 0 ([riak,riak_kv,node,gets]/one) @@ -141,22 +141,22 @@ be present, so '*.**' would work as a catch-all expression.) Each Exometer entry has a type and a set of datapoints. A filter can be given on the command line, selecting only a subset of datapoints: -\$ bin/riak-admin stat show riak.riak_kv.node.gets/one +\$ bin/riak admin stat show riak.riak_kv.node.gets/one [riak,riak_kv,node,gets]: [{one,0}] The type can also be restricted: -\$ bin/riak-admin stat show *.**/type=duration/mean,max +\$ bin/riak admin stat show *.**/type=duration/mean,max [riak,riak_core,converge_delay]: [{mean,0},{max,0}] [riak,riak_core,rebalance_delay]: [{mean,0},{max,0}] Note how multiple datapoints are separated by comma (no space). Showing disabled entries: -\$ bin/riak-admin stat show riak.riak_kv.node.gets +\$ bin/riak admin stat show riak.riak_kv.node.gets No matching stats -\$ bin/riak-admin stat show riak.riak_kv.node.gets/status=* +\$ bin/riak admin stat show riak.riak_kv.node.gets/status=* [riak,riak_kv,node,gets]: disabled -\$ bin/riak-admin stat show riak.riak_kv.node.gets/status=disabled +\$ bin/riak admin stat show riak.riak_kv.node.gets/status=disabled [riak,riak_kv,node,gets]: disabled " } @@ -171,16 +171,16 @@ updated, and have no value. The same syntax can be used as in \`stat show\`. The requested action will be performed on the matching entries. -\$ bin/riak-admin stat disable node_gets +\$ bin/riak admin stat disable node_gets == node_gets (Legacy pattern): == [riak,riak_kv,node,gets]: disabled -\$ bin/riak-admin stat enable node_gets +\$ bin/riak admin stat enable node_gets == node_gets (Legacy pattern): == [riak,riak_kv,node,gets]: enabled Wildcards can be used: -\$ bin/riak-admin stat disable riak.riak_kv.node.* +\$ bin/riak admin stat disable riak.riak_kv.node.* [riak,riak_kv,node,gets]: disabled [riak,riak_kv,node,puts]: disabled " @@ -202,7 +202,7 @@ with options: The same entry formats can be used as for all other stat subcommands. Example: -\$ bin/riak-admin stat info riak.riak_kv.node.gets +\$ bin/riak admin stat info riak.riak_kv.node.gets [riak,riak_kv,node,gets]: name = [riak,riak_kv,node,gets] type = spiral module = exometer_spiral @@ -212,7 +212,7 @@ Example: timestamp = undefined options = [{status,disabled}] -\$ bin/riak-admin stat info -type -status riak.riak_kv.node.gets +\$ bin/riak admin stat info -type -status riak.riak_kv.node.gets [riak,riak_kv,node,gets]: type = spiral status = disabled " @@ -985,7 +985,7 @@ case "$1" in diag) case "$2" in --help|-h) - echo "Usage: riak-admin diag [-d ] [-l] [-h] [--export] [check_name ...]" + echo "Usage: riak admin diag [-d ] [-l] [-h] [--export] [check_name ...]" echo "" echo "-h, --help Display help/usage dialogue" echo "-d, --level Minimum message severity level (default: notice)" From 7b349b9e499abce1fe86e781e8db978cddfab12b Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Thu, 15 Jul 2021 21:09:31 +0100 Subject: [PATCH 23/47] Update branch prep for release 3.0.7 --- rebar.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rebar.config b/rebar.config index f1b1de96a..0c44e6280 100644 --- a/rebar.config +++ b/rebar.config @@ -11,13 +11,13 @@ {deps, [ {lager_syslog, {git, "git://github.com/basho/lager_syslog.git", {tag, "3.1.0"}}}, {cluster_info, {git, "git://github.com/basho/cluster_info.git", {tag, "2.1.0"}}}, - {riak_kv, {git, "git://github.com/basho/riak_kv.git", {tag, "riak_kv-3.0.6"}}}, + {riak_kv, {git, "git://github.com/basho/riak_kv.git", {branch, "develop-3.0"}}}, {riak_auth_mods, {git, "git://github.com/basho/riak_auth_mods.git", {tag, "riak_kv-3.0.0"}}}, {riaknostic, {git, "git://github.com/basho/riaknostic.git", {tag, "riak_kv-3.0.2"}}}, % {yokozuna, {git, "git://github.com/basho/yokozuna.git", {branch, "develop-3.0"}}}, - {riak_repl, {git, "git://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.4"}}} + {riak_repl, {git, "git://github.com/basho/riak_repl.git", {branch, "develop-3.0"}}} ]}. {project_plugins, [ @@ -46,7 +46,7 @@ ]} ]}. -{relx, [{release, {riak, "3.0.6"}, +{relx, [{release, {riak, "3.0.7"}, [kernel, stdlib, lager, From 2e33ed30121aab1d99bc4c93a81a259bdf0620bc Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Thu, 15 Jul 2021 21:23:46 +0100 Subject: [PATCH 24/47] Update rebar.lock --- rebar.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/rebar.lock b/rebar.lock index 8d819c297..d5f441aa7 100644 --- a/rebar.lock +++ b/rebar.lock @@ -106,7 +106,7 @@ 1}, {<<"riak_api">>, {git,"git://github.com/basho/riak_api.git", - {ref,"94489df53296c20a8f5efb2ab3f50b248086f6e4"}}, + {ref,"982b973669b7128a23cb6bc33fc3527660de31df"}}, 1}, {<<"riak_auth_mods">>, {git,"git://github.com/basho/riak_auth_mods.git", @@ -126,11 +126,11 @@ 2}, {<<"riak_kv">>, {git,"git://github.com/basho/riak_kv.git", - {ref,"9066933ccdbec7a3f636cb4ef48631f5b7cd6b4b"}}, + {ref,"5c4c8da45725d1f9c4dc73d1164786277a6d8b23"}}, 0}, {<<"riak_pb">>, {git,"git://github.com/basho/riak_pb.git", - {ref,"db5c228385f9125185a310bacd9d51bcb0fdab5f"}}, + {ref,"3129374435d67086e05f97a92482ef7f280aa7c7"}}, 2}, {<<"riak_pipe">>, {git,"git://github.com/basho/riak_pipe.git", @@ -138,7 +138,7 @@ 1}, {<<"riak_repl">>, {git,"git://github.com/basho/riak_repl.git", - {ref,"0e0fd356c99d1074c7fca7dc9b040882ce05f1e0"}}, + {ref,"2a81fa0ee4fd26c29fc367b2e5cf97caefb40cde"}}, 0}, {<<"riak_sysmon">>, {git,"git://github.com/basho/riak_sysmon.git", @@ -146,11 +146,11 @@ 2}, {<<"riakc">>, {git,"git://github.com/basho/riak-erlang-client", - {ref,"ea4fcd062f55a67e31c7294a1b8b8ccd957526c8"}}, + {ref,"82b82c82444e8ccd0c18cc2cb629106dd3c20425"}}, 2}, {<<"riakhttpc">>, {git,"git://github.com/basho/riak-erlang-http-client", - {ref,"51b7a8781db1c9644f4464aeec92621415f8a2e6"}}, + {ref,"78bce6115d5cfb77ca3b88eb823c1eb737a017c5"}}, 1}, {<<"riaknostic">>, {git,"git://github.com/basho/riaknostic.git", From 1ae2ca8943289f622f5ede0863fc6bf1ea1237eb Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Fri, 16 Jul 2021 20:56:50 +0100 Subject: [PATCH 25/47] Add 3.0.7 release notes and locked deps --- RELEASE-NOTES.md | 8 ++++++++ rebar.config | 4 ++-- rebar.lock | 10 +++++----- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index af2969d42..714965078 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,3 +1,11 @@ +# Riak KV 3.0.7 Release Notes + +The primary change in 3.0.7 is that Riak will now run the [erlang runtime system in interactive mode, not embedded mode](http://erlang.org/doc/man/code.html). This returns Riak to the default behaviour prior to Riak KV 3.0, in order to resolve a number of problems which occurred post 3.0 when trying to dynamically load code. + +The mode used is controlled in a [pre-start script](https://github.com/basho/riak/blob/develop-3.0/rel/files/erl_codeloadingmode), changing this script or overriding the referenced environment variable for the riak user should allow the runtime to be reverted to using `embedded` mode, should this be required. + +This release also extends a prototype API to support for the use of the `nextgenrepl` API by external applications, for example to reconcile replication to an external non-riak database. The existing `fetch` api function has been extended to allow for a new response format that includes the Active Anti-Entropy Segment ID and Segment Hash for the object (e.g. to be used when recreating the Riak merkle tree in external databases). A new `push` function has been added to the api, this will push a list of object references to be queued for replication. + # Riak KV 3.0.6 Release Notes Release 3.0.5 adds [location-awareness to Riak cluster management](https://github.com/basho/riak_core/blob/riak_kv-3.0.5/docs/rack-awareness.md). The broad aim is to improve data diversity across locations (e.g. racks) to reduce the probability of data-loss should a set of nodes fail concurrently within a location. The location-awareness does not provide firm guarantees of data diversity that will always be maintained across all cluster changes - but [testing](https://github.com/basho/riak_test/pull/1353) has indicated it will generally find a cluster arrangement which is close to optimal in terms of data protection. diff --git a/rebar.config b/rebar.config index fd58fa693..bf0e363c0 100644 --- a/rebar.config +++ b/rebar.config @@ -11,13 +11,13 @@ {deps, [ {lager_syslog, {git, "git://github.com/basho/lager_syslog.git", {tag, "3.1.0"}}}, {cluster_info, {git, "git://github.com/basho/cluster_info.git", {tag, "2.1.0"}}}, - {riak_kv, {git, "git://github.com/basho/riak_kv.git", {branch, "develop-3.0"}}}, + {riak_kv, {git, "git://github.com/basho/riak_kv.git", {tag, "riak_kv-3.0.7"}}}, {riak_auth_mods, {git, "git://github.com/basho/riak_auth_mods.git", {tag, "riak_kv-3.0.0"}}}, {riaknostic, {git, "git://github.com/basho/riaknostic.git", {tag, "riak_kv-3.0.2"}}}, % {yokozuna, {git, "git://github.com/basho/yokozuna.git", {branch, "develop-3.0"}}}, - {riak_repl, {git, "git://github.com/basho/riak_repl.git", {branch, "develop-3.0"}}} + {riak_repl, {git, "git://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.7"}}} ]}. {project_plugins, [ diff --git a/rebar.lock b/rebar.lock index d5f441aa7..8eb41dda2 100644 --- a/rebar.lock +++ b/rebar.lock @@ -106,7 +106,7 @@ 1}, {<<"riak_api">>, {git,"git://github.com/basho/riak_api.git", - {ref,"982b973669b7128a23cb6bc33fc3527660de31df"}}, + {ref,"0f0a485d214ca9abc26451a0c11873097c605622"}}, 1}, {<<"riak_auth_mods">>, {git,"git://github.com/basho/riak_auth_mods.git", @@ -126,7 +126,7 @@ 2}, {<<"riak_kv">>, {git,"git://github.com/basho/riak_kv.git", - {ref,"5c4c8da45725d1f9c4dc73d1164786277a6d8b23"}}, + {ref,"578eb0a4212f183041111b8703b481935a273937"}}, 0}, {<<"riak_pb">>, {git,"git://github.com/basho/riak_pb.git", @@ -138,7 +138,7 @@ 1}, {<<"riak_repl">>, {git,"git://github.com/basho/riak_repl.git", - {ref,"2a81fa0ee4fd26c29fc367b2e5cf97caefb40cde"}}, + {ref,"75faa4a0621aa42f45cd1e6e1fa103017a8343a4"}}, 0}, {<<"riak_sysmon">>, {git,"git://github.com/basho/riak_sysmon.git", @@ -146,11 +146,11 @@ 2}, {<<"riakc">>, {git,"git://github.com/basho/riak-erlang-client", - {ref,"82b82c82444e8ccd0c18cc2cb629106dd3c20425"}}, + {ref,"07c9582c9fd97f8252e0a3e488d6ebc08a9a4172"}}, 2}, {<<"riakhttpc">>, {git,"git://github.com/basho/riak-erlang-http-client", - {ref,"78bce6115d5cfb77ca3b88eb823c1eb737a017c5"}}, + {ref,"f5d9574a834a5efe996b340107eb932504de869f"}}, 1}, {<<"riaknostic">>, {git,"git://github.com/basho/riaknostic.git", From 96ff4d42b278795a0e082c36c98f1ee59f3ed62f Mon Sep 17 00:00:00 2001 From: Andrei Zavada Date: Wed, 6 Oct 2021 13:55:03 +0300 Subject: [PATCH 26/47] pass NODETOOL_NODE_PREFIX to relx-generated script, spare some atoms (#1079) Each time riak status (and the like) is run, relx generates a unique id which is then converted to an atom ("maint1a2b3c4d-riak@127.0.0.1") in riak VM, eventually causing atom table exhaustion. Relx has been modified to optionally generate a static id if NODETOOL_NODE_PREFIX env var is present. Convenient when the caller guarantees calls are always serialized. Co-authored-by: Andrei Zavada --- rel/files/riak | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rel/files/riak b/rel/files/riak index 70fef3db6..a04281509 100755 --- a/rel/files/riak +++ b/rel/files/riak @@ -39,17 +39,17 @@ else fi case "$1" in start|console|foreground) - su - riak -c "RELX_CONFIG_PATH=${RELX_CONFIG_PATH} VMARGS_PATH=${VMARGS_PATH} RUNNER_LOG_DIR=${RUNNER_LOG_DIR} PIPE_DIR=${PIPE_DIR} ${COMMAND} ${*} -pa {{platform_lib_dir}}/patches" + su - riak -c "NODETOOL_NODE_PREFIX=${NODETOOL_NODE_PREFIX} RELX_CONFIG_PATH=${RELX_CONFIG_PATH} VMARGS_PATH=${VMARGS_PATH} RUNNER_LOG_DIR=${RUNNER_LOG_DIR} PIPE_DIR=${PIPE_DIR} ${COMMAND} ${*} -pa {{platform_lib_dir}}/patches" ;; debug) # Drop the "debug" from the args as we're going to directly call riak-debug now shift # Debug may fail if run via relx script due to use of relative start location, and also need for root access - RELX_CONFIG_PATH=${RELX_CONFIG_PATH} VMARGS_PATH=${VMARGS_PATH} RUNNER_LOG_DIR=${RUNNER_LOG_DIR} PIPE_DIR=${PIPE_DIR} ${DEBUG_COMMAND} ${*} + NODETOOL_NODE_PREFIX=${NODETOOL_NODE_PREFIX} RELX_CONFIG_PATH=${RELX_CONFIG_PATH} VMARGS_PATH=${VMARGS_PATH} RUNNER_LOG_DIR=${RUNNER_LOG_DIR} PIPE_DIR=${PIPE_DIR} ${DEBUG_COMMAND} ${*} ;; *) ESCAPED_ARGS=`echo "$@" | sed -e 's/\([\\\(\\\){}"\x27]\)/\\\\\1/g'` - su - riak -c "RELX_CONFIG_PATH=${RELX_CONFIG_PATH} VMARGS_PATH=${VMARGS_PATH} RUNNER_LOG_DIR=${RUNNER_LOG_DIR} PIPE_DIR=${PIPE_DIR} ${COMMAND} ${ESCAPED_ARGS}" + su - riak -c "NODETOOL_NODE_PREFIX=${NODETOOL_NODE_PREFIX} RELX_CONFIG_PATH=${RELX_CONFIG_PATH} VMARGS_PATH=${VMARGS_PATH} RUNNER_LOG_DIR=${RUNNER_LOG_DIR} PIPE_DIR=${PIPE_DIR} ${COMMAND} ${ESCAPED_ARGS}" ;; esac fi From eac77556327654cdc9f11ccab1c6f7dd92b2d267 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Tue, 12 Oct 2021 08:17:10 +0100 Subject: [PATCH 27/47] Mas release308 (#1082) * Changes for Release 3.0.8 * Update rebar.lock * Update THANKS * Update lock for release --- RELEASE-NOTES.md | 32 +++++++++++++++++++ THANKS | 3 ++ rebar.config | 16 +++++----- rebar.lock | 80 ++++++++++++++++++++++++------------------------ 4 files changed, 83 insertions(+), 48 deletions(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 714965078..d997fd549 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,3 +1,25 @@ +# Riak KV 3.0.8 Release Notes + +This release contains a number of stability improvements. + +- Fix to critical issue in leveled when using (non-default, but recommended, option): [leveled_reload_recalc = enabled](https://github.com/basho/riak_kv/blob/33add2a29b6880b680a407dc91828736f54c7911/priv/riak_kv.schema#L1156-L1174). If using this option, it is recommended to rebuild the ledger on each vnode at some stage after updating. + +- [Fix to an issue with cluster leave](https://github.com/basho/riak_core/issues/970) operations that could leave clusters unnecessarily unbalanced after Stage 1 of leave, and also cause unexpected safety violations in Stage 1 (with either a simple transfer or a rebalance). An option to force a rebalance in Stage 1 is also now supported. + +- The ability to [set an environment variable](https://github.com/basho/riak/pull/1079) to remove the risk of atom table exhaustion due to repeated calls to `riak status` (and other) command-line functions. + +- The default setting of the object_hash_version environment variable to reduce the opportunity for the [riak_core_capability system to falsely request downgrading to legacy](https://github.com/basho/riak_kv/issues/1656), especially when concurrently restarting many nodes. + +- An update to the versions of `recon` and `redbug` used in Riak. + +- The fixing of an issue with [connection close handling](https://github.com/basho/riak-erlang-client/pull/402) in the Riak erlang client. + +This release also contains two new features: + +- A [new aae_fold operation](https://github.com/basho/riak_kv/issues/1793) has been added which will support the prompting of read-repair for a range of keys e.g. all the keys in a bucket after a given last modified date. This is intended to allow an operator to accelerate full recovery of data following a known node outage. + +- The addition of the [`sync_on_write` property for write operations](https://github.com/basho/riak_kv/blob/develop-3.0/docs/Sync-On-Write.md). Some Riak users require flushing of writes to disk to protect against data loss in disaster scenarios, such as mass loss of power across a DC. This can have a significant impact on throughput even with hardware acceleration (e.g. flash-backed write caches). The decision to flush was previously all or nothing. It can now be set as a bucket property (and even determined on individual writes), and can be set to flush on `all` vnodes or just `one` (the coordinator), or to simply respect the `backend` configuration. If `one` is used the single flush will occur only on client-initiated writes - writes due to handoffs or replication will not be flushed. + # Riak KV 3.0.7 Release Notes The primary change in 3.0.7 is that Riak will now run the [erlang runtime system in interactive mode, not embedded mode](http://erlang.org/doc/man/code.html). This returns Riak to the default behaviour prior to Riak KV 3.0, in order to resolve a number of problems which occurred post 3.0 when trying to dynamically load code. @@ -75,6 +97,16 @@ This major release allows Riak to run on OTP versions 20, 21 and 22 - but is not Other than the limitations listed above, the release should be functionally identical to Riak KV 2.9.7. Throughput improvements may be seen as a result of the OTP 20 upgrade on some CPU-bound workloads. For disk-bound workloads, additional benefit may be achieved by upgrading further to OTP 22. +# Riak KV 2.9.10 Release Notes + +Fix to critical issue in leveled when using (non-default, but recommended, option): [leveled_reload_recalc = enabled](https://github.com/basho/riak_kv/blob/33add2a29b6880b680a407dc91828736f54c7911/priv/riak_kv.schema#L1156-L1174). + +If using this option, it is recommended to rebuild the ledger on each vnode at some stage after updating. + +# Riak KV 2.9.9 Release Notes + +Minor stability improvements to leveled backend - [see leveled release notes](https://github.com/martinsumner/leveled/releases/tag/0.9.24) for further details. + # Riak KV 2.9.8 Release Notes This release improves the performance and stability of the leveled backend and of AAE folds. These performance improvements are based on feedback from deployments with > 1bn keys per cluster. diff --git a/THANKS b/THANKS index b9a05c502..ee4fc80ee 100644 --- a/THANKS +++ b/THANKS @@ -37,6 +37,7 @@ Anders Nygren Andre Graf Andreas Hasselberg Andreas Pfotenhauer +Andrei Zavada Andrew Gopienko Andrew J. Stone Andrew Matheny @@ -487,6 +488,7 @@ Przemysław Dąbek Péter Gömöri Ragil Rajiv M. Ranganath +Dr Ramen Sen Ramón Lastres Randy Secrist Ransom Richardson @@ -571,6 +573,7 @@ Ted Nyman Thiago Avelino Thiago Carvalho Thomas +Thomas Arts Thomas Guyot-Sionnest Thomas Järvstrand Tiago Peczenyj diff --git a/rebar.config b/rebar.config index bf0e363c0..c0bbbc90a 100644 --- a/rebar.config +++ b/rebar.config @@ -9,15 +9,15 @@ {eunit_opts, [nowarn_export_all, verbose]}. {deps, [ - {lager_syslog, {git, "git://github.com/basho/lager_syslog.git", {tag, "3.1.0"}}}, - {cluster_info, {git, "git://github.com/basho/cluster_info.git", {tag, "2.1.0"}}}, - {riak_kv, {git, "git://github.com/basho/riak_kv.git", {tag, "riak_kv-3.0.7"}}}, - {riak_auth_mods, {git, "git://github.com/basho/riak_auth_mods.git", {tag, "riak_kv-3.0.0"}}}, - {riaknostic, {git, "git://github.com/basho/riaknostic.git", {tag, "riak_kv-3.0.2"}}}, + {lager_syslog, {git, "https://github.com/basho/lager_syslog.git", {tag, "3.1.0"}}}, + {cluster_info, {git, "https://github.com/basho/cluster_info.git", {tag, "2.1.0"}}}, + {riak_kv, {git, "https://github.com/basho/riak_kv.git", {tag, "riak_kv-3.0.8"}}}, + {riak_auth_mods, {git, "https://github.com/basho/riak_auth_mods.git", {tag, "riak_kv-3.0.0"}}}, + {riaknostic, {git, "https://github.com/basho/riaknostic.git", {tag, "riak_kv-3.0.2"}}}, - % {yokozuna, {git, "git://github.com/basho/yokozuna.git", {branch, "develop-3.0"}}}, + % {yokozuna, {git, "https://github.com/basho/yokozuna.git", {branch, "develop-3.0"}}}, - {riak_repl, {git, "git://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.7"}}} + {riak_repl, {git, "https://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.8"}}} ]}. {project_plugins, [ @@ -46,7 +46,7 @@ ]} ]}. -{relx, [{release, {riak, "3.0.7"}, +{relx, [{release, {riak, "3.0.8"}, [kernel, stdlib, lager, diff --git a/rebar.lock b/rebar.lock index 8eb41dda2..840170895 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,11 +1,11 @@ {"1.1.0", [{<<"basho_stats">>, - {git,"git://github.com/basho/basho_stats.git", + {git,"https://github.com/basho/basho_stats.git", {ref,"c5b7ecb92af7b050bc62d08c6fbf1aafd6bdeff0"}}, 2}, {<<"bear">>,{pkg,<<"bear">>,<<"0.8.7">>},4}, {<<"bitcask">>, - {git,"git://github.com/basho/bitcask.git", + {git,"https://github.com/basho/bitcask.git", {ref,"dd96f6dd84a1cad68ccc21959ba56b0426434d85"}}, 1}, {<<"canola">>, @@ -13,11 +13,11 @@ {ref,"2cd39378442f3dc727de22289243b3c4d56f25fd"}}, 1}, {<<"clique">>, - {git,"git://github.com/basho/clique.git", + {git,"https://github.com/basho/clique.git", {ref,"4014357e4e677164b890fdad08a45cfa54b3e8f3"}}, 2}, {<<"cluster_info">>, - {git,"git://github.com/basho/cluster_info.git", + {git,"https://github.com/basho/cluster_info.git", {ref,"389d43af7ac1550b3c01cd55b8147bcc0e20022f"}}, 0}, {<<"cuttlefish">>, @@ -29,11 +29,11 @@ {ref,"3c25f9ef61a73d5f1d2c2601da464dd03975721d"}}, 1}, {<<"eleveldb">>, - {git,"git://github.com/basho/eleveldb.git", + {git,"https://github.com/basho/eleveldb.git", {ref,"3dee1dd7173507a4ad29df1ceb99cb70d6142478"}}, 2}, {<<"exometer_core">>, - {git,"git://github.com/Feuerlabs/exometer_core.git", + {git,"https://github.com/Feuerlabs/exometer_core.git", {ref,"547f28bc93c7cb3d3f1174fb4c510667a4ebb645"}}, 2}, {<<"folsom">>,{pkg,<<"folsom">>,<<"0.8.7">>},3}, @@ -44,44 +44,44 @@ {<<"goldrush">>,{pkg,<<"goldrush">>,<<"0.1.9">>},2}, {<<"hut">>,{pkg,<<"hut">>,<<"1.2.1">>},3}, {<<"hyper">>, - {git,"git://github.com/basho/hyper", + {git,"https://github.com/basho/hyper", {ref,"d0bf3d67fd17b8ed2610b266aebd54b1027ea916"}}, 1}, {<<"ibrowse">>, - {git,"git://github.com/basho/ibrowse.git", + {git,"https://github.com/basho/ibrowse.git", {ref,"cdde6fe5e80ce71e7b5570e40a45d9f862721499"}}, 2}, {<<"kv_index_tictactree">>, {git,"https://github.com/martinsumner/kv_index_tictactree.git", - {ref,"e1f580a908014bdc2461198dd8f5e8012ec152c1"}}, + {ref,"1c297426e86542fcb9d2dd61400853a42a6e104d"}}, 1}, {<<"lager">>, {git,"git://github.com/erlang-lager/lager.git", {ref,"22e62f28e5afabe90a6f31bcde367a2b5799fc94"}}, 1}, {<<"lager_syslog">>, - {git,"git://github.com/basho/lager_syslog.git", + {git,"https://github.com/basho/lager_syslog.git", {ref,"152bb8eb254c225ebd8c579f9603f44b50df44ec"}}, 0}, {<<"leveled">>, - {git,"https://github.com/martinsumner/leveled.git", - {ref,"489d89fb16af987f0b74fb42dfa6e4f9e6c2f7f1"}}, - 1}, + {git,"https://github.com/martinsumner/leveled", + {ref,"8fa373d367e33179da12fd0b7fb1ca8ddce23ffd"}}, + 2}, {<<"lz4">>, {git,"https://github.com/martinsumner/erlang-lz4", {ref,"d407af5de71303b8e381df718d2e5d05eb775c59"}}, - 2}, + 3}, {<<"meck">>, {git,"https://github.com/eproxus/meck.git", {ref,"4ecc1ae9089edc6977e8c8c4cd41081513cc5590"}}, 4}, {<<"mochiweb">>, - {git,"git://github.com/basho/mochiweb.git", + {git,"https://github.com/basho/mochiweb.git", {ref,"8471d2197afd412fcec239dd9ad5cbe90e64eefa"}}, 2}, {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.3.0">>},3}, {<<"pbkdf2">>, - {git,"git://github.com/basho/erlang-pbkdf2.git", + {git,"https://github.com/basho/erlang-pbkdf2.git", {ref,"681494c3ea03625c909a16b1b3ca295edf892057"}}, 2}, {<<"poolboy">>, @@ -98,26 +98,26 @@ 1}, {<<"recon">>, {git,"https://github.com/ferd/recon", - {ref,"75d70c7c08926d2f24f1ee6de14ee50fe8a52763"}}, + {ref,"34194da6d9f8ed25f274e0ebb098dc9e95bcf547"}}, 1}, {<<"redbug">>, {git,"https://github.com/massemanet/redbug", - {ref,"c583d4567d1e2b8a540593ad0512559798d42565"}}, + {ref,"c466ddc4f05a34c47bcf0666899c50e40c5223d2"}}, 1}, {<<"riak_api">>, - {git,"git://github.com/basho/riak_api.git", - {ref,"0f0a485d214ca9abc26451a0c11873097c605622"}}, + {git,"https://github.com/basho/riak_api.git", + {ref,"4c1dbe06c31c93b6947ec9ab0150ea668811f726"}}, 1}, {<<"riak_auth_mods">>, - {git,"git://github.com/basho/riak_auth_mods.git", + {git,"https://github.com/basho/riak_auth_mods.git", {ref,"1eccaef8e9e6200f0b75ea9312f0a731c8d572bf"}}, 0}, {<<"riak_core">>, - {git,"git://github.com/basho/riak_core.git", - {ref,"1f7e204548148f7dda9727113838d40b559310e3"}}, + {git,"https://github.com/basho/riak_core.git", + {ref,"b36a7aedcc152b3070748972730fb2461ab49c14"}}, 1}, {<<"riak_dt">>, - {git,"git://github.com/basho/riak_dt.git", + {git,"https://github.com/basho/riak_dt.git", {ref,"d0de4694f1423dc942844fbc494604a3c6df17d6"}}, 1}, {<<"riak_ensemble">>, @@ -125,44 +125,44 @@ {ref,"573a271aa546dcdcaeaaf09b3de6949a3130e021"}}, 2}, {<<"riak_kv">>, - {git,"git://github.com/basho/riak_kv.git", - {ref,"578eb0a4212f183041111b8703b481935a273937"}}, + {git,"https://github.com/basho/riak_kv.git", + {ref,"ef092fb28d1cf32cf10cd61a4ceb3d4467701c62"}}, 0}, {<<"riak_pb">>, - {git,"git://github.com/basho/riak_pb.git", - {ref,"3129374435d67086e05f97a92482ef7f280aa7c7"}}, + {git,"https://github.com/basho/riak_pb.git", + {ref,"b7abca90e4c708073021d2c4e18c896f1a10b838"}}, 2}, {<<"riak_pipe">>, - {git,"git://github.com/basho/riak_pipe.git", + {git,"https://github.com/basho/riak_pipe.git", {ref,"8dd2a6352e58a7d8215ba5afad4cf7b9e4807714"}}, 1}, {<<"riak_repl">>, - {git,"git://github.com/basho/riak_repl.git", - {ref,"75faa4a0621aa42f45cd1e6e1fa103017a8343a4"}}, + {git,"https://github.com/basho/riak_repl.git", + {ref,"89ffda17e1fcd4f596a5c72e4ff6fc9405616327"}}, 0}, {<<"riak_sysmon">>, - {git,"git://github.com/basho/riak_sysmon.git", + {git,"https://github.com/basho/riak_sysmon.git", {ref,"726df1f4c31108bb9366fb767b480f286e51f7fc"}}, 2}, {<<"riakc">>, - {git,"git://github.com/basho/riak-erlang-client", - {ref,"07c9582c9fd97f8252e0a3e488d6ebc08a9a4172"}}, + {git,"https://github.com/basho/riak-erlang-client", + {ref,"d9fa46335fda4704fda71cbc011ab6a11e50e3f3"}}, 2}, {<<"riakhttpc">>, - {git,"git://github.com/basho/riak-erlang-http-client", - {ref,"f5d9574a834a5efe996b340107eb932504de869f"}}, + {git,"https://github.com/basho/riak-erlang-http-client", + {ref,"fd7255150967245195105f4437904538c17614b1"}}, 1}, {<<"riaknostic">>, - {git,"git://github.com/basho/riaknostic.git", + {git,"https://github.com/basho/riaknostic.git", {ref,"8aef63a89d719b22754683cce187eaeeecdd789c"}}, 0}, {<<"setup">>,{pkg,<<"setup">>,<<"2.0.2">>},3}, {<<"sext">>, - {git,"git://github.com/uwiger/sext.git", + {git,"https://github.com/uwiger/sext.git", {ref,"615eebcf975ec4b4561c6f2b2bc433dabdb2be0f"}}, 1}, {<<"sidejob">>, - {git,"git://github.com/basho/sidejob.git", + {git,"https://github.com/basho/sidejob.git", {ref,"d5e8f8450b92ab30610496ffb58f560d6a8bad6b"}}, 1}, {<<"syslog">>, @@ -170,7 +170,7 @@ {ref,"4a6c6f2c996483e86c1320e9553f91d337bcb6aa"}}, 1}, {<<"webmachine">>, - {git,"git://github.com/webmachine/webmachine.git", + {git,"https://github.com/webmachine/webmachine.git", {ref,"92225b82fc702f78cf9d23248023ec841272df80"}}, 2}]}. [ From 1cf3c5d99f77af458ab8aa816c4ec01b717dd80b Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Thu, 11 Nov 2021 11:26:38 +0000 Subject: [PATCH 28/47] Release 3.0.9 (#1084) * Logging of 2i query performance * Update deps * Add release notes Initial preparation for Riak 3.0.9 release --- RELEASE-NOTES.md | 14 ++++++++++++++ eunit.log | 47 ----------------------------------------------- rebar.config | 6 +++--- rebar.lock | 14 +++++++------- 4 files changed, 24 insertions(+), 57 deletions(-) delete mode 100644 eunit.log diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index d997fd549..14036469a 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,3 +1,17 @@ +# Riak KV 3.0.9 Release Notes + +This release contains stability, monitoring and performance improvements. + +- Fix to the [riak_core coverage planner](https://github.com/basho/riak_kv/issues/1801) to significantly reduce the CPU time required to produce a coverage plan, especially with larger ring sizes. This improves both the mean and tail-latency of secondary index queries. As a result of this change, it is recommended that larger ring sizes should be used by default, even when running relatively small clusters - for example in standard volume tests a ring size of 512 is outperforming lower ring sizes even on small (8-node) clusters. + +- Further monitoring stats have been added to track the performance of coverage queries, in particular secondary index queries. For each worker queue (e.g. vnode_worker_pool, af1_pool etc) the queue_time and work_time is now monitored with results available via riak stats. The result counts, and overall query time for secondary index queries are now also monitored via riak stats. See the [PR](https://github.com/basho/riak_kv/pull/1802#issuecomment-966160925) for a full list of stats added in this release. + +- Change to some default settings to be better adapted to running with higher concentrations of vnodes per nodes. The per-vnode cache sizes in leveled are reduced, and the [default size of the vnode_worker_pool has been reduced from 10 to 4](https://github.com/basho/riak_kv/commit/39e3d84082d2144ab300c0a6250b8ec2e9e8db65) and is now configurable via riak.conf. Exceptionally heavy users of secondary index queries (i.e. > 1% of transactions), should consider monitoring the [new queue_time and work_time statistics](https://github.com/basho/riak_kv/pull/1802#issuecomment-966160925) before accepting this new default. + +- Fix to an issue in the leveled backend when a key and (internal) sequence number would [hash to 0](https://github.com/martinsumner/leveled/issues/361). It is recommended that users of leveled uplift to this version as soon as possible to reoslve this issue. The risk is mitigated in Riak as it can generally be expected that most keys will have different sequence numbers in different vnodes, and will always have different sequence numbers when re-written - so normally anti-entropy process will recover from any occurrence. + +- More time is now given to the legacy AAE `kv_index_hashtree` process to [shut down](https://github.com/basho/riak_kv/pull/1803), to handle delays as multiple vnodes are shutdown concurrently and contend for disk and CPU resources. + # Riak KV 3.0.8 Release Notes This release contains a number of stability improvements. diff --git a/eunit.log b/eunit.log deleted file mode 100644 index 2665bc6ca..000000000 --- a/eunit.log +++ /dev/null @@ -1,47 +0,0 @@ -Eunit: basho_stats PASSED -Eunit: bear PASSED -Eunit: bitcask PASSED -Eunit: canola PASSED -Eunit: clique PASSED -Eunit: cluster_info PASSED -Eunit: cuttlefish PASSED -Eunit: ebloom PASSED -Eunit: eleveldb PASSED -Eunit: exometer_core PASSED -Eunit: folsom PASSED -Eunit: getopt PASSED -Eunit: goldrush PASSED -Eunit: hut PASSED -Eunit: hyper PASSED -Eunit: ibrowse PASSED -Eunit: kv_index_tictactree PASSED -Eunit: lager_syslog PASSED -Eunit: leveled PASSED -Eunit: lz4 PASSED -Eunit: meck PASSED -Eunit: mochiweb PASSED -Eunit: parse_trans PASSED -Eunit: pbkdf2 PASSED -Eunit: poolboy PASSED -Eunit: protobuffs PASSED -Eunit: ranch PASSED -Eunit: recon PASSED -Eunit: redbug PASSED -Eunit: riak_api PASSED -Eunit: riak_auth_mods PASSED -Eunit: riak_core PASSED -Eunit: riak_dt PASSED -Eunit: riak_ensemble PASSED -Eunit: riak_kv PASSED -Eunit: riak_pb PASSED -Eunit: riak_pipe PASSED -Eunit: riak_repl PASSED -Eunit: riak_sysmon PASSED -Eunit: riakc PASSED -Eunit: riakhttpc PASSED -Eunit: riaknostic PASSED -Eunit: setup PASSED -Eunit: sext PASSED -Eunit: sidejob PASSED -Eunit: syslog PASSED -Eunit: webmachine PASSED diff --git a/rebar.config b/rebar.config index c0bbbc90a..e17d3fb6a 100644 --- a/rebar.config +++ b/rebar.config @@ -11,13 +11,13 @@ {deps, [ {lager_syslog, {git, "https://github.com/basho/lager_syslog.git", {tag, "3.1.0"}}}, {cluster_info, {git, "https://github.com/basho/cluster_info.git", {tag, "2.1.0"}}}, - {riak_kv, {git, "https://github.com/basho/riak_kv.git", {tag, "riak_kv-3.0.8"}}}, + {riak_kv, {git, "https://github.com/basho/riak_kv.git", {branch, "develop-3.0"}}}, {riak_auth_mods, {git, "https://github.com/basho/riak_auth_mods.git", {tag, "riak_kv-3.0.0"}}}, {riaknostic, {git, "https://github.com/basho/riaknostic.git", {tag, "riak_kv-3.0.2"}}}, % {yokozuna, {git, "https://github.com/basho/yokozuna.git", {branch, "develop-3.0"}}}, - {riak_repl, {git, "https://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.8"}}} + {riak_repl, {git, "https://github.com/basho/riak_repl.git", {branch, "develop-3.0"}}} ]}. {project_plugins, [ @@ -46,7 +46,7 @@ ]} ]}. -{relx, [{release, {riak, "3.0.8"}, +{relx, [{release, {riak, "3.0.9"}, [kernel, stdlib, lager, diff --git a/rebar.lock b/rebar.lock index 840170895..3e5c3d2f0 100644 --- a/rebar.lock +++ b/rebar.lock @@ -53,7 +53,7 @@ 2}, {<<"kv_index_tictactree">>, {git,"https://github.com/martinsumner/kv_index_tictactree.git", - {ref,"1c297426e86542fcb9d2dd61400853a42a6e104d"}}, + {ref,"e77b381e5c1e608b15756b9f183ac2b364201749"}}, 1}, {<<"lager">>, {git,"git://github.com/erlang-lager/lager.git", @@ -65,7 +65,7 @@ 0}, {<<"leveled">>, {git,"https://github.com/martinsumner/leveled", - {ref,"8fa373d367e33179da12fd0b7fb1ca8ddce23ffd"}}, + {ref,"086e06979e6f6d0d37b15df883c0ce2eb3c15892"}}, 2}, {<<"lz4">>, {git,"https://github.com/martinsumner/erlang-lz4", @@ -106,7 +106,7 @@ 1}, {<<"riak_api">>, {git,"https://github.com/basho/riak_api.git", - {ref,"4c1dbe06c31c93b6947ec9ab0150ea668811f726"}}, + {ref,"11e23d47f25cf01648e2219392a30e95f37162e1"}}, 1}, {<<"riak_auth_mods">>, {git,"https://github.com/basho/riak_auth_mods.git", @@ -114,7 +114,7 @@ 0}, {<<"riak_core">>, {git,"https://github.com/basho/riak_core.git", - {ref,"b36a7aedcc152b3070748972730fb2461ab49c14"}}, + {ref,"7e0aa310ecf5ec7b9fd03df61b996c2823591950"}}, 1}, {<<"riak_dt">>, {git,"https://github.com/basho/riak_dt.git", @@ -126,7 +126,7 @@ 2}, {<<"riak_kv">>, {git,"https://github.com/basho/riak_kv.git", - {ref,"ef092fb28d1cf32cf10cd61a4ceb3d4467701c62"}}, + {ref,"44849c1bee170804d68df7ad9d244dc89236db7b"}}, 0}, {<<"riak_pb">>, {git,"https://github.com/basho/riak_pb.git", @@ -134,11 +134,11 @@ 2}, {<<"riak_pipe">>, {git,"https://github.com/basho/riak_pipe.git", - {ref,"8dd2a6352e58a7d8215ba5afad4cf7b9e4807714"}}, + {ref,"229965f8c90ff2168800a68b8fb47fa44722f67d"}}, 1}, {<<"riak_repl">>, {git,"https://github.com/basho/riak_repl.git", - {ref,"89ffda17e1fcd4f596a5c72e4ff6fc9405616327"}}, + {ref,"0e0ed28210b4720a99475802e79bb8eae999d946"}}, 0}, {<<"riak_sysmon">>, {git,"https://github.com/basho/riak_sysmon.git", From 985ccfe94a44a2e72c86a0f660645065944e098f Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Fri, 12 Nov 2021 00:30:00 +0000 Subject: [PATCH 29/47] Update lock for release --- RELEASE-NOTES.md | 4 ++-- rebar.config | 4 ++-- rebar.lock | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 14036469a..73b64bc98 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -6,9 +6,9 @@ This release contains stability, monitoring and performance improvements. - Further monitoring stats have been added to track the performance of coverage queries, in particular secondary index queries. For each worker queue (e.g. vnode_worker_pool, af1_pool etc) the queue_time and work_time is now monitored with results available via riak stats. The result counts, and overall query time for secondary index queries are now also monitored via riak stats. See the [PR](https://github.com/basho/riak_kv/pull/1802#issuecomment-966160925) for a full list of stats added in this release. -- Change to some default settings to be better adapted to running with higher concentrations of vnodes per nodes. The per-vnode cache sizes in leveled are reduced, and the [default size of the vnode_worker_pool has been reduced from 10 to 4](https://github.com/basho/riak_kv/commit/39e3d84082d2144ab300c0a6250b8ec2e9e8db65) and is now configurable via riak.conf. Exceptionally heavy users of secondary index queries (i.e. > 1% of transactions), should consider monitoring the [new queue_time and work_time statistics](https://github.com/basho/riak_kv/pull/1802#issuecomment-966160925) before accepting this new default. +- Change to some default settings to be better adapted to running with higher concentrations of vnodes per nodes. The per-vnode cache sizes in leveled are reduced, and the [default size of the vnode_worker_pool has been reduced from 10 to 5](https://github.com/basho/riak_kv/blob/0af857d57d0206b100db59043a926df2ada8f7db/priv/riak_kv.schema#L1210-L1233) and is now configurable via riak.conf. Exceptionally heavy users of secondary index queries (i.e. > 1% of transactions), should consider monitoring the [new queue_time and work_time statistics](https://github.com/basho/riak_kv/pull/1802#issuecomment-966160925) before accepting this new default. -- Fix to an issue in the leveled backend when a key and (internal) sequence number would [hash to 0](https://github.com/martinsumner/leveled/issues/361). It is recommended that users of leveled uplift to this version as soon as possible to reoslve this issue. The risk is mitigated in Riak as it can generally be expected that most keys will have different sequence numbers in different vnodes, and will always have different sequence numbers when re-written - so normally anti-entropy process will recover from any occurrence. +- Fix to an issue in the leveled backend when a key and (internal) sequence number would [hash to 0](https://github.com/martinsumner/leveled/issues/361). It is recommended that users of leveled uplift to this version as soon as possible to resolve this issue. The risk is mitigated in Riak as it can generally be expected that most keys will have different sequence numbers in different vnodes, and will always have different sequence numbers when re-written - so normal anti-entropy process will recover from any localised data loss. - More time is now given to the legacy AAE `kv_index_hashtree` process to [shut down](https://github.com/basho/riak_kv/pull/1803), to handle delays as multiple vnodes are shutdown concurrently and contend for disk and CPU resources. diff --git a/rebar.config b/rebar.config index e17d3fb6a..51e22aa09 100644 --- a/rebar.config +++ b/rebar.config @@ -11,13 +11,13 @@ {deps, [ {lager_syslog, {git, "https://github.com/basho/lager_syslog.git", {tag, "3.1.0"}}}, {cluster_info, {git, "https://github.com/basho/cluster_info.git", {tag, "2.1.0"}}}, - {riak_kv, {git, "https://github.com/basho/riak_kv.git", {branch, "develop-3.0"}}}, + {riak_kv, {git, "https://github.com/basho/riak_kv.git", {tag, "riak_kv-3.0.9"}}}, {riak_auth_mods, {git, "https://github.com/basho/riak_auth_mods.git", {tag, "riak_kv-3.0.0"}}}, {riaknostic, {git, "https://github.com/basho/riaknostic.git", {tag, "riak_kv-3.0.2"}}}, % {yokozuna, {git, "https://github.com/basho/yokozuna.git", {branch, "develop-3.0"}}}, - {riak_repl, {git, "https://github.com/basho/riak_repl.git", {branch, "develop-3.0"}}} + {riak_repl, {git, "https://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.9"}}} ]}. {project_plugins, [ diff --git a/rebar.lock b/rebar.lock index 3e5c3d2f0..1fca129c5 100644 --- a/rebar.lock +++ b/rebar.lock @@ -53,7 +53,7 @@ 2}, {<<"kv_index_tictactree">>, {git,"https://github.com/martinsumner/kv_index_tictactree.git", - {ref,"e77b381e5c1e608b15756b9f183ac2b364201749"}}, + {ref,"a3787dbf7c556cd323ae00849c1926b6dfcf15dc"}}, 1}, {<<"lager">>, {git,"git://github.com/erlang-lager/lager.git", @@ -106,7 +106,7 @@ 1}, {<<"riak_api">>, {git,"https://github.com/basho/riak_api.git", - {ref,"11e23d47f25cf01648e2219392a30e95f37162e1"}}, + {ref,"85994688946d9c201d08981e756dec564458eeb9"}}, 1}, {<<"riak_auth_mods">>, {git,"https://github.com/basho/riak_auth_mods.git", @@ -126,7 +126,7 @@ 2}, {<<"riak_kv">>, {git,"https://github.com/basho/riak_kv.git", - {ref,"44849c1bee170804d68df7ad9d244dc89236db7b"}}, + {ref,"0af857d57d0206b100db59043a926df2ada8f7db"}}, 0}, {<<"riak_pb">>, {git,"https://github.com/basho/riak_pb.git", @@ -134,11 +134,11 @@ 2}, {<<"riak_pipe">>, {git,"https://github.com/basho/riak_pipe.git", - {ref,"229965f8c90ff2168800a68b8fb47fa44722f67d"}}, + {ref,"899e26181daa0fe2590d7502652c577412140fe1"}}, 1}, {<<"riak_repl">>, {git,"https://github.com/basho/riak_repl.git", - {ref,"0e0ed28210b4720a99475802e79bb8eae999d946"}}, + {ref,"c7b4fc2ae1a24b9476c297fe077f5a19c8d30a44"}}, 0}, {<<"riak_sysmon">>, {git,"https://github.com/basho/riak_sysmon.git", From e0b059ae5435368f136714d929da77b78b356090 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Fri, 27 May 2022 11:21:58 +0100 Subject: [PATCH 30/47] Mas i1820 controls (#1110) Preparation for release 3.0.10. Uplift all changed deps to develop-3.0, add new schema controls, update release notes and readme. Minimum OTP version now OTP 22. --- README.md | 15 ++++ README.org | 22 ----- RELEASE-NOTES.md | 42 +++++++++ priv/riak.schema | 219 ++++++++++++++++++++++++++++++++++++++++++++++- rebar.config | 16 ++-- rebar.lock | 72 ++++++++-------- 6 files changed, 317 insertions(+), 69 deletions(-) create mode 100644 README.md delete mode 100644 README.org diff --git a/README.md b/README.md new file mode 100644 index 000000000..dad7615ed --- /dev/null +++ b/README.md @@ -0,0 +1,15 @@ +# Riak - a distributed, decentralised data storage system. + +To build riak, Erlang OTP 22 or higher is required. + +`make rel` will build a release which can be run via `rel/riak/bin/riak start`. Riak is primarily configured via `rel/riak/etc/riak.conf` + +To make a package, install appropriate build tools for your operating system and run `make package`. + +To create a local multi-node build environment use `make devclean; make devrel`. + +To test Riak use [Riak Test](https://github.com/basho/riak_test/blob/develop-3.0/doc/SIMPLE_SETUP.md). + +Up to date documentation is not available, but work on [documentation](https://www.tiot.jp/riak-docs/riak/kv/2.9.10/) is ongoing and the core information available in the [legacy documentation](https://docs.riak.com/riak/kv/latest/index.html) is still generally relevant. + +Issues and PRs can be tracked via [Riak Github](https://github.com/basho/riak/issues) or [Riak KV Github](https://github.com/basho/riak_kv/issues). diff --git a/README.org b/README.org deleted file mode 100644 index 7a507caac..000000000 --- a/README.org +++ /dev/null @@ -1,22 +0,0 @@ -#+SETUPFILE: "doc/basho-doc-style.iorg" - -Welcome to Riak. - -* Overview - Riak is a distributed, decentralized data storage system. - - In the wiki, you will find the "quick start" directions for setting up and - using Riak. For more information, browse the following files: - - - README: this file - - LICENSE: the license under which Riak is released - - doc/ - - admin.org: Riak Administration Guide - - architecture.txt: details about the underlying design of Riak - - basic-client.txt: slightly more detail on using Riak - - basic-setup.txt: slightly more detail on setting up Riak - - man/riak.1.gz: manual page for the riak(1) command - - man/riak-admin.1.gz manual page for the riak-admin(1) command - - raw-http-howto.txt: using the Riak HTTP interface - -Archived Branches: https://github.com/basho-archive/riak diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 73b64bc98..b09400a4b 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,3 +1,45 @@ +# Riak KV 3.0.10 Release Notes + +This release is focused on improving memory management, especially with the leveled backend, and improving the efficiency and ease of configuration of tictacaae full-sync. + +- Improved [memory management of leveled](https://github.com/martinsumner/leveled/pull/371) SST files that contain [rarely accessed data](https://github.com/martinsumner/leveled/pull/371) + +- Fix a bug whereby leveled_sst files could spend an [extended time in the delete_pending state](https://github.com/martinsumner/leveled/pull/377), causing significant short-term increases in memory usage when there are work backlogs in the penciller. + +- Change the queue for reapers and erasers so that [they overflow to disk](https://github.com/basho/riak_kv/issues/1807), rather than simply consuming more and more memory. + +- Change the replrtq (nextgenrepl) queue to use the same [overflow queue mechanism](https://github.com/basho/riak_kv/issues/1817) as used by the reaper and erasers. + +- Change the default full-sync mechanism for tictacaae (nextgenrepl) full-sync to `auto_check`, which attempts to [automatically learn and use information about modified date-ranges](https://github.com/basho/riak_kv/issues/1815) in full-sync checks. The related changes also make full-sync by default bi-directional, reducing the amount of wasted effort in full-sync queries. + +- Add [a peer discovery feature](https://github.com/basho/riak_kv/issues/1804) for replrtq (nextgenrepl) so that new nodes added to the cluster can be automatically recognised without configuration changes. By default this is disabled, and should only be enabled once both clusters have been upgraded to at least 3.0.10. + +- Allow for underlying beam memory management and scheduler configuration to be exposed via riak.conf to allow for further performance tests on these settings. Note initial tests indicate the potential for [significant improvements when using the leveled backend](https://github.com/basho/riak_kv/issues/1826). + +- Fix a potential issue whereby corrupted objects would prevent AAE (either legacy or nextgenrepl) [tree rebuilds](https://github.com/basho/riak_kv/issues/1824) from completing. + +- Improved [handling of key amnesia](https://github.com/basho/riak_kv/issues/1813), to prevent rebounding of objects, and also introduce a reader process (like reaper and eraser) to which read repairs can be queued with overflow to disk. + +Some caveats for this release exist: + +- The release does not support OTP 20, only OTP 22 is supported. Updating some long out-of-date components have led to a requirement for the OTP version to be lifted. + +- Volume and performance testing with the leveled backend now uses the following non-default settings: + +``` +erlang.schedulers_busywait = none +erlang.schedulers_busywait_dirtycpu = none +erlang.schedulers_busywait_dirtyio = none +erlang.async_threads = 4 +erlang.schedulers.force_wakeup_interval = 0 +erlang.schedulers.compaction_of_load = true +leveled_reload_recalc = enabled +``` + +- To maintain backwards compatibility with older linux versions, the [latest version of basho's leveldb](https://github.com/basho/leveldb/releases/tag/2.0.37) is not yet supported. This is likely to change in the next release, where support for older linux versions will be dropped. + +- The release process has [exposed an issue](https://github.com/basho/riak_kv/issues/1831) via a recently extended test. This issue is pre-existing, and not specific to this release. + # Riak KV 3.0.9 Release Notes This release contains stability, monitoring and performance improvements. diff --git a/priv/riak.schema b/priv/riak.schema index 8407c39d5..d0773734f 100644 --- a/priv/riak.schema +++ b/priv/riak.schema @@ -205,25 +205,240 @@ merge ]}. -%% VM scheduler collapse, part 1 of 2 +%% @doc Riak changes the VM default wakeup interval in order to reduce the +%% risk of scheduler collapse, prior to the availability of Dirty NIFs in +%% later OTP versions. When using the leveled backend exclusively (either for +%% AAE or object storage) this change is likely unnecessary, and the VM default +%% of 0 can be used. {mapping, "erlang.schedulers.force_wakeup_interval", "vm_args.+sfwi", [ {default, 500}, {datatype, integer}, merge ]}. -%% VM scheduler collapse, part 2 of 2 +%% @doc Riak changes the compaction_of_load default from true to false. This +%% is part of the strategy for preventing scheduler collapse in older VMs. +%% When using the leveled backend exclusively (either for AAE or object +%% storage), this change from the standard BEAM defaults is likely unnecessary +%% - and compaction_of_load can be re-enabled. {mapping, "erlang.schedulers.compaction_of_load", "vm_args.+scl", [ {default, "false"}, merge ]}. +%% @doc Sets the number of threads in async thread pool, valid range +%% is 0-1024. If thread support is available, the default is 64. +%% +%% More information at: http://erlang.org/doc/man/erl.html +%% +%% Large async_thread pools are likely now unnecessary if exclusively using +%% the leveled backend due to dirty NIFs, and so can be set to a much smaller +%% value (potentially 1). +{mapping, "erlang.async_threads", "vm_args.+A", [ + {default, 64}, + {datatype, integer}, + {validators, ["range:0-1024"]}, + merge +]}. + %% VM emulator ignore break signal (prevent ^C / ^Gq) {mapping, "erlang.vm.ignore_break_signal", "vm_args.+Bi", [ {default, "true"}, merge ]}. +%% @doc The VM single block carrier threshold (KB) for process heap +{mapping, "erlang.eheap_memory.sbct", "vm_args.+MHsbct", [ + {commented, 512}, + {datatype, integer}, + merge +]}. + +%% @doc The VM single block carrier threshold (KB) for binary heap +{mapping, "erlang.binary_memory.sbct", "vm_args.+MBsbct", [ + {commented, 512}, + {datatype, integer}, + merge +]}. + +%% @doc The VM multi block carrier large size for process heap +{mapping, "erlang.eheap_memory.lmbcs", "vm_args.+MHlmbcs", [ + {commented, 5120}, + {datatype, integer}, + merge +]}. + +%% @doc The VM multi block carrier large size for binary heap +{mapping, "erlang.binary_memory.lmbcs", "vm_args.+MBlmbcs", [ + {commented, 5120}, + {datatype, integer}, + merge +]}. + +%% @doc The VM multi block carrier small size for process heap +{mapping, "erlang.eheap_memory.smbcs", "vm_args.+MHsmbcs", [ + {commented, 256}, + {datatype, integer}, + merge +]}. + +%% @doc The VM multi block carrier small size for binary heap +{mapping, "erlang.binary_memory.smbcs", "vm_args.+MBsmbcs", [ + {commented, 256}, + {datatype, integer}, + merge +]}. + +%% @doc Force move on realloc for process heap +{mapping, "erlang.eheap_memory.ramv", "vm_args.+MHramv", [ + {commented, "false"}, + {datatype, {enum, [true, false]}}, + merge +]}. + +%% @doc Force move on realloc for binary heap +{mapping, "erlang.binary_memory.ramv", "vm_args.+MBramv", [ + {commented, "false"}, + {datatype, {enum, [true, false]}}, + merge +]}. + + %% @doc Set allocation strategy for binary multiblock carriers. Default is +%% not predictable - do not rely on aoffcbf being the default. For more info +%% see: +%% https://github.com/erlang/otp/blob/master/erts/emulator/internal_doc/CarrierMigration.md +{mapping, "erlang.binary_memory.as", "vm_args.+MBas", [ + {commented, "aoffcbf"}, + {datatype, {enum, [bf, aobf, aoff, aoffcbf, aoffcaobf, ageffcaoff, ageffcbf, ageffcaobf, gf]}}, + merge +]}. + +%% @doc Set allocation strategy for process multiblock carriers. Default is +%% not predictable - do not rely on aoffcbf being the default. For more info +%% see: +%% https://github.com/erlang/otp/blob/master/erts/emulator/internal_doc/CarrierMigration.md +{mapping, "erlang.eheap_memory.as", "vm_args.+MHas", [ + {commented, "aoffcbf"}, + {datatype, {enum, [bf, aobf, aoff, aoffcbf, aoffcaobf, ageffcaoff, ageffcbf, ageffcaobf, gf]}}, + merge +]}. + +%% @doc Set scheduler binding. This is either unbound (default - u) or can be +%% set to whatever the default binding condition is, in the deployed release of +%% OTP (db). +%% For more info see: https://www.erlang.org/doc/man/erl.html#+sbt +%% Note that if non-Riak work is activated on the same node - e.g. as part +%% of batch operational jobs, or monitoring - allowing schedulers to be bound +%% can result in significant and unpredictable negative outcomes. There may be +%% other ways of achieving similar performance improvements - e.g. by +%% right-sizing scheduler counts - that are lower risk than scheduler binding. +%% If a CPU topology cannot be determined, the binding will default to unbound +%% even when a binding is configured. To confirm binding, use `remote_console` +%% and view: +%% `erlang:system_info(scheduler_bindings).` +{mapping, "erlang.schedulers_binding", "vm_args.+stbt", [ + {commented, "u"}, + {datatype, {enum, [u, db]}}, + merge +]}. + +%% @doc Busy wait of schedulers +%% Sets scheduler busy wait threshold. Defaults to medium. The threshold +%% determines how long schedulers are to busy wait when running out of work +%% before going to sleep. +%% Significant improvements in efficiency may be gained by disabling busy +%% waiting +{mapping, "erlang.schedulers_busywait", "vm_args.+sbwt", [ + {commented, "none"}, + {datatype, {enum, [none, very_short, short, medium, long, very_long]}}, + merge +]}. + +%% @doc Busy wait of dirty cpu schedulers +%% Sets scheduler busy wait threshold. Defaults to short. The threshold +%% determines how long schedulers are to busy wait when running out of work +%% before going to sleep. +%% Significant improvements in efficiency may be gained by disabling busy +%% waiting +{mapping, "erlang.schedulers_busywait_dirtycpu", "vm_args.+sbwtdcpu", [ + {commented, "none"}, + {datatype, {enum, [none, very_short, short, medium, long, very_long]}}, + merge +]}. + +%% @doc Busy wait of dirty io schedulers +%% Sets scheduler busy wait threshold. Defaults to short. The threshold +%% determines how long schedulers are to busy wait when running out of work +%% before going to sleep. +%% Significant improvements in efficiency may be gained by disabling busy +%% waiting +{mapping, "erlang.schedulers_busywait_dirtyio", "vm_args.+sbwtdio", [ + {commented, "none"}, + {datatype, {enum, [none, very_short, short, medium, long, very_long]}}, + merge +]}. + +%% @doc Set the Percentage of Schedulers to be online +%% For every vCPU in the system, what percentage should have a scheduler, and +%% what percentage of those schedulers should be online by default. +%% Do not set unless guided by perfomance tests for the specific setup and +%% workload. +{mapping, "erlang.schedulers_online_percentage", "vm_args.+SP",[ + {commented, "100:75"}, + {validators, ["scheduler_percentage"]}, + merge +]}. + +%% @doc Set the Percentage of Dirty CPU Schedulers to be online +%% When using the leveled backend a relatievly low number of dirty schedulers +%% (e.g. 25%) are likely to be required due to the low proportion of NIFs in +%% use. +%% The percentages cannot exceed those of the schedulers_online_percentage +%% which will default to 100% of CPU. +%% Do not set unless guided by perfomance tests for the specific setup and +%% workload. +{mapping, "erlang.schedulers_dirtycpu_online_percentage", "vm_args.+SDPcpu",[ + {commented, "50:25"}, + {validators, ["scheduler_percentage"]}, + merge +]}. + +%% @doc Set the absolute limit of Dirty IO Schedulers to be online +%% When using the leveled backend a relatievly high number of dirty schedulers +%% may be required relative to the CPU count, depending on the concurrent disk +%% throughput possible. +%% Unlike the scheduler percentages, this is set as an abolute number between +%% 1 and 1024 (default is 10). +%% Do not set unless guided by perfomance tests for the specific setup and +%% workload. +{mapping, "erlang.schedulers_dirtyio_online", "vm_args.+SDio",[ + {commented, 10}, + {datatype, integer}, + {validators, ["scheduler_absolute"]}, + merge +]}. + +{validator, + "scheduler_percentage", + "must be A:B when B =< A and both A and B 1 < x =< 100", + fun(PercPerc) -> + case string:tokens(PercPerc, ":") of + [A, B] -> + AV = list_to_integer(A), + BV = list_to_integer(B), + AV =< 100 andalso AV > 0 andalso BV =< 100 andalso BV > 0; + _ -> + false + end + end}. + +{validator, + "scheduler_absolute", + "must be 1 to 1024", + fun(Value) -> + is_integer(Value) andalso Value =< 1024 andalso Value >= 1 + end}. + {{#devrel}} %% Because of the 'merge' keyword in the proplist below, the docs and datatype %% are pulled from the leveldb schema. diff --git a/rebar.config b/rebar.config index 51e22aa09..5e4dc8c11 100644 --- a/rebar.config +++ b/rebar.config @@ -1,7 +1,7 @@ %% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- {sub_dirs, ["rel", "apps/riak"]}. -{minimum_otp_vsn, "20.0"}. +{minimum_otp_vsn, "22.0"}. {cover_enabled, true}. @@ -9,19 +9,19 @@ {eunit_opts, [nowarn_export_all, verbose]}. {deps, [ - {lager_syslog, {git, "https://github.com/basho/lager_syslog.git", {tag, "3.1.0"}}}, + {lager_syslog, {git, "https://github.com/basho/lager_syslog.git", {branch, "develop-3.0"}}}, {cluster_info, {git, "https://github.com/basho/cluster_info.git", {tag, "2.1.0"}}}, - {riak_kv, {git, "https://github.com/basho/riak_kv.git", {tag, "riak_kv-3.0.9"}}}, - {riak_auth_mods, {git, "https://github.com/basho/riak_auth_mods.git", {tag, "riak_kv-3.0.0"}}}, - {riaknostic, {git, "https://github.com/basho/riaknostic.git", {tag, "riak_kv-3.0.2"}}}, + {riak_kv, {git, "https://github.com/basho/riak_kv.git", {branch, "develop-3.0"}}}, + {riak_auth_mods, {git, "https://github.com/basho/riak_auth_mods.git", {branch, "develop-3.0"}}}, + {riaknostic, {git, "https://github.com/basho/riaknostic.git", {branch, "develop-3.0"}}}, % {yokozuna, {git, "https://github.com/basho/yokozuna.git", {branch, "develop-3.0"}}}, - {riak_repl, {git, "https://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.9"}}} + {riak_repl, {git, "https://github.com/basho/riak_repl.git", {branch, "develop-3.0"}}} ]}. {project_plugins, [ - {rebar3_cuttlefish, {git, "https://github.com/basho/rebar3_cuttlefish", {tag, "0.2.0"}}} + {rebar3_cuttlefish, {git, "https://github.com/basho/rebar3_cuttlefish", {branch, "develop-3.0"}}} ]}. {cuttlefish, [ @@ -46,7 +46,7 @@ ]} ]}. -{relx, [{release, {riak, "3.0.9"}, +{relx, [{release, {riak, "3.0.10"}, [kernel, stdlib, lager, diff --git a/rebar.lock b/rebar.lock index 1fca129c5..37f1ff748 100644 --- a/rebar.lock +++ b/rebar.lock @@ -3,18 +3,18 @@ {git,"https://github.com/basho/basho_stats.git", {ref,"c5b7ecb92af7b050bc62d08c6fbf1aafd6bdeff0"}}, 2}, - {<<"bear">>,{pkg,<<"bear">>,<<"0.8.7">>},4}, + {<<"bear">>,{pkg,<<"bear">>,<<"1.0.0">>},4}, {<<"bitcask">>, {git,"https://github.com/basho/bitcask.git", {ref,"dd96f6dd84a1cad68ccc21959ba56b0426434d85"}}, 1}, {<<"canola">>, - {git,"git://github.com/basho/canola.git", + {git,"https://github.com/basho/canola.git", {ref,"2cd39378442f3dc727de22289243b3c4d56f25fd"}}, 1}, {<<"clique">>, {git,"https://github.com/basho/clique.git", - {ref,"4014357e4e677164b890fdad08a45cfa54b3e8f3"}}, + {ref,"3e6d031ecda8a30b7679d9da47f487e202f2dc1b"}}, 2}, {<<"cluster_info">>, {git,"https://github.com/basho/cluster_info.git", @@ -22,27 +22,24 @@ 0}, {<<"cuttlefish">>, {git,"https://github.com/basho/cuttlefish.git", - {ref,"e84070b6cf08196ece2ae54083eefae4dc4e4ecc"}}, + {ref,"a70377b83c4859f418f2cae901c577ba4c085670"}}, 3}, {<<"ebloom">>, - {git,"git://github.com/basho/ebloom.git", + {git,"https://github.com/basho/ebloom.git", {ref,"3c25f9ef61a73d5f1d2c2601da464dd03975721d"}}, 1}, {<<"eleveldb">>, {git,"https://github.com/basho/eleveldb.git", - {ref,"3dee1dd7173507a4ad29df1ceb99cb70d6142478"}}, + {ref,"4962acd94662a192456b6fd5fc5d6b40f3bce8a3"}}, 2}, - {<<"exometer_core">>, - {git,"https://github.com/Feuerlabs/exometer_core.git", - {ref,"547f28bc93c7cb3d3f1174fb4c510667a4ebb645"}}, - 2}, - {<<"folsom">>,{pkg,<<"folsom">>,<<"0.8.7">>},3}, + {<<"exometer_core">>,{pkg,<<"exometer_core">>,<<"1.6.1">>},2}, + {<<"folsom">>,{pkg,<<"folsom">>,<<"1.0.0">>},3}, {<<"getopt">>, - {git,"git://github.com/jcomellas/getopt.git", + {git,"https://github.com/jcomellas/getopt.git", {ref,"fbd210550677114a6c96cdf66e340eb248908d93"}}, 1}, {<<"goldrush">>,{pkg,<<"goldrush">>,<<"0.1.9">>},2}, - {<<"hut">>,{pkg,<<"hut">>,<<"1.2.1">>},3}, + {<<"hut">>,{pkg,<<"hut">>,<<"1.3.0">>},3}, {<<"hyper">>, {git,"https://github.com/basho/hyper", {ref,"d0bf3d67fd17b8ed2610b266aebd54b1027ea916"}}, @@ -53,19 +50,19 @@ 2}, {<<"kv_index_tictactree">>, {git,"https://github.com/martinsumner/kv_index_tictactree.git", - {ref,"a3787dbf7c556cd323ae00849c1926b6dfcf15dc"}}, + {ref,"5ca15cfe24b84923c4985ff4c6d359a6489f646a"}}, 1}, {<<"lager">>, - {git,"git://github.com/erlang-lager/lager.git", + {git,"https://github.com/erlang-lager/lager.git", {ref,"22e62f28e5afabe90a6f31bcde367a2b5799fc94"}}, 1}, {<<"lager_syslog">>, {git,"https://github.com/basho/lager_syslog.git", - {ref,"152bb8eb254c225ebd8c579f9603f44b50df44ec"}}, + {ref,"f044581b21b5d6690ac90c4d631fd836160337a0"}}, 0}, {<<"leveled">>, {git,"https://github.com/martinsumner/leveled", - {ref,"086e06979e6f6d0d37b15df883c0ce2eb3c15892"}}, + {ref,"dcb6a248edeaa95bcd27b2e3957d79c5036b2185"}}, 2}, {<<"lz4">>, {git,"https://github.com/martinsumner/erlang-lz4", @@ -79,7 +76,7 @@ {git,"https://github.com/basho/mochiweb.git", {ref,"8471d2197afd412fcec239dd9ad5cbe90e64eefa"}}, 2}, - {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.3.0">>},3}, + {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.4.1">>},3}, {<<"pbkdf2">>, {git,"https://github.com/basho/erlang-pbkdf2.git", {ref,"681494c3ea03625c909a16b1b3ca295edf892057"}}, @@ -93,7 +90,7 @@ {ref,"098efad8f85dfe556d64e2cf6ce31f2075808f67"}}, 3}, {<<"ranch">>, - {git,"git://github.com/ninenines/ranch.git", + {git,"https://github.com/ninenines/ranch.git", {ref,"d0fad24657711cf584da61b5163c29fdafbd9201"}}, 1}, {<<"recon">>, @@ -106,15 +103,15 @@ 1}, {<<"riak_api">>, {git,"https://github.com/basho/riak_api.git", - {ref,"85994688946d9c201d08981e756dec564458eeb9"}}, + {ref,"e5e1fe0d2eed500f4c93f53110d9d1efdae2b675"}}, 1}, {<<"riak_auth_mods">>, {git,"https://github.com/basho/riak_auth_mods.git", - {ref,"1eccaef8e9e6200f0b75ea9312f0a731c8d572bf"}}, + {ref,"c7f16fb0a54d9f7bc96904bc38a9cc73d23f0f41"}}, 0}, {<<"riak_core">>, {git,"https://github.com/basho/riak_core.git", - {ref,"7e0aa310ecf5ec7b9fd03df61b996c2823591950"}}, + {ref,"c54233c68ada99b4cf86618d896e0b7335f3eb5c"}}, 1}, {<<"riak_dt">>, {git,"https://github.com/basho/riak_dt.git", @@ -126,11 +123,11 @@ 2}, {<<"riak_kv">>, {git,"https://github.com/basho/riak_kv.git", - {ref,"0af857d57d0206b100db59043a926df2ada8f7db"}}, + {ref,"c28826b2f22f6bf35294104898251c4d3743c6e8"}}, 0}, {<<"riak_pb">>, {git,"https://github.com/basho/riak_pb.git", - {ref,"b7abca90e4c708073021d2c4e18c896f1a10b838"}}, + {ref,"04e8cc7daa081c23be137b1eff2a6a3fb827f8e4"}}, 2}, {<<"riak_pipe">>, {git,"https://github.com/basho/riak_pipe.git", @@ -138,25 +135,25 @@ 1}, {<<"riak_repl">>, {git,"https://github.com/basho/riak_repl.git", - {ref,"c7b4fc2ae1a24b9476c297fe077f5a19c8d30a44"}}, + {ref,"e2ad0452f4946967bf504b0112ed1fb433e1c652"}}, 0}, {<<"riak_sysmon">>, {git,"https://github.com/basho/riak_sysmon.git", - {ref,"726df1f4c31108bb9366fb767b480f286e51f7fc"}}, + {ref,"750192f923435aff5dcf474b5e748cf4ca10cc1e"}}, 2}, {<<"riakc">>, {git,"https://github.com/basho/riak-erlang-client", - {ref,"d9fa46335fda4704fda71cbc011ab6a11e50e3f3"}}, + {ref,"bdfc8f0b87e09fe946d2b0f0595d94ece43c5f1b"}}, 2}, {<<"riakhttpc">>, {git,"https://github.com/basho/riak-erlang-http-client", - {ref,"fd7255150967245195105f4437904538c17614b1"}}, + {ref,"75d35da7e2617b647ede55950e246870e8ca0d7f"}}, 1}, {<<"riaknostic">>, {git,"https://github.com/basho/riaknostic.git", - {ref,"8aef63a89d719b22754683cce187eaeeecdd789c"}}, + {ref,"c86a7a943f6c5b5b743968d183f03caa98c78cf9"}}, 0}, - {<<"setup">>,{pkg,<<"setup">>,<<"2.0.2">>},3}, + {<<"setup">>,{pkg,<<"setup">>,<<"2.1.0">>},3}, {<<"sext">>, {git,"https://github.com/uwiger/sext.git", {ref,"615eebcf975ec4b4561c6f2b2bc433dabdb2be0f"}}, @@ -166,8 +163,8 @@ {ref,"d5e8f8450b92ab30610496ffb58f560d6a8bad6b"}}, 1}, {<<"syslog">>, - {git,"git://github.com/Vagabond/erlang-syslog", - {ref,"4a6c6f2c996483e86c1320e9553f91d337bcb6aa"}}, + {git,"https://github.com/basho/erlang-syslog", + {ref,"370e795feae8f5bf84a2ee76a1e1e5a766b3930f"}}, 1}, {<<"webmachine">>, {git,"https://github.com/webmachine/webmachine.git", @@ -175,10 +172,11 @@ 2}]}. [ {pkg_hash,[ - {<<"bear">>, <<"16264309AE5D005D03718A5C82641FCC259C9E8F09ADEB6FD79CA4271168656F">>}, - {<<"folsom">>, <<"A885F0AEEE4C84270954C88A55A5A473D6B2C7493E32FFDC5765412DD555A951">>}, + {<<"bear">>, <<"430419C1126B477686CDE843E88BA0F2C7DC5CDF0881C677500074F704339A99">>}, + {<<"exometer_core">>, <<"742ED00E1F10F8BEFF61A0F43E105C78C2F982E10F1C4E740D59DF89924C65EA">>}, + {<<"folsom">>, <<"50ECC998D2149939F1D5E0AA3E32788F8ED16A58E390D81B5C0BE4CC4EF25589">>}, {<<"goldrush">>, <<"F06E5D5F1277DA5C413E84D5A2924174182FB108DABB39D5EC548B27424CD106">>}, - {<<"hut">>, <<"08D46679523043424870723923971889E8A34D63B2F946A35B46CF921D1236E7">>}, - {<<"parse_trans">>, <<"09765507A3C7590A784615CFD421D101AEC25098D50B89D7AA1D66646BC571C1">>}, - {<<"setup">>, <<"1203F4CDA11306C2E34434244576DED0A7BBFB0908D9A572356C809BD0CDF085">>}]} + {<<"hut">>, <<"71F2F054E657C03F959CF1ACC43F436EA87580696528CA2A55C8AFB1B06C85E7">>}, + {<<"parse_trans">>, <<"6E6AA8167CB44CC8F39441D05193BE6E6F4E7C2946CB2759F015F8C56B76E5FF">>}, + {<<"setup">>, <<"05F69185A5EB71474C9BC6BA892565651EC7507791F85632B7B914DBFE130510">>}]} ]. From 582bbcc848ac1dff3be7a6bbe6b85d04a5b9b87f Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Sun, 29 May 2022 23:44:08 +0100 Subject: [PATCH 31/47] Updated with tags for release --- priv/riak.schema | 14 -------------- rebar.config | 12 ++++++------ rebar.lock | 28 ++++++++++++++-------------- 3 files changed, 20 insertions(+), 34 deletions(-) diff --git a/priv/riak.schema b/priv/riak.schema index d0773734f..ba72c948f 100644 --- a/priv/riak.schema +++ b/priv/riak.schema @@ -289,20 +289,6 @@ merge ]}. -%% @doc Force move on realloc for process heap -{mapping, "erlang.eheap_memory.ramv", "vm_args.+MHramv", [ - {commented, "false"}, - {datatype, {enum, [true, false]}}, - merge -]}. - -%% @doc Force move on realloc for binary heap -{mapping, "erlang.binary_memory.ramv", "vm_args.+MBramv", [ - {commented, "false"}, - {datatype, {enum, [true, false]}}, - merge -]}. - %% @doc Set allocation strategy for binary multiblock carriers. Default is %% not predictable - do not rely on aoffcbf being the default. For more info %% see: diff --git a/rebar.config b/rebar.config index 5e4dc8c11..43ee60b27 100644 --- a/rebar.config +++ b/rebar.config @@ -9,19 +9,19 @@ {eunit_opts, [nowarn_export_all, verbose]}. {deps, [ - {lager_syslog, {git, "https://github.com/basho/lager_syslog.git", {branch, "develop-3.0"}}}, + {lager_syslog, {git, "https://github.com/basho/lager_syslog.git", {tag, "3.1.1"}}}, {cluster_info, {git, "https://github.com/basho/cluster_info.git", {tag, "2.1.0"}}}, - {riak_kv, {git, "https://github.com/basho/riak_kv.git", {branch, "develop-3.0"}}}, - {riak_auth_mods, {git, "https://github.com/basho/riak_auth_mods.git", {branch, "develop-3.0"}}}, - {riaknostic, {git, "https://github.com/basho/riaknostic.git", {branch, "develop-3.0"}}}, + {riak_kv, {git, "https://github.com/basho/riak_kv.git", {tag, "riak_kv-3.0.10"}}}, + {riak_auth_mods, {git, "https://github.com/basho/riak_auth_mods.git", {tag, "riak_kv-3.0.10"}}}, + {riaknostic, {git, "https://github.com/basho/riaknostic.git", {tag, "riak_kv-3.0.10"}}}, % {yokozuna, {git, "https://github.com/basho/yokozuna.git", {branch, "develop-3.0"}}}, - {riak_repl, {git, "https://github.com/basho/riak_repl.git", {branch, "develop-3.0"}}} + {riak_repl, {git, "https://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.10"}}} ]}. {project_plugins, [ - {rebar3_cuttlefish, {git, "https://github.com/basho/rebar3_cuttlefish", {branch, "develop-3.0"}}} + {rebar3_cuttlefish, {git, "https://github.com/basho/rebar3_cuttlefish", {tag, "0.2.1"}}} ]}. {cuttlefish, [ diff --git a/rebar.lock b/rebar.lock index 37f1ff748..9ba3878e4 100644 --- a/rebar.lock +++ b/rebar.lock @@ -14,7 +14,7 @@ 1}, {<<"clique">>, {git,"https://github.com/basho/clique.git", - {ref,"3e6d031ecda8a30b7679d9da47f487e202f2dc1b"}}, + {ref,"9ad8515dfeebf7d324b6d56eab8670c0b8995ce0"}}, 2}, {<<"cluster_info">>, {git,"https://github.com/basho/cluster_info.git", @@ -30,7 +30,7 @@ 1}, {<<"eleveldb">>, {git,"https://github.com/basho/eleveldb.git", - {ref,"4962acd94662a192456b6fd5fc5d6b40f3bce8a3"}}, + {ref,"b482410fd6d996d9d71d79c2b2fb9c0ddafc6bd1"}}, 2}, {<<"exometer_core">>,{pkg,<<"exometer_core">>,<<"1.6.1">>},2}, {<<"folsom">>,{pkg,<<"folsom">>,<<"1.0.0">>},3}, @@ -50,7 +50,7 @@ 2}, {<<"kv_index_tictactree">>, {git,"https://github.com/martinsumner/kv_index_tictactree.git", - {ref,"5ca15cfe24b84923c4985ff4c6d359a6489f646a"}}, + {ref,"d491d8efd688a04354ea41ced665e7ce8d9a8658"}}, 1}, {<<"lager">>, {git,"https://github.com/erlang-lager/lager.git", @@ -58,7 +58,7 @@ 1}, {<<"lager_syslog">>, {git,"https://github.com/basho/lager_syslog.git", - {ref,"f044581b21b5d6690ac90c4d631fd836160337a0"}}, + {ref,"5c8eb6ec82fa45ff0f4e456747e095654e342746"}}, 0}, {<<"leveled">>, {git,"https://github.com/martinsumner/leveled", @@ -103,15 +103,15 @@ 1}, {<<"riak_api">>, {git,"https://github.com/basho/riak_api.git", - {ref,"e5e1fe0d2eed500f4c93f53110d9d1efdae2b675"}}, + {ref,"a4a17fc1d9a3bf2096358b2f779e8f599fa79a30"}}, 1}, {<<"riak_auth_mods">>, {git,"https://github.com/basho/riak_auth_mods.git", - {ref,"c7f16fb0a54d9f7bc96904bc38a9cc73d23f0f41"}}, + {ref,"5e4a240791f6bbadc9b3bce4653ba08cb9aeb0ce"}}, 0}, {<<"riak_core">>, {git,"https://github.com/basho/riak_core.git", - {ref,"c54233c68ada99b4cf86618d896e0b7335f3eb5c"}}, + {ref,"e8e329e684a236dea4f7719c73230e5458bc7722"}}, 1}, {<<"riak_dt">>, {git,"https://github.com/basho/riak_dt.git", @@ -119,11 +119,11 @@ 1}, {<<"riak_ensemble">>, {git,"https://github.com/basho/riak_ensemble", - {ref,"573a271aa546dcdcaeaaf09b3de6949a3130e021"}}, + {ref,"163bafa1a29db14f242e164dbe79b3fa84c3173d"}}, 2}, {<<"riak_kv">>, {git,"https://github.com/basho/riak_kv.git", - {ref,"c28826b2f22f6bf35294104898251c4d3743c6e8"}}, + {ref,"c656fb47f3933d1c0d60f3469f7b0158d0c39e54"}}, 0}, {<<"riak_pb">>, {git,"https://github.com/basho/riak_pb.git", @@ -135,19 +135,19 @@ 1}, {<<"riak_repl">>, {git,"https://github.com/basho/riak_repl.git", - {ref,"e2ad0452f4946967bf504b0112ed1fb433e1c652"}}, + {ref,"ca79f6a7714108f4d238499679b954493b1ecf06"}}, 0}, {<<"riak_sysmon">>, {git,"https://github.com/basho/riak_sysmon.git", - {ref,"750192f923435aff5dcf474b5e748cf4ca10cc1e"}}, + {ref,"bb75c2b8fd2e1074ffded60054f3a17b745957aa"}}, 2}, {<<"riakc">>, {git,"https://github.com/basho/riak-erlang-client", - {ref,"bdfc8f0b87e09fe946d2b0f0595d94ece43c5f1b"}}, + {ref,"467deeb1eee6830a6945249bc9745900766d77c0"}}, 2}, {<<"riakhttpc">>, {git,"https://github.com/basho/riak-erlang-http-client", - {ref,"75d35da7e2617b647ede55950e246870e8ca0d7f"}}, + {ref,"a99e0401ff670177f8a1a00d089db6970448ac66"}}, 1}, {<<"riaknostic">>, {git,"https://github.com/basho/riaknostic.git", @@ -164,7 +164,7 @@ 1}, {<<"syslog">>, {git,"https://github.com/basho/erlang-syslog", - {ref,"370e795feae8f5bf84a2ee76a1e1e5a766b3930f"}}, + {ref,"b6d4db6fcaf5ce1031c77a1582e3ccd24b9ae0e8"}}, 1}, {<<"webmachine">>, {git,"https://github.com/webmachine/webmachine.git", From 9df6ccf411bd0d4be4553370f8bef483ef61e9d3 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Tue, 11 Oct 2022 19:16:48 +0100 Subject: [PATCH 32/47] Update for release 3.0.11 --- rebar.config | 4 ++-- rebar.lock | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/rebar.config b/rebar.config index 43ee60b27..293c61c2f 100644 --- a/rebar.config +++ b/rebar.config @@ -11,13 +11,13 @@ {deps, [ {lager_syslog, {git, "https://github.com/basho/lager_syslog.git", {tag, "3.1.1"}}}, {cluster_info, {git, "https://github.com/basho/cluster_info.git", {tag, "2.1.0"}}}, - {riak_kv, {git, "https://github.com/basho/riak_kv.git", {tag, "riak_kv-3.0.10"}}}, + {riak_kv, {git, "https://github.com/basho/riak_kv.git", {tag, "riak_kv-3.0.11"}}}, {riak_auth_mods, {git, "https://github.com/basho/riak_auth_mods.git", {tag, "riak_kv-3.0.10"}}}, {riaknostic, {git, "https://github.com/basho/riaknostic.git", {tag, "riak_kv-3.0.10"}}}, % {yokozuna, {git, "https://github.com/basho/yokozuna.git", {branch, "develop-3.0"}}}, - {riak_repl, {git, "https://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.10"}}} + {riak_repl, {git, "https://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.11"}}} ]}. {project_plugins, [ diff --git a/rebar.lock b/rebar.lock index 9ba3878e4..3bf58b4f1 100644 --- a/rebar.lock +++ b/rebar.lock @@ -50,7 +50,7 @@ 2}, {<<"kv_index_tictactree">>, {git,"https://github.com/martinsumner/kv_index_tictactree.git", - {ref,"d491d8efd688a04354ea41ced665e7ce8d9a8658"}}, + {ref,"31e19fa1ae2aba1ae9045e9a2d1742a1476a8be2"}}, 1}, {<<"lager">>, {git,"https://github.com/erlang-lager/lager.git", @@ -62,7 +62,7 @@ 0}, {<<"leveled">>, {git,"https://github.com/martinsumner/leveled", - {ref,"dcb6a248edeaa95bcd27b2e3957d79c5036b2185"}}, + {ref,"46644271908adad5eee10c64eb446bb2bf7da3ec"}}, 2}, {<<"lz4">>, {git,"https://github.com/martinsumner/erlang-lz4", @@ -123,7 +123,7 @@ 2}, {<<"riak_kv">>, {git,"https://github.com/basho/riak_kv.git", - {ref,"c656fb47f3933d1c0d60f3469f7b0158d0c39e54"}}, + {ref,"6ebac46bf9ead05dae716b05b2a3f5767c5288db"}}, 0}, {<<"riak_pb">>, {git,"https://github.com/basho/riak_pb.git", @@ -135,7 +135,7 @@ 1}, {<<"riak_repl">>, {git,"https://github.com/basho/riak_repl.git", - {ref,"ca79f6a7714108f4d238499679b954493b1ecf06"}}, + {ref,"53b3a4fd24172d3687112e743fab5f819f6d4545"}}, 0}, {<<"riak_sysmon">>, {git,"https://github.com/basho/riak_sysmon.git", From 2d8eca0b0b551f6d4177461fd7490ea004099e53 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Tue, 11 Oct 2022 20:09:14 +0100 Subject: [PATCH 33/47] Release notes for 3.0.11 --- RELEASE-NOTES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index b09400a4b..ae8cc7102 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,3 +1,7 @@ +# Riak KV 3.0.11 Release Notes + +A simple change to [release a bottleneck](https://github.com/martinsumner/leveled/issues/379) in 2i queries with the leveled backend. Should only be relevant to those using leveled, and attempting o(1000) 2i queries per second. + # Riak KV 3.0.10 Release Notes This release is focused on improving memory management, especially with the leveled backend, and improving the efficiency and ease of configuration of tictacaae full-sync. From f753a3119d27af63e391cf035626284c6f65ecb4 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Tue, 11 Oct 2022 20:09:51 +0100 Subject: [PATCH 34/47] Update version --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 293c61c2f..304aeafce 100644 --- a/rebar.config +++ b/rebar.config @@ -46,7 +46,7 @@ ]} ]}. -{relx, [{release, {riak, "3.0.10"}, +{relx, [{release, {riak, "3.0.11"}, [kernel, stdlib, lager, From 52e6b282b9ce56815c6adf8af471e850c00615a8 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Tue, 6 Dec 2022 18:17:55 +0000 Subject: [PATCH 35/47] Temp replace reip with reip_manual (#1122) * Temp replace reip with reip_manual Reip was broken by the missing scripts necessary to generate the config so that erl could be run without the riak node active, but with the config necessary to load riak_core. Now loading riak_core is not necessary with riak_manual .. but both the path to the ring directory, and the cluster name need to be manually passed in as command params. * Update rebar.lock * Update branch --- rebar.config | 2 +- rebar.lock | 4 +-- rel/files/riak-admin | 74 +++++++++++++++++++++++++++++++++----------- 3 files changed, 59 insertions(+), 21 deletions(-) diff --git a/rebar.config b/rebar.config index 304aeafce..46ea2ae93 100644 --- a/rebar.config +++ b/rebar.config @@ -11,7 +11,7 @@ {deps, [ {lager_syslog, {git, "https://github.com/basho/lager_syslog.git", {tag, "3.1.1"}}}, {cluster_info, {git, "https://github.com/basho/cluster_info.git", {tag, "2.1.0"}}}, - {riak_kv, {git, "https://github.com/basho/riak_kv.git", {tag, "riak_kv-3.0.11"}}}, + {riak_kv, {git, "https://github.com/basho/riak_kv.git", {branch, "develop-3.0"}}}, {riak_auth_mods, {git, "https://github.com/basho/riak_auth_mods.git", {tag, "riak_kv-3.0.10"}}}, {riaknostic, {git, "https://github.com/basho/riaknostic.git", {tag, "riak_kv-3.0.10"}}}, diff --git a/rebar.lock b/rebar.lock index 3bf58b4f1..8f9e36d5c 100644 --- a/rebar.lock +++ b/rebar.lock @@ -111,7 +111,7 @@ 0}, {<<"riak_core">>, {git,"https://github.com/basho/riak_core.git", - {ref,"e8e329e684a236dea4f7719c73230e5458bc7722"}}, + {ref,"1afd61e871b08890de99a3424aec5b56c6cc2b95"}}, 1}, {<<"riak_dt">>, {git,"https://github.com/basho/riak_dt.git", @@ -123,7 +123,7 @@ 2}, {<<"riak_kv">>, {git,"https://github.com/basho/riak_kv.git", - {ref,"6ebac46bf9ead05dae716b05b2a3f5767c5288db"}}, + {ref,"159df407282e67b57cb7a43f36ed375324fcb035"}}, 0}, {<<"riak_pb">>, {git,"https://github.com/basho/riak_pb.git", diff --git a/rel/files/riak-admin b/rel/files/riak-admin index 7a4188f9d..59b94f1d8 100755 --- a/rel/files/riak-admin +++ b/rel/files/riak-admin @@ -16,9 +16,17 @@ ERTS_PATH=$BINDIR NAME_PARAM="-name" BOOT_FILE="start_clean" +# Function to validate the node is down +node_down_check() { + if relx_nodetool "ping" > /dev/null; then + echo "Node is already running!" + exit 1 + fi +} + usage() { echo "Usage: $SCRIPT { cluster | join | leave | backup | restore | test | " - echo " reip | js-reload | erl-reload | wait-for-service | " + echo " reip | reip_manual | erl-reload | wait-for-service | " echo " ringready | transfers | force-remove | down |" echo " cluster-info | member-status | ring-status | vnode-status |" echo " aae-status | diag | stat | status | transfer-limit | reformat-indexes |" @@ -881,16 +889,6 @@ case "$1" in done ;; - js[_-]reload) - # Reload all Javascript VMs - - shift #optional names come after 'js_reload' - - # Make sure the local node is running - - relx_nodetool rpc riak_kv_js_manager reload "$@" - ;; - erl[_-]reload) # Reload user Erlang code # Make sure the local node is running @@ -909,23 +907,63 @@ case "$1" in # Make sure the local node is not running node_down_check - # Sanity check the app.config file - check_config - ES=$? - if [ "$ES" -ne 0 ]; then - exit $ES + OLDNODE=$1 + NEWNODE=$2 + + echo "" + echo "Usage $SCRIPT $ACTION is not currently supported in Riak 3.0" + echo "" + echo "As an alternative use reip_manual:" + echo "riak admin reip_manual $OLDNODE $NEWNODE '/absolute_path_to/riak/data/ring' 'cluster_name'" + echo "" + echo "Where the path should be the absolute path to the ring directory in platform_data_dir containing filenames such as:" + echo "riak_core_ring.default.20221122164111" + echo "" + echo "The cluster_name should be set to the middle part of the ring file name (e.g. 'default' in this case)," + echo "or can be found when the node is active using app_helper:get_env(riak_core, cluster_name) from remote_console." + echo "" + + exit 1;; + + # check_config + # ES=$? + # if [ "$ES" -ne 0 ]; then + # exit $ES + # fi + + # OLDNODE=$1 + # NEWNODE=$2 + # $ERTS_PATH/erl -noshell \ + # -pa $RUNNER_LIB_DIR/basho-patches \ + # -boot $BOOT_FILE \ + # $CONFIG_ARGS \ + # -eval "riak_kv_console:$ACTION(['$OLDNODE', '$NEWNODE'])" \ + # -s init stop + # ;; + + reip_manual) + ACTION=$1 + shift + if [ $# -lt 4 ]; then + echo "Usage $SCRIPT $ACTION " + exit 1 fi + # Make sure the local node is not running + node_down_check + OLDNODE=$1 NEWNODE=$2 + RINGDIR=$3 + CLUSTERNAME=$4 $ERTS_PATH/erl -noshell \ -pa $RUNNER_LIB_DIR/basho-patches \ -boot $BOOT_FILE \ - $CONFIG_ARGS \ - -eval "riak_kv_console:$ACTION(['$OLDNODE', '$NEWNODE'])" \ + -eval "riak_kv_console:$ACTION(['$OLDNODE', '$NEWNODE', '$RINGDIR', '$CLUSTERNAME'])" \ -s init stop ;; + restore) ACTION=$1 shift From 92f37a603f5dccc7b89f78374d93883c94cfb0b2 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Mon, 12 Dec 2022 18:04:36 +0000 Subject: [PATCH 36/47] 3.0.12 Preparation --- RELEASE-NOTES.md | 20 ++++++++++++++++++++ THANKS | 3 +++ rebar.config | 4 ++-- rebar.lock | 14 +++++++------- 4 files changed, 32 insertions(+), 9 deletions(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index ae8cc7102..7cebd6c67 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,3 +1,23 @@ +# Riak KV 3.0.12 Release Notes + +This is a general release of small changes and fixes: + +- A major refactoring of the leveled backend with the aim of further improving memory management, by simplifying the [summary tree within each leveled_sst file](https://github.com/martinsumner/leveled/pull/383). + +- A [fix to a critical issue with the use of PR variables](https://github.com/basho/riak_kv/pull/1836) in Riak KV (when defined via bucket properties). + +- An [update to leveldb snappy compression](https://github.com/basho/eleveldb/pull/267), to make Riak compatible with a broader set of platforms (including AWS Graviton). + +- A [new `reip_manual` console command](https://github.com/basho/riak/pull/1122) has been added to use as an alternative to `reip` which does not work in the 3.0 release stream. Use `riak admin reip_manual` in place of `riak admin reip` when performing the reip operation on a node. This now requirs that the path to the ring file, and cluster name be known by the operator. + +- Some [improvements to logging within the leveled backend](https://github.com/martinsumner/leveled/pull/385) have been made to allow for better statistics monitoring of low-level operations, and reduce the cost of writing a log within leveled. + +- A fix for a deadlock issue in [riak_repl AAE fullsync](https://github.com/basho/riak_repl/pull/818) to help when clusters with high object counts have large deltas. + +- [Two](https://github.com/basho/riak_kv/pull/1839) [other](https://github.com/basho/riak_kv/pull/1837) fixes within Riak KV. + +As part of this release, further testing of the new memory configuration options added in Riak 3.0.10 has been undertaken. It is now recommended when using the leveled backend, that if memory growth in the Riak process is a signifcant concern that the following configuration option be tested: `erlang.eheap_memory.sbct = 128`. This has been shown to reduce the memory footprint of Riak, with a small performance overhead. + # Riak KV 3.0.11 Release Notes A simple change to [release a bottleneck](https://github.com/martinsumner/leveled/issues/379) in 2i queries with the leveled backend. Should only be relevant to those using leveled, and attempting o(1000) 2i queries per second. diff --git a/THANKS b/THANKS index ee4fc80ee..2dbfb1856 100644 --- a/THANKS +++ b/THANKS @@ -189,6 +189,7 @@ Eli Klein Endre H Engel A. Sanchez Enrique +Enrique Fernández Eric Eric Cestari Eric Merritt @@ -470,6 +471,7 @@ Pavlo Baron Pedram Nimreezi Pedro Larroy Pentium10 +Peter Clark Peter Garbers Peter Gribanov Peter Lemenkov @@ -562,6 +564,7 @@ Steve Pulec Steve Strong Steve Vinoski Steve Wight +Steve Wood Steven Gravell Susan Potter Takahiko Sato diff --git a/rebar.config b/rebar.config index 46ea2ae93..61be6ae94 100644 --- a/rebar.config +++ b/rebar.config @@ -17,7 +17,7 @@ % {yokozuna, {git, "https://github.com/basho/yokozuna.git", {branch, "develop-3.0"}}}, - {riak_repl, {git, "https://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.11"}}} + {riak_repl, {git, "https://github.com/basho/riak_repl.git", {branch, "mas-i817-segments"}}} ]}. {project_plugins, [ @@ -46,7 +46,7 @@ ]} ]}. -{relx, [{release, {riak, "3.0.11"}, +{relx, [{release, {riak, "3.0.12"}, [kernel, stdlib, lager, diff --git a/rebar.lock b/rebar.lock index 8f9e36d5c..7aef61ae3 100644 --- a/rebar.lock +++ b/rebar.lock @@ -30,7 +30,7 @@ 1}, {<<"eleveldb">>, {git,"https://github.com/basho/eleveldb.git", - {ref,"b482410fd6d996d9d71d79c2b2fb9c0ddafc6bd1"}}, + {ref,"edfd3924abbb2c5780d2c4641dcc0592a221ac43"}}, 2}, {<<"exometer_core">>,{pkg,<<"exometer_core">>,<<"1.6.1">>},2}, {<<"folsom">>,{pkg,<<"folsom">>,<<"1.0.0">>},3}, @@ -50,7 +50,7 @@ 2}, {<<"kv_index_tictactree">>, {git,"https://github.com/martinsumner/kv_index_tictactree.git", - {ref,"31e19fa1ae2aba1ae9045e9a2d1742a1476a8be2"}}, + {ref,"1cbf5ecb8f87c3bedda22de2b237ee29c738b117"}}, 1}, {<<"lager">>, {git,"https://github.com/erlang-lager/lager.git", @@ -62,7 +62,7 @@ 0}, {<<"leveled">>, {git,"https://github.com/martinsumner/leveled", - {ref,"46644271908adad5eee10c64eb446bb2bf7da3ec"}}, + {ref,"9fd49879f0a03879c1199c464ae5869bf65f4f9c"}}, 2}, {<<"lz4">>, {git,"https://github.com/martinsumner/erlang-lz4", @@ -91,7 +91,7 @@ 3}, {<<"ranch">>, {git,"https://github.com/ninenines/ranch.git", - {ref,"d0fad24657711cf584da61b5163c29fdafbd9201"}}, + {ref,"9b8ed47d789412b0021bfc1f94f1c17c387c721c"}}, 1}, {<<"recon">>, {git,"https://github.com/ferd/recon", @@ -111,7 +111,7 @@ 0}, {<<"riak_core">>, {git,"https://github.com/basho/riak_core.git", - {ref,"1afd61e871b08890de99a3424aec5b56c6cc2b95"}}, + {ref,"334749a86a93ce32993e4382fcc4344d0eff79cd"}}, 1}, {<<"riak_dt">>, {git,"https://github.com/basho/riak_dt.git", @@ -123,7 +123,7 @@ 2}, {<<"riak_kv">>, {git,"https://github.com/basho/riak_kv.git", - {ref,"159df407282e67b57cb7a43f36ed375324fcb035"}}, + {ref,"52e865121d4e13e63a31a26b69a1ebfd05aa6753"}}, 0}, {<<"riak_pb">>, {git,"https://github.com/basho/riak_pb.git", @@ -135,7 +135,7 @@ 1}, {<<"riak_repl">>, {git,"https://github.com/basho/riak_repl.git", - {ref,"53b3a4fd24172d3687112e743fab5f819f6d4545"}}, + {ref,"33dbc30589dc8539da4883f700dd4b851f91a62c"}}, 0}, {<<"riak_sysmon">>, {git,"https://github.com/basho/riak_sysmon.git", From e399c7410550d4ed26d91ece8cbc71ca0219f06e Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Tue, 13 Dec 2022 08:27:35 +0000 Subject: [PATCH 37/47] 3.0.12 - clarify notes --- RELEASE-NOTES.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 7cebd6c67..9bcf81bd9 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,14 +1,14 @@ # Riak KV 3.0.12 Release Notes -This is a general release of small changes and fixes: +This is a general release of changes and fixes: - A major refactoring of the leveled backend with the aim of further improving memory management, by simplifying the [summary tree within each leveled_sst file](https://github.com/martinsumner/leveled/pull/383). -- A [fix to a critical issue with the use of PR variables](https://github.com/basho/riak_kv/pull/1836) in Riak KV (when defined via bucket properties). +- A [fix to a critical issue with the use of PR variables](https://github.com/basho/riak_kv/pull/1836) in Riak KV (when the PR value is set via bucket properties). -- An [update to leveldb snappy compression](https://github.com/basho/eleveldb/pull/267), to make Riak compatible with a broader set of platforms (including AWS Graviton). +- An [update to leveldb snappy compression](https://github.com/basho/eleveldb/pull/267), to make Riak compatible with a broader set of platforms (including AWS Graviton). Because of this change, anyone building from source will now require `cmake` to be installed. -- A [new `reip_manual` console command](https://github.com/basho/riak/pull/1122) has been added to use as an alternative to `reip` which does not work in the 3.0 release stream. Use `riak admin reip_manual` in place of `riak admin reip` when performing the reip operation on a node. This now requirs that the path to the ring file, and cluster name be known by the operator. +- A [new `reip_manual` console command](https://github.com/basho/riak/pull/1122) has been added to use as an alternative to `reip`, which does not currently work in the 3.0 release stream. Use `riak admin reip_manual` in place of `riak admin reip` when performing the reip operation on a node. This now requires that the path to the ring file, and the cluster name, to be known by the operator. - Some [improvements to logging within the leveled backend](https://github.com/martinsumner/leveled/pull/385) have been made to allow for better statistics monitoring of low-level operations, and reduce the cost of writing a log within leveled. @@ -16,7 +16,7 @@ This is a general release of small changes and fixes: - [Two](https://github.com/basho/riak_kv/pull/1839) [other](https://github.com/basho/riak_kv/pull/1837) fixes within Riak KV. -As part of this release, further testing of the new memory configuration options added in Riak 3.0.10 has been undertaken. It is now recommended when using the leveled backend, that if memory growth in the Riak process is a signifcant concern that the following configuration option be tested: `erlang.eheap_memory.sbct = 128`. This has been shown to reduce the memory footprint of Riak, with a small performance overhead. +As part of this release, further testing of the new memory configuration options added in Riak 3.0.10 has been undertaken. It is now recommended when using the leveled backend, that if memory growth in the Riak process is a signifcant concern, then the following configuration option may be tested: `erlang.eheap_memory.sbct = 128`. This has been shown to reduce the memory footprint of Riak, with a small performance overhead. # Riak KV 3.0.11 Release Notes From 950e5b8078181d796a152375733fb68431c499f1 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Fri, 16 Dec 2022 16:06:50 +0000 Subject: [PATCH 38/47] Switch branch to d-3.0 --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 61be6ae94..b02a43acc 100644 --- a/rebar.config +++ b/rebar.config @@ -17,7 +17,7 @@ % {yokozuna, {git, "https://github.com/basho/yokozuna.git", {branch, "develop-3.0"}}}, - {riak_repl, {git, "https://github.com/basho/riak_repl.git", {branch, "mas-i817-segments"}}} + {riak_repl, {git, "https://github.com/basho/riak_repl.git", {branch, "develop-3.0"}}} ]}. {project_plugins, [ From c97c30fc573943aa7620cb1f466e12353f4426be Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Fri, 16 Dec 2022 16:12:32 +0000 Subject: [PATCH 39/47] Update rebar.lock --- rebar.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rebar.lock b/rebar.lock index 7aef61ae3..556c8d13e 100644 --- a/rebar.lock +++ b/rebar.lock @@ -50,7 +50,7 @@ 2}, {<<"kv_index_tictactree">>, {git,"https://github.com/martinsumner/kv_index_tictactree.git", - {ref,"1cbf5ecb8f87c3bedda22de2b237ee29c738b117"}}, + {ref,"57a92f2c40e0f5ad2f0c3ceb791302c34ef99f79"}}, 1}, {<<"lager">>, {git,"https://github.com/erlang-lager/lager.git", @@ -62,7 +62,7 @@ 0}, {<<"leveled">>, {git,"https://github.com/martinsumner/leveled", - {ref,"9fd49879f0a03879c1199c464ae5869bf65f4f9c"}}, + {ref,"0c337b881e60d06f7441ec036ffc7e47acb5682e"}}, 2}, {<<"lz4">>, {git,"https://github.com/martinsumner/erlang-lz4", @@ -123,7 +123,7 @@ 2}, {<<"riak_kv">>, {git,"https://github.com/basho/riak_kv.git", - {ref,"52e865121d4e13e63a31a26b69a1ebfd05aa6753"}}, + {ref,"4ddb2aa80959b07061b3d367031fde0f8d4c344e"}}, 0}, {<<"riak_pb">>, {git,"https://github.com/basho/riak_pb.git", @@ -135,7 +135,7 @@ 1}, {<<"riak_repl">>, {git,"https://github.com/basho/riak_repl.git", - {ref,"33dbc30589dc8539da4883f700dd4b851f91a62c"}}, + {ref,"bbbf3c4d8fdcb9d504816ecfc23caf0330bed016"}}, 0}, {<<"riak_sysmon">>, {git,"https://github.com/basho/riak_sysmon.git", From 21f5839e221124dd3843dbe51146d2a36be64530 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Mon, 19 Dec 2022 12:21:56 +0000 Subject: [PATCH 40/47] Update to tagged versions for release --- rebar.config | 4 ++-- rebar.lock | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/rebar.config b/rebar.config index b02a43acc..d8b90e00a 100644 --- a/rebar.config +++ b/rebar.config @@ -11,13 +11,13 @@ {deps, [ {lager_syslog, {git, "https://github.com/basho/lager_syslog.git", {tag, "3.1.1"}}}, {cluster_info, {git, "https://github.com/basho/cluster_info.git", {tag, "2.1.0"}}}, - {riak_kv, {git, "https://github.com/basho/riak_kv.git", {branch, "develop-3.0"}}}, + {riak_kv, {git, "https://github.com/basho/riak_kv.git", {tag, "riak_kv-3.0.12"}}}, {riak_auth_mods, {git, "https://github.com/basho/riak_auth_mods.git", {tag, "riak_kv-3.0.10"}}}, {riaknostic, {git, "https://github.com/basho/riaknostic.git", {tag, "riak_kv-3.0.10"}}}, % {yokozuna, {git, "https://github.com/basho/yokozuna.git", {branch, "develop-3.0"}}}, - {riak_repl, {git, "https://github.com/basho/riak_repl.git", {branch, "develop-3.0"}}} + {riak_repl, {git, "https://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.12"}}} ]}. {project_plugins, [ diff --git a/rebar.lock b/rebar.lock index 556c8d13e..81c271e1e 100644 --- a/rebar.lock +++ b/rebar.lock @@ -50,7 +50,7 @@ 2}, {<<"kv_index_tictactree">>, {git,"https://github.com/martinsumner/kv_index_tictactree.git", - {ref,"57a92f2c40e0f5ad2f0c3ceb791302c34ef99f79"}}, + {ref,"4cd9ab1e20772db36f593879272dff5f02fbbeda"}}, 1}, {<<"lager">>, {git,"https://github.com/erlang-lager/lager.git", @@ -103,7 +103,7 @@ 1}, {<<"riak_api">>, {git,"https://github.com/basho/riak_api.git", - {ref,"a4a17fc1d9a3bf2096358b2f779e8f599fa79a30"}}, + {ref,"53ec7feb441c910adf80d55d4a61c9989fd1afca"}}, 1}, {<<"riak_auth_mods">>, {git,"https://github.com/basho/riak_auth_mods.git", @@ -111,7 +111,7 @@ 0}, {<<"riak_core">>, {git,"https://github.com/basho/riak_core.git", - {ref,"334749a86a93ce32993e4382fcc4344d0eff79cd"}}, + {ref,"5da9f752949779b0cd28d7312fb4224a566b5a4a"}}, 1}, {<<"riak_dt">>, {git,"https://github.com/basho/riak_dt.git", @@ -123,7 +123,7 @@ 2}, {<<"riak_kv">>, {git,"https://github.com/basho/riak_kv.git", - {ref,"4ddb2aa80959b07061b3d367031fde0f8d4c344e"}}, + {ref,"1e19c3238967b8e4784ebc35e44027654399e411"}}, 0}, {<<"riak_pb">>, {git,"https://github.com/basho/riak_pb.git", @@ -131,11 +131,11 @@ 2}, {<<"riak_pipe">>, {git,"https://github.com/basho/riak_pipe.git", - {ref,"899e26181daa0fe2590d7502652c577412140fe1"}}, + {ref,"524b76d4605da0a3dda27c4d0e08258d7ed53e69"}}, 1}, {<<"riak_repl">>, {git,"https://github.com/basho/riak_repl.git", - {ref,"bbbf3c4d8fdcb9d504816ecfc23caf0330bed016"}}, + {ref,"bfcbf630cdb925b734a31072998926e14a8bb96e"}}, 0}, {<<"riak_sysmon">>, {git,"https://github.com/basho/riak_sysmon.git", From 50e687b16e248bcd2b8670a590d9c84e70ec0cc8 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Tue, 20 Dec 2022 09:02:52 +0000 Subject: [PATCH 41/47] Represent change priorities correctly --- RELEASE-NOTES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 9bcf81bd9..7019514d6 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -2,10 +2,10 @@ This is a general release of changes and fixes: -- A major refactoring of the leveled backend with the aim of further improving memory management, by simplifying the [summary tree within each leveled_sst file](https://github.com/martinsumner/leveled/pull/383). - - A [fix to a critical issue with the use of PR variables](https://github.com/basho/riak_kv/pull/1836) in Riak KV (when the PR value is set via bucket properties). +- Changes to the leveled backend with the aim of further improving memory management, by simplifying the [summary tree within each leveled_sst file](https://github.com/martinsumner/leveled/pull/383). + - An [update to leveldb snappy compression](https://github.com/basho/eleveldb/pull/267), to make Riak compatible with a broader set of platforms (including AWS Graviton). Because of this change, anyone building from source will now require `cmake` to be installed. - A [new `reip_manual` console command](https://github.com/basho/riak/pull/1122) has been added to use as an alternative to `reip`, which does not currently work in the 3.0 release stream. Use `riak admin reip_manual` in place of `riak admin reip` when performing the reip operation on a node. This now requires that the path to the ring file, and the cluster name, to be known by the operator. From 967478a9d4c60f200e6612facd63f2f5cb1f9ec4 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Sat, 4 Feb 2023 13:55:15 +0000 Subject: [PATCH 42/47] Mas 3.0.13 preview (#1126) * planned combination for 3.0.13 https://github.com/basho/eleveldb/pull/269 to be added once issues are resolved * Release preparation - 3.0.13 * For testing of fix to KV1847 * Update with reference to conditional PUT changes * Update riak-chkconfig As per https://github.com/basho/riak/issues/1125 * Update following merge * Tags for release --- RELEASE-NOTES.md | 20 ++++++++++++++++++++ rebar.config | 6 +++--- rebar.lock | 18 +++++++++--------- rel/files/riak-chkconfig | 3 +++ 4 files changed, 35 insertions(+), 12 deletions(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 7019514d6..ed3d1af2f 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,3 +1,23 @@ +# Riak KV 3.0.13 Release Notes + +This release is focused on improving the reliability of handoffs. The speed of handoffs is critical to the recovery times of nodes following failure, and also to the time necessary to expand or contract the cluster. Controlling the speed can be managed by increasing concurrency (using `riak admin transfer-limit `), but this can often lead to handoff unreliability due to timeouts. + +- The main [handoff process has been refactored](https://github.com/basho/riak_core/pull/995) to remove some deprecated messages, simplify the naming of configuration items, improve the logging of handoffs and increase the frequency of handoff sync messages. More frequent sync messages should make the flow of handoffs from the sender more responsive to pressure at the receiver. + +- On joining a node to a cluster, there will now be an attempt to exchange metadata with the joining node before the join is staged. This should reduce the probability of events failing immediately after join, as bucket types had not yet been replicated to the joining node. + +- A [new configuration option has been added](https://github.com/basho/riak_kv/blob/fbb53630645e53af053228d526caa3c86f304066/priv/riak_kv.schema#L1469-L1487) to change Riak to only commit read repair on primary (not fallback) vnodes. In short-term failures, enabling this option will reduce the time taken for hinted handoff to complete following recovery of a failed node - as now the handoff will contain only objects changed for that partition during the outage. With the default setting of `disabled`; fallback nodes will also contain each object fetched during the outage, in-line with the behaviour in previous releases. + +- A [fix has been implemented in leveled](https://github.com/martinsumner/leveled/pull/390) to reduce failures and inefficiencies when re-building the ledger key store from the object Journal, in situations where there has been a high volume of object churn. + +- Some [helper functions have been added to riak_client](https://github.com/basho/riak_kv/blob/fbb53630645e53af053228d526caa3c86f304066/src/riak_client.erl#L949-L993), to simplify some operational tasks. These functions can be called from `riak remote_console`, e.g. `riak_client:repair_node().` - which replaces the series of commands previously required to run partition repair across all partitions on a node. `riak_client:tictacaae_suspend_node().` may be used to suspend Tictacaae AAE exchanges on a node following a failure, so that they can be re-enabled using `riak_client:tictacaae_resume_node().` once handoffs have been completed. + +There are still [outstanding](https://github.com/basho/riak_kv/issues/1846) [issues](https://github.com/basho/riak_core/issues/996) related to handoffs. + +The release also includes a [significant change to the HTTP API](https://github.com/basho/riak_kv/issues/1849). In previous releases PUT, POST and DELETE requests would all GET the object prior to starting the PUT process. This is in contrast to the Protocol Buffers API which would only GET the object in case where conditions were passed in the put (e.g. `if_none_match` or `if_not_modified`). These two APIs now have the same non-functional behaviour, the HTTP API will no longer request a GET before the PUT if the request does not contain a condition (e.g. using `If-None-Match`, `If-Match`, `If-Modified-Since` as well as a new bespoke condition `X-Riak-If-Not-Modified`). + +A vector clock being passed on a PUT using the `X-Riak-If-Not-Modified` header, will return a `409:Conflict` should the passed vector clock not match the clock found prior to updating the object. This will work as the PB API `if_not_modified` option. This is still an eventually consistent condition, parallel updates may still lead to siblings when `{allow_mult, true}`. + # Riak KV 3.0.12 Release Notes This is a general release of changes and fixes: diff --git a/rebar.config b/rebar.config index d8b90e00a..3ba4804b7 100644 --- a/rebar.config +++ b/rebar.config @@ -11,13 +11,13 @@ {deps, [ {lager_syslog, {git, "https://github.com/basho/lager_syslog.git", {tag, "3.1.1"}}}, {cluster_info, {git, "https://github.com/basho/cluster_info.git", {tag, "2.1.0"}}}, - {riak_kv, {git, "https://github.com/basho/riak_kv.git", {tag, "riak_kv-3.0.12"}}}, + {riak_kv, {git, "https://github.com/basho/riak_kv.git", {tag, "riak_kv-3.0.13"}}}, {riak_auth_mods, {git, "https://github.com/basho/riak_auth_mods.git", {tag, "riak_kv-3.0.10"}}}, {riaknostic, {git, "https://github.com/basho/riaknostic.git", {tag, "riak_kv-3.0.10"}}}, % {yokozuna, {git, "https://github.com/basho/yokozuna.git", {branch, "develop-3.0"}}}, - {riak_repl, {git, "https://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.12"}}} + {riak_repl, {git, "https://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.13"}}} ]}. {project_plugins, [ @@ -46,7 +46,7 @@ ]} ]}. -{relx, [{release, {riak, "3.0.12"}, +{relx, [{release, {riak, "3.0.13"}, [kernel, stdlib, lager, diff --git a/rebar.lock b/rebar.lock index 81c271e1e..94fb82039 100644 --- a/rebar.lock +++ b/rebar.lock @@ -50,7 +50,7 @@ 2}, {<<"kv_index_tictactree">>, {git,"https://github.com/martinsumner/kv_index_tictactree.git", - {ref,"4cd9ab1e20772db36f593879272dff5f02fbbeda"}}, + {ref,"1d982aeaf6bd57c6c00fcad67e90c27c58edcfd1"}}, 1}, {<<"lager">>, {git,"https://github.com/erlang-lager/lager.git", @@ -62,7 +62,7 @@ 0}, {<<"leveled">>, {git,"https://github.com/martinsumner/leveled", - {ref,"0c337b881e60d06f7441ec036ffc7e47acb5682e"}}, + {ref,"ee6c166b9f49884340970109f5365f581db87688"}}, 2}, {<<"lz4">>, {git,"https://github.com/martinsumner/erlang-lz4", @@ -103,7 +103,7 @@ 1}, {<<"riak_api">>, {git,"https://github.com/basho/riak_api.git", - {ref,"53ec7feb441c910adf80d55d4a61c9989fd1afca"}}, + {ref,"0f9c5783f87f3c1797fa7b3ca5088c6c3a74a13f"}}, 1}, {<<"riak_auth_mods">>, {git,"https://github.com/basho/riak_auth_mods.git", @@ -111,7 +111,7 @@ 0}, {<<"riak_core">>, {git,"https://github.com/basho/riak_core.git", - {ref,"5da9f752949779b0cd28d7312fb4224a566b5a4a"}}, + {ref,"ad546ed1ce85b48aada8446cc2ce75a129103b32"}}, 1}, {<<"riak_dt">>, {git,"https://github.com/basho/riak_dt.git", @@ -123,7 +123,7 @@ 2}, {<<"riak_kv">>, {git,"https://github.com/basho/riak_kv.git", - {ref,"1e19c3238967b8e4784ebc35e44027654399e411"}}, + {ref,"e04bc998e2f48e083a875e290ca242f2d699830d"}}, 0}, {<<"riak_pb">>, {git,"https://github.com/basho/riak_pb.git", @@ -131,11 +131,11 @@ 2}, {<<"riak_pipe">>, {git,"https://github.com/basho/riak_pipe.git", - {ref,"524b76d4605da0a3dda27c4d0e08258d7ed53e69"}}, + {ref,"bd2240266acfa824ea1fe5f3a2b2192e92e822d3"}}, 1}, {<<"riak_repl">>, {git,"https://github.com/basho/riak_repl.git", - {ref,"bfcbf630cdb925b734a31072998926e14a8bb96e"}}, + {ref,"dd890e83498fcb254de485e2a3aa195f67fbe59c"}}, 0}, {<<"riak_sysmon">>, {git,"https://github.com/basho/riak_sysmon.git", @@ -143,11 +143,11 @@ 2}, {<<"riakc">>, {git,"https://github.com/basho/riak-erlang-client", - {ref,"467deeb1eee6830a6945249bc9745900766d77c0"}}, + {ref,"3b6ad6e92dcb55833e9add829b2871be8468a68e"}}, 2}, {<<"riakhttpc">>, {git,"https://github.com/basho/riak-erlang-http-client", - {ref,"a99e0401ff670177f8a1a00d089db6970448ac66"}}, + {ref,"20cfdf4eb1f2e2f8521fd6c974d7b3e2bd9db728"}}, 1}, {<<"riaknostic">>, {git,"https://github.com/basho/riaknostic.git", diff --git a/rel/files/riak-chkconfig b/rel/files/riak-chkconfig index 9d521460e..649e6e8e0 100644 --- a/rel/files/riak-chkconfig +++ b/rel/files/riak-chkconfig @@ -21,3 +21,6 @@ CODE=" try $BINDIR/erl -noshell -boot start_clean -eval "$CODE" echo $CUTTLE_CONF + +# No explicit exit, assumed to have succeeded +exit 0 From 994efa876132cca9d87bfa6d9cb52f572f004c8d Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Mon, 13 Feb 2023 10:37:16 +0000 Subject: [PATCH 43/47] Mas 3.0.14 preview (#1127) Release 3.0.14 --- RELEASE-NOTES.md | 8 ++++++++ rebar.config | 6 +++--- rebar.lock | 14 +++++++------- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index ed3d1af2f..74662c454 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,3 +1,11 @@ +# Riak KV 3.0.14 Release Notes + +This release [fixes an issue](https://github.com/martinsumner/leveled/issues/393) whereby a failure to signal and handle back-pressure correctly by the leveled backend can cause a backlog within the store. In particular this can be triggered by handoffs (e.g. due to cluster admin operations), and lead to partition transfers stalling almost completely. The issue existed in previous releases, by may have been exacerbated by refactoring in [Riak KV 3.0.13](#riak-kv-3013-release-notes). + +An additional [minor improvement has been made to handoffs](https://github.com/basho/riak_kv/pull/1851). Previously requests to reap tombstones after deletions (where the delete_mode is not `keep`), would not be forwarded during handoffs. These tombstones would then need to be corrected by AAE (which may result in a permanent tombstone). There is now a configuration option `handoff_deletes` which can be enabled to ensure these reap requests are forwarded, reducing the AAE work required on handoff completion. + +Desipite the handoff improvements in [Riak KV 3.0.13](#riak-kv-3013-release-notes), handoff timeouts are still possible. If handoff timeouts do occur, then the first stage should be to reduce the [handoff batch threshold count](https://github.com/basho/riak_core/blob/riak_kv-3.0.14/priv/riak_core.schema#L47-L55) to a lower number than that of [the item_count in the handoff sender log](https://github.com/basho/riak_core/blob/riak_kv-3.0.14/src/riak_core_handoff_sender.erl#L474-L486). + # Riak KV 3.0.13 Release Notes This release is focused on improving the reliability of handoffs. The speed of handoffs is critical to the recovery times of nodes following failure, and also to the time necessary to expand or contract the cluster. Controlling the speed can be managed by increasing concurrency (using `riak admin transfer-limit `), but this can often lead to handoff unreliability due to timeouts. diff --git a/rebar.config b/rebar.config index 3ba4804b7..5f200062b 100644 --- a/rebar.config +++ b/rebar.config @@ -11,13 +11,13 @@ {deps, [ {lager_syslog, {git, "https://github.com/basho/lager_syslog.git", {tag, "3.1.1"}}}, {cluster_info, {git, "https://github.com/basho/cluster_info.git", {tag, "2.1.0"}}}, - {riak_kv, {git, "https://github.com/basho/riak_kv.git", {tag, "riak_kv-3.0.13"}}}, + {riak_kv, {git, "https://github.com/basho/riak_kv.git", {tag, "riak_kv-3.0.14"}}}, {riak_auth_mods, {git, "https://github.com/basho/riak_auth_mods.git", {tag, "riak_kv-3.0.10"}}}, {riaknostic, {git, "https://github.com/basho/riaknostic.git", {tag, "riak_kv-3.0.10"}}}, % {yokozuna, {git, "https://github.com/basho/yokozuna.git", {branch, "develop-3.0"}}}, - {riak_repl, {git, "https://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.13"}}} + {riak_repl, {git, "https://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.14"}}} ]}. {project_plugins, [ @@ -46,7 +46,7 @@ ]} ]}. -{relx, [{release, {riak, "3.0.13"}, +{relx, [{release, {riak, "3.0.14"}, [kernel, stdlib, lager, diff --git a/rebar.lock b/rebar.lock index 94fb82039..8d013331a 100644 --- a/rebar.lock +++ b/rebar.lock @@ -50,7 +50,7 @@ 2}, {<<"kv_index_tictactree">>, {git,"https://github.com/martinsumner/kv_index_tictactree.git", - {ref,"1d982aeaf6bd57c6c00fcad67e90c27c58edcfd1"}}, + {ref,"cd192703c2464bc0f0da12177b077c000c12b392"}}, 1}, {<<"lager">>, {git,"https://github.com/erlang-lager/lager.git", @@ -62,7 +62,7 @@ 0}, {<<"leveled">>, {git,"https://github.com/martinsumner/leveled", - {ref,"ee6c166b9f49884340970109f5365f581db87688"}}, + {ref,"bd6ec2fdb591aa199e18b1d4abeb3fe8e037926c"}}, 2}, {<<"lz4">>, {git,"https://github.com/martinsumner/erlang-lz4", @@ -103,7 +103,7 @@ 1}, {<<"riak_api">>, {git,"https://github.com/basho/riak_api.git", - {ref,"0f9c5783f87f3c1797fa7b3ca5088c6c3a74a13f"}}, + {ref,"d38c4c33b2d803ba1488017e5942c240ffa46c9b"}}, 1}, {<<"riak_auth_mods">>, {git,"https://github.com/basho/riak_auth_mods.git", @@ -111,7 +111,7 @@ 0}, {<<"riak_core">>, {git,"https://github.com/basho/riak_core.git", - {ref,"ad546ed1ce85b48aada8446cc2ce75a129103b32"}}, + {ref,"10cad320852a66d9d3c81a1f3745e7e6a5286cc2"}}, 1}, {<<"riak_dt">>, {git,"https://github.com/basho/riak_dt.git", @@ -123,7 +123,7 @@ 2}, {<<"riak_kv">>, {git,"https://github.com/basho/riak_kv.git", - {ref,"e04bc998e2f48e083a875e290ca242f2d699830d"}}, + {ref,"cdaf80eb75c2070cd569c88b0cb3e47e140a39de"}}, 0}, {<<"riak_pb">>, {git,"https://github.com/basho/riak_pb.git", @@ -131,11 +131,11 @@ 2}, {<<"riak_pipe">>, {git,"https://github.com/basho/riak_pipe.git", - {ref,"bd2240266acfa824ea1fe5f3a2b2192e92e822d3"}}, + {ref,"0fc3833b0dfb1166753b67af460eb8af6b8af1bf"}}, 1}, {<<"riak_repl">>, {git,"https://github.com/basho/riak_repl.git", - {ref,"dd890e83498fcb254de485e2a3aa195f67fbe59c"}}, + {ref,"a1b5ec0836d2ed4d24e00012a0b405a7e51c94ff"}}, 0}, {<<"riak_sysmon">>, {git,"https://github.com/basho/riak_sysmon.git", From 6eefccca85ea0d7131b08a9e5a396f2e201d5c9a Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Wed, 15 Feb 2023 09:35:36 +0000 Subject: [PATCH 44/47] Riak 3.0.15 Preview (#1128) --- RELEASE-NOTES.md | 4 ++++ rebar.config | 6 +++--- rebar.lock | 4 ++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 74662c454..2e1f75cf3 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,3 +1,7 @@ +# Riak KV 3.0.15 Release Notes + +Fix to an issue introduced with the `auto_check` feature for TictacAAE full-sync in [Riak KV 3.0.10](#riak-kv-3010-release-notes). + # Riak KV 3.0.14 Release Notes This release [fixes an issue](https://github.com/martinsumner/leveled/issues/393) whereby a failure to signal and handle back-pressure correctly by the leveled backend can cause a backlog within the store. In particular this can be triggered by handoffs (e.g. due to cluster admin operations), and lead to partition transfers stalling almost completely. The issue existed in previous releases, by may have been exacerbated by refactoring in [Riak KV 3.0.13](#riak-kv-3013-release-notes). diff --git a/rebar.config b/rebar.config index 5f200062b..40dd81d8c 100644 --- a/rebar.config +++ b/rebar.config @@ -11,13 +11,13 @@ {deps, [ {lager_syslog, {git, "https://github.com/basho/lager_syslog.git", {tag, "3.1.1"}}}, {cluster_info, {git, "https://github.com/basho/cluster_info.git", {tag, "2.1.0"}}}, - {riak_kv, {git, "https://github.com/basho/riak_kv.git", {tag, "riak_kv-3.0.14"}}}, + {riak_kv, {git, "https://github.com/basho/riak_kv.git", {tag, "riak_kv-3.0.15"}}}, {riak_auth_mods, {git, "https://github.com/basho/riak_auth_mods.git", {tag, "riak_kv-3.0.10"}}}, {riaknostic, {git, "https://github.com/basho/riaknostic.git", {tag, "riak_kv-3.0.10"}}}, % {yokozuna, {git, "https://github.com/basho/yokozuna.git", {branch, "develop-3.0"}}}, - {riak_repl, {git, "https://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.14"}}} + {riak_repl, {git, "https://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.15"}}} ]}. {project_plugins, [ @@ -46,7 +46,7 @@ ]} ]}. -{relx, [{release, {riak, "3.0.14"}, +{relx, [{release, {riak, "3.0.15"}, [kernel, stdlib, lager, diff --git a/rebar.lock b/rebar.lock index 8d013331a..363f9040e 100644 --- a/rebar.lock +++ b/rebar.lock @@ -123,7 +123,7 @@ 2}, {<<"riak_kv">>, {git,"https://github.com/basho/riak_kv.git", - {ref,"cdaf80eb75c2070cd569c88b0cb3e47e140a39de"}}, + {ref,"ed93f69c1e345620534c36282ba7fc08f8d9ed26"}}, 0}, {<<"riak_pb">>, {git,"https://github.com/basho/riak_pb.git", @@ -135,7 +135,7 @@ 1}, {<<"riak_repl">>, {git,"https://github.com/basho/riak_repl.git", - {ref,"a1b5ec0836d2ed4d24e00012a0b405a7e51c94ff"}}, + {ref,"1b3f602861a5e8045e13fcd8b1f4f5efae5cfefa"}}, 0}, {<<"riak_sysmon">>, {git,"https://github.com/basho/riak_sysmon.git", From cf9850e2a4d7b4b2f5fadb3061e73709bbefcad9 Mon Sep 17 00:00:00 2001 From: Martin Sumner Date: Thu, 22 Jun 2023 20:08:33 +0100 Subject: [PATCH 45/47] Mas 3.0.16 preview (#1138) * Update deps for 3.0.16 test * Update deps * Prepare for release Now with riak_core claim algorithm update * Add addendum to 3.0.12 release notes to highlight snappy issue * Update RELEASE-NOTES.md * Update to tagged repos --- RELEASE-NOTES.md | 20 +++++++++++++++++++- rebar.config | 6 +++--- rebar.lock | 14 +++++++------- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 2e1f75cf3..2b89deaeb 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,10 +1,22 @@ +# Riak KV 3.0.16 Release Notes + +This release includes the following updates: + +- A significant new [riak_core cluster claim algorithm](https://github.com/basho/riak_core/pull/1008) has been added - `choose_claim_v4`. The default cluster claim algorithm (`choose_claim_v2`) is unchanged, but the `choose_claim_fun` configuration option in riak.conf can be used to enable the new algorithm. The new algorithm offers significant improvements in the claim process, especially when location awareness is enabled, with the algorithm now more likely to find valid solutions and with fewer required transfers. The trade-off when compared to `choose_claim_v2` is that in some situations computational overhead of claim may increase by multiple orders of magnitude, leading to long cluster plan times. + +- The leveled backend has been updated to reduce the memory required for stores with a large number of keys, and also reduce volatility in the memory demanded, by [optimising the memory footprint of the SST process working heap](https://github.com/martinsumner/leveled/pull/403). + +- A collection of fixes have been added. A fix to timeouts in [leveled hot backups](https://github.com/martinsumner/leveled/pull/406). A fix to reap and erase queries to [prevent a backlog from overloading riak_kv_eraser or riak_kv_reaper mailboxes](https://github.com/basho/riak_kv/pull/1862) (creating rapidly increasing memory footprint). A fix to an issue with [nextgen replication when node_confirms is enabled](https://github.com/basho/riak_kv/pull/1858). The [R/W value used by nextgen replication is now configurable](https://github.com/basho/riak_kv/pull/1860), and this may be useful for preventing a backlog of replication events from overloading a cluster. + +As a result of the memory management improvements made in 3.0.16, the recommendation to consider altering the eheap single-block carrier threshold made in [Riak 3.0.12](#riak-kv-3012-release-notes) has been deprecated. With the leveled backend, memory management should now be efficient on default settings, although with a small overhead on the latency of HEAD operations in that backend. + # Riak KV 3.0.15 Release Notes Fix to an issue introduced with the `auto_check` feature for TictacAAE full-sync in [Riak KV 3.0.10](#riak-kv-3010-release-notes). # Riak KV 3.0.14 Release Notes -This release [fixes an issue](https://github.com/martinsumner/leveled/issues/393) whereby a failure to signal and handle back-pressure correctly by the leveled backend can cause a backlog within the store. In particular this can be triggered by handoffs (e.g. due to cluster admin operations), and lead to partition transfers stalling almost completely. The issue existed in previous releases, by may have been exacerbated by refactoring in [Riak KV 3.0.13](#riak-kv-3013-release-notes). +This release [fixes an issue](https://github.com/martinsumner/leveled/issues/393) whereby a failure to signal and handle back-pressure correctly by the leveled backend can cause a backlog within the store. In particular this can be triggered by handoffs (e.g. due to cluster admin operations), and lead to partition transfers stalling almost completely. The issue existed in previous releases, but may have been exacerbated by refactoring in [Riak KV 3.0.13](#riak-kv-3013-release-notes). An additional [minor improvement has been made to handoffs](https://github.com/basho/riak_kv/pull/1851). Previously requests to reap tombstones after deletions (where the delete_mode is not `keep`), would not be forwarded during handoffs. These tombstones would then need to be corrected by AAE (which may result in a permanent tombstone). There is now a configuration option `handoff_deletes` which can be enabled to ensure these reap requests are forwarded, reducing the AAE work required on handoff completion. @@ -50,6 +62,12 @@ This is a general release of changes and fixes: As part of this release, further testing of the new memory configuration options added in Riak 3.0.10 has been undertaken. It is now recommended when using the leveled backend, that if memory growth in the Riak process is a signifcant concern, then the following configuration option may be tested: `erlang.eheap_memory.sbct = 128`. This has been shown to reduce the memory footprint of Riak, with a small performance overhead. +## 3.0.12 Addendum + +Although not originally advertised in the release, the version of eleveldb used in 3.0.12 includes an upgrade of the [`snappy` compression algorithm used within eleveldb](https://github.com/basho/eleveldb/pull/267). This version of snappy is incompatible with the previous version used, and so any update to 3.0.12 and beyond where the eleveldb backend is deployed with `snappy` compression [will fail](https://github.com/basho/eleveldb/issues/273). + +To update to 3.0.12 in this case, the change should be rolled into the cluster using the ` riak admin cluster replace` command. + # Riak KV 3.0.11 Release Notes A simple change to [release a bottleneck](https://github.com/martinsumner/leveled/issues/379) in 2i queries with the leveled backend. Should only be relevant to those using leveled, and attempting o(1000) 2i queries per second. diff --git a/rebar.config b/rebar.config index 40dd81d8c..0bf6d152e 100644 --- a/rebar.config +++ b/rebar.config @@ -11,13 +11,13 @@ {deps, [ {lager_syslog, {git, "https://github.com/basho/lager_syslog.git", {tag, "3.1.1"}}}, {cluster_info, {git, "https://github.com/basho/cluster_info.git", {tag, "2.1.0"}}}, - {riak_kv, {git, "https://github.com/basho/riak_kv.git", {tag, "riak_kv-3.0.15"}}}, + {riak_kv, {git, "https://github.com/basho/riak_kv.git", {tag, "riak_kv-3.0.16"}}}, {riak_auth_mods, {git, "https://github.com/basho/riak_auth_mods.git", {tag, "riak_kv-3.0.10"}}}, {riaknostic, {git, "https://github.com/basho/riaknostic.git", {tag, "riak_kv-3.0.10"}}}, % {yokozuna, {git, "https://github.com/basho/yokozuna.git", {branch, "develop-3.0"}}}, - {riak_repl, {git, "https://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.15"}}} + {riak_repl, {git, "https://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.16"}}} ]}. {project_plugins, [ @@ -46,7 +46,7 @@ ]} ]}. -{relx, [{release, {riak, "3.0.15"}, +{relx, [{release, {riak, "3.0.16"}, [kernel, stdlib, lager, diff --git a/rebar.lock b/rebar.lock index 363f9040e..b14805352 100644 --- a/rebar.lock +++ b/rebar.lock @@ -50,7 +50,7 @@ 2}, {<<"kv_index_tictactree">>, {git,"https://github.com/martinsumner/kv_index_tictactree.git", - {ref,"cd192703c2464bc0f0da12177b077c000c12b392"}}, + {ref,"50302987d33689fb2066c831ec7376f11d4304c9"}}, 1}, {<<"lager">>, {git,"https://github.com/erlang-lager/lager.git", @@ -62,7 +62,7 @@ 0}, {<<"leveled">>, {git,"https://github.com/martinsumner/leveled", - {ref,"bd6ec2fdb591aa199e18b1d4abeb3fe8e037926c"}}, + {ref,"4142914e3fc32090747a4a7ea63087e18a74a042"}}, 2}, {<<"lz4">>, {git,"https://github.com/martinsumner/erlang-lz4", @@ -103,7 +103,7 @@ 1}, {<<"riak_api">>, {git,"https://github.com/basho/riak_api.git", - {ref,"d38c4c33b2d803ba1488017e5942c240ffa46c9b"}}, + {ref,"1ccf72103a154493e952e7a544ea3429cb342b20"}}, 1}, {<<"riak_auth_mods">>, {git,"https://github.com/basho/riak_auth_mods.git", @@ -111,7 +111,7 @@ 0}, {<<"riak_core">>, {git,"https://github.com/basho/riak_core.git", - {ref,"10cad320852a66d9d3c81a1f3745e7e6a5286cc2"}}, + {ref,"7b04e2c4731c16ffb9cfbe1ac085cb4056cb0422"}}, 1}, {<<"riak_dt">>, {git,"https://github.com/basho/riak_dt.git", @@ -123,7 +123,7 @@ 2}, {<<"riak_kv">>, {git,"https://github.com/basho/riak_kv.git", - {ref,"ed93f69c1e345620534c36282ba7fc08f8d9ed26"}}, + {ref,"e80a0699ffc5bc004b93e06c09197b79109d6844"}}, 0}, {<<"riak_pb">>, {git,"https://github.com/basho/riak_pb.git", @@ -131,11 +131,11 @@ 2}, {<<"riak_pipe">>, {git,"https://github.com/basho/riak_pipe.git", - {ref,"0fc3833b0dfb1166753b67af460eb8af6b8af1bf"}}, + {ref,"abedf61218d4ed61a3494a12e95bd4de0277468d"}}, 1}, {<<"riak_repl">>, {git,"https://github.com/basho/riak_repl.git", - {ref,"1b3f602861a5e8045e13fcd8b1f4f5efae5cfefa"}}, + {ref,"dab42494f35b5102925138b28e8a1b0492947287"}}, 0}, {<<"riak_sysmon">>, {git,"https://github.com/basho/riak_sysmon.git", From 6ae237d686f2218b9f388d8711dd897a73206447 Mon Sep 17 00:00:00 2001 From: nsaadouni Date: Fri, 18 Aug 2023 14:58:39 +0100 Subject: [PATCH 46/47] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dad7615ed..d94f062f6 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,6 @@ To create a local multi-node build environment use `make devclean; make devrel`. To test Riak use [Riak Test](https://github.com/basho/riak_test/blob/develop-3.0/doc/SIMPLE_SETUP.md). -Up to date documentation is not available, but work on [documentation](https://www.tiot.jp/riak-docs/riak/kv/2.9.10/) is ongoing and the core information available in the [legacy documentation](https://docs.riak.com/riak/kv/latest/index.html) is still generally relevant. +[legacy documentation](https://docs.riak.com/riak/kv/latest/index.html) is still generally relevant. Issues and PRs can be tracked via [Riak Github](https://github.com/basho/riak/issues) or [Riak KV Github](https://github.com/basho/riak_kv/issues). From d43e7413e1792794ef6849311c12e7f6eacd8fbc Mon Sep 17 00:00:00 2001 From: Nicholas Adams Date: Fri, 25 Aug 2023 23:02:34 +0900 Subject: [PATCH 47/47] Update riak-debug for Riak KV 3.x Bug fixes and feature improvement update - Fix a bug when retrieving leveled backend info. - Fix attempt for some OS which occasionally encounter a bug where `riak debug` attempts to write to the Riak data directory instead of the current folder. - Improve OS identification. - Avoid running blockdev on snapd mounts. --- rel/files/riak-debug | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/rel/files/riak-debug b/rel/files/riak-debug index 24b3e4ce9..1de29390b 100755 --- a/rel/files/riak-debug +++ b/rel/files/riak-debug @@ -141,6 +141,7 @@ riak_base_dir={{runner_base_dir}} riak_bin_dir={{runner_script_dir}} riak_etc_dir={{runner_etc_dir}} riak_sbin_dir=/usr/sbin +curdir=`pwd` get_cfgs=0 get_ssl_certs=0 @@ -334,7 +335,7 @@ fi if [ -z "$outfile" ]; then # If output file not specified, output to the default - outfile="`pwd`"/"${debug_dir}".tar.gz + outfile="${curdir}"/"${debug_dir}".tar.gz fi if [ '-' != "$outfile" ] && [ -f "$outfile" ]; then @@ -357,7 +358,8 @@ if [ 1 -eq $get_syscmds ]; then dump last last dump hostname hostname dump uname uname -a - dump lsb_release lsb_release + dump lsb_release lsb_release -a + dump hostnamectl hostnamectl dump ps ps aux dump vmstat vmstat 1 5 dump free free -m @@ -389,7 +391,7 @@ if [ 1 -eq $get_syscmds ]; then BLOCKDEV=/sbin/blockdev if [ -e $BLOCKDEV ]; then - for mount_point in `mount | egrep '^/' | awk '{print $1}'`; do + for mount_point in `mount | egrep '^/' | grep -v /dev/shm | grep -v /var/lib/snapd | awk '{print $1}'`; do flat_point=`echo $mount_point | sed 's:/:_:g'` dump blockdev.$flat_point $BLOCKDEV --getra $mount_point done @@ -429,6 +431,13 @@ if [ 1 -eq $get_syscmds ]; then [ -f /etc/release ] && dump release cat /etc/release [ -f /etc/redhat-release ] && dump redhat_release cat /etc/redhat-release [ -f /etc/debian_version ] && dump debian_version cat /etc/debian_version + [ -f /etc/amazon-release ] && dump amazon_release cat /etc/amazon-release + [ -f /etc/ec2_version ] && dump ec2_version cat /etc/ec2_version + [ -f /etc/centos-release ] && dump centos_release cat /etc/centos-release + [ -f /etc/oracle-release ] && dump oracle_release cat /etc/oracle-release + [ -f /etc/os-release ] && dump os_release cat /etc/os-release + [ -f /etc/issue ] && dump issue cat /etc/issue + [ -f /etc/system-release ] && dump system_release cat /etc/system-release [ -f /etc/security/limits.conf ] && dump limits.conf cat /etc/security/limits.conf [ -f /var/log/messages ] && dump messages cat /var/log/messages [ -f /var/log/syslog ] && dump messages cat /var/log/syslog @@ -729,12 +738,12 @@ if [ 1 -eq $get_logs ]; then elif [ 'riak_kv_leveled_backend' = "$backend" ]; then leveled_dir="`epath 'leveled data_root' "$riak_epaths" | sed -e 's/^\"//' -e 's/\".*$//'`" - if [ '/' != `echo "$bitcask_dir" | cut -c1` ]; then + if [ '/' != `echo "$leveled_dir" | cut -c1` ]; then # relative path. prepend base dir leveled_dir="$riak_base_dir"/"$leveled_dir" fi - if [ ! -d "$bitcask_dir" ]; then + if [ ! -d "$leveled_dir" ]; then echoerr "Unable to locate Leveled data directory. Aborting." echoerr "Using leveled data_root: $leveled_dir" exit 1