Switch integration tests to use the peer module

.github/workflows/test-make-target.yaml

@@ -112,8 +112,10 @@ jobs:
           ldap-utils \
           slapd
 
-        sudo systemctl is-active --quiet apparmor.service && sudo systemctl stop apparmor.service
-        sudo systemctl disable apparmor.service
+        sudo systemctl stop slapd
+
+        sudo aa-teardown || true
+        sudo systemctl disable --now apparmor.service
 
     - name: RUN LOW VERSION ERLANG NODE IN DOCKER
       if: inputs.make_target == 'ct-rabbit_fifo_prop'

.gitignore

@@ -116,6 +116,7 @@ rabbitmq-server-*.tar.gz
 rabbitmq-server-*.tar.bz2
 rabbitmq-server-*.tar.xz
 rabbitmq-server-*.zip
+secondary_dist/
 
 # Trace tools output.
 *-ttb

deps/amqp_client/test/system_SUITE.erl

@@ -129,12 +129,8 @@ end_per_suite(Config) ->
       ] ++ rabbit_ct_broker_helpers:teardown_steps()).
 
 ensure_amqp_client_srcdir(Config) ->
-    case rabbit_ct_helpers:get_config(Config, rabbitmq_run_cmd) of
-        undefined ->
-            rabbit_ct_helpers:ensure_application_srcdir(Config,
-                                                        amqp_client, amqp_client);
-        _ -> Config
-    end.
+    rabbit_ct_helpers:ensure_application_srcdir(Config,
+                                                amqp_client, amqp_client).
 
 create_unauthorized_user(Config) ->
     Cmd = ["add_user", ?UNAUTHORIZED_USER, ?UNAUTHORIZED_USER],
@@ -1621,10 +1617,7 @@ set_resource_alarm(Resource, Config)
     Cmd = [{"RABBITMQ_NODENAME=~ts", [Nodename]},
            "set-resource-alarm",
            {"SOURCE=~ts", [Resource]}],
-    {ok, _} = case os:getenv("RABBITMQ_RUN") of
-        false -> rabbit_ct_helpers:make(Config, SrcDir, Cmd);
-        Run -> rabbit_ct_helpers:exec([Run | Cmd])
-    end.
+    rabbit_ct_helpers:make(Config, SrcDir, Cmd).
 
 clear_resource_alarm(Resource, Config)
   when Resource =:= memory orelse Resource =:= disk ->
@@ -1633,9 +1626,6 @@ clear_resource_alarm(Resource, Config)
     Cmd = [{"RABBITMQ_NODENAME=~ts", [Nodename]},
            "clear-resource-alarm",
            {"SOURCE=~ts", [Resource]}],
-    {ok, _} = case os:getenv("RABBITMQ_RUN") of
-        false -> rabbit_ct_helpers:make(Config, SrcDir, Cmd);
-        Run -> rabbit_ct_helpers:exec([Run | Cmd])
-    end.
+    rabbit_ct_helpers:make(Config, SrcDir, Cmd).
 
 fmt(Fmt, Args) -> list_to_binary(rabbit_misc:format(Fmt, Args)).

deps/rabbit/test/amqp_client_SUITE.erl

@@ -4767,7 +4767,7 @@ message_ttl(Config) ->
 plugin(Config) ->
     Node = 0,
     Plugin = rabbitmq_amqp1_0,
-    %% rabbit/Makefile and rabbit/BUILD.bazel declare a test dependency on the rabbitmq_amqp1_0 plugin.
+    %% rabbit/Makefile declares a test dependency on the rabbitmq_amqp1_0 plugin.
     %% Therefore, we first disable, and then enable.
     ?assertEqual(ok, rabbit_ct_broker_helpers:disable_plugin(Config, Node, Plugin)),
     ?assertEqual(ok, rabbit_ct_broker_helpers:enable_plugin(Config, Node, Plugin)).

deps/rabbit/test/feature_flags_SUITE.erl

@@ -130,67 +130,16 @@ end_per_suite(Config) ->
 init_per_group(registry, Config) ->
     logger:set_primary_config(level, debug),
     rabbit_ct_helpers:run_steps(Config, []);
-init_per_group(feature_flags_v2, Config) ->
-    %% @todo Remove this entirely as that FF became required in 3.12.
-    %% `feature_flags_v2' is now required and won't work in mixed-version
-    %% clusters if the other version doesn't support it.
-    case rabbit_ct_helpers:is_mixed_versions() of
-        false ->
-            rabbit_ct_helpers:set_config(
-              Config, {enable_feature_flags_v2, true});
-        true ->
-            %% Before we run `feature_flags_v2'-related tests, we must ensure
-            %% that both umbrellas support them. Otherwise there is no point
-            %% in running them.
-            %% To determine that `feature_flags_v2' are supported, we can't
-            %% query RabbitMQ which is not started. Therefore, we check if the
-            %% source or bytecode of `rabbit_ff_controller' is present.
-            Dir1 = ?config(rabbit_srcdir, Config),
-            File1 = filename:join([Dir1, "ebin", "rabbit_ff_controller.beam"]),
-            SupportedPrimary = filelib:is_file(File1),
-            SupportedSecondary =
-                case rabbit_ct_helpers:get_config(Config, rabbitmq_run_cmd) of
-                    undefined ->
-                        %% make
-                        Dir2 = ?config(secondary_rabbit_srcdir, Config),
-                        File2 = filename:join(
-                                  [Dir2, "src", "rabbit_ff_controller.erl"]),
-                        filelib:is_file(File2);
-                    RmqRunSecondary ->
-                        %% bazel
-                        Dir2 = filename:dirname(RmqRunSecondary),
-                        Beam = filename:join(
-                                 [Dir2, "plugins", "rabbit-*",
-                                  "ebin", "rabbit_ff_controller.beam"]),
-                        case filelib:wildcard(Beam) of
-                            [_] -> true;
-                            [] -> false
-                        end
-                end,
-            case {SupportedPrimary, SupportedSecondary} of
-                {true, true} ->
-                    rabbit_ct_helpers:set_config(
-                      Config, {enable_feature_flags_v2, true});
-                {false, true} ->
-                    {skip,
-                     "Primary umbrella does not support "
-                     "feature_flags_v2"};
-                {true, false} ->
-                    {skip,
-                     "Secondary umbrella does not support "
-                     "feature_flags_v2"}
-            end
-    end;
 init_per_group(enabling_on_single_node, Config) ->
     Config1 = rabbit_ct_helpers:set_config(
                 Config,
                 [{rmq_nodes_count, 1}]),
-    rabbit_ct_helpers:run_setup_steps(Config1, [fun prepare_my_plugin/1]);
+    rabbit_ct_helpers:run_setup_steps(Config1, [fun build_my_plugin/1]);
 init_per_group(enabling_in_cluster, Config) ->
     Config1 = rabbit_ct_helpers:set_config(
                 Config,
                 [{rmq_nodes_count, 5}]),
-    rabbit_ct_helpers:run_setup_steps(Config1, [fun prepare_my_plugin/1]);
+    rabbit_ct_helpers:run_setup_steps(Config1, [fun build_my_plugin/1]);
 init_per_group(clustering, Config) ->
     Config1 = rabbit_ct_helpers:set_config(
                 Config,
@@ -207,14 +156,14 @@ init_per_group(clustering, Config) ->
                                      message_containers,
                                      quorum_queue_non_voters
                                     ]}]}),
-    rabbit_ct_helpers:run_setup_steps(Config2, [fun prepare_my_plugin/1]);
+    rabbit_ct_helpers:run_setup_steps(Config2, [fun build_my_plugin/1]);
 init_per_group(activating_plugin, Config) ->
     Config1 = rabbit_ct_helpers:set_config(
                 Config,
                 [{rmq_nodes_count, 2},
                  {rmq_nodes_clustered, true},
                  {start_rmq_with_plugins_disabled, true}]),
-    rabbit_ct_helpers:run_setup_steps(Config1, [fun prepare_my_plugin/1]);
+    rabbit_ct_helpers:run_setup_steps(Config1, [fun build_my_plugin/1]);
 init_per_group(_, Config) ->
     Config.
 
@@ -1367,19 +1316,6 @@ restart_node_with_unknown_enabled_feature_flag(Config) ->
 %% Internal helpers.
 %% -------------------------------------------------------------------
 
-prepare_my_plugin(Config) ->
-    case os:getenv("RABBITMQ_RUN") of
-        false ->
-            build_my_plugin(Config);
-        _ ->
-            MyPluginDir = filename:dirname(
-                            filename:dirname(
-                              code:where_is_file("my_plugin.app"))),
-            PluginsDir = filename:dirname(MyPluginDir),
-            rabbit_ct_helpers:set_config(
-              Config, [{rmq_plugins_dir, PluginsDir}])
-    end.
-
 build_my_plugin(Config) ->
     DataDir = filename:join(
                 filename:dirname(filename:dirname(?config(data_dir, Config))),

deps/rabbit/test/feature_flags_v2_SUITE.erl

@@ -65,17 +65,7 @@ all() ->
     ].
 
 groups() ->
-    %% Don't run testcases in parallel when Bazel is used because they fail
-    %% with various system errors in CI, like the inability to spawn system
-    %% processes or to open a TCP port.
-    UsesBazel = case os:getenv("RABBITMQ_RUN") of
-                    false -> false;
-                    _     -> true
-                end,
-    GroupOptions = case UsesBazel of
-                       false -> [parallel];
-                       true  -> []
-                   end,
+    GroupOptions = [parallel],
     Groups =
     [
      {direct, GroupOptions,
@@ -123,8 +113,6 @@ init_per_suite(Config) ->
 end_per_suite(Config) ->
     Config.
 
-init_per_group(feature_flags_v2, Config) ->
-    rabbit_ct_helpers:set_config(Config, {enable_feature_flags_v2, true});
 init_per_group(direct, Config) ->
     Config;
 init_per_group(cluster_size_1, Config) ->

deps/rabbit/test/queue_parallel_SUITE.erl

@@ -21,17 +21,7 @@ all() ->
     ].
 
 groups() ->
-    %% Don't run testcases in parallel when Bazel is used because they fail
-    %% with various system errors in CI, like the inability to spawn system
-    %% processes or to open a TCP port.
-    UsesBazel = case os:getenv("RABBITMQ_RUN") of
-                    false -> false;
-                    _     -> true
-                end,
-    GroupOptions = case UsesBazel of
-                       false -> [parallel];
-                       true  -> []
-                   end,
+    GroupOptions = [parallel],
     AllTests = [publish,
                 consume,
                 consume_first_empty,

deps/rabbit/test/quorum_queue_SUITE.erl

@@ -1813,7 +1813,7 @@ dont_leak_file_handles(Config) ->
     ok.
 
 grow_queue(Config) ->
-    [Server0, Server1, Server2, _Server3, _Server4] =
+    [Server0, Server1, _Server2, _Server3, _Server4] =
         rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
 
     Ch = rabbit_ct_client_helpers:open_channel(Config, Server0),
@@ -5394,7 +5394,7 @@ delete_queues() ->
      || Q <- rabbit_amqqueue:list()].
 
 stop_node(Config, Server) ->
-    rabbit_ct_broker_helpers:rabbitmqctl(Config, Server, ["stop"]).
+    rabbit_ct_broker_helpers:stop_node(Config, Server).
 
 get_message_bytes(Leader, QRes) ->
     case rpc:call(Leader, ets, lookup, [queue_metrics, QRes]) of

deps/rabbit/test/rabbit_access_control_SUITE.erl

@@ -79,7 +79,7 @@ init_per_testcase(Testcase, Config)
        Testcase =:= with_disabled_plugin orelse
        Testcase =:= with_disabled_plugin_plus_internal ->
     rabbit_ct_helpers:testcase_started(Config, Testcase),
-    do_init_per_testcase(Testcase, Config, [fun prepare_my_plugin/1]);
+    do_init_per_testcase(Testcase, Config, [fun build_my_plugin/1]);
 init_per_testcase(_TestCase, Config) ->
     Config.
 
@@ -327,19 +327,6 @@ with_disabled_plugin_plus_internal(Config) ->
 %% Internal helpers.
 %% -------------------------------------------------------------------
 
-prepare_my_plugin(Config) ->
-    case os:getenv("RABBITMQ_RUN") of
-        false ->
-            build_my_plugin(Config);
-        _ ->
-            MyPluginDir = filename:dirname(
-                            filename:dirname(
-                              code:where_is_file("my_auth_plugin.app"))),
-            PluginsDir = filename:dirname(MyPluginDir),
-            rabbit_ct_helpers:set_config(
-              Config, [{rmq_plugins_dir, PluginsDir}])
-    end.
-
 build_my_plugin(Config) ->
     DataDir = filename:join(
                 filename:dirname(filename:dirname(?config(data_dir, Config))),

deps/rabbit_common/mk/rabbitmq-early-plugin.mk

@@ -25,41 +25,38 @@ CT_OPTS += -hidden
 
 CT_OPTS += -kernel net_ticktime 5
 
-# Enable the following common_test hooks on GH and Concourse:
-#
-# cth_fail_fast
-#   This hook will make sure the first failure puts an end to the
-#   testsuites; ie. all remaining tests are skipped.
-#
-# cth_styledout
-#   This hook will change the output of common_test to something more
-#   concise and colored. Not used on GitHub Actions except in parallel CT.
-
-ifndef GITHUB_ACTIONS
+ifdef GITHUB_ACTIONS
+# On CI, set $RABBITMQ_CT_SKIP_AS_ERROR so that any skipped
+# testsuite/testgroup/testcase is considered an error.
+export RABBITMQ_CT_SKIP_AS_ERROR = true
+else
+# This hook will change the output of common_test to something more
+# concise and colored. Not used on GitHub Actions except in parallel
+# CT where it is hardcoded.
 CT_HOOKS += cth_styledout
 endif
+
 TEST_DEPS += cth_styledout
+dep_cth_styledout = git https://github.com/rabbitmq/cth_styledout.git master
 
-ifdef CONCOURSE
-FAIL_FAST = 1
-SKIP_AS_ERROR = 1
+ifneq ($(strip $(CT_HOOKS)),)
+CT_OPTS += -ct_hooks $(CT_HOOKS)
 endif
 
-RMQ_CI_CT_HOOKS = cth_fail_fast
-ifeq ($(FAIL_FAST),1)
-CT_HOOKS += $(RMQ_CI_CT_HOOKS)
-TEST_DEPS += $(RMQ_CI_CT_HOOKS)
+# We fetch a SECONDARY_DIST if SECONDARY_DIST_VSN is set and
+# we setup the SECONDARY_DIST variable. The location of the
+# secondary dists can be customized using SECONDARY_DIST_DIR
+# but note that it only works with an absolute path.
+ifneq ($(strip $(SECONDARY_DIST_VSN)),)
+SECONDARY_DIST_DIR ?= $(abspath $(CURDIR)/../../)/secondary_dist
+SECONDARY_DIST=$(SECONDARY_DIST_DIR)/rabbitmq_server-$(SECONDARY_DIST_VSN)
+export SECONDARY_DIST
+
+SECONDARY_DIST_DOWNLOAD_URL = https://github.com/rabbitmq/rabbitmq-server/releases/download/v$(SECONDARY_DIST_VSN)/rabbitmq-server-generic-unix-$(SECONDARY_DIST_VSN).tar.xz
+
+ifeq ($(wildcard $(SECONDARY_DIST)),)
+$(shell curl -L -o $(ERLANG_MK_TMP)/rabbitmq_server-$(SECONDARY_DIST_VSN).tar.xz $(SECONDARY_DIST_DOWNLOAD_URL) \
+    && mkdir -p $(SECONDARY_DIST) \
+    && tar -xf $(ERLANG_MK_TMP)/rabbitmq_server-$(SECONDARY_DIST_VSN).tar.xz -C $(dir $(SECONDARY_DIST)))
 endif
-
-dep_cth_fail_fast = git https://github.com/rabbitmq/cth_fail_fast.git master
-dep_cth_styledout = git https://github.com/rabbitmq/cth_styledout.git master
-
-CT_HOOKS_PARAM_VALUE = $(patsubst %,and %,$(CT_HOOKS))
-CT_OPTS += -ct_hooks $(wordlist 2,$(words $(CT_HOOKS_PARAM_VALUE)),$(CT_HOOKS_PARAM_VALUE))
-
-# On CI, set $RABBITMQ_CT_SKIP_AS_ERROR so that any skipped
-# testsuite/testgroup/testcase is considered an error.
-
-ifeq ($(SKIP_AS_ERROR),1)
-export RABBITMQ_CT_SKIP_AS_ERROR = true
 endif

deps/rabbit_common/mk/rabbitmq-run.mk

@@ -335,7 +335,7 @@ start-background-node: node-tmpdir $(DIST_TARGET)
 	  $(RABBITMQ_SERVER) \
 	  $(REDIRECT_STDIO) &
 	ERL_LIBS="$(DIST_ERL_LIBS)" \
-	  $(RABBITMQCTL) -n $(RABBITMQ_NODENAME) wait --timeout $(RMQCTL_WAIT_TIMEOUT) $(RABBITMQ_PID_FILE) kernel
+	  $(RABBITMQCTL) -n $(RABBITMQ_NODENAME) wait --timeout $(RMQCTL_WAIT_TIMEOUT) $(RABBITMQ_PID_FILE)
 
 start-background-broker: node-tmpdir $(DIST_TARGET)
 	$(maybe_enabled_plugins); \

deps/rabbit_common/src/rabbit_misc.erl

@@ -1160,7 +1160,11 @@ get_channel_operation_timeout() ->
     %% Default channel_operation_timeout set to net_ticktime + 10s to
     %% give allowance for any down messages to be received first,
     %% whenever it is used for cross-node calls with timeouts.
-    Default = (net_kernel:get_net_ticktime() + 10) * 1000,
+    Tick = case net_kernel:get_net_ticktime() of
+        {ongoing_change_to, Tick0} -> Tick0;
+        Tick0 -> Tick0
+    end,
+    Default = (Tick + 10) * 1000,
     application:get_env(rabbit, channel_operation_timeout, Default).
 
 moving_average(_Time, _HalfLife, Next, undefined) ->