From f68fc8bb94388312da8494bc64c2e861a5eab7f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Mon, 21 Oct 2024 13:09:25 +0200 Subject: [PATCH 1/2] Make CI: Add mixed version testing This is enabled on main and for pull requests. Bazel remains used in previous branches. --- .github/workflows/test-make-target.yaml | 31 ++++++- .github/workflows/test-make-tests.yaml | 7 ++ .github/workflows/test-make.yaml | 21 +++++ .github/workflows/test-mixed-versions.yaml | 2 - deps/rabbit/test/cluster_upgrade_SUITE.erl | 4 +- deps/rabbit/test/feature_flags_SUITE.erl | 2 + deps/rabbit_common/mk/rabbitmq-dist.mk | 6 +- deps/rabbit_common/mk/rabbitmq-run.mk | 6 +- .../src/rabbit_ct_broker_helpers.erl | 81 +++++++++++++++++-- .../src/rabbit_ct_helpers.erl | 19 ++++- 10 files changed, 160 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test-make-target.yaml b/.github/workflows/test-make-target.yaml index fa53cde6bab4..656364d2a281 100644 --- a/.github/workflows/test-make-target.yaml +++ b/.github/workflows/test-make-target.yaml @@ -11,6 +11,10 @@ on: metadata_store: required: true type: string + mixed_clusters: + required: false + default: false + type: boolean make_target: required: true type: string @@ -41,6 +45,31 @@ jobs: # restricted to the build jobs to avoid duplication in output. disable_problem_matchers: true + - name: MIXED CLUSTERS - FETCH SIGNING KEYS + uses: dsaltares/fetch-gh-release-asset@master + if: inputs.mixed_clusters + with: + repo: rabbitmq/signing-keys + file: rabbitmq-release-signing-key.asc + + - name: MIXED CLUSTERS - FETCH PREVIOUS VERSION + id: fetch_secondary_dist + uses: dsaltares/fetch-gh-release-asset@master + if: inputs.mixed_clusters + with: + regex: true + file: "rabbitmq-server-generic-unix-[\\d.]*\\.tar.xz" + target: ./ + + - name: MIXED CLUSTERS - SETUP SECONDARY_DIST + if: inputs.mixed_clusters + run: | + gpg --import rabbitmq-release-signing-key.asc + gpg --verify rabbitmq-server-generic-unix-*.asc rabbitmq-server-generic-unix-*.tar.xz + tar xf rabbitmq-server-generic-unix-*.tar.xz + + echo "SECONDARY_DIST=${GITHUB_WORKSPACE}/rabbitmq_server-`echo -n ${{ steps.fetch_secondary_dist.outputs.version }} | sed s/v//`" >> $GITHUB_ENV + - name: SETUP DOTNET (rabbit) uses: actions/setup-dotnet@v4 if: inputs.plugin == 'rabbit' @@ -74,7 +103,7 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: CT logs (${{ inputs.plugin }} ${{ inputs.make_target }} OTP-${{ inputs.erlang_version }} ${{ inputs.metadata_store }}) + name: CT logs (${{ inputs.plugin }} ${{ inputs.make_target }} OTP-${{ inputs.erlang_version }} ${{ inputs.metadata_store }}${{ inputs.mixed_clusters && ' mixed' || '' }}) path: | logs/ # !logs/**/log_private diff --git a/.github/workflows/test-make-tests.yaml b/.github/workflows/test-make-tests.yaml index a0142656815d..5fa4c6e43d48 100644 --- a/.github/workflows/test-make-tests.yaml +++ b/.github/workflows/test-make-tests.yaml @@ -11,6 +11,9 @@ on: metadata_store: required: true type: string + mixed_clusters: + required: true + type: boolean jobs: test-rabbit: name: Test rabbit @@ -33,6 +36,7 @@ jobs: erlang_version: ${{ inputs.erlang_version }} elixir_version: ${{ inputs.elixir_version }} metadata_store: ${{ inputs.metadata_store }} + mixed_clusters: ${{ inputs.mixed_clusters }} make_target: ${{ matrix.make_target }} plugin: rabbit @@ -43,6 +47,7 @@ jobs: erlang_version: ${{ inputs.erlang_version }} elixir_version: ${{ inputs.elixir_version }} metadata_store: ${{ inputs.metadata_store }} + mixed_clusters: ${{ inputs.mixed_clusters }} make_target: parallel-ct-set-1 plugin: rabbitmq_mqtt @@ -55,6 +60,7 @@ jobs: erlang_version: ${{ inputs.erlang_version }} elixir_version: ${{ inputs.elixir_version }} metadata_store: ${{ inputs.metadata_store }} + mixed_clusters: ${{ inputs.mixed_clusters }} make_target: ct-config_schema ct-unit plugin: rabbitmq_peer_discovery_aws @@ -110,5 +116,6 @@ jobs: erlang_version: ${{ inputs.erlang_version }} elixir_version: ${{ inputs.elixir_version }} metadata_store: ${{ inputs.metadata_store }} + mixed_clusters: ${{ inputs.mixed_clusters }} make_target: tests plugin: ${{ matrix.plugin }} diff --git a/.github/workflows/test-make.yaml b/.github/workflows/test-make.yaml index 66d940f00811..32109d64fcc6 100644 --- a/.github/workflows/test-make.yaml +++ b/.github/workflows/test-make.yaml @@ -74,6 +74,27 @@ jobs: erlang_version: ${{ matrix.erlang_version }} elixir_version: ${{ matrix.elixir_version }} metadata_store: ${{ matrix.metadata_store }} + mixed_clusters: false + + test-mixed-clusters: + name: Test mixed clusters + strategy: + fail-fast: false + matrix: + erlang_version: + - '26' +# - '27' + elixir_version: + - '1.17' + metadata_store: + - mnesia +# - khepri + uses: ./.github/workflows/test-make-tests.yaml + with: + erlang_version: ${{ matrix.erlang_version }} + elixir_version: ${{ matrix.elixir_version }} + metadata_store: ${{ matrix.metadata_store }} + mixed_clusters: true type-check: name: Type check diff --git a/.github/workflows/test-mixed-versions.yaml b/.github/workflows/test-mixed-versions.yaml index f79c4bce8833..7a97d0a5cbad 100644 --- a/.github/workflows/test-mixed-versions.yaml +++ b/.github/workflows/test-mixed-versions.yaml @@ -2,7 +2,6 @@ name: Test Mixed Version Clusters on: push: branches: - - main - v4.0.x - v3.13.x - bump-otp-* @@ -21,7 +20,6 @@ on: - '*.bzl' - '*.bazel' - .github/workflows/test-mixed-versions.yaml - pull_request: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true diff --git a/deps/rabbit/test/cluster_upgrade_SUITE.erl b/deps/rabbit/test/cluster_upgrade_SUITE.erl index 2b78f119c904..ea943f1cc0f8 100644 --- a/deps/rabbit/test/cluster_upgrade_SUITE.erl +++ b/deps/rabbit/test/cluster_upgrade_SUITE.erl @@ -55,7 +55,7 @@ init_per_testcase(Testcase, Config) -> Config1 = rabbit_ct_helpers:set_config(Config, [ {rmq_nodename_suffix, Testcase}, {rmq_nodes_count, 3}, - {force_secondary_umbrella, true} + {force_secondary, true} ]), Config2 = rabbit_ct_helpers:run_steps(Config1, rabbit_ct_broker_helpers:setup_steps() ++ @@ -139,7 +139,7 @@ upgrade_cluster(Config) -> || N <- Cluster], ct:pal(?LOW_IMPORTANCE, "Restarting cluster ~p", [Cluster]), Config1 = rabbit_ct_helpers:set_config( - Config, {force_secondary_umbrella, false}), + Config, {force_secondary, false}), [ok = rabbit_ct_broker_helpers:async_start_node(Config1, N) || N <- Cluster], [ok = rabbit_ct_broker_helpers:wait_for_async_start_node(N) diff --git a/deps/rabbit/test/feature_flags_SUITE.erl b/deps/rabbit/test/feature_flags_SUITE.erl index cf1ff3e2e7eb..72df3c0469bd 100644 --- a/deps/rabbit/test/feature_flags_SUITE.erl +++ b/deps/rabbit/test/feature_flags_SUITE.erl @@ -126,6 +126,7 @@ init_per_group(registry, Config) -> logger:set_primary_config(level, debug), rabbit_ct_helpers:run_steps(Config, []); init_per_group(feature_flags_v2, Config) -> + %% @todo Remove this entirely as that FF became required in 3.12. %% `feature_flags_v2' is now required and won't work in mixed-version %% clusters if the other version doesn't support it. case rabbit_ct_helpers:is_mixed_versions() of @@ -267,6 +268,7 @@ init_per_testcase(Testcase, Config) -> Config2 = rabbit_ct_helpers:set_config( Config1, [{rmq_nodename_suffix, Testcase}, + {secondary_enabled_plugins, "my_plugin"}, {tcp_ports_base, {skip_n_nodes, TestNumber * ClusterSize}}, {net_ticktime, 5} diff --git a/deps/rabbit_common/mk/rabbitmq-dist.mk b/deps/rabbit_common/mk/rabbitmq-dist.mk index f55fe1ef08ea..10ee9938e849 100644 --- a/deps/rabbit_common/mk/rabbitmq-dist.mk +++ b/deps/rabbit_common/mk/rabbitmq-dist.mk @@ -1,8 +1,8 @@ .PHONY: dist test-dist do-dist cli-scripts cli-escripts clean-dist -DIST_DIR = plugins -CLI_SCRIPTS_DIR = sbin -CLI_ESCRIPTS_DIR = escript +DIST_DIR ?= $(CURDIR)/plugins +CLI_SCRIPTS_DIR ?= $(CURDIR)/sbin +CLI_ESCRIPTS_DIR ?= $(CURDIR)/escript MIX = echo y | mix # Set $(DIST_AS_EZS) to a non-empty value to enable the packaging of diff --git a/deps/rabbit_common/mk/rabbitmq-run.mk b/deps/rabbit_common/mk/rabbitmq-run.mk index b3f7a3e998f9..605b67846799 100644 --- a/deps/rabbit_common/mk/rabbitmq-run.mk +++ b/deps/rabbit_common/mk/rabbitmq-run.mk @@ -19,7 +19,7 @@ TEST_TMPDIR ?= $(TMPDIR)/rabbitmq-test-instances endif # Location of the scripts controlling the broker. -RABBITMQ_SCRIPTS_DIR ?= $(CURDIR)/sbin +RABBITMQ_SCRIPTS_DIR ?= $(CLI_SCRIPTS_DIR) ifeq ($(PLATFORM),msys2) RABBITMQ_PLUGINS ?= $(RABBITMQ_SCRIPTS_DIR)/rabbitmq-plugins.bat @@ -39,7 +39,7 @@ export RABBITMQ_SCRIPTS_DIR RABBITMQCTL RABBITMQ_PLUGINS RABBITMQ_SERVER RABBITM export MAKE # We need to pass the location of codegen to the Java client ant -# process. +# process. @todo Delete? CODEGEN_DIR = $(DEPS_DIR)/rabbitmq_codegen PYTHONPATH = $(CODEGEN_DIR) export PYTHONPATH @@ -90,7 +90,7 @@ ifdef PLUGINS_FROM_DEPS_DIR RMQ_PLUGINS_DIR = $(DEPS_DIR) DIST_ERL_LIBS = $(ERL_LIBS) else -RMQ_PLUGINS_DIR = $(CURDIR)/$(DIST_DIR) +RMQ_PLUGINS_DIR = $(DIST_DIR) # We do not want to add apps/ or deps/ to ERL_LIBS # when running the release from dist. The `plugins` # directory is added to ERL_LIBS by rabbitmq-env. diff --git a/deps/rabbitmq_ct_helpers/src/rabbit_ct_broker_helpers.erl b/deps/rabbitmq_ct_helpers/src/rabbit_ct_broker_helpers.erl index 77c78cc98ac5..ff526cca9d34 100644 --- a/deps/rabbitmq_ct_helpers/src/rabbit_ct_broker_helpers.erl +++ b/deps/rabbitmq_ct_helpers/src/rabbit_ct_broker_helpers.erl @@ -433,6 +433,7 @@ start_rabbitmq_node(Master, Config, NodeConfig, I) -> %% It's unlikely we'll ever succeed to start RabbitMQ. Master ! {self(), Error}, unlink(Master); + %% @todo This might not work right now in at least some cases... {skip, _} -> %% Try again with another TCP port numbers base. NodeConfig4 = move_nonworking_nodedir_away(NodeConfig3), @@ -506,6 +507,7 @@ tcp_port_base_for_broker0(Config, I, PortsCount) -> tcp_port_base_for_broker1(Base, I, PortsCount) -> Base + I * PortsCount * ?NODE_START_ATTEMPTS. +%% @todo Refactor to simplify this... update_tcp_ports_in_rmq_config(NodeConfig, [tcp_port_amqp = Key | Rest]) -> NodeConfig1 = rabbit_ct_helpers:merge_app_env(NodeConfig, {rabbit, [{tcp_listeners, [?config(Key, NodeConfig)]}]}), @@ -626,21 +628,52 @@ write_config_file(Config, NodeConfig, _I) -> ConfigFile ++ "\": " ++ file:format_error(Reason)} end. +-define(REQUIRED_FEATURE_FLAGS, [ + %% Required in 3.11: + "virtual_host_metadata," + "quorum_queue," + "implicit_default_bindings," + "maintenance_mode_status," + "user_limits," + %% Required in 3.12: + "stream_queue," + "classic_queue_type_delivery_support," + "tracking_records_in_ets," + "stream_single_active_consumer," + "listener_records_in_ets," + "feature_flags_v2," + "direct_exchange_routing_v2," + "classic_mirrored_queue_version," %% @todo Missing in FF docs!! + %% Required in 3.12 in rabbitmq_management_agent: +% "drop_unroutable_metric," +% "empty_basic_get_metric," + %% Required in 4.0: + "stream_sac_coordinator_unblock_group," + "restart_streams," + "stream_update_config_command," + "stream_filtering," + "message_containers" %% @todo Update FF docs!! It *is* required. +]). + do_start_rabbitmq_node(Config, NodeConfig, I) -> WithPlugins0 = rabbit_ct_helpers:get_config(Config, - broker_with_plugins), + broker_with_plugins), %% @todo This is probably not used. WithPlugins = case is_list(WithPlugins0) of true -> lists:nth(I + 1, WithPlugins0); false -> WithPlugins0 end, ForceUseSecondary = rabbit_ct_helpers:get_config( - Config, force_secondary_umbrella, undefined), + Config, force_secondary, undefined), CanUseSecondary = case ForceUseSecondary of undefined -> (I + 1) rem 2 =:= 0; Override when is_boolean(Override) -> Override end, + UseSecondaryDist = case ?config(secondary_dist, Config) of + false -> false; + _ -> CanUseSecondary + end, UseSecondaryUmbrella = case ?config(secondary_umbrella, Config) of false -> false; _ -> CanUseSecondary @@ -686,8 +719,10 @@ do_start_rabbitmq_node(Config, NodeConfig, I) -> StartWithPluginsDisabled = rabbit_ct_helpers:get_config( Config, start_rmq_with_plugins_disabled), ExtraArgs2 = case StartWithPluginsDisabled of - true -> ["LEAVE_PLUGINS_DISABLED=yes" | ExtraArgs1]; - _ -> ExtraArgs1 + true -> + ["LEAVE_PLUGINS_DISABLED=1" | ExtraArgs1]; + _ -> + ExtraArgs1 end, KeepPidFile = rabbit_ct_helpers:get_config( Config, keep_pid_file_on_exit), @@ -731,7 +766,30 @@ do_start_rabbitmq_node(Config, NodeConfig, I) -> {"RABBITMQ_PLUGINS=~ts/rabbitmq-plugins", [SecScriptsDir]} | ExtraArgs4]; false -> - ExtraArgs4 + case UseSecondaryDist of + true -> + SecondaryDist = ?config(secondary_dist, Config), + SecondaryEnabledPlugins = case { + StartWithPluginsDisabled, + ?config(secondary_enabled_plugins, Config), + filename:basename(SrcDir) + } of + {true, _, _} -> ""; + {_, undefined, "rabbit"} -> ""; + {_, undefined, SrcPlugin} -> SrcPlugin; + {_, SecondaryEnabledPlugins0, _} -> SecondaryEnabledPlugins0 + end, + [{"DIST_DIR=~ts/plugins", [SecondaryDist]}, + {"CLI_SCRIPTS_DIR=~ts/sbin", [SecondaryDist]}, + {"CLI_ESCRIPTS_DIR=~ts/escript", [SecondaryDist]}, + {"RABBITMQ_SCRIPTS_DIR=~ts/sbin", [SecondaryDist]}, + {"RABBITMQ_SERVER=~ts/sbin/rabbitmq-server", [SecondaryDist]}, + {"RABBITMQ_ENABLED_PLUGINS=~ts", [SecondaryEnabledPlugins]}, + {"RABBITMQ_FEATURE_FLAGS=~ts", [?REQUIRED_FEATURE_FLAGS]} + | ExtraArgs4]; + false -> + ExtraArgs4 + end end, MakeVars = [ {"RABBITMQ_NODENAME=~ts", [Nodename]}, @@ -1285,6 +1343,10 @@ rabbitmqctl(Config, Node, Args, Timeout) -> CanUseSecondary = (I + 1) rem 2 =:= 0, BazelRunSecCmd = rabbit_ct_helpers:get_config( Config, rabbitmq_run_secondary_cmd), + UseSecondaryDist = case ?config(secondary_dist, Config) of + false -> false; + _ -> CanUseSecondary + end, UseSecondaryUmbrella = case ?config(secondary_umbrella, Config) of false -> case BazelRunSecCmd of @@ -1327,7 +1389,14 @@ rabbitmqctl(Config, Node, Args, Timeout) -> "rabbitmqctl"]) end; false -> - ?config(rabbitmqctl_cmd, Config) + case UseSecondaryDist of + true -> + SecondaryDist = ?config(secondary_dist, Config), + rabbit_misc:format( + "~ts/sbin/rabbitmqctl", [SecondaryDist]); + false -> + ?config(rabbitmqctl_cmd, Config) + end end, NodeConfig = get_node_config(Config, Node), diff --git a/deps/rabbitmq_ct_helpers/src/rabbit_ct_helpers.erl b/deps/rabbitmq_ct_helpers/src/rabbit_ct_helpers.erl index c9b351ddd6ab..162a456db7e9 100644 --- a/deps/rabbitmq_ct_helpers/src/rabbit_ct_helpers.erl +++ b/deps/rabbitmq_ct_helpers/src/rabbit_ct_helpers.erl @@ -78,6 +78,7 @@ run_setup_steps(Config, ExtraSteps) -> [ fun init_skip_as_error_flag/1, fun guess_tested_erlang_app_name/1, + fun ensure_secondary_dist/1, fun ensure_secondary_umbrella/1, fun ensure_current_srcdir/1, fun ensure_rabbitmq_ct_helpers_srcdir/1, @@ -201,6 +202,18 @@ guess_tested_erlang_app_name(Config) -> set_config(Config, {tested_erlang_app, list_to_atom(AppName)}) end. +ensure_secondary_dist(Config) -> + Path = case get_config(Config, secondary_dist) of + undefined -> os:getenv("SECONDARY_DIST"); + P -> P + end, + %% Hard fail if the path is invalid. + case Path =:= false orelse filelib:is_dir(Path) of + true -> ok; + false -> error(secondary_dist_path_invalid) + end, + set_config(Config, {secondary_dist, Path}). + ensure_secondary_umbrella(Config) -> Path = case get_config(Config, secondary_umbrella) of undefined -> os:getenv("SECONDARY_UMBRELLA"); @@ -1060,11 +1073,13 @@ convert_to_unicode_binary(Arg) when is_binary(Arg) -> Arg. is_mixed_versions() -> - os:getenv("SECONDARY_UMBRELLA") =/= false + os:getenv("SECONDARY_DIST") =/= false + orelse os:getenv("SECONDARY_UMBRELLA") =/= false orelse os:getenv("RABBITMQ_RUN_SECONDARY") =/= false. is_mixed_versions(Config) -> - get_config(Config, secondary_umbrella, false) =/= false + get_config(Config, secondary_dist, false) =/= false + orelse get_config(Config, secondary_umbrella, false) =/= false orelse get_config(Config, rabbitmq_run_secondary_cmd, false) =/= false. %% ------------------------------------------------------------------- From e89e1fa2e8a3fd7d98497474f55cc37858548fbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Fri, 25 Oct 2024 14:41:51 +0200 Subject: [PATCH 2/2] Make CI: Enable khepri mixed clusters testing --- .github/workflows/test-make.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-make.yaml b/.github/workflows/test-make.yaml index 32109d64fcc6..d2d8a54b1a26 100644 --- a/.github/workflows/test-make.yaml +++ b/.github/workflows/test-make.yaml @@ -88,7 +88,7 @@ jobs: - '1.17' metadata_store: - mnesia -# - khepri + - khepri uses: ./.github/workflows/test-make-tests.yaml with: erlang_version: ${{ matrix.erlang_version }}