Skip to content

Commit

Permalink
Merge pull request #12558 from rabbitmq/loic-make-ci-mixed-versions
Browse files Browse the repository at this point in the history
Make CI: Mixed version testing
  • Loading branch information
lhoguin authored Oct 25, 2024
2 parents 8b79ac7 + e89e1fa commit b351149
Show file tree
Hide file tree
Showing 10 changed files with 160 additions and 19 deletions.
31 changes: 30 additions & 1 deletion .github/workflows/test-make-target.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,10 @@ on:
metadata_store:
required: true
type: string
mixed_clusters:
required: false
default: false
type: boolean
make_target:
required: true
type: string
Expand Down Expand Up @@ -41,6 +45,31 @@ jobs:
# restricted to the build jobs to avoid duplication in output.
disable_problem_matchers: true

- name: MIXED CLUSTERS - FETCH SIGNING KEYS
uses: dsaltares/fetch-gh-release-asset@master
if: inputs.mixed_clusters
with:
repo: rabbitmq/signing-keys
file: rabbitmq-release-signing-key.asc

- name: MIXED CLUSTERS - FETCH PREVIOUS VERSION
id: fetch_secondary_dist
uses: dsaltares/fetch-gh-release-asset@master
if: inputs.mixed_clusters
with:
regex: true
file: "rabbitmq-server-generic-unix-[\\d.]*\\.tar.xz"
target: ./

- name: MIXED CLUSTERS - SETUP SECONDARY_DIST
if: inputs.mixed_clusters
run: |
gpg --import rabbitmq-release-signing-key.asc
gpg --verify rabbitmq-server-generic-unix-*.asc rabbitmq-server-generic-unix-*.tar.xz
tar xf rabbitmq-server-generic-unix-*.tar.xz
echo "SECONDARY_DIST=${GITHUB_WORKSPACE}/rabbitmq_server-`echo -n ${{ steps.fetch_secondary_dist.outputs.version }} | sed s/v//`" >> $GITHUB_ENV
- name: SETUP DOTNET (rabbit)
uses: actions/setup-dotnet@v4
if: inputs.plugin == 'rabbit'
Expand Down Expand Up @@ -74,7 +103,7 @@ jobs:
if: always()
uses: actions/upload-artifact@v4
with:
name: CT logs (${{ inputs.plugin }} ${{ inputs.make_target }} OTP-${{ inputs.erlang_version }} ${{ inputs.metadata_store }})
name: CT logs (${{ inputs.plugin }} ${{ inputs.make_target }} OTP-${{ inputs.erlang_version }} ${{ inputs.metadata_store }}${{ inputs.mixed_clusters && ' mixed' || '' }})
path: |
logs/
# !logs/**/log_private
Expand Down
7 changes: 7 additions & 0 deletions .github/workflows/test-make-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ on:
metadata_store:
required: true
type: string
mixed_clusters:
required: true
type: boolean
jobs:
test-rabbit:
name: Test rabbit
Expand All @@ -33,6 +36,7 @@ jobs:
erlang_version: ${{ inputs.erlang_version }}
elixir_version: ${{ inputs.elixir_version }}
metadata_store: ${{ inputs.metadata_store }}
mixed_clusters: ${{ inputs.mixed_clusters }}
make_target: ${{ matrix.make_target }}
plugin: rabbit

Expand All @@ -43,6 +47,7 @@ jobs:
erlang_version: ${{ inputs.erlang_version }}
elixir_version: ${{ inputs.elixir_version }}
metadata_store: ${{ inputs.metadata_store }}
mixed_clusters: ${{ inputs.mixed_clusters }}
make_target: parallel-ct-set-1
plugin: rabbitmq_mqtt

Expand All @@ -55,6 +60,7 @@ jobs:
erlang_version: ${{ inputs.erlang_version }}
elixir_version: ${{ inputs.elixir_version }}
metadata_store: ${{ inputs.metadata_store }}
mixed_clusters: ${{ inputs.mixed_clusters }}
make_target: ct-config_schema ct-unit
plugin: rabbitmq_peer_discovery_aws

Expand Down Expand Up @@ -110,5 +116,6 @@ jobs:
erlang_version: ${{ inputs.erlang_version }}
elixir_version: ${{ inputs.elixir_version }}
metadata_store: ${{ inputs.metadata_store }}
mixed_clusters: ${{ inputs.mixed_clusters }}
make_target: tests
plugin: ${{ matrix.plugin }}
21 changes: 21 additions & 0 deletions .github/workflows/test-make.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,27 @@ jobs:
erlang_version: ${{ matrix.erlang_version }}
elixir_version: ${{ matrix.elixir_version }}
metadata_store: ${{ matrix.metadata_store }}
mixed_clusters: false

test-mixed-clusters:
name: Test mixed clusters
strategy:
fail-fast: false
matrix:
erlang_version:
- '26'
# - '27'
elixir_version:
- '1.17'
metadata_store:
- mnesia
- khepri
uses: ./.github/workflows/test-make-tests.yaml
with:
erlang_version: ${{ matrix.erlang_version }}
elixir_version: ${{ matrix.elixir_version }}
metadata_store: ${{ matrix.metadata_store }}
mixed_clusters: true

type-check:
name: Type check
Expand Down
2 changes: 0 additions & 2 deletions .github/workflows/test-mixed-versions.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ name: Test Mixed Version Clusters
on:
push:
branches:
- main
- v4.0.x
- v3.13.x
- bump-otp-*
Expand All @@ -21,7 +20,6 @@ on:
- '*.bzl'
- '*.bazel'
- .github/workflows/test-mixed-versions.yaml
pull_request:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
Expand Down
4 changes: 2 additions & 2 deletions deps/rabbit/test/cluster_upgrade_SUITE.erl
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ init_per_testcase(Testcase, Config) ->
Config1 = rabbit_ct_helpers:set_config(Config, [
{rmq_nodename_suffix, Testcase},
{rmq_nodes_count, 3},
{force_secondary_umbrella, true}
{force_secondary, true}
]),
Config2 = rabbit_ct_helpers:run_steps(Config1,
rabbit_ct_broker_helpers:setup_steps() ++
Expand Down Expand Up @@ -139,7 +139,7 @@ upgrade_cluster(Config) ->
|| N <- Cluster],
ct:pal(?LOW_IMPORTANCE, "Restarting cluster ~p", [Cluster]),
Config1 = rabbit_ct_helpers:set_config(
Config, {force_secondary_umbrella, false}),
Config, {force_secondary, false}),
[ok = rabbit_ct_broker_helpers:async_start_node(Config1, N)
|| N <- Cluster],
[ok = rabbit_ct_broker_helpers:wait_for_async_start_node(N)
Expand Down
2 changes: 2 additions & 0 deletions deps/rabbit/test/feature_flags_SUITE.erl
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ init_per_group(registry, Config) ->
logger:set_primary_config(level, debug),
rabbit_ct_helpers:run_steps(Config, []);
init_per_group(feature_flags_v2, Config) ->
%% @todo Remove this entirely as that FF became required in 3.12.
%% `feature_flags_v2' is now required and won't work in mixed-version
%% clusters if the other version doesn't support it.
case rabbit_ct_helpers:is_mixed_versions() of
Expand Down Expand Up @@ -267,6 +268,7 @@ init_per_testcase(Testcase, Config) ->
Config2 = rabbit_ct_helpers:set_config(
Config1,
[{rmq_nodename_suffix, Testcase},
{secondary_enabled_plugins, "my_plugin"},
{tcp_ports_base, {skip_n_nodes,
TestNumber * ClusterSize}},
{net_ticktime, 5}
Expand Down
6 changes: 3 additions & 3 deletions deps/rabbit_common/mk/rabbitmq-dist.mk
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
.PHONY: dist test-dist do-dist cli-scripts cli-escripts clean-dist

DIST_DIR = plugins
CLI_SCRIPTS_DIR = sbin
CLI_ESCRIPTS_DIR = escript
DIST_DIR ?= $(CURDIR)/plugins
CLI_SCRIPTS_DIR ?= $(CURDIR)/sbin
CLI_ESCRIPTS_DIR ?= $(CURDIR)/escript
MIX = echo y | mix

# Set $(DIST_AS_EZS) to a non-empty value to enable the packaging of
Expand Down
6 changes: 3 additions & 3 deletions deps/rabbit_common/mk/rabbitmq-run.mk
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ TEST_TMPDIR ?= $(TMPDIR)/rabbitmq-test-instances
endif

# Location of the scripts controlling the broker.
RABBITMQ_SCRIPTS_DIR ?= $(CURDIR)/sbin
RABBITMQ_SCRIPTS_DIR ?= $(CLI_SCRIPTS_DIR)

ifeq ($(PLATFORM),msys2)
RABBITMQ_PLUGINS ?= $(RABBITMQ_SCRIPTS_DIR)/rabbitmq-plugins.bat
Expand All @@ -39,7 +39,7 @@ export RABBITMQ_SCRIPTS_DIR RABBITMQCTL RABBITMQ_PLUGINS RABBITMQ_SERVER RABBITM
export MAKE

# We need to pass the location of codegen to the Java client ant
# process.
# process. @todo Delete?
CODEGEN_DIR = $(DEPS_DIR)/rabbitmq_codegen
PYTHONPATH = $(CODEGEN_DIR)
export PYTHONPATH
Expand Down Expand Up @@ -90,7 +90,7 @@ ifdef PLUGINS_FROM_DEPS_DIR
RMQ_PLUGINS_DIR = $(DEPS_DIR)
DIST_ERL_LIBS = $(ERL_LIBS)
else
RMQ_PLUGINS_DIR = $(CURDIR)/$(DIST_DIR)
RMQ_PLUGINS_DIR = $(DIST_DIR)
# We do not want to add apps/ or deps/ to ERL_LIBS
# when running the release from dist. The `plugins`
# directory is added to ERL_LIBS by rabbitmq-env.
Expand Down
81 changes: 75 additions & 6 deletions deps/rabbitmq_ct_helpers/src/rabbit_ct_broker_helpers.erl
Original file line number Diff line number Diff line change
Expand Up @@ -433,6 +433,7 @@ start_rabbitmq_node(Master, Config, NodeConfig, I) ->
%% It's unlikely we'll ever succeed to start RabbitMQ.
Master ! {self(), Error},
unlink(Master);
%% @todo This might not work right now in at least some cases...
{skip, _} ->
%% Try again with another TCP port numbers base.
NodeConfig4 = move_nonworking_nodedir_away(NodeConfig3),
Expand Down Expand Up @@ -506,6 +507,7 @@ tcp_port_base_for_broker0(Config, I, PortsCount) ->
tcp_port_base_for_broker1(Base, I, PortsCount) ->
Base + I * PortsCount * ?NODE_START_ATTEMPTS.

%% @todo Refactor to simplify this...
update_tcp_ports_in_rmq_config(NodeConfig, [tcp_port_amqp = Key | Rest]) ->
NodeConfig1 = rabbit_ct_helpers:merge_app_env(NodeConfig,
{rabbit, [{tcp_listeners, [?config(Key, NodeConfig)]}]}),
Expand Down Expand Up @@ -626,21 +628,52 @@ write_config_file(Config, NodeConfig, _I) ->
ConfigFile ++ "\": " ++ file:format_error(Reason)}
end.

-define(REQUIRED_FEATURE_FLAGS, [
%% Required in 3.11:
"virtual_host_metadata,"
"quorum_queue,"
"implicit_default_bindings,"
"maintenance_mode_status,"
"user_limits,"
%% Required in 3.12:
"stream_queue,"
"classic_queue_type_delivery_support,"
"tracking_records_in_ets,"
"stream_single_active_consumer,"
"listener_records_in_ets,"
"feature_flags_v2,"
"direct_exchange_routing_v2,"
"classic_mirrored_queue_version," %% @todo Missing in FF docs!!
%% Required in 3.12 in rabbitmq_management_agent:
% "drop_unroutable_metric,"
% "empty_basic_get_metric,"
%% Required in 4.0:
"stream_sac_coordinator_unblock_group,"
"restart_streams,"
"stream_update_config_command,"
"stream_filtering,"
"message_containers" %% @todo Update FF docs!! It *is* required.
]).

do_start_rabbitmq_node(Config, NodeConfig, I) ->
WithPlugins0 = rabbit_ct_helpers:get_config(Config,
broker_with_plugins),
broker_with_plugins), %% @todo This is probably not used.
WithPlugins = case is_list(WithPlugins0) of
true -> lists:nth(I + 1, WithPlugins0);
false -> WithPlugins0
end,
ForceUseSecondary = rabbit_ct_helpers:get_config(
Config, force_secondary_umbrella, undefined),
Config, force_secondary, undefined),
CanUseSecondary = case ForceUseSecondary of
undefined ->
(I + 1) rem 2 =:= 0;
Override when is_boolean(Override) ->
Override
end,
UseSecondaryDist = case ?config(secondary_dist, Config) of
false -> false;
_ -> CanUseSecondary
end,
UseSecondaryUmbrella = case ?config(secondary_umbrella, Config) of
false -> false;
_ -> CanUseSecondary
Expand Down Expand Up @@ -686,8 +719,10 @@ do_start_rabbitmq_node(Config, NodeConfig, I) ->
StartWithPluginsDisabled = rabbit_ct_helpers:get_config(
Config, start_rmq_with_plugins_disabled),
ExtraArgs2 = case StartWithPluginsDisabled of
true -> ["LEAVE_PLUGINS_DISABLED=yes" | ExtraArgs1];
_ -> ExtraArgs1
true ->
["LEAVE_PLUGINS_DISABLED=1" | ExtraArgs1];
_ ->
ExtraArgs1
end,
KeepPidFile = rabbit_ct_helpers:get_config(
Config, keep_pid_file_on_exit),
Expand Down Expand Up @@ -731,7 +766,30 @@ do_start_rabbitmq_node(Config, NodeConfig, I) ->
{"RABBITMQ_PLUGINS=~ts/rabbitmq-plugins", [SecScriptsDir]}
| ExtraArgs4];
false ->
ExtraArgs4
case UseSecondaryDist of
true ->
SecondaryDist = ?config(secondary_dist, Config),
SecondaryEnabledPlugins = case {
StartWithPluginsDisabled,
?config(secondary_enabled_plugins, Config),
filename:basename(SrcDir)
} of
{true, _, _} -> "";
{_, undefined, "rabbit"} -> "";
{_, undefined, SrcPlugin} -> SrcPlugin;
{_, SecondaryEnabledPlugins0, _} -> SecondaryEnabledPlugins0
end,
[{"DIST_DIR=~ts/plugins", [SecondaryDist]},
{"CLI_SCRIPTS_DIR=~ts/sbin", [SecondaryDist]},
{"CLI_ESCRIPTS_DIR=~ts/escript", [SecondaryDist]},
{"RABBITMQ_SCRIPTS_DIR=~ts/sbin", [SecondaryDist]},
{"RABBITMQ_SERVER=~ts/sbin/rabbitmq-server", [SecondaryDist]},
{"RABBITMQ_ENABLED_PLUGINS=~ts", [SecondaryEnabledPlugins]},
{"RABBITMQ_FEATURE_FLAGS=~ts", [?REQUIRED_FEATURE_FLAGS]}
| ExtraArgs4];
false ->
ExtraArgs4
end
end,
MakeVars = [
{"RABBITMQ_NODENAME=~ts", [Nodename]},
Expand Down Expand Up @@ -1285,6 +1343,10 @@ rabbitmqctl(Config, Node, Args, Timeout) ->
CanUseSecondary = (I + 1) rem 2 =:= 0,
BazelRunSecCmd = rabbit_ct_helpers:get_config(
Config, rabbitmq_run_secondary_cmd),
UseSecondaryDist = case ?config(secondary_dist, Config) of
false -> false;
_ -> CanUseSecondary
end,
UseSecondaryUmbrella = case ?config(secondary_umbrella, Config) of
false ->
case BazelRunSecCmd of
Expand Down Expand Up @@ -1327,7 +1389,14 @@ rabbitmqctl(Config, Node, Args, Timeout) ->
"rabbitmqctl"])
end;
false ->
?config(rabbitmqctl_cmd, Config)
case UseSecondaryDist of
true ->
SecondaryDist = ?config(secondary_dist, Config),
rabbit_misc:format(
"~ts/sbin/rabbitmqctl", [SecondaryDist]);
false ->
?config(rabbitmqctl_cmd, Config)
end
end,

NodeConfig = get_node_config(Config, Node),
Expand Down
19 changes: 17 additions & 2 deletions deps/rabbitmq_ct_helpers/src/rabbit_ct_helpers.erl
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ run_setup_steps(Config, ExtraSteps) ->
[
fun init_skip_as_error_flag/1,
fun guess_tested_erlang_app_name/1,
fun ensure_secondary_dist/1,
fun ensure_secondary_umbrella/1,
fun ensure_current_srcdir/1,
fun ensure_rabbitmq_ct_helpers_srcdir/1,
Expand Down Expand Up @@ -201,6 +202,18 @@ guess_tested_erlang_app_name(Config) ->
set_config(Config, {tested_erlang_app, list_to_atom(AppName)})
end.

ensure_secondary_dist(Config) ->
Path = case get_config(Config, secondary_dist) of
undefined -> os:getenv("SECONDARY_DIST");
P -> P
end,
%% Hard fail if the path is invalid.
case Path =:= false orelse filelib:is_dir(Path) of
true -> ok;
false -> error(secondary_dist_path_invalid)
end,
set_config(Config, {secondary_dist, Path}).

ensure_secondary_umbrella(Config) ->
Path = case get_config(Config, secondary_umbrella) of
undefined -> os:getenv("SECONDARY_UMBRELLA");
Expand Down Expand Up @@ -1060,11 +1073,13 @@ convert_to_unicode_binary(Arg) when is_binary(Arg) ->
Arg.
is_mixed_versions() ->
os:getenv("SECONDARY_UMBRELLA") =/= false
os:getenv("SECONDARY_DIST") =/= false
orelse os:getenv("SECONDARY_UMBRELLA") =/= false
orelse os:getenv("RABBITMQ_RUN_SECONDARY") =/= false.
is_mixed_versions(Config) ->
get_config(Config, secondary_umbrella, false) =/= false
get_config(Config, secondary_dist, false) =/= false
orelse get_config(Config, secondary_umbrella, false) =/= false
orelse get_config(Config, rabbitmq_run_secondary_cmd, false) =/= false.
%% -------------------------------------------------------------------
Expand Down

0 comments on commit b351149

Please sign in to comment.