Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make CI: Mixed version testing #12558

Merged
merged 2 commits into from
Oct 25, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 30 additions & 1 deletion .github/workflows/test-make-target.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,10 @@ on:
metadata_store:
required: true
type: string
mixed_clusters:
required: false
default: false
type: boolean
make_target:
required: true
type: string
Expand Down Expand Up @@ -41,6 +45,31 @@ jobs:
# restricted to the build jobs to avoid duplication in output.
disable_problem_matchers: true

- name: MIXED CLUSTERS - FETCH SIGNING KEYS
uses: dsaltares/fetch-gh-release-asset@master
if: inputs.mixed_clusters
with:
repo: rabbitmq/signing-keys
file: rabbitmq-release-signing-key.asc

- name: MIXED CLUSTERS - FETCH PREVIOUS VERSION
id: fetch_secondary_dist
uses: dsaltares/fetch-gh-release-asset@master
if: inputs.mixed_clusters
with:
regex: true
file: "rabbitmq-server-generic-unix-[\\d.]*\\.tar.xz"
target: ./

- name: MIXED CLUSTERS - SETUP SECONDARY_DIST
if: inputs.mixed_clusters
run: |
gpg --import rabbitmq-release-signing-key.asc
gpg --verify rabbitmq-server-generic-unix-*.asc rabbitmq-server-generic-unix-*.tar.xz
tar xf rabbitmq-server-generic-unix-*.tar.xz

echo "SECONDARY_DIST=${GITHUB_WORKSPACE}/rabbitmq_server-`echo -n ${{ steps.fetch_secondary_dist.outputs.version }} | sed s/v//`" >> $GITHUB_ENV

- name: SETUP DOTNET (rabbit)
uses: actions/setup-dotnet@v4
if: inputs.plugin == 'rabbit'
Expand Down Expand Up @@ -74,7 +103,7 @@ jobs:
if: always()
uses: actions/upload-artifact@v4
with:
name: CT logs (${{ inputs.plugin }} ${{ inputs.make_target }} OTP-${{ inputs.erlang_version }} ${{ inputs.metadata_store }})
name: CT logs (${{ inputs.plugin }} ${{ inputs.make_target }} OTP-${{ inputs.erlang_version }} ${{ inputs.metadata_store }}${{ inputs.mixed_clusters && ' mixed' || '' }})
path: |
logs/
# !logs/**/log_private
Expand Down
7 changes: 7 additions & 0 deletions .github/workflows/test-make-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ on:
metadata_store:
required: true
type: string
mixed_clusters:
required: true
type: boolean
jobs:
test-rabbit:
name: Test rabbit
Expand All @@ -33,6 +36,7 @@ jobs:
erlang_version: ${{ inputs.erlang_version }}
elixir_version: ${{ inputs.elixir_version }}
metadata_store: ${{ inputs.metadata_store }}
mixed_clusters: ${{ inputs.mixed_clusters }}
make_target: ${{ matrix.make_target }}
plugin: rabbit

Expand All @@ -43,6 +47,7 @@ jobs:
erlang_version: ${{ inputs.erlang_version }}
elixir_version: ${{ inputs.elixir_version }}
metadata_store: ${{ inputs.metadata_store }}
mixed_clusters: ${{ inputs.mixed_clusters }}
make_target: parallel-ct-set-1
plugin: rabbitmq_mqtt

Expand All @@ -55,6 +60,7 @@ jobs:
erlang_version: ${{ inputs.erlang_version }}
elixir_version: ${{ inputs.elixir_version }}
metadata_store: ${{ inputs.metadata_store }}
mixed_clusters: ${{ inputs.mixed_clusters }}
make_target: ct-config_schema ct-unit
plugin: rabbitmq_peer_discovery_aws

Expand Down Expand Up @@ -110,5 +116,6 @@ jobs:
erlang_version: ${{ inputs.erlang_version }}
elixir_version: ${{ inputs.elixir_version }}
metadata_store: ${{ inputs.metadata_store }}
mixed_clusters: ${{ inputs.mixed_clusters }}
make_target: tests
plugin: ${{ matrix.plugin }}
21 changes: 21 additions & 0 deletions .github/workflows/test-make.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,27 @@ jobs:
erlang_version: ${{ matrix.erlang_version }}
elixir_version: ${{ matrix.elixir_version }}
metadata_store: ${{ matrix.metadata_store }}
mixed_clusters: false

test-mixed-clusters:
name: Test mixed clusters
strategy:
fail-fast: false
matrix:
erlang_version:
- '26'
# - '27'
elixir_version:
- '1.17'
metadata_store:
- mnesia
- khepri
uses: ./.github/workflows/test-make-tests.yaml
with:
erlang_version: ${{ matrix.erlang_version }}
elixir_version: ${{ matrix.elixir_version }}
metadata_store: ${{ matrix.metadata_store }}
mixed_clusters: true

type-check:
name: Type check
Expand Down
2 changes: 0 additions & 2 deletions .github/workflows/test-mixed-versions.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ name: Test Mixed Version Clusters
on:
push:
branches:
- main
- v4.0.x
- v3.13.x
- bump-otp-*
Expand All @@ -21,7 +20,6 @@ on:
- '*.bzl'
- '*.bazel'
- .github/workflows/test-mixed-versions.yaml
pull_request:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
Expand Down
4 changes: 2 additions & 2 deletions deps/rabbit/test/cluster_upgrade_SUITE.erl
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ init_per_testcase(Testcase, Config) ->
Config1 = rabbit_ct_helpers:set_config(Config, [
{rmq_nodename_suffix, Testcase},
{rmq_nodes_count, 3},
{force_secondary_umbrella, true}
{force_secondary, true}
]),
Config2 = rabbit_ct_helpers:run_steps(Config1,
rabbit_ct_broker_helpers:setup_steps() ++
Expand Down Expand Up @@ -139,7 +139,7 @@ upgrade_cluster(Config) ->
|| N <- Cluster],
ct:pal(?LOW_IMPORTANCE, "Restarting cluster ~p", [Cluster]),
Config1 = rabbit_ct_helpers:set_config(
Config, {force_secondary_umbrella, false}),
Config, {force_secondary, false}),
[ok = rabbit_ct_broker_helpers:async_start_node(Config1, N)
|| N <- Cluster],
[ok = rabbit_ct_broker_helpers:wait_for_async_start_node(N)
Expand Down
2 changes: 2 additions & 0 deletions deps/rabbit/test/feature_flags_SUITE.erl
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ init_per_group(registry, Config) ->
logger:set_primary_config(level, debug),
rabbit_ct_helpers:run_steps(Config, []);
init_per_group(feature_flags_v2, Config) ->
%% @todo Remove this entirely as that FF became required in 3.12.
%% `feature_flags_v2' is now required and won't work in mixed-version
%% clusters if the other version doesn't support it.
case rabbit_ct_helpers:is_mixed_versions() of
Expand Down Expand Up @@ -267,6 +268,7 @@ init_per_testcase(Testcase, Config) ->
Config2 = rabbit_ct_helpers:set_config(
Config1,
[{rmq_nodename_suffix, Testcase},
{secondary_enabled_plugins, "my_plugin"},
{tcp_ports_base, {skip_n_nodes,
TestNumber * ClusterSize}},
{net_ticktime, 5}
Expand Down
6 changes: 3 additions & 3 deletions deps/rabbit_common/mk/rabbitmq-dist.mk
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
.PHONY: dist test-dist do-dist cli-scripts cli-escripts clean-dist

DIST_DIR = plugins
CLI_SCRIPTS_DIR = sbin
CLI_ESCRIPTS_DIR = escript
DIST_DIR ?= $(CURDIR)/plugins
CLI_SCRIPTS_DIR ?= $(CURDIR)/sbin
CLI_ESCRIPTS_DIR ?= $(CURDIR)/escript
MIX = echo y | mix

# Set $(DIST_AS_EZS) to a non-empty value to enable the packaging of
Expand Down
6 changes: 3 additions & 3 deletions deps/rabbit_common/mk/rabbitmq-run.mk
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ TEST_TMPDIR ?= $(TMPDIR)/rabbitmq-test-instances
endif

# Location of the scripts controlling the broker.
RABBITMQ_SCRIPTS_DIR ?= $(CURDIR)/sbin
RABBITMQ_SCRIPTS_DIR ?= $(CLI_SCRIPTS_DIR)

ifeq ($(PLATFORM),msys2)
RABBITMQ_PLUGINS ?= $(RABBITMQ_SCRIPTS_DIR)/rabbitmq-plugins.bat
Expand All @@ -39,7 +39,7 @@ export RABBITMQ_SCRIPTS_DIR RABBITMQCTL RABBITMQ_PLUGINS RABBITMQ_SERVER RABBITM
export MAKE

# We need to pass the location of codegen to the Java client ant
# process.
# process. @todo Delete?
CODEGEN_DIR = $(DEPS_DIR)/rabbitmq_codegen
PYTHONPATH = $(CODEGEN_DIR)
export PYTHONPATH
Expand Down Expand Up @@ -90,7 +90,7 @@ ifdef PLUGINS_FROM_DEPS_DIR
RMQ_PLUGINS_DIR = $(DEPS_DIR)
DIST_ERL_LIBS = $(ERL_LIBS)
else
RMQ_PLUGINS_DIR = $(CURDIR)/$(DIST_DIR)
RMQ_PLUGINS_DIR = $(DIST_DIR)
# We do not want to add apps/ or deps/ to ERL_LIBS
# when running the release from dist. The `plugins`
# directory is added to ERL_LIBS by rabbitmq-env.
Expand Down
81 changes: 75 additions & 6 deletions deps/rabbitmq_ct_helpers/src/rabbit_ct_broker_helpers.erl
Original file line number Diff line number Diff line change
Expand Up @@ -433,6 +433,7 @@ start_rabbitmq_node(Master, Config, NodeConfig, I) ->
%% It's unlikely we'll ever succeed to start RabbitMQ.
Master ! {self(), Error},
unlink(Master);
%% @todo This might not work right now in at least some cases...
{skip, _} ->
%% Try again with another TCP port numbers base.
NodeConfig4 = move_nonworking_nodedir_away(NodeConfig3),
Expand Down Expand Up @@ -506,6 +507,7 @@ tcp_port_base_for_broker0(Config, I, PortsCount) ->
tcp_port_base_for_broker1(Base, I, PortsCount) ->
Base + I * PortsCount * ?NODE_START_ATTEMPTS.

%% @todo Refactor to simplify this...
update_tcp_ports_in_rmq_config(NodeConfig, [tcp_port_amqp = Key | Rest]) ->
NodeConfig1 = rabbit_ct_helpers:merge_app_env(NodeConfig,
{rabbit, [{tcp_listeners, [?config(Key, NodeConfig)]}]}),
Expand Down Expand Up @@ -626,21 +628,52 @@ write_config_file(Config, NodeConfig, _I) ->
ConfigFile ++ "\": " ++ file:format_error(Reason)}
end.

-define(REQUIRED_FEATURE_FLAGS, [
%% Required in 3.11:
"virtual_host_metadata,"
"quorum_queue,"
"implicit_default_bindings,"
"maintenance_mode_status,"
"user_limits,"
%% Required in 3.12:
"stream_queue,"
"classic_queue_type_delivery_support,"
"tracking_records_in_ets,"
"stream_single_active_consumer,"
"listener_records_in_ets,"
"feature_flags_v2,"
"direct_exchange_routing_v2,"
"classic_mirrored_queue_version," %% @todo Missing in FF docs!!
%% Required in 3.12 in rabbitmq_management_agent:
% "drop_unroutable_metric,"
% "empty_basic_get_metric,"
%% Required in 4.0:
"stream_sac_coordinator_unblock_group,"
"restart_streams,"
"stream_update_config_command,"
"stream_filtering,"
"message_containers" %% @todo Update FF docs!! It *is* required.
]).

do_start_rabbitmq_node(Config, NodeConfig, I) ->
WithPlugins0 = rabbit_ct_helpers:get_config(Config,
broker_with_plugins),
broker_with_plugins), %% @todo This is probably not used.
WithPlugins = case is_list(WithPlugins0) of
true -> lists:nth(I + 1, WithPlugins0);
false -> WithPlugins0
end,
ForceUseSecondary = rabbit_ct_helpers:get_config(
Config, force_secondary_umbrella, undefined),
Config, force_secondary, undefined),
CanUseSecondary = case ForceUseSecondary of
undefined ->
(I + 1) rem 2 =:= 0;
Override when is_boolean(Override) ->
Override
end,
UseSecondaryDist = case ?config(secondary_dist, Config) of
false -> false;
_ -> CanUseSecondary
end,
UseSecondaryUmbrella = case ?config(secondary_umbrella, Config) of
false -> false;
_ -> CanUseSecondary
Expand Down Expand Up @@ -686,8 +719,10 @@ do_start_rabbitmq_node(Config, NodeConfig, I) ->
StartWithPluginsDisabled = rabbit_ct_helpers:get_config(
Config, start_rmq_with_plugins_disabled),
ExtraArgs2 = case StartWithPluginsDisabled of
true -> ["LEAVE_PLUGINS_DISABLED=yes" | ExtraArgs1];
_ -> ExtraArgs1
true ->
["LEAVE_PLUGINS_DISABLED=1" | ExtraArgs1];
_ ->
ExtraArgs1
end,
KeepPidFile = rabbit_ct_helpers:get_config(
Config, keep_pid_file_on_exit),
Expand Down Expand Up @@ -731,7 +766,30 @@ do_start_rabbitmq_node(Config, NodeConfig, I) ->
{"RABBITMQ_PLUGINS=~ts/rabbitmq-plugins", [SecScriptsDir]}
| ExtraArgs4];
false ->
ExtraArgs4
case UseSecondaryDist of
true ->
SecondaryDist = ?config(secondary_dist, Config),
SecondaryEnabledPlugins = case {
StartWithPluginsDisabled,
?config(secondary_enabled_plugins, Config),
filename:basename(SrcDir)
} of
{true, _, _} -> "";
{_, undefined, "rabbit"} -> "";
{_, undefined, SrcPlugin} -> SrcPlugin;
{_, SecondaryEnabledPlugins0, _} -> SecondaryEnabledPlugins0
end,
[{"DIST_DIR=~ts/plugins", [SecondaryDist]},
{"CLI_SCRIPTS_DIR=~ts/sbin", [SecondaryDist]},
{"CLI_ESCRIPTS_DIR=~ts/escript", [SecondaryDist]},
{"RABBITMQ_SCRIPTS_DIR=~ts/sbin", [SecondaryDist]},
{"RABBITMQ_SERVER=~ts/sbin/rabbitmq-server", [SecondaryDist]},
{"RABBITMQ_ENABLED_PLUGINS=~ts", [SecondaryEnabledPlugins]},
{"RABBITMQ_FEATURE_FLAGS=~ts", [?REQUIRED_FEATURE_FLAGS]}
| ExtraArgs4];
false ->
ExtraArgs4
end
end,
MakeVars = [
{"RABBITMQ_NODENAME=~ts", [Nodename]},
Expand Down Expand Up @@ -1285,6 +1343,10 @@ rabbitmqctl(Config, Node, Args, Timeout) ->
CanUseSecondary = (I + 1) rem 2 =:= 0,
BazelRunSecCmd = rabbit_ct_helpers:get_config(
Config, rabbitmq_run_secondary_cmd),
UseSecondaryDist = case ?config(secondary_dist, Config) of
false -> false;
_ -> CanUseSecondary
end,
UseSecondaryUmbrella = case ?config(secondary_umbrella, Config) of
false ->
case BazelRunSecCmd of
Expand Down Expand Up @@ -1327,7 +1389,14 @@ rabbitmqctl(Config, Node, Args, Timeout) ->
"rabbitmqctl"])
end;
false ->
?config(rabbitmqctl_cmd, Config)
case UseSecondaryDist of
true ->
SecondaryDist = ?config(secondary_dist, Config),
rabbit_misc:format(
"~ts/sbin/rabbitmqctl", [SecondaryDist]);
false ->
?config(rabbitmqctl_cmd, Config)
end
end,

NodeConfig = get_node_config(Config, Node),
Expand Down
19 changes: 17 additions & 2 deletions deps/rabbitmq_ct_helpers/src/rabbit_ct_helpers.erl
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ run_setup_steps(Config, ExtraSteps) ->
[
fun init_skip_as_error_flag/1,
fun guess_tested_erlang_app_name/1,
fun ensure_secondary_dist/1,
fun ensure_secondary_umbrella/1,
fun ensure_current_srcdir/1,
fun ensure_rabbitmq_ct_helpers_srcdir/1,
Expand Down Expand Up @@ -201,6 +202,18 @@ guess_tested_erlang_app_name(Config) ->
set_config(Config, {tested_erlang_app, list_to_atom(AppName)})
end.

ensure_secondary_dist(Config) ->
Path = case get_config(Config, secondary_dist) of
undefined -> os:getenv("SECONDARY_DIST");
P -> P
end,
%% Hard fail if the path is invalid.
case Path =:= false orelse filelib:is_dir(Path) of
true -> ok;
false -> error(secondary_dist_path_invalid)
end,
set_config(Config, {secondary_dist, Path}).

ensure_secondary_umbrella(Config) ->
Path = case get_config(Config, secondary_umbrella) of
undefined -> os:getenv("SECONDARY_UMBRELLA");
Expand Down Expand Up @@ -1060,11 +1073,13 @@ convert_to_unicode_binary(Arg) when is_binary(Arg) ->
Arg.

is_mixed_versions() ->
os:getenv("SECONDARY_UMBRELLA") =/= false
os:getenv("SECONDARY_DIST") =/= false
orelse os:getenv("SECONDARY_UMBRELLA") =/= false
orelse os:getenv("RABBITMQ_RUN_SECONDARY") =/= false.

is_mixed_versions(Config) ->
get_config(Config, secondary_umbrella, false) =/= false
get_config(Config, secondary_dist, false) =/= false
orelse get_config(Config, secondary_umbrella, false) =/= false
orelse get_config(Config, rabbitmq_run_secondary_cmd, false) =/= false.

%% -------------------------------------------------------------------
Expand Down
Loading