Skip to content

Commit

Permalink
Check additional applications when comparing bazel and make results (#…
Browse files Browse the repository at this point in the history
…8209)

* Check additional applications when comparing bazel and make results

* Sync bazel/make for amqp_client

* Do not fail-fast in build system comparison

* promethus -> prometheus

* Regenerate BUILD.redbug

* When comparing build systems & .app files ignore empty 'registered'

It's listed as a required key in
https://www.erlang.org/doc/man/app.html, but the same docs state the
default is "[]". It seems to ignore it if it's empty.

* Copy bazel/BUILD.osiris from BUILD.bazel in the osiris repo

Normally it would be generated with `bazel run gazelle-update-repos --
-args osiris@1.5.1=github.com/rabbitmq/osiris@v1.5.1`, but in this
case we just want to match it's compilation with erlang.mk with some
manual tweaks.

* Use elixir 1.15, otherwise mix format fails

* Sync bazel/make for rabbitmq_web_dispatch, rabbitmq_management_agent

(cherry picked from commit ca1806d)

# Conflicts:
#	.github/workflows/check-build-system-equivalence.yaml
#	MODULE.bazel
  • Loading branch information
HoloRin authored and mergify[bot] committed Aug 13, 2023
1 parent d63e1fe commit 3b36275
Show file tree
Hide file tree
Showing 10 changed files with 218 additions and 44 deletions.
7 changes: 7 additions & 0 deletions .github/workflows/check-build-system-equivalence.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,17 @@ on:
- cron: '0 2 * * *'
workflow_dispatch:
env:
<<<<<<< HEAD
erlang_version: 25.3
elixir_version: 1.14
VERSION: 3.11.0
=======
erlang_version: 26.0
elixir_version: 1.15
VERSION: 3.13.0
>>>>>>> ca1806dbcd (Check additional applications when comparing bazel and make results (#8209))
PLUGINS: amqp10_common amqp10_client rabbitmq_amqp1_0 rabbitmq_auth_backend_cache rabbitmq_auth_backend_http rabbitmq_auth_backend_ldap rabbitmq_auth_backend_oauth2 rabbitmq_auth_mechanism_ssl rabbitmq_consistent_hash_exchange rabbitmq_event_exchange rabbitmq_federation rabbitmq_jms_topic_exchange rabbitmq_mqtt rabbitmq_random_exchange rabbitmq_recent_history_exchange rabbitmq_sharding rabbitmq_shovel rabbitmq_stomp rabbitmq_stream rabbitmq_trust_store rabbitmq_web_dispatch rabbitmq_management_agent rabbitmq_management rabbitmq_prometheus rabbitmq_federation_management rabbitmq_shovel_management rabbitmq_stream_management rabbitmq_top rabbitmq_tracing rabbitmq_web_mqtt rabbitmq_web_mqtt_examples rabbitmq_web_stomp rabbitmq_web_stomp_examples rabbitmq_aws rabbitmq_peer_discovery_common rabbitmq_peer_discovery_aws rabbitmq_peer_discovery_k8s rabbitmq_peer_discovery_consul rabbitmq_peer_discovery_etcd
EXTRA_PLUGINS: accept amqp_client aten base64url cowboy cowlib credentials_obfuscation cuttlefish eetcd enough gen_batch_server getopt gun jose observer_cli osiris prometheus quantile_estimator ra ranch recon redbug seshat stdout_formatter syslog sysmon_handler systemd thoas
jobs:
build-with-bazel:
name: bazel build package-generic-unix.tar.xz
Expand Down
4 changes: 4 additions & 0 deletions MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -268,6 +268,7 @@ erlang_package.hex_package(
)

erlang_package.git_package(
<<<<<<< HEAD
patch_cmds = [
"""VERSION=$(git rev-parse HEAD)
echo "Injecting ${VERSION} into Makefile..."
Expand All @@ -276,6 +277,9 @@ echo "Injecting ${VERSION} into BUILD.bazel..."
sed -i"_orig" -E '/VERSION/ s/[0-9]+\\.[0-9]+\\.[0-9]+/'${VERSION}'/' BUILD.bazel
""",
],
=======
build_file = "@rabbitmq-server//bazel:BUILD.osiris",
>>>>>>> ca1806dbcd (Check additional applications when comparing bazel and make results (#8209))
repository = "rabbitmq/osiris",
tag = "v1.6.0",
)
Expand Down
144 changes: 144 additions & 0 deletions bazel/BUILD.osiris
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
load("@rules_erlang//:erlang_app.bzl", "erlang_app", "test_erlang_app")
load("@rules_erlang//:xref2.bzl", "xref")
load("@rules_erlang//:dialyze.bzl", "DEFAULT_PLT_APPS", "dialyze", "plt")
load("@rules_erlang//:eunit2.bzl", "eunit")
load("@rules_erlang//:ct.bzl", "assert_suites2", "ct_suite")

NAME = "osiris"

VERSION = "1.5.1"

DESCRIPTION = "New project"

APP_ENV = """[
{data_dir, "/tmp/osiris"},
{port_range, {6000, 6500}},
{max_segment_size_chunks, 256000},
{replication_transport, tcp},
{replica_forced_gc_default_interval, 4999}
]"""

EXTRA_APPS = [
"sasl",
"crypto",
]

DEPS = [
"@gen_batch_server//:erlang_app",
]

RUNTIME_DEPS = [
"@seshat//:erlang_app",
]

erlang_app(
app_description = DESCRIPTION,
app_env = APP_ENV,
app_name = NAME,
app_version = VERSION,
extra_apps = EXTRA_APPS,
runtime_deps = RUNTIME_DEPS,
deps = DEPS,
)

test_erlang_app(
app_description = DESCRIPTION,
app_env = APP_ENV,
app_name = NAME,
app_version = VERSION,
extra_apps = EXTRA_APPS,
runtime_deps = RUNTIME_DEPS,
deps = DEPS,
)

xref()

PLT_APPS = DEFAULT_PLT_APPS + [
"compiler",
"crypto",
"tools",
"runtime_tools",
"mnesia",
"public_key",
"asn1",
"ssl",
"inets",
]

plt(
name = "deps_plt",
apps = PLT_APPS,
for_target = ":erlang_app",
)

dialyze(
name = "dialyze",
size = "small",
dialyzer_opts = [
"-Werror_handling",
"-Wunmatched_returns",
],
plt = ":deps_plt",
target = ":erlang_app",
)

plt(
name = "test_deps_plt",
apps = PLT_APPS + [
"eunit",
"common_test",
"debugger",
"xmerl",
"ftp",
"ssh",
"snmp",
"wx",
"syntax_tools",
],
for_target = ":test_erlang_app",
)

dialyze(
name = "dialyze_tests",
size = "small",
beam = [
f.replace("test/", "").replace(".erl", "_beam_files")
for f in glob(["test/*_SUITE.erl"])
],
dialyzer_opts = [
"-Werror_handling",
"-Wunmatched_returns",
],
plt = ":test_deps_plt",
)

eunit(
name = "eunit",
eunit_opts = [
"no_tty",
"{report, {eunit_progress, [colored, profile]}}",
],
target = ":test_erlang_app",
deps = [
"@eunit_formatters//:erlang_app",
],
)

[ct_suite(
name = f.replace("test/", "").replace(".erl", ""),
) for f in glob(
["test/*_SUITE.erl"],
exclude = ["test/osiris_SUITE.erl"],
)]

ct_suite(
name = "osiris_SUITE",
data = [
"@tls_gen//:basic",
],
test_env = {
"DEPS_DIR": "$TEST_SRCDIR/$TEST_WORKSPACE/external",
},
)

assert_suites2()
37 changes: 15 additions & 22 deletions bazel/BUILD.redbug
Original file line number Diff line number Diff line change
Expand Up @@ -25,17 +25,9 @@ erlang_bytecode(
"src/redbug_parser.erl",
"src/redbug_targ.erl",
],
outs = [
"ebin/redbug.beam",
"ebin/redbug_compiler.beam",
"ebin/redbug_dtop.beam",
"ebin/redbug_lexer.beam",
"ebin/redbug_parser.beam",
"ebin/redbug_targ.beam",
],
hdrs = [],
hdrs = [":public_and_private_hdrs"],
app_name = "redbug",
beam = [],
dest = "ebin",
erlc_opts = "//:erlc_opts",
)

Expand All @@ -57,20 +49,11 @@ filegroup(
],
)

filegroup(
name = "private_hdrs",
srcs = [],
)
filegroup(name = "private_hdrs")

filegroup(
name = "public_hdrs",
srcs = [],
)
filegroup(name = "public_hdrs")

filegroup(
name = "priv",
srcs = [],
)
filegroup(name = "priv")

filegroup(
name = "licenses",
Expand All @@ -96,13 +79,23 @@ filegroup(
erlang_app(
name = "erlang_app",
srcs = [":all_srcs"],
hdrs = [":public_hdrs"],
app_name = "redbug",
beam_files = [":beam_files"],
extra_apps = ["runtime_tools"],
license_files = [":license_files"],
priv = [":priv"],
)

alias(
name = "redbug",
actual = ":erlang_app",
visibility = ["//visibility:public"],
)

filegroup(
name = "license_files",
srcs = [
"LICENSE",
],
)
5 changes: 3 additions & 2 deletions deps/amqp_client/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ APP_EXTRA_KEYS = """%% Hex.pm package informations.
{licenses, ["MPL-2.0"]},
{links, [
{"Website", "https://www.rabbitmq.com/"},
{"GitHub", "https://github.com/rabbitmq/rabbitmq-server/deps/amqp_client"},
{"GitHub", "https://github.com/rabbitmq/rabbitmq-server/tree/main/deps/amqp_client"},
{"User guide", "https://www.rabbitmq.com/erlang-client-user-guide.html"}
]},
{build_tools, ["make", "rebar3"]},
Expand All @@ -41,7 +41,8 @@ APP_EXTRA_KEYS = """%% Hex.pm package informations.
"Makefile",
"rabbitmq-components.mk",
"README",
"README.md"
"README.md",
"src"
]}
"""

Expand Down
2 changes: 1 addition & 1 deletion deps/amqp_client/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ endef
# Release artifacts are put in $(PACKAGES_DIR).
PACKAGES_DIR ?= $(abspath PACKAGES)

LOCAL_DEPS = xmerl
LOCAL_DEPS = xmerl ssl public_key
DEPS = rabbit_common credentials_obfuscation
TEST_DEPS = rabbitmq_ct_helpers rabbit meck

Expand Down
2 changes: 1 addition & 1 deletion deps/rabbitmq_management_agent/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ define PROJECT_APP_EXTRA_KEYS
{broker_version_requirements, []}
endef

DEPS = rabbit_common rabbit
DEPS = rabbit_common rabbit rabbitmq_web_dispatch
TEST_DEPS = rabbitmq_ct_helpers rabbitmq_ct_client_helpers
LOCAL_DEPS += xmerl mnesia ranch ssl crypto public_key
DEP_EARLY_PLUGINS = rabbit_common/mk/rabbitmq-early-plugin.mk
Expand Down
2 changes: 1 addition & 1 deletion deps/rabbitmq_web_dispatch/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ define PROJECT_APP_EXTRA_KEYS
endef

LOCAL_DEPS = inets
DEPS = rabbit_common rabbit cowboy
DEPS = amqp_client rabbit_common rabbit cowboy
TEST_DEPS = rabbitmq_ct_helpers rabbitmq_ct_client_helpers

dep_cowboy = hex 2.0.0
Expand Down
48 changes: 33 additions & 15 deletions tools/compare_dist.sh
Original file line number Diff line number Diff line change
@@ -1,44 +1,62 @@
#!/usr/bin/env bash
set -euo pipefail
set -uo pipefail

GOLDEN=$1
SECOND=$2

failure_count=0

echo "Check both have INSTALL"
test -f $GOLDEN/rabbitmq_server-${VERSION}/INSTALL
test -f $SECOND/rabbitmq_server-${VERSION}/INSTALL
test -f $GOLDEN/rabbitmq_server-${VERSION}/INSTALL || ((failure_count++))
test -f $SECOND/rabbitmq_server-${VERSION}/INSTALL || ((failure_count++))

echo "Check LICENSEs"
diff \
<(grep LICENSE make.manifest) \
<(grep LICENSE bazel.manifest | grep -v ".md" | grep -v ".txt")
<(grep LICENSE bazel.manifest | grep -v ".md" | grep -v ".txt") \
|| ((failure_count++))

echo "Check plugins"
plugins_rel=rabbitmq_server-${VERSION}/plugins
diff <(grep $plugins_rel make.manifest | grep -v ".ez") <(grep $plugins_rel bazel.manifest | grep -v ".ez")
diff \
<(grep $plugins_rel make.manifest | grep -v ".ez") \
<(grep $plugins_rel bazel.manifest | grep -v ".ez") \
|| ((failure_count++))

echo "Plugins exist with same version and deps"
for p in ${PLUGINS}; do
for p in ${PLUGINS} ${EXTRA_PLUGINS}; do
echo "$p"
f="$(cd $GOLDEN && ls -d $plugins_rel/$p-*)"
test -f $GOLDEN/$f/ebin/$p.app || (echo "$GOLDEN/$f/ebin/$p.app does not exist"; exit 1)
test -d $SECOND/$f || (echo "$SECOND/$f does not exist"; exit 1)
test -f $SECOND/$f/ebin/$p.app || (echo "$SECOND/$f/ebin/$p.app does not exist"; exit 1)
test -f $GOLDEN/$f/ebin/$p.app || (echo "$GOLDEN/$f/ebin/$p.app does not exist"; ((failure_count++)))
test -d $SECOND/$f || (echo "$SECOND/$f does not exist"; ((failure_count++)))
test -f $SECOND/$f/ebin/$p.app || (echo "$SECOND/$f/ebin/$p.app does not exist"; ((failure_count++)))
./rabbitmq-server/tools/erlang_app_equal \
$GOLDEN/$f/ebin/$p.app \
$SECOND/$f/ebin/$p.app
$GOLDEN/$f/ebin/$p.app \
$SECOND/$f/ebin/$p.app \
|| ((failure_count++))
done

echo "Both have escript"
escript_rel=rabbitmq_server-${VERSION}/escript
diff <(grep $escript_rel make.manifest) <(grep $escript_rel bazel.manifest)
diff \
<(grep $escript_rel make.manifest) \
<(grep $escript_rel bazel.manifest) \
|| ((failure_count++))

echo "Both have sbin"
sbin_rel=rabbitmq_server-${VERSION}/sbin
diff <(grep $sbin_rel make.manifest) <(grep $sbin_rel bazel.manifest)
diff \
<(grep $sbin_rel make.manifest) \
<(grep $sbin_rel bazel.manifest) \
|| ((failure_count++))

echo "Both have manpages"
manpages_rel=rabbitmq_server-${VERSION}/share/man
diff <(grep $manpages_rel make.manifest) <(grep $manpages_rel bazel.manifest)
diff \
<(grep $manpages_rel make.manifest) \
<(grep $manpages_rel bazel.manifest) \
|| ((failure_count++))

echo "There were $failure_count failures."

echo "PASS"
exit $failure_count
11 changes: 9 additions & 2 deletions tools/erlang_app_equal
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,15 @@ compare(LeftMetadata, RightMetadata) ->
"Warning:\t 'id' does not match (~p != ~p)~n", [LeftId, RightId])
end,

LeftPropsMap = proplists:to_map(proplists:delete(id, LeftProps)),
RightPropsMap = proplists:to_map(proplists:delete(id, RightProps)),
FilterEmptyRegistered = fun
(registered, []) -> false;
(_, _) -> true
end,

LeftPropsMap = maps:filter(FilterEmptyRegistered,
proplists:to_map(proplists:delete(id, LeftProps))),
RightPropsMap = maps:filter(FilterEmptyRegistered,
proplists:to_map(proplists:delete(id, RightProps))),
assert_equal(
lists:sort(maps:keys(LeftPropsMap)),
lists:sort(maps:keys(RightPropsMap)),
Expand Down

0 comments on commit 3b36275

Please sign in to comment.