4.1/main: Make CI: Fix and enhance ct_master

deps/rabbit/Makefile

@@ -239,22 +239,16 @@ define ct_master.erl
 	peer:call(Pid2, persistent_term, put, [rabbit_ct_tcp_port_base, 25000]),
 	peer:call(Pid3, persistent_term, put, [rabbit_ct_tcp_port_base, 27000]),
 	peer:call(Pid4, persistent_term, put, [rabbit_ct_tcp_port_base, 29000]),
-	ct_master_fork:run("$1"),
-	Fail1 = peer:call(Pid1, cth_parallel_ct_detect_failure, has_failures, []),
-	Fail2 = peer:call(Pid2, cth_parallel_ct_detect_failure, has_failures, []),
-	Fail3 = peer:call(Pid3, cth_parallel_ct_detect_failure, has_failures, []),
-	Fail4 = peer:call(Pid4, cth_parallel_ct_detect_failure, has_failures, []),
+	[{[_], {ok, Results}}] = ct_master_fork:run("$1"),
 	peer:stop(Pid4),
 	peer:stop(Pid3),
 	peer:stop(Pid2),
 	peer:stop(Pid1),
-	if
-		Fail1 -> halt(1);
-		Fail2 -> halt(2);
-		Fail3 -> halt(3);
-		Fail4 -> halt(4);
-		true -> halt(0)
-	end
+	lists:foldl(fun
+		({_, {_, 0, {_, 0}}}, Err) -> Err + 1;
+		(What, Peer) -> halt(Peer)
+	end, 1, Results),
+	halt(0)
 endef
 
 PARALLEL_CT_SET_1_A = amqp_client unit_cluster_formation_locking_mocks unit_cluster_formation_sort_nodes unit_collections unit_config_value_encryption unit_connection_tracking
@@ -293,6 +287,7 @@ define tpl_parallel_ct_test_spec
 {logdir, "$(CT_LOGS_DIR)"}.
 {logdir, master, "$(CT_LOGS_DIR)"}.
 {create_priv_dir, all_nodes, auto_per_run}.
+{auto_compile, false}.
 
 {node, shard1, 'rabbit_shard1@localhost'}.
 {node, shard2, 'rabbit_shard2@localhost'}.

deps/rabbitmq_ct_helpers/app.bzl

@@ -11,9 +11,10 @@ def all_beam_files(name = "all_beam_files"):
         name = "other_beam",
         testonly = True,
         srcs = [
+            "src/ct_master_event_fork.erl",
             "src/ct_master_fork.erl",
+            "src/ct_master_logs_fork.erl",
             "src/cth_log_redirect_any_domains.erl",
-            "src/cth_parallel_ct_detect_failure.erl",
             "src/rabbit_control_helper.erl",
             "src/rabbit_ct_broker_helpers.erl",
             "src/rabbit_ct_config_schema.erl",
@@ -39,9 +40,10 @@ def all_test_beam_files(name = "all_test_beam_files"):
         name = "test_other_beam",
         testonly = True,
         srcs = [
+            "src/ct_master_event_fork.erl",
             "src/ct_master_fork.erl",
+            "src/ct_master_logs_fork.erl",
             "src/cth_log_redirect_any_domains.erl",
-            "src/cth_parallel_ct_detect_failure.erl",
             "src/rabbit_control_helper.erl",
             "src/rabbit_ct_broker_helpers.erl",
             "src/rabbit_ct_config_schema.erl",
@@ -103,9 +105,10 @@ def all_srcs(name = "all_srcs"):
         name = "srcs",
         testonly = True,
         srcs = [
+            "src/ct_master_event_fork.erl",
             "src/ct_master_fork.erl",
+            "src/ct_master_logs_fork.erl",
             "src/cth_log_redirect_any_domains.erl",
-            "src/cth_parallel_ct_detect_failure.erl",
             "src/rabbit_control_helper.erl",
             "src/rabbit_ct_broker_helpers.erl",
             "src/rabbit_ct_config_schema.erl",

deps/rabbitmq_ct_helpers/src/ct_master_event_fork.erl

@@ -0,0 +1,217 @@
+%%
+%% %CopyrightBegin%
+%% 
+%% Copyright Ericsson AB 2006-2024. All Rights Reserved.
+%% 
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%% 
+%% %CopyrightEnd%
+%%
+
+%%% Common Test Framework Event Handler
+%%%
+%%% This module implements an event handler that the CT Master
+%%% uses to handle status and progress notifications sent to the
+%%% master node during test runs. It also keeps track of the
+%%% details of failures which are used by the CT Master to print
+%%% a summary at the end of its run. This module may be used as a
+%%% template for other event handlers that can be plugged in to 
+%%% handle logging and reporting on the master node.
+-module(ct_master_event_fork).
+-moduledoc false.
+
+-behaviour(gen_event).
+
+%% API
+-export([start_link/0, add_handler/0, add_handler/1, stop/0]).
+-export([notify/1, sync_notify/1, get_results/0]).
+
+%% gen_event callbacks
+-export([init/1, handle_event/2, handle_call/2, 
+	 handle_info/2, terminate/2, code_change/3]).
+
+-include_lib("common_test/include/ct_event.hrl").
+-include_lib("common_test/src/ct_util.hrl").
+
+
+-record(state, {auto_skipped=[], failed=[]}).
+
+%%====================================================================
+%% gen_event callbacks
+%%====================================================================
+%%--------------------------------------------------------------------
+%% Function: start_link() -> {ok,Pid} | {error,Error} 
+%% Description: Creates an event manager.
+%%--------------------------------------------------------------------
+start_link() ->
+    gen_event:start_link({local,?CT_MEVMGR}). 
+
+%%--------------------------------------------------------------------
+%% Function: add_handler() -> ok | {'EXIT',Reason} | term()
+%% Description: Adds an event handler
+%%--------------------------------------------------------------------
+add_handler() ->
+    gen_event:add_handler(?CT_MEVMGR_REF,?MODULE,[]).
+add_handler(Args) ->
+    gen_event:add_handler(?CT_MEVMGR_REF,?MODULE,Args).
+
+%%--------------------------------------------------------------------
+%% Function: stop() -> ok
+%% Description: Stops the event manager
+%%--------------------------------------------------------------------
+stop() ->
+    case flush() of
+	{error,Reason} ->
+	    ct_master_logs_fork:log("Error",
+			       "No response from CT Master Event.\n"
+			       "Reason = ~tp\n"
+			       "Terminating now!\n",[Reason]),
+	    %% communication with event manager fails, kill it
+	    catch exit(whereis(?CT_MEVMGR_REF), kill);
+	_ ->
+	    gen_event:stop(?CT_MEVMGR_REF)
+    end.
+
+flush() ->
+    try gen_event:call(?CT_MEVMGR_REF,?MODULE,flush,1800000) of
+	flushing ->
+	    timer:sleep(1),
+	    flush();
+	done ->
+	    ok;
+	Error = {error,_} ->
+	    Error
+    catch
+	_:Reason ->
+	    {error,Reason}
+    end.
+
+%%--------------------------------------------------------------------
+%% Function: notify(Event) -> ok
+%% Description: Asynchronous notification to event manager.
+%%--------------------------------------------------------------------
+notify(Event) ->
+    gen_event:notify(?CT_MEVMGR_REF,Event).
+
+%%--------------------------------------------------------------------
+%% Function: sync_notify(Event) -> ok
+%% Description: Synchronous notification to event manager.
+%%--------------------------------------------------------------------
+sync_notify(Event) ->
+    gen_event:sync_notify(?CT_MEVMGR_REF,Event).
+
+%%--------------------------------------------------------------------
+%% Function: sync_notify(Event) -> Results
+%% Description: Get the results for auto-skipped and failed test cases.
+%%--------------------------------------------------------------------
+get_results() ->
+    gen_event:call(?CT_MEVMGR_REF,?MODULE,get_results).
+
+%%====================================================================
+%% gen_event callbacks
+%%====================================================================
+%%--------------------------------------------------------------------
+%% Function: init(Args) -> {ok, State}
+%% Description: Whenever a new event handler is added to an event manager,
+%% this function is called to initialize the event handler.
+%%--------------------------------------------------------------------
+init(_) ->
+    ct_util:mark_process(),
+    ct_master_logs_fork:log("CT Master Event Handler started","",[]),
+    {ok,#state{}}.
+
+%%--------------------------------------------------------------------
+%% Function:  
+%% handle_event(Event, State) -> {ok, State} |
+%%                               {swap_handler, Args1, State1, Mod2, Args2} |
+%%                               remove_handler
+%% Description:Whenever an event manager receives an event sent using
+%% gen_event:notify/2 or gen_event:sync_notify/2, this function is called for
+%% each installed event handler to handle the event. 
+%%--------------------------------------------------------------------
+handle_event(#event{name=start_logging,node=Node,data=RunDir},State) ->
+    ct_master_logs_fork:log("CT Master Event Handler","Got ~ts from ~w",[RunDir,Node]),
+    ct_master_logs_fork:nodedir(Node,RunDir),
+    {ok,State};
+
+handle_event(Event=#event{name=Name,node=Node,data=Data},State) ->
+    print("~n=== ~w ===~n", [?MODULE]),
+    print("~tw on ~w: ~tp~n", [Name,Node,Data]),
+    {ok,maybe_store_event(Event,State)}.
+
+%%--------------------------------------------------------------------
+%% Function: 
+%% handle_call(Request, State) -> {ok, Reply, State} |
+%%                                {swap_handler, Reply, Args1, State1, 
+%%                                  Mod2, Args2} |
+%%                                {remove_handler, Reply}
+%% Description: Whenever an event manager receives a request sent using
+%% gen_event:call/3,4, this function is called for the specified event 
+%% handler to handle the request.
+%%--------------------------------------------------------------------
+handle_call(get_results,State=#state{auto_skipped=AutoSkipped,failed=Failed}) ->
+    {ok,#{
+        auto_skipped => lists:sort(AutoSkipped),
+        failed => lists:sort(Failed)
+    },State};
+handle_call(flush,State) ->
+    case process_info(self(),message_queue_len) of
+	{message_queue_len,0} ->
+	    {ok,done,State};
+	_ ->
+	    {ok,flushing,State}
+    end.
+
+%%--------------------------------------------------------------------
+%% Function: 
+%% handle_info(Info, State) -> {ok, State} |
+%%                             {swap_handler, Args1, State1, Mod2, Args2} |
+%%                              remove_handler
+%% Description: This function is called for each installed event handler when
+%% an event manager receives any other message than an event or a synchronous
+%% request (or a system message).
+%%--------------------------------------------------------------------
+handle_info(_Info,State) ->
+    {ok,State}.
+
+%%--------------------------------------------------------------------
+%% Function: terminate(Reason, State) -> ok
+%% Description:Whenever an event handler is deleted from an event manager,
+%% this function is called. It should be the opposite of Module:init/1 and 
+%% do any necessary cleaning up. 
+%%--------------------------------------------------------------------
+terminate(_Reason,_State) ->
+    ct_master_logs_fork:log("CT Master Event Handler stopping","",[]),
+    ok.
+
+%%--------------------------------------------------------------------
+%% Function: code_change(OldVsn, State, Extra) -> {ok, NewState} 
+%% Description: Convert process state when code is changed
+%%--------------------------------------------------------------------
+code_change(_OldVsn,State,_Extra) ->
+    {ok,State}.
+
+%%--------------------------------------------------------------------
+%%% Internal functions
+%%--------------------------------------------------------------------
+
+print(_Str,_Args) ->
+%    io:format(_Str,_Args),
+    ok.
+
+maybe_store_event(#event{name=tc_done,node=Node,data={Suite,FuncOrGroup,{auto_skipped,Reason}}},State=#state{auto_skipped=Acc}) ->
+    State#state{auto_skipped=[{Node,Suite,FuncOrGroup,Reason}|Acc]};
+maybe_store_event(#event{name=tc_done,node=Node,data={Suite,FuncOrGroup,{failed,Reason}}},State=#state{failed=Acc}) ->
+    State#state{failed=[{Node,Suite,FuncOrGroup,Reason}|Acc]};
+maybe_store_event(_Event,State) ->
+    State.