DPL: add test to verify crashing workflows

Framework/Core/CMakeLists.txt

@@ -281,6 +281,11 @@ o2_add_test(O2DatabasePDG NAME test_Framework_test_O2DatabasePDG
             LABELS framework
             PUBLIC_LINK_LIBRARIES O2::Framework O2::FrameworkPhysicsSupport)
 
+o2_add_executable(crashing-workflow
+            SOURCES test/test_CrashingWorkflow.cxx
+            COMPONENT_NAME Framework
+            PUBLIC_LINK_LIBRARIES O2::Framework)
+
 # All the tests which require ROOT to work
 add_executable(o2-test-framework-root
                test/test_Root2ArrowTable.cxx
@@ -290,6 +295,7 @@ target_link_libraries(o2-test-framework-root PRIVATE O2::Catch2)
 target_link_libraries(o2-test-framework-root PRIVATE ROOT::ROOTDataFrame)
 set_property(TARGET o2-test-framework-root PROPERTY RUNTIME_OUTPUT_DIRECTORY ${outdir})
 add_test(NAME framework:root COMMAND o2-test-framework-root --skip-benchmarks)
+add_test(NAME framework:crash COMMAND  sh -e -c "PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY}:$PATH ${CMAKE_CURRENT_LIST_DIR}/test/test_AllCrashTypes.sh")
 
 o2_add_test(InfoLogger NAME test_Framework_test_InfoLogger
             SOURCES test/test_InfoLogger.cxx

Framework/Core/test/test_AllCrashTypes.sh

@@ -0,0 +1,27 @@
+#!/bin/sh -e
+echo $PATH
+printf "Testing abort-init..."
+o2-framework-crashing-workflow --crash-type=abort-init --completion-policy=quit -b --run | grep -q "Abort trap" || { echo "abort not found in init"; exit 1; }
+printf "ok\nTesting runtime-init..."
+o2-framework-crashing-workflow --crash-type=runtime-init --completion-policy=quit -b --run | grep -q "Exception caught: This is a std::runtime_error" || { printf "runtime error not found" ; exit 1; }
+printf "ok\nTesting framework-init..."
+o2-framework-crashing-workflow --crash-type=framework-init --completion-policy=quit -b --run | grep -q "Exception caught: This is a o2::framework::runtime_error" || { printf "framework error not found" ;  exit 1; }
+printf "ok\nTesting abort-run..."
+o2-framework-crashing-workflow --crash-type=abort-run --completion-policy=quit -b --run | grep -q "Program crashed (Abort trap: 6)" || { printf "abort not found" ; exit 1; }
+printf "ok\nTesting framework-run..."
+o2-framework-crashing-workflow --crash-type=framework-run --completion-policy=quit -b --run | grep -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a o2::framework::runtime_error" || { printf "framework error not found" ; exit 1; }
+printf "ok\nTesting runtime-run..."
+o2-framework-crashing-workflow --crash-type=runtime-run --completion-policy=quit --run | grep -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a std::runtime_error" || { echo "runtime error not found" ; exit 1; }
+printf "ok\n"
+
+export O2_NO_CATCHALL_EXCEPTIONS=1
+echo O2_NO_CATCHALL_EXCEPTIONS enabled
+printf "ok\nTesting runtime-init..."
+o2-framework-crashing-workflow --crash-type=runtime-init --completion-policy=quit -b --run | grep -v -q "Exception caught while in Init: This is a std::runtime_error. Exiting with 1." || { printf "runtime error not found" ; exit 1; }
+printf "ok\nTesting framework-init..."
+o2-framework-crashing-workflow --crash-type=framework-init --completion-policy=quit -b --run | grep -v -q "Exception caught while in Init: This is a o2::framework::runtime_error. Exiting with 1" || { printf "framework error not found" ;  exit 1; }
+printf "ok\nTesting framework-run..."
+o2-framework-crashing-workflow --crash-type=framework-run --completion-policy=quit -b --run | grep -v -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a o2::framework::runtime_error" || { printf "framework error not found" ; exit 1; }
+printf "ok\nTesting runtime-run..."
+o2-framework-crashing-workflow --crash-type=runtime-run --completion-policy=quit --run | grep -v -q "Unhandled o2::framework::runtime_error reached the top of main of o2-framework-crashing-workflow, device shutting down. Reason: This is a std::runtime_error" || { echo "runtime error not found" ; exit 1; }
+printf "ok"

Framework/Core/test/test_CrashingWorkflow.cxx

@@ -0,0 +1,88 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+#include "Framework/ConfigParamSpec.h"
+#include "Framework/AlgorithmSpec.h"
+#include "Framework/Configurable.h"
+#include "Framework/Logger.h"
+#include "Framework/CallbackService.h"
+#include "Framework/Signpost.h"
+
+O2_DECLARE_DYNAMIC_LOG(crash_test);
+
+using namespace o2::framework;
+
+struct WorkflowOptions {
+  Configurable<std::string> crashType{"crash-type", "fatal-init", {"how should this crash? (fatal-init, fatal-run, runtime-init, runtime-fail, abort-init, abort-run)"}};
+};
+
+#include "Framework/runDataProcessing.h"
+
+AlgorithmSpec simpleCrashingSource(std::string const& what)
+{
+  return AlgorithmSpec{adaptStateful([what](InitContext& ctx) {
+    O2_SIGNPOST_ID_FROM_POINTER(ii, crash_test, &ctx);
+    O2_SIGNPOST_START(crash_test, ii, "Init", "Starting Init");
+    O2_SIGNPOST_EVENT_EMIT(crash_test, ii, "Init", "%{public}s selected", what.c_str());
+
+    if (what == "fatal-init") {
+      LOG(fatal) << "This should have a fatal";
+    } else if (what == "runtime-init") {
+      throw std::runtime_error("This is a std::runtime_error");
+    } else if (what == "abort-init") {
+      abort();
+    } else if (what == "framework-init") {
+      throw o2::framework::runtime_error("This is a o2::framework::runtime_error");
+    } else if (what == "framework-prerun") {
+      ctx.services().get<CallbackService>().set<CallbackService::Id::PreProcessing>([](ServiceRegistryRef, int) {
+        throw o2::framework::runtime_error("This is o2::framework::runtime_error in PreProcessing");
+      });
+    } else if (what == "runtime-prerun") {
+      ctx.services().get<CallbackService>().set<CallbackService::Id::PreProcessing>([](ServiceRegistryRef, int) {
+        throw std::runtime_error("This is std::runtime_error in PreProcessing");
+      });
+    }
+    O2_SIGNPOST_END(crash_test, ii, "Init", "Init Done");
+    return adaptStateless([what](ProcessingContext& pCtx) {
+      O2_SIGNPOST_ID_FROM_POINTER(ri, crash_test, &pCtx);
+      O2_SIGNPOST_START(crash_test, ri, "Run", "Starting Run");
+      O2_SIGNPOST_EVENT_EMIT(crash_test, ri, "Run", "%{public}s selected", what.c_str());
+      if (what == "fatal-run") {
+        LOG(fatal) << "This should have a fatal";
+      } else if (what == "runtime-run") {
+        throw std::runtime_error("This is a std::runtime_error");
+      } else if (what == "abort-run") {
+        abort();
+      } else if (what == "framework-run") {
+        throw o2::framework::runtime_error("This is a o2::framework::runtime_error");
+      }
+      O2_SIGNPOST_EVENT_EMIT_ERROR(crash_test, ri, "Run", "Unknown option for crash-type: %{public}s.", what.c_str());
+      O2_SIGNPOST_END(crash_test, ri, "Init", "Run Done");
+      exit(1);
+    });
+  })};
+}
+
+// This is how you can define your processing in a declarative way
+WorkflowSpec defineDataProcessing(ConfigContext const& config)
+{
+  auto crashType = config.options().get<std::string>("crash-type");
+  DataProcessorSpec a{
+    .name = "deliberately-crashing",
+    .outputs = {OutputSpec{{"a1"}, "TST", "A1"}},
+    .algorithm = AlgorithmSpec{simpleCrashingSource(crashType)}};
+  DataProcessorSpec b{
+    .name = "B",
+    .inputs = {InputSpec{"x", "TST", "A1", Lifetime::Timeframe}},
+    .algorithm = AlgorithmSpec{adaptStateless([](ProcessingContext&) {})}};
+
+  return workflow::concat(WorkflowSpec{a, b});
+}
+