From e2a70de4d06ce040b6580fa1fe6f4f766a8d498e Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Fri, 8 Nov 2024 11:51:34 +0100 Subject: [PATCH] DPL: account for aod-parent-access-level == 0 In the case of self contained derived data, it is ok not to look in the parents for missing metadata. --- .../src/AODJAlienReaderHelpers.cxx | 23 +++++++++---------- Framework/AnalysisSupport/src/Plugin.cxx | 9 +++++++- Framework/Core/src/Plugin.cxx | 5 +++- Framework/Core/src/WorkflowHelpers.cxx | 1 - 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx index 90d88cb43626e..9c19de85739ce 100644 --- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx +++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx @@ -118,19 +118,23 @@ static inline auto extractOriginalsTuple(framework::pack, ProcessingConte return std::make_tuple(extractTypedOriginal(pc)...); } -AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback(ConfigContext const& config) +AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback(ConfigContext const& ctx) { // aod-parent-base-path-replacement is now a workflow option, so it needs to be // retrieved from the ConfigContext. This is because we do not allow workflow options // to change over start-stop-start because they can affect the topology generation. std::string parentFileReplacement; - if (config.options().isSet("aod-parent-base-path-replacement")) { - parentFileReplacement = config.options().get("aod-parent-base-path-replacement"); + if (ctx.options().isSet("aod-parent-base-path-replacement")) { + parentFileReplacement = ctx.options().get("aod-parent-base-path-replacement"); } - auto callback = AlgorithmSpec{adaptStateful([parentFileReplacement](ConfigParamRegistry const& options, - DeviceSpec const& spec, - Monitoring& monitoring, - DataProcessingStats& stats) { + int parentAccessLevel = 0; + if (ctx.options().isSet("aod-parent-access-level")) { + parentAccessLevel = ctx.options().get("aod-parent-access-level"); + } + auto callback = AlgorithmSpec{adaptStateful([parentFileReplacement, parentAccessLevel](ConfigParamRegistry const& options, + DeviceSpec const& spec, + Monitoring& monitoring, + DataProcessingStats& stats) { // FIXME: not actually needed, since data processing stats can specify that we should // send the initial value. stats.updateStats({static_cast(ProcessingStatsId::ARROW_BYTES_CREATED), DataProcessingStats::Op::Set, 0}); @@ -148,11 +152,6 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback(ConfigContext const auto maxRate = options.get("aod-max-io-rate"); - int parentAccessLevel = 0; - if (options.isSet("aod-parent-access-level")) { - parentAccessLevel = options.get("aod-parent-access-level"); - } - // create a DataInputDirector auto didir = std::make_shared(filename, &monitoring, parentAccessLevel, parentFileReplacement); if (options.isSet("aod-reader-json")) { diff --git a/Framework/AnalysisSupport/src/Plugin.cxx b/Framework/AnalysisSupport/src/Plugin.cxx index e2a9a98ab90d3..bba3499286e08 100644 --- a/Framework/AnalysisSupport/src/Plugin.cxx +++ b/Framework/AnalysisSupport/src/Plugin.cxx @@ -150,13 +150,20 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin { return results; } - // Lets try in parent files + if (!registry.isSet("aod-parent-access-level") || registry.get("aod-parent-access-level") == 0) { + LOGP(info, "No metadata found in file \"{}\" and parent level 0 prevents further lookup.", filename); + results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}}); + return results; + } + + // Lets try in parent file. auto parentFiles = (TMap*)currentFile->Get("parentFiles"); if (!parentFiles) { LOGP(info, "No metadata found in file \"{}\"", filename); results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}}); return results; } + LOGP(info, "No metadata found in file \"{}\", checking in its parents.", filename); for (auto* p : *parentFiles) { std::string parentFilename = ((TPair*)p)->Value()->GetName(); // Do the replacement. Notice this will require changing aod-parent-base-path-replacement to be diff --git a/Framework/Core/src/Plugin.cxx b/Framework/Core/src/Plugin.cxx index 91c74bafff5ad..726b12ff68365 100644 --- a/Framework/Core/src/Plugin.cxx +++ b/Framework/Core/src/Plugin.cxx @@ -141,7 +141,7 @@ struct DiscoverAODOptionsInCommandLine : o2::framework::ConfigDiscoveryPlugin { bool injectOption = true; for (size_t i = 0; i < argc; i++) { std::string_view arg = argv[i]; - if (!arg.starts_with("--aod-writer-") && arg != "--aod-parent-base-path-replacement") { + if (!arg.starts_with("--aod-writer-") && !arg.starts_with("--aod-parent-")) { continue; } std::string key = arg.data() + 2; @@ -156,6 +156,9 @@ struct DiscoverAODOptionsInCommandLine : o2::framework::ConfigDiscoveryPlugin { if (key == "aod-parent-base-path-replacement") { results.push_back(ConfigParamSpec{"aod-parent-base-path-replacement", VariantType::String, value, {R"(Replace base path of parent files. Syntax: FROM;TO. E.g. "alien:///path/in/alien;/local/path". Enclose in "" on the command line.)"}}); } + if (key == "aod-parent-access-level") { + results.push_back(ConfigParamSpec{"aod-parent-access-level", VariantType::String, value, {"Allow parent file access up to specified level. Default: no (0)"}}); + } } if (injectOption) { results.push_back(ConfigParamSpec{"aod-writer-compression", VariantType::Int, 505, {"AOD Compression options"}}); diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx index 56e9930e3b655..da9a135dc5eb8 100644 --- a/Framework/Core/src/WorkflowHelpers.cxx +++ b/Framework/Core/src/WorkflowHelpers.cxx @@ -216,7 +216,6 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext .options = {ConfigParamSpec{"aod-file-private", VariantType::String, ctx.options().get("aod-file"), {"AOD file"}}, ConfigParamSpec{"aod-max-io-rate", VariantType::Float, 0.f, {"Maximum I/O rate in MB/s"}}, ConfigParamSpec{"aod-reader-json", VariantType::String, {"json configuration file"}}, - ConfigParamSpec{"aod-parent-access-level", VariantType::String, {"Allow parent file access up to specified level. Default: no (0)"}}, ConfigParamSpec{"time-limit", VariantType::Int64, 0ll, {"Maximum run time limit in seconds"}}, ConfigParamSpec{"orbit-offset-enumeration", VariantType::Int64, 0ll, {"initial value for the orbit"}}, ConfigParamSpec{"orbit-multiplier-enumeration", VariantType::Int64, 0ll, {"multiplier to get the orbit from the counter"}},