Merge 18a9be8 into sapling-pr-archive-ktf

Common/Utils/include/CommonUtils/IRFrameSelector.h

@@ -46,6 +46,8 @@ class IRFrameSelector
   auto getIRFrames() const { return mFrames; }
   bool isSet() const { return mIsSet; }
 
+  void setOwnList(const std::vector<o2::dataformats::IRFrame>& lst, bool toBeSorted);
+
  private:
   gsl::span<const o2::dataformats::IRFrame> mFrames{}; // externally provided span of IRFrames, must be sorted in IRFrame.getMin()
   o2::dataformats::IRFrame mLastIRFrameChecked{};      // last frame which was checked

Common/Utils/src/IRFrameSelector.cxx

@@ -167,6 +167,16 @@ size_t IRFrameSelector::loadIRFrames(const std::string& fname)
   return mOwnList.size();
 }
 
+void IRFrameSelector::setOwnList(const std::vector<o2::dataformats::IRFrame>& lst, bool toBeSorted)
+{
+  clear();
+  mOwnList.insert(mOwnList.end(), lst.begin(), lst.end());
+  if (toBeSorted) {
+    std::sort(mOwnList.begin(), mOwnList.end(), [](const auto& a, const auto& b) { return a.getMin() < b.getMin(); });
+  }
+  setSelectedIRFrames(mOwnList, 0, 0, 0, false);
+}
+
 void IRFrameSelector::print(bool lst) const
 {
   LOGP(info, "Last query stopped at entry {} for IRFrame {}:{}", mLastBoundID,
@@ -183,6 +193,8 @@ void IRFrameSelector::clear()
 {
   mIsSet = false;
   mOwnList.clear();
+  mLastIRFrameChecked.getMin().clear(); // invalidate
+  mLastBoundID = -1;
   mFrames = {};
 }
 

Detectors/AOD/include/AODProducerWorkflow/AODProducerHelpers.h

@@ -18,8 +18,6 @@
 #include <boost/functional/hash.hpp>
 #include <boost/tuple/tuple.hpp>
 #include <boost/unordered_map.hpp>
-#include <string>
-#include <vector>
 #include <Framework/AnalysisHelpers.h>
 
 namespace o2::aodhelpers
@@ -55,7 +53,7 @@ auto createTableCursor(framework::ProcessingContext& pc)
   framework::Produces<T> c;
   c.resetCursor(pc.outputs()
                   .make<framework::TableBuilder>(framework::OutputForTable<T>::ref()));
-  c.setLabel(o2::aod::MetadataTrait<T>::metadata::tableLabel());
+  c.setLabel(aod::label<T::ref>());
   return c;
 }
 } // namespace o2::aodhelpers

Detectors/AOD/include/AODProducerWorkflow/AODProducerWorkflowSpec.h

@@ -21,7 +21,6 @@
 #include "DataFormatsTRD/TrackTRD.h"
 #include "DetectorsBase/GRPGeomHelper.h"
 #include "DetectorsBase/Propagator.h"
-#include "Framework/AnalysisHelpers.h"
 #include "Framework/DataProcessorSpec.h"
 #include "Framework/Task.h"
 #include "ReconstructionDataFormats/GlobalTrackID.h"
@@ -36,7 +35,6 @@
 #include <cstdint>
 #include <limits>
 #include <set>
-#include <string>
 #include <vector>
 #include <random>
 using namespace o2::framework;

Detectors/AOD/src/StandaloneAODProducer.cxx

@@ -94,7 +94,7 @@ void fillMCollisionTable(o2::steer::MCKinematicsReader const& mcreader)
 
   TFile outfile("aod.root", "UPDATE");
   {
-    TableToTree t2t(mccoltable, &outfile, aod::MetadataTrait<o2::aod::McCollisions>::metadata::tableLabel());
+    TableToTree t2t(mccoltable, &outfile, aod::description_str(aod::signature<o2::aod::McCollisions::ref>()).data());
     t2t.addAllBranches();
     t2t.process();
   }
@@ -200,12 +200,12 @@ void fillCollisionAndTrackTable()
       f.Close();
       TFile outfile("aod.root", "RECREATE");
       {
-        TableToTree t2t(colltable, &outfile, aod::MetadataTrait<o2::aod::Collisions>::metadata::tableLabel());
+        TableToTree t2t(colltable, &outfile, aod::description_str(aod::signature<o2::aod::Collisions::ref>()).data());
         t2t.addAllBranches();
         t2t.process();
       }
       {
-        TableToTree t2t(tracktable, &outfile, "Tracks" /* aod::MetadataTrait<o2::aod::Tracks>::metadata::tableLabel() */);
+        TableToTree t2t(tracktable, &outfile, aod::description_str(aod::signature<o2::aod::StoredTracks::ref>()).data());
         t2t.addAllBranches();
         t2t.process();
       }

Detectors/CTF/README.md

@@ -95,6 +95,14 @@ comma-separated list of detectors to read, Overrides skipDet
 ```
 comma-separated list of detectors to skip
 
+
+By default an exception will be thrown if detector is requested but missing in the CTF. To enable injection of the empty output in such case one should use option `--allow-missing-detectors`.
+
+```
+--ctf-data-subspec arg (=0)
+```
+allows to alter the `subSpecification` used to send the CTFDATA from the reader to decoders. Non-0 value must be used in case the data extracted by the CTF-reader should be processed and stored in new CTFs (in order to avoid clash of CTFDATA messages of the reader and writer).
+
 ```
 --max-tf arg (=-1)
 ```
@@ -141,31 +149,34 @@ There is a possibility to read remote root files directly, w/o caching them loca
 2) provide proper regex to define remote files, e.g. for the example above: `--remote-regex "^root://.+/eos/aliceo2/.+"`.
 3) pass an option `--copy-cmd no-copy`.
 
+## Selective TF reading
+
 ```
 --select-ctf-ids <id's of CTFs to select>
 ```
 This is a `ctf-reader` device local option allowing selective reading of particular CTFs. It is useful when dealing with CTF files containing multiple TFs. The comma-separated list of increasing CTFs indices must be provided in the format parsed by the `RangeTokenizer<int>`, e.g. `1,4-6,...`.
 Note that the index corresponds not to the entry of the TF in the CTF tree but to the reader own counter incremented throught all input files (e.g. if the 10 CTF files with 20 TFs each are provided for the input and the selection of TFs
 `0,2,22,66` is provided, the reader will inject to the DPL the TFs at entries 0 and 2 from the 1st CTF file, entry 5 of the second file, entry 6 of the 3d and will finish the job.
 
-For the ITS and MFT entropy decoding one can request either to decompose clusters to digits and send them instead of clusters (via `o2-ctf-reader-workflow` global options `--its-digits` and `--mft-digits` respectively)
-or to apply the noise mask to decoded clusters (or decoded digits). If the masking (e.g. via option `--its-entropy-decoder " --mask-noise "`) is requested, user should provide to the entropy decoder the noise mask file (eventually will be loaded from CCDB) and cluster patterns decoding dictionary (if the clusters were encoded with patterns IDs).
-For example,
 ```
-o2-ctf-reader-workflow --ctf-input <ctfFiles> --onlyDet ITS,MFT --its-entropy-decoder ' --mask-noise' | ...
+--ir-frames-files <root_file_with_IRFrames_to_select> --skip-skimmed-out-tf
 ```
-will decode ITS and MFT data, decompose on the fly ITS clusters to digits, mask the noisy pixels with the provided masks, recluster remaining ITS digits and send the new clusters out, together with unchanged MFT clusters.
+This option (used for skimming) allow to push to DPL only those TFs which overlap with selected BC-ranges provided via input root file (for various formats see `o2::utils::IRFrameSelector::loadIRFrames` method).
+
 ```
-o2-ctf-reader-workflow --ctf-input <ctfFiles> --onlyDet ITS,MFT --mft-digits --mft-entropy-decoder ' --mask-noise' | ...
+--ir-frames-files <root_file_with_IRFrames_to_select>
 ```
-will send decompose clusters to digits and send ben out after masking the noise for the MFT, while ITS clusters will be sent as decoded.
-
-By default an exception will be thrown if detector is requested but missing in the CTF. To enable injection of the empty output in such case one should use option `--allow-missing-detectors`.
+This option allows to push to DPL only those TFs which overlap with the `<runnumber> <range-min> <range-max>` (separators can be any whitespace, comma or semicolon) records provided via text file (assuming that there are some entries for a given run, otherwise the option is ignored).
+Multiple ranges per run and multiple runs can be mentioned in a single input file. The range limits can be indicated either as a UNIX timestamp in `ms` or as an orbit number (in the fill the run belongs to).
 
+In case an option
 ```
---ctf-data-subspec arg (=0)
+--invert-irframe-selection
 ```
-allows to alter the `subSpecification` used to send the CTFDATA from the reader to decoders. Non-0 value must be used in case the data extracted by the CTF-reader should be processed and stored in new CTFs (in order to avoid clash of CTFDATA messages of the reader and writer).
+is provided, the selections above are inverted: TFs matching some of the provided ranges will be discarded, while the rest will be pushed to the DPL
+
+At the end of the processing the `ctf-writer` will create a local file `ctf_read_ntf.txt` containing only the number of TFs pushed to the DPL.
+In case no TF passed the selections above, this file will contain 0.
 
 ## Support for externally provided encoding dictionaries
 
@@ -201,3 +212,18 @@ Additionally, one may throttle on the free SHM by providing an option to the rea
 
 Note that by default the reader reads into the memory the CTF data and prepares all output messages but injects them only once the rate-limiter allows that.
 With the option `--limit-tf-before-reading` set also the preparation of the data to inject will be conditioned by the green light from the rate-limiter.
+
+
+## Modifying ITS/MFT CTF output
+
+For the ITS and MFT entropy decoding one can request either to decompose clusters to digits and send them instead of clusters (via `o2-ctf-reader-workflow` global options `--its-digits` and `--mft-digits` respectively)
+or to apply the noise mask to decoded clusters (or decoded digits). If the masking (e.g. via option `--its-entropy-decoder " --mask-noise "`) is requested, user should provide to the entropy decoder the noise mask file (eventually will be loaded from CCDB) and cluster patterns decoding dictionary (if the clusters were encoded with patterns IDs).
+For example,
+```
+o2-ctf-reader-workflow --ctf-input <ctfFiles> --onlyDet ITS,MFT --its-entropy-decoder ' --mask-noise' | ...
+```
+will decode ITS and MFT data, decompose on the fly ITS clusters to digits, mask the noisy pixels with the provided masks, recluster remaining ITS digits and send the new clusters out, together with unchanged MFT clusters.
+```
+o2-ctf-reader-workflow --ctf-input <ctfFiles> --onlyDet ITS,MFT --mft-digits --mft-entropy-decoder ' --mask-noise' | ...
+```
+will send decompose clusters to digits and send ben out after masking the noise for the MFT, while ITS clusters will be sent as decoded.

Detectors/CTF/workflow/include/CTFWorkflow/CTFReaderSpec.h

@@ -31,8 +31,10 @@ struct CTFReaderInp {
   std::string remoteRegex{};
   std::string metricChannel{};
   std::string fileIRFrames{};
+  std::string fileRunTimeSpans{};
   std::vector<int> ctfIDs{};
   bool skipSkimmedOutTF = false;
+  bool invertIRFramesSelection = false;
   bool allowMissingDetectors = false;
   bool checkTFLimitBeforeReading = false;
   bool sup0xccdb = false;