diff --git a/docs/api-reference/tools/index.rst b/docs/api-reference/tools/index.rst index ad122d76dc1..eb41dde8d0d 100644 --- a/docs/api-reference/tools/index.rst +++ b/docs/api-reference/tools/index.rst @@ -82,16 +82,22 @@ Reference/API ============= .. automodapi:: ctapipe.tools - :no-inheritance-diagram: + :no-inheritance-diagram: + +.. automodapi:: ctapipe.tools.quickstart + :no-inheritance-diagram: .. automodapi:: ctapipe.tools.process - :no-inheritance-diagram: + :no-inheritance-diagram: + +.. automodapi:: ctapipe.tools.merge + :no-inheritance-diagram: .. automodapi:: ctapipe.tools.train_energy_regressor - :no-inheritance-diagram: + :no-inheritance-diagram: .. automodapi:: ctapipe.tools.train_particle_classifier - :no-inheritance-diagram: + :no-inheritance-diagram: .. automodapi:: ctapipe.tools.train_disp_reconstructor :no-inheritance-diagram: @@ -104,3 +110,9 @@ Reference/API .. automodapi:: ctapipe.tools.compute_irf :no-inheritance-diagram: + +.. automodapi:: ctapipe.tools.dump_instrument + :no-inheritance-diagram: + +.. automodapi:: ctapipe.tools.calculate_pixel_stats + :no-inheritance-diagram: diff --git a/docs/changes/2691.maintenance.rst b/docs/changes/2691.maintenance.rst new file mode 100644 index 00000000000..90ba1d3f46c --- /dev/null +++ b/docs/changes/2691.maintenance.rst @@ -0,0 +1,4 @@ +Add a tutorial to the docs on how to use the machine learning tools +to process dl1b (image parameters) files to dl2 for a mono analysis. + +Perform minor updates to other parts of the api docs and the user-guide. diff --git a/docs/user-guide/data_models/dl1.rst b/docs/user-guide/data_models/dl1.rst deleted file mode 100644 index 0e555449545..00000000000 --- a/docs/user-guide/data_models/dl1.rst +++ /dev/null @@ -1,134 +0,0 @@ -.. _datamodel: - -.. currentmodule:: ctapipe.io.containers - -****************** -ctapipe Data Model -****************** - -The DL1 files are HDF5 format files, with the following data set hierarchy. -The tables should be written with `pytables `_ (not ``h5py``), -ideally with the :py:class:`~ctapipe.io.HDF5TableWriter`, which ensures the unit -and other descriptive metadata are attached to the output. -Containers marked with a ``+`` should be written without their prefix (all others should use column prefixes). - -The following describes the contents of data level 1 (DL1) output files -generated by ctapipe (e.g. the ``ctapipe-process`` tool which uses the -:py:class:`~ctapipe.io.DataWriter` component to generate output data). - - -DL1 Data Model -============== - -This describes data that change per-event. -The following datasets will be written to the group ``/dl1/event/`` in the output file: - -.. list-table:: - :header-rows: 1 - - * - Group/Dataset - - Description - - Contents - * - ``subarray`` - - event-wise data pertaining to a subarray - - (group) - * - ``subarray/trigger`` - - subarray trigger information - - :py:class:`~ctapipe.containers.EventIndexContainer` +, :py:class:`~ctapipe.containers.TriggerContainer` - * - ``telescope/`` - - Per-telescope Per-event information - - (group) - * - ``telescope/parameters/tel_{TEL_ID:03d}`` - - tables of image parameters (one per telescope) - - :py:class:`~ctapipe.containers.TelEventIndexContainer` +, :py:class:`~ctapipe.containers.ImageParametersContainer` - * - ``telescope/images/tel_{TEL_ID:03d}`` - - tables of telescope images (one per telescope) - - :py:class:`~ctapipe.containers.TelEventIndexContainer` +, :py:class:`~ctapipe.containers.DL1CameraContainer` - - -DL2 Data Model -============== - -This describes data that change per-event. The following datasets will be -written to the group ``/dl2/event/stereo//`` and or -``/dl2/event/mono//``, one for each reconstruction algorithm in the -output file, where ```` is the identifier of the algorithm (e.g. -"Hillas"): - -.. list-table:: - :widths: 25 50 25 - :header-rows: 1 - - * - Group/Dataset - - Description - - Contents - * - /geometry - - shower geometry reconstruction - - :py:class:`~ctapipe.containers.EventIndexContainer`, :py:class:`~ctapipe.containers.ReconstructedGeometryContainer` - * - /energy - - shower energy reconstruction - - :py:class:`~ctapipe.containers.EventIndexContainer`, :py:class:`~ctapipe.containers.ReconstructedEnergyContainer` - * - /classification - - shower classification parameters - - :py:class:`~ctapipe.containers.EventIndexContainer`, :py:class:`~ctapipe.containers.ParticleClassificationContainer` - - -Simulation Data Model -===================== - - * - ``/simulation/event/subarray/shower`` - - true shower parameters from Monte-Carlo simulation - - :py:class:`~ctapipe.containers.EventIndexContainer` +, :py:class:`~ctapipe.containers.SimulatedShowerContainer` - * - ``/simulation/event/telescope/images/tel_{TEL_ID:03d}`` - - simulated camera images - - :py:class:`~ctapipe.containers.EventIndexContainer` +, :py:class:`~ctapipe.containers.SimulatedCameraContainer` - * - ``/simulation/event/telescope/parameters/tel_{TEL_ID:03d}`` - - Parameters derived form the simulated camera images - - :py:class:`~ctapipe.containers.EventIndexContainer` +, :py:class:`~ctapipe.containers.ImageParametersContainer` - * - ``/simulation/service/shower_distribution`` - - simulated shower distribution histograms - - :py:class:`~ctapipe.containers.SimulatedShowerDistribution` - - -Configuration Data Model -======================== - -The output file should also contain serializations of the instrument, observation (if -applicable), simulation (if applicable) configuration information, written to the -``/configuration`` group: - -.. list-table:: - :widths: 25 50 25 - :header-rows: 1 - - * - Group/Dataset - - Description - - Contents - * - ``/instrument`` - - Serialized :py:class:`ctapipe.instrument.SubarrayDescription`, using :py:meth:`ctapipe.instrument.SubarrayDescription.to_hdf` - - (group) - * - ``/instrument/subarray/layout`` - - Subarray layout info - - result of ``SubarrayDescription.to_table()`` output as HDF5 using ``astropy.table`` functionality - * - ``/instrument/telescope/optics`` - - telescope optics information - - result of ``SubarrayDescription.to_table(kind='optics')`` output as HDF5 using - ``astropy.table`` functionality - * - ``/instrument/telescope/camera/{CAMERA_ID}`` - - camera geometry information - - result of ``CameraGeometry.to_table()`` output as HDF5 using ``astropy.table`` - functionality - * - ``/simulation`` - - Monte-Carlo simulation configuration information - - (group) - * - ``/simulation/run`` - - Monte-Carlo simulation run information - - :py:class:`~ctapipe.containers.SimulationConfigContainer` - - -Core Provenance -=============== - -The root group of the file shall contain all of the "CTA Core Provenance Metadata" -headers as user attributes, with the hierarchy flattened and separated by spaces -(e.g. ``"CTA ACTIVITY NAME" = "ctapipe-process"``). diff --git a/docs/user-guide/data_models/index.rst b/docs/user-guide/data_models/index.rst index 3ee38f83d7f..8e8fb40eabb 100644 --- a/docs/user-guide/data_models/index.rst +++ b/docs/user-guide/data_models/index.rst @@ -4,6 +4,127 @@ Data Models *********** -.. toctree:: +The ``ctapipe`` output files are HDF5 format files, with the following data set hierarchy. +The tables are written with `pytables `_ (not ``h5py``). +Containers marked with a ``+`` should be written without their prefix (all others should use column prefixes). - dl1 +The following describes the contents of data level 1 (DL1) output files +generated by ctapipe (e.g. the ``ctapipe-process`` tool which uses the +:py:class:`~ctapipe.io.DataWriter` component to generate output data). + + +DL1 Data Model +============== + +This describes data that change per-event. +The following datasets will be written to the group ``/dl1/event/`` in the output file: + +.. list-table:: + :header-rows: 1 + + * - Group/Dataset + - Description + - Contents + * - ``subarray`` + - event-wise data pertaining to a subarray + - (group) + * - ``subarray/trigger`` + - subarray trigger information + - :py:class:`~ctapipe.containers.EventIndexContainer` +, :py:class:`~ctapipe.containers.TriggerContainer` + * - ``telescope/`` + - Per-telescope Per-event information + - (group) + * - ``telescope/parameters/tel_{TEL_ID:03d}`` + - tables of image parameters (one per telescope) + - :py:class:`~ctapipe.containers.TelEventIndexContainer` +, :py:class:`~ctapipe.containers.ImageParametersContainer` + * - ``telescope/images/tel_{TEL_ID:03d}`` + - tables of telescope images (one per telescope) + - :py:class:`~ctapipe.containers.TelEventIndexContainer` +, :py:class:`~ctapipe.containers.DL1CameraContainer` + + +DL2 Data Model +============== + +This describes data that change per-event. The following datasets will be +written to the group ``/dl2/event/stereo//`` and or +``/dl2/event/mono//``, one for each reconstruction algorithm in the +output file, where ```` is the identifier of the algorithm (e.g. +"Hillas"): + +.. list-table:: + :widths: 25 50 25 + :header-rows: 1 + + * - Group/Dataset + - Description + - Contents + * - /geometry + - shower geometry reconstruction + - :py:class:`~ctapipe.containers.EventIndexContainer`, :py:class:`~ctapipe.containers.ReconstructedGeometryContainer` + * - /energy + - shower energy reconstruction + - :py:class:`~ctapipe.containers.EventIndexContainer`, :py:class:`~ctapipe.containers.ReconstructedEnergyContainer` + * - /classification + - shower classification parameters + - :py:class:`~ctapipe.containers.EventIndexContainer`, :py:class:`~ctapipe.containers.ParticleClassificationContainer` + + +Simulation Data Model +===================== + + * - ``/simulation/event/subarray/shower`` + - true shower parameters from Monte-Carlo simulation + - :py:class:`~ctapipe.containers.EventIndexContainer` +, :py:class:`~ctapipe.containers.SimulatedShowerContainer` + * - ``/simulation/event/telescope/images/tel_{TEL_ID:03d}`` + - simulated camera images + - :py:class:`~ctapipe.containers.EventIndexContainer` +, :py:class:`~ctapipe.containers.SimulatedCameraContainer` + * - ``/simulation/event/telescope/parameters/tel_{TEL_ID:03d}`` + - Parameters derived form the simulated camera images + - :py:class:`~ctapipe.containers.EventIndexContainer` +, :py:class:`~ctapipe.containers.ImageParametersContainer` + * - ``/simulation/service/shower_distribution`` + - simulated shower distribution histograms + - :py:class:`~ctapipe.containers.SimulatedShowerDistribution` + + +Configuration Data Model +======================== + +The output file should also contain serializations of the instrument, observation (if +applicable), simulation (if applicable) configuration information, written to the +``/configuration`` group: + +.. list-table:: + :widths: 25 50 25 + :header-rows: 1 + + * - Group/Dataset + - Description + - Contents + * - ``/instrument`` + - Serialized :py:class:`ctapipe.instrument.SubarrayDescription`, using :py:meth:`ctapipe.instrument.SubarrayDescription.to_hdf` + - (group) + * - ``/instrument/subarray/layout`` + - Subarray layout info + - result of ``SubarrayDescription.to_table()`` output as HDF5 using ``astropy.table`` functionality + * - ``/instrument/telescope/optics`` + - telescope optics information + - result of ``SubarrayDescription.to_table(kind='optics')`` output as HDF5 using + ``astropy.table`` functionality + * - ``/instrument/telescope/camera/{CAMERA_ID}`` + - camera geometry information + - result of ``CameraGeometry.to_table()`` output as HDF5 using ``astropy.table`` + functionality + * - ``/simulation`` + - Monte-Carlo simulation configuration information + - (group) + * - ``/simulation/run`` + - Monte-Carlo simulation run information + - :py:class:`~ctapipe.containers.SimulationConfigContainer` + + +Core Provenance +=============== + +The root group of the file shall contain all of the "CTA Core Provenance Metadata" +headers as user attributes, with the hierarchy flattened and separated by spaces +(e.g. ``"CTA ACTIVITY NAME" = "ctapipe-process"``). diff --git a/docs/user-guide/index.rst b/docs/user-guide/index.rst index 2d2411f11e6..0474279f9fb 100644 --- a/docs/user-guide/index.rst +++ b/docs/user-guide/index.rst @@ -8,7 +8,7 @@ User Guide :maxdepth: 2 getting-started - tools + tools/index data_models/index examples_tutorials FAQ diff --git a/docs/user-guide/tools.rst b/docs/user-guide/tools.rst deleted file mode 100644 index 0dfda42fdcb..00000000000 --- a/docs/user-guide/tools.rst +++ /dev/null @@ -1,36 +0,0 @@ -.. _command_line_tools: - -****************** -Command-line Tools -****************** - -You can get a list of all available command-line tools by typing - -.. code-block:: sh - - ctapipe-info --tools - - -Data Processing Tools -===================== - -* ``ctapipe-quickstart``: Create some default analysis configurations and a working directory. -* ``ctapipe-process``: Process event data in any supported format from R0/R1/DL0 to DL1 or DL2 HDF5 files. -* ``ctapipe-apply-models``: Tool to apply machine learning models in bulk (as opposed to event by event). -* ``ctapipe-calculate-pixel-statistics``: Tool to aggregate statistics and detect outliers from pixel-wise image data. -* ``ctapipe-train-disp-reconstructor`` : Train the ML models for the `ctapipe.reco.DispReconstructor` (monoscopic reconstruction) -* ``ctapipe-train-energy-regressor``: Train the ML models for the `ctapipe.reco.EnergyRegressor` (energy estimation) -* ``ctapipe-train-particle-classifier``: Train the ML models for the `ctapipe.reco.ParticleClassifier` (gamma-hadron separation) -* ``ctapipe-optimize-event-selection``: Calculate gamma/hadron and direction cuts (e.g. for IRF calculation). -* ``ctapipe-compute-irf``: Calculate an IRF with or without applying a direction cut and optionally benchmarks. - -File Management Tools: -====================== -* ``ctapipe-merge``: Merge multiple ctapipe HDF5 files into one -* ``ctapipe-fileinfo``: Display information about ctapipe HDF5 output files - -Other Tools -=========== - -* ``ctapipe-info``: print information about your ctapipe installation and its command-line tools. -* ``ctapipe-dump-instrument``: writes instrumental info from any supported event input file, and writes them out as FITS or ECSV files for external use. diff --git a/docs/user-guide/tools/index.rst b/docs/user-guide/tools/index.rst new file mode 100644 index 00000000000..49ff4f7e753 --- /dev/null +++ b/docs/user-guide/tools/index.rst @@ -0,0 +1,51 @@ +.. _command_line_tools: + +****************** +Command-line Tools +****************** + +You can get a list of all available command-line tools by typing + +.. code-block:: sh + + ctapipe-info --tools + + +Data Processing Tools +===================== + +* `ctapipe-quickstart `: create some default analysis configurations and a working directory +* `ctapipe-process `: Process event data in any supported format from R0/R1/DL0 to DL1 or DL2 HDF5 files. +* `ctapipe-apply-models `: Tool to apply machine learning models in bulk (as opposed to event by event). +* `ctapipe-calculate-pixel-statistics `: Tool to aggregate statistics + and detect outliers from pixel-wise image data. +* `ctapipe-train-disp-reconstructor ` : + Train the ML models for the `~ctapipe.reco.DispReconstructor` (monoscopic reconstruction) +* `ctapipe-train-energy-regressor `: + Train the ML models for the `~ctapipe.reco.EnergyRegressor` (energy estimation) +* `ctapipe-train-particle-classifier `: + Train the ML models for the `~ctapipe.reco.ParticleClassifier` (gamma-hadron separation) +* `ctapipe-optimize-event-selection `: + Calculate gamma/hadron and direction cuts (e.g. for IRF calculation). +* `ctapipe-compute-irf `: Calculate an IRF with or without applying a direction cut + and optionally benchmarks. + +File Management Tools: +====================== +* `ctapipe-merge `: Merge multiple ctapipe HDF5 files into one +* ``ctapipe-fileinfo``: Display information about ctapipe HDF5 output files + +Other Tools +=========== + +* ``ctapipe-info``: print information about your ctapipe installation and its command-line tools. +* `ctapipe-dump-instrument `: writes instrumental info from any supported event input file, and writes them out as FITS or ECSV files for external use. + +Examples +======== +The following pages contain examples on how to use the command-line tools. + +.. toctree:: + :maxdepth: 1 + + mono_dl2 diff --git a/docs/user-guide/tools/mono_dl2.rst b/docs/user-guide/tools/mono_dl2.rst new file mode 100644 index 00000000000..fcb8d7829e9 --- /dev/null +++ b/docs/user-guide/tools/mono_dl2.rst @@ -0,0 +1,182 @@ +.. _mono_dl2: + +*************************************************************** +How to do monoscopic DL2 reconstruction using ``ctapipe`` tools +*************************************************************** + +This guide explains how to use the machine learning capabilities of ``ctapipe`` +to process files containing image parameters (data level 1b/ DL1b) to data level 2 (DL2). + +.. NOTE:: + * This guide can also be used for a stereo analysis, if the disp algorithm should + be used for direction reconstruction (Compared to the geometric reconstruction, + `~ctapipe.reco.hillas_reconstructor.HillasReconstructor`, which is used by default). + * This guide assumes you have a directory containing gamma, proton, and electron files + already containing image parameters. + * The provided commands also assume you are trying to process files in a ``bash`` shell + environment. + +Setup +===== +As always, you can run + +.. code-block:: sh + + ctapipe-quickstart + +to get some example configuration files, which you can use as a starting point to create your +desired configuration. +To keep things organized, also define an output directory and make sure it exists, for example like this:: + + OUTPUT_DIR=build + mkdir -p $OUTPUT_DIR + +Merging +======= +Decide on which fraction of your files you want to use for training each ML model: + +* Training the energy regressor requires a gamma training file (called e.g. ``gamma_merged_train_en.dl2.h5``). +* Training the particle classifier requires another gamma and a proton training file + (called e.g. ``gamma_merged_train_clf.dl2.h5`` and ``proton_merged_train.dl2.h5``). +* Training the disp reconstructor requires a gamma training file, which can be the same file + as used for the particle classifier (e.g. ``gamma_merged_train_clf.dl2.h5``). + +The remaining gamma and proton files can be used for testing (or e.g. IRF calculation). +It does not make sense to use electrons for training, so all electron files can be used for testing. + +Then save these sets of files into a total of six lists and run something like the following command +for each of these lists:: + + ctapipe-merge $GAMMA_TRAIN_EN_FILES --output $OUTPUT_DIR/gamma_merged_train_en.dl2.h5 + +where ``GAMMA_TRAIN_EN_FILES`` is an environment variable containing the list of gamma files +you want to merge into a training set, for example generated using:: + + GAMMA_TRAIN_EN_FILES=$(echo $INPUT_DIR/gamma*[0-1]*.h5) + +which will merge all gamma files in ``INPUT_DIR`` with names that start with "0" or "1". +Alternatively you could save the files into a literal file (e.g. ``gamma_train_en_files.list``), +one file name per row, which you then use like this:: + + GAMMA_TRAIN_EN_FILES=$(cat gamma_train_en_files.list) + ctapipe-merge $GAMMA_TRAIN_EN_FILES --output $OUTPUT_DIR/gamma_merged_train_en.dl2.h5 + +Using some method of specifying files, merge your gamma, proton, and electron files so that you end up with six merged files: + +* Gamma train energy, the file containing the gamma events to be used for training the energy regressor. +* Gamma train classifier, the file containing the gamma events to be used for training the particle classifier + and the disp reconstructor. +* Gamma test, the file with gamma events used for "testing" the performance of the analysis. +* Proton train, the file containing the proton events to be used for training the particle classifier. +* Proton test, the file with proton events used for "testing" the performance of the analysis. +* Electron test, the file with electron events used for "testing" the performance of the analysis. + +Training the machine learning models +==================================== +The training process has the following steps: + +1. Train an energy model on the gamma train energy file. +2. Apply the energy model to the gamma train classifier file and the proton train file. +3. Train a particle classifier on the gamma train classifier file and the proton train file. +4. Train a disp reconstructor on the gamma train classifier file. + +First define the following environment variables: + +* ``REG_CONF_FILE``, a configuration file for the energy regression training for example ``train_energy_regressor.yaml`` +* ``CLF_CONF_FILE``, a configuration file for the particle classification training for example ``train_particle_classifier.yaml`` +* ``DISP_CONF_FILE``, a configuration file for the disp reconstructor training for example ``train_disp_reconstructor.yaml`` +* ``INPUT_GAMMA_EN_FILE``, the gamma train energy file created in the previous step +* ``INPUT_GAMMA_CLF_FILE``, the gamma train classifier file created in the previous step +* ``INPUT_PROTON_FILE``, the proton train file +* ``EVAL_GAMMA_FILE``, the gamma test file +* ``EVAL_PROTON_FILE``, the proton test file +* ``EVAL_ELECTRON_FILE``, the electron test file + +Then the training of the machine learning models is done using the following commands:: + + ctapipe-train-energy-regressor --input $INPUT_GAMMA_EN_FILE \ + --output $OUTPUT_DIR/energy_regressor.pkl \ + --config $REG_CONF_FILE \ + --cv-output $OUTPUT_DIR/cv_energy.h5 \ + --provenance-log $OUTPUT_DIR/train_energy.provenance.log \ + --log-file $OUTPUT_DIR/train_energy.log \ + --log-level INFO \ + --overwrite + + ctapipe-apply-models --input $INPUT_GAMMA_CLF_FILE \ + --output $OUTPUT_DIR/gamma_train_clf.dl2.h5 \ + --reconstructor $OUTPUT_DIR/energy_regressor.pkl \ + --provenance-log $OUTPUT_DIR/apply_gamma_train_clf.provenance.log \ + --log-file $OUTPUT_DIR/apply_gamma_train_clf.log \ + --log-level INFO \ + --overwrite + + ctapipe-apply-models --input $INPUT_PROTON_FILE \ + --output $OUTPUT_DIR/proton_train_clf.dl2.h5 \ + --reconstructor $OUTPUT_DIR/energy_regressor.pkl \ + --provenance-log $OUTPUT_DIR/apply_proton_train.provenance.log \ + --log-file $OUTPUT_DIR/apply_proton_train.log \ + --log-level INFO \ + --overwrite + + ctapipe-train-particle-classifier --signal $OUTPUT_DIR/gamma_train_clf.dl2.h5 \ + --background $OUTPUT_DIR/proton_train_clf.dl2.h5 \ + --output $OUTPUT_DIR/particle_classifier.pkl \ + --config $CLF_CONF_FILE \ + --cv-output $OUTPUT_DIR/cv_particle.h5 \ + --provenance-log $OUTPUT_DIR/train_particle.provenance.log \ + --log-file $OUTPUT_DIR/train_particle.log \ + --log-level INFO \ + --overwrite + + ctapipe-train-disp-reconstructor --input $OUTPUT_DIR/gamma_train_clf.dl2.h5 \ + --output $OUTPUT_DIR/disp_reconstructor.pkl \ + --config $DISP_CONF_FILE \ + --cv-output $OUTPUT_DIR/cv_disp.h5 \ + --provenance-log $OUTPUT_DIR/train_disp.provenance.log \ + --log-file $OUTPUT_DIR/train_disp.log \ + --log-level INFO \ + --overwrite + +which will produce three trained models saved as ``$OUTPUT_DIR/energy_regressor.pkl``, ``$OUTPUT_DIR/particle_classifier.pkl``, +and ``$OUTPUT_DIR/disp_reconstructor.pkl``. +The saved model for the disp reconstruction contains both, the regressor for estimating ``norm(disp)`` and the classifier +for determining ``sign(disp)``. + +Applying the machine learning models on the test files +====================================================== +Now we can apply these trained models on the test files, ``EVAL_GAMMA_FILE``, ``EVAL_PROTON_FILE``, and ``EVAL_ELECTRON_FILE``, +to produce the final DL2 files:: + + ctapipe-apply-models --input $EVAL_GAMMA_FILE \ + --output $OUTPUT_DIR/gamma_final.dl2.h5 \ + --reconstructor $OUTPUT_DIR/energy_regressor.pkl \ + --reconstructor $OUTPUT_DIR/particle_classifier.pkl \ + --reconstructor $OUTPUT_DIR/disp_reconstructor.pkl \ + --provenance-log $OUTPUT_DIR/apply_gamma_final.provenance.log \ + --log-file $OUTPUT_DIR/apply_gamma_final.log \ + --log-level INFO \ + --overwrite + + ctapipe-apply-models --input $EVAL_PROTON_FILE \ + --output $OUTPUT_DIR/proton_final.dl2.h5 \ + --reconstructor $OUTPUT_DIR/energy_regressor.pkl \ + --reconstructor $OUTPUT_DIR/particle_classifier.pkl \ + --reconstructor $OUTPUT_DIR/disp_reconstructor.pkl \ + --provenance-log $OUTPUT_DIR/apply_proton_final.provenance.log \ + --log-file $OUTPUT_DIR/apply_proton_final.log \ + --log-level INFO \ + --overwrite + + ctapipe-apply-models --input $EVAL_ELECTRON_FILE \ + --output $OUTPUT_DIR/electron_final.dl2.h5 \ + --reconstructor $OUTPUT_DIR/energy_regressor.pkl \ + --reconstructor $OUTPUT_DIR/particle_classifier.pkl \ + --reconstructor $OUTPUT_DIR/disp_reconstructor.pkl \ + --provenance-log $OUTPUT_DIR/apply_electron_final.provenance.log \ + --log-file $OUTPUT_DIR/apply_electron_final.log \ + --log-level INFO \ + --overwrite + +which will produce ``$OUTPUT_DIR/gamma_final.dl2.h5``, ``$OUTPUT_DIR/proton_final.dl2.h5``, +and ``$OUTPUT_DIR/electron_final.dl2.h5``. diff --git a/src/ctapipe/core/tool.py b/src/ctapipe/core/tool.py index 488dd1242b2..449a13c9208 100644 --- a/src/ctapipe/core/tool.py +++ b/src/ctapipe/core/tool.py @@ -236,12 +236,14 @@ def enter_context(self, context_manager): ``context_manager.__enter__`` and return its result. This will guarantee that ``context_manager.__exit__`` is called when - `~ctapipe.core.Tool.run` finishes, even when an error occurs. + `~ctapipe.core.tool.Tool.run` finishes, even when an error occurs. """ return self._exit_stack.enter_context(context_manager) def initialize(self, argv=None): - """handle config and any other low-level setup""" + """ + Handle config and any other low-level setup. + """ self.parse_command_line(argv) self.update_logging_config() @@ -381,7 +383,7 @@ def start(self): """ Main function of the tool. - This is automatically called after `Tool.initialize` when `Tool.run` is called. + This is automatically called after `~ctapipe.core.tool.Tool.initialize` when `~ctapipe.core.tool.Tool.run` is called. """ @abstractmethod @@ -389,14 +391,14 @@ def finish(self): """ Finish up. - This is called automatically after `Tool.start` when `Tool.run` is called. + This is called automatically after `~ctapipe.core.tool.Tool.start` when `~ctapipe.core.tool.Tool.run` is called. """ self.log.info("Goodbye") def run(self, argv=None, raises=False): """Run the tool. - This automatically calls `Tool.initialize`, `Tool.start` and `Tool.finish` + This automatically calls `~ctapipe.core.tool.Tool.initialize`, `~ctapipe.core.tool.Tool.start` and `~ctapipe.core.tool.Tool.finish` Parameters ---------- diff --git a/src/ctapipe/tools/calculate_pixel_stats.py b/src/ctapipe/tools/calculate_pixel_stats.py index 18aa07632eb..6297acd2e06 100644 --- a/src/ctapipe/tools/calculate_pixel_stats.py +++ b/src/ctapipe/tools/calculate_pixel_stats.py @@ -22,6 +22,8 @@ from ctapipe.io.tableloader import TableLoader from ctapipe.monitoring.calculator import PixelStatisticsCalculator +__all__ = ["PixelStatisticsCalculatorTool"] + class PixelStatisticsCalculatorTool(Tool): """ diff --git a/src/ctapipe/tools/dump_instrument.py b/src/ctapipe/tools/dump_instrument.py index a3ad471696b..0cd3077c9bb 100644 --- a/src/ctapipe/tools/dump_instrument.py +++ b/src/ctapipe/tools/dump_instrument.py @@ -4,6 +4,7 @@ CameraGeometry.from_table()). The name of the output files are automatically generated. """ + import os import pathlib @@ -11,6 +12,8 @@ from ctapipe.core.traits import Enum, Path, Unicode from ctapipe.io import EventSource +__all__ = ["DumpInstrumentTool"] + class DumpInstrumentTool(Tool): description = Unicode(__doc__) diff --git a/src/ctapipe/tools/merge.py b/src/ctapipe/tools/merge.py index 11285ad9caf..7a9401eb113 100644 --- a/src/ctapipe/tools/merge.py +++ b/src/ctapipe/tools/merge.py @@ -1,6 +1,7 @@ """ Merge multiple ctapipe HDF5 files into one """ + import sys from argparse import ArgumentParser from collections import Counter @@ -17,6 +18,8 @@ from ..io import HDF5Merger from ..io import metadata as meta +__all__ = ["MergeTool"] + class MergeTool(Tool): """ diff --git a/src/ctapipe/tools/train_disp_reconstructor.py b/src/ctapipe/tools/train_disp_reconstructor.py index 7e42886c83e..a125ff753cf 100644 --- a/src/ctapipe/tools/train_disp_reconstructor.py +++ b/src/ctapipe/tools/train_disp_reconstructor.py @@ -1,6 +1,7 @@ """ Tool for training the DispReconstructor """ + import astropy.units as u import numpy as np @@ -12,9 +13,7 @@ from .utils import read_training_events -__all__ = [ - "TrainDispReconstructor", -] +__all__ = ["TrainDispReconstructor"] class TrainDispReconstructor(Tool):