From 1634f94edbd64ecd3b05eb00341866ef8db1cacd Mon Sep 17 00:00:00 2001 From: EmmaRenauld Date: Mon, 4 Mar 2024 09:57:41 -0500 Subject: [PATCH 01/15] Add test for divide_volume --- .../dwiml_divide_volume_into_blocs.py | 21 +++++++++++++---- .../tests/test_divide_volume_into_blocs.py | 23 ++++++++++++++++++- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/scripts_python/dwiml_divide_volume_into_blocs.py b/scripts_python/dwiml_divide_volume_into_blocs.py index bbdb1bdc..26df5706 100644 --- a/scripts_python/dwiml_divide_volume_into_blocs.py +++ b/scripts_python/dwiml_divide_volume_into_blocs.py @@ -8,16 +8,20 @@ from scilpy.io.utils import assert_inputs_exist, assert_outputs_exist, \ add_overwrite_arg +from dwi_ml.data.hdf5.utils import format_nb_blocs_connectivity + def _build_arg_parser(): p = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawTextHelpFormatter) p.add_argument('in_image', metavar='IN_FILE', help='Input file name, in nifti format.') - - p.add_argument( - 'out_filename', - help='name of the output file, which will be saved as a text file.') + p.add_argument('out_filename', + help='name of the output file, which will be saved as a ' + 'text file.') + p.add_argument('nb_blocs', nargs='+', type=int, + help="Number of blocs. Either a single int, or a list of " + "3 values.") add_overwrite_arg(p) @@ -48,11 +52,18 @@ def main(): parser = _build_arg_parser() args = parser.parse_args() + # Checks assert_inputs_exist(parser, args.in_image) assert_outputs_exist(parser, args, required=args.out_filename) + args.nb_blocs = format_nb_blocs_connectivity(args.nb_blocs) + # Loading volume = nib.load(args.in_image) - final_volume = color_mri_connectivity_blocs([6, 6, 6], volume.shape) + + # Processing + final_volume = color_mri_connectivity_blocs(args.nb_blocs, volume.shape) + + # Saving img = nib.Nifti1Image(final_volume, volume.affine) nib.save(img, args.out_filename) diff --git a/scripts_python/tests/test_divide_volume_into_blocs.py b/scripts_python/tests/test_divide_volume_into_blocs.py index 95d0e9cc..b535afac 100644 --- a/scripts_python/tests/test_divide_volume_into_blocs.py +++ b/scripts_python/tests/test_divide_volume_into_blocs.py @@ -1,8 +1,29 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +import os +import tempfile + +from dwi_ml.unit_tests.utils.data_and_models_for_tests import \ + fetch_testing_data + +data_dir = fetch_testing_data() +tmp_dir = tempfile.TemporaryDirectory() def test_help_option(script_runner): + ret = script_runner.run('dwiml_divide_volume_into_blocs.py', '--help') + assert ret.success + + +def test_run(script_runner): + os.chdir(os.path.expanduser(tmp_dir.name)) + + dwi_ml_folder = os.path.join(data_dir, 'dwi_ml_ready', 'subjX') + in_volume = os.path.join(dwi_ml_folder, 'anat', 't1.nii.gz') + + out_file = 'volume_blocs.nii.gz' + + nb_blocs = '4' ret = script_runner.run('dwiml_divide_volume_into_blocs.py', - '--help') + in_volume, out_file, nb_blocs) assert ret.success From 5d180571db2f922210fa2faeea7edb254585b19e Mon Sep 17 00:00:00 2001 From: EmmaRenauld Date: Mon, 4 Mar 2024 10:16:48 -0500 Subject: [PATCH 02/15] Use master scilpy (again...) --- requirements.txt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 8dc68002..9db0f4e0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,10 +5,13 @@ # Main dependency: scilpy # Scilpy and comet_ml both require requests. In comet: >=2.18.*, # which installs a version >2.28. Adding request version explicitely. +# +# Changed many times the scilpy version. Tried to have a fixed version, but +# currently, working on beluga only when installing manually. Reverting to +# master branch for now, easier to manage for me (Emmanuelle) # ------- requests==2.28.* -scilpy @ git+https://github.com/scilus/scilpy.git@1.7.0-dev - +git+https://github.com/scilus/scilpy.git#egg=scilpy # ------- # Other important dependencies From 7bbb430e4d6db1a28e9ca44938d4ee8c0642ed71 Mon Sep 17 00:00:00 2001 From: EmmaRenauld Date: Mon, 4 Mar 2024 10:19:26 -0500 Subject: [PATCH 03/15] Add test for connectivity_score --- .../dwiml_compute_connectivity_score.py | 10 +++++-- .../tests/test_compute_connectivity_score.py | 28 ++++++++++++++++++- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/scripts_python/dwiml_compute_connectivity_score.py b/scripts_python/dwiml_compute_connectivity_score.py index 32ac092f..965883eb 100644 --- a/scripts_python/dwiml_compute_connectivity_score.py +++ b/scripts_python/dwiml_compute_connectivity_score.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- import argparse +import logging import matplotlib.pyplot as plt import numpy as np @@ -15,7 +16,8 @@ def _build_arg_parser(): help="Matrix file (reference). .npy") p.add_argument('scored_matrix', help="Matrix being scored. .npy") - + p.add_argument('--show_now', action='store_true', + help="If set, shows the matrix with matplotlib.") add_overwrite_arg(p) return p @@ -89,7 +91,11 @@ def main(): axs[2].imshow((ref.astype(int) - m.astype(int))**2) plt.suptitle("Binary") - plt.show() + if args.show_now: + plt.show() + else: + logging.warning("Saving of figure not implemented yet! Use --show_now " + "to see the figure.") if __name__ == '__main__': diff --git a/scripts_python/tests/test_compute_connectivity_score.py b/scripts_python/tests/test_compute_connectivity_score.py index f9e6bf08..ea56bc95 100644 --- a/scripts_python/tests/test_compute_connectivity_score.py +++ b/scripts_python/tests/test_compute_connectivity_score.py @@ -1,8 +1,34 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +import os +import tempfile + +from dwi_ml.unit_tests.utils.data_and_models_for_tests import \ + fetch_testing_data + +data_dir = fetch_testing_data() +tmp_dir = tempfile.TemporaryDirectory() def test_help_option(script_runner): + ret = script_runner.run('dwiml_compute_connectivity_score.py', '--help') + assert ret.success + + +def test_execution(script_runner): + os.chdir(os.path.expanduser(tmp_dir.name)) + dwi_ml_folder = os.path.join(data_dir, 'dwi_ml_ready', 'subjX') + + # Currently no matrix in our tests. Creating one. + in_volume = os.path.join(dwi_ml_folder, 'anat', 't1.nii.gz') + streamlines = os.path.join(dwi_ml_folder, 'example_bundle', 'Fornix.trk') + matrix = 'matrix_connectivity.npy' + + nb_blocs = '4' + script_runner.run('dwiml_compute_connectivity_matrix_from_blocs.py', + in_volume, streamlines, matrix, nb_blocs, '--binary') + + # Now scoring ret = script_runner.run('dwiml_compute_connectivity_score.py', - '--help') + matrix, matrix) assert ret.success From 134c0e90bb1f0b48f52911af4ac1f2a6eabed8a3 Mon Sep 17 00:00:00 2001 From: EmmaRenauld Date: Mon, 4 Mar 2024 10:20:04 -0500 Subject: [PATCH 04/15] No test for connectivity_from_labels. --- .../tests/test_compute_connectivity_matrix_from_labels.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts_python/tests/test_compute_connectivity_matrix_from_labels.py b/scripts_python/tests/test_compute_connectivity_matrix_from_labels.py index faed1242..cb127591 100644 --- a/scripts_python/tests/test_compute_connectivity_matrix_from_labels.py +++ b/scripts_python/tests/test_compute_connectivity_matrix_from_labels.py @@ -5,3 +5,8 @@ def test_help_option(script_runner): ret = script_runner.run('dwiml_compute_connectivity_matrix_from_labels.py', '--help') assert ret.success + + +def test_execution(script_runner): + # Impossible for now, no labels file. Could use data from scilpy's tests. + pass From 4c86ddd51c54fa2b74d2a2fca87c4d032cbaec17 Mon Sep 17 00:00:00 2001 From: EmmaRenauld Date: Mon, 4 Mar 2024 10:22:44 -0500 Subject: [PATCH 05/15] Add test for print_hdf5 --- .../tests/test_print_hdf5_architecture.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 scripts_python/tests/test_print_hdf5_architecture.py diff --git a/scripts_python/tests/test_print_hdf5_architecture.py b/scripts_python/tests/test_print_hdf5_architecture.py new file mode 100644 index 00000000..9d0cc968 --- /dev/null +++ b/scripts_python/tests/test_print_hdf5_architecture.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import os + +from dwi_ml.unit_tests.utils.data_and_models_for_tests import fetch_testing_data + +data_dir = fetch_testing_data() +experiment_name = 'test_experiment' + + +def test_help_option(script_runner): + ret = script_runner.run('dwiml_print_hdf5_architecture.py', '--help') + assert ret.success + + +def test_execution(script_runner): + hdf5_file = os.path.join(data_dir, 'hdf5_file.hdf5') + ret = script_runner.run('dwiml_print_hdf5_architecture.py', hdf5_file) + assert ret.success From 73a41604d739feb1b140cfabf69643f203f4b675 Mon Sep 17 00:00:00 2001 From: EmmaRenauld Date: Mon, 4 Mar 2024 10:30:59 -0500 Subject: [PATCH 06/15] Git requirement not working without the -e --- requirements.txt | 2 +- .../tests/test_send_value_to_comet_manually.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 scripts_python/tests/test_send_value_to_comet_manually.py diff --git a/requirements.txt b/requirements.txt index 9db0f4e0..23ac4207 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ # master branch for now, easier to manage for me (Emmanuelle) # ------- requests==2.28.* -git+https://github.com/scilus/scilpy.git#egg=scilpy +-e git+https://github.com/scilus/scilpy.git#egg=scilpy # ------- # Other important dependencies diff --git a/scripts_python/tests/test_send_value_to_comet_manually.py b/scripts_python/tests/test_send_value_to_comet_manually.py new file mode 100644 index 00000000..cb127591 --- /dev/null +++ b/scripts_python/tests/test_send_value_to_comet_manually.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +def test_help_option(script_runner): + ret = script_runner.run('dwiml_compute_connectivity_matrix_from_labels.py', + '--help') + assert ret.success + + +def test_execution(script_runner): + # Impossible for now, no labels file. Could use data from scilpy's tests. + pass From 2941c2108447a0248c07ba1acdce617feb57ad86 Mon Sep 17 00:00:00 2001 From: EmmaRenauld Date: Mon, 4 Mar 2024 10:33:23 -0500 Subject: [PATCH 07/15] Add test send_value_to_comet --- ...nually_comet.py => dwiml_send_value_to_comet_manually.py} | 0 scripts_python/tests/test_send_value_to_comet_manually.py | 5 ++--- 2 files changed, 2 insertions(+), 3 deletions(-) rename scripts_python/{dwiml_send_value_manually_comet.py => dwiml_send_value_to_comet_manually.py} (100%) diff --git a/scripts_python/dwiml_send_value_manually_comet.py b/scripts_python/dwiml_send_value_to_comet_manually.py similarity index 100% rename from scripts_python/dwiml_send_value_manually_comet.py rename to scripts_python/dwiml_send_value_to_comet_manually.py diff --git a/scripts_python/tests/test_send_value_to_comet_manually.py b/scripts_python/tests/test_send_value_to_comet_manually.py index cb127591..266c2084 100644 --- a/scripts_python/tests/test_send_value_to_comet_manually.py +++ b/scripts_python/tests/test_send_value_to_comet_manually.py @@ -2,11 +2,10 @@ # -*- coding: utf-8 -*- def test_help_option(script_runner): - ret = script_runner.run('dwiml_compute_connectivity_matrix_from_labels.py', - '--help') + ret = script_runner.run('dwiml_send_value_to_comet_manually.py', '--help') assert ret.success def test_execution(script_runner): - # Impossible for now, no labels file. Could use data from scilpy's tests. + # Impossible pass From b1e6003d2970ccfacdefb469f9da4eb72d590861 Mon Sep 17 00:00:00 2001 From: EmmaRenauld Date: Mon, 4 Mar 2024 11:06:07 -0500 Subject: [PATCH 08/15] Update comet API doc --- source/getting_started.rst | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/source/getting_started.rst b/source/getting_started.rst index 3943bec6..3e889778 100644 --- a/source/getting_started.rst +++ b/source/getting_started.rst @@ -4,8 +4,7 @@ Getting started: download and installation Downloading dwi_ml ****************** -To use the DWI_ML toolkit you will need to clone the repository and install the -required dependencies:: +To use the DWI_ML toolkit you will need to clone the repository and install the required dependencies:: git clone https://github.com/scil-vital/dwi_ml.git @@ -24,23 +23,19 @@ We strongly recommend working in a virtual environment to install all dependenci pip install -r requirements.txt -- The toolkit heavily relies on deep learning methods. As such, a GPU device will be instantiated whenever one is available. DWI_ML uses PyTorch as its deep learning back end. Thus, in order to use DWI_ML deep learning methods - you will need to take a few additional steps. +- The toolkit heavily relies on deep learning methods. As such, a GPU device will be instantiated whenever one is available. DWI_ML uses PyTorch as its deep learning back end. Thus, in order to use DWI_ML deep learning methods you will need to take a few additional steps. 1. **Cuda**: - - Verify that your computer has the required capabilities in the *Pre-installation Actions* section at `cuda/cuda-installation-guide `_ - (sections 2.1 - 2.4). To find your OS version and the available GPU, check the *About* menu in your computer settings. + - Verify that your computer has the required capabilities in the *Pre-installation Actions* section at `cuda/cuda-installation-guide `_ (sections 2.1 - 2.4). To find your OS version and the available GPU, check the *About* menu in your computer settings. - - Follow the download instructions at `nvidia.com/cuda-downloads `_. - Choose the environment that fits your system in the selector. You can choose *deb(local)* for the installer type. + - Follow the download instructions at `nvidia.com/cuda-downloads `_. Choose the environment that fits your system in the selector. You can choose *deb(local)* for the installer type. - Follow the installation instructions. 2. **PyTorch**: - - Install `PyTorch`_. Use the selector under the *Start locally* section at `pytorch.org/get-started `_ to have the specific command line instructions to install PyTorch with CUDA - capabilities on your system. + - Install `PyTorch`_. Use the selector under the *Start locally* section at `pytorch.org/get-started `_ to have the specific command line instructions to install PyTorch with CUDA capabilities on your system. - Perform the suggested verifications to make sure that both `CUDA`_ and `PyTorch`_ have been correctly installed. @@ -52,8 +47,11 @@ Creating a Comet account | [comet] | api_key=YOUR-API-KEY - An API (application programming interface) is a code that gets passed in by applications, containing information to identify its user, for instance. To get an API key, see ``_. Click on the key icon, - copy value to the clipboard and save it in your file in $HOME. +Alternatively, you can add it as an environment variable. Add this to your $HOME/.bashrc file. + + | export COMET_API_KEY=YOUR-API-KEY + + An API (application programming interface) is a code that gets passed in by applications, containing information to identify its user, for instance. To get an API key, see ``_. Click on the key icon, copy value to the clipboard and save it in your file in $HOME. Installing dwi_ml From 4a6a84466e775a8aaa6b83f5ed57be71476abf92 Mon Sep 17 00:00:00 2001 From: EmmaRenauld Date: Mon, 4 Mar 2024 13:16:29 -0500 Subject: [PATCH 09/15] Add script to send entire log to comet --- .../dwiml_send_value_to_comet_from_log.py | 144 ++++++++++++++++++ .../tests/test_send_valueto_comet_from_log.py | 11 ++ 2 files changed, 155 insertions(+) create mode 100644 scripts_python/dwiml_send_value_to_comet_from_log.py create mode 100644 scripts_python/tests/test_send_valueto_comet_from_log.py diff --git a/scripts_python/dwiml_send_value_to_comet_from_log.py b/scripts_python/dwiml_send_value_to_comet_from_log.py new file mode 100644 index 00000000..156bbc92 --- /dev/null +++ b/scripts_python/dwiml_send_value_to_comet_from_log.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""Sends all values to comet from a log (expecting the log to be one value per +epoch). Note that the comet experiment is found by reading the experiment's +checkpoing (must exist). + +USE WITH CARE. This cannot be undone. +""" +import argparse +import glob +import logging +import os +from time import sleep + +import numpy as np +from comet_ml import ExistingExperiment +from scilpy.io.utils import assert_inputs_exist + +from dwi_ml.training.trainers import DWIMLAbstractTrainer + + +def _build_arg_parser(): + p = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawTextHelpFormatter) + p.add_argument('experiments_path', + help="Path to the experiments folder.") + p.add_argument('experiment_name', + help="Name of the experiment.") + p.add_argument('--logs', nargs='+', + help="Log file (s). Expected to be located inside the " + "experiments_path. If not set, uses all logs.") + p.add_argument('--metric_names', nargs='+', + help="Comet.ml's metric name(s). Must contain the same " + "number of inputs as --logs.\n" + "If not set, we will suggest you the probable name(s) " + "but we will not run the script.") + p.add_argument('--use_suggested_name', action='store_true', + help="If set and --metric_name is not set, will run with " + "the suggested name(s).") + p.add_argument('--use_best', action='store_true', + help="If set, uses only the best value in segment [0, t] " + "as value at time t. (Best = lowest).") + + return p + + +def main(): + parser = _build_arg_parser() + args = parser.parse_args() + + # Verifications + experiment_path = os.path.join(args.experiments_path, args.experiment_name) + if not os.path.isdir(experiment_path): + parser.error('Experiment does not exist: {}'.format(experiment_path)) + + # Prepare logs, with possible wildcards. + log_path = os.path.join(str(experiment_path), 'logs') + if args.logs is None: + log_paths = glob.glob(os.path.join(log_path, '*')) + else: + log_paths = [os.path.join(log_path, file) for file in args.logs] + + assert_inputs_exist(parser, log_paths) + + # Prepare associated metric names (if not given) + if args.metric_names is None: + args.metric_names = [] + if args.use_best: + print("Not sure what to suggest you for --metric_name with " + "option --use_best. Probably 'best_loss'!?") + args.use_suggested_name = False + else: + for log in log_paths: + # Based on current implementation of things: + # - comet adds train_ or validate_ as prefix + # - we add _per_epoch.npy as suffix to the log name. + base_name = os.path.basename(log) + metric_name, _ = os.path.splitext(base_name) + + # Add comet prefix + if 'tracking' in metric_name or 'valid' in metric_name: + metric_name = 'validate_' + metric_name + elif 'training' in metric_name: + metric_name = 'train_' + metric_name + + # Remove our suffix + if metric_name[-10:] == '_per_epoch': + metric_name = metric_name[:-10] + + print("Suggested --metric_name for log {}: {}" + .format(base_name, metric_name)) + args.metric_names.append(metric_name) + + # Possibly stop now + if not args.use_suggested_name: + return + + # Verify + if not len(args.metric_names) == len(log_paths): + parser.error("Expecting the same number of metrics_names (got {}) " + "than logs (got {})." + .format(len(args.metric_names), len(log_paths))) + + # Loading comet from info in checkpoint + checkpoint_state = DWIMLAbstractTrainer.load_params_from_checkpoint( + args.experiments_path, args.experiment_name) + current_states = checkpoint_state['current_states'] + comet_key = current_states['comet_key'] + + print("Found comet experiment: key {}. Loading.".format(comet_key)) + comet_exp = ExistingExperiment(previous_experiment=comet_key) + + for log, metric_name in zip(log_paths, args.metric_names): + # Loading the log + data = np.load(log) + + # Note. There is comet_exp.metrics with the metrics previously logged. + # But seems to only contain metrics logged in current session. + # Now, metrics={}. + print("Will send values for all {} epochs to metric {}" + .format(len(data), metric_name)) + best_value = np.inf + for t in range(len(data)): + value = data[t] + if args.use_best: + best_value = min(value, best_value) + value = best_value + + # Send value + comet_exp.log_metric(metric_name, value, step=t) + + # Rate limits: + # https://www.comet.com/docs/v2/api-and-sdk/python-sdk/warnings-errors/ + # 10,000 per minute = 167 per second. + # If we wait 0.01 second, max 100 call per second, will help. + # Note. Sleep on Linux seems to allow 1ms sleep. On windows, could + # fail. https://python-forum.io/thread-17019.html + sleep(0.01) + + print("Done!\n\n") + + +if __name__ == '__main__': + main() diff --git a/scripts_python/tests/test_send_valueto_comet_from_log.py b/scripts_python/tests/test_send_valueto_comet_from_log.py new file mode 100644 index 00000000..dddc82b2 --- /dev/null +++ b/scripts_python/tests/test_send_valueto_comet_from_log.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +def test_help_option(script_runner): + ret = script_runner.run('dwiml_send_value_to_comet_from_log.py', '--help') + assert ret.success + + +def test_execution(script_runner): + # Impossible + pass From 5be66a4aa9ad782914ec1531035db416816f55e2 Mon Sep 17 00:00:00 2001 From: EmmaRenauld Date: Mon, 4 Mar 2024 13:46:04 -0500 Subject: [PATCH 10/15] Changes from master scilpy update --- scripts_python/dwiml_compute_connectivity_matrix_from_blocs.py | 3 ++- .../dwiml_compute_connectivity_matrix_from_labels.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts_python/dwiml_compute_connectivity_matrix_from_blocs.py b/scripts_python/dwiml_compute_connectivity_matrix_from_blocs.py index 2f76885a..402dc4db 100644 --- a/scripts_python/dwiml_compute_connectivity_matrix_from_blocs.py +++ b/scripts_python/dwiml_compute_connectivity_matrix_from_blocs.py @@ -10,8 +10,9 @@ from dipy.io.streamline import save_tractogram from dipy.io.utils import is_header_compatible +from scilpy.io.streamlines import load_tractogram_with_reference from scilpy.io.utils import assert_inputs_exist, assert_outputs_exist, \ - load_tractogram_with_reference, add_verbose_arg, add_overwrite_arg + add_verbose_arg, add_overwrite_arg from dwi_ml.data.hdf5.utils import format_nb_blocs_connectivity from dwi_ml.data.processing.streamlines.post_processing import \ diff --git a/scripts_python/dwiml_compute_connectivity_matrix_from_labels.py b/scripts_python/dwiml_compute_connectivity_matrix_from_labels.py index 877653fd..4f390bd6 100644 --- a/scripts_python/dwiml_compute_connectivity_matrix_from_labels.py +++ b/scripts_python/dwiml_compute_connectivity_matrix_from_labels.py @@ -17,8 +17,9 @@ from dipy.io.utils import is_header_compatible from scilpy.image.labels import get_data_as_labels +from scilpy.io.streamlines import load_tractogram_with_reference from scilpy.io.utils import assert_inputs_exist, assert_outputs_exist, \ - load_tractogram_with_reference, add_verbose_arg, add_overwrite_arg + add_verbose_arg, add_overwrite_arg from dwi_ml.data.processing.streamlines.post_processing import \ find_streamlines_with_chosen_connectivity, \ From 052168eea5785015a825be91f350fe60fbe388ca Mon Sep 17 00:00:00 2001 From: EmmaRenauld Date: Mon, 4 Mar 2024 13:46:24 -0500 Subject: [PATCH 11/15] Updating comet version in requirements. Today=3.38. Tests pass" --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 23ac4207..79d3e205 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ requests==2.28.* bertviz~=1.4.0 # For transformer's visu torch==1.13.* tqdm==4.64.* -comet-ml==3.21.* +comet-ml>=3.22.0 contextlib2==21.6.0 jupyterlab>=3.6.2 # For transformer's visu IProgress>=0.4 # For jupyter with tdqm From d81bcf1fa83c4f887adade3e265d411776e860fe Mon Sep 17 00:00:00 2001 From: EmmaRenauld Date: Tue, 12 Mar 2024 14:26:43 -0400 Subject: [PATCH 12/15] Fix filename --- ...eto_comet_from_log.py => test_send_value_to_comet_from_log.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts_python/tests/{test_send_valueto_comet_from_log.py => test_send_value_to_comet_from_log.py} (100%) diff --git a/scripts_python/tests/test_send_valueto_comet_from_log.py b/scripts_python/tests/test_send_value_to_comet_from_log.py similarity index 100% rename from scripts_python/tests/test_send_valueto_comet_from_log.py rename to scripts_python/tests/test_send_value_to_comet_from_log.py From 99032be3ca4fa13931f48f199d8f7d01cd3faf60 Mon Sep 17 00:00:00 2001 From: EmmaRenauld Date: Tue, 12 Mar 2024 14:42:28 -0400 Subject: [PATCH 13/15] Used fixed commit rather than scilpy master --- requirements.txt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 79d3e205..535296e5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,12 +6,13 @@ # Scilpy and comet_ml both require requests. In comet: >=2.18.*, # which installs a version >2.28. Adding request version explicitely. # -# Changed many times the scilpy version. Tried to have a fixed version, but -# currently, working on beluga only when installing manually. Reverting to -# master branch for now, easier to manage for me (Emmanuelle) +# Changed many times the scilpy version. Tried to have latest fixed version, +# but currently, working on beluga only when installing manually. But using +# the master is not good; changes too fast for us. Using a fixed commit for +# now. # ------- requests==2.28.* --e git+https://github.com/scilus/scilpy.git#egg=scilpy +-e git+https://github.com/scilus/scilpy.git@44912280e301e00a33278a7dc19c810590e32f63#egg=scilpy # ------- # Other important dependencies From 97509632b15ffa84a4e45f0d15a89497f4ebf673 Mon Sep 17 00:00:00 2001 From: EmmaRenauld Date: Tue, 12 Mar 2024 14:58:25 -0400 Subject: [PATCH 14/15] pynvml suddenly missing. installing explicitely --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 535296e5..3fc07596 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,6 +25,7 @@ contextlib2==21.6.0 jupyterlab>=3.6.2 # For transformer's visu IProgress>=0.4 # For jupyter with tdqm nested_lookup==0.2.25 # For lists management +pynvml>=11.5.0 # ------- # Necessary but should be installed with scilpy (Last check: 01/2024): From 1a1e8eb78e8814bb6d27f004858242ac815f39ec Mon Sep 17 00:00:00 2001 From: EmmaRenauld Date: Tue, 12 Mar 2024 15:28:47 -0400 Subject: [PATCH 15/15] Using a commit before scilpy's fetcher was changed --- requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3fc07596..be6fdebd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,9 +10,11 @@ # but currently, working on beluga only when installing manually. But using # the master is not good; changes too fast for us. Using a fixed commit for # now. +# To using a commit preceding all recent changes in scilpy's test data +# management: d20d3d4917d40f698aa975f64a575dda34e0c89c # ------- requests==2.28.* --e git+https://github.com/scilus/scilpy.git@44912280e301e00a33278a7dc19c810590e32f63#egg=scilpy +-e git+https://github.com/scilus/scilpy.git@d20d3d4917d40f698aa975f64a575dda34e0c89c#egg=scilpy # ------- # Other important dependencies