From 6076416e2a6ec9de39f558fd3cd246220644d5d9 Mon Sep 17 00:00:00 2001 From: danibene <34680344+danibene@users.noreply.github.com> Date: Fri, 15 Mar 2024 11:50:34 -0400 Subject: [PATCH 01/19] add readthedocs badge --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 28906fc..3a12f68 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +[![Documentation Status](https://readthedocs.org/projects/equiadapt/badge/?version=latest)](https://equiadapt.readthedocs.io/en/latest/?badge=latest) +


From 2f8394a274dd7c8c95ea198c38ec5a67792aca4c Mon Sep 17 00:00:00 2001 From: danibene <34680344+danibene@users.noreply.github.com> Date: Fri, 15 Mar 2024 12:05:11 -0400 Subject: [PATCH 02/19] add docs and pypi badges --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3a12f68..2095ef0 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Documentation Status](https://readthedocs.org/projects/equiadapt/badge/?version=latest)](https://equiadapt.readthedocs.io/en/latest/?badge=latest) +[![Documentation Status](https://readthedocs.org/projects/equiadapt/badge/?version=latest)](https://equiadapt.readthedocs.io/en/latest/?badge=latest) ![PyPI](https://img.shields.io/pypi/v/PACKAGE?label=pypi%20package)


From 92bf196e55b8a3110004375415665840e25375df Mon Sep 17 00:00:00 2001 From: danibene <34680344+danibene@users.noreply.github.com> Date: Fri, 15 Mar 2024 12:07:52 -0400 Subject: [PATCH 03/19] have badge link to pypi --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2095ef0..f898e4d 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Documentation Status](https://readthedocs.org/projects/equiadapt/badge/?version=latest)](https://equiadapt.readthedocs.io/en/latest/?badge=latest) ![PyPI](https://img.shields.io/pypi/v/PACKAGE?label=pypi%20package) +[![Documentation Status](https://readthedocs.org/projects/equiadapt/badge/?version=latest)](https://equiadapt.readthedocs.io/en/latest/?badge=latest) [![PyPI](https://img.shields.io/pypi/v/PACKAGE?label=pypi%20package)](https://pypi.org/project/equiadapt/)


From e22fcc087b4bfaec06027e59835a4d3c7b44e5d9 Mon Sep 17 00:00:00 2001 From: danibene <34680344+danibene@users.noreply.github.com> Date: Fri, 15 Mar 2024 12:12:18 -0400 Subject: [PATCH 04/19] try testing with python 3.11 and all OSs --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b95f5ab..c107a13 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,10 +62,11 @@ jobs: python: - "3.7" # oldest Python supported by PSF - "3.10" # newest Python that is stable + - "3.11" platform: - ubuntu-latest - # - macos-latest - # - windows-latest + - macos-latest + - windows-latest runs-on: ${{ matrix.platform }} steps: - uses: actions/checkout@v3 From 77b2c26015cc3b36f996a90203496df51d5289d2 Mon Sep 17 00:00:00 2001 From: danibene <34680344+danibene@users.noreply.github.com> Date: Fri, 15 Mar 2024 12:22:35 -0400 Subject: [PATCH 05/19] only test all OSs --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c107a13..351ae15 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,6 @@ jobs: python: - "3.7" # oldest Python supported by PSF - "3.10" # newest Python that is stable - - "3.11" platform: - ubuntu-latest - macos-latest From 3dfaad5c9b4f39bb874122af5531e6e4587f25fc Mon Sep 17 00:00:00 2001 From: danibene <34680344+danibene@users.noreply.github.com> Date: Fri, 15 Mar 2024 14:05:24 -0400 Subject: [PATCH 06/19] update version in python requires + tests --- .github/workflows/ci.yml | 1 + setup.cfg | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 351ae15..c107a13 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,6 +62,7 @@ jobs: python: - "3.7" # oldest Python supported by PSF - "3.10" # newest Python that is stable + - "3.11" platform: - ubuntu-latest - macos-latest diff --git a/setup.cfg b/setup.cfg index 1c2b0cd..5bbf872 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,7 +35,7 @@ packages = find_namespace: include_package_data = True # Require a min/specific Python version (comma-separated conditions) -python_requires = >=3.7, <3.11 +python_requires = >=3.7, <3.12 # Add here dependencies of your project (line-separated), e.g. requests>=2.2,<3.0. # Version specifiers like >=2.2,<3.0 avoid problems due to API changes in From 792bc163a0264090697c58014d91c48bead5f62d Mon Sep 17 00:00:00 2001 From: Siba Smarak Panigrahi Date: Mon, 18 Mar 2024 13:33:23 +0530 Subject: [PATCH 07/19] Update README.md --- README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index f898e4d..324cd7a 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@
# About -EquiAdapt is a [PyTorch](https://pytorch.org) package that provides a flexible and efficient way to make *any* neural network architecture (including large foundation models) equivariant, instead of redesigning and training from scratch. This is done by learning to canonicalize transformed inputs, before feeding them to the prediction model. You can play with this concept in the provided [tutorial](tutorials/images/instance_segmentation_group_equivariant_canonicalization.ipynb) for equivariant adaptation of the Segment-Anything Model (SAM, [Kirillov et. al, 2023](https://arxiv.org/abs/2304.02643)) and images from Microsoft COCO ([Lin et. al, 2014](https://arxiv.org/abs/1405.0312)) dataset for instance segmentation. +EquiAdapt is a [PyTorch](https://pytorch.org) package that provides a flexible and efficient way to make *any* neural network architecture (including large foundation models) equivariant, instead of redesigning and training from scratch. This is done by learning to canonicalize transformed inputs, before feeding them to the prediction model. You can play with this concept in the provided [tutorial](https://github.com/arnab39/equiadapt/blob/main/tutorials/images/instance_segmentation_group_equivariant_canonicalization.ipynb) for equivariant adaptation of the Segment-Anything Model (SAM, [Kirillov et. al, 2023](https://arxiv.org/abs/2304.02643)) and images from Microsoft COCO ([Lin et. al, 2014](https://arxiv.org/abs/1405.0312)) dataset for instance segmentation. To learn more about this from a blog, check out: [How to make your foundation model equivariant](https://mila.quebec/en/article/how-to-make-your-foundation-model-equivariant/) @@ -77,7 +77,7 @@ canonicalization_network = ESCNNEquivariantNetwork(...) ``` 2. Wrap it using `equiadapt` wrappers to form a `canonicalizer`. - To create your custom canonicalizer, you must inherit `BaseCanonicalization` and define `canonicalize()` and, optionally, `invert_canonicalization()`. Please refer to [this](equiadapt/images/canonicalization) for custom image canonicalizers. + To create your custom canonicalizer, you must inherit `BaseCanonicalization` and define `canonicalize()` and, optionally, `invert_canonicalization()`. Please refer to [this](https://github.com/arnab39/equiadapt/tree/main/equiadapt/images/canonicalization) for custom image canonicalizers. ``` canonicalizer = GroupEquivariantImageCanonicalization(canonicalization_network, ...) ``` @@ -139,15 +139,15 @@ Note that this might not be a complete list of dependencies. If you encounter an # Running equiadapt using example code -We provide [examples](examples) to run equiadapt in different data domains and tasks to achieve equivariance. +We provide [examples](https://github.com/arnab39/equiadapt/tree/main/examples) to run equiadapt in different data domains and tasks to achieve equivariance. - Image: - - Classification: [Link](examples/images/classification/README.md) - - Segmentation: [Link](examples/images/segmentation/README.md) + - Classification: [Link](https://github.com/arnab39/equiadapt/tree/main/examples/images/classification/README.md) + - Segmentation: [Link](https://github.com/arnab39/equiadapt/tree/main/examples/images/segmentation/README.md) - Point Cloud: - - Classification: [Link](examples/pointcloud/classification/README.md) - - Part Segmentation: [Link](examples/pointcloud/part_segmentation/README.md) -- Nbody Dynamics: [Link](examples/nbody/README.md) + - Classification: [Link](https://github.com/arnab39/equiadapt/tree/main/examples/pointcloud/classification/README.md) + - Part Segmentation: [Link](https://github.com/arnab39/equiadapt/tree/main/examples/pointcloud/part_segmentation/README.md) +- Nbody Dynamics: [Link](https://github.com/arnab39/equiadapt/tree/main/examples/nbody/README.md) Our examples use `hydra` to configure hyperparameters. Follow the hydra setup instructions to create a .env file with the paths to store all the data, wandb logs and checkpoints. @@ -160,7 +160,7 @@ Create a `.env` file in the root of the project with the following content: export CHECKPOINT_PATH="/path/to/your/checkpoint/directory" ``` -You can also find [tutorials](tutorials) on how to use equiadapt with minimalistic changes to your own code. +You can also find [tutorials](https://github.com/arnab39/equiadapt/blob/main/tutorials/) on how to use equiadapt with minimalistic changes to your own code. @@ -206,7 +206,7 @@ For questions related to this code, please raise an issue and you can mail us at # Contributing -You can check out the [contributor's guide](CONTRIBUTING.md). +You can check out the [contributor's guide](https://github.com/arnab39/equiadapt/blob/main/CHANGELOG.md). This project uses `pre-commit`, you can install it before making any changes:: From 552146bf4c998c0b2ccc8ca52586137d3eb7af92 Mon Sep 17 00:00:00 2001 From: danibene <34680344+danibene@users.noreply.github.com> Date: Mon, 18 Mar 2024 23:41:06 -0400 Subject: [PATCH 08/19] update newest python version to 3.12 --- .github/workflows/ci.yml | 3 +-- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c107a13..8c04bef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,8 +61,7 @@ jobs: matrix: python: - "3.7" # oldest Python supported by PSF - - "3.10" # newest Python that is stable - - "3.11" + - "3.12" # newest Python that is stable platform: - ubuntu-latest - macos-latest diff --git a/setup.cfg b/setup.cfg index 5bbf872..942b851 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,7 +35,7 @@ packages = find_namespace: include_package_data = True # Require a min/specific Python version (comma-separated conditions) -python_requires = >=3.7, <3.12 +python_requires = >=3.7 # Add here dependencies of your project (line-separated), e.g. requests>=2.2,<3.0. # Version specifiers like >=2.2,<3.0 avoid problems due to API changes in From 4c5e712ce9ebc46af84aca865fa3252c219f4b25 Mon Sep 17 00:00:00 2001 From: sibasmarak Date: Fri, 5 Apr 2024 06:48:55 -0400 Subject: [PATCH 09/19] correct segmentation eval; transfer learning with pretrained canonicalizer and finetuned pred network --- equiadapt/images/__init__.py | 4 + .../canonicalization_networks/__init__.py | 4 + .../custom_nonequivariant_networks.py | 102 +++++++++++++++++- .../opt_group_equivariant.yaml | 2 +- .../configs/checkpoint/default.yaml | 2 + .../images/classification/inference_utils.py | 8 +- examples/images/classification/model.py | 3 +- examples/images/classification/train.py | 8 +- examples/images/classification/train_utils.py | 18 ++++ examples/images/common/utils.py | 6 +- .../opt_group_equivariant.yaml | 4 +- .../configs/checkpoint/default.yaml | 2 + .../images/segmentation/inference_utils.py | 82 +++++++++----- examples/images/segmentation/train.py | 8 +- examples/images/segmentation/train_utils.py | 17 +++ 15 files changed, 226 insertions(+), 44 deletions(-) diff --git a/equiadapt/images/__init__.py b/equiadapt/images/__init__.py index b670541..bdc5971 100644 --- a/equiadapt/images/__init__.py +++ b/equiadapt/images/__init__.py @@ -22,6 +22,8 @@ RotationEquivariantConvLift, RotoReflectionEquivariantConv, RotoReflectionEquivariantConvLift, + WideResNet50Network, + WideResNet101Network, custom_equivariant_networks, custom_group_equivariant_layers, custom_nonequivariant_networks, @@ -51,6 +53,8 @@ "OptimizedGroupEquivariantImageCanonicalization", "OptimizedSteerableImageCanonicalization", "ResNet18Network", + "WideResNet50Network", + "WideResNet101Network", "RotationEquivariantConv", "RotationEquivariantConvLift", "RotoReflectionEquivariantConv", diff --git a/equiadapt/images/canonicalization_networks/__init__.py b/equiadapt/images/canonicalization_networks/__init__.py index 13b33d0..f48b390 100644 --- a/equiadapt/images/canonicalization_networks/__init__.py +++ b/equiadapt/images/canonicalization_networks/__init__.py @@ -16,6 +16,8 @@ from equiadapt.images.canonicalization_networks.custom_nonequivariant_networks import ( ConvNetwork, ResNet18Network, + WideResNet50Network, + WideResNet101Network, ) from equiadapt.images.canonicalization_networks.escnn_networks import ( ESCNNEquivariantNetwork, @@ -34,6 +36,8 @@ "ESCNNWideBasic", "ESCNNWideBottleneck", "ResNet18Network", + "WideResNet101Network", + "WideResNet50Network", "RotationEquivariantConv", "RotationEquivariantConvLift", "RotoReflectionEquivariantConv", diff --git a/equiadapt/images/canonicalization_networks/custom_nonequivariant_networks.py b/equiadapt/images/canonicalization_networks/custom_nonequivariant_networks.py index 73b5508..6c1d25d 100644 --- a/equiadapt/images/canonicalization_networks/custom_nonequivariant_networks.py +++ b/equiadapt/images/canonicalization_networks/custom_nonequivariant_networks.py @@ -110,7 +110,7 @@ def __init__( out_vector_size (int, optional): The size of the output vector of the network. Defaults to 128. """ super().__init__() - self.resnet18 = torchvision.models.resnet18(weights=None) + self.resnet18 = torchvision.models.resnet18(weights="DEFAULT") self.resnet18.fc = nn.Sequential( nn.Linear(512, out_vector_size), ) @@ -128,3 +128,103 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: torch.Tensor: The output of the network. It has the shape (batch_size, 1). """ return self.resnet18(x) + + +class WideResNet101Network(nn.Module): + """ + This class represents a neural network based on the WideResNetNetwork architecture. + + The network uses a pre-trained WideResNet model. The final fully connected layer of the WideResNet101 model is replaced with a new fully connected layer. + + Attributes: + resnet18 (torchvision.models.ResNet): The ResNet-18 model. + out_vector_size (int): The size of the output vector of the network. + """ + + def __init__( + self, + in_shape: tuple, + out_channels: int, + kernel_size: int, + num_layers: int = 2, + out_vector_size: int = 128, + ): + """ + Initializes the ResNet18Network instance. + + Args: + in_shape (tuple): The shape of the input data. It should be a tuple of the form (in_channels, height, width). + out_channels (int): The number of output channels of the first convolutional layer. + kernel_size (int): The size of the kernel of the convolutional layers. + num_layers (int, optional): The number of convolutional layers. Defaults to 2. + out_vector_size (int, optional): The size of the output vector of the network. Defaults to 128. + """ + super().__init__() + self.wideresnet = torchvision.models.wide_resnet101_2(weights="DEFAULT") + self.wideresnet.fc = nn.Sequential( + nn.Linear(2048, out_vector_size), + ) + + self.out_vector_size = out_vector_size + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """ + Performs a forward pass through the network. + + Args: + x (torch.Tensor): The input data. It should have the shape (batch_size, in_channels, height, width). + + Returns: + torch.Tensor: The output of the network. It has the shape (batch_size, 1). + """ + return self.wideresnet(x) + + +class WideResNet50Network(nn.Module): + """ + This class represents a neural network based on the WideResNetNetwork architecture. + + The network uses a pre-trained WideResNet model. The final fully connected layer of the WideResNet50 model is replaced with a new fully connected layer. + + Attributes: + resnet18 (torchvision.models.ResNet): The ResNet-18 model. + out_vector_size (int): The size of the output vector of the network. + """ + + def __init__( + self, + in_shape: tuple, + out_channels: int, + kernel_size: int, + num_layers: int = 2, + out_vector_size: int = 128, + ): + """ + Initializes the ResNet18Network instance. + + Args: + in_shape (tuple): The shape of the input data. It should be a tuple of the form (in_channels, height, width). + out_channels (int): The number of output channels of the first convolutional layer. + kernel_size (int): The size of the kernel of the convolutional layers. + num_layers (int, optional): The number of convolutional layers. Defaults to 2. + out_vector_size (int, optional): The size of the output vector of the network. Defaults to 128. + """ + super().__init__() + self.wideresnet = torchvision.models.wide_resnet50_2(weights="DEFAULT") + self.wideresnet.fc = nn.Sequential( + nn.Linear(2048, out_vector_size), + ) + + self.out_vector_size = out_vector_size + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """ + Performs a forward pass through the network. + + Args: + x (torch.Tensor): The input data. It should have the shape (batch_size, in_channels, height, width). + + Returns: + torch.Tensor: The output of the network. It has the shape (batch_size, 1). + """ + return self.wideresnet(x) diff --git a/examples/images/classification/configs/canonicalization/opt_group_equivariant.yaml b/examples/images/classification/configs/canonicalization/opt_group_equivariant.yaml index 986eae3..8d5d1e1 100644 --- a/examples/images/classification/configs/canonicalization/opt_group_equivariant.yaml +++ b/examples/images/classification/configs/canonicalization/opt_group_equivariant.yaml @@ -1,5 +1,5 @@ canonicalization_type: opt_group_equivariant -network_type: cnn # Options for canonization method 1) cnn 2) wideresnet +network_type: cnn # Options for canonization method 1) cnn 2) non_equivariant_wrn_50 3) non_equivariant_wrn_101 4) non_equivariant_resnet18 network_hyperparams: kernel_size: 7 # Kernel size for the canonization network out_channels: 16 # Number of output channels for the canonization network diff --git a/examples/images/classification/configs/checkpoint/default.yaml b/examples/images/classification/configs/checkpoint/default.yaml index b479fe7..57075ff 100644 --- a/examples/images/classification/configs/checkpoint/default.yaml +++ b/examples/images/classification/configs/checkpoint/default.yaml @@ -2,3 +2,5 @@ checkpoint_path: ${oc.env:CHECKPOINT_PATH} # Path to save checkpoints checkpoint_name: "" # Model checkpoint name, should be left empty for training and dynamically allocated later save_canonized_images: 0 # Whether to save canonized images (1) or not (0) strict_loading: 1 # Whether to strictly load the model (1) or not (0) +prediction_network_checkpoint_path: null # Path to load prediction network checkpoints +prediction_network_checkpoint_name: null # Path to load prediction network checkpoint name diff --git a/examples/images/classification/inference_utils.py b/examples/images/classification/inference_utils.py index 9095df2..617399e 100644 --- a/examples/images/classification/inference_utils.py +++ b/examples/images/classification/inference_utils.py @@ -61,7 +61,9 @@ def get_inference_metrics(self, x: torch.Tensor, y: torch.Tensor): ] # check if the accuracy per class is nan - acc_per_class = [0.0 if math.isnan(acc) else acc for acc in acc_per_class] + acc_per_class = [ + torch.tensor(0.0) if math.isnan(acc) else acc for acc in acc_per_class + ] # Update metrics with accuracy per class metrics.update( @@ -151,7 +153,9 @@ def get_inference_metrics(self, x: torch.Tensor, y: torch.Tensor): ] # check if the accuracy per class is nan - acc_per_class = [0.0 if math.isnan(acc) else acc for acc in acc_per_class] + acc_per_class = [ + torch.tensor(0.0) if math.isnan(acc) else acc for acc in acc_per_class + ] # Update metrics with accuracy per class metrics.update( diff --git a/examples/images/classification/model.py b/examples/images/classification/model.py index 1586713..6fd9a49 100644 --- a/examples/images/classification/model.py +++ b/examples/images/classification/model.py @@ -64,7 +64,7 @@ def training_step(self, batch: torch.Tensor): assert (num_channels, height, width) == self.image_shape training_metrics = {} - loss = 0.0 + loss, acc = 0.0, 0.0 # canonicalize the input data # For the vanilla model, the canonicalization is the identity transformation @@ -101,7 +101,6 @@ def training_step(self, batch: torch.Tensor): acc = (preds == y).float().mean() training_metrics.update({"train/task_loss": task_loss, "train/acc": acc}) - training_metrics.update({"train/task_loss": task_loss, "train/acc": acc}) # Add prior regularization loss if the prior weight is non-zero if self.hyperparams.experiment.training.loss.prior_weight: diff --git a/examples/images/classification/train.py b/examples/images/classification/train.py index 50a2ff5..779f985 100644 --- a/examples/images/classification/train.py +++ b/examples/images/classification/train.py @@ -75,13 +75,13 @@ def train_images(hyperparams: DictConfig) -> None: if not hyperparams["experiment"]["run_mode"] == "test": hyperparams["checkpoint"]["checkpoint_name"] = ( - wandb_run.id + str(wandb_run.id) + "_" - + wandb_run.name + + str(wandb_run.name) + "_" - + wandb_run.sweep_id + + str(wandb_run.sweep_id) + "_" - + wandb_run.group + + str(wandb_run.group) ) # set seed diff --git a/examples/images/classification/train_utils.py b/examples/images/classification/train_utils.py index 6e9fc5b..f4f7996 100644 --- a/examples/images/classification/train_utils.py +++ b/examples/images/classification/train_utils.py @@ -2,6 +2,7 @@ import dotenv import pytorch_lightning as pl +import torch from model import ImageClassifierPipeline from omegaconf import DictConfig from prepare import ( @@ -39,6 +40,23 @@ def get_model_pipeline(hyperparams: DictConfig) -> pl.LightningModule: hyperparams=hyperparams, strict=hyperparams.checkpoint.strict_loading, ) + + # load a different (finetuned) prediction network + # to test the transfer learning capabilities of the canonicalizer + if hyperparams.checkpoint.prediction_network_checkpoint_path: + model.load_state_dict( + torch.load( + open( + hyperparams.checkpoint.prediction_network_checkpoint_path + + "/" + + hyperparams.checkpoint.prediction_network_checkpoint_name + + ".ckpt", + mode="rb", + ) + )["state_dict"], + strict=False, + ) + model.freeze() model.eval() else: diff --git a/examples/images/common/utils.py b/examples/images/common/utils.py index a1ce933..59dde53 100644 --- a/examples/images/common/utils.py +++ b/examples/images/common/utils.py @@ -17,6 +17,8 @@ ESCNNSteerableNetwork, ESCNNWRNEquivariantNetwork, ResNet18Network, + WideResNet50Network, + WideResNet101Network, ) @@ -46,7 +48,9 @@ def get_canonicalization_network( }, "opt_group_equivariant": { "cnn": ConvNetwork, - "resnet18": ResNet18Network, + "non_equivariant_resnet18": ResNet18Network, + "non_equivariant_wrn_101": WideResNet101Network, + "non_equivariant_wrn_50": WideResNet50Network, }, "opt_steerable": { "cnn": ConvNetwork, diff --git a/examples/images/segmentation/configs/canonicalization/opt_group_equivariant.yaml b/examples/images/segmentation/configs/canonicalization/opt_group_equivariant.yaml index 986eae3..b0b4f5d 100644 --- a/examples/images/segmentation/configs/canonicalization/opt_group_equivariant.yaml +++ b/examples/images/segmentation/configs/canonicalization/opt_group_equivariant.yaml @@ -1,5 +1,5 @@ canonicalization_type: opt_group_equivariant -network_type: cnn # Options for canonization method 1) cnn 2) wideresnet +network_type: cnn # Options for canonization method 1) cnn 2) non_equivariant_wrn network_hyperparams: kernel_size: 7 # Kernel size for the canonization network out_channels: 16 # Number of output channels for the canonization network @@ -9,6 +9,6 @@ group_type: rotation # Type of group for the canonization network num_rotations: 4 # Number of rotations for the canonization network beta: 1.0 # Beta parameter for the canonization network input_crop_ratio: 0.8 # Ratio at which we crop the input to the canonicalization -resize_shape: 96 # Resize shape for the input +resize_shape: 128 # Resize shape for the input learn_ref_vec: False # Whether to learn the reference vector artifact_err_wt: 0 # Weight for rotation artifact error (specific to image data, for non C4 rotation, for non-equivariant canonicalization networks) diff --git a/examples/images/segmentation/configs/checkpoint/default.yaml b/examples/images/segmentation/configs/checkpoint/default.yaml index a3e9fbd..8470d12 100644 --- a/examples/images/segmentation/configs/checkpoint/default.yaml +++ b/examples/images/segmentation/configs/checkpoint/default.yaml @@ -2,3 +2,5 @@ checkpoint_path: ${oc.env:CHECKPOINT_PATH} # Path to save checkpoints checkpoint_name: "" # Model checkpoint name, should be left empty and dynamically allocated later save_canonized_images: 0 # Whether to save canonized images (1) or not (0) strict_loading: 1 # Whether to strictly load the model (1) or not (0) +prediction_network_checkpoint_path: null # Path to load prediction network checkpoints +prediction_network_checkpoint_name: null # Path to load prediction network checkpoint name diff --git a/examples/images/segmentation/inference_utils.py b/examples/images/segmentation/inference_utils.py index a60770e..97ce758 100644 --- a/examples/images/segmentation/inference_utils.py +++ b/examples/images/segmentation/inference_utils.py @@ -76,6 +76,46 @@ def __init__( self.pad = transforms.Pad(math.ceil(in_shape[-2] * 0.4), padding_mode="edge") self.crop = transforms.CenterCrop((in_shape[-2], in_shape[-1])) + def forward( + self, x: torch.Tensor, targets: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + # canonicalize the input data + # For the vanilla model, the canonicalization is the identity transformation + _, _, _, outputs = super().forward(x, targets) + + if self.canonicalizer.canonicalization_info_dict: + # if non-identity canonicalization is used, then the outputs will be transformed + rotation_angles = self.canonicalizer.canonicalization_info_dict[ + "group_element" + ]["rotation"] + reflections = ( + self.canonicalizer.canonicalization_info_dict["group_element"][ + "reflection" + ] + if "reflection" + in self.canonicalizer.canonicalization_info_dict["group_element"] + else [None] * len(rotation_angles) + ) + + image_width = x[0].shape[1] + outputs = [ + dict( + boxes=( + flip_boxes(rotate_boxes(output["boxes"], degree, image_width)) + if reflection + else rotate_boxes(output["boxes"], degree, image_width) + ), + labels=output["labels"], + scores=output["scores"], + masks=output["masks"], + ) + for degree, reflection, output in zip( + rotation_angles, reflections, outputs + ) + ] + + return None, None, None, outputs + def get_group_element_wise_maps( self, images: torch.Tensor, targets: torch.Tensor ) -> dict: @@ -105,13 +145,13 @@ def get_group_element_wise_maps( _, _, _, outputs = self.forward(images_rot, targets_transformed) Map = MeanAveragePrecision(iou_type="segm") - targets = [ + _targets = [ dict( boxes=target["boxes"], labels=target["labels"], masks=target["masks"], ) - for target in targets + for target in targets_transformed ] outputs = [ dict( @@ -122,7 +162,7 @@ def get_group_element_wise_maps( ) for output in outputs ] - Map.update(outputs, targets) + Map.update(outputs, _targets) map_dict[rot] = Map.compute() @@ -191,30 +231,18 @@ def get_inference_metrics( for i in range(self.num_group_elements): metrics.update( { - f"test/map_group_element_{i}": max(map_dict[i]["map"], 0.0), - f"test/map_small_group_element_{i}": max( - map_dict[i]["map_small"], 0.0 - ), - f"test/map_medium_group_element_{i}": max( - map_dict[i]["map_medium"], 0.0 - ), - f"test/map_large_group_element_{i}": max( - map_dict[i]["map_large"], 0.0 - ), - f"test/map_50_group_element_{i}": max(map_dict[i]["map_50"], 0.0), - f"test/map_75_group_element_{i}": max(map_dict[i]["map_75"], 0.0), - f"test/mar_1_group_element_{i}": max(map_dict[i]["mar_1"], 0.0), - f"test/mar_10_group_element_{i}": max(map_dict[i]["mar_10"], 0.0), - f"test/mar_100_group_element_{i}": max(map_dict[i]["mar_100"], 0.0), - f"test/mar_small_group_element_{i}": max( - map_dict[i]["mar_small"], 0.0 - ), - f"test/mar_medium_group_element_{i}": max( - map_dict[i]["mar_medium"], 0.0 - ), - f"test/mar_large_group_element_{i}": max( - map_dict[i]["mar_large"], 0.0 - ), + f"test/map_group_element_{i}": map_dict[i]["map"], + f"test/map_small_group_element_{i}": map_dict[i]["map_small"], + f"test/map_medium_group_element_{i}": map_dict[i]["map_medium"], + f"test/map_large_group_element_{i}": map_dict[i]["map_large"], + f"test/map_50_group_element_{i}": map_dict[i]["map_50"], + f"test/map_75_group_element_{i}": map_dict[i]["map_75"], + f"test/mar_1_group_element_{i}": map_dict[i]["mar_1"], + f"test/mar_10_group_element_{i}": map_dict[i]["mar_10"], + f"test/mar_100_group_element_{i}": map_dict[i]["mar_100"], + f"test/mar_small_group_element_{i}": map_dict[i]["mar_small"], + f"test/mar_medium_group_element_{i}": map_dict[i]["mar_medium"], + f"test/mar_large_group_element_{i}": map_dict[i]["mar_large"], } ) diff --git a/examples/images/segmentation/train.py b/examples/images/segmentation/train.py index c243147..b32261d 100644 --- a/examples/images/segmentation/train.py +++ b/examples/images/segmentation/train.py @@ -87,13 +87,13 @@ def train_images(hyperparams: DictConfig) -> None: if not hyperparams["experiment"]["run_mode"] == "test": hyperparams["checkpoint"]["checkpoint_name"] = ( - wandb_run.id + str(wandb_run.id) + "_" - + wandb_run.name + + str(wandb_run.name) + "_" - + wandb_run.sweep_id + + str(wandb_run.sweep_id) + "_" - + wandb_run.group + + str(wandb_run.group) ) # set seed diff --git a/examples/images/segmentation/train_utils.py b/examples/images/segmentation/train_utils.py index dc282e9..48e6d10 100644 --- a/examples/images/segmentation/train_utils.py +++ b/examples/images/segmentation/train_utils.py @@ -2,6 +2,7 @@ import dotenv import pytorch_lightning as pl +import torch from model import ImageSegmentationPipeline from omegaconf import DictConfig from prepare import COCODataModule @@ -33,6 +34,22 @@ def get_model_pipeline(hyperparams: DictConfig) -> pl.LightningModule: hyperparams=hyperparams, strict=hyperparams.checkpoint.strict_loading, ) + + # load a different (finetuned) prediction network + # to test the transfer learning capabilities of the canonicalizer + if hyperparams.checkpoint.prediction_network_checkpoint_path: + model.load_state_dict( + torch.load( + open( + hyperparams.checkpoint.prediction_network_checkpoint_path + + "/" + + hyperparams.checkpoint.prediction_network_checkpoint_name + + ".ckpt", + mode="rb", + ) + )["state_dict"], + strict=False, + ) model.freeze() model.eval() else: From 68be8c40d70261373648aa3808783972cf420de8 Mon Sep 17 00:00:00 2001 From: danibene <34680344+danibene@users.noreply.github.com> Date: Thu, 11 Apr 2024 06:23:54 -0400 Subject: [PATCH 10/19] run "pre-commit run --all-files" --- examples/images/classification/train.py | 3 +-- examples/images/segmentation/train.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/examples/images/classification/train.py b/examples/images/classification/train.py index 779f985..9d7a731 100644 --- a/examples/images/classification/train.py +++ b/examples/images/classification/train.py @@ -4,12 +4,11 @@ import omegaconf import pytorch_lightning as pl import torch +import wandb from omegaconf import DictConfig, OmegaConf from pytorch_lightning.loggers import WandbLogger from train_utils import get_model_data_and_callbacks, get_trainer, load_envs -import wandb - def train_images(hyperparams: DictConfig) -> None: diff --git a/examples/images/segmentation/train.py b/examples/images/segmentation/train.py index b32261d..b70cf52 100644 --- a/examples/images/segmentation/train.py +++ b/examples/images/segmentation/train.py @@ -4,12 +4,11 @@ import omegaconf import pytorch_lightning as pl import torch +import wandb from omegaconf import DictConfig, OmegaConf from pytorch_lightning.loggers import WandbLogger from train_utils import get_model_data_and_callbacks, get_trainer, load_envs -import wandb - def train_images(hyperparams: DictConfig) -> None: From 4d6f8f3f9ce83cb9f5b3100fdbe1c22674165a5f Mon Sep 17 00:00:00 2001 From: danibene <34680344+danibene@users.noreply.github.com> Date: Thu, 11 Apr 2024 07:06:13 -0400 Subject: [PATCH 11/19] update contributor's guide with info on code checks --- CONTRIBUTING.md | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9afc304..245d2c9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -155,17 +155,29 @@ This can easily be done via [Anaconda] or [Miniconda] and detailed [here](https: `git log --graph --decorate --pretty=oneline --abbrev-commit --all` to look for recurring communication patterns. +#### Run code checks -5. Please check that your changes don't break any unit tests with: +Please make sure to see the validation messages from pre-commit and fix any +eventual issues. This should automatically use [flake8]/[black] to check/fix +the code style in a way that is compatible with the project. - ``` - tox - ``` +To run pre-commit manually, you can use: + +``` +pre-commit run --all-files +``` + +Please also check that your changes don't break any unit tests with: + +``` +tox +``` + +(after having installed [tox] with `pip install tox` or `pipx`). - (after having installed [tox] with `pip install tox` or `pipx`). +You can also use [tox] to run several other pre-configured tasks in the +repository. Try `tox -av` to see a list of the available checks. - You can also use [tox] to run several other pre-configured tasks in the - repository. Try `tox -av` to see a list of the available checks. ### Submit your contribution From bf537396de41326859500ed1c60e63bbbc527d87 Mon Sep 17 00:00:00 2001 From: danibene <34680344+danibene@users.noreply.github.com> Date: Thu, 11 Apr 2024 07:07:47 -0400 Subject: [PATCH 12/19] add PR template inspired by https://github.com/neuropsychology/NeuroKit/blob/97b5a97e660c867d01ffc683031e38c35ff7d034/.github/PULL_REQUEST_TEMPLATE.md --- .github/pull_request_template.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..ca47eb6 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,20 @@ +This is a template for making a pull-request. You can remove the text and sections and write your own thing if you wish, just make sure you give enough information about how and why. If you have any issues or difficulties, don't hesitate to open an issue. + + +# Description + +The aim is to add this feature ... + +# Proposed Changes + +I changed the `foo()` function so that ... + + +# Checklist + +Here are some things to check before creating the pull request. If you encounter any issues, don't hesitate to ask for help :) + +- [ ] I have read the [contributor's guide](https://github.com/arnab39/equiadapt/blob/main/CONTRIBUTING.md). +- [ ] The base branch of my pull request is the `dev` branch, not the `main` branch. +- [ ] I ran the [code checks](https://github.com/arnab39/equiadapt/blob/main/CONTRIBUTING.md#implement-your-changes) on the files I added or modified and fixed the errors. +- [ ] I updated the [changelog](https://github.com/arnab39/equiadapt/blob/main/CHANGELOG.md). From 5583bc7c091bba1861f4385adc45f4d8ddb22612 Mon Sep 17 00:00:00 2001 From: danibene <34680344+danibene@users.noreply.github.com> Date: Thu, 11 Apr 2024 07:08:41 -0400 Subject: [PATCH 13/19] update changelog with changes from this PR --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a443b94..cb94f3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- Added pull request template. ### Fixed ### Changed +- Updated `CONTRIBUTING.md` with more information on how to run the code checks. ### Removed From 99c533f3817c92efa13ed552f3a9a76c01ac22bf Mon Sep 17 00:00:00 2001 From: danibene <34680344+danibene@users.noreply.github.com> Date: Thu, 11 Apr 2024 07:10:04 -0400 Subject: [PATCH 14/19] add changes from https://github.com/arnab39/equiadapt/commit/f39eb87e99200941e350f0b4695507cc95395cc5 to changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb94f3a..be24ee5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added pull request template. +- Added test for discrete invert canonicalization, fixing minor bugs. ### Fixed +- Fixed inverse canonicalization key issue group -> group_element. +- Fixed scalar group element key: 0 -> rotation. ### Changed - Updated `CONTRIBUTING.md` with more information on how to run the code checks. From 8bf1da389a1ff94afc26ef377b47250cc35c457f Mon Sep 17 00:00:00 2001 From: sibasmarak Date: Mon, 27 May 2024 16:02:27 -0400 Subject: [PATCH 15/19] Added EquiOptAdapt paper details --- .pre-commit-config.yaml | 6 +++--- README.md | 17 +++++++++++++++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7f783d0..9a01735 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ exclude: '^docs/conf.py' repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: trailing-whitespace - id: check-added-large-files @@ -40,7 +40,7 @@ repos: - id: isort - repo: https://github.com/psf/black - rev: 24.2.0 + rev: 24.4.2 hooks: - id: black language_version: python3 @@ -66,7 +66,7 @@ repos: # Check for type errors with mypy: - repo: https://github.com/pre-commit/mirrors-mypy - rev: 'v1.9.0' + rev: 'v1.10.0' hooks: - id: mypy args: [--disallow-untyped-defs, --ignore-missing-imports] diff --git a/README.md b/README.md index 324cd7a..5fdf567 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,8 @@ You can clone this repository and manually install it with: ## Setup Conda environment for examples +The recommended way is to manually create an environment and install the dependencies from the `min_conda_env.yaml` file. + To create a conda environment with the necessary packages: ``` @@ -168,7 +170,7 @@ You can also find [tutorials](https://github.com/arnab39/equiadapt/blob/main/tut # Related papers and Citations -For more insights on this library refer to our original paper on the idea: [Equivariance with Learned Canonicalization Function (ICML 2023)](https://proceedings.mlr.press/v202/kaba23a.html) and how to extend it to make any existing large pre-trained model equivariant: [Equivariant Adaptation of Large Pretrained Models (NeurIPS 2023)](https://proceedings.neurips.cc/paper_files/paper/2023/hash/9d5856318032ef3630cb580f4e24f823-Abstract-Conference.html). +For more insights on this library refer to our original paper on the idea: [Equivariance with Learned Canonicalization Function (ICML 2023)](https://proceedings.mlr.press/v202/kaba23a.html) and how to extend it to make any existing large pre-trained model equivariant: [Equivariant Adaptation of Large Pretrained Models (NeurIPS 2023)](https://proceedings.neurips.cc/paper_files/paper/2023/hash/9d5856318032ef3630cb580f4e24f823-Abstract-Conference.html). An improved approach for designing canonicalization network, which allows non-equivariant and expressive models as equivariant networks is presented in [Improved Canonicalization for Model Agnostic Equivariance (CVPR 2024: EquiVision Workshop)](https://arxiv.org/abs/2405.14089). If you find this library or the associated papers useful, please cite the following papers: @@ -197,6 +199,17 @@ If you find this library or the associated papers useful, please cite the follow } ``` +``` +@inproceedings{ + panigrahi2024improved, + title={Improved Canonicalization for Model Agnostic Equivariance}, + author={Siba Smarak Panigrahi and Arnab Kumar Mondal}, + booktitle={CVPR 2024 Workshop on Equivariant Vision: From Theory to Practice}, + year={2024}, + url={https://arxiv.org/abs/2405.14089} +} +``` + # Contact For questions related to this code, please raise an issue and you can mail us at: @@ -206,7 +219,7 @@ For questions related to this code, please raise an issue and you can mail us at # Contributing -You can check out the [contributor's guide](https://github.com/arnab39/equiadapt/blob/main/CHANGELOG.md). +You can check out the [contributor's guide](https://github.com/arnab39/equiadapt/blob/main/CONTRIBUTING.md). This project uses `pre-commit`, you can install it before making any changes:: From 2d355ac6ec487575a0e13d132e0275c030bb0040 Mon Sep 17 00:00:00 2001 From: danibene <34680344+danibene@users.noreply.github.com> Date: Mon, 27 May 2024 22:13:34 -0400 Subject: [PATCH 16/19] change os tested for python 3.7 following https://github.com/actions/runner-images/issues/9770#issuecomment-2085623315 --- .github/workflows/ci.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c04bef..0775954 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,14 +58,16 @@ jobs: test: needs: prepare strategy: + fail-fast: false matrix: - python: - - "3.7" # oldest Python supported by PSF - - "3.12" # newest Python that is stable - platform: - - ubuntu-latest - - macos-latest - - windows-latest + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + os: [ubuntu-latest, macos-latest, windows-latest] + exclude: # Python < v3.8 does not support Apple Silicon ARM64. + - python-version: "3.7" + os: macos-latest + include: # So run those legacy versions on Intel CPUs. + - python-version: "3.7" + os: macos-13 runs-on: ${{ matrix.platform }} steps: - uses: actions/checkout@v3 From 00b0e86c218f277e1e1e44a6f04bc0abcabd9fa4 Mon Sep 17 00:00:00 2001 From: danibene <34680344+danibene@users.noreply.github.com> Date: Mon, 27 May 2024 22:17:20 -0400 Subject: [PATCH 17/19] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index be24ee5..2de1c9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Updated `CONTRIBUTING.md` with more information on how to run the code checks. +- Changed the OS used to test Python 3.7 on GitHub actions (macos-latest -> macos-13). ### Removed From 102f73f135c74447aee9049e8a254eb6f66a856c Mon Sep 17 00:00:00 2001 From: danibene <34680344+danibene@users.noreply.github.com> Date: Mon, 27 May 2024 22:22:49 -0400 Subject: [PATCH 18/19] use variable names from previous version of ci file --- .github/workflows/ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0775954..2b77530 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,14 +60,14 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] - os: [ubuntu-latest, macos-latest, windows-latest] + python: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + platform: [ubuntu-latest, macos-latest, windows-latest] exclude: # Python < v3.8 does not support Apple Silicon ARM64. - - python-version: "3.7" - os: macos-latest + - python: "3.7" + platform: macos-latest include: # So run those legacy versions on Intel CPUs. - - python-version: "3.7" - os: macos-13 + - python: "3.7" + platform: macos-13 runs-on: ${{ matrix.platform }} steps: - uses: actions/checkout@v3 From 2b0e434873e30fea8d62bd6bcaaebdd9667e852c Mon Sep 17 00:00:00 2001 From: sibasmarak Date: Wed, 29 May 2024 18:14:53 -0400 Subject: [PATCH 19/19] Updated CHANGELOG.md --- CHANGELOG.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2de1c9d..ba5a48f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,22 +5,23 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.1.2] - 2024-05-29 ### Added +- Added canonicalization with optimization approach. +- Added evaluating transfer learning capabilities of canonicalizer. - Added pull request template. -- Added test for discrete invert canonicalization, fixing minor bugs. +- Added test for discrete invert canonicalization. ### Fixed -- Fixed inverse canonicalization key issue group -> group_element. -- Fixed scalar group element key: 0 -> rotation. +- Fixed segmentation evaluation for non-identity canonicalizers. +- Fixed minor bugs in inverse canonicalization for discrete groups. ### Changed +- Updated `README.md` with [Improved Canonicalization for Model Agnostic Equivariance](https://arxiv.org/abs/2405.14089) ([EquiVision](https://equivision.github.io/), CVPR 2024 workshop) paper details. - Updated `CONTRIBUTING.md` with more information on how to run the code checks. - Changed the OS used to test Python 3.7 on GitHub actions (macos-latest -> macos-13). -### Removed - ## [0.1.1] - 2024-03-15 ### Changed