Merge pull request #49 from Vivswan/develop

v1.0.7

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## 1.0.7
+* Fixed `GeLU` backward function equation.
+
 ## 1.0.6
 
 * `Model` is subclass of `BackwardModule` for additional functionality.

README.md

@@ -37,7 +37,9 @@ Documentation: [https://analogvnn.readthedocs.io/](https://analogvnn.readthedocs
 
 ## Abstract
 
-![3 Layered Linear Photonic Analog Neural Network](https://github.com/Vivswan/AnalogVNN/raw/release/docs/_static/analogvnn_model.png)
+![3 Layered Linear Photonic Analog Neural Network](docs/_static/analogvnn_model.png)
+
+[//]: # (![3 Layered Linear Photonic Analog Neural Network](https://github.com/Vivswan/AnalogVNN/raw/release/docs/_static/analogvnn_model.png))
 
 **AnalogVNN** is a simulation framework built on PyTorch which can simulate the effects of
 optoelectronic noise, limited precision, and signal normalization present in photonic

analogvnn/graph/AccumulateGrad.py

@@ -1,4 +1,4 @@
-from typing import Dict, Union, Callable
+from typing import Dict, Union, Callable, List
 
 import torch
 from torch import nn
@@ -9,6 +9,10 @@
 __all__ = ['AccumulateGrad']
 
 
+def _get_index(tensor: torch.Tensor, tensor_list: List[torch.Tensor]) -> int:
+    return [(i.shape == tensor.shape and torch.all(torch.eq(i, tensor))) for i in tensor_list].index(True)
+
+
 class AccumulateGrad:
     """AccumulateGrad is a module that accumulates the gradients of the outputs of the module it is attached to.
 
@@ -74,14 +78,14 @@ def __call__(  # noqa: C901
             if forward_out_arg is True and isinstance(forward_in_arg, int) and not isinstance(forward_in_arg, bool):
                 forward_inputs = forward_input_output_graph[predecessor].inputs.args
                 forward_outputs = forward_input_output_graph[self.module].outputs.args
-                forward_out_arg = forward_inputs.index(forward_outputs[forward_in_arg])
+                forward_out_arg = _get_index(forward_outputs[forward_in_arg], forward_inputs)
                 grad_output = grad_output[forward_out_arg]
 
             # 7
             if forward_out_arg is True and isinstance(forward_in_kwarg, str):
                 forward_inputs = forward_input_output_graph[predecessor].inputs.args
                 forward_outputs = forward_input_output_graph[self.module].outputs.kwargs
-                forward_out_arg = forward_inputs.index(forward_outputs[forward_in_kwarg])
+                forward_out_arg = _get_index(forward_outputs[forward_in_kwarg], forward_inputs)
                 grad_output = grad_output[forward_out_arg]
 
             # 1
@@ -92,7 +96,7 @@ def __call__(  # noqa: C901
                     if forward_inputs[i] not in forward_outputs:
                         continue
 
-                    value_index = forward_outputs.index(forward_inputs[i])
+                    value_index = _get_index(forward_inputs[i], forward_outputs)
                     if value_index not in grad_inputs_args:
                         grad_inputs_args[value_index] = torch.zeros_like(grad_output[i])
                     grad_inputs_args[value_index] += grad_output[i]
@@ -103,7 +107,7 @@ def __call__(  # noqa: C901
                 forward_inputs = forward_input_output_graph[predecessor].inputs.args
                 forward_outputs = forward_input_output_graph[self.module].outputs.kwargs
                 for i in forward_outputs:
-                    value_index = forward_inputs.index(forward_outputs[i])
+                    value_index = _get_index(forward_outputs[i], forward_inputs)
 
                     if i not in grad_inputs_kwargs:
                         grad_inputs_kwargs[i] = torch.zeros_like(grad_output[value_index])

analogvnn/graph/ForwardGraph.py

@@ -73,8 +73,15 @@ def calculate(
             inputs = (inputs,)
 
         if not self.graph_state.use_autograd_graph and is_training:
+            value_tensor = False
             for i in inputs:
+                if not isinstance(i, torch.Tensor):
+                    continue
                 i.requires_grad = True
+                value_tensor = True
+
+            if not value_tensor:
+                raise ValueError('At least one input must be a tensor.')
 
         input_output_graph = self._pass(self.INPUT, *inputs)
         if is_training:
@@ -102,8 +109,14 @@ def _pass(self, from_node: GraphEnum, *inputs: Tensor) -> Dict[GraphEnum, InputO
             if module != from_node:
                 inputs = self.parse_args_kwargs(input_output_graph, module, predecessors)
                 if not self.graph_state.use_autograd_graph:
-                    inputs.args = [self._detach_tensor(i) for i in inputs.args]
-                    inputs.kwargs = {k: self._detach_tensor(v) for k, v in inputs.kwargs.items()}
+                    inputs.args = [
+                        self._detach_tensor(i) if isinstance(i, torch.Tensor) else i
+                        for i in inputs.args
+                    ]
+                    inputs.kwargs = {
+                        k: self._detach_tensor(v) if isinstance(v, torch.Tensor) else v
+                        for k, v in inputs.kwargs.items()
+                    }
                 input_output_graph[module] = InputOutput(inputs=inputs)
 
             if isinstance(module, GraphEnum):

analogvnn/nn/activation/Gaussian.py

@@ -68,6 +68,6 @@ def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]:
 
         x = self.inputs
         grad = (1 / 2) * (
-                (1 + torch.erf(x / math.sqrt(2))) + x * ((2 / math.sqrt(math.pi)) * torch.exp(-torch.pow(x, 2)))
+                (1 + torch.erf(x / math.sqrt(2))) + x * (math.sqrt(2 * math.pi) * torch.exp(-torch.pow(x, 2) / 2))
         )
         return grad_output * grad

pyproject.toml

@@ -1,7 +1,7 @@
 # pyproject-build
 
 [build-system]
-requires = ["wheel", "setuptools>=61.0.0", "flit_core >=3.2,<4"]
+requires = ["wheel>=0.38.0", "setuptools>=61.0.0", "flit_core>=3.2,<4"]
 build-backend = "flit_core.buildapi"
 
 [tool.flit.module]
@@ -19,7 +19,7 @@ where = ["analogvnn"]
 [project]
 # $ pip install analogvnn
 name = "analogvnn"
-version = "1.0.6"
+version = "1.0.7"
 description = "A fully modular framework for modeling and optimizing analog/photonic neural networks"
 readme = "README.md"
 requires-python = ">=3.7"

requirements.txt

@@ -1,16 +1,19 @@
---extra-index-url https://download.pytorch.org/whl/cu118
+--extra-index-url https://download.pytorch.org/whl/cu121
 torch
 torchvision
 torchaudio
-numpy
+numpy>=1.22.2
 scipy
 networkx
 importlib-metadata; python_version < '3.8'
 
 # Full
+wheel>=0.38.0
 tensorflow>=2.0.0
 tensorboard>=2.0.0
 torchinfo
 # conda install graphviz python-graphviz pydot pydotplus python-dotenv
 # conda install --channel conda-forge pygraphviz
 graphviz
+pillow>=10.0.1 # not directly required, pinned by Snyk to avoid a vulnerability
+werkzeug>=3.0.1 # not directly required, pinned by Snyk to avoid a vulnerability

requirements/requirements-dev.txt

@@ -1,6 +1,6 @@
 # Development
 flit
-setuptools>=61.0.0
+setuptools>=65.5.1
 build # building the package {pyproject-build}
 twine # to publish on pypi {twine upload --repository-url=https://test.pypi.org/legacy/ dist/*} {twine upload dist/*}
 johnnydep # to see dependencies {johnnydep <package>}

requirements/requirements-docs.txt

@@ -11,3 +11,5 @@ sphinx-notfound-page
 sphinx-inline-tabs
 sphinxext-opengraph
 sphinxcontrib-katex  # math
+tornado>=6.3.3 # not directly required, pinned by Snyk to avoid a vulnerability
+pillow>=10.0.1 # not directly required, pinned by Snyk to avoid a vulnerability