GeLU backward eq fix

Vivswan · Nov 22, 2023 · 38a966c · 38a966c
1 parent 0d444e5
commit 38a966c
Show file tree

Hide file tree

Showing 4 changed files with 6 additions and 3 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,8 @@
 # Changelog
 
+## 1.0.7
+* Fixed `GeLU` backward function equation.
+
 ## 1.0.6
 
 * `Model` is subclass of `BackwardModule` for additional functionality.

diff --git a/analogvnn/nn/activation/Gaussian.py b/analogvnn/nn/activation/Gaussian.py
@@ -68,6 +68,6 @@ def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]:
 
         x = self.inputs
         grad = (1 / 2) * (
-                (1 + torch.erf(x / math.sqrt(2))) + x * ((2 / math.sqrt(math.pi)) * torch.exp(-torch.pow(x, 2)))
+                (1 + torch.erf(x / math.sqrt(2))) + x * (math.sqrt(2 * math.pi) * torch.exp(-torch.pow(x, 2) / 2))
         )
         return grad_output * grad
diff --git a/pyproject.toml b/pyproject.toml
@@ -19,7 +19,7 @@ where = ["analogvnn"]
 [project]
 # $ pip install analogvnn
 name = "analogvnn"
-version = "1.0.6"
+version = "1.0.7"
 description = "A fully modular framework for modeling and optimizing analog/photonic neural networks"
 readme = "README.md"
 requires-python = ">=3.7"

diff --git a/requirements.txt b/requirements.txt
@@ -1,4 +1,4 @@
---extra-index-url https://download.pytorch.org/whl/cu118
+--extra-index-url https://download.pytorch.org/whl/cu121
 torch
 torchvision
 torchaudio