From a2093531a6e9ae9efb727eff9b5ea72b8d4dcd2f Mon Sep 17 00:00:00 2001
From: Jakub <jakub@schwenkbeck.de>
Date: Wed, 4 Dec 2024 19:57:00 +0100
Subject: [PATCH] sinusoidal positional encoding implemented as util for
 embedded layer in the future

---
 src/math/positional_encoding.rs | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 src/math/positional_encoding.rs

diff --git a/src/math/positional_encoding.rs b/src/math/positional_encoding.rs
new file mode 100644
index 0000000..3339987
--- /dev/null
+++ b/src/math/positional_encoding.rs
@@ -0,0 +1,9 @@
+pub fn sinusoidal_pos_encoding(pos: usize, index: usize, embedding: usize) -> f32 {
+    let divisor = 10000f32.powf(2.0 * (index / embedding) as f32);
+
+    if index % 2 == 0 {
+        (pos as f32 / divisor).sin()
+    } else {
+        (pos as f32 / divisor).cos()
+    }
+}