Activation unit

.gitignore

@@ -6,6 +6,7 @@ simvectors/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/
+.pytest_cache/
 *.py[cod]
 *$py.class
 
@@ -22,10 +23,9 @@ dist
 # Jupyter Notebook
 .ipynb_checkpoints
 
-# Ignore everything in .vscode except launch.json
+# Ignore everything in .vscode with some exceptions
 .vscode/*
 !.vscode/launch.json
-!.vscode/settings.json
 !.vscode/c_cpp_properties.json
 
 simvectors

.gitlab-ci.yml

@@ -26,8 +26,9 @@ generate_testvectors:
   stage: test
   script:
     - !reference [.setup_test, script]
-    - python testGenerator.py -H 1 -S 64 -E 64 -P 64
-    - python testGenerator.py -H 1 -S 128 -E 192 -P 256
+    - python testGenerator.py -H 1 -S 64 -E 64 -P 64 -F 64 --activation gelu
+    - python testGenerator.py -H 1 -S 128 -E 192 -P 256 -F 256 --activation gelu
+    - python testGenerator.py -H 1 -S 192 -E 256 -P 128 -F 128 --activation relu
   artifacts:
     paths:
       - simvectors
@@ -42,13 +43,22 @@ run_sim:
     - S: 64
       E: 64
       P: 64
+      F: 64
+      activation: gelu
     - S: 128
       E: 192
       P: 256
+      F: 256
+      activation: gelu
+    - S: 192
+      E: 256
+      P: 128
+      F: 128
+      activation: relu
   script:
     - make bender
-    - make sim VSIM_FLAGS=-c s=$S e=$E p=$P bias=1
-    - ./modelsim/return_status.sh modelsim/build/transcript $S $E ita_tb
+    - make sim VSIM_FLAGS=-c s=$S e=$E p=$P f=$F bias=1 activation=$activation
+    - ./modelsim/return_status.sh modelsim/build/transcript $S $E $P $F ita_tb
 
 run_hwpe_sim:
   stage: sim
@@ -59,10 +69,19 @@ run_hwpe_sim:
     - S: 64
       E: 64
       P: 64
+      F: 64
+      activation: gelu
     - S: 128
       E: 192
       P: 256
+      F: 256
+      activation: gelu
+    - S: 192
+      E: 256
+      P: 128
+      F: 128
+      activation: relu
   script:
     - make bender
-    - make sim VSIM_FLAGS=-c DEBUG=OFF target=sim_ita_hwpe_tb s=$S e=$E p=$P bias=1
-    - ./modelsim/return_status.sh modelsim/build/transcript $S $E hwpe_tb
+    - make sim VSIM_FLAGS=-c DEBUG=OFF target=sim_ita_hwpe_tb s=$S e=$E p=$P f=$F bias=1 activation=$activation
+    - ./modelsim/return_status.sh modelsim/build/transcript $S $E $P $F hwpe_tb

Bender.yml

@@ -50,6 +50,10 @@ sources:
   - src/ita_weight_controller.sv
   - src/ita.sv
   - src/ita_max_finder.sv
+  - src/ita_gelu.sv
+  - src/ita_relu.sv
+  - src/ita_activation.sv
+  - src/ita_requantization_controller.sv
 
   # HWPE sources
   - target: ita_hwpe
@@ -72,6 +76,7 @@ sources:
       - src/tb/ita_tb.sv
       - src/tb/clk_rst_gen.sv
       - src/tb/rst_gen.sv
+      - src/tb/activation_tb.sv
 
   # HWPE TB sources
   - target: ita_hwpe_test

Makefile

@@ -21,10 +21,19 @@ target ?= sim_ita_tb
 
 no_stalls ?= 0
 s ?= 64
-p ?= 64
-e ?= 64
+e ?= 128
+p ?= 192
+f ?= 256
 bias ?= 0
-vlog_defs += -DNO_STALLS=$(no_stalls) -DSEQ_LENGTH=$(s) -DPROJ_SPACE=$(p) -DEMBED_SIZE=$(e) -DBIAS=$(bias)
+activation ?= identity
+ifeq ($(activation), gelu)
+	activation_int = 1
+else ifeq ($(activation), relu)
+	activation_int = 2
+else
+	activation_int = 0
+endif
+vlog_defs += -DNO_STALLS=$(no_stalls) -DSEQ_LENGTH=$(s) -DEMBED_SIZE=$(e) -DPROJ_SPACE=$(p) -DFF_SIZE=$(f) -DBIAS=$(bias) -DACTIVATION=$(activation_int)
 
 ifeq ($(target), sim_ita_hwpe_tb)
 	BENDER_TARGETS += -t ita_hwpe -t ita_hwpe_test