simplify 1b config (#181)

PrimeIntellect-ai · Dec 21, 2024 · 163ddb8 · 163ddb8
1 parent 3387a27
commit 163ddb8
Show file tree

Hide file tree

Showing 5 changed files with 5 additions and 85 deletions.
diff --git a/configs/1B/H100.toml b/configs/1B/H100.toml
@@ -3,10 +3,11 @@ project = "debug_1B_zero_band"
 type_model = "llama2"
 
 [train]
-micro_bs = 16  
+micro_bs = 32
+reshard_after_forward = true
 
 [optim]
-batch_size = 2048
+batch_size = 1024
 warmup_steps = 1000
-total_steps = 88_000
-lr = 4e-4
+total_steps = 8192
+lr = 7e-4
diff --git a/configs/1B/H100_c4.toml b/configs/1B/H100_c4.toml
diff --git a/configs/1B/H100_llama2_edu.toml b/configs/1B/H100_llama2_edu.toml
diff --git a/configs/1B/H100_llama2_edu_no_feat.toml b/configs/1B/H100_llama2_edu_no_feat.toml
diff --git a/configs/1B/H100_llama3.toml b/configs/1B/H100_llama3.toml