.

jedmccaleb · jedmccaleb · commit fe8b7e7a6587 · 2023-04-23T18:02:52.000-07:00
diff --git a/configs/local_setup.yml b/configs/local_setup.yml
@@ -1,6 +1,7 @@
 # Suggested data paths when using GPT-NeoX locally
 {
   "global_num_gpus":1,
+  "num_gpus": 1,
   "data-path": "data/enwik8/enwik8_text_document",
 
   # or for weighted datasets:
diff --git a/megatron/training.py b/megatron/training.py
@@ -613,7 +613,7 @@ def setup_model_and_optimizer(neox_args, use_cache=False, iteration=None):
 
         params=neox_args.deepspeed_config
         params["gradient_accumulation_steps"]=1 # JED figure out what is broken
-        params["train_batch_size"]=16 # JED figure out what is broken
+        #params["train_batch_size"]=16 # JED figure out what is broken
        
         
         model, optimizer, _, lr_scheduler = deepspeed.initialize(

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,7 @@`
`1`	`1`	`# Suggested data paths when using GPT-NeoX locally`
`2`	`2`	`{`
`3`	`3`	`"global_num_gpus":1,`
	`4`	`+ "num_gpus": 1,`
`4`	`5`	`"data-path": "data/enwik8/enwik8_text_document",`
`5`	`6`
`6`	`7`	`# or for weighted datasets:`