Merge branch 'main' into rwitten_v4_convergence

Author: Rafi Witten

.github/workflows/UnitTests.yml

@@ -23,8 +23,8 @@ on:
     branches: [ "main" ]
   workflow_dispatch:
   schedule:
-    # Run the job every 60 mins
-    - cron:  '*/60 * * * *'
+    # Run the job every 2 hours
+    - cron:  '*/120 * * * *'
 
 jobs:
   # IF YOU MODIFY THIS, YOU SHOULD ALSO ADD CORRESPONDING MODICATIONS TO 'gpu' job

end_to_end/llama_load_and_test.sh

@@ -1,6 +1,9 @@
 set -e
 idx=$(date +%Y-%m-%d-%H-%M)
-pip install torch
-gsutil cp -r gs://maxtext-llama/llama2-7b/meta-ckpt /tmp/
-python3 MaxText/convert_llama_ckpt.py --base-model-path /tmp/meta-ckpt --model-size 7b --maxtext-model-path gs://maxtext-llama/test/${idx}/decode-ckpt-maxtext/
-python3 MaxText/decode.py MaxText/configs/base.yml run_name=runner_${idx} base_output_directory=gs://runner-maxtext-logs dataset_path=gs://maxtext-dataset load_from_other_directory=gs://maxtext-llama/test/${idx}/decode-ckpt-maxtext/ load_from_other_directory_step=0 per_device_batch_size=1 model_name='llama2-7b' assets_path=gs://maxtext-llama/llama2-7b ici_tensor_parallelism=4 steps=1 max_prefill_predict_length=4  max_target_length=16 async_checkpointing=false prompt="I love to" autoregressive_decode_assert="read. I love to write. I love to share." attention=dot_product
+#TODO(internal bug -- migrate to XLML)
+#pip install torch
+#gsutil cp -r gs://maxtext-llama/llama2-7b/meta-ckpt /tmp/
+#python3 MaxText/convert_llama_ckpt.py --base-model-path /tmp/meta-ckpt --model-size 7b --maxtext-model-path gs://maxtext-llama/test/${idx}/decode-ckpt-maxtext/
+#python3 MaxText/decode.py MaxText/configs/base.yml run_name=runner_${idx} base_output_directory=gs://runner-maxtext-logs dataset_path=gs://maxtext-dataset load_from_other_directory=gs://maxtext-llama/test/${idx}/decode-ckpt-maxtext/ load_from_other_directory_step=0 per_device_batch_size=1 model_name='llama2-7b' assets_path=gs://maxtext-llama/llama2-7b ici_tensor_parallelism=4 steps=1 max_prefill_predict_length=4  max_target_length=16 async_checkpointing=false prompt="I love to" autoregressive_decode_assert="read. I love to write. I love to share." attention=dot_product
+
+python3 MaxText/decode.py MaxText/configs/base.yml run_name=runner_${idx} base_output_directory=gs://runner-maxtext-logs dataset_path=gs://maxtext-dataset load_from_other_directory=gs://maxtext-llama/test/2024-01-12-17-46/decode-ckpt-maxtext/ load_from_other_directory_step=0 per_device_batch_size=1 model_name='llama2-7b' assets_path=gs://maxtext-llama/llama2-7b ici_tensor_parallelism=4 steps=1 max_prefill_predict_length=4  max_target_length=16 async_checkpointing=false prompt="I love to" autoregressive_decode_assert="read. I love to write. I love to share." attention=dot_product

end_to_end/test_decode.sh

@@ -25,6 +25,4 @@ fi
 python3 MaxText/decode.py MaxText/configs/base.yml run_name=$RUN_NAME\
     steps=50 enable_checkpointing=False metrics_file=/tmp/${RUN_NAME}_metrics.txt \
     base_output_directory=$OUTPUT_PATH dataset_path=$DATASET_PATH \
-    attention=dot_product ici_tensor_parallelism=${ICI_TENSOR_PARALLELISM} add_eos=False
-
-python3 end_to_end/eval_assert.py metrics_average /tmp/${RUN_NAME}_metrics.txt $NUM_TOKEN_THRESHOLD num_tokens
+    attention=dot_product ici_tensor_parallelism=${ICI_TENSOR_PARALLELISM} add_eos=False