Run tests with pytest, not bash

.circleci/config.yml

@@ -25,6 +25,9 @@ jobs:
           name: Run Pytorch scripts
           command: ./scripts/run_pytorch.sh
           no_output_timeout: 1h
+      - store_test_results:
+          path: test-results
+
 
 workflows:
   version: 2

scripts/install.sh

@@ -9,6 +9,8 @@ conda install -y pytorch torchvision -c pytorch
 # Also install torchaudio
 conda install -y -c pytorch torchaudio
 
+conda install -y pytest
+
 # Dependencies required to load models
 conda install -y regex pillow tqdm boto3 requests numpy\
     h5py scipy matplotlib unidecode ipython pyyaml

scripts/run_pytorch.sh

@@ -3,33 +3,21 @@ set -e
 . ~/miniconda3/etc/profile.d/conda.sh
 conda activate base
 
-ALL_FILE=$(find *.md ! -name README.md)
-TEMP_PY="temp.py"
-CUDAS="nvidia"
+ALL_FILES=$(find *.md ! -name README.md)
+PYTHON_CODE_DIR="python_code"
 
-for f in $ALL_FILE
-do
-  echo "Running pytorch example in $f"
-  # FIXME: NVIDIA models checkoints are on cuda
-  if [[ $f = $CUDAS* ]]; then
-    echo "...skipped due to cuda checkpoints."
-  elif [[ $f = "pytorch_fairseq_translation"* ]]; then
-    echo "...temporarily disabled"
-  # FIXME: torch.nn.modules.module.ModuleAttributeError: 'autoShape' object has no attribute 'fuse'
-  elif [[ $f = "ultralytics_yolov5"* ]]; then
-    echo "...temporarily disabled"
-  elif [[ $f = "huggingface_pytorch-transformers"* ]]; then
-    echo "...temporarily disabled"
-  # FIXME: TypeError: compose() got an unexpected keyword argument 'strict'
-  elif [[ $f = "pytorch_fairseq_roberta"* ]]; then
-    echo "...temporarily disabled"
-  # FIXME: rate limiting
-  else
-    sed -n '/^```python/,/^```/ p' < $f | sed '/^```/ d' > $TEMP_PY
-    python $TEMP_PY
+mkdir $PYTHON_CODE_DIR
 
-    if [ -f "$TEMP_PY" ]; then
-      rm $TEMP_PY
-    fi
-  fi
+# Quick rundown: for each file we extract the python code that's within
+# the ``` markers and we put that code in a corresponding .py file in $PYTHON_CODE_DIR
+# Then we execute each of these python files with pytest in test_run_python_code.py
+for f in $ALL_FILES
+do
+  f_no_ext=${f%.md}  # remove .md extension
+  out_py=$PYTHON_CODE_DIR/$f_no_ext.py
+  echo "Extracting Python code from $f into $out_py"
+  sed -n '/^```python/,/^```/ p' < $f | sed '/^```/ d' > $out_py
 done
+
+pytest -v -s test_run_python_code.py
+rm -r $PYTHON_CODE_DIR

test_run_python_code.py

@@ -0,0 +1,24 @@
+import pytest
+import glob
+
+
+ALL_FILES = glob.glob("python_code/*.py")
+
+
+@pytest.mark.parametrize('file_path', ALL_FILES)
+def test_run_file(file_path):
+    if 'nvidia' in file_path:
+        # FIXME: NVIDIA models checkoints are on cuda
+        pytest.skip("temporarily disabled")
+    if 'pytorch_fairseq_translation' in file_path:
+        pytest.skip("temporarily disabled")
+    if 'ultralytics_yolov5' in file_path:
+        # FIXME torch.nn.modules.module.ModuleAttributeError: 'autoShape' object has no attribute 'fuse
+        pytest.skip("temporarily disabled")
+    if 'huggingface_pytorch-transformers' in file_path:
+        # FIXME torch.nn.modules.module.ModuleAttributeError: 'autoShape' object has no attribute 'fuse
+        pytest.skip("temporarily disabled")
+    if 'pytorch_fairseq_roberta' in file_path:
+        pytest.skip("temporarily disabled")
+
+    exec(open(file_path).read())