Add infra for publishing PyPI wheel (PolusAI#227)

* initial setup for PyPi publishing

* update tests to run w/o initial global config

* update pytest run

* skip problematic test cases for pypi build step

* try dependency resolution

* fix ppe8 error and update dep list

.github/workflows/fuzzy_compile_weekly.yml

@@ -80,13 +80,13 @@ jobs:
 
       - name: Install Workflow Inference Compiler
         if: always()
-        run: cd workflow-inference-compiler/ && pip install ".[all]"
+        run: cd workflow-inference-compiler/ && pip install ".[all_except_runner_src]"
 
       - name: Install Molecular Modeling Workflows
         if: always()
         # Also run mm-workflows command to generate
         # mm-workflows/autogenerated/schemas/config_schemas.json
-        # NOTE: Use ".[test]" instead of ".[all]"
+        # NOTE: Use ".[test]" instead of ".[all_except_runner_src]"
         # We do not want or need to install the workflow_deps extra.
         # (Many of the packages conflict with pypy.)
         run: cd mm-workflows/ && pip install ".[test]" && mm-workflows --generate_schemas

.github/workflows/lint_and_test.yml

@@ -162,13 +162,13 @@ jobs:
 
       - name: Install Workflow Inference Compiler
         if: always()
-        run: cd workflow-inference-compiler/ && pip install ".[all]"
+        run: cd workflow-inference-compiler/ && pip install ".[all_except_runner_src]"
 
       - name: Install Molecular Modeling Workflows
         if: always()
         # Also run mm-workflows command to generate
         # mm-workflows/autogenerated/schemas/config_schemas.json
-        # NOTE: Use ".[test]" instead of ".[all]"
+        # NOTE: Use ".[test]" instead of ".[all_except_runner_src]"
         # We do not want or need to install the workflow_deps extra.
         # (Many of the packages conflict with pypy.)
         run: cd mm-workflows/ && pip install ".[test]" && mm-workflows --generate_schemas

.github/workflows/lint_and_test_macos.yml

@@ -86,13 +86,13 @@ jobs:
 
       - name: Install Workflow Inference Compiler
         if: always()
-        run: cd workflow-inference-compiler/ && pip install ".[all]"
+        run: cd workflow-inference-compiler/ && pip install ".[all_except_runner_src]"
 
       - name: Install Molecular Modeling Workflows
         if: always()
         # Also run mm-workflows command to generate
         # mm-workflows/autogenerated/schemas/config_schemas.json
-        # NOTE: Use ".[test]" instead of ".[all]"
+        # NOTE: Use ".[test]" instead of ".[all_except_runner_src]"
         # We do not want or need to install the workflow_deps extra.
         # (Many of the packages conflict with pypy.)
         run: cd mm-workflows/ && pip install ".[test]" && mm-workflows --generate_schemas

.github/workflows/run_workflows.yml

@@ -140,13 +140,13 @@ jobs:
 
     - name: Install Workflow Inference Compiler
       if: always()
-      run: cd workflow-inference-compiler/ && pip install ".[all]"
+      run: cd workflow-inference-compiler/ && pip install ".[all_except_runner_src]"
 
     - name: Install Molecular Modeling Workflows
       if: always()
       # Also run mm-workflows command to generate
       # mm-workflows/autogenerated/schemas/config_schemas.json
-      # NOTE: Use ".[test]" instead of ".[all]"
+      # NOTE: Use ".[test]" instead of ".[all_except_runner_src]"
       # We do not want or need to install the workflow_deps extra.
       # (Many of the packages conflict with pypy.)
       run: cd mm-workflows/ && pip install ".[test]" && mm-workflows --generate_schemas

.github/workflows/run_workflows_weekly.yml

@@ -103,13 +103,13 @@ jobs:
 
     - name: Install Workflow Inference Compiler
       if: always()
-      run: cd workflow-inference-compiler/ && pip install ".[all]"
+      run: cd workflow-inference-compiler/ && pip install ".[all_except_runner_src]"
 
     - name: Install Molecular Modeling Workflows
       if: always()
       # Also run mm-workflows command to generate
       # mm-workflows/autogenerated/schemas/config_schemas.json
-      # NOTE: Use ".[test]" instead of ".[all]"
+      # NOTE: Use ".[test]" instead of ".[all_except_runner_src]"
       # We do not want or need to install the workflow_deps extra.
       # (Many of the packages conflict with pypy.)
       run: cd mm-workflows/ && pip install ".[test]" && mm-workflows --generate_schemas

.github/workflows/test_and_publish_pypi.yml

@@ -0,0 +1,40 @@
+name: Test and Publish PyPi
+
+on:
+  push:
+    tags:
+      - v[0-9]+.[0-9]+.[0-9]+
+      - v[0-9]+.[0-9]+.[0-9]+-dev[0-9]+
+
+jobs:
+  package:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout 🛎️
+        uses: actions/checkout@v3
+
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
+      - name: Install wic 🔧
+        run: |
+          pip install .[test]
+      - name: Test with pytest
+        run: |
+          python -m pytest -k test_cwl_docker_extract
+          python -m pytest -m "not skip_pypi_ci" -vv
+
+      - name: Build wic
+        run: |
+          python -m pip install twine build
+
+      - name: Publish to PyPi
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+        run: |
+          python -m build
+          twine upload dist/*

pyproject.toml

@@ -36,7 +36,10 @@ dependencies = [
     "typeguard",
     "pydantic>=2.6",
     "docker",
-    "podman" # FYI also need uidmap to run podman rootless
+    # FYI also need uidmap to run podman rootless
+    "podman", 
+    # We are using the official release for these packages for now
+    "toil[cwl]",
 ]
 
 [project.readme]
@@ -57,6 +60,8 @@ test = [
     "pytest-parallel",
     "coverage",
     "mypy",
+    "numpy",
+    "scipy",
     "pylint",
     "autopep8",
     "pre-commit",
@@ -110,6 +115,7 @@ doc = [
 ]
 plots = ["matplotlib"]
 cyto = ["ipycytoscape"] # only for DAG visualization
+all_except_runner_src = ["workflow-inference-compiler[test,doc,plots,cyto,mypy-types]"]
 all = ["workflow-inference-compiler[test,doc,plots,cyto,runners-src,mypy-types]"]
 
 [project.scripts]

tests/test_examples.py

@@ -30,7 +30,7 @@
 # Look in each directory of search_paths_wic tag in global_config.json
 # for separate config_ci.json files and combine them.
 config_ci: Json = {}
-global_config = wic.input_output.read_config_from_disk(Path(get_args().config_file))
+global_config = wic.input_output.get_config(Path(get_args().config_file), Path(get_args().config_file))
 search_paths_wic_tag = global_config['search_paths_wic']
 for _yml_namespaces in search_paths_wic_tag:
     yml_dirs = search_paths_wic_tag[_yml_namespaces]
@@ -121,6 +121,7 @@ def test_run_inlined_workflows_on_push(yml_path_str: str, yml_path: Path, cwl_ru
 
 
 # partial failure tests
+@pytest.mark.skip_pypi_ci
 @pytest.mark.parametrize("yml_path_str, yml_path", yml_paths_partial_failure)
 def test_run_partial_failures_pass(yml_path_str: str, yml_path: Path, cwl_runner: str) -> None:
     """Run workflows allowing partial failures. yml files of workflows which are known to have failure steps"""

tests/test_fuzzy_compile.py

@@ -20,6 +20,7 @@
 from .test_setup import tools_cwl, yml_paths, validator, wic_strategy
 
 
+@pytest.mark.skip_pypi_ci
 class TestFuzzyCompile(unittest.TestCase):
 
     @pytest.mark.slow

tests/test_setup.py

@@ -19,7 +19,7 @@
 
 args = wic.cli.get_args()
 # Just read from the disk and pass around config object
-global_config = io.read_config_from_disk(Path(args.config_file))
+global_config = io.get_config(Path(args.config_file), Path(args.config_file))
 tools_cwl = wic.plugins.get_tools_cwl(global_config, quiet=args.quiet)
 wic.api.pythonapi.global_config = tools_cwl  # Use path fallback in the CI
 yml_paths = wic.plugins.get_yml_paths(global_config)