[Python 3.9] Overhaul RMG-RMS Python-Julia dependencies (juliaup and pythoncall), Add Tests for Arkane Network Algorithms

.conda/meta.yaml

@@ -14,7 +14,6 @@ requirements:
     - {{ compiler('c') }} # [unix]
   host:
     - cython >=0.25.2
-    - lpsolve55
     - numpy
     - openbabel >=3
     - pydas >=1.0.2
@@ -39,7 +38,6 @@ requirements:
     - h5py
     - jinja2
     - jupyter
-    - lpsolve55
     - markupsafe
     - matplotlib >=1.5
     - mopac

.github/workflows/CI.yml

@@ -33,8 +33,8 @@ on:
   schedule:
     # * is a special character in YAML so you have to quote this string
     - cron: "0 8 * * *"
-  # runs on all branches on both RMG-Py and forks
-  push:
+  # allow running on RMG-Py on a pushed branch, only if triggered manually
+  workflow_dispatch:
   # runs on PRs against RMG-Py (and anywhere else, but we add this for RMG-Py)
   pull_request:
 
@@ -51,66 +51,15 @@ env:
 
 
 jobs:
-  build-osx:
-    runs-on: macos-13
-    # skip scheduled runs from forks
-    if: ${{ !( github.repository != 'ReactionMechanismGenerator/RMG-Py' && github.event_name == 'schedule' ) }}
-    defaults:
-      run:
-        shell: bash -l {0}
-    steps:
-      - name: Checkout RMG-Py
-        uses: actions/checkout@v4
-
-      # Step to create a custom condarc.yml before setting up conda
-      - name: Create custom conda config file
-        run: |
-          RUNNER_CWD=$(pwd)
-          echo "channels:" > $RUNNER_CWD/condarc.yml
-          echo "  - conda-forge" >> $RUNNER_CWD/condarc.yml
-          echo "  - rmg" >> $RUNNER_CWD/condarc.yml
-          echo "  - cantera" >> $RUNNER_CWD/condarc.yml
-          echo "show_channel_urls: true" >> $RUNNER_CWD/condarc.yml
-
-      # configures the mamba environment manager and builds the environment
-      - name: Setup Miniforge Python 3.7
-        uses: conda-incubator/setup-miniconda@v3
-        with:
-          environment-file: environment.yml
-          miniforge-variant: Miniforge3
-          miniforge-version: latest
-          python-version: 3.7
-          condarc-file: condarc.yml
-          activate-environment: rmg_env
-          use-mamba: true
-
-      # list the environment for debugging purposes
-      - name: mamba info
-        run: |
-          mamba info
-          mamba list
-
-      # Clone RMG-database
-      - name: Clone RMG-database
-        run: |
-          cd ..
-          git clone -b $RMG_DATABASE_BRANCH https://github.com/ReactionMechanismGenerator/RMG-database.git
-
-      # modify env variables as directed in the RMG installation instructions
-      - name: Set Environment Variables
-        run: |
-          RUNNER_CWD=$(pwd)
-          echo "PYTHONPATH=$RUNNER_CWD/RMG-Py:$PYTHONPATH" >> $GITHUB_ENV
-          echo "$RUNNER_CWD/RMG-Py" >> $GITHUB_PATH
-
-      # RMG build step
-      - name: make RMG
-        run: |
-          make clean
-          make
-
-  build-and-test-linux:
-    runs-on: ubuntu-latest
+  build-and-test:
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9"]
+        os: [macos-13, macos-latest, ubuntu-latest]
+        test_julia: [yes, no]
+    runs-on: ${{ matrix.os }}
+    name: ${{ matrix.os }} Build and Test Python ${{ matrix.python-version }} (Julia? ${{ matrix.test_julia }})
     # skip scheduled runs from forks
     if: ${{ !( github.repository != 'ReactionMechanismGenerator/RMG-Py' && github.event_name == 'schedule' ) }}
     env: 
@@ -134,13 +83,13 @@ jobs:
           echo "show_channel_urls: true" >> $RUNNER_CWD/condarc.yml
 
       # configures the mamba environment manager and builds the environment
-      - name: Setup Miniforge Python 3.7
+      - name: Setup Mambaforge Python ${{ matrix.python-version }}
         uses: conda-incubator/setup-miniconda@v3
         with:
           environment-file: environment.yml
           miniforge-variant: Miniforge3
           miniforge-version: latest
-          python-version: 3.7
+          python-version: ${{ matrix.python-version }}
           condarc-file: condarc.yml
           activate-environment: rmg_env
           use-mamba: true
@@ -170,12 +119,39 @@ jobs:
           make clean
           make
 
+      # Setup Juliaup
+      - name: Set Julia paths
+        if: matrix.test_julia == 'yes'
+        run: |
+          # echo "JULIAUP_DEPOT_PATH=$CONDA/envs/rmg_env/.julia" >> $GITHUB_ENV
+          # echo "JULIAUP_DEPOT_PATH=$CONDA/envs/rmg_env/.julia" >> $GITHUB_PATH
+          # echo "JULIA_DEPOT_PATH=$CONDA/envs/rmg_env/.julia" >> $GITHUB_ENV
+          # echo "JULIA_DEPOT_PATH=$CONDA/envs/rmg_env/.julia" >> $GITHUB_PATH
+          # echo "JULIA_CONDAPKG_EXE=$CONDA/condabin/mamba" >> $GITHUB_ENV
+          # echo "JULIA_CONDAPKG_EXE=$CONDA/condabin/mamba" >> $GITHUB_PATH
+          # echo "JULIA_CONDAPKG_EXE=$CONDA/condabin/mamba" >> $GITHUB_ENV
+          # echo "JULIA_CONDAPKG_EXE=$CONDA/condabin/mamba" >> $GITHUB_PATH
+          echo "JULIA_CONDAPKG_BACKEND=Current" >> $GITHUB_ENV
+          # echo "JULIA_CONDAPKG_BACKEND=Current" >> $GITHUB_PATH
+
+      - name: Setup Juliaup
+        if: matrix.test_julia == 'yes'
+        uses: julia-actions/install-juliaup@v2
+        with:
+          channel: '1.9'
+
+      - name: Check Julia version
+        if: matrix.test_julia == 'yes'
+        run: julia --version
+
       # RMS installation and linking to Julia
       - name: Install and link Julia dependencies
+        if: matrix.test_julia == 'yes'
         timeout-minutes: 120 # this usually takes 20-45 minutes (or hangs for 6+ hours).
+        # JULIA_CONDAPKG_EXE points to the existing conda/mamba to avoid JuliaCall from installing their own. See https://juliapy.github.io/PythonCall.jl/stable/pythoncall/#If-you-already-have-a-Conda-environment.
         run: |
-          python -c "import julia; julia.install(); import diffeqpy; diffeqpy.install()"
-          julia -e 'using Pkg; Pkg.add(PackageSpec(name="ReactionMechanismSimulator",rev="for_rmg")); using ReactionMechanismSimulator'
+          mamba install conda-forge::pyjuliacall
+          julia -e 'using Pkg; Pkg.add(Pkg.PackageSpec(name="ReactionMechanismSimulator", url="https://github.com/hwpang/ReactionMechanismSimulator.jl.git", rev="fix_installation")); using ReactionMechanismSimulator'
 
       - name: Install Q2DTor
         run: echo "" | make q2dtor
@@ -192,7 +168,7 @@ jobs:
         run: |
           for regr_test in aromatics liquid_oxidation nitrogen oxidation sulfur superminimal RMS_constantVIdealGasReactor_superminimal RMS_CSTR_liquid_oxidation RMS_liquidSurface_ch4o2cat fragment RMS_constantVIdealGasReactor_fragment minimal_surface;
           do
-            if python-jl rmg.py test/regression/"$regr_test"/input.py; then
+            if python rmg.py test/regression/"$regr_test"/input.py; then
               echo "$regr_test" "Executed Successfully"
             else
               echo "$regr_test" "Failed to Execute" | tee -a $GITHUB_STEP_SUMMARY
@@ -282,7 +258,7 @@ jobs:
 
             echo "<details>"
             # Compare the edge and core
-            if python-jl scripts/checkModels.py \
+            if python scripts/checkModels.py \
                 "$regr_test-core" \
                 $REFERENCE/"$regr_test"/chemkin/chem_annotated.inp \
                 $REFERENCE/"$regr_test"/chemkin/species_dictionary.txt \
@@ -299,7 +275,7 @@ jobs:
             cat "$regr_test-core.log" || (echo "Dumping the whole log failed, please download it from GitHub actions. Here are the first 100 lines:" && head -n100 "$regr_test-core.log")
             echo "</details>"
             echo "<details>"
-            if python-jl scripts/checkModels.py \
+            if python scripts/checkModels.py \
                 "$regr_test-edge" \
                 $REFERENCE/"$regr_test"/chemkin/chem_edge_annotated.inp \
                 $REFERENCE/"$regr_test"/chemkin/species_edge_dictionary.txt \
@@ -320,7 +296,7 @@ jobs:
             if [ -f test/regression/"$regr_test"/regression_input.py ];
             then
               echo "<details>"
-              if python-jl rmgpy/tools/regression.py \
+              if python rmgpy/tools/regression.py \
                 test/regression/"$regr_test"/regression_input.py \
                 $REFERENCE/"$regr_test"/chemkin \
                 test/regression/"$regr_test"/chemkin
@@ -384,7 +360,7 @@ jobs:
     # who knows ¯\_(ツ)_/¯
     #
     # taken from https://github.com/docker/build-push-action
-    needs: build-and-test-linux
+    needs: build-and-test
     runs-on: ubuntu-latest
     if: github.ref == 'refs/heads/main' && github.repository == 'ReactionMechanismGenerator/RMG-Py'
     steps:

.github/workflows/docs.yml

@@ -26,13 +26,13 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Setup Mambaforge Python 3.7
-        uses: conda-incubator/setup-miniconda@v3
+      - name: Setup Mambaforge Python 3.9
+        uses: conda-incubator/setup-miniconda@v2
         with:
           environment-file: environment.yml
           miniforge-variant: Miniforge3
           miniforge-version: latest
-          python-version: 3.7
+          python-version: 3.9
           activate-environment: rmg_env
           use-mamba: true
 
@@ -60,12 +60,6 @@ jobs:
           make clean
           make
 
-      - name: Install and link Julia dependencies
-        timeout-minutes: 120 # this usually takes 20-45 minutes (or hangs for 6+ hours).
-        run: |
-          python -c "import julia; julia.install(); import diffeqpy; diffeqpy.install()"
-          julia -e 'using Pkg; Pkg.add(PackageSpec(name="ReactionMechanismSimulator",rev="for_rmg")); using ReactionMechanismSimulator'
-
       - name: Checkout gh-pages Branch
         uses: actions/checkout@v2
         with:

.gitignore

@@ -47,7 +47,7 @@ nbproject/*
 .vscode/*
 
 # Unit test files
-.coverage
+.coverage*
 testing/*
 htmlcov/*
 
@@ -103,6 +103,7 @@ test/rmgpy/test_data/temp_dir_for_testing/cantera/chem001.yaml
 rmgpy/test_data/copied_kinetic_lib/
 testing/qm/*
 test_log.txt
+rmgpy/tools/data/flux/flux/1/*.dot
 
 # example directory - save the inputs but not the outputs
 # cantera input files
@@ -117,3 +118,4 @@ examples/**/dictionary.txt
 examples/**/reactions.py
 examples/**/RMG_libraries
 examples/**/species
+examples/**/plots

Dockerfile

@@ -13,12 +13,12 @@ RUN ln -snf /bin/bash /bin/sh
 #  - libxrender1 required by RDKit
 RUN apt-get update && \
     apt-get install -y \
-    make \
-    gcc \
-    wget \
-    git \
-    g++ \
-    libxrender1 && \
+        make \
+        gcc \
+        wget \
+        git \
+        g++ \
+        libxrender1 && \
     apt-get autoremove -y && \
     apt-get clean -y
 
@@ -28,10 +28,6 @@ RUN wget "https://github.com/conda-forge/miniforge/releases/latest/download/Mini
     rm Miniforge3-Linux-x86_64.sh
 ENV PATH="$PATH:/miniforge/bin"
 
-# Set solver backend to mamba for speed
-RUN conda install -n base conda-libmamba-solver && \
-    conda config --set solver libmamba
-
 # Set Bash as the default shell for following commands
 SHELL ["/bin/bash", "-c"]
 
@@ -50,8 +46,7 @@ RUN git clone --single-branch --branch ${RMG_Py_Branch} --depth 1 https://github
 
 WORKDIR /rmg/RMG-Py
 # build the conda environment
-RUN conda env create --file environment.yml && \
-    conda clean --all --yes
+RUN conda env create --file environment.yml
 
 # This runs all subsequent commands inside the rmg_env conda environment
 #
@@ -60,6 +55,10 @@ RUN conda env create --file environment.yml && \
 # in a Dockerfile build script)
 SHELL ["conda", "run", "--no-capture-output", "-n", "rmg_env", "/bin/bash", "-c"]
 
+RUN conda install -c conda-forge julia=1.9.1 pyjulia>=0.6 && \
+    conda install -c rmg pyrms diffeqpy && \
+    conda clean --all --yes
+
 # Set environment variables as directed in the RMG installation instructions
 ENV RUNNER_CWD=/rmg
 ENV PYTHONPATH="$RUNNER_CWD/RMG-Py:$PYTHONPATH"
@@ -70,16 +69,15 @@ ENV PATH="$RUNNER_CWD/RMG-Py:$PATH"
 # setting this env variable fixes an issue with Julia precompilation on Windows
 ENV JULIA_CPU_TARGET="x86-64,haswell,skylake,broadwell,znver1,znver2,znver3,cascadelake,icelake-client,cooperlake,generic"
 RUN make && \
-    julia -e 'using Pkg; Pkg.add(PackageSpec(name="PyCall",rev="master")); Pkg.add(PackageSpec(name="ReactionMechanismSimulator",rev=ENV["rmsbranch"])); using ReactionMechanismSimulator' && \
-    python -c "import julia; julia.install(); import diffeqpy; diffeqpy.install()"
+    julia -e 'ENV["JULIA_CONDAPKG_BACKEND"] = "Current"; using Pkg; Pkg.add(Pkg.PackageSpec(name="ReactionMechanismSimulator", url="https://github.com/hwpang/ReactionMechanismSimulator.jl.git", rev="fix_installation")); using ReactionMechanismSimulator'
 
 # RMG-Py should now be installed and ready - trigger precompilation and test run
-RUN python-jl rmg.py examples/rmg/minimal/input.py
+RUN python rmg.py examples/rmg/minimal/input.py
 # delete the results, preserve input.py
 RUN mv examples/rmg/minimal/input.py . && \
     rm -rf examples/rmg/minimal/* && \
     mv input.py examples/rmg/minimal/
 
 # when running this image, open an interactive bash terminal inside the conda environment
-RUN echo "source activate rmg_env" > ~/.bashrc
+RUN echo "source activate rmg_env" >~/.bashrc
 ENTRYPOINT ["/bin/bash", "--login"]

Makefile

@@ -62,16 +62,16 @@ decython:
 	find . -name *.pyc -exec rm -f '{}' \;
 
 test-all:
-	python-jl -m pytest
+	python -m pytest
 
 test test-unittests:
-	python-jl -m pytest -m "not functional and not database"
+	python -m pytest -m "not functional and not database"
 
 test-functional:
-	python-jl -m pytest -m "functional"
+	python -m pytest -m "functional"
 
 test-database:
-	python-jl -m pytest -m "database"
+	python -m pytest -m "database"
 
 eg0: all
 	mkdir -p testing/eg0