move PyMPDATA-examples to examples; simplify dependency logic; execute notebooks from devops_tests (#400)

Author: slayoo

.binder/postBuild

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# mimicking what happens on Colab: packages are fetched from PyPI, only notebooks from the repo
+
+set -e
+shopt -s extglob
+rm -rfv !("examples")

.binder/requirements.txt

@@ -0,0 +1 @@
+PyMPDATA-examples

.github/workflows/pdoc_index_workaround.py

@@ -9,7 +9,7 @@
 
 modules = [
     import_module(module, reload=True)
-    for module in ('PyMPDATA', 'PyMPDATA-examples/PyMPDATA_examples')
+    for module in ('PyMPDATA', 'examples/PyMPDATA_examples')
 ]
 
 with open('html/index.html', 'w', encoding='utf-8') as index:

.github/workflows/tests+pypi.yml

@@ -18,7 +18,7 @@ jobs:
   precommit:
     strategy:
       matrix:
-        package-dir: [".", "PyMPDATA-examples"]
+        package-dir: [".", "examples"]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -46,8 +46,7 @@ jobs:
         python-version: 3.9
     - name: Generate coverage report
       run: |
-        pip install -e .[tests]
-        pip install -e ./PyMPDATA-examples
+        pip install -e .[tests] ./examples
         pip install pytest-cov
         pytest --durations=10 -We tests/unit_tests --cov-report=xml --cov=PyMPDATA
     - name: Upload coverage to Codecov
@@ -71,22 +70,17 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install pylint nbqa pdoc3 # (pdoc3 for checking the .github/workflows/pdoc_index_workaround.py)
-        pip install -e ./PyMPDATA-examples[tests]
-        pip install -e .[tests]
+        pip install -e .[tests] ./examples[tests]
     - run: |
-        pylint --max-module-lines=500 --disable=fixme $(git ls-files '*.py')
+        pylint --max-module-lines=500 --disable=fixme $(git ls-files '*.py' | grep -v ^examples)
     - run: |
         # TODO #299
-        cd PyMPDATA-examples
         pylint --disable=fixme --max-module-lines=200 \
-               --disable=too-many-locals,too-many-branches,duplicate-code,invalid-name,missing-function-docstring,missing-module-docstring,too-many-arguments,missing-class-docstring,too-many-instance-attributes $(git ls-files '*.py')
-        cd ..
+               --disable=too-many-locals,too-many-branches,duplicate-code,invalid-name,missing-function-docstring,missing-module-docstring,too-many-arguments,missing-class-docstring,too-many-instance-attributes $(git ls-files '*.py' | grep ^examples)
     - run: |
         # TODO #299
-        cd PyMPDATA-examples
-        nbqa pylint --disable=fixme,trailing-whitespace,wrong-import-position,ungrouped-imports \
+        nbqa pylint --disable=fixme,trailing-whitespace,wrong-import-position,wrong-import-order,ungrouped-imports \
                     --disable=duplicate-code,invalid-name,missing-module-docstring,line-too-long,missing-function-docstring,redefined-outer-name,too-many-arguments,no-member,too-many-locals,cell-var-from-loop,missing-class-docstring $(git ls-files '*.ipynb')
-        cd ..
   pdoc:
     strategy:
       matrix:
@@ -100,11 +94,12 @@ jobs:
       - uses: actions/setup-python@v2
         with:
           python-version: 3.9
-      - run: |
+      - env:
+          JUPYTER_PLATFORM_DIRS: 1
+        run: |
           pip3 install pdoc3
-          pip install -e .
-          pip install -e . PyMPDATA-examples
-          python -We -m pdoc --html PyMPDATA PyMPDATA-examples/PyMPDATA_examples
+          pip install -e . ./examples
+          python -We -m pdoc --html PyMPDATA examples/PyMPDATA_examples
           python -We .github/workflows/pdoc_index_workaround.py
       - if: ${{ github.ref == 'refs/heads/main' && matrix.platform == 'ubuntu-latest' }}
         uses: JamesIves/github-pages-deploy-action@4.1.1
@@ -145,8 +140,7 @@ jobs:
       - run: |
           python -We -c "import PyMPDATA"
       - run: |
-          pip install -e ./PyMPDATA-examples
-          pip install -e .[tests]
+          pip install -e .[tests] ./examples
       # https://github.com/numba/numba/issues/6350#issuecomment-728174860
       - if: startsWith(matrix.platform, 'ubuntu-')
         run: echo NUMBA_THREADING_LAYER=omp >> $GITHUB_ENV
@@ -161,7 +155,7 @@ jobs:
     strategy:
       matrix:
         platform: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ["3.7", "3.10"]
+        python-version: ["3.8", "3.10"]
       fail-fast: false
     runs-on: ${{ matrix.platform }}
     steps:
@@ -172,12 +166,8 @@ jobs:
       - uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
-      - run: |
-          pip install -e ./PyMPDATA-examples[tests]
-      - run: |
-          python -We -c "import PyMPDATA_examples"
-      - run: |
-          pip install -e .[tests]
+      - run: pip install -e .[tests] ./examples
+      - run: pip install -r tests/devops_tests/requirements.txt
       - if: matrix.platform == 'ubuntu-latest'
         run: |
           sudo apt-get install libblitz0-dev libboost-thread-dev libboost-date-time-dev libboost-system-dev libboost-iostreams-dev libboost-timer-dev libboost-filesystem-dev
@@ -194,9 +184,7 @@ jobs:
       - if: matrix.platform == 'ubuntu-latest'
         run: echo NUMBA_THREADING_LAYER=omp >> $GITHUB_ENV
 
-      - env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: pytest --durations=10 -p no:unraisableexception -We PyMPDATA-examples
+      - run: pytest --durations=10 -p no:unraisableexception -We tests/devops_tests/test_run_notebooks.py
 
   devops:
     runs-on: ubuntu-latest
@@ -206,14 +194,15 @@ jobs:
           submodules: recursive
           fetch-depth: 0  # https://github.com/pypa/setuptools_scm/issues/480
       - run: pip install -r tests/devops_tests/requirements.txt
+      - run: pip install -e . ./examples
       - env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: pytest --durations=10 -v -s -We -p no:unraisableexception tests/devops_tests
+        run: pytest --durations=10 -v -s -We -p no:unraisableexception -k "not run_notebooks" tests/devops_tests
 
   dist:
     strategy:
       matrix:
-        package-dir: [".", "PyMPDATA-examples"]
+        package-dir: [".", "examples"]
       fail-fast: false
     runs-on: ubuntu-latest
     needs: [tests, examples, devops]

.gitmodules

@@ -1,7 +1,4 @@
-[submodule "PyMPDATA-examples"]
-	path = PyMPDATA-examples
-	url = git@github.com:open-atmos/PyMPDATA-examples.git
-	shallow = true
 [submodule "tests/devops_tests"]
 	path = tests/devops_tests
-	url = git@github.com:open-atmos/devops_tests.git
+	url = https://github.com/open-atmos/devops_tests
+	shallow = true

PyMPDATA-examples

@@ -0,0 +1,62 @@
+import numpy as np
+import PyMPDATA_examples.Arabas_and_Farhat_2020.Black_Scholes_1973 as BS
+
+
+def _phi(
+    S: [np.ndarray, float],
+    gamma: float,
+    H: float,
+    I: float,
+    r: float,
+    b: float,
+    var: float,
+    T: float,
+):
+    lmbd = (-r + gamma * b + 0.5 * gamma * (gamma - 1) * var) * T
+    d = -(np.log(S / H) + (b + (gamma - 0.5) * var) * T) / np.sqrt(var * T)
+    kappa = 2 * b / var + (2 * gamma - 1)
+    return (
+        np.exp(lmbd)
+        * np.power(S, gamma)
+        * (
+            BS.N(d)
+            - pow((I / S), kappa) * BS.N(d - 2 * np.log(I / S) / np.sqrt(var * T))
+        )
+    )
+
+
+def c_amer(
+    S: [np.ndarray, float],
+    K: [float, np.ndarray],
+    T: float,
+    r: float,
+    b: float,
+    sgma: float,
+):
+    if b >= r:
+        return BS.c_euro(S, K=K, T=T, r=r, b=b, sgma=sgma)
+
+    var = sgma * sgma
+    beta = (0.5 - b / var) + np.sqrt(pow((b / var - 0.5), 2) + 2 * r / var)
+    BInf = beta / (beta - 1) * K
+    B0 = np.maximum(K, r / (r - b) * K)
+    ht = -(b * T + 2 * sgma * np.sqrt(T)) * B0 / (BInf - B0)
+    I = B0 + (BInf - B0) * (1 - np.exp(ht))
+    alpha = (I - K) * pow(I, -beta)
+
+    return np.where(
+        S >= I,
+        S - K,
+        alpha * np.power(S, beta)
+        + (
+            -alpha * _phi(S, gamma=beta, H=I, I=I, r=r, b=b, var=var, T=T)
+            + _phi(S, gamma=1, H=I, I=I, r=r, b=b, var=var, T=T)
+            - _phi(S, gamma=1, H=K, I=I, r=r, b=b, var=var, T=T)
+            - K * _phi(S, gamma=0, H=I, I=I, r=r, b=b, var=var, T=T)
+            + K * _phi(S, gamma=0, H=K, I=I, r=r, b=b, var=var, T=T)
+        ),
+    )
+
+
+def p_amer(S: [np.ndarray, float], K: float, T: float, r: float, b: float, sgma: float):
+    return c_amer(K, K=S, T=T, r=r - b, b=-b, sgma=sgma)

examples/PyMPDATA_examples/Arabas_and_Farhat_2020/Bjerksund_and_Stensland_1993.py

@@ -0,0 +1,18 @@
+import numpy as np
+from scipy.special import erf  # pylint: disable=no-name-in-module
+
+
+def N(x: float):
+    return (1 + erf(x / np.sqrt(2))) / 2
+
+
+def c_euro(S: np.ndarray, K: float, T: float, r: float, b: float, sgma: float):
+    d1 = (np.log(S / K) + (b + sgma * sgma / 2) * T) / sgma / np.sqrt(T)
+    d2 = d1 - sgma * np.sqrt(T)
+    return S * np.exp(b - r) * N(d1) - K * np.exp(-r * T) * N(d2)
+
+
+def p_euro(S: np.ndarray, K: float, T: float, r: float, b: float, sgma: float):
+    d1 = (np.log(S / K) + (b + sgma * sgma / 2) * T) / sgma / np.sqrt(T)
+    d2 = d1 - sgma * np.sqrt(T)
+    return K * np.exp(-r * T) * N(-d2) - S * np.exp((b - r) * T) * N(-d1)

examples/PyMPDATA_examples/Arabas_and_Farhat_2020/Black_Scholes_1973.py

@@ -0,0 +1 @@
+from .simulation import Simulation

examples/PyMPDATA_examples/Arabas_and_Farhat_2020/__init__.py

@@ -0,0 +1,76 @@
+import numpy as np
+from joblib import Parallel, delayed, parallel_backend
+from PyMPDATA_examples.Arabas_and_Farhat_2020.setup1_european_corridor import Settings
+from PyMPDATA_examples.Arabas_and_Farhat_2020.simulation import Simulation
+from PyMPDATA_examples.utils.error_norms import L2
+
+
+def compute(simulation):
+    output = []
+    for n_iters in (1, 2):
+        simulation.run(n_iters)
+        output.append(
+            {
+                "n_iters": n_iters,
+                "log2_C": np.log2(simulation.C),
+                "log2_C_opt": np.log2(simulation.settings.C_opt),
+                "log2_l2": np.log2(simulation.l2),
+                "log2_l2_opt": np.log2(simulation.settings.l2_opt),
+                "err2": error_L2_norm(
+                    simulation.solvers,
+                    simulation.settings,
+                    simulation.S,
+                    simulation.nt,
+                    n_iters,
+                ),
+            }
+        )
+    return output
+
+
+def convergence_in_space(num=8):
+    with parallel_backend("threading", n_jobs=-2):
+        data = Parallel(verbose=10)(
+            delayed(compute)(
+                Simulation(Settings(l2_opt=2**log2_l2_opt, C_opt=2**log2_C_opt))
+            )
+            for log2_C_opt in np.linspace(-9.5, -6, num=num)
+            for log2_l2_opt in range(1, 4)
+        )
+        result = {}
+        for pair in data:
+            for datum in pair:
+                label = f" $\\lambda^2\\approx2^{{{datum['log2_l2_opt']}}}$"
+                key = ("upwind" + label, "MPDATA" + label)[datum["n_iters"] - 1]
+                if key not in result:
+                    result[key] = ([], [])
+                result[key][0].append(datum["log2_C"])
+                result[key][1].append(datum["err2"])
+        return result
+
+
+def convergence_in_time(num=13):
+    with parallel_backend("threading", n_jobs=-2):
+        data = Parallel(verbose=10)(
+            delayed(compute)(
+                Simulation(Settings(l2_opt=2**log2_l2_opt, C_opt=2**log2_C_opt))
+            )
+            for log2_C_opt in np.log2((0.01, 0.005, 0.0025))
+            for log2_l2_opt in np.linspace(1.1, 3.5, num=num)
+        )
+        result = {}
+        for pair in data:
+            for datum in pair:
+                label = f" $C\\approx{2**(datum['log2_C_opt']):.4f}$"
+                key = ("upwind" + label, "MPDATA" + label)[datum["n_iters"] - 1]
+                if key not in result:
+                    result[key] = ([], [])
+                result[key][0].append(datum["log2_l2"])
+                result[key][1].append(datum["err2"])
+        return result
+
+
+def error_L2_norm(solvers, settings, S, nt, n_iters: int):
+    numerical = solvers[n_iters].advectee.get()
+    analytical = settings.analytical_solution(S)
+    return L2(numerical, analytical, nt)

examples/PyMPDATA_examples/Arabas_and_Farhat_2020/analysis_figures_2_and_3.py

@@ -0,0 +1,48 @@
+import numpy as np
+from joblib import Parallel, delayed, parallel_backend
+from PyMPDATA_examples.Arabas_and_Farhat_2020.analysis_figures_2_and_3 import (
+    error_L2_norm,
+)
+from PyMPDATA_examples.Arabas_and_Farhat_2020.setup2_american_put import Settings
+from PyMPDATA_examples.Arabas_and_Farhat_2020.simulation import Simulation
+
+
+def compute_row(simulations):
+    S0 = simulations[0].settings.S0
+    T = simulations[0].settings.T
+    for i in range(1, len(simulations)):
+        assert simulations[i].settings.T == T
+        assert simulations[i].settings.S0 == S0
+    row = [T, S0]
+    f = None
+    for simulation in simulations:
+        f = simulation.run(n_iters=2)
+        row.append(
+            error_L2_norm(
+                simulation.solvers,
+                simulation.settings,
+                simulation.S,
+                simulation.nt,
+                n_iters=2,
+            )
+        )
+        np.testing.assert_almost_equal(simulation.S[simulation.ix_match], S0)
+    row.append(f[simulations[-1].ix_match])
+    row.append(simulations[0].settings.analytical_solution(S0))
+    row.append(simulations[0].settings.analytical_solution(S0, amer=False))
+    return row
+
+
+def table_1_data():
+    with parallel_backend("threading", n_jobs=-2):
+        result = Parallel(verbose=10)(
+            delayed(compute_row)(
+                tuple(
+                    Simulation(Settings(T=T, C_opt=C_opt, S0=S0))
+                    for C_opt in (0.02, 0.01, 0.005)
+                )
+            )
+            for T in (0.25, 0.5, 3)
+            for S0 in (80, 90, 100, 110, 120)
+        )
+        return result

examples/PyMPDATA_examples/Arabas_and_Farhat_2020/analysis_table_1.py

@@ -0,0 +1 @@
+colors = ("purple", "teal", "turquoise")