Skip to content

ci: add pixi to CI workflow #177

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 17 commits into from
Jun 11, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 19 additions & 19 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
@@ -19,10 +19,13 @@ jobs:
pymakeCI-os-intel:
name: pymake CI intel on different OSs
runs-on: ${{ matrix.os }}
env:
FC: intel-classic
FC_V: "2021.7"
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-2019]
os: [ubuntu-latest, macos-13, windows-2019]
defaults:
run:
shell: bash
@@ -31,53 +34,50 @@ jobs:
- name: Checkout repo
uses: actions/checkout@v4

- name: Setup Intel Fortran Classic
- name: Setup ${{ env.FC }} ${{ env.FC_V }}
uses: fortran-lang/setup-fortran@v1
with:
compiler: intel-classic
version: "2021.7"
compiler: ${{ env.FC }}
version: ${{ env.FC_V }}

- name: Setup Graphviz
if: runner.os == 'Linux'
uses: ts-graphviz/setup-graphviz@v2

- name: Set up Python
uses: actions/setup-python@v5
- name: Setup pixi
uses: prefix-dev/setup-pixi@v0.8.1
with:
python-version: "3.12"
pixi-version: v0.19.1
manifest-path: "pixi.toml"

- name: Install python packages
- name: pixi post-install
working-directory: pymake
run: |
python -m pip install --upgrade pip
pip install ".[test]"
pixi run postinstall

- name: Download examples for pytest runs
run: |
.github/common/download-examples.sh
pixi run download-examples

- name: test on Linux
if: runner.os == 'Linux'
working-directory: ./autotest
run: |
pytest -v -n=auto --dist=loadfile -m="base or regression" --durations=0 --cov=pymake --cov-report=xml
pixi run autotest

- name: test on MacOS
if: runner.os == 'macOS'
working-directory: ./autotest
run: |
pytest -v -n=auto --dist=loadfile -m="base" --durations=0 --cov=pymake --cov-report=xml
pixi run autotest-base

- name: test on Windows
if: runner.os == 'Windows'
working-directory: ./autotest
shell: cmd
run: |
pytest -v -m="base" --durations=0 --cov=pymake --cov-report=xml --basetemp=pytest_temp
pixi run autotest-Windows

- name: Print coverage report before upload
working-directory: ./autotest
run: |
coverage report
pixi run coverage-report

- name: Upload coverage to Codecov
uses: codecov/codecov-action@v4
51 changes: 25 additions & 26 deletions .github/workflows/pymake-gcc.yml
Original file line number Diff line number Diff line change
@@ -24,17 +24,12 @@ jobs:
matrix:
include:
# test latest gcc and python
- {os: ubuntu-latest, gcc: 13, python: "3.12"}
- {os: windows-latest, gcc: 13, python: "3.12"}
- {os: macos-latest, gcc: 13, python: "3.12"}
# test latest gcc and previous python
- {os: ubuntu-latest, gcc: 13, python: "3.11"}
- {os: ubuntu-latest, gcc: 13, python: "3.10"}
- {os: ubuntu-latest, gcc: 13, python: 3.9}
- {os: ubuntu-latest, gcc: 13, python: 3.8}
- {os: ubuntu-latest, gcc: 13}
- {os: windows-latest, gcc: 13}
- {os: macos-latest, gcc: 13}
# test latest python and previous gcc
- {os: ubuntu-latest, gcc: 12, python: "3.12"}
- {os: ubuntu-latest, gcc: 11, python: "3.12"}
- {os: ubuntu-latest, gcc: 12}
- {os: ubuntu-latest, gcc: 11}
defaults:
run:
shell: bash
@@ -47,15 +42,16 @@ jobs:
if: runner.os == 'Linux'
uses: ts-graphviz/setup-graphviz@v2

- name: Setup Python
uses: actions/setup-python@v5
- name: Setup pixi
uses: prefix-dev/setup-pixi@v0.8.1
with:
python-version: ${{ matrix.python }}
pixi-version: v0.19.1
manifest-path: "pixi.toml"

- name: Install python packages
- name: pixi post-install
working-directory: pymake
run: |
python -m pip install --upgrade pip
pip install ".[test]"
pixi run postinstall

- name: Setup GNU Fortran
uses: fortran-lang/setup-fortran@main
@@ -64,24 +60,27 @@ jobs:
version: ${{ matrix.gcc }}

- name: Download examples for pytest runs
run: .github/common/download-examples.sh
run: |
pixi run download-examples

- name: Install make
if: runner.os == 'Windows'
run: choco install make

- name: Run pytest
working-directory: ./autotest
- name: Run pytest on Linux and MacOS
if: runner.os != 'Windows'
run: |
markers="base"
if [[ ${{ matrix.gcc }} == 13 ]]; then
markers="base or regression"
fi
pytest -v --dist=loadfile -n=auto --durations=0 --basetemp=pytest_temp --cov=pymake --cov-report=xml --cov-append -m "$markers"
pixi run autotest-base

- name: test on Windows
if: runner.os == 'Windows'
shell: pwsh
run: |
pixi run autotest-Windows

- name: Print coverage report before upload
working-directory: ./autotest
run: coverage report
run: |
pixi run coverage-report

- name: Upload coverage to Codecov
uses: codecov/codecov-action@v4
17 changes: 6 additions & 11 deletions .github/workflows/pymake-linting-install.yml
Original file line number Diff line number Diff line change
@@ -20,22 +20,17 @@ jobs:
- name: Checkout repo
uses: actions/checkout@v4

- name: Set up Python
uses: actions/setup-python@v5
- name: Setup pixi
uses: prefix-dev/setup-pixi@v0.8.1
with:
python-version: "3.12"

- name: Setup Graphviz
uses: ts-graphviz/setup-graphviz@v2

- name: Install packages
run: pip install ".[lint]"
pixi-version: v0.19.1
manifest-path: "pixi.toml"

- name: Lint
run: ruff check .
run: pixi run check-lint

- name: Check format
run: ruff format . --check
run: pixi run check-format

pymake_setup:
name: standard installation
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -24,6 +24,9 @@ var/
.installed.cfg
*.egg

# pixi
.pixi/

# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
80 changes: 0 additions & 80 deletions autotest/ci_setup.py
Original file line number Diff line number Diff line change
@@ -12,9 +12,6 @@
mf6_exdir = temp_pth / "mf6examples"
if mf6_exdir.is_dir():
shutil.rmtree(mf6_exdir)
gsflow_exdir = temp_pth / "gsflowexamples"
if gsflow_exdir.is_dir():
shutil.rmtree(gsflow_exdir)


def download_mf6_examples(verbose=False):
@@ -71,83 +68,6 @@ def examples_list(verbose=False):
return


def download_gsflow_examples(verbose=False):
"""Download gsflow examples and return location of folder"""

examples_dirs = (
"sagehen",
"acfb_dyn_params",
"acfb_water_use",
)

target = "gsflow"
pm = pymake.Pymake(verbose=True)
pm.target = target

# download the gsflow release
pm.download_target(target, download_path=temp_pth)
assert pm.download, f"could not download {target} distribution"

# get program dictionary
prog_dict = pymake.usgs_program_data.get_target(target)

# set path to example
temp_download_dir = temp_pth / prog_dict.dirname
temp_dir = temp_download_dir / "data"

print(f"copying files to...{gsflow_exdir}")
shutil.copytree(temp_dir, gsflow_exdir)

# create list of examples to test and edit files if necessary
src_folders = []
for ex_dir in examples_dirs:
out_path = gsflow_exdir / ex_dir

if "sagehen" in ex_dir:
out_path = out_path / "linux"
modify_gsflow_sagehen(out_path)
elif "acfb" in ex_dir:
shutil.copy(
out_path / "control/control", out_path / "gsflow.control"
)

# add final example path to src_folders list
src_folders.append(out_path)

print(f"removing...{temp_download_dir} directory")
shutil.rmtree(temp_download_dir)

# create a list of gsflow examples
fpth = gsflow_exdir / "gsflowexamples.txt"
f = open(fpth, "w")
for idx, folder in enumerate(src_folders):
if verbose:
if idx == 0:
print(f"\n\nGSFLOW examples:\n{78 * '-'}")
print(f"{idx + 1:>3d}: {folder}")
f.write(f"{os.path.abspath(folder)}\n")
f.close()

return gsflow_exdir.resolve()


def modify_gsflow_sagehen(temp_pth):
fpth = temp_pth / "gsflow.control"
with open(fpth) as f:
lines = f.readlines()
with open(fpth, "w") as f:
idx = 0
while idx < len(lines):
line = lines[idx]
if "end_time" in line:
line += "6\n1\n1981\n"
idx += 3
f.write(line)
idx += 1
return


if __name__ == "__main__":
mf6pth = download_mf6_examples(verbose=True)
examples_list(verbose=True)
gsflowpth = download_gsflow_examples(verbose=True)
10 changes: 4 additions & 6 deletions autotest/test_build.py
Original file line number Diff line number Diff line change
@@ -69,8 +69,7 @@ def test_build(function_tmpdir, target: str) -> None:
pm = pymake.Pymake(verbose=True)
pm.target = target
pm.inplace = True
if system() == "Darwin":
pm.syslibs = "-Wl,-ld_classic"
fc = os.environ.get("FC", "gfortran")
assert (
pymake.build_apps(
target,
@@ -87,8 +86,9 @@ def test_build(function_tmpdir, target: str) -> None:
@pytest.mark.parametrize("target", targets_meson)
def test_meson_build(function_tmpdir, target: str) -> None:
kwargs = {}
if system() == "Darwin":
kwargs["LDFLAGS"] = "-Wl,-ld_classic"
cc = os.environ.get("CC", "gcc")
fc = os.environ.get("FC", "gfortran")
pymake.linker_update_environment(cc=cc, fc=fc)
with set_dir(function_tmpdir), set_env(**kwargs):
assert (
pymake.build_apps(
@@ -113,8 +113,6 @@ def test_makefile_build(function_tmpdir, target: str) -> None:
pm.inplace = True
pm.dryrun = True
pm.makeclean = False
if system() == "Darwin":
pm.syslibs = "-Wl,-ld_classic"

with set_dir(function_tmpdir):
pm.download_target(target)
14 changes: 11 additions & 3 deletions autotest/test_cli_cmds.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
import os
import pathlib as pl
import subprocess
from platform import system

import pytest
from flaky import flaky
from modflow_devtools.misc import set_dir
from modflow_devtools.misc import set_dir, set_env

from pymake import linker_update_environment

RERUNS = 3

@@ -106,10 +109,15 @@ def test_mfpymake(function_tmpdir, meson: bool) -> None:
"--verbose",
"-fc",
]
fc = "gfortran"
if os.environ.get("FC") is None:
cmd.append("gfortran")
cmd.append(fc)
else:
cmd.append(os.environ.get("FC"))
fc = os.environ.get("FC")
cmd.append(fc)

linker_update_environment(fc=fc)

if meson:
cmd.append("--meson")
run_cli_cmd(cmd)
4 changes: 1 addition & 3 deletions autotest/test_gridgen.py
Original file line number Diff line number Diff line change
@@ -31,9 +31,7 @@ def pm(module_tmpdir, target) -> pymake.Pymake:
pm.target = str(target)
pm.appdir = str(module_tmpdir)
pm.cc = environ.get("CXX", "g++")
pm.fc = environ.get("FC", "gfortran")
if system() == "Darwin":
pm.syslibs = "-Wl,-ld_classic"
pm.fc = None
pm.inplace = True
pm.makeclean = True
yield pm
14 changes: 7 additions & 7 deletions autotest/test_mf6_existing_meson.py
Original file line number Diff line number Diff line change
@@ -1,24 +1,21 @@
import os
import sys
from pathlib import Path
from platform import system
from typing import List

import pytest
from modflow_devtools.ostags import get_binary_suffixes

import pymake
from pymake import linker_update_environment


@pytest.fixture(scope="module")
def targets() -> List[Path]:
target = "mf6"
ext = ""
shared_ext = ".so"
ext, shared_ext = get_binary_suffixes()
executables = [target, "zbud6", "mf5to6", "libmf6"]
if sys.platform.lower() == "win32":
ext = ".exe"
shared_ext = ".dll"
elif sys.platform.lower() == "darwin":
shared_ext = ".dylib"
for idx, _ in enumerate(executables[:3]):
executables[idx] += ext
executables[3] += shared_ext
@@ -40,6 +37,7 @@ def pm(workspace, targets) -> pymake.Pymake:
pm = pymake.Pymake(verbose=True)
pm.target = str(targets[0])
pm.appdir = str(workspace / "bin")
pm.fc = os.environ.get("FC", "gfortran")
pm.meson = True
pm.makeclean = True
pm.mesondir = str(workspace)
@@ -83,6 +81,8 @@ def test_build_with_existing_meson(pm, module_tmpdir, workspace, targets):
pm.download_target(targets[0], download_path=module_tmpdir)
assert pm.download, f"could not download {targets[0]} distribution"

linker_update_environment(cc=cc, fc=fc)

# make modflow 6 with existing meson.build file
returncode = pymake.meson_build(
workspace,
10,491 changes: 10,491 additions & 0 deletions pixi.lock

Large diffs are not rendered by default.

51 changes: 51 additions & 0 deletions pixi.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
[project]
name = "pymake"
channels = ["conda-forge"]
platforms = ["win-64", "linux-64", "linux-aarch64", "osx-arm64", "osx-64"]
version = "1.2.10.dev0"

[dependencies]
appdirs = "*"
codespell = "*"
coverage = "*"
filelock = "*"
flaky = "*"
flopy = "*"
matplotlib = "*"
meson = "1.3.0"
modflow-devtools = "*"
networkx = "*"
ninja = "*"
numpy = "*"
pip = "*"
pydotplus = "*"
pytest = "!=8.1.0"
pytest-benchmark = "*"
pytest-cov = "*"
pytest-dotenv = "*"
pytest-order = "*"
pytest-xdist = "*"
python = "3.9.*"
requests = "*"
ruff = "*"

[tasks]
postinstall = "pip install --no-build-isolation --no-deps --disable-pip-version-check -e ."

# format
check-lint = "ruff check ."
fix-lint = "ruff check . --fix"
check-format = "ruff format . --check"
fix-format = "ruff format ."

# build
test = "meson test --verbose --no-rebuild -C"

# test
download-examples = {cmd = "python ci_setup.py", cwd = "autotest"}
autotest = { cmd = "pytest -v -n auto --dist=loadfile -m='base or regression' --durations 0 --cov=pymake --cov-report=xml --keep-failed .failed", cwd = "autotest" }
autotest-base = { cmd = "pytest -v -n auto --dist=loadfile -m='base' --durations 0 --cov=pymake --cov-report=xml --keep-failed .failed", cwd = "autotest" }
autotest-Windows = { cmd = "pytest -v -m='base' --durations 0 --cov=pymake --cov-report=xml --basetemp=$RUNNER_TEMP/pytest_temp --keep-failed .failed", cwd = "autotest" }

# coverage report
coverage-report = { cmd = "coverage report", cwd = "autotest"}
1 change: 1 addition & 0 deletions pymake/__init__.py
Original file line number Diff line number Diff line change
@@ -21,6 +21,7 @@
from .pymake_base import get_temporary_directories, main
from .pymake_build_apps import build_apps
from .pymake_parser import parser
from .utils._compiler_switches import linker_update_environment
from .utils._meson_build import meson_build, meson_install, meson_setup
from .utils.download import (
download_and_unzip,
74 changes: 72 additions & 2 deletions pymake/utils/_compiler_switches.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
"""Private functions for setting c/c++ and fortran compiler flags and
appropriate linker flags for defined targets.
"""Public and private functions for setting c/c++ and fortran compiler
flags and appropriate linker flags for defined targets.
"""

import os
import sys
from subprocess import check_output

from ._compiler_language_files import (
_get_c_files,
@@ -17,6 +18,28 @@
)


def linker_update_environment(cc="gcc", fc="gfortran", verbose=False):
"""Add additional LDFLAGS to the environment based on OS and compilers.
Parameters
----------
fc : str
fortran compiler
cc : str
c or cpp compiler
verbose : bool
boolean for verbose output to terminal
Returns
-------
None
"""
syslibs = _darwin_syslibs(cc, fc, verbose=verbose)
if syslibs is not None:
environ = os.environ
environ.update({"LDFLAGS": syslibs})


def _check_gnu_switch_available(switch, compiler="gfortran", verbose=False):
"""Determine if a specified GNU compiler switch exists. Not all switches
will be detected, for example '-O2' adn '-fbounds-check=on'.
@@ -971,6 +994,10 @@ def _set_syslibs(
if default_syslibs:
syslibs.append("-lc")

darwin_options = _darwin_syslibs(cc, fc, verbose)
if darwin_options is not None:
syslibs.append(darwin_options)

# add additional syslibs for select programs
if target == "triangle":
if osname in ("linux", "darwin"):
@@ -1033,3 +1060,46 @@ def _get_os_macro(osname=None):
else:
os_macro = None
return os_macro


def _darwin_syslibs(cc, fc, verbose=False):
"""Get additional syslibs for Darwin systems using gcc
tool chain and command line tools greater than 14
Parameters
----------
cc : str
c compiler
fc : str
fortran compiler
verbose : bool
boolean for verbose output to terminal
Returns
-------
linker_str : str
additiona; linker flags for darwin systems. Default is None
"""
linker_str = None
if _get_osname() == "darwin":
if cc in (
"gcc",
"g++",
) or fc in ("gfortran",):
cmd = ["pkgutil", "--pkg-info=com.apple.pkg.CLTools_Executables"]
out_lines = check_output(cmd).decode("utf-8").splitlines()
version = None
tag = "version: "
for line in out_lines:
if line.startswith(tag):
version = line.replace(tag, "")
break
if version is not None:
major = int(version.split(".")[0])
if major > 14:
linker_str = "-Wl,-ld_classic"
if verbose:
print(f"C/C++ compiler: {cc} \nFortran compiler: {fc}")
print(f"Command line tools version: {version}")
print(f"Command line tools major version number: {major}")
return linker_str
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -33,7 +33,7 @@ requires-python = ">=3.8"
dependencies = [
"numpy",
"requests",
"networkx >= 2.6.3",
"networkx",
"meson",
"ninja",
"pydotplus",