From dd71314ec74d4fe255d3301e46ad7568a0ca7ea3 Mon Sep 17 00:00:00 2001 From: Luis Fabregas <48292540+luisfabib@users.noreply.github.com> Date: Fri, 2 Jul 2021 22:58:03 +0200 Subject: [PATCH] Patch v0.13.2 (#191) * Enable code analysis by GitHub's CodeQL (#186) * examples: fix rendering of RST code-blocks in notebooks (#179) (#184) * Fix call to pipwin executable during PyPI installation (#187) * `snlls`: Fix violation of boundaries due to float-point round-off errors (#188) * docs: fix automated example plots in the models reference (#190) * VERSION: bump to v0.13.2 * Implement automated PyPI and Anaconda build and publish workflow (#185) * Update CHANGELOG for v0.13.2 --- .../conda_build_publish_package/Dockerfile | 11 ++++ .../conda_build_publish_package/README.md | 63 +++++++++++++++++++ .../conda_build_publish_package/action.yml | 21 +++++++ .../conda_build_publish_package/entrypoint.sh | 50 +++++++++++++++ .github/workflows/codeql-analysis.yml | 57 +++++++++++++++++ .github/workflows/package_upload.yml | 57 +++++++++++++++++ CHANGELOG.md | 11 ++++ VERSION | 2 +- conda.recipe/build.sh | 1 + conda.recipe/meta.yaml | 6 +- deerlab/dd_models.py | 3 +- deerlab/snlls.py | 10 +-- deerlab/utils/utils.py | 2 +- examples/basic/ex_bootstrapping.py | 12 ++-- examples/basic/ex_fitting_4pdeer.py | 12 ++-- examples/basic/ex_fitting_4pdeer_2p1.py | 12 ++-- examples/basic/ex_fitting_5pdeer.py | 12 ++-- examples/basic/ex_global_different_deer.py | 14 ++--- examples/basic/ex_global_fitting_4pdeer.py | 14 ++--- examples/basic/ex_restraints_4pdeer.py | 10 +-- setup.py | 8 +-- 21 files changed, 329 insertions(+), 59 deletions(-) create mode 100644 .github/actions/conda_build_publish_package/Dockerfile create mode 100644 .github/actions/conda_build_publish_package/README.md create mode 100644 .github/actions/conda_build_publish_package/action.yml create mode 100644 .github/actions/conda_build_publish_package/entrypoint.sh create mode 100644 .github/workflows/codeql-analysis.yml create mode 100644 .github/workflows/package_upload.yml create mode 100644 conda.recipe/build.sh diff --git a/.github/actions/conda_build_publish_package/Dockerfile b/.github/actions/conda_build_publish_package/Dockerfile new file mode 100644 index 000000000..1acbcfd82 --- /dev/null +++ b/.github/actions/conda_build_publish_package/Dockerfile @@ -0,0 +1,11 @@ +FROM continuumio/miniconda3:4.7.10 + +LABEL "repository"="https://github.com/m0nhawk/conda-package-publish-action" +LABEL "maintainer"="Andrew Prokhorenkov " + +RUN conda install -y anaconda-client conda-build + + +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] diff --git a/.github/actions/conda_build_publish_package/README.md b/.github/actions/conda_build_publish_package/README.md new file mode 100644 index 000000000..da267334c --- /dev/null +++ b/.github/actions/conda_build_publish_package/README.md @@ -0,0 +1,63 @@ +# Build and Publish Anaconda Package + +A Github Action to publish your software package to an Anaconda repository. + +### Example workflow to publish to conda every time you make a new release + +```yaml +name: publish_conda + +on: + release: + types: [published] + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: publish-to-conda + uses: maxibor/conda-package-publish-action@v1.1 + with: + subDir: 'conda' + AnacondaToken: ${{ secrets.ANACONDA_TOKEN }} + platforms: 'win-64 osx-64 linux-64' + python: '3.6 3.8' +``` + +### Example project structure + +``` +. +├── LICENSE +├── README.md +├── myproject +│   ├── __init__.py +│   └── myproject.py +├── conda +│   ├── build.sh +│   └── meta.yaml +├── .github +│   └── workflows +│   └── publish_conda.yml +├── .gitignore +``` +### Inputs + +The action takes the following + +- `AnacondaToken` - Anaconda access Token (see below) + +- `subDir` - (Optional) Sub-directory with conda recipe. Default: `.` + +- `platforms` - (Optional) Platforms to build and publish. Default: `win-64 osx-64 linux-64`. + +- `python` - (Optional) Python versions to build and publish. Default: `3.8`. + +### ANACONDA_TOKEN + +1. Get an Anaconda token (with read and write API access) at `anaconda.org/USERNAME/settings/access` +2. Add it to the Secrets of the Github repository as `ANACONDA_TOKEN` + +### Build Channels +By Default, this Github Action will search for conda build dependancies (on top of the standard channels) in `conda-forge` and `bioconda` diff --git a/.github/actions/conda_build_publish_package/action.yml b/.github/actions/conda_build_publish_package/action.yml new file mode 100644 index 000000000..877c90c56 --- /dev/null +++ b/.github/actions/conda_build_publish_package/action.yml @@ -0,0 +1,21 @@ +name: 'Publish Conda package to Anaconda.org' +description: 'Build and Publish conda package to Anaconda' +author: 'Andrew Prokhorenkov, modified by Maxime Borry, modified by Luis Fabregas' +branding: + icon: 'package' + color: 'purple' +inputs: + subDir: + description: 'Sub-directory with conda recipe' + default: '.' + AnacondaToken: + description: 'Anaconda access Token' + platforms: + description: 'Platforms to build and publish [osx linux win]' + default: 'win-64 osx-64 linux-64' + python: + description: 'Python version to build and publish' + default: '3.8' +runs: + using: 'docker' + image: 'Dockerfile' diff --git a/.github/actions/conda_build_publish_package/entrypoint.sh b/.github/actions/conda_build_publish_package/entrypoint.sh new file mode 100644 index 000000000..c45b9c864 --- /dev/null +++ b/.github/actions/conda_build_publish_package/entrypoint.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +set -ex +set -o pipefail + +go_to_build_dir() { + if [ ! -z $INPUT_SUBDIR ]; then + cd $INPUT_SUBDIR + fi +} + +check_if_meta_yaml_file_exists() { + if [ ! -f meta.yaml ]; then + echo "meta.yaml must exist in the directory that is being packaged and published." + exit 1 + fi +} + +build_package(){ + + IFS=' ' read -ra PYTHON <<< "$INPUT_PYTHON" + IFS=' ' read -ra PLATFORMS <<< "$INPUT_PLATFORMS" + + for python in "${PYTHON[@]}"; do + conda build -c conda-forge -c bioconda --output-folder . --python $python . + done + for platform in "${PLATFORMS[@]}"; do + for filename in /$platform/*.tar.bz2; do + conda convert -p $platform linux-64/*.tar.bz2 + done + done +} + +upload_package(){ + + IFS=' ' read -ra PLATFORMS <<< "$INPUT_PLATFORMS" + + export ANACONDA_API_TOKEN=$INPUT_ANACONDATOKEN + + for platform in "${PLATFORMS[@]}"; do + for filename in ./"$platform"/*.tar.bz2; do + anaconda upload $filename + done + done +} + +go_to_build_dir +check_if_meta_yaml_file_exists +build_package +upload_package diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 000000000..84fb1b3ab --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,57 @@ +name: "CodeQL" + +on: + push: + branches: [ main, release/v0.13.0 ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ main ] + schedule: + - cron: '33 11 * * 0' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'python' ] + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/.github/workflows/package_upload.yml b/.github/workflows/package_upload.yml new file mode 100644 index 000000000..c29636cc5 --- /dev/null +++ b/.github/workflows/package_upload.yml @@ -0,0 +1,57 @@ +name: Build & Upload Python Package + +on: + release: + types: [published] + +jobs: + + pypi-build-n-publish: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v1 + - name: Set up Python 3.8 + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Install dependencies + run: | + apt-get install python3-pip -y + python -m pip install setuptools --user + python -m pip install build --user + python -m pip install twine --user + python -m pip install conda --user + - name: PyPI - Build and publish + run: | + python setup.py sdist + python -m twine upload dist/* + - name: Publish distribution to PyPI + uses: pypa/gh-action-pypi-publish@master + with: + password: ${{ secrets.PYPI_API_TOKEN }} + + conda-build-n-publish: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v1 + - name: Set up Python 3.8 + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Get DeerLab version + run: echo "DEERLAB_VERSION=$(cat VERSION)" >> $GITHUB_ENV + - name: Update version in Conda metadata + uses: jacobtomlinson/gha-find-replace@master + with: + find: "VERSION" + replace: ${{env.DEERLAB_VERSION}} + include: "conda.recipe/meta.yaml" + - name: Build & Publish to Anaconda + uses: ./.github/actions/conda_build_publish_package + with: + subdir: 'conda.recipe' + anacondatoken: ${{ secrets.ANACONDA_TOKEN }} + platforms: 'osx-64 linux-32 linux-64 win-32 win-64' + python: '3.6 3.7 3.8 3.9' \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e0a7c9c7..e9496f0ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,15 @@ +Release v0.13.2 - July 2021 +--------------------------------- + +#### Overall changes + +- Fixed an error appearing during installation in Windows systems. If during installation a ``python`` executable alias was not created, the call to the ``pipwin`` manager returned an error and the installation failed to download and install Numpy, SciPy and cvxopt. ([#187](https://github.com/JeschkeLab/DeerLab/pull/187)). +- Fixed the rendering of certain code-blocks in the documentation examples which were appearing as plain text ([#179](https://github.com/JeschkeLab/DeerLab/issues/179), [#184](https://github.com/JeschkeLab/DeerLab/pull/184)). +- Fixed the execution and rendering of the model examples in the documentation ([#189](https://github.com/JeschkeLab/DeerLab/issues/189), [#190](https://github.com/JeschkeLab/DeerLab/pull/190)). +- Fixed a bug in ``snlls`` where one of the linear least-squares solvers can return results that violate the boundary conditions due to float-point round-off errors ([#177](https://github.com/JeschkeLab/DeerLab/issues/177), [#188](https://github.com/JeschkeLab/DeerLab/pull/188)). + + Release v0.13.1 - May 2021 --------------------------------- diff --git a/VERSION b/VERSION index b561134c6..7f00a577f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.13.1 +v0.13.2 diff --git a/conda.recipe/build.sh b/conda.recipe/build.sh new file mode 100644 index 000000000..fec5047cc --- /dev/null +++ b/conda.recipe/build.sh @@ -0,0 +1 @@ +$PYTHON setup.py install # Python command to install the script. \ No newline at end of file diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 6e8789625..913b9c1e4 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,11 +1,9 @@ -{% set version = "v0.13.0" %} - package: name: deerlab - version: {{ version }} + version: VERSION source: - git_rev: {{ version }} + git_rev: VERSION git_url: https://github.com/JeschkeLab/DeerLab.git requirements: diff --git a/deerlab/dd_models.py b/deerlab/dd_models.py index 30257dacb..94d79b95d 100644 --- a/deerlab/dd_models.py +++ b/deerlab/dd_models.py @@ -59,8 +59,7 @@ def docstr_example(fcnstr): import numpy as np model = dl.{fcnstr} r = np.linspace(2,5,400) - info = model() - par0 = info['Start'] + par0 = model.start P = model(r,par0) plt.figure(figsize=[6,3]) plt.plot(r,P) diff --git a/deerlab/snlls.py b/deerlab/snlls.py index 8e018f212..f15246f43 100644 --- a/deerlab/snlls.py +++ b/deerlab/snlls.py @@ -293,6 +293,9 @@ def snlls(y, Amodel, par0, lb=None, ub=None, lbl=None, ubl=None, nnlsSolver='cvx elif nnlsSolver == 'cvx': linSolver = lambda AtA, Aty: cvxnnls(AtA, Aty, tol=lin_tol, maxiter=lin_maxiter) parseResult = lambda result: result + + # Ensure correct formatting and shield against float-point errors + validateResult = lambda result: np.maximum(lbl,np.minimum(ubl,np.atleast_1d(result))) # ---------------------------------------------------------- # Containers for alpha-update checks @@ -319,9 +322,9 @@ def linear_problem(A,optimize_alpha,alpha): # Solve the linear least-squares problem result = linSolver(AtA, Aty) - linfit = parseResult(result) - linfit = np.atleast_1d(linfit) - + result = parseResult(result) + linfit = validateResult(result) + return linfit, alpha #=========================================================================== @@ -566,4 +569,3 @@ def _plot(subsets,y,yfit,show): plt.close() return fig # =========================================================================================== - diff --git a/deerlab/utils/utils.py b/deerlab/utils/utils.py index b9272b6e8..d61476bc3 100644 --- a/deerlab/utils/utils.py +++ b/deerlab/utils/utils.py @@ -409,7 +409,7 @@ def Jacobian(fcn, x0, lb, ub): """ J = opt._numdiff.approx_derivative(fcn,x0,method='2-point',bounds=(lb,ub)) - J = np.atleast_2d(J) + J = np.atleast_2d(J) return J #=============================================================================== diff --git a/examples/basic/ex_bootstrapping.py b/examples/basic/ex_bootstrapping.py index 406ab5484..80c25eb9d 100644 --- a/examples/basic/ex_bootstrapping.py +++ b/examples/basic/ex_bootstrapping.py @@ -11,12 +11,12 @@ import deerlab as dl # %% [markdown] -# Uncomment and use the following lines if you have experimental data: -# -# t,Vexp = dl.deerload('my\path\4pdeer_data.DTA') -# Vexp = dl.correctphase(Vexp) -# t = dl.correctzerotime(Vexp,t) -# +# Uncomment and use the following lines if you have experimental data:: +# +# t,Vexp = dl.deerload('my\path\4pdeer_data.DTA') +# Vexp = dl.correctphase(Vexp) +# t = dl.correctzerotime(Vexp,t) +# # %% [markdown] # In this example we will use simulated data instead: diff --git a/examples/basic/ex_fitting_4pdeer.py b/examples/basic/ex_fitting_4pdeer.py index b12a58823..f4aeac97a 100644 --- a/examples/basic/ex_fitting_4pdeer.py +++ b/examples/basic/ex_fitting_4pdeer.py @@ -15,12 +15,12 @@ # Load and pre-process data # --------------------------- # -# Uncomment and use the following lines if you have experimental data: -# -# t, Vexp = dl.deerload('my\path\4pdeer_data.DTA') -# Vexp = dl.correctphase(Vexp) -# t = dl.correctzerotime(Vexp,t) -# +# Uncomment and use the following lines if you have experimental data:: +# +# t, Vexp = dl.deerload('my\path\4pdeer_data.DTA') +# Vexp = dl.correctphase(Vexp) +# t = dl.correctzerotime(Vexp,t) +# # In this example we will use simulated data instead. # %% [markdown] diff --git a/examples/basic/ex_fitting_4pdeer_2p1.py b/examples/basic/ex_fitting_4pdeer_2p1.py index a0edcf6dd..7f948091b 100644 --- a/examples/basic/ex_fitting_4pdeer_2p1.py +++ b/examples/basic/ex_fitting_4pdeer_2p1.py @@ -15,12 +15,12 @@ # Load and pre-process data # --------------------------- # -# Uncomment and use the following lines if you have experimental data: -# -# t,Vexp = dl.deerload('my\path\4pdeer_data.DTA') -# Vexp = dl.correctphase(Vexp) -# t = dl.correctzerotime(Vexp,t) -# +# Uncomment and use the following lines if you have experimental data:: +# +# t,Vexp = dl.deerload('my\path\4pdeer_data.DTA') +# Vexp = dl.correctphase(Vexp) +# t = dl.correctzerotime(Vexp,t) +# # In this example we will use simulated data instead: # %% diff --git a/examples/basic/ex_fitting_5pdeer.py b/examples/basic/ex_fitting_5pdeer.py index db319b91c..3c3d59da1 100644 --- a/examples/basic/ex_fitting_5pdeer.py +++ b/examples/basic/ex_fitting_5pdeer.py @@ -15,12 +15,12 @@ # Load and pre-process data # --------------------------- # -# Uncomment and use the following lines if you have experimental data: -# -# t,Vexp = dl.deerload('my\path\5pdeer_data.DTA') -# Vexp = dl.correctphase(Vexp) -# t = dl.correctzerotime(Vexp,t) -# +# Uncomment and use the following lines if you have experimental data:: +# +# t,Vexp = dl.deerload('my\path\5pdeer_data.DTA') +# Vexp = dl.correctphase(Vexp) +# t = dl.correctzerotime(Vexp,t) +# # In this example we will use simulated data instead. # %% diff --git a/examples/basic/ex_global_different_deer.py b/examples/basic/ex_global_different_deer.py index ab9cebfa6..dae179851 100644 --- a/examples/basic/ex_global_different_deer.py +++ b/examples/basic/ex_global_different_deer.py @@ -15,13 +15,13 @@ # Load and pre-process data # --------------------------- # -# Uncomment and use the following lines if you have experimental data: -# -# datasets = ('file1.DTA','file2.DTA','file3.DTA') -# data = [dl.deerload(ds) for ds in datasets] -# t = [_[0] for _ in data] -# V = [_[1] for _ in data] -# +# Uncomment and use the following lines if you have experimental data:: +# +# datasets = ('file1.DTA','file2.DTA','file3.DTA') +# data = [dl.deerload(ds) for ds in datasets] +# t = [_[0] for _ in data] +# V = [_[1] for _ in data] +# # %% [markdown] # Simulate data diff --git a/examples/basic/ex_global_fitting_4pdeer.py b/examples/basic/ex_global_fitting_4pdeer.py index 147b41eb9..5fb251c16 100644 --- a/examples/basic/ex_global_fitting_4pdeer.py +++ b/examples/basic/ex_global_fitting_4pdeer.py @@ -15,13 +15,13 @@ # Load and pre-process data # --------------------------- # -# Uncomment and use the following lines if you have experimental data: -# -# datasets = ('file1.DTA','file2.DTA','file3.DTA') -# data = [dl.deerload(ds) for ds in datasets] -# t = [_[0] for _ in data] -# V = [_[1] for _ in data] -# +# Uncomment and use the following lines if you have experimental data:: +# +# datasets = ('file1.DTA','file2.DTA','file3.DTA') +# data = [dl.deerload(ds) for ds in datasets] +# t = [_[0] for _ in data] +# V = [_[1] for _ in data] +# # %% [markdown] # Simulate data diff --git a/examples/basic/ex_restraints_4pdeer.py b/examples/basic/ex_restraints_4pdeer.py index 28a4abdc3..f7c3ec6e5 100644 --- a/examples/basic/ex_restraints_4pdeer.py +++ b/examples/basic/ex_restraints_4pdeer.py @@ -12,11 +12,11 @@ import deerlab as dl # %% [markdown] -# Uncomment and use the following lines if you have experimental data: -# -# t,Vexp = dl.deerload('my\path\4pdeer_data.DTA') -# Vexp = dl.correctphase(Vexp) -# t = dl.correctzerotime(Vexp,t) +# Uncomment and use the following lines if you have experimental data:: +# +# t,Vexp = dl.deerload('my\path\4pdeer_data.DTA') +# Vexp = dl.correctphase(Vexp) +# t = dl.correctzerotime(Vexp,t) # # %% [markdown]# diff --git a/setup.py b/setup.py index c4c650bf9..1c94800e4 100644 --- a/setup.py +++ b/setup.py @@ -43,11 +43,11 @@ def install_dependencies(develop_mode=False): # Download and install pre-compiled binaries for Windows-systems if platform.system() == 'Windows': # Refresh the pipwin cache to get the latest repo status - subprocess.run(['pipwin','refresh'],check=False) + subprocess.run([executable,'-m','pipwin','refresh'],check=False) # Install Numpy,SciPy, CVXopt linked to MKL from Gohlken's repository - subprocess.run(['pipwin','install','numpy','--filter=mkl'],check=False) - subprocess.run(['pipwin','install','scipy'],check=False) - subprocess.run(['pipwin','install','cvxopt'],check=False) + subprocess.run([executable,'-m','pipwin','install','numpy','--filter=mkl'],check=False) + subprocess.run([executable,'-m','pipwin','install','scipy'],check=False) + subprocess.run([executable,'-m','pipwin','install','cvxopt'],check=False) #----------------------------------------------------------------------- class install_routine(install):