diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a0e3b057..09f73441 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,7 +9,7 @@ jobs: strategy: matrix: - python: ["3.11", "3.10", "3.9", "3.8"] + python: ["3.12", "3.11", "3.10", "3.9"] steps: - uses: actions/checkout@v1 @@ -19,11 +19,9 @@ jobs: - name: Run Tests (Linux) run: | - python -m pip install --upgrade pip setuptools wheel - python -m pip install pytest-cov numpy "Cython<3.0.4" - python setup.py build_ext -i - python -m pip install . - pytest --cov-report html --cov-report xml --cov-report annotate --cov=pixell pixell/tests/ -s + python -m pip install --upgrade pip setuptools wheel meson ninja meson-python cython numpy + python -m pip install --no-build-isolation --editable '.[test]' + pytest --cov --cov-report html --cov-report xml --cov-report annotate -s - uses: codecov/codecov-action@v2 with: @@ -44,7 +42,7 @@ jobs: - uses: actions/checkout@v1 - uses: actions/setup-python@v1 with: - python-version: "3.9" + python-version: "3.10" - name: Install Dependencies (MacOS) run: | @@ -58,141 +56,58 @@ jobs: run: | ln -s $FC $(dirname $(which $FC))/gfortran echo "Using FC=$FC CXX=$CXX CC=$CC" - python setup.py build_ext -i python -m pip install . - name: Run Tests (MacOS) run: | - pytest pixell/tests/ -s - + pytest -s - build_wheels_linux: - name: Build wheels on Linux - runs-on: ubuntu-latest + build_wheels: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} strategy: - # Ensure that a wheel builder finishes even if another fails - fail-fast: false matrix: - os: [ubuntu-20.04] - cp: [cp38, cp39, cp310, cp311] - include: - - cp: cp38 - numpyver: "1.20" - - cp: cp39 - numpyver: "1.20" - - cp: cp310 - numpyver: "1.22" - - cp: cp311 - numpyver: "1.22" + # macos-13 is an intel runner, macos-14 is apple silicon + os: [ubuntu-latest, macos-13] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - - uses: actions/setup-python@v2 - name: Install Python - with: - python-version: '3.8' - - - name: Install cibuildwheel and other dependencies - run: | - python -m pip install -U pip - python -m pip install -U setuptools - python -m pip install --upgrade pip setuptools wheel - python -m pip install pytest-cov numpy "Cython<3.0.4" - python -m pip install cibuildwheel - - - name: Test and build wheels - run: | - ln -s $FC $(dirname $(which $FC))/gfortran - bash scripts/build_wheels.sh + - name: Build wheels + uses: pypa/cibuildwheel@v2.19.1 env: - CIBW_ENVIRONMENT: OMP_NUM_THREADS=2 - OPENBLAS_NUM_THREADS=2 - CIBW_BEFORE_BUILD: "python -m pip install numpy==${{ matrix.numpyver }} scipy 'cython<3.0.4'" - CIBW_BUILD_VERBOSITY: 3 - # If users are still on 32 bit systems or PPC they should build from source. - CIBW_BUILD: "${{ matrix.cp }}-manylinux_{x86_64,aarch64}" - CIBW_TEST_REQUIRES: pytest - CIBW_TEST_COMMAND: > - cd / - pytest {project}/pixell/tests -s - PYTHON: "python" - PIP: "pip" + FC: gfortran-12 + CC: gcc-12 + MACOSX_DEPLOYMENT_TARGET: "13.0" - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: + name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} path: ./wheelhouse/*.whl - build_wheels_macos: - name: Build wheels on MacOS - runs-on: macos-12 - env: - CC: gcc-12 - CXX: gcc-12 - FC: gfortran-12 - DUCC0_NUM_THREADS: 2 + build_wheels_macos_arm: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} strategy: - # Ensure that a wheel builder finishes even if another fails - fail-fast: false matrix: - os: [macos-12] - # We don't build 3.7 wheels for MacOS because that's x86 only. - cp: [cp38, cp39, cp310, cp311] - include: - - cp: cp38 - numpyver: "1.20" - - cp: cp39 - numpyver: "1.20" - - cp: cp310 - numpyver: "1.22" - - cp: cp311 - numpyver: "1.22" + # macos-13 is an intel runner, macos-14 is apple silicon + os: [macos-14] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - - uses: actions/setup-python@v2 - name: Install Python - with: - python-version: '3.9' - - - name: Install cibuildwheel and other dependencies - run: | - python -m pip install -U pip - python -m pip install -U setuptools - python -m pip install --upgrade pip setuptools wheel - python -m pip install pytest-cov numpy "Cython<3.0.4" - python -m pip install cibuildwheel - - - name: Test and build wheels - run: | - ln -s $FC $(dirname $(which $FC))/gfortran - bash scripts/build_wheels.sh + - name: Build wheels + uses: pypa/cibuildwheel@v2.19.1 env: - CIBW_ENVIRONMENT: OMP_NUM_THREADS=2 - OPENBLAS_NUM_THREADS=2 - CIBW_BEFORE_BUILD: "python -m pip install numpy==${{ matrix.numpyver }} scipy 'cython<3.0.4'" - CIBW_BUILD_VERBOSITY: 3 - # To build for both x86_64 and arm64, uncomment the following lines. - # That would require a fortran compiler that can cross-comile from - # x86_64 to arm64, as well as an OMP library that's cross-compiled. - # CIBW_BUILD: "${{ matrix.cp }}-macosx_{x86_64,arm64}" - CIBW_BUILD: "${{ matrix.cp }}-macosx_x86_64" - #CIBW_ARCHS_MACOS: "x86_64 arm64" - CIBW_ARCHS_MACOS: "x86_64" - CIBW_TEST_REQUIRES: pytest - CIBW_TEST_COMMAND: > - cd / - pytest {project}/pixell/tests -s - # Cannot test arm64 on non-native runner - CIBW_TEST_SKIP: "*_arm64" - PYTHON: "python" - PIP: "pip" + FC: gfortran-12 + CC: gcc-12 + MACOSX_DEPLOYMENT_TARGET: "14.0" - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: + name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} path: ./wheelhouse/*.whl - + build_sdist: name: Build source distribution @@ -203,7 +118,7 @@ jobs: - uses: actions/setup-python@v2 name: Install Python with: - python-version: '3.8' + python-version: '3.10' - name: Build sdist run: | @@ -212,14 +127,15 @@ jobs: python -m pip install --upgrade pip setuptools wheel python -m pip install "Cython<3.0.4" python -m pip install numpy - python setup.py sdist + python -m pip install build + python -m build . --sdist - uses: actions/upload-artifact@v2 with: path: dist/*.tar.gz upload_pypi: - needs: [build_wheels_linux, build_wheels_macos, build_sdist] + needs: [build_wheels, build_sdist, build_wheels_macos_arm] runs-on: ubuntu-latest # upload to PyPI on every tag starting with 'v' if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 87eebf02..3e0822da 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -7,7 +7,7 @@ version: 2 build: os: ubuntu-22.04 tools: - python: "3.7" + python: "3.9" # Build documentation in the docs/ directory with Sphinx sphinx: diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 97a6b68e..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1,24 +0,0 @@ -include AUTHORS.rst -include CONTRIBUTING.rst -include HISTORY.rst -include LICENSE -include README.rst -include requirements.txt -include optional-requirements.txt -include requirements_dev.txt - -recursive-include cython * -recursive-include fortran * -recursive-include pixell/tests * -recursive-include pixell/tests/data * -recursive-include scripts * -recursive-include * *.ttf -recursive-exclude cython sharp.c -recursive-exclude fortran interpol_*.f90 -recursive-exclude * temporary_map.fits -recursive-exclude * __pycache__ -recursive-exclude * *.py[co] - -recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif -include versioneer.py -include pixell/_version.py diff --git a/Makefile b/Makefile index 9da19b15..ed243fb7 100644 --- a/Makefile +++ b/Makefile @@ -27,12 +27,10 @@ export PRINT_HELP_PYSCRIPT BROWSER := python -c "$$BROWSER_PYSCRIPT" python = python --include options.mk # Main targets: - develop: - OPT="" $(python) setup.py build_ext --inplace $(build_opts) + OPT="" $(python) -m pip install --no-build-isolation -e . help: @$(python) -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) @@ -52,12 +50,10 @@ lint: ## check style with flake8 flake8 pixell tests test: ## run tests quickly with the default Python - $(python) setup.py test $(build_opts) + pytest coverage: ## check code coverage quickly with the default Python - coverage run --source pixell setup.py test - coverage report -m - coverage html + pytest --cov --cov-report=html $(BROWSER) htmlcov/index.html docs: ## generate Sphinx HTML documentation, including API docs @@ -75,9 +71,8 @@ release: dist ## package and upload a release twine upload dist/* dist: clean ## builds source and wheel package - $(python) setup.py sdist $(build_opts) - $(python) setup.py bdist_wheel $(build_opts) + $(python) build ls -l dist install: clean ## install the package to the active Python's site-packages - $(python) setup.py install $(build_opts) + pip install . diff --git a/README.rst b/README.rst index 193e56dc..0f76952b 100644 --- a/README.rst +++ b/README.rst @@ -25,14 +25,13 @@ pixell Dependencies ------------ -* Python>=3.7 but Python 3.12 is not currently supported +* Python>=3.9. * gcc/gfortran or Intel compilers (clang might not work out of the box), if compiling from source * ducc0_, healpy, Cython, astropy, numpy, scipy, matplotlib, pyyaml, h5py, Pillow (Python Image Library) On MacOS, and other systems with non-traditional environments, you should specify the following standard environment variables: * ``CC``: C compiler (example: ``gcc``) -* ``CXX``: C++ compiler (example: ``g++``) * ``FC``: Fortran compiler (example: ``gfortran``) We recommend using ``gcc`` installed from Homebrew to access these compilers on @@ -56,6 +55,13 @@ have both efficiency and performance cores, you may wish to set ``OMP_NUM_THREAD the number of performance cores in your system. This will ensure that the efficiency cores are not used for the parallelized parts of ``pixell`` and ``ducc0``. +You can check the threading behaviour (and the installation of ``pixell``) by running +the benchmark script: + +.. code-block:: console + + $ benchmark-pixell-runner + Installing ---------- @@ -64,59 +70,40 @@ Make sure your ``pip`` tool is up-to-date. To install ``pixell``, run: .. code-block:: console $ pip install pixell --user - $ test-pixell -This will install a pre-compiled binary suitable for your system (only Linux and Mac OS X with Python>=3.7 are supported). Note that you need ``~/.local/bin`` to be in your ``PATH`` for the latter ``test-pixell`` to work. +This will install a pre-compiled binary suitable for your system (only Linux and Mac OS X with Python>=3.9 are supported). -If you require more control over your installation, e.g. using Intel compilers, please see the section below on compiling from source. The ``test-pixell`` command will run a suite of unit tests. +If you require more control over your installation, e.g. using Intel compilers, please see the section below on compiling from source. Compiling from source (advanced / development workflow) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -For compilation instructions specific to NERSC/cori, see NERSC_. - -For all other, below are general instructions. - -First, download the source distribution or ``git clone`` this repository. You can work from ``master`` or checkout one of the released version tags (see the Releases section on Github). Then change into the cloned/source directory. +The easiest way to install from source is to use the ``pip`` tool, +with the ``--no-binary`` flag. This will download the source distribution +and compile it for you. Don't forget to make sure you have CC and FC set +if you have any problems. +For all other cases, below are general instructions. -Run ``setup.py`` -~~~~~~~~~~~~~~~~ +First, download the source distribution or ``git clone`` this repository. You +can work from ``master`` or checkout one of the released version tags (see the +Releases section on Github). Then change into the cloned/source directory. -If not using Intel compilers (see below), build the package using +Once downloaded, you can install using ``pip install .`` inside the project +directory. We use the ``meson`` build system, which should be understood by +``pip`` (it will build in an isolated environment). -.. code-block:: console - - $ python setup.py build_ext -i +We suggest you then test the installation by running the unit tests. You +can do this by running ``pytest``. -You may now test the installation: +To run an editable install, you will need to do so in a way that does not +have build isolation (as the backend build system, `meson` and `ninja`, actually +perform micro-builds on usage in this case): .. code-block:: console - - $ py.test pixell/tests/ -If the tests pass, you can install the package (optionally with ``-e`` if you would like to edit the files after installation) - -.. code-block:: console - - $ python setup.py install --user - - -Intel compilers -~~~~~~~~~~~~~~~ - -Intel compilers require you to modify the build step above as follows - -.. code-block:: console - - $ python setup.py build_ext -i --fcompiler=intelem --compiler=intelem - -On some systems, further specification might be required (make sure to get a fresh copy of the repository before trying out a new install method), e.g.: - -.. code-block:: console - - $ LDSHARED="icc -shared" LD=icc LINKCC=icc CC=icc python setup.py build_ext -i --fcompiler=intelem --compiler=intelem - + $ pip install --upgrade pip meson ninja meson-python cython numpy + $ pip install --no-build-isolation --editable . Contributions diff --git a/docs/conf.py b/docs/conf.py index 4b20cc70..4de60943 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -60,7 +60,7 @@ # the built documents. # # The short X.Y version. -version = pixell.__version__ +version = ".".join(pixell.__version__.split(".")[:2]) # The full version, including alpha/beta/rc tags. release = pixell.__version__ diff --git a/docs/nersc.rst b/docs/nersc.rst deleted file mode 100644 index e90dc470..00000000 --- a/docs/nersc.rst +++ /dev/null @@ -1,51 +0,0 @@ -================== -NERSC Installation -================== - -If you have not set a Python environment already, we recommend using this module: - -.. code:: shell - - module load python/3.7-anaconda-2019.07 - -You can put this line in your .bashrc.ext in order to load it automatically when -logging in. - -Then, clone pixell and install as follows, - -.. code:: shell - - git clone https://github.com/simonsobs/pixell.git - cd pixell - python setup.py build_ext -i --fcompiler=intelem --compiler=intelem - -Make sure to test the installation - -.. code:: shell - - py.test - -which should display no errors. - - -and then finally symbolically link pixell into your Python path - -.. code:: shell - - pip install -e . --user - - -To update your installation, - - -.. code:: shell - - git pull origin master - python setup.py build_ext -i --fcompiler=intelem --compiler=intelem - py.test - pip install -e . --user - - -Note that all of this assumes you will be using the default Intel suite on -NERSC. If for some reason you have set up your environment to use GNU, then you -should not include `--fcompiler=intelem --compiler=intelem` in any of the above. diff --git a/meson.build b/meson.build new file mode 100644 index 00000000..b8d28333 --- /dev/null +++ b/meson.build @@ -0,0 +1,133 @@ +# Main build file for pixell, helping with building the fortran, +# c, and cython extensions. +project( + 'pixell', + ['c', 'fortran', 'cython'], + version: run_command( + [ + # Note that _version.py doesn't actually 'dynamically' set the + # version number. We call it at build time and set it as a meson + # property. + 'python', + 'pixell/_version.py' + ], + check: true, + ).stdout().strip() +) + +py = import('python').find_installation(pure: false) + +# Dependencies +py_dep = py.dependency() +omp_dep = dependency('openmp') + +# Libraries +cc = meson.get_compiler('c') +c_m_dep = cc.find_library('m', required: true) + +fc = meson.get_compiler('fortran') +fortran_m_dep = fc.find_library('m', required: true) + +# Directories +library_install_dir = py.get_install_dir() / 'pixell' + +# Includes + +# Need to massage these into relative paths to keep meson happy. +# It does not allow for absolute paths. +incdir_numpy = run_command( + py, + ['-c', 'import numpy; import os; print(os.path.relpath(numpy.get_include()))'], + check: true +).stdout().strip() + +incdir_f2py = run_command( + py, + ['-c', 'import numpy.f2py; import os; print(os.path.relpath(numpy.f2py.get_include()))'], + check: true +).stdout().strip() + +# Build fortran extensions + +# Pixell oddity - need to run the makefile (make -C fortran) +# to generate specific fortran files before we can build them. +# TODO: put those commands in this meson file instead. +run_command('make', '-C', 'fortran', check: true) + +fortran_include = include_directories(incdir_numpy, incdir_f2py) + +fortran_sources = { + 'fortran/interpol_32.f90': '_interpol_32', + 'fortran/interpol_64.f90': '_interpol_64', + 'fortran/colorize.f90': '_colorize', + 'fortran/array_ops_32.f90': '_array_ops_32', + 'fortran/array_ops_64.f90': '_array_ops_64', +} + +foreach source_name, module_name : fortran_sources + f2py_output = custom_target( + input: source_name, + output: [module_name + '-f2pywrappers2.f90', module_name + 'module.c'], + command: [py, '-m', 'numpy.f2py', '@INPUT@', '-m', module_name, '--lower'], + ) + + py.extension_module( + module_name, + [source_name, f2py_output], + incdir_f2py / 'fortranobject.c', + include_directories: fortran_include, + dependencies: [py_dep, omp_dep, c_m_dep, fortran_m_dep], + install: true, + subdir: 'pixell' + ) +endforeach + + +# Build c(ython) extensions. + +# Before building cython, we must build shared libraries for all of +# the underlying c code that those cython extensions rely on. + +helper_sources = { + 'cython/cmisc_core.c': '_cmisc_shared', + 'cython/distances_core.c': '_distances_shared', + 'cython/srcsim_core.c': '_srcsim_shared', +} + +linkables = [] + +foreach source_name, module_name : helper_sources + linkables += shared_library( + module_name, + source_name, + install: true, + install_dir: library_install_dir, + include_directories: [incdir_numpy], + dependencies: [omp_dep, c_m_dep], + ) +endforeach + +# Now we can build cython and link our shared libraries to them. + +cython_sources = { + 'cython/cmisc.pyx': 'cmisc', + 'cython/distances.pyx': 'distances', + 'cython/srcsim.pyx': 'srcsim', +} + +foreach source_name, module_name : cython_sources + cython_module = py.extension_module( + module_name, + source_name, + include_directories: ['cython', incdir_numpy], + dependencies: [py_dep, omp_dep, c_m_dep], + link_with: linkables, + install: true, + subdir: 'pixell', + ) +endforeach + +# The actual python install itself is left up to a helper build +# script deifned in pixell/ +subdir('pixell') +subdir('scripts') \ No newline at end of file diff --git a/pixell/__init__.py b/pixell/__init__.py index a5ba7486..8b0bf876 100644 --- a/pixell/__init__.py +++ b/pixell/__init__.py @@ -2,8 +2,12 @@ """Top-level package for pixell.""" +from importlib.metadata import version, PackageNotFoundError + +try: + __version__ = version("pixell") +except PackageNotFoundError: + __version__ = "unknown" + __author__ = """Simons Observatory Collaboration Analysis Library Task Force""" __email__ = '' -from ._version import get_versions -__version__ = get_versions()['version'] -del get_versions diff --git a/pixell/_version.py b/pixell/_version.py index 4553e96a..b3832598 100644 --- a/pixell/_version.py +++ b/pixell/_version.py @@ -518,3 +518,7 @@ def get_versions(): return {"version": "0+unknown", "full-revisionid": None, "dirty": None, "error": "unable to compute version", "date": None} + + +if __name__ == "__main__": + print(get_versions()["version"]) \ No newline at end of file diff --git a/pixell/curvedsky.py b/pixell/curvedsky.py index 155f2bbf..4d3d074b 100644 --- a/pixell/curvedsky.py +++ b/pixell/curvedsky.py @@ -2,7 +2,9 @@ full sky.""" from __future__ import print_function, division import numpy as np, os, warnings -from . import enmap, powspec, wcsutils, utils, bunch, cmisc +from . import enmap, powspec, wcsutils, utils, bunch + +from . import cmisc # Initialize DUCC's thread num variable from OMP's if it's not already set. # This must be done before importing ducc0 for the first time. Doing this # limits wasted memory from ducc allocating too big a thread pool. For computes diff --git a/pixell/enmap.py b/pixell/enmap.py index 1575545c..837bff90 100644 --- a/pixell/enmap.py +++ b/pixell/enmap.py @@ -48,7 +48,10 @@ def __array_finalize__(self, obj): def __repr__(self): return "ndmap(%s,%s)" % (np.asarray(self), wcsutils.describe(self.wcs)) def __str__(self): return repr(self) - def __array_wrap__(self, arr, context=None): + def __array_wrap__(self, arr, context=None, return_scalar=False): + # In the future need to support `return_scalar`, but that is seemingly + # undocumented and not actually supported in numpy 2.0? So for now we + # just ignore it. if arr.ndim < 2: return arr return ndmap(arr, self.wcs) def __reduce__(self): @@ -2729,7 +2732,9 @@ def fix_endian(map): """Make endianness of array map match the current machine. Returns the result.""" if map.dtype.byteorder not in ['=','<' if sys.byteorder == 'little' else '>']: - map = map.byteswap(True).newbyteorder() + # Compatibility to numpy 2.0 + map = map.byteswap(True) + map = map.view(map.dtype.newbyteorder()) map.dtype = utils.fix_dtype_mpi4py(map.dtype) return map diff --git a/pixell/enplot.py b/pixell/enplot.py index 11a4b763..f1d5bae3 100644 --- a/pixell/enplot.py +++ b/pixell/enplot.py @@ -542,6 +542,14 @@ def draw_colorbar(crange, width, args): fmt = "%g" labels, boxes = [], [] for val in crange: + # Val could be a one-element array. In NumPy 1.25 this is not + # acceptable to string formatters. + + try: + val = val[0] + except (TypeError, IndexError): + pass + labels.append(fmt % val) boxes.append(font.getbbox(labels[-1])[-2:]) boxes = np.array(boxes,int) diff --git a/pixell/meson.build b/pixell/meson.build new file mode 100644 index 00000000..c77a7246 --- /dev/null +++ b/pixell/meson.build @@ -0,0 +1,38 @@ +python_sources = [ + '__init__.py', + '_version.py', + 'aberration.py', + 'analysis.py', + 'array_ops.py', + 'bunch.py', + 'cgrid.py', + 'colorize.py', + 'colors.py', + 'coordinates.py', + 'curvedsky.py', + 'enmap.py', + 'enplot.py', + 'fft.py', + 'interpol.py', + 'lensing.py', + 'memory.py', + 'mpi.py', + 'multimap.py', + 'old_aberration.py', + 'pointsrcs.py', + 'powspec.py', + 'reproject.py', + 'resample.py', + 'tilemap.py', + 'uharm.py', + 'utils.py', + 'wavelets.py', + 'wcsutils.py', + # Not technically a python source file, but is required for pixell + 'arial.ttf', +] + +py.install_sources( + python_sources, + subdir: 'pixell' +) \ No newline at end of file diff --git a/pixell/tests/__init__.py b/pixell/tests/__init__.py deleted file mode 100644 index 8ce61302..00000000 --- a/pixell/tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .test_geom import * -from .test_io import * -from .test_pixell import * diff --git a/pixell/tests/pixel_tests.py b/pixell/tests/pixel_tests.py deleted file mode 100644 index de561ef9..00000000 --- a/pixell/tests/pixel_tests.py +++ /dev/null @@ -1,182 +0,0 @@ -from __future__ import print_function -import matplotlib -matplotlib.use('Agg') -from pixell import enmap,curvedsky,wcsutils,reproject -import numpy as np -import itertools,yaml,pickle,os,sys -import matplotlib.pyplot as plt - -TEST_DIR = os.path.dirname(__file__) -DATA_PREFIX = os.path.join(TEST_DIR, 'data/') - -""" -This script generates a set of reference values against which -pixell tests will be done. - -""" - - -def get_reference_pixels(shape): - """For a given 2D array, return a list of pixel indices - corresponding to locations of a pre-determined and fixed - pattern of reference pixels. - - e.g even x even - 1100110011 - 1100110011 - 0000000000 - 0000000000 - 1100110011 - 1100110011 - 0000000000 - 0000000000 - 1100110011 - 1100110011 - - e,g. odd x odd - 110010011 - 110010011 - 000000000 - 000000000 - 110010011 - 000000000 - 000000000 - 110010011 - 110010011 - - e.g even x odd - 110010011 - 110010011 - 000000000 - 000000000 - 110010011 - 110010011 - 000000000 - 000000000 - 110010011 - 110010011 - - requires N>=5 in each axis - """ - Ny,Nx = shape[-2:] - assert (Ny>=5) and (Nx>=5), "Tests are not implemented for arrays with a dimension<5." - """Given N, return 0,1,{x},N-2,N-1, where {x} is N//2-1,N//2 if N is even - and {x} is N//2 if N is odd. - """ - midextremes = lambda N: [0,1,N//2-1,N//2,N-2,N-1] if N%2==0 else [0,1,N//2,N-2,N-1] - ys = midextremes(Ny) - xs = midextremes(Nx) - pixels = np.array(list(itertools.product(ys,xs))) - return pixels - -def mask(arr,pixels,val=0): - """Mask an array arr based on array pixels of (y,x) pixel coordinates of (Npix,2)""" - arr[...,pixels[:,0],pixels[:,1]] = val - return arr - -def get_pixel_values(arr,pixels): - """Get values of arr at pixels specified in pixels (Npix,2)""" - return arr[...,pixels[:,0],pixels[:,1]] - -def get_meansquare(arr): - return np.mean(arr*2.) - -def save_mask_image(filename,shape): - """Save a minimal plot of an array masked by the currently implemented reference - pixel geometry - - e.g. - > shape = (11,12) - > save_mask_image("test_mask.png",shape) - """ - arr = np.zeros(shape) - pixels = get_reference_pixels(shape) - masked = mask(arr,pixels,val=1) - fig = plt.figure() - im = plt.imshow(masked,cmap='rainbow') - ax = plt.gca() - ax.set_xticks(np.arange(0,shape[1])+0.5); - ax.set_yticks(np.arange(0,shape[0])+0.5); - ax.grid(which='major',color='w', linestyle='-', linewidth=5) - ax.tick_params(axis='x', colors=(0,0,0,0)) - ax.tick_params(axis='y', colors=(0,0,0,0)) - for spine in im.axes.spines.values(): - spine.set_edgecolor((0,0,0,0)) - plt.savefig(filename, bbox_inches='tight') - -def get_spectrum(ntype,noise,lmax,lmax_pad): - ells = np.arange(0,lmax+lmax_pad) - if ntype=="white": return np.ones(shape=(ells.size,))*(noise**2.)*((np.pi/180./60.)**2.) - if ntype=="white_dl": - spec = np.zeros(shape=(ells.size,)) - spec[2:] = (noise**2.)*((np.pi/180./60.)**2.)*2.*np.pi/ells[2:]/(ells+1.)[2:] - return spec - raise NotImplementedError - -def get_spectra(yml_section,lmax,lmax_pad): - spectra = {} - for s in yml_section: - spectra[s['name']] = get_spectrum(s['type'],s['noise'],lmax,lmax_pad) - return spectra - -def get_geometries(yml_section): - geos = {} - for g in yml_section: - if g['type']=='fullsky': - geos[g['name']] = enmap.fullsky_geometry(res=np.deg2rad(g['res_arcmin']/60.),proj=g['proj']) - elif g['type']=='pickle': - geos[g['name']] = pickle.load(open(DATA_PREFIX+"%s"%g['filename'],'rb')) - else: - raise NotImplementedError - return geos - -def generate_map(shape,wcs,powspec,lmax,seed): - return curvedsky.rand_map(shape, wcs, powspec, lmax=lmax, dtype=np.float64, seed=seed, spin=[0,2], method="auto", verbose=False) - -def check_equality(imap1,imap2): - assert np.all(imap1.shape==imap2.shape) - assert wcsutils.equal(imap1.wcs,imap2.wcs) - try: - assert np.all(np.isclose(imap1,imap2)) - except: - from orphics import io - io.plot_img(imap1,"i1.png",lim=[-1.5,2]) - io.plot_img(imap2,"i2.png",lim=[-1.5,2]) - io.plot_img((imap1-imap2)/imap1,"ip.png",lim=[-0.1,0.1]) - assert 1==0 - - -def get_extraction_test_results(yaml_file): - print("Starting tests from ",yaml_file) - with open(yaml_file) as f: - config = yaml.safe_load(f) - geos = get_geometries(config['geometries']) - lmax = config['lmax'] ; lmax_pad = config['lmax_pad'] - spectra = get_spectra(config['spectra'],lmax,lmax_pad) - seed = config['seed'] - - results = {} - for g in geos.keys(): - results[g] = {} - for s in spectra.keys(): - results[g][s] = {} - imap = generate_map(geos[g][0][-2:],geos[g][1],spectra[s],lmax,seed) - - # Do write and read test - filename = "temporary_map.fits" # NOT THREAD SAFE - enmap.write_map(filename,imap) - imap_in = enmap.read_map(filename) - check_equality(imap,imap_in) - for e in config['extracts']: - print("Doing test for extract ",e['name']," with geometry ",g," and spectrum ",s,"...") - if e['type']=='slice': - box = np.deg2rad(np.array(e['box_deg'])) - cutout = enmap.read_map(filename,box=box) - cutout_internal = imap.submap(box=box) - check_equality(cutout,cutout_internal) - pixels = get_reference_pixels(cutout.shape) - results[g][s]['refpixels'] = get_pixel_values(cutout,pixels) - results[g][s]['meansquare'] = get_meansquare(cutout) - - os.remove(filename) - return results,config['result_name'] diff --git a/pixell/tilemap.py b/pixell/tilemap.py index cab00540..22766ce9 100644 --- a/pixell/tilemap.py +++ b/pixell/tilemap.py @@ -71,7 +71,10 @@ def __array_finalize__(self, obj): def __repr__(self): return "TileMap(%s,%s)" % (np.asarray(self), str(self.geometry)) def __str__(self): return repr(self) - def __array_wrap__(self, arr, context=None): + def __array_wrap__(self, arr, context=None, return_scalar=False): + # In the future need to support `return_scalar`, but that is seemingly + # undocumented and not actually supported in numpy 2.0? So for now we + # just ignore it. return TileMap(arr, self.geometry) def __getitem__(self, sel): # Split sel into normal and wcs parts. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..496d5f26 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,92 @@ +[build-system] +build-backend = 'mesonpy' +requires = ['meson-python', 'numpy', 'cython', 'versioneer[toml]', 'build'] + +[project] +dynamic = ["version"] +name = 'pixell' +description = "A rectangular pixel map manipulation and harmonic analysis library derived from Sigurd Naess' enlib." +readme = 'README.rst' +requires-python = '>=3.9' +license = {file = 'LICENSE'} +authors = [ + {name = "Simons Observatory Collaboration Analysis Library Task Force"}, +] +maintainers = [ + {name = "Mathew Madhavacheril", email = "mathewsyriac@gmail.com"} +] +dependencies = [ + 'numpy>=1.20.0', + 'astropy>=2.0', + 'setuptools>=39', + 'h5py>=2.7', + 'scipy>=1.0', + 'python_dateutil>=2.7', + 'cython', + 'healpy>=1.13', + 'matplotlib>=2.0', + 'pyyaml>=5.0', + 'Pillow>=5.3.0, != 10.4.0', + 'pytest-cov>=2.6', + 'coveralls>=1.5', + 'pytest>=4.6', + 'ducc0>=0.34.0', + 'numba>=0.54.0' +] + +[project.scripts] +# See scripts/meson.build for installation +benchmark-pixell-runner = "pixell.scripts.benchmark_pixell_runner:main" +benchmark-pixell = "pixell.scripts.benchmark_pixell:main" + +[project.optional-dependencies] +test = [ + 'pip>=9.0', + 'bumpversion>=0.5', + 'wheel>=0.30', + 'watchdog>=0.8', + 'flake8>=3.5', + 'coverage>=4.5', + 'Sphinx>=1.7', + 'twine>=1.10', + 'numpy>=1.20', + 'astropy>=2.0', + 'setuptools>=39.2', + 'h5py>=2.7', + 'scipy>=1.0', + 'python_dateutil>=2.7', + 'cython', + 'matplotlib>=2.0', + 'pyyaml>=5.0', + 'pytest-cov>=2.6', + 'coveralls>=1.5', + 'pytest>=4.6' +] + +[tool.pytest.ini_options] +testpaths = [ + "tests", +] + +[tool.cibuildwheel] +test-requires = "pytest" +test-command = "pytest {project}/tests" +build = "cp39-* cp310-* cp311-* cp312-*" +skip = "*i686* *musllinux*" + +[tool.coverage.run] +source = [ + "pixell" +] + +[tool.coverage.report] +exclude_lines = ["pragma: no cover"] +exclude_also = ["if TYPE_CHECKING:"] + +[tool.versioneer] +VCS = "git" +style = "pep440" +versionfile_source = "pixell/_version.py" +versionfile_build = "pixell/_version.py" +tag_prefix = "v" +parentdir_prefix = "pixell-" \ No newline at end of file diff --git a/scripts/benchmark_pixell.py b/scripts/benchmark_pixell.py new file mode 100644 index 00000000..7e31416d --- /dev/null +++ b/scripts/benchmark_pixell.py @@ -0,0 +1,21 @@ +import subprocess +import multiprocessing +import sys +from pathlib import Path + + +def main(): + max_threads = multiprocessing.cpu_count() + assert max_threads >= 1 + + def run_benchmark(nthreads): + subprocess.call( + [sys.executable, Path(__file__).parent / "benchmark_pixell_runner.py"], + env={"OMP_NUM_THREADS": str(nthreads)}, + ) + + print("Single threaded alm test:") + run_benchmark(1) + + print(f"Multi-threaded alm test with {max_threads} threads:") + run_benchmark(max_threads) diff --git a/scripts/benchmark_pixell_runner.py b/scripts/benchmark_pixell_runner.py new file mode 100644 index 00000000..cbed72ce --- /dev/null +++ b/scripts/benchmark_pixell_runner.py @@ -0,0 +1,31 @@ +""" +Core benchmark script to be evaluated with various threading +configurations. See benchmark_pixell.py for the actual benchmark +that varies the number of threads. +""" + +from pixell import curvedsky, enmap, utils +import time +import numpy as np + + +def main(): + np.random.seed(100) + shape, wcs = enmap.fullsky_geometry(res=12.0 * utils.arcmin) + imap = enmap.enmap(np.random.random(shape), wcs) + + nsims = 40 + lmax = int(6000 * (2.0 / 16.0)) + + t0 = time.time() + for i in range(nsims): + alm = curvedsky.map2alm(imap, lmax=lmax) + curvedsky.alm2map(alm, enmap.empty(shape, wcs)) + t1 = time.time() + + total = t1 - t0 + print(f"{total:.4f} seconds.") + + +if __name__ == "__main__": + main() diff --git a/scripts/build_wheels.sh b/scripts/build_wheels.sh deleted file mode 100755 index a19b71ad..00000000 --- a/scripts/build_wheels.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# Adapted from scikit-learn -# https://raw.githubusercontent.com/scikit-learn/scikit-learn/21312644df0a6b4c6f3c27a74ac9d26cf49c2304/build_tools/wheels/build_wheels.sh - - -#!/bin/bash - -set -e -set -x - -# The version of the built dependencies are specified -# in the pyproject.toml file, while the tests are run -# against the most recent version of the dependencies - -python -m pip install --upgrade pip setuptools wheel -python -m pip install cibuildwheel -python setup.py build_ext -i -python -m pip install . -py.test --cov=pixell pixell/tests/ -s -find . -type f -iname '*.so' -print -delete -rm -rf _deps/ -python -m cibuildwheel --output-dir wheelhouse -ls wheelhouse diff --git a/scripts/generate_tests/pixel_test_generator.py b/scripts/generate_tests/pixel_test_generator.py index 53364be9..33f4af25 100644 --- a/scripts/generate_tests/pixel_test_generator.py +++ b/scripts/generate_tests/pixel_test_generator.py @@ -1,6 +1,6 @@ from pixell import sharp import sys -sys.path.append('../../pixell/tests') +sys.path.append('../../tests') import pixel_tests as ptests import pickle import os diff --git a/scripts/meson.build b/scripts/meson.build new file mode 100644 index 00000000..301363ff --- /dev/null +++ b/scripts/meson.build @@ -0,0 +1,11 @@ +# Install the script in a new directory, pixell/scripts. + +python_scripts = [ + 'benchmark_pixell_runner.py', + 'benchmark_pixell.py', +] + +py.install_sources( + python_scripts, + subdir: 'pixell/scripts' +) \ No newline at end of file diff --git a/scripts/omp_hello.c b/scripts/omp_hello.c deleted file mode 100644 index fb471f52..00000000 --- a/scripts/omp_hello.c +++ /dev/null @@ -1,38 +0,0 @@ -/****************************************************************************** -* FILE: omp_hello.c -* DESCRIPTION: -* OpenMP Example - Hello World - C/C++ Version -* In this simple example, the master thread forks a parallel region. -* All threads in the team obtain their unique thread number and print it. -* The master thread only prints the total number of threads. Two OpenMP -* library routines are used to obtain the number of threads and each -* thread's number. -* AUTHOR: Blaise Barney 5/99 -* LAST REVISED: 04/06/05 -******************************************************************************/ -#include -#include -#include - -int main (int argc, char *argv[]) -{ - int nthreads, tid; - - /* Fork a team of threads giving them their own copies of variables */ - #pragma omp parallel private(nthreads, tid) - { - - /* Obtain thread number */ - tid = omp_get_thread_num(); - printf("Hello World from thread = %d\n", tid); - - /* Only master thread does this */ - if (tid == 0) - { - nthreads = omp_get_num_threads(); - printf("Number of threads = %d\n", nthreads); - } - - } /* All threads join master thread and disband */ - -} diff --git a/scripts/omp_hello.f90 b/scripts/omp_hello.f90 deleted file mode 100644 index bd560cfc..00000000 --- a/scripts/omp_hello.f90 +++ /dev/null @@ -1,15 +0,0 @@ -! Hello world test program for fortran. - -program example - use :: omp_lib - implicit none - - ! Set number of threads to use. - call omp_set_num_threads(2) - - !$omp parallel - - print '("Thread: ", i0)', omp_get_thread_num() - - !$omp end parallel -end program example diff --git a/scripts/test-pixell b/scripts/test-pixell deleted file mode 100755 index c34bb52f..00000000 --- a/scripts/test-pixell +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python - -import os,sys -import unittest - -# Must do the thread setup before we import pixell tests, otherwise those -# dependent on the threadpool will fail. - -import multiprocessing -max_threads = multiprocessing.cpu_count() -assert max_threads>=1 - -# On MacOS default values of `DUCC0_NUM_THREADS` (used for run-time thread number selection -# as ducc0 uses pthreads not OMP) are not correctly handled, we must set it to a valid -# integer value. -if sys.platform == "darwin": - if "DUCC0_NUM_THREADS" not in os.environ: - os.environ["DUCC0_NUM_THREADS"] = str(max_threads) - print(f"Setting DUCC0_NUM_THREADS (not present) to {max_threads}.") - - try: - ducc0_num_threads = int(os.environ["DUCC0_NUM_THREADS"]) - except ValueError: - # We have a non-integer value for this environment variable. - # Ducc0 does not currently (2023-10-16) have error handling for this - # (at least on MacOS 13), so we need to set it to a valid value. - os.environ["DUCC0_NUM_THREADS"] = str(max_threads) - -from pixell.tests import * - -unittest.main(exit=False) - -def run_alm_benchmark(nthreads): - os.system(f"""OMP_NUM_THREADS={nthreads} python -c 'from pixell import curvedsky,enmap,utils ; -import time ; -import numpy as np ; - -np.random.seed(100) ; -shape,wcs = enmap.fullsky_geometry(res=12.*utils.arcmin) ; -imap = enmap.enmap(np.random.random(shape),wcs) ; - -nsims = 40 ; -lmax = int(6000 * (2./16.)); - -t0 = time.time() ; -for i in range(nsims): alm = curvedsky.map2alm(imap,lmax=lmax) ; omap = curvedsky.alm2map(alm,enmap.empty(shape,wcs)) ; -t1 = time.time(); - -total = t1-t0; -print(f"{{total:.4f}} seconds."); -' - """) - -print("Single threaded alm test:") -run_alm_benchmark(1) - -if max_threads==1: - print("Multi-threading not detected.") -else: - print(f"Multi-threaded alm test with {max_threads} threads:") - run_alm_benchmark(max_threads) - - diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 1cdee40b..00000000 --- a/setup.cfg +++ /dev/null @@ -1,20 +0,0 @@ -[bumpversion] -current_version = 0.24.0 -commit = True -tag = True - -[bdist_wheel] -universal = 1 - -[flake8] -exclude = docs - -[aliases] - -[versioneer] -VCS = git -style = pep440 -versionfile_source = pixell/_version.py -versionfile_build = pixell/_version.py -tag_prefix = v -parentdir_prefix = pixell- diff --git a/setup.py b/setup.py deleted file mode 100644 index c8c0b424..00000000 --- a/setup.py +++ /dev/null @@ -1,284 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -"""The setup script.""" -from __future__ import print_function -import setuptools -from setuptools import find_packages -from distutils.errors import DistutilsError -from numpy.distutils.core import setup, Extension, build_ext, build_src -import versioneer -import os, sys -import subprocess as sp -import numpy as np - -build_ext = build_ext.build_ext -build_src = build_src.build_src - -CYTHON_NOT_FOUND_MSG = """Cython not found. -Please run either one of the following commands to install Cython first - mamba install cython - conda install cython - pip install cython -""" - - -compile_opts = { - #'extra_compile_args': ['-std=c99','-fopenmp', '-Wno-strict-aliasing', '-g', '-O0', '-fPIC', '-fsanitize=address', '-fsanitize=undefined'], - 'extra_compile_args': ['-std=c99','-fopenmp', '-Wno-strict-aliasing', '-g', '-Ofast', '-fPIC'], - 'extra_f90_compile_args': ['-fopenmp', '-Wno-conversion', '-Wno-tabs', '-fPIC'], - 'f2py_options': ['skip:', 'map_border', 'calc_weights', ':'], - 'extra_link_args': ['-fopenmp', '-g', '-fPIC', '-fno-lto'] - } - -# Set compiler options -# Windows -if sys.platform == 'win32': - raise DistutilsError('Windows is not supported.') -elif sys.platform == 'darwin' or sys.platform == 'linux': - environment = os.environ - - if not 'CC' in environment: - environment["CC"] = "gcc" - - if not "CXX" in environment: - environment["CXX"] = "g++" - - if not "FC" in environment: - environment["FC"] = "gfortran" - - # Now, try out our environment! - c_return = sp.call([environment["CC"], *compile_opts["extra_compile_args"], "scripts/omp_hello.c", "-o", "/tmp/pixell-cc-test"], env=environment) - - try: - os.remove("/tmp/pixell-cc-test") - except OSError: - pass - - if c_return != 0: - raise EnvironmentError( - "Your C compiler does not support the following flags, required by pixell: " - f"{' '.join(compile_opts['extra_compile_args'])}" - ". Consider setting the value of environment variable CC to a known good gcc install. " - "The built-in Apple clang does not support OpenMP. Use Homebrew to install either gcc or llvm. " - f"Current value of $CC is {environment['CC']}.", - ) - else: - print(f"C compiler found ({environment['CC']}) and supports OpenMP.") - - cxx_return = sp.call([environment["CXX"], *compile_opts["extra_compile_args"], "scripts/omp_hello.c", "-o", "/tmp/pixell-cxx-test"], env=environment) - - try: - os.remove("/tmp/pixell-cxx-test") - except OSError: - pass - - if cxx_return != 0: - raise EnvironmentError( - "Your CXX compiler does not support the following flags, required by pixell: " - f"{' '.join(compile_opts['extra_compile_args'])}" - ". Consider setting the value of environment variable CXX to a known good gcc install. " - "The built-in Apple clang does not support OpenMP. Use Homebrew to install either gcc or llvm. " - f"Current value of $CXX is {environment['CXX']}.", - ) - else: - print(f"CXX compiler found ({environment['CXX']}) and supports OpenMP.") - - fc_return = sp.call([environment["FC"], *compile_opts["extra_f90_compile_args"], "scripts/omp_hello.f90", "-o", "/tmp/pixell-fc-test"], env=environment) - - try: - os.remove("/tmp/pixell-fc-test") - except OSError: - pass - - if fc_return != 0: - raise EnvironmentError( - "Your Fortran compiler does not support the following flags, required by pixell: " - f"{' '.join(compile_opts['extra_f90_compile_args'])}" - ". Consider setting the value of environment variable FC to a known good gfortran install." - f"Current value of $FC is {environment['FC']}.", - ) - else: - print(f"Fortran compiler found ({environment['FC']}) and supports OpenMP.") - - # Why do we remove -fPIC here? - compile_opts['extra_link_args'] = ['-fopenmp'] -else: - raise EnvironmentError("Unknown platform. Please file an issue on GitHub.") - -with open('README.rst') as readme_file: - readme = readme_file.read() - -with open('HISTORY.rst') as history_file: - history = history_file.read() - -requirements = ['numpy>=1.20.0', - 'astropy>=2.0', - 'setuptools>=39', - 'h5py>=2.7', - 'scipy>=1.0', - 'python_dateutil>=2.7', - 'cython<3.0.4', - 'healpy>=1.13', - 'matplotlib>=2.0', - 'pyyaml>=5.0', - 'Pillow>=5.3.0', - 'pytest-cov>=2.6', - 'coveralls>=1.5', - 'pytest>=4.6', - 'ducc0>=0.34.0', - 'numba>=0.54.0'] - - -test_requirements = ['pip>=9.0', - 'bumpversion>=0.5', - 'wheel>=0.30', - 'watchdog>=0.8', - 'flake8>=3.5', - 'coverage>=4.5', - 'Sphinx>=1.7', - 'twine>=1.10', - 'numpy>=1.20', - 'astropy>=2.0', - 'setuptools>=39.2', - 'h5py>=2.7,<=2.10', - 'scipy>=1.0', - 'python_dateutil>=2.7', - 'cython<3.0.4', - 'matplotlib>=2.0', - 'pyyaml>=5.0', - 'pytest-cov>=2.6', - 'coveralls>=1.5', - 'pytest>=4.6'] - -# Why are we doing this instead of allowing the environment to do this? We should just use -O3 and -fPIC. -fcflags = os.getenv('FCFLAGS') -if fcflags is None or fcflags.strip() == '': - fcflags = ['-O3','-fPIC'] - #fcflags = ['-O0','-fPIC', '-fsanitize=address', '-fsanitize=undefined'] -else: - print('User supplied fortran flags: ', fcflags) - print('These will supersede other optimization flags.') - fcflags = fcflags.split() - -compile_opts['extra_f90_compile_args'].extend(fcflags) -compile_opts['extra_f77_compile_args'] = compile_opts['extra_f90_compile_args'] - -def presrc(): - # Create f90 files for f2py. - if sp.call('make -C fortran', shell=True) != 0: - raise DistutilsError('Failure in the fortran source-prep step.') - -def prebuild(): - # Handle cythonization - no_cython = sp.call('cython --version',shell=True) - if no_cython: - raise DistutilsError(CYTHON_NOT_FOUND_MSG) - - if sp.call('make -C cython', shell=True) != 0: - raise DistutilsError('Failure in the cython pre-build step.') - - -class CustomBuild(build_ext): - def run(self): - print("Running build...") - prebuild() - # Then let setuptools do its thing. - return build_ext.run(self) - -class CustomSrc(build_src): - def run(self): - print("Running src...") - presrc() - # Then let setuptools do its thing. - return build_src.run(self) - -class CustomEggInfo(setuptools.command.egg_info.egg_info): - def run(self): - print("Running EggInfo...") - presrc() - prebuild() - return setuptools.command.egg_info.egg_info.run(self) - -# Cascade your overrides here. -cmdclass = { - 'build_ext': CustomBuild, - 'build_src': CustomSrc, - 'egg_info': CustomEggInfo, -} -cmdclass = versioneer.get_cmdclass(cmdclass) - - -setup( - author="Simons Observatory Collaboration Analysis Library Task Force", - author_email='mathewsyriac@gmail.com', - classifiers=[ - 'Development Status :: 2 - Pre-Alpha', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: BSD License', - 'Natural Language :: English', - "Programming Language :: Python :: 2", - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - ], - description="pixell", - package_dir={"pixell": "pixell"}, - entry_points={ - }, - ext_modules=[ - Extension('pixell.cmisc', - sources=['cython/cmisc.c','cython/cmisc_core.c'], - libraries=['m'], - include_dirs=[np.get_include()], - **compile_opts), - Extension('pixell.distances', - sources=['cython/distances.c','cython/distances_core.c'], - libraries=['m'], - include_dirs=[np.get_include()], - **compile_opts), - Extension('pixell.srcsim', - sources=['cython/srcsim.c','cython/srcsim_core.c'], - libraries=['m'], - include_dirs=[np.get_include()], - **compile_opts), - Extension('pixell._interpol_32', - sources=['fortran/interpol_32.f90'], - **compile_opts), - Extension('pixell._interpol_64', - sources=['fortran/interpol_64.f90'], - **compile_opts), - Extension('pixell._colorize', - sources=['fortran/colorize.f90'], - **compile_opts), - Extension('pixell._array_ops_32', - sources=['fortran/array_ops_32.f90'], - **compile_opts), - Extension('pixell._array_ops_64', - sources=['fortran/array_ops_64.f90'], - **compile_opts), - ], - include_dirs = [], - library_dirs = [], - install_requires=requirements, - extras_require = {'fftw':['pyFFTW>=0.10'],'mpi':['mpi4py>=2.0']}, - license="BSD license", - long_description=readme + '\n\n' + history, - package_data={'pixell': ['pixell/tests/data/*.fits','pixell/tests/data/*.dat','pixell/tests/data/*.pkl']}, - include_package_data=True, - data_files=[('pixell', ['pixell/arial.ttf'])], - keywords='pixell', - name='pixell', - packages=find_packages(), - test_suite='pixell.tests', - tests_require=test_requirements, - url='https://github.com/simonsobs/pixell', - version=versioneer.get_version(), - zip_safe=False, - cmdclass=cmdclass, - scripts=['scripts/test-pixell'] -) - -print('\n[setup.py request was successful.]') diff --git a/pixell/tests/data/MM_041121.pkl b/tests/data/MM_041121.pkl similarity index 100% rename from pixell/tests/data/MM_041121.pkl rename to tests/data/MM_041121.pkl diff --git a/pixell/tests/data/MM_lensed_071123.fits b/tests/data/MM_lensed_071123.fits similarity index 100% rename from pixell/tests/data/MM_lensed_071123.fits rename to tests/data/MM_lensed_071123.fits diff --git a/pixell/tests/data/MM_offset_grad_071123.fits b/tests/data/MM_offset_grad_071123.fits similarity index 100% rename from pixell/tests/data/MM_offset_grad_071123.fits rename to tests/data/MM_offset_grad_071123.fits diff --git a/pixell/tests/data/MM_offset_obs_pos_071123.fits b/tests/data/MM_offset_obs_pos_071123.fits similarity index 100% rename from pixell/tests/data/MM_offset_obs_pos_071123.fits rename to tests/data/MM_offset_obs_pos_071123.fits diff --git a/pixell/tests/data/MM_offset_raw_pos_071123.fits b/tests/data/MM_offset_raw_pos_071123.fits similarity index 100% rename from pixell/tests/data/MM_offset_raw_pos_071123.fits rename to tests/data/MM_offset_raw_pos_071123.fits diff --git a/pixell/tests/data/MM_unlensed_071123.fits b/tests/data/MM_unlensed_071123.fits similarity index 100% rename from pixell/tests/data/MM_unlensed_071123.fits rename to tests/data/MM_unlensed_071123.fits diff --git a/pixell/tests/data/annot.txt b/tests/data/annot.txt similarity index 100% rename from pixell/tests/data/annot.txt rename to tests/data/annot.txt diff --git a/pixell/tests/data/cosmo2017_10K_acc3_lensedCls.dat b/tests/data/cosmo2017_10K_acc3_lensedCls.dat similarity index 100% rename from pixell/tests/data/cosmo2017_10K_acc3_lensedCls.dat rename to tests/data/cosmo2017_10K_acc3_lensedCls.dat diff --git a/pixell/tests/data/cutsky_geometry_scaled_20x_9.4_arcminute_pixel_shape_wcs_tuple.pkl b/tests/data/cutsky_geometry_scaled_20x_9.4_arcminute_pixel_shape_wcs_tuple.pkl similarity index 100% rename from pixell/tests/data/cutsky_geometry_scaled_20x_9.4_arcminute_pixel_shape_wcs_tuple.pkl rename to tests/data/cutsky_geometry_scaled_20x_9.4_arcminute_pixel_shape_wcs_tuple.pkl diff --git a/pixell/tests/data/test_scalCls.dat b/tests/data/test_scalCls.dat similarity index 100% rename from pixell/tests/data/test_scalCls.dat rename to tests/data/test_scalCls.dat diff --git a/pixell/tests/test_geom.py b/tests/test_geom.py similarity index 100% rename from pixell/tests/test_geom.py rename to tests/test_geom.py diff --git a/pixell/tests/test_io.py b/tests/test_io.py similarity index 100% rename from pixell/tests/test_io.py rename to tests/test_io.py diff --git a/pixell/tests/test_pixell.py b/tests/test_pixell.py similarity index 87% rename from pixell/tests/test_pixell.py rename to tests/test_pixell.py index aaf191a1..f55aa346 100644 --- a/pixell/tests/test_pixell.py +++ b/tests/test_pixell.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - """Tests for `pixell` package.""" import unittest @@ -24,13 +21,180 @@ import pickle import os,sys -try: # when invoked directly... - import pixel_tests as ptests -except ImportError: # when imported through py.test - from . import pixel_tests as ptests +import matplotlib +matplotlib.use('Agg') +import numpy as np +import itertools,yaml,pickle,os,sys +import matplotlib.pyplot as plt + +TEST_DIR = os.path.dirname(__file__) +DATA_PREFIX = os.path.join(TEST_DIR, 'data/') + +def get_reference_pixels(shape): + """For a given 2D array, return a list of pixel indices + corresponding to locations of a pre-determined and fixed + pattern of reference pixels. + + e.g even x even + 1100110011 + 1100110011 + 0000000000 + 0000000000 + 1100110011 + 1100110011 + 0000000000 + 0000000000 + 1100110011 + 1100110011 + + e,g. odd x odd + 110010011 + 110010011 + 000000000 + 000000000 + 110010011 + 000000000 + 000000000 + 110010011 + 110010011 + + e.g even x odd + 110010011 + 110010011 + 000000000 + 000000000 + 110010011 + 110010011 + 000000000 + 000000000 + 110010011 + 110010011 + + requires N>=5 in each axis + """ + Ny,Nx = shape[-2:] + assert (Ny>=5) and (Nx>=5), "Tests are not implemented for arrays with a dimension<5." + """Given N, return 0,1,{x},N-2,N-1, where {x} is N//2-1,N//2 if N is even + and {x} is N//2 if N is odd. + """ + midextremes = lambda N: [0,1,N//2-1,N//2,N-2,N-1] if N%2==0 else [0,1,N//2,N-2,N-1] + ys = midextremes(Ny) + xs = midextremes(Nx) + pixels = np.array(list(itertools.product(ys,xs))) + return pixels + +def mask(arr,pixels,val=0): + """Mask an array arr based on array pixels of (y,x) pixel coordinates of (Npix,2)""" + arr[...,pixels[:,0],pixels[:,1]] = val + return arr + +def get_pixel_values(arr,pixels): + """Get values of arr at pixels specified in pixels (Npix,2)""" + return arr[...,pixels[:,0],pixels[:,1]] + +def get_meansquare(arr): + return np.mean(arr*2.) + +def save_mask_image(filename,shape): + """Save a minimal plot of an array masked by the currently implemented reference + pixel geometry + + e.g. + > shape = (11,12) + > save_mask_image("test_mask.png",shape) + """ + arr = np.zeros(shape) + pixels = get_reference_pixels(shape) + masked = mask(arr,pixels,val=1) + fig = plt.figure() + im = plt.imshow(masked,cmap='rainbow') + ax = plt.gca() + ax.set_xticks(np.arange(0,shape[1])+0.5); + ax.set_yticks(np.arange(0,shape[0])+0.5); + ax.grid(which='major',color='w', linestyle='-', linewidth=5) + ax.tick_params(axis='x', colors=(0,0,0,0)) + ax.tick_params(axis='y', colors=(0,0,0,0)) + for spine in im.axes.spines.values(): + spine.set_edgecolor((0,0,0,0)) + plt.savefig(filename, bbox_inches='tight') + +def get_spectrum(ntype,noise,lmax,lmax_pad): + ells = np.arange(0,lmax+lmax_pad) + if ntype=="white": return np.ones(shape=(ells.size,))*(noise**2.)*((np.pi/180./60.)**2.) + if ntype=="white_dl": + spec = np.zeros(shape=(ells.size,)) + spec[2:] = (noise**2.)*((np.pi/180./60.)**2.)*2.*np.pi/ells[2:]/(ells+1.)[2:] + return spec + raise NotImplementedError + +def get_spectra(yml_section,lmax,lmax_pad): + spectra = {} + for s in yml_section: + spectra[s['name']] = get_spectrum(s['type'],s['noise'],lmax,lmax_pad) + return spectra + +def get_geometries(yml_section): + geos = {} + for g in yml_section: + if g['type']=='fullsky': + geos[g['name']] = enmap.fullsky_geometry(res=np.deg2rad(g['res_arcmin']/60.),proj=g['proj']) + elif g['type']=='pickle': + geos[g['name']] = pickle.load(open(DATA_PREFIX+"%s"%g['filename'],'rb')) + else: + raise NotImplementedError + return geos + +def generate_map(shape,wcs,powspec,lmax,seed): + return curvedsky.rand_map(shape, wcs, powspec, lmax=lmax, dtype=np.float64, seed=seed, spin=[0,2], method="auto", verbose=False) + +def check_equality(imap1,imap2): + assert np.all(imap1.shape==imap2.shape) + assert wcsutils.equal(imap1.wcs,imap2.wcs) + try: + assert np.all(np.isclose(imap1,imap2)) + except: + from orphics import io + io.plot_img(imap1,"i1.png",lim=[-1.5,2]) + io.plot_img(imap2,"i2.png",lim=[-1.5,2]) + io.plot_img((imap1-imap2)/imap1,"ip.png",lim=[-0.1,0.1]) + assert 1==0 + + +def get_extraction_test_results(yaml_file): + print("Starting tests from ",yaml_file) + with open(yaml_file) as f: + config = yaml.safe_load(f) + geos = get_geometries(config['geometries']) + lmax = config['lmax'] ; lmax_pad = config['lmax_pad'] + spectra = get_spectra(config['spectra'],lmax,lmax_pad) + seed = config['seed'] + + results = {} + for g in geos.keys(): + results[g] = {} + for s in spectra.keys(): + results[g][s] = {} + imap = generate_map(geos[g][0][-2:],geos[g][1],spectra[s],lmax,seed) + + # Do write and read test + filename = "temporary_map.fits" # NOT THREAD SAFE + enmap.write_map(filename,imap) + imap_in = enmap.read_map(filename) + check_equality(imap,imap_in) + for e in config['extracts']: + print("Doing test for extract ",e['name']," with geometry ",g," and spectrum ",s,"...") + if e['type']=='slice': + box = np.deg2rad(np.array(e['box_deg'])) + cutout = enmap.read_map(filename,box=box) + cutout_internal = imap.submap(box=box) + check_equality(cutout,cutout_internal) + pixels = get_reference_pixels(cutout.shape) + results[g][s]['refpixels'] = get_pixel_values(cutout,pixels) + results[g][s]['meansquare'] = get_meansquare(cutout) + + os.remove(filename) + return results,config['result_name'] -TEST_DIR = ptests.TEST_DIR -DATA_PREFIX = ptests.DATA_PREFIX lens_version = '071123' def get_offset_result(res=1.,dtype=np.float64,seed=1): @@ -328,7 +492,7 @@ def test_pixels(self): """Runs reference pixel and mean-square comparisons on extracts from randomly generated maps""" print("Testing reference pixels...") - results,rname = ptests.get_extraction_test_results(TEST_DIR+"/tests.yml") + results,rname = get_extraction_test_results(TEST_DIR+"/tests.yml") cresults = pickle.load(open(DATA_PREFIX+"%s.pkl" % rname,'rb')) assert sorted(results.keys())==sorted(cresults.keys()) for g in results.keys(): @@ -957,7 +1121,3 @@ def test_tilemap(self): assert m1c.geometry.nactive == 2 assert np.allclose(m1c, 1) - -if __name__ == '__main__': - unittest.main() - test_sim_slice() diff --git a/pixell/tests/tests.yml b/tests/tests.yml similarity index 100% rename from pixell/tests/tests.yml rename to tests/tests.yml