diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 00000000..00be950e --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,13 @@ +name: lint + +on: + push: + branches: + - lsst-dev + - main + - master + pull_request: + +jobs: + call-workflow: + uses: lsst/rubin_workflows/.github/workflows/lint.yaml@main diff --git a/.github/workflows/lsst-tests-ngmix-upstream.yml b/.github/workflows/lsst-tests-ngmix-upstream.yml index fb7a93ae..4acf818c 100644 --- a/.github/workflows/lsst-tests-ngmix-upstream.yml +++ b/.github/workflows/lsst-tests-ngmix-upstream.yml @@ -3,15 +3,19 @@ name: lsst-tests-ngmix-upstream on: push: branches: + - lsst-dev + - main - master pull_request: null + schedule: + - cron: 0 23 * * 4 jobs: lsst-tests-ngmix-upstream: name: lsst-tests-ngmix-upstream strategy: matrix: - pyver: ["3.10"] + pyver: ["3.11"] runs-on: "ubuntu-latest" diff --git a/.github/workflows/lsst-tests.yml b/.github/workflows/lsst-tests.yml index ff2f509c..24944c54 100644 --- a/.github/workflows/lsst-tests.yml +++ b/.github/workflows/lsst-tests.yml @@ -3,15 +3,19 @@ name: lsst-tests on: push: branches: + - lsst-dev + - main - master pull_request: null + schedule: + - cron: 0 23 * * 4 jobs: lsst-tests: name: lsst-tests strategy: matrix: - pyver: ["3.10"] + pyver: ["3.11"] runs-on: "ubuntu-latest" diff --git a/.github/workflows/rebase_checker.yaml b/.github/workflows/rebase_checker.yaml new file mode 100644 index 00000000..2bd02c50 --- /dev/null +++ b/.github/workflows/rebase_checker.yaml @@ -0,0 +1,31 @@ +name: "Check that the 'lsst-dev' branch was not merged into the development branch" +on: pull_request + +jobs: + rebase-checker: + runs-on: ubuntu-latest + + steps: + - name: Check that 'lsst-dev' is not merged into the development branch + uses: gsactions/commit-message-checker@v2 + with: + excludeDescription: "true" # optional: this excludes the description body of a pull request + excludeTitle: "true" # optional: this excludes the title of a pull request + checkAllCommitMessages: "true" # optional: this checks all commits associated with a pull request + accessToken: ${{ secrets.GITHUB_TOKEN }} # github access token is only required if checkAllCommitMessages is true + # Check for patterns that emerge from + # 1) git pull origin lsst-dev, as well as + # 2) hitting "Update branch" button on GitHub. + pattern: ^(?!Merge branch 'lsst-dev') + error: | + "This step failed because you merged the 'lsst-dev' branch into the development branch, + likely by clicking the 'Update branch' button on the GitHub pull request page. + + In order to bring this pull request to a mergeable state, + update the 'lsst-dev' branch locally and rebase against it. + See https://developer.lsst.io/work/flow.html#pushing-code for detailed instructions. + + To avoid this error in the future, rebase against the latest 'lsst-dev' branch + by following the instructions above, or + use the little down arrow on the right side of 'Update branch' and click + 'Update with rebase' option." diff --git a/.github/workflows/shear_test.yml b/.github/workflows/shear_test.yml index 6a4b8b37..1115ba6a 100644 --- a/.github/workflows/shear_test.yml +++ b/.github/workflows/shear_test.yml @@ -3,6 +3,8 @@ name: shear-tests on: push: branches: + - lsst-dev + - main - master workflow_dispatch: null @@ -11,7 +13,7 @@ jobs: name: shear-tests strategy: matrix: - pyver: ["3.10"] + pyver: ["3.11"] runs-on: "ubuntu-latest" diff --git a/.github/workflows/tests-ngmix-upstream.yml b/.github/workflows/tests-ngmix-upstream.yml index d27e2e92..99595dba 100644 --- a/.github/workflows/tests-ngmix-upstream.yml +++ b/.github/workflows/tests-ngmix-upstream.yml @@ -3,6 +3,8 @@ name: tests-ngmix-upstream on: push: branches: + - lsst-dev + - main - master pull_request: null @@ -11,7 +13,7 @@ jobs: name: tests-ngmix-upstream strategy: matrix: - pyver: ["3.9", "3.10"] + pyver: ["3.11"] runs-on: "ubuntu-latest" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 74c87bdd..939bc1cf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -3,6 +3,8 @@ name: tests on: push: branches: + - lsst-dev + - main - master pull_request: null @@ -11,7 +13,7 @@ jobs: name: tests strategy: matrix: - pyver: ["3.9", "3.10"] + pyver: ["3.11"] runs-on: "ubuntu-latest" diff --git a/.gitignore b/.gitignore index a8939884..28784ce4 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ wheels/ .installed.cfg *.egg MANIFEST +version.py # PyInstaller # Usually these files are written by a python script from a template diff --git a/README.md b/README.md index 39da8158..7cd72d7a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,41 @@ # metadetect -[![tests](https://github.com/esheldon/metadetect/actions/workflows/tests.yml/badge.svg)](https://github.com/esheldon/metadetect/actions/workflows/tests.yml) [![shear-tests](https://github.com/esheldon/metadetect/actions/workflows/shear_test.yml/badge.svg)](https://github.com/esheldon/metadetect/actions/workflows/shear_test.yml) +[![tests](https://github.com/lsst-dm/metadetect/actions/workflows/tests.yml/badge.svg)](https://github.com/lsst-dm/metadetect/actions/workflows/tests.yml) +[![shear-tests](https://github.com/lsst-dm/metadetect/actions/workflows/shear_test.yml/badge.svg)](https://github.com/lsst-dm/metadetect/actions/workflows/shear_test.yml) +[![lsst-tests](https://github.com/lsst-dm/metadetect/actions/workflows/lsst-tests.yml/badge.svg)](https://github.com/lsst-dm/metadetect/actions/workflows/lsst-tests.yml) +[![lsst-tests](https://github.com/lsst-dm/metadetect/actions/workflows/lsst-tests-ngmix-upstream.yml/badge.svg)](https://github.com/lsst-dm/metadetect/actions/workflows/lsst-tests-ngmix-upstream.yml) -Library for meta-detection, combining detection and metacalibration +Library for meta-detection, combining detection and metacalibration. +The algorithm is explained in detail in [Sheldon et al., (2020)](https://ui.adsabs.harvard.edu/abs/2020ApJ...902..138S/abstract) and its applicability with LSST data structures is demonstrated using simulations in [Sheldon et al., (2023)](https://ui.adsabs.harvard.edu/abs/2023OJAp....6E..17S/abstract). + +## Shared-fork model + +This repository is a fork of the original metadetection repository for packaging and distributing the `metadetect` code with, and for use within, LSST Science Pipelines (e.g., in `drp_tasks`). + +### Motivation + +We use a fork of this repository instead of declaring it as a dependency in `rubin-env` because the LSST-specific code in this repository uses some of the core packages of the LSST Science Pipelines themselves. +Having a fork allows us keeps the dependency graph cleaner and simpler, while enabling the Science Pipelines +team members to make any API changes in a consistent manner without breakage. +Any significant algorithmic change is expected to happen in the upstream package and merged into this fork. + +The LSST-specific unit tests cannot be run on Jenkins because it has additional dependencies that are not available within `rubin-env` (e.g., `descwl-shear-sims`). +However, these tests are the most relevant ones for the LSST organization as they would indicate any breakage. +Therefore, we run these tests on GitHub Actions at least once weekly, using the latest weekly through `stackvana`. +While this does not help to catch breakage _before_ it is merged to the default branch, it helps identify it +soon after the change. +The workflow failures can be of two types: `ERRORS` typically due to incorrect APIs and `FAILURES` due to inaccurate results. +The latter may need to be fixed upstream after discussing with the original authors and are generally not within the scope of the LSST DM team. + +### Style differences + +This package differs from other LSST DM packages in the organization and coding styles. +Fixing these to adhere to the [LSST dev-guide](https://developer.lsst.io/python/style.html) is unnecessary code churn and makes it harder to pull in changes from the upstream. + + +- The directory organization of the package differs from typical LSST DM repository structure, but follows more of the community standard. +- The package uses `pytest` for unit tests instead of `unittest` package. +- Import statements need not be at the beginning of the module. On-the-fly imports are permitted so as to +not require having all the (optional) dependencies available. +- This LSST-specific code from this package is imported as `metadetect.lsst` as opposed to `lsst.metadetect`. +- The docstrings cannot be all parsed by `sphinx`, and cannot result in a clean, fully-built documentation. diff --git a/SConstruct b/SConstruct new file mode 100644 index 00000000..f7b7c124 --- /dev/null +++ b/SConstruct @@ -0,0 +1,4 @@ +# -*- python -*- +from lsst.sconsUtils import scripts +# Python-only package +scripts.BasicSConstruct("metadetect", disableCc=True, noCfgFile=True, versionModuleName="%s/version.py") diff --git a/metadetect/__init__.py b/metadetect/__init__.py index 1225c511..cd54cb1b 100644 --- a/metadetect/__init__.py +++ b/metadetect/__init__.py @@ -1,11 +1,13 @@ # flake8: noqa -from ._version import __version__ +try: + from .version import * # generated by scons +except ImportError: + from ._version import * # package is installed with setup.py for GHA from .metadetect import ( do_metadetect, Metadetect, ) -from . import detect from . import metadetect from . import fitting diff --git a/metadetect/lsst/tests/test_import.py b/metadetect/lsst/tests/test_import.py new file mode 100644 index 00000000..7b8adb45 --- /dev/null +++ b/metadetect/lsst/tests/test_import.py @@ -0,0 +1,49 @@ +# This file is part of metadetect. +# +# LSST Data Management System +# This product includes software developed by the +# LSST Project (http://www.lsst.org/). +# See COPYRIGHT file at the top of the source tree. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the LSST License Statement and +# the GNU General Public License along with this program. If not, +# see . + +import unittest + +from lsst.utils.tests import ImportTestCase + + +class MetadetectImportTestCase(ImportTestCase): + """Test that every file can be imported. + + metadetect package has dependencies on packages that are not + in rubin-env. Routines that needs those packages (for upstream repo) + import them on the fly. This test case acts as a place to document those + files that can or cannot be imported. + """ + + PACKAGES = { + "metadetect.lsst", + "metadetect", + } + SKIP_FILES = { + "metadetect": { + # This depends on the meds package, and are not needed for LSST. + "detect.py", + } + } + + +if __name__ == "__main__": + unittest.main() diff --git a/metadetect/metadetect.py b/metadetect/metadetect.py index 9f9547cf..911ffa0e 100644 --- a/metadetect/metadetect.py +++ b/metadetect/metadetect.py @@ -6,7 +6,6 @@ from ngmix.gexceptions import BootPSFFailure import esutil as eu -from . import detect from . import fitting from . import procflags from . import shearpos @@ -429,6 +428,8 @@ def _go_bands(self, shear_bands, mcal_res, det_bands): return _result def _go_bands_with_color(self, shear_bands, mcal_res, det_bands): + from . import detect + _result = {} for shear_str, shear_mbobs in mcal_res.items(): if not self._fitter_is_wavg[0]: @@ -750,6 +751,8 @@ def _do_detect(self, mbobs, det_bands): """ use a MEDSifier to run detection """ + from . import detect + t0 = time.time() det_mbobs = ngmix.MultiBandObsList() for band in det_bands: diff --git a/metadetect/mfrac.py b/metadetect/mfrac.py index 473e2c75..72230e2c 100644 --- a/metadetect/mfrac.py +++ b/metadetect/mfrac.py @@ -1,7 +1,6 @@ import ngmix import numpy as np -from .detect import CatalogMEDSifier from .defaults import BMASK_EDGE @@ -45,6 +44,8 @@ def measure_mfrac( mfracs : np.ndarray The weighted averages at each input location. """ + from .detect import CatalogMEDSifier + if fwhm is None: fwhm = 1.2 diff --git a/metadetect/tests/SConscript b/metadetect/tests/SConscript new file mode 100644 index 00000000..fa12e265 --- /dev/null +++ b/metadetect/tests/SConscript @@ -0,0 +1,13 @@ +# -*- python -*- +from lsst.sconsUtils import scripts + +# Do not attempt automated test discovery. +# Limit to metadetect/tests directory, as most tests in metadetect/lsst/tests/ +# cannot be run in rubin-env environment. +scripts.BasicSConscript.tests( + pyList=None, + pySingles=[ + "metadetect/lsst/tests/test_import.py", + "metadetect/tests/test_lsst_configs.py", + ], +) diff --git a/metadetect/tests/test_metadetect.py b/metadetect/tests/test_metadetect.py index 0770633e..6ca0f24c 100644 --- a/metadetect/tests/test_metadetect.py +++ b/metadetect/tests/test_metadetect.py @@ -13,7 +13,6 @@ import ngmix import numpy as np -from .. import detect from .. import metadetect from .. import fitting from .. import procflags @@ -102,6 +101,10 @@ def test_detect(ntrial=1, show=False): """ just test the detection """ + pytest.importorskip("meds") + pytest.importorskip("sxdes") + from .. import detect + rng = np.random.RandomState(seed=45) tm0 = time.time() @@ -143,6 +146,10 @@ def test_detect_masking(ntrial=1, show=False): """ just test the detection """ + pytest.importorskip("meds") + pytest.importorskip("sxdes") + from .. import detect + rng = np.random.RandomState(seed=45) sim = Sim(rng) @@ -209,6 +216,8 @@ def test_metadetect_coadd_faster(model): """ test coadding is faster """ + pytest.importorskip("sxdes") + config = {} config.update(copy.deepcopy(TEST_METADETECT_CONFIG)) del config["model"] @@ -257,6 +266,8 @@ def test_metadetect_smoke(model): """ test full metadetection """ + pytest.importorskip("sxdes") + ntrial = 1 rng = np.random.RandomState(seed=116) @@ -289,6 +300,8 @@ def test_metadetect_uberseg(model): """ test full metadetection """ + pytest.importorskip("sxdes") + ntrial = 1 rng = np.random.RandomState(seed=116) @@ -324,6 +337,8 @@ def test_metadetect_mfrac(model): """ test full metadetection w/ mfrac """ + pytest.importorskip("sxdes") + ntrial = 1 rng = np.random.RandomState(seed=53341) @@ -496,6 +511,8 @@ def test_metadetect_nodet_flags_some(model): ) @pytest.mark.parametrize("model", ["pgauss", "ksigma"]) def test_metadetect_fitter_fwhm_smooth(model): + pytest.importorskip("sxdes") + nband = 3 rng = np.random.RandomState(seed=116) @@ -537,6 +554,8 @@ def test_metadetect_fitter_fwhm_smooth(model): @pytest.mark.parametrize("model", ["pgauss", "ksigma"]) def test_metadetect_fitter_fwhm_reg(model): + pytest.importorskip("sxdes") + nband = 3 rng = np.random.RandomState(seed=116) @@ -577,6 +596,8 @@ def test_metadetect_fitter_fwhm_reg(model): def test_metadetect_fitter_multi_meas(): + pytest.importorskip("sxdes") + nband = 3 rng = np.random.RandomState(seed=116) @@ -653,6 +674,8 @@ def test_metadetect_flux(model, nband, nshear): """ test full metadetection w/ fluxes """ + pytest.importorskip("sxdes") + ntrial = 1 rng = np.random.RandomState(seed=116) @@ -698,6 +721,8 @@ def test_metadetect_multiband(model, det_bands, coadd): """ test full metadetection w/ multiple bands """ + pytest.importorskip("sxdes") + nband = 3 ntrial = 1 rng = np.random.RandomState(seed=116) @@ -762,6 +787,8 @@ def test_metadetect_multiband(model, det_bands, coadd): def test_metadetect_with_color_is_same(): + pytest.importorskip("sxdes") + model = "wmom" nband = 3 ntrial = 1 diff --git a/metadetect/tests/test_mfrac.py b/metadetect/tests/test_mfrac.py index 79782687..e5c3606f 100644 --- a/metadetect/tests/test_mfrac.py +++ b/metadetect/tests/test_mfrac.py @@ -1,10 +1,12 @@ import numpy as np import ngmix - -from ..mfrac import measure_mfrac +import pytest def test_measure_mfrac_neg_bbox(): + pytest.importorskip("meds") + from ..mfrac import measure_mfrac + rng = np.random.RandomState(seed=100) cen = (201-1)/2 mfrac = rng.uniform(size=(201, 201), low=0.2, high=0.8) diff --git a/shear_meas_test/test_shear_meas.py b/shear_meas_test/test_shear_meas.py index c3f429f0..6d9ca469 100644 --- a/shear_meas_test/test_shear_meas.py +++ b/shear_meas_test/test_shear_meas.py @@ -405,6 +405,8 @@ def _color_key_func(fluxes): ] ) def test_shear_meas_color(model, snr, ngrid, ntrial): + pytest.importorskip("sxdes") + nsub = max(ntrial // 100, 8) nitr = ntrial // nsub rng = np.random.RandomState(seed=116) @@ -494,6 +496,8 @@ def test_shear_meas_color(model, snr, ngrid, ntrial): ] ) def test_shear_meas_simple(model, snr, ngrid, ntrial): + pytest.importorskip("sxdes") + nsub = max(ntrial // 128, 8) nitr = ntrial // nsub rng = np.random.RandomState(seed=116) @@ -578,6 +582,8 @@ def test_shear_meas_simple(model, snr, ngrid, ntrial): ] ) def test_shear_meas_timing(model, snr, ngrid): + pytest.importorskip("sxdes") + rng = np.random.RandomState(seed=116) seeds = rng.randint(low=1, high=2**29, size=1) mdet_seeds = rng.randint(low=1, high=2**29, size=1) diff --git a/ups/eupspkg.cfg.sh b/ups/eupspkg.cfg.sh new file mode 100644 index 00000000..8ac50470 --- /dev/null +++ b/ups/eupspkg.cfg.sh @@ -0,0 +1,7 @@ +build() { + # Work around SIP on MacOSX + export DYLD_LIBRARY_PATH=$LSST_LIBRARY_PATH + python setup.py pytest --addopts "metadetect/tests/ metadetect/lsst/tests/test_import.py" + default_build +} + diff --git a/ups/metadetect.table b/ups/metadetect.table new file mode 100644 index 00000000..3c27d181 --- /dev/null +++ b/ups/metadetect.table @@ -0,0 +1,5 @@ +setupRequired(meas_base) +setupRequired(meas_deblender) +setupRequired(pex_exceptions) + +envPrepend(PYTHONPATH, ${PRODUCT_DIR})