Merge pull request #1 from lsst-dm/tickets/DM-41142

DM-41142: Make metadetect a EUPS package

.github/workflows/lint.yaml

@@ -0,0 +1,13 @@
+name: lint
+
+on:
+  push:
+    branches:
+      - lsst-dev
+      - main
+      - master
+  pull_request:
+
+jobs:
+  call-workflow:
+    uses: lsst/rubin_workflows/.github/workflows/lint.yaml@main

.github/workflows/lsst-tests-ngmix-upstream.yml

@@ -3,15 +3,19 @@ name: lsst-tests-ngmix-upstream
 on:
   push:
     branches:
+      - lsst-dev
+      - main
       - master
   pull_request: null
+  schedule:
+    - cron: 0 23 * * 4
 
 jobs:
   lsst-tests-ngmix-upstream:
     name: lsst-tests-ngmix-upstream
     strategy:
       matrix:
-        pyver: ["3.10"]
+        pyver: ["3.11"]
 
     runs-on: "ubuntu-latest"
 

.github/workflows/lsst-tests.yml

@@ -3,15 +3,19 @@ name: lsst-tests
 on:
   push:
     branches:
+      - lsst-dev
+      - main
       - master
   pull_request: null
+  schedule:
+    - cron: 0 23 * * 4
 
 jobs:
   lsst-tests:
     name: lsst-tests
     strategy:
       matrix:
-        pyver: ["3.10"]
+        pyver: ["3.11"]
 
     runs-on: "ubuntu-latest"
 

.github/workflows/rebase_checker.yaml

@@ -0,0 +1,31 @@
+name: "Check that the 'lsst-dev' branch was not merged into the development branch"
+on: pull_request
+
+jobs:
+  rebase-checker:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check that 'lsst-dev' is not merged into the development branch
+        uses: gsactions/commit-message-checker@v2
+        with:
+          excludeDescription: "true"  # optional: this excludes the description body of a pull request
+          excludeTitle: "true"  # optional: this excludes the title of a pull request
+          checkAllCommitMessages: "true"  # optional: this checks all commits associated with a pull request
+          accessToken: ${{ secrets.GITHUB_TOKEN }}  # github access token is only required if checkAllCommitMessages is true
+          # Check for patterns that emerge from
+          # 1) git pull origin lsst-dev, as well as
+          # 2) hitting "Update branch" button on GitHub.
+          pattern: ^(?!Merge branch 'lsst-dev')
+          error: |
+            "This step failed because you merged the 'lsst-dev' branch into the development branch,
+             likely by clicking the 'Update branch' button on the GitHub pull request page.
+
+             In order to bring this pull request to a mergeable state,
+             update the 'lsst-dev' branch locally and rebase against it.
+             See https://developer.lsst.io/work/flow.html#pushing-code for detailed instructions.
+
+             To avoid this error in the future, rebase against the latest 'lsst-dev' branch
+             by following the instructions above, or
+             use the little down arrow on the right side of 'Update branch' and click
+             'Update with rebase' option."

.github/workflows/shear_test.yml

@@ -3,6 +3,8 @@ name: shear-tests
 on:
   push:
     branches:
+      - lsst-dev
+      - main
       - master
   workflow_dispatch: null
 
@@ -11,7 +13,7 @@ jobs:
     name: shear-tests
     strategy:
       matrix:
-        pyver: ["3.10"]
+        pyver: ["3.11"]
 
     runs-on: "ubuntu-latest"
 

.github/workflows/tests-ngmix-upstream.yml

@@ -3,6 +3,8 @@ name: tests-ngmix-upstream
 on:
   push:
     branches:
+      - lsst-dev
+      - main
       - master
   pull_request: null
 
@@ -11,7 +13,7 @@ jobs:
     name: tests-ngmix-upstream
     strategy:
       matrix:
-        pyver: ["3.9", "3.10"]
+        pyver: ["3.11"]
 
     runs-on: "ubuntu-latest"
 

.github/workflows/tests.yml

@@ -3,6 +3,8 @@ name: tests
 on:
   push:
     branches:
+      - lsst-dev
+      - main
       - master
   pull_request: null
 
@@ -11,7 +13,7 @@ jobs:
     name: tests
     strategy:
       matrix:
-        pyver: ["3.9", "3.10"]
+        pyver: ["3.11"]
 
     runs-on: "ubuntu-latest"
 

.gitignore

@@ -24,6 +24,7 @@ wheels/
 .installed.cfg
 *.egg
 MANIFEST
+version.py
 
 # PyInstaller
 #  Usually these files are written by a python script from a template

README.md

@@ -1,5 +1,41 @@
 # metadetect
 
-[![tests](https://github.com/esheldon/metadetect/actions/workflows/tests.yml/badge.svg)](https://github.com/esheldon/metadetect/actions/workflows/tests.yml) [![shear-tests](https://github.com/esheldon/metadetect/actions/workflows/shear_test.yml/badge.svg)](https://github.com/esheldon/metadetect/actions/workflows/shear_test.yml)
+[![tests](https://github.com/lsst-dm/metadetect/actions/workflows/tests.yml/badge.svg)](https://github.com/lsst-dm/metadetect/actions/workflows/tests.yml)
+[![shear-tests](https://github.com/lsst-dm/metadetect/actions/workflows/shear_test.yml/badge.svg)](https://github.com/lsst-dm/metadetect/actions/workflows/shear_test.yml)
+[![lsst-tests](https://github.com/lsst-dm/metadetect/actions/workflows/lsst-tests.yml/badge.svg)](https://github.com/lsst-dm/metadetect/actions/workflows/lsst-tests.yml)
+[![lsst-tests](https://github.com/lsst-dm/metadetect/actions/workflows/lsst-tests-ngmix-upstream.yml/badge.svg)](https://github.com/lsst-dm/metadetect/actions/workflows/lsst-tests-ngmix-upstream.yml)
 
-Library for meta-detection, combining detection and metacalibration
+Library for meta-detection, combining detection and metacalibration.
+The algorithm is explained in detail in [Sheldon et al., (2020)](https://ui.adsabs.harvard.edu/abs/2020ApJ...902..138S/abstract) and its applicability with LSST data structures is demonstrated using simulations in [Sheldon et al., (2023)](https://ui.adsabs.harvard.edu/abs/2023OJAp....6E..17S/abstract).
+
+## Shared-fork model
+
+This repository is a fork of the original metadetection repository for packaging and distributing the `metadetect` code with, and for use within, LSST Science Pipelines (e.g., in `drp_tasks`).
+
+### Motivation
+
+We use a fork of this repository instead of declaring it as a dependency in `rubin-env` because the LSST-specific code in this repository uses some of the core packages of the LSST Science Pipelines themselves.
+Having a fork allows us keeps the dependency graph cleaner and simpler, while enabling the Science Pipelines
+team members to make any API changes in a consistent manner without breakage.
+Any significant algorithmic change is expected to happen in the upstream package and merged into this fork.
+
+The LSST-specific unit tests cannot be run on Jenkins because it has additional dependencies that are not available within `rubin-env` (e.g., `descwl-shear-sims`).
+However, these tests are the most relevant ones for the LSST organization as they would indicate any breakage.
+Therefore, we run these tests on GitHub Actions at least once weekly, using the latest weekly through `stackvana`.
+While this does not help to catch breakage _before_ it is merged to the default branch, it helps identify it
+soon after the change.
+The workflow failures can be of two types: `ERRORS` typically due to incorrect APIs and `FAILURES` due to inaccurate results.
+The latter may need to be fixed upstream after discussing with the original authors and are generally not within the scope of the LSST DM team.
+
+### Style differences
+
+This package differs from other LSST DM packages in the organization and coding styles.
+Fixing these to adhere to the [LSST dev-guide](https://developer.lsst.io/python/style.html) is unnecessary code churn and makes it harder to pull in changes from the upstream.
+
+
+- The directory organization of the package differs from typical LSST DM repository structure, but follows more of the community standard.
+- The package uses `pytest` for unit tests instead of `unittest` package.
+- Import statements need not be at the beginning of the module. On-the-fly imports are permitted so as to
+not require having all the (optional) dependencies available.
+- This LSST-specific code from this package is imported as `metadetect.lsst` as opposed to `lsst.metadetect`.
+- The docstrings cannot be all parsed by `sphinx`, and cannot result in a clean, fully-built documentation.

SConstruct

@@ -0,0 +1,4 @@
+# -*- python -*-
+from lsst.sconsUtils import scripts
+# Python-only package
+scripts.BasicSConstruct("metadetect", disableCc=True, noCfgFile=True, versionModuleName="%s/version.py")

metadetect/__init__.py

@@ -1,11 +1,13 @@
 # flake8: noqa
-from ._version import __version__
+try:
+    from .version import *  # generated by scons
+except ImportError:
+    from ._version import *  # package is installed with setup.py for GHA
 
 from .metadetect import (
     do_metadetect,
     Metadetect,
 )
-from . import detect
 from . import metadetect
 from . import fitting
 

metadetect/lsst/tests/test_import.py

@@ -0,0 +1,49 @@
+# This file is part of metadetect.
+#
+# LSST Data Management System
+# This product includes software developed by the
+# LSST Project (http://www.lsst.org/).
+# See COPYRIGHT file at the top of the source tree.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the LSST License Statement and
+# the GNU General Public License along with this program.  If not,
+# see <https://www.lsstcorp.org/LegalNotices/>.
+
+import unittest
+
+from lsst.utils.tests import ImportTestCase
+
+
+class MetadetectImportTestCase(ImportTestCase):
+    """Test that every file can be imported.
+
+    metadetect package has dependencies on packages that are not
+    in rubin-env. Routines that needs those packages (for upstream repo)
+    import them on the fly. This test case acts as a place to document those
+    files that can or cannot be imported.
+    """
+
+    PACKAGES = {
+        "metadetect.lsst",
+        "metadetect",
+    }
+    SKIP_FILES = {
+        "metadetect": {
+            # This depends on the meds package, and are not needed for LSST.
+            "detect.py",
+        }
+    }
+
+
+if __name__ == "__main__":
+    unittest.main()

metadetect/metadetect.py

@@ -6,7 +6,6 @@
 from ngmix.gexceptions import BootPSFFailure
 import esutil as eu
 
-from . import detect
 from . import fitting
 from . import procflags
 from . import shearpos
@@ -429,6 +428,8 @@ def _go_bands(self, shear_bands, mcal_res, det_bands):
         return _result
 
     def _go_bands_with_color(self, shear_bands, mcal_res, det_bands):
+        from . import detect
+
         _result = {}
         for shear_str, shear_mbobs in mcal_res.items():
             if not self._fitter_is_wavg[0]:
@@ -750,6 +751,8 @@ def _do_detect(self, mbobs, det_bands):
         """
         use a MEDSifier to run detection
         """
+        from . import detect
+
         t0 = time.time()
         det_mbobs = ngmix.MultiBandObsList()
         for band in det_bands:

metadetect/mfrac.py

@@ -1,7 +1,6 @@
 import ngmix
 import numpy as np
 
-from .detect import CatalogMEDSifier
 from .defaults import BMASK_EDGE
 
 
@@ -45,6 +44,8 @@ def measure_mfrac(
     mfracs : np.ndarray
         The weighted averages at each input location.
     """
+    from .detect import CatalogMEDSifier
+
     if fwhm is None:
         fwhm = 1.2
 

metadetect/tests/SConscript

@@ -0,0 +1,13 @@
+# -*- python -*-
+from lsst.sconsUtils import scripts
+
+# Do not attempt automated test discovery.
+# Limit to metadetect/tests directory, as most tests in metadetect/lsst/tests/
+# cannot be run in rubin-env environment.
+scripts.BasicSConscript.tests(
+    pyList=None,
+    pySingles=[
+        "metadetect/lsst/tests/test_import.py",
+        "metadetect/tests/test_lsst_configs.py",
+    ],
+)