Continue 1.0 process

.github/workflows/unit-tests.yml

@@ -25,9 +25,9 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - name: Python 3.11 with devdeps and online catalogs
-            python: '3.11'
-            toxenv: py311-test-devdeps-online
+          - name: Python 3.12 with devdeps and online catalogs
+            python: '3.12-dev'
+            toxenv: py312-test-devdeps-online
 
           - name: Python 3.8 with old deps (online)
             python: '3.8'
@@ -58,13 +58,10 @@ jobs:
       with:
         python-version: ${{ matrix.python }}
         cache: pip
-        cache-dependency-path: '**/setup.cfg'
+        cache-dependency-path: '**/pyproject.toml'
     - name: Install base dependencies
       run: |
-        python -m pip install --upgrade pip
-        python -m pip install tox
-    - name: Print Python, pip, setuptools, and tox versions
-      run: |
+        python -m pip install --upgrade --quiet pip setuptools tox
         python -c "import sys; print(f'Python {sys.version}')"
         python -c "import pip; print(f'pip {pip.__version__}')"
         python -c "import setuptools; print(f'setuptools {setuptools.__version__}')"
@@ -78,7 +75,7 @@ jobs:
         path: .tox
         # setup.cfg and pyproject.toml have versioning info that would
         # impact the tox environment.
-        key: ${{ github.ref }}-${{ matrix.toxenv }}-tox-${{ hashFiles('setup.cfg') }}
+        key: ${{ github.ref }}-${{ matrix.toxenv }}-tox-${{ hashFiles('pyproject.toml') }}
     - name: Cache astropy cache
       uses: actions/cache@v3
       with:
@@ -106,13 +103,13 @@ jobs:
       with:
         path: ~/conda_pkgs_dir
         key:
-          ${{ runner.os }}-${{ runner.arch }}-conda-${{ hashFiles('pyproject.toml', 'setup.cfg') }}
+          ${{ runner.os }}-${{ runner.arch }}-conda-${{ hashFiles('pyproject.toml') }}
     - name: Cache env
       uses: actions/cache@v3
       with:
         path: ${{ env.CONDA }}/envs
         key:
-          ${{ github.ref }}-${{ runner.os }}-${{ runner.arch }}-${{ steps.get-date.outputs.today }}-conda-envs-${{ hashFiles('pyproject.toml', 'setup.cfg') }}
+          ${{ github.ref }}-${{ runner.os }}-${{ runner.arch }}-${{ steps.get-date.outputs.today }}-conda-envs-${{ hashFiles('pyproject.toml') }}
     - uses: conda-incubator/setup-miniconda@v2
       with:
         miniforge-variant: Mambaforge
@@ -148,7 +145,7 @@ jobs:
     - name: Run Pytest and generate coverage
       shell: bash -l {0}
       run: |
-        pytest --pyargs ./tests ./docs --remote-data --cov astropop --cov-config=./setup.cfg
+        pytest --pyargs ./tests ./docs --remote-data --cov astropop --cov-config=./pyproject.toml
         coverage xml -o ./coverage.xml
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v3

astropop/_db.py

@@ -6,7 +6,7 @@
 from astropy.table import Table
 
 from .logger import logger
-from .py_utils import check_iterable, broadcast
+from .py_utils import broadcast
 
 
 __all__ = ['SQLDatabase', 'SQLTable', 'SQLRow', 'SQLColumn', 'SQLColumnMap']
@@ -75,7 +75,7 @@ def add_column(self, name):
 
     def get_column_name(self, item, add_columns=False):
         """Get the column name for a given keyword."""
-        if check_iterable(item):
+        if not np.isscalar(item):
             return [self.get_column_name(i) for i in item]
 
         item = item.lower()
@@ -88,7 +88,7 @@ def get_column_name(self, item, add_columns=False):
 
     def get_keyword(self, item):
         """Get the keyword for a given column."""
-        if check_iterable(item):
+        if not np.isscalar(item):
             return [self.get_keyword(i) for i in item]
 
         item = item.lower()

astropop/astrometry/astrometrynet.py

@@ -5,7 +5,6 @@
 improvements.
 """
 
-
 import os
 import shutil
 from subprocess import CalledProcessError
@@ -249,6 +248,9 @@
         'To astrometry.cfg file. If no depths are given, use these.'
     )
 }
+# These are the parameters that can be set in the astrometry.cfg file
+_conf_file = ['inparallel', 'minwidth', 'maxwidth', 'depths',
+              'add_path', 'autoindex', 'index']
 
 
 def get_options_help():
@@ -621,6 +623,7 @@ def _updated_config(self, config=None):
 
     def _write_config(self, fname, config=None):
         """Create the astrometry.cfg file for the run."""
+        # write the new cfg
         f = open(fname, 'w')
         config = config or self.config
         for k, v in config.items():
@@ -643,12 +646,27 @@ def _pop_config(self, options):
         """Pop the config from options."""
         options = options.copy()
         cfg = {}
-        for i in ['minwidth', 'maxwidth', 'inparallel', 'autoindex', 'index',
-                  'add_path', 'depths']:
+        for i in _conf_file:
             if i in options:
                 cfg[i] = options.pop(i)
         return options, cfg
 
+    def _get_args(self, root, fname, options, output_dir,
+                  correspond):
+        args = [self._command, fname, '--dir', output_dir]
+        # if any of the options is from config file, rewrite it.
+        if any([i in _conf_file for i in options.keys()]):
+            # pop some options and put them in config
+            options, cfg = self._pop_config(options)
+            cfg = self._updated_config(cfg)
+            config_file = os.path.join(output_dir, root + '.cfg')
+            self._write_config(config_file, cfg)
+            args += ['--config', config_file]
+
+        args += self._parse_options(options)
+        args += ['--corr', correspond]
+        return args
+
     def _run_solver(self, filename, options, output_dir=None, **kwargs):
         """Run the astrometry.net localy using the given params.
 
@@ -658,19 +676,12 @@ def _run_solver(self, filename, options, output_dir=None, **kwargs):
         basename = os.path.basename(filename)
         root, _ = os.path.splitext(basename)
         output_dir, tmp_dir = self._get_output_dir(root, output_dir)
-        solved_file = os.path.join(output_dir, root + '.solved')
+        # Ensure the output directory exist
+        os.makedirs(output_dir, exist_ok=True)
         correspond = os.path.join(output_dir, root + '.corr')
-
-        # pop some options and put them in config
-        options, cfg = self._pop_config(options)
-        cfg = self._updated_config(cfg)
-        config_file = os.path.join(output_dir, root + '.cfg')
-        self._write_config(config_file, cfg)
-
-        args = [self._command, filename, '--dir', output_dir]
-        args += ['--config', config_file]
-        args += self._parse_options(options)
-        args += ['--corr', correspond]
+        solved_file = os.path.join(output_dir, root + '.solved')
+        args = self._get_args(root, filename, options, output_dir=output_dir,
+                              correspond=correspond)
 
         try:
             process, _, _ = run_command(args, **kwargs)

astropop/astrometry/coords_utils.py

@@ -4,16 +4,15 @@
 import numpy as np
 from astropy.coordinates import SkyCoord
 from astropy import units
-from ..py_utils import check_iterable
 
 __all__ = ['guess_coordinates']
 
 
 def guess_coordinates(ra, dec, skycoord=True):
     """Try to guess the format or ra and dec passed."""
     # TODO: this needs a refactor!
     # process as lists of coordinates
-    if check_iterable(ra) and check_iterable(dec):
+    if not np.isscalar(ra) and not np.isscalar(dec):
         if len(ra) != len(dec):
             raise ValueError('RA and Dec do not match in dimensions.')
         try:
@@ -22,7 +21,7 @@
             s = None
         except ValueError:
             s = SkyCoord(ra, dec, unit=(units.hourangle, units.deg))
-    elif check_iterable(ra) or check_iterable(dec):
+    elif not np.isscalar(ra) or not np.isscalar(dec):
         raise ValueError('RA and Dec do not match in dimensions.')
 
     # process as single coordinates

astropop/catalogs/_online_tools.py

@@ -7,7 +7,7 @@
 from astropy.coordinates import SkyCoord, Angle
 from astropy.units import UnitTypeError
 from ..astrometry.coords_utils import guess_coordinates
-from ..py_utils import string_fix, process_list
+from ..py_utils import string_fix
 
 
 MAX_PARALLEL_QUERY = 30
@@ -32,7 +32,7 @@ def _fix_query_table(table):
     for i in table.columns:
         tdtype = table[i].dtype.char
         if tdtype in ('b', 'B', 'S', 'a', 'O'):
-            row = process_list(string_fix, table[i])
+            row = [string_fix(r) for r in table[i]]
             table[i] = np.array(row, dtype=str)
     return table
 

astropop/catalogs/vizier.py

@@ -214,7 +214,8 @@ class NewViz(VizierSourcesCatalog):
             def __init__(self, *args, **kwargs):
                 super(NewViz, self).__init__(filename, *args, **kwargs)
 
-        conf = yaml.safe_load(open(filename, 'r'))
+        with open(filename, 'r') as f:
+            conf = yaml.safe_load(f)
         available_filters = conf.get('available_filters', None)
         if available_filters:
             available_filters = list(available_filters.keys())

astropop/file_collection.py

@@ -13,7 +13,6 @@
 from .fits_utils import _fits_extensions, \
                         _fits_extensions_with_compress
 from .framedata import check_framedata
-from .py_utils import check_iterable
 from .logger import logger
 
 __all__ = ['FitsFileGroup']
@@ -25,23 +24,23 @@ def list_fits_files(location, fits_extensions=None,
     if fits_extensions is None:
         fits_extensions = _fits_extensions
 
-    if not check_iterable(fits_extensions):
+    if np.isscalar(fits_extensions):
         fits_extensions = [fits_extensions]
 
     f = []
     for i in fits_extensions:
         files = glob.glob(os.path.join(location, '**/*'+i), recursive=True)
         # filter only glob include
         if glob_include is not None:
-            if not check_iterable(glob_include):
+            if np.isscalar(glob_include):
                 glob_include = [glob_include]
             for inc in glob_include:
                 files = [i for i in files if fnmatch.fnmatch(i, inc)]
         f.extend(files)
 
     # Filter excluded files
     if glob_exclude is not None:
-        if not check_iterable(glob_exclude):
+        if np.isscalar(glob_exclude):
             glob_exclude = [glob_exclude]
         for exc in glob_exclude:
             f = [i for i in f if not fnmatch.fnmatch(i, exc)]

astropop/image/imcombine.py

@@ -7,7 +7,7 @@
 from astropy.stats import mad_std
 
 from ..framedata import FrameData, check_framedata
-from ..py_utils import check_iterable, check_number
+from ..py_utils import check_number
 from ..logger import logger
 from ._tools import merge_header
 
@@ -71,7 +71,7 @@ def _sigma_clip(data, threshold=3, cen_func='median', dev_func='mad_std',
     if check_number(threshold):
         slow = threshold
         shigh = threshold
-    elif check_iterable(threshold):
+    elif not np.isscalar(threshold):
         slow, shigh = threshold
     else:
         raise TypeError(f'Sigma clipping threshold {threshold} not'
@@ -250,7 +250,7 @@ def set_sigma_clip(self, sigma_limits=None,
             self._sigma_dev_func = None
             return
 
-        if check_iterable(sigma_limits):
+        if not np.isscalar(sigma_limits):
             if len(sigma_limits) not in (1, 2):
                 raise ValueError('Invalid sigma clipping thresholds'
                                  r' {sigma_limits}')

astropop/math/_deriv.py

@@ -38,13 +38,6 @@ def _deriv_mod_1(x, y):
         return numerical_derivative(np.mod, 1)(x, y)
 
 
-@np.vectorize
-def _deriv_fabs(x):
-    if x >= 0:
-        return 1
-    return -1
-
-
 @np.vectorize
 def _deriv_copysign(x, y):
     if x >= 0:
@@ -81,14 +74,12 @@ def _deriv_copysign(x, y):
     'arctan2': (lambda y, x: x/(x**2+y**2),  # Correct for x == 0
                 lambda y, x: -y/(x**2+y**2)),  # Correct for x == 0
     'arctanh': (lambda x: 1/(1-x**2)),
-    'copysign': (_deriv_copysign,
-                 lambda x, y: 0),
     'cos': (lambda x: -np.sin(x)),
     'cosh': (np.sinh),
+    'copysign': (_deriv_copysign, lambda x, y: 0),
     'exp': (np.exp),
     'expm1': (np.exp),
     'exp2': (lambda x: np.exp2(x)*np.log(2)),
-    'fabs': (_deriv_fabs),
     'hypot': [lambda x, y: x/np.hypot(x, y),
               lambda x, y: y/np.hypot(x, y)],
     'log': (lambda x: 1/x),  # for np, log=ln
@@ -125,7 +116,7 @@ def propagate_1(func, fx, x, sx):
     if func not in derivatives.keys():
         raise ValueError(f'func {func} not in derivatives.')
 
-    if hasattr(derivatives[func], '__len__'):
+    if np.size(derivatives[func]) != 1:
         raise ValueError(f'func {func} is not a 1 variable function.')
 
     try:
@@ -137,7 +128,7 @@ def propagate_1(func, fx, x, sx):
         if len(shape) == 0:
             return np.nan
         else:
-            return np.empty(shape).fill(np.nan)
+            return np.full(shape, fill_value=np.nan)
 
 
 def propagate_2(func, fxy, x, y, sx, sy):
@@ -163,7 +154,7 @@ def propagate_2(func, fxy, x, y, sx, sy):
     if func not in derivatives.keys():
         raise ValueError(f'func {func} not in derivatives.')
 
-    if len(derivatives[func]) != 2:
+    if np.size(derivatives[func]) != 2:
         raise ValueError(f'func {func} is not a 2 variable function.')
 
     deriv_x, deriv_y = derivatives[func]
@@ -206,7 +197,7 @@ def numerical_derivative(func, arg_ref, step=STEP_SIZE):
     - Implementation based on `uncertainties` package.
     """
     if not callable(func):
-        raise ValueError(f'function {func} not callable.')
+        raise TypeError(f'function {func} not callable.')
 
     # If reference variable is in kwargs (str) instead of args (int).
     change_kwargs = isinstance(arg_ref, str)