Merge branch 'spacetelescope:main' into fix-ifu

.github/workflows/ci_deprecate.yml

@@ -0,0 +1,66 @@
+name: Deprecate Notebook
+
+on:
+  workflow_dispatch:
+    inputs:
+      notebook_name:
+        description: 'The name of the notebook to deprecate (e.g., example.ipynb)'
+        required: true
+        default: 'example.ipynb'
+
+jobs:
+  deprecate:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Find notebook path
+        id: find_path
+        run: |
+          NOTEBOOK_NAME="${{ github.event.inputs.notebook_name }}"
+          NOTEBOOK_PATH=$(find ./notebooks -name "$NOTEBOOK_NAME" -type f)
+          if [ -z "$NOTEBOOK_PATH" ]; then
+            echo "::error::Notebook '${NOTEBOOK_NAME}' not found in the notebooks directory."
+            exit 1
+          fi
+          echo "notebook_path=$NOTEBOOK_PATH" >> $GITHUB_ENV
+
+      # - name: Check for deprecated tag
+      #  id: check_deprecated
+      #  run: |
+      #    notebook_path="${{ env.notebook_path }}"
+      #    if jq '.metadata.deprecated == true' "$notebook_path"; then
+      #      echo "::error::Notebook '${{ env.notebook_path }}' is already flagged as deprecated."
+      #     exit 0
+      #    fi
+
+      - name: Add deprecated tag with timestamp and removal date
+        run: |
+          notebook_path="${{ env.notebook_path }}"
+          timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+          removal_date=$(date -u -d "$timestamp + 30 days" +"%Y-%m-%dT%H:%M:%SZ")
+          jq --arg ts "$timestamp" --arg rd "$removal_date" \
+            '.metadata.deprecated = { "status": true, "timestamp": $ts, "removal_date": $rd }' \
+            "$notebook_path" > temp.ipynb && mv temp.ipynb "$notebook_path"
+      
+      - name: Add deprecation banner with timestamp and removal date
+        run: |
+          notebook_path="${{ env.notebook_path }}"
+          timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+          removal_date=$(date -u -d "$timestamp + 30 days" +"%Y-%m-%d")
+          BANNER_CELL=$(jq -n \
+            --arg text "<div style='border: 3px solid red; padding: 10px; text-align: center; font-weight: bold; color: red;'>⚠️ This notebook is scheduled for deprecation as of $timestamp and is planned for removal by $removal_date. Future use is discouraged.</div>" \
+            '{"cell_type": "markdown", "metadata": {"deprecation": true}, "source": [$text]}')
+          jq ".cells |= [$BANNER_CELL] + ." "$notebook_path" > temp.ipynb && mv temp.ipynb "$notebook_path"
+
+
+      - name: Commit and push to gh-storage branch
+        run: |
+          git config --global user.name "github-actions"
+          git config --global user.email "github-actions@github.com"
+          git checkout -B gh-storage
+          git add "${{ env.notebook_path }}"
+          git commit -m "Deprecate notebook ${{ env.notebook_path }}"
+          git push origin gh-storage --force

.github/workflows/ci_move_deprecated_notebooks.yml

@@ -0,0 +1,82 @@
+name: Move Deprecated Notebooks
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 3 * * *'  # Runs daily at 3 AM UTC
+
+jobs:
+  check_and_move:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          ref: gh-storage  # Start from the gh-storage branch
+
+      - name: Set up date variables
+        id: date_setup
+        run: |
+          CURRENT_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+          echo "current_date=$CURRENT_DATE" >> $GITHUB_ENV
+      - name: Find all deprecated notebooks
+        id: find_deprecated
+        run: |
+          # Find all notebooks with the deprecation flag set
+          DEPRECATED_NOTEBOOKS=$(find ./notebooks -name "*.ipynb" -exec jq -r 'select(.metadata.deprecated.status == true) | input_filename' {} \;)
+          if [ -z "$DEPRECATED_NOTEBOOKS" ]; then
+            echo "No deprecated notebooks found."
+            exit 0
+          fi
+          echo "deprecated_notebooks=$DEPRECATED_NOTEBOOKS" >> $GITHUB_ENV
+      - name: Process deprecated notebooks
+        run: |
+          current_date="${{ env.current_date }}"
+          deprecated_notebooks="${{ env.deprecated_notebooks }}"
+          
+          for notebook_path in $deprecated_notebooks; do
+            # Extract the removal date from the notebook metadata
+            removal_date=$(jq -r '.metadata.deprecated.removal_date' "$notebook_path")
+            # Check if the current date is past the removal date
+            if [[ "$current_date" > "$removal_date" ]]; then
+              echo "Notebook $notebook_path is past the deprecation date ($removal_date). Moving to 'deprecated' branch."
+              # Determine the notebook's directory
+              notebook_dir=$(dirname "$notebook_path")
+            else
+              echo "Notebook $notebook_path is not past the deprecation date ($removal_date)."
+            fi
+          done
+              # Checkout deprecated branch
+              git fetch origin deprecated
+              git checkout deprecated || git checkout -b deprecated
+
+              # Move the entire folder to the deprecated branch
+              git mv "$notebook_dir" .
+
+              # Commit changes on deprecated branch
+              git add .
+              git commit -m "Moved deprecated notebook $notebook_path to deprecated branch"
+
+              # Push changes to the deprecated branch
+              git push origin deprecated
+
+              # Checkout main branch
+              git checkout main
+
+              # Remove the folder from main
+              git rm -r "$notebook_dir"
+
+              # Commit changes on main branch
+              git add .
+              git commit -m "Removed deprecated notebook $notebook_path from main branch"
+
+              # Push changes to the main branch
+              git push origin main
+
+              # Return to gh-storage branch
+              git checkout gh-storage
+            else
+              echo "Notebook $notebook_path is not past the deprecation date ($removal_date)."
+            fi
+          done

.github/workflows/weekly_html_accessibility_check.yml

@@ -3,11 +3,26 @@ on:
   schedule:
     - cron: '0 4 * * 0' # 0400 UTC every Sunday
   workflow_dispatch:
+    inputs:
+      total_error_limit:
+        required: false
+        description: 'The maximum total allowed number of HTML accessibility errors. To skip this testing requirement, enter the value "-1". If not explicitly specified, the default value is "0".'
+        default: 0
+        type: string
+      total_warning_limit:
+        required: false
+        description: 'The maximum total allowed number of HTML accessibility warnings. To skip this testing requirement, enter the value "-1". If not explicitly specified, the default value is "0".'
+        default: 0
+        type: string
 
 jobs:
   Scheduled:
-   uses: spacetelescope/notebook-ci-actions/.github/workflows/html_accessibility_check.yml@main
+   uses: spacetelescope/notebook-ci-actions/.github/workflows/html_accessibility_check.yml@v3
    with:
      target_url: https://spacetelescope.github.io/${{ github.event.repository.name }}/
+     python-version: ${{ vars.PYTHON_VERSION }}
+     total_error_limit: ${{ inputs.total_error_limit || 0 }}
+     total_warning_limit: ${{ inputs.total_warning_limit || 0 }}
+
    secrets:
      A11YWATCH_TOKEN: ${{ secrets.A11YWATCH_TOKEN }}

.gitignore

@@ -9,6 +9,15 @@ __pycache__/
 # C extensions
 *.so
 
+
+
+# FITS files
+*.fits
+
+
+
+
+
 # Distribution / packaging
 .Python
 build/

notebooks/MIRI/MIRI_LRS_spectral_extraction/miri_lrs_advanced_extraction_part1.ipynb

@@ -15,6 +15,7 @@
     "**Cross-intrument:** NIRSpec, MIRI.<br>\n",
     "**Documentation:** This notebook is part of a STScI's larger [post-pipeline Data Analysis Tools Ecosystem](https://jwst-docs.stsci.edu/jwst-post-pipeline-data-analysis) and can be [downloaded](https://github.com/spacetelescope/dat_pyinthesky/tree/main/jdat_notebooks/MRS_Mstar_analysis) directly from the [JDAT Notebook Github directory](https://github.com/spacetelescope/jdat_notebooks).<br>\n",
     "\n",
+    "\n",
     "### Introduction: Spectral extraction in the JWST calibration pipeline\n",
     "\n",
     "The JWST calibration pipeline performs spectrac extraction for all spectroscopic data using basic default assumptions that are tuned to produce accurately calibrated spectra for the majority of science cases. This default method is a simple fixed-width boxcar extraction, where the spectrum is summed over a number of pixels along the cross-dispersion axis, over the valid wavelength range. An aperture correction is applied at each pixel along the spectrum to account for flux lost from the finite-width aperture. \n",
@@ -144,7 +145,7 @@
     "if not os.path.exists(\"data/\"):\n",
     "    print(\"Unpacking Data\")\n",
     "    with tarfile.open('./data.tar.gz', \"r:gz\") as tar:\n",
-    "        tar.extractall()"
+    "        tar.extractall(filter='data')"
    ]
   },
   {

notebooks/MIRI/MIRI_LRS_spectral_extraction/requirements.txt

@@ -1,5 +1,4 @@
 jdaviz >= 3.6.0
 astropy >= 5.3.1
 jwst >= 1.11.3
-specreduce >= 1.3.0
-
+specreduce >= 1.3.0