Skip to content

Commit 34eef1a

Browse files
haticekaratayhaticekaratay
authored
Merge branch 'main' into NIRSpec_NSClean_BOTS
2 parents f11d2d7 + 7401159 commit 34eef1a

34 files changed

+4449
-10148
lines changed

‎.github/workflows/ci_html_build.yml

Lines changed: 121 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,127 @@
11
name: Build HTML on merge
2-
on:
3-
pull_request:
4-
branches:
5-
- main
6-
types: [closed]
72

8-
workflow_dispatch:
3+
on:
4+
pull_request_target:
5+
types: [closed]
6+
paths:
7+
- '**.ipynb'
98

9+
env:
10+
CASJOBS_PW: ${{ secrets.CASJOBS_PW }}
11+
CASJOBS_USERID: ${{ secrets.CASJOBS_USERID }}
1012
jobs:
11-
Generate_HTML:
13+
prepare_matrix:
14+
if: github.event.pull_request.merged == true
15+
runs-on: ubuntu-latest
16+
outputs:
17+
matrix: ${{ steps.set-matrix.outputs.matrix }}
18+
steps:
19+
- uses: actions/checkout@v3
20+
with:
21+
fetch-depth: 0
22+
ref: ${{ github.event.pull_request.head.sha }}
23+
24+
- id: set-matrix
25+
run: |
26+
files=$(git diff --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} '*.ipynb')
27+
#files_json=$(echo $files | jq -R -s -c 'split("\n")[:-1]')
28+
files_json=$(echo "$files" | jq -R -s -c 'split("\n")[:-1]')
29+
echo "::set-output name=matrix::${files_json}"
30+
31+
execute-notebooks:
1232
if: github.event.pull_request.merged == true
13-
uses: spacetelescope/notebook-ci-actions/.github/workflows/ci_builder.yml@v3
14-
with:
15-
python-version: ${{ vars.PYTHON_VERSION }}
16-
# permissions:
17-
# contents: write
33+
needs: prepare_matrix
34+
runs-on: ubuntu-latest
35+
strategy:
36+
fail-fast: false
37+
matrix:
38+
notebook: ${{fromJson(needs.prepare_matrix.outputs.matrix)}}
39+
40+
steps:
41+
- uses: actions/checkout@v3
42+
43+
- name: Setup Python
44+
uses: actions/setup-python@v3
45+
with:
46+
python-version: ${{ vars.PYTHON_VERSION }}
47+
48+
- name: Install Dependencies (if requirements.txt exists) and Execute Notebook
49+
run: |
50+
notebook="${{ matrix.notebook }}"
51+
dir=$(dirname "$notebook")
52+
if [ -f "$dir/requirements.txt" ]; then
53+
pip install -r "$dir/requirements.txt"
54+
fi
55+
pip install notebook
56+
jupyter nbconvert --to notebook --execute --inplace "$notebook"
57+
58+
- name: Commit modified file on current branch
59+
run: |
60+
git config user.name 'CI Bot'
61+
git config user.email 'action@github.com'
62+
git add "${{ matrix.notebook }}"
63+
git commit -m "Storing executed notebook ${{ matrix.notebook }}"
64+
65+
- name: Checkout only the file to the target branch
66+
run: |
67+
git fetch
68+
git pull
69+
git checkout -f gh-storage
70+
git checkout @{-1} "${{ matrix.notebook }}"
71+
72+
- name: Commit and push modifications to target branch
73+
run: |
74+
git commit -m "Storing executed notebook ${{ matrix.notebook }}"
75+
76+
MAX_RETRIES=5
77+
RETRY_DELAY=10s
78+
for i in $(seq 1 $MAX_RETRIES); do
79+
git push origin gh-storage --force && break || {
80+
echo "Push $i failed... waiting $RETRY_DELAY"
81+
sleep $RETRY_DELAY
82+
}
83+
done
84+
env:
85+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
86+
87+
88+
generate_html:
89+
if: github.event.pull_request.merged == true
90+
needs: execute-notebooks
91+
runs-on: ubuntu-latest
92+
permissions:
93+
contents: write
94+
steps:
95+
- name: Checkout repository
96+
uses: actions/checkout@v4
97+
98+
- name: Set up Python ${{ vars.PYTHON_VERSION }}
99+
uses: actions/setup-python@v5
100+
with:
101+
python-version: ${{ vars.PYTHON_VERSION }}
102+
103+
- name: Install dependencies
104+
run: |
105+
pip install ghp-import
106+
pip install jupyter-book==v0.15.1
107+
pip install myst-nb
108+
pip install astroid
109+
pip install nbval
110+
#pip install bs4
111+
#pip install lxml
112+
## test to bypass the jupyter-book lower version
113+
pip install jsonschema==4.6.0
114+
PATH="${PATH}:${HOME}/.local/bin"
115+
116+
- name: Build HTML
117+
run: |
118+
git fetch
119+
git checkout origin/gh-storage -- notebooks/
120+
jupyter-book build .
18121
122+
# Push the book's HTML to github-pages
123+
- name: GitHub Pages action
124+
uses: peaceiris/actions-gh-pages@v3
125+
with:
126+
github_token: ${{ secrets.GITHUB_TOKEN }}
127+
publish_dir: ./_build/html

‎.github/workflows/ci_runner.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ on:
2525

2626
jobs:
2727
NotebookExecutionValidation:
28-
uses: spacetelescope/notebook-ci-actions/.github/workflows/ci_runner.yml@v3
28+
uses: spacetelescope/notebook-ci-actions/.github/workflows/ci_runner.yml@v4
2929
with:
3030
python-version: ${{ vars.PYTHON_VERSION }}
3131
permissions:

‎_toc.yml

Lines changed: 9 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -49,22 +49,12 @@ parts:
4949
title: RGB images with Imviz
5050
- file: notebooks/specviz_notebookGUI_interaction/specviz_notebook_gui_interaction_redshift.ipynb
5151
title: Specviz Simple Demo
52-
#- file: notebooks/cube_fitting/cube_fitting.ipynb
53-
# title: IFU Cube Line Maps
5452
- caption: MIRI
5553
chapters:
56-
#- file: notebooks/example_notebook
57-
# title: TEST SECTION
58-
# sections:
59-
# - file: notebooks/example_notebook/example_notebook.ipynb
60-
# title: Section 1
6154
- file: notebooks/MRS_Mstar_analysis/JWST_Mstar_dataAnalysis_runpipeline.ipynb
6255
title: MRS Mstar - Run Pipeline
6356
- file: notebooks/MRS_Mstar_analysis/JWST_Mstar_dataAnalysis_PointSourceDetectorBasedExtraction.ipynb
6457
title: MRS Mstar - Optimal Extraction
65-
#sections:
66-
#- file: notebooks/MRS_Mstar_analysis/JWST_Mstar_dataAnalysis_runpipeline.ipynb
67-
# title: Run Pipeline
6858
- file: notebooks/MRS_Mstar_analysis/JWST_Mstar_dataAnalysis_analysis.ipynb
6959
title: MRS Mstar - Data Analysis
7060
- file: notebooks/MIRI_IFU_YSOs_in_the_LMC/isha_nayak_ysos_in_the_lmc.ipynb
@@ -81,38 +71,19 @@ parts:
8171
title: Cross-Filter PSF Matching
8272
- file: notebooks/psf_photometry/NIRCam_PSF_Photometry_Example.ipynb
8373
title: PSF Photometry
84-
#- file: https://github.com/spacetelescope/jdat_notebooks/tree/main/notebooks/preimaging
85-
# title: Preimaging - MIRAGE
86-
#sections:
87-
#- file: notebooks/preimaging/preimaging_01_mirage.ipynb
88-
# title: Preimaging - MIRAGE
89-
#- file: notebooks/preimaging/preimaging_02_calwebb.ipynb
90-
# title: Preimaging - Pipeline Stages 1, 2
91-
#- file: notebooks/preimaging/preimaging_03_association.ipynb
92-
# title: Preimaging - Pipeline Stage 3
9374
- caption: NIRISS
9475
chapters:
9576
- file: notebooks/NIRISS_WFSS_postpipeline/00_Optimal_extraction.ipynb
9677
title: WFSS Spectra - Optimal Extraction
97-
#sections:
98-
#- file: notebooks/NIRISS_WFSS_postpipeline/00. Optimal extraction.ipynb
99-
# title: WFSS Spectra - Optimal Extraction
100-
# 00_Optimal_extraction.ipynb 01_Combine_and_normalize_1D_spectra.ipynb 02_Cross_correlation_template.ipynb 03_Spatially_resolved_emission_line_map.ipynb
10178
- file: notebooks/NIRISS_WFSS_postpipeline/01_Combine_and_normalize_1D_spectra.ipynb
10279
title: WFSS Spectra - Combine and Normalize 1D Spectra
10380
- file: notebooks/NIRISS_WFSS_postpipeline/02_Cross_correlation_template.ipynb
10481
title: WFSS Spectra - Cross-Correlation Template
10582
- file: notebooks/NIRISS_WFSS_postpipeline/03_Spatially_resolved_emission_line_map.ipynb
10683
title: WFSS Spectra - Emission Line Map
107-
#- file: notebooks/niriss_ami_binary/1_niriss_ami_binary.ipynb
108-
# title: AMI - MIRAGE
109-
#sections:
110-
#- file: notebooks/niriss_ami_binary/1_niriss_ami_binary.ipynb
111-
# title: Notebook 1
112-
#- file: notebooks/niriss_ami_binary/2_niriss_ami_binary.ipynb
113-
# title: AMI - Pipeline
114-
#- file: notebooks/soss-transit-spectroscopy/HAT-P-1b-PART3-data-analysis.ipynb
115-
# title: SOSS Transit Spectroscopy
84+
- file: notebooks/niriss_imaging/niriss-imaging-tutorial.ipynb
85+
title: NIRISS Imaging Tutorial
86+
11687
- caption: NIRSpec
11788
chapters:
11889
- file: notebooks/IFU_cube_continuum_fit/NGC4151_FeII_ContinuumFit.ipynb
@@ -129,3 +100,9 @@ parts:
129100
title: BOTS Time Series Observations
130101
- file: notebooks/galaxy_redshift/redshift_fitting.ipynb
131102
title: Redshift and Template Fitting
103+
- file: notebooks/NIRSpec_NSClean/FS_NSClean_example.ipynb
104+
title: FS Products with NSClean
105+
- file: notebooks/NIRSpec_NSClean/IFU_NSClean_example.ipynb
106+
title: IFU Products with NSClean
107+
- file: notebooks/NIRSpec_NSClean/MOS_NSClean_example.ipynb
108+
title: MOS Products with NSClean

0 commit comments

Comments
 (0)