Updating branch with main

.gitattributes

@@ -1,2 +1,4 @@
-tutorial/* linguist-vendored
+tutorial/* linguist-documentation=true
+*.ipynb linguist-documentation=true
+docs/* linguist-documentation=true
 src/PyHyperScattering/_version.py export-subst

.github/workflows/codeql-analysis.yml

@@ -43,7 +43,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -54,7 +54,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v2
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -68,4 +68,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2

.github/workflows/main.yml

@@ -1,14 +1,18 @@
-name: PyHyperScattering
+name: PyHyperScattering CI Testing
 
-on: [push]
+on: 
+  push:
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ main ]
 
 jobs:
   build:
 
     runs-on: ${{ matrix.os}}
     strategy:
       matrix:
-        python-version: [ '3.7', '3.8', '3.9', '3.10']
+        python-version: ['3.10','3.11']
         os: [ubuntu-latest, macOS-latest, windows-latest]
 
     steps:
@@ -33,6 +37,7 @@ jobs:
         pip install flake8 pytest coverage
         pip install pybind11 wheel
         pip install -r requirements.txt
+        pip install -r requirements-bluesky.txt
         pip install dask
     - name: Lint with flake8
       run: |
@@ -49,6 +54,12 @@ jobs:
         unzip cyrsoxs-example.zip
         wget https://github.com/usnistgov/PyHyperScattering/releases/download/0.0.0-example-data/mask-test-pack.zip
         unzip mask-test-pack.zip
+        wget https://github.com/usnistgov/PyHyperScattering/releases/download/0.0.0-example-data/PSS300nm_C_ccd100.zip
+        unzip PSS300nm_C_ccd100.zip
+        wget https://github.com/usnistgov/PyHyperScattering/releases/download/0.0.0-example-data/smi_example.zip
+        unzip smi_example.zip
+        wget https://github.com/usnistgov/PyHyperScattering/releases/download/0.0.0-example-data/CMS_giwaxs_series.zip
+        unzip CMS_giwaxs_series.zip
     - name: Fetch and unzip example data (Windows)
       if: ${{ matrix.os == 'windows-latest' }}
       run: |
@@ -58,6 +69,12 @@ jobs:
         unzip cyrsoxs-example.zip
         C:\msys64\usr\bin\wget.exe https://github.com/usnistgov/PyHyperScattering/releases/download/0.0.0-example-data/mask-test-pack.zip
         unzip mask-test-pack.zip
+        C:\msys64\usr\bin\wget.exe https://github.com/usnistgov/PyHyperScattering/releases/download/0.0.0-example-data/PSS300nm_C_ccd100.zip
+        unzip PSS300nm_C_ccd100.zip
+        C:\msys64\usr\bin\wget.exe https://github.com/usnistgov/PyHyperScattering/releases/download/0.0.0-example-data/smi_example.zip
+        unzip smi_example.zip
+        C:\msys64\usr\bin\wget.exe https://github.com/usnistgov/PyHyperScattering/releases/download/0.0.0-example-data/CMS_giwaxs_series.zip
+        unzip CMS_giwaxs_series.zip
     - name: Test with pytest
       run: |
         #pytest -v
@@ -70,7 +87,7 @@ jobs:
     - name: Upload coverage report
       uses: actions/upload-artifact@v2
       with:
-        name: coverage-${{ matrix.python-version }}
+        name: coverage-${{ matrix.os}}-${{ matrix.python-version }}.html
         path: htmlcov/index.html
       # Use always() to always run this step to publish test results when there are test failures
       if: ${{ always() }}

.github/workflows/publish-docs.yml

@@ -0,0 +1,15 @@
+name: "Publish Documentation to NIST Pages"
+
+on: [push, pull_request, delete]
+
+jobs:
+  docs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: usnistgov/Docs4NIST@0.5
+        with:
+          docs-folder: docs/
+          formats: |-
+            epub
+            pdf
+          pip-requirements: 'requirements.txt'

.gitignore

@@ -12,18 +12,18 @@ venv/*
 OrientationRunDec2019/**
 **/*.p
 **/.*
-**/*.ipynb
 *.png
 *.h5
 *.nxs
+.ipynb_checkpoints
 Example/**
 example-data/**
 src/__pycache__
 *.egg-info
 src/PyHyperScattering/__pycache__
 src/PyHyperScattering/.ipynb_checkpoints/*
 dist
-docs/_build/*
+dcs/_build/*
 docs/_build/html/*
 *.pyc
 example_data/11012/Dark_56367-AI.txt

README.md

@@ -1,10 +1,12 @@
-# PyHyperScattering
+![# PyHyperScattering](https://user-images.githubusercontent.com/875623/234083915-e62ee2d4-ad7f-4d91-a847-18fa54652ffc.png)
+
 Python utilities for loading, reducing, slicing, and plotting hyperspectral scattering datasets.
 
 This is a package approaching scope-completeness, but still under extremely active development and notably without any guarantee of API stability (we do try to not change things without reason, though). An increasing number of parts of it are unit-tested for functionality, stability, and/or scientific correctness, but large swaths are not. Its documentation is certainly lacking and we are actively seeking help in generating good documentation (see the Issues page on github for areas).  Use at your own risk.  If you're interested in contributing, please contact Peter Beaucage (peter.beaucage@nist.gov).
 
+The (quite incomplete) documentation is located at https://pages.nist.gov/PyHyperScattering, and the tutorials in the repository are occasionally helpful.  Several core developers are active on the NIST RSoXS slack, Nikea, and NSLS2 slacks and welcome DMs with questions, or email Peter Beaucage.
+
 [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/usnistgov/PyHyperScattering/HEAD)
-[![Documentation Status](https://readthedocs.org/projects/pyhyperscattering/badge/?version=latest)](https://pyhyperscattering.readthedocs.io/en/latest/?badge=latest)
 ![Unit Tests](https://github.com/usnistgov/PyHyperScattering/actions/workflows/main.yml/badge.svg)
 ![CodeQL](https://github.com/usnistgov/PyHyperScattering/actions/workflows/codeql-analysis.yml/badge.svg)
 

docs/conf.py

@@ -12,17 +12,21 @@
 #
 import os
 import sys
-sys.path.insert(0, os.path.abspath('../src/'))
+
+sys.path.insert(0, os.path.abspath("../src/"))
 
 
 # -- Project information -----------------------------------------------------
 
-project = 'PyHyperScattering'
-copyright = ': Official Contribution of the US Government.  Not subject to copyright in the United States.'
-author = 'Peter Beaucage'
+project = "PyHyperScattering"
+copyright = (
+    ": Official Contribution of the US Government.  Not subject to copyright in the United States."
+)
+author = "Peter Beaucage"
 
 # The full version, including alpha/beta/rc tags
 from PyHyperScattering import __version__
+
 release = __version__
 
 
@@ -32,31 +36,54 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-'sphinx.ext.autodoc',
-'sphinx.ext.coverage',
-'sphinx.ext.napoleon'
+    "sphinx.ext.autodoc",
+    "sphinx.ext.coverage",
+    "sphinx.ext.napoleon",  # numpydoc and google docstrings
 ]
 
+# Ignore annoying type exception warnings which often come from newlines
+nitpick_ignore = [("py:class", "type")]
+
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
 
 # -- Options for HTML output -------------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'sphinx_rtd_theme'
+# html_theme = 'alabaster'
+
+html_theme = "pydata_sphinx_theme"
+html_logo = "source/_static/Logo_PyHyperO9_Light.svg"
+html_theme_options = {
+    "logo": {
+        "image_light": "source/_images/Logo_PyHyperO9_Light.svg",
+        "image_dark": "source/_images/Logo_PyHyperO10_Dark.svg",
+    },
+    "github_url": "https://github.com/usnistgov/PyHyperScattering",
+    "collapse_navigation": True,
+    #   "external_links": [
+    #       {"name": "Learn", "url": "https://numpy.org/numpy-tutorials/"},
+    #       {"name": "NEPs", "url": "https://numpy.org/neps"}
+    #       ],
+    "header_links_before_dropdown": 6,
+    # Add light/dark mode and documentation version switcher:
+    "navbar_end": ["theme-switcher", "navbar-icon-links"],
+
+}
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["source/_static"]
 
 import sys
-sys.path.append('../src/')
+
+sys.path.append("../src/")

docs/index.rst

@@ -3,31 +3,60 @@
    You can adapt this file completely to your liking, but it should at least
    contain the root `toctree` directive.
 
-Welcome to PyHyperScattering's documentation!
-=============================================
 
-To play with PyHyperScattering's tutorial notebooks interactively, use Binder:
-.. image:: https://mybinder.org/badge_logo.svg
- :target: https://mybinder.org/v2/gh/usnistgov/PyHyperScattering/HEAD
+.. only:: html 
 
- At this stage, the tutorials may be a better overview than this (incomplete, but growing) documentation.
-
-
-.. toctree::
-   :maxdepth: 4
-   :caption: Contents:
-
-   source/intro
-   source/loading
-   source/integration
-   source/learning-to-fly
-   source/utilities
-   source/modules
+    .. image:: /source/_static/Logo_PyHyperO10_Dark.svg
+        :class: only-dark
+        :alt: PyHyperScattering Logo
+        :width: 800
 
+    .. image:: /source/_static/Logo_PyHyperO9_Light.svg
+        :class: only-light
+        :alt: PyHyperScattering Logo
+        :width: 800
 
-
-Indices and tables
-==================
+.. toctree::
+   :maxdepth: 1
+   :hidden:
+
+   Getting Started <source/getting_started/idx_getting_started>
+   User Guide <source/user_guide/idx_user_guide>
+   API Reference <source/reference_API/idx_reference_API>
+   Development <source/development/idx_development>
+   Release notes <source/release_notes/idx_release_notes>
+
+About
+-----
+PyHyperScattering aims to make working with hyperspectral x-ray and neutron scattering data easy" 
+to make programs that work with such data a combination of simple, logical commands with minimal 
+‘cruft’. In the era of modern computing, there is no reason you should have to think about for loops 
+and how you’re storing different intermediate data products - you should be able to go immediately 
+from raw data to an analysis with clear commands, punch down to the data you need for your science 
+quickly. The goal is for these tools to make the mechanics of hyperspectral scattering easier and 
+in so doing, more reproducible, explainable, and robust.
+
+PyHyperScattering is an open-source collaboration maintained by the `National Institute of 
+Standards and Technology (NIST) <https://www.nist.gov/>`_. This package is under active 
+development, and the team welcome DMs with questions on the NIST RSoXS slack, Nikea, and NSLS2 slack 
+channels, or by email to `Dr. Peter Beaucage <mailto:peter.beaucage@nist.gov>`_. 
+For more information about contributing, development philosophy, and licensing, 
+see :ref:`the Development page <Development>`.
+
+Documentation
+-------------
+.. list-table:: 
+   :widths: 30 30
+   :header-rows: 0
+
+   * - :ref:`Getting Started <Getting_Started>` Tutorials to help you get your analysis up and running. Beginners should start here. 
+     - :ref:`User Guide <User_Guide>` A collection of How-To guides (recipes) for specific data reduction, analysis, and visualization tasks.
+   * - :ref:`API Reference <Reference_API>` Detailed technical reference; presents documentation at the function, class, and module level. 
+     - :ref:`Development <Development>` Information and resources regarding the scope and development philosophy of this project, along with information on contributing and licensing.
+
+
+Sitemap
+--------
 
 * :ref:`genindex`
 * :ref:`modindex`