From 1a48f263444b332c4b414e69783c6a0dc1c6b2ff Mon Sep 17 00:00:00 2001 From: Peter Beaucage Date: Mon, 24 Apr 2023 14:27:40 -0400 Subject: [PATCH 01/49] Add logo to readme --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 64f1c3ae..1d7259ae 100755 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ -# PyHyperScattering +![# PyHyperScattering](https://user-images.githubusercontent.com/875623/234083915-e62ee2d4-ad7f-4d91-a847-18fa54652ffc.png) + Python utilities for loading, reducing, slicing, and plotting hyperspectral scattering datasets. This is a package approaching scope-completeness, but still under extremely active development and notably without any guarantee of API stability (we do try to not change things without reason, though). An increasing number of parts of it are unit-tested for functionality, stability, and/or scientific correctness, but large swaths are not. Its documentation is certainly lacking and we are actively seeking help in generating good documentation (see the Issues page on github for areas). Use at your own risk. If you're interested in contributing, please contact Peter Beaucage (peter.beaucage@nist.gov). From 6b20b80a6a55f7858d36f9e0892800bc1886d1bd Mon Sep 17 00:00:00 2001 From: Peter Beaucage Date: Fri, 11 Aug 2023 14:24:46 -0400 Subject: [PATCH 02/49] Create doc publish workflow --- .github/workflows/publish-docs.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .github/workflows/publish-docs.yml diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml new file mode 100644 index 00000000..0c046491 --- /dev/null +++ b/.github/workflows/publish-docs.yml @@ -0,0 +1,14 @@ +name: "Publish Documentation to NIST Pages" + +on: [push, pull_request, delete] + +jobs: + docs: + runs-on: ubuntu-latest + steps: + - uses: usnistgov/NISTtheDocs2Death@0.3 + with: + docs-folder: docs/ + formats: |- + epub + pdf From f2fa9859e7eabc80475e09dd1a4c967e81fafdad Mon Sep 17 00:00:00 2001 From: Josh A <34893923+J-avery32@users.noreply.github.com> Date: Sun, 3 Sep 2023 23:27:08 -0700 Subject: [PATCH 03/49] Update SST1RSoXSLoader.py Change equality check to assignment --- src/PyHyperScattering/SST1RSoXSLoader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PyHyperScattering/SST1RSoXSLoader.py b/src/PyHyperScattering/SST1RSoXSLoader.py index e80033a3..cc9c04b6 100644 --- a/src/PyHyperScattering/SST1RSoXSLoader.py +++ b/src/PyHyperScattering/SST1RSoXSLoader.py @@ -167,7 +167,7 @@ def read_json(self,jsonfile): json_dict['sdd'] = data[1]['RSoXS_WAXS_SDD'] else: - json_dict['rsoxs_config'] == 'unknown' + json_dict['rsoxs_config'] = 'unknown' warnings.warn('RSoXS_Config is neither SAXS or WAXS. Check json file',stacklevel=2) if json_dict['sdd'] == None: From 2886a4be43ca0ae87e5e763e3d4db2e56302a14e Mon Sep 17 00:00:00 2001 From: Peter Beaucage Date: Wed, 6 Sep 2023 16:42:01 -0400 Subject: [PATCH 04/49] Add pip requirements file link to NISTthedocs script --- .github/workflows/publish-docs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index 0c046491..0bbe31ef 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -12,3 +12,4 @@ jobs: formats: |- epub pdf + pip-requirements: requirements.txt From be4d2ad714c999e6bc67c7c247a645346ae10671 Mon Sep 17 00:00:00 2001 From: Peter Beaucage Date: Thu, 7 Sep 2023 12:59:35 -0400 Subject: [PATCH 05/49] syntax fix pip requirements --- .github/workflows/publish-docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index 0bbe31ef..afc36053 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -12,4 +12,4 @@ jobs: formats: |- epub pdf - pip-requirements: requirements.txt + pip-requirements: 'requirements.txt' From 840acd0952bef6194fa7a035b94252017edb49a9 Mon Sep 17 00:00:00 2001 From: Peter Beaucage Date: Thu, 7 Sep 2023 13:07:37 -0400 Subject: [PATCH 06/49] Use newer version of NTD2D --- .github/workflows/publish-docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index afc36053..22b79312 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -6,7 +6,7 @@ jobs: docs: runs-on: ubuntu-latest steps: - - uses: usnistgov/NISTtheDocs2Death@0.3 + - uses: usnistgov/NISTtheDocs2Death@0.4 with: docs-folder: docs/ formats: |- From 533b5aae85732917bd1c3435feaf0bed3ccf7d44 Mon Sep 17 00:00:00 2001 From: Peter Beaucage Date: Thu, 7 Sep 2023 13:29:27 -0400 Subject: [PATCH 07/49] Add sphinx rtd theme to dependencies --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 7d7c332f..8fa43a32 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,4 @@ scipy pillow xarray tqdm +sphinx-rtd-theme From 9d220f0a278a87e2eb98734620f6e0e847d8a1d0 Mon Sep 17 00:00:00 2001 From: Peter Beaucage Date: Thu, 7 Sep 2023 16:09:38 -0400 Subject: [PATCH 08/49] Try to fix RTD theme by commenting static dir --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 08f3a2fc..3617c897 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -56,7 +56,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +# try commenting out? html_static_path = ['_static'] import sys sys.path.append('../src/') From 0d29f9a21595b7fee3f2b2550af41b49d339fd61 Mon Sep 17 00:00:00 2001 From: Peter Beaucage Date: Thu, 7 Sep 2023 17:20:39 -0400 Subject: [PATCH 09/49] try explicitly including rtd theme --- docs/conf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 3617c897..784351c6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -34,7 +34,8 @@ extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.coverage', -'sphinx.ext.napoleon' +'sphinx.ext.napoleon', +'sphinx_rtd_theme' ] # Add any paths that contain templates here, relative to this directory. From f394b1f93b31ad49214ad9f41fdb5455aee65530 Mon Sep 17 00:00:00 2001 From: Peter Beaucage Date: Thu, 7 Sep 2023 17:29:11 -0400 Subject: [PATCH 10/49] Update conf.py --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 784351c6..62eac3e2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -57,7 +57,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -# try commenting out? html_static_path = ['_static'] +html_static_path = ['_static'] import sys sys.path.append('../src/') From 16f595c9fab3697141073c5f9842fbc1cec16ca4 Mon Sep 17 00:00:00 2001 From: Peter Beaucage Date: Thu, 7 Sep 2023 17:43:34 -0400 Subject: [PATCH 11/49] Update conf.py --- docs/conf.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 62eac3e2..4da273ee 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -35,7 +35,6 @@ 'sphinx.ext.autodoc', 'sphinx.ext.coverage', 'sphinx.ext.napoleon', -'sphinx_rtd_theme' ] # Add any paths that contain templates here, relative to this directory. @@ -52,7 +51,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'sphinx_rtd_theme' +html_theme = 'piccolo_theme' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, From 42e3a46ea5f397f69d16da788a2cec5adba77c91 Mon Sep 17 00:00:00 2001 From: Peter Beaucage Date: Thu, 7 Sep 2023 17:45:12 -0400 Subject: [PATCH 12/49] Update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8fa43a32..66996210 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,4 +11,4 @@ scipy pillow xarray tqdm -sphinx-rtd-theme +piccolo-theme From 001623135bf0ae6605cd79061ca9bfcebb1200d9 Mon Sep 17 00:00:00 2001 From: Peter Beaucage Date: Fri, 8 Sep 2023 00:05:58 -0400 Subject: [PATCH 13/49] switch to alabaster --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 4da273ee..5f7470b9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -51,7 +51,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'piccolo_theme' +html_theme = 'alabaster' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, From f3d13167bb52303bffa3c28a3aaed31897738fc8 Mon Sep 17 00:00:00 2001 From: Andrew Levin Date: Fri, 8 Sep 2023 15:26:48 -0400 Subject: [PATCH 14/49] in progress --- src/PyHyperScattering/SST1RSoXSLoader.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/PyHyperScattering/SST1RSoXSLoader.py b/src/PyHyperScattering/SST1RSoXSLoader.py index e80033a3..1f7c2b64 100644 --- a/src/PyHyperScattering/SST1RSoXSLoader.py +++ b/src/PyHyperScattering/SST1RSoXSLoader.py @@ -126,8 +126,12 @@ def loadSingleImage(self,filepath,coords=None, return_q=False,image_slice=None,u def read_json(self,jsonfile): json_dict = {} with open(jsonfile) as f: - data = json.load(f) - meas_time =datetime.datetime.fromtimestamp(data[1]['time']) + try: + data = json.load(f) + meas_time = datetime.datetime.fromtimestamp(data[1]['time']) + except KeyError: # Expected for cycle 2 2023 metadata update + data = [0, json.load(f)] # Quick fix to just load as the second element of a list + meas_time = datetime.datetime.fromtimestamp(data[1]['time']) json_dict['sample_name'] = data[1]['sample_name'] if data[1]['RSoXS_Main_DET'] == 'SAXS': json_dict['rsoxs_config'] = 'saxs' @@ -247,9 +251,14 @@ def loadMd(self,filepath): else: cwd = pathlib.Path(dirPath) - json_fname = list(cwd.glob('*.jsonl')) - json_dict = self.read_json(json_fname[0]) - + try: + json_fname = list(cwd.glob('*.jsonl')) + json_dict = self.read_json(json_fname[0]) + except IndexError: # For cycle 2 2023 + + # Changed '*.jsonl' to '*.json' + json_fname = list(cwd.glob('*.json')) + json_dict = self.read_json(json_fname[0]) + baseline_fname = list(cwd.glob('*baseline.csv')) baseline_dict = self.read_baseline(baseline_fname[0]) From 2cf92a81f7181cdc20ce631bd5f4a42b68d428ae Mon Sep 17 00:00:00 2001 From: Andrew Levin Date: Sun, 10 Sep 2023 13:24:12 -0400 Subject: [PATCH 15/49] updated to loader to work for data pre and post cycle 2 2023 --- src/PyHyperScattering/SST1RSoXSLoader.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/PyHyperScattering/SST1RSoXSLoader.py b/src/PyHyperScattering/SST1RSoXSLoader.py index 1f7c2b64..10915853 100644 --- a/src/PyHyperScattering/SST1RSoXSLoader.py +++ b/src/PyHyperScattering/SST1RSoXSLoader.py @@ -9,6 +9,7 @@ import json #from pyFAI import azimuthalIntegrator import numpy as np +# from json.decoder import JSONDecodeError class SST1RSoXSLoader(FileLoader): @@ -125,14 +126,16 @@ def loadSingleImage(self,filepath,coords=None, return_q=False,image_slice=None,u def read_json(self,jsonfile): json_dict = {} - with open(jsonfile) as f: - try: + try: + with open(jsonfile) as f: data = json.load(f) meas_time = datetime.datetime.fromtimestamp(data[1]['time']) - except KeyError: # Expected for cycle 2 2023 metadata update - data = [0, json.load(f)] # Quick fix to just load as the second element of a list + except KeyError: # Expected for cycle 2 2023 metadata update + with open(jsonfile) as f: + data = [0, json.load(f)] # quick fix to put the data in a list as the second element meas_time = datetime.datetime.fromtimestamp(data[1]['time']) - json_dict['sample_name'] = data[1]['sample_name'] + + json_dict['sample_name'] = data[1]['sample_name'] if data[1]['RSoXS_Main_DET'] == 'SAXS': json_dict['rsoxs_config'] = 'saxs' # discrepency between what is in .json and actual @@ -154,7 +157,7 @@ def read_json(self,jsonfile): json_dict['beamcenter_y'] = data[1]['RSoXS_SAXS_BCY'] json_dict['sdd'] = data[1]['RSoXS_SAXS_SDD'] - elif data[1]['RSoXS_Main_DET'] == 'WAXS': + elif (data[1]['RSoXS_Main_DET'] == 'WAXS') | (data[1]['RSoXS_Main_DET'] == 'waxs_det'): # additional name for for cycle 2 2023 json_dict['rsoxs_config'] = 'waxs' if (meas_time > datetime.datetime(2020,11,16)) and (meas_time < datetime.datetime(2021,1,15)): json_dict['beamcenter_x'] = 400.46 @@ -257,7 +260,7 @@ def loadMd(self,filepath): except IndexError: # For cycle 2 2023 + # Changed '*.jsonl' to '*.json' json_fname = list(cwd.glob('*.json')) - json_dict = self.read_json(json_fname[0]) + json_dict = self.read_json(json_fname[0]) baseline_fname = list(cwd.glob('*baseline.csv')) baseline_dict = self.read_baseline(baseline_fname[0]) From 4f94b221140ea24c4b30eab4577fe4c9b1bf8439 Mon Sep 17 00:00:00 2001 From: Andrew Levin Date: Sun, 10 Sep 2023 11:31:34 -0600 Subject: [PATCH 16/49] update comments --- src/PyHyperScattering/SST1RSoXSLoader.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/PyHyperScattering/SST1RSoXSLoader.py b/src/PyHyperScattering/SST1RSoXSLoader.py index 10915853..43f400a6 100644 --- a/src/PyHyperScattering/SST1RSoXSLoader.py +++ b/src/PyHyperScattering/SST1RSoXSLoader.py @@ -9,7 +9,6 @@ import json #from pyFAI import azimuthalIntegrator import numpy as np -# from json.decoder import JSONDecodeError class SST1RSoXSLoader(FileLoader): @@ -126,13 +125,16 @@ def loadSingleImage(self,filepath,coords=None, return_q=False,image_slice=None,u def read_json(self,jsonfile): json_dict = {} - try: + + # Try / except statment to be compatible with local rsoxs data before + # and after SST1 cycle 2 2023 + try: # For earlier cyles with open(jsonfile) as f: data = json.load(f) meas_time = datetime.datetime.fromtimestamp(data[1]['time']) - except KeyError: # Expected for cycle 2 2023 metadata update + except KeyError: # For later cycles (confirmed working for cycle 2 2023) with open(jsonfile) as f: - data = [0, json.load(f)] # quick fix to put the data in a list as the second element + data = [0, json.load(f)] # Quick fix to load the json in a list meas_time = datetime.datetime.fromtimestamp(data[1]['time']) json_dict['sample_name'] = data[1]['sample_name'] @@ -254,12 +256,13 @@ def loadMd(self,filepath): else: cwd = pathlib.Path(dirPath) - try: + # Another try/except statement to be compatible with local data pre and + # post SST1 cycle 2 2023 + try: # For earlier data json_fname = list(cwd.glob('*.jsonl')) json_dict = self.read_json(json_fname[0]) - except IndexError: # For cycle 2 2023 + - # Changed '*.jsonl' to '*.json' - json_fname = list(cwd.glob('*.json')) + except IndexError: # For later data (works for cycle 2 2023) + json_fname = list(cwd.glob('*.json')) # Changed '*.jsonl' to '*.json' json_dict = self.read_json(json_fname[0]) baseline_fname = list(cwd.glob('*baseline.csv')) From 622425c6f9ebb01771b77ee2c2276c2b015d3d4d Mon Sep 17 00:00:00 2001 From: Peter Beaucage Date: Mon, 11 Sep 2023 10:51:09 -0400 Subject: [PATCH 17/49] Add new docs link to readme and remove readthedocs. --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1d7259ae..d540a632 100755 --- a/README.md +++ b/README.md @@ -4,8 +4,9 @@ Python utilities for loading, reducing, slicing, and plotting hyperspectral scat This is a package approaching scope-completeness, but still under extremely active development and notably without any guarantee of API stability (we do try to not change things without reason, though). An increasing number of parts of it are unit-tested for functionality, stability, and/or scientific correctness, but large swaths are not. Its documentation is certainly lacking and we are actively seeking help in generating good documentation (see the Issues page on github for areas). Use at your own risk. If you're interested in contributing, please contact Peter Beaucage (peter.beaucage@nist.gov). +The (quite incomplete) documentation is located at https://pages.nist.gov/PyHyperScattering, and the tutorials in the repository are occasionally helpful. Several core developers are active on the NIST RSoXS slack, Nikea, and NSLS2 slacks and welcome DMs with questions, or email Peter Beaucage. + [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/usnistgov/PyHyperScattering/HEAD) -[![Documentation Status](https://readthedocs.org/projects/pyhyperscattering/badge/?version=latest)](https://pyhyperscattering.readthedocs.io/en/latest/?badge=latest) ![Unit Tests](https://github.com/usnistgov/PyHyperScattering/actions/workflows/main.yml/badge.svg) ![CodeQL](https://github.com/usnistgov/PyHyperScattering/actions/workflows/codeql-analysis.yml/badge.svg) From 2b27bd8ec1c409e6e2090dcd608bbda0a9cfd3af Mon Sep 17 00:00:00 2001 From: Peter Beaucage Date: Mon, 11 Sep 2023 10:51:33 -0400 Subject: [PATCH 18/49] remove old theme from requirements --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 66996210..7d7c332f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,4 +11,3 @@ scipy pillow xarray tqdm -piccolo-theme From f8395f11edc4b36bd995970fa653e7cb0dba399b Mon Sep 17 00:00:00 2001 From: Patel Date: Mon, 11 Sep 2023 13:59:29 -0400 Subject: [PATCH 19/49] Attempt pydata theme --- docs/conf.py | 30 ++++++++++++++++++------------ docs/index.rst | 7 +++---- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 5f7470b9..13751818 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,17 +12,21 @@ # import os import sys -sys.path.insert(0, os.path.abspath('../src/')) + +sys.path.insert(0, os.path.abspath("../src/")) # -- Project information ----------------------------------------------------- -project = 'PyHyperScattering' -copyright = ': Official Contribution of the US Government. Not subject to copyright in the United States.' -author = 'Peter Beaucage' +project = "PyHyperScattering" +copyright = ( + ": Official Contribution of the US Government. Not subject to copyright in the United States." +) +author = "Peter Beaucage" # The full version, including alpha/beta/rc tags from PyHyperScattering import __version__ + release = __version__ @@ -32,18 +36,18 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ -'sphinx.ext.autodoc', -'sphinx.ext.coverage', -'sphinx.ext.napoleon', + "sphinx.ext.autodoc", + "sphinx.ext.coverage", + "sphinx.ext.napoleon", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # -- Options for HTML output ------------------------------------------------- @@ -51,12 +55,14 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' +# html_theme = 'alabaster' +html_theme = "pydata_sphinx_theme" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] import sys -sys.path.append('../src/') + +sys.path.append("../src/") diff --git a/docs/index.rst b/docs/index.rst index 7ac5abad..1390437a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,10 +7,9 @@ Welcome to PyHyperScattering's documentation! ============================================= To play with PyHyperScattering's tutorial notebooks interactively, use Binder: -.. image:: https://mybinder.org/badge_logo.svg - :target: https://mybinder.org/v2/gh/usnistgov/PyHyperScattering/HEAD - - At this stage, the tutorials may be a better overview than this (incomplete, but growing) documentation. +.. image:: mybinder.org/badge_logo.svg + :target: mybinder.org/v2/gh/usnistgov/PyHyperScattering/HEAD +At this stage, the tutorials may be a better overview than this (incomplete, but growing) documentation. .. toctree:: From b56bce3c833269e3c37ca64b66114cbd0e369627 Mon Sep 17 00:00:00 2001 From: Patel Date: Mon, 11 Sep 2023 14:01:39 -0400 Subject: [PATCH 20/49] Attempt pydata theme add to requirements --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 7d7c332f..0ef3748e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,4 @@ scipy pillow xarray tqdm +pydata_sphinx_theme \ No newline at end of file From fbc7ce08d66766a2899ae217ea1676cbff5ba19b Mon Sep 17 00:00:00 2001 From: Patel Date: Mon, 11 Sep 2023 15:40:53 -0400 Subject: [PATCH 21/49] Logo on landing page --- docs/index.rst | 10 + docs/source/_static/Logo_PyHyperO10_Dark.svg | 6458 ++++++++++++++++ docs/source/_static/Logo_PyHyperO9_Light.svg | 7008 ++++++++++++++++++ 3 files changed, 13476 insertions(+) create mode 100644 docs/source/_static/Logo_PyHyperO10_Dark.svg create mode 100644 docs/source/_static/Logo_PyHyperO9_Light.svg diff --git a/docs/index.rst b/docs/index.rst index 1390437a..42d359ee 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,6 +6,16 @@ Welcome to PyHyperScattering's documentation! ============================================= +.. image:: /source/_static/Logo_PyHyperO10_Dark.svg + :width: 600 + :class: only-dark + :alt: PyHyperScattering Logo + +.. image:: /source/_static/Logo_PyHyperO9_Light.svg + :width: 600 + :class: only-light + :alt: PyHyperScattering Logo + To play with PyHyperScattering's tutorial notebooks interactively, use Binder: .. image:: mybinder.org/badge_logo.svg :target: mybinder.org/v2/gh/usnistgov/PyHyperScattering/HEAD diff --git a/docs/source/_static/Logo_PyHyperO10_Dark.svg b/docs/source/_static/Logo_PyHyperO10_Dark.svg new file mode 100644 index 00000000..bbadc22c --- /dev/null +++ b/docs/source/_static/Logo_PyHyperO10_Dark.svg @@ -0,0 +1,6458 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/source/_static/Logo_PyHyperO9_Light.svg b/docs/source/_static/Logo_PyHyperO9_Light.svg new file mode 100644 index 00000000..442e5815 --- /dev/null +++ b/docs/source/_static/Logo_PyHyperO9_Light.svg @@ -0,0 +1,7008 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From a87d02644b0d4ff4df200c8daa4b2c5e9009e31a Mon Sep 17 00:00:00 2001 From: Patel Date: Mon, 11 Sep 2023 17:05:57 -0400 Subject: [PATCH 22/49] BaselineReorg --- docs/conf.py | 19 ++++++++++++++++++- docs/index.rst | 29 ++++++++++++++--------------- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 13751818..73d30902 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -57,11 +57,28 @@ # # html_theme = 'alabaster' html_theme = "pydata_sphinx_theme" +html_logo = "source/_static/Logo_PyHyperO9_Light.svg" +html_theme_options = { + "logo": { + "image_light": "source/_images/Logo_PyHyperO9_Light.svg", + "image_dark": "source/_images/Logo_PyHyperO10_Dark.svg", + }, + "github_url": "https://github.com/usnistgov/PyHyperScattering", + "collapse_navigation": True, + # "external_links": [ + # {"name": "Learn", "url": "https://numpy.org/numpy-tutorials/"}, + # {"name": "NEPs", "url": "https://numpy.org/neps"} + # ], + "header_links_before_dropdown": 6, + # Add light/dark mode and documentation version switcher: + "navbar_end": ["theme-switcher", "navbar-icon-links"], +} + # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] +html_static_path = ["source/_static"] import sys diff --git a/docs/index.rst b/docs/index.rst index 42d359ee..72086706 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,31 +7,30 @@ Welcome to PyHyperScattering's documentation! ============================================= .. image:: /source/_static/Logo_PyHyperO10_Dark.svg - :width: 600 :class: only-dark :alt: PyHyperScattering Logo .. image:: /source/_static/Logo_PyHyperO9_Light.svg - :width: 600 :class: only-light :alt: PyHyperScattering Logo -To play with PyHyperScattering's tutorial notebooks interactively, use Binder: -.. image:: mybinder.org/badge_logo.svg - :target: mybinder.org/v2/gh/usnistgov/PyHyperScattering/HEAD +.. To play with PyHyperScattering's tutorial notebooks interactively, use Binder: +.. .. image:: mybinder.org/badge_logo.svg +.. :target: mybinder.org/v2/gh/usnistgov/PyHyperScattering/HEAD + At this stage, the tutorials may be a better overview than this (incomplete, but growing) documentation. -.. toctree:: - :maxdepth: 4 - :caption: Contents: - - source/intro - source/loading - source/integration - source/learning-to-fly - source/utilities - source/modules +.. .. toctree:: +.. :maxdepth: 4 +.. :caption: Contents: + +.. source/intro +.. source/loading +.. source/integration +.. source/learning-to-fly +.. source/utilities +.. source/modules From 2b27ff8b198e269afa8790e28b602fe377962e26 Mon Sep 17 00:00:00 2001 From: Patel Date: Mon, 11 Sep 2023 17:45:52 -0400 Subject: [PATCH 23/49] reorg skeleton --- docs/index.rst | 36 ++++++++++--------- docs/source/development/index.rst | 2 ++ .../{intro.rst => getting_started/index.rst} | 2 +- .../{ => getting_started}/integration.rst | 0 .../{ => getting_started}/learning-to-fly.rst | 0 docs/source/{ => getting_started}/loading.rst | 0 .../{ => getting_started}/utilities.rst | 0 .../{modules.rst => reference_API/index.rst} | 0 docs/source/release_notes/index.rst | 4 +++ docs/source/user_guide/index.rst | 2 ++ 10 files changed, 29 insertions(+), 17 deletions(-) create mode 100644 docs/source/development/index.rst rename docs/source/{intro.rst => getting_started/index.rst} (99%) rename docs/source/{ => getting_started}/integration.rst (100%) rename docs/source/{ => getting_started}/learning-to-fly.rst (100%) rename docs/source/{ => getting_started}/loading.rst (100%) rename docs/source/{ => getting_started}/utilities.rst (100%) rename docs/source/{modules.rst => reference_API/index.rst} (100%) create mode 100644 docs/source/release_notes/index.rst create mode 100644 docs/source/user_guide/index.rst diff --git a/docs/index.rst b/docs/index.rst index 72086706..66441b84 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,34 +6,38 @@ Welcome to PyHyperScattering's documentation! ============================================= +.. toctree:: + :maxdepth: 1 + :hidden: + + Getting Started + User Guide + API reference + Development + Release notes + .. image:: /source/_static/Logo_PyHyperO10_Dark.svg :class: only-dark :alt: PyHyperScattering Logo + :width: 600 .. image:: /source/_static/Logo_PyHyperO9_Light.svg :class: only-light :alt: PyHyperScattering Logo + :width: 600 -.. To play with PyHyperScattering's tutorial notebooks interactively, use Binder: -.. .. image:: mybinder.org/badge_logo.svg -.. :target: mybinder.org/v2/gh/usnistgov/PyHyperScattering/HEAD - -At this stage, the tutorials may be a better overview than this (incomplete, but growing) documentation. +**TEXT TODO** - -.. .. toctree:: -.. :maxdepth: 4 -.. :caption: Contents: - -.. source/intro -.. source/loading -.. source/integration -.. source/learning-to-fly -.. source/utilities -.. source/modules +Tutorials +--------- +To play with PyHyperScattering's tutorial notebooks interactively, use |Binder|_ +.. |Binder| image:: https://static.mybinder.org/badge_logo.svg +.. _Binder: https://mybinder.org/v2/gh/usnistgov/PyHyperScattering/HEAD +At this stage, the tutorials may be a better overview than this (incomplete, but growing) documentation. + Indices and tables ================== diff --git a/docs/source/development/index.rst b/docs/source/development/index.rst new file mode 100644 index 00000000..079c6552 --- /dev/null +++ b/docs/source/development/index.rst @@ -0,0 +1,2 @@ +Development and Contributing Info: TODO +============================================================== \ No newline at end of file diff --git a/docs/source/intro.rst b/docs/source/getting_started/index.rst similarity index 99% rename from docs/source/intro.rst rename to docs/source/getting_started/index.rst index e8fbc9da..d838f391 100644 --- a/docs/source/intro.rst +++ b/docs/source/getting_started/index.rst @@ -1,4 +1,4 @@ -Introduction +Getting Started ================= PyHyperScattering aims to make working with hyperspectral x-ray and neutron scattering data easy, to make programs that work with such data a combination of simple, logical commands with minimal 'cruft'. In the era of modern computing, there is no reason you should have to think about for loops and how you're storing different intermediate data products - you should be able to go immediately from raw data to an analysis with clear commands, punch down to the data you need for your science quickly. The goal is for these tools to make the mechanics of hyperspectral scattering easier and in so doing, more reproducible, explainable, and robust. diff --git a/docs/source/integration.rst b/docs/source/getting_started/integration.rst similarity index 100% rename from docs/source/integration.rst rename to docs/source/getting_started/integration.rst diff --git a/docs/source/learning-to-fly.rst b/docs/source/getting_started/learning-to-fly.rst similarity index 100% rename from docs/source/learning-to-fly.rst rename to docs/source/getting_started/learning-to-fly.rst diff --git a/docs/source/loading.rst b/docs/source/getting_started/loading.rst similarity index 100% rename from docs/source/loading.rst rename to docs/source/getting_started/loading.rst diff --git a/docs/source/utilities.rst b/docs/source/getting_started/utilities.rst similarity index 100% rename from docs/source/utilities.rst rename to docs/source/getting_started/utilities.rst diff --git a/docs/source/modules.rst b/docs/source/reference_API/index.rst similarity index 100% rename from docs/source/modules.rst rename to docs/source/reference_API/index.rst diff --git a/docs/source/release_notes/index.rst b/docs/source/release_notes/index.rst new file mode 100644 index 00000000..518c269a --- /dev/null +++ b/docs/source/release_notes/index.rst @@ -0,0 +1,4 @@ +Release Notes +================= + +Preparing for v1.0 release, which includes breaking changes that make xarray Datasets the underlying data structure. \ No newline at end of file diff --git a/docs/source/user_guide/index.rst b/docs/source/user_guide/index.rst new file mode 100644 index 00000000..44c1bf01 --- /dev/null +++ b/docs/source/user_guide/index.rst @@ -0,0 +1,2 @@ +USER GUIDE +=========== \ No newline at end of file From b94fc524f85002330169a07b469f22710868ead6 Mon Sep 17 00:00:00 2001 From: Patel Date: Mon, 11 Sep 2023 18:56:36 -0400 Subject: [PATCH 24/49] Fin skeleton --- docs/index.rst | 67 +++++++++++-------- docs/source/development/idx_development.rst | 29 ++++++++ docs/source/development/index.rst | 2 - .../{index.rst => idx_getting_started.rst} | 23 ++++--- docs/source/getting_started/integration.rst | 2 + .../getting_started/learning-to-fly.rst | 2 + docs/source/getting_started/loading.rst | 2 + docs/source/getting_started/utilities.rst | 1 + .../{ => reference_API}/PyHyperScattering.rst | 0 .../{index.rst => idx_reference_API.rst} | 6 +- .../{index.rst => idx_release_notes.rst} | 2 + docs/source/user_guide/idx_user_guide.rst | 5 ++ docs/source/user_guide/index.rst | 2 - 13 files changed, 101 insertions(+), 42 deletions(-) create mode 100644 docs/source/development/idx_development.rst delete mode 100644 docs/source/development/index.rst rename docs/source/getting_started/{index.rst => idx_getting_started.rst} (73%) rename docs/source/{ => reference_API}/PyHyperScattering.rst (100%) rename docs/source/reference_API/{index.rst => idx_reference_API.rst} (51%) rename docs/source/release_notes/{index.rst => idx_release_notes.rst} (88%) create mode 100644 docs/source/user_guide/idx_user_guide.rst delete mode 100644 docs/source/user_guide/index.rst diff --git a/docs/index.rst b/docs/index.rst index 66441b84..48488234 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,19 +3,6 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Welcome to PyHyperScattering's documentation! -============================================= - -.. toctree:: - :maxdepth: 1 - :hidden: - - Getting Started - User Guide - API reference - Development - Release notes - .. image:: /source/_static/Logo_PyHyperO10_Dark.svg :class: only-dark :alt: PyHyperScattering Logo @@ -26,20 +13,46 @@ Welcome to PyHyperScattering's documentation! :alt: PyHyperScattering Logo :width: 600 -**TEXT TODO** - -Tutorials ---------- - -To play with PyHyperScattering's tutorial notebooks interactively, use |Binder|_ - -.. |Binder| image:: https://static.mybinder.org/badge_logo.svg -.. _Binder: https://mybinder.org/v2/gh/usnistgov/PyHyperScattering/HEAD - -At this stage, the tutorials may be a better overview than this (incomplete, but growing) documentation. - -Indices and tables -================== +.. toctree:: + :maxdepth: 1 + :hidden: + + Getting Started + User Guide + API Reference + Development + Release notes + +About +----- +PyHyperScattering aims to make working with hyperspectral x-ray and neutron scattering data easy" +to make programs that work with such data a combination of simple, logical commands with minimal +‘cruft’. In the era of modern computing, there is no reason you should have to think about for loops +and how you’re storing different intermediate data products - you should be able to go immediately +from raw data to an analysis with clear commands, punch down to the data you need for your science +quickly. The goal is for these tools to make the mechanics of hyperspectral scattering easier and +in so doing, more reproducible, explainable, and robust. + +PyHyperScattering is a product of the `National Institute of Standards and Technology (NIST) +`_. This package is under active development, and the team welcome DMs +with questions on the NIST RSoXS slack, Nikea, and NSLS2 slack channels, or by email to +`Dr. Peter Beaucage `_. For more information about contributing, +development philosophy, and licensing, see :ref:`the Development page `. + +Documentation +------------- +.. list-table:: + :widths: 30 30 + :header-rows: 0 + + * - :ref:`Getting Started ` Tutorials to help you get your analysis up and running. Beginners should start here. + - :ref:`User Guide ` A collection of How-To guides (recipes) for specific data reduction, analysis, and visualization tasks. + * - :ref:`API Reference ` Detailed technical reference; presents documentation at the function, class, and module level. + - :ref:`Development ` Information and resources regarding the scope and development philosophy of this project, along with information on contributing and licensing. + + +Sitemap +-------- * :ref:`genindex` * :ref:`modindex` diff --git a/docs/source/development/idx_development.rst b/docs/source/development/idx_development.rst new file mode 100644 index 00000000..708ad2f1 --- /dev/null +++ b/docs/source/development/idx_development.rst @@ -0,0 +1,29 @@ +.. _Development: + +Development and Contributing Info: TODO +============================================================== + +PyHyperScattering is a product of the `National Institute of Standards and Technology (NIST) +`_. This package is under active development, and the team welcome DMs +with questions on the NIST RSoXS slack, Nikea, and NSLS2 slack channels, or by email to +`Dr. Peter Beaucage `_. For more information about contributing, +development philosophy, and licensing, see :ref:`the Development page `. + + +Scope and Package Outline +------------------------- +TODO + +License +------- +This software was developed by employees of the National Institute of Standards and Technology (NIST), an agency of the Federal Government and is being made available as a public service. Pursuant to title 17 United States Code Section 105, works of NIST employees are not subject to copyright protection in the United States. This software may be subject to foreign copyright. Permission in the United States and in foreign countries, to the extent that NIST may hold copyright, to use, copy, modify, create derivative works, and distribute this software and its documentation without fee is hereby granted on a non-exclusive basis, provided that this notice and disclaimer of warranty appears in all copies. +THE SOFTWARE IS PROVIDED 'AS IS' WITHOUT ANY WARRANTY OF ANY KIND, EITHER EXPRESSED, IMPLIED, OR STATUTORY, INCLUDING, BUT NOT LIMITED TO, ANY WARRANTY THAT THE SOFTWARE WILL CONFORM TO SPECIFICATIONS, ANY IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND FREEDOM FROM INFRINGEMENT, AND ANY WARRANTY THAT THE DOCUMENTATION WILL CONFORM TO THE SOFTWARE, OR ANY WARRANTY THAT THE SOFTWARE WILL BE ERROR FREE. IN NO EVENT SHALL NIST BE LIABLE FOR ANY DAMAGES, INCLUDING, BUT NOT LIMITED TO, DIRECT, INDIRECT, SPECIAL OR CONSEQUENTIAL DAMAGES, ARISING OUT OF, RESULTING FROM, OR IN ANY WAY CONNECTED WITH THIS SOFTWARE, WHETHER OR NOT BASED UPON WARRANTY, CONTRACT, TORT, OR OTHERWISE, WHETHER OR NOT INJURY WAS SUSTAINED BY PERSONS OR PROPERTY OR OTHERWISE, AND WHETHER OR NOT LOSS WAS SUSTAINED FROM, OR AROSE OUT OF THE RESULTS OF, OR USE OF, THE SOFTWARE OR SERVICES PROVIDED HEREUNDER. +With respect to the example data package, the following terms apply: +The data/work is provided by NIST as a public service and is expressly provided “AS IS.” NIST MAKES NO WARRANTY OF ANY KIND, EXPRESS, IMPLIED OR STATUTORY, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT AND DATA ACCURACY. NIST does not warrant or make any representations regarding the use of the data or the results thereof, including but not limited to the correctness, accuracy, reliability or usefulness of the data. NIST SHALL NOT BE LIABLE AND YOU HEREBY RELEASE NIST FROM LIABILITY FOR ANY INDIRECT, CONSEQUENTIAL, SPECIAL, OR INCIDENTAL DAMAGES (INCLUDING DAMAGES FOR LOSS OF BUSINESS PROFITS, BUSINESS INTERRUPTION, LOSS OF BUSINESS INFORMATION, AND THE LIKE), WHETHER ARISING IN TORT, CONTRACT, OR OTHERWISE, ARISING FROM OR RELATING TO THE DATA (OR THE USE OF OR INABILITY TO USE THIS DATA), EVEN IF NIST HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. +To the extent that NIST may hold copyright in countries other than the United States, you are hereby granted the non-exclusive irrevocable and unconditional right to print, publish, prepare derivative works and distribute the NIST data, in any medium, or authorize others to do so on your behalf, on a royalty-free basis throughout the world. +You may improve, modify, and create derivative works of the data or any portion of the data, and you may copy and distribute such modifications or works. Modified works should carry a notice stating that you changed the data and should note the date and nature of any such change. Please explicitly acknowledge the National Institute of Standards and Technology as the source of the data: Data citation recommendations are provided at https://www.nist.gov/open/license. +Permission to use this data is contingent upon your acceptance of the terms of this agreement and upon your providing appropriate acknowledgments of NIST’s creation of the data/work. + +Contributing +------------ +Contributions are welcome! Please view our `Contributer Guidelines `_ on github. \ No newline at end of file diff --git a/docs/source/development/index.rst b/docs/source/development/index.rst deleted file mode 100644 index 079c6552..00000000 --- a/docs/source/development/index.rst +++ /dev/null @@ -1,2 +0,0 @@ -Development and Contributing Info: TODO -============================================================== \ No newline at end of file diff --git a/docs/source/getting_started/index.rst b/docs/source/getting_started/idx_getting_started.rst similarity index 73% rename from docs/source/getting_started/index.rst rename to docs/source/getting_started/idx_getting_started.rst index d838f391..25452017 100644 --- a/docs/source/getting_started/index.rst +++ b/docs/source/getting_started/idx_getting_started.rst @@ -1,3 +1,5 @@ +.. _Getting_Started: + Getting Started ================= @@ -9,22 +11,25 @@ It grew out of the NIST RSoXS program, but aims to be broadly applicable. If it The tools PyHyper provides are basically divided into three categories: -loading ----------------- -get your data from a source - files on disk or a network connection to a DataBroker or whatever - into a standardized structure we call a raw xarray. +:ref:`Loading Data ` +------------------------------- +Get your data from a source - files on disk or a network connection to a DataBroker or whatever - into a standardized structure we call a raw xarray. The metadata should be loaded and converted to a standardized set of terms, intensity corrections applied, and the frames stacked along whatever dimensions you want. -integration ----------------- -convert your data from pixel space or qx/qy space to chi-q space - perfect for slicing. generally tools here are built on pyFAI, though some variants also use warp_polar from numpy. +:ref:`Integration ` +----------------------------------- +Convert your data from pixel space or qx/qy space to chi-q space - perfect for slicing. generally tools here are built on pyFAI, though some variants also use warp_polar from numpy. -utility modules ------------------- -these are pre-canned routines for common analyses, such as RSoXS anisotropic ratio or peak/background fitting. the intent here is that the barrier to building your own code is low, and contributions in module space are especially welcomed and encouraged. +:ref:`Custom Analysis ` +------------------------------------ +Slice your data along specified dimensions and visualize the results. +:ref:`Utilities ` +-------------------------------- +These are pre-canned routines for common analyses, such as RSoXS anisotropic ratio or peak/background fitting. the intent here is that the barrier to building your own code is low, and contributions in module space are especially welcomed and encouraged. Have fun! \ No newline at end of file diff --git a/docs/source/getting_started/integration.rst b/docs/source/getting_started/integration.rst index adfce5d5..84d1feb9 100644 --- a/docs/source/getting_started/integration.rst +++ b/docs/source/getting_started/integration.rst @@ -1,3 +1,5 @@ +.. _integration: + Integration: raw intensity to I(q) ==================================== diff --git a/docs/source/getting_started/learning-to-fly.rst b/docs/source/getting_started/learning-to-fly.rst index 2326a225..fdf9f1a2 100644 --- a/docs/source/getting_started/learning-to-fly.rst +++ b/docs/source/getting_started/learning-to-fly.rst @@ -1,3 +1,5 @@ +.. _analysis: + Learning to Fly ================= diff --git a/docs/source/getting_started/loading.rst b/docs/source/getting_started/loading.rst index 86015c77..1be28983 100644 --- a/docs/source/getting_started/loading.rst +++ b/docs/source/getting_started/loading.rst @@ -1,3 +1,5 @@ +.. _loading: + Loading: getting your data into PyHyperScattering ============================================================== diff --git a/docs/source/getting_started/utilities.rst b/docs/source/getting_started/utilities.rst index dae62c06..78f77930 100644 --- a/docs/source/getting_started/utilities.rst +++ b/docs/source/getting_started/utilities.rst @@ -1,3 +1,4 @@ +.. _utilities: Utility Modules ================= diff --git a/docs/source/PyHyperScattering.rst b/docs/source/reference_API/PyHyperScattering.rst similarity index 100% rename from docs/source/PyHyperScattering.rst rename to docs/source/reference_API/PyHyperScattering.rst diff --git a/docs/source/reference_API/index.rst b/docs/source/reference_API/idx_reference_API.rst similarity index 51% rename from docs/source/reference_API/index.rst rename to docs/source/reference_API/idx_reference_API.rst index b5d4e4de..6f5e9db3 100644 --- a/docs/source/reference_API/index.rst +++ b/docs/source/reference_API/idx_reference_API.rst @@ -1,7 +1,9 @@ -API Documentation +.. _Reference_API: + +API Reference ================= .. toctree:: - :maxdepth: 4 + :maxdepth: 3 PyHyperScattering diff --git a/docs/source/release_notes/index.rst b/docs/source/release_notes/idx_release_notes.rst similarity index 88% rename from docs/source/release_notes/index.rst rename to docs/source/release_notes/idx_release_notes.rst index 518c269a..5e4d6e0e 100644 --- a/docs/source/release_notes/index.rst +++ b/docs/source/release_notes/idx_release_notes.rst @@ -1,3 +1,5 @@ +.. _Release_Notes: + Release Notes ================= diff --git a/docs/source/user_guide/idx_user_guide.rst b/docs/source/user_guide/idx_user_guide.rst new file mode 100644 index 00000000..d22ef88e --- /dev/null +++ b/docs/source/user_guide/idx_user_guide.rst @@ -0,0 +1,5 @@ +.. _User_Guide: + +User Guide +=========== +Work in progress cookbook for specific tasks. \ No newline at end of file diff --git a/docs/source/user_guide/index.rst b/docs/source/user_guide/index.rst deleted file mode 100644 index 44c1bf01..00000000 --- a/docs/source/user_guide/index.rst +++ /dev/null @@ -1,2 +0,0 @@ -USER GUIDE -=========== \ No newline at end of file From 225ee49ea0783b7fca486c07c9601654b5d064aa Mon Sep 17 00:00:00 2001 From: Patel Date: Mon, 11 Sep 2023 18:57:22 -0400 Subject: [PATCH 25/49] Contributing Above Skeleton --- docs/source/development/idx_development.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/source/development/idx_development.rst b/docs/source/development/idx_development.rst index 708ad2f1..733716ad 100644 --- a/docs/source/development/idx_development.rst +++ b/docs/source/development/idx_development.rst @@ -14,6 +14,10 @@ Scope and Package Outline ------------------------- TODO +Contributing +------------ +Contributions are welcome! Please view our `Contributer Guidelines `_ on github. + License ------- This software was developed by employees of the National Institute of Standards and Technology (NIST), an agency of the Federal Government and is being made available as a public service. Pursuant to title 17 United States Code Section 105, works of NIST employees are not subject to copyright protection in the United States. This software may be subject to foreign copyright. Permission in the United States and in foreign countries, to the extent that NIST may hold copyright, to use, copy, modify, create derivative works, and distribute this software and its documentation without fee is hereby granted on a non-exclusive basis, provided that this notice and disclaimer of warranty appears in all copies. @@ -24,6 +28,3 @@ To the extent that NIST may hold copyright in countries other than the United St You may improve, modify, and create derivative works of the data or any portion of the data, and you may copy and distribute such modifications or works. Modified works should carry a notice stating that you changed the data and should note the date and nature of any such change. Please explicitly acknowledge the National Institute of Standards and Technology as the source of the data: Data citation recommendations are provided at https://www.nist.gov/open/license. Permission to use this data is contingent upon your acceptance of the terms of this agreement and upon your providing appropriate acknowledgments of NIST’s creation of the data/work. -Contributing ------------- -Contributions are welcome! Please view our `Contributer Guidelines `_ on github. \ No newline at end of file From 141321fd589cd46599062a7ef4a1e9fdf7563335 Mon Sep 17 00:00:00 2001 From: Patel Date: Mon, 11 Sep 2023 19:09:49 -0400 Subject: [PATCH 26/49] fix markdown errors 1 --- docs/source/getting_started/idx_getting_started.rst | 9 +++++++++ docs/source/reference_API/PyHyperScattering.rst | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/source/getting_started/idx_getting_started.rst b/docs/source/getting_started/idx_getting_started.rst index 25452017..2e52248f 100644 --- a/docs/source/getting_started/idx_getting_started.rst +++ b/docs/source/getting_started/idx_getting_started.rst @@ -3,6 +3,15 @@ Getting Started ================= +.. toctree:: + :maxdepth: 1 + :hidden: + + Loading Data + Integration + Custom Analysis + Utilities + PyHyperScattering aims to make working with hyperspectral x-ray and neutron scattering data easy, to make programs that work with such data a combination of simple, logical commands with minimal 'cruft'. In the era of modern computing, there is no reason you should have to think about for loops and how you're storing different intermediate data products - you should be able to go immediately from raw data to an analysis with clear commands, punch down to the data you need for your science quickly. The goal is for these tools to make the mechanics of hyperspectral scattering easier and in so doing, more reproducible, explainable, and robust. It grew out of the NIST RSoXS program, but aims to be broadly applicable. If it's scattering data, you should be able to load it, reduce it, and slice it. diff --git a/docs/source/reference_API/PyHyperScattering.rst b/docs/source/reference_API/PyHyperScattering.rst index 51541037..0bd9ddd5 100644 --- a/docs/source/reference_API/PyHyperScattering.rst +++ b/docs/source/reference_API/PyHyperScattering.rst @@ -77,7 +77,7 @@ PyHyperScattering.integrate module :undoc-members: :show-inheritance: - PyHyperScattering.integrate.WPIntegrator module + PyHyperScattering.integrate.WPIntegrator module -------------------------------------------------------------------------------- .. automodule:: PyHyperScattering.integrate.WPIntegrator From c53966a25f90c532e564a8ee11d188996478f366 Mon Sep 17 00:00:00 2001 From: Patel Date: Mon, 11 Sep 2023 19:18:13 -0400 Subject: [PATCH 27/49] fix sphinx errors 2 --- docs/conf.py | 3 +++ docs/source/getting_started/utilities.rst | 1 + 2 files changed, 4 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index 73d30902..166acd22 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -41,6 +41,9 @@ "sphinx.ext.napoleon", ] +# Ignore annoying type exception warnings which often come from newlines +nitpick_ignore = [("py:class", "type")] + # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] diff --git a/docs/source/getting_started/utilities.rst b/docs/source/getting_started/utilities.rst index 78f77930..a64ca6f1 100644 --- a/docs/source/getting_started/utilities.rst +++ b/docs/source/getting_started/utilities.rst @@ -1,4 +1,5 @@ .. _utilities: + Utility Modules ================= From eeba6de6a947243420727193aa8408a5e50f7142 Mon Sep 17 00:00:00 2001 From: Patel Date: Mon, 11 Sep 2023 19:24:36 -0400 Subject: [PATCH 28/49] Test without autodoc --- .../reference_API/PyHyperScattering.rst | 146 ------------------ 1 file changed, 146 deletions(-) diff --git a/docs/source/reference_API/PyHyperScattering.rst b/docs/source/reference_API/PyHyperScattering.rst index 0bd9ddd5..c1e2cdda 100644 --- a/docs/source/reference_API/PyHyperScattering.rst +++ b/docs/source/reference_API/PyHyperScattering.rst @@ -1,149 +1,3 @@ PyHyperScattering package ========================= -Submodules ----------- - -PyHyperScattering.load module ------------------------------ - -.. automodule:: PyHyperScattering.load - :members: - :undoc-members: - :show-inheritance: - - PyHyperScattering.load.FileLoader module - ---------------------------------------------------- - - .. autoclass:: PyHyperScattering.load.FileLoader - :members: - :undoc-members: - :show-inheritance: - - PyHyperScattering.load.ALS11012RSoXSLoader module - ------------------------------------------------------------------- - - .. autoclass:: PyHyperScattering.load.ALS11012RSoXSLoader - :members: - :undoc-members: - :show-inheritance: - - PyHyperScattering.load.SST1RSoXSLoader module - --------------------------------------------------------- - - .. autoclass:: PyHyperScattering.load.SST1RSoXSLoader - :members: - :undoc-members: - :show-inheritance: - - - PyHyperScattering.load.SST1RSoXSDB module - ----------------------------------------------------- - - .. autoclass:: PyHyperScattering.load.SST1RSoXSDB - :members: - :undoc-members: - :show-inheritance: - - PyHyperScattering.load.cyrsoxsLoader module - ------------------------------------------------------------------- - - .. autoclass:: PyHyperScattering.load.cyrsoxsLoader - :members: - :undoc-members: - :show-inheritance: - -PyHyperScattering.integrate module ----------------------------------- - -.. automodule:: PyHyperScattering.integrate - :members: - :undoc-members: - :show-inheritance: - - PyHyperScattering.integrate.PFEnergySeriesIntegrator module - ------------------------------------------------------------------------------------- - - .. automodule:: PyHyperScattering.PFEnergySeriesIntegrator - :members: - :undoc-members: - :show-inheritance: - - PyHyperScattering.integrate.PFGeneralIntegrator module - -------------------------------------------------------------------------------- - - .. automodule:: PyHyperScattering.PFGeneralIntegrator - :members: - :undoc-members: - :show-inheritance: - - PyHyperScattering.integrate.WPIntegrator module - -------------------------------------------------------------------------------- - - .. automodule:: PyHyperScattering.integrate.WPIntegrator - :members: - :undoc-members: - :show-inheritance: - -PyHyperScattering.util module ------------------------------ - -.. automodule:: PyHyperScattering.util - :members: - :undoc-members: - :show-inheritance: - - PyHyperScattering.util.Fitting module - ------------------------------------------------- - - .. automodule:: PyHyperScattering.util.Fitting - :members: - :undoc-members: - :show-inheritance: - - PyHyperScattering.util.HDR module - --------------------------------------------- - - .. automodule:: PyHyperScattering.util.HDR - :members: - :undoc-members: - :show-inheritance: - - PyHyperScattering.util.IntegrationUtils module - ---------------------------------------------------------- - - .. automodule:: PyHyperScattering.util.IntegrationUtils - :members: - :undoc-members: - :show-inheritance: - - PyHyperScattering.util.FileIO module - ----------------------------------------------- - - .. automodule:: PyHyperScattering.util.FileIO - :members: - :undoc-members: - :show-inheritance: - - - - PyHyperScattering.util.RSoXS module - ----------------------------------------------- - - .. automodule:: PyHyperScattering.util.RSoXS - :members: - :undoc-members: - :show-inheritance: - - - - - - -Module contents ---------------- - -.. automodule:: PyHyperScattering - :members: - :undoc-members: - :show-inheritance: From 16cde7427ab66555a89b3cbaace45839f2a37f4a Mon Sep 17 00:00:00 2001 From: Patel Date: Mon, 11 Sep 2023 19:39:23 -0400 Subject: [PATCH 29/49] restore autodoc --- .../reference_API/PyHyperScattering.rst | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) diff --git a/docs/source/reference_API/PyHyperScattering.rst b/docs/source/reference_API/PyHyperScattering.rst index c1e2cdda..0bd9ddd5 100644 --- a/docs/source/reference_API/PyHyperScattering.rst +++ b/docs/source/reference_API/PyHyperScattering.rst @@ -1,3 +1,149 @@ PyHyperScattering package ========================= +Submodules +---------- + +PyHyperScattering.load module +----------------------------- + +.. automodule:: PyHyperScattering.load + :members: + :undoc-members: + :show-inheritance: + + PyHyperScattering.load.FileLoader module + ---------------------------------------------------- + + .. autoclass:: PyHyperScattering.load.FileLoader + :members: + :undoc-members: + :show-inheritance: + + PyHyperScattering.load.ALS11012RSoXSLoader module + ------------------------------------------------------------------- + + .. autoclass:: PyHyperScattering.load.ALS11012RSoXSLoader + :members: + :undoc-members: + :show-inheritance: + + PyHyperScattering.load.SST1RSoXSLoader module + --------------------------------------------------------- + + .. autoclass:: PyHyperScattering.load.SST1RSoXSLoader + :members: + :undoc-members: + :show-inheritance: + + + PyHyperScattering.load.SST1RSoXSDB module + ----------------------------------------------------- + + .. autoclass:: PyHyperScattering.load.SST1RSoXSDB + :members: + :undoc-members: + :show-inheritance: + + PyHyperScattering.load.cyrsoxsLoader module + ------------------------------------------------------------------- + + .. autoclass:: PyHyperScattering.load.cyrsoxsLoader + :members: + :undoc-members: + :show-inheritance: + +PyHyperScattering.integrate module +---------------------------------- + +.. automodule:: PyHyperScattering.integrate + :members: + :undoc-members: + :show-inheritance: + + PyHyperScattering.integrate.PFEnergySeriesIntegrator module + ------------------------------------------------------------------------------------- + + .. automodule:: PyHyperScattering.PFEnergySeriesIntegrator + :members: + :undoc-members: + :show-inheritance: + + PyHyperScattering.integrate.PFGeneralIntegrator module + -------------------------------------------------------------------------------- + + .. automodule:: PyHyperScattering.PFGeneralIntegrator + :members: + :undoc-members: + :show-inheritance: + + PyHyperScattering.integrate.WPIntegrator module + -------------------------------------------------------------------------------- + + .. automodule:: PyHyperScattering.integrate.WPIntegrator + :members: + :undoc-members: + :show-inheritance: + +PyHyperScattering.util module +----------------------------- + +.. automodule:: PyHyperScattering.util + :members: + :undoc-members: + :show-inheritance: + + PyHyperScattering.util.Fitting module + ------------------------------------------------- + + .. automodule:: PyHyperScattering.util.Fitting + :members: + :undoc-members: + :show-inheritance: + + PyHyperScattering.util.HDR module + --------------------------------------------- + + .. automodule:: PyHyperScattering.util.HDR + :members: + :undoc-members: + :show-inheritance: + + PyHyperScattering.util.IntegrationUtils module + ---------------------------------------------------------- + + .. automodule:: PyHyperScattering.util.IntegrationUtils + :members: + :undoc-members: + :show-inheritance: + + PyHyperScattering.util.FileIO module + ----------------------------------------------- + + .. automodule:: PyHyperScattering.util.FileIO + :members: + :undoc-members: + :show-inheritance: + + + + PyHyperScattering.util.RSoXS module + ----------------------------------------------- + + .. automodule:: PyHyperScattering.util.RSoXS + :members: + :undoc-members: + :show-inheritance: + + + + + + +Module contents +--------------- + +.. automodule:: PyHyperScattering + :members: + :undoc-members: + :show-inheritance: From 30e8c7e23fc2a32c1685f4561e40de7349d07b5a Mon Sep 17 00:00:00 2001 From: Patel Date: Mon, 11 Sep 2023 19:57:26 -0400 Subject: [PATCH 30/49] Attempt SVG Workaround for pdf --- docs/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 48488234..7c446536 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,12 +3,12 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -.. image:: /source/_static/Logo_PyHyperO10_Dark.svg +.. image:: /source/_static/Logo_PyHyperO10_Dark.* :class: only-dark :alt: PyHyperScattering Logo :width: 600 -.. image:: /source/_static/Logo_PyHyperO9_Light.svg +.. image:: /source/_static/Logo_PyHyperO9_Light.* :class: only-light :alt: PyHyperScattering Logo :width: 600 From 501adaed405d7b6dafcef064abc04ccf86250510 Mon Sep 17 00:00:00 2001 From: Patel Date: Mon, 11 Sep 2023 20:03:40 -0400 Subject: [PATCH 31/49] Give up and use png --- docs/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 7c446536..6cb0d564 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,12 +3,12 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -.. image:: /source/_static/Logo_PyHyperO10_Dark.* +.. image:: /source/_static/Logo_PyHyperO10_Dark.png :class: only-dark :alt: PyHyperScattering Logo :width: 600 -.. image:: /source/_static/Logo_PyHyperO9_Light.* +.. image:: /source/_static/Logo_PyHyperO9_Light.png :class: only-light :alt: PyHyperScattering Logo :width: 600 From e2616117b5324c448a4adefc55b721eac7883f81 Mon Sep 17 00:00:00 2001 From: Patel Date: Mon, 11 Sep 2023 20:20:04 -0400 Subject: [PATCH 32/49] No image --- docs/index.rst | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 6cb0d564..964a92f0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,16 +3,6 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -.. image:: /source/_static/Logo_PyHyperO10_Dark.png - :class: only-dark - :alt: PyHyperScattering Logo - :width: 600 - -.. image:: /source/_static/Logo_PyHyperO9_Light.png - :class: only-light - :alt: PyHyperScattering Logo - :width: 600 - .. toctree:: :maxdepth: 1 :hidden: From 72902776ff58459767976fb50ec62844d92cb885 Mon Sep 17 00:00:00 2001 From: Patel Date: Mon, 11 Sep 2023 20:31:57 -0400 Subject: [PATCH 33/49] added Binder link back --- docs/source/getting_started/idx_getting_started.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/getting_started/idx_getting_started.rst b/docs/source/getting_started/idx_getting_started.rst index 2e52248f..294a2118 100644 --- a/docs/source/getting_started/idx_getting_started.rst +++ b/docs/source/getting_started/idx_getting_started.rst @@ -16,6 +16,8 @@ PyHyperScattering aims to make working with hyperspectral x-ray and neutron scat It grew out of the NIST RSoXS program, but aims to be broadly applicable. If it's scattering data, you should be able to load it, reduce it, and slice it. +Run tutorials using `Binder: `_ + The tools PyHyper provides are basically divided into three categories: From a479aceab103e131da3a8290e38c2e454537924d Mon Sep 17 00:00:00 2001 From: Patel Date: Mon, 11 Sep 2023 21:15:57 -0400 Subject: [PATCH 34/49] SVG only for html --- docs/index.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 964a92f0..055cc3c6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,6 +3,19 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. + +.. only:: html + + .. image:: /source/_static/Logo_PyHyperO10_Dark.svg + :class: only-dark + :alt: PyHyperScattering Logo + :width: 800 + + .. image:: /source/_static/Logo_PyHyperO9_Light.svg + :class: only-light + :alt: PyHyperScattering Logo + :width: 800 + .. toctree:: :maxdepth: 1 :hidden: From 9b1c476837979bd889ff8e4eefe9919a1dbd848e Mon Sep 17 00:00:00 2001 From: Patel Date: Mon, 11 Sep 2023 21:29:46 -0400 Subject: [PATCH 35/49] Added conf comment --- docs/conf.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 166acd22..20ce6115 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -38,7 +38,7 @@ extensions = [ "sphinx.ext.autodoc", "sphinx.ext.coverage", - "sphinx.ext.napoleon", + "sphinx.ext.napoleon", # numpydoc and google docstrings ] # Ignore annoying type exception warnings which often come from newlines @@ -59,6 +59,7 @@ # a list of builtin themes. # # html_theme = 'alabaster' + html_theme = "pydata_sphinx_theme" html_logo = "source/_static/Logo_PyHyperO9_Light.svg" html_theme_options = { @@ -75,8 +76,8 @@ "header_links_before_dropdown": 6, # Add light/dark mode and documentation version switcher: "navbar_end": ["theme-switcher", "navbar-icon-links"], -} +} # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, From 806ad0662bf4f92a97dba3b069a4aae961084a74 Mon Sep 17 00:00:00 2001 From: Patel Date: Tue, 12 Sep 2023 11:53:05 -0400 Subject: [PATCH 36/49] Renamed Func, add deprecated stub --- src/PyHyperScattering/SST1RSoXSDB.py | 304 ++++++++++++++++----------- 1 file changed, 176 insertions(+), 128 deletions(-) diff --git a/src/PyHyperScattering/SST1RSoXSDB.py b/src/PyHyperScattering/SST1RSoXSDB.py index 6960fa94..00ac0be9 100644 --- a/src/PyHyperScattering/SST1RSoXSDB.py +++ b/src/PyHyperScattering/SST1RSoXSDB.py @@ -13,6 +13,7 @@ import asyncio import time import copy + try: os.environ["TILED_SITE_PROFILES"] = "/nsls2/software/etc/tiled/profiles" from tiled.client import from_profile @@ -22,7 +23,8 @@ from databroker.queries import RawMongo, Key, FullText, Contains, Regex except Exception: print( - "Imports failed. Are you running on a machine with proper libraries for databroker, tiled, etc.?" + "Imports failed. Are you running on a machine with proper libraries for databroker," + " tiled, etc.?" ) import copy @@ -39,19 +41,19 @@ class SST1RSoXSDB: md_loading_is_quick = True pix_size_1 = 0.06 pix_size_2 = 0.06 - + md_lookup = { - 'sam_x':'RSoXS Sample Outboard-Inboard', - 'sam_y':'RSoXS Sample Up-Down', - 'sam_z':'RSoXS Sample Downstream-Upstream', - 'sam_th':'RSoXS Sample Rotation', - 'polarization':'en_polarization_setpoint', - 'energy':'en_energy_setpoint', - 'exposure':'RSoXS Shutter Opening Time (ms)' #md['detector']+'_cam_acquire_time' - } + "sam_x": "RSoXS Sample Outboard-Inboard", + "sam_y": "RSoXS Sample Up-Down", + "sam_z": "RSoXS Sample Downstream-Upstream", + "sam_th": "RSoXS Sample Rotation", + "polarization": "en_polarization_setpoint", + "energy": "en_energy_setpoint", + "exposure": "RSoXS Shutter Opening Time (ms)", # md['detector']+'_cam_acquire_time' + } md_secondary_lookup = { - 'energy':'en_monoen_setpoint', - } + "energy": "en_monoen_setpoint", + } def __init__( self, @@ -76,10 +78,11 @@ def __init__( use_precise_positions (bool): if False, rounds sam_x and sam_y to 1 digit. If True, keeps default rounding (4 digits). Needed for spiral scans to work with readback positions. use_chunked_loading (bool): if True, returns Dask backed arrays for further Dask processing. if false, behaves in conventional Numpy-backed way """ - + if corr_mode == None: warnings.warn( - "Correction mode was not set, not performing *any* intensity corrections. Are you sure this is " + "Correction mode was not set, not performing *any* intensity corrections. Are you" + " sure this is " + "right? Set corr_mode to 'none' to suppress this warning.", stacklevel=2, ) @@ -94,9 +97,15 @@ def __init__( else: self.c = catalog if use_chunked_loading: - raise SyntaxError('use_chunked_loading is incompatible with externally supplied catalog. when creating the catalog, pass structure_clients = "dask" as a kwarg.') + raise SyntaxError( + "use_chunked_loading is incompatible with externally supplied catalog. when" + ' creating the catalog, pass structure_clients = "dask" as a kwarg.' + ) if len(catalog_kwargs) != 0: - raise SyntaxError('catalog_kwargs is incompatible with externally supplied catalog. pass those kwargs to whoever gave you the catalog you passed in.') + raise SyntaxError( + "catalog_kwargs is incompatible with externally supplied catalog. pass those" + " kwargs to whoever gave you the catalog you passed in." + ) self.dark_subtract = dark_subtract self.dark_pedestal = dark_pedestal self.exposure_offset = exposure_offset @@ -131,7 +140,18 @@ def runSearch(self, **kwargs): q = RawMongo(**kwargs) return self.c.search(q) - def summarize_run( + def summarize_run(*args, **kwargs): + warnings.warn( + ( + "summarize_run has been renamed to searchCatalog. This will stop working in" + " PyHyperScattering 1.0.0 and later." + ), + DeprecationWarning, + stacklevel=2, + ) + return self.searchCatalog(*args, **kwargs) + + def searchCatalog( self, outputType: str = "default", cycle: str = None, @@ -157,13 +177,13 @@ def summarize_run( underlying metadata scheme. Ex1: All of the carbon,fluorine,or oxygen scans for a single sample series in the most recent cycle: - bsCatalogReduced4 = db_loader.summarize_run(sample="bBP_", institution="NIST", cycle = "2022-2", plan="carbon|fluorine|oxygen") + bsCatalogReduced4 = db_loader.searchCatalog(sample="bBP_", institution="NIST", cycle = "2022-2", plan="carbon|fluorine|oxygen") Ex2: Just all of the scan Ids for a particular sample: - bsCatalogReduced4 = db_loader.summarize_run(sample="BBP_PFP09A", outputType='scans') + bsCatalogReduced4 = db_loader.searchCatalog(sample="BBP_PFP09A", outputType='scans') Ex3: Complex Search with custom parameters - bsCatalogReduced3 = db_loader.summarize_run(['angle', '-1.6', 'numeric'], outputType='all',sample="BBP_", cycle = "2022-2", + bsCatalogReduced3 = db_loader.searchCatalog(['angle', '-1.6', 'numeric'], outputType='all',sample="BBP_", cycle = "2022-2", institution="NIST",plan="carbon", userOutputs = [["Exposure Multiplier", "exptime", r'catalog.start'], ["Stop Time","time",r'catalog.stop']]) @@ -198,7 +218,7 @@ def summarize_run( Ex2: passing in grazing=[0,'numeric'] would match grazing==0 Ex3: create kwargs first, then pass it into the function. kwargs = {'2weird metadata label': "Bob", 'grazing': 0, 'angle':-1.6} - db_loader.summarize_run(sample="BBP_PFP09A", outputType='scans', **kwargs) + db_loader.searchCatalog(sample="BBP_PFP09A", outputType='scans', **kwargs) userOutputs (list of lists, optional): Additional metadata to be added to output can be specified as a list of lists. Each sub-list specifies a metadata field as a 3 element list of format: [Output column title (str), Metadata label (str), Metadata Source (raw str)], @@ -239,7 +259,9 @@ def summarize_run( userSearchList.append([userLabel, value[0], value[1]]) else: # bad user input raise ValueError( - f"Error parsing a keyword search term, check the format. Skipped argument: {value} ") + "Error parsing a keyword search term, check the format. Skipped argument:" + f" {value} " + ) # combine the lists of lists fullSearchList = defaultSearchDetails + userSearchList @@ -251,11 +273,11 @@ def summarize_run( # Iterate through search terms sequentially, reducing the size of the catalog based on successful matches reducedCatalog = bsCatalog - for _,searchSeries in tqdm(df_SearchDet.iterrows(),total = df_SearchDet.shape[0], desc = "Running catalog search..."): - + for _, searchSeries in tqdm( + df_SearchDet.iterrows(), total=df_SearchDet.shape[0], desc="Running catalog search..." + ): # Skip arguments with value None, and quits if the catalog was reduced to 0 elements if (searchSeries[1] is not None) and (len(reducedCatalog) > 0): - # For numeric entries, do Key equality if "numeric" in str(searchSeries[2]): reducedCatalog = reducedCatalog.search( @@ -278,25 +300,23 @@ def summarize_run( regexString = reg_prefix + str(searchSeries[1]) + reg_postfix # Search/reduce the catalog - reducedCatalog = reducedCatalog.search( - Regex(searchSeries[0], regexString) - ) + reducedCatalog = reducedCatalog.search(Regex(searchSeries[0], regexString)) # If a match fails, notify the user which search parameter yielded 0 results if len(reducedCatalog) == 0: warnString = ( f"Catalog reduced to zero when attempting to match {searchSeries}\n" - +f"If this is a user-provided search parameter, check spelling/syntax." + + f"If this is a user-provided search parameter, check spelling/syntax." ) warnings.warn(warnString, stacklevel=2) return pd.DataFrame() ### Part 2: Build and return output dataframe - if (outputType == "scans"): + if outputType == "scans": # Branch 2.1, if only scan IDs needed, build and return a 1-column dataframe scan_ids = [] - for scanEntry in tqdm(reducedCatalog.values(), desc = "Building scan list"): + for scanEntry in tqdm(reducedCatalog.values(), desc="Building scan list"): scan_ids.append(scanEntry.start["scan_id"]) return pd.DataFrame(scan_ids, columns=["Scan ID"]) @@ -317,7 +337,7 @@ def summarize_run( ["sample_id", "sample_id", r"catalog.start", "default"], ["bar_spot", "bar_spot", r"catalog.start", "ext_msmt"], ["plan", "plan_name", r"catalog.start", "default"], - ["detector", "RSoXS_Main_DET", r"catalog.start", "default"], + ["detector", "RSoXS_Main_DET", r"catalog.start", "default"], ["polarization", "pol", r'catalog.start["plan_args"]', "default"], ["sample_rotation", "angle", r"catalog.start", "ext_msmt"], ["exit_status", "exit_status", r"catalog.stop", "default"], @@ -344,7 +364,13 @@ def summarize_run( activeOutputValues.append(userOutEntry) activeOutputLabels.append(userOutEntry[0]) else: # bad user input - raise ValueError(f"Error parsing user-provided output request {userOutEntry}, check the format.", stacklevel=2) + raise ValueError( + ( + f"Error parsing user-provided output request {userOutEntry}, check the" + " format." + ), + stacklevel=2, + ) # Add any user-provided search terms for userSearchEntry in userSearchList: @@ -361,17 +387,17 @@ def summarize_run( # Build output dataframe as a list of lists outputList = [] - # Outer loop: Catalog entries - for scanEntry in tqdm(reducedCatalog.values(),desc = "Retrieving results..."): + # Outer loop: Catalog entries + for scanEntry in tqdm(reducedCatalog.values(), desc="Retrieving results..."): singleScanOutput = [] # Pull the start and stop docs once - + currentCatalogStart = scanEntry.start currentCatalogStop = scanEntry.stop - + currentScanID = currentCatalogStart["scan_id"] - + # Inner loop: append output values for outputEntry in activeOutputValues: outputVariableName = outputEntry[0] @@ -379,8 +405,10 @@ def summarize_run( metaDataSource = outputEntry[2] try: # Add the metadata value depending on where it is located - if metaDataLabel == 'time': - singleScanOutput.append(datetime.datetime.fromtimestamp(currentCatalogStart['time'])) + if metaDataLabel == "time": + singleScanOutput.append( + datetime.datetime.fromtimestamp(currentCatalogStart["time"]) + ) # see Zen of Python # 9,8 for justification elif metaDataSource == r"catalog.start": singleScanOutput.append(currentCatalogStart[metaDataLabel]) @@ -397,27 +425,38 @@ def summarize_run( else: if debugWarnings: warnings.warn( - f'Failed to locate metadata for {outputVariableName} in scan {currentScanID}.', - stacklevel=2) + ( + f"Failed to locate metadata for {outputVariableName} in" + f" scan {currentScanID}." + ), + stacklevel=2, + ) missesDuringLoad = True except (KeyError, TypeError): if debugWarnings: warnings.warn( - f'Failed to locate metadata for {outputVariableName} in scan {currentScanID}.', - stacklevel=2) + ( + f"Failed to locate metadata for {outputVariableName} in scan" + f" {currentScanID}." + ), + stacklevel=2, + ) missesDuringLoad = True singleScanOutput.append("N/A") # Append to the filled output list for this entry to the list of lists outputList.append(singleScanOutput) - - + # Convert to dataframe for export if missesDuringLoad: - warnings.warn( - f'One or more missing field(s) during this load were replaced with "N/A". Re-run with debugWarnings=True to see details.', - stacklevel=2) + warnings.warn( + ( + f'One or more missing field(s) during this load were replaced with "N/A". ' + f" Re-run with debugWarnings=True to see details." + ), + stacklevel=2, + ) return pd.DataFrame(outputList, columns=activeOutputLabels) def background(f): @@ -481,10 +520,10 @@ def loadSeries( scans.append(loaded) label_val = loaded.__getattr__(meta_dim) try: - if len(label_val)>1 and type(label_val) != str: + if len(label_val) > 1 and type(label_val) != str: label_val = label_val.mean() except TypeError: - pass # assume if there is no len, then this is a single value and everything is fine + pass # assume if there is no len, then this is a single value and everything is fine label_vals.append(label_val) assert len(axes) == axes.count( axes[0] @@ -537,72 +576,77 @@ def loadRun( ) md = self.loadMd(run) - + monitors = self.loadMonitors(run) - + if dims is None: - if ('NEXAFS' or 'nexafs') in md['start']['plan_name']: - raise NotImplementedError(f"Scan {md['start']['scan_id']} is a {md['start']['plan_name']} NEXAFS scan. NEXAFS loading is not yet supported.") # handled case change in "NEXAFS" - elif ('full' in md['start']['plan_name'] or 'short' in md['start']['plan_name'] or 'custom_rsoxs_scan' in md['start']['plan_name']) and dims is None: - dims = ['energy'] - elif 'spiralsearch' in md['start']['plan_name'] and dims is None: - dims = ['sam_x','sam_y'] - elif 'count' in md['start']['plan_name'] and dims is None: - dims = ['epoch'] + if ("NEXAFS" or "nexafs") in md["start"]["plan_name"]: + raise NotImplementedError( + f"Scan {md['start']['scan_id']} is a {md['start']['plan_name']} NEXAFS scan. " + " NEXAFS loading is not yet supported." + ) # handled case change in "NEXAFS" + elif ( + "full" in md["start"]["plan_name"] + or "short" in md["start"]["plan_name"] + or "custom_rsoxs_scan" in md["start"]["plan_name"] + ) and dims is None: + dims = ["energy"] + elif "spiralsearch" in md["start"]["plan_name"] and dims is None: + dims = ["sam_x", "sam_y"] + elif "count" in md["start"]["plan_name"] and dims is None: + dims = ["epoch"] else: axes_to_include = [] rsd_cutoff = 0.005 # begin with a list of the things that are primary streams - axis_list = list(run['primary']['data'].keys()) + axis_list = list(run["primary"]["data"].keys()) # next, knock out anything that has 'image', 'fullframe' in it - these aren't axes - axis_list = [x for x in axis_list if 'image' not in x] - axis_list = [x for x in axis_list if 'fullframe' not in x] - axis_list = [x for x in axis_list if 'stats' not in x] - axis_list = [x for x in axis_list if 'saturated' not in x] - axis_list = [x for x in axis_list if 'under_exposed' not in x] + axis_list = [x for x in axis_list if "image" not in x] + axis_list = [x for x in axis_list if "fullframe" not in x] + axis_list = [x for x in axis_list if "stats" not in x] + axis_list = [x for x in axis_list if "saturated" not in x] + axis_list = [x for x in axis_list if "under_exposed" not in x] # knock out any known names of scalar counters - axis_list = [x for x in axis_list if 'Beamstop' not in x] - axis_list = [x for x in axis_list if 'Current' not in x] - - - + axis_list = [x for x in axis_list if "Beamstop" not in x] + axis_list = [x for x in axis_list if "Current" not in x] + # now, clean up duplicates. - axis_list = [x for x in axis_list if 'setpoint' not in x] + axis_list = [x for x in axis_list if "setpoint" not in x] # now, figure out what's actually moving. we use a relative standard deviation to do this. # arbitrary cutoff of 0.5% motion = it moved intentionally. for axis in axis_list: std = np.std(run["primary"]["data"][axis]) mean = np.mean(run["primary"]["data"][axis]) - rsd = std/mean - + rsd = std / mean + if rsd > rsd_cutoff: axes_to_include.append(axis) # next, construct the reverse lookup table - best mapping we can make of key to pyhyper word # we start with the lookup table used by loadMd() - reverse_lut = {v: k for k, v in self.md_lookup.items()} - reverse_lut_secondary = {v: k for k, v in self.md_secondary_lookup.items()} + reverse_lut = {v: k for k, v in self.md_lookup.items()} + reverse_lut_secondary = {v: k for k, v in self.md_secondary_lookup.items()} reverse_lut.update(reverse_lut_secondary) # here, we broaden the table to make a value that default sources from '_setpoint' actually match on either # the bare value or the readback value. reverse_lut_adds = {} for k in reverse_lut.keys(): - if 'setpoint' in k: - reverse_lut_adds[k.replace('_setpoint','')] = reverse_lut[k] - reverse_lut_adds[k.replace('_setpoint','_readback')] = reverse_lut[k] + if "setpoint" in k: + reverse_lut_adds[k.replace("_setpoint", "")] = reverse_lut[k] + reverse_lut_adds[k.replace("_setpoint", "_readback")] = reverse_lut[k] reverse_lut.update(reverse_lut_adds) - + pyhyper_axes_to_use = [] for x in axes_to_include: try: pyhyper_axes_to_use.append(reverse_lut[x]) except KeyError: pyhyper_axes_to_use.append(x) - dims = pyhyper_axes_to_use - - ''' + dims = pyhyper_axes_to_use + + """ elif dims == None: # use the dim tols to define the dimensions # dims = [] @@ -626,8 +670,7 @@ def loadRun( dims[i] = 'en_energy' if len(dims) == 0: raise NotImplementedError('You have not entered any dimensions; please enter at least one, or use None rather than an empty list') - ''' - + """ data = run["primary"]["data"][md["detector"] + "_image"] if ( @@ -656,9 +699,7 @@ def loadRun( def subtract_dark(img, pedestal=100, darks=None): return img + pedestal - darks[int(img.dark_id.values)] - data = data.groupby("time").map( - subtract_dark, darks=dark, pedestal=self.dark_pedestal - ) + data = data.groupby("time").map(subtract_dark, darks=dark, pedestal=self.dark_pedestal) dims_to_join = [] dim_names_to_join = [] @@ -710,7 +751,10 @@ def subtract_dark(img, pedestal=100, darks=None): except Exception as e: warnings.warn( - "Monitor streams loaded successfully, but could not be correlated to images. Check monitor stream for issues, probable metadata change.", + ( + "Monitor streams loaded successfully, but could not be correlated to images. " + " Check monitor stream for issues, probable metadata change." + ), stacklevel=2, ) retxr.attrs.update(md) @@ -802,7 +846,7 @@ def loadMonitors( # At this stage monitors has dimension time and all streams as data variables # the time dimension inherited all time values from all streams # the data variables (Mesh current, sample current etc.) are all sparse, with lots of nans - + # if there are no monitors, return an empty xarray Dataset if monitors is None: return xr.Dataset() @@ -818,15 +862,9 @@ def loadMonitors( try: primary_time = entry.primary.data["time"].values except AttributeError: - if ( - type(entry.primary.data["time"]) - == tiled.client.array.DaskArrayClient - ): + if type(entry.primary.data["time"]) == tiled.client.array.DaskArrayClient: primary_time = entry.primary.data["time"].read().compute() - elif ( - type(entry.primary.data["time"]) - == tiled.client.array.ArrayClient - ): + elif type(entry.primary.data["time"]) == tiled.client.array.ArrayClient: primary_time = entry.primary.data["time"].read() # If we want to exclude values for when the shutter was opening or closing @@ -834,14 +872,10 @@ def loadMonitors( if useShutterThinning: # Create new data variable to hold shutter toggle values that are thinned # Shutter Toggle stream is 1 when open (or opening) and 0 when closed (or closing) - monitors["RSoXS Shutter Toggle_thinned"] = monitors[ - "RSoXS Shutter Toggle" - ] + monitors["RSoXS Shutter Toggle_thinned"] = monitors["RSoXS Shutter Toggle"] # Perform thinning to remove edge cases where shutter may be partially open or closed - monitors[ - "RSoXS Shutter Toggle_thinned" - ].values = scipy.ndimage.binary_erosion( + monitors["RSoXS Shutter Toggle_thinned"].values = scipy.ndimage.binary_erosion( monitors["RSoXS Shutter Toggle"].values, iterations=n_thinning_iters, border_value=0, @@ -849,9 +883,9 @@ def loadMonitors( # Filter monitors to only include timepoints where shutter was open (as determined by thinning) # Drop any remaining missing values along the time axis - monitors = monitors.where( - monitors["RSoXS Shutter Toggle_thinned"] > 0 - ).dropna("time") + monitors = monitors.where(monitors["RSoXS Shutter Toggle_thinned"] > 0).dropna( + "time" + ) # Bin the indexes in 'time' based on the intervales between timepoints in 'primary_time' and evaluate their mean # Then rename the 'time_bin' dimension that results to 'time' @@ -872,7 +906,11 @@ def loadMonitors( except Exception as e: # raise e # for testing warnings.warn( - "Error while time-integrating monitors onto images. Usually, this indicates a problem with the monitors, this is a critical error if doing normalization otherwise fine to ignore.", + ( + "Error while time-integrating monitors onto images. Usually, this" + " indicates a problem with the monitors, this is a critical error if doing" + " normalization otherwise fine to ignore." + ), stacklevel=2, ) return monitors @@ -910,9 +948,7 @@ def loadMd(self, run): meas_time < datetime.datetime(2022, 7, 7) ): # these params determined by Camille from Igor - md[ - "beamcenter_x" - ] = 498 # not the best estimate; I didn't have great data + md["beamcenter_x"] = 498 # not the best estimate; I didn't have great data md["beamcenter_y"] = 498 md["sdd"] = 512.12 # GUESS; SOMEONE SHOULD CONFIRM WITH A BCP MAYBE?? else: @@ -943,7 +979,10 @@ def loadMd(self, run): else: md["rsoxs_config"] = "unknown" warnings.warn( - f'RSoXS_Config is neither SAXS or WAXS. Looks to be {start["RSoXS_Config"]}. Might want to check that out.', + ( + f'RSoXS_Config is neither SAXS or WAXS. Looks to be {start["RSoXS_Config"]}. ' + " Might want to check that out." + ), stacklevel=2, ) @@ -952,19 +991,20 @@ def loadMd(self, run): elif md["rsoxs_config"] == "waxs": md["detector"] = "Wide Angle CCD Detector" else: - warnings.warn( - f"Cannot auto-hint detector type without RSoXS config.", stacklevel=2 - ) + warnings.warn(f"Cannot auto-hint detector type without RSoXS config.", stacklevel=2) # items coming from baseline baseline = run["baseline"]["data"] # items coming from primary try: - primary = run['primary']['data'] - except (KeyError,HTTPStatusError): - raise Exception('No primary stream --> probably you caught run before image was written. Try again.') - + primary = run["primary"]["data"] + except (KeyError, HTTPStatusError): + raise Exception( + "No primary stream --> probably you caught run before image was written. Try" + " again." + ) + md_lookup = copy.deepcopy(self.md_lookup) md_secondary_lookup = copy.deepcopy(self.md_secondary_lookup) @@ -988,7 +1028,11 @@ def loadMd(self, run): md[phs] = blval.mean().round(4) if blval.var() > 0: warnings.warn( - f"While loading {rsoxs} to infill metadata entry for {phs}, found beginning and end values unequal: {baseline[rsoxs]}. It is possible something is messed up.", + ( + f"While loading {rsoxs} to infill metadata entry for {phs}, found" + f" beginning and end values unequal: {baseline[rsoxs]}. It is" + " possible something is messed up." + ), stacklevel=2, ) except (KeyError, HTTPStatusError): @@ -1005,12 +1049,20 @@ def loadMd(self, run): md[phs] = blval.mean().round(4) if blval.var() > 0: warnings.warn( - f"While loading {md_secondary_lookup[phs]} to infill metadata entry for {phs}, found beginning and end values unequal: {baseline[rsoxs]}. It is possible something is messed up.", + ( + f"While loading {md_secondary_lookup[phs]} to infill" + f" metadata entry for {phs}, found beginning and end" + f" values unequal: {baseline[rsoxs]}. It is possible" + " something is messed up." + ), stacklevel=2, ) except (KeyError, HTTPStatusError): warnings.warn( - f"Could not find {rsoxs} in either baseline or primary. Needed to infill value {phs}. Setting to None.", + ( + f"Could not find {rsoxs} in either baseline or primary. " + f" Needed to infill value {phs}. Setting to None." + ), stacklevel=2, ) md[phs] = None @@ -1101,11 +1153,7 @@ def loadSingleImage(self, filepath, coords=None, return_q=False, **kwargs): # img = (img-darkimg+self.dark_pedestal)/corr if return_q: qpx = ( - 2 - * np.pi - * 60e-6 - / (headerdict["sdd"] / 1000) - / (headerdict["wavelength"] * 1e10) + 2 * np.pi * 60e-6 / (headerdict["sdd"] / 1000) / (headerdict["wavelength"] * 1e10) ) qx = (np.arange(1, img.size[0] + 1) - headerdict["beamcenter_x"]) * qpx qy = (np.arange(1, img.size[1] + 1) - headerdict["beamcenter_y"]) * qpx From cd1bc7035bf2e6827c551f43ba691d06bc7322c7 Mon Sep 17 00:00:00 2001 From: Patel Date: Tue, 12 Sep 2023 11:56:28 -0400 Subject: [PATCH 37/49] Troubleshoot summarize_run 1 --- src/PyHyperScattering/SST1RSoXSDB.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/PyHyperScattering/SST1RSoXSDB.py b/src/PyHyperScattering/SST1RSoXSDB.py index 00ac0be9..4f319af7 100644 --- a/src/PyHyperScattering/SST1RSoXSDB.py +++ b/src/PyHyperScattering/SST1RSoXSDB.py @@ -140,7 +140,11 @@ def runSearch(self, **kwargs): q = RawMongo(**kwargs) return self.c.search(q) - def summarize_run(*args, **kwargs): + def summarize_run(self, *args, **kwargs): + """Deprecated function for searching the bluesky catalog for a run. Replaced by searchCatalog() + + To be removed in PyHyperScattering 1.0.0+. + """ warnings.warn( ( "summarize_run has been renamed to searchCatalog. This will stop working in" From a3ad8ac4eaf1dcabe10024ed9162c78e7b3b8ee2 Mon Sep 17 00:00:00 2001 From: Patel Date: Tue, 12 Sep 2023 12:12:05 -0400 Subject: [PATCH 38/49] Pin numexpr for CI test --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 7d7c332f..d302d15e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,4 @@ scipy pillow xarray tqdm +numexpr<2.8.5 From 3ceb6cb530d14b8fc2f318ddc1990c0a29f373f9 Mon Sep 17 00:00:00 2001 From: Peter Beaucage Date: Tue, 12 Sep 2023 12:32:18 -0400 Subject: [PATCH 39/49] add note to numexpr pin --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index d302d15e..42f13e1c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,4 +11,6 @@ scipy pillow xarray tqdm +# the following pin is due to a security update to numexpr: https://github.com/pydata/numexpr/issues/442 +# consider removing once this is resolved numexpr<2.8.5 From e7380e13d0e1440249e0a368cf8e637b6e375909 Mon Sep 17 00:00:00 2001 From: Patel Date: Tue, 12 Sep 2023 16:15:41 -0400 Subject: [PATCH 40/49] Re-add Peter suggestions I forgot to accept --- docs/index.rst | 11 ++++++----- docs/source/development/idx_development.rst | 12 ++++++------ docs/source/release_notes/idx_release_notes.rst | 4 ++++ 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 055cc3c6..1ab9b8ef 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -36,11 +36,12 @@ from raw data to an analysis with clear commands, punch down to the data you nee quickly. The goal is for these tools to make the mechanics of hyperspectral scattering easier and in so doing, more reproducible, explainable, and robust. -PyHyperScattering is a product of the `National Institute of Standards and Technology (NIST) -`_. This package is under active development, and the team welcome DMs -with questions on the NIST RSoXS slack, Nikea, and NSLS2 slack channels, or by email to -`Dr. Peter Beaucage `_. For more information about contributing, -development philosophy, and licensing, see :ref:`the Development page `. +PyHyperScattering is an open-source collaboration maintained by the `National Institute of +Standards and Technology (NIST) `_. This package is under active +development, and the team welcome DMs with questions on the NIST RSoXS slack, Nikea, and NSLS2 slack +channels, or by email to `Dr. Peter Beaucage `_. +For more information about contributing, development philosophy, and licensing, +see :ref:`the Development page `. Documentation ------------- diff --git a/docs/source/development/idx_development.rst b/docs/source/development/idx_development.rst index 733716ad..5483bc24 100644 --- a/docs/source/development/idx_development.rst +++ b/docs/source/development/idx_development.rst @@ -3,11 +3,11 @@ Development and Contributing Info: TODO ============================================================== -PyHyperScattering is a product of the `National Institute of Standards and Technology (NIST) -`_. This package is under active development, and the team welcome DMs -with questions on the NIST RSoXS slack, Nikea, and NSLS2 slack channels, or by email to -`Dr. Peter Beaucage `_. For more information about contributing, -development philosophy, and licensing, see :ref:`the Development page `. +PyHyperScattering is an open-source collaboration maintained by the `National Institute of +Standards and Technology (NIST) `_. This package is under active +development, and the team welcome DMs with questions on the NIST RSoXS slack, Nikea, and NSLS2 slack +channels, or by email to `Dr. Peter Beaucage `_. For more information +about contributing, development philosophy, and licensing, see :ref:`the Development page `. Scope and Package Outline @@ -16,7 +16,7 @@ TODO Contributing ------------ -Contributions are welcome! Please view our `Contributer Guidelines `_ on github. +Contributions are welcome! Please view our `Contributor Guidelines `_ on github. License ------- diff --git a/docs/source/release_notes/idx_release_notes.rst b/docs/source/release_notes/idx_release_notes.rst index 5e4d6e0e..37e0567f 100644 --- a/docs/source/release_notes/idx_release_notes.rst +++ b/docs/source/release_notes/idx_release_notes.rst @@ -3,4 +3,8 @@ Release Notes ================= +Release notes for each release are on the `GitHub releases page ` + +Major changes are discussed here: + Preparing for v1.0 release, which includes breaking changes that make xarray Datasets the underlying data structure. \ No newline at end of file From 21b0b5df6ce13a7051fff43d4009a8568af98d3e Mon Sep 17 00:00:00 2001 From: Camille Bishop Date: Thu, 14 Sep 2023 16:19:26 -0400 Subject: [PATCH 41/49] Fixed Error Handling for integrateImage energy When attempting to integrate data from cycle 2022-2, was getting an IndexError when integrateImageStack called integrateSingleImage. The indexes for finding the energy were switched from what they needed to be. Perhaps this block of code isn't running for other cycles, or the index structure is different. Changes are at lines 62 and 63. --- src/PyHyperScattering/PFEnergySeriesIntegrator.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/PyHyperScattering/PFEnergySeriesIntegrator.py b/src/PyHyperScattering/PFEnergySeriesIntegrator.py index 2926782c..77c6490d 100755 --- a/src/PyHyperScattering/PFEnergySeriesIntegrator.py +++ b/src/PyHyperScattering/PFEnergySeriesIntegrator.py @@ -57,7 +57,10 @@ def integrateSingleImage(self,img): for i,n in enumerate(img.indexes[multiindex_name].names): if n == 'energy': idx_of_energy = i - en = float(getattr(img,multiindex_name).values[idx_of_energy][0]) + try: + en = float(getattr(img,multiindex_name).values[idx_of_energy][0]) # this does not work for 2022-2 data; does it work for other cycles? + except IndexError: + en = float(getattr(img,multiindex_name).values[0][idx_of_energy]) except KeyError: pass if en is not None: From d7eb2d1405e49b7918a4548edb75128cfde442d1 Mon Sep 17 00:00:00 2001 From: Jonathan Guyer Date: Tue, 5 Dec 2023 11:05:55 -0500 Subject: [PATCH 42/49] Rename doc-building action OISM has no sense of humor --- .github/workflows/publish-docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index 22b79312..0e999d85 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -6,7 +6,7 @@ jobs: docs: runs-on: ubuntu-latest steps: - - uses: usnistgov/NISTtheDocs2Death@0.4 + - uses: usnistgov/Docs4NIST@0.5 with: docs-folder: docs/ formats: |- From 0023c69637f18303d5fdc4995178417b9f3f6e39 Mon Sep 17 00:00:00 2001 From: Phong Date: Tue, 5 Mar 2024 17:43:25 -0800 Subject: [PATCH 43/49] Implemented dask backing under use_chunked_loading flag --- src/PyHyperScattering/SST1RSoXSLoader.py | 56 ++++++++++-------------- 1 file changed, 24 insertions(+), 32 deletions(-) diff --git a/src/PyHyperScattering/SST1RSoXSLoader.py b/src/PyHyperScattering/SST1RSoXSLoader.py index a75d937a..6119b359 100644 --- a/src/PyHyperScattering/SST1RSoXSLoader.py +++ b/src/PyHyperScattering/SST1RSoXSLoader.py @@ -9,7 +9,7 @@ import json #from pyFAI import azimuthalIntegrator import numpy as np - +from dask_image.imread import imread class SST1RSoXSLoader(FileLoader): ''' @@ -21,7 +21,7 @@ class SST1RSoXSLoader(FileLoader): pix_size_1 = 0.06 pix_size_2 = 0.06 - def __init__(self,corr_mode=None,user_corr_func=None,dark_pedestal=100,exposure_offset=0,constant_md={},): + def __init__(self,corr_mode=None,user_corr_func=None,dark_pedestal=100,exposure_offset=0,constant_md={},use_chunked_loading=False): ''' Args: corr_mode (str): origin to use for the intensity correction. Can be 'expt','i0','expt+i0','user_func','old',or 'none' @@ -29,6 +29,7 @@ def __init__(self,corr_mode=None,user_corr_func=None,dark_pedestal=100,exposure_ dark_pedestal (numeric): value to subtract(/add, if negative) to the whole image. this should match the instrument setting for suitcased tiffs, typically 100. exposure_offset (numeric): value to add to the exposure time. Measured at 2ms with the piezo shutter in Dec 2019 by Jacob Thelen, NIST constant_md (dict): values to insert into every metadata load. + use_chunked_loading (bool): flag to use chunked loading with dask or not. ''' if corr_mode == None: @@ -38,13 +39,13 @@ def __init__(self,corr_mode=None,user_corr_func=None,dark_pedestal=100,exposure_ else: self.corr_mode = corr_mode - self.constant_md = constant_md - self.dark_pedestal = dark_pedestal self.user_corr_func = user_corr_func self.exposure_offset = exposure_offset + self.use_chunked_loading = use_chunked_loading # self.darks = {} + # def loadFileSeries(self,basepath): # try: # flist = list(basepath.glob('*primary*.tiff')) @@ -60,8 +61,6 @@ def __init__(self,corr_mode=None,user_corr_func=None,dark_pedestal=100,exposure_ # # return out - - def loadSingleImage(self,filepath,coords=None, return_q=False,image_slice=None,use_cached_md=False,**kwargs): ''' HELPER FUNCTION that loads a single image and returns an xarray with either pix_x / pix_y dimensions (if return_q == False) or qx / qy (if return_q == True) @@ -80,7 +79,14 @@ def loadSingleImage(self,filepath,coords=None, return_q=False,image_slice=None,u raise NotImplementedError('Image slicing is not supported for SST1') if use_cached_md != False: raise NotImplementedError('Caching of metadata is not supported for SST1') - img = Image.open(filepath) + + # Use chunked loading if flag is set + if self.use_chunked_loading: + img = imread(str(filepath)) + if img.ndim == 3: + img = img[:, :, 0] # Select the first channel if the image is multi-channel + else: + img = np.array(Image.open(filepath)) headerdict = self.loadMd(filepath) # two steps in this pre-processing stage: @@ -112,7 +118,8 @@ def loadSingleImage(self,filepath,coords=None, return_q=False,image_slice=None,u # # step 2: dark subtraction # this is already done in the suitcase, but we offer the option to add/subtract a pedestal. - image_data = (np.array(img)-self.dark_pedestal)/corr + image_data = (img-self.dark_pedestal)/corr + image_data = img if return_q: qpx = 2*np.pi*60e-6/(headerdict['sdd']/1000)/(headerdict['wavelength']*1e10) qx = (np.arange(1,img.size[0]+1)-headerdict['beamcenter_y'])*qpx @@ -125,19 +132,10 @@ def loadSingleImage(self,filepath,coords=None, return_q=False,image_slice=None,u def read_json(self,jsonfile): json_dict = {} - - # Try / except statment to be compatible with local rsoxs data before - # and after SST1 cycle 2 2023 - try: # For earlier cyles - with open(jsonfile) as f: - data = json.load(f) - meas_time = datetime.datetime.fromtimestamp(data[1]['time']) - except KeyError: # For later cycles (confirmed working for cycle 2 2023) - with open(jsonfile) as f: - data = [0, json.load(f)] # Quick fix to load the json in a list - meas_time = datetime.datetime.fromtimestamp(data[1]['time']) - - json_dict['sample_name'] = data[1]['sample_name'] + with open(jsonfile) as f: + data = json.load(f) + meas_time =datetime.datetime.fromtimestamp(data[1]['time']) + json_dict['sample_name'] = data[1]['sample_name'] if data[1]['RSoXS_Main_DET'] == 'SAXS': json_dict['rsoxs_config'] = 'saxs' # discrepency between what is in .json and actual @@ -159,7 +157,7 @@ def read_json(self,jsonfile): json_dict['beamcenter_y'] = data[1]['RSoXS_SAXS_BCY'] json_dict['sdd'] = data[1]['RSoXS_SAXS_SDD'] - elif (data[1]['RSoXS_Main_DET'] == 'WAXS') | (data[1]['RSoXS_Main_DET'] == 'waxs_det'): # additional name for for cycle 2 2023 + elif data[1]['RSoXS_Main_DET'] == 'WAXS': json_dict['rsoxs_config'] = 'waxs' if (meas_time > datetime.datetime(2020,11,16)) and (meas_time < datetime.datetime(2021,1,15)): json_dict['beamcenter_x'] = 400.46 @@ -176,7 +174,7 @@ def read_json(self,jsonfile): json_dict['sdd'] = data[1]['RSoXS_WAXS_SDD'] else: - json_dict['rsoxs_config'] = 'unknown' + json_dict['rsoxs_config'] == 'unknown' warnings.warn('RSoXS_Config is neither SAXS or WAXS. Check json file',stacklevel=2) if json_dict['sdd'] == None: @@ -256,15 +254,9 @@ def loadMd(self,filepath): else: cwd = pathlib.Path(dirPath) - # Another try/except statement to be compatible with local data pre and - # post SST1 cycle 2 2023 - try: # For earlier data - json_fname = list(cwd.glob('*.jsonl')) - json_dict = self.read_json(json_fname[0]) - except IndexError: # For later data (works for cycle 2 2023) - json_fname = list(cwd.glob('*.json')) # Changed '*.jsonl' to '*.json' - json_dict = self.read_json(json_fname[0]) - + json_fname = list(cwd.glob('*.jsonl')) + json_dict = self.read_json(json_fname[0]) + baseline_fname = list(cwd.glob('*baseline.csv')) baseline_dict = self.read_baseline(baseline_fname[0]) From 805df350b340c0428db41de536abf52ee58fb5a8 Mon Sep 17 00:00:00 2001 From: Phong Date: Tue, 5 Mar 2024 18:13:50 -0800 Subject: [PATCH 44/49] Minor changes to img dimensions from dask_imread --- src/PyHyperScattering/SST1RSoXSLoader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PyHyperScattering/SST1RSoXSLoader.py b/src/PyHyperScattering/SST1RSoXSLoader.py index 6119b359..be50f638 100644 --- a/src/PyHyperScattering/SST1RSoXSLoader.py +++ b/src/PyHyperScattering/SST1RSoXSLoader.py @@ -84,7 +84,7 @@ def loadSingleImage(self,filepath,coords=None, return_q=False,image_slice=None,u if self.use_chunked_loading: img = imread(str(filepath)) if img.ndim == 3: - img = img[:, :, 0] # Select the first channel if the image is multi-channel + img = img[0, :, :] # Select the first channel if the image is multi-channel else: img = np.array(Image.open(filepath)) From 0bc289b447fa480003dcf2bab90eff76eedf089e Mon Sep 17 00:00:00 2001 From: Phong Date: Wed, 6 Mar 2024 18:15:07 -0800 Subject: [PATCH 45/49] Trying alternative method to convert image to dask array because dask.imread.imread does not behave well with subsequent processing --- src/PyHyperScattering/SST1RSoXSLoader.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/PyHyperScattering/SST1RSoXSLoader.py b/src/PyHyperScattering/SST1RSoXSLoader.py index be50f638..a9ad5ff7 100644 --- a/src/PyHyperScattering/SST1RSoXSLoader.py +++ b/src/PyHyperScattering/SST1RSoXSLoader.py @@ -9,7 +9,7 @@ import json #from pyFAI import azimuthalIntegrator import numpy as np -from dask_image.imread import imread +import dask.array as da class SST1RSoXSLoader(FileLoader): ''' @@ -80,13 +80,11 @@ def loadSingleImage(self,filepath,coords=None, return_q=False,image_slice=None,u if use_cached_md != False: raise NotImplementedError('Caching of metadata is not supported for SST1') - # Use chunked loading if flag is set + img = np.array(Image.open(filepath)) + if self.use_chunked_loading: - img = imread(str(filepath)) - if img.ndim == 3: - img = img[0, :, :] # Select the first channel if the image is multi-channel - else: - img = np.array(Image.open(filepath)) + img = da.from_array(img, chunks='auto') + headerdict = self.loadMd(filepath) # two steps in this pre-processing stage: From 399d07ec31b2464858b0827900dde9f4f473bd6a Mon Sep 17 00:00:00 2001 From: Phong Date: Thu, 7 Mar 2024 16:07:33 -0800 Subject: [PATCH 46/49] Finalizing changes to chunk loading for SST1RSoXSLoader after testing --- src/PyHyperScattering/SST1RSoXSLoader.py | 44 ++++++++++++++++-------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/src/PyHyperScattering/SST1RSoXSLoader.py b/src/PyHyperScattering/SST1RSoXSLoader.py index a9ad5ff7..571394d0 100644 --- a/src/PyHyperScattering/SST1RSoXSLoader.py +++ b/src/PyHyperScattering/SST1RSoXSLoader.py @@ -11,6 +11,7 @@ import numpy as np import dask.array as da + class SST1RSoXSLoader(FileLoader): ''' Loader for TIFF files from NSLS-II SST1 RSoXS instrument @@ -29,7 +30,6 @@ def __init__(self,corr_mode=None,user_corr_func=None,dark_pedestal=100,exposure_ dark_pedestal (numeric): value to subtract(/add, if negative) to the whole image. this should match the instrument setting for suitcased tiffs, typically 100. exposure_offset (numeric): value to add to the exposure time. Measured at 2ms with the piezo shutter in Dec 2019 by Jacob Thelen, NIST constant_md (dict): values to insert into every metadata load. - use_chunked_loading (bool): flag to use chunked loading with dask or not. ''' if corr_mode == None: @@ -39,13 +39,14 @@ def __init__(self,corr_mode=None,user_corr_func=None,dark_pedestal=100,exposure_ else: self.corr_mode = corr_mode + self.constant_md = constant_md + self.dark_pedestal = dark_pedestal self.user_corr_func = user_corr_func self.exposure_offset = exposure_offset self.use_chunked_loading = use_chunked_loading # self.darks = {} - # def loadFileSeries(self,basepath): # try: # flist = list(basepath.glob('*primary*.tiff')) @@ -61,6 +62,8 @@ def __init__(self,corr_mode=None,user_corr_func=None,dark_pedestal=100,exposure_ # # return out + + def loadSingleImage(self,filepath,coords=None, return_q=False,image_slice=None,use_cached_md=False,**kwargs): ''' HELPER FUNCTION that loads a single image and returns an xarray with either pix_x / pix_y dimensions (if return_q == False) or qx / qy (if return_q == True) @@ -79,12 +82,11 @@ def loadSingleImage(self,filepath,coords=None, return_q=False,image_slice=None,u raise NotImplementedError('Image slicing is not supported for SST1') if use_cached_md != False: raise NotImplementedError('Caching of metadata is not supported for SST1') - + img = np.array(Image.open(filepath)) if self.use_chunked_loading: img = da.from_array(img, chunks='auto') - headerdict = self.loadMd(filepath) # two steps in this pre-processing stage: @@ -117,7 +119,6 @@ def loadSingleImage(self,filepath,coords=None, return_q=False,image_slice=None,u # # step 2: dark subtraction # this is already done in the suitcase, but we offer the option to add/subtract a pedestal. image_data = (img-self.dark_pedestal)/corr - image_data = img if return_q: qpx = 2*np.pi*60e-6/(headerdict['sdd']/1000)/(headerdict['wavelength']*1e10) qx = (np.arange(1,img.size[0]+1)-headerdict['beamcenter_y'])*qpx @@ -130,10 +131,19 @@ def loadSingleImage(self,filepath,coords=None, return_q=False,image_slice=None,u def read_json(self,jsonfile): json_dict = {} - with open(jsonfile) as f: - data = json.load(f) - meas_time =datetime.datetime.fromtimestamp(data[1]['time']) - json_dict['sample_name'] = data[1]['sample_name'] + + # Try / except statment to be compatible with local rsoxs data before + # and after SST1 cycle 2 2023 + try: # For earlier cyles + with open(jsonfile) as f: + data = json.load(f) + meas_time = datetime.datetime.fromtimestamp(data[1]['time']) + except KeyError: # For later cycles (confirmed working for cycle 2 2023) + with open(jsonfile) as f: + data = [0, json.load(f)] # Quick fix to load the json in a list + meas_time = datetime.datetime.fromtimestamp(data[1]['time']) + + json_dict['sample_name'] = data[1]['sample_name'] if data[1]['RSoXS_Main_DET'] == 'SAXS': json_dict['rsoxs_config'] = 'saxs' # discrepency between what is in .json and actual @@ -155,7 +165,7 @@ def read_json(self,jsonfile): json_dict['beamcenter_y'] = data[1]['RSoXS_SAXS_BCY'] json_dict['sdd'] = data[1]['RSoXS_SAXS_SDD'] - elif data[1]['RSoXS_Main_DET'] == 'WAXS': + elif (data[1]['RSoXS_Main_DET'] == 'WAXS') | (data[1]['RSoXS_Main_DET'] == 'waxs_det'): # additional name for for cycle 2 2023 json_dict['rsoxs_config'] = 'waxs' if (meas_time > datetime.datetime(2020,11,16)) and (meas_time < datetime.datetime(2021,1,15)): json_dict['beamcenter_x'] = 400.46 @@ -172,7 +182,7 @@ def read_json(self,jsonfile): json_dict['sdd'] = data[1]['RSoXS_WAXS_SDD'] else: - json_dict['rsoxs_config'] == 'unknown' + json_dict['rsoxs_config'] = 'unknown' warnings.warn('RSoXS_Config is neither SAXS or WAXS. Check json file',stacklevel=2) if json_dict['sdd'] == None: @@ -252,9 +262,15 @@ def loadMd(self,filepath): else: cwd = pathlib.Path(dirPath) - json_fname = list(cwd.glob('*.jsonl')) - json_dict = self.read_json(json_fname[0]) - + # Another try/except statement to be compatible with local data pre and + # post SST1 cycle 2 2023 + try: # For earlier data + json_fname = list(cwd.glob('*.jsonl')) + json_dict = self.read_json(json_fname[0]) + except IndexError: # For later data (works for cycle 2 2023) + json_fname = list(cwd.glob('*.json')) # Changed '*.jsonl' to '*.json' + json_dict = self.read_json(json_fname[0]) + baseline_fname = list(cwd.glob('*baseline.csv')) baseline_dict = self.read_baseline(baseline_fname[0]) From dba10cd8cac221d594f27c042c25c1e62d5f47a0 Mon Sep 17 00:00:00 2001 From: Peter Beaucage Date: Sun, 17 Mar 2024 10:12:14 -0400 Subject: [PATCH 47/49] Apply suggestions from code review --- src/PyHyperScattering/SST1RSoXSLoader.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/PyHyperScattering/SST1RSoXSLoader.py b/src/PyHyperScattering/SST1RSoXSLoader.py index 571394d0..fd209e38 100644 --- a/src/PyHyperScattering/SST1RSoXSLoader.py +++ b/src/PyHyperScattering/SST1RSoXSLoader.py @@ -9,7 +9,11 @@ import json #from pyFAI import azimuthalIntegrator import numpy as np -import dask.array as da +try: + import dask.array as da +except ImportError: + print('Could not import Dask. Chunked loading may not work. Install Dask or pyhyperscattering[performance] if this is desired.') + class SST1RSoXSLoader(FileLoader): @@ -121,7 +125,7 @@ def loadSingleImage(self,filepath,coords=None, return_q=False,image_slice=None,u image_data = (img-self.dark_pedestal)/corr if return_q: qpx = 2*np.pi*60e-6/(headerdict['sdd']/1000)/(headerdict['wavelength']*1e10) - qx = (np.arange(1,img.size[0]+1)-headerdict['beamcenter_y'])*qpx + qx = (np.arange(1,img.shape[0]+1)-headerdict['beamcenter_y'])*qpx qy = (np.arange(1,img.size[1]+1)-headerdict['beamcenter_x'])*qpx # now, match up the dims and coords return xr.DataArray(image_data,dims=['qy','qx'],coords={'qy':qy,'qx':qx},attrs=headerdict) From 3f22b887249d21b8b3510de37f257d02334e9e07 Mon Sep 17 00:00:00 2001 From: Peter Beaucage Date: Sun, 17 Mar 2024 10:13:10 -0400 Subject: [PATCH 48/49] Fix numpy/pillow size/shape convention --- src/PyHyperScattering/SST1RSoXSLoader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PyHyperScattering/SST1RSoXSLoader.py b/src/PyHyperScattering/SST1RSoXSLoader.py index fd209e38..a1118aad 100644 --- a/src/PyHyperScattering/SST1RSoXSLoader.py +++ b/src/PyHyperScattering/SST1RSoXSLoader.py @@ -126,7 +126,7 @@ def loadSingleImage(self,filepath,coords=None, return_q=False,image_slice=None,u if return_q: qpx = 2*np.pi*60e-6/(headerdict['sdd']/1000)/(headerdict['wavelength']*1e10) qx = (np.arange(1,img.shape[0]+1)-headerdict['beamcenter_y'])*qpx - qy = (np.arange(1,img.size[1]+1)-headerdict['beamcenter_x'])*qpx + qy = (np.arange(1,img.shape[1]+1)-headerdict['beamcenter_x'])*qpx # now, match up the dims and coords return xr.DataArray(image_data,dims=['qy','qx'],coords={'qy':qy,'qx':qx},attrs=headerdict) else: From 5e97d8964ed7ce3d3e115562667cecd6452c3982 Mon Sep 17 00:00:00 2001 From: Peter Beaucage Date: Sun, 17 Mar 2024 10:18:19 -0400 Subject: [PATCH 49/49] Flip qx/qy which are inverted in pillow/numpy transition --- src/PyHyperScattering/SST1RSoXSLoader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/PyHyperScattering/SST1RSoXSLoader.py b/src/PyHyperScattering/SST1RSoXSLoader.py index a1118aad..dc6b523c 100644 --- a/src/PyHyperScattering/SST1RSoXSLoader.py +++ b/src/PyHyperScattering/SST1RSoXSLoader.py @@ -125,8 +125,8 @@ def loadSingleImage(self,filepath,coords=None, return_q=False,image_slice=None,u image_data = (img-self.dark_pedestal)/corr if return_q: qpx = 2*np.pi*60e-6/(headerdict['sdd']/1000)/(headerdict['wavelength']*1e10) - qx = (np.arange(1,img.shape[0]+1)-headerdict['beamcenter_y'])*qpx - qy = (np.arange(1,img.shape[1]+1)-headerdict['beamcenter_x'])*qpx + qx = (np.arange(1,img.shape[1]+1)-headerdict['beamcenter_y'])*qpx + qy = (np.arange(1,img.shape[0]+1)-headerdict['beamcenter_x'])*qpx # now, match up the dims and coords return xr.DataArray(image_data,dims=['qy','qx'],coords={'qy':qy,'qx':qx},attrs=headerdict) else: