Merge branch 'main' into 129-feat-sst1rsoxsdb-normalize-dataset-retur…

…n-names-to-100

.github/workflows/main.yml

@@ -24,12 +24,11 @@ jobs:
     - name: Cache pip
       uses: actions/cache@v2
       with:
-        # This path is specific to Ubuntu
-        path: ~/.cache/pip
+        path: ${{ env.pythonLocation }}
         # Look to see if there is a cache hit for the corresponding requirements file
-        key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+        key: ${{ runner.os }}-py${{ matrix.python-version }}-pip-${{ hashFiles('requirements.txt') }}
         restore-keys: |
-         ${{ runner.os }}-pip-
+         ${{ runner.os }}-py${{ matrix.python-version }}-pip-
           ${{ runner.os }}-
     - name: Install dependencies
       run: |
@@ -76,6 +75,8 @@ jobs:
         C:\msys64\usr\bin\wget.exe https://github.com/usnistgov/PyHyperScattering/releases/download/0.0.0-example-data/CMS_giwaxs_series.zip
         unzip CMS_giwaxs_series.zip
     - name: Test with pytest
+      env: 
+         TILED_API_KEY: ${{ secrets.TILED_API_KEY }}
       run: |
         #pytest -v
         #temporarily disabling coverage for memory usage

requirements-bluesky.txt

@@ -1,2 +1,3 @@
-tiled[client]>=0.1.0a74
+tiled[all]>=0.1.0a74
 databroker[all]>=2.0.0b10
+bottleneck

requirements.txt

@@ -3,7 +3,7 @@ astropy
 fabio
 h5py
 nodejs
-numpy
+numpy<2
 pandas
 # pygix fails to improt if silx > 2.0.0 
 silx==2.0.0

src/PyHyperScattering/SST1RSoXSDB.py

@@ -609,17 +609,17 @@ def loadRun(
             else:
                 axes_to_include = []
                 rsd_cutoff = 0.005
-            
+
                 # begin with a list of the things that are primary streams
                 axis_list = list(run["primary"]["data"].keys())
-                
+
                 # next, knock out anything that has 'image', 'fullframe' in it - these aren't axes
                 axis_list = [x for x in axis_list if "image" not in x]
                 axis_list = [x for x in axis_list if "fullframe" not in x]
                 axis_list = [x for x in axis_list if "stats" not in x]
                 axis_list = [x for x in axis_list if "saturated" not in x]
                 axis_list = [x for x in axis_list if "under_exposed" not in x]
-                
+
                 # knock out any known names of scalar counters
                 axis_list = [x for x in axis_list if "Beamstop" not in x]
                 axis_list = [x for x in axis_list if "Current" not in x]
@@ -638,10 +638,10 @@ def loadRun(
                         rsd = 0
                     else:
                         rsd = std / motion
-                    #print(f'Evaluating {axis} for inclusion as a dimension with rsd {rsd}...')
+                    # print(f'Evaluating {axis} for inclusion as a dimension with rsd {rsd}...')
                     if rsd > rsd_cutoff:
                         axes_to_include.append(axis)
-                        #print(f'    --> it was included')
+                        # print(f'    --> it was included')
 
                 # next, construct the reverse lookup table - best mapping we can make of key to pyhyper word
                 # we start with the lookup table used by loadMd()
@@ -697,7 +697,7 @@ def loadRun(
             data = run["primary"]["data"].read()[md["detector"] + "_image"]
         elif isinstance(data,tiled.client.array.DaskArrayClient):
             data = run["primary"]["data"].read()[md["detector"] + "_image"]
-        
+
         data = data.astype(int)  # convert from uint to handle dark subtraction
 
         if self.dark_subtract:
@@ -1122,10 +1122,46 @@ def loadMd(self, run):
                             md[phs] = None
         md["epoch"] = md["meas_time"].timestamp()
 
+        # looking at exposure tests in the stream and issuing warnings
+        if "Wide Angle CCD Detector_under_exposed" in md:
+            if np.any(md["Wide Angle CCD Detector_under_exposed"]):
+                message = "\nWide Angle CCD Detector is reported as underexposed\n"
+                message += "at one or more energies per definitions here:\n"
+                message += "https://github.com/NSLS-II-SST/rsoxs/blob/10c2c41b695c1db552f62decdde571472b71d981/rsoxs/Base/detectors.py#L110-L119\n"
+                if np.all(md["Wide Angle CCD Detector_under_exposed"]):
+                    message += "Wide Angle CCD Detector is reported as underexposed at all energies."
+                else:
+                    idx = np.where(md["Wide Angle CCD Detector_under_exposed"])
+                    warning_e = md["energy"][idx]
+                    message += f"Affected energies include: \n{warning_e}"
+                warnings.warn(message, stacklevel=2)
+        else:
+            warnings.warn(
+                "'Wide Angle CCD Detector_under_exposed' not found in stream."
+            )
+        if "Wide Angle CCD Detector_saturated" in md:
+            if np.any(md["Wide Angle CCD Detector_saturated"]):
+                message = "\nWide Angle CCD Detector is reported as saturated\n"
+                message += "at one or more energies per definitions here:\n"
+                message += "https://github.com/NSLS-II-SST/rsoxs/blob/10c2c41b695c1db552f62decdde571472b71d981/rsoxs/Base/detectors.py#L110-L119\n"
+                if np.all(md["Wide Angle CCD Detector_saturated"]):
+                    message += "\tWide Angle CCD Detector is reported as saturated at all energies."
+                else:
+                    idx = np.where(md["Wide Angle CCD Detector_saturated"])
+                    warning_e = md["energy"][idx]
+                    message += f"Affected energies include: \n{warning_e}"
+                warnings.warn(message, stacklevel=2)
+        else:
+            warnings.warn(
+                "'Wide Angle CCD Detector_saturated' not found in stream."
+            )
+        md["epoch"] = md["meas_time"].timestamp()
+
         try:
             md["wavelength"] = 1.239842e-6 / md["energy"]
         except TypeError:
             md["wavelength"] = None
+
         md["sampleid"] = start["scan_id"]
 
         md["dist"] = md["sdd"] / 1000

tests/test_SST1DBLoader.py

@@ -10,8 +10,10 @@
         SKIP_DB_TESTING=False
     except tiled.profiles.ProfileNotFound:
         try:
-            client = tiled.client.from_uri('https://tiled-demo.blueskyproject.io')
-            SKIP_DB_TESTING=True # waiting on test data to be posted to this server
+            import os
+            api_key = os.environ['TILED_API_KEY']
+            client = tiled.client.from_uri('https://tiled.nsls2.bnl.gov',api_key=api_key)
+            SKIP_DB_TESTING=False
         except Exception:
             SKIP_DB_TESTING=True
 except ImportError:
@@ -32,10 +34,12 @@
 @pytest.fixture(autouse=True,scope='module')
 def sstdb():
     try:
-        catalog = tiled.client.from_profile('rsoxs')
+        client = tiled.client.from_profile('rsoxs')
     except tiled.profiles.ProfileNotFound:
-        catalog = tiled.client.from_uri('https://tiled-demo.blueskyproject.io')['rsoxs']['raw']
-    sstdb = SST1RSoXSDB(catalog=catalog,corr_mode='none')
+        import os
+        api_key = os.environ['TILED_API_KEY']
+        client = tiled.client.from_uri('https://tiled.nsls2.bnl.gov',api_key=api_key)['rsoxs']['raw']
+    sstdb = SST1RSoXSDB(catalog=client,corr_mode='none')
     return sstdb
 
 @must_have_tiled
@@ -64,3 +68,10 @@ def test_SST1DB_load_snake_scan_explicit_dims(sstdb):
     assert type(run) == xr.DataArray
     assert 'sam_th' in run.indexes
     assert 'polarization' in run.indexes
+
+@must_have_tiled
+def test_SST1DB_exposurewarnings(sstdb):
+    with pytest.warns(UserWarning, match="Wide Angle CCD Detector is reported as underexposed"):
+        sstdb.loadRun(83192)
+    with pytest.warns(UserWarning, match="Wide Angle CCD Detector is reported as saturated"):
+        sstdb.loadRun(67522)