DM-48486 instrument specific fault loss accounting

notebooks_tsqr/NightLog.ipynb

@@ -42,7 +42,7 @@
     "# 2024-12-08: Exposures(738) time: 2:40:54\n",
     "# 2024-12-10: Exposures(614) time: 2:45:17, has Good\n",
     "# 2024-12-11: Exposures(612) time: 2:15:08\n",
-    "day_obs = \"2024-12-01\"  # TODO Change to 'YESTERDAY' and 'TODAY' to test with default before push\n",
+    "day_obs = \"YESTERDAY\"  # TODO Change to 'YESTERDAY' and 'TODAY' to test with default before push\n",
     "# Total number of days of data to display (ending on day_obs)\n",
     "number_of_days = \"1\"  # TODO Change to '1' to test with default before push\n",
     "period = \"2h\"  # TODO change to 4h before push\n",
@@ -325,12 +325,14 @@
    ]
   },
   {
-   "cell_type": "markdown",
+   "cell_type": "code",
+   "execution_count": null,
    "id": "20",
    "metadata": {},
+   "outputs": [],
    "source": [
-    "- (1) There is no practical way to get detector read-out time.  A value of 2.41 seconds per exposure is used.\n",
-    "- (2) There is currently no simple way to get slew times.  We SlewTime to find its way into the Consolidated Database eventually. "
+    "# - (1) There is no practical way to get detector read-out time.  A value of 2.41 seconds per exposure is used.\n",
+    "# - (2) There is currently no simple way to get slew times.  We SlewTime to find its way into the Consolidated Database #eventually."
    ]
   },
   {

python/lsst/ts/logging_and_reporting/all_sources.py

@@ -183,7 +183,7 @@ def dayobs_range(self):
 
     # This will have to be async def night_tally_observation_gaps
     # if efd_src is used.
-    def night_tally_observation_gaps(self, verbose=False):
+    def SAVE_night_tally_observation_gaps(self, verbose=False):
         # observable is between 18deg twilights
         total_observable_hours = self.alm_src.night_hours
         used_instruments = set()
@@ -331,9 +331,12 @@ def exposed_instruments(self):
                 used_instruments.add(instrum)
         return used_instruments
 
-    # From night_tally_observation_gaps()
+    # modified from night_tally_observation_gaps()
     # Internal times in decimal hours. Render as HH:MM:SS
     def time_account(self, verbose=False):
+        """Report on how instrument time is partitioned over the
+        observing night."""
+
         # between 18deg twilights
         total_observable_hours = self.alm_src.night_hours
         used_instruments = self.exposed_instruments()
@@ -355,7 +358,7 @@ def time_account(self, verbose=False):
             ("C", "const*<nar>", "Readout time(1)", "Total exposure readout time"),
             ("D", "(ConsDB later)", "Slew time(2)", "Total slew time"),
             ("E", "<nar>", "Time loss due to fault(3)", ""),
-            ("F", "<nar>", "Time loss due to weather(3)", ""),
+            ("F", "<nar>", "Time loss due to weather(4)", ""),
             ("G", "A-(B+C+D+F)", "Idle time", 'Time doing "nothing"'),
             ("H", "<nar>", "Number of exposures", ""),
             ("I", "D", "Number of slews", ""),
@@ -372,8 +375,14 @@ def time_account(self, verbose=False):
                 "to find its way into the Consolidated Database eventually"
             ),
             "3": (
-                "A fault is not associated with a specific Instrument so "
-                "is counted as applying to all instruments."
+                "A fault loss is assumed to be associated with a specific "
+                "Instrument so is counted as applying to just that instrument."
+            ),
+            "4": (
+                "A weather loss is not associated with a specific Instrument so "
+                "is counted as applying to all instruments. "
+                "Different telescopes might have different weather (clouds) "
+                "so this might be wrong occasionaly."
             ),
         }
 
@@ -410,9 +419,10 @@ def time_account(self, verbose=False):
             # TODO despite unreliability, use messages values.
             ltypes = set([r["time_lost_type"] for r in self.nar_src.records])
             ltime = defaultdict(int)
-            for t in ltypes:
+            for lt in ltypes:
                 for r in self.nar_src.records:
-                    ltime[t] += r["time_lost"]
+                    if r["instrument"] == instrument:
+                        ltime[lt] += r["time_lost"]
             # Loss due to FAULT
             if "fault" in ltypes:
                 loss_fault = ltime["fault"]  # hours
@@ -428,9 +438,7 @@ def time_account(self, verbose=False):
             if pd.notna(slew_hours):
                 act_hours += slew_hours
             if pd.notna(loss_fault):
-                # Never add this since it might apply to a different instrument
-                # act_hours += loss_fault
-                pass
+                act_hours += loss_fault
             if pd.notna(loss_weather):
                 act_hours += loss_weather
             idle_hours = total_observable_hours - act_hours
@@ -454,6 +462,7 @@ def rt(hours):  # Render Time
                 exposure_hours
                 + readout_hours
                 + (0 if pd.isna(slew_hours) else slew_hours)
+                + loss_fault
                 + loss_weather
                 + idle_hours
             )

python/lsst/ts/logging_and_reporting/consdb.py

@@ -85,7 +85,7 @@ def __init__(
     # where part of the name comes from the instrument. It turns
     # out EITHER lowcase or camel case is ok for table.
     # Identifiers in Postgresql are case insenstive unless quoted.
-    def get_instruments(self, include=None):
+    def get_instruments(self, include=None) -> dict:
         url = f"{self.server}/{self.service}/schema"
         ok, result, code = self.protected_get(url)
         if not ok:  # failure
@@ -213,7 +213,7 @@ def query(self, sql):
     # DM-48072 Add a visit1_exposure table to link visits and exposures
     # In the meantime KT says assume visit_id = exposure_id
     # In the presence of snaps, exposure_id and visit_id many-to-one.
-    def get_exposures(self, instrument):
+    def get_exposures(self, instrument) -> pd.DataFrame:
         """SIDE-EFFECT: cache results in self.exposures[instrument]"""
         # DM-47573
         exposure_out = [

python/lsst/ts/logging_and_reporting/source_adapters.py

@@ -749,16 +749,44 @@ def obs_date(rec):
 
     # figure out instrument name from telescope name
     def add_instrument(self, records):
-        lut = {
-            "AuxTel": "LATISS",
-            "MainTel": "LSSTCam",
-            "telescope-1": "LSSComCam",
-        }
-        found = [r["components"] for r in records if r["components"]]
-        # Above got "MainTel" only for 2024-12-01 with
-        # two instruments: LSSTComCam, LSSTCam
-        # TODO finish -- if it is even possible!!!
-        return lut, found
+        """Add 'instrument' field to records (SIDE-EFFECT)
+        Narrativelog gives Telescope (field="components") but not Instrument,
+        but we need to report by Instrument since that is what is used for
+        exposures.
+        Therefore, we must map Telescope to the Instrument that is assumed to
+        be on the Telescope.
+        We always assume Telescope=AuxTel means Instrument=LATISS
+        For Telescope=MainTel (aka Simonyi) the Instrument assumed is different
+        depending on the dayobs.
+        Prior to dayobs=2025-01-19 we assume Instrument=LSSTComCam
+          from then on we assume Instrument=LSSTCam.
+        For any Telescope value other than AuxTel or MainTel we ignore the data
+          (but warn about what Telescope we are ignoring).
+        """
+
+        LSST_DAYOBS = 20250120
+        for rec in records:
+            dayobs = int(rec["date_added"][:8].replace("-", ""))
+            if rec["components"] is None:
+                instrument = None
+            elif rec["components"][0] == "AuxTel":
+                instrument = "LATISS"
+            elif rec["components"][0] == "MainTel":
+                instrument = "lsst"
+            elif rec["components"][0] == "Simonyi":
+                instrument = "lsst"
+            else:
+                instrument = None
+
+            if instrument == "lsst":
+                if dayobs >= LSST_DAYOBS:
+                    rec["instrument"] = "LSSTCam"
+                else:  # dayobs < LSST_DAYOBS
+                    rec["instrument"] = "LSSTComCam"
+            else:
+                rec["instrument"] = instrument
+
+        return records
 
     def get_records(
         self,
@@ -818,13 +846,18 @@ def get_records(
             qparams["offset"] += len(page)
         # END: while
 
-        self.records = recs
-        # TODO Calc instrument for record based upon other fields!!!
-        # figure out instrument name from telescope name
-        # #!self.add_instrument(recs);
-        pam.markup_errors(recs)
+        self.records = self.add_instrument(recs)
+        pam.markup_errors(self.records)
         return status
 
+    def verify_records(self):
+        telescope_fault_loss = [
+            (r["date_added"], r["time_lost"], r["time_lost_type"], r["components"])
+            for r in self.records
+            if r["time_lost"] > 0
+        ]
+        return telescope_fault_loss
+
     # END: class NarrativelogAdapter