diff --git a/notebooks_tsqr/NightLog.ipynb b/notebooks_tsqr/NightLog.ipynb index e0ba746..571e642 100644 --- a/notebooks_tsqr/NightLog.ipynb +++ b/notebooks_tsqr/NightLog.ipynb @@ -26,10 +26,10 @@ "# day_obs values: TODAY, YESTERDAY, YYYY-MM-DD\n", "# Report on observing nights that start upto but not included this day.\n", "#!day_obs = '2024-09-25' # Value to use for local testing (Summit)\n", - "day_obs = \"YESTERDAY\" # TODO Change to 'YESTERDAY' to test with default before push\n", + "day_obs = \"TODAY\" # TODO Change to 'YESTERDAY' to test with default before push\n", "\n", "# Total number of days of data to display (ending on day_obs)\n", - "number_of_days = \"3\" # TODO Change to '1' to test with default before push" + "number_of_days = \"1\" # TODO Change to '1' to test with default before push" ] }, { @@ -81,16 +81,10 @@ { "cell_type": "code", "execution_count": 4, - "id": "3", + "id": "ec883835-42d3-40ae-a014-8b89befe4263", "metadata": {}, "outputs": [], "source": [ - "# Normalize Parameters (both explicit Times Squares params, in implicit ones)\n", - "limit = 5000 # YAGNI for Auto get more if this isn't enough to get all requested DAYS\n", - "response_timeout = 3.05 # seconds, how long to wait for connection\n", - "read_timeout = 20 # seconds\n", - "timeout = (float(response_timeout), float(read_timeout))\n", - "\n", "date = ut.get_datetime_from_dayobs_str(day_obs)\n", "# date: is EXCLUSIVE (upto, but not including)\n", "days = int(number_of_days)\n", @@ -120,10 +114,27 @@ "name": "stdout", "output_type": "stream", "text": [ - "DEBUG add_exposure_flag instrument='LSSTComCamSim' set()\n", - "DEBUG add_exposure_flag instrument='LATISS' set()\n", - "DEBUG add_exposure_flag instrument='LSSTComCam' {'junk', 'questionable', 'unknown', 'good'}\n", - "DEBUG add_exposure_flag instrument='LSSTCam' {'unknown'}\n" + "DBG get_records endpoint='https://usdf-rsp-dev.slac.stanford.edu/nightreport/reports'\n", + "DBG get_records qstr: is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=100&min_day_obs=20241111&max_day_obs=20241112\n", + "DBG get_records len(page)=0 len(recs)=0\n", + "DBG get_exposures endpoint='https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures'\n", + "DBG get_exposures qstr: registry=1&instrument=LSSTComCamSim&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241111&max_day_obs=20241112\n", + "DBG get_exposures len(page)=0 len(recs)=0\n", + "DBG get_exposures endpoint='https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures'\n", + "DBG get_exposures qstr: registry=1&instrument=LATISS&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241111&max_day_obs=20241112\n", + "DBG get_exposures len(page)=0 len(recs)=0\n", + "DBG get_exposures endpoint='https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures'\n", + "DBG get_exposures qstr: registry=1&instrument=LSSTComCam&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241111&max_day_obs=20241112\n", + "DBG get_exposures len(page)=5 len(recs)=5\n", + "DBG get_exposures endpoint='https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures'\n", + "DBG get_exposures qstr: registry=1&instrument=LSSTCam&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241111&max_day_obs=20241112\n", + "DBG get_exposures len(page)=0 len(recs)=0\n", + "DBG get_records: endpoint='https://usdf-rsp-dev.slac.stanford.edu/exposurelog/messages'\n", + "DBG get_records qstr: is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=2500&min_day_obs=20241111&max_day_obs=20241112\n", + "NarrativeLogAdapter(server_url='https://usdf-rsp-dev.slac.stanford.edu', max_dayobs='2024-11-12', min_dayobs='2024-11-11', limit=None\n", + "Using endpoint='https://usdf-rsp-dev.slac.stanford.edu/narrativelog/messages'\n", + "Using url='https://usdf-rsp-dev.slac.stanford.edu/narrativelog/messages?is_human=either&is_valid=true&order_by=-date_added&offset=0&limit=1000&min_date_added=2024-11-11T00%3A00%3A00&max_date_added=2024-11-12T00%3A00%3A00'\n", + "Loaded data from sources in 3.7 seconds\n" ] } ], @@ -133,8 +144,8 @@ " server_url=server,\n", " min_dayobs=min_day_obs,\n", " max_dayobs=max_day_obs,\n", - " verbose=False, # TODO change to False before push\n", - " limit=limit,\n", + " verbose=True, # TODO change to False before push\n", + " limit=5000,\n", " exclude_instruments = [] # TODO change to empty list before push\n", ")\n", "allrep = AllReports(allsrc=allsrc)" @@ -153,7 +164,7 @@ { "data": { "text/markdown": [ - "# Showing data for 2024-11-05 to 2024-11-08 for LSSTComCamSim, LATISS, LSSTComCam, LSSTCam" + "# Showing data for 2024-11-11 to 2024-11-12 for LSSTComCamSim, LATISS, LSSTComCam, LSSTCam" ], "text/plain": [ "" @@ -232,7 +243,7 @@ { "data": { "text/markdown": [ - "**Almanac for the observing night starting 2024-11-05**" + "**Almanac for the observing night starting 2024-11-11**" ], "text/plain": [ "" @@ -246,80 +257,80 @@ "text/html": [ "\n", - "\n", + "
\n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", "
(times in UTC)(times in UTC)
Moon Rise2024-11-05 12:26:23.360Moon Rise2024-11-10 17:43:00.925
Moon Set2024-11-05 02:21:05.324Moon Set2024-11-11 06:41:58.146
Moon Illumination12%(% illuminated)Moon Illumination71%(% illuminated)
Astronomical Twilight (morning)2024-11-05 08:16:31.085(-18 degrees)Astronomical Twilight (morning)2024-11-11 08:10:48.514(-18 degrees)
Astronomical Twilight (evening)2024-11-05 00:36:18.653(-18 degrees)Astronomical Twilight (evening)2024-11-11 00:42:56.117(-18 degrees)
Solar Midnight2024-11-05 04:26:24.970Solar Midnight2024-11-11 04:26:52.295
Nautical Twilight (morning)2024-11-05 08:48:16.492(-12 degrees)Nautical Twilight (morning)2024-11-11 08:43:18.209(-12 degrees)
Nautical Twilight (evening)2024-11-05 00:04:33.140(-12 degrees)Nautical Twilight (evening)2024-11-11 00:10:26.410(-12 degrees)
Civil Twilight (morning)2024-11-05 09:18:46.299(-6 degrees)Civil Twilight (morning)2024-11-11 09:14:21.799(-6 degrees)
Civil Twilight (evening)2024-11-04 23:34:03.177(-6 degrees)Civil Twilight (evening)2024-11-10 23:39:22.778(-6 degrees)
Sun Rise2024-11-05 09:48:23.825Sun Rise2024-11-11 09:44:24.900
Sun Set2024-11-04 23:04:25.434Sun Set2024-11-10 23:09:19.593
\n" ], "text/plain": [ - "" + "" ] }, "metadata": {}, @@ -406,82 +417,70 @@ " \n", " \n", " LSSTComCam\n", - " LSSTCam\n", " \n", " \n", " \n", " \n", " Total Night\n", - " 7:40:12\n", - " 7:40:12\n", + " 7:27:52\n", " \n", " \n", " Total Exposure\n", - " 4:28:06\n", - " 7:30:13\n", + " 0:00:00\n", " \n", " \n", " Slew time(1)\n", " 0:00:00\n", - " 0:00:00\n", " \n", " \n", " Readout time(2)\n", - " 0:23:36\n", - " 1:13:54\n", + " 0:00:11\n", " \n", " \n", " Time loss to fault\n", " NA\n", - " NA\n", " \n", " \n", " Time loss to weather\n", " NA\n", - " NA\n", " \n", " \n", " Idle time\n", - " 3:12:06\n", - " 0:09:59\n", + " 7:27:52\n", " \n", " \n", " Number of exposures\n", - " 616\n", - " 1928\n", + " 5\n", " \n", " \n", " Mean readout time\n", " 0.000639\n", - " 0.000639\n", " \n", " \n", " Number of slews(1)\n", " 0\n", - " 0\n", " \n", " \n", " Mean Slew time(1)\n", " 0:00:00\n", - " 0:00:00\n", " \n", " \n", "\n", "" ], "text/plain": [ - " LSSTComCam LSSTCam\n", - "Total Night 7:40:12 7:40:12\n", - "Total Exposure 4:28:06 7:30:13\n", - "Slew time(1) 0:00:00 0:00:00\n", - "Readout time(2) 0:23:36 1:13:54\n", - "Time loss to fault NA NA\n", - "Time loss to weather NA NA\n", - "Idle time 3:12:06 0:09:59\n", - "Number of exposures 616 1928\n", - "Mean readout time 0.000639 0.000639\n", - "Number of slews(1) 0 0\n", - "Mean Slew time(1) 0:00:00 0:00:00" + " LSSTComCam\n", + "Total Night 7:27:52\n", + "Total Exposure 0:00:00\n", + "Slew time(1) 0:00:00\n", + "Readout time(2) 0:00:11\n", + "Time loss to fault NA\n", + "Time loss to weather NA\n", + "Idle time 7:27:52\n", + "Number of exposures 5\n", + "Mean readout time 0.000639\n", + "Number of slews(1) 0\n", + "Mean Slew time(1) 0:00:00" ] }, "metadata": {}, @@ -527,67 +526,7 @@ { "data": { "text/markdown": [ - "- [OBS-648](https://rubinobs.atlassian.net/browse/OBS-648)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- [OBS-321](https://rubinobs.atlassian.net/browse/OBS-321)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- [OBS-646](https://rubinobs.atlassian.net/browse/OBS-646)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- [/rubinobs-lfa-cp/LOVE/OLE/2024/11/04/LOVE_OLE_2024-11-05T04:06:17.057.png](https://s3.cp.lsst.org/rubinobs-lfa-cp/LOVE/OLE/2024/11/04/LOVE_OLE_2024-11-05T04:06:17.057.png)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- [OBS-649](https://rubinobs.atlassian.net/browse/OBS-649)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- [obs-647](https://rubinobs.atlassian.net/browse/obs-647)" + "- [OBS-651](https://rubinobs.atlassian.net/browse/OBS-651)" ], "text/plain": [ "" @@ -599,7 +538,7 @@ { "data": { "text/markdown": [ - "- [obs-321](https://rubinobs.atlassian.net/browse/obs-321)" + "- [OBS-614](https://rubinobs.atlassian.net/browse/OBS-614)" ], "text/plain": [ "" @@ -611,7 +550,7 @@ { "data": { "text/markdown": [ - "- [obs-611](https://rubinobs.atlassian.net/browse/obs-611)" + "- [OBS-659](https://rubinobs.atlassian.net/browse/OBS-659)" ], "text/plain": [ "" @@ -640,7 +579,7 @@ { "data": { "text/markdown": [ - "No tickets found using: [API Data](https://usdf-rsp-dev.slac.stanford.edu/nightreport/reports?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=100&min_day_obs=20241105&max_day_obs=20241108) in `confluence_url`." + "No tickets found using: [API Data](https://usdf-rsp-dev.slac.stanford.edu/nightreport/reports?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=100&min_day_obs=20241111&max_day_obs=20241112) in `confluence_url`." ], "text/plain": [ "" @@ -688,8 +627,9 @@ "id": "21", "metadata": {}, "source": [ - "- NOTE: *modified all_sources.py:uniform_field_counts()*\n", - "- NOTE: Each tally count is now a link to a detailed table of exposures." + "- Should we replace zero with empty string?\n", + "- TODO: Do not show empty tables. ie no table when \"0 total exposures\"\n", + "- TODO: Add link to exposures detail when A field value (e.g. observation_reason=science) is clicked on. The detail should include all exposures counted int the total for that field value." ] }, { @@ -710,133 +650,13 @@ "metadata": {}, "output_type": "display_data" }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Observation Type\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
\n", - "
" - ], - "text/plain": [ - "Empty DataFrame\n", - "Columns: []\n", - "Index: []" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Observation Reason\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
\n", - "
" - ], - "text/plain": [ - "Empty DataFrame\n", - "Columns: []\n", - "Index: []" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Science Program\n" - ] - }, { "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
\n", - "
" + "text/markdown": [ + "**LATISS: 0 total exposures**" ], "text/plain": [ - "Empty DataFrame\n", - "Columns: []\n", - "Index: []" + "" ] }, "metadata": {}, @@ -845,7 +665,7 @@ { "data": { "text/markdown": [ - "**LATISS: 0 total exposures**" + "**LSSTComCam: 5 total exposures**" ], "text/plain": [ "" @@ -882,17 +702,29 @@ " \n", " \n", " \n", + " good\n", + " questionable\n", + " junk\n", + " unknown\n", + " total\n", " \n", " \n", " \n", + " \n", + " bias\n", + " 0\n", + " 0\n", + " 0\n", + " 5\n", + " 5\n", + " \n", " \n", "\n", "" ], "text/plain": [ - "Empty DataFrame\n", - "Columns: []\n", - "Index: []" + " good questionable junk unknown total\n", + "bias 0 0 0 5 5" ] }, "metadata": {}, @@ -926,17 +758,29 @@ " \n", " \n", " \n", + " good\n", + " questionable\n", + " junk\n", + " unknown\n", + " total\n", " \n", " \n", " \n", + " \n", + " bias\n", + " 0\n", + " 0\n", + " 0\n", + " 5\n", + " 5\n", + " \n", " \n", "\n", "" ], "text/plain": [ - "Empty DataFrame\n", - "Columns: []\n", - "Index: []" + " good questionable junk unknown total\n", + "bias 0 0 0 5 5" ] }, "metadata": {}, @@ -970,7428 +814,35 @@ " \n", " \n", " \n", + " good\n", + " questionable\n", + " junk\n", + " total\n", + " \n", + " \n", + " unknown\n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", - " \n", - "\n", - "" - ], - "text/plain": [ - "Empty DataFrame\n", - "Columns: []\n", - "Index: []" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "**LSSTComCam: 616 total exposures**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Observation Type\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
goodquestionablejunkunknowntotal
acq502161168
science3108286
bias0007474
focus10089
cwfs102230233
dark1004546
\n", - "
" - ], - "text/plain": [ - " good questionable junk unknown total\n", - "acq 5 0 2 161 168\n", - "science 3 1 0 82 86\n", - "bias 0 0 0 74 74\n", - "focus 1 0 0 8 9\n", - "cwfs 1 0 2 230 233\n", - "dark 1 0 0 45 46" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Observation Reason\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
goodquestionablejunkunknowntotal
intra1016163
intra_spherical_alignment0001616
extra_sitcom-8260003838
comcam-usdf-checkout0003030
comcam_twilight_flat3002427
object2002830
extra_spherical_alignment0001616
focus_sweep_cam_dz10089
infocus0026264
check00099
infocus_sitcom-8260003636
extra0016162
infocus_spherical_alignment0001616
intra_sitcom-8260003838
science1105456
comcam-daily-calibration0008888
comcam_manual_alignment30025
acq0001313
\n", - "
" - ], - "text/plain": [ - " good questionable junk unknown total\n", - "intra 1 0 1 61 63\n", - "intra_spherical_alignment 0 0 0 16 16\n", - "extra_sitcom-826 0 0 0 38 38\n", - "comcam-usdf-checkout 0 0 0 30 30\n", - "comcam_twilight_flat 3 0 0 24 27\n", - "object 2 0 0 28 30\n", - "extra_spherical_alignment 0 0 0 16 16\n", - "focus_sweep_cam_dz 1 0 0 8 9\n", - "infocus 0 0 2 62 64\n", - "check 0 0 0 9 9\n", - "infocus_sitcom-826 0 0 0 36 36\n", - "extra 0 0 1 61 62\n", - "infocus_spherical_alignment 0 0 0 16 16\n", - "intra_sitcom-826 0 0 0 38 38\n", - "science 1 1 0 54 56\n", - "comcam-daily-calibration 0 0 0 88 88\n", - "comcam_manual_alignment 3 0 0 2 5\n", - "acq 0 0 0 13 13" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Science Program\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
goodquestionablejunkunknowntotal
BLOCK-T940004343
BLOCK-T1500004545
unknown5005257
BLOCK-T2200008888
BLOCK-T2460003030
BLOCK-T600003535
BLOCK-T2153002427
BLOCK-T8800099
BLOCK-T921002930
BLOCK-T249000115115
BLOCK-T2521047681
PP-SURVEY1105456
\n", - "
" - ], - "text/plain": [ - " good questionable junk unknown total\n", - "BLOCK-T94 0 0 0 43 43\n", - "BLOCK-T150 0 0 0 45 45\n", - "unknown 5 0 0 52 57\n", - "BLOCK-T220 0 0 0 88 88\n", - "BLOCK-T246 0 0 0 30 30\n", - "BLOCK-T60 0 0 0 35 35\n", - "BLOCK-T215 3 0 0 24 27\n", - "BLOCK-T88 0 0 0 9 9\n", - "BLOCK-T92 1 0 0 29 30\n", - "BLOCK-T249 0 0 0 115 115\n", - "BLOCK-T252 1 0 4 76 81\n", - "PP-SURVEY 1 1 0 54 56" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "**LSSTCam: 1928 total exposures**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Observation Type\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
goodquestionablejunkunknowntotal
unknown00011
flat000656656
bias000395395
dark000876876
\n", - "
" - ], - "text/plain": [ - " good questionable junk unknown total\n", - "unknown 0 0 0 1 1\n", - "flat 0 0 0 656 656\n", - "bias 0 0 0 395 395\n", - "dark 0 0 0 876 876" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Observation Reason\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
goodquestionablejunkunknowntotal
preimage00066
unknown00011
flat00010661066
bias000109109
sflat_hi0005050
dark000696696
\n", - "
" - ], - "text/plain": [ - " good questionable junk unknown total\n", - "preimage 0 0 0 6 6\n", - "unknown 0 0 0 1 1\n", - "flat 0 0 0 1066 1066\n", - "bias 0 0 0 109 109\n", - "sflat_hi 0 0 0 50 50\n", - "dark 0 0 0 696 696" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Science Program\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
goodquestionablejunkunknowntotal
E21840001010
E2236000100100
unknown000334334
E218000033
E2136000201201
E22290002424
E2230000201201
E223400011
E218300033
E2016000448448
E223500011
E2181000181181
E2237000421421
\n", - "
" - ], - "text/plain": [ - " good questionable junk unknown total\n", - "E2184 0 0 0 10 10\n", - "E2236 0 0 0 100 100\n", - "unknown 0 0 0 334 334\n", - "E2180 0 0 0 3 3\n", - "E2136 0 0 0 201 201\n", - "E2229 0 0 0 24 24\n", - "E2230 0 0 0 201 201\n", - "E2234 0 0 0 1 1\n", - "E2183 0 0 0 3 3\n", - "E2016 0 0 0 448 448\n", - "E2235 0 0 0 1 1\n", - "E2181 0 0 0 181 181\n", - "E2237 0 0 0 421 421" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "for instrum in allsrc.exp_src.exposures.keys():\n", - " md(f'**{instrum}: {len(allsrc.exp_src.exposures[instrum])} total exposures**')\n", - " df_dict = allsrc.fields_count_exposure(instrum)\n", - " for field_name,df in df_dict.items():\n", - " print(field_name.title().replace('_',' '))\n", - " display(df.T)" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "22", - "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "### Experimental Exposure field Tally DISASBLED" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "**LSSTComCamSim: 0 total exposures**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "**LATISS: 0 total exposures**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "**LSSTComCam: 616 total exposures**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "**LSSTCam: 1928 total exposures**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# Exposure Report\n", - "if experimental.get(\"exposure_type_tally\"): \n", - " # md(\"### Experimental Exposure field Tally ENABLED\")\n", - " md(\"Tally exposures with specific values of selected fields\")\n", - "else:\n", - " md(\"### Experimental Exposure field Tally DISASBLED\")\n", - " \n", - "for instrum in allsrc.exp_src.exposures.keys():\n", - " md(f'**{instrum}: {len(allsrc.exp_src.exposures[instrum])} total exposures**')\n", - " if 0 == len(allsrc.exp_src.exposures[instrum]):\n", - " continue\n", - "\n", - " # Facet counts\n", - " if experimental.get(\"exposure_type_tally\"): \n", - " counts,cc,f = allsrc.uniform_exposure_field_counts(instrum)\n", - " for fname, tally in []: # counts.items():\n", - " md(f\"{fname}\")\n", - " df = pd.DataFrame.from_dict(tally, orient=\"index\").T\n", - " wdf = ut.wrap_dataframe_columns(df)\n", - " display(HTML(wdf.to_html(render_links=True, escape=False)))" - ] - }, - { - "cell_type": "markdown", - "id": "261925dc-1171-4429-b6a8-3bf62d71904f", - "metadata": {}, - "source": [ - "Note: *Added counts of Exposures with messages*" - ] - }, - { - "cell_type": "markdown", - "id": "20820900-05ab-4ca5-8df5-9f725c0ee570", - "metadata": {}, - "source": [ - "## Narrative Log BETA " - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "26", - "metadata": { - "editable": true, - "jupyter": { - "source_hidden": true - }, - "slideshow": { - "slide_type": "" - }, - "tags": [] - }, - "outputs": [ - { - "data": { - "text/markdown": [ - "- **2024-11-05 10:06:04.409120** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Done for tonight. All TMA and CSCs are closed as requested.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 09:48:51.572165** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
2024 11 05 09:47:37.238 Hardpoint #2 hit high limit.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 09:46:08.288296** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
NO fault! Locking pin engaged and turning OFF PS. M1M3 lowered and will disabled.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 09:31:44.430342** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Retracted the mirror cover. Controlling with EUI again. Will move to Horizon...! Mirror will keep being raise per Petr's request.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 09:25:45.527290** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
We realized that we need retracted (open) for the mirror cover on Tuesday. Will enable the MTMount to retract the mirror.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 09:12:16.703310** Time Lost: 0.2; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
measure failed. Stop tracking and the rotator back to 0.  Rotator not commanded to 0,  so we tried to 1) reenable MTRotator CSC 2) cycle MTRotator CSC but not working. Then 3) move 38 first (where CCW) then it failed. Then  move to 0 and then it worked.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- Link: [obs-611](https://rubinobs.atlassian.net/browse/obs-611)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 09:01:37.017558** Time Lost: 0.5; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
We have an issue with MTDome slit while we closed. \r\n",
-       "\r\n",
-       "\r\n",
-       "- We closed the left shutter and then right shutter, the right one was completed (and click sounds there) but the left one was not done (definitely longer than usual). We could hear some sounds to keep closing it.\r\n",
-       "- We opened the left one again, then stop (click the open button once again) and closed the left on again (click the close button). The left one is closing again but take more then few minutes.\r\n",
-       "- At some point, two lights off but I can hear some unusual big sounds (not the simple click sounds) from the dome cabinet, another one from the cabinet with the open/closing button.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- Link: [obs-647](https://rubinobs.atlassian.net/browse/obs-647)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 08:59:27.669499** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Check with Petr. K, we will not lower the mirror and try to go to horizon. We will monitor if all forces on the HPs are fine.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 08:46:06.336612** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Starting parking the TMA on Az direction first. (-32 deg). Parking thru EUI.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 08:45:28.633575** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
BLOCK-T220 (Daytime calibration for Comcam) is done. Starting parking for TMA.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 08:00:07.024341** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Starting Block-T220.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 07:56:07.467968** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Comcam Daytime DayCalibration. T220.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 07:54:17.883749** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
We closed the MTDome and run the HVAC system. Keep setpoint 9C.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 05:53:17.114015** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: None\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Now we are going to apply the modes determined by the MTAOS component.\r\n",
-       "\r\n",
-       "\r\n",
-       "It worked but it is still not as a good as our best case.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 05:41:05.421205** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: None\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
We are doing block 249 to do the MTAOS loop\r\n",
-       "\r\n",
-       "\r\n",
-       "We are using \r\n",
-       "\r\n",
-       "\r\n",
-       "filter: \"r_03\"\r\n",
-       "exposure_time: 30\r\n",
-       "max_iter: 1\r\n",
-       "mode: \"FAM\"\r\n",
-       "program: \"BLOCK-T249\"\r\n",
-       "reason: \"SITCOM-826\"\r\n",
-       "note: \"optical_state_hexapods\"\r\n",
-       "used_dofs: [0, 3, 4, 5, 8, 9]\r\n",
-       "apply_corrections: false\r\n",
-       "use_ocps: true
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 05:36:07.239689** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: None\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
It was hard to draw a conclusion from the title sweep.  We are going to tr to run MTAOS system to find the best optical point instead.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 05:29:56.274032** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
-Apply Cam_dz: -1500 with apply_dof.py.\r\n",
-       "\r\n",
-       "\r\n",
-       "-Another focus_sweep_comcam.py with following config.\r\n",
-       "axis: v\r\n",
-       "focus_window: 0.0666\r\n",
-       "n_steps: 9\r\n",
-       "program: \"BLOCK-T92\"\r\n",
-       "hexapod: \"Camera\"\r\n",
-       "reason: \"focus_sweep_cam_dry\"
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 05:27:30.087917** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Now we are going to do a tilt sweep on all intrafocal donuts.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 05:20:17.796629** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
We gave set_dof.py\r\n",
-       "Cam_dx:-126.1\r\n",
-       "Cam_dy: 690.41\r\n",
-       "Cam_dz: 2723.51\r\n",
-       "Cam_rx: 0.01\r\n",
-       "Cam_ry: 2.26e-3\r\n",
-       "M2_dz: -1967.92\r\n",
-       "\r\n",
-       "\r\n",
-       "Then We focus_sweep_comcam.py with (part of BLOCK-T92, not on the night plan for tonight). \r\n",
-       "axis: z\r\n",
-       "focus_window: 300\r\n",
-       "n_steps: 9\r\n",
-       "program: \"BLOCK-T92\"\r\n",
-       "hexapod: \"Camera\"\r\n",
-       "reason: \"focus_sweep_cam_dz\"
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 05:07:37.076344** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
exitFault for MTDome and cycle MTDome CSC again.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 05:06:00.484543** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Track_target.py timeout. MTDome seems fault. We cycled the MTDome.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 04:55:23.292922** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
MTMount and MTPtg were fault. \r\n",
-       "Name: SofteMotion Axis Fault system: Azimuth, code: 101 04:50:30.176 05/11/24\r\n",
-       "The softmotion axis went to fault. \r\n",
-       "Then following\r\n",
-       "Name: Extrapolation/system: Azimuth/name: Time Exceeded 04:50:30.176 05/11/24 code: 160\r\n",
-       "The defined extrapolation time was exceeded.\r\n",
-       "Then elevation axis was fault with code: 401 for the softmotion axis went to fault.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 04:47:51.175339** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
We took a series of three images with no changes with PSFs that varied from 1.79 to 1.73.  So we shouldn't pay attention to anything less than the first significant figure.\r\n",
-       "\r\n",
-       "\r\n",
-       "Now doing the sweep over a larger area.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 04:47:38.703920** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Looks like we missed the minimum so we are going to scan over a larger range.  We also are applying a title that was extracted from the laser tracker test but was left out after we lost the state.  Our first attempt was unsuccessful.\r\n",
-       "\r\n",
-       "\r\n",
-       "We are attempting to go back to our previous best case.  We are close but not the same.  We are doing a quick check of taking a serfies of images to see how much variance we get just from the atmosphere.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 04:47:25.662723** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
The previous step got us to 1.78\". So now we are we going to run the parameter sweep for moving the hexapods in the opposite directions. This is to remove spherical aberration.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 04:46:52.843549** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
The donuts look better now but the PSF looks worse than our worse focus in our in focus cases.  That is probably because the focus point changed some, so we will rescan for the focus point.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 04:06:19.079415** Time Lost: 1.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['Dome']\n", - "\n", - "
The MTDome AMCS is going to fault from time to time. I noticed that every time it goes to fault there is a jump in the azimuth position. I am wondering if this is due to some encoder issue or something like that.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- Link: [OBS-646](https://rubinobs.atlassian.net/browse/OBS-646)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- Link: [/rubinobs-lfa-cp/LOVE/OLE/2024/11/04/LOVE_OLE_2024-11-05T04:06:17.057.png](https://s3.cp.lsst.org/rubinobs-lfa-cp/LOVE/OLE/2024/11/04/LOVE_OLE_2024-11-05T04:06:17.057.png)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 03:51:04.771798** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Dome faulted and we moved it back into position.\r\n",
-       "\r\n",
-       "\r\n",
-       "Then we saw that we didn't move back exactly to the best focus position.  So, we tried the position in the hexapod before the one we tried:\r\n",
-       "\r\n",
-       "\r\n",
-       "Cam_dx:-126.1\r\n",
-       "Cam_dy: 690.41\r\n",
-       "Cam_dz: 223.51\r\n",
-       "Cam_rx: 0.01\r\n",
-       "Cam_ry: 2.26e-3\r\n",
-       "M2_dz: 532.08
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 03:41:05.904448** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
We are setting the optical state to the the same position as the image 41:\r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "Cam_dx:-126.1\r\n",
-       "Cam_dy: 690.41\r\n",
-       "Cam_dz: 223.51\r\n",
-       "Cam_rx: 0.01\r\n",
-       "Cam_ry: 2.26e-3\r\n",
-       "M2_dz: 565.42
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 03:39:48.698092** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
After this sweep we found the minimum in focus to be at  Image 41 with the Camera Hexapod at zposition was 223 microns and with  a PSF of 1.8\".
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 03:25:05.167546** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
After restoring and moving the dome back to 160 degrees, we started rerunning the parameter march.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 03:11:09.961074** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
The potinting component and the dome went to fault when after tracking we hit the 180 degree limit.  We had to restore the component.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 03:00:58.888565** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
We took a triplet after restoring the optical state found by the laser alignment.  The focus was close to the alignment.\r\n",
-       "\r\n",
-       "\r\n",
-       "Now we move to T60 and scan through a movement of both hexapods. We will move the two hexapods both together in the same and then in opposite directions.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 02:43:59.587656** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
We ran a set_dof with Cam_dz: 190.18 --- this is the absolute osition of cam hex when in focus
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 02:41:52.390816** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
I think we accumulated offsets in hexapods :/ that screwed up the triplet taking now. the accumulation was due to failed scripts before
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 02:36:33.436739** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Ok Tiago fixed the config fail , and the script failed with a new error - \r\n",
-       "--------------\r\n",
-       "\r\n",
-       "\r\n",
-       "AttributeError: 'Remote' object has no attribute 'evt_imageInOODS'
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 02:27:24.593881** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
take_aos_sequence_comcam config failed with the following error \r\n",
-       "------------------------\r\n",
-       "\r\n",
-       "\r\n",
-       "tcs_ready_to_take_data=self.mtcs.tcs_ready_to_take_data,\r\n",
-       "                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "AttributeError: 'MTCS' object has no attribute 'tcs_ready_to_take_data'
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 02:22:34.853711** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Taking a triplet now --- we saw a beautiful unwinding spiral <3
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 02:21:38.571326** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Taking ACQ with ComCam with 1 sec Exp
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 02:15:30.721275** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
enabling dome following now
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 02:15:23.818506** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Ok dome is not reacting -- we disabled dome following and moved it to our Az position
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 02:13:00.531123** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
re-enabling dome following
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 02:12:16.429220** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Dome entered a FAULT state --- running exitFault
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 02:10:02.342493** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Started tracking at the position we are at with \r\n",
-       "----------------\r\n",
-       "component: MTPtg\r\n",
-       "cmd: startTracking
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 02:08:35.766532** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Now the Align worked fpor Camera. Next we executed a run_command with with -\r\n",
-       "-------------------\r\n",
-       "\r\n",
-       "\r\n",
-       "component: LaserTracker:1\r\n",
-       "cmd: measurePoint\r\n",
-       "parameters:\r\n",
-       "  collection: A\r\n",
-       "  pointgroup: M1M3\r\n",
-       "  target: p2
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 02:00:29.576596** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
We are doing Lasrer Tracker Alignment now -- succsessful for M2, failed for Camera . Moved the Rotator to 0 and re-enabled CCW following because it was found to not follow CCW
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 01:15:39.676421** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
move p2p to az=160, el=65/
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 01:15:26.620523** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Camhexapod faulted, compensation mode failed.\r\n",
-       "Recover and re-enabled the compensation mode.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 01:00:07.391914** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
TMA was still tracking so faulted. \r\n",
-       "We recovered MTPtg.\r\n",
-       "MTRotator faulted because MTMount stopped tracking \r\n",
-       "Recover MTRotator after recovering CCW to follow\r\n",
-       "cmd: enableCameraCableWrapFollowing
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 00:52:12.912966** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Opening the mirror cover.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 00:51:26.178743** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Done with the TMA Checkout.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 00:49:16.581543** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
exitFault due to the fault code from MTDome.\r\n",
-       "Even track_target is ignoring MTDome, MTDomeTrajectory, due to tracking: 60sec, MTDome and MTDomeTrajectory should be ENABLED and cleared with the faults.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 00:46:10.713352** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
There were typo in the configuration for track_target with ignore : mtdome and mtdometrajectory. stopped the script and re-run.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 00:28:28.185741** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Enabling hexapods and MTAOS. Home both axis.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 00:24:42.862724** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
reset alarm from EUI read out the elevation ok. Enabling MTMount again
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 00:22:17.448025** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
changed to CSC mode, elevation read wrong again. switching back to EUI mode, turn on the elevation drive to readout correct position.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 00:19:40.732169** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Enable MTMount
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 00:17:34.870941** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
We had issue with TMA positive limit switch. It usually means elevation position is read out wrong, so we ignored the limit switch and turned on elevation drive, moved to 80, changed settings back to default.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 10:10:28.412396** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Parking with static support with disabled M1M3.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 10:07:57.622784** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
M1M3 was fault. 2024-11-06 10:03:50.047 [050002] Force controller X Moment Limit - applied 4004.25 N, expected -4000.00 N to 4000.00 N\r\n",
-       "Also MTHexapod2 was fault while try to move to origin. will do later. M1M3 is in disabled. Will park with EUI.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 09:47:53.046133** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Start Parking Az: 148, El: 0.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 09:40:26.777500** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
BLOCK-T240 is done. We will park the TMA.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 09:34:20.472294** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
T-241 done without issue. T-240  - move_p2p is running. el 50 to 80. az fixed to 0.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 09:10:43.434671** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
move_p2p.py have az:0 el: 80 inside the block. Slewing.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 08:53:24.072877** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Put the queue for BLOCK-T231, the Az brake test, but for az: -78 deg, el: 30deg.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 08:52:12.978631** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
T-241, M2 open loop test  done without any issue.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 08:38:25.565116** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Scheduler still taking i band data.  Restarting scheduler with fixed yaml to omit i
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 08:37:59.501983** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
234 is the first of the DM sequence.  Ignore the out-of-focus i exposures
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 08:37:33.802270** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
we opened the m1m3 mirror cover again.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 08:25:08.525169** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
T-220 is ongoing.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 07:38:22.100641** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: None\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Trying the scheduler again
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 07:37:06.228112** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: None\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Offset 1 degree in alt; no obvious waffle.  Taking 30 15s exposures
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 07:34:56.254100** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: None\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Back to r;  forgot to offset the star so still waffles
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 07:27:19.822096** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: None\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Saw a ghost!  Stop the sequence, and take a pinhole 120s exposure (BLOCK-T251)
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 07:24:36.896987** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: None\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Scheduler failed again;  restarting tracking and take 30 15s exposures
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 07:22:35.957301** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: None\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
trying again on scheduler; stopping sequence of 15s exposures
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 07:20:46.851463** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: None\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
While waiting for the scheduler, resume tracking and take 30 15s exposures
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 07:11:37.400246** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: None\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Problems enabling the FB scheduler
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 07:05:09.775237** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: None\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Slewing to DM dither field
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 06:16:43.332637** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: None\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Taking data to test effect of exposure time
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 06:05:03.173660** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: None\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Elevation below 50; returning to c. 65 for exposure time studies
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 05:49:14.068126** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: None\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Stepping back to zero in 500 micron steps, followed by a set_dof (should be a NOOP)
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 05:43:46.145565** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
BLOCK-T249  close_loop_comcam.py used_dof [5] (n.b. different field)
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 05:41:40.843075** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: None\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
getting to 2500 microns in y using 500 micron steps
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 05:36:42.575098** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: None\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
m2 hexapod faulted again.  Big moves in y (but not out of range)
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 05:20:57.614436** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
m2 hexapod faulted
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 05:16:53.731713** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
perturbing degrees of freedom and observing response of wavefront
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 05:14:54.045805** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
issued track_target command to slew back to 160...  stopped and slewing back to -75
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 05:14:25.186568** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Images look comparable to those at az=165.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 04:32:52.763184** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Slew to \"morning\" azimuth finished with timeout.  Rerunning track_target.py
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 04:25:23.194034** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
database filled and rendered e.g. LOVE inoperable.  Carlos resized the PV, and we're back
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- Link: [OBS-649](https://rubinobs.atlassian.net/browse/OBS-649)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 02:55:46.084959** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
run close_loop_comcam.py but with only used_dofs:[5]
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 02:54:12.938425** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
offset_camera_hexapod\r\n",
-       "z: 400
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 02:51:44.254084** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Getting worse. Rerun set_dof.py
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 02:45:10.179289** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Issueing MTAOS correction and then another iteration of close_loop_comcam !
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 02:37:52.944744** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Running close-loop_comcam using --\r\n",
-       "\r\n",
-       "\r\n",
-       "filter: \"r_03\"\r\n",
-       "exposure_time: 30\r\n",
-       "max_iter: 1\r\n",
-       "mode: \"FAM\"\r\n",
-       "program: \"BLOCK-T249\"\r\n",
-       "reason: \"SITCOM-826\"\r\n",
-       "note: \"optical_state_hexapods\"\r\n",
-       "used_dofs: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\r\n",
-       "apply_corrections: false\r\n",
-       "use_ocps: true
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 02:35:42.629283** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
We generally had repeat issues of the MTDome cRio where the dome stopped following in between and we had to recycle MTDome after clearing errors -- this has happed once so far.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- Link: [OBS-646](https://rubinobs.atlassian.net/browse/OBS-646)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 02:33:41.181099** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Something went wrong again - spherical has gone worse again :(
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 02:32:07.717605** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Setting dof again to the same hexapod positions and taking anither triplet
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 02:28:39.513902** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
START OF BLOCK-T249 \r\n",
-       "\r\n",
-       "\r\n",
-       "Cycling MTAOS
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 02:27:43.118004** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
END OF BLOCK-T60 \r\n",
-       "\r\n",
-       "\r\n",
-       "setting dof to new hexapod positions and taking an aos triplet.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 02:25:54.873490** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Dome successfully recovered ! and we tracked the target  !
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 02:21:52.227673** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Commanding a move to MTDome to 155 deg and it moved ! \r\n",
-       "\r\n",
-       "\r\n",
-       "Enable dome following again !
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 02:16:42.745343** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Dome is not moving. Its still braked with seals engaged. Cycling MTDOME \r\n",
-       "\r\n",
-       "\r\n",
-       "1. we first disabled dome following.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 02:12:38.163491** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
We are doing track_target again hoping the dome follows
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 02:10:25.127728** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
We were tracking for too long -- we overshot the AZ setpoint position by 2 degrees. We now check  that MTDome in parking state because its in FAULTYS. Guess track_target was waiting for confirmation from MTDome which it was not recieving. \r\n",
-       "\r\n",
-       "\r\n",
-       "MTDome recovery in progress -- \r\n",
-       "1. we disabled dome following \r\n",
-       "2. exitFault for dome \r\n",
-       "3. dome now with breaks engaged
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 01:53:20.352283** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
going back t o Az =160 deg
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 01:52:23.328249** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
New hexapod absolute positions -- \r\n",
-       "\r\n",
-       "\r\n",
-       "Cam_dx: -141.85\r\n",
-       "Cam_dy: 673.96\r\n",
-       "Cam_dz: 1972.47\r\n",
-       "Cam_rx: 7.76e-3\r\n",
-       "Cam_ry: 0\r\n",
-       "M2_dz: -1265.88
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 01:49:06.023132** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
The best positions found through live analysis are in seq 84 !
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 01:44:30.471912** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
We got a spherical zernike coeff of 0.001 !
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 01:38:33.627742** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Now doing a parameter march again with -- \r\n",
-       "\r\n",
-       "\r\n",
-       "exp_time: 30.0\r\n",
-       "dofs: [1.0, 0.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]\r\n",
-       "step_sequence: [-1750, -2000]\r\n",
-       "program: \"BLOCK-T60\"\r\n",
-       "reason: \"spherical_alignment\"
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 01:37:53.802799** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Applying the last set_dof again -- \r\n",
-       "\r\n",
-       "\r\n",
-       "Cam_dx: -141.85\r\n",
-       "Cam_dy: 673.96\r\n",
-       "Cam_dz: 222.47\r\n",
-       "Cam_rx: 7.76e-3\r\n",
-       "Cam_ry: 0\r\n",
-       "M2_dz: 484.12
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 01:34:47.301432** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
We are trying again even if the hexapods did not like this big move !
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 01:32:53.302179** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Recovered both hexapods and enabled compensation mode again
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 01:30:45.685818** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Naturally the parameter_march_comcam script failed becasue of this.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 01:30:13.819125** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Both Hexapods FAULTED --- with low-level controller going to FAULT.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 01:27:11.150742** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
We are doing three more steps in the same direction !  parameter_march now with -- \r\n",
-       "\r\n",
-       "\r\n",
-       "exp_time: 30.0\r\n",
-       "dofs: [1.0, 0.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]\r\n",
-       "step_sequence: [-1750, -2000]\r\n",
-       "program: \"BLOCK-T60\"\r\n",
-       "reason: \"spherical_alignment\"
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 01:21:10.402284** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Spherical has come down the coma and astigmatism is still high .....
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 01:18:29.540167** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
keyboard  check !
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 01:15:02.118472** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
parameter_march_comcam.py\r\n",
-       "\r\n",
-       "\r\n",
-       "exp_time: 30.0\r\n",
-       "dofs: [1.0, 0.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]\r\n",
-       "step_sequence: [-1000, -1500]\r\n",
-       "program: \"BLOCK-T60\"\r\n",
-       "reason: \"spherical_alignment\"
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 01:13:18.007510** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
since we were near az=180, we track_target again to el=65, az=160 after parameter_march_comcam.py was done.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 01:12:14.092249** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
sORRY it took only 3 sets not 7 ! \r\n",
-       "\r\n",
-       "\r\n",
-       "Also we are hitting the limit of +180 deg in Azimuth ! \r\n",
-       "\r\n",
-       "\r\n",
-       "We are going back to 160 deg in Az
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 01:08:38.871962** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
set_dof.py config:\r\n",
-       "\r\n",
-       "\r\n",
-       "Cam_dx: -141.85\r\n",
-       "Cam_dy: 673.96\r\n",
-       "Cam_dz: 222.47\r\n",
-       "Cam_rx: 7.76e-3\r\n",
-       "Cam_ry: 0\r\n",
-       "M2_dz: 484.12
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 01:04:18.065893** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: None\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Running parameter_march_comca.py with following configs --\r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "exp_time: 30.0\r\n",
-       "dofs: [1.0, 0.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]\r\n",
-       "step_sequence: [-100, -250, -500]\r\n",
-       "program: \"BLOCK-T60\"\r\n",
-       "note: \"spherical_alignment\"\r\n",
-       "\r\n",
-       "\r\n",
-       "----------------------------\r\n",
-       "It will start taking 7 sets !
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 01:02:27.169129** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: None\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Took 7 sets (intra , extra , in-focus)-- punchline - rather than looking at PSF we should be looking at z11 !  \r\n",
-       "\r\n",
-       "\r\n",
-       "END OF BLOCK-T92  & START OF BLOCK-T60 (Again !)
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 00:47:33.035796** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Starting with spherical alignment first with following configs -- \r\n",
-       "\r\n",
-       "\r\n",
-       "exp_time: 30.0\r\n",
-       "dofs: [1.0, 0.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]\r\n",
-       "range: 200\r\n",
-       "n_steps: 7\r\n",
-       "program: \"BLOCK-T92\"\r\n",
-       "note: \"spherical_alignment\"\r\n",
-       "\r\n",
-       "\r\n",
-       "-----------------\r\n",
-       "\r\n",
-       "\r\n",
-       "its a parameter_march_comcam.py
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 00:44:31.951518** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
STARTING BLOCK-T92 \r\n",
-       "\r\n",
-       "\r\n",
-       "But not exactly the same as the test case for today --> Guillem is giving the new configs
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 00:43:51.150776** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
END OF BLOCK-T60 (with deviations)
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 00:43:05.222060** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Guillem: We still have some spericals though not as bad as yesterday ! so the triplet looks good
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 00:41:50.935383** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
take_aos_sequence_comcam.py with program: \"BLOCK-T88\" (not in the test case)
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 00:40:29.569310** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Taking a AOS triplet ---->  with dz=1500
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 00:37:25.461173** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Taking another 30s image with comcam in r-band --- > Looks better ! so the offset that LaserTracker gave is the one ? ...... it still can be better though !
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 00:35:49.830675** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
We are changing hexapod offsets again --- \r\n",
-       "\r\n",
-       "\r\n",
-       "Cam_dx: -141.85\r\n",
-       "Cam_dy: 673.96\r\n",
-       "Cam_dz: 222.47\r\n",
-       "Cam_rx: 7.76e-3\r\n",
-       "Cam_ry: 0\r\n",
-       "M2_dz: 484.12
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 00:34:16.742230** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Now we are taking image again with ComComa in the correct filter (r-band)
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 00:32:51.930574** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
focus slightly improved but not there yet -- BUT we realised now that we are in i-band and not r-band as intended. Also the ComCam image taking failed with \"command ack not seen \"
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 00:31:20.471743** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
salIndex: 101256\r\n",
-       "\r\n",
-       "\r\n",
-       "raise base.AckTimeoutError(\r\n",
-       "lsst.ts.salobj.base.AckTimeoutError: msg='Timed out waiting for command acknowledgement', ackcmd=(ackcmd private_seqNum=2080556893, ack=<SalRetCode.CMD_NOACK: -301>, error=0, result='No command acknowledgement seen')
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 00:29:22.233514** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Guillem removed 200 microns to see if we get a focused image now because last dof did NOT give us great focus. So set dof to \r\n",
-       "\r\n",
-       "\r\n",
-       "Cam_dx: -141.85\r\n",
-       "Cam_dy: 673.96\r\n",
-       "Cam_dz: -6.0\r\n",
-       "Cam_rx: 7.76e-3\r\n",
-       "Cam_ry: 0\r\n",
-       "M2_dz: 484.12 \r\n",
-       "\r\n",
-       "\r\n",
-       "Taking ComCam image
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 00:24:41.044641** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
STARTED BLOCK-T60 --- Manual Alignment \r\n",
-       "\r\n",
-       "\r\n",
-       "Setting dof to -\r\n",
-       "Cam_dx: -141.85\r\n",
-       "Cam_dy: 673.96\r\n",
-       "Cam_dz: 222.47\r\n",
-       "Cam_rx: 7.76e-3\r\n",
-       "Cam_ry: 0\r\n",
-       "M2_dz: 484.12
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 00:21:53.898841** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Disabled hexapods compensation and enabled it again
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 00:20:33.538950** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Trackking target Az= 160 deg and El =65 deg now \r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "Alternatively ..... current 5 spikes observed at 21:03 and 21:14 CLT  almost to > 4 A.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 00:18:05.010700** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
21:14 it jumped again on camerahexapods.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- Link: [OBS-648](https://rubinobs.atlassian.net/browse/OBS-648)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 00:14:51.075990** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
20 out of 20 Twilight Flats taken ---- END OF BLOCK-T215
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 00:08:41.890903** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
20:54 Chile time : Hexapod jumped and went out of focus
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- Link: [OBS-648](https://rubinobs.atlassian.net/browse/OBS-648)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-06 00:03:21.484995** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Somehow hexapods moved -- now we are out of focus since seq number 18 !
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 23:46:35.360832** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Ok ! Mirror is open to the skies and starting the twilight flats for real
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 23:44:20.996249** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Stopped TMA tracking to open mirror covers
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 23:43:01.689159** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Oops -- we forgot to open Mirror covers !
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 23:42:14.005441** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
STARTED BLOCK-T215
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 23:41:13.483358** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
MTMount in DEBUG mode now --- Starting Twilight Flats
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 23:39:22.573574** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Preparing to go on sky --- enabled dome following
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 23:35:58.150714** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Utilising the time to do a quick ComCam checkout -- it will take 10 Biases
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 23:30:39.116055** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Guillem : \" In case we lose them, here are the laser offsets - we need to log this too\" \r\n",
-       "\r\n",
-       "\r\n",
-       "Cam_dx:-141.85\r\n",
-       "Cam_dy: 673.96\r\n",
-       "Cam_dz: 222.47\r\n",
-       "Cam_rx: 7.76e-3\r\n",
-       "Cam_ry: 0\r\n",
-       "M2_dz: 484.12
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 23:16:48.265551** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
M1M3 covers deployed --- so that folks can open the shutters
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 23:11:34.190430** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
HVAC systems. config changed to night-time -- downdraft fan ON \r\n",
-       "\r\n",
-       "\r\n",
-       "HP and Sandrine are on their way to open the dome shutters.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 23:05:22.077580** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Turned off laser tracker and put CSC into standby
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 23:03:39.379450** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Laser Tracker turned OFF ---- END OF BLOCK-T88
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 23:00:39.441462** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Pointing Laser at M1M3  --DONE\r\n",
-       "-------------------\r\n",
-       "\r\n",
-       "\r\n",
-       "component: LaserTracker:1\r\n",
-       "cmd: measurePoint\r\n",
-       "parameters:\r\n",
-       "  collection: A\r\n",
-       "  pointgroup: M1M3\r\n",
-       "  target: p4
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 22:59:34.115635** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Aligning Camera with Laser Tracker --  DONE\r\n",
-       "----------------------\r\n",
-       "\r\n",
-       "\r\n",
-       "target: Camera\r\n",
-       "tolerance_linear: 0.055\r\n",
-       "tolerance_angular: 0.002\r\n",
-       "program: \"BLOCK-T88\"\r\n",
-       "reason: \"BLOCK-T88 - Camera Alignment\"
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 22:52:35.621477** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
M2 alignment with laserTracker --- DONE\r\n",
-       "\r\n",
-       "\r\n",
-       "------------------\r\n",
-       "target: M2\r\n",
-       "tolerance_linear: 0.055\r\n",
-       "tolerance_angular: 0.002\r\n",
-       "program: \"BLOCK-T88\"\r\n",
-       "reason: \"BLOCK-T88 - M2 Alignment\"
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 22:52:23.689176** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: None\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Aligning Camera with Laser Tracker -- \r\n",
-       "----------------------\r\n",
-       "\r\n",
-       "\r\n",
-       "target: Camera\r\n",
-       "tolerance_linear: 0.055\r\n",
-       "tolerance_angular: 0.002\r\n",
-       "program: \"BLOCK-T88\"\r\n",
-       "reason: \"BLOCK-T88 - Camera Alignment\"
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 22:43:26.738410** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Starting with BLOCK-T88 now with DOME closed \r\n",
-       "----------------\r\n",
-       "Starting alignment of M2 with LaserTracker \r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "--------------------------\r\n",
-       "Note: we will point laser somewhere else after the test !
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 22:39:34.263264** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
M1M3 Cover open
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 22:38:51.422338** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Hexapods in compensation mode now
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 22:38:16.037156** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
All systems green -- proceeding with the alignment test that Guillem wants
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 22:33:00.271780** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
TMA in position -- HP also update the setting set to default from El parking set
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 22:31:13.490710** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Made the dome 1 C warmer ---> setpoint updated to 10C
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 22:30:58.905091** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
home both axis, track_target to el=65, az=160 again
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 22:27:05.025426** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Track_target failed with time-out --- > we were moving a long way in Az with parking settings. I think it was too slow !
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 22:02:59.176186** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Tracking target at Az= 160 and El= 65 -- > incidently this is the position where we will be doing the laser--tracker alignment !
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 22:00:17.966162** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
MTMount ENABLED and now homing both axes
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 21:58:38.346172** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Doing TMA checkout at El = 70
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 21:57:51.371601** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
M1M3 raised!
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 21:52:22.784645** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Enabling CSCs
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 21:50:05.350758** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Raising M1M3
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 21:50:00.135343** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Petr checked the failed FA and they seem nothing extra-ordinary. \r\n",
-       "\"FAs are failing randomly,but nothing serious. Let's raise\"\r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "We are ready for TMA checkouts
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 21:49:08.499006** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Tested bump test individually, FA 235 failed but others passed. 235 also seems to be overshooting. Proceed. raising mirror.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 21:37:07.444704** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Enabled hexapods
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 21:25:50.963854** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
FA Bump tests failed for : 233, 235, 330 \r\n",
-       "------------------\r\n",
-       "\r\n",
-       "\r\n",
-       "Actuators [233, 235, 330] FAILED the bump test.\r\n",
-       "\r\n",
-       "\r\n",
-       "SAA (Single Actuator Axes) Failures:\r\n",
-       "  None\r\n",
-       "DAA (Dual Actuator Axes) Failures:\r\n",
-       "  - Actuator ID 233: Pri Index 67\r\n",
-       "  - Actuator ID 235: Pri Index 69\r\n",
-       "  - Actuator ID 330: Pri Index 107
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 20:29:11.956808** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Guillem just wanted to check something.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 20:25:02.879879** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Hexapods zeroe-d from ATQue
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 20:19:39.095765** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Meanwhile ENABLing both Hexs for a quick liquid tests for Guillem
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 20:19:05.432036** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Trying the bump tests again.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 20:18:38.373635** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Scheduler 1 ENABLED with  maintel_fbs_sit_block_t87.yaml
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 20:18:04.176264** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Bump tests started and failed.... \r\n",
-       "\r\n",
-       "\r\n",
-       "------------------------\r\n",
-       "\r\n",
-       "\r\n",
-       " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 205, in next_ackcmd\r\n",
-       "    raise base.AckTimeoutError(\r\n",
-       "lsst.ts.salobj.base.AckTimeoutError: msg='Timed out waiting for command acknowledgement', ackcmd=(ackcmd private_seqNum=94448011, ack=<SalRetCode.CMD_NOACK: -301>, error=0, result='No command acknowledgement seen')\r\n",
-       "\r\n",
-       "\r\n",
-       "-----------------\r\n",
-       "\r\n",
-       "\r\n",
-       "Because Scheduler 1 is in STANDBY :P
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 20:14:43.610786** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Petr is running the FCU at 200 RPM...
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 20:14:21.842718** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Petr: at zenith the glycol pressure is enough to cycle the glycol flow around in circle. Not possible at horizon. Also the cold air at bottom should be circulated around the entire mirror volume. this is where the air temperatures and pressure matter. Aim to get the glycol temp 2-2.5 C lesser than ambient
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 20:06:46.107154** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Meanwhile now M1M3 temperature is about 12 C
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 20:06:23.539745** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Petr started it through the GUI and now its ENABLED
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 20:04:25.811414** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
ALL M1M3 Interlocks cleared !
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 20:01:15.269872** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
The M1M3 Air valve appears to be closed. Perhaps we should have waited for the pressure to go completely down before attempting to enable it again ? .....
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 19:54:02.726598** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
M1M3 Faulted again with this in CSC error log \r\n",
-       "\r\n",
-       "\r\n",
-       "Faulted (262149): Interlock Aux Power Bus Off
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 19:51:33.447245** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Need to remove TMA Motion lock
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 19:50:21.299980** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
M1M3 Faulted while moving it to the CSC mode from the GUI
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 19:47:42.461302** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
HP breakaway test done. All passed
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 19:33:23.399964** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Started simultaneous hardpoint breakaway test !
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 19:26:21.792285** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
M1M3 ENABLED and entering the Eng mode. waiting for the pressure to stabilize
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 19:25:01.359218** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
El at 70 ! Starting the Hardpoingt breakaway tests first through the M1M3GUI --- it will be faster !
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 19:15:02.269975** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Moving el=70
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 19:14:27.766520** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
home elevation drive
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 19:13:28.645792** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Eduardo says - \"the oil is perfect !\"
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 19:11:14.961212** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Done. Moving Elevation to 15 deg so we can home the axis.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 18:59:59.053302** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Moving in Az with parking settings --- homed Az earlier through EUI
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 18:57:31.593655** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Moving by 60 deg in azimuth with M1M3 DISABLED and TMA at horizon --- Day crew requests this check to check the oil drain under the TMA
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 18:47:07.403178** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Locking pins out !
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 18:45:13.654985** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
So the TMA at the moment is at horizon with locking pins engaged and the M1M3 is disabled. The plan now is to move TMA out of parking position to 70 with M1M3 DISABLED and on static support. We do bump tests first and then raise it and proceed to TMA checkout.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 18:39:18.595409** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Dome handed over
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 18:37:04.217053** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
TMA handed over !
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 18:28:36.531398** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Dome handover went good it seems ! From Wouter : \"The MTDome CSC reports that the current state of the MTDome azimuth rotation is PARKED. This means no error, position=328º and the brakes are engaged. All is well to start moving the dome.\"
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 18:22:42.392759** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
TMA PS ON and OSS is in progress
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 18:20:40.473584** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
HP is atL7 for Dome Handover while Robinson is starting the power supply.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 18:19:20.578657** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['TMA_EUI']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
h3. Error/Failure explanation:\r\n",
-       "Elevation Error code 410 : Positive Software limit ; Safety Matrix --> El +\r\n",
-       "h3. What were you doing when the failure occurred?:\r\n",
-       "We were trying to bring the elevation from zenith to within operational limits (80 deg) using Parking settings. The problem could be because the Elevation (non-homed) was reading 92 deg \r\n",
-       "h3. Detailed description of the steps followed that caused the error/failure:\r\n",
-       "We changed the TWO positive software limit switches in Elevation settings to FALSE,  tried to reset alarm. I think we over rode the EL+ alarm in safety matrix. We turned on the El drives , moved to 80, released the over ride and Reset alarm. The positive Software limit disappeared. We did have an STO though. We reverted the changed settings back and put the TMA motion settings to default. We then proceeded to enable the MTMount through CSC and that worked.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 17:13:01.501901** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
General M1M3 tidbits from Petr : \"You can run both tests, there shall not be any safety issue with the glass. But better if you have time to run those at zenith. HP are clear, we already have those in breakaway. I question a bit FA bump tests. We of course ran those at horizon with the surrogate, but now we have glass\r\n",
-       "2:05\r\n",
-       "Best still if you will have time to move to zenith before running the tests. We can do horizon tests later. \"
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 16:25:36.813603** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
M2Hex back to zero position
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 16:12:22.166525** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: None\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
M2Hex Warm ups going on -- today the max position reached in each axis is 4500 um
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 15:46:03.862203** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
hexapods ENABLED, rotator DISABLED
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 15:36:20.480965** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
turning on rotator T161
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-05 15:21:33.631597** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Turning on the hexapods cabinet and PXI.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 08:48:49.461019** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Shut down rotator
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 08:44:35.085235** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Turning off hexapods
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 08:41:25.795068** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Taking ComCam daytime calibrations, BLOCK-T220, while telescope is slewing to park position.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 08:34:46.875485** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Going to telescope parking position. Elevation 90, azimuth -32.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 08:29:50.458408** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MTMount']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
h3. Error/Failure explanation:\r\n",
-       "Internal Alarm in Elevation OMT 08:26:05.572 07/11/24 code: 123 system:123\r\n",
-       "A timeout was ocurred when:Power off softmotion axis. In state: \"Disabling Axis\"\r\n",
-       "h3. What were you doing when the failure occurred?:\r\n",
-       "While control over from EUI, elevation axis was fault. \r\n",
-       "h3. Detailed description of the steps followed that caused the error/failure:\r\n",
-       "Reset alarm.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- Link: [obs-321](https://rubinobs.atlassian.net/browse/obs-321)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 08:24:23.293399** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Starting closing telescope around 05:23 Chile
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 08:18:40.265899** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
HVAC to daytime setting, 11 C
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 08:14:20.837183** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Dome shutters closed
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 08:14:12.648169** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Starting process to close dome. Mirror covers.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 07:58:28.186780** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
2024-11-06 seqnum 252 PSF FWHM 0.93\"
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 07:49:44.911984** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Setting degrees of freedom and taking image. Likely last of the night.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 07:41:06.564453** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Closed loop test in i band. BLOCK-T249. First 5 modes corresponding to M2 hexapod degrees of freedom.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 07:35:54.665379** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Applying offset to camera hexapod z corresponding to expected focus for i filter from batoid. Cam_dz -657.301. This brings image into focus, PSF FWHM 1.13\"
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 07:35:54.665379** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Applying offset to camera hexapod z corresponding to expected focus for i filter from batoid. Cam_dz -657.301. This brings image into focus, PSF FWHM 1.13\"
\n", - "\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "Script failed salindex 100274\r\n", - "\r\n", - "\r\n", - "
Traceback (most recent call last):\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/asyncio/tasks.py", line 500, in wait_for\r\n",
-       "    return fut.result()\r\n",
-       "           ^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 239, in _get_next_ackcmd\r\n",
-       "    await self._next_ack_event.wait()\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/asyncio/locks.py", line 213, in wait\r\n",
-       "    await fut\r\n",
-       "asyncio.exceptions.CancelledError\r\n",
-       "\r\n",
-       "\r\n",
-       "The above exception was the direct cause of the following exception:\r\n",
-       "\r\n",
-       "\r\n",
-       "Traceback (most recent call last):\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 189, in next_ackcmd\r\n",
-       "    ackcmd = await self._wait_task\r\n",
-       "             ^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 214, in _basic_next_ackcmd\r\n",
-       "    ackcmd = await asyncio.wait_for(\r\n",
-       "             ^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/asyncio/tasks.py", line 502, in wait_for\r\n",
-       "    raise exceptions.TimeoutError() from exc\r\n",
-       "TimeoutError\r\n",
-       "\r\n",
-       "\r\n",
-       "During handling of the above exception, another exception occurred:\r\n",
-       "\r\n",
-       "\r\n",
-       "Traceback (most recent call last):\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py", line 631, in do_run\r\n",
-       "    await self._run_task\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/base_take_image.py", line 249, in run\r\n",
-       "    await self.camera.take_imgtype(\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/base_camera.py", line 1005, in take_imgtype\r\n",
-       "    return await self.expose(camera_exposure=camera_exposure)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/base_camera.py", line 1115, in expose\r\n",
-       "    exp_ids = await self.handle_take_images(camera_exposure=camera_exposure)\r\n",
-       "              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/base_camera.py", line 1136, in handle_take_images\r\n",
-       "    return await self._handle_take_images(camera_exposure=camera_exposure)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/base_camera.py", line 1207, in _handle_take_images\r\n",
-       "    exp_ids += await self._handle_snaps(camera_exposure)\r\n",
-       "               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/base_camera.py", line 1245, in _handle_snaps\r\n",
-       "    await self.camera.cmd_takeImages.start(timeout=take_images_timeout)\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 487, in start\r\n",
-       "    return await cmd_info.next_ackcmd(timeout=timeout)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 205, in next_ackcmd\r\n",
-       "    raise base.AckTimeoutError(\r\n",
-       "lsst.ts.salobj.base.AckTimeoutError: msg='Timed out waiting for command acknowledgement', ackcmd=(ackcmd private_seqNum=2080556893, ack=&lt;SalRetCode.CMD_NOACK: -301&gt;, error=0, result='No command acknowledgement seen')\r\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 07:27:32.893837** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Taking images in g and i of same field containing NGC 1097. The g image is close to in focus, PSF FWHM 1.12\". The i image is out of focus donuts. The change in focus was not applied for these images.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 07:24:18.474546** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
The NGC 1097 field was going out of dome azimuth range for end of night. Moving to new field.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 07:13:55.089605** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Cycle MTAOS. Camera hexapod in compensation mode.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 06:54:46.875898** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
We are currently at elevation 42 deg. Moving to a new target at higher elevation. NGC 1097
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 06:47:30.498016** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Updating configuration for closed loop ComCam. M2 Coma.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 06:41:41.943501** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Applied corrections with all signs flipped for camera z, r_x, r_y. Image quality got worse. Realized that we probably should not have applied sign flip in z.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 06:28:38.416437** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Re-set optical state. Running a closed loop on camera z, r_x, r_y.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 06:27:41.080039** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Will apply a large trefoil to M2 to help determine sign of correction. Then run closed loop again. Observe that trefoil goes up significantly.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 06:10:33.043244** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Applying correction for M2 trefoil.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 06:01:11.997062** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Updating closed-loop configuration. Starting closed loop tests with M2 trefoil.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 05:49:43.570704** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Corrections are in the same direction and getting successively smaller. Hopefully converging.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 05:48:50.547384** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Check the FCU data until now tonight. Sandrin T. hit Edit FCU Fans Speed and Set constant again. 02:47CLT/05:48UTC
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 05:33:16.776222** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Image quality getting worse. Will flip signs and try again. Cycling MTAOS to pick up changes. Re-set degrees of freedom.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 05:26:33.698446** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Estimated corrections from first iteration look reasonable. Gain at 0.5. Applying correction and running another iteration. Will see if the sign is correct.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 05:19:51.974083** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Starting closed loop tests with M2 astigmatism degrees of freedom.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 05:17:50.352813** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Re-setting DoF after observing tilt degrees of freedom running away.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 05:13:16.694891** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Multiple iterations of closed loop. Appears to be running away in tilt. M2 dr_x in camera dr_y.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 05:04:41.373665** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Resuming BLOCK-249. Closed loop testing.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 04:54:44.043443** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
h3. Error/Failure explanation:\r\n",
-       "Internal Alarm in Elevation OMT error. Happened 04:43:42.938 07/11/24 code 499\r\n",
-       "Elevation \r\n",
-       "description Tracing command not accepted by axies More info:  Command not allowed in \"errorStop (stopping \" state  Allowed commands:  \r\n",
-       "h3. What were you doing when the failure occurred?:\r\n",
-       "While we are doing scheduler. We recover MTMount, MTPtg. MTRotator. \r\n",
-       "\r\n",
-       "h3. Detailed description of the steps followed that caused the error/failure:\r\n",
-       "MtPtg has Error MTMount replied error for trackTarget command (1734071833) Error no:1 message:Failed.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- Link: [OBS-321](https://rubinobs.atlassian.net/browse/OBS-321)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 04:53:00.371418** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Home both axes.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 04:38:18.951610** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Seqnum 163 shows trails. Suspect an issue with slew and settle timing.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 04:28:38.776238** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Need to rebuild MTAOS. Starting the scheduler configuration for in situ science pipelines commissioning observations, BLOCK-T248\r\n",
-       "\r\n",
-       "\r\n",
-       "30 visits in r band.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 04:25:58.816332** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Good image quality at seqnum 150, PSF FWHM 1.3-1.4 in this sequence. Donuts look good with reduced coma. Recording this state.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 04:25:58.816332** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Good image quality at seqnum 150, PSF FWHM 1.3-1.4 in this sequence. Donuts look good with reduced coma. Recording this state.
\n", - "\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "Script failed. salindex 100144\r\n", - "\r\n", - "\r\n", - "
Traceback (most recent call last):\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py", line 631, in do_run\r\n",
-       "    await self._run_task\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/run_command.py", line 170, in run\r\n",
-       "    await getattr(self.remote, f"cmd_{self.cmd}").start(\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 487, in start\r\n",
-       "    return await cmd_info.next_ackcmd(timeout=timeout)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 191, in next_ackcmd\r\n",
-       "    raise base.AckError(msg="Command failed", ackcmd=ackcmd)\r\n",
-       "lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=1472167480, ack=&lt;SalRetCode.CMD_FAILED: -302&gt;, error=1, result='Failed: axis 1 is out of bounds for array of dimension 1')\r\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 04:25:58.816332** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Good image quality at seqnum 150, PSF FWHM 1.3-1.4 in this sequence. Donuts look good with reduced coma. Recording this state.
\n", - "\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "Script failed. salindex 100144\r\n", - "\r\n", - "\r\n", - "
Traceback (most recent call last):\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py", line 631, in do_run\r\n",
-       "    await self._run_task\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/run_command.py", line 170, in run\r\n",
-       "    await getattr(self.remote, f"cmd_{self.cmd}").start(\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 487, in start\r\n",
-       "    return await cmd_info.next_ackcmd(timeout=timeout)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 191, in next_ackcmd\r\n",
-       "    raise base.AckError(msg="Command failed", ackcmd=ackcmd)\r\n",
-       "lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=1472167480, ack=&lt;SalRetCode.CMD_FAILED: -302&gt;, error=1, result='Failed: axis 1 is out of bounds for array of dimension 1')\r\n",
-       "
\n", - "\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "Running closed loop again. Another timeout error. salindex 100143\r\n", - "\r\n", - "\r\n", - "
Traceback (most recent call last):\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/asyncio/tasks.py", line 500, in wait_for\r\n",
-       "    return fut.result()\r\n",
-       "           ^^^^^^^^^^^^\r\n",
-       "asyncio.exceptions.CancelledError\r\n",
-       "\r\n",
-       "\r\n",
-       "The above exception was the direct cause of the following exception:\r\n",
-       "\r\n",
-       "\r\n",
-       "Traceback (most recent call last):\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py", line 631, in do_run\r\n",
-       "    await self._run_task\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/maintel/base_close_loop.py", line 572, in run\r\n",
-       "    await self.arun(True)\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/maintel/base_close_loop.py", line 508, in arun\r\n",
-       "    wavefront_error = await self.mtcs.rem.mtaos.evt_wavefrontError.next(\r\n",
-       "                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/read_topic.py", line 681, in next\r\n",
-       "    return await self._next(timeout=timeout)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/read_topic.py", line 693, in _next\r\n",
-       "    return await asyncio.wait_for(self._next_task, timeout=timeout)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/asyncio/tasks.py", line 502, in wait_for\r\n",
-       "    raise exceptions.TimeoutError() from exc\r\n",
-       "TimeoutError\r\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 04:14:03.993296** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Take another close loop again.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 04:13:46.169869** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Tiago tried run following config. to runOFC on MTAOS. But failed 100142. \r\n",
-       "component: MTAOS\r\n",
-       "cmd: runOFC\r\n",
-       "parameters:\r\n",
-       "  config: \"comp_dof_idx:\\n  M1M3Bend:\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  M2Bend:\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  camHexPos:\\n  - 1.0\\n  - 1.0\\n  - 1.0\\n  - 1.0\\n  - 1.0\\n  m2HexPos:\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\nfilter_name: r_03\\nrotation_angle: -2.0\\n\"
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 04:13:46.169869** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Tiago tried run following config. to runOFC on MTAOS. But failed 100142. \r\n",
-       "component: MTAOS\r\n",
-       "cmd: runOFC\r\n",
-       "parameters:\r\n",
-       "  config: \"comp_dof_idx:\\n  M1M3Bend:\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  M2Bend:\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  camHexPos:\\n  - 1.0\\n  - 1.0\\n  - 1.0\\n  - 1.0\\n  - 1.0\\n  m2HexPos:\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\n  - 0.0\\nfilter_name: r_03\\nrotation_angle: -2.0\\n\"
\n", - "\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "Script failed, timeout error salindex 100141\r\n", - "\r\n", - "\r\n", - "
Traceback (most recent call last):\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/asyncio/tasks.py", line 500, in wait_for\r\n",
-       "    return fut.result()\r\n",
-       "           ^^^^^^^^^^^^\r\n",
-       "asyncio.exceptions.CancelledError\r\n",
-       "\r\n",
-       "\r\n",
-       "The above exception was the direct cause of the following exception:\r\n",
-       "\r\n",
-       "\r\n",
-       "Traceback (most recent call last):\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py", line 631, in do_run\r\n",
-       "    await self._run_task\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/maintel/base_close_loop.py", line 572, in run\r\n",
-       "    await self.arun(True)\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/maintel/base_close_loop.py", line 508, in arun\r\n",
-       "    wavefront_error = await self.mtcs.rem.mtaos.evt_wavefrontError.next(\r\n",
-       "                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/read_topic.py", line 681, in next\r\n",
-       "    return await self._next(timeout=timeout)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/read_topic.py", line 693, in _next\r\n",
-       "    return await asyncio.wait_for(self._next_task, timeout=timeout)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/asyncio/tasks.py", line 502, in wait_for\r\n",
-       "    raise exceptions.TimeoutError() from exc\r\n",
-       "TimeoutError\r\n",
-       "d\r\n",
-       "2024/11/07 04:06:14 TAI\r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 04:04:24.224386** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Signs are on the tilts are as they were.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 03:57:26.918979** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Running track target. Moving to a new field. Cycling MTAOS. Set DoF. Take image.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 03:57:26.918979** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Running track target. Moving to a new field. Cycling MTAOS. Set DoF. Take image.
\n", - "\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "Script failed, salindex 100133\r\n", - "\r\n", - "\r\n", - "
Traceback (most recent call last):\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py", line 631, in do_run\r\n",
-       "    await self._run_task\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/maintel/base_close_loop.py", line 572, in run\r\n",
-       "    await self.arun(True)\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/maintel/base_close_loop.py", line 517, in arun\r\n",
-       "    dof_offset = await self.compute_ofc_offsets(rotation_angle)\r\n",
-       "                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/maintel/base_close_loop.py", line 452, in compute_ofc_offsets\r\n",
-       "    await self.mtcs.rem.mtaos.cmd_runOFC.set_start(\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 416, in set_start\r\n",
-       "    return await self.start(timeout=timeout, wait_done=wait_done)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 487, in start\r\n",
-       "    return await cmd_info.next_ackcmd(timeout=timeout)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 191, in next_ackcmd\r\n",
-       "    raise base.AckError(msg="Command failed", ackcmd=ackcmd)\r\n",
-       "lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=1418882047, ack=&lt;SalRetCode.CMD_FAILED: -302&gt;, error=1, result="Failed: ufunc 'multiply' did not contain a loop with signature matching types (dtype('&lt;U4'), dtype('float64')) -&gt; None")\r\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 03:51:16.572998** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Revert the sign in tilt. Pushing a new configuration. Cycling MTAOS again. Resetting DoF.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 03:35:38.452522** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Guillem will flip sign in dr_x for next set of tests, starting with M2. Cycling the MTAOS again
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 03:35:38.452522** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Guillem will flip sign in dr_x for next set of tests, starting with M2. Cycling the MTAOS again
\n", - "\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "Script has failed. SALINDEX 100121\r\n", - "\r\n", - "\r\n", - "
Traceback (most recent call last):\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py", line 631, in do_run\r\n",
-       "    await self._run_task\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/maintel/base_close_loop.py", line 572, in run\r\n",
-       "    await self.arun(True)\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/maintel/base_close_loop.py", line 517, in arun\r\n",
-       "    dof_offset = await self.compute_ofc_offsets(rotation_angle)\r\n",
-       "                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/maintel/base_close_loop.py", line 452, in compute_ofc_offsets\r\n",
-       "    await self.mtcs.rem.mtaos.cmd_runOFC.set_start(\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 416, in set_start\r\n",
-       "    return await self.start(timeout=timeout, wait_done=wait_done)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 487, in start\r\n",
-       "    return await cmd_info.next_ackcmd(timeout=timeout)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 191, in next_ackcmd\r\n",
-       "    raise base.AckError(msg="Command failed", ackcmd=ackcmd)\r\n",
-       "lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=1418882047, ack=&lt;SalRetCode.CMD_FAILED: -302&gt;, error=1, result='Failed: not enough values to unpack (expected 2, got 0)')\r\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 03:30:24.574922** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Changed sign x for camera and M2. \r\n",
-       "\r\n",
-       "\r\n",
-       "Changed z in M2
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 03:28:06.167214** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Starting another triplet iteration with closed loop to evaluate convergence.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 03:27:03.275754** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Sandrine notes that when we apply the corrections from yesterday in another position, we get different delivered image quality. Suspect there might be some azimuth dependence.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 03:24:49.934340** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Sign change appears to have worked based on applied offsets. The code change was correct.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 03:22:32.672432** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Guillem is updating MTAOS code in real time to apply sign flips. We see that corrections appear reasonable. Applying corrections.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 03:20:03.387665** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
After running MTAOS test, the corrections are small, but when applied, the image quality is getting worse. Expect that there is a sign error.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 03:17:41.656546** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
MTAOS going to standby, and then enable.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 03:14:06.744761** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Keith Bechtol starting late shift and making log entries.\r\n",
-       "\r\n",
-       "\r\n",
-       "Achieved 1.39\" FWHM on seqnum 116
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 03:13:43.904838** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Starting BLOCK-T249, running closed loop.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 02:53:06.893392** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Tiago is sending MTAOAS to standby in order to update it.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 02:50:13.343549** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Run track_target.py az: 70 el: -70 with ignore MTAOS . Tag with BLOCK-T250.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 02:46:03.070301** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
salIndex: 100100 failed with the same issue. \r\n",
-       "Tiago R. said DDS issue but more deliverey side.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 02:45:56.445030** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
set_dof.py with the same dofs was ran. Take another take_aos_sequence_comcam.py for baseline.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 02:43:59.165552** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Run M1M3_B15: -0.25  then run take_aos_sequence_comcam.py for baseline.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 02:38:53.874163** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Run apply_dof.py M1M3_B14:-0.4. M1M3_B15: 0.25  then run take_aos_sequence_comcam.py.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 02:38:44.680897** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Run apply_dof.py M1M3_B13: reset and M1M3_B14: 0.4.  then run take_aos_sequence_comcam.py. but failed. \r\n",
-       "\r\n",
-       "\r\n",
-       "salIndex: 100089 2024/11/07 02:28:58 TAI failed same timeout error. \r\n",
-       " take_aos_sequence_comcam.py. \r\n",
-       "    raise base.AckTimeoutError(\r\n",
-       "lsst.ts.salobj.base.AckTimeoutError: msg='Timed out waiting for command acknowledgement', ackcmd=(ackcmd private_seqNum=2080556893, ack=<SalRetCode.CMD_NOACK: -301>, error=0, result='No command acknowledgement seen')
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 02:37:15.823658** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Run apply_dof.py M1M3_B14: 0.4.  then run take_aos_sequence_comcam.py.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 02:32:42.273037** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
take_aos_sequence_comcam.py : for reference Imag \"ref_after_error\"
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 02:31:49.972191** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Need to run set_dof.py. \r\n",
-       "[-1.26588e+03,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  1.93497e+03,  4.85815e+03,  6.73960e+02,\r\n",
-       "        7.76000e-03,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  0.00000e+00]
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 02:24:24.098817** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Run apply_dof.py M1M3_B12:-0.5. M1M3_B13: 0.4 then run take_aos_sequence_comcam.py.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 02:18:21.375598** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
run apply_dof.py M1M3_B11:-0.5. M1M3_B12:0.5 then run take_aos_sequence_comcam.py.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 02:17:18.971698** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Apply_dof.py M1M3_B11:0.5. then back to  take_aos_sequence_comcam.py.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 02:14:50.663760** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Apply_dof.py M1M3_B10:-0.5. then back to  take_aos_sequence_comcam.py. for baseline.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 02:08:51.873248** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Apply B9:0.5 and take_aos_sequence_comcam.py.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 02:07:28.371003** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Run Set_dof.py with same one.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 02:05:10.087429** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Re run take_aos_sequence_comcam.py. again
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 02:04:56.686035** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
salIndex: 100069 2024/11/07 02:03:29 TAI\r\n",
-       "maintel/take_aos_sequence_comcam.py fault. Timeout error. \r\n",
-       "\r\n",
-       "\r\n",
-       "    raise base.AckTimeoutError(\r\n",
-       "lsst.ts.salobj.base.AckTimeoutError: msg='Timed out waiting for command acknowledgement', ackcmd=(ackcmd private_seqNum=2080556893, ack=<SalRetCode.CMD_NOACK: -301>, error=0, result='No command acknowledgement seen')
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 02:02:04.603428** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Apply_dof.py M1M3_B8:-0.5 M1M3_B9: 0.5. then back to  take_aos_sequence_comcam.py.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 01:58:31.260127** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Apply_dof.py M1M3_B7:-1 M1M3_B8: 0.5. then back to  take_aos_sequence_comcam.py.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 01:54:53.974754** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
apply_dof.py M1M3_B5: -0.75 then back to  take_aos_sequence_comcam.py
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 01:54:49.172070** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Apply_dof.py M1M3_B6: -1 M1M3_B7:1. then back to  take_aos_sequence_comcam.py.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 01:52:26.010270** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
apply_dof.py M1M3_B6: 1 then back to  take_aos_sequence_comcam.py. at baseline.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 01:44:21.272738** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Apply dof:  M1M3_B4: -0.75 M1M3_B5: ​0.75 then run another take_aos_sequence_comcam.py.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 01:42:55.301879** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Apply dof. M1M3_B3: -1 M1M3_B4: 0.75\r\n",
-       "Then run another take_aos_sequence_comcam.py.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 01:39:50.223463** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
apply_dof.py for M1M3_B2: -1 M1M3_B2: 3\r\n",
-       "Then run another take_aos_sequence_comcam.py.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 01:36:38.406724** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
apply_dof.py for M1M3_B1: -1 M1M3_B2: 1. \r\n",
-       "Then run another take_aos_sequence_comcam.py.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 01:35:23.724746** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
take_aos_sequence_comcam.py with \r\n",
-       "filter: \"r_03\"\r\n",
-       "exposure_time: 30\r\n",
-       "dz: 1500\r\n",
-       "n_sequences: 1\r\n",
-       "program: \"BLOCK-T252\"\r\n",
-       "note: \"M1M3_B1_1\"
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 01:34:39.605571** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['Camera Control Software']\n", - " - primary_hardware_components: ['Science Cameras']\n", - "\n", - "
CCCamera stopped responding to commands after twilite flats.  Also would not transition to different summary states.  In the end, had to restart the mcm and ocs-bridge services on comcam-mcm node.  many alerts could then be cleared (including fp stuck in integrating state) and return to normal state.\r\n",
-       "\r\n",
-       "\r\n",
-       "Tony J. will record an OBS ticket to further investigate the cause.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 01:34:03.961002** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Start BLOCK-T252.\r\n",
-       "apply_dof.py on M1M3_B1: 1
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 01:30:02.172747** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
take_aos_sequence_comcam.py.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 01:29:30.570420** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
set_dof.py\r\n",
-       "dofs: [-1.26588e+03,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  1.93497e+03,  4.85815e+03,  6.73960e+02,\r\n",
-       "        7.76000e-03,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  0.00000e+00]
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 01:25:57.520281** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
apply_dof.py with M1M3_B1: 1. \r\n",
-       "take_aos_sequence_comcam.py\r\n",
-       "filter: \"r\"\r\n",
-       "exposure_time: 30.0\r\n",
-       "max_iter: 1\r\n",
-       "program: \"BLOCK-T183\"\r\n",
-       "reason: \"SITCOM-826\"\r\n",
-       "note: \"optical_state_m1m3_bending\"\r\n",
-       "used_dofs: [10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29]\r\n",
-       "apply_corrections: false
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 01:23:31.704514** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
set_dof for the following dof.py\r\n",
-       "dof: [-1.26588e+03,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  1.89747e+03,  4.85815e+03,  6.73960e+02,\r\n",
-       "        7.76000e-03,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  0.00000e+00,  0.00000e+00,  0.00000e+00,\r\n",
-       "        0.00000e+00,  0.00000e+00]
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 01:21:59.238714** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Back to BLOKC-T252. M1M3 Bending mode again. Take another triplet to check.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 01:19:58.001108** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Step 9. Sequence number 32 is the best focus.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 01:16:47.635370** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Run focus_sweep_comcam.py with config.\r\n",
-       "axis: z\r\n",
-       "focus_window: 300\r\n",
-       "n_steps: 9\r\n",
-       "program: \"BLOCK-T92\"\r\n",
-       "hexapod: \"Camera\"\r\n",
-       "reason: \"focus_sweep_cam_dz\"
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 01:15:52.333293** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
take_image_comcam.py to check we are in focus.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 01:15:41.266040** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Run set_dof.py with the same one.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 01:12:52.162267** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
We need to switch to BLOCK-T92. Take a single image with 10sec exp. time to check first.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 01:11:54.240935** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
apply_dof.py . Reset the dof for B1.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 01:07:19.132628** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Run take_ao_sequence_comcam for T252. M1M3_01
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 01:06:55.020804** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
apply_dof.py M1M3_B1: 1
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 01:06:13.386874** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Moving to BLOCK-T252. M1M3 Bending mode. But will use take_ao_sequence_comcam. py. to take image tonight.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 01:00:55.104084** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Take triplet images with take_ao_sequence_comcam.py.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 01:00:37.920199** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Take_image_comcam with 10 sec. Looks good.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 00:58:23.105041** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
run take_aos_sesquence_comcam.py. but abort it. \r\n",
-       "Will apply another set_dof.py with the same offset.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 00:54:22.420277** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Run enable_dome_following. Now the dome is moving.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 00:53:11.634019** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Filter r_03 in position. But found that MTDome is not following
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 00:51:33.932785** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
take_aos_sequence_comcam.py with the same setup.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 00:50:45.971328** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Reenabled CCCamera. Seems work
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 00:50:25.571624** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Removed all the lock and switch to enterControl. CCCamera on Standby.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 00:49:18.497228** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Tony J. restarted it. Brian checked it is a nominal state with no alarm.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 00:47:11.915573** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Brian set it on the \"READY\" state. Alarms seem cleared.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 00:44:49.772214** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
run_command.py again for enter control for CCCamera. It failed. \r\n",
-       "component: CCCamera\r\n",
-       "cmd: enterControl \r\n",
-       "\r\n",
-       "\r\n",
-       "lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=886868127, ack=<SalRetCode.CMD_FAILED: -302>, error=0, result='Error : ACTION and CONFIGURATION commands are only accepted in READY state')
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 00:43:36.786594** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Run_command.py was failed and it turned out it was locked by Brian's lock.
\n" + " \n", + " 5\n", + " 0\n", + " 0\n", + " 0\n", + " 5\n", + " \n", + " \n", + "\n", + "" ], "text/plain": [ - "" + " good questionable junk total\n", + "unknown \n", + "5 0 0 0 5" ] }, "metadata": {}, @@ -8400,15 +851,7 @@ { "data": { "text/markdown": [ - "- **2024-11-07 00:40:38.953203** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Same issue. Tony J. suspects it is due to  the network issue. Ask IT support with Baden. \r\n",
-       "File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 205, in next_ackcmd\r\n",
-       "    raise base.AckTimeoutError(\r\n",
-       "lsst.ts.salobj.base.AckTimeoutError: msg='Timed out waiting for command acknowledgement', ackcmd=(ackcmd private_seqNum=262260873, ack=<SalRetCode.CMD_NOACK: -301>, error=0, result='No command acknowledgement seen')
\n" + "**LSSTCam: 0 total exposures**" ], "text/plain": [ "" @@ -8416,17 +859,29 @@ }, "metadata": {}, "output_type": "display_data" - }, + } + ], + "source": [ + "for instrum in allsrc.exp_src.exposures.keys():\n", + " md(f'**{instrum}: {len(allsrc.exp_src.exposures[instrum])} total exposures**')\n", + " df_dict = allsrc.fields_count_exposure(instrum)\n", + " for field_name,df in df_dict.items():\n", + " if df.empty:\n", + " continue\n", + " print(field_name.title().replace('_',' '))\n", + " display(df.T)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "22", + "metadata": {}, + "outputs": [ { "data": { "text/markdown": [ - "- **2024-11-07 00:39:21.926242** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Tony J. joined.CCOODS cycle. CCHeasder Service is enabled. CCCamera enabled. \r\n",
-       "Sent  CCCamera to disabled. Failed
\n" + "### Experimental Exposure field Tally DISASBLED" ], "text/plain": [ "" @@ -8438,12 +893,7 @@ { "data": { "text/markdown": [ - "- **2024-11-07 00:38:06.620819** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Cycle CCCamera, but change state even failed.
\n" + "**LSSTComCamSim: 0 total exposures**" ], "text/plain": [ "" @@ -8455,13 +905,7 @@ { "data": { "text/markdown": [ - "- **2024-11-07 00:34:07.950295** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Ping Kevin F. \r\n",
-       "He mentioned that The command is received on CCS but never ack'd  and OCS give up
\n" + "**LATISS: 0 total exposures**" ], "text/plain": [ "" @@ -8473,26 +917,7 @@ { "data": { "text/markdown": [ - "- **2024-11-07 00:28:38.068331** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
salIndex: 100025  take_aos_sequence_comcam.py\r\n",
-       "Same timeout error fault. The first image was not even taken. \r\n",
-       "\r\n",
-       "\r\n",
-       "env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/maintel/comcam.py\", line 232, in setup_filter\r\n",
-       "    await self.rem.cccamera.cmd_setFilter.set_start(\r\n",
-       "  File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 416, in set_start\r\n",
-       "    return await self.start(timeout=timeout, wait_done=wait_done)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 487, in start\r\n",
-       "    return await cmd_info.next_ackcmd(timeout=timeout)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 205, in next_ackcmd\r\n",
-       "    raise base.AckTimeoutError(\r\n",
-       "lsst.ts.salobj.base.AckTimeoutError: msg='Timed out waiting for command acknowledgement', ackcmd=(ackcmd private_seqNum=1434847998, ack=<SalRetCode.CMD_NOACK: -301>, error=0, result='No command acknowledgement seen')
\n" + "**LSSTComCam: 5 total exposures**" ], "text/plain": [ "" @@ -8504,12 +929,7 @@ { "data": { "text/markdown": [ - "- **2024-11-07 00:26:36.715368** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Re-running take_aos_sequence_comcam.py failed.
\n" + "**LSSTCam: 0 total exposures**" ], "text/plain": [ "" @@ -8517,36 +937,71 @@ }, "metadata": {}, "output_type": "display_data" + } + ], + "source": [ + "# Exposure Report\n", + "if experimental.get(\"exposure_type_tally\"): \n", + " # md(\"### Experimental Exposure field Tally ENABLED\")\n", + " md(\"Tally exposures with specific values of selected fields\")\n", + "else:\n", + " md(\"### Experimental Exposure field Tally DISASBLED\")\n", + " \n", + "for instrum in allsrc.exp_src.exposures.keys():\n", + " md(f'**{instrum}: {len(allsrc.exp_src.exposures[instrum])} total exposures**')\n", + " if 0 == len(allsrc.exp_src.exposures[instrum]):\n", + " continue\n", + "\n", + " # Facet counts\n", + " if experimental.get(\"exposure_type_tally\"): \n", + " counts,cc,f = allsrc.uniform_exposure_field_counts(instrum)\n", + " for fname, tally in []: # counts.items():\n", + " md(f\"{fname}\")\n", + " df = pd.DataFrame.from_dict(tally, orient=\"index\").T\n", + " wdf = ut.wrap_dataframe_columns(df)\n", + " display(HTML(wdf.to_html(render_links=True, escape=False)))" + ] + }, + { + "cell_type": "markdown", + "id": "261925dc-1171-4429-b6a8-3bf62d71904f", + "metadata": {}, + "source": [ + "Note: *Added counts of Exposures with messages*" + ] + }, + { + "cell_type": "markdown", + "id": "20820900-05ab-4ca5-8df5-9f725c0ee570", + "metadata": {}, + "source": [ + "## Narrative Log BETA " + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "26", + "metadata": { + "editable": true, + "jupyter": { + "source_hidden": true }, - { - "data": { - "text/markdown": [ - "- **2024-11-07 00:25:45.608939** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
salIndex: 100024  - Started at: 2024-11-07T00:23:30.896Z\r\n",
-       "maintel/take_aos_sequence_comcam.py failed. Timeout error. \r\n",
-       "lsst.ts.salobj.base.AckTimeoutError: msg='Timed out waiting for command acknowledgement', ackcmd=(ackcmd private_seqNum=1434847998, ack=<SalRetCode.CMD_NOACK: -301>, error=0, result='No command acknowledgement seen')
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" + "slideshow": { + "slide_type": "" }, + "tags": [] + }, + "outputs": [ { "data": { "text/markdown": [ - "- **2024-11-07 00:23:42.878064** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", + "- **2024-11-11 04:18:44.671682** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
BLOCK-T60. dz:1500. exposure_time:30. take_aos_sequence_comcam.py. \r\n",
-       "note: \"check\"
\n" + "
MTDome telemetry returning to normal
\n" ], "text/plain": [ "" @@ -8558,13 +1013,25 @@ { "data": { "text/markdown": [ - "- **2024-11-07 00:23:05.536179** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", + "- **2024-11-11 04:17:45.689723** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Start focus at the position El;77, Az:-71.. Running set_dof.py \r\n",
-       "dofs: [-1.26588e+03, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 1.97247e+03, 4.85815e+03, 6.73960e+02, 7.76000e-03, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00]
\n" + "
Tried move_rotator script, but failed. Can't move the rotator to enabled mode. \r\n",
+       "The difference between the rotator and CCW positions is out of range to enable CSC.   \r\n",
+       "\r\n",
+       "\r\n",
+       "Alternative procedure: move CCW to rotator position through the EUI \r\n",
+       "\tPut MTRotator and MTMount in Disabled \r\n",
+       "\tTake control in the EUI \r\n",
+       "\tCCW, reset alarm, turn on, select 1, input the rotator position of 5.8 deg, move, reset alarm, turn off \r\n",
+       "\tMain axes, reset alarm\r\n",
+       "\r\n",
+       "\r\n",
+       "Send MTMount to Enabled \r\n",
+       "Then  MTForceError.MTM2 fault\r\n",
+       "\t (the range is not tuned for the AOS tests)
\n" ], "text/plain": [ "" @@ -8576,22 +1043,12 @@ { "data": { "text/markdown": [ - "- **2024-11-07 00:20:16.860484** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", + "- **2024-11-11 04:09:33.289017** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
START OF BLOCK-T60 \r\n",
-       "\r\n",
-       "\r\n",
-       "starting from last night's best position (instead of doing lasertracker measurements today): \r\n",
-       "\r\n",
-       "\r\n",
-       "set_dof.py\r\n",
-       "\r\n",
-       "\r\n",
-       "----------------\r\n",
-       "[-1.26588e+03, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 1.97247e+03, 4.85815e+03, 6.73960e+02, 7.76000e-03, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00, 0.00000e+00]
\n" + "
Rotator position is 5.874 and the CCW position is 3.674.
\n" ], "text/plain": [ "" @@ -8603,24 +1060,16 @@ { "data": { "text/markdown": [ - "- **2024-11-07 00:10:04.009756** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", + "- **2024-11-11 04:06:39.542003** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
END OF BLOCK- T215 --- not quite successful :( \r\n",
-       "\r\n",
+       "
MTPtg wen to fault\r\n",
        "\r\n",
-       "Running track_target to \r\n",
        "\r\n",
-       "\r\n",
-       "track_azel:\r\n",
-       " az: -70\r\n",
-       " el: 78\r\n",
-       "rot_value: 0\r\n",
-       "program: \"BLOCK-T250\"\r\n",
-       "reason: \"TMACheckout\"\r\n",
-       "rot_type: PhysicalSky
\n"
+       " Error Rotator replied error for trackTarget command (1735273186) Error no:1 message:Failed: Must be in Enabled state\r\n",
+       "0
\n" ], "text/plain": [ "" @@ -8632,15 +1081,15 @@ { "data": { "text/markdown": [ - "- **2024-11-07 00:08:49.060858** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", + "- **2024-11-11 04:06:06.989743** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Basically we missed Az=0 and and the Az was moving more -ve degrees .... \r\n",
+       "
MTRotator wen to fault\r\n",
        "\r\n",
        "\r\n",
-       "Caused the script to fail
\n"
+       "Camera cable wrap not following closely enough: error # 3 = 2.2011443473732264 > 2.2 deg
\n" ], "text/plain": [ "" @@ -8652,12 +1101,12 @@ { "data": { "text/markdown": [ - "- **2024-11-07 00:06:42.577876** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", + "- **2024-11-11 03:58:38.468255** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
twilight flat script failed with command ack not seen
\n" + "
tack taerget to come back to center of the dome
\n" ], "text/plain": [ "" @@ -8669,15 +1118,19 @@ { "data": { "text/markdown": [ - "- **2024-11-07 00:03:21.097009** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", + "- **2024-11-11 03:45:56.009488** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
ok back in business-- \r\n",
+       "
parameter march comcam\r\n",
        "\r\n",
        "\r\n",
-       "MTMount back, running twilight flats block again
\n"
+       "exp_time: 30.0\r\n",
+       "dof_index: 4 \r\n",
+       "step_sequence: [-0.015, -0.0175, -0.02, -0.025, -0.03]\r\n",
+       "program: \"BLOCK-T279\"\r\n",
+       "reason: \"alignment_sweep_m2ry\"
\n" ], "text/plain": [ "" @@ -8689,18 +1142,20 @@ { "data": { "text/markdown": [ - "- **2024-11-07 00:00:31.399917** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", + "- **2024-11-11 03:34:30.254260** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
The axes are OFF but the ACW is still tracking, we cannot turn it OFF \r\n",
+       "
parameter carch comcam\r\n",
        "\r\n",
        "\r\n",
-       "1.we stopped ACW tracking from EUI \r\n",
-       "\r\n",
-       "\r\n",
-       "Still waiting for MTMount to get ENABLED--- Lots of alarms preventing it.
\n"
+       "exp_time: 30.0\r\n",
+       "dof_index: 4 \r\n",
+       "range: 0.03\r\n",
+       "n_steps: 7\r\n",
+       "program: \"BLOCK-T279\"\r\n",
+       "reason: \"alignment_sweep_m2ry\"
\n" ], "text/plain": [ "" @@ -8712,23 +1167,28 @@ { "data": { "text/markdown": [ - "- **2024-11-06 23:57:18.801287** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", + "- **2024-11-11 03:16:35.071705** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
It seems that there were some interlocks preventing CCW from ENABLING and hence MTMount was not getting ENABLED. \r\n",
-       "Resolved by - \r\n",
-       "\r\n",
-       "\r\n",
-       "1. Enabling the CCW from EUI \r\n",
-       "2. ENABLING Mtmount from CSC\r\n",
+       "
set dof\r\n",
        "\r\n",
        "\r\n",
-       "BUT FAILED \r\n",
-       "\r\n",
-       "\r\n",
-       "begin_enable failed; remaining in state <State.DISABLED: 1>: Command BothAxesResetAlarm(sequence_id=5847129, command_code=<CommandCode.BOTH_AXES_RESET_ALARM: 37>, source=<Source.CSC: 1>, timestamp=1730937369.664346) failed: ExpectedError('Not reseted subsystems: \\nThe command with sequenceID: 5847129 and operationManagerSequenceID: 3955613 from commander AzimuthAxis was rejected by CableWrap subsystem with explanation: The commands allowed in Tracking are: Stop\\n\\nNot accepted command: 306. Timestamp 1730937378.884\\nActive Interlocks: \\n')
\n"
+       "dofs:\r\n",
+       "  [-1.26588000e+03, 0.00000000e+00, -4.30089012e+02,  0.0,\r\n",
+       "       0.0,  1.80668450e+03, -1.45225000e+02,  7.93919936e+01,\r\n",
+       "        0.0, 0.0,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  1.02255131e+00, -1.10015581e+00,\r\n",
+       "        2.73888782e-02,  4.87750072e-02,  8.95215371e-01,  0.0,\r\n",
+       "       0.0,  0.0,  0.0,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00]
\n" ], "text/plain": [ "" @@ -8740,16 +1200,28 @@ { "data": { "text/markdown": [ - "- **2024-11-06 23:53:47.533579** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 03:15:53.281303** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
1. we saw that CCW was not `tracking` on the EUI \r\n",
-       "2. so we did a run_command with cmd: enableCameraCableWrapFollowing \r\n",
+       "
set dof \r\n",
        "\r\n",
        "\r\n",
-       "That resolved it but sjortly after the MTMount+MTPtg Faulted
\n"
+       "dofs:\r\n",
+       "  [-1.26588000e+03, 0.00000000e+00, -4.30089012e+02,  0.0,\r\n",
+       "       0.0,  1.80668450e+03, -1.45225000e+02,  7.93919936e+01,\r\n",
+       "        0.0, 0.0,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  1.02255131e+00, -1.10015581e+00,\r\n",
+       "        2.73888782e-02,  4.87750072e-02,  8.95215371e-01,  1.36704232e-01,\r\n",
+       "       -3.87806584e-01,  2.04433613e-02,  6.66169035e-02,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00]
\n" ], "text/plain": [ "" @@ -8761,25 +1233,24 @@ { "data": { "text/markdown": [ - "- **2024-11-06 23:52:20.404572** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 03:15:38.996277** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Reason why MTMount and MTPtg Faulted \r\n",
-       "\r\n",
-       "\r\n",
-       "---------------------------------\r\n",
-       "\r\n",
-       "\r\n",
-       "Error MTMount replied error for trackTarget command (1733740004) Error no:1 message:Failed: Statechart is busy\r\n",
-       "Not accepted com\r\n",
-       "\r\n",
-       "\r\n",
-       "---------------\r\n",
+       "
sending track target ignoring the MTDome to track with MTA and continue the test\r\n",
        "\r\n",
        "\r\n",
-       "MTMount enabling failed trying again
\n"
+       "track_azel:\r\n",
+       " az: -110\r\n",
+       " el: 80\r\n",
+       "rot_value: 0\r\n",
+       "rot_type: PhysicalSky\r\n",
+       "program: \"BLOCK-T279\"\r\n",
+       "reason: \"BlockT279\"\r\n",
+       "ignore: \r\n",
+       "  - mtdome\r\n",
+       "  - mtdometrajectory
\n" ], "text/plain": [ "" @@ -8791,12 +1262,21 @@ { "data": { "text/markdown": [ - "- **2024-11-06 23:50:25.564567** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 03:14:45.095094** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
MTMount faulted because MTPTg was being rejected by the mount --- > both Faulted and recovering now
\n" + "
we send an other tack target to try the dome following but failed\r\n",
+       "\r\n",
+       "\r\n",
+       "track_azel:\r\n",
+       " az: -110\r\n",
+       " el: 80\r\n",
+       "rot_value: 0\r\n",
+       "rot_type: PhysicalSky\r\n",
+       "program: \"BLOCK-T279\"\r\n",
+       "reason: \"BlockT279\"
\n" ], "text/plain": [ "" @@ -8808,12 +1288,20 @@ { "data": { "text/markdown": [ - "- **2024-11-06 23:46:00.095540** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 03:13:56.582609** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
MTRotator Faulted because CCW did not follow :/ --- > MTPTg Faulted.
\n" + "
take image comcam\r\n",
+       "\r\n",
+       "\r\n",
+       "exp_times: 30\r\n",
+       "nimages: 1\r\n",
+       "image_type: \"ACQ\"\r\n",
+       "program: \"BLOCK-T248\"\r\n",
+       "reason: \"Checking\"\r\n",
+       "filter: \"r_03\"
\n" ], "text/plain": [ "" @@ -8825,21 +1313,24 @@ { "data": { "text/markdown": [ - "- **2024-11-06 23:39:45.796587** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 03:12:00.840609** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
ConsDB issue !\r\n",
+       "
close loop comcam\r\n",
        "\r\n",
        "\r\n",
-       "--------------------------------------------------\r\n",
-       "\r\n",
-       "\r\n",
-       "  File \"/net/obs-env/auto_base_packages/ts_externalscripts/python/lsst/ts/externalscripts/base_take_twilight_flats.py\", line 341, in get_new_exptime\r\n",
-       "    new_exp_time = self.config.target_sky_counts * exp_time / sky_counts\r\n",
-       "                   ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^~~~~~~~~~~~\r\n",
-       "TypeError: unsupported operand type(s) for /: 'int' and 'NoneType'
\n"
+       "filter: \"r_03\"\r\n",
+       "exposure_time: 30\r\n",
+       "max_iter: 1\r\n",
+       "mode: \"FAM\"\r\n",
+       "program: \"BLOCK-T279\"\r\n",
+       "reason: \"SITCOM-826\"\r\n",
+       "note: \"align_m2rxry\"\r\n",
+       "used_dofs: [3, 4]\r\n",
+       "apply_corrections: true\r\n",
+       "use_ocps: true
\n" ], "text/plain": [ "" @@ -8851,31 +1342,15 @@ { "data": { "text/markdown": [ - "- **2024-11-06 23:38:54.726149** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 03:11:39.590255** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
START OF BLOCK-T215 \r\n",
+       "
offset m2 hexapod\r\n",
        "\r\n",
        "\r\n",
-       "Twilight flats in g-band \r\n",
-       "\r\n",
-       "\r\n",
-       "---------------\r\n",
-       "\r\n",
-       "\r\n",
-       "id: BLOCK-T215\r\n",
-       "override:\r\n",
-       " run_filter: \"g_01\"\r\n",
-       " run_target_sky_counts: 20000\r\n",
-       " run_max_counts: 40000\r\n",
-       " run_n_flats: 20\r\n",
-       " run_dither: -180\r\n",
-       " run_max_exp: 30\r\n",
-       " run_target_el: 65.0\r\n",
-       " run_target_az: 70.0\r\n",
-       " run_point_directly: True
\n"
+       "v: -0.02
\n" ], "text/plain": [ "" @@ -8887,12 +1362,12 @@ { "data": { "text/markdown": [ - "- **2024-11-06 23:24:49.614996** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 03:01:50.658093** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Mirror covers, hexapods in compensation mode, dome following enables and MTMount already in position to take twilight flats in g-band.
\n" + "
sending enable dome following, to keep the dome moving
\n" ], "text/plain": [ "" @@ -8904,12 +1379,12 @@ { "data": { "text/markdown": [ - "- **2024-11-06 21:26:11.253078** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 02:58:40.383744** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Scheduler 1 cycled !
\n" + "
the dome is in position, we try one more time ignoring MTDome and DomeTrajectory. TMA is tracking now
\n" ], "text/plain": [ "" @@ -8921,12 +1396,24 @@ { "data": { "text/markdown": [ - "- **2024-11-06 20:48:39.087867** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 02:48:38.761384** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
closing mirror covers till we open them again !
\n" + "
tack target failed\r\n",
+       "\r\n",
+       "\r\n",
+       "dome_az = await self.rem.mtdome.tel_azimuth.next(\r\n",
+       "              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
+       "  File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/read_topic.py\", line 681, in next\r\n",
+       "    return await self._next(timeout=timeout)\r\n",
+       "\r\n",
+       "\r\n",
+       "timeout\r\n",
+       "\r\n",
+       "\r\n",
+       "we try again
\n" ], "text/plain": [ "" @@ -8938,26 +1425,21 @@ { "data": { "text/markdown": [ - "- **2024-11-06 20:21:54.923026** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 02:45:50.800643** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
We are move_p2p with updated config becuase Tiago is testing MTAOS and hexapods are resting (DISABLED) -\r\n",
-       "----------------------------\r\n",
+       "
probably the moon has scater light, moving to new direction\r\n",
        "\r\n",
        "\r\n",
-       "az: 0\r\n",
-       "el: 65\r\n",
-       "move_timeout: 3600\r\n",
-       "ignore:: 65\r\n",
-       "move_timeout: 3600\r\n",
-       "ignore:\r\n",
-       " - mtdome\r\n",
-       " - mtdometrajectory\r\n",
-       " - mtaos\r\n",
-       " - mthexapod_1\r\n",
-       " - mthexapod_2
\n"
+       "track_azel:\r\n",
+       " az: -110\r\n",
+       " el: 80\r\n",
+       "rot_value: 0\r\n",
+       "rot_type: PhysicalSky\r\n",
+       "program: \"BLOCK-T279\"\r\n",
+       "reason: \"BlockT279\"
\n" ], "text/plain": [ "" @@ -8969,12 +1451,24 @@ { "data": { "text/markdown": [ - "- **2024-11-06 20:03:39.125844** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 02:37:05.292736** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
State cycling MTQueue (OFFLINE to back ) for faster LOVE <3
\n" + "
close loop comcam\r\n",
+       "\r\n",
+       "\r\n",
+       "filter: \"r_03\"\r\n",
+       "exposure_time: 30\r\n",
+       "max_iter: 2\r\n",
+       "mode: \"FAM\"\r\n",
+       "program: \"BLOCK-T279\"\r\n",
+       "reason: \"SITCOM-826\"\r\n",
+       "note: \"align_m2rxry\"\r\n",
+       "used_dofs: [3, 4]\r\n",
+       "apply_corrections: true\r\n",
+       "use_ocps: true
\n" ], "text/plain": [ "" @@ -8986,12 +1480,12 @@ { "data": { "text/markdown": [ - "- **2024-11-06 19:50:29.957137** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 02:21:57.464292** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Just for record , we are making the TMA move at Default settings which are still at 1%. However, as mentioned before, Brian has created a specific setting set called \"NewSpeedSetting2percent\" if we are cleared to upgrade the TMA speed to 2%
\n" + "
Starting BLOCK-T249
\n" ], "text/plain": [ "" @@ -9003,19 +1497,12 @@ { "data": { "text/markdown": [ - "- **2024-11-06 19:44:58.575006** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 02:02:33.991707** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Track_ target seems to run a 3rd time. We will do move+p2p next to \r\n",
-       "-------------\r\n",
-       "az: 0\r\n",
-       "el: 75\r\n",
-       "move_timeout: 3600\r\n",
-       "ignore:\r\n",
-       " - mtdome\r\n",
-       " - mtdometrajectory
\n" + "
we set dof
\n" ], "text/plain": [ "" @@ -9027,18 +1514,18 @@ { "data": { "text/markdown": [ - "- **2024-11-06 19:43:50.583099** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 02:02:15.668207** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
It still failed with same error as before -- but MTAOS was ENABLED \r\n",
-       "------------------------\r\n",
+       "
parameter march comcam failed\r\n",
        "\r\n",
        "\r\n",
-       "  File \"/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/remote_group.py\", line 505, in check_component_state\r\n",
-       "    raise RuntimeError(\r\n",
-       "RuntimeError: mtaos state is <State.STANDBY: 5>, expected <State.ENABLED: 2>
\n"
+       "salindex 101515\r\n",
+       "\r\n",
+       "\r\n",
+       "    self.camera.rem.ccoods.evt_imageInOODS.flush()
\n" ], "text/plain": [ "" @@ -9050,12 +1537,12 @@ { "data": { "text/markdown": [ - "- **2024-11-06 19:40:51.723031** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 01:59:43.854201** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Track target failed because MTAOS was DISABLED (not by us ). Running again
\n" + "
Starting BLOCK-T279
\n" ], "text/plain": [ "" @@ -9067,20 +1554,12 @@ { "data": { "text/markdown": [ - "- **2024-11-06 19:39:11.693711** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 01:55:13.836699** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
TMA checkout in progress-- tracking target \r\n",
-       "-----------------------------------\r\n",
-       "track_azel:\r\n",
-       " az: 145\r\n",
-       " el: 77\r\n",
-       "rot_value: 0\r\n",
-       "program: \"BLOCK-T250\"\r\n",
-       "reason: \"TMACheckout\"\r\n",
-       "rot_type: PhysicalSkytrack_azel:
\n" + "
The dome has an issue, is disconnecting form the CSC, we check the telemetry if is still connected and reset the dome sending the dome to standby then to Enable again, be patient to confirm the telemetry is back.
\n" ], "text/plain": [ "" @@ -9092,12 +1571,12 @@ { "data": { "text/markdown": [ - "- **2024-11-06 19:23:23.287722** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", + "- **2024-11-11 01:48:32.828089** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Hardpoint test passed !
\n" + "
Homing the Dom and moving TMA to -90 Az to prepare for BLOCK-279
\n" ], "text/plain": [ "" @@ -9109,12 +1588,12 @@ { "data": { "text/markdown": [ - "- **2024-11-06 19:22:26.252986** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", + "- **2024-11-11 01:39:23.513944** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
ENABLING MTmount
\n" + "
Debugging the Dome, we are taken some images for AOS tests with out dome following
\n" ], "text/plain": [ "" @@ -9126,12 +1605,12 @@ { "data": { "text/markdown": [ - "- **2024-11-06 19:14:03.157404** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 01:10:35.770067** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Started the Hard-point breakaway test from the M1M3GUI-- Simultaneous testing !
\n" + "
Dome is reading wrong position. still having issue with az drive.
\n" ], "text/plain": [ "" @@ -9143,12 +1622,12 @@ { "data": { "text/markdown": [ - "- **2024-11-06 18:57:42.870367** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 01:10:17.726281** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
We note now that the M1M3 Cell pressure improving slightly as we move to zenith !
\n" + "
while dome is having a problem, we decided to stop tracking and take triplet image to check focusing.
\n" ], "text/plain": [ "" @@ -9160,12 +1639,12 @@ { "data": { "text/markdown": [ - "- **2024-11-06 18:54:32.264023** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 00:57:35.568717** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Ok we are moving TMA straight to 90 from o deg on parking settings !!
\n" + "
212202 FAULT in Azimuth control software status : problem with ethercat connection.
\n" ], "text/plain": [ "" @@ -9177,12 +1656,12 @@ { "data": { "text/markdown": [ - "- **2024-11-06 18:49:58.137902** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 00:55:28.974017** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
TMA PS turning on
\n" + "
MTDome still enabled, but had AMCS azimuth drive fault and has error. will run exitFault
\n" ], "text/plain": [ "" @@ -9194,12 +1673,15 @@ { "data": { "text/markdown": [ - "- **2024-11-06 18:49:00.492078** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 00:52:11.926041** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Dome handed over
\n" + "
When hexapod faulted, there was huge vibration in motor currents, and all we tried to do was to move a bit to focus.  \r\n",
+       "\r\n",
+       "\r\n",
+       "salindex=101495 2024-11-11T00:44:17.736Z
\n" ], "text/plain": [ "" @@ -9211,12 +1693,7 @@ { "data": { "text/markdown": [ - "- **2024-11-06 18:25:03.332479** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
The dome was handeed over at 328.4 Deg --- moved now to Az=328
\n" + "- Link: [OBS-659](https://rubinobs.atlassian.net/browse/OBS-659)" ], "text/plain": [ "" @@ -9228,19 +1705,12 @@ { "data": { "text/markdown": [ - "- **2024-11-06 18:22:19.433603** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 00:46:57.433925** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Exit Fault failed -- expected !!! \r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 191, in next_ackcmd\r\n",
-       "    raise base.AckError(msg=\"Command failed\", ackcmd=ackcmd)\r\n",
-       "lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=612720053, ack=<SalRetCode.CMD_FAILED: -302>, error=1, result='Failed: Command exitFault was sent for an incorrect state.')
\n" + "
running close_loop_comcam again.
\n" ], "text/plain": [ "" @@ -9252,15 +1722,12 @@ { "data": { "text/markdown": [ - "- **2024-11-06 17:58:34.013548** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 00:45:54.702501** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
So the M1M3 Cell pressure is degrading -- today its 116.96 PSI. Its close to 117 but  not exactly. \r\n",
-       "\r\n",
-       "\r\n",
-       "Petr says: \"Regarding pressure: let's stop anything when we reach 115 PSI. Lower mirror, go on static supports to zenith. They will check tomorrow what's going on. All FAs seems healthy, there will be some leak\"
\n" + "
recovered hexapod and enabled compensation mode.
\n" ], "text/plain": [ "" @@ -9272,12 +1739,12 @@ { "data": { "text/markdown": [ - "- **2024-11-06 17:57:42.806362** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 00:44:55.871835** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Robinson - Turning on the OSS !
\n" + "
Hexapod 2 faulted. recovering.
\n" ], "text/plain": [ "" @@ -9289,12 +1756,12 @@ { "data": { "text/markdown": [ - "- **2024-11-06 17:47:35.041041** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 00:41:29.942358** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Both Hex turned on --- Cam Hex is in DISABLED while M2Hex started its warm-up
\n" + "
stopping the scheduler
\n" ], "text/plain": [ "" @@ -9306,12 +1773,13 @@ { "data": { "text/markdown": [ - "- **2024-11-06 17:35:58.189695** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 00:36:00.059077** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Brian updated the default setting set in TMA EUI -- > I think to 2% speed !
\n" + "
tried take_image_comcam, worked. \r\n",
+       "trying resume again
\n" ], "text/plain": [ "" @@ -9323,12 +1791,12 @@ { "data": { "text/markdown": [ - "- **2024-11-06 17:30:47.463921** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 00:30:32.668602** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
TEA is all clear so we are going to proceed with hexapod and rotator startups --- but keep the hexapods DISABLED till we actually need to use them !
\n" + "
Re-enabling the scheduler and try if other command than resume.py works.
\n" ], "text/plain": [ "" @@ -9340,19 +1808,12 @@ { "data": { "text/markdown": [ - "- **2024-11-06 17:21:00.858089** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 00:28:37.322827** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
So Eduardo wants to check teh oil drain undersneath the tMA today as well --- all efforts are towards releasing the TMA early to the observers so that they can do the Az motion for the cj=heck --- \r\n",
-       "\r\n",
-       "\r\n",
-       "PLAN : \r\n",
-       "\r\n",
-       "\r\n",
-       "1. get TMA out of horizon and take it to 70 degrees with M1M3 lowered. \r\n",
-       "2. We will move in Az at parking settings since the M1M3 cannot be raised before hardpoint Breakaway tests
\n" + "
Faulted the scheduler again. Pinging Tiago and Tony Johnson.
\n" ], "text/plain": [ "" @@ -9364,25 +1825,14 @@ { "data": { "text/markdown": [ - "- **2024-11-06 16:57:44.936219** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 00:24:16.888755** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
M1M3 Bump tests are done ! we report failures for the following FAs - \r\n",
-       "\r\n",
-       "\r\n",
-       "-------------------------------\r\n",
-       "\r\n",
-       "\r\n",
-       "Actuators [135, 330] FAILED the bump test.\r\n",
-       "\r\n",
-       "\r\n",
-       "SAA (Single Actuator Axes) Failures:\r\n",
-       "  None\r\n",
-       "DAA (Dual Actuator Axes) Failures:\r\n",
-       "  - Actuator ID 135: Pri Index 34\r\n",
-       "  - Actuator ID 330: Pri Index 107
\n" + "
Scheduler 1 faulted \r\n",
+       "raise base.AckError(msg=\"Command failed\", ackcmd=ackcmd)\r\n",
+       "lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=9743385, ack=<SalRetCode.CMD_FAILED: -302>, error=1, result=\"Failed: 502, message='Bad Gateway', url=URL('http://ccs.lsst.org/ImageUtilities/rest/imageNameService?n=1&sourceIndex=248&source=BlockT')\")
\n" ], "text/plain": [ "" @@ -9394,12 +1844,7 @@ { "data": { "text/markdown": [ - "- **2024-11-06 16:02:06.548405** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
We already started bump tests. The telescope is at Horizon but I confirmed with Petr that we should be good anyways.
\n" + "- Link: [OBS-614](https://rubinobs.atlassian.net/browse/OBS-614)" ], "text/plain": [ "" @@ -9411,12 +1856,12 @@ { "data": { "text/markdown": [ - "- **2024-11-06 16:00:03.341535** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 00:17:30.976422** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Started the M1M3 FA Bump tests --- Note that the TMA is at horizon.
\n" + "
starting BLOCK-T248
\n" ], "text/plain": [ "" @@ -9428,19 +1873,15 @@ { "data": { "text/markdown": [ - "- **2024-11-06 15:38:53.935820** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 00:04:41.106757** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Summary of activities going on at this moment:\r\n",
-       "* Roberto T. working on M2 tangent links\r\n",
-       "* Manuel and Miguel working on the return lines for the OSS\r\n",
-       "* Brian, Tomi, and Claudio working on changing a filter in ComCam\r\n",
-       "* No work inside M1m3\r\n",
-       "\r\n",
-       "\r\n",
-       "With everything going up, I will try to start bump tests to try to gain some time. Hexapods and Rotator should remain off for now.
\n" + "
take_image_comcam \r\n",
+       "exp time 30sec\r\n",
+       "r_03 band\r\n",
+       "1 acq image
\n" ], "text/plain": [ "" @@ -9452,23 +1893,25 @@ { "data": { "text/markdown": [ - "- **2024-11-07 23:56:37.372735** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 00:04:21.585925** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Today we learned some important lessons:\r\n",
-       "1. The laser tracker seems to be blocked by the G-band filter, as it should.\r\n",
-       "2. When trying to run twilight flats, if the script seems stale, CHECK THE DOME. \r\n",
-       "\r\n",
-       "\r\n",
-       "We realized a bit too late that the dome was not publishing telemetry. \r\n",
-       "We were fooled because we had LOVE with the Dome display in front of us. \r\n",
-       "It gave us the idea the the dome was following the telescope just fine. \r\n",
-       "However, it was only until we opened Chronograf that we realized that the dome was not publishing telemetry at all. \r\n",
-       "The dashboard confirmed that we had ``212202=Problem with ethercat connection`` problem. \r\n",
-       "We tried to state cycle many times, even with pauses. No success. \r\n",
-       "So we are now rebooting the MTDome cRIO.
\n" + "
starting BLOCK-T249 with set_dof\r\n",
+       "[-1.26588000e+03, -2.08991571e-08,  4.95136945e-08,  5.53397010e-03,\r\n",
+       "       -2.40923443e-03,  1.80668450e+03,  4.85815000e+03,  6.73960000e+02,\r\n",
+       "        3.16024784e-03, -7.47932190e-04,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  1.02255131e+00, -1.10015581e+00,\r\n",
+       "        2.73888782e-02,  4.87750072e-02,  8.95215371e-01,  1.36704232e-01,\r\n",
+       "       -3.87806584e-01,  2.04433613e-02,  6.66169035e-02,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00]
\n" ], "text/plain": [ "" @@ -9480,13 +1923,12 @@ { "data": { "text/markdown": [ - "- **2024-11-07 23:51:28.538724** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 20:25:43.425602** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
we first took a flat image in g_01 and 0.1 seconds with the laser on and worked.\r\n",
-       "Then we tried to take a flat image (BLOCK-T215) but did not ingest the image.
\n" + "
Dome handover was finally completed after rebooting the cRIO.  But the telemetry was slow, and only a few minutes after successfully moving the dome under CSC control, we lost the telemetry, around 19:43 UT
\n" ], "text/plain": [ "" @@ -9498,19 +1940,7 @@ { "data": { "text/markdown": [ - "- **2024-11-07 23:18:36.474799** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
2024/11/07 23:05:26 TAI\r\n",
-       "Salindex 100366\r\n",
-       "Mirror covers got stuck when trying to close them.\r\n",
-       "we ran again the run command to close them again and it worked \r\n",
-       "\r\n",
-       "\r\n",
-       "base.AckError(msg=\"Command failed\", ackcmd=ackcmd)\r\n",
-       "lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=252964407, ack=<SalRetCode.CMD_FAILED: -302>, error=1, result='Failed: Command with id 189576 has timed out after not receiving accept for 500 ms.')
\n" + "- Link: [OBS-651](https://rubinobs.atlassian.net/browse/OBS-651)" ], "text/plain": [ "" @@ -9522,12 +1952,62 @@ { "data": { "text/markdown": [ - "- **2024-11-07 23:10:56.584268** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: None\n", + "- **2024-11-11 20:25:43.425602** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
Dome handover was finally completed after rebooting the cRIO.  But the telemetry was slow, and only a few minutes after successfully moving the dome under CSC control, we lost the telemetry, around 19:43 UT
\n", + "\n", + " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
HVAC turned off and Fans turned on,
\n" + "sending exitFault resulted in:\r\n", + "\r\n", + "\r\n", + "Error in run\r\n", + "
Traceback (most recent call last):\r\n",
+       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/asyncio/tasks.py", line 500, in wait_for\r\n",
+       "    return fut.result()\r\n",
+       "           ^^^^^^^^^^^^\r\n",
+       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 239, in _get_next_ackcmd\r\n",
+       "    await self._next_ack_event.wait()\r\n",
+       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/asyncio/locks.py", line 213, in wait\r\n",
+       "    await fut\r\n",
+       "asyncio.exceptions.CancelledError\r\n",
+       "\r\n",
+       "\r\n",
+       "The above exception was the direct cause of the following exception:\r\n",
+       "\r\n",
+       "\r\n",
+       "Traceback (most recent call last):\r\n",
+       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 189, in next_ackcmd\r\n",
+       "    ackcmd = await self._wait_task\r\n",
+       "             ^^^^^^^^^^^^^^^^^^^^^\r\n",
+       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 214, in _basic_next_ackcmd\r\n",
+       "    ackcmd = await asyncio.wait_for(\r\n",
+       "             ^^^^^^^^^^^^^^^^^^^^^^^\r\n",
+       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/asyncio/tasks.py", line 502, in wait_for\r\n",
+       "    raise exceptions.TimeoutError() from exc\r\n",
+       "TimeoutError\r\n",
+       "\r\n",
+       "\r\n",
+       "During handling of the above exception, another exception occurred:\r\n",
+       "\r\n",
+       "\r\n",
+       "Traceback (most recent call last):\r\n",
+       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py", line 631, in do_run\r\n",
+       "    await self._run_task\r\n",
+       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/run_command.py", line 170, in run\r\n",
+       "    await getattr(self.remote, f"cmd_{self.cmd}").start(\r\n",
+       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 487, in start\r\n",
+       "    return await cmd_info.next_ackcmd(timeout=timeout)\r\n",
+       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
+       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 205, in next_ackcmd\r\n",
+       "    raise base.AckTimeoutError(\r\n",
+       "lsst.ts.salobj.base.AckTimeoutError: msg='Timed out waiting for command acknowledgement', ackcmd=(ackcmd private_seqNum=612720053, ack=&lt;SalRetCode.CMD_NOACK: -301&gt;, error=0, result='No command acknowledgement seen')\r\n",
+       "
\n" ], "text/plain": [ "" @@ -9539,20 +2019,12 @@ { "data": { "text/markdown": [ - "- **2024-11-07 22:52:30.372258** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: None\n", + "- **2024-11-11 19:41:28.960982** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
We ran M2 bump tests which triggered a couple of alarms. The alarms would come and go quickly. \r\n",
-       "Finally, we performed alignment with the lasertracker but it seemed that the scripts went too fast. \r\n",
-       "Later we realized that the LaserTracker was in a weird state. \r\n",
-       "\r\n",
-       "\r\n",
-       "It is important to notice that we realized that the LaserTracker was powered off almost at 17:30. Sandrine and David J. went up to power ir up. TMA with locking pins and access gate open. \r\n",
-       "\r\n",
-       "\r\n",
-       "Now we have the lasertracker finally running. We should go up soon to open the dome to go on sky.
\n" + "
Having difficulty with the dome handover.  MTDome cannot transition to enabled after putting into remote mode.  Goes from disabled to FAULT.  Dashboard reports \"212202=Problem with ethercat connection.\".  David S reset the cRIO remotely (since I could not access the brick machine - though I have in the past without issue).
\n" ], "text/plain": [ "" @@ -9564,12 +2036,12 @@ { "data": { "text/markdown": [ - "- **2024-11-07 21:43:58.959073** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 18:58:49.940826** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
We are now running M2 bump tests in MTQueue and need to take measurements with the laser tracker before we go on Sky. ATQueue and OCSQueue seems unusable. So we are a bit stuck for now.
\n" + "
Actuators [218, 233, 235, 238, 424, 427, 428, 429, 430, 433, 434, 435] FAILED the bump test.
\n" ], "text/plain": [ "" @@ -9581,12 +2053,12 @@ { "data": { "text/markdown": [ - "- **2024-11-07 21:27:18.450753** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 18:56:04.282979** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Mirror covers are open in prep for laser tracker measurements
\n" + "
Bump test appeared to have more failures than usual, perhaps due to the power glitch that happened when switching between generator #2 and generator #1.  Attempting to state cycle MTM1M3 was unsuccessful, cannot transition from Standby to Start.  It faults.  David noticed the air pressure is low, 112 psi and dropping
\n" ], "text/plain": [ "" @@ -9598,12 +2070,12 @@ { "data": { "text/markdown": [ - "- **2024-11-07 19:44:38.779952** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: None\n", + "- **2024-11-11 18:11:43.168782** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
TMA is recovered
\n" + "
The dome handover is going to happen at 4pm, since electrical team will check that generators are operational for the night.
\n" ], "text/plain": [ "" @@ -9615,12 +2087,12 @@ { "data": { "text/markdown": [ - "- **2024-11-07 19:39:26.159204** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: None\n", + "- **2024-11-11 17:46:30.473695** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
The bump test is finished, and everything went okay
\n" + "
Running bump test ignoring FA 215.
\n" ], "text/plain": [ "" @@ -9632,12 +2104,12 @@ { "data": { "text/markdown": [ - "- **2024-11-07 19:22:29.373337** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 17:30:31.927662** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Elevation drives failed, probably due to the overheating in the cabinets. David Jimenez will go and check it.
\n" + "
Laser tracker was turned one. We are re-enabling MTM3 SS and TS. Turning FCUs on at 300 RPM.
\n" ], "text/plain": [ "" @@ -9649,12 +2121,12 @@ { "data": { "text/markdown": [ - "- **2024-11-07 19:20:30.048428** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 17:17:55.174260** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Dome handover done with no issues
\n" + "
Ran M2 hex warmup while at lunch.  Finished and back in the control room.
\n" ], "text/plain": [ "" @@ -9666,12 +2138,12 @@ { "data": { "text/markdown": [ - "- **2024-11-07 18:35:11.217925** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 16:00:11.794739** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Starting dome handover
\n" + "
Starting Cam Hex warmup (T4).
\n" ], "text/plain": [ "" @@ -9683,13 +2155,12 @@ { "data": { "text/markdown": [ - "- **2024-11-07 18:15:10.284357** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 15:35:57.769718** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Petr and Brian stopped the SAL Script to run the Hardpoint Breakaway tests. \r\n",
-       "They will run these tests via EUI instead.
\n" + "
Powering on hexapods and rotator electronics.
\n" ], "text/plain": [ "" @@ -9701,13 +2172,13 @@ { "data": { "text/markdown": [ - "- **2024-11-07 18:13:58.715631** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 14:50:59.255566** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
We got confirmation from Petr that we are good to go on HardPoint Breakaway tests (BLOCK-T145). \r\n",
-       "Starting it now.
\n" + "
Shouldn't have put M1M3 into standby after this probably, discussing with Petr. (M1M3 back to Enabled, which made air and power back on.)\r\n",
+       "M1M3 power has some errors.
\n" ], "text/plain": [ "" @@ -9719,12 +2190,12 @@ { "data": { "text/markdown": [ - "- **2024-11-07 18:05:04.447679** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 14:49:41.971912** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
While M2Hexapods are warming up, Marina, Gonzalo and I went up to Level 8 for the TMA and Dome Handover.
\n" + "
Disabled FA215 back, turned air off, power off (main->aux)
\n" ], "text/plain": [ "" @@ -9736,12 +2207,12 @@ { "data": { "text/markdown": [ - "- **2024-11-07 17:54:20.667900** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 14:46:30.269168** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
We are warming-up the Hexapods now
\n" + "
Enabled FA215 back, (had it disabled from configuration in lower level, couldn't change disable/enable state of FA on EUI. Petr had to change), tested bump test on FA215, failed. ~100 N instead of 222 N.
\n" ], "text/plain": [ "" @@ -9753,12 +2224,12 @@ { "data": { "text/markdown": [ - "- **2024-11-07 17:03:53.871075** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 14:44:02.085067** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Jacques, Brian, Marina, Gonzalo and I had a quick chat and we all agreed to leave the set point today at 11ºC.
\n" + "
Visual inspection with FA215 looked good, Christian and Roberto opened the FA215's valve.
\n" ], "text/plain": [ "" @@ -9770,12 +2241,12 @@ { "data": { "text/markdown": [ - "- **2024-11-07 16:07:41.295262** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 13:24:52.800785** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
running BLOCK-T246 Taking bias images
\n" + "
LOTO on, Christian gets into the M1M3 cell to inspect FA 215
\n" ], "text/plain": [ "" @@ -9787,12 +2258,12 @@ { "data": { "text/markdown": [ - "- **2024-11-07 16:03:18.586752** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 13:10:27.057259** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Rotator and Hexapods startup went well with no issues
\n" + "
TMA parked at zenith, az=-31. locking pin in. PS off, OSS off.
\n" ], "text/plain": [ "" @@ -9804,15 +2275,12 @@ { "data": { "text/markdown": [ - "- **2024-11-07 15:43:51.013520** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 13:08:09.819671** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Gonzalo and Marina are now in the control room. Both are powering up the hexapods and the rotator. We might leave the warm-ups for later following Te-Wei's suggestion ([here|https://rubin-obs.slack.com/archives/C07QJMQ01FU/p1730994153557769]). \r\n",
-       "\r\n",
-       "\r\n",
-       "Meanwhile, I am addressing the temperatures in preparation for tonight.
\n" + "
MTPtg ENABLED, MTMount DISABLED
\n" ], "text/plain": [ "" @@ -9824,13 +2292,13 @@ { "data": { "text/markdown": [ - "- **2024-11-07 14:46:42.064697** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-11 12:58:37.079807** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
There was a fault on M1M3 just now and I recovered. \r\n",
-       "It was associated with work inside the cell where Felipe D. closed the DAA438 valve while checking for leaks.
\n" + "
Day crew came. Need TMA zenith and az=-31 for access to the M1M3 cell.\r\n",
+       "PS on, switched to the Parking set, David Jimenez moving TMA in EUI.
\n" ], "text/plain": [ "" @@ -9889,8 +2357,8 @@ "text/markdown": [ "## Project-Wide Night(s) Report \n", "- Run on logs and databases from **https://usdf-rsp-dev.slac.stanford.edu/**\n", - "- Report **3 observing night(s)** with the last reported night starting on **2024-11-07 12:00:00**.\n", - "- min_dayobs='2024-11-05', max_dayobs='2024-11-08'\n", + "- Report **1 observing night(s)** with the last reported night starting on **2024-11-11 12:00:00**.\n", + "- min_dayobs='2024-11-11', max_dayobs='2024-11-12'\n", "- Using ***Prototype* Logging and Reporting** Version: **0.1.dev145+gf2a5765.d20241014**\n", "- have_consdb = True\n", "- Imported lsst.ts.logging_and_reporting from local packages.\n" @@ -9963,7 +2431,7 @@ " \n", " nightreport/reports\n", " 0\n", - " https://usdf-rsp-dev.slac.stanford.edu/nightreport/reports?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=100&min_day_obs=20241105&max_day_obs=20241108\n", + " https://usdf-rsp-dev.slac.stanford.edu/nightreport/reports?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=100&min_day_obs=20241111&max_day_obs=20241112\n", " \n", " \n", " exposurelog/instruments\n", @@ -9973,32 +2441,32 @@ " \n", " exposurelog/exposures.LSSTComCamSim\n", " 0\n", - " https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LSSTComCamSim&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241105&max_day_obs=20241108\n", + " https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LSSTComCamSim&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241111&max_day_obs=20241112\n", " \n", " \n", " exposurelog/exposures.LATISS\n", " 0\n", - " https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LATISS&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241105&max_day_obs=20241108\n", + " https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LATISS&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241111&max_day_obs=20241112\n", " \n", " \n", " exposurelog/exposures.LSSTComCam\n", - " 616\n", - " https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LSSTComCam&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241105&max_day_obs=20241108\n", + " 5\n", + " https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LSSTComCam&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241111&max_day_obs=20241112\n", " \n", " \n", " exposurelog/exposures.LSSTCam\n", - " 1928\n", - " https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LSSTCam&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241105&max_day_obs=20241108\n", + " 0\n", + " https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LSSTCam&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241111&max_day_obs=20241112\n", " \n", " \n", " exposurelog/messages\n", - " 16\n", - " https://usdf-rsp-dev.slac.stanford.edu/exposurelog/messages?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=2500&min_day_obs=20241105&max_day_obs=20241108\n", + " 0\n", + " https://usdf-rsp-dev.slac.stanford.edu/exposurelog/messages?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=2500&min_day_obs=20241111&max_day_obs=20241112\n", " \n", " \n", " narrativelog/messages\n", - " 402\n", - " https://usdf-rsp-dev.slac.stanford.edu/narrativelog/messages?is_human=either&is_valid=true&order_by=-date_added&offset=0&limit=1000&min_date_added=2024-11-05T00%3A00%3A00&max_date_added=2024-11-08T00%3A00%3A00\n", + " 62\n", + " https://usdf-rsp-dev.slac.stanford.edu/narrativelog/messages?is_human=either&is_valid=true&order_by=-date_added&offset=0&limit=1000&min_date_added=2024-11-11T00%3A00%3A00&max_date_added=2024-11-12T00%3A00%3A00\n", " \n", " \n", " efd/targets\n", @@ -10135,7 +2603,7 @@ { "data": { "text/markdown": [ - "- Endpoint: https://usdf-rsp-dev.slac.stanford.edu/nightreport/reports?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=100&min_day_obs=20241105&max_day_obs=20241108" + "- Endpoint: https://usdf-rsp-dev.slac.stanford.edu/nightreport/reports?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=100&min_day_obs=20241111&max_day_obs=20241112" ], "text/plain": [ "" @@ -10155,7 +2623,7 @@ { "data": { "text/markdown": [ - "### Overview for Service: `exposurelog` [16]" + "### Overview for Service: `exposurelog` [0]" ], "text/plain": [ "" @@ -10174,7 +2642,7 @@ { "data": { "text/markdown": [ - "- Endpoint: https://usdf-rsp-dev.slac.stanford.edu/exposurelog/messages?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=2500&min_day_obs=20241105&max_day_obs=20241108" + "- Endpoint: https://usdf-rsp-dev.slac.stanford.edu/exposurelog/messages?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=2500&min_day_obs=20241111&max_day_obs=20241112" ], "text/plain": [ "" @@ -10188,13 +2656,13 @@ "output_type": "stream", "text": [ "None\n", - "- Got 16 records. \n" + "- Got 0 records. \n" ] }, { "data": { "text/markdown": [ - "### Overview for Service: `narrativelog` [402]" + "### Overview for Service: `narrativelog` [62]" ], "text/plain": [ "" @@ -10213,7 +2681,7 @@ { "data": { "text/markdown": [ - "- Endpoint: https://usdf-rsp-dev.slac.stanford.edu/narrativelog/messages?is_human=either&is_valid=true&order_by=-date_added&offset=0&limit=1000&min_date_added=2024-11-05T00%3A00%3A00&max_date_added=2024-11-08T00%3A00%3A00" + "- Endpoint: https://usdf-rsp-dev.slac.stanford.edu/narrativelog/messages?is_human=either&is_valid=true&order_by=-date_added&offset=0&limit=1000&min_date_added=2024-11-11T00%3A00%3A00&max_date_added=2024-11-12T00%3A00%3A00" ], "text/plain": [ "" @@ -10227,7 +2695,7 @@ "output_type": "stream", "text": [ "None\n", - "- Got 402 records. \n" + "- Got 62 records. \n" ] } ], @@ -10277,228 +2745,7 @@ "execution_count": 21, "id": "24", "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "### Instrument: LSSTComCam (16)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "* CC_O_20241105_000019 : 2024-11-06T00:05:06.599261\n", - " -  G `We suddenly got out of focus ! the hexapods moved ! FocusZ changed from 0.22 to 0.44`" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "* CC_O_20241105_000026 : 2024-11-06T00:12:34.276724\n", - " -  G `Hexapods moved again and the focus deteriorated !`" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "* CC_O_20241105_000033 : 2024-11-06T00:38:34.849868\n", - " -  G `BLOCK-88`" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "* CC_O_20241105_000034 : 2024-11-06T00:38:34.811449\n", - " -  G `BLOCK-88`" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "* CC_O_20241105_000035 : 2024-11-06T00:38:34.771987\n", - " -  G `BLOCK-88`" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "* CC_O_20241105_000036 : 2024-11-06T00:38:34.731047\n", - " -  G `BLOCK-88`" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "* CC_O_20241105_000219 : 2024-11-06T07:28:40.085767\n", - " -  G `Waffle`" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "* CC_O_20241105_000220 : 2024-11-06T07:31:07.333966\n", - " -  G `BLOCK-T251 pinhole filter`" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "* CC_O_20241106_000032 : 2024-11-07T01:20:39.419111\n", - " -  G `The best one. Step 9 for BLOCK-T92.`" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "* CC_O_20241106_000042 : 2024-11-07T01:28:28.520157\n", - " -  R `The best one. Step 9 for BLOCK-T92.`" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "* CC_O_20241106_000043 : 2024-11-07T01:29:00.025889\n", - " -  R `The best one. Step 9 for BLOCK-T92.`" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "* CC_O_20241106_000044 : 2024-11-07T01:29:42.989914\n", - " -  R `The best one. Step 9 for BLOCK-T92.`" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "* CC_O_20241106_000045 : 2024-11-07T01:31:23.489238\n", - " -  G `Looks good, donuts`" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "* CC_O_20241106_000078 : 2024-11-07T02:08:04.439645\n", - " -  R `Looks good, donuts`" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "* CC_O_20241106_000163 : 2024-11-07T04:41:14.472781\n", - " -  ? `Looks good, donuts`" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "* CC_O_20241106_000178 : 2024-11-07T05:01:19.057283\n", - " -  G `Lost tracking during the exposure. Streaks`" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "# Time Log\n", "allrep.exp_rep.time_log_as_markdown()" @@ -10526,7 +2773,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Finished 2024-11-08 23:27:38.490611\n" + "Finished 2024-11-11 20:44:55.082635\n" ] } ], diff --git a/notebooks_tsqr/dashboard.ipynb b/notebooks_tsqr/dashboard.ipynb index 1f3aeb2..43271f4 100644 --- a/notebooks_tsqr/dashboard.ipynb +++ b/notebooks_tsqr/dashboard.ipynb @@ -46,22 +46,15 @@ "text": [ "Try connect to each endpoint of https://summit-lsp.lsst.codes/nightreport \n", "Try connect to each endpoint of https://summit-lsp.lsst.codes/narrativelog \n", - "DEBUG add_exposure_flag instrument='LSSTComCamSim' set()\n", - "DEBUG add_exposure_flag instrument='LSSTComCam' {'unknown'}\n", - "DEBUG add_exposure_flag instrument='LATISS' set()\n", "Try connect to each endpoint of https://summit-lsp.lsst.codes/exposurelog \n", "Try connect to each endpoint of https://usdf-rsp-dev.slac.stanford.edu/nightreport \n", "Try connect to each endpoint of https://usdf-rsp-dev.slac.stanford.edu/narrativelog \n", - "DEBUG add_exposure_flag instrument='LSSTComCamSim' set()\n", - "DEBUG add_exposure_flag instrument='LATISS' set()\n", - "DEBUG add_exposure_flag instrument='LSSTComCam' {'unknown'}\n", - "DEBUG add_exposure_flag instrument='LSSTCam' {'unknown'}\n", "Try connect to each endpoint of https://usdf-rsp-dev.slac.stanford.edu/exposurelog \n" ] }, { "ename": "ConnectTimeout", - "evalue": "HTTPSConnectionPool(host='tucson-teststand.lsst.codes', port=443): Max retries exceeded with url: /nightreport/reports?limit=2 (Caused by ConnectTimeoutError(, 'Connection to tucson-teststand.lsst.codes timed out. (connect timeout=1.05)'))", + "evalue": "HTTPSConnectionPool(host='tucson-teststand.lsst.codes', port=443): Max retries exceeded with url: /nightreport/reports?limit=2 (Caused by ConnectTimeoutError(, 'Connection to tucson-teststand.lsst.codes timed out. (connect timeout=1.05)'))", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", @@ -77,25 +70,25 @@ "File \u001b[0;32m/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/urllib3/connectionpool.py:1060\u001b[0m, in \u001b[0;36mHTTPSConnectionPool._validate_conn\u001b[0;34m(self, conn)\u001b[0m\n\u001b[1;32m 1059\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mgetattr\u001b[39m(conn, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msock\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m): \u001b[38;5;66;03m# AppEngine might not have `.sock`\u001b[39;00m\n\u001b[0;32m-> 1060\u001b[0m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1062\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m conn\u001b[38;5;241m.\u001b[39mis_verified:\n", "File \u001b[0;32m/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/urllib3/connection.py:363\u001b[0m, in \u001b[0;36mHTTPSConnection.connect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 361\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mconnect\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 362\u001b[0m \u001b[38;5;66;03m# Add certificate verification\u001b[39;00m\n\u001b[0;32m--> 363\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msock \u001b[38;5;241m=\u001b[39m conn \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_new_conn\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 364\u001b[0m hostname \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhost\n", "File \u001b[0;32m/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/urllib3/connection.py:179\u001b[0m, in \u001b[0;36mHTTPConnection._new_conn\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 178\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m SocketTimeout:\n\u001b[0;32m--> 179\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m ConnectTimeoutError(\n\u001b[1;32m 180\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 181\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mConnection to \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m timed out. (connect timeout=\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m)\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 182\u001b[0m \u001b[38;5;241m%\u001b[39m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhost, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtimeout),\n\u001b[1;32m 183\u001b[0m )\n\u001b[1;32m 185\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m SocketError \u001b[38;5;28;01mas\u001b[39;00m e:\n", - "\u001b[0;31mConnectTimeoutError\u001b[0m: (, 'Connection to tucson-teststand.lsst.codes timed out. (connect timeout=1.05)')", + "\u001b[0;31mConnectTimeoutError\u001b[0m: (, 'Connection to tucson-teststand.lsst.codes timed out. (connect timeout=1.05)')", "\nDuring handling of the above exception, another exception occurred:\n", "\u001b[0;31mMaxRetryError\u001b[0m Traceback (most recent call last)", "File \u001b[0;32m/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/requests/adapters.py:667\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m 666\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 667\u001b[0m resp \u001b[38;5;241m=\u001b[39m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 668\u001b[0m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 669\u001b[0m \u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 670\u001b[0m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 671\u001b[0m \u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 672\u001b[0m \u001b[43m \u001b[49m\u001b[43mredirect\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 673\u001b[0m \u001b[43m \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 674\u001b[0m \u001b[43m \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 675\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 676\u001b[0m \u001b[43m \u001b[49m\u001b[43mretries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmax_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 677\u001b[0m \u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 678\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 679\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 681\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (ProtocolError, \u001b[38;5;167;01mOSError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m err:\n", "File \u001b[0;32m/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/urllib3/connectionpool.py:801\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m 799\u001b[0m e \u001b[38;5;241m=\u001b[39m ProtocolError(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mConnection aborted.\u001b[39m\u001b[38;5;124m\"\u001b[39m, e)\n\u001b[0;32m--> 801\u001b[0m retries \u001b[38;5;241m=\u001b[39m \u001b[43mretries\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mincrement\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 802\u001b[0m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43merror\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43me\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_pool\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_stacktrace\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msys\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexc_info\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m2\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[1;32m 803\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 804\u001b[0m retries\u001b[38;5;241m.\u001b[39msleep()\n", "File \u001b[0;32m/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/urllib3/util/retry.py:594\u001b[0m, in \u001b[0;36mRetry.increment\u001b[0;34m(self, method, url, response, error, _pool, _stacktrace)\u001b[0m\n\u001b[1;32m 593\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m new_retry\u001b[38;5;241m.\u001b[39mis_exhausted():\n\u001b[0;32m--> 594\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m MaxRetryError(_pool, url, error \u001b[38;5;129;01mor\u001b[39;00m ResponseError(cause))\n\u001b[1;32m 596\u001b[0m log\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mIncremented Retry for (url=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m): \u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m\"\u001b[39m, url, new_retry)\n", - "\u001b[0;31mMaxRetryError\u001b[0m: HTTPSConnectionPool(host='tucson-teststand.lsst.codes', port=443): Max retries exceeded with url: /nightreport/reports?limit=2 (Caused by ConnectTimeoutError(, 'Connection to tucson-teststand.lsst.codes timed out. (connect timeout=1.05)'))", + "\u001b[0;31mMaxRetryError\u001b[0m: HTTPSConnectionPool(host='tucson-teststand.lsst.codes', port=443): Max retries exceeded with url: /nightreport/reports?limit=2 (Caused by ConnectTimeoutError(, 'Connection to tucson-teststand.lsst.codes timed out. (connect timeout=1.05)'))", "\nDuring handling of the above exception, another exception occurred:\n", "\u001b[0;31mConnectTimeout\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[2], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m dash \u001b[38;5;241m=\u001b[39m Dashboard()\n\u001b[0;32m----> 2\u001b[0m score, working \u001b[38;5;241m=\u001b[39m \u001b[43mdash\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreport\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 3\u001b[0m \u001b[38;5;66;03m# On 9/12/2024 this gets two failed connects.\u001b[39;00m\n\u001b[1;32m 4\u001b[0m \u001b[38;5;66;03m# The are usdf: exposurelog/instruments,exposurelog/exposures\u001b[39;00m\n\u001b[1;32m 5\u001b[0m \u001b[38;5;66;03m# When usdfdev if fully functional, there should be zero failed connects (with both VPNs active).\u001b[39;00m\n", "File \u001b[0;32m~/develop/ts_logging_and_reporting/python/lsst/ts/logging_and_reporting/dashboard.py:70\u001b[0m, in \u001b[0;36mDashboard.report\u001b[0;34m(self, timeout, verbose)\u001b[0m\n\u001b[1;32m 68\u001b[0m server_all_good \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m 69\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m adapter \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39madapters:\n\u001b[0;32m---> 70\u001b[0m service \u001b[38;5;241m=\u001b[39m \u001b[43madapter\u001b[49m\u001b[43m(\u001b[49m\u001b[43mserver_url\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mserver\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 71\u001b[0m service\u001b[38;5;241m.\u001b[39mverbose \u001b[38;5;241m=\u001b[39m verbose\n\u001b[1;32m 72\u001b[0m stats, aag \u001b[38;5;241m=\u001b[39m service\u001b[38;5;241m.\u001b[39mcheck_endpoints(timeout\u001b[38;5;241m=\u001b[39mtimeout)\n", - "File \u001b[0;32m~/develop/ts_logging_and_reporting/python/lsst/ts/logging_and_reporting/source_adapters.py:326\u001b[0m, in \u001b[0;36mNightReportAdapter.__init__\u001b[0;34m(self, server_url, min_dayobs, max_dayobs, limit, verbose)\u001b[0m\n\u001b[1;32m 324\u001b[0m \u001b[38;5;66;03m# Load the data (records) we need from relevant endpoints\u001b[39;00m\n\u001b[1;32m 325\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmin_date:\n\u001b[0;32m--> 326\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhack_reconnect_after_idle\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 327\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstatus[\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprimary_endpoint] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_records()\n", - "File \u001b[0;32m~/develop/ts_logging_and_reporting/python/lsst/ts/logging_and_reporting/source_adapters.py:154\u001b[0m, in \u001b[0;36mSourceAdapter.hack_reconnect_after_idle\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 152\u001b[0m qparams \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mdict\u001b[39m(limit\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2\u001b[39m) \u001b[38;5;66;03m# API requires > 1 !\u001b[39;00m\n\u001b[1;32m 153\u001b[0m url \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mendpoint\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m?\u001b[39m\u001b[38;5;132;01m{\u001b[39;00murlencode(qparams)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m--> 154\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[43mrequests\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 155\u001b[0m validate_response(response, url)\n\u001b[1;32m 156\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m response\u001b[38;5;241m.\u001b[39mstatus_code\n", + "File \u001b[0;32m~/develop/ts_logging_and_reporting/python/lsst/ts/logging_and_reporting/source_adapters.py:333\u001b[0m, in \u001b[0;36mNightReportAdapter.__init__\u001b[0;34m(self, server_url, min_dayobs, max_dayobs, limit, verbose)\u001b[0m\n\u001b[1;32m 331\u001b[0m \u001b[38;5;66;03m# Load the data (records) we need from relevant endpoints\u001b[39;00m\n\u001b[1;32m 332\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmin_date:\n\u001b[0;32m--> 333\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhack_reconnect_after_idle\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 334\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstatus[\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprimary_endpoint] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_records()\n", + "File \u001b[0;32m~/develop/ts_logging_and_reporting/python/lsst/ts/logging_and_reporting/source_adapters.py:161\u001b[0m, in \u001b[0;36mSourceAdapter.hack_reconnect_after_idle\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 159\u001b[0m qparams \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mdict\u001b[39m(limit\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2\u001b[39m) \u001b[38;5;66;03m# API requires > 1 !\u001b[39;00m\n\u001b[1;32m 160\u001b[0m url \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mendpoint\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m?\u001b[39m\u001b[38;5;132;01m{\u001b[39;00murlencode(qparams)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m--> 161\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[43mrequests\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 162\u001b[0m validate_response(response, url)\n\u001b[1;32m 163\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m response\u001b[38;5;241m.\u001b[39mstatus_code\n", "File \u001b[0;32m/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/requests/api.py:73\u001b[0m, in \u001b[0;36mget\u001b[0;34m(url, params, **kwargs)\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget\u001b[39m(url, params\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 63\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124;03m\"\"\"Sends a GET request.\u001b[39;00m\n\u001b[1;32m 64\u001b[0m \n\u001b[1;32m 65\u001b[0m \u001b[38;5;124;03m :param url: URL for the new :class:`Request` object.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 70\u001b[0m \u001b[38;5;124;03m :rtype: requests.Response\u001b[39;00m\n\u001b[1;32m 71\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m---> 73\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mget\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/requests/api.py:59\u001b[0m, in \u001b[0;36mrequest\u001b[0;34m(method, url, **kwargs)\u001b[0m\n\u001b[1;32m 55\u001b[0m \u001b[38;5;66;03m# By using the 'with' statement we are sure the session is closed, thus we\u001b[39;00m\n\u001b[1;32m 56\u001b[0m \u001b[38;5;66;03m# avoid leaving sockets open which can trigger a ResourceWarning in some\u001b[39;00m\n\u001b[1;32m 57\u001b[0m \u001b[38;5;66;03m# cases, and look like a memory leak in others.\u001b[39;00m\n\u001b[1;32m 58\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m sessions\u001b[38;5;241m.\u001b[39mSession() \u001b[38;5;28;01mas\u001b[39;00m session:\n\u001b[0;32m---> 59\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msession\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/requests/sessions.py:589\u001b[0m, in \u001b[0;36mSession.request\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m 584\u001b[0m send_kwargs \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m 585\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtimeout\u001b[39m\u001b[38;5;124m\"\u001b[39m: timeout,\n\u001b[1;32m 586\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mallow_redirects\u001b[39m\u001b[38;5;124m\"\u001b[39m: allow_redirects,\n\u001b[1;32m 587\u001b[0m }\n\u001b[1;32m 588\u001b[0m send_kwargs\u001b[38;5;241m.\u001b[39mupdate(settings)\n\u001b[0;32m--> 589\u001b[0m resp \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprep\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43msend_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 591\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m resp\n", "File \u001b[0;32m/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/requests/sessions.py:703\u001b[0m, in \u001b[0;36mSession.send\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m 700\u001b[0m start \u001b[38;5;241m=\u001b[39m preferred_clock()\n\u001b[1;32m 702\u001b[0m \u001b[38;5;66;03m# Send the request\u001b[39;00m\n\u001b[0;32m--> 703\u001b[0m r \u001b[38;5;241m=\u001b[39m \u001b[43madapter\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 705\u001b[0m \u001b[38;5;66;03m# Total elapsed time of the request (approximately)\u001b[39;00m\n\u001b[1;32m 706\u001b[0m elapsed \u001b[38;5;241m=\u001b[39m preferred_clock() \u001b[38;5;241m-\u001b[39m start\n", "File \u001b[0;32m/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/requests/adapters.py:688\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m 685\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(e\u001b[38;5;241m.\u001b[39mreason, ConnectTimeoutError):\n\u001b[1;32m 686\u001b[0m \u001b[38;5;66;03m# TODO: Remove this in 3.0.0: see #2811\u001b[39;00m\n\u001b[1;32m 687\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(e\u001b[38;5;241m.\u001b[39mreason, NewConnectionError):\n\u001b[0;32m--> 688\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m ConnectTimeout(e, request\u001b[38;5;241m=\u001b[39mrequest)\n\u001b[1;32m 690\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(e\u001b[38;5;241m.\u001b[39mreason, ResponseError):\n\u001b[1;32m 691\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m RetryError(e, request\u001b[38;5;241m=\u001b[39mrequest)\n", - "\u001b[0;31mConnectTimeout\u001b[0m: HTTPSConnectionPool(host='tucson-teststand.lsst.codes', port=443): Max retries exceeded with url: /nightreport/reports?limit=2 (Caused by ConnectTimeoutError(, 'Connection to tucson-teststand.lsst.codes timed out. (connect timeout=1.05)'))" + "\u001b[0;31mConnectTimeout\u001b[0m: HTTPSConnectionPool(host='tucson-teststand.lsst.codes', port=443): Max retries exceeded with url: /nightreport/reports?limit=2 (Caused by ConnectTimeoutError(, 'Connection to tucson-teststand.lsst.codes timed out. (connect timeout=1.05)'))" ] } ], diff --git a/python/lsst/ts/logging_and_reporting/all_sources.py b/python/lsst/ts/logging_and_reporting/all_sources.py index d065055..cc61979 100644 --- a/python/lsst/ts/logging_and_reporting/all_sources.py +++ b/python/lsst/ts/logging_and_reporting/all_sources.py @@ -25,12 +25,13 @@ import datetime as dt import itertools from collections import Counter, defaultdict +from urllib.parse import urlencode import lsst.ts.logging_and_reporting.almanac as alm import lsst.ts.logging_and_reporting.efd as efd import lsst.ts.logging_and_reporting.source_adapters as sad +import lsst.ts.logging_and_reporting.utils as ut import pandas as pd -from lsst.ts.logging_and_reporting.utils import hhmmss class AllSources: @@ -48,7 +49,7 @@ def __init__( ): self.verbose = verbose self.exclude_instruments = exclude_instruments or [] - + ut.tic() # Load data for all needed sources for the selected dayobs range. self.nig_src = sad.NightReportAdapter( server_url=server_url, @@ -87,6 +88,8 @@ def __init__( self.min_date = self.nig_src.min_date self.max_dayobs = self.nig_src.max_dayobs self.min_dayobs = self.nig_src.min_dayobs + if verbose: + print(f"Loaded data from sources in {ut.toc():0.1f} seconds") # END init @@ -122,8 +125,8 @@ async def night_tally_observation_gaps(self, verbose=False): if len(used_instruments) == 0: return { "": { - "Total night": hhmmss(total_observable_hrs), - "Idle time": hhmmss(total_observable_hrs), + "Total night": ut.hhmmss(total_observable_hrs), + "Idle time": ut.hhmmss(total_observable_hrs), } } if verbose: @@ -146,21 +149,11 @@ async def night_tally_observation_gaps(self, verbose=False): # f"using date range {self.min_date} to {self.max_date}. " # ) + # Merlin might add this to consdb or efd .... eventually num_slews = 0 total_slew_seconds = 0 mean_slew = 0 - # ## TODO remove - # if targets.empty: - # num_slews = 0 - # total_slew_seconds = 0 - # mean_slew = 0 - # else: - # num_slews = targets[["slewTime"] - # ].astype(bool).sum(axis=0).squeeze() - # total_slew_seconds = targets[["slewTime"]].sum().squeeze() - # mean_slew = slew_hrs / num_slews - # per Merlin: There is no practical way to get actual detector read # time. He has done some experiments and inferred that it is # 2.3 seconds. He recommends hardcoding the value. @@ -188,17 +181,17 @@ async def night_tally_observation_gaps(self, verbose=False): ) instrument_tally[instrument] = { - "Total Night": hhmmss(total_observable_hrs), # (a) - "Total Exposure": hhmmss(exposure_hrs), # (b) - "Slew time(1)": hhmmss(slew_hrs), # (g) - "Readout time(2)": hhmmss(detector_hrs), # (e) + "Total Night": ut.hhmmss(total_observable_hrs), # (a) + "Total Exposure": ut.hhmmss(exposure_hrs), # (b) + "Slew time(1)": ut.hhmmss(slew_hrs), # (g) + "Readout time(2)": ut.hhmmss(detector_hrs), # (e) "Time loss to fault": "NA", "Time loss to weather": "NA", - "Idle time": hhmmss(idle_hrs), # (i=a-b-e-g) + "Idle time": ut.hhmmss(idle_hrs), # (i=a-b-e-g) "Number of exposures": num_exposures, # (c) "Mean readout time": mean_detector_hrs, # (f=e/c) "Number of slews(1)": num_slews, # (d) - "Mean Slew time(1)": hhmmss(mean_slew), # (g/d) + "Mean Slew time(1)": ut.hhmmss(mean_slew), # (g/d) } # Composition to combine Exposure and Efd (blackboard) @@ -268,9 +261,6 @@ def lost_type(rec): day_tl[day]["weather"] = sum( [r["time_lost"] for r in day_grp if "weather" == r["time_lost_type"]] ) - # TODO remove - # for type, type_grp in itertools.groupby(day_grp, key=lost_type): - # day_tl[day][type] = sum([r["time_lost"] for r in type_grp]) return day_tl def get_observation_gaps(self): @@ -324,38 +314,37 @@ def get_slews(self): pass - # TODO remove - # def tally_exposure_flags(self, instrument): - # # ... make sure we have counts for all three - # tally = Counter(good=0, questionable=0, junk=0) - # tally.update([r["exposure_flag"] - # for r in self.exp_src.messages[instrument]]) - # df = pd.DataFrame.from_dict(tally, orient="index").T - # return df # .style.hide(axis="index") - @property def urls(self): return self.nar_src.urls | self.exp_src.urls - # TODO remove print DEBUG - def flag_count_exposures(self, instrument, fieldname): + def flag_count_exposures(self, instrument, field_name): + def gen_link(field_value, text): + oneday = dt.timedelta(days=1) + qparams = { + "day_obs": ut.datetime_to_dayobs(self.max_date - oneday), + "number_of_days": (self.max_date - self.min_date).days, + "instrument": instrument, + field_name: field_value, + } + url = f"{self.server_url}/times-square/github/" + url += "lsst-ts/ts_logging_and_reporting/ExposureDetail" + url += f"?{urlencode(qparams)}" + return f"{field_name}" + records = self.exp_src.exposures[instrument] - # fieldname: observation_type, observation_reason, science_program - field_values = {r[fieldname] for r in records} + # field_name: observation_type, observation_reason, science_program + field_values = {r[field_name] for r in records} # Values of rec["exposure_flag"] eflag_values = ["good", "questionable", "junk", "unknown"] - # print(f'DEBUG flag_count_exposures {fieldname=}') - # print(f'DEBUG flag_count_exposures {field_values=}') - # print(f'DEBUG flag_count_exposures {eflag_values=}') table_recs = dict() for field in field_values: for eflag in eflag_values: - # print(f'DEBUG flag_count {field=} {eflag=}') # Initialize to zeros counter = Counter({f: 0 for f in eflag_values}) counter.update( - [r["exposure_flag"] for r in records if r[fieldname] == field] + [r["exposure_flag"] for r in records if r[field_name] == field] ) counter.update(dict(total=counter.total())) table_recs[field] = counter diff --git a/python/lsst/ts/logging_and_reporting/dashboard.py b/python/lsst/ts/logging_and_reporting/dashboard.py index bace03e..7d35186 100644 --- a/python/lsst/ts/logging_and_reporting/dashboard.py +++ b/python/lsst/ts/logging_and_reporting/dashboard.py @@ -10,7 +10,7 @@ class Dashboard: # TODO Move to its own file (utils.py). we need for any of our sources. """ - timeout = (1.05, 2) # connect, read (seconds) + timeout = (5.05, 2) # connect, read (seconds) envs = dict( # key, server summit="https://summit-lsp.lsst.codes", diff --git a/python/lsst/ts/logging_and_reporting/reports.py b/python/lsst/ts/logging_and_reporting/reports.py index a04021c..d0547ee 100644 --- a/python/lsst/ts/logging_and_reporting/reports.py +++ b/python/lsst/ts/logging_and_reporting/reports.py @@ -23,7 +23,7 @@ from urllib.parse import urlparse import pandas as pd -from IPython.display import Markdown, display +from IPython.display import HTML, Markdown, display def md(markdown_str, color=None, background=None): @@ -76,6 +76,16 @@ def mdfragmentlink(url, remove="!/"): return f"[{urlparse(url).fragment.replace(remove, '')}]({url})" +def display_error(text, fgcolor="black", bgcolor="red", size="1em"): + style = "" + style += f"color: {fgcolor}; " + style += f"background-color: {bgcolor}; " + style += f"font-size: {size}; " + + elem = "pre" + return display(HTML(f"<{elem} {style=}>{text}")) + + def htmlcode(text, fgcolor="black", bgcolor="white", size="1em", left=0): style = "" style += f"color: {fgcolor}; " diff --git a/python/lsst/ts/logging_and_reporting/source_adapters.py b/python/lsst/ts/logging_and_reporting/source_adapters.py index e176996..aa646d3 100644 --- a/python/lsst/ts/logging_and_reporting/source_adapters.py +++ b/python/lsst/ts/logging_and_reporting/source_adapters.py @@ -38,18 +38,18 @@ import copy import datetime as dt import itertools +import traceback from abc import ABC from collections import defaultdict from urllib.parse import urlencode -import lsst.ts.logging_and_reporting.exceptions as ex import lsst.ts.logging_and_reporting.parse_message as pam import lsst.ts.logging_and_reporting.reports as rep import lsst.ts.logging_and_reporting.utils as ut import pandas as pd import requests -MAX_CONNECT_TIMEOUT = 3.1 # seconds +MAX_CONNECT_TIMEOUT = 5.1 # seconds MAX_READ_TIMEOUT = 180 # seconds summit = "https://summit-lsp.lsst.codes" usdf = "https://usdf-rsp-dev.slac.stanford.edu" @@ -67,21 +67,30 @@ def all_endpoints(server): return list(endpoints) -def validate_response(response, endpoint_url, verbose=False): +def invalid_response(response, endpoint_url, timeout=None, verbose=False): + """Return error string if invalid, else return None""" if verbose: - print(f"DEBUG validate_response {endpoint_url=}") + print(f"DEBUG invalid_response {endpoint_url=}") if response.ok: - return True + return None else: - msg = f"{endpoint_url=} {response.status_code=} {response.reason}" + msg = f"{endpoint_url=} {timeout=} " + msg += f"{response.status_code=} {response.reason}" try: msg += f" {response.json()}" except Exception as err: msg += f" {response.text}; {str(err)}" + if verbose: - print(f"DEBUG validate_response ERROR {msg=}") + print(f"DEBUG invalid_response ERROR {msg=}") - raise ex.StatusError(msg) + # We want to continue when one source gets errors because + # other sources may not. + # raise ex.StatusError(msg) + rep.display_error(msg) + msg2 = "".join(traceback.format_stack(limit=4)) + rep.display_error(msg2) + return msg class SourceAdapter(ABC): @@ -152,7 +161,7 @@ def hack_reconnect_after_idle(self): qparams = dict(limit=2) # API requires > 1 ! url = f"{endpoint}?{urlencode(qparams)}" response = requests.get(url, timeout=self.timeout) - validate_response(response, url) + invalid_response(response, url, timeout=self.timeout) return response.status_code def get_status(self, endpoint=None): @@ -235,12 +244,14 @@ def check_endpoints(self, timeout=None, verbose=True): for ep in self.endpoints: url = f"{self.server}/{self.service}/{ep}" try: - r = requests.get(url, timeout=(timeout or self.timeout)) - validate_response(r, url) + to = timeout or self.timeout + response = requests.get(url, timeout=to) + if invalid_response(response, url, timeout=to): + url_http_status_code[url] = "GET error" except Exception: url_http_status_code[url] = "GET error" else: - url_http_status_code[url] = r.status_code + url_http_status_code[url] = response.status_code return url_http_status_code, all( [v == 200 for v in url_http_status_code.values()] ) @@ -422,7 +433,8 @@ def get_records( print(f"DBG get_records qstr: {urlencode(qparams)}") url = f"{endpoint}?{urlencode(qparams)}" response = requests.get(url, timeout=self.timeout) - validate_response(response, url) + if invalid_response(response, url, timeout=self.timeout): + break page = response.json() if self.verbose: print(f"DBG get_records {len(page)=} {len(recs)=}") @@ -631,7 +643,8 @@ def get_records( if self.verbose: print(f"Using {url=}") response = requests.get(url, timeout=self.timeout) - validate_response(response, url) + if invalid_response(response, url, timeout=self.timeout): + break page = response.json() recs += page if len(page) < self.limit: @@ -737,10 +750,6 @@ def add_exposure_flag_to_exposures(self): print(f"add_exposure_flag_to_exposures {rec=}") new_recs.append(new_rec) self.exposures[instrument] = new_recs - print( - f"DEBUG add_exposure_flag {instrument=} " - f'{set([r["exposure_flag"] for r in new_recs])}' - ) return count @property @@ -869,12 +878,14 @@ def check_endpoints(self, timeout=None, verbose=True): qstr = "?instrument=na" if ep == "exposures" else "" url = f"{self.server}/{self.service}/{ep}{qstr}" try: - r = requests.get(url, timeout=to) - validate_response(r, url) + response = requests.get(url, timeout=to) + err = invalid_response(response, url, timeout=to) + if err: + url_http_status_code[url] = err except Exception: url_http_status_code[url] = "GET error" else: - url_http_status_code[url] = r.status_code + url_http_status_code[url] = response.status_code allgood_p = all([v == 200 for v in url_http_status_code.values()]) return url_http_status_code, allgood_p @@ -883,9 +894,17 @@ def get_instruments(self): recs = dict(dummy=[]) error = None # try: - r = requests.get(url, timeout=self.timeout) - validate_response(r, url) - recs = r.json() + response = requests.get(url, timeout=self.timeout) + err = invalid_response(response, url, timeout=self.timeout) + if err: + status = dict( + endpoint_url=url, + number_of_records=len(recs), + error=err, + ) + return status + + recs = response.json() # except Exception as err: # error = str(err) # else: @@ -930,7 +949,8 @@ def get_exposures(self, instrument): print(f"DBG get_exposures qstr: {urlencode(qparams)}") url = f"{endpoint}?{urlencode(qparams)}" response = requests.get(url, timeout=self.timeout) - validate_response(response, url) + if invalid_response(response, url, timeout=self.timeout): + break page = response.json() recs += page if self.verbose: @@ -1008,7 +1028,8 @@ def get_records( print(f"DBG get_records qstr: {urlencode(qparams)}") url = f"{endpoint}?{urlencode(qparams)}" response = requests.get(url, timeout=self.timeout) - validate_response(response, url) + if invalid_response(response, url, timeout=self.timeout): + break page = response.json() recs += page if len(page) < self.limit: