diff --git a/notebooks_tsqr/NightLog.ipynb b/notebooks_tsqr/NightLog.ipynb index 0702bdf..ad95766 100644 --- a/notebooks_tsqr/NightLog.ipynb +++ b/notebooks_tsqr/NightLog.ipynb @@ -26,7 +26,7 @@ "# day_obs values: TODAY, YESTERDAY, YYYY-MM-DD\n", "# Report on observing nights that start upto but not included this day.\n", "#!day_obs = '2024-09-25' # Value to use for local testing (Summit)\n", - "day_obs = \"YESTERDAY\" # TODO Change to 'YESTERDAY' to test with default before push\n", + "day_obs = \"2024-11-09\" # TODO Change to 'YESTERDAY' to test with default before push\n", "\n", "# Total number of days of data to display (ending on day_obs)\n", "number_of_days = \"1\" # TODO Change to '1' to test with default before push\n", @@ -112,28 +112,28 @@ "output_type": "stream", "text": [ "DBG get_records endpoint='https://usdf-rsp-dev.slac.stanford.edu/nightreport/reports'\n", - "DBG get_records qstr: is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=100&min_day_obs=20241110&max_day_obs=20241111\n", - "DBG get_records len(page)=2 len(recs)=0\n", - "DBG get_records-2 len(page)=2 len(recs)=2\n", - "DBG get_records-2 len(self.records)=2 status={'endpoint_url': 'https://usdf-rsp-dev.slac.stanford.edu/nightreport/reports?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=100&min_day_obs=20241110&max_day_obs=20241111', 'number_of_records': 2, 'error': None}\n", + "DBG get_records qstr: is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=100&min_day_obs=20241109&max_day_obs=20241110\n", + "DBG get_records len(page)=3 len(recs)=0\n", + "DBG get_records-2 len(page)=3 len(recs)=3\n", + "DBG get_records-2 len(self.records)=3 status={'endpoint_url': 'https://usdf-rsp-dev.slac.stanford.edu/nightreport/reports?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=100&min_day_obs=20241109&max_day_obs=20241110', 'number_of_records': 3, 'error': None}\n", "DBG get_exposures endpoint='https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures'\n", - "DBG get_exposures qstr: registry=1&instrument=LSSTComCamSim&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241110&max_day_obs=20241111\n", + "DBG get_exposures qstr: registry=1&instrument=LSSTComCamSim&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241109&max_day_obs=20241110\n", "DBG get_exposures len(page)=0 len(recs)=0\n", "DBG get_exposures endpoint='https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures'\n", - "DBG get_exposures qstr: registry=1&instrument=LATISS&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241110&max_day_obs=20241111\n", + "DBG get_exposures qstr: registry=1&instrument=LATISS&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241109&max_day_obs=20241110\n", "DBG get_exposures len(page)=0 len(recs)=0\n", "DBG get_exposures endpoint='https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures'\n", - "DBG get_exposures qstr: registry=1&instrument=LSSTComCam&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241110&max_day_obs=20241111\n", - "DBG get_exposures len(page)=239 len(recs)=239\n", + "DBG get_exposures qstr: registry=1&instrument=LSSTComCam&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241109&max_day_obs=20241110\n", + "DBG get_exposures len(page)=292 len(recs)=292\n", "DBG get_exposures endpoint='https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures'\n", - "DBG get_exposures qstr: registry=1&instrument=LSSTCam&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241110&max_day_obs=20241111\n", - "DBG get_exposures len(page)=0 len(recs)=0\n", + "DBG get_exposures qstr: registry=1&instrument=LSSTCam&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241109&max_day_obs=20241110\n", + "DBG get_exposures len(page)=1338 len(recs)=1338\n", "DBG get_records: endpoint='https://usdf-rsp-dev.slac.stanford.edu/exposurelog/messages'\n", - "DBG get_records qstr: is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=2500&min_day_obs=20241110&max_day_obs=20241111\n", - "NarrativeLogAdapter(server_url='https://usdf-rsp-dev.slac.stanford.edu', max_dayobs='2024-11-11', min_dayobs='2024-11-10', limit=None\n", + "DBG get_records qstr: is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=2500&min_day_obs=20241109&max_day_obs=20241110\n", + "NarrativeLogAdapter(server_url='https://usdf-rsp-dev.slac.stanford.edu', max_dayobs='2024-11-10', min_dayobs='2024-11-09', limit=None\n", "Using endpoint='https://usdf-rsp-dev.slac.stanford.edu/narrativelog/messages'\n", - "Using url='https://usdf-rsp-dev.slac.stanford.edu/narrativelog/messages?is_human=either&is_valid=true&order_by=-date_added&offset=0&limit=1000&min_date_added=2024-11-10T00%3A00%3A00&max_date_added=2024-11-11T00%3A00%3A00'\n", - "Loaded data from sources in 4.0 seconds\n" + "Using url='https://usdf-rsp-dev.slac.stanford.edu/narrativelog/messages?is_human=either&is_valid=true&order_by=-date_added&offset=0&limit=1000&min_date_added=2024-11-09T00%3A00%3A00&max_date_added=2024-11-10T00%3A00%3A00'\n", + "Loaded data from sources in 5.6 seconds\n" ] } ], @@ -163,7 +163,7 @@ { "data": { "text/markdown": [ - "# Showing data for 2024-11-10 to 2024-11-11 for LSSTComCamSim, LATISS, LSSTComCam, LSSTCam" + "# Showing data for 2024-11-09 to 2024-11-10 for LSSTComCamSim, LATISS, LSSTComCam, LSSTCam" ], "text/plain": [ "" @@ -218,7 +218,7 @@ { "data": { "text/markdown": [ - "### Telescope: AuxTel" + "### Telescope: Simonyi" ], "text/plain": [ "" @@ -231,7 +231,23 @@ "data": { "text/markdown": [ "```\n", + "We started around 12:00h CLT powering up the rotator and hexapods. Then we did the M1M3 HP breakaway test passing all HPs. We ran this from EUI to run all HPs at the same time. We ran bump test with the script, and 5 FAs failed (215, 227, 235, 409's primaries and 330 secondary). M1M3 faulted when we put it to disabled state to switch to EUI mode to run the individual FA bump test, it is known issue. OBS-652. We waited air pressure to be lowered to Enable back. Meanwhile we ran hexapod warmups, and comcam usdf checkout. \n", + "We re-ran the individual bump test, FA 330 passed, 227, 235, 409 seemed to be within the range, but Petr identified FA 215 as bad and he asked to close its valve from the M1M3 cell, Brian helped us to performed this action in the cell.\n", + "During the hexapod warmups, M2 passed without issues, but then during CamHex warm-up, M2Hex showed oscillations in X, Y, and Z and continued oscillating post CamHex warmup. Brian checked this via the EUI.\n", + "We turned off OSS, put LOTO on, for Guillem and Andy to turn on the laser tracker. M1M3 faulted because it was enabled while they entered the cell. After recovering M1M3, we disabled the FA215, Brian turned off the air valve for it. Then we checked the dome moving, telemetry and shutter. After raising M1M3 with FA215 disabled, after that we went through TMA checkout. \n", + "We opened the mirror cover, ran BLOCK-T88 alignment test, turned off the laser tracker. We closed the mirror cover again for the dome shutter opening. We also kept the HVAC setpoint 10°C, turned off HVAC, turn the VEC-04 on, opened the dome shutter at 20:00h CLT, watching the temperature. We opened the mirror cover, and made dome to follow.\n", + "We moved TMA and dome to be 60 deg away from the sun. We tried to take sample image before the twilight flat script, then Camhexapod faulted, due to u band offset not applied to the hexapod. We tried to take images without recovering hexapod, but it faulted MTMount and MTPtg. We updated the hexapod configuration, recover all, took some flats and darks. It was too late to run the BLOCK-215. Then we applied the best offset from last night, with set_dof and ran T235. \n", + "We took some images in between and we found that we were pointing close to the Moon (images 34, 35, 36, 37), we did a 30-degree shift to avoid interference and proceeded with T249.\n", + "Applying bending mode 1 and 3 to M2, ran T270 with recycling MTAOS to compare the +-90 degrees. T273 is same test as T270 but for the M1M3. Also during this time Brian adjusted the azimuth limit to 220°.\n", + "Handing over to late night shift...\n", + "T227\n", + "\n", + "\n", + "--------- AOS Summary ---------\n", "\n", + "To answer the big question—no, we didn't break the image quality record tonight, as poor seeing prevailed, with SOAR measurements reaching around 1.85 arcsec. Despite the challenging conditions, we successfully resolved the rotation issue identified yesterday with the bending modes and closed the loop for the first time using all 20 M2 bending modes. We also managed to close the loop with M1M3, using its first 9 modes. We then took at some NGC 55 in different bands, which would be interesting to reduce if possible tomorrow.\n", + "\n", + "When we attempted all 20 modes for M1M3, we encountered force clipping due to differing stress limits between MTAOS and M1M3. We started attempting to fix the tilts and hexapod misalignments, but midway through, we closed the dome due to temperature shifts. Rapid analysis fell over during the night, due to images being passed to it by standardscripts before they were ingested. The issue was resolved, but RA kept slow for the rest of he night. We later reopened, giving us a brief 45 minutes on sky which were dedicated to science validation.\n", "```" ], "text/plain": [ @@ -244,7 +260,16 @@ { "data": { "text/markdown": [ - "Telescope Status: " + "Telescope Status: MTM1M3 ENABLED - lowered and enabled to prevent to go to fault\n", + "MTM2 ENABLED in closed loop\n", + "MTAOS STANDBY\n", + "MTDome DISABLED\n", + "MTDomeTrajectory STANDBY\n", + "MTMount DISABLED\n", + "MTHexapods in STANDBY w/ cabinets and PXIs powered off\n", + "MTPtg ENABLED\n", + "MTRotator needs to be back at 0º (when parking at horizon), in STANDBY w/ cabinets and PXIs powered off\n", + "CCCamera, CCOODS, CCHeaderService in ENABLED" ], "text/plain": [ "" @@ -256,7 +281,7 @@ { "data": { "text/markdown": [ - "*Authors: *" + "*Authors: HyeYun Park, Gonzalo Aravena, Guillem Megias Homar, Brian Stalder, Andrew Connolly, Carlos Morales, Kate Napier*" ], "text/plain": [ "" @@ -268,7 +293,66 @@ { "data": { "text/markdown": [ - "### Telescope: Simonyi" + "```\n", + "We started around 12:00h CLT powering up the rotator and hexapods. Then we did the M1M3 HP breakaway test passing all HPs. We ran this from EUI to run all HPs at the same time. We ran bump test with the script, and 5 FAs failed (215, 227, 235, 409's primaries and 330 secondary). M1M3 faulted when we put it to disabled state to switch to EUI mode to run the individual FA bump test, it is known issue. OBS-652. We waited air pressure to be lowered to Enable back. Meanwhile we ran hexapod warmups, and comcam usdf checkout. \n", + "We re-ran the individual bump test, FA 330 passed, 227, 235, 409 seemed to be within the range, but Petr identified FA 215 as bad and he asked to close its valve from the M1M3 cell, Brian helped us to performed this action in the cell.\n", + "During the hexapod warmups, M2 passed without issues, but then during CamHex warm-up, M2Hex showed oscillations in X, Y, and Z and continued oscillating post CamHex warmup. Brian checked this via the EUI.\n", + "We turned off OSS, put LOTO on, for Guillem and Andy to turn on the laser tracker. M1M3 faulted because it was enabled while they entered the cell. After recovering M1M3, we disabled the FA215, Brian turned off the air valve for it. Then we checked the dome moving, telemetry and shutter. After raising M1M3 with FA215 disabled, after that we went through TMA checkout. \n", + "We opened the mirror cover, ran BLOCK-T88 alignment test, turned off the laser tracker. We closed the mirror cover again for the dome shutter opening. We also kept the HVAC setpoint 10°C, turned off HVAC, turn the VEC-04 on, opened the dome shutter at 20:00h CLT, watching the temperature. We opened the mirror cover, and made dome to follow.\n", + "We moved TMA and dome to be 60 deg away from the sun. We tried to take sample image before the twilight flat script, then Camhexapod faulted, due to u band offset not applied to the hexapod. We tried to take images without recovering hexapod, but it faulted MTMount and MTPtg. We updated the hexapod configuration, recover all, took some flats and darks. It was too late to run the BLOCK-215. Then we applied the best offset from last night, with set_dof and ran T235. \n", + "We took some images in between and we found that we were pointing close to the Moon (images 34, 35, 36, 37), we did a 30-degree shift to avoid interference and proceeded with T249.\n", + "Applying bending mode 1 and 3 to M2, ran T270 with recycling MTAOS to compare the +-90 degrees. T273 is same test as T270 but for the M1M3. Also during this time Brian adjusted the azimuth limit to 220°.\n", + "Handing over to late night shift...\n", + "T227\n", + "\n", + "\n", + "Around 1:45am, the dew point temperature was too close to the outside temperature. The humidity was 65%. We followed the recommended weather closure limits and decided to close the dome. We continued to monitor the weather conditions. Meanwhile we prepared the TMA for the next test which was a series of u-band images. \n", + "After approximately 30 minutes, we evaluated that the temperatures had stabilized and the humidity had significantly dropped. We reopened the dome and continued to run the blocks until 5am. We closed the dome and performed BLOCK-T241 and BLOCK-T220. \n", + "\n", + "\n", + "--------- AOS Summary ---------\n", + "\n", + "To answer the big question—no, we didn't break the image quality record tonight, as poor seeing prevailed, with SOAR measurements reaching around 1.85 arcsec. Despite the challenging conditions, we successfully resolved the rotation issue identified yesterday with the bending modes and closed the loop for the first time using all 20 M2 bending modes. We also managed to close the loop with M1M3, using its first 9 modes. We then took at some NGC 55 in different bands, which would be interesting to reduce if possible tomorrow.\n", + "\n", + "When we attempted all 20 modes for M1M3, we encountered force clipping due to differing stress limits between MTAOS and M1M3. We started attempting to fix the tilts and hexapod misalignments, but midway through, we closed the dome due to temperature shifts. Rapid analysis fell over during the night, due to images being passed to it by standardscripts before they were ingested. The issue was resolved, but RA kept slow for the rest of he night. We later reopened, giving us a brief 45 minutes on sky which were dedicated to science validation.\n", + "```" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "Telescope Status: MTM1M3 ENABLED - lowered and disabled\n", + "MTM2 ENABLED in closed loop\n", + "MTAOS STANDBY\n", + "MTDome DISABLED\n", + "MTDomeTrajectory STANDBY\n", + "MTMount DISABLED\n", + "MTHexapods in STANDBY w/ cabinets and PXIs powered off\n", + "MTPtg ENABLED\n", + "MTRotator needs to be back at 0º (when parking at horizon), in STANDBY w/ cabinets and PXIs powered off\n", + "CCCamera, CCOODS, CCHeaderService in ENABLED\n", + "\n", + "Mirror lowered\n", + "HVAC set point = 10 deg C\n", + "FCU Fans still on " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "*Authors: HyeYun Park, Gonzalo Aravena, Guillem Megias Homar, Brian Stalder, Andrew Connolly, Carlos Morales, Kate Napier*" ], "text/plain": [ "" @@ -281,13 +365,15 @@ "data": { "text/markdown": [ "```\n", - "In the morning, there was a power outage, and by 8:10 AM, we switched to backup power. This caused the OSS to go into fault, the PS to turn off, alarms on the chillers, and required a restart of compressor #2. While bringing up the drives and PXI for the hexapods and rotator, we cleared the interlock from the chillers, reset the PS, and restored the OSS. \n", - "We ran M2hexapod warm-ups, enabling the hexapod under test while keeping the other disabled. \n", - "Around 13:20h CLT, Jaques noticed that compressor #2 was not working properly, and HPs air compressors were not reaching pressures over 117 PSI, so we restarted it, and after the restart, M1M3 pressure reached 125 PSI but just for couple of minutes, then it was oscillating between 116 and 117 PSI, so we decided to turn on the LaserTracker first. To do this, we had to turn off the OSS, LOTO on, M1M3 standby, laser tracker on, then put back to M1M3 Enabled, LOTO off, and OSS on, then M1M3 pressure reached 125 PSI again.\n", - "We then ran the CamHex warmup and once this finished, we started the HP breakaway test. M1M3 HP breakaway test done with EUI to run all HPs at the same time.\n", - "We ran the bump test after this. FA 235 P, 238 Y, 305 Y failed bump test but when we re-ran individually, we saw 235 and 238 are slightly overshooting and 305 passed. We raised the mirror. FA 215 that failed yesterday was kept disabled.\n", - "During the bump test, we unparked the TMA to elevation 80 using EUI and conducted the TMA and dome parallel tests while taking sample acquisition images in the r and u bands with lights off. Brian adjusted the default system speed to 5%. An alignment test (T88) was performed, and the laser tracker was turned off. \n", - "To avoid the Sun, the dome was moved to 140°. Earthquake happened and MTMount and M1M3 went into fault, with M1M3 Force controller X Moment Limit error. We recovered MTMount, M1M3, M1M3TS, and MTRotator. Clearing interlocks were needed for M1M3. HVAC was turned off, VEC04 on, we moved dome to az=220 so that we can reach the dome cabinet. Then the shutter and mirror covers were opened. After opening the shutter we moved dome away from sun again az=110. We ran track_target to check it’s working since we recovered many systems, with same az but to el=60. Twilight flats in the U band began at 20:25h CLT (~-2.5° Sun’s altitude), followed by darks, and an AOS test (BLOCK-T249, T248) was performed. During scheduled script run (resume.py), we had bad gateway issue (OBS-614, 623). Re-resuming 3rd time worked. We stopped the script to focus better, re-ran the script. During this, we got a azimuth drive failure with the MTDome and exitFault didn’t clear the error. Tiago recycled MTDome, reset cRIO, and re-home and defined the zeroAz there to recover the dome. While dome is being fixed, we took images and ran some test that could be done with fixed dome position (still tracking). We are back to on-sky but missed the good timing for T248. Moved onto BLOCK-Txx with late night shift.\n", + "We started around 12:00h CLT powering up the rotator and hexapods. M1M3 HP breakaway test done passing all HPs. We ran this from EUI to run all HPs at the same time. We ran bump test with the script, and 5 FAs failed (215, 227, 235, 409's primaries and 330 secondary). M1M3 faulted when we put it to disabled state to switch to EUI mode to run the individual FA bump test, it is known issue. OBS-652. We waited air pressure to be lowered to Enable back. Meanwhile we ran hexapod warmups, and comcam usdf checkout. \n", + "We re-ran the individual bump test, FA 330 passed, 227, 235, 409 seemed to be within the range, but Petr identified FA 215 as bad and he asked to close its valve from the M1M3 cell, Brian agreed to performed this action in the cell. During the hexapod warmups, M2 passed without issues, but then during CamHex warm-up, M2Hex showed oscillations in X, Y, and Z and continued oscillating post CamHex warmup. Brian checked this via the EUI. \n", + "After that, we turned off OSS, put LOTO on, for Guillem and Andy to turn on the laser tracker. M1M3 faulted because it was enabled while they entered the cell. After recovering M1M3, we disabled the FA215, Brian turned off the air valve for it. \n", + "We checked the dome moving, telemetry and shutter. After raising M1M3 with FA215 disabled, we went through TMA checkout. \n", + "We opened the mirror cover, ran BLOCK-T88 alignment test, turned off the laser tracker. We closed the mirror cover again for the dome shutter opening. We kept the HVAC setpoint 10°C, turned off HVAC, turn the VEC-04 on, opened the dome shutter at 20:00h CLT, watching the temperature. We opened the mirror cover, and made dome to follow. We moved TMA and dome to be 60 deg away from the sun. We tried to take sample image before the twilight flat script, then Camhexapod faulted, due to u band offset not applied to the hexapod. We tried to take images without recovering hexapod, but it faulted MTMount and MTPtg. We updated the hexapod configuration, recover all, took some flats and darks. It was too late to run the BLOCK-215.\n", + "We applied the best offset from last night, with set_dof and ran T235. Then we took some images in between, found out we were pointing the Moon, we did a 30-degree shift to avoid interference and proceeded with T249.\n", + "Applying bending mode 1 and 3 to M2, ran T270 with recycling MTAOS to compare the +-90 degrees. T273 is same test as T270 but for the M1M3. Also during this time Brian adjusted the azimuth limit to 220°.\n", + "Handing over to late night shift...\n", + "T227\n", "```" ], "text/plain": [ @@ -312,7 +398,7 @@ { "data": { "text/markdown": [ - "*Authors: HyeYun Park, Gonzalo Aravena, Elana Urbach, Guillem Megias Homar, Andrew Rasmussen, Kate Napier, Carlos Morales*" + "*Authors: HyeYun Park, Gonzalo Aravena, Guillem Megias Homar, Brian Stalder, Andrew Connolly, Carlos Morales, Kate Napier*" ], "text/plain": [ "" @@ -349,7 +435,7 @@ { "data": { "text/markdown": [ - "**Almanac for the observing night starting 2024-11-10**" + "**Almanac for the observing night starting 2024-11-09**" ], "text/plain": [ "" @@ -363,80 +449,80 @@ "text/html": [ "\n", - "\n", + "
\n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", "
(times in UTC)(times in UTC)
Moon Rise2024-11-09 16:37:10.226Moon Rise2024-11-09 16:37:10.235
Moon Set2024-11-10 06:08:59.597Moon Set2024-11-09 05:33:17.186
Moon Illumination60%(% illuminated)Moon Illumination49%(% illuminated)
Astronomical Twilight (morning)2024-11-10 08:11:41.944(-18 degrees)Astronomical Twilight (morning)2024-11-09 08:12:36.896(-18 degrees)
Astronomical Twilight (evening)2024-11-10 00:41:49.376(-18 degrees)Astronomical Twilight (evening)2024-11-09 00:40:42.782(-18 degrees)
Solar Midnight2024-11-10 04:26:45.656Solar Midnight2024-11-09 04:26:39.852
Nautical Twilight (morning)2024-11-10 08:44:04.416(-12 degrees)Nautical Twilight (morning)2024-11-09 08:44:52.119(-12 degrees)
Nautical Twilight (evening)2024-11-10 00:09:26.878(-12 degrees)Nautical Twilight (evening)2024-11-09 00:08:27.518(-12 degrees)
Civil Twilight (morning)2024-11-10 09:15:02.510(-6 degrees)Civil Twilight (morning)2024-11-09 09:15:44.702(-6 degrees)
Civil Twilight (evening)2024-11-09 23:38:28.725(-6 degrees)Civil Twilight (evening)2024-11-08 23:37:34.858(-6 degrees)
Sun Rise2024-11-10 09:45:01.340Sun Rise2024-11-09 09:45:39.224
Sun Set2024-11-09 23:08:29.791Sun Set2024-11-08 23:07:40.214
\n" ], "text/plain": [ - "" + "" ] }, "metadata": {}, @@ -522,71 +608,83 @@ " \n", " \n", " \n", + " LSSTCam\n", " LSSTComCam\n", " \n", " \n", " \n", " \n", " Total Night\n", - " 7:29:52\n", + " 7:31:54\n", + " 7:31:54\n", " \n", " \n", " Total Exposure\n", - " 1:21:59\n", + " 5:33:42\n", + " 1:57:57\n", " \n", " \n", " Slew time(1)\n", " 0:00:00\n", + " 0:00:00\n", " \n", " \n", " Readout time(2)\n", - " 0:09:09\n", + " 0:51:17\n", + " 0:11:11\n", " \n", " \n", " Time loss to fault\n", " NA\n", + " NA\n", " \n", " \n", " Time loss to weather\n", " NA\n", + " NA\n", " \n", " \n", " Idle time\n", - " 6:07:53\n", + " 1:58:11\n", + " 5:33:56\n", " \n", " \n", " Number of exposures\n", - " 239\n", + " 1338\n", + " 292\n", " \n", " \n", " Mean readout time\n", " 0.000639\n", + " 0.000639\n", " \n", " \n", " Number of slews(1)\n", " 0\n", + " 0\n", " \n", " \n", " Mean Slew time(1)\n", " 0:00:00\n", + " 0:00:00\n", " \n", " \n", "\n", "" ], "text/plain": [ - " LSSTComCam\n", - "Total Night 7:29:52\n", - "Total Exposure 1:21:59\n", - "Slew time(1) 0:00:00\n", - "Readout time(2) 0:09:09\n", - "Time loss to fault NA\n", - "Time loss to weather NA\n", - "Idle time 6:07:53\n", - "Number of exposures 239\n", - "Mean readout time 0.000639\n", - "Number of slews(1) 0\n", - "Mean Slew time(1) 0:00:00" + " LSSTCam LSSTComCam\n", + "Total Night 7:31:54 7:31:54\n", + "Total Exposure 5:33:42 1:57:57\n", + "Slew time(1) 0:00:00 0:00:00\n", + "Readout time(2) 0:51:17 0:11:11\n", + "Time loss to fault NA NA\n", + "Time loss to weather NA NA\n", + "Idle time 1:58:11 5:33:56\n", + "Number of exposures 1338 292\n", + "Mean readout time 0.000639 0.000639\n", + "Number of slews(1) 0 0\n", + "Mean Slew time(1) 0:00:00 0:00:00" ] }, "metadata": {}, @@ -648,7 +746,7 @@ { "data": { "text/markdown": [ - "No tickets found using: [API Data](https://usdf-rsp-dev.slac.stanford.edu/nightreport/reports?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=100&min_day_obs=20241110&max_day_obs=20241111) in `confluence_url`." + "No tickets found using: [API Data](https://usdf-rsp-dev.slac.stanford.edu/nightreport/reports?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=100&min_day_obs=20241109&max_day_obs=20241110) in `confluence_url`." ], "text/plain": [ "" @@ -734,7 +832,7 @@ { "data": { "text/markdown": [ - "**LSSTComCam: 239 total exposures**" + "**LSSTComCam: 292 total exposures**" ], "text/plain": [ "" @@ -784,40 +882,40 @@ " 0\n", " 0\n", " 0\n", - " 94\n", - " 94\n", + " 105\n", + " 105\n", " \n", " \n", " bias\n", " 0\n", " 0\n", " 0\n", - " 10\n", - " 10\n", + " 32\n", + " 32\n", " \n", " \n", " cwfs\n", " 0\n", " 0\n", " 0\n", - " 81\n", - " 81\n", + " 95\n", + " 95\n", " \n", " \n", " dark\n", " 0\n", " 0\n", " 0\n", - " 46\n", - " 46\n", + " 32\n", + " 32\n", " \n", " \n", " science\n", " 0\n", " 0\n", " 0\n", - " 8\n", - " 8\n", + " 28\n", + " 28\n", " \n", " \n", "\n", @@ -825,11 +923,11 @@ ], "text/plain": [ " good questionable junk unknown total\n", - "acq 0 0 0 94 94\n", - "bias 0 0 0 10 10\n", - "cwfs 0 0 0 81 81\n", - "dark 0 0 0 46 46\n", - "science 0 0 0 8 8" + "acq 0 0 0 105 105\n", + "bias 0 0 0 32 32\n", + "cwfs 0 0 0 95 95\n", + "dark 0 0 0 32 32\n", + "science 0 0 0 28 28" ] }, "metadata": {}, @@ -872,52 +970,44 @@ " \n", " \n", " \n", - " checking\n", - " 0\n", - " 0\n", - " 0\n", - " 6\n", - " 6\n", - " \n", - " \n", - " comcam-usdf-checkout\n", + " acq\n", " 0\n", " 0\n", " 0\n", - " 10\n", - " 10\n", + " 33\n", + " 33\n", " \n", " \n", - " comcam_twilight_flat\n", + " comcam-daily-calibration\n", " 0\n", " 0\n", " 0\n", - " 87\n", - " 87\n", + " 44\n", + " 44\n", " \n", " \n", - " extra\n", + " comcam-usdf-checkout\n", " 0\n", " 0\n", " 0\n", - " 3\n", - " 3\n", + " 10\n", + " 10\n", " \n", " \n", - " extra_alignment_sweep_m2rx\n", + " dark\n", " 0\n", " 0\n", " 0\n", - " 7\n", - " 7\n", + " 10\n", + " 10\n", " \n", " \n", - " extra_alignment_sweep_m2ry\n", + " extra\n", " 0\n", " 0\n", " 0\n", - " 17\n", - " 17\n", + " 25\n", + " 25\n", " \n", " \n", " extra_best_m2_ry\n", @@ -928,7 +1018,7 @@ " 1\n", " \n", " \n", - " extra_sitcom-826\n", + " extra_m2_ry\n", " 0\n", " 0\n", " 0\n", @@ -936,31 +1026,31 @@ " 12\n", " \n", " \n", - " infocus\n", + " extra_sitcom-826\n", " 0\n", " 0\n", " 0\n", - " 3\n", - " 3\n", + " 8\n", + " 8\n", " \n", " \n", - " infocus_alignment_sweep_m2rx\n", + " infocus\n", " 0\n", " 0\n", " 0\n", - " 7\n", - " 7\n", + " 25\n", + " 25\n", " \n", " \n", - " infocus_alignment_sweep_m2ry\n", + " infocus_best_m2_ry\n", " 0\n", " 0\n", " 0\n", - " 17\n", - " 17\n", + " 1\n", + " 1\n", " \n", " \n", - " infocus_sitcom-826\n", + " infocus_m2_ry\n", " 0\n", " 0\n", " 0\n", @@ -968,86 +1058,85 @@ " 12\n", " \n", " \n", - " intra\n", + " infocus_sitcom-826\n", " 0\n", " 0\n", " 0\n", - " 3\n", - " 3\n", + " 8\n", + " 8\n", " \n", " \n", - " intra_alignment_sweep_m2rx\n", + " intra\n", " 0\n", " 0\n", " 0\n", - " 7\n", - " 7\n", + " 26\n", + " 26\n", " \n", " \n", - " intra_alignment_sweep_m2ry\n", + " intra_best_m2_ry\n", " 0\n", " 0\n", " 0\n", - " 18\n", - " 18\n", + " 1\n", + " 1\n", " \n", " \n", - " intra_best_m2_ry\n", + " intra_m2_ry\n", " 0\n", " 0\n", " 0\n", - " 1\n", - " 1\n", + " 12\n", + " 12\n", " \n", " \n", " intra_sitcom-826\n", " 0\n", " 0\n", " 0\n", - " 12\n", - " 12\n", + " 10\n", + " 10\n", " \n", " \n", " science\n", " 0\n", " 0\n", " 0\n", - " 8\n", - " 8\n", + " 28\n", + " 28\n", " \n", " \n", - " tmacheckout\n", + " snap_investigation\n", " 0\n", " 0\n", " 0\n", - " 8\n", - " 8\n", + " 26\n", + " 26\n", " \n", " \n", "\n", "" ], "text/plain": [ - " good questionable junk unknown total\n", - "checking 0 0 0 6 6\n", - "comcam-usdf-checkout 0 0 0 10 10\n", - "comcam_twilight_flat 0 0 0 87 87\n", - "extra 0 0 0 3 3\n", - "extra_alignment_sweep_m2rx 0 0 0 7 7\n", - "extra_alignment_sweep_m2ry 0 0 0 17 17\n", - "extra_best_m2_ry 0 0 0 1 1\n", - "extra_sitcom-826 0 0 0 12 12\n", - "infocus 0 0 0 3 3\n", - "infocus_alignment_sweep_m2rx 0 0 0 7 7\n", - "infocus_alignment_sweep_m2ry 0 0 0 17 17\n", - "infocus_sitcom-826 0 0 0 12 12\n", - "intra 0 0 0 3 3\n", - "intra_alignment_sweep_m2rx 0 0 0 7 7\n", - "intra_alignment_sweep_m2ry 0 0 0 18 18\n", - "intra_best_m2_ry 0 0 0 1 1\n", - "intra_sitcom-826 0 0 0 12 12\n", - "science 0 0 0 8 8\n", - "tmacheckout 0 0 0 8 8" + " good questionable junk unknown total\n", + "acq 0 0 0 33 33\n", + "comcam-daily-calibration 0 0 0 44 44\n", + "comcam-usdf-checkout 0 0 0 10 10\n", + "dark 0 0 0 10 10\n", + "extra 0 0 0 25 25\n", + "extra_best_m2_ry 0 0 0 1 1\n", + "extra_m2_ry 0 0 0 12 12\n", + "extra_sitcom-826 0 0 0 8 8\n", + "infocus 0 0 0 25 25\n", + "infocus_best_m2_ry 0 0 0 1 1\n", + "infocus_m2_ry 0 0 0 12 12\n", + "infocus_sitcom-826 0 0 0 8 8\n", + "intra 0 0 0 26 26\n", + "intra_best_m2_ry 0 0 0 1 1\n", + "intra_m2_ry 0 0 0 12 12\n", + "intra_sitcom-826 0 0 0 10 10\n", + "science 0 0 0 28 28\n", + "snap_investigation 0 0 0 26 26" ] }, "metadata": {}, @@ -1090,12 +1179,28 @@ " \n", " \n", " \n", - " BLOCK-T215\n", + " BLOCK-T220\n", + " 0\n", + " 0\n", + " 0\n", + " 44\n", + " 44\n", + " \n", + " \n", + " BLOCK-T227\n", + " 0\n", + " 0\n", + " 0\n", + " 6\n", + " 6\n", + " \n", + " \n", + " BLOCK-T235\n", " 0\n", " 0\n", " 0\n", - " 87\n", - " 87\n", + " 3\n", + " 3\n", " \n", " \n", " BLOCK-T246\n", @@ -1106,52 +1211,68 @@ " 10\n", " \n", " \n", - " BLOCK-T248\n", + " BLOCK-T249\n", " 0\n", " 0\n", " 0\n", - " 17\n", - " 17\n", + " 42\n", + " 42\n", " \n", " \n", - " BLOCK-T249\n", + " BLOCK-T270\n", " 0\n", " 0\n", " 0\n", - " 11\n", - " 11\n", + " 10\n", + " 10\n", " \n", " \n", - " BLOCK-T250\n", + " BLOCK-T273\n", " 0\n", " 0\n", " 0\n", - " 6\n", - " 6\n", + " 18\n", + " 18\n", " \n", " \n", - " BLOCK-T279\n", + " BLOCK-T276\n", " 0\n", " 0\n", " 0\n", - " 98\n", - " 98\n", + " 26\n", + " 26\n", " \n", " \n", " BLOCK-T92\n", " 0\n", " 0\n", " 0\n", - " 2\n", - " 2\n", + " 71\n", + " 71\n", + " \n", + " \n", + " BLOCK-T94\n", + " 0\n", + " 0\n", + " 0\n", + " 3\n", + " 3\n", " \n", " \n", " PP-SURVEY\n", " 0\n", " 0\n", " 0\n", - " 8\n", - " 8\n", + " 28\n", + " 28\n", + " \n", + " \n", + " unknown\n", + " 0\n", + " 0\n", + " 0\n", + " 31\n", + " 31\n", " \n", " \n", "\n", @@ -1159,14 +1280,18 @@ ], "text/plain": [ " good questionable junk unknown total\n", - "BLOCK-T215 0 0 0 87 87\n", + "BLOCK-T220 0 0 0 44 44\n", + "BLOCK-T227 0 0 0 6 6\n", + "BLOCK-T235 0 0 0 3 3\n", "BLOCK-T246 0 0 0 10 10\n", - "BLOCK-T248 0 0 0 17 17\n", - "BLOCK-T249 0 0 0 11 11\n", - "BLOCK-T250 0 0 0 6 6\n", - "BLOCK-T279 0 0 0 98 98\n", - "BLOCK-T92 0 0 0 2 2\n", - "PP-SURVEY 0 0 0 8 8" + "BLOCK-T249 0 0 0 42 42\n", + "BLOCK-T270 0 0 0 10 10\n", + "BLOCK-T273 0 0 0 18 18\n", + "BLOCK-T276 0 0 0 26 26\n", + "BLOCK-T92 0 0 0 71 71\n", + "BLOCK-T94 0 0 0 3 3\n", + "PP-SURVEY 0 0 0 28 28\n", + "unknown 0 0 0 31 31" ] }, "metadata": {}, @@ -1175,7 +1300,7 @@ { "data": { "text/markdown": [ - "**LSSTCam: 0 total exposures**" + "**LSSTCam: 1338 total exposures**" ], "text/plain": [ "" @@ -1183,120 +1308,319 @@ }, "metadata": {}, "output_type": "display_data" - } - ], - "source": [ - "# Exposure Report\n", - "for instrum in allsrc.exp_src.exposures.keys():\n", - " md(f'**{instrum}: {len(allsrc.exp_src.exposures[instrum])} total exposures**')\n", - " df_dict = allsrc.fields_count_exposure(instrum)\n", - " for field_name,df in df_dict.items():\n", - " if df.empty:\n", - " continue\n", - " print(field_name.title().replace('_',' '))\n", - " display(df)" - ] - }, - { - "cell_type": "markdown", - "id": "20820900-05ab-4ca5-8df5-9f725c0ee570", - "metadata": {}, - "source": [ - "## Narrative Log BETA " - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "26", - "metadata": { - "editable": true, - "jupyter": { - "source_hidden": true }, - "slideshow": { - "slide_type": "" + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Observation Type\n" + ] }, - "tags": [] - }, - "outputs": [ { "data": { - "text/markdown": [ - "- **2024-11-10 11:07:15.143285** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: None\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
goodquestionablejunkunknowntotal
bias000380380
dark000128128
flat000830830
\n", + "
" ], "text/plain": [ - "" + " good questionable junk unknown total\n", + "bias 0 0 0 380 380\n", + "dark 0 0 0 128 128\n", + "flat 0 0 0 830 830" ] }, "metadata": {}, "output_type": "display_data" }, { - "data": { - "text/markdown": [ - "- **2024-11-10 10:56:40.425009** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: None\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "
Hexapod shutdown
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" + "name": "stdout", + "output_type": "stream", + "text": [ + "Observation Reason\n" + ] }, { "data": { - "text/markdown": [ - "- **2024-11-10 10:02:18.321268** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
goodquestionablejunkunknowntotal
bias0002020
block-t233000119119
dark0001616
eo_opsim000115115
flat00010651065
test00033
\n", + "
" ], "text/plain": [ - "" + " good questionable junk unknown total\n", + "bias 0 0 0 20 20\n", + "block-t233 0 0 0 119 119\n", + "dark 0 0 0 16 16\n", + "eo_opsim 0 0 0 115 115\n", + "flat 0 0 0 1065 1065\n", + "test 0 0 0 3 3" ] }, "metadata": {}, "output_type": "display_data" }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Science Program\n" + ] + }, { "data": { - "text/markdown": [ - "- **2024-11-10 09:38:04.028897** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: None\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
goodquestionablejunkunknowntotal
BLOCK-T233000119119
BLOCK-T262000115115
E74800010571057
test00033
unknown0004444
\n", + "
" ], "text/plain": [ - "" + " good questionable junk unknown total\n", + "BLOCK-T233 0 0 0 119 119\n", + "BLOCK-T262 0 0 0 115 115\n", + "E748 0 0 0 1057 1057\n", + "test 0 0 0 3 3\n", + "unknown 0 0 0 44 44" ] }, "metadata": {}, "output_type": "display_data" + } + ], + "source": [ + "# Exposure Report\n", + "for instrum in allsrc.exp_src.exposures.keys():\n", + " md(f'**{instrum}: {len(allsrc.exp_src.exposures[instrum])} total exposures**')\n", + " df_dict = allsrc.fields_count_exposure(instrum)\n", + " for field_name,df in df_dict.items():\n", + " if df.empty:\n", + " continue\n", + " print(field_name.title().replace('_',' '))\n", + " display(df)" + ] + }, + { + "cell_type": "markdown", + "id": "20820900-05ab-4ca5-8df5-9f725c0ee570", + "metadata": {}, + "source": [ + "## Narrative Log BETA " + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "26", + "metadata": { + "editable": true, + "jupyter": { + "source_hidden": true + }, + "slideshow": { + "slide_type": "" }, + "tags": [] + }, + "outputs": [ { "data": { "text/markdown": [ - "- **2024-11-10 09:13:16.168956** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", + "- **2024-11-09 10:57:28.581261** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
we home axes in the EUI, now we are moving the MTMount to inicial position
\n" + "
Retracting the mirror covers before parking at horizon
\n" ], "text/plain": [ "" @@ -1308,15 +1632,12 @@ { "data": { "text/markdown": [ - "- **2024-11-10 09:07:18.662342** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", + "- **2024-11-09 10:49:51.806841** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
home both exes failed \r\n",
-       "\r\n",
-       "\r\n",
-       "lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=849682263, ack=<SalRetCode.CMD_FAILED: -302>, error=1, result='Failed: Connection closed before command finished')
\n" + "
lower the mirror
\n" ], "text/plain": [ "" @@ -1328,12 +1649,12 @@ { "data": { "text/markdown": [ - "- **2024-11-10 09:02:27.882534** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", + "- **2024-11-09 10:48:26.067541** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Homeming the exes
\n" + "
Completed hexapod shutdown
\n" ], "text/plain": [ "" @@ -1345,12 +1666,12 @@ { "data": { "text/markdown": [ - "- **2024-11-10 09:01:02.698747** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", + "- **2024-11-09 10:39:35.374175** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
we change manually the setting set to 5% in the EUI, now sending the mount to enable vis set summary state
\n" + "
rotator shutdown
\n" ], "text/plain": [ "" @@ -1362,15 +1683,12 @@ { "data": { "text/markdown": [ - "- **2024-11-10 08:57:51.000615** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-09 10:34:13.672391** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
The apply setting set to 5% via run command fault\r\n",
-       "\r\n",
-       "\r\n",
-       "Command AzimuthPower(sequence_id=2959445, command_code=<CommandCode.AZIMUTH_POWER: 101>, source=<Source.CSC: 1>, timestamp=1731228958.3681328, on=False)=b'2959445\\n101\\n1\\n1731228958.3681328\\n0\\r\\n' failed: Command failed due to an alarm.
\n" + "
Run BLOCK-T241
\n" ], "text/plain": [ "" @@ -1382,12 +1700,12 @@ { "data": { "text/markdown": [ - "- **2024-11-10 08:47:35.160859** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-09 10:33:59.347865** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
sending MTMount to enable to start the test M2 closed-loop break-out brake test during TMA slew - 5% Settings
\n" + "
Enable the rotator and MTMount
\n" ], "text/plain": [ "" @@ -1399,12 +1717,12 @@ { "data": { "text/markdown": [ - "- **2024-11-10 08:46:53.127538** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-09 10:26:17.324498** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Dome Close
\n" + "
Apply parking and 2% settings in the EUI
\n" ], "text/plain": [ "" @@ -1416,12 +1734,15 @@ { "data": { "text/markdown": [ - "- **2024-11-10 08:18:17.273822** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-09 10:15:45.081468** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Closing the Dome
\n" + "
Remember: Cycle the thermal system when MTM1M3 faults \r\n",
+       "\r\n",
+       "\r\n",
+       "Set the FCU fans to 300 RPM
\n" ], "text/plain": [ "" @@ -1433,47 +1754,12 @@ { "data": { "text/markdown": [ - "- **2024-11-10 08:18:17.273822** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-09 10:08:35.183649** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Closing the Dome
\n", - "\n", - " - components: ['MainTel']\n", - " - primary_software_components: ['None']\n", - " - primary_hardware_components: ['None']\n", - "\n", - "Enable scheduler Fail\r\n", - "SALINDEX 101283\r\n", - "\r\n", - "\r\n", - "Error in run\r\n", - "
Traceback (most recent call last):\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/csc_utils.py", line 163, in set_summary_state\r\n",
-       "    await cmd.start(timeout=timeout)\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 487, in start\r\n",
-       "    return await cmd_info.next_ackcmd(timeout=timeout)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 191, in next_ackcmd\r\n",
-       "    raise base.AckError(msg="Command failed", ackcmd=ackcmd)\r\n",
-       "lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=1698289930, ack=&lt;SalRetCode.CMD_FAILED: -302&gt;, error=1, result="Failed: BLOCK-T262 not in the list of observing blocks. Currently defined are: {'BLOCK-T91', 'BLOCK-T208', 'BLOCK-T248_27', 'BLOCK-T248_2', 'BLOCK-T248_12', 'BLOCK-T62', 'BLOCK-T170', 'BLOCK-T87', 'BLOCK-T194', 'BLOCK-T248_16', 'BLOCK-T240', 'BLOCK-T75', 'BLOCK-T185', 'BLOCK-T248_17', 'BLOCK-T203', 'BLOCK-T222', 'BLOCK-303', 'BLOCK-T192', 'BLOCK-T145', 'BLOCK-T4', 'BLOCK-T248_22', 'BLOCK-T227', 'BLOCK-T248_7', 'BLOCK-T188', 'BLOCK-T184', 'BLOCK-304', 'BLOCK-T186', 'BLOCK-T160', 'BLOCK-T248_14', 'BLOCK-T248_23', 'BLOCK-T223', 'BLOCK-T248_1', 'BLOCK-T248_25', 'BLOCK-T248_13', 'BLOCK-T187', 'BLOCK-T74', 'BLOCK-T248_8', 'BLOCK-T89', 'BLOCK-T220', 'BLOCK-T205', 'BLOCK-T204', 'BLOCK-302', 'BLOCK-T248_0', 'BLOCK-T248_20', 'BLOCK-T202', 'BLOCK-T86', 'BLOCK-T238', 'BLOCK-T216', 'BLOCK-T172', 'BLOCK-T176', 'BLOCK-T193', 'BLOCK-T248_6', 'BLOCK-T248_9', 'BLOCK-T248_4', 'BLOCK-T248_18', 'BLOCK-T177', 'BLOCK-T237', 'BLOCK-T248_10', 'BLOCK-T248_11', 'BLOCK-T215', 'BLOCK-T248_3', 'BLOCK-T175', 'BLOCK-T248_5', 'BLOCK-T231', 'BLOCK-T248_26', 'BLOCK-T248_28', 'BLOCK-T248_29', 'BLOCK-T248_21', 'BLOCK-T144', 'BLOCK-T248_15', 'BLOCK-T246', 'BLOCK-T3', 'BLOCK-T248_24', 'BLOCK-T248_19'}")\r\n",
-       "\r\n",
-       "\r\n",
-       "The above exception was the direct cause of the following exception:\r\n",
-       "\r\n",
-       "\r\n",
-       "Traceback (most recent call last):\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py", line 631, in do_run\r\n",
-       "    await self._run_task\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/scheduler/set_desired_state.py", line 217, in run\r\n",
-       "    await self.set_desired_state()\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/scheduler/set_desired_state.py", line 189, in set_desired_state\r\n",
-       "    await salobj.set_summary_state(\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/csc_utils.py", line 165, in set_summary_state\r\n",
-       "    raise RuntimeError(\r\n",
-       "RuntimeError: Error on cmd=cmd_start, initial_state=5: msg='Command failed', ackcmd=(ackcmd private_seqNum=1698289930, ack=&lt;SalRetCode.CMD_FAILED: -302&gt;, error=1, result="Failed: BLOCK-T262 not in the list of observing blocks. Currently defined are: {'BLOCK-T91', 'BLOCK-T208', 'BLOCK-T248_27', 'BLOCK-T248_2', 'BLOCK-T248_12', 'BLOCK-T62', 'BLOCK-T170', 'BLOCK-T87', 'BLOCK-T194', 'BLOCK-T248_16', 'BLOCK-T240', 'BLOCK-T75', 'BLOCK-T185', 'BLOCK-T248_17', 'BLOCK-T203', 'BLOCK-T222', 'BLOCK-303', 'BLOCK-T192', 'BLOCK-T145', 'BLOCK-T4', 'BLOCK-T248_22', 'BLOCK-T227', 'BLOCK-T248_7', 'BLOCK-T188', 'BLOCK-T184', 'BLOCK-304', 'BLOCK-T186', 'BLOCK-T160', 'BLOCK-T248_14', 'BLOCK-T248_23', 'BLOCK-T223', 'BLOCK-T248_1', 'BLOCK-T248_25', 'BLOCK-T248_13', 'BLOCK-T187', 'BLOCK-T74', 'BLOCK-T248_8', 'BLOCK-T89', 'BLOCK-T220', 'BLOCK-T205', 'BLOCK-T204', 'BLOCK-302', 'BLOCK-T248_0', 'BLOCK-T248_20', 'BLOCK-T202', 'BLOCK-T86', 'BLOCK-T238', 'BLOCK-T216', 'BLOCK-T172', 'BLOCK-T176', 'BLOCK-T193', 'BLOCK-T248_6', 'BLOCK-T248_9', 'BLOCK-T248_4', 'BLOCK-T248_18', 'BLOCK-T177', 'BLOCK-T237', 'BLOCK-T248_10', 'BLOCK-T248_11', 'BLOCK-T215', 'BLOCK-T248_3', 'BLOCK-T175', 'BLOCK-T248_5', 'BLOCK-T231', 'BLOCK-T248_26', 'BLOCK-T248_28', 'BLOCK-T248_29', 'BLOCK-T248_21', 'BLOCK-T144', 'BLOCK-T248_15', 'BLOCK-T246', 'BLOCK-T3', 'BLOCK-T248_24', 'BLOCK-T248_19'}")\r\n",
-       "
\n" + "
raise the mirror
\n" ], "text/plain": [ "" @@ -1485,12 +1771,12 @@ { "data": { "text/markdown": [ - "- **2024-11-10 07:06:07.394198** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-09 10:02:27.713135** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
try again
\n" + "
Move to az = -32 and el = 75
\n" ], "text/plain": [ "" @@ -1502,16 +1788,13 @@ { "data": { "text/markdown": [ - "- **2024-11-10 07:05:56.102733** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-09 09:58:24.872494** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
We track target and MTPtg wen to fault\r\n",
-       "\r\n",
-       "\r\n",
-       "Error MTMount replied error for trackTarget command (1735226902) Error no:1 message:Failed: Command with id 2791462 has timed o\r\n",
-       "0
\n" + "
MTMount in enabled \r\n",
+       "home both axes
\n" ], "text/plain": [ "" @@ -1523,15 +1806,12 @@ { "data": { "text/markdown": [ - "- **2024-11-10 07:02:48.211865** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-09 09:51:41.635028** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
File \"/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/maintel/mtcs.py\", line 935, in open_m1_cover\r\n",
-       "    f\"Open mirror cover command failed with {ack.ack!r}::{ack.error}. \"\r\n",
-       "                                             ^^^^^^^\r\n",
-       "AttributeError: 'AckError' object has no attribute 'ack'
\n" + "
Clearing interlocks since MTM1M3 faulted
\n" ], "text/plain": [ "" @@ -1543,12 +1823,13 @@ { "data": { "text/markdown": [ - "- **2024-11-10 07:02:14.963653** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-09 09:51:08.422324** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
we have good conditions to open de the Dome
\n" + "
MTM1M3 Fault \r\n",
+       "Force controller Z Moment Limit - applied -4460.39 N, expected -4000.00 N to 4000.00 N
\n" ], "text/plain": [ "" @@ -1560,12 +1841,12 @@ { "data": { "text/markdown": [ - "- **2024-11-10 06:28:07.790689** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-09 09:45:34.129916** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=612720053, ack=<SalRetCode.CMD_FAILED: -302>, error=1, result='Failed: Command exitFault was sent for an incorrect state.')
\n" + "
Manually apply move_p2p
\n" ], "text/plain": [ "" @@ -1577,12 +1858,16 @@ { "data": { "text/markdown": [ - "- **2024-11-10 06:12:14.793572** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-09 09:45:26.878025** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
We are waiting to evaluate the weather conditions to determine whether it is safe to open the dome.  In the meantime, we are preparing the telescope to slew to the next position to prepare for the event of reopening the dome.
\n" + "
Failed with the following error:\r\n",
+       "lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=94448011, ack=<SalRetCode.CMD_FAILED: -302>, error=1, result='Failed: Block BLOCK-T231 is not in the list of observing blocks. Current observing blocks are: BLOCK-T262,BLOCK-T263,BLOCK-T228,BLOCK-T189,BLOCK-T246,BLOCK-T169,BLOCK-T233,BLOCK-T220,BLOCK-T68,BLOCK-302,BLOCK-T3,BLOCK-T4,BLOCK-303.')\r\n",
+       "\r\n",
+       "\r\n",
+       "2024/11/09 09:44:34 TAI
\n" ], "text/plain": [ "" @@ -1594,26 +1879,12 @@ { "data": { "text/markdown": [ - "- **2024-11-10 05:46:55.681732** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-09 09:44:37.120574** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
In touch with Camera Team Yousuke, Stuart, and John since the LSSTCam Stg1TXVBulb temp got hotter than -20 deg C.  \r\n",
-       "\r\n",
-       "\r\n",
-       "Kate and Carlos went down to the first floor to the room just right of the main building entrance and followed this procedure with instruction from the Camera Team \r\n",
-       "Procedure for when the LSSTCam temp raises above -20 deg C \r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "press the green REF button \r\n",
-       "it should turn gray \r\n",
-       "wait two minutes \r\n",
-       "press REF button again \r\n",
-       "should now be green \r\n",
-       "wait for a minute or two for COMP1 and COMP2 to become green
\n" + "
Running  BLOCK-T231
\n" ], "text/plain": [ "" @@ -1625,38 +1896,13 @@ { "data": { "text/markdown": [ - "- **2024-11-10 05:27:18.498075** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-09 09:25:13.533714** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
closing the mirror covers \r\n",
-       "\r\n",
-       "\r\n",
-       "MTMount is Enabled\r\n",
-       "Stop tracking \r\n",
-       "close_mirror_covers\r\n",
-       "\r\n",
-       "\r\n",
-       "time out on the close_mirror_covers so proceed manually \r\n",
-       "\r\n",
-       "\r\n",
-       "MTMount to Disabled \r\n",
-       "Fault \r\n",
-       "Send to Standby \r\n",
-       "\r\n",
-       "\r\n",
-       "MTPtg fault \r\n",
-       "Send to standby \r\n",
-       "\r\n",
-       "\r\n",
-       "Go to EUI \r\n",
-       "\r\n",
-       "\r\n",
-       "(forgot to post this earlier) \r\n",
-       "\r\n",
-       "\r\n",
-       "Dome is closed at 2:20
\n" + "
Brian opened the mixing valve \r\n",
+       "Setting HVAC set point to 10 deg C
\n" ], "text/plain": [ "" @@ -1668,12 +1914,12 @@ { "data": { "text/markdown": [ - "- **2024-11-10 04:58:41.074077** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-09 09:10:11.517864** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Engaging the brakes
\n" + "
We are manually starting the move_p2p and sleep sequence
\n" ], "text/plain": [ "" @@ -1685,12 +1931,18 @@ { "data": { "text/markdown": [ - "- **2024-11-10 04:56:44.860411** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-09 09:01:24.673825** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Park the dome at 328 degrees
\n" + "
Failed a second time \r\n",
+       "\r\n",
+       "\r\n",
+       "lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=94448011, ack=<SalRetCode.CMD_FAILED: -302>, error=1, result='Failed: Block BLOCK-T227 is not in the list of observing blocks. Current observing blocks are: BLOCK-T262,BLOCK-T263,BLOCK-T228,BLOCK-T189,BLOCK-T246,BLOCK-T169,BLOCK-T233,BLOCK-T220,BLOCK-T68,BLOCK-302,BLOCK-T3,BLOCK-T4,BLOCK-303.')\r\n",
+       "\r\n",
+       "\r\n",
+       "2024/11/09 09:00:52 TAI
\n" ], "text/plain": [ "" @@ -1702,12 +1954,12 @@ { "data": { "text/markdown": [ - "- **2024-11-10 04:55:16.250852** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-09 09:00:05.151407** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Disable dome following failed because accidentally selected script for auxtel
\n" + "
Erik is now done with recovering MTAOS, so we will try the add_block script again
\n" ], "text/plain": [ "" @@ -1719,12 +1971,15 @@ { "data": { "text/markdown": [ - "- **2024-11-10 04:54:55.418285** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-09 08:59:23.795776** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Try disable dome again
\n" + "
Block-T227 add_block failed \r\n",
+       "\r\n",
+       "\r\n",
+       "lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=94448011, ack=<SalRetCode.CMD_FAILED: -302>, error=1, result='Failed: Not allowed in state=<State.STANDBY: 5>')
\n" ], "text/plain": [ "" @@ -1736,12 +1991,12 @@ { "data": { "text/markdown": [ - "- **2024-11-10 04:53:57.923822** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-09 08:58:04.157702** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Disabling dome following
\n" + "
Run maintel/scheduler/add_block.py that is part of Block-T227
\n" ], "text/plain": [ "" @@ -1753,12 +2008,12 @@ { "data": { "text/markdown": [ - "- **2024-11-10 04:51:51.688325** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-09 08:57:25.310055** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Stop tracking and then move the rotator to zero
\n" + "
Put MTRotator in enabled
\n" ], "text/plain": [ "" @@ -1770,18 +2025,12 @@ { "data": { "text/markdown": [ - "- **2024-11-10 04:50:21.246177** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-09 08:54:38.918942** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Moving to close the dome \r\n",
-       "Outside temp: 9.43 \r\n",
-       "M1M3: 13.11 \r\n",
-       "M2: 10.93 \r\n",
-       "\r\n",
-       "\r\n",
-       "Humidity: 65%
\n" + "
Home_both_axes
\n" ], "text/plain": [ "" @@ -1793,28 +2042,20 @@ { "data": { "text/markdown": [ - "- **2024-11-10 04:42:24.407995** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", + "- **2024-11-09 08:54:19.006322** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
set dof\r\n",
+       "
The set_summary_state.py script failed again. \r\n",
        "\r\n",
        "\r\n",
-       "dofs: \r\n",
-       "  [-1.26588000e+03,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "       -1.50000000e-02,  1.73617628e+03,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  1.08892279e+00, -1.22705647e+00,\r\n",
-       "       -5.29640509e-02, -1.28094313e-03,  3.54122613e-01,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00]
\n"
+       "In the EUI, apply parking settings \r\n",
+       "Apply 5% settings (these are applied on top of parking settings, so both are applied) \r\n",
+       "\r\n",
+       "\r\n",
+       "Trying set_summary_state.py again and got an error about the mirror cover\r\n",
+       "MirrorCoverLocksResetAlarm(sequence_id=1807865, command_code=<CommandCode.MIRROR_COVER_LOCKS_RESET_ALARM: 1505>, source=<Source.CSC: 1>, timestamp=1731142271.49153, drive=-1) failed: ExpectedError('Command with id 1807865 has timed out after not receiving accept for 500 ms.')\")
\n" ], "text/plain": [ "" @@ -1826,12 +2067,12 @@ { "data": { "text/markdown": [ - "- **2024-11-10 04:30:57.033254** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", + "- **2024-11-09 08:45:53.572294** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
I create a ticket for this https://rubinobs.atlassian.net/browse/OBS-658
\n" + "
Mount didn't get enabled so trying again
\n" ], "text/plain": [ "" @@ -1843,12 +2084,12 @@ { "data": { "text/markdown": [ - "- **2024-11-10 04:22:38.042474** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-09 08:45:23.004582** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
The Rapid Analysis is only working intermittently
\n" + "
Home both axes
\n" ], "text/plain": [ "" @@ -1860,12 +2101,16 @@ { "data": { "text/markdown": [ - "- **2024-11-10 04:22:24.456412** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", + "- **2024-11-09 08:44:29.907115** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
RubinTV has some Issues, we are missing so many images
\n" + "
Changing to EUI mode \r\n",
+       "Selecting NewSpeedSettings5percent \r\n",
+       "\r\n",
+       "\r\n",
+       "Put MTMount back to enabled
\n" ], "text/plain": [ "" @@ -1877,20 +2122,12 @@ { "data": { "text/markdown": [ - "- **2024-11-10 04:19:41.896724** Time Lost: 0.0; Time Lost Type: fault;\n", - " - components: ['MainTel']\n", + "- **2024-11-09 08:42:34.468895** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Parameter MArch ComCam\r\n",
-       "\r\n",
-       "\r\n",
-       "exp_time: 30.0\r\n",
-       "dof_index: 4\r\n",
-       "range: 0.03\r\n",
-       "n_steps: 7\r\n",
-       "program: \"BLOCK-T92\"\r\n",
-       "reason: \"m2_ry\"
\n" + "
Sending MTMount to Disabled with set_summary_state
\n" ], "text/plain": [ "" @@ -1902,19 +2139,1459 @@ { "data": { "text/markdown": [ - "- **2024-11-10 04:08:37.884809** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-09 08:38:02.440302** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
Sending MTAOS to Offline using set_summary_state
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 08:18:30.935099** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
Stop_tracking\r\n",
+       "\r\n",
+       "\r\n",
+       "Then the run_command to close the mirror covers
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 08:17:12.245455** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
Rerun maintel/close_mirror_covers  \r\n",
+       "\r\n",
+       "\r\n",
+       "The elapsed time was 515 sec and the telescope did not stop tracking.  \r\n",
+       "We quit the script and will continue with the run_command.py
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 08:06:53.658734** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
Making sure MTPtg, MTRotator, MTDome, and MTDomeTrajectory are all enabled
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 08:05:59.847800** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
maintel/close_mirror_covers script failed \r\n",
+       "\r\n",
+       "\r\n",
+       "AssertionError: The following components are not enabled: ['mtptg', 'mtrotator', 'mtdome', 'mtdometrajectory'].
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 08:04:19.809586** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
Closing the mirror covers following Block-T152 \r\n",
+       "\r\n",
+       "\r\n",
+       "Track_target failed, but we knew it was not tracking\r\n",
+       "\r\n",
+       "\r\n",
+       "Using the new maintel/close_mirror_cover script
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 08:02:02.867032** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
Running BLOCK-T196
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 08:00:14.745484** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
Move the dome back to the safe position \r\n",
+       "Disable dome following
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 08:00:06.405036** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
MTMount and MTPtg went to fault \r\n",
+       "Scheduler went to fault\r\n",
+       "\r\n",
+       "\r\n",
+       "Put them in standby \r\n",
+       "\r\n",
+       "\r\n",
+       "stop the scheduler \r\n",
+       "\r\n",
+       "\r\n",
+       "Recover MTMount first  \r\n",
+       "- trying to put MTMount in disabled in CSC, but not working \r\n",
+       "- Now try set_summary_state, this worked \r\n",
+       "\r\n",
+       "\r\n",
+       "Home both axes \r\n",
+       "\r\n",
+       "\r\n",
+       "Recover MTPtg
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 06:43:58.009171** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
we got the same error with the scheduler\r\n",
+       "\r\n",
+       "\r\n",
+       " lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=9743175, ack=<SalRetCode.CMD_FAILED: -302>, error=1, result=\"Failed: 502, message='Bad Gateway', url=URL('http://ccs.lsst.org/ImageUtilities/rest/imageNameService?n=1&sourceIndex=248&source=BlockT')\")\r\n",
+       "0
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 06:41:53.470210** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
We improve the configuration adding the g_band
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 06:09:46.728537** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
Starting the BLOCK-T248\r\n",
+       "tracking target\r\n",
+       "\r\n",
+       "\r\n",
+       "\r\n",
+       "\r\n",
+       "slew_icrs:\r\n",
+       "  dec: -28.100000\r\n",
+       "  ra: 3.5417 \r\n",
+       "rot_value: 0\r\n",
+       "rot_type: PhysicalSky\r\n",
+       "program: \"BLOCK-T248\"\r\n",
+       "reason: \"slew_to_target\"
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 06:07:46.832750** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
Trying track_target now with the following configuration: \r\n",
+       "slew_icrs:\r\n",
+       "  dec: -28.100000\r\n",
+       "  ra: 3.5417 \r\n",
+       "rot_value: 0\r\n",
+       "rot_type: PhysicalSky\r\n",
+       "program: \"BLOCK-T248\"\r\n",
+       "reason: \"slew_to_target\"\r\n",
+       "\r\n",
+       "\r\n",
+       "That worked
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 06:04:09.975612** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
Track_target using the following configuration: \r\n",
+       "\r\n",
+       "\r\n",
+       "track_azel:\r\n",
+       " az: 276.1479248364837\r\n",
+       " el: 76.92015741526181\r\n",
+       "rot_value: 0\r\n",
+       "rot_type: PhysicalSky\r\n",
+       "program: \"BLOCK-T248\"\r\n",
+       "reason: \"slew_to_target\"\r\n",
+       "\r\n",
+       "\r\n",
+       "Error: \r\n",
+       "lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=137258246, ack=<SalRetCode.CMD_FAILED: -302>, error=6611, result='Rejected : Azimuth out of range')
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 05:50:23.801764** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
Put the laser tracker in DISABLED since we have already turned it off.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 05:46:03.559239** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
Running Block-T249\r\n",
+       "\r\n",
+       "\r\n",
+       "close_loop_comcam.py\r\n",
+       "max_iter: 2\r\n",
+       "apply_corrections: true
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 05:39:19.122486** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
Turning off the laser now.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 05:36:29.971024** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
We are working on Block T-273 bending modes 4 and 5.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 05:17:01.547778** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
We noticed around 1:45am that the StgTXVBulb chiller1/Maq20 temp got warmer than -20 deg C.  Sean had let us know to inform him if the temp got warmer than 20 deg.  Carlos ended up driving to wake him up after we were unable to reach him via phone.
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 05:15:22.535025** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
Finished running Block T-270.  Now doing track_target to az = -90 and el = 75
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:59:37.046793** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: None\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
When running bending mode B19 with M2_B19: 0.025, got an error when running take_aos_sequence_comcam \r\n",
+       "\r\n",
+       "\r\n",
+       "lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=788397162, ack=<SalRetCode.CMD_FAILED: -302>, error=1, result=\"Failed: Failed to apply correction to: ['m2']. \")
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:36:19.507029** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
filter: \"r_03\"\r\n",
+       "exposure_time: 30.0\r\n",
+       "max_iter: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: \"closed_loop_m2_b18\"\r\n",
+       "used_dofs: [47, 48, 49]\r\n",
+       "apply_corrections: true\r\n",
+       "use_ocps: true\r\n",
+       "mode: FAM
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:36:01.900553** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
M2_B18: 0.1
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:35:50.757649** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
dofs:\r\n",
+       " [-1.26588000e+03, -2.08991571e-08,  4.95136945e-08,  5.53397010e-03,\r\n",
+       "       -2.40923443e-03,  1.76380420e+03,  4.85815000e+03,  6.73960000e+02,\r\n",
+       "        3.16024784e-03, -7.47932190e-04,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  1.08892279e+00, -1.22705647e+00,\r\n",
+       "       -5.29640509e-02, -1.28094313e-03,  3.54122613e-01,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00]
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:33:45.630929** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
filter: 'r_03'\r\n",
+       "exposure_time: 30\r\n",
+       "n_sequences: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: 'closed_loop_m2_b17'
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:33:35.971754** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
filter: \"r_03\"\r\n",
+       "exposure_time: 30.0\r\n",
+       "max_iter: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: \"closed_loop_m2_b17\"\r\n",
+       "used_dofs: [45, 46]\r\n",
+       "apply_corrections: true\r\n",
+       "use_ocps: true\r\n",
+       "mode: FAM
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:33:26.108690** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
M2_B17: 0.1
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:33:14.684086** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
dofs:\r\n",
+       " [-1.26588000e+03, -2.08991571e-08,  4.95136945e-08,  5.53397010e-03,\r\n",
+       "       -2.40923443e-03,  1.76380420e+03,  4.85815000e+03,  6.73960000e+02,\r\n",
+       "        3.16024784e-03, -7.47932190e-04,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  1.08892279e+00, -1.22705647e+00,\r\n",
+       "       -5.29640509e-02, -1.28094313e-03,  3.54122613e-01,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00]
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:33:04.253959** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
filter: 'r_03'\r\n",
+       "exposure_time: 30\r\n",
+       "n_sequences: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: 'closed_loop_m2_b16'
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:31:00.270369** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
filter: \"r_03\"\r\n",
+       "exposure_time: 30.0\r\n",
+       "max_iter: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: \"closed_loop_m2_b16\"\r\n",
+       "used_dofs: [45, 46]\r\n",
+       "apply_corrections: true\r\n",
+       "use_ocps: true\r\n",
+       "mode: FAM
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:29:54.003553** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
M2_B16: 0.1
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:29:46.141232** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
dofs:\r\n",
+       " [-1.26588000e+03, -2.08991571e-08,  4.95136945e-08,  5.53397010e-03,\r\n",
+       "       -2.40923443e-03,  1.76380420e+03,  4.85815000e+03,  6.73960000e+02,\r\n",
+       "        3.16024784e-03, -7.47932190e-04,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  1.08892279e+00, -1.22705647e+00,\r\n",
+       "       -5.29640509e-02, -1.28094313e-03,  3.54122613e-01,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00]
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:29:37.119254** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
filter: 'r_03'\r\n",
+       "exposure_time: 30\r\n",
+       "n_sequences: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: 'closed_loop_m2_b15'
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:29:29.884463** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
filter: \"r_03\"\r\n",
+       "exposure_time: 30.0\r\n",
+       "max_iter: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: \"closed_loop_m2_b15\"\r\n",
+       "used_dofs: [43, 44]\r\n",
+       "apply_corrections: true\r\n",
+       "use_ocps: true\r\n",
+       "mode: FAM
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:29:18.432060** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
M2_B15: 0.1
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:29:09.514689** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
dofs:\r\n",
+       " [-1.26588000e+03, -2.08991571e-08,  4.95136945e-08,  5.53397010e-03,\r\n",
+       "       -2.40923443e-03,  1.76380420e+03,  4.85815000e+03,  6.73960000e+02,\r\n",
+       "        3.16024784e-03, -7.47932190e-04,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  1.08892279e+00, -1.22705647e+00,\r\n",
+       "       -5.29640509e-02, -1.28094313e-03,  3.54122613e-01,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00]
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:15:05.584461** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
filter: 'r_03'\r\n",
+       "exposure_time: 30\r\n",
+       "n_sequences: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: 'closed_loop_m2_b14'
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:14:52.688773** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
filter: \"r_03\"\r\n",
+       "exposure_time: 30.0\r\n",
+       "max_iter: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: \"closed_loop_m2_b14\"\r\n",
+       "used_dofs: [43, 44]\r\n",
+       "apply_corrections: true\r\n",
+       "use_ocps: true\r\n",
+       "mode: FAM
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:11:50.688693** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
M2_B14: 0.1
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:11:41.706316** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
dofs:\r\n",
+       " [-1.26588000e+03, -2.08991571e-08,  4.95136945e-08,  5.53397010e-03,\r\n",
+       "       -2.40923443e-03,  1.76380420e+03,  4.85815000e+03,  6.73960000e+02,\r\n",
+       "        3.16024784e-03, -7.47932190e-04,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  1.08892279e+00, -1.22705647e+00,\r\n",
+       "       -5.29640509e-02, -1.28094313e-03,  3.54122613e-01,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00]
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:11:23.991029** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=788397162, ack=<SalRetCode.CMD_FAILED: -302>, error=1, result=\"Failed: Failed to apply correction to: ['m2']. \")\r\n",
+       "d\r\n",
+       "\r\n",
+       "\r\n",
+       "\r\n",
+       "\r\n",
+       "filter: \"r_03\"\r\n",
+       "exposure_time: 30.0\r\n",
+       "max_iter: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: \"closed_loop_m2_b14\"\r\n",
+       "used_dofs: [43, 44]\r\n",
+       "apply_corrections: true\r\n",
+       "use_ocps: true\r\n",
+       "mode: FAM
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:10:51.093300** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
M2_B14: 0.4
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:10:39.656661** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
dofs:\r\n",
+       " [-1.26588000e+03, -2.08991571e-08,  4.95136945e-08,  5.53397010e-03,\r\n",
+       "       -2.40923443e-03,  1.76380420e+03,  4.85815000e+03,  6.73960000e+02,\r\n",
+       "        3.16024784e-03, -7.47932190e-04,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  1.08892279e+00, -1.22705647e+00,\r\n",
+       "       -5.29640509e-02, -1.28094313e-03,  3.54122613e-01,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00]
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:10:23.576595** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
filter: 'r_03'\r\n",
+       "exposure_time: 30\r\n",
+       "n_sequences: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: 'closed_loop_m2_b13'
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:10:04.725985** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
filter: \"r_03\"\r\n",
+       "exposure_time: 30.0\r\n",
+       "max_iter: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: \"closed_loop_m2_b13\"\r\n",
+       "used_dofs: [42, 43]\r\n",
+       "apply_corrections: true\r\n",
+       "use_ocps: true\r\n",
+       "mode: FAM
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:04:24.068064** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
M2_B13: 0.5
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:04:11.862557** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
dofs:\r\n",
+       " [-1.26588000e+03, -2.08991571e-08,  4.95136945e-08,  5.53397010e-03,\r\n",
+       "       -2.40923443e-03,  1.76380420e+03,  4.85815000e+03,  6.73960000e+02,\r\n",
+       "        3.16024784e-03, -7.47932190e-04,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  1.08892279e+00, -1.22705647e+00,\r\n",
+       "       -5.29640509e-02, -1.28094313e-03,  3.54122613e-01,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00]
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:04:01.657665** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
filter: 'r_03'\r\n",
+       "exposure_time: 30\r\n",
+       "n_sequences: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: 'closed_loop_m2_b12'
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:03:48.263860** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
filter: \"r_03\"\r\n",
+       "exposure_time: 30.0\r\n",
+       "max_iter: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: \"closed_loop_m2_b12\"\r\n",
+       "used_dofs: [39, 40, 41]\r\n",
+       "apply_corrections: true\r\n",
+       "use_ocps: true\r\n",
+       "mode: FAM
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:03:35.121628** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
M2_B12: 0.5
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:03:20.575445** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
dofs:\r\n",
+       " [-1.26588000e+03, -2.08991571e-08,  4.95136945e-08,  5.53397010e-03,\r\n",
+       "       -2.40923443e-03,  1.76380420e+03,  4.85815000e+03,  6.73960000e+02,\r\n",
+       "        3.16024784e-03, -7.47932190e-04,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  1.08892279e+00, -1.22705647e+00,\r\n",
+       "       -5.29640509e-02, -1.28094313e-03,  3.54122613e-01,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00]
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 04:03:11.615535** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
filter: 'r_03'\r\n",
+       "exposure_time: 30\r\n",
+       "n_sequences: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: 'closed_loop_m2_b11'
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 03:56:01.476308** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
filter: \"r_03\"\r\n",
+       "exposure_time: 30.0\r\n",
+       "max_iter: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: \"closed_loop_m2_b11\"\r\n",
+       "mode: 'FAM'\r\n",
+       "used_dofs: [39, 40, 41]\r\n",
+       "apply_corrections: true\r\n",
+       "use_ocps: true
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 03:53:42.658280** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
M2_B11: 0.5
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 03:53:30.585564** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
dofs:\r\n",
+       " [-1.26588000e+03, -2.08991571e-08,  4.95136945e-08,  5.53397010e-03,\r\n",
+       "       -2.40923443e-03,  1.76380420e+03,  4.85815000e+03,  6.73960000e+02,\r\n",
+       "        3.16024784e-03, -7.47932190e-04,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  1.08892279e+00, -1.22705647e+00,\r\n",
+       "       -5.29640509e-02, -1.28094313e-03,  3.54122613e-01,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00]
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 03:53:16.707421** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
filter: 'r_03'\r\n",
+       "exposure_time: 30\r\n",
+       "n_sequences: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: 'closed_loop_m2_b10'
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 03:46:34.004780** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
filter: \"r_03\"\r\n",
+       "exposure_time: 30.0\r\n",
+       "max_iter: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: \"closed_loop_m2_b10\"\r\n",
+       "used_dofs: [39, 40, 41]\r\n",
+       "mode: 'FAM'\r\n",
+       "apply_corrections: true\r\n",
+       "use_ocps: true
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 03:46:23.202006** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
M2_B10: 0.5
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 03:46:12.031371** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
dofs:\r\n",
+       " [-1.26588000e+03, -2.08991571e-08,  4.95136945e-08,  5.53397010e-03,\r\n",
+       "       -2.40923443e-03,  1.76380420e+03,  4.85815000e+03,  6.73960000e+02,\r\n",
+       "        3.16024784e-03, -7.47932190e-04,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  1.08892279e+00, -1.22705647e+00,\r\n",
+       "       -5.29640509e-02, -1.28094313e-03,  3.54122613e-01,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00]
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 03:45:59.636251** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
filter: 'r_03'\r\n",
+       "exposure_time: 30\r\n",
+       "n_sequences: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: 'closed_loop_m2_b9'
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 03:44:01.068947** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
filter: \"r_03\"\r\n",
+       "exposure_time: 30.0\r\n",
+       "max_iter: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: \"closed_loop_m2_b9\"\r\n",
+       "used_dofs: [37, 38]\r\n",
+       "mode: 'FAM'\r\n",
+       "apply_corrections: true\r\n",
+       "use_ocps: true
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 03:40:41.662009** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
M2_B9: 0.5
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 03:40:29.259894** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
dofs:\r\n",
+       " [-1.26588000e+03, -2.08991571e-08,  4.95136945e-08,  5.53397010e-03,\r\n",
+       "       -2.40923443e-03,  1.76380420e+03,  4.85815000e+03,  6.73960000e+02,\r\n",
+       "        3.16024784e-03, -7.47932190e-04,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  1.08892279e+00, -1.22705647e+00,\r\n",
+       "       -5.29640509e-02, -1.28094313e-03,  3.54122613e-01,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00]
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 03:40:13.670872** Time Lost: 0.0; Time Lost Type: fault;\n", + " - components: ['MainTel']\n", + " - primary_software_components: ['None']\n", + " - primary_hardware_components: ['None']\n", + "\n", + "
filter: 'r_03'\r\n",
+       "exposure_time: 30\r\n",
+       "n_sequences: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: 'closed_loop_m2_b8'
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 03:39:41.673519** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Parameter March ComCam\r\n",
-       "\r\n",
-       "\r\n",
-       "exp_time: 30.0\r\n",
-       "dof_index: 3\r\n",
-       "range: 0.03\r\n",
-       "n_steps: 7\r\n",
-       "program: \"BLOCK-T92\"
\n" + "
B6 have issues with B8
\n" ], "text/plain": [ "" @@ -1926,86 +3603,37 @@ { "data": { "text/markdown": [ - "- **2024-11-10 04:08:37.884809** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-09 03:37:39.489312** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
Parameter March ComCam\r\n",
-       "\r\n",
-       "\r\n",
-       "exp_time: 30.0\r\n",
-       "dof_index: 3\r\n",
-       "range: 0.03\r\n",
-       "n_steps: 7\r\n",
-       "program: \"BLOCK-T92\"
\n", - "\n", + "
filter: \"r_03\"\r\n",
+       "exposure_time: 30.0\r\n",
+       "max_iter: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: \"closed_loop_m2_b8\"\r\n",
+       "used_dofs: [37, 38]\r\n",
+       "apply_corrections: true\r\n",
+       "use_ocps: true\r\n",
+       "mode: FAM
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/markdown": [ + "- **2024-11-09 03:37:29.895522** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "the comand was parameter march comcam\r\n", - "\r\n", - "\r\n", - "exp_time: 30.0\r\n", - "dof_index: 3\r\n", - "range: 0.04\r\n", - "n_steps: 7\r\n", - "program: \"BLOCK-T92\"\r\n", - "\r\n", - "\r\n", - "\r\n", - "\r\n", - "\r\n", - "\r\n", - "\r\n", - "\r\n", - "\r\n", - "\r\n", - "Error while trying to return telescope to its original position.\r\n", - "
Traceback (most recent call last):\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_externalscripts/python/lsst/ts/externalscripts/base_parameter_march.py", line 489, in cleanup\r\n",
-       "    await self.tcs.rem.mtaos.cmd_offsetDOF.start(data=offset_dof_data)\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 487, in start\r\n",
-       "    return await cmd_info.next_ackcmd(timeout=timeout)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 191, in next_ackcmd\r\n",
-       "    raise base.AckError(msg="Command failed", ackcmd=ackcmd)\r\n",
-       "lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=788397165, ack=&lt;SalRetCode.CMD_FAILED: -302&gt;, error=1, result="Failed: Failed to apply correction to: ['m2hex', 'camhex']. ")\r\n",
-       "\r\n",
-       "\r\n",
-       "2024/11/10 03:48:45 TAI\r\n",
-       "\r\n",
-       "\r\n",
-       "Returning telescope to original position by moving -0.02 back along dofs vector [0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\r\n",
-       " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\r\n",
-       " 0. 0.].\r\n",
-       "\r\n",
-       "\r\n",
-       "2024/11/10 03:48:45 TAI\r\n",
-       "\r\n",
-       "\r\n",
-       "Error in run\r\n",
-       "Traceback (most recent call last):\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py", line 631, in do_run\r\n",
-       "    await self._run_task\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/base_block_script.py", line 302, in run\r\n",
-       "    await self.run_block()\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_externalscripts/python/lsst/ts/externalscripts/base_parameter_march.py", line 477, in run_block\r\n",
-       "    await self.parameter_march()\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_externalscripts/python/lsst/ts/externalscripts/base_parameter_march.py", line 406, in parameter_march\r\n",
-       "    await self.take_images()\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_externalscripts/python/lsst/ts/externalscripts/maintel/parameter_march_comcam.py", line 109, in take_images\r\n",
-       "    await self.mtcs.offset_camera_hexapod(x=0, y=0, z=z_offset, u=0, v=0)\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/maintel/mtcs.py", line 2314, in offset_camera_hexapod\r\n",
-       "    await self.rem.mtaos.cmd_offsetDOF.start(\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 487, in start\r\n",
-       "    return await cmd_info.next_ackcmd(timeout=timeout)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 191, in next_ackcmd\r\n",
-       "    raise base.AckError(msg="Command failed", ackcmd=ackcmd)\r\n",
-       "lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=788397164, ack=&lt;SalRetCode.CMD_FAILED: -302&gt;, error=1, result="Failed: Failed to apply correction to: ['m2hex']. ")\r\n",
-       "
\n" + "
M2_B8: 0.5
\n" ], "text/plain": [ "" @@ -2017,18 +3645,15 @@ { "data": { "text/markdown": [ - "- **2024-11-10 04:02:46.350815** Time Lost: 0.0; Time Lost Type: fault;\n", + "- **2024-11-09 03:37:17.495582** Time Lost: 0.0; Time Lost Type: fault;\n", " - components: ['MainTel']\n", " - primary_software_components: ['None']\n", " - primary_hardware_components: ['None']\n", "\n", - "
set dof\r\n",
-       "\r\n",
-       "\r\n",
-       "dofs: \r\n",
-       "  [-1.26588000e+03, 0,  0, 0,\r\n",
-       "       0,  1.73617628e+03,  0,  0,\r\n",
-       "        0, 0,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "
dofs:\r\n",
+       " [-1.26588000e+03, -2.08991571e-08,  4.95136945e-08,  5.53397010e-03,\r\n",
+       "       -2.40923443e-03,  1.76380420e+03,  4.85815000e+03,  6.73960000e+02,\r\n",
+       "        3.16024784e-03, -7.47932190e-04,  0.00000000e+00,  0.00000000e+00,\r\n",
        "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
        "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
        "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
@@ -2050,12 +3675,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 03:57:01.973324** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 03:36:53.661397** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
we recover and enable de compensation mode of M2 and Camera Hexapods
\n"
+       "
RuntimeError: ComCam does not support CWFS mode.
\n"
       ],
       "text/plain": [
        ""
@@ -2067,12 +3692,16 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 03:53:33.982161** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 03:36:07.785062** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Camera Hexapod and M2 hexapod Fault
\n"
+       "
filter: 'r_03'\r\n",
+       "exposure_time: 30\r\n",
+       "n_sequences: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: 'closed_loop_m2_b8'
\n"
       ],
       "text/plain": [
        ""
@@ -2084,24 +3713,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 03:52:11.926367** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 03:35:53.756859** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
close loop \r\n",
-       "\r\n",
-       "\r\n",
-       "filter: \"r_03\"\r\n",
-       "exposure_time: 30\r\n",
-       "max_iter: 2\r\n",
-       "mode: \"FAM\"\r\n",
-       "program: \"BLOCK-T92\"\r\n",
-       "reason: \"SITCOM-826\"\r\n",
-       "note: \"hexapod_focus_z\"\r\n",
-       "used_dofs: [5]\r\n",
-       "apply_corrections: false\r\n",
-       "use_ocps: true
\n"
+       "
M2_B8: 0.5
\n"
       ],
       "text/plain": [
        ""
@@ -2113,24 +3730,46 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 03:51:41.374050** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 03:35:42.616659** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
close loop\r\n",
-       "\r\n",
-       "\r\n",
-       "filter: \"r_03\"\r\n",
+       "
dofs:\r\n",
+       " [-1.26588000e+03, -2.08991571e-08,  4.95136945e-08,  5.53397010e-03,\r\n",
+       "       -2.40923443e-03,  1.76380420e+03,  4.85815000e+03,  6.73960000e+02,\r\n",
+       "        3.16024784e-03, -7.47932190e-04,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  1.08892279e+00, -1.22705647e+00,\r\n",
+       "       -5.29640509e-02, -1.28094313e-03,  3.54122613e-01,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00]
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/markdown": [
+       "- **2024-11-09 03:35:04.895651** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "    - components: ['MainTel']\n",
+       "    - primary_software_components: ['None']\n",
+       "    - primary_hardware_components: ['None']\n",
+       "\n",
+       "
filter: 'r_03'\r\n",
        "exposure_time: 30\r\n",
-       "max_iter: 2\r\n",
-       "mode: \"FAM\"\r\n",
-       "program: \"BLOCK-T92\"\r\n",
-       "reason: \"SITCOM-826\"\r\n",
-       "note: \"hexapod_focus_z\"\r\n",
-       "used_dofs: [5]\r\n",
-       "apply_corrections: false\r\n",
-       "use_ocps: true
\n"
+       "n_sequences: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: 'closed_loop_m2_b7'
\n"
       ],
       "text/plain": [
        ""
@@ -2142,12 +3781,20 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 03:51:16.807621** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 03:33:39.871546** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
We are making the align sweep
\n"
+       "
filter: \"r_03\"\r\n",
+       "exposure_time: 30.0\r\n",
+       "max_iter: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: \"closed_loop_m2_b7\"\r\n",
+       "used_dofs: [35, 36]\r\n",
+       "mode: 'FAM'\r\n",
+       "apply_corrections: true\r\n",
+       "use_ocps: true
\n"
       ],
       "text/plain": [
        ""
@@ -2159,19 +3806,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 03:48:35.600009** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 03:28:36.507811** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
take image\r\n",
-       "\r\n",
-       "\r\n",
-       "exp_times: 30\r\n",
-       "nimages: 1\r\n",
-       "image_type: ACQ\r\n",
-       "filter: \"r_03\"\r\n",
-       "note: \"check\"
\n"
+       "
M2_B7: 1
\n"
       ],
       "text/plain": [
        ""
@@ -2183,18 +3823,15 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 03:47:08.771872** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 03:28:28.875609** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
set dof\r\n",
-       "\r\n",
-       "\r\n",
-       "dofs: \r\n",
-       "  [-1.26588000e+03, 0,  0, 0,\r\n",
-       "       0,  1.73617628e+03,  0,  0,\r\n",
-       "        0, 0,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "
dofs:\r\n",
+       " [-1.26588000e+03, -2.08991571e-08,  4.95136945e-08,  5.53397010e-03,\r\n",
+       "       -2.40923443e-03,  1.76380420e+03,  4.85815000e+03,  6.73960000e+02,\r\n",
+       "        3.16024784e-03, -7.47932190e-04,  0.00000000e+00,  0.00000000e+00,\r\n",
        "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
        "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
        "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
@@ -2216,19 +3853,16 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 03:39:59.414930** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 03:28:18.845520** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
take image comcam\r\n",
-       "\r\n",
-       "\r\n",
-       "exp_times: 30\r\n",
-       "nimages: 1\r\n",
-       "image_type: ACQ\r\n",
-       "filter: \"r_03\"\r\n",
-       "note: \"check\"
\n"
+       "
filter: 'r_03'\r\n",
+       "exposure_time: 30\r\n",
+       "n_sequences: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: 'closed_loop_m2_b6'
\n"
       ],
       "text/plain": [
        ""
@@ -2240,12 +3874,20 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 03:38:09.276826** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 03:27:55.375864** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
we move to other field and we set new dof
\n"
+       "
filter: \"r_03\"\r\n",
+       "exposure_time: 30.0\r\n",
+       "max_iter: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: \"closed_loop_m2_b6\"\r\n",
+       "used_dofs: [35, 36]\r\n",
+       "mode: 'FAM'\r\n",
+       "apply_corrections: true\r\n",
+       "use_ocps: true
\n"
       ],
       "text/plain": [
        ""
@@ -2257,12 +3899,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 03:37:33.078494** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 03:27:47.572524** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
We recover ComCam, restarting the deamond via CCS
\n"
+       "
M2_B6: 1
\n"
       ],
       "text/plain": [
        ""
@@ -2274,58 +3916,77 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 03:37:33.078494** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 03:27:38.377713** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
We recover ComCam, restarting the deamond via CCS
\n",
-       "\n",
+       "
dofs:\r\n",
+       " [-1.26588000e+03, -2.08991571e-08,  4.95136945e-08,  5.53397010e-03,\r\n",
+       "       -2.40923443e-03,  1.76380420e+03,  4.85815000e+03,  6.73960000e+02,\r\n",
+       "        3.16024784e-03, -7.47932190e-04,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  1.08892279e+00, -1.22705647e+00,\r\n",
+       "       -5.29640509e-02, -1.28094313e-03,  3.54122613e-01,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00]
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/markdown": [
+       "- **2024-11-09 03:27:24.975762** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "Brian is debugging the CCamera error but failed\r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "\r\n",
-       "2024/11/10 03:16:49 TAI\r\n",
-       "\r\n",
-       "\r\n",
-       "Configure started\r\n",
-       "\r\n",
-       "\r\n",
-       "2024/11/10 03:16:49 TAI\r\n",
-       "\r\n",
-       "\r\n",
-       "Read historical data in 0.00 sec\r\n",
-       "\r\n",
-       "\r\n",
-       "2024/11/10 03:16:50 TAI\r\n",
-       "\r\n",
-       "\r\n",
-       "Setting final state to <ScriptState.FAILED: 10>\r\n",
-       "\r\n",
-       "\r\n",
-       "2024/11/10 03:16:50 TAI\r\n",
+       "
take aos sequence\r\n",
        "\r\n",
        "\r\n",
-       "Error in run\r\n",
-       "
Traceback (most recent call last):\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py", line 631, in do_run\r\n",
-       "    await self._run_task\r\n",
-       "  File "/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/run_command.py", line 170, in run\r\n",
-       "    await getattr(self.remote, f"cmd_{self.cmd}").start(\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 487, in start\r\n",
-       "    return await cmd_info.next_ackcmd(timeout=timeout)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File "/opt/lsst/software/stack/conda/envs/lsst-scipipe-9.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py", line 191, in next_ackcmd\r\n",
-       "    raise base.AckError(msg="Command failed", ackcmd=ackcmd)\r\n",
-       "lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=886868127, ack=&lt;SalRetCode.CMD_FAILED: -302&gt;, error=0, result='Error : Switching to Normal mode is not allowed when the AlertState is ALARM: ALARM')\r\n",
+       "filter: 'r_03'\r\n",
+       "exposure_time: 30\r\n",
+       "n_sequences: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: 'closed_loop_m2_b5'
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/markdown": [
+       "- **2024-11-09 03:27:00.459358** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "    - components: ['MainTel']\n",
+       "    - primary_software_components: ['None']\n",
+       "    - primary_hardware_components: ['None']\n",
+       "\n",
+       "
close loop comcam\r\n",
        "\r\n",
        "\r\n",
-       "
\n"
+       "filter: \"r_03\"\r\n",
+       "exposure_time: 30.0\r\n",
+       "max_iter: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: \"closed_loop_m2_b5\"\r\n",
+       "used_dofs: [30, 31, 32, 33, 34]\r\n",
+       "apply_corrections: true\r\n",
+       "mode: 'FAM'\r\n",
+       "use_ocps: true
\n"
       ],
       "text/plain": [
        ""
@@ -2337,17 +3998,48 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 03:15:16.058419** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 03:26:41.403324** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
enabled failed :\r\n",
+       "
apply dof \r\n",
        "\r\n",
        "\r\n",
+       "M2_B5: 0.5
\n"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/markdown": [
+       "- **2024-11-09 03:26:19.342172** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "    - components: ['MainTel']\n",
+       "    - primary_software_components: ['None']\n",
+       "    - primary_hardware_components: ['None']\n",
+       "\n",
+       "
set dof \r\n",
        "\r\n",
        "\r\n",
-       "lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=886868127, ack=<SalRetCode.CMD_FAILED: -302>, error=0, result='Error : Switching to Normal mode is not allowed when the AlertState is ALARM: ALARM')
\n"
+       "dofs:\r\n",
+       " [-1.26588000e+03, -2.08991571e-08,  4.95136945e-08,  5.53397010e-03,\r\n",
+       "       -2.40923443e-03,  1.76380420e+03,  4.85815000e+03,  6.73960000e+02,\r\n",
+       "        3.16024784e-03, -7.47932190e-04,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  1.08892279e+00, -1.22705647e+00,\r\n",
+       "       -5.29640509e-02, -1.28094313e-03,  3.54122613e-01,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00]
\n"
       ],
       "text/plain": [
        ""
@@ -2359,16 +4051,19 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 03:15:00.748224** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 03:25:47.578324** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
we try to send to enable CCCamera with run command\r\n",
+       "
take aos sequence comcam\r\n",
        "\r\n",
        "\r\n",
-       "component: CCCamera\r\n",
-       "cmd: enterControl
\n"
+       "filter: 'r_03'\r\n",
+       "exposure_time: 30\r\n",
+       "n_sequences: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: 'closed_loop_m2_b4'
\n"
       ],
       "text/plain": [
        ""
@@ -2380,15 +4075,23 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 03:12:59.823615** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 03:25:25.171060** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
CCCamera fault this probably is an filter issues and the rotator was at 70 degress\r\n",
+       "
close loop comcam\r\n",
        "\r\n",
        "\r\n",
-       "MCM has entered fault state. Cause: Controlled subsystem comcam-rebpower has gone into FAULT. Cause: ALARM: Channel R22/Reb1/hvbias/IbefSwch value 0.1536 above high limit 0.15
\n"
+       "filter: \"r_03\"\r\n",
+       "exposure_time: 30.0\r\n",
+       "max_iter: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: \"closed_loop_m2_b4\"\r\n",
+       "used_dofs: [32, 33]\r\n",
+       "apply_corrections: true\r\n",
+       "mode: 'FAM'\r\n",
+       "use_ocps: true
\n"
       ],
       "text/plain": [
        ""
@@ -2400,15 +4103,15 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 03:06:42.580422** Time Lost: 0.0; Time Lost Type: fault;\n",
-       "    - components: None\n",
+       "- **2024-11-09 03:25:08.720926** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Tried to take a u-band image of NGC 55 and then ComCam faulted \r\n",
+       "
apply dof:\r\n",
        "\r\n",
        "\r\n",
-       "MCM has entered fault state. Cause: Controlled subsystem comcam-fcs has gone into FAULT. Cause: position after move off by -0.145000 mm
\n"
+       "M2_B4: 1
\n"
       ],
       "text/plain": [
        ""
@@ -2420,16 +4123,51 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 02:53:53.112357** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 03:24:48.369776** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
MTptg fualt\r\n",
+       "
set dof\r\n",
        "\r\n",
        "\r\n",
-       "Error MTMount replied error for trackTarget command (1735083855) Error no:1 message:Failed: Command with id 2433374 has timed o\r\n",
-       "0
\n"
+       "dofs:\r\n",
+       " [-1.26588000e+03, -2.08991571e-08,  4.95136945e-08,  5.53397010e-03,\r\n",
+       "       -2.40923443e-03,  1.76380420e+03,  4.85815000e+03,  6.73960000e+02,\r\n",
+       "        3.16024784e-03, -7.47932190e-04,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  1.08892279e+00, -1.22705647e+00,\r\n",
+       "       -5.29640509e-02, -1.28094313e-03,  3.54122613e-01,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00]\r\n",
+       "\r\n",
+       "\r\n",
+       "close loop comcam\r\n",
+       "filter: \"r_03\"\r\n",
+       "exposure_time: 30.0\r\n",
+       "max_iter: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "reason: \"SITCOM-826\"\r\n",
+       "note: \"closed_loop_m2_b3\"\r\n",
+       "used_dofs: [32, 33]\r\n",
+       "apply_corrections: true\r\n",
+       "mode: 'FAM'\r\n",
+       "use_ocps: true\r\n",
+       "\r\n",
+       "\r\n",
+       "take aos sequece comcam\r\n",
+       "\r\n",
+       "\r\n",
+       "filter: 'r_03'\r\n",
+       "exposure_time: 30\r\n",
+       "n_sequences: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: 'closed_loop_m2_b3'
\n"
       ],
       "text/plain": [
        ""
@@ -2441,12 +4179,15 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 02:42:17.563402** Time Lost: 0.0; Time Lost Type: fault;\n",
-       "    - components: ['MainTel']\n",
+       "- **2024-11-09 03:10:06.169362** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "    - components: None\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
m1m3 fault, to much tension in the bending mode
\n"
+       "
we apply M2_B2: 1 and \r\n",
+       "\r\n",
+       "\r\n",
+       "M2_B1: 1 plot a diference in chonograf
\n"
       ],
       "text/plain": [
        ""
@@ -2458,12 +4199,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 02:20:08.398776** Time Lost: 0.0; Time Lost Type: fault;\n",
-       "    - components: ['MainTel']\n",
+       "- **2024-11-09 03:00:34.791986** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "    - components: None\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
We have a persistent Alarm in M2 Axial force error out of normal range. MTForceError.MTM2
\n"
+       "
take aos sequence comcam
\n"
       ],
       "text/plain": [
        ""
@@ -2475,12 +4216,15 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 02:17:44.398008** Time Lost: 0.0; Time Lost Type: fault;\n",
-       "    - components: ['MainTel']\n",
+       "- **2024-11-09 03:00:12.905070** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "    - components: None\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
We are making a second iteration for M2 for all the modes
\n"
+       "
apply _dof\r\n",
+       "\r\n",
+       "\r\n",
+       "M2_B3: 1
\n"
       ],
       "text/plain": [
        ""
@@ -2492,16 +4236,30 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 02:10:42.177737** Time Lost: 0.0; Time Lost Type: fault;\n",
-       "    - components: ['MainTel']\n",
+       "- **2024-11-09 02:59:55.409990** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "    - components: None\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
exp_times: 30\r\n",
-       "nimages: 1\r\n",
-       "image_type: \"ACQ\"\r\n",
-       "program: \"BLOCK-T249\"\r\n",
-       "filter: \"r_03\"
\n"
+       "
we set \r\n",
+       "\r\n",
+       "\r\n",
+       "\r\n",
+       "\r\n",
+       "dofs:\r\n",
+       " [-1.26588000e+03, -2.08991571e-08,  4.95136945e-08,  5.53397010e-03,\r\n",
+       "       -2.40923443e-03,  1.76380420e+03,  4.85815000e+03,  6.73960000e+02,\r\n",
+       "        3.16024784e-03, -7.47932190e-04,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  1.08892279e+00, -1.22705647e+00,\r\n",
+       "       -5.29640509e-02, -1.28094313e-03,  3.54122613e-01,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00]
\n"
       ],
       "text/plain": [
        ""
@@ -2513,20 +4271,19 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 02:10:33.153659** Time Lost: 0.0; Time Lost Type: fault;\n",
-       "    - components: ['MainTel']\n",
+       "- **2024-11-09 02:59:25.689320** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "    - components: None\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
filter: \"r_03\"\r\n",
-       "exposure_time: 30.0\r\n",
-       "max_iter: 2\r\n",
-       "program: BLOCK-T249\r\n",
-       "mode: \"FAM\"\r\n",
-       "note: \"closed_loop_m2\"\r\n",
-       "used_dofs: [5, 30, 31, 32, 33, 34, 35, 36, 37, 38]\r\n",
-       "apply_corrections: true\r\n",
-       "use_ocps: true
\n"
+       "
we take a sequence \r\n",
+       "\r\n",
+       "\r\n",
+       "filter: 'r_03'\r\n",
+       "exposure_time: 30\r\n",
+       "n_sequences: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: 'closed_loop_m2_b2'
\n"
       ],
       "text/plain": [
        ""
@@ -2538,16 +4295,27 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 02:10:23.962634** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 02:47:46.053009** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
exp_times: 30\r\n",
-       "nimages: 1\r\n",
-       "image_type: \"ACQ\"\r\n",
-       "program: \"BLOCK-T249\"\r\n",
-       "filter: \"r_03\"
\n"
+       "
Best dof:\r\n",
+       "\r\n",
+       "\r\n",
+       "[-1.26588000e+03, -2.08991571e-08,  4.95136945e-08,  5.53397010e-03,\r\n",
+       "       -2.40923443e-03,  1.76380420e+03,  4.85815000e+03,  6.73960000e+02,\r\n",
+       "        3.16024784e-03, -7.47932190e-04,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  1.08892279e+00, -1.22705647e+00,\r\n",
+       "       -5.29640509e-02, -1.28094313e-03,  3.54122613e-01,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00]
\n"
       ],
       "text/plain": [
        ""
@@ -2559,19 +4327,22 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 02:10:08.361919** Time Lost: 0.0; Time Lost Type: fault;\n",
-       "    - components: ['MainTel']\n",
+       "- **2024-11-09 02:46:25.593499** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "    - components: None\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
filter: \"r_03\"\r\n",
+       "
close loop comcam\r\n",
+       "\r\n",
+       "\r\n",
+       "filter: \"r_03\"\r\n",
        "exposure_time: 30.0\r\n",
-       "max_iter: 2\r\n",
-       "program: BLOCK-T249\r\n",
-       "mode: \"FAM\"\r\n",
-       "note: \"closed_loop_m2\"\r\n",
-       "used_dofs: [30, 31, 32, 33, 34, 35, 36, 37, 38]\r\n",
+       "max_iter: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: \"closed_loop_m2_b2\"\r\n",
+       "used_dofs: [30, 31]\r\n",
        "apply_corrections: true\r\n",
+       "mode: 'FAM'\r\n",
        "use_ocps: true
\n"
       ],
       "text/plain": [
@@ -2584,52 +4355,15 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 01:52:50.171392** Time Lost: 0.0; Time Lost Type: fault;\n",
-       "    - components: ['MainTel']\n",
-       "    - primary_software_components: ['None']\n",
-       "    - primary_hardware_components: ['None']\n",
-       "\n",
-       "
take image\r\n",
-       "\r\n",
-       "\r\n",
-       "exp_times: 30\r\n",
-       "nimages: 1\r\n",
-       "image_type: \"ACQ\"\r\n",
-       "program: \"BLOCK-T249\"\r\n",
-       "filter: \"r_03\"
\n"
-      ],
-      "text/plain": [
-       ""
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/markdown": [
-       "- **2024-11-10 01:52:02.027183** Time Lost: 0.0; Time Lost Type: fault;\n",
-       "    - components: ['MainTel']\n",
+       "- **2024-11-09 02:46:03.387698** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "    - components: None\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
apply dof\r\n",
+       "
best state for the dof, we apply dof\r\n",
        "\r\n",
        "\r\n",
-       "dofs:\r\n",
-       " [-1.26588000e+03, -2.08991571e-08,  4.95136945e-08,  5.53397010e-03,\r\n",
-       "       -2.40923443e-03,  1.80668450e+03,  4.85815000e+03,  6.73960000e+02,\r\n",
-       "        3.16024784e-03, -7.47932190e-04,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  1.08892279e+00, -1.22705647e+00,\r\n",
-       "       -5.29640509e-02, -1.28094313e-03,  3.54122613e-01,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00]
\n"
+       "M2_B2: 1
\n"
       ],
       "text/plain": [
        ""
@@ -2641,19 +4375,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 01:51:38.095241** Time Lost: 0.0; Time Lost Type: fault;\n",
-       "    - components: ['MainTel']\n",
+       "- **2024-11-09 02:42:52.936473** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "    - components: None\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
take aos sequence \r\n",
-       "\r\n",
-       "\r\n",
-       "filter: 'r_03'\r\n",
-       "exposure_time: 30\r\n",
-       "n_sequences: 1\r\n",
-       "program: 'BLOCK-T273'\r\n",
-       "note: 'closed_loop_m1m2_b3b7'
\n"
+       "
checking the image with take AOS sequence
\n"
       ],
       "text/plain": [
        ""
@@ -2665,21 +4392,21 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 01:51:10.866632** Time Lost: 0.0; Time Lost Type: fault;\n",
-       "    - components: ['MainTel']\n",
+       "- **2024-11-09 02:37:05.804300** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "    - components: None\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
close loop\r\n",
+       "
looking for a correction on Z  close loop comcam\r\n",
        "\r\n",
        "\r\n",
        "filter: \"r_03\"\r\n",
        "exposure_time: 30.0\r\n",
        "max_iter: 1\r\n",
-       "program: BLOCK-T273\r\n",
-       "mode: \"FAM\"\r\n",
-       "note: \"closed_loop_m1m3_b3b7\"\r\n",
-       "used_dofs: [10, 11, 12, 13, 14, 15, 16]\r\n",
+       "mode: 'FAM'\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: \"closed_loop_cam_z\"\r\n",
+       "used_dofs: [5]\r\n",
        "apply_corrections: true\r\n",
        "use_ocps: true
\n"
       ],
@@ -2693,16 +4420,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 01:50:54.798269** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 02:26:06.719836** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
apply dof\r\n",
-       "\r\n",
-       "\r\n",
-       "M1M3_B3: 1\r\n",
-       "M1M3_B7: 1
\n"
+       "
apply dof
\n"
       ],
       "text/plain": [
        ""
@@ -2714,17 +4437,17 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 01:50:35.176076** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 02:25:34.510310** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
set dof\r\n",
+       "
set Dof\r\n",
        "\r\n",
        "\r\n",
        "dofs:\r\n",
        " [-1.26588000e+03, -2.08991571e-08,  4.95136945e-08,  5.53397010e-03,\r\n",
-       "       -2.40923443e-03,  1.80668450e+03,  4.85815000e+03,  6.73960000e+02,\r\n",
+       "       -2.40923443e-03,  1.72766900e+03,  4.85815000e+03,  6.73960000e+02,\r\n",
        "        3.16024784e-03, -7.47932190e-04,  0.00000000e+00,  0.00000000e+00,\r\n",
        "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
        "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
@@ -2747,19 +4470,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 01:50:07.473244** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 02:25:18.093071** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
take aos sequence\r\n",
-       "\r\n",
-       "\r\n",
-       "filter: 'r_03'\r\n",
-       "exposure_time: 30\r\n",
-       "n_sequences: 1\r\n",
-       "program: 'BLOCK-T227'\r\n",
-       "note: 'closed_loop_m1m2_b1b4'
\n"
+       "
enable dome following
\n"
       ],
       "text/plain": [
        ""
@@ -2771,28 +4487,22 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 01:49:18.023116** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 02:20:33.937835** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
set dof\r\n",
-       "\r\n",
-       "\r\n",
-       "dofs:\r\n",
-       " [-1.26588000e+03, -2.08991571e-08,  4.95136945e-08,  5.53397010e-03,\r\n",
-       "       -2.40923443e-03,  1.80668450e+03,  4.85815000e+03,  6.73960000e+02,\r\n",
-       "        3.16024784e-03, -7.47932190e-04,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  1.08892279e+00, -1.22705647e+00,\r\n",
-       "       -5.29640509e-02, -1.28094313e-03,  3.54122613e-01,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00]
\n"
+       "
we track target to\r\n",
+       "track_azel:\r\n",
+       " az: -70\r\n",
+       " el: 75\r\n",
+       "rot_value: 0\r\n",
+       "rot_type: PhysicalSky\r\n",
+       "program: \"BLOCK-T250\"\r\n",
+       "reason: \"TMACheckout\"\r\n",
+       "ignore:\r\n",
+       " - mtdome\r\n",
+       " - mtdometrajectory
\n"
       ],
       "text/plain": [
        ""
@@ -2804,28 +4514,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 01:48:51.693368** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 02:13:14.929887** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
set dof\r\n",
-       "\r\n",
-       "\r\n",
-       "dofs:\r\n",
-       " [-1.26588000e+03, -2.08991571e-08,  4.95136945e-08,  5.53397010e-03,\r\n",
-       "       -2.40923443e-03,  1.80668450e+03,  4.85815000e+03,  6.73960000e+02,\r\n",
-       "        3.16024784e-03, -7.47932190e-04,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  1.08892279e+00, -1.22705647e+00,\r\n",
-       "       -5.29640509e-02, -1.28094313e-03,  3.54122613e-01,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00]
\n"
+       "
the close loop fail ,  and move the hexapod out of focus so now we will set dof again
\n"
       ],
       "text/plain": [
        ""
@@ -2837,17 +4531,15 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 01:48:30.595705** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 02:09:23.452470** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
take aos sequence \r\n",
-       "filter: 'r_03'\r\n",
-       "exposure_time: 30\r\n",
-       "n_sequences: 1\r\n",
-       "program: 'BLOCK-T227'\r\n",
-       "note: 'closed_loop_m1m2_b1b4'
\n"
+       "
Mtmount goe3s to fault and MTPtg\r\n",
+       "\r\n",
+       "\r\n",
+       "Command BothAxesStop(sequence_id=981994, command_code=<CommandCode.BOTH_AXES_STOP: 32>, source=<Source.CSC: 1>, timestamp=1731118098.0973594)=b'981994\\n32\\n1\\n1731118098.0973594\\r\\n' rejected: The commands allowed in Fault are: Reset
\n"
       ],
       "text/plain": [
        ""
@@ -2859,13 +4551,22 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 01:48:08.273193** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 02:07:15.980266** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
set dof\r\n",
-       "apply dof
\n"
+       "
running again close loop comcam\r\n",
+       "\r\n",
+       "\r\n",
+       "filter: \"r_03\"\r\n",
+       "exposure_time: 30.0\r\n",
+       "max_iter: 1\r\n",
+       "mode: 'FAM'\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: \"closed_loop_m2_b1\"\r\n",
+       "used_dofs: [30, 31]\r\n",
+       "apply_corrections: true
\n"
       ],
       "text/plain": [
        ""
@@ -2877,19 +4578,13 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 01:46:11.171459** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 02:06:21.075111** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
aos sequence\r\n",
-       "\r\n",
-       "\r\n",
-       "filter: 'r_03'\r\n",
-       "exposure_time: 30\r\n",
-       "n_sequences: 1\r\n",
-       "program: 'BLOCK-T227'\r\n",
-       "note: 'closed_loop_m1m2_b1b4'
\n"
+       "
lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=1434847998, ack=<SalRetCode.CMD_NOPERM: -300>, error=0, result='Ack : NO Command rejected: Invalid filter: r')\r\n",
+       "d
\n"
       ],
       "text/plain": [
        ""
@@ -2901,16 +4596,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 01:41:09.381875** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 02:06:16.548304** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
apply dof\r\n",
-       "\r\n",
-       "\r\n",
-       "M1M3_B1: 1\r\n",
-       "M1M3_B4: 1
\n"
+       "
Running again T270
\n"
       ],
       "text/plain": [
        ""
@@ -2922,12 +4613,13 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 01:40:37.020662** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 02:02:54.349712** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
close loop comcam
\n"
+       "
the last script fail\r\n",
+       "RuntimeError: ComCam does not support CWFS mode.
\n"
       ],
       "text/plain": [
        ""
@@ -2939,12 +4631,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 01:40:26.628887** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 02:02:12.101126** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
apply dof
\n"
+       "
Before swaping to Carlos and Kate, here is a brief summary: After homing the MTDome, we obtained a couple of images with take_image_comcam to verify the position and focus. We took one image each in the g, r, and i bands to check if laser was visible (images 50, 51, 52). Following this, we executed BLOCK-T235 three times, which involved taking triplets and verifying the images. We then proceeded with preparations for the AOS closed-loop M2 science verification (BLOCK-T270). The telescope and dome were set to their home positions, and tracking was enabled.
\n"
       ],
       "text/plain": [
        ""
@@ -2956,12 +4648,21 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 01:23:57.635380** Time Lost: 0.0; Time Lost Type: fault;\n",
-       "    - components: None\n",
+       "- **2024-11-09 02:00:40.985826** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
We cycle MTAOS to update M1M3 sing
\n"
+       "
clsoe loop comcam\r\n",
+       "\r\n",
+       "\r\n",
+       "filter: \"r\"\r\n",
+       "exposure_time: 30.0\r\n",
+       "max_iter: 1\r\n",
+       "program: BLOCK-T270\r\n",
+       "note: \"closed_loop_m2_b1\"\r\n",
+       "used_dofs: [30, 31]\r\n",
+       "apply_corrections: true
\n"
       ],
       "text/plain": [
        ""
@@ -2973,12 +4674,15 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 01:12:55.916923** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 02:00:05.531463** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
moving to T273. same test but for M1M3
\n"
+       "
Apply dof\r\n",
+       "\r\n",
+       "\r\n",
+       "M2_B1: 1
\n"
       ],
       "text/plain": [
        ""
@@ -2990,12 +4694,19 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 01:09:14.380827** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 01:57:47.720054** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
set_dof, apply_dof, close_loop_comcam again
\n"
+       "
take aos sequence\r\n",
+       "\r\n",
+       "\r\n",
+       "ilter: 'r_03'\r\n",
+       "exposure_time: 30\r\n",
+       "n_sequences: 1\r\n",
+       "program: 'BLOCK-T270'\r\n",
+       "note: 'triplet_check_2'
\n"
       ],
       "text/plain": [
        ""
@@ -3007,12 +4718,26 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 01:06:56.460239** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 01:56:59.556554** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
el=60, az=-125
\n"
+       "
Set DOF\r\n",
+       "dofs:\r\n",
+       " [-1.26588000e+03, -2.08991571e-08,  4.95136945e-08,  5.53397010e-03,\r\n",
+       "       -2.40923443e-03,  1.72766900e+03,  4.85815000e+03,  6.73960000e+02,\r\n",
+       "        3.16024784e-03, -7.47932190e-04,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  1.08892279e+00, -1.22705647e+00,\r\n",
+       "       -5.29640509e-02, -1.28094313e-03,  3.54122613e-01,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
+       "        0.00000000e+00,  0.00000000e+00]
\n"
       ],
       "text/plain": [
        ""
@@ -3024,13 +4749,18 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 01:06:20.093188** Time Lost: 0.0; Time Lost Type: fault;\n",
-       "    - components: ['MainTel']\n",
+       "- **2024-11-09 01:53:03.880591** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "    - components: None\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Found out Rotator was also disabled. \r\n",
-       "Recovering, home both axis, track_target again
\n"
+       "
focus seep\r\n",
+       "axis: z\r\n",
+       "focus_window: 300\r\n",
+       "n_steps: 9\r\n",
+       "program: \"BLOCK-T92\"\r\n",
+       "hexapod: \"Camera\"\r\n",
+       "reason: \"focus_sweep_cam_dz\"
\n"
       ],
       "text/plain": [
        ""
@@ -3042,12 +4772,15 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 01:02:32.357890** Time Lost: 0.0; Time Lost Type: fault;\n",
-       "    - components: ['MainTel']\n",
+       "- **2024-11-09 01:50:02.333038** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "    - components: None\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
MTMount and MTPtg faulted. Maybe we had to stop tracking when cycling MTAOS?
\n"
+       "
off set camera hexapod\r\n",
+       "\r\n",
+       "\r\n",
+       "z:300
\n"
       ],
       "text/plain": [
        ""
@@ -3059,12 +4792,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 01:01:06.664005** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 01:49:41.578522** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Cycling MTAOS
\n"
+       "
Runing BLOCK-T92 we run a focus sweep_comcam
\n"
       ],
       "text/plain": [
        ""
@@ -3076,12 +4809,14 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 00:59:32.237588** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 01:48:45.942842** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
set_dof again
\n"
+       "
Correction, instead of BLOCK-T270, we will start BLOCK-T249.  Because we are out of focus.\r\n",
+       "aply degres of freeedom\r\n",
+       "take image
\n"
       ],
       "text/plain": [
        ""
@@ -3093,12 +4828,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 00:56:23.917829** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 01:12:46.383461** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Running T270. comparing +-90
\n"
+       "
MTMount and MTPtg failed
\n"
       ],
       "text/plain": [
        ""
@@ -3110,12 +4845,13 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 00:51:06.718661** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 01:10:31.070559** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Cycling MTAOS
\n"
+       "
Following Guillem's words - we will start a new test case BLOCK-T270.\r\n",
+       "Since we are not in focus in Z, we will close the aos loop and correct the focus in z.
\n"
       ],
       "text/plain": [
        ""
@@ -3127,12 +4863,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 00:50:23.106805** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 00:58:06.163820** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
NOTE: New morning az limit = 220
\n"
+       "
Starting triplet sequence
\n"
       ],
       "text/plain": [
        ""
@@ -3144,12 +4880,19 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 00:33:53.099455** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 00:34:20.452661** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Runnign T249
\n"
+       "
From Guillem: \r\n",
+       "\r\n",
+       "\r\n",
+       "Here is the state 20251025 for the hexapods, when it seemed there was no spherical, to understand if the spherical was due to our own misalignments or come from some temperature effect:set_dof.py\r\n",
+       "\r\n",
+       "\r\n",
+       "Cam_z: 1056.\r\n",
+       "Cam_rx: -0.0333
\n"
       ],
       "text/plain": [
        ""
@@ -3161,12 +4904,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 00:27:52.869196** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 00:32:28.868650** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
we stopped T249 thinking we messed up the hexapod (actually we didn't), now since it is stopped we need to redo the set_dof and rerun close_loop_comcam
\n"
+       "
Tiago is still trying to recover the dome and we are a bit vignetted. So we will take a sequence of images in all the filters to check if the laser is visible at all.
\n"
       ],
       "text/plain": [
        ""
@@ -3178,13 +4921,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 00:20:59.939925** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 00:23:58.266917** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
it was too close to moon!\r\n",
-       "moving to el=70, az=-70
\n"
+       "
We too a few darks and we will start getting a triplet.
\n"
       ],
       "text/plain": [
        ""
@@ -3196,12 +4938,28 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 00:11:17.546614** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 00:23:40.825758** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
BLOCK-T235 started.
\n"
+       "
Continuing the summary...\r\n",
+       "\r\n",
+       "\r\n",
+       "Once M1M3 was recovered, we started preparation to go on sky with twilight flats. The temperatures inside, outside, and on M1M3 seemed fine and we proceeded to open the dome shutter near 20h CLT approx. Maybe a bit later. \r\n",
+       "\r\n",
+       "\r\n",
+       "We aligned the telescope using the laser tracker and poked the dome to ensure it was working fine. Everything seemed ok until here. We fired up the block for twilight flats. It did not start taking flats in the first two attempts because there was too much light. We changed the minimum exposure time to 0.1 s using the new parameter that Elana added to the script. That worked. We started taking twilight flats successfully. \r\n",
+       "\r\n",
+       "\r\n",
+       "One detail to keep in mind is that the images were all flagged as `acq` instead of `flats`. We might need to review that. \r\n",
+       "\r\n",
+       "\r\n",
+       "On top of that, near image 10 in the flat sequence, the dome started presenting problems again. The watcher alarm that Wouter implemented worked very well. We started the faults with problems on the drives followed by ethercat issues. \r\n",
+       "\r\n",
+       "\r\n",
+       "We needed to reboot the cRIO remotely to recover connection. However, the telemetry was still slow. \r\n",
+       "Right now, we are in a situation similar to what happened yesterday. But we are trying to get on sky anyway.
\n"
       ],
       "text/plain": [
        ""
@@ -3213,25 +4971,18 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 00:10:28.439117** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 00:15:11.590150** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
since the hexapod faulted, putting set_dof again:\r\n",
-       "[-1.26588000e+03, -2.08991571e-08,  4.95136945e-08,  5.53397010e-03,\r\n",
-       "       -2.40923443e-03,  1.80668450e+03,  4.85815000e+03,  6.73960000e+02,\r\n",
-       "        3.16024784e-03, -7.47932190e-04,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  1.08892279e+00, -1.22705647e+00,\r\n",
-       "       -5.29640509e-02, -1.28094313e-03,  3.54122613e-01,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00,  0.00000000e+00,  0.00000000e+00,\r\n",
-       "        0.00000000e+00,  0.00000000e+00]
\n"
+       "
Let me try to summarize everything that happened at the end of the day and beginning of the night. This might require more than one OLE entry. Bear with me. \r\n",
+       "\r\n",
+       "\r\n",
+       "In the afternoon, after we ran bump tests we performed the last checkouts. When getting ready to perform a few slew-and-track movements, just to check the health of the system, the M1M3 CSC crashed. That triggered a Support System Heartbeat Loss. Brian connected to the M1M3 cRIO and recovered the CSC. A few minutes later (half hour?), the M1M3 CSC crashed again. That happened during the Evening Tailgate Meeting. Tiago and Erik stayed longer with us (summit) and Tiago explained a bit more the issues with DDS and the importance of avoiding using EUIs and Jupyter Notebooks to control the system via DDS. \r\n",
+       "\r\n",
+       "\r\n",
+       "A few minutes after the Evening Tailgate Meeting (near 18:15 CLT - ish), the M1M3 CSC crashed again but it seems to be something related to one of the M1M3 ILCs timeout. The CSC was running but, for unknown reason, the daemon was not connected. Petr helped recovering.
\n"
       ],
       "text/plain": [
        ""
@@ -3243,12 +4994,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 00:06:39.845478** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 23:58:50.814807** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
enable dome following
\n"
+       "
we weren't tracking, MTPtg faulted. recover and track_trarget.
\n"
       ],
       "text/plain": [
        ""
@@ -3260,12 +5011,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 00:05:11.472746** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 23:57:00.374461** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Let's not point to the west next time
\n"
+       "
5 more acq, 10 darks
\n"
       ],
       "text/plain": [
        ""
@@ -3277,12 +5028,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 00:02:18.688791** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 23:56:50.203339** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
5 acq, 10 darks with 15sec exp
\n"
+       "
track_target again el=75, az=-51
\n"
       ],
       "text/plain": [
        ""
@@ -3294,12 +5045,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 00:00:44.143953** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 23:56:35.950976** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
home both axis.
\n"
+       "
taking 5 images flat
\n"
       ],
       "text/plain": [
        ""
@@ -3311,12 +5062,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 00:00:40.204220** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 23:56:06.282199** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
MTPtg faulted, found it was due to homing.
\n"
+       "
recovering, put hexapods into compensation mode.
\n"
       ],
       "text/plain": [
        ""
@@ -3328,13 +5079,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 00:00:13.280304** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 23:55:41.547940** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
MTPtg faulted again. trying with different az and el\r\n",
-       "el=75, az=-54
\n"
+       "
updating hexapod csc config.
\n"
       ],
       "text/plain": [
        ""
@@ -3346,12 +5096,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 23:36:05.054879** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 23:55:32.434482** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Stopped the script while taking darks, re-ran the same script
\n"
+       "
hexapod failed due to filter offset not applied for u band.
\n"
       ],
       "text/plain": [
        ""
@@ -3363,12 +5113,13 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 23:25:37.633929** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 23:47:51.707450** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Starting BLOCK-T215 twilight flat with u_02 band
\n"
+       "
CamHexapod failed\r\n",
+       "There was a change in the configuration that was not uploaded
\n"
       ],
       "text/plain": [
        ""
@@ -3380,12 +5131,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 23:09:32.380070** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 23:28:56.361834** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
stop tracking
\n"
+       "
take sample image with u_02 filter.
\n"
       ],
       "text/plain": [
        ""
@@ -3397,12 +5148,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 23:05:08.791820** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 23:26:22.166408** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
testing track_target to el=60, az=103
\n"
+       "
moving to az=-51 to be 60deg away from sun
\n"
       ],
       "text/plain": [
        ""
@@ -3414,12 +5165,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 22:54:59.371984** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 23:18:18.303049** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Enabling dome following
\n"
+       "
track_target to el=75, az=-80 to have dome following the TMA position.
\n"
       ],
       "text/plain": [
        ""
@@ -3431,12 +5182,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 22:52:46.990669** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 23:13:46.623367** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Open the mirror cover
\n"
+       "
enable dome following
\n"
       ],
       "text/plain": [
        ""
@@ -3448,12 +5199,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 22:24:13.669923** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 23:06:03.923673** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
raise mirror
\n"
+       "
dome shutter open, mirror cover open
\n"
       ],
       "text/plain": [
        ""
@@ -3465,12 +5216,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 22:19:41.270722** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 22:25:00.883558** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Enable Rotator
\n"
+       "
Closing the mirror cover.
\n"
       ],
       "text/plain": [
        ""
@@ -3482,12 +5233,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 22:19:35.171470** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 22:21:14.474644** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
home both axis.
\n"
+       "
sending LaserTracker.1 to standby
\n"
       ],
       "text/plain": [
        ""
@@ -3499,12 +5250,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 22:18:59.319565** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 22:18:29.042696** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Turning HVAC off
\n"
+       "
Turning off the laser tracker.
\n"
       ],
       "text/plain": [
        ""
@@ -3516,12 +5267,13 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 22:17:02.045751** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 22:16:13.004937** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
recovering
\n"
+       "
point laser at m1m3 : step 3 in T88:\r\n",
+       "target: p2 instead of p4
\n"
       ],
       "text/plain": [
        ""
@@ -3533,12 +5285,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 22:16:58.407344** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 21:57:55.669068** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Faulted M1M3, MTMount
\n"
+       "
Starting BLOCK-T88
\n"
       ],
       "text/plain": [
        ""
@@ -3550,12 +5302,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 22:16:49.520545** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 21:57:44.634039** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
EARTHQUAKE
\n"
+       "
Laser tracker recycled.
\n"
       ],
       "text/plain": [
        ""
@@ -3567,13 +5319,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 22:15:22.997034** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 21:52:49.181689** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
MTMount and MTM1M3 failed right after the dome reached the Az\r\n",
-       "(earthquake?)
\n"
+       "
mirror cover opened.
\n"
       ],
       "text/plain": [
        ""
@@ -3585,12 +5336,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 22:06:26.263416** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 21:49:51.096370** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
move dome to az=140
\n"
+       "
moving to el=75, az=-80 with tracking.
\n"
       ],
       "text/plain": [
        ""
@@ -3602,12 +5353,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 21:04:32.671370** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 21:49:37.947189** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
close mirror cover
\n"
+       "
Hexapods in compensation mode.
\n"
       ],
       "text/plain": [
        ""
@@ -3619,12 +5370,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 20:47:45.176489** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 21:49:29.371780** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Finished T88. Turning off the laser tracker and put into standby
\n"
+       "
Before that, Enabled MTCSs and Laser tracker..
\n"
       ],
       "text/plain": [
        ""
@@ -3636,12 +5387,13 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 20:33:38.563011** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 21:39:27.174971** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
alignment failed on camera, move rotator to 0 and run again
\n"
+       "
Moving to el=75, az=-80 track_target as TMA Checkout (BLOCK-T250)\r\n",
+       "and it's initial position for the T88.
\n"
       ],
       "text/plain": [
        ""
@@ -3653,12 +5405,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 20:20:15.509259** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 21:35:23.319628** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Starting T88 alignment at az=104, el=78
\n"
+       "
home both axis.
\n"
       ],
       "text/plain": [
        ""
@@ -3670,12 +5422,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 20:19:09.520193** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 21:34:21.646070** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
(Ignore) Taking another set of images but now in the u_02 band
\n"
+       "
Command BothAxesStop(sequence_id=1882068, command_code=<CommandCode.BOTH_AXES_STOP: 32>, source=<Source.CSC: 1>, timestamp=1731187920.7486072) failed in disable_devices; skipping all remaining disable commands and going directly to giving up command and stopping the heartbeat command: ExpectedError('Azimuth commands are not allowed when powering. Wait until powering finishes\\nNot accepted command: 102')
\n"
       ],
       "text/plain": [
        ""
@@ -3687,12 +5439,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 20:17:55.082635** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 21:31:08.925381** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Opening the mirror cover
\n"
+       "
Enable MTMount
\n"
       ],
       "text/plain": [
        ""
@@ -3704,12 +5456,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 20:16:12.130533** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 21:30:03.948240** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
stop_tracking
\n"
+       "
Turning off the elevation drive and changing the setting
\n"
       ],
       "text/plain": [
        ""
@@ -3721,12 +5473,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 20:16:06.723307** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 21:28:34.369563** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
took another 3 images with u_02
\n"
+       "
Unpark TMA to el=80
\n"
       ],
       "text/plain": [
        ""
@@ -3738,12 +5490,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 20:16:05.763494** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 21:27:12.704713** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Taking 3 images of 5 seconds in r_03 BLOCK-T250 for the TMA Checkout
\n"
+       "
Freeing the locking pins
\n"
       ],
       "text/plain": [
        ""
@@ -3755,12 +5507,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 19:58:56.840424** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 21:26:57.205807** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Track_target to az=103, el=75
\n"
+       "
OSS back on
\n"
       ],
       "text/plain": [
        ""
@@ -3772,12 +5524,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 19:56:20.569860** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 21:25:56.688699** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
home both axis
\n"
+       "
FA 215's valve closed.
\n"
       ],
       "text/plain": [
        ""
@@ -3789,15 +5541,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 19:54:46.514388** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 20:37:20.510673** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Bump test finished and we got some FA fails:\r\n",
-       "235 Primary\r\n",
-       "238 Secondary\r\n",
-       "305 Secondary
\n"
+       "
Moving dome to 328-> 300 -> 328
\n"
       ],
       "text/plain": [
        ""
@@ -3809,12 +5558,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 19:52:49.151510** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 20:26:32.480906** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
raise m1m3
\n"
+       "
Enable MTDome for checking
\n"
       ],
       "text/plain": [
        ""
@@ -3826,15 +5575,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 19:52:20.249460** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 20:18:54.602622** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Retesting FAs\r\n",
-       "235 slight overshoot\r\n",
-       "238 within the range \r\n",
-       "305 passed
\n"
+       "
Re-enabling the scheduler1
\n"
       ],
       "text/plain": [
        ""
@@ -3846,12 +5592,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 19:47:03.384874** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 20:13:51.173942** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Enabling compensation mode for both hexapods
\n"
+       "
M1M3 was faulting to enable due to displacement sensor
\n"
       ],
       "text/plain": [
        ""
@@ -3863,12 +5609,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 19:27:10.552243** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 19:58:23.079787** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Move MTDome to check
\n"
+       "
M1M3 faulted again. Reset interlock
\n"
       ],
       "text/plain": [
        ""
@@ -3880,12 +5626,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 19:23:58.559368** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 19:57:49.651219** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Rotator failed to be ENABLED. Clearing interlock from GUI
\n"
+       "
M1M3 faulted, recycled M1M3 and M1M3TS.
\n"
       ],
       "text/plain": [
        ""
@@ -3897,12 +5643,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 19:21:14.927976** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 19:51:07.619404** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Failed to ENABLE MTMount due to wrong readout of elevation position. Re-running set_summary_state
\n"
+       "
Running BLOCK-T246 comcam usdf checkout
\n"
       ],
       "text/plain": [
        ""
@@ -3914,12 +5660,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 19:18:46.864275** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 19:46:53.981702** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
TMA setting back to default.
\n"
+       "
M1M3 went to offline, Petr is looking into it
\n"
       ],
       "text/plain": [
        ""
@@ -3931,12 +5677,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 19:14:26.219241** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 19:46:14.330231** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Enabling MTCS
\n"
+       "
LOTO off, Turning on OSS
\n"
       ],
       "text/plain": [
        ""
@@ -3948,12 +5694,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 19:10:37.885963** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 19:35:54.914340** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Moving to el=80
\n"
+       "
M1M3 faulted.
\n"
       ],
       "text/plain": [
        ""
@@ -3965,12 +5711,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 19:10:15.843745** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 19:27:35.865108** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Turning on the elevation drive.
\n"
+       "
Hexapod warmups done, vibration detected. (individual bump test done during the camera hexapod warmups but it vibrated even after the bump test was done)
\n"
       ],
       "text/plain": [
        ""
@@ -3982,12 +5728,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 19:08:32.966107** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 19:26:41.912688** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Freeing locking pin
\n"
+       "
Turned off OSS, LOTO, for laser tracker turning on
\n"
       ],
       "text/plain": [
        ""
@@ -3999,12 +5745,20 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 18:58:27.630240** Time Lost: 0.0; Time Lost Type: fault;\n",
-       "    - components: None\n",
+       "- **2024-11-09 19:06:04.001999** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Stopped the BLOCK-T144, running check_actuators with ignoring FAs that passed already, and FA 215.
\n"
+       "
Individual bump test also failed\r\n",
+       "\r\n",
+       "\r\n",
+       "215: P 112 expected 222\r\n",
+       "         Y -108 expected 0\r\n",
+       "227: P -215 expected -222\r\n",
+       "235: P 228 expected 222\r\n",
+       "330: P, Y 228 expected 222\r\n",
+       "409: P -230 expected -222
\n"
       ],
       "text/plain": [
        ""
@@ -4016,12 +5770,22 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 18:39:52.412678** Time Lost: 0.0; Time Lost Type: fault;\n",
-       "    - components: None\n",
+       "- **2024-11-09 18:54:00.775027** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Starting BLOCK-T144 M1M3 bump test. FA215 is disabled.
\n"
+       "
Original FAs that failed bump test\r\n",
+       "\r\n",
+       "\r\n",
+       "SAA (Single Actuator Axes) Failures:\r\n",
+       "  None\r\n",
+       "DAA (Dual Actuator Axes) Failures:\r\n",
+       "  - Actuator ID 215: Pri Index 50\r\n",
+       "  - Actuator ID 227: Pri Index 61\r\n",
+       "  - Actuator ID 235: Pri Index 69\r\n",
+       "  - Actuator ID 330: Sec Index 80\r\n",
+       "  - Actuator ID 409: Pri Index 123
\n"
       ],
       "text/plain": [
        ""
@@ -4033,12 +5797,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 18:29:36.340560** Time Lost: 0.0; Time Lost Type: fault;\n",
-       "    - components: None\n",
+       "- **2024-11-09 18:53:08.662061** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
HP breakaway test done. all passed.
\n"
+       "
Entered engineering mode on M1M3, running individual bump test for 215, 227, 235, 330, 409
\n"
       ],
       "text/plain": [
        ""
@@ -4050,12 +5814,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 18:14:29.059653** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 18:50:13.109309** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
re-enabling scheduler 1 with t28.yaml
\n"
+       "
Hexapod warmups started
\n"
       ],
       "text/plain": [
        ""
@@ -4067,12 +5831,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 18:13:14.498888** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 18:50:04.994392** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
M1M3 HP breakaway test starting. Pressure 125.
\n"
+       "
we waited for the M1M3 air pressure to go down, and then re-enabled.
\n"
       ],
       "text/plain": [
        ""
@@ -4084,12 +5848,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 18:01:35.635582** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 18:48:19.429723** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Running camhex warmup
\n"
+       "
Some of the actuators failed at the end of the bump test but was not possible to force individual test from the GUI after entering to engineering mode, probably because we were using the CSC in the bump test. Therefore we disabled m1m3 from CSC, this went to fault, and HP compressor pressure started to decrease. We are waiting now for this to increase the pressure again and doing the individual bump test to those actuators
\n"
       ],
       "text/plain": [
        ""
@@ -4101,12 +5865,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 18:01:21.675068** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 18:26:42.058728** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
LOTO off. We turn OSS back o, M1M3 back to enabled. Pressure back to 125
\n"
+       "
M1M3 faulted when putting into disabled, we cycled M1M3TS, and recovered M1M3
\n"
       ],
       "text/plain": [
        ""
@@ -4118,12 +5882,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 17:35:16.175574** Time Lost: 0.0; Time Lost Type: fault;\n",
-       "    - components: None\n",
+       "- **2024-11-09 17:07:48.100301** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Turning off OSS, put M1M3 into standby, m2hexapod into disabled.
\n"
+       "
re-running M1M3 bump test (T144)
\n"
       ],
       "text/plain": [
        ""
@@ -4135,12 +5899,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 17:34:39.874306** Time Lost: 0.0; Time Lost Type: fault;\n",
-       "    - components: None\n",
+       "- **2024-11-09 17:06:54.604913** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
M2hexapod warmup done
\n"
+       "
Enabling scheduler first. t87.yaml
\n"
       ],
       "text/plain": [
        ""
@@ -4152,13 +5916,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 17:21:54.505224** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 17:02:52.153808** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Right after air compressor #2 was restarted, it increased close to 9,000\r\n",
-       "Couple of minutes later it started to decrease a bit to 8,900 and all the JP air compressor started to oscillate around 116.00 - 117.20 PSI
\n"
+       "
Exit engineering mode. Running BLOCK-T144 M1M3 Bump test.
\n"
       ],
       "text/plain": [
        ""
@@ -4170,12 +5933,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 17:09:36.380593** Time Lost: 0.0; Time Lost Type: fault;\n",
-       "    - components: None\n",
+       "- **2024-11-09 15:58:04.907021** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
restarted compressor 2 from level 1
\n"
+       "
M1M3 HP breakaway test all passed.
\n"
       ],
       "text/plain": [
        ""
@@ -4187,12 +5950,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 16:58:32.938245** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 15:41:00.876927** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Starting Warm-up for M2 Hexapod and CamHexapod
\n"
+       "
Starting HP breakaway test on M1M3 (BLOCK-T145)
\n"
       ],
       "text/plain": [
        ""
@@ -4204,12 +5967,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 15:58:34.237203** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 15:35:16.447985** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
ComCam usdf checkout (BLOCK-T246) done
\n"
+       "
Enble M1M3
\n"
       ],
       "text/plain": [
        ""
@@ -4221,12 +5984,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 15:51:10.530313** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 15:16:52.897987** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Cleared the interlock of chillers, turning OSS on.
\n"
+       "
Turning on the rotator
\n"
       ],
       "text/plain": [
        ""
@@ -4238,13 +6001,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 15:23:52.530045** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 15:15:42.700719** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
We had PS and OSS off for TMA, due to power outage.\r\n",
-       "Turning back PS on
\n"
+       "
Powering up Rotator and Hexapods
\n"
       ],
       "text/plain": [
        ""
@@ -4256,12 +6018,12 @@
     {
      "data": {
       "text/markdown": [
-       "- **2024-11-10 15:23:52.522869** Time Lost: 0.0; Time Lost Type: fault;\n",
+       "- **2024-11-09 13:23:44.532962** Time Lost: 0.0; Time Lost Type: fault;\n",
        "    - components: ['MainTel']\n",
        "    - primary_software_components: ['None']\n",
        "    - primary_hardware_components: ['None']\n",
        "\n",
-       "
Powering up both rotator and hexapods
\n"
+       "
u-band installed in ComCam (replacing g).  OSS and PS back on.  moving back to zenith.
\n"
       ],
       "text/plain": [
        ""
@@ -4320,8 +6082,8 @@
       "text/markdown": [
        "## Project-Wide Night(s) Report \n",
        "- Run on logs and databases from **https://usdf-rsp-dev.slac.stanford.edu/**\n",
-       "- Report **1 observing night(s)** with the last reported night starting on **2024-11-10 12:00:00**.\n",
-       "- min_dayobs='2024-11-10', max_dayobs='2024-11-11'\n",
+       "- Report **1 observing night(s)** with the last reported night starting on **2024-11-09 12:00:00**.\n",
+       "- min_dayobs='2024-11-09', max_dayobs='2024-11-10'\n",
        "- Using ***Prototype* Logging and Reporting** Version: **0.1.dev145+gf2a5765.d20241014**\n",
        "- have_consdb = True\n",
        "- Imported lsst.ts.logging_and_reporting from local packages.\n"
@@ -4393,8 +6155,8 @@
        "  \n",
        "    \n",
        "      nightreport/reports\n",
-       "      2\n",
-       "      https://usdf-rsp-dev.slac.stanford.edu/nightreport/reports?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=100&min_day_obs=20241110&max_day_obs=20241111\n",
+       "      3\n",
+       "      https://usdf-rsp-dev.slac.stanford.edu/nightreport/reports?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=100&min_day_obs=20241109&max_day_obs=20241110\n",
        "    \n",
        "    \n",
        "      exposurelog/instruments\n",
@@ -4404,32 +6166,32 @@
        "    \n",
        "      exposurelog/exposures.LSSTComCamSim\n",
        "      0\n",
-       "      https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LSSTComCamSim&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241110&max_day_obs=20241111\n",
+       "      https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LSSTComCamSim&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241109&max_day_obs=20241110\n",
        "    \n",
        "    \n",
        "      exposurelog/exposures.LATISS\n",
        "      0\n",
-       "      https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LATISS&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241110&max_day_obs=20241111\n",
+       "      https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LATISS&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241109&max_day_obs=20241110\n",
        "    \n",
        "    \n",
        "      exposurelog/exposures.LSSTComCam\n",
-       "      239\n",
-       "      https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LSSTComCam&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241110&max_day_obs=20241111\n",
+       "      292\n",
+       "      https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LSSTComCam&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241109&max_day_obs=20241110\n",
        "    \n",
        "    \n",
        "      exposurelog/exposures.LSSTCam\n",
-       "      0\n",
-       "      https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LSSTCam&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241110&max_day_obs=20241111\n",
+       "      1338\n",
+       "      https://usdf-rsp-dev.slac.stanford.edu/exposurelog/exposures?registry=1&instrument=LSSTCam&order_by=-timespan_end&offset=0&limit=2500&min_day_obs=20241109&max_day_obs=20241110\n",
        "    \n",
        "    \n",
        "      exposurelog/messages\n",
        "      0\n",
-       "      https://usdf-rsp-dev.slac.stanford.edu/exposurelog/messages?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=2500&min_day_obs=20241110&max_day_obs=20241111\n",
+       "      https://usdf-rsp-dev.slac.stanford.edu/exposurelog/messages?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=2500&min_day_obs=20241109&max_day_obs=20241110\n",
        "    \n",
        "    \n",
        "      narrativelog/messages\n",
-       "      150\n",
-       "      https://usdf-rsp-dev.slac.stanford.edu/narrativelog/messages?is_human=either&is_valid=true&order_by=-date_added&offset=0&limit=1000&min_date_added=2024-11-10T00%3A00%3A00&max_date_added=2024-11-11T00%3A00%3A00\n",
+       "      214\n",
+       "      https://usdf-rsp-dev.slac.stanford.edu/narrativelog/messages?is_human=either&is_valid=true&order_by=-date_added&offset=0&limit=1000&min_date_added=2024-11-09T00%3A00%3A00&max_date_added=2024-11-10T00%3A00%3A00\n",
        "    \n",
        "    \n",
        "      efd/targets\n",
@@ -4547,7 +6309,7 @@
     {
      "data": {
       "text/markdown": [
-       "### Overview for Service: `nightreport` [2]"
+       "### Overview for Service: `nightreport` [3]"
       ],
       "text/plain": [
        ""
@@ -4566,7 +6328,7 @@
     {
      "data": {
       "text/markdown": [
-       "- Endpoint: https://usdf-rsp-dev.slac.stanford.edu/nightreport/reports?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=100&min_day_obs=20241110&max_day_obs=20241111"
+       "- Endpoint: https://usdf-rsp-dev.slac.stanford.edu/nightreport/reports?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=100&min_day_obs=20241109&max_day_obs=20241110"
       ],
       "text/plain": [
        ""
@@ -4580,7 +6342,7 @@
      "output_type": "stream",
      "text": [
       "None\n",
-      "- Got 2 records.  \n"
+      "- Got 3 records.  \n"
      ]
     },
     {
@@ -4605,7 +6367,7 @@
     {
      "data": {
       "text/markdown": [
-       "- Endpoint: https://usdf-rsp-dev.slac.stanford.edu/exposurelog/messages?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=2500&min_day_obs=20241110&max_day_obs=20241111"
+       "- Endpoint: https://usdf-rsp-dev.slac.stanford.edu/exposurelog/messages?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=2500&min_day_obs=20241109&max_day_obs=20241110"
       ],
       "text/plain": [
        ""
@@ -4625,7 +6387,7 @@
     {
      "data": {
       "text/markdown": [
-       "### Overview for Service: `narrativelog` [150]"
+       "### Overview for Service: `narrativelog` [214]"
       ],
       "text/plain": [
        ""
@@ -4644,7 +6406,7 @@
     {
      "data": {
       "text/markdown": [
-       "- Endpoint: https://usdf-rsp-dev.slac.stanford.edu/narrativelog/messages?is_human=either&is_valid=true&order_by=-date_added&offset=0&limit=1000&min_date_added=2024-11-10T00%3A00%3A00&max_date_added=2024-11-11T00%3A00%3A00"
+       "- Endpoint: https://usdf-rsp-dev.slac.stanford.edu/narrativelog/messages?is_human=either&is_valid=true&order_by=-date_added&offset=0&limit=1000&min_date_added=2024-11-09T00%3A00%3A00&max_date_added=2024-11-10T00%3A00%3A00"
       ],
       "text/plain": [
        ""
@@ -4658,7 +6420,7 @@
      "output_type": "stream",
      "text": [
       "None\n",
-      "- Got 150 records.  \n"
+      "- Got 214 records.  \n"
      ]
     }
    ],
@@ -4712,7 +6474,7 @@
     {
      "data": {
       "text/markdown": [
-       "No exposurelog records found 2024-11-10 to 2024-11-11. "
+       "No exposurelog records found 2024-11-09 to 2024-11-10. "
       ],
       "text/plain": [
        ""
@@ -4724,7 +6486,7 @@
     {
      "data": {
       "text/markdown": [
-       "Used [API Data](https://usdf-rsp-dev.slac.stanford.edu/exposurelog/messages?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=2500&min_day_obs=20241110&max_day_obs=20241111)"
+       "Used [API Data](https://usdf-rsp-dev.slac.stanford.edu/exposurelog/messages?is_human=either&is_valid=true&order_by=-day_obs&offset=0&limit=2500&min_day_obs=20241109&max_day_obs=20241110)"
       ],
       "text/plain": [
        ""
@@ -4761,7 +6523,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Finished 2024-11-11 22:31:53.852033\n"
+      "Finished 2024-11-12 01:53:58.149098\n"
      ]
     }
    ],
diff --git a/python/lsst/ts/logging_and_reporting/source_adapters.py b/python/lsst/ts/logging_and_reporting/source_adapters.py
index 0b1c646..a3e0e43 100644
--- a/python/lsst/ts/logging_and_reporting/source_adapters.py
+++ b/python/lsst/ts/logging_and_reporting/source_adapters.py
@@ -583,11 +583,11 @@ def obs_date(rec):
                 attrstr += f" Time Lost: {rec.get('time_lost')};"
                 attrstr += f" Time Lost Type: {rec.get('time_lost_type')};"
                 new = rec.get("error_message")
+                mdstr = ""
                 if new:
                     msg = new
                 else:
                     msg = rep.htmlcode(rec["message_text"].strip())
-                    mdstr = ""
                     mdstr += f"- {attrstr}"
                 for fname in show_fields:
                     mdstr += f"\n    - {fname}: {rec.get(fname)}"