diff --git a/src/lvmcryo/handlers/ln2.py b/src/lvmcryo/handlers/ln2.py index 9e2e0e9..104655a 100644 --- a/src/lvmcryo/handlers/ln2.py +++ b/src/lvmcryo/handlers/ln2.py @@ -25,6 +25,7 @@ from lvmopstools.devices.specs import spectrograph_pressures, spectrograph_temperatures from lvmopstools.devices.thermistors import read_thermistors +from lvmopstools.retrier import Retrier from sdsstools.utils import GatheringTaskGroup from lvmcryo.config import ValveConfig, get_internal_config @@ -182,13 +183,15 @@ async def check( """ + retrier = Retrier(max_attempts=3, delay=1) + # Check O2 alarms. if not self.alerts_route: self.log.warning("No alerts route provided. Not checking O2 alarms.") else: try: self.log.info("Checking for O2 alarms ...") - if await o2_alert(self.alerts_route): + if await retrier(o2_alert)(self.alerts_route): self.fail("O2 alarm detected.") else: self.log.debug("No O2 alarms reported.") @@ -207,7 +210,7 @@ async def check( if max_temperature is not None: self.log.info("Checking LN2 temperatures ...") try: - spec_temperatures = await spectrograph_temperatures() + spec_temperatures = await retrier(spectrograph_temperatures)() except Exception as err: self.fail(f"Failed reading spectrograph temperatures: {err}") else: @@ -227,7 +230,7 @@ async def check( if max_pressure is not None: self.log.info("Checking pressures ...") try: - spec_pressures = await spectrograph_pressures() + spec_pressures = await retrier(spectrograph_pressures)() except Exception as err: self.fail(f"Failed reading spectrograph pressures: {err}") else: diff --git a/src/lvmcryo/handlers/valve.py b/src/lvmcryo/handlers/valve.py index 31636b1..7bc61c8 100644 --- a/src/lvmcryo/handlers/valve.py +++ b/src/lvmcryo/handlers/valve.py @@ -18,18 +18,18 @@ from rich.progress import TaskID from lvmopstools.clu import CluClient +from lvmopstools.retrier import Retrier from lvmcryo.config import get_internal_config from lvmcryo.handlers.thermistor import ThermistorHandler -from lvmcryo.tools import ( - cancel_task, - get_fake_logger, -) +from lvmcryo.tools import cancel_task, get_fake_logger if TYPE_CHECKING: from sdsstools.configuration import Configuration + from lvmcryo.tools import TimerProgressBar + __all__ = [ "ValveHandler", @@ -40,6 +40,7 @@ ] +@Retrier(max_attempts=3, delay=1) async def outlet_info(actor: str, outlet: str) -> dict[str, Any]: """Retrieves outlet information from the NPS.""" @@ -51,6 +52,7 @@ async def outlet_info(actor: str, outlet: str) -> dict[str, Any]: return cmd.replies.get("outlet_info") +@Retrier(max_attempts=3, delay=1) async def valve_on_off( actor: str, outlet_name: str, @@ -118,6 +120,7 @@ async def valve_on_off( return +@Retrier(max_attempts=3, delay=1) async def cancel_nps_threads(actor: str, thread_id: int | None = None): """Cancels a script thread in an NPS. @@ -136,6 +139,7 @@ async def cancel_nps_threads(actor: str, thread_id: int | None = None): await client.send_command(actor, command_string) +@Retrier(max_attempts=3, delay=1) async def close_all_valves(config: Configuration | None = None, dry_run: bool = False): """Closes all the outlets.""" @@ -155,10 +159,6 @@ async def close_all_valves(config: Configuration | None = None, dry_run: bool = ) -if TYPE_CHECKING: - from lvmcryo.tools import TimerProgressBar - - @dataclass class ValveHandler: """Handles a valve, including opening and closing, timeouts, and thermistors.