Skip to content

Commit

Permalink
Add retries for safe commands
Browse files Browse the repository at this point in the history
  • Loading branch information
albireox committed Nov 29, 2024
1 parent 1dacde3 commit e567d12
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 11 deletions.
9 changes: 6 additions & 3 deletions src/lvmcryo/handlers/ln2.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@

from lvmopstools.devices.specs import spectrograph_pressures, spectrograph_temperatures
from lvmopstools.devices.thermistors import read_thermistors
from lvmopstools.retrier import Retrier
from sdsstools.utils import GatheringTaskGroup

from lvmcryo.config import ValveConfig, get_internal_config
Expand Down Expand Up @@ -182,13 +183,15 @@ async def check(
"""

retrier = Retrier(max_attempts=3, delay=1)

# Check O2 alarms.
if not self.alerts_route:
self.log.warning("No alerts route provided. Not checking O2 alarms.")
else:
try:
self.log.info("Checking for O2 alarms ...")
if await o2_alert(self.alerts_route):
if await retrier(o2_alert)(self.alerts_route):
self.fail("O2 alarm detected.")
else:
self.log.debug("No O2 alarms reported.")
Expand All @@ -207,7 +210,7 @@ async def check(
if max_temperature is not None:
self.log.info("Checking LN2 temperatures ...")
try:
spec_temperatures = await spectrograph_temperatures()
spec_temperatures = await retrier(spectrograph_temperatures)()
except Exception as err:
self.fail(f"Failed reading spectrograph temperatures: {err}")
else:
Expand All @@ -227,7 +230,7 @@ async def check(
if max_pressure is not None:
self.log.info("Checking pressures ...")
try:
spec_pressures = await spectrograph_pressures()
spec_pressures = await retrier(spectrograph_pressures)()
except Exception as err:
self.fail(f"Failed reading spectrograph pressures: {err}")
else:
Expand Down
16 changes: 8 additions & 8 deletions src/lvmcryo/handlers/valve.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,18 +18,18 @@
from rich.progress import TaskID

from lvmopstools.clu import CluClient
from lvmopstools.retrier import Retrier

from lvmcryo.config import get_internal_config
from lvmcryo.handlers.thermistor import ThermistorHandler
from lvmcryo.tools import (
cancel_task,
get_fake_logger,
)
from lvmcryo.tools import cancel_task, get_fake_logger


if TYPE_CHECKING:
from sdsstools.configuration import Configuration

from lvmcryo.tools import TimerProgressBar


__all__ = [
"ValveHandler",
Expand All @@ -40,6 +40,7 @@
]


@Retrier(max_attempts=3, delay=1)
async def outlet_info(actor: str, outlet: str) -> dict[str, Any]:
"""Retrieves outlet information from the NPS."""

Expand All @@ -51,6 +52,7 @@ async def outlet_info(actor: str, outlet: str) -> dict[str, Any]:
return cmd.replies.get("outlet_info")


@Retrier(max_attempts=3, delay=1)
async def valve_on_off(
actor: str,
outlet_name: str,
Expand Down Expand Up @@ -118,6 +120,7 @@ async def valve_on_off(
return


@Retrier(max_attempts=3, delay=1)
async def cancel_nps_threads(actor: str, thread_id: int | None = None):
"""Cancels a script thread in an NPS.
Expand All @@ -136,6 +139,7 @@ async def cancel_nps_threads(actor: str, thread_id: int | None = None):
await client.send_command(actor, command_string)


@Retrier(max_attempts=3, delay=1)
async def close_all_valves(config: Configuration | None = None, dry_run: bool = False):
"""Closes all the outlets."""

Expand All @@ -155,10 +159,6 @@ async def close_all_valves(config: Configuration | None = None, dry_run: bool =
)


if TYPE_CHECKING:
from lvmcryo.tools import TimerProgressBar


@dataclass
class ValveHandler:
"""Handles a valve, including opening and closing, timeouts, and thermistors.
Expand Down

0 comments on commit e567d12

Please sign in to comment.