Merge pull request #180 from akrherz/ia_rwis

Prevent IA_RWIS data being processed via SHEF

.pre-commit-config.yaml

@@ -2,11 +2,11 @@ ci:
   autoupdate_schedule: quarterly
 repos:
   - repo: https://github.com/psf/black
-    rev: 23.10.0
+    rev: 23.10.1
     hooks:
     - id: black
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: "v0.1.0"
+    rev: "v0.1.2"
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

parsers/pywwa/workflows/gini2gis.py

@@ -55,8 +55,8 @@ def do_legacy_ir(sat, tmpfn):
         fh.write(WORLDFILE_FORMAT % sat.metadata)
 
     cmd = (
-        "pqinsert -i -p 'gis c %s gis/images/awips%s/%s GIS/sat/awips%s/%s png' "
-        "%s.png"
+        "pqinsert -i -p "
+        "'gis c %s gis/images/awips%s/%s GIS/sat/awips%s/%s png' %s.png"
     ) % (
         sat.metadata["valid"].strftime("%Y%m%d%H%M"),
         sat.awips_grid(),
@@ -68,8 +68,8 @@ def do_legacy_ir(sat, tmpfn):
     subprocess.call(cmd, shell=True)
 
     cmd = (
-        "pqinsert -i -p 'gis c %s gis/images/awips%s/%s GIS/sat/awips%s/%s wld' "
-        "%s.wld"
+        "pqinsert -i -p "
+        "'gis c %s gis/images/awips%s/%s GIS/sat/awips%s/%s wld' %s.wld"
     ) % (
         sat.metadata["valid"].strftime("%Y%m%d%H%M"),
         sat.awips_grid(),
@@ -100,8 +100,8 @@ def write_gispng(sat, tmpfn):
         fh.write(WORLDFILE_FORMAT % sat.metadata)
 
     cmd = (
-        "pqinsert -i -p 'gis %s %s gis/images/awips%s/%s GIS/sat/awips%s/%s png' "
-        "%s.png"
+        "pqinsert -i -p "
+        "'gis %s %s gis/images/awips%s/%s GIS/sat/awips%s/%s png' %s.png"
     ) % (
         get_ldm_routes(sat),
         sat.metadata["valid"].strftime("%Y%m%d%H%M"),
@@ -114,8 +114,8 @@ def write_gispng(sat, tmpfn):
     subprocess.call(cmd, shell=True)
 
     cmd = (
-        "pqinsert -i -p 'gis %s %s gis/images/awips%s/%s GIS/sat/awips%s/%s wld' "
-        "%s.wld"
+        "pqinsert -i -p "
+        "'gis %s %s gis/images/awips%s/%s GIS/sat/awips%s/%s wld' %s.wld"
     ) % (
         get_ldm_routes(sat),
         sat.metadata["valid"].strftime("%Y%m%d%H%M"),
@@ -143,7 +143,8 @@ def write_metadata(sat, tmpfn):
         json.dump(metadata, fh)
 
     cmd = (
-        "pqinsert -i -p 'gis c %s gis/images/awips%s/%s GIS/sat/%s json' %s.json"
+        "pqinsert -i -p "
+        "'gis c %s gis/images/awips%s/%s GIS/sat/%s json' %s.json"
     ) % (
         sat.metadata["valid"].strftime("%Y%m%d%H%M"),
         sat.awips_grid(),

parsers/pywwa/workflows/shef_parser.py

@@ -42,6 +42,8 @@
 # a queue for saving database IO
 CURRENT_QUEUE = {}
 U1980 = utc(1980)
+# Networks that can come via SHEF backdoors
+DOUBLEBACKED_NETWORKS = ["ISUSM", "IA_RWIS"]
 
 
 DIRECTMAP = {
@@ -120,9 +122,10 @@ def load_stations(txn):
         SELECT id, s.iemid, network, tzname, a.value as pedts from stations s
         LEFT JOIN station_attributes a on (s.iemid = a.iemid and
         a.attr = 'PEDTS') WHERE (network ~* 'COOP'
-        or network ~* 'DCP' or network = 'ISUSM') and tzname is not null
-        ORDER by network ASC
-        """
+        or network ~* 'DCP' or network = ANY(%s))
+        and tzname is not null ORDER by network ASC
+        """,
+        (DOUBLEBACKED_NETWORKS,),
     )
 
     # A sentinel to know if we later need to remove things in the case of a
@@ -391,7 +394,7 @@ def update_current_queue(element: SHEFElement, product_id: str):
 def process_site_time(accesstxn, prod, sid, ts, elements: List[SHEFElement]):
     """Ingest for IEMAccess."""
     network = get_network(prod, sid, elements)
-    if network in ["ISUSM", None]:
+    if network is None or network in DOUBLEBACKED_NETWORKS:
         return
 
     localts = get_localtime(sid, ts)

pyproject.toml

@@ -4,4 +4,5 @@ line-length = 79
 [tool.ruff]
 select = ["E", "F", "I"]
 target-version = "py39"
+line-length = 79