Omnibus

.github/workflows/pydep.yml

@@ -22,6 +22,17 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
+    - name: Run IEM Database container
+      run: |
+        docker run -d --name iem_database -p 5432:5432 ghcr.io/akrherz/iem_database:test_data
+        until docker exec iem_database pg_isready -h localhost; do
+          sleep 6
+        done
+
+    - name: Run Memcached container
+      run: |
+        docker run -d --name iem_memcached -p 11211:11211 memcached:1.6.9
+
     - name: Add /etc/hosts entries
       run: |
         cat .github/workflows/etchosts.txt | sudo tee -a /etc/hosts
@@ -49,25 +60,12 @@ jobs:
     - name: Setup data
       run: sh .github/setupdata.sh
 
-    - name: Setup Postgres
-      run: |
-        git clone --depth 1 https://github.com/akrherz/iem-database.git database
-        git clone --depth 1 https://github.com/akrherz/ci_tooling.git .ci_tooling
-        . .ci_tooling/postgres.sh
-        cd database; sh bootstrap.sh
-        python schema_manager.py
-
-    - name: Setup Memcached
-      run: |
-        . .ci_tooling/memcached.sh
-
     - name: Build and test
       run: |
         python -m pip install . --upgrade --no-deps
         python -m pytest --cov=pydep tests
         python -m coverage xml
 
-
     - name: Upload codecov
       if: ${{ env.PYTHON_VERSION == '3.12' }}
       uses: codecov/codecov-action@v4

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: "v0.8.4"
+    rev: "v0.8.6"
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

scripts/cligen/daily_clifile_editor.py

@@ -97,7 +97,7 @@ def get_sts_ets_at_localhour(dt: date, local_hour):
 
 def iemre_bounds_check(clidf: pd.DataFrame, col: str, lower, upper) -> None:
     """Make sure our data is within bounds, if not, exit!"""
-    if clidf[col].min() <= lower or clidf[col].max() > upper:
+    if clidf[col].min() < lower or clidf[col].max() > upper:
         LOG.warning(
             "FATAL: iemre failure %s %.3f to %.3f [%.3f to %.3f]",
             col,

scripts/cligen/map_clifile_points.py

@@ -3,6 +3,7 @@
 import click
 import geopandas as gpd
 import numpy as np
+import rasterio as rio
 from matplotlib.colors import BoundaryNorm
 from pyiem.database import get_sqlalchemy_conn
 from pyiem.plot import MapPlot, get_cmap
@@ -20,30 +21,46 @@
 @click.option("--year", default=2024, help="Year to plot")
 def main(year: int):
     """make maps, not war."""
+    # https://zenodo.org/records/11078865
+    with rio.open("/home/akrherz/Downloads/GloRESatE/GloRESatE.tif") as src:
+        gloresate = src.read(1)
+        aff = src.transform
+
+    def find_ij(lon, lat):
+        """figure out the grid index."""
+        return ~aff * (lon, lat)
+
     with get_sqlalchemy_conn("idep") as pgconn:
         pts = gpd.read_postgis(
             text("""
     with climo as (
         select climate_file_id, avg(rfactor) as rr from
         climate_file_yearly_summary
         GROUP by climate_file_id)
-    select geom, (rfactor - rr) / rr  * 100. as ratio from
-    climate_files f, climate_file_yearly_summary s, climo c
-    WHERE f.id = s.climate_file_id and s.year = :year
-    and f.id = c.climate_file_id and f.scenario = 0
+    select geom, rr from climate_files f, climo c
+    WHERE f.id = c.climate_file_id and f.scenario = 0
 """),
             pgconn,
             params={"year": year},
             geom_col="geom",
         )
+    for idx, row in pts.iterrows():
+        lon, lat = row["geom"].x, row["geom"].y
+        i, j = find_ij(lon, lat)
+        pts.at[idx, "gloresate"] = gloresate[int(j), int(i)]
+
+    pts["ratio"] = (pts["rr"] - pts["gloresate"]) / pts["gloresate"] * 100.0
+
+    print(pts["ratio"].describe())
+
     cmap = get_cmap("RdBu")
     # clevs = [0, 250, 500, 750, 1000, 1500, 2000, 2500, 3000, 4000, 5000]
     clevs = np.arange(-60, 61, 20)
     norm = BoundaryNorm(clevs, cmap.N)
     mp = MapPlot(
         logo="dep",
         sector="conus",
-        title=f"{year} R-Factor % Difference from 2007-2024 Average",
+        title="R-Factor % Difference DEP 2007-2024 Minus GloRESatE",
         subtitle=f"Total Climate Files: {len(pts)}",
         caption="Daily Erosion Project",
     )
@@ -64,7 +81,7 @@ def main(year: int):
         units="%",
         extend="both",
     )
-    mp.fig.savefig(f"{year}_departure.png")
+    mp.fig.savefig("gloresate_departure.png")
 
 
 if __name__ == "__main__":