diff --git a/.github/workflows/pydep.yml b/.github/workflows/pydep.yml index 9e134c20..3b5e086b 100644 --- a/.github/workflows/pydep.yml +++ b/.github/workflows/pydep.yml @@ -22,6 +22,17 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Run IEM Database container + run: | + docker run -d --name iem_database -p 5432:5432 ghcr.io/akrherz/iem_database:test_data + until docker exec iem_database pg_isready -h localhost; do + sleep 6 + done + + - name: Run Memcached container + run: | + docker run -d --name iem_memcached -p 11211:11211 memcached:1.6.9 + - name: Add /etc/hosts entries run: | cat .github/workflows/etchosts.txt | sudo tee -a /etc/hosts @@ -49,25 +60,12 @@ jobs: - name: Setup data run: sh .github/setupdata.sh - - name: Setup Postgres - run: | - git clone --depth 1 https://github.com/akrherz/iem-database.git database - git clone --depth 1 https://github.com/akrherz/ci_tooling.git .ci_tooling - . .ci_tooling/postgres.sh - cd database; sh bootstrap.sh - python schema_manager.py - - - name: Setup Memcached - run: | - . .ci_tooling/memcached.sh - - name: Build and test run: | python -m pip install . --upgrade --no-deps python -m pytest --cov=pydep tests python -m coverage xml - - name: Upload codecov if: ${{ env.PYTHON_VERSION == '3.12' }} uses: codecov/codecov-action@v4 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ad912439..e74bd307 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.8.4" + rev: "v0.8.6" hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] diff --git a/scripts/cligen/daily_clifile_editor.py b/scripts/cligen/daily_clifile_editor.py index dfb992ee..5213fcf9 100644 --- a/scripts/cligen/daily_clifile_editor.py +++ b/scripts/cligen/daily_clifile_editor.py @@ -97,7 +97,7 @@ def get_sts_ets_at_localhour(dt: date, local_hour): def iemre_bounds_check(clidf: pd.DataFrame, col: str, lower, upper) -> None: """Make sure our data is within bounds, if not, exit!""" - if clidf[col].min() <= lower or clidf[col].max() > upper: + if clidf[col].min() < lower or clidf[col].max() > upper: LOG.warning( "FATAL: iemre failure %s %.3f to %.3f [%.3f to %.3f]", col, diff --git a/scripts/cligen/map_clifile_points.py b/scripts/cligen/map_clifile_points.py index 6fa8f926..862fc9a9 100644 --- a/scripts/cligen/map_clifile_points.py +++ b/scripts/cligen/map_clifile_points.py @@ -3,6 +3,7 @@ import click import geopandas as gpd import numpy as np +import rasterio as rio from matplotlib.colors import BoundaryNorm from pyiem.database import get_sqlalchemy_conn from pyiem.plot import MapPlot, get_cmap @@ -20,6 +21,15 @@ @click.option("--year", default=2024, help="Year to plot") def main(year: int): """make maps, not war.""" + # https://zenodo.org/records/11078865 + with rio.open("/home/akrherz/Downloads/GloRESatE/GloRESatE.tif") as src: + gloresate = src.read(1) + aff = src.transform + + def find_ij(lon, lat): + """figure out the grid index.""" + return ~aff * (lon, lat) + with get_sqlalchemy_conn("idep") as pgconn: pts = gpd.read_postgis( text(""" @@ -27,15 +37,22 @@ def main(year: int): select climate_file_id, avg(rfactor) as rr from climate_file_yearly_summary GROUP by climate_file_id) - select geom, (rfactor - rr) / rr * 100. as ratio from - climate_files f, climate_file_yearly_summary s, climo c - WHERE f.id = s.climate_file_id and s.year = :year - and f.id = c.climate_file_id and f.scenario = 0 + select geom, rr from climate_files f, climo c + WHERE f.id = c.climate_file_id and f.scenario = 0 """), pgconn, params={"year": year}, geom_col="geom", ) + for idx, row in pts.iterrows(): + lon, lat = row["geom"].x, row["geom"].y + i, j = find_ij(lon, lat) + pts.at[idx, "gloresate"] = gloresate[int(j), int(i)] + + pts["ratio"] = (pts["rr"] - pts["gloresate"]) / pts["gloresate"] * 100.0 + + print(pts["ratio"].describe()) + cmap = get_cmap("RdBu") # clevs = [0, 250, 500, 750, 1000, 1500, 2000, 2500, 3000, 4000, 5000] clevs = np.arange(-60, 61, 20) @@ -43,7 +60,7 @@ def main(year: int): mp = MapPlot( logo="dep", sector="conus", - title=f"{year} R-Factor % Difference from 2007-2024 Average", + title="R-Factor % Difference DEP 2007-2024 Minus GloRESatE", subtitle=f"Total Climate Files: {len(pts)}", caption="Daily Erosion Project", ) @@ -64,7 +81,7 @@ def main(year: int): units="%", extend="both", ) - mp.fig.savefig(f"{year}_departure.png") + mp.fig.savefig("gloresate_departure.png") if __name__ == "__main__":