extract_ESSD_benchmark_training_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a59fd50e",
   "metadata": {},
   "source": [
    "## Preambula"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f315457",
   "metadata": {},
   "outputs": [],
   "source": [
    "import xarray as xr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e6768772",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cfcdba58",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3cf89025",
   "metadata": {},
   "outputs": [],
   "source": [
    "import climetlab as cml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e6a3fde",
   "metadata": {},
   "outputs": [],
   "source": [
    "countries = ['france', 'austria', 'netherlands', 'germany', 'belgium']  # Switzerland will not be available online"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5c938cf2",
   "metadata": {},
   "source": [
    "## Extracting the countries t2m data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac22c4f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "fcs_t2m = list()\n",
    "obs_t2m = list()\n",
    "\n",
    "sources_list = []\n",
    "land_usage_source = None\n",
    "land_usage_legend = None\n",
    "model_altitude_source = None\n",
    "\n",
    "for country in countries:\n",
    "    \n",
    "    ds = cml.load_dataset('EUPPBench-training-data-stations-reforecasts-surface', country)\n",
    "    \n",
    "    fcs = ds.to_xarray()\n",
    "    \n",
    "    model_altitude_source = fcs.attrs['model altitude source']\n",
    "\n",
    "    fcs_t2m.append(fcs.t2m.load())\n",
    "    fcs.close()\n",
    "    \n",
    "    obs = ds.get_observations_as_xarray()\n",
    "    \n",
    "    sources_list.append(obs.attrs['source'])\n",
    "    land_usage_source = obs.attrs['land usage source']\n",
    "    land_usage_legend = obs.attrs['land usage legend']\n",
    "\n",
    "    \n",
    "    obs_t2m.append(obs.t2m.load())\n",
    "    \n",
    "    obs.close()\n",
    "       \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d2a55c03",
   "metadata": {},
   "outputs": [],
   "source": [
    "sources = '; '.join(sources_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb6f5813",
   "metadata": {},
   "outputs": [],
   "source": [
    "essd_fcs = xr.concat(fcs_t2m, dim='station_id')\n",
    "essd_obs = xr.concat(obs_t2m, dim='station_id')\n",
    "essd_obs.attrs['sources'] = sources\n",
    "essd_obs.attrs['land usage source'] = land_usage_source\n",
    "essd_obs.attrs['land usage legend'] = land_usage_legend\n",
    "essd_fcs.attrs['model altitude source'] = model_altitude_source\n",
    "essd_fcs.attrs['land usage source'] = land_usage_source\n",
    "essd_fcs.attrs['land usage legend'] = land_usage_legend\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4f2ebd0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop data that overlaps with test period for the forecasts\n",
    "fcs_test_times = essd_fcs.time.sel(time = slice('2018', '2019'))\n",
    "fcs_dummy = xr.where((essd_fcs.year == 20) & (essd_fcs.time.isin(fcs_test_times)), np.nan, essd_fcs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "776dde72",
   "metadata": {},
   "outputs": [],
   "source": [
    "fcs_dummy.attrs = essd_fcs.attrs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8cc601b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop data that overlaps with test period for the observations\n",
    "obs_test_times = essd_obs.time.sel(time = slice('2018', '2019'))\n",
    "obs_dummy = xr.where((essd_obs.year == 20) & (essd_obs.time.isin(obs_test_times)), np.nan, essd_obs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d61041b",
   "metadata": {},
   "outputs": [],
   "source": [
    "obs_dummy.attrs = essd_obs.attrs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1614ca1e",
   "metadata": {},
   "source": [
    "## Reordering the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ecda3890",
   "metadata": {},
   "outputs": [],
   "source": [
    "essd_fcs = fcs_dummy.squeeze(drop=True).transpose('station_id', 'time', 'year', 'step', 'number')\n",
    "essd_obs = obs_dummy.transpose('station_id', 'time', 'year', 'step')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd03899b",
   "metadata": {},
   "source": [
    "## Dropping stations with no data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82c508bb",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "nodata = np.load('stationd_id_nodata.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8777c1e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "essd_obs = essd_obs.isel(station_id=~nodata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c933eabd",
   "metadata": {},
   "outputs": [],
   "source": [
    "essd_fcs = essd_fcs.isel(station_id=~nodata)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "26b69a7f",
   "metadata": {},
   "source": [
    "## Final test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96ede3a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "essd_fcs.name = 't2m'\n",
    "essd_obs.name = 't2m'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c318589f",
   "metadata": {},
   "outputs": [],
   "source": [
    "essd_fcs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "710a2302",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "essd_obs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47a04776",
   "metadata": {},
   "outputs": [],
   "source": [
    "station_id_order = np.loadtxt('station_id_order.txt').astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9542bbef",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert(np.all(essd_obs.station_id.to_numpy() == essd_fcs.station_id.to_numpy()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ddd11eef",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert(np.all(station_id_order == essd_fcs.station_id.to_numpy()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ea63ff1",
   "metadata": {},
   "source": [
    "## Saving the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5caa178",
   "metadata": {},
   "outputs": [],
   "source": [
    "essd_fcs.to_netcdf('ESSD_benchmark_training_data_forecasts.nc')\n",
    "essd_obs.to_netcdf('ESSD_benchmark_training_data_observations.nc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf47c161",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}