From ecec6a835863d6197618d1ce0d22ef208f04d14d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guilherme=20Castel=C3=A3o?= Date: Mon, 18 Mar 2024 08:00:24 -0600 Subject: [PATCH] Syntax update using numpy and scipy (#196) * fix: np.product() is deprecated Replaced by np.prod() * fix: gaussian_filter() moving to scipy.ndimage The `scipy.ndimage.filters` namespace is deprecated and will be removed in SciPy 2.0.0 * Explicit requirement on numpy and scipy I just noticed that although numpy and scipy are used explicitly in sup3r, there was no explicit dependency on those. The respective minimum versions where defined based on the syntax update in the previous commit. * Holding TensorFlow to <2.16 There is something going wrong with TF-2.16 (see PR #198). --- requirements.txt | 4 +++- sup3r/bias/bias_calc.py | 2 +- sup3r/bias/bias_transforms.py | 2 +- sup3r/models/abstract.py | 4 ++-- sup3r/models/multi_step.py | 2 +- sup3r/pipeline/forward_pass.py | 18 +++++++++--------- sup3r/postprocessing/file_handling.py | 4 ++-- sup3r/preprocessing/batch_handling.py | 2 +- sup3r/preprocessing/data_handling/base.py | 4 ++-- .../data_handling/dual_data_handling.py | 4 ++-- .../data_handling/exo_extraction.py | 2 +- .../data_handling/exogenous_data_handling.py | 8 ++++---- .../data_handling/nc_data_handling.py | 2 +- sup3r/qa/stats.py | 2 +- sup3r/utilities/interpolate_log_profile.py | 4 ++-- sup3r/utilities/interpolation.py | 4 ++-- sup3r/utilities/pytest.py | 2 +- sup3r/utilities/regridder.py | 4 ++-- sup3r/utilities/utilities.py | 2 +- 19 files changed, 39 insertions(+), 37 deletions(-) diff --git a/requirements.txt b/requirements.txt index 017d898fa..0f83724a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,9 +6,11 @@ NREL-farms>=1.0.4 google-auth-oauthlib==0.5.3 pytest>=5.2 pillow>=10.0 -tensorflow>2.4 +tensorflow>2.4,<2.16 xarray>=2023.0 netCDF4==1.5.8 +numpy>=1.7.0 dask>=2022.0 sphinx>=7.0 +scipy>=1.0.0 pandas>=2.0 diff --git a/sup3r/bias/bias_calc.py b/sup3r/bias/bias_calc.py index c20f25b23..2ea899ac4 100644 --- a/sup3r/bias/bias_calc.py +++ b/sup3r/bias/bias_calc.py @@ -15,7 +15,7 @@ import rex from rex.utilities.fun_utils import get_fun_call_str from scipy import stats -from scipy.ndimage.filters import gaussian_filter +from scipy.ndimage import gaussian_filter from scipy.spatial import KDTree import sup3r.preprocessing.data_handling diff --git a/sup3r/bias/bias_transforms.py b/sup3r/bias/bias_transforms.py index 3995a0f3b..1d5c14614 100644 --- a/sup3r/bias/bias_transforms.py +++ b/sup3r/bias/bias_transforms.py @@ -6,7 +6,7 @@ import numpy as np from rex import Resource -from scipy.ndimage.filters import gaussian_filter +from scipy.ndimage import gaussian_filter logger = logging.getLogger(__name__) diff --git a/sup3r/models/abstract.py b/sup3r/models/abstract.py index 43498127f..e54182de0 100644 --- a/sup3r/models/abstract.py +++ b/sup3r/models/abstract.py @@ -114,7 +114,7 @@ def get_s_enhance_from_layers(self): if hasattr(self, '_gen'): s_enhancements = [getattr(layer, '_spatial_mult', 1) for layer in self._gen.layers] - s_enhance = int(np.product(s_enhancements)) + s_enhance = int(np.prod(s_enhancements)) return s_enhance # pylint: disable=E1101 @@ -125,7 +125,7 @@ def get_t_enhance_from_layers(self): if hasattr(self, '_gen'): t_enhancements = [getattr(layer, '_temporal_mult', 1) for layer in self._gen.layers] - t_enhance = int(np.product(t_enhancements)) + t_enhance = int(np.prod(t_enhancements)) return t_enhance @property diff --git a/sup3r/models/multi_step.py b/sup3r/models/multi_step.py index a500c20db..b1074d822 100644 --- a/sup3r/models/multi_step.py +++ b/sup3r/models/multi_step.py @@ -449,7 +449,7 @@ def preflight(self): msg = ('Solar and wind spatial enhancements must be equivalent but ' 'received models that do spatial enhancements of ' '{} (solar) and {} (wind)'.format(s_enh, w_enh)) - assert np.product(s_enh) == np.product(w_enh), msg + assert np.prod(s_enh) == np.prod(w_enh), msg s_t_feat = self.spatial_solar_models.lr_features s_o_feat = self.spatial_solar_models.hr_out_features diff --git a/sup3r/pipeline/forward_pass.py b/sup3r/pipeline/forward_pass.py index 18a214979..28bdf0dc5 100644 --- a/sup3r/pipeline/forward_pass.py +++ b/sup3r/pipeline/forward_pass.py @@ -100,8 +100,8 @@ def __init__(self, self.time_steps = time_steps self.s_enhancements = s_enhancements self.t_enhancements = t_enhancements - self.s_enhance = np.product(self.s_enhancements) - self.t_enhance = np.product(self.t_enhancements) + self.s_enhance = np.prod(self.s_enhancements) + self.t_enhance = np.prod(self.t_enhancements) self.dummy_time_index = np.arange(time_steps) self.temporal_slice = temporal_slice self.temporal_pad = temporal_pad @@ -775,8 +775,8 @@ def __init__(self, models = getattr(model, 'models', [model]) self.s_enhancements = [model.s_enhance for model in models] self.t_enhancements = [model.t_enhance for model in models] - self.s_enhance = np.product(self.s_enhancements) - self.t_enhance = np.product(self.t_enhancements) + self.s_enhance = np.prod(self.s_enhancements) + self.t_enhance = np.prod(self.t_enhancements) self.output_features = model.hr_out_features assert len(self.output_features) > 0, 'No output features!' @@ -841,7 +841,7 @@ def preflight(self): hr_data_shape = (self.grid_shape[0] * self.s_enhance, self.grid_shape[1] * self.s_enhance, ) - self.gids = np.arange(np.product(hr_data_shape)) + self.gids = np.arange(np.prod(hr_data_shape)) self.gids = self.gids.reshape(hr_data_shape) out = self.fwp_slicer.get_spatial_slices() @@ -1475,15 +1475,15 @@ def _get_step_enhance(self, step): s_enhance = 1 t_enhance = 1 else: - s_enhance = np.product( + s_enhance = np.prod( self.strategy.s_enhancements[:model_step]) - t_enhance = np.product( + t_enhance = np.prod( self.strategy.t_enhancements[:model_step]) elif combine_type.lower() in ('output', 'layer'): - s_enhance = np.product( + s_enhance = np.prod( self.strategy.s_enhancements[:model_step + 1]) - t_enhance = np.product( + t_enhance = np.prod( self.strategy.t_enhancements[:model_step + 1]) return s_enhance, t_enhance diff --git a/sup3r/postprocessing/file_handling.py b/sup3r/postprocessing/file_handling.py index ea9901962..445cf866f 100644 --- a/sup3r/postprocessing/file_handling.py +++ b/sup3r/postprocessing/file_handling.py @@ -677,7 +677,7 @@ def invert_uv_features(cls, data, features, lat_lon, max_workers=None): logger.debug('Found heights {} for output features {}' .format(heights, features)) - proc_mem = 4 * np.product(data.shape[:-1]) + proc_mem = 4 * np.prod(data.shape[:-1]) n_procs = len(heights) max_workers = estimate_max_workers(max_workers, proc_mem, n_procs) @@ -797,7 +797,7 @@ def _write_output(cls, data, features, lat_lon, times, out_file, data, features = cls._transform_output(data.copy(), features, lat_lon, max_workers) gids = (gids if gids is not None - else np.arange(np.product(lat_lon.shape[:-1]))) + else np.arange(np.prod(lat_lon.shape[:-1]))) meta = pd.DataFrame({'gid': gids.flatten(), 'latitude': lat_lon[..., 0].flatten(), 'longitude': lat_lon[..., 1].flatten()}) diff --git a/sup3r/preprocessing/batch_handling.py b/sup3r/preprocessing/batch_handling.py index 394d36931..31f0d0563 100644 --- a/sup3r/preprocessing/batch_handling.py +++ b/sup3r/preprocessing/batch_handling.py @@ -10,7 +10,7 @@ import numpy as np from rex.utilities import log_mem -from scipy.ndimage.filters import gaussian_filter +from scipy.ndimage import gaussian_filter from sup3r.preprocessing.data_handling.h5_data_handling import ( DataHandlerDCforH5, diff --git a/sup3r/preprocessing/data_handling/base.py b/sup3r/preprocessing/data_handling/base.py index d1950a5b2..4386217b8 100644 --- a/sup3r/preprocessing/data_handling/base.py +++ b/sup3r/preprocessing/data_handling/base.py @@ -679,7 +679,7 @@ def grid_mem(self): int Number of bytes for a single feature array at a single time step """ - grid_mem = np.product(self.grid_shape) + grid_mem = np.prod(self.grid_shape) # assuming feature arrays are float32 (4 bytes) return 4 * grid_mem @@ -986,7 +986,7 @@ def size(self): size : int Number of total elements contained in data array """ - return np.product(self.requested_shape) + return np.prod(self.requested_shape) def cache_data(self, cache_file_paths): """Cache feature data to file and delete from memory diff --git a/sup3r/preprocessing/data_handling/dual_data_handling.py b/sup3r/preprocessing/data_handling/dual_data_handling.py index 15692f1b0..ddb963483 100644 --- a/sup3r/preprocessing/data_handling/dual_data_handling.py +++ b/sup3r/preprocessing/data_handling/dual_data_handling.py @@ -383,7 +383,7 @@ def grid_mem(self): int Number of bytes for a single feature array at a single time step """ - grid_mem = np.product(self.lr_grid_shape) + grid_mem = np.prod(self.lr_grid_shape) # assuming feature arrays are float32 (4 bytes) return 4 * grid_mem @@ -452,7 +452,7 @@ def shape(self): @property def size(self): """Get low_res size""" - return np.product(self.shape) + return np.prod(self.shape) @property def hr_required_shape(self): diff --git a/sup3r/preprocessing/data_handling/exo_extraction.py b/sup3r/preprocessing/data_handling/exo_extraction.py index cf965199c..1129b8eec 100644 --- a/sup3r/preprocessing/data_handling/exo_extraction.py +++ b/sup3r/preprocessing/data_handling/exo_extraction.py @@ -505,7 +505,7 @@ def get_data(self): df = pd.DataFrame({'topo': self.source_data.flatten(), 'gid_target': self.nn}) - n_target = np.product(self.hr_shape[:-1]) + n_target = np.prod(self.hr_shape[:-1]) df = df[df['gid_target'] != n_target] df = df.sort_values('gid_target') df = df.groupby('gid_target').mean() diff --git a/sup3r/preprocessing/data_handling/exogenous_data_handling.py b/sup3r/preprocessing/data_handling/exogenous_data_handling.py index a17accc5c..1c522f160 100644 --- a/sup3r/preprocessing/data_handling/exogenous_data_handling.py +++ b/sup3r/preprocessing/data_handling/exogenous_data_handling.py @@ -508,12 +508,12 @@ def _get_single_step_enhance(self, step): s_enhance = 1 t_enhance = 1 else: - s_enhance = np.product(s_enhancements[:model_step]) - t_enhance = np.product(t_enhancements[:model_step]) + s_enhance = np.prod(s_enhancements[:model_step]) + t_enhance = np.prod(t_enhancements[:model_step]) elif combine_type.lower() in ('output', 'layer'): - s_enhance = np.product(s_enhancements[:model_step + 1]) - t_enhance = np.product(t_enhancements[:model_step + 1]) + s_enhance = np.prod(s_enhancements[:model_step + 1]) + t_enhance = np.prod(t_enhancements[:model_step + 1]) else: msg = ('Received exo_kwargs entry without valid combine_type ' diff --git a/sup3r/preprocessing/data_handling/nc_data_handling.py b/sup3r/preprocessing/data_handling/nc_data_handling.py index 629a40760..475f88f02 100644 --- a/sup3r/preprocessing/data_handling/nc_data_handling.py +++ b/sup3r/preprocessing/data_handling/nc_data_handling.py @@ -13,7 +13,7 @@ import pandas as pd import xarray as xr from rex import Resource -from scipy.ndimage.filters import gaussian_filter +from scipy.ndimage import gaussian_filter from scipy.spatial import KDTree from scipy.stats import mode diff --git a/sup3r/qa/stats.py b/sup3r/qa/stats.py index 58d8eeccf..8dcc6b682 100644 --- a/sup3r/qa/stats.py +++ b/sup3r/qa/stats.py @@ -8,7 +8,7 @@ import pandas as pd import psutil from rex.utilities.fun_utils import get_fun_call_str -from scipy.ndimage.filters import gaussian_filter +from scipy.ndimage import gaussian_filter from sup3r.preprocessing.feature_handling import Feature from sup3r.qa.utilities import ( diff --git a/sup3r/utilities/interpolate_log_profile.py b/sup3r/utilities/interpolate_log_profile.py index 4b4165426..5b854f1df 100644 --- a/sup3r/utilities/interpolate_log_profile.py +++ b/sup3r/utilities/interpolate_log_profile.py @@ -543,7 +543,7 @@ def _get_timestep_interp_input(cls, lev_array, var_array, idt): """ array_shape = var_array.shape - shape = (array_shape[-3], np.product(array_shape[-2:])) + shape = (array_shape[-3], np.prod(array_shape[-2:])) h_t = lev_array[idt].reshape(shape).T var_t = var_array[idt].reshape(shape).T mask = ~np.isnan(h_t) & ~np.isnan(var_t) @@ -648,7 +648,7 @@ def interp_var_to_height(cls, array_shape = var_array.shape # Flatten h_array and var_array along lat, long axis - shape = (len(levels), array_shape[-4], np.product(array_shape[-2:])) + shape = (len(levels), array_shape[-4], np.prod(array_shape[-2:])) out_array = np.zeros(shape, dtype=np.float32).T total_checks = [] diff --git a/sup3r/utilities/interpolation.py b/sup3r/utilities/interpolation.py index e061d7579..bd9761608 100644 --- a/sup3r/utilities/interpolation.py +++ b/sup3r/utilities/interpolation.py @@ -347,12 +347,12 @@ def interp_to_level(cls, var_array, lev_array, levels): array_shape = var_array.shape # Flatten h_array and var_array along lat, long axis - shape = (len(levels), array_shape[-4], np.product(array_shape[-2:])) + shape = (len(levels), array_shape[-4], np.prod(array_shape[-2:])) out_array = np.zeros(shape, dtype=np.float32).T # iterate through time indices for idt in range(array_shape[0]): - shape = (array_shape[-3], np.product(array_shape[-2:])) + shape = (array_shape[-3], np.prod(array_shape[-2:])) h_tmp = lev_array[idt].reshape(shape).T var_tmp = var_array[idt].reshape(shape).T not_nan = ~np.isnan(h_tmp) & ~np.isnan(var_tmp) diff --git a/sup3r/utilities/pytest.py b/sup3r/utilities/pytest.py index 899bd3b99..4580c9840 100644 --- a/sup3r/utilities/pytest.py +++ b/sup3r/utilities/pytest.py @@ -166,7 +166,7 @@ def make_fake_h5_chunks(td): lon, lat = np.meshgrid(lon, lat) low_res_lat_lon = np.dstack((lat, lon)) - gids = np.arange(np.product(shape[:2])) + gids = np.arange(np.prod(shape[:2])) gids = gids.reshape(shape[:2]) low_res_times = pd_date_range( diff --git a/sup3r/utilities/regridder.py b/sup3r/utilities/regridder.py index 22ad199fb..fa086a230 100644 --- a/sup3r/utilities/regridder.py +++ b/sup3r/utilities/regridder.py @@ -161,7 +161,7 @@ def weights(self): dists = np.array(self.distances, dtype=np.float32) mask = dists < self.MIN_DISTANCE if mask.sum() > 0: - logger.info(f'{np.sum(mask)} of {np.product(mask.shape)} ' + logger.info(f'{np.sum(mask)} of {np.prod(mask.shape)} ' 'distances are zero.') dists[mask] = self.MIN_DISTANCE weights = 1 / dists @@ -360,7 +360,7 @@ def interpolate(cls, distance_chunk, values): dists = np.array(distance_chunk, dtype=np.float32) mask = dists < cls.MIN_DISTANCE if mask.sum() > 0: - logger.info(f'{np.sum(mask)} of {np.product(mask.shape)} ' + logger.info(f'{np.sum(mask)} of {np.prod(mask.shape)} ' 'distances are zero.') dists[mask] = cls.MIN_DISTANCE weights = 1 / dists diff --git a/sup3r/utilities/utilities.py b/sup3r/utilities/utilities.py index 0e155ba45..1e4c7b4a3 100644 --- a/sup3r/utilities/utilities.py +++ b/sup3r/utilities/utilities.py @@ -21,7 +21,7 @@ from scipy import ndimage as nd from scipy.interpolate import RegularGridInterpolator, interp1d from scipy.ndimage import zoom -from scipy.ndimage.filters import gaussian_filter +from scipy.ndimage import gaussian_filter np.random.seed(42)