diff --git a/requirements.txt b/requirements.txt index 017d898fa..0f83724a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,9 +6,11 @@ NREL-farms>=1.0.4 google-auth-oauthlib==0.5.3 pytest>=5.2 pillow>=10.0 -tensorflow>2.4 +tensorflow>2.4,<2.16 xarray>=2023.0 netCDF4==1.5.8 +numpy>=1.7.0 dask>=2022.0 sphinx>=7.0 +scipy>=1.0.0 pandas>=2.0 diff --git a/sup3r/bias/bias_calc.py b/sup3r/bias/bias_calc.py index c20f25b23..2ea899ac4 100644 --- a/sup3r/bias/bias_calc.py +++ b/sup3r/bias/bias_calc.py @@ -15,7 +15,7 @@ import rex from rex.utilities.fun_utils import get_fun_call_str from scipy import stats -from scipy.ndimage.filters import gaussian_filter +from scipy.ndimage import gaussian_filter from scipy.spatial import KDTree import sup3r.preprocessing.data_handling diff --git a/sup3r/bias/bias_transforms.py b/sup3r/bias/bias_transforms.py index 3995a0f3b..1d5c14614 100644 --- a/sup3r/bias/bias_transforms.py +++ b/sup3r/bias/bias_transforms.py @@ -6,7 +6,7 @@ import numpy as np from rex import Resource -from scipy.ndimage.filters import gaussian_filter +from scipy.ndimage import gaussian_filter logger = logging.getLogger(__name__) diff --git a/sup3r/models/abstract.py b/sup3r/models/abstract.py index 43498127f..e54182de0 100644 --- a/sup3r/models/abstract.py +++ b/sup3r/models/abstract.py @@ -114,7 +114,7 @@ def get_s_enhance_from_layers(self): if hasattr(self, '_gen'): s_enhancements = [getattr(layer, '_spatial_mult', 1) for layer in self._gen.layers] - s_enhance = int(np.product(s_enhancements)) + s_enhance = int(np.prod(s_enhancements)) return s_enhance # pylint: disable=E1101 @@ -125,7 +125,7 @@ def get_t_enhance_from_layers(self): if hasattr(self, '_gen'): t_enhancements = [getattr(layer, '_temporal_mult', 1) for layer in self._gen.layers] - t_enhance = int(np.product(t_enhancements)) + t_enhance = int(np.prod(t_enhancements)) return t_enhance @property diff --git a/sup3r/models/multi_step.py b/sup3r/models/multi_step.py index a500c20db..b1074d822 100644 --- a/sup3r/models/multi_step.py +++ b/sup3r/models/multi_step.py @@ -449,7 +449,7 @@ def preflight(self): msg = ('Solar and wind spatial enhancements must be equivalent but ' 'received models that do spatial enhancements of ' '{} (solar) and {} (wind)'.format(s_enh, w_enh)) - assert np.product(s_enh) == np.product(w_enh), msg + assert np.prod(s_enh) == np.prod(w_enh), msg s_t_feat = self.spatial_solar_models.lr_features s_o_feat = self.spatial_solar_models.hr_out_features diff --git a/sup3r/pipeline/forward_pass.py b/sup3r/pipeline/forward_pass.py index 18a214979..28bdf0dc5 100644 --- a/sup3r/pipeline/forward_pass.py +++ b/sup3r/pipeline/forward_pass.py @@ -100,8 +100,8 @@ def __init__(self, self.time_steps = time_steps self.s_enhancements = s_enhancements self.t_enhancements = t_enhancements - self.s_enhance = np.product(self.s_enhancements) - self.t_enhance = np.product(self.t_enhancements) + self.s_enhance = np.prod(self.s_enhancements) + self.t_enhance = np.prod(self.t_enhancements) self.dummy_time_index = np.arange(time_steps) self.temporal_slice = temporal_slice self.temporal_pad = temporal_pad @@ -775,8 +775,8 @@ def __init__(self, models = getattr(model, 'models', [model]) self.s_enhancements = [model.s_enhance for model in models] self.t_enhancements = [model.t_enhance for model in models] - self.s_enhance = np.product(self.s_enhancements) - self.t_enhance = np.product(self.t_enhancements) + self.s_enhance = np.prod(self.s_enhancements) + self.t_enhance = np.prod(self.t_enhancements) self.output_features = model.hr_out_features assert len(self.output_features) > 0, 'No output features!' @@ -841,7 +841,7 @@ def preflight(self): hr_data_shape = (self.grid_shape[0] * self.s_enhance, self.grid_shape[1] * self.s_enhance, ) - self.gids = np.arange(np.product(hr_data_shape)) + self.gids = np.arange(np.prod(hr_data_shape)) self.gids = self.gids.reshape(hr_data_shape) out = self.fwp_slicer.get_spatial_slices() @@ -1475,15 +1475,15 @@ def _get_step_enhance(self, step): s_enhance = 1 t_enhance = 1 else: - s_enhance = np.product( + s_enhance = np.prod( self.strategy.s_enhancements[:model_step]) - t_enhance = np.product( + t_enhance = np.prod( self.strategy.t_enhancements[:model_step]) elif combine_type.lower() in ('output', 'layer'): - s_enhance = np.product( + s_enhance = np.prod( self.strategy.s_enhancements[:model_step + 1]) - t_enhance = np.product( + t_enhance = np.prod( self.strategy.t_enhancements[:model_step + 1]) return s_enhance, t_enhance diff --git a/sup3r/postprocessing/file_handling.py b/sup3r/postprocessing/file_handling.py index ea9901962..445cf866f 100644 --- a/sup3r/postprocessing/file_handling.py +++ b/sup3r/postprocessing/file_handling.py @@ -677,7 +677,7 @@ def invert_uv_features(cls, data, features, lat_lon, max_workers=None): logger.debug('Found heights {} for output features {}' .format(heights, features)) - proc_mem = 4 * np.product(data.shape[:-1]) + proc_mem = 4 * np.prod(data.shape[:-1]) n_procs = len(heights) max_workers = estimate_max_workers(max_workers, proc_mem, n_procs) @@ -797,7 +797,7 @@ def _write_output(cls, data, features, lat_lon, times, out_file, data, features = cls._transform_output(data.copy(), features, lat_lon, max_workers) gids = (gids if gids is not None - else np.arange(np.product(lat_lon.shape[:-1]))) + else np.arange(np.prod(lat_lon.shape[:-1]))) meta = pd.DataFrame({'gid': gids.flatten(), 'latitude': lat_lon[..., 0].flatten(), 'longitude': lat_lon[..., 1].flatten()}) diff --git a/sup3r/preprocessing/batch_handling.py b/sup3r/preprocessing/batch_handling.py index 394d36931..31f0d0563 100644 --- a/sup3r/preprocessing/batch_handling.py +++ b/sup3r/preprocessing/batch_handling.py @@ -10,7 +10,7 @@ import numpy as np from rex.utilities import log_mem -from scipy.ndimage.filters import gaussian_filter +from scipy.ndimage import gaussian_filter from sup3r.preprocessing.data_handling.h5_data_handling import ( DataHandlerDCforH5, diff --git a/sup3r/preprocessing/data_handling/base.py b/sup3r/preprocessing/data_handling/base.py index d1950a5b2..4386217b8 100644 --- a/sup3r/preprocessing/data_handling/base.py +++ b/sup3r/preprocessing/data_handling/base.py @@ -679,7 +679,7 @@ def grid_mem(self): int Number of bytes for a single feature array at a single time step """ - grid_mem = np.product(self.grid_shape) + grid_mem = np.prod(self.grid_shape) # assuming feature arrays are float32 (4 bytes) return 4 * grid_mem @@ -986,7 +986,7 @@ def size(self): size : int Number of total elements contained in data array """ - return np.product(self.requested_shape) + return np.prod(self.requested_shape) def cache_data(self, cache_file_paths): """Cache feature data to file and delete from memory diff --git a/sup3r/preprocessing/data_handling/dual_data_handling.py b/sup3r/preprocessing/data_handling/dual_data_handling.py index 15692f1b0..ddb963483 100644 --- a/sup3r/preprocessing/data_handling/dual_data_handling.py +++ b/sup3r/preprocessing/data_handling/dual_data_handling.py @@ -383,7 +383,7 @@ def grid_mem(self): int Number of bytes for a single feature array at a single time step """ - grid_mem = np.product(self.lr_grid_shape) + grid_mem = np.prod(self.lr_grid_shape) # assuming feature arrays are float32 (4 bytes) return 4 * grid_mem @@ -452,7 +452,7 @@ def shape(self): @property def size(self): """Get low_res size""" - return np.product(self.shape) + return np.prod(self.shape) @property def hr_required_shape(self): diff --git a/sup3r/preprocessing/data_handling/exo_extraction.py b/sup3r/preprocessing/data_handling/exo_extraction.py index cf965199c..1129b8eec 100644 --- a/sup3r/preprocessing/data_handling/exo_extraction.py +++ b/sup3r/preprocessing/data_handling/exo_extraction.py @@ -505,7 +505,7 @@ def get_data(self): df = pd.DataFrame({'topo': self.source_data.flatten(), 'gid_target': self.nn}) - n_target = np.product(self.hr_shape[:-1]) + n_target = np.prod(self.hr_shape[:-1]) df = df[df['gid_target'] != n_target] df = df.sort_values('gid_target') df = df.groupby('gid_target').mean() diff --git a/sup3r/preprocessing/data_handling/exogenous_data_handling.py b/sup3r/preprocessing/data_handling/exogenous_data_handling.py index a17accc5c..1c522f160 100644 --- a/sup3r/preprocessing/data_handling/exogenous_data_handling.py +++ b/sup3r/preprocessing/data_handling/exogenous_data_handling.py @@ -508,12 +508,12 @@ def _get_single_step_enhance(self, step): s_enhance = 1 t_enhance = 1 else: - s_enhance = np.product(s_enhancements[:model_step]) - t_enhance = np.product(t_enhancements[:model_step]) + s_enhance = np.prod(s_enhancements[:model_step]) + t_enhance = np.prod(t_enhancements[:model_step]) elif combine_type.lower() in ('output', 'layer'): - s_enhance = np.product(s_enhancements[:model_step + 1]) - t_enhance = np.product(t_enhancements[:model_step + 1]) + s_enhance = np.prod(s_enhancements[:model_step + 1]) + t_enhance = np.prod(t_enhancements[:model_step + 1]) else: msg = ('Received exo_kwargs entry without valid combine_type ' diff --git a/sup3r/preprocessing/data_handling/nc_data_handling.py b/sup3r/preprocessing/data_handling/nc_data_handling.py index 629a40760..475f88f02 100644 --- a/sup3r/preprocessing/data_handling/nc_data_handling.py +++ b/sup3r/preprocessing/data_handling/nc_data_handling.py @@ -13,7 +13,7 @@ import pandas as pd import xarray as xr from rex import Resource -from scipy.ndimage.filters import gaussian_filter +from scipy.ndimage import gaussian_filter from scipy.spatial import KDTree from scipy.stats import mode diff --git a/sup3r/qa/stats.py b/sup3r/qa/stats.py index 58d8eeccf..8dcc6b682 100644 --- a/sup3r/qa/stats.py +++ b/sup3r/qa/stats.py @@ -8,7 +8,7 @@ import pandas as pd import psutil from rex.utilities.fun_utils import get_fun_call_str -from scipy.ndimage.filters import gaussian_filter +from scipy.ndimage import gaussian_filter from sup3r.preprocessing.feature_handling import Feature from sup3r.qa.utilities import ( diff --git a/sup3r/utilities/interpolate_log_profile.py b/sup3r/utilities/interpolate_log_profile.py index 4b4165426..5b854f1df 100644 --- a/sup3r/utilities/interpolate_log_profile.py +++ b/sup3r/utilities/interpolate_log_profile.py @@ -543,7 +543,7 @@ def _get_timestep_interp_input(cls, lev_array, var_array, idt): """ array_shape = var_array.shape - shape = (array_shape[-3], np.product(array_shape[-2:])) + shape = (array_shape[-3], np.prod(array_shape[-2:])) h_t = lev_array[idt].reshape(shape).T var_t = var_array[idt].reshape(shape).T mask = ~np.isnan(h_t) & ~np.isnan(var_t) @@ -648,7 +648,7 @@ def interp_var_to_height(cls, array_shape = var_array.shape # Flatten h_array and var_array along lat, long axis - shape = (len(levels), array_shape[-4], np.product(array_shape[-2:])) + shape = (len(levels), array_shape[-4], np.prod(array_shape[-2:])) out_array = np.zeros(shape, dtype=np.float32).T total_checks = [] diff --git a/sup3r/utilities/interpolation.py b/sup3r/utilities/interpolation.py index e061d7579..bd9761608 100644 --- a/sup3r/utilities/interpolation.py +++ b/sup3r/utilities/interpolation.py @@ -347,12 +347,12 @@ def interp_to_level(cls, var_array, lev_array, levels): array_shape = var_array.shape # Flatten h_array and var_array along lat, long axis - shape = (len(levels), array_shape[-4], np.product(array_shape[-2:])) + shape = (len(levels), array_shape[-4], np.prod(array_shape[-2:])) out_array = np.zeros(shape, dtype=np.float32).T # iterate through time indices for idt in range(array_shape[0]): - shape = (array_shape[-3], np.product(array_shape[-2:])) + shape = (array_shape[-3], np.prod(array_shape[-2:])) h_tmp = lev_array[idt].reshape(shape).T var_tmp = var_array[idt].reshape(shape).T not_nan = ~np.isnan(h_tmp) & ~np.isnan(var_tmp) diff --git a/sup3r/utilities/pytest.py b/sup3r/utilities/pytest.py index 899bd3b99..4580c9840 100644 --- a/sup3r/utilities/pytest.py +++ b/sup3r/utilities/pytest.py @@ -166,7 +166,7 @@ def make_fake_h5_chunks(td): lon, lat = np.meshgrid(lon, lat) low_res_lat_lon = np.dstack((lat, lon)) - gids = np.arange(np.product(shape[:2])) + gids = np.arange(np.prod(shape[:2])) gids = gids.reshape(shape[:2]) low_res_times = pd_date_range( diff --git a/sup3r/utilities/regridder.py b/sup3r/utilities/regridder.py index 22ad199fb..fa086a230 100644 --- a/sup3r/utilities/regridder.py +++ b/sup3r/utilities/regridder.py @@ -161,7 +161,7 @@ def weights(self): dists = np.array(self.distances, dtype=np.float32) mask = dists < self.MIN_DISTANCE if mask.sum() > 0: - logger.info(f'{np.sum(mask)} of {np.product(mask.shape)} ' + logger.info(f'{np.sum(mask)} of {np.prod(mask.shape)} ' 'distances are zero.') dists[mask] = self.MIN_DISTANCE weights = 1 / dists @@ -360,7 +360,7 @@ def interpolate(cls, distance_chunk, values): dists = np.array(distance_chunk, dtype=np.float32) mask = dists < cls.MIN_DISTANCE if mask.sum() > 0: - logger.info(f'{np.sum(mask)} of {np.product(mask.shape)} ' + logger.info(f'{np.sum(mask)} of {np.prod(mask.shape)} ' 'distances are zero.') dists[mask] = cls.MIN_DISTANCE weights = 1 / dists diff --git a/sup3r/utilities/utilities.py b/sup3r/utilities/utilities.py index 0e155ba45..1e4c7b4a3 100644 --- a/sup3r/utilities/utilities.py +++ b/sup3r/utilities/utilities.py @@ -21,7 +21,7 @@ from scipy import ndimage as nd from scipy.interpolate import RegularGridInterpolator, interp1d from scipy.ndimage import zoom -from scipy.ndimage.filters import gaussian_filter +from scipy.ndimage import gaussian_filter np.random.seed(42)