diff --git a/README.md b/README.md index 22fe1417..13bd9256 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,12 @@ Usage: -i INPUT_FILE(S) -o OUTPUT_FILE -m nesdis -k maskout -t 0. ``` For method option (-m) of bias and uncertainty calculation (default/nesdis), deafult means to set bias and uncertainty as 0.0 and nesdis means to use NESDIS bias and uncertainty calculation method. For maskout option (-k) default/maskout, default means to keep all missing values and maskout means to not write out missing values. For thinning option, the value should be within 0.0 and 1.0 depending how much data will be thinned, and 0.0 means without any thining. +The AOD converter normally produces a single IODA output file, but it can also produce two additional output files, one containing only smoke-affected obs and one containing only dust-affected obs. This functionality uses flagging in the Aerosol Data Product (ADP) files created as companion datasets for the native AOD550 observation files. To activate the smoke and dust processing, use the --adp_mask flag followed by the ADP file(s) that correspond to the AOD file(s) specified with -i. + +``` +Usage (with smoke and dust processing): -i AOD_INPUT_FILE(S) --adp_mask ADP_INPUT_FILE(S) --adp_conf_lvl 'medhigh' -o OUTPUT_FILE -m nesdis -k maskout -n 0.0 +``` +The ADP files contain a confidence level for the smoke and dust flagging. The confidence level of the smoke- and dust-affected obs retained in the smoke and dust IODA output files can be specified using --adp_conf_lvl [level], where level can be "low", "med", "medhigh", or "high". The default is "medhigh". ## land diff --git a/src/compo/viirs_aod2ioda.py b/src/compo/viirs_aod2ioda.py index 09cb257a..72ad72b7 100755 --- a/src/compo/viirs_aod2ioda.py +++ b/src/compo/viirs_aod2ioda.py @@ -17,6 +17,50 @@ from collections import defaultdict, OrderedDict from pyiodaconv.orddicts import DefaultOrderedDict + +def flag_by_confidence_lvl(qfarr, aerosol_type, conf_lvl='MedHigh', print_diag=False): + + # ADP smoke confidence is defined in (counting from 0) bit 2-3, dust in bit 4-5 + # input array contains decimal representations of the binary values + # 10 (2 in base10) is medium confidence and 00 (0 in base10) is high confidence + # 01 (1 in base10) is low confidence and 11 (3 in base10) is missing/bad data + + if conf_lvl == 'med' or conf_lvl == 'medhigh': + conf_lvl = 'MedHigh' + else: + conf_lvl = conf_lvl.title() + + if aerosol_type == 'Smoke': + rhtrim = np.floor(qfarr/4) # trim off the 2 least significant bits + elif aerosol_type == 'Dust': + rhtrim = np.floor(qfarr/16) # trim off the 4 least significant bits + else: + print(f'ERROR: aerosol_type {aerosol_type} not valid') + + conf = np.mod(rhtrim, 4) # trim off all except the 2 least significant bits of the remaining value; this gives the value what we want + + if conf_lvl == 'MedHigh': + inv_conf_mask = np.logical_or(conf == 0, conf == 2) + elif conf_lvl == 'High': + inv_conf_mask = conf == 0 + elif conf_lvl == 'Med': + inv_conf_mask = conf == 2 + elif conf_lvl == 'Low': + inv_conf_mask = conf == 1 + else: + print(f'ERROR: No configuration for confidence level {conf_lvl}') + + if print_diag: + for i, val in enumerate(conf_mask): + qfarr_i = int(qfarr[i]) + conf_i = int(conf[i]) + # use bitmasks to convert bin() output for negative vals to two's complement + print(f'{i} Orig dvalue: {qfarr_i} Orig bvalue: {bin(qfarr_i & 0b11111111)} {aerosol_type} bvalue:' + '{bin(conf_i & 0b11)} mask_value: {val}') + + return inv_conf_mask # true elements here are values we want to keep (invert for an actual mask) + + locationKeyList = [ ("latitude", "float"), ("longitude", "float"), @@ -25,12 +69,15 @@ obsvars = ["aerosolOpticalDepth"] channels = [4] + # A dictionary of global attributes. More filled in further down. AttrData = {} AttrData['ioda_object_type'] = 'AOD' # A dictionary of variable dimensions. DimDict = {} +DimDict_smoke = {} +DimDict_dust = {} # A dictionary of variable names and their dimensions. VarDims = {'aerosolOpticalDepth': ['Location', 'Channel']} @@ -45,17 +92,25 @@ class AOD(object): - def __init__(self, filenames, method, mask, thin): + def __init__(self, filenames, method, mask, thin, adp_mask, adp_conf_lvl): self.filenames = filenames self.mask = mask self.method = method self.thin = thin + self.adp_mask = adp_mask + self.adp_conf_lvl = adp_conf_lvl + if self.adp_conf_lvl is None: + self.adp_conf_lvl = 'MedHigh' # default to including Medium and High Confidence obs based on ADP data self.varDict = defaultdict(lambda: defaultdict(dict)) self.outdata = defaultdict(lambda: DefaultOrderedDict(OrderedDict)) self.varAttrs = DefaultOrderedDict(lambda: DefaultOrderedDict(dict)) + if self.adp_mask is not None: + self.outdata_smoke = defaultdict(lambda: DefaultOrderedDict(OrderedDict)) + self.outdata_dust = defaultdict(lambda: DefaultOrderedDict(OrderedDict)) self._read() def _read(self): + # set up variable names for IODA for iodavar in obsvars: self.varDict[iodavar]['valKey'] = iodavar, obsValName @@ -79,9 +134,31 @@ def _read(self): self.outdata[self.varDict[iodavar]['errKey']] = np.array([], dtype=np.float32) self.outdata[self.varDict[iodavar]['qcKey']] = np.array([], dtype=np.int32) - # loop through input filenamess - for f in self.filenames: - ncd = nc.Dataset(f, 'r') + if self.adp_mask is not None: + self.outdata_smoke[('latitude', metaDataName)] = np.array([], dtype=np.float32) + self.outdata_smoke[('longitude', metaDataName)] = np.array([], dtype=np.float32) + self.outdata_smoke[('dateTime', metaDataName)] = np.array([], dtype=object) + for iodavar in obsvars: + self.outdata_smoke[self.varDict[iodavar]['valKey']] = np.array([], dtype=np.float32) + self.outdata_smoke[self.varDict[iodavar]['errKey']] = np.array([], dtype=np.float32) + self.outdata_smoke[self.varDict[iodavar]['qcKey']] = np.array([], dtype=np.int32) + + self.outdata_dust[('latitude', metaDataName)] = np.array([], dtype=np.float32) + self.outdata_dust[('longitude', metaDataName)] = np.array([], dtype=np.float32) + self.outdata_dust[('dateTime', metaDataName)] = np.array([], dtype=object) + for iodavar in obsvars: + self.outdata_dust[self.varDict[iodavar]['valKey']] = np.array([], dtype=np.float32) + self.outdata_dust[self.varDict[iodavar]['errKey']] = np.array([], dtype=np.float32) + self.outdata_dust[self.varDict[iodavar]['qcKey']] = np.array([], dtype=np.int32) + + # loop through input filenames + # both filenames and corresponding adp_mask filename(s) need to be lists, i believe + for n, f in enumerate(self.filenames): + try: + ncd = nc.Dataset(f, 'r') + except FileNotFoundError: + print(f'AOD data file {f} not found, continuing to next file in list or ending if at end of list.') + continue gatts = {attr: getattr(ncd, attr) for attr in ncd.ncattrs()} base_datetime = datetime.strptime(gatts["time_coverage_end"], '%Y-%m-%dT%H:%M:%SZ') self.satellite = gatts["satellite_name"] @@ -107,7 +184,70 @@ def _read(self): qcall = ncd.variables['QCAll'][:].ravel().astype('int32') obs_time = np.full(np.shape(qcall), base_datetime, dtype=object) - if self.mask == "maskout": + + ncd.close() + + # apply ADP mask if indicated by user + if self.adp_mask is not None: + + # Check if ADP file start/end times match AOD file (creation time can differ slightly) + # Using negative indexing makes the paths irrelevant for the comparison + if self.adp_mask[n][-53:-20] != f[-53:-20]: + print(f'ADP data file {self.adp_mask[n]} start and end times do not match those of AOD file {f}') + print(f'Continuing to next AOD and ADP file (or ending)') + continue + + print('Processing the following AOD and ADP files:') + print(f) + print(self.adp_mask[n]) + print() + + # open ADP file corresponding to AOD file. + try: + ncd_adp = nc.Dataset(self.adp_mask[n], 'r') + except FileNotFoundError: + # this will skip creating output for both the AOD and ADP at the selected time; + # however, that should be fine, as when this code runs we are interested only in + # the ADP-filtered output rather than the full AOD output. + print(f'ADP data file {self.adp_mask[n]} not found, continuing to next AOD file on list (or ending).') + continue + + # Get ADP smoke/dust masks + # These are 1=True=Smoke/Dust and 0=False=No Smoke/Dust. The 1 values are the ones + # we want to *keep*, so to mask out all the non-smoke/non-dust values below, + # invert the mask elements here + adpmasksmoke = np.logical_not(ncd_adp.variables['Smoke'][:].ravel().astype(bool)) + adpmaskdust = np.logical_not(ncd_adp.variables['Dust'][:].ravel().astype(bool)) + + qcflag = ncd_adp.variables['QC_Flag'][:, :].ravel() + + # As above, in the confidence masks, med-high confidence is marked by True mask elements, + # due to the line conf_mask = np.logical_or(conf == 0, conf == 2) in the get functions, + # so invert the masks the same as above + conf_mask_smoke = np.logical_not(flag_by_confidence_lvl(qcflag, 'Smoke', self.adp_conf_lvl)) + conf_mask_dust = np.logical_not(flag_by_confidence_lvl(qcflag, 'Dust', self.adp_conf_lvl)) + + ncd_adp.close() + + # apply masks from ADP data including QC masks + vals_smoke = np.ma.masked_where(np.logical_or(adpmasksmoke, conf_mask_smoke), vals) + lons_smoke = np.ma.masked_where(np.logical_or(adpmasksmoke, conf_mask_smoke), lons) + lats_smoke = np.ma.masked_where(np.logical_or(adpmasksmoke, conf_mask_smoke), lats) + errs_smoke = np.ma.masked_where(np.logical_or(adpmasksmoke, conf_mask_smoke), errs) + qcpath_smoke = np.ma.masked_where(np.logical_or(adpmasksmoke, conf_mask_smoke), qcpath) + qcall_smoke = np.ma.masked_where(np.logical_or(adpmasksmoke, conf_mask_smoke), qcall) + obs_time_smoke = np.ma.masked_where(np.logical_or(adpmasksmoke, conf_mask_smoke), obs_time) + + vals_dust = np.ma.masked_where(np.logical_or(adpmaskdust, conf_mask_dust), vals) + lons_dust = np.ma.masked_where(np.logical_or(adpmaskdust, conf_mask_dust), lons) + lats_dust = np.ma.masked_where(np.logical_or(adpmaskdust, conf_mask_dust), lats) + errs_dust = np.ma.masked_where(np.logical_or(adpmaskdust, conf_mask_dust), errs) + qcpath_dust = np.ma.masked_where(np.logical_or(adpmaskdust, conf_mask_dust), qcpath) + qcall_dust = np.ma.masked_where(np.logical_or(adpmaskdust, conf_mask_dust), qcall) + obs_time_dust = np.ma.masked_where(np.logical_or(adpmaskdust, conf_mask_dust), obs_time) + + # Remove masked elements if 'maskout' is indictated + if self.mask == 'maskout': mask = np.logical_not(vals.mask) vals = vals[mask] lons = lons[mask] @@ -117,31 +257,83 @@ def _read(self): qcall = qcall[mask] obs_time = obs_time[mask] - ncd.close() - - # apply thinning mask + if self.adp_mask is not None: + + mask_smoke = np.logical_not(vals_smoke.mask) + vals_smoke = vals_smoke[mask_smoke] + lons_smoke = lons_smoke[mask_smoke] + lats_smoke = lats_smoke[mask_smoke] + errs_smoke = errs_smoke[mask_smoke] + qcpath_smoke = qcpath_smoke[mask_smoke] + qcall_smoke = qcall_smoke[mask_smoke] + obs_time_smoke = obs_time_smoke[mask_smoke] + + mask_dust = np.logical_not(vals_dust.mask) + vals_dust = vals_dust[mask_dust] + lons_dust = lons_dust[mask_dust] + lats_dust = lats_dust[mask_dust] + errs_dust = errs_dust[mask_dust] + qcpath_dust = qcpath_dust[mask_dust] + qcall_dust = qcall_dust[mask_dust] + obs_time_dust = obs_time_dust[mask_dust] + + # Apply thinning mask if self.thin > 0.0: + mask_thin = np.random.uniform(size=len(lons)) > self.thin + vals = vals[mask_thin] lons = lons[mask_thin] lats = lats[mask_thin] - vals = vals[mask_thin] errs = errs[mask_thin] qcpath = qcpath[mask_thin] qcall = qcall[mask_thin] obs_time = obs_time[mask_thin] + if self.adp_mask is not None: + + # switch smoke and dust to just use mask_thin if moved above maskout block + mask_thin_smoke = np.random.uniform(size=len(lons_smoke)) > self.thin + vals_smoke = vals_smoke[mask_thin_smoke] + lons_smoke = lons_smoke[mask_thin_smoke] + lats_smoke = lats_smoke[mask_thin_smoke] + errs_smoke = errs_smoke[mask_thin_smoke] + qcpath_smoke = qcpath_smoke[mask_thin_smoke] + qcall_smoke = qcall_smoke[mask_thin_smoke] + obs_time_smoke = obs_time_smoke[mask_thin_smoke] + + mask_thin_dust = np.random.uniform(size=len(lons_dust)) > self.thin + vals_dust = vals_dust[mask_thin_dust] + lons_dust = lons_dust[mask_thin_dust] + lats_dust = lats_dust[mask_thin_dust] + errs_dust = errs_dust[mask_thin_dust] + qcpath_dust = qcpath_dust[mask_thin_dust] + qcall_dust = qcall_dust[mask_thin_dust] + obs_time_dust = obs_time_dust[mask_thin_dust] + + # calculate and store ob errors # defined surface type and uncertainty if self.method == "nesdis": errs = 0.111431 + 0.128699*vals # over land (dark) errs[qcpath % 2 == 1] = 0.00784394 + 0.219923*vals[qcpath % 2 == 1] # over ocean errs[qcpath % 4 == 2] = 0.0550472 + 0.299558*vals[qcpath % 4 == 2] # over bright land + # also need to make these changes for smoke and dust filtered arrays if smoke and dust filtering indicated by user + if self.adp_mask is not None: + errs_smoke = 0.111431 + 0.128699*vals_smoke # over land (dark) + errs_smoke[qcpath_smoke % 2 == 1] = 0.00784394 + 0.219923*vals_smoke[qcpath_smoke % 2 == 1] # over ocean + errs_smoke[qcpath_smoke % 4 == 2] = 0.0550472 + 0.299558*vals_smoke[qcpath_smoke % 4 == 2] # over bright land + + errs_dust = 0.111431 + 0.128699*vals_dust # over land (dark) + errs_dust[qcpath_dust % 2 == 1] = 0.00784394 + 0.219923*vals_dust[qcpath_dust % 2 == 1] # over ocean + errs_dust[qcpath_dust % 4 == 2] = 0.0550472 + 0.299558*vals_dust[qcpath_dust % 4 == 2] # over bright land + # Write out data self.outdata[('latitude', metaDataName)] = np.append(self.outdata[('latitude', metaDataName)], np.array(lats, dtype=np.float32)) self.outdata[('longitude', metaDataName)] = np.append(self.outdata[('longitude', metaDataName)], np.array(lons, dtype=np.float32)) self.outdata[('dateTime', metaDataName)] = np.append(self.outdata[('dateTime', metaDataName)], np.array(obs_time, dtype=object)) for iodavar in obsvars: + self.outdata[self.varDict[iodavar]['valKey']] = np.append( self.outdata[self.varDict[iodavar]['valKey']], np.array(vals, dtype=np.float32)) self.outdata[self.varDict[iodavar]['errKey']] = np.append( @@ -149,33 +341,69 @@ def _read(self): self.outdata[self.varDict[iodavar]['qcKey']] = np.append( self.outdata[self.varDict[iodavar]['qcKey']], np.array(qcall, dtype=np.int32)) + if self.adp_mask is not None: + + self.outdata_smoke[('latitude', metaDataName)] = np.append( + self.outdata_smoke[('latitude', metaDataName)], np.array(lats_smoke, dtype=np.float32)) + self.outdata_smoke[('longitude', metaDataName)] = np.append( + self.outdata_smoke[('longitude', metaDataName)], np.array(lons_smoke, dtype=np.float32)) + self.outdata_smoke[('dateTime', metaDataName)] = np.append( + self.outdata_smoke[('dateTime', metaDataName)], np.array(obs_time_smoke, dtype=object)) + + self.outdata_dust[('latitude', metaDataName)] = np.append( + self.outdata_dust[('latitude', metaDataName)], np.array(lats_dust, dtype=np.float32)) + self.outdata_dust[('longitude', metaDataName)] = np.append( + self.outdata_dust[('longitude', metaDataName)], np.array(lons_dust, dtype=np.float32)) + self.outdata_dust[('dateTime', metaDataName)] = np.append( + self.outdata_dust[('dateTime', metaDataName)], np.array(obs_time_dust, dtype=object)) + + for iodavar in obsvars: + + self.outdata_smoke[self.varDict[iodavar]['valKey']] = np.append( + self.outdata_smoke[self.varDict[iodavar]['valKey']], np.array(vals_smoke, dtype=np.float32)) + self.outdata_smoke[self.varDict[iodavar]['errKey']] = np.append( + self.outdata_smoke[self.varDict[iodavar]['errKey']], np.array(errs_smoke, dtype=np.float32)) + self.outdata_smoke[self.varDict[iodavar]['qcKey']] = np.append( + self.outdata_smoke[self.varDict[iodavar]['qcKey']], np.array(qcall_smoke, dtype=np.int32)) + + self.outdata_dust[self.varDict[iodavar]['valKey']] = np.append( + self.outdata_dust[self.varDict[iodavar]['valKey']], np.array(vals_dust, dtype=np.float32)) + self.outdata_dust[self.varDict[iodavar]['errKey']] = np.append( + self.outdata_dust[self.varDict[iodavar]['errKey']], np.array(errs_dust, dtype=np.float32)) + self.outdata_dust[self.varDict[iodavar]['qcKey']] = np.append( + self.outdata_dust[self.varDict[iodavar]['qcKey']], np.array(qcall_dust, dtype=np.int32)) + DimDict['Location'] = len(self.outdata[('latitude', metaDataName)]) DimDict['Channel'] = np.array(channels) + if self.adp_mask is not None: + DimDict_smoke['Location'] = len(self.outdata_smoke[('latitude', metaDataName)]) + DimDict_smoke['Channel'] = np.array(channels) + DimDict_dust['Location'] = len(self.outdata_dust[('latitude', metaDataName)]) + DimDict_dust['Channel'] = np.array(channels) + def main(): # get command line arguments - # Usage: python blah.py -i /path/to/obs/2021060801.nc /path/to/obs/2021060802.nc ... -t Analysis_time /path/to/obs/2021060823.nc - # -o /path/to/ioda/20210608.nc - # where the input obs could be for any desired interval to concatenated together. Analysis time is generally the midpoint of - # analysis window. parser = argparse.ArgumentParser( - description=('Read VIIRS aerosol optical depth file(s) and Converter' - ' of native NetCDF format for observations of optical' + description=('Read VIIRS aerosol optical depth file(s) and convert' + ' native NetCDF format for observations of optical' ' depth from VIIRS AOD550 to IODA-V2 netCDF format.') ) parser.add_argument( '-i', '--input', - help="path of viirs aod input file(s)", + help="path of viirs AOD input file(s)", type=str, nargs='+', required=True) parser.add_argument( '-o', '--output', - help="name of ioda-v2 output file", + help="""name of base IODA-V2 output file; if ADP smoke and dust processing is activated, smoke and dust filenames + will have '_smoke' or '_dust' added either before the filetype suffix or, if there is no suffix, at the end + of the filename""", type=str, required=True) parser.add_argument( '-m', '--method', - help="calculation error method: nesdis/default, default=none", + help="obs error calculation method: nesdis/default, default=none", type=str, required=True) parser.add_argument( '-k', '--mask', @@ -186,18 +414,61 @@ def main(): help="percentage of random thinning fro 0.0 to 1.0. Zero indicates" " no thinning is performed. (default: %(default)s)", type=float, default=0.0) + parser.add_argument( + '--adp_mask', + help="""activate ADP-based separation of smoke and dust affected obs from full AOD dataset and specify the path of + ADP input file(s). ADP files should each correspond to an AOD file specified with -i or --input""", + type=str, nargs='+', required=False) + parser.add_argument( + '--adp_conf_lvl', + help="set ADP confidence level(s) of smoke and dust obs to keep; use 'low', 'med', 'medhigh', or 'high': default='medhigh'", + type=str, required=False) args = parser.parse_args() # setup the IODA writer - - # Read in the AOD data - aod = AOD(args.input, args.method, args.mask, args.thin) - - # write everything out - - writer = iconv.IodaWriter(args.output, locationKeyList, DimDict) - writer.BuildIoda(aod.outdata, VarDims, aod.varAttrs, AttrData) + # Read in the AOD data and perform requested processing + aod = AOD(args.input, args.method, args.mask, args.thin, args.adp_mask, args.adp_conf_lvl) + + # Write everything out. Use try-except because if all obs are filtered out of + # one of the datasets due to either masking or thinning or a combination, an IndexError will be thrown + if not args.adp_mask: + try: + print('writing full AOD data') + writer = iconv.IodaWriter(args.output, locationKeyList, DimDict) + writer.BuildIoda(aod.outdata, VarDims, aod.varAttrs, AttrData) + print(f"AOD ({DimDict['Location']} obs) written to: {args.output}") + except IndexError: + print('viirs_aod2ioda.py: No obs to write for non smoke- and dust-filtered AOD at this time') + + else: + orig_pn = os.path.dirname(args.output) + orig_fn = os.path.basename(args.output) + smoke_fn = os.path.splitext(orig_fn)[0] + '_smoke' + os.path.splitext(orig_fn)[-1] + smoke_fullpath = os.path.join(orig_pn, smoke_fn) + dust_fn = os.path.splitext(orig_fn)[0] + '_dust' + os.path.splitext(orig_fn)[-1] + dust_fullpath = os.path.join(orig_pn, dust_fn) + + try: + writer = iconv.IodaWriter(args.output, locationKeyList, DimDict) + writer.BuildIoda(aod.outdata, VarDims, aod.varAttrs, AttrData) + print(f"AOD ({DimDict['Location']} obs) written to: {args.output}") + except IndexError: + print('viirs_aod2ioda.py: No obs to write for full AOD at this time') + + try: + writer = iconv.IodaWriter(smoke_fullpath, locationKeyList, DimDict_smoke) + writer.BuildIoda(aod.outdata_smoke, VarDims, aod.varAttrs, AttrData) + print(f"Smoke AOD ({DimDict_smoke['Location']} obs) written to: {smoke_fullpath}") + except IndexError: + print('viirs_aod2ioda.py: No obs to write for smoke-filtered AOD at this time') + + try: + writer = iconv.IodaWriter(dust_fullpath, locationKeyList, DimDict_dust) + writer.BuildIoda(aod.outdata_dust, VarDims, aod.varAttrs, AttrData) + print(f"Dust AOD ({DimDict_dust['Location']} obs) written to: {dust_fullpath}") + except IndexError: + print('viirs_aod2ioda.py: No obs to write for dust-filtered AOD at this time') if __name__ == '__main__': diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index f90341f5..5a6da51f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -99,6 +99,8 @@ list( APPEND test_input testinput/madis_2021010100.nc testinput/gpm_dpr_20240401-S105223-E122536.sample.hdf5 testinput/Pandora57s1_BoulderCO_L2_rnvs3p1-8.txt + testinput/JRR-AOD_v3r2_j01_s20240110T202044Z_e20240110T202044Z.sample.nc + testinput/JRR-ADP_v3r2_j01_s20240110T202044Z_e20240110T202044Z.sample.nc ) list( APPEND test_output @@ -191,6 +193,7 @@ list( APPEND test_output testoutput/madis_snod_2021010100.nc testoutput/20240401T105230_PT1M_dpr_gpm.nc testoutput/Pandora57s1_BoulderCO_L2_rnvs3p1-8.nc + testoutput/obs.20240110T202044Z_PT1M_aod_smoke.nc ) if( iodaconv_gnssro_ENABLED ) @@ -1251,6 +1254,22 @@ ecbuild_add_test( TARGET test_${PROJECT_NAME}_viirs_aod -n 0.0" viirs_aod.nc ${IODA_CONV_COMP_TOL}) +ecbuild_add_test( TARGET test_${PROJECT_NAME}_viirs_smoke_dust + TYPE SCRIPT + ENVIRONMENT "PYTHONPATH=${IODACONV_PYTHONPATH}" + COMMAND bash + ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh + netcdf + "${Python3_EXECUTABLE} ${CMAKE_BINARY_DIR}/bin/viirs_aod2ioda.py + -i testinput/JRR-AOD_v3r2_j01_s20240110T202044Z_e20240110T202044Z.sample.nc + --adp_mask testinput/JRR-ADP_v3r2_j01_s20240110T202044Z_e20240110T202044Z.sample.nc + --adp_conf_lvl 'medhigh' + -o testrun/obs.20240110T202044Z_PT1M_aod.nc + -m nesdis + -k maskout + -n 0.0" + obs.20240110T202044Z_PT1M_aod_smoke.nc ${IODA_CONV_COMP_TOL}) + ecbuild_add_test( TARGET test_${PROJECT_NAME}_satbias_viirs_aod TYPE SCRIPT COMMAND bash diff --git a/test/testinput/JRR-ADP_v3r2_j01_s20240110T202044Z_e20240110T202044Z.sample.nc b/test/testinput/JRR-ADP_v3r2_j01_s20240110T202044Z_e20240110T202044Z.sample.nc new file mode 100644 index 00000000..2e69d6f5 --- /dev/null +++ b/test/testinput/JRR-ADP_v3r2_j01_s20240110T202044Z_e20240110T202044Z.sample.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c31a7f2cc00cc02bfc3c4d6259846358a83b25753da94fc20f354555620e0d04 +size 118032 diff --git a/test/testinput/JRR-AOD_v3r2_j01_s20240110T202044Z_e20240110T202044Z.sample.nc b/test/testinput/JRR-AOD_v3r2_j01_s20240110T202044Z_e20240110T202044Z.sample.nc new file mode 100644 index 00000000..fa1fd3c7 --- /dev/null +++ b/test/testinput/JRR-AOD_v3r2_j01_s20240110T202044Z_e20240110T202044Z.sample.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70a669416b2a3a38427dae68e428c54c060e0a899318e147c5a56e13c92c18e8 +size 574094 diff --git a/test/testoutput/obs.20240110T202044Z_PT1M_aod_smoke.nc b/test/testoutput/obs.20240110T202044Z_PT1M_aod_smoke.nc new file mode 100644 index 00000000..612ea8db --- /dev/null +++ b/test/testoutput/obs.20240110T202044Z_PT1M_aod_smoke.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:724ecc04f5dd1c327a33b59795c86e0c901e3313c5d62553603e55436639e8dd +size 14268