From 6a98a1aa5088713a8dec47499793d068f50fab80 Mon Sep 17 00:00:00 2001
From: blake-j-allen <blake.j.allen@noaa.gov>
Date: Wed, 4 Sep 2024 19:55:43 +0000
Subject: [PATCH 01/15] modified to add identification and separate output of
 smoke and dust affected observations

---
 src/compo/viirs_aod2ioda.py | 313 +++++++++++++++++++++++++++++++++---
 1 file changed, 292 insertions(+), 21 deletions(-)

diff --git a/src/compo/viirs_aod2ioda.py b/src/compo/viirs_aod2ioda.py
index 09cb257ac..6945fb794 100755
--- a/src/compo/viirs_aod2ioda.py
+++ b/src/compo/viirs_aod2ioda.py
@@ -13,10 +13,56 @@
 import numpy as np
 import os
 
+import glob
 import pyiodaconv.ioda_conv_engines as iconv
 from collections import defaultdict, OrderedDict
 from pyiodaconv.orddicts import DefaultOrderedDict
 
+def get_confidence_qc_mask(qfarr,aerosol_type,conf_lvl='MedHigh'):
+
+    #ADP smoke confidence is defined in (counting from 0) bit 2-3, dust in bit 4-5
+    #input array contains decimal representations of the binary values
+    #10 (2 in base10) is medium confidence and 00 (0 in base10) is high confidence
+    #01 (1 in base10) is low confidence and 11 (3 in base10) is missing/bad data
+
+    if conf_lvl == 'med' or conf_lvl == 'medhigh':
+      conf_lvl = 'MedHigh'
+    if conf_lvl == 'low':
+      conf_lvl = 'Low'
+    if conf_lvl == 'high':
+      conf_lvl ='High'
+
+    if aerosol_type == 'Smoke':
+        rhtrim = np.floor(qfarr/4) #trim off the 2 least significant bits
+    elif aerosol_type == 'Dust':
+        rhtrim = np.floor(qfarr/16) #trim off the 4 least significant bits
+    else:
+        print(f'ERROR: aerosol_type {aerosol_type} not valid')
+
+    conf = np.mod(rhtrim,4) #trim off all except the 2 least significant bits of the remaining value; this gives the value what we want
+
+    if conf_lvl == 'MedHigh':
+        conf_mask = np.logical_or(conf == 0, conf == 2)
+    elif conf_lvl == 'High':
+        conf_mask = conf == 0
+    elif conf_lvl == 'Med':
+        conf_mask = conf == 2
+    elif conf_lvl == 'Low':
+        conf_mask = conf == 1
+    else:
+        print(f'ERROR: No configuration for confidence level {conf_lvl}')
+
+    #diagnostic test printout
+    #binary output needs converted to twos complement to directly check negative values
+    # for i,val in enumerate(conf_mask):
+    #     qfarr_i = int(qfarr[i])
+    #     conf_i = int(conf[i])
+    #     #use bitmasks to convert bin() output for negative vals to two's complement
+    #     print(f'{i} Orig dvalue: {qfarr_i} Orig bvalue: {bin(qfarr_i & 0b11111111)}  {aerosol_type} bvalue: {bin(conf_i & 0b11)} mask_value: {val}')
+    return conf_mask #true elements here are values we want to keep (invert for an actual mask)
+
+
+
 locationKeyList = [
     ("latitude", "float"),
     ("longitude", "float"),
@@ -25,12 +71,15 @@
 
 obsvars = ["aerosolOpticalDepth"]
 channels = [4]
+
 # A dictionary of global attributes.  More filled in further down.
 AttrData = {}
 AttrData['ioda_object_type'] = 'AOD'
 
 # A dictionary of variable dimensions.
 DimDict = {}
+DimDict_smoke = {}
+DimDict_dust = {}
 
 # A dictionary of variable names and their dimensions.
 VarDims = {'aerosolOpticalDepth': ['Location', 'Channel']}
@@ -45,17 +94,26 @@
 
 
 class AOD(object):
-    def __init__(self, filenames, method, mask, thin):
+    def __init__(self, filenames, method, mask, thin, adp_mask, adp_qc_lvl):
         self.filenames = filenames
         self.mask = mask
         self.method = method
         self.thin = thin
+        self.adp_mask = adp_mask
+        self.adp_qc_lvl = adp_qc_lvl
+        if self.adp_qc_lvl is None:
+          self.adp_qc_lvl = 'MedHigh' #default to including Medium and High Confidence obs based on ADP data
         self.varDict = defaultdict(lambda: defaultdict(dict))
         self.outdata = defaultdict(lambda: DefaultOrderedDict(OrderedDict))
         self.varAttrs = DefaultOrderedDict(lambda: DefaultOrderedDict(dict))
+        if self.adp_mask:
+            self.outdata_smoke = defaultdict(lambda: DefaultOrderedDict(OrderedDict))
+            self.outdata_dust = defaultdict(lambda: DefaultOrderedDict(OrderedDict))
         self._read()
 
+
     def _read(self):
+
         # set up variable names for IODA
         for iodavar in obsvars:
             self.varDict[iodavar]['valKey'] = iodavar, obsValName
@@ -79,9 +137,30 @@ def _read(self):
             self.outdata[self.varDict[iodavar]['errKey']] = np.array([], dtype=np.float32)
             self.outdata[self.varDict[iodavar]['qcKey']] = np.array([], dtype=np.int32)
 
+        if self.adp_mask:
+            self.outdata_smoke[('latitude', metaDataName)] = np.array([], dtype=np.float32)
+            self.outdata_smoke[('longitude', metaDataName)] = np.array([], dtype=np.float32)
+            self.outdata_smoke[('dateTime', metaDataName)] = np.array([], dtype=object)
+            for iodavar in obsvars:
+                self.outdata_smoke[self.varDict[iodavar]['valKey']] = np.array([], dtype=np.float32)
+                self.outdata_smoke[self.varDict[iodavar]['errKey']] = np.array([], dtype=np.float32)
+                self.outdata_smoke[self.varDict[iodavar]['qcKey']] = np.array([], dtype=np.int32)
+
+            self.outdata_dust[('latitude', metaDataName)] = np.array([], dtype=np.float32)
+            self.outdata_dust[('longitude', metaDataName)] = np.array([], dtype=np.float32)
+            self.outdata_dust[('dateTime', metaDataName)] = np.array([], dtype=object)
+            for iodavar in obsvars:
+                self.outdata_dust[self.varDict[iodavar]['valKey']] = np.array([], dtype=np.float32)
+                self.outdata_dust[self.varDict[iodavar]['errKey']] = np.array([], dtype=np.float32)
+                self.outdata_dust[self.varDict[iodavar]['qcKey']] = np.array([], dtype=np.int32)
+
         # loop through input filenamess
         for f in self.filenames:
-            ncd = nc.Dataset(f, 'r')
+            try:
+                ncd = nc.Dataset(f, 'r')
+            except FileNotFoundError:
+                print(f'AOD data file {f} not found, continuing to next file in list or ending if at end of list.')
+                continue
             gatts = {attr: getattr(ncd, attr) for attr in ncd.ncattrs()}
             base_datetime = datetime.strptime(gatts["time_coverage_end"], '%Y-%m-%dT%H:%M:%SZ')
             self.satellite = gatts["satellite_name"]
@@ -107,7 +186,75 @@ def _read(self):
 
             qcall = ncd.variables['QCAll'][:].ravel().astype('int32')
             obs_time = np.full(np.shape(qcall), base_datetime, dtype=object)
-            if self.mask == "maskout":
+
+            ncd.close()
+
+            #apply ADP mask if indicated by user
+            if self.adp_mask:
+
+                #Set path and filename of adp data corresponding to aod data
+                #Note: this works for the directory structure that exists for these on hera
+                #another approach would be to have the masked output filenames specified as a CLI argument
+
+                #the creation time can differ between the AOD and ADP filenames,
+                #break up fname and reconstruct it from pieces after using glob to get corresponding ADP fname
+                adp_fn_ctime_to_end =  f.replace('aod', 'adp').replace("JRR-AOD", "JRR-ADP").split('_')[-1]
+                adp_fn_before_ctime =  f.replace('aod', 'adp').replace("JRR-AOD", "JRR-ADP")[:-19]
+                adp_fn = glob.glob(adp_fn_before_ctime+'*')
+                if len(adp_fn)==1:
+                  adp_fn = adp_fn[0]
+                  print(f'Found ADP file: {adp_fn}')
+                elif len(adp_fn)>1:
+                  print(f'AOD file: {f}')
+                  print(f'ADP files: {adp_fn}')
+                  print(f'ERROR: Too many ADP files found for AOD file, skipping granule.')
+                  continue
+
+                try:
+                    ncd_adp = nc.Dataset(adp_fn, 'r')
+                except FileNotFoundError:
+                    #this will skip creating output for both the AOD and ADP at the selected time;
+                    #however, that should be fine, as when this code runs we are interested only in
+                    #the ADP-filtered output rather than the full AOD output.
+                    print(f'ADP data file {adp_fn} not found, continuing to next AOD file on list (or ending).')
+                    continue
+
+                #Get ADP smoke/dust masks
+                #These are 1=True=Smoke/Dust and 0=False=No Smoke/Dust. The 1 values are the ones
+                #we want to *keep*, so to mask out all the non-smoke/non-dust values below,
+                #invert the mask elements here
+                adpmasksmoke = np.logical_not(ncd_adp.variables['Smoke'][:].ravel().astype(bool))
+                adpmaskdust = np.logical_not(ncd_adp.variables['Dust'][:].ravel().astype(bool))
+
+                qcflag = ncd_adp.variables['QC_Flag'][:,:].ravel()
+
+                #As above, in the confidence masks, med-high confidence is marked by True mask elements,
+                #due to the line conf_mask = np.logical_or(conf == 0, conf == 2) in the get functions,
+                #so invert the masks the same as above
+                conf_mask_smoke = np.logical_not(get_confidence_qc_mask(qcflag,'Smoke',self.adp_qc_lvl))
+                conf_mask_dust = np.logical_not(get_confidence_qc_mask(qcflag,'Dust',self.adp_qc_lvl))
+
+                ncd_adp.close()
+
+                #apply masks from ADP data including QC masks
+                vals_smoke = np.ma.masked_where(np.logical_or(adpmasksmoke,conf_mask_smoke),vals)
+                lons_smoke = np.ma.masked_where(np.logical_or(adpmasksmoke,conf_mask_smoke),lons)
+                lats_smoke = np.ma.masked_where(np.logical_or(adpmasksmoke,conf_mask_smoke),lats)
+                errs_smoke = np.ma.masked_where(np.logical_or(adpmasksmoke,conf_mask_smoke),errs)
+                qcpath_smoke = np.ma.masked_where(np.logical_or(adpmasksmoke,conf_mask_smoke),qcpath)
+                qcall_smoke = np.ma.masked_where(np.logical_or(adpmasksmoke,conf_mask_smoke),qcall)
+                obs_time_smoke = np.ma.masked_where(np.logical_or(adpmasksmoke,conf_mask_smoke),obs_time)
+
+                vals_dust = np.ma.masked_where(np.logical_or(adpmaskdust,conf_mask_dust),vals)
+                lons_dust = np.ma.masked_where(np.logical_or(adpmaskdust,conf_mask_dust),lons)
+                lats_dust = np.ma.masked_where(np.logical_or(adpmaskdust,conf_mask_dust),lats)
+                errs_dust = np.ma.masked_where(np.logical_or(adpmaskdust,conf_mask_dust),errs)
+                qcpath_dust = np.ma.masked_where(np.logical_or(adpmaskdust,conf_mask_dust),qcpath)
+                qcall_dust = np.ma.masked_where(np.logical_or(adpmaskdust,conf_mask_dust),qcall)
+                obs_time_dust = np.ma.masked_where(np.logical_or(adpmaskdust,conf_mask_dust),obs_time)
+
+            #Remove masked elements if 'maskout' is indictated
+            if self.mask == 'maskout':
                 mask = np.logical_not(vals.mask)
                 vals = vals[mask]
                 lons = lons[mask]
@@ -117,31 +264,84 @@ def _read(self):
                 qcall = qcall[mask]
                 obs_time = obs_time[mask]
 
-            ncd.close()
+                if self.adp_mask:
+
+                    mask_smoke = np.logical_not(vals_smoke.mask)
+                    vals_smoke = vals_smoke[mask_smoke]
+                    lons_smoke = lons_smoke[mask_smoke]
+                    lats_smoke = lats_smoke[mask_smoke]
+                    errs_smoke = errs_smoke[mask_smoke]
+                    qcpath_smoke = qcpath_smoke[mask_smoke]
+                    qcall_smoke = qcall_smoke[mask_smoke]
+                    obs_time_smoke = obs_time_smoke[mask_smoke]
 
-            # apply thinning mask
+                    mask_dust = np.logical_not(vals_dust.mask)
+                    vals_dust = vals_dust[mask_dust]
+                    lons_dust = lons_dust[mask_dust]
+                    lats_dust = lats_dust[mask_dust]
+                    errs_dust = errs_dust[mask_dust]
+                    qcpath_dust = qcpath_dust[mask_dust]
+                    qcall_dust = qcall_dust[mask_dust]
+                    obs_time_dust = obs_time_dust[mask_dust]
+
+
+            #Apply thinning mask
             if self.thin > 0.0:
+
                 mask_thin = np.random.uniform(size=len(lons)) > self.thin
+                vals = vals[mask_thin]
                 lons = lons[mask_thin]
                 lats = lats[mask_thin]
-                vals = vals[mask_thin]
                 errs = errs[mask_thin]
                 qcpath = qcpath[mask_thin]
                 qcall = qcall[mask_thin]
                 obs_time = obs_time[mask_thin]
 
+                if self.adp_mask:
+
+                    #switch smoke and dust to just use mask_thin if moved above maskout block
+                    mask_thin_smoke = np.random.uniform(size=len(lons_smoke)) > self.thin
+                    vals_smoke = vals_smoke[mask_thin_smoke]
+                    lons_smoke = lons_smoke[mask_thin_smoke]
+                    lats_smoke = lats_smoke[mask_thin_smoke]
+                    errs_smoke = errs_smoke[mask_thin_smoke]
+                    qcpath_smoke = qcpath_smoke[mask_thin_smoke]
+                    qcall_smoke = qcall_smoke[mask_thin_smoke]
+                    obs_time_smoke = obs_time_smoke[mask_thin_smoke]
+
+                    mask_thin_dust = np.random.uniform(size=len(lons_dust)) > self.thin
+                    vals_dust = vals_dust[mask_thin_dust]
+                    lons_dust = lons_dust[mask_thin_dust]
+                    lats_dust = lats_dust[mask_thin_dust]
+                    errs_dust = errs_dust[mask_thin_dust]
+                    qcpath_dust = qcpath_dust[mask_thin_dust]
+                    qcall_dust = qcall_dust[mask_thin_dust]
+                    obs_time_dust = obs_time_dust[mask_thin_dust]
+
+            # calculate and store ob errors
             # defined surface type and uncertainty
             if self.method == "nesdis":
                 errs = 0.111431 + 0.128699*vals    # over land (dark)
                 errs[qcpath % 2 == 1] = 0.00784394 + 0.219923*vals[qcpath % 2 == 1]  # over ocean
                 errs[qcpath % 4 == 2] = 0.0550472 + 0.299558*vals[qcpath % 4 == 2]   # over bright land
 
+                #also need to make these changes for smoke and dust filtered arrays if smoke and dust filtering indicated by user
+                if self.adp_mask:
+                    errs_smoke = 0.111431 + 0.128699*vals_smoke    # over land (dark)
+                    errs_smoke[qcpath_smoke % 2 == 1] = 0.00784394 + 0.219923*vals_smoke[qcpath_smoke % 2 == 1]  # over ocean
+                    errs_smoke[qcpath_smoke % 4 == 2] = 0.0550472 + 0.299558*vals_smoke[qcpath_smoke % 4 == 2]   # over bright land
+
+                    errs_dust = 0.111431 + 0.128699*vals_dust    # over land (dark)
+                    errs_dust[qcpath_dust % 2 == 1] = 0.00784394 + 0.219923*vals_dust[qcpath_dust % 2 == 1]  # over ocean
+                    errs_dust[qcpath_dust % 4 == 2] = 0.0550472 + 0.299558*vals_dust[qcpath_dust % 4 == 2]   # over bright land
+
             #  Write out data
             self.outdata[('latitude', metaDataName)] = np.append(self.outdata[('latitude', metaDataName)], np.array(lats, dtype=np.float32))
             self.outdata[('longitude', metaDataName)] = np.append(self.outdata[('longitude', metaDataName)], np.array(lons, dtype=np.float32))
             self.outdata[('dateTime', metaDataName)] = np.append(self.outdata[('dateTime', metaDataName)], np.array(obs_time, dtype=object))
 
             for iodavar in obsvars:
+
                 self.outdata[self.varDict[iodavar]['valKey']] = np.append(
                     self.outdata[self.varDict[iodavar]['valKey']], np.array(vals, dtype=np.float32))
                 self.outdata[self.varDict[iodavar]['errKey']] = np.append(
@@ -149,17 +349,46 @@ def _read(self):
                 self.outdata[self.varDict[iodavar]['qcKey']] = np.append(
                     self.outdata[self.varDict[iodavar]['qcKey']], np.array(qcall, dtype=np.int32))
 
-        DimDict['Location'] = len(self.outdata[('latitude', metaDataName)])
+            if self.adp_mask:
+
+                self.outdata_smoke[('latitude', metaDataName)] = np.append(self.outdata_smoke[('latitude', metaDataName)], np.array(lats_smoke, dtype=np.float32))
+                self.outdata_smoke[('longitude', metaDataName)] = np.append(self.outdata_smoke[('longitude', metaDataName)], np.array(lons_smoke, dtype=np.float32))
+                self.outdata_smoke[('dateTime', metaDataName)] = np.append(self.outdata_smoke[('dateTime', metaDataName)], np.array(obs_time_smoke, dtype=object))
+
+                self.outdata_dust[('latitude', metaDataName)] = np.append(self.outdata_dust[('latitude', metaDataName)], np.array(lats_dust, dtype=np.float32))
+                self.outdata_dust[('longitude', metaDataName)] = np.append(self.outdata_dust[('longitude', metaDataName)], np.array(lons_dust, dtype=np.float32))
+                self.outdata_dust[('dateTime', metaDataName)] = np.append(self.outdata_dust[('dateTime', metaDataName)], np.array(obs_time_dust, dtype=object))
+
+                for iodavar in obsvars:
+
+                    self.outdata_smoke[self.varDict[iodavar]['valKey']] = np.append(
+                        self.outdata_smoke[self.varDict[iodavar]['valKey']], np.array(vals_smoke, dtype=np.float32))
+                    self.outdata_smoke[self.varDict[iodavar]['errKey']] = np.append(
+                        self.outdata_smoke[self.varDict[iodavar]['errKey']], np.array(errs_smoke, dtype=np.float32))
+                    self.outdata_smoke[self.varDict[iodavar]['qcKey']] = np.append(
+                        self.outdata_smoke[self.varDict[iodavar]['qcKey']], np.array(qcall_smoke, dtype=np.int32))
+
+                    self.outdata_dust[self.varDict[iodavar]['valKey']] = np.append(
+                        self.outdata_dust[self.varDict[iodavar]['valKey']], np.array(vals_dust, dtype=np.float32))
+                    self.outdata_dust[self.varDict[iodavar]['errKey']] = np.append(
+                        self.outdata_dust[self.varDict[iodavar]['errKey']], np.array(errs_dust, dtype=np.float32))
+                    self.outdata_dust[self.varDict[iodavar]['qcKey']] = np.append(
+                        self.outdata_dust[self.varDict[iodavar]['qcKey']], np.array(qcall_dust, dtype=np.int32))
+
+
+        DimDict['Location'] = len(self.outdata[('latitude', metaDataName)]) 
         DimDict['Channel'] = np.array(channels)
 
+        if self.adp_mask:
+            DimDict_smoke['Location'] = len(self.outdata_smoke[('latitude', metaDataName)]) 
+            DimDict_smoke['Channel'] = np.array(channels)
+            DimDict_dust['Location'] = len(self.outdata_dust[('latitude', metaDataName)]) 
+            DimDict_dust['Channel'] = np.array(channels)
+
 
 def main():
 
     # get command line arguments
-    # Usage: python blah.py -i /path/to/obs/2021060801.nc /path/to/obs/2021060802.nc ... -t Analysis_time /path/to/obs/2021060823.nc
-    # -o /path/to/ioda/20210608.nc
-    # where the input obs could be for any desired interval to concatenated together. Analysis time is generally the midpoint of
-    # analysis window.
     parser = argparse.ArgumentParser(
         description=('Read VIIRS aerosol optical depth file(s) and Converter'
                      ' of native NetCDF format for observations of optical'
@@ -171,7 +400,7 @@ def main():
         type=str, nargs='+', required=True)
     parser.add_argument(
         '-o', '--output',
-        help="name of ioda-v2 output file",
+        help="name of base ioda-v2 output file; smoke and dust files will add '_smoke' or '_dust' suffix",
         type=str, required=True)
     parser.add_argument(
         '-m', '--method',
@@ -186,19 +415,61 @@ def main():
         help="percentage of random thinning fro 0.0 to 1.0. Zero indicates"
         " no thinning is performed. (default: %(default)s)",
         type=float, default=0.0)
+    parser.add_argument(
+	'--adp_mask',
+	help="activate ADP-based separation of smoke and dust affected obs from full AOD dataset",
+	action='store_true',
+	default=False)
+    parser.add_argument(
+	'--adp_qc_lvl',
+	help="set ADP QC confidence level for smoke and dust obs, use 'low','med', 'medhigh',or 'high': default='medhigh'",
+	type=str, required=False)
 
     args = parser.parse_args()
 
     # setup the IODA writer
-
-    # Read in the AOD data
-    aod = AOD(args.input, args.method, args.mask, args.thin)
-
-    # write everything out
-
-    writer = iconv.IodaWriter(args.output, locationKeyList, DimDict)
-    writer.BuildIoda(aod.outdata, VarDims, aod.varAttrs, AttrData)
-
+    # Read in the AOD data and perform requested processing
+    aod = AOD(args.input, args.method, args.mask, args.thin, args.adp_mask, args.adp_qc_lvl)
+
+    # Write everything out. Use try-except because if all obs are filtered out of
+    # one of the datasets due to either masking or thinning or a combination, an IndexError will be thrown
+    if not args.adp_mask:
+        try:
+            print('writing full AOD data')
+            writer = iconv.IodaWriter(args.output, locationKeyList, DimDict)
+            writer.BuildIoda(aod.outdata, VarDims, aod.varAttrs, AttrData)
+            print(f"AOD ({DimDict['Location']} obs) written to: {args.output}")
+        except IndexError:
+            print('viirs_aod2ioda.py: No obs to write for non smoke- and dust-filtered AOD at this time')
+
+    else:
+        orig_pn = os.path.dirname(args.output)
+        orig_fn = os.path.basename(args.output)
+        smoke_fn =  orig_fn.split('.')[0] + '_smoke' + '.' + '.'.join(orig_fn.split('.')[1:])
+        smoke_fullpath = os.path.join(orig_pn, smoke_fn)
+        dust_fn =  orig_fn.split('.')[0] + '_dust' + '.' + '.'.join(orig_fn.split('.')[1:])
+        dust_fullpath = os.path.join(orig_pn, dust_fn)
+
+        try:
+            writer = iconv.IodaWriter(args.output, locationKeyList, DimDict)
+            writer.BuildIoda(aod.outdata, VarDims, aod.varAttrs, AttrData)
+            print(f"AOD ({DimDict['Location']} obs) written to: {args.output}")
+        except IndexError:
+            print('viirs_aod2ioda.py: No obs to write for full AOD at this time')
+
+        try:
+            writer = iconv.IodaWriter(smoke_fullpath, locationKeyList, DimDict_smoke)
+            writer.BuildIoda(aod.outdata_smoke, VarDims, aod.varAttrs, AttrData)
+            print(f"Smoke AOD ({DimDict_smoke['Location']} obs) written to: {smoke_fullpath}")
+        except IndexError:
+            print('viirs_aod2ioda.py: No obs to write for smoke-filtered AOD at this time')
+
+        try:
+            writer = iconv.IodaWriter(dust_fullpath, locationKeyList, DimDict_dust)
+            writer.BuildIoda(aod.outdata_dust, VarDims, aod.varAttrs, AttrData)
+            print(f"Dust AOD ({DimDict_dust['Location']} obs) written to: {dust_fullpath}")
+        except IndexError:
+            print('viirs_aod2ioda.py: No obs to write for dust-filtered AOD at this time')
 
 if __name__ == '__main__':
     main()

From 2966d03aa2eb20c46773613492bd778f34c24563 Mon Sep 17 00:00:00 2001
From: blake-j-allen <Blake.J.Allen@noaa.gov>
Date: Tue, 10 Sep 2024 19:24:37 +0000
Subject: [PATCH 02/15] minor changes as needed to pass coding norm test

---
 src/compo/viirs_aod2ioda.py | 188 ++++++++++++++++++------------------
 1 file changed, 96 insertions(+), 92 deletions(-)

diff --git a/src/compo/viirs_aod2ioda.py b/src/compo/viirs_aod2ioda.py
index 6945fb794..7af1d61b6 100755
--- a/src/compo/viirs_aod2ioda.py
+++ b/src/compo/viirs_aod2ioda.py
@@ -18,28 +18,29 @@
 from collections import defaultdict, OrderedDict
 from pyiodaconv.orddicts import DefaultOrderedDict
 
-def get_confidence_qc_mask(qfarr,aerosol_type,conf_lvl='MedHigh'):
 
-    #ADP smoke confidence is defined in (counting from 0) bit 2-3, dust in bit 4-5
-    #input array contains decimal representations of the binary values
-    #10 (2 in base10) is medium confidence and 00 (0 in base10) is high confidence
-    #01 (1 in base10) is low confidence and 11 (3 in base10) is missing/bad data
+def get_confidence_qc_mask(qfarr, aerosol_type, conf_lvl='MedHigh'):
+
+    # ADP smoke confidence is defined in (counting from 0) bit 2-3, dust in bit 4-5
+    # input array contains decimal representations of the binary values
+    # 10 (2 in base10) is medium confidence and 00 (0 in base10) is high confidence
+    # 01 (1 in base10) is low confidence and 11 (3 in base10) is missing/bad data
 
     if conf_lvl == 'med' or conf_lvl == 'medhigh':
-      conf_lvl = 'MedHigh'
+        conf_lvl = 'MedHigh'
     if conf_lvl == 'low':
-      conf_lvl = 'Low'
+        conf_lvl = 'Low'
     if conf_lvl == 'high':
-      conf_lvl ='High'
+        conf_lvl = 'High'
 
     if aerosol_type == 'Smoke':
-        rhtrim = np.floor(qfarr/4) #trim off the 2 least significant bits
+        rhtrim = np.floor(qfarr/4)  # trim off the 2 least significant bits
     elif aerosol_type == 'Dust':
-        rhtrim = np.floor(qfarr/16) #trim off the 4 least significant bits
+        rhtrim = np.floor(qfarr/16)  # trim off the 4 least significant bits
     else:
         print(f'ERROR: aerosol_type {aerosol_type} not valid')
 
-    conf = np.mod(rhtrim,4) #trim off all except the 2 least significant bits of the remaining value; this gives the value what we want
+    conf = np.mod(rhtrim, 4)  # trim off all except the 2 least significant bits of the remaining value; this gives the value what we want
 
     if conf_lvl == 'MedHigh':
         conf_mask = np.logical_or(conf == 0, conf == 2)
@@ -52,15 +53,14 @@ def get_confidence_qc_mask(qfarr,aerosol_type,conf_lvl='MedHigh'):
     else:
         print(f'ERROR: No configuration for confidence level {conf_lvl}')
 
-    #diagnostic test printout
-    #binary output needs converted to twos complement to directly check negative values
-    # for i,val in enumerate(conf_mask):
-    #     qfarr_i = int(qfarr[i])
-    #     conf_i = int(conf[i])
-    #     #use bitmasks to convert bin() output for negative vals to two's complement
-    #     print(f'{i} Orig dvalue: {qfarr_i} Orig bvalue: {bin(qfarr_i & 0b11111111)}  {aerosol_type} bvalue: {bin(conf_i & 0b11)} mask_value: {val}')
-    return conf_mask #true elements here are values we want to keep (invert for an actual mask)
-
+    # diagnostic test printout
+    # binary output needs converted to twos complement to directly check negative values
+    #  for i,val in enumerate(conf_mask):
+    #      qfarr_i = int(qfarr[i])
+    #      conf_i = int(conf[i])
+    #      #use bitmasks to convert bin() output for negative vals to two's complement
+    #      print(f'{i} Orig dvalue: {qfarr_i} Orig bvalue: {bin(qfarr_i & 0b11111111)}  {aerosol_type} bvalue: {bin(conf_i & 0b11)} mask_value: {val}')
+    return conf_mask  # true elements here are values we want to keep (invert for an actual mask)
 
 
 locationKeyList = [
@@ -102,7 +102,7 @@ def __init__(self, filenames, method, mask, thin, adp_mask, adp_qc_lvl):
         self.adp_mask = adp_mask
         self.adp_qc_lvl = adp_qc_lvl
         if self.adp_qc_lvl is None:
-          self.adp_qc_lvl = 'MedHigh' #default to including Medium and High Confidence obs based on ADP data
+            self.adp_qc_lvl = 'MedHigh'  # default to including Medium and High Confidence obs based on ADP data
         self.varDict = defaultdict(lambda: defaultdict(dict))
         self.outdata = defaultdict(lambda: DefaultOrderedDict(OrderedDict))
         self.varAttrs = DefaultOrderedDict(lambda: DefaultOrderedDict(dict))
@@ -111,7 +111,6 @@ def __init__(self, filenames, method, mask, thin, adp_mask, adp_qc_lvl):
             self.outdata_dust = defaultdict(lambda: DefaultOrderedDict(OrderedDict))
         self._read()
 
-
     def _read(self):
 
         # set up variable names for IODA
@@ -189,71 +188,71 @@ def _read(self):
 
             ncd.close()
 
-            #apply ADP mask if indicated by user
+            # apply ADP mask if indicated by user
             if self.adp_mask:
 
-                #Set path and filename of adp data corresponding to aod data
-                #Note: this works for the directory structure that exists for these on hera
-                #another approach would be to have the masked output filenames specified as a CLI argument
+                # Set path and filename of adp data corresponding to aod data
+                # Note: this works for the directory structure that exists for these on hera
+                # another approach would be to have the masked output filenames specified as a CLI argument
 
-                #the creation time can differ between the AOD and ADP filenames,
-                #break up fname and reconstruct it from pieces after using glob to get corresponding ADP fname
-                adp_fn_ctime_to_end =  f.replace('aod', 'adp').replace("JRR-AOD", "JRR-ADP").split('_')[-1]
-                adp_fn_before_ctime =  f.replace('aod', 'adp').replace("JRR-AOD", "JRR-ADP")[:-19]
+                # the creation time can differ between the AOD and ADP filenames,
+                # break up fname and reconstruct it from pieces after using glob to get corresponding ADP fname
+                adp_fn_ctime_to_end = f.replace('aod', 'adp').replace("JRR-AOD", "JRR-ADP").split('_')[-1]
+                adp_fn_before_ctime = f.replace('aod', 'adp').replace("JRR-AOD", "JRR-ADP")[:-19]
                 adp_fn = glob.glob(adp_fn_before_ctime+'*')
-                if len(adp_fn)==1:
-                  adp_fn = adp_fn[0]
-                  print(f'Found ADP file: {adp_fn}')
-                elif len(adp_fn)>1:
-                  print(f'AOD file: {f}')
-                  print(f'ADP files: {adp_fn}')
-                  print(f'ERROR: Too many ADP files found for AOD file, skipping granule.')
-                  continue
+                if len(adp_fn) == 1:
+                    adp_fn = adp_fn[0]
+                    print(f'Found ADP file: {adp_fn}')
+                elif len(adp_fn) > 1:
+                    print(f'AOD file: {f}')
+                    print(f'ADP files: {adp_fn}')
+                    print(f'ERROR: Too many ADP files found for AOD file, skipping granule.')
+                    continue
 
                 try:
                     ncd_adp = nc.Dataset(adp_fn, 'r')
                 except FileNotFoundError:
-                    #this will skip creating output for both the AOD and ADP at the selected time;
-                    #however, that should be fine, as when this code runs we are interested only in
-                    #the ADP-filtered output rather than the full AOD output.
+                    # this will skip creating output for both the AOD and ADP at the selected time;
+                    # however, that should be fine, as when this code runs we are interested only in
+                    # the ADP-filtered output rather than the full AOD output.
                     print(f'ADP data file {adp_fn} not found, continuing to next AOD file on list (or ending).')
                     continue
 
-                #Get ADP smoke/dust masks
-                #These are 1=True=Smoke/Dust and 0=False=No Smoke/Dust. The 1 values are the ones
-                #we want to *keep*, so to mask out all the non-smoke/non-dust values below,
-                #invert the mask elements here
+                # Get ADP smoke/dust masks
+                # These are 1=True=Smoke/Dust and 0=False=No Smoke/Dust. The 1 values are the ones
+                # we want to *keep*, so to mask out all the non-smoke/non-dust values below,
+                # invert the mask elements here
                 adpmasksmoke = np.logical_not(ncd_adp.variables['Smoke'][:].ravel().astype(bool))
                 adpmaskdust = np.logical_not(ncd_adp.variables['Dust'][:].ravel().astype(bool))
 
-                qcflag = ncd_adp.variables['QC_Flag'][:,:].ravel()
+                qcflag = ncd_adp.variables['QC_Flag'][:, :].ravel()
 
-                #As above, in the confidence masks, med-high confidence is marked by True mask elements,
-                #due to the line conf_mask = np.logical_or(conf == 0, conf == 2) in the get functions,
-                #so invert the masks the same as above
-                conf_mask_smoke = np.logical_not(get_confidence_qc_mask(qcflag,'Smoke',self.adp_qc_lvl))
-                conf_mask_dust = np.logical_not(get_confidence_qc_mask(qcflag,'Dust',self.adp_qc_lvl))
+                # As above, in the confidence masks, med-high confidence is marked by True mask elements,
+                # due to the line conf_mask = np.logical_or(conf == 0, conf == 2) in the get functions,
+                # so invert the masks the same as above
+                conf_mask_smoke = np.logical_not(get_confidence_qc_mask(qcflag, 'Smoke', self.adp_qc_lvl))
+                conf_mask_dust = np.logical_not(get_confidence_qc_mask(qcflag, 'Dust', self.adp_qc_lvl))
 
                 ncd_adp.close()
 
-                #apply masks from ADP data including QC masks
-                vals_smoke = np.ma.masked_where(np.logical_or(adpmasksmoke,conf_mask_smoke),vals)
-                lons_smoke = np.ma.masked_where(np.logical_or(adpmasksmoke,conf_mask_smoke),lons)
-                lats_smoke = np.ma.masked_where(np.logical_or(adpmasksmoke,conf_mask_smoke),lats)
-                errs_smoke = np.ma.masked_where(np.logical_or(adpmasksmoke,conf_mask_smoke),errs)
-                qcpath_smoke = np.ma.masked_where(np.logical_or(adpmasksmoke,conf_mask_smoke),qcpath)
-                qcall_smoke = np.ma.masked_where(np.logical_or(adpmasksmoke,conf_mask_smoke),qcall)
-                obs_time_smoke = np.ma.masked_where(np.logical_or(adpmasksmoke,conf_mask_smoke),obs_time)
-
-                vals_dust = np.ma.masked_where(np.logical_or(adpmaskdust,conf_mask_dust),vals)
-                lons_dust = np.ma.masked_where(np.logical_or(adpmaskdust,conf_mask_dust),lons)
-                lats_dust = np.ma.masked_where(np.logical_or(adpmaskdust,conf_mask_dust),lats)
-                errs_dust = np.ma.masked_where(np.logical_or(adpmaskdust,conf_mask_dust),errs)
-                qcpath_dust = np.ma.masked_where(np.logical_or(adpmaskdust,conf_mask_dust),qcpath)
-                qcall_dust = np.ma.masked_where(np.logical_or(adpmaskdust,conf_mask_dust),qcall)
-                obs_time_dust = np.ma.masked_where(np.logical_or(adpmaskdust,conf_mask_dust),obs_time)
-
-            #Remove masked elements if 'maskout' is indictated
+                # apply masks from ADP data including QC masks
+                vals_smoke = np.ma.masked_where(np.logical_or(adpmasksmoke, conf_mask_smoke), vals)
+                lons_smoke = np.ma.masked_where(np.logical_or(adpmasksmoke, conf_mask_smoke), lons)
+                lats_smoke = np.ma.masked_where(np.logical_or(adpmasksmoke, conf_mask_smoke), lats)
+                errs_smoke = np.ma.masked_where(np.logical_or(adpmasksmoke, conf_mask_smoke), errs)
+                qcpath_smoke = np.ma.masked_where(np.logical_or(adpmasksmoke, conf_mask_smoke), qcpath)
+                qcall_smoke = np.ma.masked_where(np.logical_or(adpmasksmoke, conf_mask_smoke), qcall)
+                obs_time_smoke = np.ma.masked_where(np.logical_or(adpmasksmoke, conf_mask_smoke), obs_time)
+
+                vals_dust = np.ma.masked_where(np.logical_or(adpmaskdust, conf_mask_dust), vals)
+                lons_dust = np.ma.masked_where(np.logical_or(adpmaskdust, conf_mask_dust), lons)
+                lats_dust = np.ma.masked_where(np.logical_or(adpmaskdust, conf_mask_dust), lats)
+                errs_dust = np.ma.masked_where(np.logical_or(adpmaskdust, conf_mask_dust), errs)
+                qcpath_dust = np.ma.masked_where(np.logical_or(adpmaskdust, conf_mask_dust), qcpath)
+                qcall_dust = np.ma.masked_where(np.logical_or(adpmaskdust, conf_mask_dust), qcall)
+                obs_time_dust = np.ma.masked_where(np.logical_or(adpmaskdust, conf_mask_dust), obs_time)
+
+            # Remove masked elements if 'maskout' is indictated
             if self.mask == 'maskout':
                 mask = np.logical_not(vals.mask)
                 vals = vals[mask]
@@ -284,8 +283,7 @@ def _read(self):
                     qcall_dust = qcall_dust[mask_dust]
                     obs_time_dust = obs_time_dust[mask_dust]
 
-
-            #Apply thinning mask
+            # Apply thinning mask
             if self.thin > 0.0:
 
                 mask_thin = np.random.uniform(size=len(lons)) > self.thin
@@ -299,7 +297,7 @@ def _read(self):
 
                 if self.adp_mask:
 
-                    #switch smoke and dust to just use mask_thin if moved above maskout block
+                    # switch smoke and dust to just use mask_thin if moved above maskout block
                     mask_thin_smoke = np.random.uniform(size=len(lons_smoke)) > self.thin
                     vals_smoke = vals_smoke[mask_thin_smoke]
                     lons_smoke = lons_smoke[mask_thin_smoke]
@@ -325,7 +323,7 @@ def _read(self):
                 errs[qcpath % 2 == 1] = 0.00784394 + 0.219923*vals[qcpath % 2 == 1]  # over ocean
                 errs[qcpath % 4 == 2] = 0.0550472 + 0.299558*vals[qcpath % 4 == 2]   # over bright land
 
-                #also need to make these changes for smoke and dust filtered arrays if smoke and dust filtering indicated by user
+                # also need to make these changes for smoke and dust filtered arrays if smoke and dust filtering indicated by user
                 if self.adp_mask:
                     errs_smoke = 0.111431 + 0.128699*vals_smoke    # over land (dark)
                     errs_smoke[qcpath_smoke % 2 == 1] = 0.00784394 + 0.219923*vals_smoke[qcpath_smoke % 2 == 1]  # over ocean
@@ -351,13 +349,19 @@ def _read(self):
 
             if self.adp_mask:
 
-                self.outdata_smoke[('latitude', metaDataName)] = np.append(self.outdata_smoke[('latitude', metaDataName)], np.array(lats_smoke, dtype=np.float32))
-                self.outdata_smoke[('longitude', metaDataName)] = np.append(self.outdata_smoke[('longitude', metaDataName)], np.array(lons_smoke, dtype=np.float32))
-                self.outdata_smoke[('dateTime', metaDataName)] = np.append(self.outdata_smoke[('dateTime', metaDataName)], np.array(obs_time_smoke, dtype=object))
+                self.outdata_smoke[('latitude', metaDataName)] = np.append(
+                    self.outdata_smoke[('latitude', metaDataName)], np.array(lats_smoke, dtype=np.float32))
+                self.outdata_smoke[('longitude', metaDataName)] = np.append(
+                    self.outdata_smoke[('longitude', metaDataName)], np.array(lons_smoke, dtype=np.float32))
+                self.outdata_smoke[('dateTime', metaDataName)] = np.append(
+                    self.outdata_smoke[('dateTime', metaDataName)], np.array(obs_time_smoke, dtype=object))
 
-                self.outdata_dust[('latitude', metaDataName)] = np.append(self.outdata_dust[('latitude', metaDataName)], np.array(lats_dust, dtype=np.float32))
-                self.outdata_dust[('longitude', metaDataName)] = np.append(self.outdata_dust[('longitude', metaDataName)], np.array(lons_dust, dtype=np.float32))
-                self.outdata_dust[('dateTime', metaDataName)] = np.append(self.outdata_dust[('dateTime', metaDataName)], np.array(obs_time_dust, dtype=object))
+                self.outdata_dust[('latitude', metaDataName)] = np.append(
+                    self.outdata_dust[('latitude', metaDataName)], np.array(lats_dust, dtype=np.float32))
+                self.outdata_dust[('longitude', metaDataName)] = np.append(
+                    self.outdata_dust[('longitude', metaDataName)], np.array(lons_dust, dtype=np.float32))
+                self.outdata_dust[('dateTime', metaDataName)] = np.append(
+                    self.outdata_dust[('dateTime', metaDataName)], np.array(obs_time_dust, dtype=object))
 
                 for iodavar in obsvars:
 
@@ -375,14 +379,13 @@ def _read(self):
                     self.outdata_dust[self.varDict[iodavar]['qcKey']] = np.append(
                         self.outdata_dust[self.varDict[iodavar]['qcKey']], np.array(qcall_dust, dtype=np.int32))
 
-
-        DimDict['Location'] = len(self.outdata[('latitude', metaDataName)]) 
+        DimDict['Location'] = len(self.outdata[('latitude', metaDataName)])
         DimDict['Channel'] = np.array(channels)
 
         if self.adp_mask:
-            DimDict_smoke['Location'] = len(self.outdata_smoke[('latitude', metaDataName)]) 
+            DimDict_smoke['Location'] = len(self.outdata_smoke[('latitude', metaDataName)])
             DimDict_smoke['Channel'] = np.array(channels)
-            DimDict_dust['Location'] = len(self.outdata_dust[('latitude', metaDataName)]) 
+            DimDict_dust['Location'] = len(self.outdata_dust[('latitude', metaDataName)])
             DimDict_dust['Channel'] = np.array(channels)
 
 
@@ -416,14 +419,14 @@ def main():
         " no thinning is performed. (default: %(default)s)",
         type=float, default=0.0)
     parser.add_argument(
-	'--adp_mask',
-	help="activate ADP-based separation of smoke and dust affected obs from full AOD dataset",
-	action='store_true',
-	default=False)
+        '--adp_mask',
+        help="activate ADP-based separation of smoke and dust affected obs from full AOD dataset",
+        action='store_true',
+        default=False)
     parser.add_argument(
-	'--adp_qc_lvl',
-	help="set ADP QC confidence level for smoke and dust obs, use 'low','med', 'medhigh',or 'high': default='medhigh'",
-	type=str, required=False)
+        '--adp_qc_lvl',
+        help="set ADP QC confidence level for smoke and dust obs, use 'low','med', 'medhigh',or 'high': default='medhigh'",
+        type=str, required=False)
 
     args = parser.parse_args()
 
@@ -445,9 +448,9 @@ def main():
     else:
         orig_pn = os.path.dirname(args.output)
         orig_fn = os.path.basename(args.output)
-        smoke_fn =  orig_fn.split('.')[0] + '_smoke' + '.' + '.'.join(orig_fn.split('.')[1:])
+        smoke_fn = orig_fn.split('.')[0] + '_smoke' + '.' + '.'.join(orig_fn.split('.')[1:])
         smoke_fullpath = os.path.join(orig_pn, smoke_fn)
-        dust_fn =  orig_fn.split('.')[0] + '_dust' + '.' + '.'.join(orig_fn.split('.')[1:])
+        dust_fn = orig_fn.split('.')[0] + '_dust' + '.' + '.'.join(orig_fn.split('.')[1:])
         dust_fullpath = os.path.join(orig_pn, dust_fn)
 
         try:
@@ -471,5 +474,6 @@ def main():
         except IndexError:
             print('viirs_aod2ioda.py: No obs to write for dust-filtered AOD at this time')
 
+
 if __name__ == '__main__':
     main()

From b02aa6c3147e3bf5f8f5cf117190c101bd55f991 Mon Sep 17 00:00:00 2001
From: blake-j-allen <Blake.J.Allen@noaa.gov>
Date: Tue, 10 Sep 2024 21:02:08 +0000
Subject: [PATCH 03/15] 1.) Changed name of get_confidence_qc_mask() to
 flag_confidence_lvl() for clarity.  2.) Renamed variable conf_mask in
 flag_confidence_lvl() to inv_conf_mask to make clear that it is the inverse
 of a mask rather than a mask.  3.) Placed flag_confidence_lvl() diagnostic
 output block in if block (not run by default) instead of leaving it in as a
 comment block (perhaps this diagnostic prinout block should just be removed,
 though).

---
 src/compo/viirs_aod2ioda.py | 31 ++++++++++++++++---------------
 1 file changed, 16 insertions(+), 15 deletions(-)

diff --git a/src/compo/viirs_aod2ioda.py b/src/compo/viirs_aod2ioda.py
index 7af1d61b6..39c7cbeea 100755
--- a/src/compo/viirs_aod2ioda.py
+++ b/src/compo/viirs_aod2ioda.py
@@ -19,7 +19,7 @@
 from pyiodaconv.orddicts import DefaultOrderedDict
 
 
-def get_confidence_qc_mask(qfarr, aerosol_type, conf_lvl='MedHigh'):
+def flag_by_confidence_lvl(qfarr, aerosol_type, conf_lvl='MedHigh',print_diag=False):
 
     # ADP smoke confidence is defined in (counting from 0) bit 2-3, dust in bit 4-5
     # input array contains decimal representations of the binary values
@@ -43,24 +43,25 @@ def get_confidence_qc_mask(qfarr, aerosol_type, conf_lvl='MedHigh'):
     conf = np.mod(rhtrim, 4)  # trim off all except the 2 least significant bits of the remaining value; this gives the value what we want
 
     if conf_lvl == 'MedHigh':
-        conf_mask = np.logical_or(conf == 0, conf == 2)
+        inv_conf_mask = np.logical_or(conf == 0, conf == 2)
     elif conf_lvl == 'High':
-        conf_mask = conf == 0
+        inv_conf_mask = conf == 0
     elif conf_lvl == 'Med':
-        conf_mask = conf == 2
+        inv_conf_mask = conf == 2
     elif conf_lvl == 'Low':
-        conf_mask = conf == 1
+        inv_conf_mask = conf == 1
     else:
         print(f'ERROR: No configuration for confidence level {conf_lvl}')
 
-    # diagnostic test printout
-    # binary output needs converted to twos complement to directly check negative values
-    #  for i,val in enumerate(conf_mask):
-    #      qfarr_i = int(qfarr[i])
-    #      conf_i = int(conf[i])
-    #      #use bitmasks to convert bin() output for negative vals to two's complement
-    #      print(f'{i} Orig dvalue: {qfarr_i} Orig bvalue: {bin(qfarr_i & 0b11111111)}  {aerosol_type} bvalue: {bin(conf_i & 0b11)} mask_value: {val}')
-    return conf_mask  # true elements here are values we want to keep (invert for an actual mask)
+    if print_diag:
+       for i,val in enumerate(conf_mask):
+          qfarr_i = int(qfarr[i])
+          conf_i = int(conf[i])
+          # use bitmasks to convert bin() output for negative vals to two's complement
+          print(f'{i} Orig dvalue: {qfarr_i} Orig bvalue: {bin(qfarr_i & 0b11111111)}  {aerosol_type} bvalue:'
+                 '{bin(conf_i & 0b11)} mask_value: {val}')
+
+    return inv_conf_mask  # true elements here are values we want to keep (invert for an actual mask)
 
 
 locationKeyList = [
@@ -230,8 +231,8 @@ def _read(self):
                 # As above, in the confidence masks, med-high confidence is marked by True mask elements,
                 # due to the line conf_mask = np.logical_or(conf == 0, conf == 2) in the get functions,
                 # so invert the masks the same as above
-                conf_mask_smoke = np.logical_not(get_confidence_qc_mask(qcflag, 'Smoke', self.adp_qc_lvl))
-                conf_mask_dust = np.logical_not(get_confidence_qc_mask(qcflag, 'Dust', self.adp_qc_lvl))
+                conf_mask_smoke = np.logical_not(flag_by_confidence_lvl(qcflag, 'Smoke', self.adp_qc_lvl))
+                conf_mask_dust = np.logical_not(flag_by_confidence_lvl(qcflag, 'Dust', self.adp_qc_lvl))
 
                 ncd_adp.close()
 

From fe2b80f7dec0f6733120674fe6d9776d2fb6f0c5 Mon Sep 17 00:00:00 2001
From: blake-j-allen <Blake.J.Allen@noaa.gov>
Date: Tue, 10 Sep 2024 22:12:26 +0000
Subject: [PATCH 04/15] * Change construction of smoke and dust output
 filenames to work for more general set of original AOD output filenames

---
 src/compo/viirs_aod2ioda.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/compo/viirs_aod2ioda.py b/src/compo/viirs_aod2ioda.py
index 39c7cbeea..d2433dec5 100755
--- a/src/compo/viirs_aod2ioda.py
+++ b/src/compo/viirs_aod2ioda.py
@@ -449,9 +449,9 @@ def main():
     else:
         orig_pn = os.path.dirname(args.output)
         orig_fn = os.path.basename(args.output)
-        smoke_fn = orig_fn.split('.')[0] + '_smoke' + '.' + '.'.join(orig_fn.split('.')[1:])
+        smoke_fn = os.path.splitext(orig_fn)[0] + '_smoke' + os.path.splitext(orig_fn)[-1]
         smoke_fullpath = os.path.join(orig_pn, smoke_fn)
-        dust_fn = orig_fn.split('.')[0] + '_dust' + '.' + '.'.join(orig_fn.split('.')[1:])
+        dust_fn = os.path.splitext(orig_fn)[0] + '_dust' + os.path.splitext(orig_fn)[-1]
         dust_fullpath = os.path.join(orig_pn, dust_fn)
 
         try:

From b0a08b995b366754fc2f65c23d9605edee8111a0 Mon Sep 17 00:00:00 2001
From: blake-j-allen <Blake.J.Allen@noaa.gov>
Date: Tue, 10 Sep 2024 23:12:41 +0000
Subject: [PATCH 05/15] Changed ADP filename specification

* Previously the program derived the ADP filename from the AOD filename, but it assumed the files were stored in a certain directory structure

* Previously, --adp_mask was a boolean CLI argument

* Now ADP filename(s) specified by user via CLI argument --adp_mask=<file or files>
---
 src/compo/viirs_aod2ioda.py | 53 +++++++++++++++----------------------
 1 file changed, 22 insertions(+), 31 deletions(-)

diff --git a/src/compo/viirs_aod2ioda.py b/src/compo/viirs_aod2ioda.py
index d2433dec5..7a2f85e2a 100755
--- a/src/compo/viirs_aod2ioda.py
+++ b/src/compo/viirs_aod2ioda.py
@@ -107,7 +107,7 @@ def __init__(self, filenames, method, mask, thin, adp_mask, adp_qc_lvl):
         self.varDict = defaultdict(lambda: defaultdict(dict))
         self.outdata = defaultdict(lambda: DefaultOrderedDict(OrderedDict))
         self.varAttrs = DefaultOrderedDict(lambda: DefaultOrderedDict(dict))
-        if self.adp_mask:
+        if self.adp_mask is not None:
             self.outdata_smoke = defaultdict(lambda: DefaultOrderedDict(OrderedDict))
             self.outdata_dust = defaultdict(lambda: DefaultOrderedDict(OrderedDict))
         self._read()
@@ -137,7 +137,7 @@ def _read(self):
             self.outdata[self.varDict[iodavar]['errKey']] = np.array([], dtype=np.float32)
             self.outdata[self.varDict[iodavar]['qcKey']] = np.array([], dtype=np.int32)
 
-        if self.adp_mask:
+        if self.adp_mask is not None:
             self.outdata_smoke[('latitude', metaDataName)] = np.array([], dtype=np.float32)
             self.outdata_smoke[('longitude', metaDataName)] = np.array([], dtype=np.float32)
             self.outdata_smoke[('dateTime', metaDataName)] = np.array([], dtype=object)
@@ -154,8 +154,9 @@ def _read(self):
                 self.outdata_dust[self.varDict[iodavar]['errKey']] = np.array([], dtype=np.float32)
                 self.outdata_dust[self.varDict[iodavar]['qcKey']] = np.array([], dtype=np.int32)
 
-        # loop through input filenamess
-        for f in self.filenames:
+        # loop through input filenames
+        # both filenames and corresponding adp_mask filename(s) need to be lists, i believe
+        for n, f in enumerate(self.filenames):
             try:
                 ncd = nc.Dataset(f, 'r')
             except FileNotFoundError:
@@ -190,28 +191,18 @@ def _read(self):
             ncd.close()
 
             # apply ADP mask if indicated by user
-            if self.adp_mask:
-
-                # Set path and filename of adp data corresponding to aod data
-                # Note: this works for the directory structure that exists for these on hera
-                # another approach would be to have the masked output filenames specified as a CLI argument
-
-                # the creation time can differ between the AOD and ADP filenames,
-                # break up fname and reconstruct it from pieces after using glob to get corresponding ADP fname
-                adp_fn_ctime_to_end = f.replace('aod', 'adp').replace("JRR-AOD", "JRR-ADP").split('_')[-1]
-                adp_fn_before_ctime = f.replace('aod', 'adp').replace("JRR-AOD", "JRR-ADP")[:-19]
-                adp_fn = glob.glob(adp_fn_before_ctime+'*')
-                if len(adp_fn) == 1:
-                    adp_fn = adp_fn[0]
-                    print(f'Found ADP file: {adp_fn}')
-                elif len(adp_fn) > 1:
-                    print(f'AOD file: {f}')
-                    print(f'ADP files: {adp_fn}')
-                    print(f'ERROR: Too many ADP files found for AOD file, skipping granule.')
+            if self.adp_mask is not None:
+
+                # Check if ADP file start/end times match AOD file (creation time can differ slightly)
+                # Using negative indexing makes the paths irrelevant for the comparison
+                if self.adp_mask[n][-53:-20] != f[-53:-20]:
+                    print(f'ADP data file {adp_fn[n]} start and end times do not match those of AOD file {f}')
+                    print(f'Continuing to next AOD and ADP file (or ending)')
                     continue
 
+                # open ADP file corresponding to AOD file.
                 try:
-                    ncd_adp = nc.Dataset(adp_fn, 'r')
+                    ncd_adp = nc.Dataset(adp_fn[n], 'r')
                 except FileNotFoundError:
                     # this will skip creating output for both the AOD and ADP at the selected time;
                     # however, that should be fine, as when this code runs we are interested only in
@@ -264,7 +255,7 @@ def _read(self):
                 qcall = qcall[mask]
                 obs_time = obs_time[mask]
 
-                if self.adp_mask:
+                if self.adp_mask is not None:
 
                     mask_smoke = np.logical_not(vals_smoke.mask)
                     vals_smoke = vals_smoke[mask_smoke]
@@ -296,7 +287,7 @@ def _read(self):
                 qcall = qcall[mask_thin]
                 obs_time = obs_time[mask_thin]
 
-                if self.adp_mask:
+                if self.adp_mask is not None:
 
                     # switch smoke and dust to just use mask_thin if moved above maskout block
                     mask_thin_smoke = np.random.uniform(size=len(lons_smoke)) > self.thin
@@ -325,7 +316,7 @@ def _read(self):
                 errs[qcpath % 4 == 2] = 0.0550472 + 0.299558*vals[qcpath % 4 == 2]   # over bright land
 
                 # also need to make these changes for smoke and dust filtered arrays if smoke and dust filtering indicated by user
-                if self.adp_mask:
+                if self.adp_mask is not None:
                     errs_smoke = 0.111431 + 0.128699*vals_smoke    # over land (dark)
                     errs_smoke[qcpath_smoke % 2 == 1] = 0.00784394 + 0.219923*vals_smoke[qcpath_smoke % 2 == 1]  # over ocean
                     errs_smoke[qcpath_smoke % 4 == 2] = 0.0550472 + 0.299558*vals_smoke[qcpath_smoke % 4 == 2]   # over bright land
@@ -348,7 +339,7 @@ def _read(self):
                 self.outdata[self.varDict[iodavar]['qcKey']] = np.append(
                     self.outdata[self.varDict[iodavar]['qcKey']], np.array(qcall, dtype=np.int32))
 
-            if self.adp_mask:
+            if self.adp_mask is not None:
 
                 self.outdata_smoke[('latitude', metaDataName)] = np.append(
                     self.outdata_smoke[('latitude', metaDataName)], np.array(lats_smoke, dtype=np.float32))
@@ -383,7 +374,7 @@ def _read(self):
         DimDict['Location'] = len(self.outdata[('latitude', metaDataName)])
         DimDict['Channel'] = np.array(channels)
 
-        if self.adp_mask:
+        if self.adp_mask is not None:
             DimDict_smoke['Location'] = len(self.outdata_smoke[('latitude', metaDataName)])
             DimDict_smoke['Channel'] = np.array(channels)
             DimDict_dust['Location'] = len(self.outdata_dust[('latitude', metaDataName)])
@@ -421,9 +412,9 @@ def main():
         type=float, default=0.0)
     parser.add_argument(
         '--adp_mask',
-        help="activate ADP-based separation of smoke and dust affected obs from full AOD dataset",
-        action='store_true',
-        default=False)
+        help="""activate ADP-based separation of smoke and dust affected obs from full AOD dataset and specify the path of ADP
+              file(s). ADP files should each correspond to an AOD file specified with -i or --input""",
+        type=str, required=False)
     parser.add_argument(
         '--adp_qc_lvl',
         help="set ADP QC confidence level for smoke and dust obs, use 'low','med', 'medhigh',or 'high': default='medhigh'",

From 0c339b96c30b3c8366f05af85ae9509e44dbb3ce Mon Sep 17 00:00:00 2001
From: blake-j-allen <Blake.J.Allen@noaa.gov>
Date: Wed, 11 Sep 2024 01:11:53 +0000
Subject: [PATCH 06/15] Small changes to satisfy coding norm test

---
 src/compo/viirs_aod2ioda.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/compo/viirs_aod2ioda.py b/src/compo/viirs_aod2ioda.py
index 7a2f85e2a..2e7b12f4c 100755
--- a/src/compo/viirs_aod2ioda.py
+++ b/src/compo/viirs_aod2ioda.py
@@ -19,7 +19,7 @@
 from pyiodaconv.orddicts import DefaultOrderedDict
 
 
-def flag_by_confidence_lvl(qfarr, aerosol_type, conf_lvl='MedHigh',print_diag=False):
+def flag_by_confidence_lvl(qfarr, aerosol_type, conf_lvl='MedHigh', print_diag=False):
 
     # ADP smoke confidence is defined in (counting from 0) bit 2-3, dust in bit 4-5
     # input array contains decimal representations of the binary values
@@ -54,12 +54,12 @@ def flag_by_confidence_lvl(qfarr, aerosol_type, conf_lvl='MedHigh',print_diag=Fa
         print(f'ERROR: No configuration for confidence level {conf_lvl}')
 
     if print_diag:
-       for i,val in enumerate(conf_mask):
-          qfarr_i = int(qfarr[i])
-          conf_i = int(conf[i])
-          # use bitmasks to convert bin() output for negative vals to two's complement
-          print(f'{i} Orig dvalue: {qfarr_i} Orig bvalue: {bin(qfarr_i & 0b11111111)}  {aerosol_type} bvalue:'
-                 '{bin(conf_i & 0b11)} mask_value: {val}')
+        for i, val in enumerate(conf_mask):
+            qfarr_i = int(qfarr[i])
+            conf_i = int(conf[i])
+            # use bitmasks to convert bin() output for negative vals to two's complement
+            print(f'{i} Orig dvalue: {qfarr_i} Orig bvalue: {bin(qfarr_i & 0b11111111)}  {aerosol_type} bvalue:'
+                  '{bin(conf_i & 0b11)} mask_value: {val}')
 
     return inv_conf_mask  # true elements here are values we want to keep (invert for an actual mask)
 

From 9027e15272ae0466239f21572f5c73948fc49337 Mon Sep 17 00:00:00 2001
From: blake-j-allen <Blake.J.Allen@noaa.gov>
Date: Wed, 11 Sep 2024 04:04:20 +0000
Subject: [PATCH 07/15] A couple more small changes

* Fixed variable name for ADP filenames

* Added output of filenames currently being processed

* Made --adp_mask filename specification accept multiple filenames (and made it produce a list of strings instead of a string)
---
 src/compo/viirs_aod2ioda.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/compo/viirs_aod2ioda.py b/src/compo/viirs_aod2ioda.py
index 2e7b12f4c..c9f4f2195 100755
--- a/src/compo/viirs_aod2ioda.py
+++ b/src/compo/viirs_aod2ioda.py
@@ -196,18 +196,23 @@ def _read(self):
                 # Check if ADP file start/end times match AOD file (creation time can differ slightly)
                 # Using negative indexing makes the paths irrelevant for the comparison
                 if self.adp_mask[n][-53:-20] != f[-53:-20]:
-                    print(f'ADP data file {adp_fn[n]} start and end times do not match those of AOD file {f}')
+                    print(f'ADP data file {self.adp_mask[n]} start and end times do not match those of AOD file {f}')
                     print(f'Continuing to next AOD and ADP file (or ending)')
                     continue
 
+                print('Processing the following AOD and ADP files:')
+                print(f)
+                print(self.adp_mask[n])
+                print()
+
                 # open ADP file corresponding to AOD file.
                 try:
-                    ncd_adp = nc.Dataset(adp_fn[n], 'r')
+                    ncd_adp = nc.Dataset(self.adp_mask[n], 'r')
                 except FileNotFoundError:
                     # this will skip creating output for both the AOD and ADP at the selected time;
                     # however, that should be fine, as when this code runs we are interested only in
                     # the ADP-filtered output rather than the full AOD output.
-                    print(f'ADP data file {adp_fn} not found, continuing to next AOD file on list (or ending).')
+                    print(f'ADP data file {self.adp_mask[n]} not found, continuing to next AOD file on list (or ending).')
                     continue
 
                 # Get ADP smoke/dust masks
@@ -414,7 +419,7 @@ def main():
         '--adp_mask',
         help="""activate ADP-based separation of smoke and dust affected obs from full AOD dataset and specify the path of ADP
               file(s). ADP files should each correspond to an AOD file specified with -i or --input""",
-        type=str, required=False)
+        type=str, nargs='+', required=False)
     parser.add_argument(
         '--adp_qc_lvl',
         help="set ADP QC confidence level for smoke and dust obs, use 'low','med', 'medhigh',or 'high': default='medhigh'",

From 0bb944a325f13c272f5a03b1ae5d788816c0f873 Mon Sep 17 00:00:00 2001
From: blake-j-allen <Blake.J.Allen@noaa.gov>
Date: Wed, 11 Sep 2024 06:10:43 +0000
Subject: [PATCH 08/15] Argument parser clarifications and documentation

* Argument parser help messages made clearer

* Also for clarity, argument --adp_qc_lvl, variable adp_qc_lvl, and attribute AOD.adp_qc_lvl renamed to --adp_conf_lvl, adp_conf_lvl, and AOD.adp_conf_lvl, respectively

* README updated with usage information for the smoke and dust processing
---
 README.md                   |  6 ++++++
 src/compo/viirs_aod2ioda.py | 34 ++++++++++++++++++----------------
 2 files changed, 24 insertions(+), 16 deletions(-)

diff --git a/README.md b/README.md
index 22fe14176..58470f899 100644
--- a/README.md
+++ b/README.md
@@ -108,6 +108,12 @@ Usage: <converter.py> -i INPUT_FILE(S) -o OUTPUT_FILE -m nesdis -k maskout -t 0.
 ```
 For method option (-m) of bias and uncertainty calculation (default/nesdis), deafult means to set bias and uncertainty as 0.0 and nesdis means to use NESDIS bias and uncertainty calculation method. For maskout option (-k) default/maskout, default means to keep all missing values and maskout means to not write out missing values. For thinning option, the value should be within 0.0 and 1.0 depending how much data will be thinned, and 0.0 means without any thining.
 
+The AOD converter normally produces a single IODA output file, but it can also produce two additional output files, one containing only smoke-affected obs and one containing only dust-affected obs. This functionality uses flagging in the Aerosol Data Product (ADP) files created as companion datasets for the native AOD550 observation files. To activate the smoke and dust processing, use the --adp_mask flag followed by the ADP file(s) that correspond to the AOD file(s) specified with -i.
+
+```
+Usage (with smoke and dust processing): <converter.py> -i AOD_INPUT_FILE(S) --adp_mask ADP_INPUT_FILE(S) --adp_conf_lvl 'medhigh' -o OUTPUT_FILE -m nesdis -k maskout -t 0.0
+```
+The ADP files contain a confidence level for the smoke and dust flagging. The confidence level of the smoke- and dust-affected obs retained in the smoke and dust IODA output files can be specified using --adp_conf_lvl [level], where level can be "low", "med", "medhigh", or "high". The default is "medhigh".
 
 ## land
 
diff --git a/src/compo/viirs_aod2ioda.py b/src/compo/viirs_aod2ioda.py
index c9f4f2195..8f8134ca6 100755
--- a/src/compo/viirs_aod2ioda.py
+++ b/src/compo/viirs_aod2ioda.py
@@ -95,15 +95,15 @@ def flag_by_confidence_lvl(qfarr, aerosol_type, conf_lvl='MedHigh', print_diag=F
 
 
 class AOD(object):
-    def __init__(self, filenames, method, mask, thin, adp_mask, adp_qc_lvl):
+    def __init__(self, filenames, method, mask, thin, adp_mask, adp_conf_lvl):
         self.filenames = filenames
         self.mask = mask
         self.method = method
         self.thin = thin
         self.adp_mask = adp_mask
-        self.adp_qc_lvl = adp_qc_lvl
-        if self.adp_qc_lvl is None:
-            self.adp_qc_lvl = 'MedHigh'  # default to including Medium and High Confidence obs based on ADP data
+        self.adp_conf_lvl = adp_conf_lvl
+        if self.adp_conf_lvl is None:
+            self.adp_conf_lvl = 'MedHigh'  # default to including Medium and High Confidence obs based on ADP data
         self.varDict = defaultdict(lambda: defaultdict(dict))
         self.outdata = defaultdict(lambda: DefaultOrderedDict(OrderedDict))
         self.varAttrs = DefaultOrderedDict(lambda: DefaultOrderedDict(dict))
@@ -227,8 +227,8 @@ def _read(self):
                 # As above, in the confidence masks, med-high confidence is marked by True mask elements,
                 # due to the line conf_mask = np.logical_or(conf == 0, conf == 2) in the get functions,
                 # so invert the masks the same as above
-                conf_mask_smoke = np.logical_not(flag_by_confidence_lvl(qcflag, 'Smoke', self.adp_qc_lvl))
-                conf_mask_dust = np.logical_not(flag_by_confidence_lvl(qcflag, 'Dust', self.adp_qc_lvl))
+                conf_mask_smoke = np.logical_not(flag_by_confidence_lvl(qcflag, 'Smoke', self.adp_conf_lvl))
+                conf_mask_dust = np.logical_not(flag_by_confidence_lvl(qcflag, 'Dust', self.adp_conf_lvl))
 
                 ncd_adp.close()
 
@@ -390,21 +390,23 @@ def main():
 
     # get command line arguments
     parser = argparse.ArgumentParser(
-        description=('Read VIIRS aerosol optical depth file(s) and Converter'
-                     ' of native NetCDF format for observations of optical'
+        description=('Read VIIRS aerosol optical depth file(s) and convert'
+                     ' native NetCDF format for observations of optical'
                      ' depth from VIIRS AOD550 to IODA-V2 netCDF format.')
     )
     parser.add_argument(
         '-i', '--input',
-        help="path of viirs aod input file(s)",
+        help="path of viirs AOD input file(s)",
         type=str, nargs='+', required=True)
     parser.add_argument(
         '-o', '--output',
-        help="name of base ioda-v2 output file; smoke and dust files will add '_smoke' or '_dust' suffix",
+        help="""name of base IODA-V2 output file; if ADP smoke and dust processing is activated, smoke and dust filenames
+              will have '_smoke' or '_dust' added either before the filetype suffix or, if there is no suffix, at the end
+              of the filename""",
         type=str, required=True)
     parser.add_argument(
         '-m', '--method',
-        help="calculation error method: nesdis/default, default=none",
+        help="obs error calculation method: nesdis/default, default=none",
         type=str, required=True)
     parser.add_argument(
         '-k', '--mask',
@@ -417,19 +419,19 @@ def main():
         type=float, default=0.0)
     parser.add_argument(
         '--adp_mask',
-        help="""activate ADP-based separation of smoke and dust affected obs from full AOD dataset and specify the path of ADP
-              file(s). ADP files should each correspond to an AOD file specified with -i or --input""",
+        help="""activate ADP-based separation of smoke and dust affected obs from full AOD dataset and specify the path of
+              ADP input file(s). ADP files should each correspond to an AOD file specified with -i or --input""",
         type=str, nargs='+', required=False)
     parser.add_argument(
-        '--adp_qc_lvl',
-        help="set ADP QC confidence level for smoke and dust obs, use 'low','med', 'medhigh',or 'high': default='medhigh'",
+        '--adp_conf_lvl',
+        help="set ADP confidence level(s) of smoke and dust obs to keep; use 'low', 'med', 'medhigh', or 'high': default='medhigh'",
         type=str, required=False)
 
     args = parser.parse_args()
 
     # setup the IODA writer
     # Read in the AOD data and perform requested processing
-    aod = AOD(args.input, args.method, args.mask, args.thin, args.adp_mask, args.adp_qc_lvl)
+    aod = AOD(args.input, args.method, args.mask, args.thin, args.adp_mask, args.adp_conf_lvl)
 
     # Write everything out. Use try-except because if all obs are filtered out of
     # one of the datasets due to either masking or thinning or a combination, an IndexError will be thrown

From 5161a8b6beb8dbedf2f8edc301435c7e763df8c1 Mon Sep 17 00:00:00 2001
From: blake-j-allen <Blake.J.Allen@noaa.gov>
Date: Thu, 12 Sep 2024 17:53:08 +0000
Subject: [PATCH 09/15] removed unused glob import

---
 src/compo/viirs_aod2ioda.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/compo/viirs_aod2ioda.py b/src/compo/viirs_aod2ioda.py
index 8f8134ca6..f1213dd21 100755
--- a/src/compo/viirs_aod2ioda.py
+++ b/src/compo/viirs_aod2ioda.py
@@ -13,7 +13,6 @@
 import numpy as np
 import os
 
-import glob
 import pyiodaconv.ioda_conv_engines as iconv
 from collections import defaultdict, OrderedDict
 from pyiodaconv.orddicts import DefaultOrderedDict

From fa437f782278aece1b2f73a352050f95b017f4c9 Mon Sep 17 00:00:00 2001
From: BlakeJAllen-NOAA <150301684+BlakeJAllen-NOAA@users.noreply.github.com>
Date: Thu, 12 Sep 2024 12:04:00 -0600
Subject: [PATCH 10/15] Update src/compo/viirs_aod2ioda.py

simplify code for changing case of conf_lvl in function flag_by_confidence_lvl

Co-authored-by: Cory Martin <cory.r.martin@noaa.gov>
---
 src/compo/viirs_aod2ioda.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/compo/viirs_aod2ioda.py b/src/compo/viirs_aod2ioda.py
index f1213dd21..72ad72b78 100755
--- a/src/compo/viirs_aod2ioda.py
+++ b/src/compo/viirs_aod2ioda.py
@@ -27,10 +27,8 @@ def flag_by_confidence_lvl(qfarr, aerosol_type, conf_lvl='MedHigh', print_diag=F
 
     if conf_lvl == 'med' or conf_lvl == 'medhigh':
         conf_lvl = 'MedHigh'
-    if conf_lvl == 'low':
-        conf_lvl = 'Low'
-    if conf_lvl == 'high':
-        conf_lvl = 'High'
+    else:
+        conf_lvl = conf_lvl.title()
 
     if aerosol_type == 'Smoke':
         rhtrim = np.floor(qfarr/4)  # trim off the 2 least significant bits

From cd9e3630b999eef4dc69660c1052d35e79f550f4 Mon Sep 17 00:00:00 2001
From: BenjaminRuston <benr@ucar.edu>
Date: Wed, 23 Oct 2024 14:31:26 -0700
Subject: [PATCH 11/15] add a stub to show how a ctest should be added

---
 test/CMakeLists.txt | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 595b4fa46..c294b16be 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -95,6 +95,7 @@ list( APPEND test_input
   testinput/viirs_j1_l1b_albedo_geoloc_202108050600.nc
   testinput/viirs_j1_l1b_albedo_obsval_202108050600.nc
   testinput/madis_2021010100.nc
+  testinput/viirs_aod_smoke_dust_inputfile.nc
 )
 
 list( APPEND test_output
@@ -183,6 +184,7 @@ list( APPEND test_output
   testoutput/viirs_j1_l1b_albedo_2021080506.nc
   testoutput/imsfv3grid_scf.nc
   testoutput/madis_snod_2021010100.nc
+  testoutput/obs.20200912T200000Z_PT1H_aod_smoke_dust.nc
 )
 
 if( iodaconv_gnssro_ENABLED )
@@ -1231,6 +1233,20 @@ ecbuild_add_test( TARGET  test_${PROJECT_NAME}_viirs_aod
                           -n 0.0"
                           viirs_aod.nc ${IODA_CONV_COMP_TOL})
 
+ecbuild_add_test( TARGET  test_${PROJECT_NAME}_viirs_aod
+                  TYPE    SCRIPT
+                  ENVIRONMENT "PYTHONPATH=${IODACONV_PYTHONPATH}"
+                  COMMAND bash
+                  ARGS    ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh
+                          netcdf
+                          "${Python3_EXECUTABLE} ${CMAKE_BINARY_DIR}/bin/viirs_aod2ioda.py
+                          -i testinput/viirs_aod_smoke_dust_inputfile.nc
+                          -o testrun/obs.20200912T200000Z_PT1H_aod_smoke_dust.nc
+                          -m nesdis
+                          -k maskout
+                          -n 0.0"
+                          obs.20200912T200000Z_PT1H_aod_smoke_dust.nc ${IODA_CONV_COMP_TOL})
+
 ecbuild_add_test( TARGET  test_${PROJECT_NAME}_satbias_viirs_aod
                   TYPE    SCRIPT
                   COMMAND bash

From 7da71243a3edfbd9a8101d0ebc9f7d9e42d77560 Mon Sep 17 00:00:00 2001
From: Benjamin Ruston <93737224+BenjaminRuston@users.noreply.github.com>
Date: Wed, 23 Oct 2024 16:13:53 -0700
Subject: [PATCH 12/15] Update test/CMakeLists.txt

---
 test/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index c294b16be..1216975db 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -1233,7 +1233,7 @@ ecbuild_add_test( TARGET  test_${PROJECT_NAME}_viirs_aod
                           -n 0.0"
                           viirs_aod.nc ${IODA_CONV_COMP_TOL})
 
-ecbuild_add_test( TARGET  test_${PROJECT_NAME}_viirs_aod
+ecbuild_add_test( TARGET  test_${PROJECT_NAME}_viirs_smoke_dust
                   TYPE    SCRIPT
                   ENVIRONMENT "PYTHONPATH=${IODACONV_PYTHONPATH}"
                   COMMAND bash

From 99146f5b61678419f4c24ce64e57734b34f58f23 Mon Sep 17 00:00:00 2001
From: BenjaminRuston <benr@ucar.edu>
Date: Thu, 24 Oct 2024 18:18:40 -0700
Subject: [PATCH 13/15] initial guess at some sampled files for dust and smoke
 from AOD ctest

---
 ...-ADP_v3r2_j01_s20240110T202044Z_e20240110T202044Z.sample.nc | 3 +++
 ...-AOD_v3r2_j01_s20240110T202044Z_e20240110T202044Z.sample.nc | 3 +++
 2 files changed, 6 insertions(+)
 create mode 100644 test/testinput/JRR-ADP_v3r2_j01_s20240110T202044Z_e20240110T202044Z.sample.nc
 create mode 100644 test/testinput/JRR-AOD_v3r2_j01_s20240110T202044Z_e20240110T202044Z.sample.nc

diff --git a/test/testinput/JRR-ADP_v3r2_j01_s20240110T202044Z_e20240110T202044Z.sample.nc b/test/testinput/JRR-ADP_v3r2_j01_s20240110T202044Z_e20240110T202044Z.sample.nc
new file mode 100644
index 000000000..2e69d6f54
--- /dev/null
+++ b/test/testinput/JRR-ADP_v3r2_j01_s20240110T202044Z_e20240110T202044Z.sample.nc
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c31a7f2cc00cc02bfc3c4d6259846358a83b25753da94fc20f354555620e0d04
+size 118032
diff --git a/test/testinput/JRR-AOD_v3r2_j01_s20240110T202044Z_e20240110T202044Z.sample.nc b/test/testinput/JRR-AOD_v3r2_j01_s20240110T202044Z_e20240110T202044Z.sample.nc
new file mode 100644
index 000000000..fa1fd3c7d
--- /dev/null
+++ b/test/testinput/JRR-AOD_v3r2_j01_s20240110T202044Z_e20240110T202044Z.sample.nc
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70a669416b2a3a38427dae68e428c54c060e0a899318e147c5a56e13c92c18e8
+size 574094

From 537ef230fb1a3ef6d75008887e54b182687cf3b4 Mon Sep 17 00:00:00 2001
From: BenjaminRuston <benr@ucar.edu>
Date: Thu, 24 Oct 2024 18:29:16 -0700
Subject: [PATCH 14/15] add sampled files and what may be a working ctest

---
 test/CMakeLists.txt                                 | 13 ++++++++-----
 .../obs.20240110T202044Z_PT1M_aod_smoke.nc          |  3 +++
 2 files changed, 11 insertions(+), 5 deletions(-)
 create mode 100644 test/testoutput/obs.20240110T202044Z_PT1M_aod_smoke.nc

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index c294b16be..354015873 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -95,7 +95,8 @@ list( APPEND test_input
   testinput/viirs_j1_l1b_albedo_geoloc_202108050600.nc
   testinput/viirs_j1_l1b_albedo_obsval_202108050600.nc
   testinput/madis_2021010100.nc
-  testinput/viirs_aod_smoke_dust_inputfile.nc
+  testinput/JRR-AOD_v3r2_j01_s20240110T202044Z_e20240110T202044Z.sample.nc
+  testinput/JRR-ADP_v3r2_j01_s20240110T202044Z_e20240110T202044Z.sample.nc
 )
 
 list( APPEND test_output
@@ -184,7 +185,7 @@ list( APPEND test_output
   testoutput/viirs_j1_l1b_albedo_2021080506.nc
   testoutput/imsfv3grid_scf.nc
   testoutput/madis_snod_2021010100.nc
-  testoutput/obs.20200912T200000Z_PT1H_aod_smoke_dust.nc
+  testoutput/obs.20240110T202044Z_PT1M_aod_smoke.nc
 )
 
 if( iodaconv_gnssro_ENABLED )
@@ -1240,12 +1241,14 @@ ecbuild_add_test( TARGET  test_${PROJECT_NAME}_viirs_aod
                   ARGS    ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh
                           netcdf
                           "${Python3_EXECUTABLE} ${CMAKE_BINARY_DIR}/bin/viirs_aod2ioda.py
-                          -i testinput/viirs_aod_smoke_dust_inputfile.nc
-                          -o testrun/obs.20200912T200000Z_PT1H_aod_smoke_dust.nc
+                          -i testinput/JRR-AOD_v3r2_j01_s20240110T202044Z_e20240110T202044Z.sample.nc
+                          --adp_mask testinput/JRR-ADP_v3r2_j01_s20240110T202044Z_e20240110T202044Z.sample.nc
+                          --adp_conf_lvl 'medhigh'
+                          -o testrun/obs.20240110T202044Z_PT1M_aod.nc
                           -m nesdis
                           -k maskout
                           -n 0.0"
-                          obs.20200912T200000Z_PT1H_aod_smoke_dust.nc ${IODA_CONV_COMP_TOL})
+                          obs.20240110T202044Z_PT1M_aod_smoke.nc ${IODA_CONV_COMP_TOL})
 
 ecbuild_add_test( TARGET  test_${PROJECT_NAME}_satbias_viirs_aod
                   TYPE    SCRIPT
diff --git a/test/testoutput/obs.20240110T202044Z_PT1M_aod_smoke.nc b/test/testoutput/obs.20240110T202044Z_PT1M_aod_smoke.nc
new file mode 100644
index 000000000..612ea8dbb
--- /dev/null
+++ b/test/testoutput/obs.20240110T202044Z_PT1M_aod_smoke.nc
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:724ecc04f5dd1c327a33b59795c86e0c901e3313c5d62553603e55436639e8dd
+size 14268

From 084e554549680f5db08ba88796981c41d2ed2649 Mon Sep 17 00:00:00 2001
From: BlakeJAllen-NOAA <150301684+BlakeJAllen-NOAA@users.noreply.github.com>
Date: Fri, 25 Oct 2024 13:58:10 -0600
Subject: [PATCH 15/15] Update README.md

fixed typo

Co-authored-by: Benjamin Ruston <93737224+BenjaminRuston@users.noreply.github.com>
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 58470f899..13bd9256e 100644
--- a/README.md
+++ b/README.md
@@ -111,7 +111,7 @@ For method option (-m) of bias and uncertainty calculation (default/nesdis), dea
 The AOD converter normally produces a single IODA output file, but it can also produce two additional output files, one containing only smoke-affected obs and one containing only dust-affected obs. This functionality uses flagging in the Aerosol Data Product (ADP) files created as companion datasets for the native AOD550 observation files. To activate the smoke and dust processing, use the --adp_mask flag followed by the ADP file(s) that correspond to the AOD file(s) specified with -i.
 
 ```
-Usage (with smoke and dust processing): <converter.py> -i AOD_INPUT_FILE(S) --adp_mask ADP_INPUT_FILE(S) --adp_conf_lvl 'medhigh' -o OUTPUT_FILE -m nesdis -k maskout -t 0.0
+Usage (with smoke and dust processing): <converter.py> -i AOD_INPUT_FILE(S) --adp_mask ADP_INPUT_FILE(S) --adp_conf_lvl 'medhigh' -o OUTPUT_FILE -m nesdis -k maskout -n 0.0
 ```
 The ADP files contain a confidence level for the smoke and dust flagging. The confidence level of the smoke- and dust-affected obs retained in the smoke and dust IODA output files can be specified using --adp_conf_lvl [level], where level can be "low", "med", "medhigh", or "high". The default is "medhigh".