From 051592f5478627b855f2f4eafde58aaefe6c36ec Mon Sep 17 00:00:00 2001 From: aradhakrishnanGFDL Date: Tue, 1 Oct 2024 15:45:45 -0400 Subject: [PATCH 1/4] print to logger debug --- catalogbuilder/intakebuilder/gfdlcrawler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/catalogbuilder/intakebuilder/gfdlcrawler.py b/catalogbuilder/intakebuilder/gfdlcrawler.py index 00455f2..4399cf3 100644 --- a/catalogbuilder/intakebuilder/gfdlcrawler.py +++ b/catalogbuilder/intakebuilder/gfdlcrawler.py @@ -80,9 +80,9 @@ def crawlLocal(projectdir, dictFilter,dictFilterIgnore,logger,configyaml,slow): #if our filename expectations are not met compared to the output_file_path_template in config, skip the loop. TODO revisit for statics if ("static" not in filename): if ((len(filename.split('.'))-1) != len(set_ftemplate)): - print("Skipping ",filename) + logger.debug("Skipping "+filename) continue - logger.info(dirpath+"/"+filename) + logger.debug(dirpath+"/"+filename) dictInfo = {} dictInfo = getinfo.getProject(projectdir, dictInfo) # get info from filename From aeafc17e02daf6d8cf9fe6f660a25eeb4c8a9032 Mon Sep 17 00:00:00 2001 From: aradhakrishnanGFDL Date: Thu, 3 Oct 2024 17:52:25 -0400 Subject: [PATCH 2/4] verbose option added, many of the print changed to logger.debug and info --- catalogbuilder/intakebuilder/CSVwriter.py | 2 +- catalogbuilder/intakebuilder/configparser.py | 15 +++++++-------- catalogbuilder/intakebuilder/gfdlcrawler.py | 4 ++-- catalogbuilder/scripts/gen_intake_gfdl.py | 19 ++++++++++++++----- 4 files changed, 24 insertions(+), 16 deletions(-) diff --git a/catalogbuilder/intakebuilder/CSVwriter.py b/catalogbuilder/intakebuilder/CSVwriter.py index f231bfd..900dc68 100644 --- a/catalogbuilder/intakebuilder/CSVwriter.py +++ b/catalogbuilder/intakebuilder/CSVwriter.py @@ -47,7 +47,7 @@ def listdict_to_csv(dict_info,headerlist, csvfile, overwrite, append,slow): if overwrite: with open(csvfile, 'w') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=headerlist) - print("writing..") + logger.debug("writing..") writer.writeheader() for data in dict_info: if len(data.keys()) > 2: diff --git a/catalogbuilder/intakebuilder/configparser.py b/catalogbuilder/intakebuilder/configparser.py index a7db0f7..0789cf3 100644 --- a/catalogbuilder/intakebuilder/configparser.py +++ b/catalogbuilder/intakebuilder/configparser.py @@ -1,7 +1,7 @@ import yaml import os class Config: - def __init__(self, config): + def __init__(self, config,logger): self.config = config with open(self.config, 'r') as file: configfile = yaml.safe_load(file) @@ -9,33 +9,32 @@ def __init__(self, config): self.input_path = configfile['input_path'] except: self.input_path = None - print("input_path does not exist in config") + logger.debug("input_path does not exist in config") pass try: self.output_path = configfile['output_path'] - #print("output_path :",self.output_path) except: self.output_path = None - print("output_path does not exist in config") + logger.debug("output_path does not exist in config") pass try: self.headerlist = configfile['headerlist'] - print("headerlist :",self.headerlist) + logger.debug("headerlist :"+(str)(self.headerlist)) except: raise KeyError("headerlist does not exist in config") try: self.output_path_template = configfile['output_path_template'] - print("output_path_template :",self.output_path_template) + logger.debug("output_path_template :"+(str)(self.output_path_template)) except: raise KeyError("output_path_template does not exist in config") try: self.output_file_template = configfile['output_file_template'] - print("output_file_template :", self.output_file_template) + logger.debug("output_file_template :"+ (str)(self.output_file_template)) except: raise KeyError("output_file_template does not exist in config") try: self.schema = configfile['schema'] - print("schema:", self.schema) + logger.info("schema:"+ self.schema) except: self.schema = None pass diff --git a/catalogbuilder/intakebuilder/gfdlcrawler.py b/catalogbuilder/intakebuilder/gfdlcrawler.py index 4399cf3..ed6f4ba 100644 --- a/catalogbuilder/intakebuilder/gfdlcrawler.py +++ b/catalogbuilder/intakebuilder/gfdlcrawler.py @@ -57,7 +57,7 @@ def crawlLocal(projectdir, dictFilter,dictFilterIgnore,logger,configyaml,slow): if ( len(set_ftemplate) > 0 ): missingcols = [col for col in diffcols if col not in set_ftemplate] missingcols.remove("path") #because we get this anyway - print("Missing cols from metadata sources:", missingcols) + logger.debug("Missing cols from metadata sources:", missingcols) #TODO INCLUDE filter in traversing through directories at the top @@ -75,7 +75,7 @@ def crawlLocal(projectdir, dictFilter,dictFilterIgnore,logger,configyaml,slow): # logger.debug("Skipping hidden file", filepath) # continue if not filename.endswith(".nc"): - logger.debug("FILE does not end with .nc. Skipping", filepath) + logger.debug("FILE does not end with .nc. Skipping "+ filepath) continue #if our filename expectations are not met compared to the output_file_path_template in config, skip the loop. TODO revisit for statics if ("static" not in filename): diff --git a/catalogbuilder/scripts/gen_intake_gfdl.py b/catalogbuilder/scripts/gen_intake_gfdl.py index 5f4f239..aa57630 100755 --- a/catalogbuilder/scripts/gen_intake_gfdl.py +++ b/catalogbuilder/scripts/gen_intake_gfdl.py @@ -9,6 +9,7 @@ logger = logging.getLogger('local') logger.setLevel(logging.INFO) +logging.basicConfig(stream=sys.stdout) try: from catalogbuilder.intakebuilder import gfdlcrawler, CSVwriter, builderconfig, configparser, getinfo @@ -30,11 +31,17 @@ #template_path = os.path.join(package_dir, '../cats/gfdl_template.json') def create_catalog(input_path=None, output_path=None, config=None, filter_realm=None, filter_freq=None, filter_chunk=None, - overwrite=False, append=False, slow = False): + overwrite=False, append=False, slow = False, verbose=False): + if verbose: + logger.setLevel(logging.DEBUG) + logger.info("Verbose log activated.") + else: + logger.setLevel(logging.INFO) + logger.info("[Mostly] silent log activated") configyaml = None # TODO error catching if (config is not None): - configyaml = configparser.Config(config) + configyaml = configparser.Config(config,logger) if(input_path is None): input_path = configyaml.input_path if(output_path is None): @@ -42,7 +49,7 @@ def create_catalog(input_path=None, output_path=None, config=None, filter_realm= if((input_path is None) or (output_path is None)): sys.exit("Missing: input_path or output_path. Pass it in the config yaml or as command-line option") if config is None or not configyaml.schema: - print("We will use catalog builder catalogbuilder/cats/gfdl_template.json as your json schema") + logger.info("Default schema: catalogbuilder/cats/gfdl_template.json") template_path = os.path.join(package_dir, '../cats/gfdl_template.json') else: template_path = configyaml.schema @@ -51,7 +58,8 @@ def create_catalog(input_path=None, output_path=None, config=None, filter_realm= sys.exit("Input path does not exist. Adjust configuration.") if not os.path.exists(Path(output_path).parent.absolute()): sys.exit("Output path parent directory does not exist. Adjust configuration.") - print("input path: ",input_path, " output path: ", output_path) + logger.info("input path: "+ input_path) + logger.info( " output path: "+ output_path) project_dir = input_path csv_path = "{0}.csv".format(output_path) json_path = "{0}.json".format(output_path) @@ -78,7 +86,7 @@ def create_catalog(input_path=None, output_path=None, config=None, filter_realm= ''' dictInfo = {} project_dir = project_dir.rstrip("/") - logger.info("Calling gfdlcrawler.crawlLocal") + logger.debug("Calling gfdlcrawler.crawlLocal") list_files = gfdlcrawler.crawlLocal(project_dir, dictFilter, dictFilterIgnore, logger, configyaml,slow) #Grabbing data from template JSON, changing CSV path to match output path, and dumping data in new JSON with open(template_path, "r") as jsonTemplate: @@ -139,6 +147,7 @@ def create_catalog(input_path=None, output_path=None, config=None, filter_realm= @click.option('--overwrite', is_flag=True, default=False) @click.option('--append', is_flag=True, default=False) @click.option('--slow','-s', is_flag=True, default=False) +@click.option('--verbose/--silent', default=False, is_flag=True) #default has silent option. Use --verbose for detailed logging def create_catalog_cli(**kwargs): return create_catalog(**kwargs) From a0f97c6fdbe33b462127357a4b614d12bc477afd Mon Sep 17 00:00:00 2001 From: aradhakrishnanGFDL Date: Thu, 3 Oct 2024 17:52:55 -0400 Subject: [PATCH 3/4] logger message --- catalogbuilder/intakebuilder/CSVwriter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/catalogbuilder/intakebuilder/CSVwriter.py b/catalogbuilder/intakebuilder/CSVwriter.py index 900dc68..aea159a 100644 --- a/catalogbuilder/intakebuilder/CSVwriter.py +++ b/catalogbuilder/intakebuilder/CSVwriter.py @@ -56,7 +56,7 @@ def listdict_to_csv(dict_info,headerlist, csvfile, overwrite, append,slow): if append: with open(csvfile, 'a') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=headerlist) - print("writing (without header)..") + logger.info("writing (without header)..") for data in dict_info: if len(data.keys()) > 2: writer.writerow(data) @@ -70,7 +70,7 @@ def listdict_to_csv(dict_info,headerlist, csvfile, overwrite, append,slow): if user_input.lower() == 'y': with open(csvfile, 'w') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=headerlist) - print("writing..") + logger.debug("writing..") writer.writeheader() for data in dict_info: if len(data.keys()) > 2: @@ -91,7 +91,7 @@ def listdict_to_csv(dict_info,headerlist, csvfile, overwrite, append,slow): else: with open(csvfile, 'w') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=headerlist) - print("writing..") + logger.debug("writing..") writer.writeheader() for data in dict_info: if len(data.keys()) > 2: From b15e72897cc01b25d5ec47e496d2c95909ec562f Mon Sep 17 00:00:00 2001 From: aradhakrishnanGFDL Date: Thu, 3 Oct 2024 17:57:52 -0400 Subject: [PATCH 4/4] rm logger msg here --- catalogbuilder/intakebuilder/CSVwriter.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/catalogbuilder/intakebuilder/CSVwriter.py b/catalogbuilder/intakebuilder/CSVwriter.py index aea159a..537f338 100644 --- a/catalogbuilder/intakebuilder/CSVwriter.py +++ b/catalogbuilder/intakebuilder/CSVwriter.py @@ -47,7 +47,6 @@ def listdict_to_csv(dict_info,headerlist, csvfile, overwrite, append,slow): if overwrite: with open(csvfile, 'w') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=headerlist) - logger.debug("writing..") writer.writeheader() for data in dict_info: if len(data.keys()) > 2: @@ -56,7 +55,6 @@ def listdict_to_csv(dict_info,headerlist, csvfile, overwrite, append,slow): if append: with open(csvfile, 'a') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=headerlist) - logger.info("writing (without header)..") for data in dict_info: if len(data.keys()) > 2: writer.writerow(data) @@ -70,7 +68,6 @@ def listdict_to_csv(dict_info,headerlist, csvfile, overwrite, append,slow): if user_input.lower() == 'y': with open(csvfile, 'w') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=headerlist) - logger.debug("writing..") writer.writeheader() for data in dict_info: if len(data.keys()) > 2: @@ -91,7 +88,6 @@ def listdict_to_csv(dict_info,headerlist, csvfile, overwrite, append,slow): else: with open(csvfile, 'w') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=headerlist) - logger.debug("writing..") writer.writeheader() for data in dict_info: if len(data.keys()) > 2: