Allow user to specify fields to exclude from feat gen/inference (#566)

* Allow user to specify fields to exclude from feat gen/inference

* Fix typo in config

config.defaults.yaml

@@ -2243,3 +2243,7 @@ inference:
     # - 852
     # - 853
   path_to_preds:
+
+# Some fields may already be completed and should be excluded from subsequent feature generation/inference
+fields_to_exclude:
+  # - 296

tools/combine_preds.py

@@ -16,6 +16,11 @@
     DEFAULT_PREDS_PATH = BASE_DIR
 
 config_fields = config["inference"].get("fields_to_run")
+fields_to_exclude = config.get("fields_to_exclude")
+
+if (config_fields is not None) and (fields_to_exclude is not None):
+    # Drop fields_to_exclude from config_fields
+    config_fields = list(set(config_fields).difference(fields_to_exclude))
 
 
 def combine_preds(
@@ -112,7 +117,9 @@ def combine_preds(
 
     preds_to_save = None
     counter = 0
-    print(f"Processing {len(fields_dnn_dict) - len(done_fields)} fields/files...")
+    print(
+        f"Processing {len(set(fields_dnn_dict).difference(done_fields))} fields/files..."
+    )
 
     for field in fields_dnn_dict.keys():
         if field not in done_fields:

tools/generate_features_job_submission.py

@@ -12,7 +12,13 @@
 BASE_DIR = pathlib.Path.cwd()
 config = parse_load_config()
 
-fields_to_run = config['feature_generation']['fields_to_run']
+fields_to_run = config['feature_generation'].get('fields_to_run')
+fields_to_exclude = config.get('fields_to_exclude')
+
+if (fields_to_run is not None) and (fields_to_exclude is not None):
+    # Drop fields_to_exclude from fields_to_run
+    fields_to_run = list(set(fields_to_run).difference(fields_to_exclude))
+
 path_to_features = config['feature_generation']['path_to_features']
 if path_to_features is not None:
     BASE_DIR = pathlib.Path(path_to_features)

tools/run_inference_job_submission.py

@@ -81,7 +81,13 @@ def main():
 
     slurmDir = str(BASE_DIR / dirname)
 
-    fields = config['inference']['fields_to_run']
+    fields = config['inference'].get('fields_to_run')
+    fields_to_exclude = config.get('fields_to_exclude')
+
+    if (fields is not None) and (fields_to_exclude is not None):
+        # Drop fields_to_exclude from fields
+        fields = list(set(fields).difference(fields_to_exclude))
+
     algorithm = args.algorithm
 
     subDir = os.path.join(slurmDir, filetype)