diff --git a/config.defaults.yaml b/config.defaults.yaml index 909921f..94eca38 100644 --- a/config.defaults.yaml +++ b/config.defaults.yaml @@ -2243,3 +2243,7 @@ inference: # - 852 # - 853 path_to_preds: + +# Some fields may already be completed and should be excluded from subsequent feature generation/inference +fields_to_exclude: + # - 296 diff --git a/tools/combine_preds.py b/tools/combine_preds.py index 2d65513..9ab0479 100755 --- a/tools/combine_preds.py +++ b/tools/combine_preds.py @@ -16,6 +16,11 @@ DEFAULT_PREDS_PATH = BASE_DIR config_fields = config["inference"].get("fields_to_run") +fields_to_exclude = config.get("fields_to_exclude") + +if (config_fields is not None) and (fields_to_exclude is not None): + # Drop fields_to_exclude from config_fields + config_fields = list(set(config_fields).difference(fields_to_exclude)) def combine_preds( @@ -112,7 +117,9 @@ def combine_preds( preds_to_save = None counter = 0 - print(f"Processing {len(fields_dnn_dict) - len(done_fields)} fields/files...") + print( + f"Processing {len(set(fields_dnn_dict).difference(done_fields))} fields/files..." + ) for field in fields_dnn_dict.keys(): if field not in done_fields: diff --git a/tools/generate_features_job_submission.py b/tools/generate_features_job_submission.py index 21353b4..5af39c2 100755 --- a/tools/generate_features_job_submission.py +++ b/tools/generate_features_job_submission.py @@ -12,7 +12,13 @@ BASE_DIR = pathlib.Path.cwd() config = parse_load_config() -fields_to_run = config['feature_generation']['fields_to_run'] +fields_to_run = config['feature_generation'].get('fields_to_run') +fields_to_exclude = config.get('fields_to_exclude') + +if (fields_to_run is not None) and (fields_to_exclude is not None): + # Drop fields_to_exclude from fields_to_run + fields_to_run = list(set(fields_to_run).difference(fields_to_exclude)) + path_to_features = config['feature_generation']['path_to_features'] if path_to_features is not None: BASE_DIR = pathlib.Path(path_to_features) diff --git a/tools/run_inference_job_submission.py b/tools/run_inference_job_submission.py index 4ed67c4..44a8721 100755 --- a/tools/run_inference_job_submission.py +++ b/tools/run_inference_job_submission.py @@ -81,7 +81,13 @@ def main(): slurmDir = str(BASE_DIR / dirname) - fields = config['inference']['fields_to_run'] + fields = config['inference'].get('fields_to_run') + fields_to_exclude = config.get('fields_to_exclude') + + if (fields is not None) and (fields_to_exclude is not None): + # Drop fields_to_exclude from fields + fields = list(set(fields).difference(fields_to_exclude)) + algorithm = args.algorithm subDir = os.path.join(slurmDir, filetype)