From 5170353be3f64a29329be73010e693fa36f8acfd Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Wed, 25 Jan 2023 21:46:06 -0500 Subject: [PATCH 001/449] Adding new sorting method to data_containers --- jwql/website/apps/jwql/data_containers.py | 31 +++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 3f334273f..4af78eaf4 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -948,6 +948,37 @@ def get_preview_images_by_rootname(rootname): return preview_images +def get_proposals_by_category(instrument): + """Return a dictionary of program numbers based on category type + Parameters + ---------- + instrument : str + Name of the JWST instrument, with first letter capitalized + (e.g. ``Fgs``) + Returns + ------- + category_sorted_dict : dict + Dictionary with category as the key and a list of program id's as the value + """ + + service = "Mast.Jwst.Filtered.{}".format(instrument) + params = {"columns": "program, category", + "filters": []} + response = Mast.service_request_async(service, params) + results = response[0].json()['data'] + + # Get all unique dictionaries + unique_results = list(map(dict, set(tuple(sorted(sub.items())) for sub in results))) + + # Seperate the catagories from programs + categories = set([sub['category'] for sub in unique_results]) + + # build dict where key in catagory and value is a list of programs belonging to the category key. + category_sorted_dict = {} + for category in categories: + category_sorted_dict[category] = [d['program'] for d in unique_results if d['category'] == category] + + return category_sorted_dict def get_proposal_info(filepaths): """Builds and returns a dictionary containing various information From 42c3ab464cd22e701d5f6717cde4085356f15c27 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Mon, 30 Jan 2023 17:09:31 -0500 Subject: [PATCH 002/449] Adding updates to html and views. Need to test population of dropdown on server --- jwql/website/apps/jwql/templates/archive.html | 12 ++++++++++++ jwql/website/apps/jwql/views.py | 5 ++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/templates/archive.html b/jwql/website/apps/jwql/templates/archive.html index c6ff05f5b..7c584a3f3 100644 --- a/jwql/website/apps/jwql/templates/archive.html +++ b/jwql/website/apps/jwql/templates/archive.html @@ -45,6 +45,18 @@

Archived {{ inst }} Images

+ +
+ Sort by Category:
+ + +
+ {% for category in prop_by_cat %} + category + {% endfor %} +
+ +
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 268c3b3ba..2d303acdc 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -70,6 +70,7 @@ from .data_containers import get_explorer_extension_names from .data_containers import get_header_info from .data_containers import get_image_info +from .data_containers import get_proposals_by_category from .data_containers import get_thumbnails_all_instruments from .data_containers import random_404_page from .data_containers import text_scrape @@ -307,8 +308,10 @@ def archived_proposals(request, inst): inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()] template = 'archive.html' + proposals_by_category = get_proposals_by_category(inst) context = {'inst': inst, - 'base_url': get_base_url()} + 'base_url': get_base_url(), + 'props_by_cat': proposals_by_category} return render(request, template, context) From e23047a1efdb2121ff19971e6164b1792a8b7422 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Tue, 31 Jan 2023 14:08:11 -0500 Subject: [PATCH 003/449] Adding header to test context variable --- jwql/website/apps/jwql/templates/archive.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/templates/archive.html b/jwql/website/apps/jwql/templates/archive.html index 7c584a3f3..cc0f9f819 100644 --- a/jwql/website/apps/jwql/templates/archive.html +++ b/jwql/website/apps/jwql/templates/archive.html @@ -17,7 +17,7 @@

Archived {{ inst }} Images

- +

{{ prop_by_cat }}

From 0ba372df45af419065f9f2b25a27f170295aed97 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Tue, 31 Jan 2023 14:11:11 -0500 Subject: [PATCH 004/449] fixing typo --- jwql/website/apps/jwql/templates/archive.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jwql/website/apps/jwql/templates/archive.html b/jwql/website/apps/jwql/templates/archive.html index cc0f9f819..d78bfc816 100644 --- a/jwql/website/apps/jwql/templates/archive.html +++ b/jwql/website/apps/jwql/templates/archive.html @@ -17,7 +17,7 @@

Archived {{ inst }} Images

-

{{ prop_by_cat }}

+

{{ props_by_cat }}

@@ -51,7 +51,7 @@

{{ prop_by_cat }}

- {% for category in prop_by_cat %} + {% for category in props_by_cat %} category {% endfor %}
From 2f0190753cf812640bcf5ec0edf30a33354df6fd Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Tue, 31 Jan 2023 14:39:22 -0500 Subject: [PATCH 005/449] looping through dictionary keys --- jwql/website/apps/jwql/templates/archive.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/templates/archive.html b/jwql/website/apps/jwql/templates/archive.html index d78bfc816..1f061b9e2 100644 --- a/jwql/website/apps/jwql/templates/archive.html +++ b/jwql/website/apps/jwql/templates/archive.html @@ -51,7 +51,7 @@

{{ props_by_cat }}

- {% for category in props_by_cat %} + {% for category in props_by_cat.keys %} category {% endfor %}
From 58fc7f864aadf9ccf4758834d55d86a997379f7e Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Tue, 31 Jan 2023 14:43:51 -0500 Subject: [PATCH 006/449] Fixing typo --- jwql/website/apps/jwql/templates/archive.html | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jwql/website/apps/jwql/templates/archive.html b/jwql/website/apps/jwql/templates/archive.html index 1f061b9e2..9e050e6c6 100644 --- a/jwql/website/apps/jwql/templates/archive.html +++ b/jwql/website/apps/jwql/templates/archive.html @@ -17,7 +17,6 @@

Archived {{ inst }} Images

-

{{ props_by_cat }}

@@ -51,7 +50,7 @@

{{ props_by_cat }}

- {% for category in props_by_cat.keys %} + {% for category in props_by_cat.keys() %} category {% endfor %}
From 3ddd36a53fad293c20cfb3f6945405f909854add Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Tue, 31 Jan 2023 15:30:12 -0500 Subject: [PATCH 007/449] Adding new div --- jwql/website/apps/jwql/templates/archive.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/templates/archive.html b/jwql/website/apps/jwql/templates/archive.html index 9e050e6c6..a728db5b3 100644 --- a/jwql/website/apps/jwql/templates/archive.html +++ b/jwql/website/apps/jwql/templates/archive.html @@ -49,7 +49,7 @@

Archived {{ inst }} Images

Sort by Category:
-
+
{% for category in props_by_cat.keys() %} category {% endfor %} From 714d24e3ee53559bc48e3e324c0da775d97fa899 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Tue, 31 Jan 2023 15:46:38 -0500 Subject: [PATCH 008/449] Adding brackets to see if it changes text value --- jwql/website/apps/jwql/templates/archive.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/templates/archive.html b/jwql/website/apps/jwql/templates/archive.html index a728db5b3..60210686d 100644 --- a/jwql/website/apps/jwql/templates/archive.html +++ b/jwql/website/apps/jwql/templates/archive.html @@ -51,7 +51,7 @@

Archived {{ inst }} Images

{% for category in props_by_cat.keys() %} - category + {{ category }} {% endfor %}
From e6129870b0c4b87e2c1f2346dd49897ddfe3e234 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Thu, 2 Feb 2023 16:13:29 -0500 Subject: [PATCH 009/449] Moving filtering to views.py --- jwql/website/apps/jwql/templates/archive.html | 12 ------------ jwql/website/apps/jwql/views.py | 10 ++++++---- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/jwql/website/apps/jwql/templates/archive.html b/jwql/website/apps/jwql/templates/archive.html index 60210686d..249093912 100644 --- a/jwql/website/apps/jwql/templates/archive.html +++ b/jwql/website/apps/jwql/templates/archive.html @@ -44,18 +44,6 @@

Archived {{ inst }} Images

- -
- Sort by Category:
- - -
- {% for category in props_by_cat.keys() %} - {{ category }} - {% endfor %} -
- -
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 2d303acdc..16173f1b7 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -308,10 +308,8 @@ def archived_proposals(request, inst): inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()] template = 'archive.html' - proposals_by_category = get_proposals_by_category(inst) context = {'inst': inst, - 'base_url': get_base_url(), - 'props_by_cat': proposals_by_category} + 'base_url': get_base_url()} return render(request, template, context) @@ -360,9 +358,13 @@ def archived_proposals_ajax(request, inst): exp_types = [exposure_type for observation in all_entries for exposure_type in observation.exptypes.split(',')] exp_types = sorted(list(set(exp_types))) + # Get all proposals based on category type + proposals_by_category = get_proposals_by_category(inst) + # The naming conventions for dropdown_menus are tightly coupled with the code, this should be changed down the line. dropdown_menus = {'look': THUMBNAIL_FILTER_LOOK, - 'exp_type': exp_types} + 'exp_type': exp_types, + 'cat_type': proposals_by_category.keys()} thumbnails_dict = {} for proposal_num in proposal_nums: From 6bd3c29f8bf31a9928772eb2429342ff04c09b45 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Wed, 8 Feb 2023 11:28:03 -0500 Subject: [PATCH 010/449] Added subprocess-able command to run the pipeline with multiprocessing --- jwql/shared_tasks/run_pipeline.py | 229 ++++++++++++++++++++++++++++++ 1 file changed, 229 insertions(+) create mode 100644 jwql/shared_tasks/run_pipeline.py diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py new file mode 100644 index 000000000..6e7a49102 --- /dev/null +++ b/jwql/shared_tasks/run_pipeline.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python + +import argparse +from astropy.io import fits +from collections import OrderedDict +from copy import deepcopy +from glob import glob +import os +import shutil +import sys + +from jwst import datamodels +from jwst.dq_init import DQInitStep +from jwst.dark_current import DarkCurrentStep +from jwst.firstframe import FirstFrameStep +from jwst.group_scale import GroupScaleStep +from jwst.ipc import IPCStep +from jwst.jump import JumpStep +from jwst.lastframe import LastFrameStep +from jwst.linearity import LinearityStep +from jwst.persistence import PersistenceStep +from jwst.pipeline.calwebb_detector1 import Detector1Pipeline +from jwst.ramp_fitting import RampFitStep +from jwst.refpix import RefPixStep +from jwst.rscd import RscdStep +from jwst.saturation import SaturationStep +from jwst.superbias import SuperBiasStep + +from jwql.instrument_monitors.pipeline_tools import PIPELINE_STEP_MAPPING, get_pipeline_steps +from jwql.utils.logging_functions import configure_logging +from jwql.utils.permissions import set_permissions +from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path + + +def run_pipe(input_file_name, short_name, work_directory, instrument, outputs): + """Run the steps of ``calwebb_detector1`` on the input file, saving the result of each + step as a separate output file, then return the name-and-path of the file as reduced + in the reduction directory. + """ + input_file_basename = os.path.basename(input_file_name) + start_dir = os.path.dirname(input_file_name) + status_file_name = short_name + "_status.txt" + status_file = os.path.join(work_directory, status_file_name) + uncal_file = os.path.join(work_directory, input_file_basename) + + try: + copy_files([input_file_name], work_directory) + set_permissions(uncal_file) + + steps = get_pipeline_steps(instrument) + first_step_to_be_run = True + for step_name in steps: + kwargs = {} + if step_name in ['jump', 'rate']: + kwargs = {'maximum_cores': 'all'} + if steps[step_name]: + output_file_name = short_name + "_{}.fits".format(step_name) + output_file = os.path.join(work_directory, output_file_name) + # skip already-done steps + if not os.path.isfile(output_file): + if first_step_to_be_run: + model = PIPELINE_STEP_MAPPING[step_name].call(input_file_name, **kwargs) + first_step_to_be_run = False + else: + model = PIPELINE_STEP_MAPPING[step_name].call(model, **kwargs) + + if step_name != 'rate': + # Make sure the dither_points metadata entry is at integer (was a + # string prior to jwst v1.2.1, so some input data still have the + # string entry. + # If we don't change that to an integer before saving the new file, + # the jwst package will crash. + try: + model.meta.dither.dither_points = int(model.meta.dither.dither_points) + except TypeError: + # If the dither_points entry is not populated, then ignore this + # change + pass + model.save(output_file) + else: + try: + model[0].meta.dither.dither_points = int(model[0].meta.dither.dither_points) + except TypeError: + # If the dither_points entry is not populated, then ignore this change + pass + model[0].save(output_file) + + done = True + for output in outputs: + output_name = "{}_{}.fits".format(short_name, output) + output_check_file = os.path.join(work_directory, output_name) + if not os.path.isfile(output_check_file): + done = False + if done: + break + except Exception as e: + with open(status_file, "w") as statfile: + statfile.write("EXCEPTION\n") + statfile.write(e) + sys.exit(1) + + with open(status_file, "w") as statfile: + statfile.write("DONE\n") + # Done. + + +def run_save_jump(input_file_name, short_name, work_directory, instrument, ramp_fit=True, save_fitopt=True): + """Call ``calwebb_detector1`` on the provided file, running all + steps up to the ``ramp_fit`` step, and save the result. Optionally + run the ``ramp_fit`` step and save the resulting slope file as well. + """ + input_file_basename = os.path.basename(input_file_name) + start_dir = os.path.dirname(input_file_name) + status_file_name = short_name + "_status.txt" + status_file = os.path.join(work_directory, status_file_name) + uncal_file = os.path.join(work_directory, input_file_basename) + + try: + # Find the instrument used to collect the data + datamodel = datamodels.RampModel(uncal_file) + instrument = datamodel.meta.instrument.name.lower() + + # If the data pre-date jwst version 1.2.1, then they will have + # the NUMDTHPT keyword (with string value of the number of dithers) + # rather than the newer NRIMDTPT keyword (with an integer value of + # the number of dithers). If so, we need to update the file here so + # that it doesn't cause the pipeline to crash later. Both old and + # new keywords are mapped to the model.meta.dither.dither_points + # metadata entry. So we should be able to focus on that. + if isinstance(datamodel.meta.dither.dither_points, str): + # If we have a string, change it to an integer + datamodel.meta.dither.dither_points = int(datamodel.meta.dither.dither_points) + elif datamodel.meta.dither.dither_points is None: + # If the information is missing completely, put in a dummy value + datamodel.meta.dither.dither_points = 1 + + # Switch to calling the pipeline rather than individual steps, + # and use the run() method so that we can set parameters + # progammatically. + model = Detector1Pipeline() + + # Always true + if instrument == 'nircam': + model.refpix.odd_even_rows = False + + # Default CR rejection threshold is too low + model.jump.rejection_threshold = 15 + + # Turn off IPC step until it is put in the right place + model.ipc.skip = True + + model.jump.save_results = True + model.jump.output_dir = os.getcwd() + model.jump.maximum_cores = 'all' + jump_output = uncal_file.replace('uncal', 'jump') + + # Check to see if the jump version of the requested file is already + # present + run_jump = not os.path.isfile(jump_output) + + if ramp_fit: + model.ramp_fit.save_results = True + model.ramp_fit.maximum_cores = 'all' + # model.save_results = True + model.output_dir = os.getcwd() + # pipe_output = os.path.join(output_dir, input_file_only.replace('uncal', 'rate')) + pipe_output = uncal_file.replace('uncal', '0_ramp_fit') + run_slope = not os.path.isfile(pipe_output) + if save_fitopt: + model.ramp_fit.save_opt = True + fitopt_output = uncal_file.replace('uncal', 'fitopt') + run_fitopt = not os.path.isfile(fitopt_output) + else: + model.ramp_fit.save_opt = False + fitopt_output = None + run_fitopt = False + else: + model.ramp_fit.skip = True + pipe_output = None + fitopt_output = None + run_slope = False + run_fitopt = False + + if run_jump or (ramp_fit and run_slope) or (save_fitopt and run_fitopt): + model.run(datamodel) + else: + print(("Files with all requested calibration states for {} already present in " + "output directory. Skipping pipeline call.".format(uncal_file))) + except Exception as e: + with open(status_file, "w") as statfile: + statfile.write("EXCEPTION\n") + statfile.write(e) + sys.exit(1) + + with open(status_file, "w") as statfile: + statfile.write("DONE\n") + # Done. + + +if __name__ == '__main__': + file_help = 'Input file to calibrate' + pipe_help = 'Pipeline type to run (valid values are "jump" and "cal")' + out_help = 'Comma-separated list of output extensions (for cal only, otherwise just "all")' + parser = argparse.ArgumentParser(description='Run local calibration') + parser.add_argument('pipe', metavar='PIPE', type=str, help=pipe_help) + parser.add_argument('outputs', metavar='OUTPUTS', type=str, help=out_help) + parser.add_argument('input_file', metavar='FILE', type=str, help=file_help) + args = parser.parse_args() + + input_files = glob(args.input_file) + if len(input_files) == 0: + raise FileNotFoundError("Pattern {} produced no input files!".format(args.input_file)) + for input_file in input_files: + if not os.path.isfile(input_file): + print("ERROR: Can't find input file {}".format(input_file)) + continue + instrument = get_instrument(input_file) + if instrument == 'unknown': + raise ValueError("Can't figure out instrument for {}".format(input_file)) + + pipe_type = args.pipe + if pipe_type not in ['jump', 'cal']: + raise ValueError("Unknown calibration type {}".format(pipe_type)) + + if pipe_type == 'jump': + run_save_jump(input_file, instrument) + elif pipe_type == 'cal': + outputs = args.outputs.split(",") + run_pipe(input_file, instrument, outputs) From 32bca1a7d7cdf16256119808805633e81ae77505 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Wed, 8 Feb 2023 11:39:39 -0500 Subject: [PATCH 011/449] Updated subprocess file to properly handle arguments --- jwql/shared_tasks/run_pipeline.py | 46 ++++++++++++++++--------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 6e7a49102..864aa56a9 100644 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -32,13 +32,13 @@ from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path -def run_pipe(input_file_name, short_name, work_directory, instrument, outputs): +def run_pipe(input_file, short_name, work_directory, instrument, outputs): """Run the steps of ``calwebb_detector1`` on the input file, saving the result of each step as a separate output file, then return the name-and-path of the file as reduced in the reduction directory. """ - input_file_basename = os.path.basename(input_file_name) - start_dir = os.path.dirname(input_file_name) + input_file_basename = os.path.basename(input_file) + start_dir = os.path.dirname(input_file) status_file_name = short_name + "_status.txt" status_file = os.path.join(work_directory, status_file_name) uncal_file = os.path.join(work_directory, input_file_basename) @@ -199,31 +199,33 @@ def run_save_jump(input_file_name, short_name, work_directory, instrument, ramp_ if __name__ == '__main__': file_help = 'Input file to calibrate' + path_help = 'Directory in which to do the calibration' + ins_help = 'Instrument that was used to produce the input file' pipe_help = 'Pipeline type to run (valid values are "jump" and "cal")' out_help = 'Comma-separated list of output extensions (for cal only, otherwise just "all")' + name_help = 'Input file name with no path or extensions' parser = argparse.ArgumentParser(description='Run local calibration') parser.add_argument('pipe', metavar='PIPE', type=str, help=pipe_help) parser.add_argument('outputs', metavar='OUTPUTS', type=str, help=out_help) + parser.add_argument('working_path', metavar='PATH', type=str, help=path_help) + parser.add_argument('instrument', metavar='INSTRUMENT', type=str, help=ins_help) parser.add_argument('input_file', metavar='FILE', type=str, help=file_help) + parser.add_argument('short_name', metavar='NAME', type=str, help=name_help) args = parser.parse_args() - input_files = glob(args.input_file) - if len(input_files) == 0: - raise FileNotFoundError("Pattern {} produced no input files!".format(args.input_file)) - for input_file in input_files: - if not os.path.isfile(input_file): - print("ERROR: Can't find input file {}".format(input_file)) - continue - instrument = get_instrument(input_file) - if instrument == 'unknown': - raise ValueError("Can't figure out instrument for {}".format(input_file)) - - pipe_type = args.pipe - if pipe_type not in ['jump', 'cal']: - raise ValueError("Unknown calibration type {}".format(pipe_type)) + if not os.path.isfile(args.input_file): + raise FileNotFoundError("No input file {}".format(args.input_file)) - if pipe_type == 'jump': - run_save_jump(input_file, instrument) - elif pipe_type == 'cal': - outputs = args.outputs.split(",") - run_pipe(input_file, instrument, outputs) + input_file = args.input_file + instrument = args.instrument + short_name = args.short_name + working_path = args.working_path + pipe_type = args.pipe + if pipe_type not in ['jump', 'cal']: + raise ValueError("Unknown calibration type {}".format(pipe_type)) + + if pipe_type == 'jump': + run_save_jump(input_file, short_name, working_path, instrument, ramp_fit=True, save_fitopt=True) + elif pipe_type == 'cal': + outputs = args.outputs.split(",") + run_pipe(input_file, short_name, work_directory, instrument, outputs) From eb0009707a09caf0d5ba1c13beb108ad1c51c025 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Wed, 8 Feb 2023 12:28:44 -0500 Subject: [PATCH 012/449] Updating shared tasks to call the subprocess script --- jwql/shared_tasks/run_pipeline.py | 3 + jwql/shared_tasks/shared_tasks.py | 259 ++++++++---------------------- 2 files changed, 72 insertions(+), 190 deletions(-) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 864aa56a9..7253025fe 100644 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -194,6 +194,9 @@ def run_save_jump(input_file_name, short_name, work_directory, instrument, ramp_ with open(status_file, "w") as statfile: statfile.write("DONE\n") + statfile.write("{}\n".format(jump_output)) + statfile.write("{}\n".format(pipe_output)) + statfile.write("{}\n".format(fitopt_output)) # Done. diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index dd40c3e21..836a03bb5 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -80,6 +80,7 @@ def some_function(some_arguments): import os import redis import shutil +import subprocess import sys from astropy.io import fits @@ -247,110 +248,53 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, reduction_path : str The path at which the reduced data file(s) may be found. """ - msg = "*****CELERY: Starting {} calibration task for {}" + msg = "Starting {} calibration task for {}" logging.info(msg.format(instrument, input_file_name)) config = get_config() input_dir = os.path.join(config['transfer_dir'], "incoming") cal_dir = os.path.join(config['outputs'], "calibrated_data") output_dir = os.path.join(config['transfer_dir'], "outgoing") - logging.info("*****CELERY: Input from {}, calibrate in {}, output to {}".format(input_dir, cal_dir, output_dir)) - - input_file = os.path.join(deepcopy(input_dir), input_file_name) - if not os.path.isfile(input_file): - logging.error("*****CELERY: File {} not found!".format(input_file)) - raise FileNotFoundError("{} not found".format(input_file)) - - uncal_file = os.path.join(deepcopy(cal_dir), input_file_name) - ensure_dir_exists(deepcopy(cal_dir)) - logging.info("*****CELERY: Copying {} to {}".format(input_file, cal_dir)) - copy_files([input_file], deepcopy(cal_dir)) - set_permissions(uncal_file) - - # Check for exposures with too many groups - max_groups = config.get("max_groups", 1000) - with fits.open(uncal_file) as inf: - total_groups = inf[0].header["NINTS"] * inf[0].header["NGROUPS"] - if total_groups > max_groups: - msg = "File {} has {} groups (greater than maximum of {})" - logging.error(msg.format(os.path.basename(uncal_file), total_groups, max_groups)) - raise ValueError(msg.format(os.path.basename(uncal_file), total_groups, max_groups)) - - steps = get_pipeline_steps(instrument) - - first_step_to_be_run = True - for step_name in steps: - kwargs = {} - if step_name in step_args: - kwargs = step_args[step_name] - if steps[step_name]: - output_filename = short_name + "_{}.fits".format(step_name) - logging.info("*****CELERY: Output File Name is {}".format(output_filename)) - output_file = os.path.join(cal_dir, deepcopy(output_filename)) - logging.info("*****CELERY: Creating output file {}".format(output_file)) - transfer_file = os.path.join(output_dir, output_filename) - # skip already-done steps - if not os.path.isfile(output_file): - logging.info("*****CELERY: Running Pipeline Step {}".format(step_name)) - if first_step_to_be_run: - model = PIPELINE_STEP_MAPPING[step_name].call(uncal_file, **kwargs) - first_step_to_be_run = False - else: - model = PIPELINE_STEP_MAPPING[step_name].call(model, **kwargs) - - if step_name != 'rate': - # Make sure the dither_points metadata entry is at integer (was a - # string prior to jwst v1.2.1, so some input data still have the - # string entry. - # If we don't change that to an integer before saving the new file, - # the jwst package will crash. - try: - model.meta.dither.dither_points = int(model.meta.dither.dither_points) - except TypeError: - # If the dither_points entry is not populated, then ignore this - # change - pass - logging.info("*****CELERY: Saving to {}".format(output_file)) - model.save(output_file) - else: - try: - model[0].meta.dither.dither_points = int(model[0].meta.dither.dither_points) - except TypeError: - # If the dither_points entry is not populated, then ignore this change - pass - logging.info("*****CELERY: Saving to {}".format(output_file)) - model[0].save(output_file) - else: - logging.info("*****CELERY: File {} exists".format(output_filename)) - if not os.path.isfile(transfer_file): - logging.info("*****CELERY: Copying {} to {}".format(output_file, output_dir)) - copy_files([output_file], output_dir) - else: - logging.info("*****CELERY: File {} already exists".format(transfer_file)) - set_permissions(transfer_file) - - # Check for everything being done (in which case we don't need to run - # subsequent pipeline steps) - done = True - for ext in ext_or_exts: - if not os.path.isfile("{}_{}.fits".format(short_name, ext)): - done = False - if done: - print("*****CELERY: Created all files in {}. Finished.".format(ext_or_exts)) - break - - logging.info("*****CELERY: Removing local files.") + msg = "Input from {}, calibrate in {}, output to {}" + logging.info(msg.format(input_dir, cal_dir, output_dir)) + + input_file = os.path.join(input_dir, input_file_name) + current_dir = os.path.dirname(__file__) + cmd_name = os.path.join(current_dir, "run_pipeline.py") + outputs = ",".join(ext_or_exts) + result_file = os.path.join(cal_dir, short_name+"_status.txt") + calibrated_files = ["{}_{}.fits".format(short_name, ext) for ext in ext_or_exts] + + result = subprocess.run(["python", cmd_name, "cal", outputs, cal_dir, instrument, input_file, short_name], + env=os.environ.copy(), shell=True) + + with open(result_file, 'r') as inf: + status = inf.readlines() + + if status[0].strip() == "DONE": + logging.info("Subprocess reports successful finish.") + else: + logging.error("ERROR: {}".format(status[1].strip())) + raise ValueError(status[1].strip()) + + for file in calibrated_files: + if not os.path.isfile(os.path.join(cal_dir, file)): + logging.error("ERROR: {} not found".format(file)) + raise FileNotFoundError(file) + copy_files([os.path.join(cal_dir, file)], output_dir) + set_permissions(os.path.join(output_dir, file)) + + logging.info("Removing local files.") files_to_remove = glob(uncal_file.replace("_uncal.fits", "*")) for file_name in files_to_remove: logging.info("\tRemoving {}".format(file_name)) os.remove(file_name) - logging.info("*****CELERY: Finished calibration.") - return output_dir + logging.info("Finished calibration.") @celery_app.task(name='jwql.shared_tasks.shared_tasks.calwebb_detector1_save_jump') -def calwebb_detector1_save_jump(input_file_name, ramp_fit=True, save_fitopt=True): +def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save_fitopt=True): """Call ``calwebb_detector1`` on the provided file, running all steps up to the ``ramp_fit`` step, and save the result. Optionally run the ``ramp_fit`` step and save the resulting slope file as well. @@ -386,119 +330,54 @@ def calwebb_detector1_save_jump(input_file_name, ramp_fit=True, save_fitopt=True Name of the saved file containing the output after ramp-fitting is performed (if requested). Otherwise ``None``. """ - config = get_config() - msg = "*****CELERY: Started Save Jump Task on {}. ramp_fit={}, save_fitopt={}" + msg = "Started Save Jump Task on {}. ramp_fit={}, save_fitopt={}" logging.info(msg.format(input_file_name, ramp_fit, save_fitopt)) + config = get_config() - input_file = os.path.join(config["transfer_dir"], "incoming", input_file_name) + input_dir = os.path.join(config["transfer_dir"], "incoming") + cal_dir = os.path.join(config['outputs'], "calibrated_data") + output_dir = os.path.join(config['transfer_dir'], "outgoing") + msg = "Input from {}, calibrate in {}, output to {}" + logging.info(msg.format(input_dir, cal_dir, output_dir)) + + input_file = os.path.join(input_dir, input_file_name) if not os.path.isfile(input_file): - logging.error("*****CELERY: File {} not found!".format(input_file)) + logging.error("File {} not found!".format(input_file)) raise FileNotFoundError("{} not found".format(input_file)) - cal_dir = os.path.join(config['outputs'], "calibrated_data") - uncal_file = os.path.join(cal_dir, input_file_name) - short_name = input_file_name.replace("_uncal", "").replace("_0thgroup", "") + short_name = input_file_name.replace("_uncal", "").replace("_0thgroup", "").replace(".fits", "") ensure_dir_exists(cal_dir) - copy_files([input_file], cal_dir) - set_permissions(uncal_file) - - # Check for exposures with too many groups - max_groups = config.get("max_groups", 1000) - with fits.open(uncal_file) as inf: - total_groups = inf[0].header["NINTS"] * inf[0].header["NGROUPS"] - if total_groups > max_groups: - msg = "File {} has {} groups (greater than maximum of {})" - logging.error(msg.format(os.path.basename(uncal_file), total_groups, max_groups)) - raise ValueError(msg.format(os.path.basename(uncal_file), total_groups, max_groups)) - output_dir = os.path.join(config["transfer_dir"], "outgoing") + + cmd_name = os.path.join(os.path.dirname(__file__), "run_pipeline.py") + result_file = os.path.join(cal_dir, short_name+"_status.txt") - log_config = os.path.join(output_dir, "celery_pipeline_log.cfg") - - # Find the instrument used to collect the data - datamodel = datamodels.RampModel(uncal_file) - instrument = datamodel.meta.instrument.name.lower() - - # If the data pre-date jwst version 1.2.1, then they will have - # the NUMDTHPT keyword (with string value of the number of dithers) - # rather than the newer NRIMDTPT keyword (with an integer value of - # the number of dithers). If so, we need to update the file here so - # that it doesn't cause the pipeline to crash later. Both old and - # new keywords are mapped to the model.meta.dither.dither_points - # metadata entry. So we should be able to focus on that. - if isinstance(datamodel.meta.dither.dither_points, str): - # If we have a string, change it to an integer - datamodel.meta.dither.dither_points = int(datamodel.meta.dither.dither_points) - elif datamodel.meta.dither.dither_points is None: - # If the information is missing completely, put in a dummy value - datamodel.meta.dither.dither_points = 1 - - # Switch to calling the pipeline rather than individual steps, - # and use the run() method so that we can set parameters - # progammatically. - model = Detector1Pipeline() - - # Always true - if instrument == 'nircam': - model.refpix.odd_even_rows = False - - # Default CR rejection threshold is too low - model.jump.rejection_threshold = 15 - - # Turn off IPC step until it is put in the right place - model.ipc.skip = True - - model.jump.save_results = True - model.jump.output_dir = cal_dir - jump_output = os.path.join(cal_dir, input_file.replace('uncal', 'jump')) - - model.logcfg = log_config - - # Check to see if the jump version of the requested file is already - # present - run_jump = not os.path.isfile(jump_output) - - if ramp_fit: - model.ramp_fit.save_results = True - # model.save_results = True - model.output_dir = cal_dir - # pipe_output = os.path.join(output_dir, input_file_only.replace('uncal', 'rate')) - pipe_output = os.path.join(cal_dir, input_file.replace('uncal', '0_ramp_fit')) - run_slope = not os.path.isfile(pipe_output) - if save_fitopt: - model.ramp_fit.save_opt = True - fitopt_output = os.path.join(cal_dir, input_file.replace('uncal', 'fitopt')) - run_fitopt = not os.path.isfile(fitopt_output) - else: - model.ramp_fit.save_opt = False - fitopt_output = None - run_fitopt = False - else: - model.ramp_fit.skip = True - pipe_output = None - fitopt_output = None - run_slope = False - run_fitopt = False - - # Call the pipeline if any of the files at the requested calibration - # states are not present in the output directory - logging.info("*****CELERY: Running save_jump pipeline") - if run_jump or (ramp_fit and run_slope) or (save_fitopt and run_fitopt): - model.run(datamodel) - else: - print(("Files with all requested calibration states for {} already present in " - "output directory. Skipping pipeline call.".format(input_file))) + result = subprocess.run(["python", cmd_name, "jump", "all", cal_dir, instrument, input_file, short_name], + env=os.environ.copy(), shell=True) - calibrated_files = glob(uncal_file.replace("_uncal.fits", "*")) - logging.info("*****CELERY: Pipeline Output is {}".format(calibrated_files)) - copy_files(calibrated_files, output_dir) + with open(result_file, 'r') as inf: + status = inf.readlines() - logging.info("*****CELERY: Removing local files.") + if status[0].strip() == "DONE": + logging.info("Subprocess reports successful finish.") + else: + logging.error("ERROR: {}".format(status[1].strip())) + raise ValueError(status[1].strip()) + + for line in status[1:]: + file = line.strip() + if not os.path.isfile(os.path.join(cal_dir, file)): + logging.error("ERROR: {} not found".format(file)) + raise FileNotFoundError(file) + copy_files([os.path.join(cal_dir, file)], output_dir) + set_permissions(os.path.join(output_dir, file)) + + logging.info("Removing local files.") for file_name in calibrated_files: logging.info("\tRemoving {}".format(file_name)) os.remove(file_name) - logging.info("*****CELERY: Finished pipeline") + logging.info("Finished pipeline") return jump_output, pipe_output, fitopt_output @@ -618,7 +497,7 @@ def start_pipeline(input_file, short_name, ext_or_exts, instrument, jump_pipe=Fa ramp_fit = True elif "fitopt" in ext: save_fitopt = True - result = calwebb_detector1_save_jump.delay(input_file, ramp_fit=ramp_fit, save_fitopt=save_fitopt) + result = calwebb_detector1_save_jump.delay(input_file, instrument, ramp_fit=ramp_fit, save_fitopt=save_fitopt) else: result = run_calwebb_detector1.delay(input_file, short_name, ext_or_exts, instrument) return result From 0212a9d345020aa96b6c572efea314fec99c565d Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Wed, 8 Feb 2023 12:33:42 -0500 Subject: [PATCH 013/449] Removing python from subprocess call --- jwql/shared_tasks/shared_tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index 836a03bb5..e8182d001 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -265,7 +265,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, result_file = os.path.join(cal_dir, short_name+"_status.txt") calibrated_files = ["{}_{}.fits".format(short_name, ext) for ext in ext_or_exts] - result = subprocess.run(["python", cmd_name, "cal", outputs, cal_dir, instrument, input_file, short_name], + result = subprocess.run([cmd_name, "cal", outputs, cal_dir, instrument, input_file, short_name], env=os.environ.copy(), shell=True) with open(result_file, 'r') as inf: @@ -352,7 +352,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save cmd_name = os.path.join(os.path.dirname(__file__), "run_pipeline.py") result_file = os.path.join(cal_dir, short_name+"_status.txt") - result = subprocess.run(["python", cmd_name, "jump", "all", cal_dir, instrument, input_file, short_name], + result = subprocess.run([cmd_name, "jump", "all", cal_dir, instrument, input_file, short_name], env=os.environ.copy(), shell=True) with open(result_file, 'r') as inf: From 5b0312656c1635371ddcbd1f737792c203ee73cc Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Wed, 8 Feb 2023 14:10:15 -0500 Subject: [PATCH 014/449] Adding command printout and result printout --- jwql/shared_tasks/shared_tasks.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index e8182d001..297c07a5a 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -265,9 +265,14 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, result_file = os.path.join(cal_dir, short_name+"_status.txt") calibrated_files = ["{}_{}.fits".format(short_name, ext) for ext in ext_or_exts] + msg = "Running {} cal {} {} {} {} {}" + logging.info(msg.format(cmd_name, outputs, cal_dir, instrument, input_file, short_name)) + result = subprocess.run([cmd_name, "cal", outputs, cal_dir, instrument, input_file, short_name], env=os.environ.copy(), shell=True) + logging.info("Subprocess result was {}".format(result)) + with open(result_file, 'r') as inf: status = inf.readlines() From e84138ba2eb3284b2c132ba447a07e165f80375c Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Wed, 8 Feb 2023 14:26:30 -0500 Subject: [PATCH 015/449] Now properly handling a single requested extension --- jwql/shared_tasks/shared_tasks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index 297c07a5a..4943871d2 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -251,6 +251,8 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, msg = "Starting {} calibration task for {}" logging.info(msg.format(instrument, input_file_name)) config = get_config() + if isinstance(ext_or_exts, str): + ext_or_exts = [ext_or_exts] input_dir = os.path.join(config['transfer_dir'], "incoming") cal_dir = os.path.join(config['outputs'], "calibrated_data") From 391315c81c1fbabd8b39bf8bfdfd94d72547ebae Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Wed, 8 Feb 2023 15:10:13 -0500 Subject: [PATCH 016/449] Logging more status information earlier --- jwql/shared_tasks/run_pipeline.py | 44 ++++++++++++++++++++++++------- jwql/shared_tasks/shared_tasks.py | 21 +++++++++------ 2 files changed, 48 insertions(+), 17 deletions(-) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 7253025fe..091f4fbf6 100644 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -43,6 +43,9 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs): status_file = os.path.join(work_directory, status_file_name) uncal_file = os.path.join(work_directory, input_file_basename) + with open(status_file, 'a+') as status_file: + status_file.write("Starting pipeline\n") + try: copy_files([input_file_name], work_directory) set_permissions(uncal_file) @@ -50,6 +53,8 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs): steps = get_pipeline_steps(instrument) first_step_to_be_run = True for step_name in steps: + with open(status_file, 'a+') as status_file: + status_file.write("Running step {}\n".format(step_name)) kwargs = {} if step_name in ['jump', 'rate']: kwargs = {'maximum_cores': 'all'} @@ -94,13 +99,14 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs): if done: break except Exception as e: - with open(status_file, "w") as statfile: + with open(status_file, "a+") as statfile: statfile.write("EXCEPTION\n") - statfile.write(e) + statfile.write("{}\n".format(e)) + statfile.write("FAILED") sys.exit(1) - with open(status_file, "w") as statfile: - statfile.write("DONE\n") + with open(status_file, "a+") as statfile: + statfile.write("SUCCEEDED") # Done. @@ -114,6 +120,9 @@ def run_save_jump(input_file_name, short_name, work_directory, instrument, ramp_ status_file_name = short_name + "_status.txt" status_file = os.path.join(work_directory, status_file_name) uncal_file = os.path.join(work_directory, input_file_basename) + + with open(status_file, 'a+') as status_file: + status_file.write("Starting pipeline\n") try: # Find the instrument used to collect the data @@ -187,16 +196,17 @@ def run_save_jump(input_file_name, short_name, work_directory, instrument, ramp_ print(("Files with all requested calibration states for {} already present in " "output directory. Skipping pipeline call.".format(uncal_file))) except Exception as e: - with open(status_file, "w") as statfile: + with open(status_file, "a+") as statfile: statfile.write("EXCEPTION\n") - statfile.write(e) + statfile.write("{}\n".format(e)) + statfile.write("FAILED") sys.exit(1) - with open(status_file, "w") as statfile: - statfile.write("DONE\n") + with open(status_file, "a+") as statfile: statfile.write("{}\n".format(jump_output)) statfile.write("{}\n".format(pipe_output)) statfile.write("{}\n".format(fitopt_output)) + statfile.write("SUCCEEDED") # Done. @@ -224,11 +234,27 @@ def run_save_jump(input_file_name, short_name, work_directory, instrument, ramp_ short_name = args.short_name working_path = args.working_path pipe_type = args.pipe + outputs = args.outputs + + status_file = os.path.join(working_path, short_name+"_status.txt") + with open(status_file, 'a+') as out_file: + out_file.write("Starting Process\n") + out_file.write("\tpipeline is {}\n".format(pipe_type)) + out_file.write("\toutputs is {}\n".format(outputs)) + out_file.write("\tworking_path is {}\n".format(working_path)) + out_file.write("\tinstrument is {}\n".format(instrument)) + out_file.write("\tinput_file is {}\n".format(input_file)) + out_file.write("\tshort_name is {}\n".format(short_name)) + if pipe_type not in ['jump', 'cal']: raise ValueError("Unknown calibration type {}".format(pipe_type)) if pipe_type == 'jump': + with open(status_file, 'a+') as out_file: + out_file.write("Running jump pipeline.\n") run_save_jump(input_file, short_name, working_path, instrument, ramp_fit=True, save_fitopt=True) elif pipe_type == 'cal': - outputs = args.outputs.split(",") + with open(status_file, 'a+') as out_file: + out_file.write("Running cal pipeline.\n") + outputs = outputs.split(",") run_pipe(input_file, short_name, work_directory, instrument, outputs) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index 4943871d2..d6f992162 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -278,11 +278,13 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, with open(result_file, 'r') as inf: status = inf.readlines() - if status[0].strip() == "DONE": + if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") else: - logging.error("ERROR: {}".format(status[1].strip())) - raise ValueError(status[1].strip()) + logging.error("Pipeline subprocess failed.") + for line in status: + logging.error("\t{}".format(line.strip())) + raise ValueError("Pipeline Failed") for file in calibrated_files: if not os.path.isfile(os.path.join(cal_dir, file)): @@ -365,13 +367,16 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save with open(result_file, 'r') as inf: status = inf.readlines() - if status[0].strip() == "DONE": + if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") else: - logging.error("ERROR: {}".format(status[1].strip())) - raise ValueError(status[1].strip()) - - for line in status[1:]: + logging.error("Pipeline subprocess failed.") + for line in status: + logging.error("\t{}".format(line.strip())) + raise ValueError("Pipeline Failed") + + + for line in status[-4:-1]: file = line.strip() if not os.path.isfile(os.path.join(cal_dir, file)): logging.error("ERROR: {} not found".format(file)) From 4b3804280723d3f28d94d7f90e04f713bfe7ce6f Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Wed, 8 Feb 2023 15:25:35 -0500 Subject: [PATCH 017/449] Even More Logging --- jwql/shared_tasks/run_pipeline.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 091f4fbf6..41dc98a10 100644 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -8,6 +8,7 @@ import os import shutil import sys +import time from jwst import datamodels from jwst.dq_init import DQInitStep @@ -209,8 +210,12 @@ def run_save_jump(input_file_name, short_name, work_directory, instrument, ramp_ statfile.write("SUCCEEDED") # Done. +#CompletedProcess(args=['/home/svc_jwqladm_mon/jwql/jwql/jwql/shared_tasks/run_pipeline.py', 'cal', 'refpix', '/internal/data1/outputs/ops/calibrated_data', 'niriss', '/grp/jwst/ins/jwql/transfer/ops/incoming/jw01499040001_02201_00001_nis_uncal.fits', 'jw01499040001_02201_00001_nis'], returncode=126) if __name__ == '__main__': + with open("/internal/data1/outputs/ops/calibrated_data/general_status.txt", "a+") as status_file: + status_file.write("Started at {}".format(time.ctime())) + file_help = 'Input file to calibrate' path_help = 'Directory in which to do the calibration' ins_help = 'Instrument that was used to produce the input file' @@ -224,10 +229,14 @@ def run_save_jump(input_file_name, short_name, work_directory, instrument, ramp_ parser.add_argument('instrument', metavar='INSTRUMENT', type=str, help=ins_help) parser.add_argument('input_file', metavar='FILE', type=str, help=file_help) parser.add_argument('short_name', metavar='NAME', type=str, help=name_help) + + with open("/internal/data1/outputs/ops/calibrated_data/general_status.txt", "a+") as status_file: + status_file.write("Created argument parser at {}".format(time.ctime())) + args = parser.parse_args() - - if not os.path.isfile(args.input_file): - raise FileNotFoundError("No input file {}".format(args.input_file)) + + with open("/internal/data1/outputs/ops/calibrated_data/general_status.txt", "a+") as status_file: + status_file.write("Finished parsing args at {}".format(time.ctime())) input_file = args.input_file instrument = args.instrument @@ -246,6 +255,9 @@ def run_save_jump(input_file_name, short_name, work_directory, instrument, ramp_ out_file.write("\tinput_file is {}\n".format(input_file)) out_file.write("\tshort_name is {}\n".format(short_name)) + if not os.path.isfile(args.input_file): + raise FileNotFoundError("No input file {}".format(args.input_file)) + if pipe_type not in ['jump', 'cal']: raise ValueError("Unknown calibration type {}".format(pipe_type)) From 48a4c7954601136345afc9a15e769c459d82ea83 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Wed, 8 Feb 2023 15:41:55 -0500 Subject: [PATCH 018/449] Made the subprocess script executable --- jwql/shared_tasks/run_pipeline.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 jwql/shared_tasks/run_pipeline.py diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py old mode 100644 new mode 100755 From 6d8ef9a5d76fbbc56bdf08219f66f8ee004c885f Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Wed, 8 Feb 2023 15:48:07 -0500 Subject: [PATCH 019/449] Adding more logging --- jwql/shared_tasks/run_pipeline.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 41dc98a10..2e8f840d0 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -214,7 +214,7 @@ def run_save_jump(input_file_name, short_name, work_directory, instrument, ramp_ if __name__ == '__main__': with open("/internal/data1/outputs/ops/calibrated_data/general_status.txt", "a+") as status_file: - status_file.write("Started at {}".format(time.ctime())) + status_file.write("Started at {}\n".format(time.ctime())) file_help = 'Input file to calibrate' path_help = 'Directory in which to do the calibration' @@ -231,12 +231,18 @@ def run_save_jump(input_file_name, short_name, work_directory, instrument, ramp_ parser.add_argument('short_name', metavar='NAME', type=str, help=name_help) with open("/internal/data1/outputs/ops/calibrated_data/general_status.txt", "a+") as status_file: - status_file.write("Created argument parser at {}".format(time.ctime())) + status_file.write("Created argument parser at {}\n".format(time.ctime())) - args = parser.parse_args() + try: + args = parser.parse_args() + except Exception as e: + with open("/internal/data1/outputs/ops/calibrated_data/general_status.txt", "a+") as status_file: + status_file.write("Error parsing arguments.\n") + status_file.write("{}".format(e)) + raise e with open("/internal/data1/outputs/ops/calibrated_data/general_status.txt", "a+") as status_file: - status_file.write("Finished parsing args at {}".format(time.ctime())) + status_file.write("Finished parsing args at {}\n".format(time.ctime())) input_file = args.input_file instrument = args.instrument From a4f56fb16acc92be11eac4f61658711066416489 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Wed, 8 Feb 2023 16:02:31 -0500 Subject: [PATCH 020/449] No longer calling sys.argc because oops --- jwql/shared_tasks/run_pipeline.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 2e8f840d0..bd8c48bff 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -215,6 +215,7 @@ def run_save_jump(input_file_name, short_name, work_directory, instrument, ramp_ if __name__ == '__main__': with open("/internal/data1/outputs/ops/calibrated_data/general_status.txt", "a+") as status_file: status_file.write("Started at {}\n".format(time.ctime())) + status_file.write("\targv={}".format(sys.argv)) file_help = 'Input file to calibrate' path_help = 'Directory in which to do the calibration' From eced199202abfaf2f8290ecc0cbdf16781eb9fc7 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Wed, 8 Feb 2023 16:18:12 -0500 Subject: [PATCH 021/449] Added a look for the general log file --- jwql/shared_tasks/run_pipeline.py | 5 +++-- jwql/shared_tasks/shared_tasks.py | 6 ++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index bd8c48bff..54668472b 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -213,9 +213,10 @@ def run_save_jump(input_file_name, short_name, work_directory, instrument, ramp_ #CompletedProcess(args=['/home/svc_jwqladm_mon/jwql/jwql/jwql/shared_tasks/run_pipeline.py', 'cal', 'refpix', '/internal/data1/outputs/ops/calibrated_data', 'niriss', '/grp/jwst/ins/jwql/transfer/ops/incoming/jw01499040001_02201_00001_nis_uncal.fits', 'jw01499040001_02201_00001_nis'], returncode=126) if __name__ == '__main__': - with open("/internal/data1/outputs/ops/calibrated_data/general_status.txt", "a+") as status_file: + with open("/internal/data1/outputs/ops/calibrated_data/general_status.txt", "w") as status_file: status_file.write("Started at {}\n".format(time.ctime())) - status_file.write("\targv={}".format(sys.argv)) + status_file.write("\targv={}\n".format(sys.argv)) + status_file.write("\toriginal argv={}\n".format(sys.orig_argv)) file_help = 'Input file to calibrate' path_help = 'Directory in which to do the calibration' diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index d6f992162..c1c666dae 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -364,6 +364,12 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save result = subprocess.run([cmd_name, "jump", "all", cal_dir, instrument, input_file, short_name], env=os.environ.copy(), shell=True) + if not os.path.isfile(result_file): + logging.error("Result file was not created.") + with open(os.path.join(cal_dir, "general_status.txt")) as status_file: + for line in status_file.readlines(): + logging.error(line.strip()) + with open(result_file, 'r') as inf: status = inf.readlines() From 2faf973350e4c9fe72378738c6e152659347741b Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Wed, 8 Feb 2023 17:14:37 -0500 Subject: [PATCH 022/449] Actually delivering arguments hopefully --- jwql/shared_tasks/shared_tasks.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index c1c666dae..e873a490a 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -270,11 +270,17 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, msg = "Running {} cal {} {} {} {} {}" logging.info(msg.format(cmd_name, outputs, cal_dir, instrument, input_file, short_name)) - result = subprocess.run([cmd_name, "cal", outputs, cal_dir, instrument, input_file, short_name], - env=os.environ.copy(), shell=True) + cmd = "{} cal {} '{}' {} {} {}".format(cmd_name, outputs, cal_dir, instrument, input_file, short_name) + result = subprocess.run(cmd, env=os.environ.copy(), shell=True) logging.info("Subprocess result was {}".format(result)) - + + if not os.path.isfile(result_file): + logging.error("Result file was not created.") + with open(os.path.join(cal_dir, "general_status.txt")) as status_file: + for line in status_file.readlines(): + logging.error(line.strip()) + with open(result_file, 'r') as inf: status = inf.readlines() @@ -361,8 +367,8 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save cmd_name = os.path.join(os.path.dirname(__file__), "run_pipeline.py") result_file = os.path.join(cal_dir, short_name+"_status.txt") - result = subprocess.run([cmd_name, "jump", "all", cal_dir, instrument, input_file, short_name], - env=os.environ.copy(), shell=True) + cmd = "{} cal {} '{}' {} {} {}".format(cmd_name, outputs, cal_dir, instrument, input_file, short_name) + result = subprocess.run(cmd, env=os.environ.copy(), shell=True) if not os.path.isfile(result_file): logging.error("Result file was not created.") From 99981c423eb3cd639fa58d0f870c9f82d1440d59 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Wed, 8 Feb 2023 17:15:01 -0500 Subject: [PATCH 023/449] Remove the original-argv call --- jwql/shared_tasks/run_pipeline.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 54668472b..c2c17ba16 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -216,7 +216,6 @@ def run_save_jump(input_file_name, short_name, work_directory, instrument, ramp_ with open("/internal/data1/outputs/ops/calibrated_data/general_status.txt", "w") as status_file: status_file.write("Started at {}\n".format(time.ctime())) status_file.write("\targv={}\n".format(sys.argv)) - status_file.write("\toriginal argv={}\n".format(sys.orig_argv)) file_help = 'Input file to calibrate' path_help = 'Directory in which to do the calibration' From a8fabc14a103d2dc77544040363b561cd0ac2944 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Wed, 8 Feb 2023 17:30:26 -0500 Subject: [PATCH 024/449] Wrapped pipeline call in a try block for more info --- jwql/shared_tasks/run_pipeline.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index c2c17ba16..432381301 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -268,12 +268,18 @@ def run_save_jump(input_file_name, short_name, work_directory, instrument, ramp_ if pipe_type not in ['jump', 'cal']: raise ValueError("Unknown calibration type {}".format(pipe_type)) - if pipe_type == 'jump': - with open(status_file, 'a+') as out_file: - out_file.write("Running jump pipeline.\n") - run_save_jump(input_file, short_name, working_path, instrument, ramp_fit=True, save_fitopt=True) - elif pipe_type == 'cal': + try: + if pipe_type == 'jump': + with open(status_file, 'a+') as out_file: + out_file.write("Running jump pipeline.\n") + run_save_jump(input_file, short_name, working_path, instrument, ramp_fit=True, save_fitopt=True) + elif pipe_type == 'cal': + with open(status_file, 'a+') as out_file: + out_file.write("Running cal pipeline.\n") + outputs = outputs.split(",") + run_pipe(input_file, short_name, work_directory, instrument, outputs) + except Exception as e: with open(status_file, 'a+') as out_file: - out_file.write("Running cal pipeline.\n") - outputs = outputs.split(",") - run_pipe(input_file, short_name, work_directory, instrument, outputs) + out_file.write("Exception when starting pipeline.\n") + out_file.write("{}".format(e)) + raise e From bd4a31528f9cb2e6f56c29715c76aa476e0b19aa Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Wed, 8 Feb 2023 17:51:34 -0500 Subject: [PATCH 025/449] typo --- jwql/shared_tasks/run_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 432381301..133ad0f44 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -277,7 +277,7 @@ def run_save_jump(input_file_name, short_name, work_directory, instrument, ramp_ with open(status_file, 'a+') as out_file: out_file.write("Running cal pipeline.\n") outputs = outputs.split(",") - run_pipe(input_file, short_name, work_directory, instrument, outputs) + run_pipe(input_file, short_name, working_path, instrument, outputs) except Exception as e: with open(status_file, 'a+') as out_file: out_file.write("Exception when starting pipeline.\n") From e4c774386e81895c6222d1ebbac1ad72b01c3e2f Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Wed, 8 Feb 2023 20:54:50 -0500 Subject: [PATCH 026/449] more logging --- jwql/shared_tasks/run_pipeline.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 133ad0f44..f45b0394f 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -253,14 +253,14 @@ def run_save_jump(input_file_name, short_name, work_directory, instrument, ramp_ outputs = args.outputs status_file = os.path.join(working_path, short_name+"_status.txt") - with open(status_file, 'a+') as out_file: + with open(status_file, 'w') as out_file: out_file.write("Starting Process\n") - out_file.write("\tpipeline is {}\n".format(pipe_type)) - out_file.write("\toutputs is {}\n".format(outputs)) - out_file.write("\tworking_path is {}\n".format(working_path)) - out_file.write("\tinstrument is {}\n".format(instrument)) - out_file.write("\tinput_file is {}\n".format(input_file)) - out_file.write("\tshort_name is {}\n".format(short_name)) + out_file.write("\tpipeline is {} ({})\n".format(pipe_type, type(pipe_type))) + out_file.write("\toutputs is {} ({})\n".format(outputs, type(outputs))) + out_file.write("\tworking_path is {} ({})\n".format(working_path, type(working_path))) + out_file.write("\tinstrument is {} ({})\n".format(instrument, type(instrument))) + out_file.write("\tinput_file is {} ({})\n".format(input_file, type(input_file))) + out_file.write("\tshort_name is {} ({})\n".format(short_name, type(short_name))) if not os.path.isfile(args.input_file): raise FileNotFoundError("No input file {}".format(args.input_file)) @@ -281,5 +281,5 @@ def run_save_jump(input_file_name, short_name, work_directory, instrument, ramp_ except Exception as e: with open(status_file, 'a+') as out_file: out_file.write("Exception when starting pipeline.\n") - out_file.write("{}".format(e)) + out_file.write("{}\n".format(e)) raise e From 21eda36e8c0767b6cc9c17674560a06e85c22d63 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Wed, 8 Feb 2023 21:12:50 -0500 Subject: [PATCH 027/449] Not accidentally overwriting the file name with the file handle --- jwql/shared_tasks/run_pipeline.py | 47 ++++++++++++++++--------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index f45b0394f..909f9eced 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -44,18 +44,21 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs): status_file = os.path.join(work_directory, status_file_name) uncal_file = os.path.join(work_directory, input_file_basename) - with open(status_file, 'a+') as status_file: - status_file.write("Starting pipeline\n") + with open(status_file, 'a+') as status_f: + status_f.write("Running run_pipe\n") + status_f.write("\t input_file_basename is {} ({})\n".format(input_file_basename, type(input_file_basename))) + status_f.write("\t start_dir is {} ({})\n".format(start_dir, type(start_dir))) + status_f.write("\t uncal_file is {} ({})\n".format(uncal_file, type(uncal_file))) - try: + try: copy_files([input_file_name], work_directory) set_permissions(uncal_file) steps = get_pipeline_steps(instrument) first_step_to_be_run = True for step_name in steps: - with open(status_file, 'a+') as status_file: - status_file.write("Running step {}\n".format(step_name)) + with open(status_file, 'a+') as status_f: + status_f.write("Running step {}\n".format(step_name)) kwargs = {} if step_name in ['jump', 'rate']: kwargs = {'maximum_cores': 'all'} @@ -100,14 +103,14 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs): if done: break except Exception as e: - with open(status_file, "a+") as statfile: - statfile.write("EXCEPTION\n") - statfile.write("{}\n".format(e)) - statfile.write("FAILED") + with open(status_file, "a+") as status_f: + status_f.write("EXCEPTION\n") + status_f.write("{}\n".format(e)) + status_f.write("FAILED") sys.exit(1) - with open(status_file, "a+") as statfile: - statfile.write("SUCCEEDED") + with open(status_file, "a+") as status_f: + status_f.write("SUCCEEDED") # Done. @@ -122,8 +125,8 @@ def run_save_jump(input_file_name, short_name, work_directory, instrument, ramp_ status_file = os.path.join(work_directory, status_file_name) uncal_file = os.path.join(work_directory, input_file_basename) - with open(status_file, 'a+') as status_file: - status_file.write("Starting pipeline\n") + with open(status_file, 'a+') as status_f: + status_f.write("Starting pipeline\n") try: # Find the instrument used to collect the data @@ -197,17 +200,17 @@ def run_save_jump(input_file_name, short_name, work_directory, instrument, ramp_ print(("Files with all requested calibration states for {} already present in " "output directory. Skipping pipeline call.".format(uncal_file))) except Exception as e: - with open(status_file, "a+") as statfile: - statfile.write("EXCEPTION\n") - statfile.write("{}\n".format(e)) - statfile.write("FAILED") + with open(status_file, "a+") as status_f: + status_f.write("EXCEPTION\n") + status_f.write("{}\n".format(e)) + status_f.write("FAILED") sys.exit(1) - with open(status_file, "a+") as statfile: - statfile.write("{}\n".format(jump_output)) - statfile.write("{}\n".format(pipe_output)) - statfile.write("{}\n".format(fitopt_output)) - statfile.write("SUCCEEDED") + with open(status_file, "a+") as status_f: + status_f.write("{}\n".format(jump_output)) + status_f.write("{}\n".format(pipe_output)) + status_f.write("{}\n".format(fitopt_output)) + status_f.write("SUCCEEDED") # Done. #CompletedProcess(args=['/home/svc_jwqladm_mon/jwql/jwql/jwql/shared_tasks/run_pipeline.py', 'cal', 'refpix', '/internal/data1/outputs/ops/calibrated_data', 'niriss', '/grp/jwst/ins/jwql/transfer/ops/incoming/jw01499040001_02201_00001_nis_uncal.fits', 'jw01499040001_02201_00001_nis'], returncode=126) From bcc414c2bf0125869642c438bf178f370ea0e040 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Wed, 8 Feb 2023 21:26:01 -0500 Subject: [PATCH 028/449] typo --- jwql/shared_tasks/run_pipeline.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 909f9eced..5922de488 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -51,7 +51,7 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs): status_f.write("\t uncal_file is {} ({})\n".format(uncal_file, type(uncal_file))) try: - copy_files([input_file_name], work_directory) + copy_files([input_file], work_directory) set_permissions(uncal_file) steps = get_pipeline_steps(instrument) @@ -68,7 +68,7 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs): # skip already-done steps if not os.path.isfile(output_file): if first_step_to_be_run: - model = PIPELINE_STEP_MAPPING[step_name].call(input_file_name, **kwargs) + model = PIPELINE_STEP_MAPPING[step_name].call(input_file, **kwargs) first_step_to_be_run = False else: model = PIPELINE_STEP_MAPPING[step_name].call(model, **kwargs) @@ -114,13 +114,13 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs): # Done. -def run_save_jump(input_file_name, short_name, work_directory, instrument, ramp_fit=True, save_fitopt=True): +def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=True, save_fitopt=True): """Call ``calwebb_detector1`` on the provided file, running all steps up to the ``ramp_fit`` step, and save the result. Optionally run the ``ramp_fit`` step and save the resulting slope file as well. """ - input_file_basename = os.path.basename(input_file_name) - start_dir = os.path.dirname(input_file_name) + input_file_basename = os.path.basename(input_file) + start_dir = os.path.dirname(input_file) status_file_name = short_name + "_status.txt" status_file = os.path.join(work_directory, status_file_name) uncal_file = os.path.join(work_directory, input_file_basename) From 1ee002b6b29a0557dd97b2bb61d347f2b48d4a1e Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Thu, 9 Feb 2023 09:03:34 -0500 Subject: [PATCH 029/449] adding dropdowns and logic for proposal selection --- jwql/website/apps/jwql/data_containers.py | 16 ++++++++-------- jwql/website/apps/jwql/views.py | 3 ++- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 4af78eaf4..7a5502879 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -970,15 +970,10 @@ def get_proposals_by_category(instrument): # Get all unique dictionaries unique_results = list(map(dict, set(tuple(sorted(sub.items())) for sub in results))) - # Seperate the catagories from programs - categories = set([sub['category'] for sub in unique_results]) + # Make a dictionary of {program: category} to pull from + proposals_by_category = {d['program']:d['category'] for d in unique_results} - # build dict where key in catagory and value is a list of programs belonging to the category key. - category_sorted_dict = {} - for category in categories: - category_sorted_dict[category] = [d['program'] for d in unique_results if d['category'] == category] - - return category_sorted_dict + return proposals_by_category def get_proposal_info(filepaths): """Builds and returns a dictionary containing various information @@ -1454,6 +1449,9 @@ def thumbnails_ajax(inst, proposal, obs_num=None): data_dict['file_data'] = {} exp_types = [] + # Get category for each instrument proposal {prop_id: category} + proposals_by_category = get_proposals_by_category(inst) + # Gather data for each rootname, and construct a list of all observations # in the proposal for rootname in rootnames: @@ -1487,6 +1485,7 @@ def thumbnails_ajax(inst, proposal, obs_num=None): exp_start = [expstart for fname, expstart in zip(filenames, columns['expstart']) if rootname in fname][0] exp_type = [exp_type for fname, exp_type in zip(filenames, columns['exp_type']) if rootname in fname][0] exp_types.append(exp_type) + # Viewed is stored by rootname in the Model db. Save it with the data_dict # THUMBNAIL_FILTER_LOOK is boolean accessed according to a viewed flag try: @@ -1502,6 +1501,7 @@ def thumbnails_ajax(inst, proposal, obs_num=None): data_dict['file_data'][rootname]['available_files'] = available_files data_dict['file_data'][rootname]["viewed"] = viewed data_dict['file_data'][rootname]["exp_type"] = exp_type + data_dict['file_data'][rootname]["cat_type"] = proposals_by_category[filename_dict['program_id']] # We generate thumbnails only for rate and dark files. Check if these files # exist in the thumbnail filesystem. In the case where neither rate nor diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 16173f1b7..a91f8b950 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -360,11 +360,12 @@ def archived_proposals_ajax(request, inst): # Get all proposals based on category type proposals_by_category = get_proposals_by_category(inst) + unique_cat_types = list(set(proposals_by_category.values())) # The naming conventions for dropdown_menus are tightly coupled with the code, this should be changed down the line. dropdown_menus = {'look': THUMBNAIL_FILTER_LOOK, 'exp_type': exp_types, - 'cat_type': proposals_by_category.keys()} + 'cat_type': unique_cat_types} thumbnails_dict = {} for proposal_num in proposal_nums: From 3e29109a9790f9a4868f0549efc108788701cc52 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Thu, 9 Feb 2023 09:54:07 -0500 Subject: [PATCH 030/449] Fixing undefined variable --- jwql/shared_tasks/shared_tasks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index e873a490a..9edb95a28 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -300,7 +300,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, set_permissions(os.path.join(output_dir, file)) logging.info("Removing local files.") - files_to_remove = glob(uncal_file.replace("_uncal.fits", "*")) + files_to_remove = glob(os.path.join(cal_dir, short_name+"*")) for file_name in files_to_remove: logging.info("\tRemoving {}".format(file_name)) os.remove(file_name) @@ -397,7 +397,8 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save set_permissions(os.path.join(output_dir, file)) logging.info("Removing local files.") - for file_name in calibrated_files: + files_to_remove = glob(os.path.join(cal_dir, short_name+"*")) + for file_name in files_to_remove: logging.info("\tRemoving {}".format(file_name)) os.remove(file_name) From 66e47b19e36293ea31473a163cc0125796b6fba1 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Thu, 9 Feb 2023 10:23:55 -0500 Subject: [PATCH 031/449] Added a bit more logging --- jwql/shared_tasks/shared_tasks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index 9edb95a28..21993eb60 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -296,6 +296,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, if not os.path.isfile(os.path.join(cal_dir, file)): logging.error("ERROR: {} not found".format(file)) raise FileNotFoundError(file) + logging.info("Copying output file {}".format(file)) copy_files([os.path.join(cal_dir, file)], output_dir) set_permissions(os.path.join(output_dir, file)) @@ -390,6 +391,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save for line in status[-4:-1]: file = line.strip() + logging.info("Copying output file {}".format(file)) if not os.path.isfile(os.path.join(cal_dir, file)): logging.error("ERROR: {} not found".format(file)) raise FileNotFoundError(file) From 5a23752b26364f839d1ca7ed661083aa3a5089d2 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Thu, 9 Feb 2023 11:06:25 -0500 Subject: [PATCH 032/449] Added logging during subprocess (hopefully) --- jwql/shared_tasks/run_pipeline.py | 4 +++- jwql/shared_tasks/shared_tasks.py | 29 +++++++++++++++++++++++------ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 5922de488..fa79ff2f2 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -57,6 +57,7 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs): steps = get_pipeline_steps(instrument) first_step_to_be_run = True for step_name in steps: + sys.stderr.write("Running step {}\n".format(step_name)) with open(status_file, 'a+') as status_f: status_f.write("Running step {}\n".format(step_name)) kwargs = {} @@ -101,6 +102,7 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs): if not os.path.isfile(output_check_file): done = False if done: + sys.stderr.write("Done pipeline.\n") break except Exception as e: with open(status_file, "a+") as status_f: @@ -125,6 +127,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T status_file = os.path.join(work_directory, status_file_name) uncal_file = os.path.join(work_directory, input_file_basename) + sys.stderr.write("Starting pipeline\n") with open(status_file, 'a+') as status_f: status_f.write("Starting pipeline\n") @@ -213,7 +216,6 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T status_f.write("SUCCEEDED") # Done. -#CompletedProcess(args=['/home/svc_jwqladm_mon/jwql/jwql/jwql/shared_tasks/run_pipeline.py', 'cal', 'refpix', '/internal/data1/outputs/ops/calibrated_data', 'niriss', '/grp/jwst/ins/jwql/transfer/ops/incoming/jw01499040001_02201_00001_nis_uncal.fits', 'jw01499040001_02201_00001_nis'], returncode=126) if __name__ == '__main__': with open("/internal/data1/outputs/ops/calibrated_data/general_status.txt", "w") as status_file: diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index 21993eb60..de3fc9e0d 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -80,7 +80,7 @@ def some_function(some_arguments): import os import redis import shutil -import subprocess +from subprocess import Popen, PIPE, run, STDOUT import sys from astropy.io import fits @@ -198,6 +198,15 @@ def create_task_log_handler(logger, propagate): cfg_file.write("handler = append:{}\n".format(log_file_name)) +def log_subprocess_output(pipe): + """ + If a subprocess STDOUT has been set to subprocess.PIPE, this function will log each + line to the logging output. + """ + for line in iter(pipe.readline, b''): # b'\n'-separated lines + logging.info("\t{}".format(line)) + + @after_setup_task_logger.connect def after_setup_celery_task_logger(logger, **kwargs): """ This function sets the 'celery.task' logger handler and formatter """ @@ -270,9 +279,12 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, msg = "Running {} cal {} {} {} {} {}" logging.info(msg.format(cmd_name, outputs, cal_dir, instrument, input_file, short_name)) - cmd = "{} cal {} '{}' {} {} {}".format(cmd_name, outputs, cal_dir, instrument, input_file, short_name) - result = subprocess.run(cmd, env=os.environ.copy(), shell=True) - + cmd = "{} cal {} '{}' {} {} {}" + cmd = cmd.format(cmd_name, outputs, cal_dir, instrument, input_file, short_name) + process = Popen(cmd, shell=True, executable="/bin/bash", stderr=PIPE) + with process.stderr: + log_subprocess_output(process.stderr) + result = process.wait() logging.info("Subprocess result was {}".format(result)) if not os.path.isfile(result_file): @@ -368,8 +380,13 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save cmd_name = os.path.join(os.path.dirname(__file__), "run_pipeline.py") result_file = os.path.join(cal_dir, short_name+"_status.txt") - cmd = "{} cal {} '{}' {} {} {}".format(cmd_name, outputs, cal_dir, instrument, input_file, short_name) - result = subprocess.run(cmd, env=os.environ.copy(), shell=True) + cmd = "{} cal {} '{}' {} {} {}" + cmd = cmd.format(cmd_name, outputs, cal_dir, instrument, input_file, short_name) + process = Popen(cmd, shell=True, executable="/bin/bash", stderr=PIPE) + with process.stderr: + log_subprocess_output(process.stderr) + result = process.wait() + logging.info("Subprocess result was {}".format(result)) if not os.path.isfile(result_file): logging.error("Result file was not created.") From 564e1f7d6f084c487a9de345c0720681e287bf0d Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Thu, 9 Feb 2023 11:54:15 -0500 Subject: [PATCH 033/449] Hopefully now properly formatting logged output --- jwql/shared_tasks/shared_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index de3fc9e0d..bef393c78 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -204,7 +204,7 @@ def log_subprocess_output(pipe): line to the logging output. """ for line in iter(pipe.readline, b''): # b'\n'-separated lines - logging.info("\t{}".format(line)) + logging.info("\t{}".format(str(line).strip())) @after_setup_task_logger.connect From 7f47b2d651ceb7fec589751994071d193a457d8b Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Thu, 9 Feb 2023 12:40:48 -0500 Subject: [PATCH 034/449] Decoding byte string to UTF8 before logging --- jwql/shared_tasks/shared_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index bef393c78..49c1e695f 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -204,7 +204,7 @@ def log_subprocess_output(pipe): line to the logging output. """ for line in iter(pipe.readline, b''): # b'\n'-separated lines - logging.info("\t{}".format(str(line).strip())) + logging.info("\t{}".format(line.decode('UTF-8').strip())) @after_setup_task_logger.connect From 77a3ee00b75f6cd321c1128f10e783aee8d464f4 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Thu, 9 Feb 2023 13:23:51 -0500 Subject: [PATCH 035/449] adding updates to proposal thumbnail data --- jwql/website/apps/jwql/static/js/jwql.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index 3440a0eb7..04c9388ef 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -624,9 +624,10 @@ function update_archive_page(inst, base_url) { viewed = data.thumbnails.viewed[i]; exp_types = data.thumbnails.exp_types[i]; obs_time = data.thumbnails.obs_time[i]; + cat_type = data.thumbnails.cat_type[i]; // Build div content - content = '
'; + content = '
'; content += ''; From e4be56e16bc4a9a646a2340bd5b03c2b82dd5d41 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Thu, 9 Feb 2023 16:47:28 -0500 Subject: [PATCH 036/449] Moving calls and assignment of category type from filenames to thumbnails --- jwql/website/apps/jwql/data_containers.py | 4 ---- jwql/website/apps/jwql/views.py | 5 +++++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 7a5502879..b39749190 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -1449,9 +1449,6 @@ def thumbnails_ajax(inst, proposal, obs_num=None): data_dict['file_data'] = {} exp_types = [] - # Get category for each instrument proposal {prop_id: category} - proposals_by_category = get_proposals_by_category(inst) - # Gather data for each rootname, and construct a list of all observations # in the proposal for rootname in rootnames: @@ -1501,7 +1498,6 @@ def thumbnails_ajax(inst, proposal, obs_num=None): data_dict['file_data'][rootname]['available_files'] = available_files data_dict['file_data'][rootname]["viewed"] = viewed data_dict['file_data'][rootname]["exp_type"] = exp_type - data_dict['file_data'][rootname]["cat_type"] = proposals_by_category[filename_dict['program_id']] # We generate thumbnails only for rate and dark files. Check if these files # exist in the thumbnail filesystem. In the case where neither rate nor diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index a91f8b950..218a86d9f 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -353,6 +353,7 @@ def archived_proposals_ajax(request, inst): thumb_exp_types = [] proposal_obs_times = [] thumb_obs_time = [] + cat_types = [] # Get a set of all exposure types used in the observations associated with this proposal exp_types = [exposure_type for observation in all_entries for exposure_type in observation.exptypes.split(',')] @@ -397,12 +398,16 @@ def archived_proposals_ajax(request, inst): proposal_obs_times = [observation.obsstart for observation in prop_entries] thumb_obs_time.append(max(proposal_obs_times)) + # Add category type to list based on proposal number + cat_types.append(proposals_by_category[proposal_num]) + thumbnails_dict['proposals'] = proposal_nums thumbnails_dict['thumbnail_paths'] = thumbnail_paths thumbnails_dict['num_files'] = total_files thumbnails_dict['viewed'] = proposal_viewed thumbnails_dict['exp_types'] = thumb_exp_types thumbnails_dict['obs_time'] = thumb_obs_time + thumbnails_dict['cat_types'] = cat_types context = {'inst': inst, 'num_proposals': num_proposals, From 336d7aa1e6986a4edd9102a32859de628764a7dc Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Fri, 10 Feb 2023 09:49:53 -0500 Subject: [PATCH 037/449] Migrating in fixes to the bad pixel monitor --- .../common_monitors/bad_pixel_monitor.py | 45 +++++++++++++++---- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py index 427623715..4f749dacf 100755 --- a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py +++ b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py @@ -781,10 +781,12 @@ def process(self, illuminated_raw_files, illuminated_slope_files, flat_file_coun badpix_types_from_darks = ['HOT', 'RC', 'OTHER_BAD_PIXEL', 'TELEGRAPH'] illuminated_obstimes = [] if illuminated_raw_files: + logging.info("Found {} uncalibrated flat fields".format(len(illuminated_raw_files))) badpix_types.extend(badpix_types_from_flats) out_exts = defaultdict(lambda: ['jump', '0_ramp_fit']) in_files = [] for uncal_file, rate_file in zip(illuminated_raw_files, illuminated_slope_files): + logging.info("\tChecking illuminated raw file {} with rate file {}".format(uncal_file, rate_file)) self.get_metadata(uncal_file) if rate_file == 'None': short_name = os.path.basename(uncal_file).replace('_uncal.fits', '') @@ -801,14 +803,25 @@ def process(self, illuminated_raw_files, illuminated_slope_files, flat_file_coun if needs_calibration: in_files.append(local_uncal_file) else: - logging.info("Calibrated files already exist for {}".format(short_name)) - outputs = run_parallel_pipeline(in_files, "uncal", out_exts, self.instrument, jump_pipe=True) + logging.info("\t\tCalibrated files already exist for {}".format(short_name)) + else: + logging.info("\tRate file found for {}".format(uncal_file)) + + outputs = {} + if len(in_files) > 0: + logging.info("Running pipeline for {} files".format(len(in_files))) + outputs = run_parallel_pipeline(in_files, "uncal", out_exts, self.instrument, jump_pipe=True) + index = 0 + logging.info("Checking files post-calibration") for uncal_file, rate_file in zip(illuminated_raw_files, illuminated_slope_files): + logging.info("\tChecking files {}, {}".format(os.path.basename(uncal_file), os.path.basename(rate_file))) local_uncal_file = os.path.join(self.data_dir, os.path.basename(uncal_file)) if local_uncal_file in outputs: + logging.info("\t\tAdding calibrated file.") illuminated_slope_files[index] = deepcopy(outputs[local_uncal_file][1]) else: + logging.info("\t\tCalibration was skipped for file") self.get_metadata(illuminated_raw_files[index]) local_ramp_file = local_uncal_file.replace("uncal", "0_ramp_fit") if hasattr(self, 'nints') and self.nints > 1: @@ -822,9 +835,11 @@ def process(self, illuminated_raw_files, illuminated_slope_files, flat_file_coun # Get observation time for all files illuminated_obstimes.append(instrument_properties.get_obstime(uncal_file)) + logging.info("Trimming unfound files.") index = 0 while index < len(illuminated_raw_files): if illuminated_slope_files[index] is None or illuminated_slope_files[index] == 'None': + logging.info("\tRemoving {}".format(illuminated_raw_files[index])) del illuminated_raw_files[index] del illuminated_slope_files[index] del illuminated_obstimes[index] @@ -842,6 +857,7 @@ def process(self, illuminated_raw_files, illuminated_slope_files, flat_file_coun dark_fitopt_files = [] dark_obstimes = [] if dark_raw_files: + logging.info("Found {} uncalibrated darks".format(len(dark_raw_files))) index = 0 badpix_types.extend(badpix_types_from_darks) # In this case we need to run the pipeline on all input files, @@ -850,17 +866,19 @@ def process(self, illuminated_raw_files, illuminated_slope_files, flat_file_coun in_files = [] out_exts = defaultdict(lambda: ['jump', 'fitopt', '0_ramp_fit']) for uncal_file, rate_file in zip(dark_raw_files, dark_slope_files): + logging.info("Checking dark file {} with rate file {}".format(uncal_file, rate_file)) self.get_metadata(uncal_file) - logging.info('Calling pipeline for {} {}'.format(uncal_file, rate_file)) - logging.info("Copying raw file to {}".format(self.data_dir)) - copy_files([uncal_file], self.data_dir) - local_uncal_file = os.path.join(self.data_dir, os.path.basename(uncal_file)) short_name = os.path.basename(uncal_file).replace('_uncal.fits', '') + local_uncal_file = os.path.join(self.data_dir, os.path.basename(uncal_file)) + if not os.path.isfile(local_uncal_file): + logging.info("\tCopying raw file to {}".format(self.data_dir)) + copy_files([uncal_file], self.data_dir) if hasattr(self, 'nints') and self.nints > 1: out_exts[short_name] = ['jump', 'fitopt', '1_ramp_fit'] local_processed_files = [local_uncal_file.replace("uncal", x) for x in out_exts[short_name]] calibrated_data = [os.path.isfile(x) for x in local_processed_files] if not all(calibrated_data): + logging.info('\tCalling pipeline for {} {}'.format(uncal_file, rate_file)) in_files.append(local_uncal_file) dark_jump_files.append(None) dark_fitopt_files.append(None) @@ -872,16 +890,25 @@ def process(self, illuminated_raw_files, illuminated_slope_files, flat_file_coun dark_slope_files[index] = deepcopy(local_processed_files[2]) dark_obstimes.append(instrument_properties.get_obstime(uncal_file)) index += 1 - outputs = run_parallel_pipeline(in_files, "uncal", out_exts, self.instrument, jump_pipe=True) + + outputs = {} + if len(in_files) > 0: + logging.info("Running pipeline for {} files".format(len(in_files))) + outputs = run_parallel_pipeline(in_files, "uncal", out_exts, self.instrument, jump_pipe=True) + index = 0 + logging.info("Checking files post-calibration") for uncal_file, rate_file in zip(dark_raw_files, dark_slope_files): + logging.info("\tChecking files {}, {}".format(uncal_file, rate_file)) local_uncal_file = os.path.join(self.data_dir, os.path.basename(uncal_file)) short_name = os.path.basename(uncal_file).replace('_uncal.fits', '') if local_uncal_file in outputs: + logging.info("\t\tAdding calibrated files") dark_jump_files[index] = outputs[local_uncal_file][0] dark_fitopt_files[index] = outputs[local_uncal_file][1] dark_slope_files[index] = deepcopy(outputs[local_uncal_file][2]) else: + logging.info("\t\tCalibration skipped for file") self.get_metadata(local_uncal_file) local_ramp_file = local_uncal_file.replace("uncal", "0_ramp_fit") if hasattr(self, 'nints') and self.nints > 1: @@ -893,10 +920,12 @@ def process(self, illuminated_raw_files, illuminated_slope_files, flat_file_coun if not os.path.isfile(local_ramp_file): dark_slope_files[index] = None index += 1 - + index = 0 + logging.info("Trimming unfound files.") while index < len(dark_raw_files): if dark_jump_files[index] is None or dark_fitopt_files[index] is None or dark_slope_files[index] is None: + logging.info("\tRemoving {}".format(dark_raw_files[index])) del dark_raw_files[index] del dark_jump_files[index] del dark_fitopt_files[index] From 504fe9208e96f59643b84c2373b293bb149e3be0 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Fri, 10 Feb 2023 15:13:58 -0500 Subject: [PATCH 038/449] Handle the case where a list is empty --- .../common_monitors/bad_pixel_monitor.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py index 4f749dacf..ef42ec0ad 100755 --- a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py +++ b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py @@ -846,8 +846,11 @@ def process(self, illuminated_raw_files, illuminated_slope_files, flat_file_coun else: index += 1 - min_illum_time = min(illuminated_obstimes) - max_illum_time = max(illuminated_obstimes) + min_illum_time = 0. + max_illum_time = 0. + if len(illuminated_obstimes) > 0: + min_illum_time = min(illuminated_obstimes) + max_illum_time = max(illuminated_obstimes) mid_illum_time = instrument_properties.mean_time(illuminated_obstimes) # Dark files - Run calwebb_detector1 on all uncal files, saving the From 78865b2a9bb65ab6a2055f4727b3d2c91146d821 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Fri, 10 Feb 2023 15:17:09 -0500 Subject: [PATCH 039/449] Trying the full fix --- .../common_monitors/bad_pixel_monitor.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py index ef42ec0ad..4f749dacf 100755 --- a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py +++ b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py @@ -846,11 +846,8 @@ def process(self, illuminated_raw_files, illuminated_slope_files, flat_file_coun else: index += 1 - min_illum_time = 0. - max_illum_time = 0. - if len(illuminated_obstimes) > 0: - min_illum_time = min(illuminated_obstimes) - max_illum_time = max(illuminated_obstimes) + min_illum_time = min(illuminated_obstimes) + max_illum_time = max(illuminated_obstimes) mid_illum_time = instrument_properties.mean_time(illuminated_obstimes) # Dark files - Run calwebb_detector1 on all uncal files, saving the From 53a3ae0f7fb8bf5b4c28726cc868e0ff50f717b7 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Fri, 10 Feb 2023 15:21:51 -0500 Subject: [PATCH 040/449] Again fixing empty list problem --- .../common_monitors/bad_pixel_monitor.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py index 4f749dacf..262950a53 100755 --- a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py +++ b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py @@ -846,9 +846,13 @@ def process(self, illuminated_raw_files, illuminated_slope_files, flat_file_coun else: index += 1 - min_illum_time = min(illuminated_obstimes) - max_illum_time = max(illuminated_obstimes) - mid_illum_time = instrument_properties.mean_time(illuminated_obstimes) + min_illum_time = 0. + max_illum_time = 0. + mid_illum_time = 0. + if len(illuminated_obstimes) > 0: + min_illum_time = min(illuminated_obstimes) + max_illum_time = max(illuminated_obstimes) + mid_illum_time = instrument_properties.mean_time(illuminated_obstimes) # Dark files - Run calwebb_detector1 on all uncal files, saving the # Jump step output. If corresponding rate file is 'None', then also From 38b86351df28639ff69ba69193c77f5933dcf0be Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Fri, 10 Feb 2023 16:54:29 -0500 Subject: [PATCH 041/449] Fixed a typo in the save jump pipeline --- jwql/shared_tasks/shared_tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index 49c1e695f..7876febee 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -380,8 +380,8 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save cmd_name = os.path.join(os.path.dirname(__file__), "run_pipeline.py") result_file = os.path.join(cal_dir, short_name+"_status.txt") - cmd = "{} cal {} '{}' {} {} {}" - cmd = cmd.format(cmd_name, outputs, cal_dir, instrument, input_file, short_name) + cmd = "{} cal all '{}' {} {} {}" + cmd = cmd.format(cmd_name, cal_dir, instrument, input_file, short_name) process = Popen(cmd, shell=True, executable="/bin/bash", stderr=PIPE) with process.stderr: log_subprocess_output(process.stderr) From 2333a14dc4635b074f019bf27d11222b28e7e376 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Fri, 10 Feb 2023 17:01:46 -0500 Subject: [PATCH 042/449] Oops. Forgot to call the 'jump' script when using the jump task. Fixed. --- jwql/shared_tasks/shared_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index 7876febee..0d22f7bbd 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -380,7 +380,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save cmd_name = os.path.join(os.path.dirname(__file__), "run_pipeline.py") result_file = os.path.join(cal_dir, short_name+"_status.txt") - cmd = "{} cal all '{}' {} {} {}" + cmd = "{} jump all '{}' {} {} {}" cmd = cmd.format(cmd_name, cal_dir, instrument, input_file, short_name) process = Popen(cmd, shell=True, executable="/bin/bash", stderr=PIPE) with process.stderr: From 9f15ad867571d1b92dc4c79156a94f151d1f89b4 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Fri, 10 Feb 2023 17:14:58 -0500 Subject: [PATCH 043/449] In the jump pipeline, we still need to copy the input file into the working directory. --- jwql/shared_tasks/run_pipeline.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index fa79ff2f2..368ecb1bb 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -132,6 +132,9 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T status_f.write("Starting pipeline\n") try: + copy_files([input_file], work_directory) + set_permissions(uncal_file) + # Find the instrument used to collect the data datamodel = datamodels.RampModel(uncal_file) instrument = datamodel.meta.instrument.name.lower() From 4b02a0d0eb03409c737158e00dff1190685f96d1 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Fri, 10 Feb 2023 18:03:12 -0500 Subject: [PATCH 044/449] No longer saving outputs in instead of the working directory --- jwql/shared_tasks/run_pipeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 368ecb1bb..8dc7ade34 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -169,7 +169,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T model.ipc.skip = True model.jump.save_results = True - model.jump.output_dir = os.getcwd() + model.jump.output_dir = work_directory model.jump.maximum_cores = 'all' jump_output = uncal_file.replace('uncal', 'jump') @@ -181,7 +181,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T model.ramp_fit.save_results = True model.ramp_fit.maximum_cores = 'all' # model.save_results = True - model.output_dir = os.getcwd() + model.output_dir = work_directory # pipe_output = os.path.join(output_dir, input_file_only.replace('uncal', 'rate')) pipe_output = uncal_file.replace('uncal', '0_ramp_fit') run_slope = not os.path.isfile(pipe_output) From 698f3f83209fc58c63de941d82cca0cb44ff5da7 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Fri, 10 Feb 2023 19:02:55 -0500 Subject: [PATCH 045/449] Properly naming return files --- jwql/shared_tasks/shared_tasks.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index 0d22f7bbd..3732b14cc 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -405,7 +405,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save logging.error("\t{}".format(line.strip())) raise ValueError("Pipeline Failed") - + files = {"jump_output": None, "pipe_output": None, "fitopt_output": None} for line in status[-4:-1]: file = line.strip() logging.info("Copying output file {}".format(file)) @@ -414,6 +414,12 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save raise FileNotFoundError(file) copy_files([os.path.join(cal_dir, file)], output_dir) set_permissions(os.path.join(output_dir, file)) + if "jump" in file: + files["jump_output"] = os.path.join(output_dir, file) + if "ramp" in file: + files["pipe_output"] = os.path.join(output_dir, file) + if "fitopt" in file: + files["fitopt_output"] = os.path.join(output_dir, file) logging.info("Removing local files.") files_to_remove = glob(os.path.join(cal_dir, short_name+"*")) @@ -422,7 +428,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save os.remove(file_name) logging.info("Finished pipeline") - return jump_output, pipe_output, fitopt_output + return files["jump_output"], files["pipe_output"], files["fitopt_output"] def prep_file(input_file, in_ext): From 418397499291ae73ffde73370379a7be47af8127 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Mon, 13 Feb 2023 13:18:45 -0500 Subject: [PATCH 046/449] Added a path for including files that have existing representations in central storage --- .../common_monitors/bad_pixel_monitor.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py index 262950a53..06197c6c5 100755 --- a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py +++ b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py @@ -806,6 +806,25 @@ def process(self, illuminated_raw_files, illuminated_slope_files, flat_file_coun logging.info("\t\tCalibrated files already exist for {}".format(short_name)) else: logging.info("\tRate file found for {}".format(uncal_file)) + if os.path.isfile(rate_file): + copy_files([rate_file], self.data_dir) + else: + logging.warning("\tRate file {} doesn't actually exist".format(rate_file)) + short_name = os.path.basename(uncal_file).replace('_uncal.fits', '') + local_uncal_file = os.path.join(self.data_dir, os.path.basename(uncal_file)) + logging.info('Calling pipeline for {}'.format(uncal_file)) + logging.info("Copying raw file to {}".format(self.data_dir)) + copy_files([uncal_file], self.data_dir) + if hasattr(self, 'nints') and self.nints > 1: + out_exts[short_name] = ['jump', '1_ramp_fit'] + needs_calibration = False + for file_type in out_exts[short_name]: + if not os.path.isfile(local_uncal_file.replace("uncal", file_type)): + needs_calibration = True + if needs_calibration: + in_files.append(local_uncal_file) + else: + logging.info("\t\tCalibrated files already exist for {}".format(short_name)) outputs = {} if len(in_files) > 0: @@ -824,11 +843,17 @@ def process(self, illuminated_raw_files, illuminated_slope_files, flat_file_coun logging.info("\t\tCalibration was skipped for file") self.get_metadata(illuminated_raw_files[index]) local_ramp_file = local_uncal_file.replace("uncal", "0_ramp_fit") + local_rateints_file = local_uncal_file.replace("uncal", "rateints") if hasattr(self, 'nints') and self.nints > 1: local_ramp_file = local_ramp_file.replace("0_ramp_fit", "1_ramp_fit") if os.path.isfile(local_ramp_file): + logging.info("\t\t\tFound local ramp file") illuminated_slope_files[index] = local_ramp_file + elif os.path.isfile(local_rateints_file): + logging.info("\t\t\tFound local rateints file") + illuminated_slope_files[index] = local_rateints_file else: + logging.info("\t\t\tNo local files found") illuminated_slope_files[index] = None index += 1 From fb6de278d249feadbddcc3a73f588c17b3d36f9b Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 13 Feb 2023 13:38:00 -0500 Subject: [PATCH 047/449] Update landing text. Add Clarke to dev list --- .../static/img/dev-Clarke_Melanie_MESA.jpg | Bin 0 -> 444526 bytes jwql/website/apps/jwql/templates/about.html | 34 ++++++++++-------- jwql/website/apps/jwql/templates/home.html | 7 ++-- 3 files changed, 22 insertions(+), 19 deletions(-) create mode 100644 jwql/website/apps/jwql/static/img/dev-Clarke_Melanie_MESA.jpg diff --git a/jwql/website/apps/jwql/static/img/dev-Clarke_Melanie_MESA.jpg b/jwql/website/apps/jwql/static/img/dev-Clarke_Melanie_MESA.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2864e7becdc023c165d6a6108e9f49a3cad7a6ac GIT binary patch literal 444526 zcmbq)*H;tV6K&|dgH-7?6zNsEARPh;9i*4gs{(=uA4=~DH6Sg75_(4vkS0|~DAJTp zf}m6t5wE{>AMPJ;XP(YFbM~Ba#Df2;p5 z`ya^v7r_4x{~ZD_kQ2reY!eYM00(?lkb#Jpk5L9d!mkZtawU}wdC9Dk zLndHZ(aPdByg)7&nvlEwUC^p>@$8;1IIj&OKe9ssanIjn6{@N}XA|}a3r~b$_x`Q{ zsEPh1XCPt#XacJKXq<-93M>*~tES~q8zURQaOJg7oZW}5aW9`kUy0+s=g4iVj^U^r zfV`lQX3*tHkY&Bylp0_yp8&S_-f)ARG$vp}Q8}ScQeSl2q^*Z64h|Dg=FQl3)37Te zhTU{O!5uSYg`^d&75@<~s}srF_w%)Ce;kwG2ep+=4PDNh1 zW=t_*aS2^eF1V^m?pUlgZx|rDSk1b8b>FN~S6$XbUW}JSm_kD)_gRb870H-Zt*K&_ zKC#1ysJ1;|z@voT=ErgnU^HtUP;e}6KCVl!9}^_BMUAVdUpiKLYKnu3lOb3#J$ zAALb2*|3i=M9oT0p!Z~pm@(8)W7KfFN~74wop;F6rAXi}zyu)j)h?rE<&J0Z8|qz6 zQ^#4RW`WVXPIQFx1b^-110^M;x)T54{nEJgNJ-%gBsSo6AaY+iUzhd@c$AP|BC_2P z6e#MnzVVTqW`Ht3x(fvaMGKYsx{KpkEd*bGO2`}eT@pU9f4ew^uSWmIKm*(6p<){g zO(cA1=2qqbVNx5+Q{@1W47x7}_pypQ<^k>1dw&58RoI#c_o7;--2S){Gtvjzf?tV&opn`99oU56msTp;jc~yn0%D;?>yL(_EaYpMf@fVPQ z9(y*P2?><(wh|LbGg}VgYr-!cp7k5g0|lV_;4xYCR!qP2t87KtH7waI(;|H$RFi44T@98x3OyXV z_kOEO-VEd4$sJC<`wJlYej~H^7to4>>uZd|-j-EeG~%aY6X)LMpeE-v{^Cw1x>jwn0pSo|o9%mM3_nK9IZXX>{`F48 z-L^Z>j?bKZ(nCRX5|!99(egFt*@($T=pr)yDn6bT&gWULIS*QM)51XjPbUeV*~J*W zsWS{%wy&ZIX!sF+gyHC@k*>TyX&VJ0I{(s5LneOY`JBtixSy-UtVx*m!8!a@iH}p) zz(tDLHBy3{gJg8w1ES44#k=#VIE!J5{*0>N1R$&EqU$7^HsC^fbf;x+( z>H#^5japQVUM0qlwywHJaNYk&s49{XIM35QW;fdxGxya!G?vu$fKL@awb*LWQ8shD zxVKVM5h8nR)*NEvIKfpHoBl?-qlX!&N~vF|Y*Z~}l(0Y;K~5tpQ<^|PVAPAuG7S`4 zg0cb)zacbpwTMpu^rJ*eU9~20b)c%LJ9Kl%R=LnQS3)=Kt};uN;@G{F-Zxq8wU#zb zML-K$Y4gozdXfJz)QpncU)U=CQM$2M*@(6tP-R}4i+37=Sh{GHAtvWs7~aR(5{7ox z*(FEE)tF>sOc*zl83l_zGcDzeC&U7wS0W9L>$RVa@9Q2Gh&gVThFLkFR&V;JcXqbK zhfm4wj|}f@G^W-{&9dQM! zo{vkdaT5lz4TlF|abiiXrEWLJ{s)ke2LIzwGf?seD~>c~n4JQm+3kIxW6M4kJo;21 zV%8VbcMt&*10gevQZRZ(i(*CLlwX6)1Ki>8v&su>GJQ>>TFfFh z+uiN~21(_CDPod$l4j7$b2~`btLo4A;c`oaA+OocaQrBn^7w2>YoEiqM;*{OL);Ru zc;r&FQ77>8bql@n+c0y~EL~kOELgRs;;}JOq2~qJC@yoJXWTkAjs+Ue;*WNbIy zS1z-ldiF>cY6<`gL%hoebph&3g%8iH$7bTRi~Hv-)WoLR*SFsL=bKKMu%`gK8d?Jy z=omr37`p%BKJyV-^LrDAC+PoNSKfL~z{P3y2I`u>-51YK?dE-uctVVjRl-HSc>nGJ zYwe~d-r8*A$o1UB?EcfzU}}4m{mEB9*RHNp7*j`ySZdUdcB|_fN#XbR?T{^}R=lRa z6pX@yfEW~q9-U>fFcG-c3j0z26Nkg#BR6Hb63ms4eWg$E4?f_GjsOP8uC-_^fCtcF zO}ZLlqXVxLvp8?SH@mx+J#cegoY$RSr#Qsnmz;P=X4P)*J)Rn*XmlcFhtGYZUbl{o zmT!cw(`xstcKE`(;ErcNjQw?lSjWdnWM@964waJ)nW;5fRP${(7wlB%E$0ohxDyj+ zM$NJQ1=OtDr^b|9VD*>)420x)BT9Edrnro(ic=o!9pC&f>n3mM;}e9l+L+H5kq1eT zfLl4l319Ow-E14bE957s`V(7+5m%_N98q;f$2P~B?pFTnJ< zc$M1i4<7zhm4pZ4DVfiWI~h(p1G?+*J@nCJ6JM9Tw~)Y$@D2}JQUkIBDUPP3D(j~D zaHk?sc)m>rae_)BVJ*^hKjz3?yG*W3QgEJeMZGp8Umli=z%Gi9Fe9^H-rll8zFVhW z4{SUYp!wMI$BU^#h|=xNK-h0B^SBYG8ffI1eI;eQMN{iv@t2J<92@mX$1~9kSkq`dRgVX z%t3xs?R~rw%TBE94rXHRS>H1f&0pv^?cNsfs6%tB`{(QM?WwEmB_=`!jis`nS>G^) z-u|8{OT_19Q%T<5N_ci(ft4sn>G75bX$Xzyo@3i_0j0b%La4jJtj4c+4fCW)(6Yu< zm(dr6kp^HLBm~kAkIbaXvea-1M@w_!icvN^)=uO9-FL-!CCOW(U(9L6Hn{7J()c`j zy~8fB6=zdNM$JaZgRQM7`lqQ?`694GQTznAdr z`sG}o%Gl4EIGWdEy|0CL2Rsx$F6#}7yHlw){YYI#ia(*blyDs}`a}}GsYlnJ-#j&o z9Lm2v>12X^j$@q41vf;)JL;*H>a8oSJrkNP$CD+fSQf)yG6rvil#egRG)(rHO4?R}?_vFejL;o zK22~uH`ldt{|`%|T`<$}qa4crm>lC67kAK{iv6`SFlNrmsrT9>^;a5|;2t7EHbNQR z;K@a z)#y_*Hdapp9Tx6qww#%{J|RkXS{j|(avEDT-`c$6SZ!AUtV3aKOhFGNrwQ)Qh^z|n zxAJ`>@oJtCB`}76UbZq(SG2&P__RL)BR?LpCx7K_g?bJk96bBqc2q%@V1JAvagDj7 z^@Sj6(5t3O0N|_E($$1&%6t6!`-KtsSIgo458wH2UtZN5RBzC?Dc?Mh@mXnpI!<9J zK;6h|n3u~gpBCb0)YKpC=da1+v7My( z1b)8oe2&DP#W|z(GCxiXO1ELvp#0>~fq^W!<)a7O5tvzFKih^st9j zmz+TZkP}^$$HR$p1lb(SM_3-zpRfI3g#|jD22$NXzDNqZwc4UD`qo0oc|I4*8=S$d zep#2dd?zz9BZ(cC%rL7Jnfd@+_cF^WwRUEX?7Z?_|(@how_n@q0zP z(*D1cggc<6O$To-aaiPCJvODz7QZN3_bZ#vNklu>n?}U!?GEm>OqHkIr=txAFqNr* zfs<+>*4D+?cbUM&ph!>~H2*X4+&_l)ORR8JHQG63{1?2|;-A!S+JJdGy`4MXjH*z9slG0k`&FB%kw&FSc8k}5d~38Qe7ep9Z1Ebq zDhRJUB~MGD+*gTFII8xE7KHLs3e+u98SrL5MO-_E7%Mmh9f2w=tCo!8*##Wg#c=AH89maTd!E9O zQ(oFs#TyM{W-_<(k>#Kd$mY!<#>?y_P$uC!u$3?ueh&h{VP!VK_in_5A|{0o10Q3Q zem)P~stopXvczYs$$xYEy!-ZPrrS~MsW*F|z3h9lH%oh)BhpGIgR4ag3ylg-c(e89 zU8jwXCsp)-*i&)mxRdOU%!abnOSNd-tP&=ymW!ZCE|JZz*YdYQ`{LrdX6TsIh$cr` zq(zg}KAQ_AP6jI*?V?Fy7B$^lVD_^>2EAaWVy@{m{59A)cJV8EXg_~G?tquZ6asDJ ziIMM?W&H8PS1*N=<*gT0cSl;k!++7pw3>l%Vgtpt)@-5uow?9D@;TEn||aPOIabT z@|?3$C%&f79E8AXV*>RS=d3Pa6VaXpqkSoPu!UI;xcnL3`$^05tYX|}@YxN#!O3b- z8PXnz{n%X_IS_=`KuPYCyNtxyS!{%^HyCx9TuMT*^^Jv+_5*rQj^U9?kms`7{X$== znhEMtAIEMD1-wmHWWVmwagx0?7RUIdh}sEZZs1;eLY4tIit85Ib(Gvt*i=y)gHA#s z5Aa94c*sD6UrEFtQ@`7b;Wxc9kWiH;#pyBB&s%M4{DeFSQ!9XLY=n46QTapN>=*J0 z8vWn1iou;deYL?bC|=K{!L$feL}uY)7OLu^C@{Q3g;EaMBd3ISNtjWG=2dk;jqZix zz{^htu(rg%iGlV;n=h#-Wtnvah1$2z9kTRKr*_eqc@QHpV)_M?)>5_00X0!S#f}T)B zFeXganI*^T%Fnz6FwuK_ueb_IXzGumqimS%wPD>WoRpBz>MZik^`>x^Zox-f)V0dl zvA8ceee0A|yE60_5WBtQ$4?%7E9S|RqAli2!Dj(mFg4#aR#x_6@3miPvGOqI(1 zrNs6Dxmh+8iYKXNiGP|+AyC78bb^p_Sl;S>tuV3JU6L^zauc4~r|m#RVo%6nBdL)8 z?S2dvb@XNspU?lWre5KiG5O3;fByT!Ng<5v-8Qbicd?ViYg+rhC3w<##6>@QzPQ+J zaRl~8XFmML0}D@BWx2M%fu~wR528(HJfbR$e>uzisFp4_6iv-t5( z_0j&`DQyeolDFzYqyxi~si9v{?9WhdY$4XjfTMMb=(#jI#$s3bu9uAlDrx|VlpzL6 z$vy_NeeO=R;6UG&rhV{30W~ovby{s{44EM*0d77{XsjPzO3Aj8d`n2K8eU)K$`M zm}+f?t8YjQiIE&J#tqMH4*@JcEF?3Rn_`T0A60Sw|) z)1%&K^-gO`qCZ?P@N8*ONbjJ*WFXy1sOfz>`>{5$HU0P5jCD&%l=7Ph#%vKV!(x?* z*+hsM#ROpM6DB6%QYRHVm;2aM$s$5bI69#)ohfBb{_v0K7j)-Lc*upsCa#k+bE0L| zIBSk%fWTd+xEbsCUs3^IgQ4|1l9SL!BxN9i^DaD%{S{aiq`;;icR?&WMf;JTUw_ZU zCB(bNq*FnGrxA$f=PjHG~A2(b&3lM3doMaRU9?^dZcv$Tqyo0VxXEEWF2HxD^|6)bt@ zX;rZXPMb!pMn?<$$UfE9UbaJig~})UZu?X+g@!FjwA-4Ny-?^k>RP^@J#NR~?d7i` ztVUZ9L;mR>5GjRrG*p`7PLNu2OmA;>U0C_120|84$_T#aBxsPW#ivJ=dK^;r?rf$$ zd{Z}T`_iVS1akGGu5<%2=+s{0?rl>LszySU?-RN?ENG3|x@IAAAId;#)r{9aX=5yV z{pBQFj!2^-o=EY0iCp7QaAK-`p;FRxy;Iw|W^+tfj|quT^{o+GRQV6ts(f27a}i99 zLkm0YOF_oOH_OKx*IzQ-6)Mh_Ia1UZi2{|iEMjb8OBIqy216lyw952 z{6h6(`Q$utpQl9F*oi);nuByQ%(V6WMCIA0U_m~huU1Y))34vH?y|#w#t`*BGDxlJ z<#A2<#ecHik^K4i)k%l`#PCB`26BmS)kHruf|Xu!YgCH7E-CW7VkN8oBXUGXHXcje z)Nt7^R}jIO<nv^^{W-zqup5Kg;2SP!$WW~q`6d6uS=F~AYQ zY1paX)FTIOLVo6DA0pA`^`A%rkEwREMic`T1}19*K)Qrt@qET}aQrx0h4}V;9oc8o z)#ovpVW6!bzTLwKfn3KS3*S!;y-LDy#R zYt05k$gC_~w&gFeZQ(Yu|}t#CU@1GShkgWeg_?>JN1_%K19JOZ2kgZv(v*Orf=sISvh(v#f7DOo8GL# zZ)gQRjAb@brPkB0jC>8|$96L-hMgdFg}P$EHp6D0=G_sN70+oQ4$Jc;?r2WHW_8E;>Ws)jj|j$|H{| zrcM8gmyC~1PkLQjh3N{~L>M($D^T1H`G?%){1U zrg-BH{O3i~H{+lywqW@_*U;-%5`VCSQD&ase;3c0uf_PUd>rsNX*_60c1D37p$FGHXL_2jzIMNAmV|oG>GO|MbP)v$UAIwCtKEj>G1X`+1WN?2{*{NEa7+<*a{`E4q zUGu2?GZ8);A#9Lbb-Iy#SG{AL<@b7HMwy&|hReR`udHdO`J$6=Cnl%(srJwo0+w!H zxdmGb)03s+e$TtqCfiS~IH_)B+th&Yct@zWZ9oj%`K;rRqsHsT5PObmmce^7?aOzB z`FMZvdckBD)%WlfOcCrB^rcqYjIF2M8vULn0-tATePlHH>fwYUX^m>PkgpsuBv?nS z#O?qiM4EC$Qoy4LiS2`#YE{pojZjYq3GloY4vb2_sGLH8N~v!RqOGr&=(Kck13>?2x*yYaUj)o@RPFlV%`-GrVZ zZ?yktQ?4Z-PezSRB>nl#^!|Vb{`JzJK(rHPuEug~ugH;yQBK7Pi~kmy;o=u};_7y_ z4sR!1i}w}otX=s{pOE6@IrRm4yVbrKR)Fdf59}OZF#D*?=fadHP&brU&XzNlnET1m z?i~8+`XCKlVcD)=Z;Z^v2ohjwQ4Q?46vwBq>A>m;R<&2L`VrkVzn;6TFnw8Vo(Vpa zgfKq%qPR<{cqSwz7Gl+0wO!@(7tpsKnV#>}Oa7mYS0UOlTK^?27$Y1mPobQxokPY= zdhvb=d}ZsfqQLkL&Fd%zt8X~}g2TjX=dBqK=e@^_82-^6-MN%ndv#H&(vX)_;k;IM z_xRN}K#j4^wh2>5@P|RIPv!1)qxFIrUV6py(0j-%#ug&l+=!f-6is=?N4vf}5^c+A zUf$+c)C$=%2Gw56MD_(|!!;Xh zr7(8|qQpdr=zr(S>i4$fHd)?rNa#nrDwg;Fo3Wfb*it={Xp}@gY(`NSE4WAcT_jm0 zL4n)K8r=|`rIXWR4(e=YTNTKewjDuS@rd!?=<6!EOYw2cCIq(bq_rRU0W>JSF@Y%KU8ivJ4` zLylEw*~!{|CDxW<+m@shk9jfuYgXZsdvUIEvqzBfO@)_xysCbs;bO2|3XG~y9lHPU zJ^aP%&8DoQe?^ba1m=YU+G~BLy{OXhN0A!L-#*x0dC-qKwFlqHL_r^r*T~yF7Q{oY zYqt?vF@3y(Y@QB+)=;_CE}=RdAV!S`I`ync7950=clW~dGgKe0UJ!fNza3wOCp3~! zx}^@IUV2+$Gb`6ZO`tDpLT>~Ut~_W!EM-Y2ogLPJ$Pz z3gVv}Yk=X$>W=_c^#-+8Xli+S@M!iUPc?(>#30#yb?9-RY*(b>X-E@UP>#)b+$&>$ z_*k6zWK)M`##{x&XG(vwswwd5(KA!zPc2DqO@3%}15@2PL`QYim=Y$f;b;i7C(_PQ z@8M5GcMj-?Rmv^Y9+PMm!}$1(4~^<5W4_6erP};V4#y!hZ?Z;q$;zGY${w5itj4u7 z)U`5idY?-6)#MZ~k$ok!a-#5&d{cEh;hXz&qIgXby||7+!HN--o>S~#FG8@s%=N~z zR|!*Jth=wo@XYmCN#YN3pVgqB1BVd%&{&CW2gmiW|3bc|uSZ@ho_czig{QBD9*ib0rX!?B+f@gi8gBVUU`b6zd)X@(tWTWFX+!atb$t%A zTLk}k+-MW)Y&sDDqFfWO{6F;EWl~FcW0+Iq=rJ`q4&Pn=M#Q@f?uoc5xpAC^T}1bk+GP6% z6r9JX6a-+!Cka+#4(;# z!IsT{BwQ+#H-#_CDZ;pZrOvt7vNoXUG=Ye;5`(P--<~gYNfTfR#MOrbyqu1Cq_#ST zbMLH=2Fgd>6)?96KjT>B8l11-Yggou#epB@o^4rS&tJ-h?eZv816K|p1(-^4M?O!KFKe31>9XtF!M5fkE^k8M>k zkUS^V@e&AXW|m;p9b@n-nMSBpnVq3#*6*dK$%d7F4ZcRO+DZ5g$IU8cgHV=wDtUNm zJmSTFBFab2W-8uG6OsoHlO6{C1#s;v9tkKHh0gAKEcg1@fh*svrqQXn%ELgf0=gD* zL9&z&AOCdj>_mK1L|t;Vu^}y>q7C&{yoF%`#1u1emO-N(Qcbw?K4!9d7^ptC3fwMA zkRBhcwIuzVg`SSMsWL*OJVFj_x3QjhaPGBO+iz^YAP;e>=C@;p(W;R9ll8l$0VuH~Qt>HAykz{Sv zw}MKrRW~6arQM4e&Pjd^y)hhCIh{AS>0b$Kz0li4PlPG_l3r}ReMdhw(WoTR@zr(k zLtDNb9mE6s6jk7x(C%806+ay?{VP_LwScv=uDC2s7HzdSR*JeDx;>2~LNN&|ZF@Rg zu||Re(DnZc+$#m;NaSSYJrp3GDniazSeh`8OJtM$=KYuQ{Y*nYB-e|Qh$r~nI!=sL zmW9E+jK(ir^BucIHzk5BZ6Z9tZkKj&0&rjl$9>{mB)b2jlis?swG9<04i;*rg&B$0 zi?N0p5)dS?a!$IL3?3*-I@fc%T4!{NR_Gb8e}(o_pxztgeqC5LD&+7sflbag&u~Bf z3%K8L7M*V&gLu?#rDRolJI*dO_g`=obj+aP*8MHnk@HRPzz4*>o_v70L z?<;5vDcYB~{YN6bmI$N#526DuwbnYFnzouYNi}rbzz5biY25XXPx!iif!fZ-d!`KRw%pV+GwXwm+Y;l7lmMOwp`xo-n}2~m(V15ySU|Hr%@W!EZT{e$&LIKb#6-v$xNg_ z`>(hy5*`-Igz8onTGKE?9hc~p9AKrgsapXMoi}N7ZMow07T_le7g?0M}rjHsg zv1=bnGnMBo(h6;XWbM-U&mZhq&H{49J)ad2o7VZe5sFN;RgLqe9QZmlp5T_rbXD6B zI#hsdgVppy*VO~S7eCeTjIpOq!c|rf)cXTswuC;Nbhoh9q1?AV`ZhMtrc|RQf6lXx z>p4d(Ij4$ekT$dhWj*+?|2(vz#2G=e1k@L{9{(6i zxAHgv#xTsq5+o|!PRZk;4$p& z#w}U+v+r&Pcha#LJ&gwn@vc^ZK{7c-yaCxVttmSW8s_@TXQyJ8HFk;pCAl9gE1@po zfOIWVFcjz*P$-fVRt;`x^A9P#QbaEfiqEyn!t8uLt*&aZwDR6E;&n~a=|r!!CQC;j zhfwBP$?J$%u>>r$apel;@}{th_~6U9mQh;KO>|nzk34X)ibH~+w5 zkWHyFyyuze)fD^EzjP5rCxULaM16yg{nJkhJhX#n&>TW0v{!t4g1#%MtC&XPpu6oxDa9^ zq1A?DwmImJcdbL;8p(I>gWH$xYbdGwa@g1!gxioGphwXOO;VAfZJm;&CqkVPo}uAR zb_<+K|E(r*>N77X50x?*eP52oYxM>h<=9f`87z7vVyAw~<`O0@l4D0dqV2~d7OvDe zW4SAPm#+Z+$4(X&ud)FGKatEKhq@IH%S*mLY3z@FTa~eW-W5_w%R2jRrVBCN!lXjX z5~`9V+?fa?j|&XgFA3|Cut;fDb^c9FSSWK?AP=++3qfbA_e;v2sv2eJLGuVYhI1r7 z%cKZf|9D}UG@A^ZdsTUppP{Z=$byGhLJWDEnrFfREgp*9v9`f+#rR;e2&G>d1;$Rg zwKt_n&o>&>zIp*0O=$CPr#J$za4={z(1Qii9M zwirjo)1bOpjy_(-UX1C?F*dvm-^h$Mhxd2dOAP(fQ#vJq`Oq=o7;`qXk_iZso}qiN%Ru8dSq=On~^Z^ztQ))}hc^_?=Unxo`07{O&B}~YH-CSVIcvekU4hEJgd_v&(bOh<>S@M$ znWt+zvnNugK9U@0vt)ZI_JkUG{ssxdDD2Y}ldOlV;Fm9hTu1Qdc_lc?d*zb--B7K| zyYmvleUo-Z3K8esg9i*liu~wrqg~zpPMhMaENt0m1Bv)#^2;Fr1(Dh#Jmu%`^@iJa zrgV>xhmA`+hu|HshbVX^M#+E9OI>k<^khAO@W~!a)R8EVff;LWfi4i zkfC)pWT7~W<0X^DmmxAzL!Ji&j&^ldfiyZ-^^PX}&BzYK%*asr^qRSA;g=UP$iaU< z8oi$s*ex7(z54OMOFcTZsPaF`V=v1+$bZ6XW0&Gfmku8xEC)t#GO>q7lwX#L(NhjJeY8NUv(=e6Bg zV~$fWC}hXY5FwP;2CZl-r~_zhsP z9OXFU1n@on*+Y=se=qsZzB7s`!l^w-iCx$j-bOG89=N;!jgLV-@QCV6Qph^SS>=*v z{HY!t;_`lneInyk?iBrU^k)!o3OjcA>$y$tImb_!7-n|BordByvN6}X#vZlGoW68M z^)-h_OxfdBZ&sWra+`v%5*1s?+M6}wVnXOns_1=h%<>bbiTdr#YR$bYp_w(q!9K*WdnqUukV`Bdoy7 zdi>iVQqd)fi$NT~Wlbg$f~l39{BkZdOH#un%W%syhxQT@>Go_{o5uLzscX(JQ-C z4sz|%y~4`@@6HdxSSx}?$_kYm#ogx>nzMA+q@H9G7)dnop;fbwvAz!*v7Gk|Gqeb0 ztACH0VDX?Azrp|JTRP2hb?Op{Q;3e-V?p=zG|=IH0V3(Kvcc*x59Wm&@bcjY|2Rl$ zJzv$7uE@%pE9f)eOl&J^EWy#Et_FqipH)pKY$tz7U0)wTijbxp+BU;^3kgna=KQ*=hkn+Ef02`+xL6)&{IpHPuf&y97|O{Q$zIqF11 z?u)@2J0>k04&3xfwdjTR5X-W3G;~%ye*uCoK9;K<>w8NavXxFhMxA_t_4+}I%v#Qj z;DakEUt9{YPP009g1HX%^+gSqp9VYKK-Tn(-%cp@^8*P`O;@Y;Z0@^48H=^u!f!mp zT?mEAGoWn+|Kt@ zbgGH0Huoj|a;DZO=Q zzV8o`des^kG6QPccm+1gPdJ2WAS(5G5b@OOjq4Pd3t_FT2pfo_kbqPWk;R=Xrwr4q zd4UXM&CqG{xoWSE>M#9V>T_XBCXn`ds&V>Z;=gA4T5kO0u7Zo0H}?C*Vjl#XE8xei z)L$Dz6(a@F5rsgVo28q@KuIoq*m&M{B_jZn(da!QWR)4h%r(}&fxWveyuzoW##ApT zm&fFugduKdl}H*DBzE&uNAcg^li7bmNtm}=?kk+EzaX0&eO6O4J|VP9VcxH#FE%t@ zdt4Y_$^8I}Vt zmwsyU3lrxMST?eDg}evAoDh1Jp*lB~*!LzfR)RxZ^xp#t>~5BxU8~SIaRu_^RZXTI z8RtBg=PaBOULZ_xs=>-I3)cK#~tR4dSk{0d^RMHfmwDM>0Qp?8p$s zDMc!NFt^NUHW2h+=VW`CLPWH#_I>w7u54t~yeevT!_ka&U_pv=GGYm?AZ0)G$vQJR zWVhtIknespPrbs)=`NlPsMi(ZDKjWCHoeXqQP!H|rwH6f2wtx_7}O)c6*08^qo4!Q7ft_FMBO zz2wJj9}ifbA$$f=1kJ)9PN);;;+K};{uXr#6t4)KC<}Aj7S#SJe5Bfz@P74Zqsd{D zc2;`6TC?}}Uk5eqH@z^A4aAL;fr|#ZM=qm~V^bYJGvZq_Ip9~nST{NDz5UPj>ZP0F z8?pdnmV>eYP}-c36qm=u_WJKMzxJQe8fg2=F@APk7RdD5LSF3xxp?VE2=PII&V~^G zH2s#!j`C%<^r;~LAwtUoDhrOw8jp==Kt6n+hF3xpO!jOCpXA~vC@|yOJxPK z668rAO@0E8;mfFH_)IfO0pSIO(Km2(Sl~DWhunm{G~S7Q%m{q>N1P6;PiJW5c4GEm zz6bE_Vv6HO_;nR$K5|L#5VZ4kz_ac#{{eZQ>AT<}T0Q?N<{v*&l3pP+9^HPitPOlY zW&hbUTAwg_=%s-bT-pHyeFeUJ;Lb;4d>WuXa##qWTwxZkU+@gTbyYz=U-%;L$seej zQBm@~_UN?o=+TjB+2W-qPaZh;VISNx4|yxX zzgn1h&k>2>RY9}3tlVqtKq2ZW4TVEqk@9=TUM7Eu!C#fF!de0j1)XYfCMhbt=0OLh@V?wKK5T(?fomFMeTmgd(;?pm)HbZMr^+HSoIUOaGY zcIlTH?0sbdD8lr@mK+*x3Z!hXSk?-itj%ph3yn}gHhwobKpIUaDOW|Kebov}k~!$p z_8-+xqHOe}IRr5{we%3!pU`Ka=BU%R5dt}qYR~gvCysg^Xji>8mHf==WxsYpr|Zzv zhD2$wfq6lw$J0L`Y$Q4aQcoV~u2wh<;7TATaxpQGLpaLU8V3CG?0!&fCu+f90+k3g zd?wPW?4=^eKqWKmUZwW)Z9-S%Aa~gu_x$}m=E~7O|K?3skw)I14~80xI|S2d%tbG9 z*y-56F$%3WCy6(p6 zI;~vA(3$H#ezm%k1e>||N!CNb5{ek-S#XfIFtLh2DgGUAqXQe(DVD~+N)^^BF< zrnIm*8@o=qi=XA7pPKBZ&dUGcJ)rw`bHP`E@^X%|w1$D1_{EJ;be5mOZpI zy^etUCNi?*^h#wiD|lnhf;BvyF#3a%{I4c-obROQ*K;SwXG6^&evuN<0l)F;Gzhmy zNGzM${nk(Zq@6#lZ<3ta@B(`2hxH1Az|oKF=P06+44q6SdI2W*$(;QswlKv@%WQiC zQH)0=+q>$_cBHq<%#f?+Q?(?v@3iLD$U`2pb6pz^719V}X7d5<2oXE_2#SDU#q1uU z6QK&yB__{>eHm#2?JM8bUwTHKug@~H7#Ik>=%&*XlZCO?@(4)9fFk8bX`Mo86PE*# zZ=YYgeG}Pz8eHb5hreCdO78Vq|XqgG_%5IE)rAdv3zqw)fla`nx;7$iaIG(XRrF(fMUQ zSSm|$`$R$N1e?%WLuqYN5q5fAt6DpI7^tZvL!zAk?jeT2jxwaIS$hPCEH<0UyCd1( zetQZ(Pv5GJFwQ*;b++7JZK~dwk!0y=f@H{*4|Wm}(T_z*^;I9f)fyUg{w36@>e%;b zRn_MI0A)a$zpq}MYnOI#N~?wx+HwIgtg+1ICU~jyisGgF(KhrN($3we+fxe#7^JmC zQKd%E3~Ctp&Koj+#~dsZ=KiQ8Obe{Kni_3lfDi$eA7So}SSqj&t^t$NCTA=W*5X zZZ?P1ty>=(h~sh)Ie3Jg-5xTlzkPnif&jkg^zL{~ zueiBq*vVrSmvrDzqWeOUADxV;7mVfkafthd0rzI8ZUS6WhC3ousETQT%C^M!IFG*oKV1fKxT6wry zbnNe#=&5CIqLP!L3b5-We9TmmR0E0ghJSvp{h9U)~)-fM;|UB zqSZ9(nL$-ia*AY(5%oKrbwF5Bt5cSx7rAPNY9C0l&&WAr-|1^5W(4Bxs#gzl z%k5f~lCn~`Kt(cLB+AkuYH7fhVG<`2H7&GLUbRH#w=`EvkSGyYP_wxLsKYG(q(Qb^E5`m4wnf~VOgLnJ>2 z=T-f7rBNKRl|ouNiO@YjV?;~L;P?a*cqif7U|Uog8k>r&PM`oFlcqAj;C~JYlQSwh z{{Rl&j`03-cdOc+LC$9&yP4BXILJ-QgF!W-{e1=KSS3Y;O{T6^nV|y=n0yp=8EkN|9ATLG&pkYm@|PY=JBZ*nVL>ip$Gv(%SCjjYg>KuG&rj4b-sESEaTP#Dn>B z!$*h9xepG-=|?iR)HM#BP2izpgRv^k=RAG;c#(xiW%`ytEZ@cueBQdTTSO?TObNcY zf;C8^YeM5wYoz3ha0t|0G=@?LIGG@lBLGO@Jt~cJds@|~6G#BYxnrTBwG}Jbgkga) za!TcxN%hT-Z;xHMiFcj{RyfB_c<%_a#q|FC*4{Ux&GZSkXeP0EM%{t7vQV8x6xFWv zeamD-d|~HClU1im-m6ujloqG&SpM##f2^jVhkoBK^%^y*01kW{Wn zS;p7~JTT82nWNPmzOwS}n&)yZo#B;cU)80Xb(?MkQaX-eC_9zpt`xdhRP7i0HC63XtlbPZ< zd36n@YOA56KH2?LBWU}`TP=kHPK}+d1~&zgn^meNnR3PDWR#`|3@0twfy$b_dbhV# zE=^}!dI%=ip%Th*rO)PQB!Q&I-WMYJ`_YSz1|h?Sio}zA8Y;S2Fou=hifDJ|wocxp0ejYQCdWP(3=7mq}GE zA=GLaAd+z{8jYux+_ksEbZ;7yHA{QZK}B5FYD`Sj0DPL70o?Z+!i_n;Kh%dpzMu2@ zV^@)M{{X5zB>IA?oC|!S%}X!8!|kJeS$jDyW&4k5o)nQNXPmL-wDnL+9o}W12oipMb@%;0G^ZI<7kxvaSI9!jWxVf`$sGO6C(G!~d{{Vr> zrCM3px5iOZXD%qdu&TkO!n1F*XG?3E)N0sQTJ;i_pw%g$rB$Aol4`DInzJ^7&;epr zGF_EA_1@jBuPtiYg5tTQT$V};P=OR`>>8pXGYHCn8b};h%j!nATY2|RI&0PD@}8aG z>(;$2Tf=$u^fy_~LY)a#JzHR{p+<2 zLYipD(Pay!SR)49$<#eF-bOf$d3dt7t#fUn;-a)udZBI}l{qB-?1;b+vz(nJq8DU- znEE@DgVLQ=ba#v1&cChb7vAJG=pJ2dT#FBj`$p;YwW=;svqwiY2|(Xyq4IXAnKE@;HEwrMBZ zD~VM6Q_hm&k1ghv11P2g+1!Q5cBrDAHW}EK84v+IO2SJ0%1tj`n+9d9lp4AcKvQgK zwMJ(@#JPcjcHB#CS0Ce>mALIe+vU0Eh?+EOR?e%RYh!CN{krK&f%<03xAg4MA|WP7 zdDm2a@K7FqA9B_FJ%*6g8L3jF1*(9>s?<;d)2UJQ7gHsm00sbPI#7$1sc}Mr&_1n0 zR;yJtBAB5_mJCLoi5Ls1rL&Cj+tlg(3tdms

@B%Uo-UaUL^AQ-s#brCyHZ#l?p# zPfDyU4arbL1V$Q6%)lx| zE1IX5)1=f;*1EZ@>s5=hQde;`6Hz0mDNxvmBn%P8PyKQAi&rz0^!u80vy&y-Ba`w& zr$c9&aok>ds=PY=m+}5z#ei`lwg9hk|`pzn{x99kve=vAbwVE)~#ECRh?=z zdP`eafFmkY#KS3GtZd!#II>$@QA-LV3@Frlc&G=c9Rh|p%E({_K{K#2IIMK@p}1Mx zFNtzn3f$z}$8nQ#-V!b9Q^(?5TT;*5=dW^s%JU~ES_lhn300CO_k}fM5))9cR<}o5 zPW7rVe(Iu|9A^r4$eEKMkRi0Ic;2p_m8J5WfbCFx%S>U8#A)9WW@Co?Lz{HTwXynh z&U!;nhjR?xBSu>{b*NR$)wQJ#P-^kpu6)%h(+Y=4x)9*;8HLMMWAKr1Xe!%XQKqd= zD^MBMvOqZlAVl}=Cw>DKHx4RYCCIp@)oL=*MTVrJAqxqRH<$)gU=YCLuTwad2H#RT z$<6q^to*ces;3@0!dmpSZ1Vcyv_Yq1J4(3)M_r+AAnYQTH9j(;IquRCtlup<_f`Ei z6sgmu3P>c@eTt5(U;=j;*kEyHwnI+Z)KpUSYwAW|A!&}<`dD`$L?0VLBL4uzJ@5Rh zd;b7W<9`%-ZTv6)0E>cu1i!+ctACdcfB0Bj#O2>osN&qmnzM>OPk?Rtt(0mUX2P^YWFEo z1um!|LQbLyKBF?3ifzJ9dtSA*g-Vw8iE~z?tpzUnR*ZGF$zp-Vz8`W#7WbUIq6+xZ*@-Iy}ip*sG7~{%l+hriUF}dL2nr+RxS6rKvWF zlsMa4DrehJuP1Q*hCPXc9}}`ysiu`im8VnG2wFz4a?75O!*DfjPm{x%G$`t)bD}HL zqv(2)A*n9PKpjv{7|TI`03B2O?}zhEo-bzDajkolad?frF&%qUlzLHQ6r1(y=OMy0 z=m|B3ky5Q*Ay=E-`C)vDy_*_N<$|XAXX^$w!H|1{u@Mg?yQa1z^|{Ky=`wH+dG-BIq__n-^7wktjCYRx9(pt48NokU*2+~zR1~C(f#?z>91%I;80WA~D;^94dVgJo|i)h}_Dcz1iF3IJx+Ia?G7ZUDAzY9coN@H}12MP|^1@ zt=uBA0&MyB0r3jjl}4Qm2i%ZU2?z6Oo|uAFYKSRS4Ej`4si=<1K~^KP zboQL>iQ`Q9<$h_jJx<~sM~(hL&hh^M$~?cR9Ld@z$U_fXX^qk!L0BTceiD`SxE>(!PugeP6N^31nqE%F2swYWi3dMr=lQA42e7C06H9DizZGIWG z$2j*m=G+IA->t@Vs-7wpYaeG-e3soeM6N5}u2pEi8ohPGZ>f?k5g3vh^-4OP=~A@U zthE59QbM+(q(}lxfI-!rzyy=SwXPzMuTHB{qf&?tpfQ4?3oOhTB*`K~$c_m;Kj(Oy zD|@JKt_w7sWW{{ua`&TCs_*uD3C^{^}EKXFpE!Krx$9L>6o^YY&u(m=i+a z(MrokhydHA5|uJcnUy%wOKc#R#Psn6-MxFNHngf;(*yfK>jy=NqCQmtSVoj?uvkXy z7{KClp*<+Km%v%ed7WKu2XmG29#N&MW0J0|oBC_3q+kof$IoeKRKAgvl3#Vzpd{pl z50gI`>}k_lPSq4E)TDKma730dq<&BqPGcrU1hn^;O!sYXMb@nfOp2N(2TxIE$R`^B zH-iJKi#|%LeJ%BmqC1?|US52Hri}r+Ud0WH8QgnAHTP*{T3v-3DsP}4A8^*g2IR24 zO(u%K3Zopd`h$9!)rapIiJppssbt6mK$AQCpl!r=6q@qZ=9MO5+BAYjg;p>@8OfOk z`gh|x{;l??YxQ5MeA@z7M!g8{%74?#%OFL0v(B3=71^H^Dk)DD1DY(xD`-=6{!#+P!rL%Wu{wA zK+5Xc%;^L{jZw4+gEBA)*;m%7LY-4@6xyA_JkkwJK+_Ei9YAh$$R~Y71H(_!zNT_+ z9nLzxuI(Jm&F*{C-Z42iHLO$Le#bVn>l}WtQ`oSv7EBvMMZ+)^(zTOCfBRSC>kGe5 z;_*rJt6o#0v^0=I)~Mc@4FDBY1z3@Y%#4K&>zAc_ep-;;jlFjjiNK|Apj2iHAd*NT z06)S8I8lD4x=%}?zf@fh;`^A#XoEY3@-3UXI@7g{&u-{!ea?l8txt?5!mYt3-E@Dd z3$#S`0r)>PvZc#8#ibA_I?*<@UDW!CUDdVJZCEB)5^yI291~-7x0W|=N!&wKprF$f z+KB^7`x6F4l0GST8sp$!St_AmLRKu1aG8ow(=)Zv*hZ{%P9xcO%|z6 zTKg_LY#*x}udXK@9cD zLHA+8+1-DSi|GicYH9(Htw0DcN|^xUM3I~{jAJ|(ay8Q`3~Hw7iI7M+AMtjBKR{W% z>+fGrTc8^GsHCXa4x=9<;ky)pQ=i1~c{8W^yxfe+{#ov|Ep?c?_chM2RXCCE!4VU$ zIfZJRw9#-Gy>_mS00A|YE{GsTwq|1>#;6Pc94(`6$#qSYho;u_lBp_nJxW9cC8q;O zh>#@e>go!>{WE<Q zk`K?3NgvAYuB|CsUCTWc^%K(s0x%1z#$lK%84a)=>jyKmqjc5j2P;yd(l3HklpQ4C z&XC_l-yntKE54}uhfkSxbD!wi(AlR~Pfqs!ORDNMoLIK`4wLON?5rfK3#?_uO2$>g zvzR8wcr=Cyb$g?9^&PD@Z+6J~&8xnIm zjErMAIN%w>b$IR$qV_F0l34WFX0qz6&BMqC&6MjXS`#x3kWs^nu@9G*aw3m{5~?N$ z>X0T#gYAg#9lpFDR2NMH6N&aGJ_g=?xiC0+cqbEFbn}LFQ;-%8zcJw4IfmFJL+m&JCSYg4N(zl`m9++_^~&j(aEdVJA1RNPPheo-MFmA; zNFf59Aa&`HC-MWUCLrmLy2~>|e~IXE&Pmn2lk4=CoNp6@r+77W9cub)HjbT0P_mNZ zjH@~j%I_8p1i}57j6uvdw0tL=)wQd0PPMQrO|7Y`24;5FgQ&@! zT*U|+1omgQ_wNPpJ5x<+(q_mmhp2W05+k@U6p36hBVu?ayXmD`2~sIE5V-`V;DAOB z(cMqX3^%7vCCk;yxLKmM9u94G*{h9OtXiQ_@50MfU0zA{Di+HqGSzWaeDn3oStl^4 z6PfVoG_Ji`UYG7v*3!C6>S}03Mjf)?Ml?i$Fe8q6bwR1srBm0dQHW<&sXA7(gHGHz z6RSy+41rTusMCYW}txeTJtvaGAKh+T5Xt4TW5-Y-Dq=ZVf42rZg?3?O2zcnx8hCJ6k$h zS?(x)9=ERy;E)$ui3^{VTgJfPwN&zyrF&B1v3fKu>NQj}))9*|sACZdRI?88oiZDO zr%rgsO?o+NGoZIJv-bH`KNRBld}EALz8M+ZrtR;N+duSt@fHlsK}5CnBAcIPUz722gyR;u09q&+)&G^u~8rAV=d zW@4+}K5l>!TjN%bFR7}Sbfjt0sFiMudiPc8#=6=-0-X>wF#$owgOlOcsNS9+Yklk&@3XF^itTHR~h)Gbg~t5rXG+L}UW)sUKEAnNKo zSFQQw{{S(sTF6?OcA}kQZj^#Aq?C*xWdTWQgb>e0IBp(`LFW8*Eon)V3*O>lMT>j8lu3ad5)nRRu;g<$@a2#Zs%F zlRI@7p4b5JMbF{V=7k!9sc%l3bR!j{^(pFMRH?9#i?pdPJN zYky9s%MZO`*#XxvsGLqjbi~}Y_m~=Lk#6+s(y3CgM5fhIPf9sCdTb(K$UY!(=f{&f z7WVdO_jN6^eHSzV*L5#mw`$<%PQD;lXeEq> z)WpuN#yS!v){4Dzkk{)YmJST#H*O z+Hx)OtAE4SwP~(374i)_`7+iV%Nbsog9_V5PA9a%L`P%#`2)7LkKQvd2QW-AlM9mq z2e(X11(My_N`+v+s188i5LgeC$57J-J4fc$PFaDEtI=MV@}6$GxS^@exWjGUa!SVS z9G2nziz+JIcCm9ffEVAKwp-Vx%^x{cfZBH;1@~DwhR+q7 zTFl(WxTL0O(Wt$G>S@90`Mb*r2M|H&&JJVBEbek{Psi`SV^59XXfDMRT&$;7RA#Y? zIfCRuh^=Aw3X&Thtm;izb z&LCu`%x$!^Q$+!ln8T7wX*s|mVsrBb5je@>jZs}{8kJ>014Mz8j5t_sl1@9x+`;1` z`i0cK0h#_QZGB!QtemaxoLcF}@zz!oX*`T+mbDwUf0H>~hgt<%U$$ES!tw1aIf;s| zH@|9s6WZ0RgG$rBYL%(AD@-qEcdOekpExiR)0uQCJ5| zO2WoQ~n2@3RiT{Bv$ zl+{&SI>ko{(wdpG3MK(D`G@zeeJG=q(H<+MYY&cK-mS~Hb>roHt8a>2k1xU;gfuga;?t!|-BPo|@!f(ej8z$6k3 z1{;`HerZ*fy;wH&f?Lp~P}8MWndv2*u_cN`!Q29Hp}4VYeK%!wA4{UUcesT;OdelZ zoTr^5dm_z2y<*iFuN9)6bK&&rtlMT3RfP(+Y6yD0iV=rig&VqdZBl^!qgclu80&mq9*HA;O* z==MgZC>W>8m3^CB9a(yg2&2bu>B4Vq;)$vhIkuZ;LZqaD78;TTit6^f%E_wM>3KlmiinwOY0OuwN8^(txEbdu9|2y5J52p7By~qYDvr9si#qC zRSfzlfG4JmmQqF(YTW7)0M4J6fIkQNnCmB2c-JP>>E9E{wj85pUp~V5A=$#K!><96 zU%Sa{v9)gMkJ$KFrC&0uVz>0c&QQw?WLt}yr>k+V(t5Qj%GW@EHl0D~$Wp|hmNH{v zyN(>(TG-n<`qdp$)`~isWII+2+C?CvR+beEL5UJXh;>i;%fO$3)1FbPdH6i5n&IeZ zbu$odoCCJ3agP50eVsee3bS`2-Egx`QjbI`o5VX?j>Yh6q^7o&(9~8gO-gk&0vfe7 z*+_x`jf}2NQK_Yit5ioM|sU36E=DC?xxaL!F zGA;DCDJ#Flw#>Ki?dw>vUlR&0#HaYiF4l{vTCHKeZa}D7a6>*+_qsP4b)i)hy*6Yd z5LuuVBWWeJTt|5vSgkD9*i>r3mi5}Aii!%80iG1$#44N^-yg*GVD)pUwYoRS7~Yz4 z{B8c2@yxyt4&FmCQD9ai{jeU>Q%f6I%M#-pIuV7 zwMvdwkb3Bp@r^(;L0HLv3`m%W8?)*4$Mk)@(fj`ZPB+;cGWDF!ujXO#?kcVQo>8&C zp`%vT<)zrYuWH*M9ThBQHLFZQropjjm{^GwcXMlTb#3^1!lhvEO=*cH5Nab!Y@|ut zNHPe6rEe~{6{+5_EF#xgxY4B$>CvbO0|bIFMj+{#<1=}+?lHO4uC8a%&7sGgjmz}a zLj}(^GK=jdrnSn5&^0KmRnKG^!XQ>k^%zP(qCitrdY-GZ870_}u#Dgm1{C9AJK{$V zEs^~SdW$?NgHucg!G^$rs%Hls=Nvbjr-yLdE}8M(W4p|;DcI^-T(rmXJmTp z(Xh{KzPMj;gU*knnCk*lD-c=yCZp2ympZjmlMr^3>HEPrq>s!-D6)7?j zdyysz5HdgpU|zB&HFXtv7WWHgnmFRLubSOcJ5TOhjP{U;GUvT11v<>2Mp|ml=}wxvyE27^mL@@1 z1N+2Bmg(w{R$}BS04l5zG6t1kWMmlx6TEl?f_NkK&zkk3dVx4yLb0#KFQ6+u^xK{G zdh|xvsp);YIsnZVv6ZEEM15#kemhoy{YD#_my2-Jt`3Tni4KKfm%JH*HXoM*cq4%A z(%(U9E$S88Re*5%wFZvlx8BCKVF{COA zi+F<=L<<=*AnX${9y?I1!Qjb1mvTX!@emGs9rAbLN|e!&m4t#IiN@f@LEq`%XolQKgfsQA3u4h45IAAb@z!ey8<=j!^2xc8?pQmgHk>*rRh|#Hp#X zm0y!uy8&U6X(Js9FJ%jIs1P{tV>^KMI3DEs#6WJL>n^24Ha3+SSb# zT7{0iJK>N807rqmNIRbWO9bw|Vsam-TGx2~X15RG8v1#Zbcc>E0Jftu-(v`^<=(UW zb-&|oGsP!y%|gTa>j9Bm4ua+CTnEacktzu4W3fI!B6rVlv(+)RBH5^bv!$0)i22(v zVTh5^N`)tWl6cUrF_Xl1ex7yukH@U_LvNg9j{3P>>nTy^{{YWzTCboOAodtA8J`}7 zF(nkJy1B5V$b3FOr%s9G^y;NITBBI3?()@NSr4Ii3Jk$IKpO+|aGIi3rOXW{6qb-w z6ttNzX{<9~oSfyEvyvhRyQ@4Chg|60eH>0Bp>o$3=iD!MCz$VQor(<8ms7uUD_+Bu z?6aSJND6PQR#Za`YXBfUWH3Ecr%st#XoV^x207J`(ByU}q=0;Z5;!j3QLR#oRsR56 zNy8w|r`R|c+p#;Jtnry&RefCIy;ADOSNU&AWO6tkBL4s{W2YuK_wlT!$nk28URugk zG#uyF$)$e3Z1%WU4V5*4;?=N5S{PpIp*<3(m8Rn2aseEduQ--Z5;ejIA+njt9%5En9m#OQcFA>OBo*E3jpzuS@};w1y<) z##gaYSz3`+qQ5W;>Z($qcC<~(4N}Kb6j20^;b@7HOv2oV!g5<-yOMyD|3&v3d0TSusX zkXEHc(oxz0P$06zh(1Jc`uXcEpExvvl(e}@bk4m@X_b9It{4U}1V8{uB$9X-e~ITy zIk!#zq`EzR;T7EeMaOgW_{9Pdqi0Q-T$wAkjUJ+CrDWEMS9-iDh6OASA|EN+;zDTi zI&DabrPUb}5uI%T2@oWl0V5|6BY{2~adXRV6ct8Fl}%k4D#T!j8Gs4zv=Y0Vx%$V! z*5kiYdhy6>+pVXu#d5aWoob!h*E4D8QT5cQ*IqJhwIqY_agC2^KPvsr`PUeg%d#Q_M{ z5OY#6#d%Ap6{wmTr8ycHfdGkq;Cp2ANTpQhC# ztNOfGqPMn7Gl+0oIvcvF%b^poyHQUTT)PjhA9CuqllJp$e{+1C*H?#f2r|X}SGTlc zQjIsOP=GhmH8lLK)O3T@-GnL23wAvesazoS3dEMc+ik&tJ8{zIe@{7o@UQ&&twtNg z`G;0H9A&~2xJJq&N!Fu6z`j-_-Fd5Pq zz>dZwoc7E>;u<$D_n(IZ>RJm?rV8qHRsazsfM+on7%)!?HZLCHRPk8n=WOd|S+k_g z3uz@+)kVSrP8C{>j?mE4?UwM*=J~bt*J6@ec61g{%1e<M74ge9yO$A45H+ z0N|>*LNNyrea=3=%i3ZaN^-jKwVSt)+`U^&-5iAR%#Kr1sw+~@*&D>F*8MLO&?^e6 z__2B9n=Tj?fwZDPVZgk9Q6E!Ezz&4UFP@@y_a;5K4yBbJO@kX2nv0mM#I}2E1e4B zxhAe)u3dXe1*+jU8-ZGdM?JT<;fjjc`aRbqy4Na`rH+pN5EBg{ff@KB8c$BKtjq|a z#*k4nQZNrsN&z#pun<51aCK{n)xZ=UEsT10^P57q-bHL`t<%eF+^5IsR~aW<5JS$%x58#5eoxA&R=KZ1 zC|FdysZ54of(14CL3Rce7x=rxMCu$v%vzm0dhJrRDhkO_QJkSA$R<@==Wq@d8g_vk zYNtBjof+xBKI!B5zX#?0N$T~?zD>Wcn{P(;k?UT!_Wn%0e{)65%;Q-}WK>0qY*Cd# z`*ekZ?6c?U}5l2a$CeuF2AVjiKh;!#SAKyTFS2TK(HJ z@ZU^Q{A5i%&iq@_L)o(i7RN_yOBRMrfSl4X0G*A>ZMur7jhDH1rLPMqGL&z z*Tffh3C%ghK3`Ewu7gUOhtYDOVv@6NtieQ{%OroZ=F_A908`xk{{UA#M(VlnoGwSq zbvX|wgIIyUsFP&HM$;RqmxI2z(xc>^F}c+fgJ-&p;4 z;VbF3rv;quHJPz~^^NTeEbcR|>yCLb0>;{E^sLrGN-EPyMRbZlAWV%xBV+t0I=n=) zysPQarrxon>H!S@0Gzi`(q{_?Y;Dw#o=?jwo4rI-tx0IBD;5Po6AC&(+>@xlhz&ZL zQ;z*Da&BeB>e}k3H0B}pxpwlZam-CidXtNHD^gcRYi4H30l?*qo8MsHFsA3 zn?6@7>C~F;nw3#-p+Px7CvXWeVg?DGHM!oas0_oTlYH8ndOP%Q*gRgI0yEN1#Oe7IbXrF%wFwS`MTm z>pw9PAAxgJPe$6_oho!nhGG&632cg|=G8yMJj4zq<#nr9xxK4ViKfK>P~>`U2FP@X zQUo%@PQV5XnMbX5=C-1mstc`uGd8Yx=EbsDG*REuN{56P10qToxG{y0$P7gYF&jjjzy>%ua*OsODD?BE&;mqSIhpsRc+N4J5$?kO}$&?-Do( zU^5V;6Fa<RHOcsYzs=RZ6R83(CW&v)5f#2I^Vu zRkhlS1P<|!aX~}^+rn2)Bod?B7%_rR(?b)E794@$V?cP7pj7@r6*0z@w!RB;$=2pJT-((Mr`m24rc=Zn? zMAXT&Zb`Gi{V@|0Kdg6STAd?k3b-IPB6{XF5@HVGM%j#(S4OBeCV--8`lHrpP$7Z; z0N!FrU?6QXA_*(Puf*1#LT75#^jMqRpH9xEm1{z{wUeJtQIfkAzYwxEE*<2_^EqgQ zN92T6D}yr&8)GC7u*SduOmE*$4jD?p5NQNaF`Y(BY8!XQat>!4UHWmuxA~VPXCC2; z9ez)9*QYg?;KQVvHkRqJQY0WH)z%WlpypTa@R^YrpOaFiYd@yBCQK>G0Drs;Oy>i^ z5)?bC=}Mc_a}?L=2ByFp03`R5_8?BKbmtQSoi0nl{_~Pf=EFGV+h537p}k|23 zvw0>loK*%OASzI}fQ5LP)&{u`mWvA1C>N&;tU@U+)7Ca}4i02TVgbyB;PN2y1B72e`h?pT-VCn|VeEr>O`75QXpIZ+Fmrb~re%kQC5!V}4LR}`i zLN}M^AvS8YaXro_#kjn~85)yCJQb46XHt%nJ;-5*h!8ZPOC+pmOKPS=mN4X)j+tQx z1|VTr5!EEFS#y3H&3e1_hst#Kb`ZX5JwJaq#C1EBDT{RY+H3y+8sM^NP^~`1NZYCr zH<*|I07A48HCOHm3JQ4%6p9D%Vr&WmNTEe=wGoj#vf(lpdb zYHV_R%zkc=J%$sn^fVvm@2DPs)D5icWxtKns@mE1C1snR$aVEyOK_DshRaMd8bYtX zM}Gn~0hCFB3eHd!xrJtRT{NgQS==icg-MM600AQ=Grpn3{Mgc?%jqh!uC{7u7DmV# zaT62lC-T5LwZH2(hxE^r^#1@ml;`i)!N12WThgz+#rFoYbkkO%thXB3rT}hd-t{_& z*3y>~4_(#j{GOJ;E%IVdpQB4Qdl@*y-vaD#8D=6m$T78V2O;P1F?&w_Br&fzq zHPs+h5vQvalrTCB9F;0a876e_wRHPeAvk)RgPJp2cFv6q4X!Iwir;lh`5AWdCdN9c zCnr*5+{@l^eBcBk(V0vvW(VS18{2vwwMJ1})lDh^1gK$%nGzcl18f|F!PRKjtx!!x zq?1uC=AN*6R-Ul=#GUG9a%xiXuOwa8|7~73y2aD^E}6Ad%Z0!PrECtoe15R&U-> z59&oaSDHv>Ud0090{{fI<;Hq4zESIcDE(1UuP#E}{Q$}0WT_kE^T?61?r$rwn=+S*U~AdD^g`$paMV{Dketq zW4xX#+~(S)D|ht6CBStd0fk^-0@_GZqz>B{0wbvJ${)#Wbo)kl1K9fZaCZLWiCq?1abz}=A}C3y)kl&=+wBYWT5GIa?p&^ zm>7u|>|pQ;(4kJhrPXox&q}p=WB`;3BoLrkdHoRK8(CFSeAt;M)6BI5e|i+?tk z58ceP73)UEO_txtSk_9b=(!7JF#>SC>#!+XC(1#E6F&zwhT__TP}Ni-YJ@#FQ>c-) z^PCJ5hzr5A>Iz`mI#QFt(pPI;#lq%x;JFRk zHK;`(j2jm3)lECT$P`pvqyjt?<+sg-0@9EYikebyR3T$~`E;txW46)r4a{~t zJh*5;{+X#(lDVe?RI?_MLY*o}8OVk09a@~+=5vfg)Bby?yIeY*z3S@GV>tzzI~Vx# zZjDuK9%btmg|)1$0#waQmI5L~jeWa|Pemva8n%@#ATc3|9P0p`jz5HvAxVf-=#=W3 zt4`T!)}{&4aJu7Y$O3R-qC|xoO~mOs z!VlfGEvoc!;jpJBoU)looJ*@T(lI0*SE*HYYsWd%2V8gsYQb3Z zf%XB>7HsJeST-rJWH0@L!X^w*zMTuDL_(y?F#xi%!H_mTUnkoV)mLkZF;hWBW%S7D zZA~U%i9Rzqoba^&0K_g){{YNW{{Y{w{{X~%TkAi!>A|`Wh5cLn`1DEs30+9#T^IV2 zM4PT|f1SE@tHpsHr9RRuF5Mjpc4o_-xURZ_+aA+c*|ulvU24XDa!xr#Lf*Y9Zs=|) zD%FFgR*O_%3t$K}8*>>ua4$Nvqh8fktu?5Wu9m1)`9!Bl+f>K`3D`k7o*qB(I_az{ znegtc(dJxth--1HdynbRU^A0)-8(&8!KSWRg{!BV7bR=2$Q4&ZA5pPR1329ExYW0{ zM{3lPopV-!Ree7)s1c+=I)+KfjQuq64=XBZ)U~ZdJvyva(15Awxd5Pv1Z||p)^y`B zdP~l_O}WD-tGt6>6JKwd{{Y0EMa@f_QgEvuoa<&g1(>K6k`|j;ng@0g$Rw?75i#?L zh#jdt8njZN3UwnZ9Uz@?1e}scGBc)o#{3g|PU4NpvsE;yQCdogMD(>KhDHP#I+hOY zh}s64*Es6_M+@oK6XF;=V^@~!dop=N4npoDk;6C#B*1?k94NV0lH7^gy``UKTB~+3 zE8C%>mtfvmQd z>LfJv!Ox9SM+z4WacpMTMF#^yld>*_a&dw5?T#LUo)o9b=5oMS*0 zTqhTjW0gJbHM9VvKwH0&V8Se2$=%PRr&4VLAnes&z8bkM{{RZrXjG?Gi!wbor&UUp z1zMFwsHRkd1=UL~pd3mxqm*e-s~sVnSFjg1j&MVYCiIa24Sf=Qp&ar$mJCGx%_#kpTkK8^Zo zZj0z!<}c>^-AUFoEez!R0(or$RcN>a?QGpkV=|M^#WgE^$-tD4ay3OUOP0z7Az7+6 z+LzW{%mdV<4&VY{m?dgx4Un*AL}UpVz}3g3Z=?=;(+&yLIOm$W z*>tkGM?%~ZxLRz=^nE*%CkCsyV4_JR>0%HV$-oeJ8~s6j zBXq;+x!gCY^}6fIF7Uj}4Xn#Z%POrZF5`PwXoRi~zy0RVN>DYD=8?FVpK!>};D-D? zg<3SLvsD#63TsdY!jXa{vjF%R?Zz#+mGuEyl28(^2BM_u)T1XSCnf;LZ{irA%eW66 z;I{g)!}bJu7N-p(R%a`lmEo7`KT!$K(iXDNt@i%_kF>A7t6W)SwpM<$fKOpgTS2IS zjZv-@hh#m(4!_V(w)|Wv*9Mh0Whm~WP&2p^PuzI!coTmax4^oc$0}{2&X$%>E8}#( zi^XbhIPL8UTb8L6Hc#&ur-=A~c2mDnnTV44B4}EOz1g4xJMJfVh}h0gU}K4oT^D_j zDJ*ySd=E3U>F?MA-h@7!vM#gftCVavsq@28!ZSLNVt8(@K?Rg*t5ZVCKg~aOxJ8RM zDw~d}FotK?lJ*ghK)7o3Xj;_uYqzLS`bee}F@2&W&Iu>aw-vj}&ezEn#p$0!txZ}< zT>voz4B%ox@rcsYzJ+IUUcGdesa*4_JQJ02jIJAi{!`Ny;1hE2@%K3RMT(momyfxk zz1?_iMJC;prK!))P*fC47~?IdQMzjjo0OyKuuXLRL!{G#l8d_#0MuYc`I4&zcc(_` zf~`bfUa+NbSOV=vpp6RhG$v*SWa=bg9~^$9I$a8XE9%DC@hsuyJ3NYx#cnxuWm>#n zb$gxj=)6~Db9nRywP<3a`o&_xeFzEOweur35U^ zOaQKD?F7g%21$|*@(AP8`#J9p()w)1qphW>p~(2>7vr2omUQ_j{{We6_sF70QoZC_ zuUvThNY@Ud<{@b?X=QlsM>Z>~#jk9%U;;`81fR&TST_CUHX8tUT2(3)>0Z>Vw3R5m zUsR7w3VMSU7S>Rk%0sCzNcV#RFGJ5tn}( zO$cl6R-hgiMwG)MunQ#2NPGNSLg>-0ZRwItC3+@v>C<*mDm0jY0V)Z>E7guIHLYn^ zr+&LFWw)w?jUujuoeVHHW7Kyz$TP-!eP8pgd7t%p!mzsU)$jP1oyPfh7O}zkj~0~2 z9WFC(lUCO({yt7SKga6Pa%QH54I5^uxVe=O)tIiR>WWKvKYrCz)mHTwlR7Gjix}yK zhyzd&oxtK7N)2yoRjut-hp4To3h8NNDp|V{0WrCZ$r3=T{8T!ffz=PFEY4fRxX&P? zaUC$uz~&l!w?J*!QM_Ad*QW>Li&4an&;lILv%ss(T$cax@kFzeTrI>aHXYTpz6NmP$w16QOWgLRo35C9(VJMJ^y3DOIO zplj3CuzElwZrkJ@qcZ{~LGEX#NxE5?!BO;s)O)tJD>Ii}-~RwL=TLP3)V;1#6@kmC zr%6q>M#7a4Y#mDo%$m@u>rofy;~EfcYK06OlErt)7&@^7{m@6e02L6=dq$;1(@YOf z)R@vK;YOT;oa9N|%#A~RENzZ0)3fTYO!*mc-c83SYR0Q5*snuP^e@eEAofU!ybZ$y zwV`Ci%BDY2@9493#q>RMN~*0@`IHzdGa8`8k+>RT zd4OYvGbid8`47|I6yRCBS1Ryn=W(3fqu#Y+RNuQQuJLjooH(Xj+G*y00JNmkfbOi z!5BC^AbduJ$kyfA#Mqq)qhm6bSuu>QV@fT{C3L-k5l+I>1?wo8Wd8us{?h|L2PLTx zOt8+|?f2U~%t0BI;OAwHeYE7xKC&`#k%6>G$>HgqJ)@Vz`fanw?QJK8x%%1QjXA2S z0k@smlb=C#eKuA16xPIBizO3b(=sJ7GbOu+sFdm)D>EtpXDdHVJ4bl%BS?X2Q!L85 zb%YK;IUx5q>~Wn$edC1x0O~w0^B$z}Yq|c+g4x6i6!^_r?i|Y05d{@$WPa@Mf~voJ zJyI(N5zGhfQc=}b)PbbALGOvi#Kg~do#TwHH$*iMbZRF~OtX8R5HJBSAhcmheHQ8d zK)Vk^PafU6wsZ%{l-9AmXb^*oVz7HgVkL_e5l`Z0$iiT!ycI|aXdrbTn>pL)aXA>7 z5J-U=?jfXV?THxMw6KC_AMWkL2dT%{T&sa$@!mekqtq^3`OC0ptu?eW-*T{8p+@%d z(4^DKu~h0>avH{1c@!z25U42{ol{cO1oYCg7-l%C0UkuC)c}&@cJC_e(0mC&B zH57cn6R`tMN|Hnk_X0YXkMTa^{BKvQ_<4L!r)Sd}{4Qbdu zwn@~P0$NgIY7sG0*kav7kQ6_a?3hb=H`=B^qz&OuBF42c#)uM#d@gG zxxS0Gm1{k8wQZ4E1*%fCp{71kGvwdl{%1p!TJuVXpx#{5rEN?ag;JKX0ILx$5Y#1r z0zmsUbK<&nIgQ^fu^MU7rCF|A6{y0n0(7YXRBA>1&Eh9x3-uMq_%}jv_|p1sqi?94 zW9PwPJo8}+yK2q-J9-s8^|JJ9C6#Ti#fGsIDATj{OGGX_QZpY6=9iSLZSAO9Q>p7# zNBm9G%-3x~SsE5pl1NZU0K|xv+mI(6(+tEHdm?g_U%c>EUmiDQ$+%`&7FS4X2u*;58E_BTK zE#@4`<@sF;YP{B~uo`Oe`ZU7ZCAyQRQ|Z*^2_)(RQ6KLK%Q1u@4&F1b1_x^#m@3)qR7je zBicmbOT*=dnI-0*#Vty-^=hG2eNAIRnF2`%GEO3Pku$;zOKVDkowa1AQILw%^y-am zrL!$E0QM6YJXE+>)PFAN?+4@eICd5mrQ}a9!Q^@wti2C@e;PQkLm~eFeMR+^w9neu z*qIwKUCsz2@)P(a4rOsljU=U0L#S#&(^9ceWJ%6rClU03yc1WI)3~~<9g3k^RVk`~ zG?CIs5~M^x?T+y*t#Lj9)gHWI^>3=wcldV~+U41s+`Sof8@r*ufs>4=w5~Ucc3|ny z4WtV$>QCFsy{(XyEz#|rZS9)n?kc^|HC=s21#=1u6EiC_m{BvG`i`qohc2f^jWl9g zKxdF7u>i3LR>bZb->OP3Rc~L*j%#DncI7JX%*(A{?^OemupgG(VNY-%mKk`xMkX`) zA(<(kh-)zw%ETroPGyYeyzf8dgTrC~qDd!MI}PAxU=BWm*Z~v6BNVxIuOsQ6FC67I zXuXvQTsg_OT6oBIlO+`^XqAP!wp8v~WRcg^@vgOm4|yq65rUt{5wwz20tA3q#Gb4f zJwU5@CII13x}<5>t5q2VCn(y18i-KcJ?`)7k#6(E=2@I|y;P;fClNwb>c2k0-{ zShNelqRP*Iq|@aL^BIXH!RjG|IvI#r6Wro@fQdN*Bi|>9E3G<&u~J}1R?;J3t4wG6 z+i#c!4zY3@T#ub}!tRypc6D-Sp=_b4O>`<|rly2Cr4011urUdEPnb`Xh{TMSZqf0D z;i~m0Emk^}Mv`+1BN&eBI}z^$#B^6pI;x5l90f32B!iQXbx1##r(jO}bvmVEsMk2w zrG83!8adNRoAjou=Kc;L?O2yCtT7&R=@Ek0GfVVxq=s1$+6&}`A3!}tL>U^20ziW* z3>cUlAdn=INN6eO*CPx`LQ27H1`HAj{XEQ#_#1G_e`Uq)Y*L#9x6ODO>m{gc0@V?A zBu-$8L7l(|e7nzhkMG~)I%Y}UV0ij={$o491EzLn5P-;Uq3`BAVNa4|u*$UGPgvZi z1CW28IK@{vaPg}8)Y$5pLz`id7HU$kFl06^lt2soJXhiSb_s(L?X9b|R5aVHk^oZ& zPjGT_W4RlAxt;~O^u;eupa9e$sc0Z+Ac>M@euK9AHh^*t5zZ5viED?~(ahwBqTPh$ za{feFTBx_;I-dN;S0<-+jMqfhB-u}#w9Cq1yjD*h%A#p39V$pEsK70>u1Py!^+baM z94WZ3L738I1QmeBm}P+gG++WIV^qQHKs;5O{(aHeZ$8$gFFM^ry<67BRGMV8OV?<8 zWc$kOL>U^+BuC&}!PwL#+FweLMj!zh0y_<`J7?zMpD$Ww*qUS&*7`{B4F0RPe8Gj= zhqwGGxJL-3)*gGqI)@Zpdy?_o?VL@md>i*Mv@-PTG`3^!C8<*W-$`tixpCmj5#3WH zGXp;(zo%t-1zJDr)F-OQSOikVr{x5D4=a%}%lYLB^{mZ$cM(q3qPayoF*;&k!6!ZZ zK+7vB^*BuF25S0Y!gcs>SU7Hu+;Uqe%By2C*G_ZsE;jwq)sl0ugr{{SgavE+#A&(2LxkW|j+!NT(S9|dN%u1`D6_Hi4Z=PTT7>sMaITuZs^y_Hvv z=RL7$Hj>Kq+ATGzeC8+RO698*cD4+JjZ9Q^kTi%`fS``d4DSQ-I<~i>)v1fYO|ep% zqQ0spJtm^Il3mCFfzzc(nFmi5PMqJ@yTy8k$Sm%sm2!IaDe&cEZ;$zdlxARS+*>H~ z4n@c8TdvyKtRk@`bK3WKBhv{p84+7$Mb@*uXJ&efp_ZxGt_` zK24u@l+=>Lr)x!4Y*tIN)^4mNcDymq@De4EfZ5QcK38&#l~t&u(K3RWE_xDlo|lj^ zROT^~wZ*7UHGxT=)YnmJmj_HCEdVhjgCt4V92|}dofrBO`g-dFUC#Ow)}{1AsM?&0 z%I$LXzSbCIXAL^io;}HUefNkyfN7vX&aAXumA-6Lvs=vCQKjcCsZML!8V{)#)usx% z7*NN#b=-kDkO#}vinXfJs7*y$s#FlKrIpEH@{zHEL;^-2=^|UL)1HxZL$BDqL&4=e zL(l22UOHXWI-N1v-ZJwy#9g8c-`ya-f<_ z%PeFG#3}ab4YvI=`c&pT=ZH6hb52XdH83pj+j!d?Jrrpzcz{SCt46i3`*~XChim5J zk^pS23?UBWPWd-8wo3GLa|1@2Fj};lbQy2;WEjB&G1KA>sp_rW>7w6>qi1fpbL5b4S5O4v2z?~jn{*3;Uct2f!tooVLM{`a=!mvJ^SIl&-?BIoMDAArj&ULf% zRSK#$sf~J-vTQR%>Lth-6vTHEKFTUqv^?qUsuniSTJoZrY!r};jZ6bL3?BnKK^cX! z-c|JIP^hb_r9(oRrUs{~vgayeNQ06AoE%3FJHNxXGwRysChE4ASNe@t6I%T7{*35x z;^XS9{t-Z8q3dj*s38}RlwI@i&!=5iXwv(4NQ2~zNt+&FcUGhN=7lhnt@OM+0WMnt zOB`g7KvMvccp9C-uwzD*3QJIidb8F`-~=9V-WE)e--MeZ={3ydolN5zS~^+XFzA@G zk80-M+0@D9TRIrQT5~wgW=&J{Z3dmIGKx>A`$_wjg-HciBLRw~B%yIqkkquP7+q~T zfPw^x8i+BcBuFA>2h_O*Xg-Z~?liirwEmF!U z)Ec8k$D!k^CT9bG17|KSsVb9DD2J$mRf(1cW&{EOoS4GQk%T|O?SJH)kNx*!{{V{l zzPca7NB$B20H20BPoMsG{%$vG{4(rvU0%C!EQ;Kvd^2k25$DZDq^2m_Yii`8!jX++ z1%X=qvSGPSDDwx?@$nF0d|AtSMR4x}(5Osm2Xw%O1Wf$HXxU`xlJWK176=@Ql++=D zcW$HJ01@(?@JS$Xpx;&+I|<``Gvu6qnqS>V2fK3jF@Gx5qhpeB%C}QvAT{|NOzQP3 z0(mPO`pAfm_p3#VJ}?mn@S#$gQeQ}r6$~jPoQWWwxtN*GNY2f|>t9jcsB|l(WFTP; zzNHs)FiNbbK*YceGD3+l+>$tv&+5{;qtuV3>fLV1CUDQEzAaA922(iAPx79Xg@4VUy=wKFkcY7;L`VS5M&h+wqj6lCzlKlTt7Srwpb#a8Na`Sn1_%Z^ za&0ME)4cpRU@n-JEYjdkOo$q>7*!BpZ#*h|lGh0?uk+Bv)4-bL%lCD-^-8oIk6FQ~ zuX5TdW@Fv#`gGAzs9 zzUBbQpS<7?uI9bgemc%w#rSjeDK#{+xqAlv#2U$(KFZSz&r!$pkX8rSCdep|NRvNO zfsvI2DI|hO#P*#30JZus$&v>AOr!!#W+Qm-<{KUM@x1h@9U|V=snl)WFQI!iH7xR7 znw)zcjH2T;@)3YnbS;;JD+PKC+XLuHh2wP3w6e_w)qAF?Ah9}k{(3XW{wrb)t5N=PH9Dk_Qp0A!Fh1RQT5wv;J^{6$w{Uh3tlD2O0H zi6(LyVD%3G#3qV#FX~4RryomQ4C@x7xbBBD>0Xy4y<;Oo9a?()khg7GPV>mb6p^aB zmq;qE;S{tAT15o|9~m3Ap=Uv2axavFnqj1cG}=?W-q+cXpqNyK6URri>}n z^(nL#?x~2#0t|@Ui2S_@Ur`*tkMh1>d-&^>ak*~{%5@GUn{OdmI1bnQx6#bf^Z?eh zHmqA4fnJL2nn(Uz=45=yVfCi2reYgyG7)6wQ2fI%qY=3yP&`jw&YfCx>8U6wKU2uo z;1FP#Aoc)&P9#M1q1W{@?{zm*9Js}1KXYG7)|B$#R-cB(Z2;0I(q(f|E?;Sp6%`Ra z5NJ%4`pQQ|AXQZ9(g^_R$4Y{Gfh8zDw z$|EeU{{WgIdUwp9l%-Lipg1X2CNK_}4HCMeaV#byq-q>Dd~2v`>nBuom!ET)Rq5US zSD9*;7UTCUxtlq))LT0bC-}tbWlcYk*baUks*+b6CAaAS3#L0h}nF zoxy@wkV1kaz+)UQuB}eog(zGmk5QBpk)}W)wCo_s&&pK%+bhves7@)u_)Kxi{Y9r@ zpFSRDlvUr;jyL`W}7TDhAQCd%NPJo2qYflV~i5|fvd`~F*F=CS;#nz&R3~N zE1utDE#O_{8Oaa3z#A~Nk*_wli&V%liG>>%;wmX5RnV;(b##D^%&O2nblrB4Op}oZ zVFx@}Y+MS?T$Lpm)|?s8NP;>*GG{uY&X9F*`DXI_x}6xgpp5knYVde>@qCJ7Rhs)+ zwQW}$G_xtQy7Sj<-_u@^kjW`~1O_G> zY~<%Xj;sv(!Qp-#ZH^V!TtwyEw_!Kyha|dW&-uDmZK1(-j5WfN`^U6oEfg*U)_ups zpl9G^O**GuMm|tM5#xTq6Eb%lSsW1Tt!@?|kQWiKh@JcG1EgnDM+?{19nLe!IQDsK;F(nUJxtZDfn#RP%LKr-0HuUk6*+5`K+8Ln5fmJZkJa5OFm(r4bpj#|SGXe{ zBRGg4(&8_Swida(Z&D?mUaPi(g-gzdHIR z##O7X5ql>Iih|pstgQj$E`~YT$y-{sIaQ%&O(AWw;ULyWv3EgqrfINpC{ZLX_;y@E|5r6Bx%T5pBWqE?r{btMyv2@oL{9~2;|%0bqV9# zwzBxnY(EP=&D~x5a2`Tgx=;Ihcy;R?lATnfnz*hLTf~S+nbU7Zi7G%21_9C?6>wADS+uhy_Cf z8Gr%Yfd}Qu@8`g1>QXfb0is!EM2P)=paab0fhI39*4B1*o}IgS+S?qKlVeLd$Q1p< ztb16x;R`5CHcCN^F*qL6*oGsxWXyo)Rwhqrf@9bK1~Nfx3>KaObigsw2f)X%{!_Nn zV@hX5{DU`xa6L^80^?O~2OwB#SW(8d&Zn-4e!?Qctz?=(HU>PH!Xa2ka`;lgDNbQ& znN>fX++YKrFmb-pNE}4zNm$m1D^5PDA+eDW9V1f`G7855?yjqo@eZV4wYQNZ% z_(ib3$CQClvQb#h7RupHl8mdeKWXz77kP*hisw}dqe|W$zi&~}e zECGg%I3RDn-r2-KSsJ)2bPlrht#O+#TK3Aeh*}I{V(Zh=ReH>}2$q+u307Dm3mL@d8a&FsG0Wmf zrPMtsYC@sxBxTDAN}EKK+3{Q108|jikl^*aSH8S3R z%bMkEa-BD2st&S30Imp#gR28_G651mx_ugRPMBnL&#s(HOM=nm-D>Chd~bK6l+LZ4 zWEp4Nud8~HBB6$QqK*2NuV`4i1}moJd4oa)e)uxjc z=#$h)C`ln=0`aBbuGK8-HOo`Cpp_QkBump_3kL)dCJ4>|lfMW50IJ`s9&y&(j+v{L z=5XFO{#4eMW$~V#=bmQnHKx~?Z0PJj*A_BeQnS{YU@NKtJ|xyikjZm#cV3+|uUym= zq*GlW1RcPPnJhPtymZ6LY-rltT$(InVs(oDhj-c<| zyNSc|oTu#G+t$?!Iwb@Y7R?<$yCsp`u()6>QC?!KEr04L`K1#`)B>wQC?b587zD^C zz{bEy%m#Okv}sKcp|!|BG=}cP>E9is{qw2^h3St=f5jR1vw8mj7t}^Ret+^Dx0UOC zhaas@c=CrvgDq2I*IH2!3#fn@03u{ce4vQ(0;ez+DB49bAU4)TSOM9W8Y*KdQz(js09;HJRB$11n&z=5{0f-}ixlIAZQ~yQ@4Wj`cHtJD|NkZYl4f-SJMFZ|BEc< zs_&^cYPB|LGt}WE6#AAkCbC~*?cOAo)uPm^`cDOhoPt;|CkOJ*#Kij@<(k{vCB+o4 zNSI_|q<{>scrzIre=uP1*<9)W08Wdh+dOWKy1WmlujLfch08N%RLfYc+0BmkmUafT zPF(hyr7*W8fF~XNzBHz^%{HT_Oym$rkA2r~wl=^cg*8pvTr0bJ^g@y85U?3RGbdIL zj03rdfV2Hh*yfyPpWmU?orPQ0MLgR1{Ese~Xro6xBDCZ@GbzE%Ww1f9X>4GbKh92K z2q)so$l{|4t_kS_7)8to1jL@xQ+>p0cJ`q~r8S_1QHH48>TbmSb|L`p^?Dd+8GT~>v6XwkYJ%hjaQDQ!OK6^Tf}6flS8fmA^mZqI`dK?WdX z5!kPchU4K*q_(FzUPCF~J)0yR^FK{6x?*&>e=X}DR^6qqkAW91s*X)e?k$qalJa0nBSHzdIm?<5G0GXjkwv(cy$ zSmX@%@3|Xi_(nLpa&A-2cqdBv-%fNkQ%{ZZdqZSU>?c%@=JQ%BwV16tBDR)=a}_o9 z+9mxdKHekc0VR5kLfNUyDpWZpGzMP;Y1Q``-=R8#B6u*O7m95wt|_jzS;rTJ!WMLhUO{ zkXNq?rb3n<+B?|Z!xJA2yXIGoRHY$Cpj{*#Lt=X?g*{$+phm_|0@AQGk*bR93sZGK zP=!}z=_DMVO@x5sILi-}UFzqq~c6K8a2&Ym13gP^yno@bsCB{%GAt|1OTCqrtUY$lH4C{L1`^z zuTWOIscJ2rsixHFv5uOGA%QS65C?7KXHQr;UrQVHf5`cE#U*o03|lJE(#osILe5s2 z+mK1i%J;=;;uf)Vj3+|8*Y3t8vcQ>-5?gZY_kyZaS4mh?^28hwJ*F_tpRS%9TTx0p zzthkIi5{Y#&4NrpB4R%<+z!Ko*8=NoRO^38V^w&aKH&NNFw^4~-G<9`@wXG2@z<1T z-nuJVX2Q+F#^;+#Tj3U0jbb55fL8+D|bJe_s+aylobD?OUuV;EG7pm1O zqv?X=OAumb$=@4+5_I1+dj9}{)^4-q^sA*<+&_SHca>$>+~+*=kXL4Tawb*<2-AM1 zV^e6ds;xwoR_ra?aY()pfH<)YPGD{tn4;@Kk!Y!^m}X+Ca!6uGj4Fj1rzA&FpMf?O z!o5#cy~rrE=_&a&7GQL=kW>tq1w=q-Q^O03!!WqVAnF$99O_o~g$|x^PA#dEozFd( zSp35uUeic;Whzff-Kmjc`V%8nGDK8dsATqr5i$!(DusHT6e-+OEV>IxXTF@Am@8C;eA$iF0?Qk^iW6m;~E zRsbjFh!SyvBRoTCS--SZMH)^L(!FG=wK}Rh(x?b=CIFC30q-n${^7yWjy1=*mnq`? zN#T4`l2hf#t4E#e=B-w?PIB`zg&c0B{{T%ex~kpE>aLU3K1?rM#6kR2m8sUYb|C2x z34%#rtI(QchE|;Cc*o4Rn)Mcddt%4aCa4knwNlI(jF}A}4Im!ocp!D!r_*+)1kLI8 z$1dqdH{IphSQitsJQj{8FXGy1vWVTcpUd!=8X48i0p~|k8IQL^LBH3TQ$)fxyXb3r zi&2uI%hRZV(CFEQpd<-`M@TpkfV0~@7s1Aa3{W={Y`hua)dT6V98WX{~r?%&BCWvJ@fmOX)%SD}>dXn>x64}bs1^|NW zyRm+}M3Y6)E z0aTbFBhzALIts=l7?s7J@bv0-Z!Y?e>c>thBS(!Vs5w`!bz_$=PW=z^cN#8BsO?(S zxejzD9300%_A&GMB4WKyKsxlw4p0QGg=gjzUZpas4M6S)AShw2lOX9J2_QfUWDEsm zacUCWRv@`ZYjrTQP_FK9JD+c<`BS_{-N zc?~MPv+IpTcJe8cOLR@rqBvC877uwpj%n5uAJ?aB9-TZwo!NmaP>Jf;9{&Kt?6m^7 zmzC)vIi`JAS7S`WAl+J~NF+>71jGoAGlQwUPw7V_=!X{Me6hHjl(9M9H&FNzKdJu!%fnB^RQ~|Df7L(p@w~s`wNs9A{{X0tndb8y3b`1Gq0TR%mWskG zmZCWG>(pw^BqpcG*s@i9rxf5+*}4S;0q|>TjI{0;^%Fv%KzGa17(0*wGH^uy0CpIQ zlm7r`d}%c6)9*bx^y)bh0bp^QM@~w%-FM09HQb$~>GGRcT|(dFMTx&k&FAXs@;;w& zZa;SYJ^K|-Y38!7mWpZQR;^VlO*oq`uAqs5DH-4ZiUBnHw@lz=h~5eJ0N|k=QBpTv zs~|%5+CzG=16g7{G5}*CL2Ls%7z7qozLGbx`f0ttdP=JC_HwWB?I-G3lAtwtE*+#L zuP6!jg5OnEBD}B$Ndhx7*aPJD*DXaH018w`re%N~V=?ni+Ze&gobjhMpe|J@nNv=s z8mL66)kTTG?;)6GU>PT1IliCy3wSS3sdD*JIR=JR>$$j@`w~pcwCyf4rAku02gQ@j zqf(iu>O~sO@q3z~n&NU{nA*Mn2yk(M38kZGc#7#7#>x5Hw9NLO}vH@ zz+%#Lu3st?6vo6$j-suUypRSYYylzjmHkIm!Xq`m&dL#4We2qx)kq!5oZpx`x9R@QTCN!NPWy~w9S$0lAWX~4M+0C z&hQ|{@^UdhSsZHW1zLulDVEfovD|rs;EYFkJPG)nMlYV`F3llDjBZ_WZrJ23@d7Ol zI>iP14Jh(TS1dz}dyp{1NqvNbgk@C(F^R|%?~TdHAMB_ToQ^g!dwAR0KDqWEKn4c@ zF1OOVvOOt6Y%1l1w@QtxwW@>#>Z5(4>lfYZmo^sm!K^z%iACCArXpkJUbMg?NHR{% zg(3&G&=g7cJP}B$g2#3Rbc28Z+=v*17$dfEEO_WoHOlFJo4JQ0vtJz?I)>_jRqQ}G zt?X1B(<;doNnX8m0}53i8C0m?C(pQvO-`vu>5>6b%PZ1jYxVxc>W`k^u zva@2Nj`Y^1pe8k|LgYa{KMP|P^J~SVT4q&eBS%@hW#ET2YBgPI6Jp3enNR{eS zHuXRi6#_Pg5`mBq3ka6nnCZ?&@y04Asw*0Wsg|iz<-`M(f@Du^fCssd+(h-us%yQ} zsOlF4;&WSbKT^eOe&AqWWivn}im!U?$7Z^z!y+owhFO7_NkU|Yv{x0HwMVT18Hpey zl?3br=k1j<2e4VFC;=(4ECTG?l>#K3?${$r z9AM1t?TpIx*A6L=u3{HbleCP^m+WGTYsn_me;sWSsc}FHTs6U+Q6!iWW3=G$J+i7%av6)7nwCt0ci+4$OvbFlPM$dWQDCm{`uJ9r zLHYBKb1rPzcW0q+<9=A%0T?K+TE~E@8725QnUM+Y8Kmb*sqy5Tk75Q0k-Hx|Y2)gP z1d}AlE$`Ex9sKNkNo9L}t9frqG=7x&U5AD3(A?ld#A^;!b^VzuT&uDtU`$^GI+Vqyos?fD;z zs4JyGq(-&TG313C%)ve2`(T2}^*9=ubjgxG$d5nX`W_iyu56A^#cXjdms%y|(ah=P zO|Ckc_92c6mR(&A%TRz1tuW(A1&`6sCW3r?uVl2JwU_L)zc6}dnsT>M(c^d zfxwo)%dwocClGw^`J-Sx%WODEx@(evO0{<>hKh5V>mn3i_NOGV?*PiP5lKSd>T)%1Ek}GFeFa$6^4)&SMdY zA_!4xtNL{mDPZ}S8%*i$ah-ws5?57c>ehL;D(l}AxpzD>Htz6xTDr8cbo4FM&`y`Q z)pqb~ie`nE2aFwNDTHG%!GM|Y9eIYY#4fG2518 zPYXXe-Y{ff0q*jVJD2<>O67`O?taKjJY?o5{FSr{O!$(&KWh&y{sT>S}_IS zer5n-Oo*9)_#H}ZOy^1Oyx{rmBgB2qLZ3~QkU(Gn@Aohx`T#eATN+3_R^Eenp4(b?V$|uFg+5%-_m(7a^m_AD2%)t*b(fdp2ul_PR8g zp5}Z0-2A(K%^8lsd)p;PsS_in36&$cIPtg4xY{x}j=IwHfncNn<-t%!K2fB=B*rtp zQ6hRt9Z1F4<5c%^Id0|0?qlEf=IPXZmsGS?krpcx4WbrniGsTYwwM$N!4Q|9!!8ig z4vcj$5s!R^jAOP>Zqvk8^%A0^QV^}uL;=)0#$qub7z4T6f?v{4PI}SxD`Q8boh7q4 zOup1?^G;W0kobcI*@KeoM>0v~G$sDpFK6E7W-^ zi;XY@mu6*vpa~L80CkDYE8AGn{6%?kr$stgQtU`3Cs17@c|FYcD|7m4>3!~!{{V>o zr%HLp1y53~@#JVlglsoBvvgH+Ee#Dqu;tjVXcZ?Ahkw!l8A*1ZA}Og+sbTRID@$^f zP#TU20HX&_oP!_*>=@%w86avN_V``$hpLb0C1h5n$~vg%Y~W8(1_ns!!5BO|{YT5? zysxW$efnqmX21?TYmo80y^ejQ45j=RPnP-*x1ySPZz_(pzhqDg$|}Tk)m?*T)QHF` z%SN<3#`fyc+M-*AR;gLkf{4hCuS!Ena-s^1Zb1Z&6VR@!ZE1bXU20RL`Y8|zt5sFk zLDf2h5;X!g5GGEP`=$K_#N%CH;yGSdc>e${4x1j0hueKEBy8MsKdSxR8|-y^k)oxz z;uSLLPVj*MP{e#xmcor{A_&(~(geJNj1?(@ERPiChS>EBAaYsFIM8c*0E7u z0#a8X2m`BB#sJd>M`429cjj5WNBWG`<&`OF<<+pB7gNKL=qCQ3Y}$1x%Z>G##QK@x z>`Fz9^(KE~iJ20mUDXvX$$}XYn&g%*pnUC|13ic$CmV5dwYORp!%HhJs)h$_%nrkN zoSBFRb=l6AaGsBHZkFwC7~cx@LaDKy z3l*%^XD9L3FS)f!RVb3LR-L+rb{(y*oY*+85kRw>aE=VB;BTpj+7|+mlck< z)>J|!Wa&G$NCN=KT@ODzk1f;P9|C5>)Z~;dFBJ@p+R+`5t0<_18s>@;OtQB8UAwX>vZ(ab%rLB=nvt+NK+K#FMl-^@o4S;4(w%2CgH-{#Lkfya zEE!U=XEPCi1WRYv1L`L%=@(G)`iX&CK$D$e=q(Vci-6qIuvM*#YWPCt`suOaqwxn; zcWFNJBwrLWGSv*MRgl0rJ4&#EW3)j8Z{TTVQC6aqBxmoo(H~+CK+n}$&#Qi@Wc6<| zhScIp($wQOYu2&%Lb}s6w7>#vHsk9q1}qhVkbCb>+}e@xf|v>UEyYuCx}=y-NMo%*By01r=Ah8GzQO1IH& zRC%jAUc#*3X{M?w)U+ZVlDG#R<;=>`MFgmk?*!3$l$<&hFdLWQc*uAlMrHX6XPDdr>x%6V;ors1F zJ7~+FetKy{qSBT&W~svF1ejUTqRCpN$Z=4aq{}Td3Tal3 zvFmzd@La7HOs0w~owI zt&s6|rMFdGt7wUXtph-Wg;b$PRAy)OWID8f7h=pHi2w}9_k#<>l?7V&bp1G5RD$Xm zU}Vh53P2}F8hW4=##o}?MfiO>+3U|#ZgNaZoU*4K#@yD*wYOTfrFzv_dTZuwTPmuB zT(N$TnkzU0S`S-ZFg=LA0dEJG5Q1s?H7Zky+!X>r4I(BC$rG?6Nj$|}Y>`=%RjG2S zA&Qcvb`9fJW*0H!a7Ox_;G7!AC+TlVc+IW%FI%bjPWC%VPvW$3c|@U>^X`4Q+ID^Y zq4%MKPE67ORzBS)AQLV9N?|DUKs71O@N2S3y5-RaJIBb5O9y^bzHzwBUeY{!GsI`$* zwQ9;M*$Y0hhMB~V&8u6{Yr0EnX}nvjYDJZT3bcqlLPm2x-7q-CI@PXiS?;RTR;gLa zKA@&Y%cRJFHjO6})FA0&xsC20$oh?iXOf2#Zzg9?_(sgA!uU*5UcF8^iKn4sGiTV0iX^L- zwb1(aZH+AyK~<+!o^}5KKoyistO%=8&a=O%P3XGQ0-6eCfn^H}4hwCt&Pjqn2jN!k zof~=-=o-N3nuq{E&crei)B`e{=gds-2>QF~CnD8qZ%IxgZ;_`zJ3gJOKOK)*~PS%Y(rCL=Po}-feN?gUd zlMOPfsabVz1|$}*^|#fH6f;kQ}0@c_6YXkf*4Jdv||G z+%1zi4_1>`Q~>Jr5n6PR4@)c^G9-~YpmDPwUVQqWU3hm!dPB$fmS(ps>IDi{dZz45 zG#0-P>8~8Jrd!#sv2Cb~y=`p;{{R(3-R*HwbC)P;lxC*1?q2G`jk1JlodTvhMIa(SNZhsr$8u=W< z6C#~;YCno(WZ<}+larZ->khqpP=VEf?#=?P`hdT(iw%ZHpNP-QRkfk%(yAPl3_zw( zi3D#lK;LYW6M>ExSzOzy)dGv7HDBIUtmKy}#uqpWNreCs2J;iZN2z@4sl7M)L;EV7 zA<53+olD>xI}=Bf;Let<4p4IWVRDV;REl(RxtH8b)|oQ2wt~e^^YB; z{$0|ob7(XyScyfHr!8c)=dgS&S#rq0z#=|kjd5&-u8ma@RMQM)b!u%vyO3Z>&H*7r zLoeqJ;)_$JLYs$cV$+q@kP4V%B;x}SAe@EM?xA$UgY^%tJcFNg6OAvYoL{OxPkgtm zJY|hZIT;AnxoEJ0wk$5eQEf`J(+jHDteX$ebNoq5fcdos0+r1wz&%Apsq;3Jdc@QL z35)PT9prx45B3sRm2wPu?b`V=zf=K+gM5PEK*nnjABX^_PlU z<5=7ExmQgn=Hph0O3tR0i!5GuC8laqMH;mhEHP(zQsLc;w8)W4At^Fy5X2Y&m_E#6 zcb&2U+Z;Ho4Q8QHq%|7esMCU1Od4cJ={?8hCUF{gV!BDh9UTq&eC8ZOUx{h+1#VrQ zW5PKBW51CGj!QQ(DmNmnmZn26azfsYt$Y3BKt9#g+K$ju9XCaFD#W-30oda*AVGuV z{kRULWvJDXr8>0InGoe)at~MxzE9b*2Ew z31}cvGF~YLNF;xz24FCMF;I0(4CG>C&vBf1QM41k2h4@i2FfG3fB`#;auo7SW0_+u=^w9ehTXBZK{+||_4qoqq}uS}4oh(U$acj*%t0B^Z5#u)OBFJQX2 zQ;UsHy`!(z4Vy-Vw5yz9;cVWzzigQKC@GrigA`FFC;E1e&xm}v)#_6ZhC$T12cUpK z-(wku7|daKF5;wGoq1?bHj|JfK`=te$UXjO>VA5IysDhXLd54?eO(S0lZTUKF=oIc zFW#inwk2R@9a6Q)a@N!cq|eAyxrp!awOYodW|7l45wgv)n;dlTU8^JgP54GUMM*6QM_9dqf_Qz}>z zAx3pDF&G)&aCi-3;}ly&A#m|60BYdgJ7A*`hQ-;1E^x&B4f{Nf=Pj-lE(&d zrAFLrL;!mcAoh>16pmA2QFk_um)qvv$wRZ0rblA;E3=JerO$rB(eW$=NZppr&6EXGwj2>xXDfPj?o8%G7AC@kvN$E2t9|_X33K$iD8r(+<_gS4fC1K zHy_b>ak@drIS)DxZ$_R+G1{-ixE{UEO;?Tu@NPbRh7i{ab|{z?e3p2Xb`wc;mt9gU z^AY*F;aJFmAj}ds_Y)>S^R_-^618NcP-;|akDwER%zFvPb;zHgl^jHYWRFHU8gjIY zDLHw?6>2pERV`j)Rw*lOv}C0k6+X9I^q;f1rddDf9m@)Ip+J@lBXF`#2S^@9!^xc^ zRJsYNssS(wWAjGa{Kw39+lqfhGA#0q&M|A4@{dji9z*KcO>HQ|%G7dfon^`B(S=)k zfCyLVi;7DX-?&T+i9@NyHJ6ry0yM{va!Chqm?wD85;+7gB1G;+$2$T(+NXR*XNmrK ze}~n;o+paj;bw;{QmNPtM_XzTS|n0oGV>V{6O|dr%VeaRb=wjw}3$O-UCCw=!A8biHqq(Rj_8A&Cbp-R3z9z0g_t|*ebdCidzxVBKX zOJrCwb|2y}5K}Wfrao)~8i0+CF}J*lf`0v{Q3s6}4JJF1(-L#w$s_3i2%h{aeQ~X2 z+SysReHoUiVsbc^r~w6V<%<4`+_^cni%JkFu*r{afRsdx50q8F5D3c3liwJOdws{- zf~Y)6(g|RoZLu4Up`U!?b(!O#(7b*xc8zMgHvZTg`_-QSWfl$0ukEOFU^~Gio;}Gj z9$>KS1vB5zN-+##GDdxm{v2S)%uZx*MQ@mAIs`}5&e4zNMl+Me9P`7O{Pcnpz7e793k5GEuqF;Q<~}Guk8Mm03kKlLuH5a{$1=1IRHk;2pyY zq?a%cVh_3B6POb}SdKV4YsM++@-7*jvddJdX>xq}*RR@Y8=sd|4x2EeVkf$-Ta>5} z`3}q%3j{~RM2ny`vh_evBROdp*v{fIyO1*`rxCqEfrrdSGZP@e?!@=R!0u1wJN-#( z=k*(%Zrah+%lC1N9Ihcv;YPf)tE0;0Yi)HxSKZr|Lc{Le`v8QD!FQ4=YBDuKSjkXS z;KVm<13zfqBLH!tIntqKI!g)IkT(;YncM6l6-b3;Mus;Z%(+NvR7Qo{HR;!My5+AD zV{Ns=9xY>!0~E!1v5dDp9HUZZ!yeAQ-}M5?00*1#oMxb}=qh%t~E z>?1w@0Ap>A3m=q@oDI(8V8md=LF`8X-2xyeEg}zZ#&j9%Z@6^(tqx*{U}w}t1s+rKMUWbMwEbcuU^5XQ{{WXCY^Vh6 zXY}vwIKo-ds^`hRXFGB~cWNot(zq%?_;64-!8*m;;|tLnp#@OIO}9vxb!S=-<} zIeXtvR`t}}(T}r#>Q-`&(W#9$ki;1P z5T~TeM`1I?f8sw@>gdsvqK!4F+zIg@86?MSPke^VFRT8gb3Bjn0_FStC$q?EkkaD( zcOQ}=XveIA_u zFbKp;7~$V5wx#`*Nh^&(m71cSs8v1Vu-*t<%pB_Bsrq2&ncXMH>Zbnx04&?Y>caS& zDQ?@-L2mLsZOXP<;<=|!R^POwT)3>XO_Ea8o(rvB6oh~bmQ_~KYK?A{g(p-i)YTcJ z$z~7+bDhpGI2V}Er&`|9of>+r#bWTiIuu622|GqRi7IAUjLzd7HRhWYIZG9l$73Ib zV%`f{)O0N_zO^xFSXtLtpWioAu`1Oz93#m4q-F=l;nn>zwCbn|R@neVKy=hq+~gA^ z%*$sGBpv{_xURI8i9-6TKC&PmO0F>a3qY7*?5%bKjS=U9C$q+=#=`CyM%o7KA)BA@Q+*QaM{ zP%JAFO(-I8brRAB0m;A|V#}3!7k1Bit}VsgK~R|`X_k6y7%&7(gpT+M0n)y4`qHh6 zzfxT516zgB;#<6blvU%`uxUD7O0}E-t3`{fXEWR_!iL9$6!?2%#!4s8<$mxxUskB} z(vearGE)aYYH9?6a&=(;09el#D`lp(l*Kxg7&-?9j1n02Om0-j+)hX!1srM@OfWem zEHrZdlVI5ALDS{CoOK&(IZki+Pi0kQ1IH0I)hi7jTcON_d;CQ}MFa-~TOcsiC>9Mx zLscR$SYv!i&vh~GM6J0>3>uwq6xMG-ipN!4u`trJG6ZMFB3m>5uQIf2^#ePXa!Ar< z&p*Va%Wpef)kjDbp=F@7Jyi&jrKKPX9`X&I#Vi<-A`vc?i_)u7e}00YmXnhjkAmCw z@rVb97OWQ3qGQluNNod5Ncw|=Be=-O<4*V;&5neBhgRP_$_tx*JPT`aL-zVuALaUc z$z=^#$Jj+dOBk?aWBSBq#NE%jz3`U0Ie6y>3B1N97~{4m-)7@<0eL7MAKoP#iJWO}oNAs3q$ELie6Y1RkptB=a zirD6S;;$OZ``Uo+56JVd&{dsI&^dC3-G`~ri%>|=S@wfrqy|17mc(-Ex7}b0;*~ZZ z>d9CPre~-zZu5Z#{S2C*^ifoQ>Pah#jjANRI6dDk}|U}Gd7ai4%e$IlcB0Nj%U%@#Ij0M z@oW%BpJ6$VlOa$NCL;hCKh6mm^V)HV3DMhDuepPc(Xcc1IDxi!(@ZX)RHe~}{{X|h z-X1mh4H_0XW$r)bgDa8<;JldI)Gm%Gm8YYz zl30vP;0O`?bNnYKg^yTLu9WHut*;@1lTBB#W*~u>fdgp7o+zDg>h~|?eGNWodw}t3 z^!ITnbnj(bITd4x3{ zqo!aJAOHyOG65rlZ(5z&)1y=}g3RHDKnOFe#H@q!bpk=1fCP*@cFn$`mt3l1@t#3P znBa1m{v#IFZzRvSD2*DF>v9ac%SzZNa-RBiRwi@#8z3hi1?HC*Hg`1bs;0kGt!k~N zN}alN;BvG1?Jv~lEiFWbaRen)w{ku19XjZNOZ(A?2%)`6>-X-GmmWK^RAp$+S)ae6K zFwR(D!1iAe^Knt;DK?cHJx2|30+{J_gkW?oKmsr$s&_mXH0$u%e^Z@OlGhC_x6twKvTAdc@ z8KF<`5t$M3B>dKDS|wKy4aFp;Ac{txsGTq{3k5QJk$`7Xv*q^dZBLk4+fG1wVsZQg;cwB=w14`XcvcI?^ zwO?KN8R3Jk)R={3cdrD=;q&9d&6vMXcbhd)u~jw(;(mm!0!;SF^xm!U=JEx($`kH7b>b~Q$Ycl znG!HHEUbb->@o>B;5Ep(wkF3%I@!{EK0$LgL+m-{JCw~Zer>Klbyl58pXO|WPo~>V z&h?}iU|tezSS+G`6cU@%uS_#idW!WWK?q4%iJ6u%2n3uB@x-*y)w#E*)P+F0Xb?_Q zOpj0@#0}0j?%$U_x<%7oMajN~dV$mKC&;f--0G(8vMS_Nxo0tI%&kYeb#+@H-2Dvs zeNj4<*<0?E!dcdMOcOo>%BiVG3P!rUBrwDg1WCaJ42Xjr;1W&MKTny^xacCDoo+;h zV8#OYI336*eL_bT{{ZmJ>o*tA;C)Ne>(?;b*6XfkJ=x>D>YaLtxpO;=aSGqU@hrUY zCD=Rb`v@hEAg3bg0dqTXr>f+snS7vbeD*+W$6;qC#TBAF$f+s?U&8@C| zdr!kEgeV;g&!)x9v+9KkK?O(%V6HVtZVugb>#c4`%#6A^tg^h0Daf=nIL&I7XytLL zd)6~Cvg3XA{2F?Gl4RJSGYVLZg_oQU$?M!PK!h|ocv7YjK!8SfG9;42=9v+wsutyc z)<9;eu6xUYED#+sxsb>5!2~uqOMbEX@y~hcbvji;lyM*UbH;ftKCd3n;u3W-cOvFU zDgMgqpS1gYo1puW03<8AMshJD=Q>unl_;#9n*}YVbh%x}zvVJF_S=ap?1dm4hia@k zo!GI-Fb3c*H=VyI{JF{gUVr{m#s2`-mB0931d-%^qk{bx@W1e$HCO2mdNTKIRi&S( z-9ACmo?XYYax&2izp1IEoVx{G))fV2twdI-YW2p~fn}(!q_o8(aU-sb78;dA`w=it z*opkZ^_9}-Aw@Mv2(?j(VrE7F_A+35=e#be^z-WKwvSwMtaJU$yPTJ2>CZE#wCiYM zH%~I`e-WzI9ySnFLM=NgvQtnRel+X4#j6ZI2s!x!GO)Nb!>_DWs}OZDRlo$}x!4|~ z2M2{$mNRLhh zRHw7{>zHIf486V7rBCcbWBbSaPxSL5 zVYuk-NKnkq*TS)!Tr&xLR-J_7omSt!hp5$SY_ZYjIvWBHRShtk11yn(Ub&ZMDV`yDZ|#}Y=#h4bN>LS6#}^cHdW;gHo^qL zKuTl0wT0$=;s#=VNHs!T!bDdihForQpqP%95%{5y{RvfXrPOu8IXTi+Iu0)n*J zWpu*K$esMm>>w$R_dpp20G#;M>`X@kQKshEHO8$>BXOg!!yeeoc?*c+Pr6Ch?t7Wl zj#pN#tclR;&(qR4zRxt}zKxwaG}A&_y{#tY!jSG%buJ@5sw#y~w}7c#(*zZ4(jZnM0bc&tTSFZLRMFO_9 zM19GLkF8LENYto+sUucI1~QUMM{j+$c`Dpet$X-zUPVntAgr&PgcY0QIxJvq6T z%Ca=mxW@&HoX1m@VY0{J0(J!B z>ykTm&t;!%a9wSH83?g~)a{TL^5CqE%o8}-nR)@z9%t8Xx9@WfP#lW~kMJE;>FQ}_ ztB+ir=42%w$qq zM71)yssW%DVWhFdmS)uv-v$Aho)wR#{y#Rk7dF_`<8QQs8af+y*lmjHz$@uSMqP3< z?M3>vRp_@dBP=Ok%!CL}$*HNNt(=kz4*B_j#y~x!$QX&?g=|915=w$6>98l;z9vC{ zPa2uSxeo-{>fcbfzD7E@MJ`{#Rf)@Ka2=Q8*W`6`Yn8e)ZuD49MCY6~XGK#67E7>~ zmva)1rA#m**a^nbunUOnSk7_|7-hFgYARGFt0U^BSwSkq%!zD4h=G%VACP^+HF{x{ z^r!WQz`5L+U7Yu~gzMsBf0?u?Y$C{pW*()w)tK!3TkwVY^+amZuW+8*jfjzh9j+=G zVQJG2eL#mfQL!=nC%!+uD;-+Xdf$oH1cq=3Z84{F-gOCx$C)SP%&(>^t1e%$TbfkV zZe!Msiu`l$+^W`MyF>0!rGfa9dqBLGRwwQ}qFLG^1dXIjl2j{i3^a+4n|1;W$P*Dc zJt|eqm5@jS5(2Rxj`;d{GmHt;;q^nO7>L2+T~fs3+T3oxJKEB-lZ5rMLW$#QeUqlX zqdWJ41jVZ$KrvmeA6p~_d}Jdek%PDxh}@|o+!&8(Z_5QLs?$s~vIW$mVL21NV^4wK z!DYi|i|2Aapl;%FOSi-9RZEdj@jtk}X|-Iw?z>$JSKKNVE3$(H9I!lq3SxY|YQHo= z1INFe;DNOC=jKrzGrT1oVb~9pdGDXJ5+D+`b$ZBWX7Sxp$F&M!avHhUbV_YE zLSm$f)bg+}C<66?%~B#YufTf*&wr6z^u&>nNsiOxNYgtX&Dg|B&2EI?K_HIWNCIS$ zoDv6rfA*=}9iw+{v@JC%=W18dp{j^uSL!&g=Umb2g7cqrxumfv0GSx_{=wr72T};m zOz(~K_yg@2_l_p6Kw&+UW_QW@{{T;X#|%#(z^|Yj&tW*24Y8ktU=$km0z*LSc&qIW zkBWk98)L{8FJmEL=lVf*g35mAxWNFx^Z7=Zm^mlL`#>BDT$qU+_t^cFf1vsZ>IaR^ zRkS&7uL5-0XFWWFCXHIj&3O4LR*+2hSTQOCNWNIp35YY27?~U^H87?m#F;yY$sYLb z7qOn8rwXBt@M8uhJ^Ob*`otgJKVypBqtE%nD%z)*R?iC>!~8@d7^ zvZ_|(wIuU&aBU{1+?`t#4u0l|`q||=Mi35a?jK~7_(FrrxkN(Xib`4^tT=Et)4%v_ zk*W!U0AQERMS5yw2#g8opX9S1+u%UQ4R;6~t;nV7YNX3k zQ#%t=P+^E=-*7UpG@kk4C+l8Hu={YE+^PpkoV+dl* zi@;)F$fx4k=;Ev?EV3ksk3V^noIvt^R(ACgqG~EM>C_1j+KzPgI35q2gYuZVQO2qN z0GCG_6^(16S9Y>nE)cb9yMSF*)5Z!AhpZ$_!c2rixP+7!nbaU4T!{XjK!N>F+ku`K zkU+XLDukblQcW4w1@VQo4IsXf>IPH^C%k7r%nh~x9wE4>&WIBQk9=HA?_Hjx)D*|f}j$26SEx2REsuNEti%gHx%6J(X+4DlF<7*@)yO0M+i)C93G zK>HOM8&_JD|=weR%8h8iOS5VGz{byF~>ye>{$Ew5PZNp;In{C9;)j<+u#)l}X_;(XCr9lQei6e3`80-eb zuwU9V=`$juK7*NHqcD0#N$w}!I}^iK#`+t|b9j~fJYHL450IlygKJ&b(Z^de>3|pB zRSKAvU0%a%l=hbi)ij@Ia-?EFny)3O^t3Fa0wWu!-ZF-MpII3U4aNPqsq0lwP(w-r z7ID%uB1UA1@>G6WYQB$pqqEFxat-S|Uufl)m{xbb*xqS!LbzI@%5i5QDBy-5CD{6YMJAw%wGBdZB#>~S|<6rH|5k!RbP(cAelDo{T0pGI~KH^u3e^Y*p z`qq5UGnH}c`F3mi@@VK}TeuM2R8n<+zN9U(C*@wnL6oiK<7GZ`=`w80fY&^dr9c{$ zWL6O%u8|s~fMnwUd1;pqiKN<(8OfSPhe(u1~}1vo%EA-?=REj zH#%`wit#MoF~;L}jNZ^~Z@IIQoA)dpdEEfBevzZFCcb-EZn+{O+wlad z7p>jpKk)?JtnwVyglIa2M?K;+)(h-I3NebpUa*LR5jZmWxt6H%%^yi|>lFbw~*_8_{}dKw*}X6BDs3##?`jT++AoEyI4+zY58su99@- z6;p%Pdug_nkg_IMZZuUFwZWa@!=YfW2>_2c6Q7c|sZ=Jqo};A|QrZHj87iU)`A7m| zI{=`Pta))-c6Di{vL528cBpdo3*@Pm5CZiGT+BcnCeN>~u59pM@X+ZONNU;7W?xgY zc+G2Eha|N~Dq!RLZ`#eU-*8lY+i6HvxtyBhm=f}mj?feJJC+nFQ$`^fO0bME2O)AK z3DRT`4DA5$Ev1ltZ*x^#=%DV5rZuxV;E^E06B}j+%t6$NV04|lC+KrCpXhYzpEqAr zrMIzpVXnyB;goQ0t&Y+V0I&h44c z2fV4^)1&c)Jz^4wDo{kVLlUeB%mX{)!pPMD)jqfItv{jeg!ERW+k0HAighQHbM8lj zQvH*yjZ$LZ=B3n2wqOmpvbj(=<-RfO0iM!Bx8le4VX+~#3#PbcC#8WTs)aBl2$H6I zokSinZ)a6`#m(Y@_bKRZiPr{VY+eEJ^rF|1~afd zLA#wF`e5ZL<)f2Fn(~!of0Hx08SZh87iV8XiSlDVxUZkFmvk7U)BA+2%X7igX;CaL z%P2NWZ_ARSC^cLL>4jAclDo-(M3E#ck@Ap9JV$qMrkfJ9$P5TpeL9A44w*4Mik|C{ z(gf1rk3Wg9+e=VuwtyME!gQ()Wm-Y zld*s}>U}5huA1`xsBb%xUC)j{T|Cgc#xnUu?QS_@_QYwfi${%7QarTwGf7Ep%J4-h zTq)2&6W``}aJ3c6+_I#IrK}P?T$s)^EGZs4_u?L9TIPj1K7zSZSbC0t74*h>i5OOr z1a4+ob}d&{yL8iu@-C6{-jY>sQTe}6xURN8QD|>3S%*sAD%DlndCfZd6|G!NNf)6s zpPY~YFTeK@EFtEkP^R|oZOuxJPPgK!N2^H`AKd~(l`JK<15x>87IVr{kgS&irAN1F zPfS~Z(ys-(0x->jPgF-p1BYLs{W9u5OZf*(YClV-Y2w;kFJk;)?p)O44%xr8K}*iF zLddRWl%cTbl?|4m_9b)KAFLd<6b%KpbiJikjY@SHPK=RJ4yE-MAOK8(2Vt?6Rm^Hy zR%*2ZTpE;a04Yl%n$pcpbl8lOq~;_TmYzPkf2YMaexrJ)$8z*@QPf@^hsFA9k-l1p z*r`OQ3345-R*P*?Nl0_FlZRa%ZnOM_KE`KVv-1}9{{RVaZJwv86umUKInrOL%97he z05K$NGI$G^6uhNta%mb=4M(X`-%gfgc%~_|E)JxZG4dGFdR3Wxj+WZ$7WY`R`Hv7S zu2qF|ioaKoU*rwm)w7FhJv8W3#dAhTk|??Y~Q(Mxwp4kiBvJ_Fl8A)I+*A^G5M!f2Bp<{`GqS)WHqlYThMxur3u2j ze4FSF3yt=Ya3iRbHC(=jc0%eN1lh95_Cnk=V|mtzM;Q{zA(v(Byc$ zaat0;=eY>3z_yNQc%s_J%D59C{t0c&QH{bbx=(Fip@BLJ13F~j0*!zJe86UUWx6%% zUmjoRvl^6xI}G|^p^?}_X;ZNz$o>_B!snUL{!jGd)BcM6XYd}G^%E-xt$K~4^G3wVR;?_74cMl<2eH7{EG>QktMz=q+XOYh%(9hMjr>jp-JhJ^2E8A5w zBm1T<&vlHjN2AQz?#KlZR^2Ita>q`p)oG~)NF_qgZQHmUMMT`Fs%lfFfPN*F%Ty+s zieONw2Rf%O^Df(9ag(3oY14ie!G5KfQR%YAEB zm9Xe^%(mzSa*3Z@Dnfr|6^S%xyehfWtQ2&HLIVMVFaSNg=6Gw$8lzCrUHw{=@YOpx zCRF{#05LyeSuXp24gUaW0i zi3gmJl(gH^E?CBZ25p@l0!oxi6f*iBtZPfZt7=LPMP8_h4QUN za85gWVur6HXCCKTc`CPC-0HR2s+MW!!)vevEZeZwEyU;`6RKU>!*qO&Px%-)pV7Za z`;O2z%yaht0IB}HyKTTzLbC&rt*=0ii=0;v*wxK-{ne(p0s(OY0qKzJuEiriujjv) z$xuMvWX{_HHuodWH};mt>-@j#`tEVDQWqn`w7xZ=y* zNe5~2@bDS@On2;1*Mg)R^qhG2-`_aeH=Z}9C5B_cos8$-1N>udGJdpm5uI(}`!%@* z_YRGlS%RBS_0Xe4FA>dgpE8IGYKa?7D35V^%6z?H0tKx$SPI2~X(nU8o|(YgdF&z> z(1@!okaxsJV;LrQI4$oSYmZTLY;hXBFWS9@sZURtbS{H5boRDKSa;$su_LFImNOHD zK?zbT<5W14G>63rPDyTT-7TikbE; zSN-Z;sM2c!24m(D;c8SU1O-yN=_IyH=1xe)XSaPT>DGF;)zq~@JsoN^6G?pxzHz0B ztf?$cI#fs;895drIiC{XzePP?x38vLD&N!=vFzjWvUe-a#N+iWUdrRzCg(A8HeX_` z);`(Zed^#IP+iVMGf;}oUJ6pA(4-!%EEFAG$dJP_ap3Wag;4%Jf`H6ujVzJXP(fgK z8A4ffh$aLWJYvru(QY1MKZB)Sz)-=ahh~~Z7oLx~s?ocBaX8E-5jot?QSEZc8y$`Sevv9mC(_D< zLV@{L*X?CU{!-&D{l0Hz0E5WpM{x}-(g$(EN{Q<4OX zN0KMo5>Fi*n?*-wd3-@uv>m$MHd(qGlve!+u|xs4smznc8h|fy(z6%~Q32gzGc)r> zMUw=<_89x?9jCW^M;`#}3slk?WPz12IhbEQ`+jX*JsmxIeLdp;08+HI`6iZb&o98I zj;eY2;C1zN?Oi*X3D|#OuSS(BMkhWUfp(%)k28XT)t6v^PD>mJ5%ZBUcK-1>@J|`i z?NWoP>ZTrQ8G;v1Ae@q9;tUPso|P*Cii3tDnr7)WZQ`0~)BZ(X4O_E77OUCd)yy;Q zt)g2gl8-=`fit{E%(-By%F47s7tGON{-1&&X;2Jb)Q1wM(y3JC(I}kR`C&u025)5#f^*57N#^(B3mH8tOni*M2 zr>kPv9ouBO(udbVP%^k0$}>#XC^;CB5b=_U_$yQhS=g8!4{tl?%x5`(MkmuSK43@y z9ORr1@tBNnoSr5=r@o##+tS{ZaLwMXYUt!wu)BBo?*(BYx%^{4kkl)=yH-2|i1oPw zd>M6)YXLFgR;&rbNaDI=EM<_vTL*w%pQl}Y4J;bq?n|As%&=HGqXMkU4SY8BtYCvA18kJq z3m0G@L&%0a2741qn0tTT0HxMc6AUD39&@q%WDU3sy)#^qkx8_)Z9WHc8*kMZ(!XDMPqgXO37;A4a{^`<&r@huP}p9nzo}-=4~=*%3^4x3S)lN^Kt=$T}0A- zf=R%PNW>g3PNiZ!ARx&jP!d4f8Pyri4%}+46+mKObl^vwgn58u=Xi)bH(dteyZ->@ zE_;Y94jc2CF_y0yr@rsM8k}^-P9hU;wHkbNRQ=RxsU&m-%Xn zcBxm?km;;35rdH+oum80ckC6|1az6W^e$#ot~4_E)00P$azzU5vcIiNn%Veh{E=Rk zx7aU<7CuT0+g`MU6|F3QqNt{SGy#G~{{S{;w8;>A1xH%o8-V45J0WnVeZ>C&34tR3 zdt{EL&*___7H{x=#Wqb~ zcr9NU+gkxhuALx~u@Fg&dF(uWj1Cmu^A)FBk_xpBo~nsVfuu3Qwn6s_JdjGj;hfIs zcLvP1+Z@u(rs-a!q*mh?)si$V40}D!eHv8Tp|x_;86|69lMq$zWHo50B3bc`Ds<^W zz;a}gOldpAb|Oqd>5OMOlqcplb=oMZr8z3kP|lzXZy*VsfW%Id*rlD*t}$9IT+PVO zF{N2dU5;UND`V$VLi~MIXeHKaAQu7a2(OlHF^>hTrbI}A=&NedgpBkKqaBM#1on_e zkPNdl6|O6?Ks5`Vl=KcsJ9PmDJQ#_P4D`?D^ByfqnksR;qkCAFSbgm53mJHevJ(X2 zL0MHMfGavvx^>Dk;xMy!(t0_80onXhY z(hT{MW5^Rca6hPpRmigBBGVpFm@63Rn!?HL+BTp8+y&TSD+V_^te5B_hh38OwTFOz`uVhyhZBn)Er+S@ufl;&z z@JBKhCSVMg)wEoPQcFqSBoX(B1dM&+VPrMUKae95GQmax11f%COz)HTU{oQ<`Y0Vt z>kl903b}TcPaM%JHZZu?*bXQ*{a-5wT}@q^dG;O)28*Q82I=yGpC}iap{X;7hK>{3^=1Hy(CmQ#VPV$ZFX7O@`QQ4-IJYjXRGcBtz| zSSK|=?cfc7*hx8zg@LNkO0^pF=~G(9l`3iproXxj$1|()!$|`%yyg|u`Z>G&M@M;n z#r^}w50DNQFbV(_D;FfT_A%b=oaI~h4sy|EDYretN=4}(AYj2v3YHassG)R&f*_L8 zA*M?LX8-{nBjwr%6+#fiszDn9oimeOq!)W7R&Wua%$!JIL*co&dj7zfgTp>yIDkXGHV{y{w*%D6pC*O+{B_s7{1py;I&#Z0XN| z#dR(&FKhlKEzPd0gG|&rRM2x!p*y!yk|S&k5Z+JvpPRw>=7%SHYO00?8` zxA$I$;p)*zq#Pr;$v{xIsNo9W6$4@hsWTl_3mb}+>@I85M)h8xN-9ZKg)mz`mpLJE z;jzZKaWAKSl|G%mvew3%8~V5fp{I&EA7O`or<7|=EWG<;TkWc5SgDYLcro+rCp)#z zIFT{d)14cYNL3XOYAzXpT$l`gRRnAyipd6lGMcYhE1GP;+m`fI`x>z<7G#wZre*|< zI=J%(^q7se8+Nh}jl_-BLadASG8WdWe zs@2pFML_im2-ZN2mm+f!C0AzIP41=rDfK^qa@BPwjq?7Rb)y?=qgXn8e~@Gy9%oMG zb@bKb%*%bjC}#n=oOZM@IZuKYZ0FEn4P1bu=XdoUmCY-=M^}4$aE_Xfs%W;dX|O#s z>Z#MD^%EfJD+KOQO8C{bYd57*g*Lhbu?2NH#S(kU7z`wk2h2enXh&N5o%D6|k<%QW zBdf(Mb!R&VANh(qe2o#WPVDnKBHTVPQ+Xe7+^yP3Ftt9p`)ak;Qpf@@3D(Er3brYYH%6RHY;a13J|iPWF_Hv665*S@G}+g7bnX;inP`<*L8q5!i%Ouwbc zrUy`SoEa^W4$YWfL46nLS0U=(Ty{4Dh;l9;$bav&bLhW!6&*37ux{MCx;5Civu*jc zTAZr)h+#~#U3Q!bhDeNqoi)YCtwjS;6v?}JGHH6Le7(l zrnxKyB_ma34yj0!F_-HjP$CGE1aYLAy$j;}SE*MWzT3hGx4`mXwK`wXvJPD$16A;QyPW)TEJ6JVLsrwT8k`MTWuUkvboGp!fMv_Eeg}B zrlG6~Jc5A%l|m%!kvYpwzzLd`ZEO7LL2b!)Vb>-LRMGRg&&z-!065gwJ}dP*cU!6- zR~;P7>G%0-m1;?I6V`*{88k4nxItu(J;DB9hKp~e;%E+ST%iZ4@(gPQptgCex~+Bi z*sW9_#nEarAOTXE>K~T~%2h&xm>}Ux8AmLqQiYw`TpvYQKpI`$o}h0{g6yOaWRRq6 z1n_R;{{Z6I(*BUJv2Zdu4_7m}UU}Qk41ObxgU5AwM*jdG{yv^H=j>WdO2dhIE3P%@ zgL1D`H!wV2lTpI$drVQeqSY1(RCQn=z|wFCFh*ypJIIn2P~?qsS^aXfDpGKvwK>wX zsunQAKfTPX!mtEnkhzKV0rg|XDg83@%-fh(`Ijx~_D$@)jx)*duuF>YJ@K%*dKUP; zu5Z5VY;l_jg(RAvP4r5drh^84Novle?p1Wxsj8hoBhzFv3T0#&Co(qOwH-D~Zd}za z4P~sf*4@D`Maiz{OaWC`mGsGNSP~*gGEWF+LOPYzezWmzE%f=ztmkqrfb@G^y-{xm z#;a2=Ut5rD*=;Jw)Z{;LEz_n~-V!-tijStBa}xG6*9!WTI)bIeWo9bTUZSuXH7GC# zQ36$B0TU7oPXzM>rD*M48J70(eGgQyA!*Z5@>2*1U5S$#p6vL6SjQO*_goE|TKCwHBxAEhJ=b zQst}8Sn1F)q-Gj6%yF6DL;Q*SXX$U~Q>0uokl<(iOL88oJDqclj%w<=cJIFq-imeE zfu(h?y<;ygI)#c^vm`-N!!c`I)}K$r*LrDD=C+DzEOQFR)qw@H)2@FGtk6kNstL!+ zF#rg8a1?V6Yk5|UX63Z4g*8MSK>~c6JFJL&!G z-L3AJbfJvrdMd0yyq;aiEyl)pKW9QOcGefsh(OIn*c2E_nfqe^A6-KDtEXO)w6XAeJ(}DFh)WsHl(|3ct%(*O{{WvzwR3Q}d_NMpw?dyHQzRn&dc{6AMRhGg z#vFgx)ZGWa7Ys7MF>}6EWrK!v3noz3Egx$Mi{$|a5nnh7h5DQg) z@YaP-v&aMWs+YSAKfy^qadcZ>xx}{<_?-}7OdwZIySzSWX<`m1Kp0jKfj+V!q zz>V^72N|EBTypnUb1+o0qs6Ukbq@}iSC+Z8>az1aP_uw*YMc=jXjz9);8 zy*o_By-x3fvWqFDWe`k)F+xsaWx_KOHRfREMk0F*49rA+(9}?7NdSV;*pt}rkFJk& zz|*{GU`Pwtp-y9m8unL_D&C5EOjmS1xKejwz`NOuz< zKg9N#j}hn|reEO%Pm*UbCLkYtdCZ(MM>7LEZ8Hb9<_1q-IM#2fJ)EsvZ=|fu?nA`4 zd4(URjw_MK)r|lXPjxG|Xt1_S2ujwLT2vtDS(%_)FhT}iA2+6iT-@p>sH7PrFc<`Q z24)zm7{+)9l#tYJS;n7I@~uFw04j`jBVuw%V12V3I{yGwojmB@PJWyEQ_T1+UR76} z@#_2uyt=jSL}7`kXGWE7L5j@$)ST3ze+ukF3awHWpP4Z0PjX=dwa0jO68H0}x|~ZT(|)ajv2YEv$_I2LK2m&H<4=@q$`P@TT#*oQtY`BA+)q?&Y-(qac&_D`r0VN_dt?;K@Cz9-lJO|KfD{1E%1Iy-ld%5r z8~2<}IEv|tRiL4X30SKtWDP(;Zkz>){KUxGA*f$XJv-x^tACZM_3Js#y`bG&nN%t~ zjIon#Hg+=eoDC4nyKK`ai!F&*)b`JrFhD+U#P!^$4)_Y73=;}Sl0>NxCx0_LYPn|K zS5Nwj)T~1@e$_uh&k*9F*xu406o9!ct2d> zTj^$1WFUqTrX&$DxH3-rdjq!?zKC?^lJeWJsbSfXLcb?<9zAmQn)4`YZj3I_;pf|X z7NWG(u%^3aO+y0^ULzg8V6vzrkpN)tI6u?5$MXfgb_jHUb!gS+2O6U|pXHGoac=sO z;F)dF>d!TbwPRlI7*<7Gs(0PUva4>Yl`zyx+vE1vRgWo&b+m|(KGT%5AW3;8XG^fsHUo zhycifF@ZXuYCFWSm-h8(xCnX+ja5~kqJfC{Ovs4pnK{W-iQs=wH24QddiT?9J{?Lm zw;bQIiy6nI97`crY~IJr;`(-2CSNrRScy@kbsn7Q$-u^A zzR|u^2;(<2Tk#ZXBlHSrVocO>nradca%Hy>yonqneJQ`GYpj_3ubgp=hPw>QQy+$E zG1oF)P0zBr^L!iYWsB|xDdrv2n&cp*caI}E4LX9AMq&w2G{7Ud5PMCJ)2E#g`;V(*a-|zgcZrd(mxnF|h1A@0j-5bmidc z2MW8(KAEZLR>#@Qv&1?@fsVa1#2j4hym~o7V%0L%K)`8j+*4fOTMRvlQ5#}F;C@X; zqWXbfA|Ov}eGj(eZPT{^QF4_CVx)SkS{tidHbi!e6fj z^%OZHP$Y&hSx3`xohNY^3IW*qZP?b=Murm#Xu7H~H}Yd!D@Ox*vlZexOq$9vr@fD? zlwGWcp=oLu!_Tt~Hl$jPPy4`VvH=mkkPma-SmHZmi5NUe{Kk%g&C?~Lq{uy|=YVoH zF}OP`G95Vncct3d+;5p>a(!&dEvglF(-d_yah{jzMaCwWDV^2xBtSVL?^E=uDvAR+Z!opBbt0yLvHFnA0R3 zGclR_NrT_tPIZbeZYr>%Y&0Eo4?FB>a1v1)A3 z`JEUKqVNu*o7)tHOaQD`rO!u$V|O{k%?~B+0^t)2kR!Im~0W$8(&H7u`Ei ztx}?alnYJHObieN7$lR72^l;-jAm~Z$6hZEsZRwsIptQq`vHGrgOIU^QuC>^$#J1= zFl>gH8$^p7*`SIX7jr&&^EnD!0bbGd=+KZ*oy zjcA*0NrgQIrFc0;;a=Au7_BYQ+}?RiUW%zEB!;x9Ai*eFzyRc=cDM>FGgu zHr}?GNTn1r`v;iUkz^nAlOrN9T0PW-CNVf;fwyzq=Qx4?0Eoe704j!OE(nrGX&pLq z$9dCt?g>iL$BOR71VJ{k3@bq7o;`hf$L zjQ;?!lX-i+jj^R~IY3^>6}>=`bPRTqf$oG)BZZQd2(YkfWkvxQRFkm5EGyJIGAT zkHQj21*&N&u^W?&=Lzh+zSgL`Z|U#zFh{;Fpoi`ArIXRo|Jb zKr7hGTGmmb-#Bozm`yTu?m|W_DOwR=hjDQj9BY+fOnrsJixRJ2~OC7O-(3#v{TaIDbb_rbDI(e2c zk;nI0ODe@#yao(RZdh_H1AWB=khqknpvGaP+g#89Gh>m^v4+k^fWAG4Wf{w5 zo4K`lI_AxN-7PXOq`h->S`BLfRha7%Ulm?iJrD>ALozUB1dI_QSNKc}Z6}L;wZc@l zMy4s6XuU-km&y5^B0v$;Vqy#vjuF45-Y0LWzg7GRJwfG9>GvI>&Ib0$#$h_v7;(#k zX$p!jNh)e8E61E|9(1a>6p5txr4jt#B{qJ0@= zdQb@7Tth@bh&zEG&IC^$tiGUrn&|c7=NakG73%fAC#%g!wz)`h-X6{`t8a}`sugYB zy-)1w2k|;|3c8gwr6CM2XNbTMjLlx!veHtu(VP}g&}tB%2`~;e!5bdhPMZC-#cNA? z;L;W8+m%G9tknX$5;Xo|NFrFD<_4}jto>H;3|^1*L!NY_Cu+Ys&Eb5;bB}K{4yI+M zHj+xnrmcbrG*xUu#_$^y_z0DejQ998M=H>>q!IIOJ@z1W`hR;#UM!rw-m)&R6jVuh zI$~$skWTOkCQr*6xG?<>@_b$~*FK+eth`%?rNph4;>#yD`5O-kbzJLmRp7mETA~5W zkI0A*oG>Oo8r)M__Qsn6;ffGvsCsM|eqP0AC8O;e6L@K@2c@bN(?Bz{#!1Fx0tga& zYQb+q+<&P2kEz`^i79$FhK<6dM*-;6sifo;h|>@VcGyXcQGt`iew{iASy@U_rEWzs1CVP~ zW%*Qeh6IAl%CZT-1j{g=T7OmbD|E+_{U?13=G5nWisw#mpE&0koTk8I^zNlPYyLuY z_b#H5ro0hfwMHr|Qt54Zg_^LY^o0tw9IUwm04N~NucE|>>J4B200y3u?DDRXb(?#W zQ_sCqI!6&-fvFc?q34G#%xBUa~@qzgO?-2HRYr=^>XZJ%d7F8MO9== zWu>WhoA(kV&ic*nY$#Sz?WUq2rqtLcrohXkMg*En#0g<5GawQ;+u@F3PMs?DjsE~= zxMZnZIy$m0pbC}%oeZng=*0b?LSxi!qi1!~j$`!O2dBM4;ayO*)V!;EeGj$Ab*Wvc zNp+~x$F9HUe%-e1XEp$|NH!Q$DuIfPipqPOXjm!=-w~BSMmwi3)eoe%ApFk;2hSyNI z+-h{ED)%17)W)Zl?20z(a{6y_HZ!$!TCi?4e3od@(XC=tc3})J6?LfHQ9h*_h*8>$ zwKft6R#TDzpHX^rg5C`0NYxUTF{n$_T2&U6WT9N^U=Eh+K@lihUUIV_lDs2*cIwws z^**I~(bO)UbpHU6aJ_ytm6N+>&TUKi9Gj-GKnmDy)W_94>S0Q2Sc*@s6yCqLL|dqt zHp!CujyUz5t(lp~>2dwerBvz$3OV-J0c!KBuCArp&9AiM zjUTjqiFz(AgW@OoEKM;Mr1>%dWzC&&J3~XBNOdBVN#2V==5KxFj}6QBQ+@j z6%J=W0Y^?Zk%7hcjr|S%NPS20f2giQ9FwN}zIew>vH7MTTxOcwoiUUe!W#r+eH)7_%er=TE2=wDdbFCdfR1V8thB=!8gj&r!G!P|-%{nm z(wb4KDORj$WQR1gfOQtrCs85^P)D&mUv2#-aDI;Uv*}NYb!$~T*E^GR(^ZCzj!m2X zNYCmeY6^B|4a4MIO?MpTx%L~OiIuXDwByULtpEj#{FSa&l^bh?bTi?4wD(-XDLDbgLu-e%yB1KdbMOKIcfKRv4R^Ii9 zP|RkLg&~`AK^mjB3_+DU$rF_GHml7)2#obsrMpyjMqy5x)54`iAW2}PgP%B^MG5}^ z4qZ-?Zgl$-q`2HDDfK${Qnvbo$$7n_%+IZJhGn68bvS+PO&e)tnOj1-D&j?XxXS2r zD>z&wtI#pWW+_SsWr)sox0PafU(umDCQYXkmQ|^*^YX45)0-=_e-A-Mzm`)`Re|(T?hhscB1krBP|~6|<_&hL^OT1Af?8e0D&ecsxu#cHTZ>w>;XziW zNvBa~q^pvX83KBMg}L9m)UQ~jT2*;{N)%B|AvI0zk*vOn4M;YiWwjHgNXXy`{vr3R z;{8VTUsar!D`v)@7vk1~OE%=|52H?krIqjLG9uNzbrhJ|WUts2imV6VU_{D#n$DWO zpnWKUa=V`U<2@l~>7=f_w^LK1U416j(vYfVHv=F7P6tp@F*ysCU&v>F z1K|1>$MZku!|odHC)R$ibi0yYq1O(cSL-JNXWi7d$~fhSqsbFS(~q`;MLn&$`k}Df z3-jFBdH!9Dv&(s&`)<+n65Q;6<4 zZHhW&+i-kRRv$IkXINJdY!o&-#LTQCuy&H{Q6q;XvZFf=%k(=cZ#JvOGo^Lmk-!l$_v zls>j29=TYQ)RGy8!d&U|?d9;gB$|$(KnN8{873eYBw}+U$v?#aa4i>NDj`szYCt3N zDA=51++qy;&58EALkS04=`9&08AW zw>q1iOS(M|H=uTI$1a$5#iDAZ1} z0FOB!VCHkM916M|?or0D`qMy-dz@Q{a#}Ctd|x2kEz)-zwV{w2XnZO4t!L9%SeOfm zn&9gZYDkO#XX5QqMP&e}ognW9AcKjJea7H#CQ9^BJrPl$)Jaby7&=JQutfZ)e)G~# z&q5i|e8GSWE&rkx>kfJ59Wch;w#LrE$=wDk0g7pg%Z<5o< z*ra1}bum)%IZnb~^R+7<>pRc^NFUHE}@)x6**GN+c{M@fShg3 zs#dzEb4dwBwW=yBp*q-w9WAC~dBMR16$G9GU0vw~E=~2B%KCf7b?9)un(AzGMN`P* znD%)k61|4E9c+BgE$h2&O+6GorAkzc;mT94LVyAQW1~@nB5G!Y`G|safh5QoGq9QS z0LE=@RM@3j)Pbm304#EYmUB-uRKJ>oBjsb`5#C_-_S5Y#JV{xU zE|%T_24~n#6Mvx|!WUE5(-K}UDOVw&AH|w%<=W#`tfDMPatan|yDTKYNS8J(4|cjh zpgTlNN9G8|O-fhHg9rYz+XujaPYWw9u**0i3>lH!Odb8cxyWcAtncPI=U;gDFXFj8 zyOHvqXT&pZ^M*Gu+7-5WpK;OS9Gp$TX#gl=O_NhckKWn?R8zX9W4QM!LaT3sRM75MGk z4nbp1_x}K1e!6;jc9N*8g8<)ZEGZeuciU!CN+6s;3}m#Rel%W3TvVl0)IT#y9^mXj z9rY|g&hx^vQq}HFMi5m>2S!+6GFwbE`C}4shRIQ1O#X#|{8J_*)ADx}X=r2lX-)*PY>MoFR+I7jd-Lg3rG~HrN(>31} zHCoX_J4)q*CZQRH%mly_Ds~hM0F9~!WxVf==NQ^ElROUBmDMFhDQ!S25XNHwi3cKb zbpsIyFviX=!TL11rwi6Zae?@-U-lB4mN_t4(cA zXE5W}uB3#M*^;hBh@CG&7%>F3+P2ACBe{SS2g&~c6pp@&Q*fpj5J?Ju#C9fQzE2R- zr&^PdrE~e34~~!o1GX`loJSJ)zfV;Wh>+rCSCsI6L3?F8{CbYH9Z$Wbca`NPIdCoOQ&k4OlccH+ zmM*K*8exC{F@km>n7$Xm@%S$q-J_I`W|C-7Mr}qiA~Q4CY)u@?MqanIbgGM{LRdf7 zaj}yoKQ*sa5|Im*jab}&b|iNbKXIrMI2|cgtn^)!8Y5czfS9cZ=Uf~Uv}y62Hy18L zQ4h%WTC9nvh|jXB3LzCT{b2T3hNuL6qb&jQRLebqSXOc3Mn4wSzEo0jL>a_n?fHp} z>@)1K!z&vXdiW+rpi1YY`P(NTW;@Es(jC;jyq+8mUJX=LkgcUmMqY6<6D$nL zf#h~o`iXC6kms(gJB7xub6^ZxTALh@uJZS25`Y zJRk!BUTX!2!4#^aFhb-+%S1`ci3Ikksf>* ziO*mv&E89quwQZPOacJp z8$q4YzsZtZMkgm8zQy{KDVmrBh|UDg!aim9`t2k0LVpKOs--t}1du5eQrrBh-q?xH zX(l*|qrkc5GS~cuj~C?^m_=gSIP+HWLzH4eB{uA~_HB<;3m54>F(??9M{=K;r%0v) zKa;2t-`qAcx%v#@cQq~R01#B7V4W`k#QA^*&^ynBjxIMbDe_~gcUyY{WD^CSXjP?} zVPPT$upKY?k6}T9z3V56-@ z{tN*zydSPQ?9?%N1zgK9phdrCs6|KIy0p2^uVQ@64*gSNWI!OL9@0qx!i9?hj}L>? zw@SY#nL3ErFVaa<_JVitIF7Y+ax|C&bd1QDo{{GrLIm&mlrFjSYms#;iEia!>L;C! zh`OVCYt)m(sYf~mez>*AwnBp)o4Zc15oQoQ6>c1m-cw7#!YkS8T@%xsG1dgmK#(^l zVjy-pr-QAzl2Bw;MYmGJu7a^ZgM8?zv?0Jod@#JvxQgQWIB7oS9Z)(4$Dwb_2Gvh4kgmy2r#V(~|XY;I*=KBL%J4qd_CwWW)Erv7q{(gifC+Ns8HW({(4SV%;&G2PM7e=(5Z94)a#R+?YJfaK;cT~140R>f#uHb3Gfr0@W9nSry z>b8w_SR$X8jRG)#nI;I#sP;G~jiKuw)*mP1ojScdUyDW7a$X&S#=3(~i0-Q5cwbEN zmYOwqyYzyq2(&hyZnq4i402-ZsOCE?$qgFqSX5=KS@h_6#2^p{Y3|1a$pEnVxE{AB z{5^CPsX8ccQA?;Y=T^FiCO{hl(}5y3QfFNF2DcRd01r-E)vl*w7O{{Y%3a^Y2~8i0~@0uQ9$C~zRGh&Uj&lcl!$TBmwQ`}7@OKW8=S%(Qt*ZPg)q54iaBp-`JX#h`|Lq#i&y8Vs9 z60xH6bwO=oTDqo|5VB@A$P+yvbAoR9)do?l2BZi%3d$tG)Kq3cFug&4Xi?I?k9Ugl{LQEBQUz2Or^?l2CbeZS8q-;OPCs9d z>0WaRme-cGYuo-ED4QQeOKM#UEmoM)0C{j9BrcKx#;zA~`ZViZ*;PPAR-)-p1pqk+ zK}wM#qeS%+Btn2d9AjSp00j@L&!`?B^?}t+A!eN}G4z)lqP5-~&a3cFTgR^8+=S!u z!*(-3H!e-oODbev78I0sv*K(35UO7}rDbngs$Ef`O0g#4j-gB%dJ=aIhgO`J#6gnJ zE$#mR2WN3(ZqH7m-slpnz`N=(lNw_<0Ae6V7Tyu{SJRF=^v%R~zOM0cIg2*>luF}$ zS;o-*L#%lmhmUKW%x-7bOzO)WROD2sHRZY-cI3nki-K9fDg-VH4EzL@d5^1{CO)QAz!)svBB(9kx z&Y9!yp`TQrOC3P^gX{kQD(Xiu`g`P7cJO%O_=W8~%@D=vLDl=eyl#VZWFrdwdJ3ae z{CiGqgO}>`x=KwzL*3rm+$BneqASv5T9qj?6+Q}?GXQBl;EZj+Rp?e6vWG9E)hJV= zQK~G#sEj}nA^{3lQn1IFj-^-pPP$8h(Ek9akEh?MPB(?qolcGYM)8fye1C=Uj0%|8 z8TPR8YkmGwB-qJuz6f;dXsJNBa^2vef`}u^T60cSLz-LFKZ>eaopmay4@X~Gfdr*c z3rl-J8!0@J_OjD!YTm0GK~k7%t*MNaA(g*`WCek_)v?sq{V@FuUES&h{#DXns4Lu0 zisW=#1FAdcZr+_6*n4C)^iP9o@j|;@Y@wZ977pD@X)47nt71}O-v0mr{56HeTN-tn z(yH_;R$};SRj8shSV2K6~H5= zMxm$FpG*3S(^qf}&!t{HSD9r00LA0#r%O4{BB^mTxrZ-xAF$uW%EVztevYpuX0D&` zRpUOiUsswI)dE5P01gx=Rpr-JX_6^aWOmU3#Z_VHmIeS7Tm=r%mfg2iHg8Vm>Hh$< z2}-oB#ie_B(gh(9%-=HURT_!ZU}_O0@m~J`6kFd=yl3gprny~9smJZ{-b2f>^|Y>W zZQy=H^4#iH_4x)y{{T-mt5(_Kyq1TNd+Oq}={>?y*h~;C%%f{k($#DF`t?HBrpU=! zl}DBu(n_+%#70Pg1&15*l(%d9pTntAh*cfcbsADaD%D&%2^;KpEfd0B^!L&4tImvX zXmtyrT{Gak&i5ImozzZKns?}O>X@A}-N^Ud=6cjwYKm*w&dzC+WQtxW4h!nr7B$CIw{oDh2d~{!=X`7H%a-z9ZE6v}+XAWBii)V#1RaHka<%Wq8p z%SFjaXA3S?djg#{0DjQ_0NCFJ{@S(w0KFgZo<95)@BPJZ z(D77ueQLa;6H3KfHY?b#Y3rneQtQblxSf@n$*>ITCcEU#5*D}E%M1ge>n1jr*B0w!{2+9!iA`S+RF>+a46)wS+H!@j9FMFwBz zSsc4Uu7%!mje2#LYezdqlV+WZpU2H&71L>oq85fB69xb!liHd{l!~IFl7Jm(>oUbj z%M#LDu_j0)9YD6yUXthp(Wt5yOs<%M14KZbgcyVJ^$r?Nhr0u-KU7?nIPSGh6~?$< zNUmDx!Z`@%P(8Eb^{9;;E>NPDQskF)Z`o@t)$RlugDjXz4^>X!qjKuPrr zlLJYTM{^M(0LI)6cJ(do=tU>4^((bE6oF<{ARTzpLZ}%ei1v_p%OgfTWS}!^ z=-aVwwTc<(uG(4GYg%d>PdFy%t+_?mV{mGV(3>O<-IZLcl$;UhgsxAsFp3 z0&rglR-6Jy)&bkhi5^Gt>iJ5FM3JO|NIu=cGqhxH{_aC=r_l#RM*jfOk5svc@of%U z$Z6Wt-r}5g(vuxgqFkZC`RTBbF%qtYGh!_ejps+vQ zneYh2iBED{A(UOSHO)O$6)6z3Mv*P*2r9(j5D(UHNf{hvHyi3}_#Mu+@G-(q8_DPE zyO3E_m=?7&cZTs-%183;2(-N4 zjo#ELY>^2d?QB$RXqeZN5DytCm>=3&Zj4w7nsd~{a6rg2vdNM%BWM%A)GD=obrPIY zN}5r~!;MgMgO!c9_7lhF6KI~D8g({x4!(<(HjIUzXIO;`icPm*yaoa-Ni3GRSOCb( z{ye-&q_Z>CFfu0?2M0+qJD&sQ-HnWbV`%&1BQhsD4E>{mt{*a*Jx0H?w$azANoxN9 zG2-hnv6HVj$^olDuOwdo08NI#iy2%dWpN$?6A*tl&>Qz)D$Xzf0FL@Z13ib=NfQel zT!j<2ZR0x(h@T@rUAS?+h`O1;INm?ib69T5~Q? z<;g_-;TkNlN?aL~%m=})Xfg_3oCY;@RBy~xOfHd#p5jdV#PA*3)v75Kau%DDq#l!v zqyr%Kn2f~bxE}Ier1AaEo!seW9}$D$L&vo1a*l6O`|`AQige-j^z-W2K*Tbf5YQ;}KMt?VL3A`_Eh)LYU?sDF_e4)YQOQQT9b zP3i#7%8FZ20N^a(52RoT@FR_u`kl=>)JhRuXnNHbM@drJR1+Oce=yZJpBh>}rHqXF zpW;>2)a`OR7rJxP9xc_YoLiby-m9N>{?qm;B=u|WU1z%9jg%eq*U&^Y_R!|S4Qp#; zmOD97R+_Z3X+_wHVVD!x7}$yTE5O$=_zFpE0IO2wnPE{3EjY4n#h`3EYxDKtG(sF1PSLkLYu3+V7=z0%VX9aU8I5-083?qI<81$fYYCD3kaV~>Z&ODm6fE|j-jZLy(drfEH_zdD;1 zkyL5>%ezXICY6h23{1IXLPcYt%XB2L3aX=13<)3Bd`jt!jP%xP^C3~{4!0VP@~U%~ z&PFke?Fw_jbw5nJPb}GYne_>r$4Rzk;aB7uZOr5K8VwRV2*%1Vhbz{k#dO@VWUk%^ zIB$B>ncTX`Y1mW{M z+?pv#y|pvw<*gRIWK#}lU+l|HQwzmq3@|B<$K&Eeb5wc=zv~w!0R&IeNSONI_y!5% zWlEPSHPNZL0ZC!D=KujC!5&1;JLAe*b2|0RZePe9>J=+I?PBqo;=%zCZ`#;M->Ybs zJ=kdjh7iobrUpVj2~pmel&nmg0KmW_82Z7(O3B-PH(pyrR8ncvoDxI1fDSi1p7Wok z?GVhd#!Guhe|{vNckVzpZf1wZ*mEm8LJl)8GP4 zBXAf(tVA>bzI@fuS4m^4eqFOT>ey-@;!Ns1Yfh6@g;fAFf=*PA;EeBq_wBzDd`p(q zy`^?-D2~gm)pe%c^9X3G-kX`P_79SM7nXiR)yot6M9;1U1yy^UB*2nDdcR2_0Ithg-avy1xSpMA!kHEs?KDy_>e`NO~6RF*48)+T& z8Ob}Coa1cH*n-?GdNpgi16Wf4fT2(pFf*Q+7|dzgJ5L`sdg$tR0#}!dUV^(r>K3;h zt)K*P9&q(Sua|fjk0n5h1P8`f9i|Uc%!0U7G9$VC+qR=4r}%fB%VKzi`8X)dzO6@` zI@59{e9C}hx19Ed>FV{*KFduPaySOHTLKKJR57oe^>;BiU6<0#RXKox?!zD6W@dX# z#C(>YG#D+kZL3fMd-(%$PM*@co*+(G^zsd4WAka!+kdq&xtIblGxH9j`M(_2D^dPQ z&M_$t&Bx@LnWGCn1!AV>y{iBgy>9>iVe z$F#{pq^1w!0^yZHff0f>`izaw*BF5S@G7mzZ8eo@qL~DsQ{OYvVl{7{c7PkVVyPV| zQPIuWi!lVPHKX{O&r-+M46-6Bfw}@w{aTLwC2&Lq-b3aFsFhH2neIG!9f`wth=Zs) z!4|MnAY_0B0~|+u0=@j0AQPqw1@<`Zu4StBXFEIvMrw0abCFQj%nomri)J7x%pyv>C*hE?ky9&UlIgfBWO4vzwrzafE4mHYveva#^cL6;G4le4 zt+PLf5;*PJenoR`HR<_CGGG%U5fP_vapP~zccA3CZAE|+Ai}mh2@(3i$Q_lj;5|d> zZxZ4bcx!TY>2eNX?V~)0F0OUG8t&vjn%Ua4-qKwfx#VEQJw2X^)BAZymy*h@c!7ci ztwvb3wkkreV7Q+D0KzlcC7e^OZgMHjrCTap(A%T$}%V-1xz!u3$+T#A%8B|(OYSr9{V*Z&lfjH?d6c9|1 zPs|{3qMcm3bEO|cA6Y4-lCDY7Tt1ufTHQCIx0bH>{{StOR~K7S-S5g-UWTaOawJkj zG{4Z;k&*eWkg5HxUiO_q={o83puk|LjAZGON|3RfL@bR}>UGlQzb&Nxr5o*R*GvYL zU|F34sUQ$(k*angM2v8le}E{`F-m+9mM;Q5<uxTS7Aq zomzk-Oc5Y7NcICyQ$_wP{5Wj=9CIxmS~tRa!*7HY4R?Nr)u0fMy|ai@Etj**|sP*(@1h8$LsAcKoQ&B&$|hl@+U$09GzX;po7~ zRt63LkOydL7cl1cmp-?raZ#;(bPGvbfT~Yh7Sc(HB1YZyK;Q+{uciKzQTnA$MsE}8 zZfZK`(%p^DW5@S+j&qw|Pm5RtB?z{R{{XbDPU>&D4^^mG2xlzi(CPB{{=Yu;UzK`} zl&V+tlJQYQQwgB1oRSK$DtD0t=_HV{g=)u>UDqO}su!VYnI)YbK%Gw~FlA2X0|Spa zJ#Gx%0rgMNeu;JSbC!K!{Vw#9*HL*FPxvns>E|Th(CLL8>Ne@o%)ODqXx-#{{A=#4 z5#d}!T%3%`RX*(}SRG)tsclA$I<TDD zRP>WP1jHq$+_Ht!rZxw)&mp(5G1`B}=PO zrRgm&l~#y&U`9;S0ktvMNgx1p;VT2(${zNG|r97Z`v7nH8)(Wvz*s?)e) zSkinoI*!1of}|J(#%2UU;^+8;eMAq{c2`9@i_3q`oG+-uY5aGYblZ|}`00ZkX1)=OVnS=?!-wT03;nh zQ5s6+bbTt96p=@wbv-)4!6r^urHL6hz}qqjQ_1u>v(ug*^sn_d#QKlRKcC&|4wCri z6UfNk+|SdC*D27#pK^(~m=H!jW2nlOmsP7;EiknWGybfizPWPDR#bq&a1JAVM+?UA^vj~?&*k4w zHM$4JxNWYPbWfzVf8q@uMbmp6LeC~<)KO<0o4V92wyv|-{HNU;pT@H7Kj^B_cs#^N zTe=s#w$|F6$)Q@3o3l}Xd7*%%_L^@!P<0@Iq)Zys)#X|hDw}}Rs$AT)1X@q6)t7Rp zK!_d*)Y#S1ILZG2@i^*N(dMW4jr}ZZ^Bt^=?Pm0!s+B&WII6h4&NEXcu4QVE;neQ7 z&4os%D`njhELAnY8Z{Aa%|T~~z%sh4b#Z6P>Ri%CuVZu)ucQWxi(0gvH2ERYiz{gW zLjq)mW*ok~<+YX2wl7knwLd}3v&gCg6R47~x`7ZN$T0vm9^-5E6t8l$@O;~w9Al8* z)*N<<9baCpK-Xh&<#mwJikF!{YYUd|I3fHeL6DA)X>!`L<*!0Ya#?rtsot?jrhNR5%n+s0Fe2&{)*rJGx!wpPd4AF z;J-@y_-lT&{{YW{=Jy?a-Yv$q*5ZUwY3lP9`J8ISvVy^eGMs8Vq=ASX4VZh39WOlU(@wa!I9m;wnT zlGyje2q%SH&XDx$R3oO5rom%zF}|QTAZ&7cIZtg`k=ROt&(UGbXXWS%KnAXF_V4w?LQ%;BWWu-bu${2d2HW;(#QOS)% z@{y6*fFKu06+uv`P*GSZ>dDIL5gj5s9Ot;Vc% zk*}p>>DpC6f>aSYM5=4m4OZT^mr|(et125h^x*W81c~2r2mxySTi>;Jnk96rjV?5b zx>q>TYS|0rUmn(#iCJ%T4iGlfojx(@@OIi39-y*pEK*?*=1` z^y;@P-OB3KjQk}wrC0AGJuy|QbUW0p$t!IEhSsG`0&0qB@uXbM21vJY8*ctDf=x4aEA-vU?C5!;$K7uw@I2F?Eci zJGH{KcN?v)0RjZpI6)2%>4$Fc1G!`e(q@;5VULvTai{F-bag_p^PH>=!Fs6A86U%6=H7!!M! z6Z2@Y$mTZol~Jx*r)hLzq5#2@Wyw4XbT+g#tXADUs?l8Aa^0t? zy_j{p32uu73?wXwkV6n66BFY0lKM=ls{uezuy)&g?K6@{I5RkSKnOv=gFm4p#KA0k z9b3lIcxis4Dsl`Sq^m!iY4VLNdYlU=;uVTxrYbg zbaAU)*xuwA9a(;rYz8KEb8p~{Yq>AdRl2{%s<}v=grRnwVU^Aa;I+?kinE$k8MY%# z8PXxP0BwwI2k8Vh{-%~wDn`T|h&aiS^j~q_0O48T_;vSr9ueF7y;~fBnVq~3{`D{A z*feP{OkrqT0@EaYgsZ@uOm`8FhErlyS~c|UKS`K7LEI2Gz#K_aLc7U?2Oo2QB#(bT zLnDKqPcP^3?xyk<@SFmE9l{VkFba)UY2wqcvbdFFOa|0va21^MO zq?0o`zS#c&!bHze5;W*ek&;Bj9AKTinZ_nRnH*Yqr%<)Fvv~giNvQFTBZI{F@;R}? zv*=c{y2Iw~Ju<1OqgrOv&T@;>MwN+zRIW&r9x^{8DD739oRuS`Hw&K~5@U1p`EWSg zAW+UHRYIl7@1%Xel>}ljkqwS7n3L1&4srDR$n-Sdx2?na!IFDCGVWXK;Mw39m~~RC zZE%{Ym8}Y0rD~)&0R$(^2{O{B7+t|VkGPo{VxrXwUeZ)qt7@xF z^#aP~s3lYa7y!G-JI+ox_~=##1M08UUsgJorK_R9I%A8u$SU5@$j6|UB{roN#JBs6T@r+i>{?sWhqu~MB%H0Y#;sOq3< zEOj!(>WKq=@jFQ{IM0qjiS`|0+2VLM)3sIm8kRF}ENr61+686*0B2#QNb^L8P z4uC2I`CH5SbK-qHP8L86KqtVS*^Q1qhI6+QIE^_VvW9Jd)ftY&PDE$e0C;Bicm`il z`Z3cC4kyBETIU=*w}+l`{W7^cX<3%_%(JGg>r1c3>AHeg(+l=&JV2IH=_4TmZ9>+y zP#Z#ozzj)@Vn(f^0Papf$OnPFMx|7^9-8!urFqGfZC>o5d?-_o zkE5yPqIRXdi&Lk{HK1gMD+pWhF8~UaL<_hcE~!wIrl5%sX9vMC+7F+6z~L>$y=(Lc zz|vhxPBk<}Hedvvqmi^q=0t1OgT`xlvrg+G zKdLRaWEKZ7HEKi=8N_Zb9G;@7zow!va75;Qqp_G02NF=Qw4kcKh^k3Cp0OH1*-Fk1 zbnVo4>=n8gTu-Q7Os#f!cl=xgu`tVwDp0ja!JlJ5OO1a{W1d^>d+7>1q(Pd$uWb11ROvIdG_MPOQrpB#5f&} zov&o`*3!Rhc5^RiiP=8YZL|2Jb)=?!g5VJM8AAU6c$ky;G}C2ctQa~;kOraH?UUa> zrg+p;p(2>9$59Za4^||Aa3VL#&cc3vEQP75#no!JBBQHYDL#_Z%C^6C8_)HCeZ?D; zP^)XWktvvZ94!GKTasj=f(+CkkTN#uY<;?0Co==vGt(;48l^!~5FRwU3BnLglbGHQ zf(OfC!P29zT{~IVE?8U#^pLt$?rVi;30&5?+cLc-5J!n7_Ib-jMN(59XFoR74lyJH z6PW&A_2Y&B7!o{Bd7n5~nUl8CAQH1(O8Wc4Q2j5#(mmu3!zBFI<#J*MhYP zs8Bbnp4i*D20LuA;J_RVmWI~0g)GP^mIT2%nYKD$0yo&mA|vws@_{VNlsT0V`?^lE zAI#KPgfuJaElr(CB*d!RQm~l^9Dh)f<7r7q&yvfk>WZ3N9<71N$DGKF$(pbol**|z1eu4p=rX#5$a@v(CmLMQzkO0$B!8stGlf07{ z7#M~K?sX0&_TRg|+$bp!o+Fh+X;YvqSCNK|8q8WjwnwonbnRKNu`vKZnMfDSdKqR| zfMG;tKQ!b{eB(W)RH|I1&Z?BD1tX-p3D?|^PrmUo!yAw(nq2{A zuB|JKg^&En(Fsf2;v*B}eS_ES?!yBf!HQP`r{yb&neQC>2Dy(TR6dz+hF+W|W*zik>+|TVLuKxfn zacXI-5A#$I-#0kp5AY(EK#OJsGM&ulU9f62d zI1F5usC6Am2rD~j5eLrfuy@;wqPM=!KQlvjoelL_@?hDto;!&bieRVL8R)~V=E7Ag zFLiqIh$#>T7kQ6|)4N(rr&hxN!NlSYHt=>66M?{WX|;D`7UDsihQJ$S5;Y%kbGDg0 zSo6B?#<=pRRV!So!7!LBw5)9FRESBxv_u`*DcAC=SxZ(N+rq}4=oH$4l$$( za7hj2)`hb=+Lq7H>*-&N_h*4bK~B8F+NuSR7?|q80y~|vpUgJcY!CdcWzj_!uP7e_0qd`C|QNAlF(|PPAcfstmUq3Rco9C z9lVv_xsH!*ZE&EKM-#bFe@UGONr_YS)wdC)&BeVy;RXn8GpICTdbTHd&d^CBc*Sq3 z-=qzr<+*&b2hp*KO47D4=A+3asZz5H+l8>E%%pyP1DIROYV?I3U2&>8I&4=~qXJ^*5lr zFX|IFN0z4#D}Qf~8uBS8hq~oT@ttDC7I+nXQ@vsKr8jmYE)6gMGAWrNs@A-FUrwbe zBBi?3Bxz+Nq~yUlhK-ql5@10LqWZGjdiS)e?&(gTUW$Rpk)&#N5IhWcGQ4zu#Qy-M z+*hdIRDVZbPrNS+FAZPM>+WgW&CnWvoTq4JqO}2(PnA^2t?HQ=1M~6(l=qVYDsw#+ zH&-<@(?C%Xm2s}SEj5ksHJLjNGn@v`CQnT{jn1Zasu$-awK6Wn$?6gpQO@&$3&w!` zRrJRxe@%UU=^rBE`Z=9D%jk8vsdJ2KI=n@U-0QEVCzn!K)KhKrcD6!t4-%huWs%5D zV$}u8(^RKe6%9}j6r#o|1JVExa!zv$znL8*t}5D9^;BFn!y~IM00v=NAgp0We%iNH z1NLH}VNSqTNKS{w33Vf2r;-pVf{`qOE6xw!W50A(M8U!k^(!{7B;K9KJM6r>w z={!=o57VYWe~IrO%KCicxm``;97ln04Q*qo$!X$bRI!}0y8i&Qy?WCjh^V(#wg^PN z8hzCSQVoI4<=q?GDikO>(-r8`Z{7n;lAr5Ilq6{+1JnA9@N+dvRyAr})FPvbh_wRh z5KJP$&`BbDU`ZfB)vmc3y-e#zS$eCRk)_9Yu7^){@Uh#Y)2CDBYkrULgJzzEi+ana zPL z!`iz&dTvoFbQdi0&;f18)IG5xN#FsX1|x zRuGO3rq;jSeix|7%KW?;Z(FS`HogV1aNix`qt*9x}|U#S*bLvn9c-cV1d{f!%XmxaBhgh1X5Z@0Mt?Qod3<$t8#gz%#TPRV8<27vmMn%V z^YzlNNS&Bp^q2)T?Zu(IOQTY)7AvFFSJc9tOuwy5{{VV{oU|)aV=@?FcwtS*wYRNx zMYSiV=u;gf*oXfBRM5}>3LLV+#zC3jktzSv}m+KFuT|VO6yM*-r z0F`BDQlY^)9#0>Jw;!k+TQ85=Zz9a5GfXb#?d#)cyeoJZ+ORAKl=N&Cm7M!Hnp|2GygV=Czl_feyBU>blpPaSz2p?wb4=uX zcDL6~fT3|slD!6XCCQS8G_FL#Vm1+(GADa_PeShTX@pxgWm{N)tdsl7fQ?yZGrTbI z0eHRrEPW>Rhw2}$3F4hF;n}(_<5-q?ZOuM8$tklpITlk`DGXk{2o_06x~x-7OXY;$ zSr3et#n<-qE)gvm=_O0HYD#Jn0mQaIB;qIe5{g3^LzuOfsdH0Ws+Oj!q_eR&C08d4 zClLVdP7b8;kE)wrN;`c`>8CWFVRxyOxA^`Z$46DJm9rt$(^`gA?Rvu2*;Yl00l!m9a$tp5njr}oh7YGRjE=d zsJ&NH>N8IC%K}xW20$lizm`d zfTC4Ynej2Jg&e+>%HG~ktLP=A0#bs~b#lxZbg|OJ%ZX<@2F3vpn|&JV7a7=+DEMf;1OG~3+O1;49DIo-?R(wJ;764DpBN*-Tap=?jC4Yst z{{Y0F{sy|O{{Z6e%CG*-b8r3&{{Ye{f1eS5?UmR601r?90KTYymmX?5knP*u48hCT z$EQVjT}XSX+scAM?3J9tah;-o{Vw|4UBDS5>+o>+NqCP$Llt&1dPyhpjHr&#PrQxD z*nk^L6R7DM?*Ptyjv#jX@568O*}=M_#X3^&t)81~<1+sM)9+TehX%{z7Ao1*)70eL z*Nu8JbrzcMsLPaUBajLXi4eNosTo@b)dYK9+*<)k(y23i&2LQGFe^o=WN zF0O?etQ|v09WgmD2L65HhsT(GDE|Q6XUu+uIA>P)7dW%Td6z&r209&v7&%(H7(FmW zqF*NAdXaf<&$>5DI^*Y9sd?6UML1-MNe5e)dKdRa!ku>Ma?3qlh!7q>ifRD>1D^2; zuGmtgXPa&cx!nn4^6j6 zT(`MhNwDB$HEC<-`>8B5YyG`@)QPH7ZbOu~>@Z&$P`IN}5~ySQ(g0qrnK;2Ge0T@8 z@x;{W>t?M>Q8dBS`vSUtW}lh|NRd7Xnc=;!kfr@MZ1oi+)^Z*(%EOINtxQFAbZg$G zg9i7tNOuE~^ z-DkNmrqGKWbk#sr1O_pgk)6bdCS;GH--dLQdUaTtnrs4{mNv(7PGkis|E+{2LdGluKg&8AlNELQD>q%OZAS`do1rYk0y5@;n&_D|f%#DrCq zTH)CqN@)**HV`mNdmfhhV}b0b(-rDf1bT*&p_wp%$N^x8I3S(d0P&^2L_=?r{W)^J zoY1Sg$T=?eQo6&(xi0{?HLP%YoO7G;q8dEJJw7GJ@M}8LUux}% zqgcD>yxkp$e3-&SQ%b4I{2*9!9%5asGfiB(7B5klnc90(?xH+{u>c9-Z7P11HA$5p zn{PYrbC{8c?jYF`c3Sn4O|Hj_GNY5(^DTNM?yTj-SgU1E|5=;!aAW zwz+^XDy+1X zRu(`Sh@CUML6IkY{KjEPtxYwqxgh{(I8sO%VB1DcNZ%)QVpdu?R|M(^2KBvu9NgN1 zV*5E4^X{s(n(J*isb2TnT3uU+Y|0$bC^WBNL^&h_iIK1owv*#O%k2#(0RyfB5+r=Y zl<)!=>Gr)$l2u_gQ)zrww_HycfZQuescTSZ zEmr2NGuLsePON|ibJ)RLe+(T`I$cF9J`IGdoH|@9gM(s^6XW@H+H|VXt3PDTx8v6m zZz8fx1<=e3zS3YK#kNXo3@pxi%<837C@2&tm{pIYn4RN2ivDS7Q&R5DH5~OboylV% z19ps(1b7?EMRCqMty1yw9R4$Rf0A4%dih5uyH;l;Z4^O`Y_}JW+_@zN_Gi^vXr#il zIUx}uE~aP;xQ+4$kD;G32F5yd6Ao0%Dg(ZdXTE1_3H~Vu^8DNvvd>S72sZw&7|7+& zxC)=;q$hs%liI>$SBiOsr&Q3G>0x`rw z*=jiHW=N7rDzFD>Ab-Mh_tdcWG&-klRS&htNR{`tQD>6z38ka%Shgas-fHUDxTzm6 zZ1&0!8SYW%G2!uT)!EWfnPeW_vj#>d?0g*{Q&>f;5JkkMve_T&nFjzG#$tR#phzKbiXoe%<)Fr6 z1c*~6dvCOe1(~gakhIE%(~~KYpE>;wpFKp8#p{c=uiL`A_IV{PMDk;0RIKY%t+JD2 zmpv5&Q0-UlMdO58dElvcLFAN3`bfJ&x{yr=h?;)jy_DqcsXeyPC*wn9xARKK!Cylq7S?bVIy+e6dSXAE2bkriEl(Z^~zyVD}#JVCo`Ihyr zl!bL-6y$<)Be*?)`(VHVSaFrGT5XVLA_yRi!2T$Zb!YwNKbF~CFOnUJWxcAd!`Af@ z$+HOc_DZF{k2IJ#h6}ipYs^IXHB%%D*JjN0_WJ$j`A)}p98Q+#l~Ljdz#>p)Tck_c9v-zgNk~vFd5&Hmmg;uFq0yQ+` zjO`uv@??-Ul0o6Ex`tPfW?(?xM$_B$8}GrEbZXRHR;cBc8qY5mc@ZF{2-Yz%`G%{+ z*)uWv#0-pyOnFFassO~C;z5$^CvhU;~;6CDs-Wm zkJa6d^Py-vzR-*T)m-_gDjy_8>N>nrWaoD!*@VS}Vl}}g~V+tFnp2Iy|=i9ac z#=~0H;+Pca{FBtj5sVNsu$dTBAmR>@A1Q5$7Th&n>p@TZk*yh!HYBp5T1rsDmT} zq?wOtV=y35g_Q;F%kd+0|HKZvJ_Ij6j{*yjS5X|kvX0{gz=Ur zWuy7m(5^lV)7EkS08q*81gNEf1VR%fF}`#41qitEFF*<84L5Q zjCvbJv?+dgiR>V)-__Cc#HLI*x6Y9G@nNnZ^3Mx^qrm|1BkBq?? z%}rb+yygnRW!6l1vRgAd#K15k z2^-3jJ%)S1J7DkznlM7rJK`{N+!Hg~^G}I@c)ns^<#Cb~2a43+p!TdUGQPJO)du*A zzUA$$u-GM}gk!SQ!G@RZ_$d6aru{P@0}@ZU?K=VebKixvXv^s?!zMK&PJ3!3#y28% z$phxSN<;RAUhZ1Im6WBNK|>zTKOPZOR&4cUFR|t0pypu6lU{n_>5AIcYT}Ak^vhKz!zx9;i0vl}#um zicA2oCW8ipt&EH=06T1{+MO!Pa^)e(fTyK(Y5*_`AWDJ*K_eyv^q@aeUtxdzT6!sq ziQLYkO!m4)@;Z+}pglv-f2_5-AXmS;Q2G!4X3kGY4}@&0CPQB&NE(OLbHj zoi&xRS|sD@HiIA+c664Fg=p#d`ges?S(DmHoSn~Tjy3!07j=2v3i_GY+RsfdwCNu( z%KM}-FJ-l-iCQRE!=vvJ;3>2a(g?WRh z&rk#pVhr#NBn?>^P!QD=h0-Jt>PZB_TmVid!yJL0AK?#A7Z3b9ICj4y0;1F+q7p>uRnqAI5~N_xP7ISO zx#xC9zcQ&xm~i9;rWOM4^A#KX44&A z_*mC@mPT*#^iU3%DYuWF+cNR4OaX7Ue=!B|=PbK=w$*MzXs2~TtJ~8n9;&3$RtjW= zkOGD<-cHRGt-GzwG++>0BLx9Ut5gyl2_z78oiGa_1pKoc0sfPDe;(ssQhib99TMRO zrNB7vLwG+B>8BdGnN@s?s}+}4;9BH-6{Ho%V>C6bx{bRh>a!|Py@qNfO9#5p?aS*A z-MUh#=_^25fRK!=M9wl8YC2*@qdXaSth~P7m1YfC-ZRx?ZAoq$0+mswL+BcSErXCG z5yR#5@zg$u0v7 zZ*qd(l{cX3Ql(l|#+OH@tyzdVhz!yQI2i#9Hk?(a!p@RZT2yK~mDIQuDXRuuBu1t) zmcTw&W_kYr!z1ZinQ_md?q}8=CShLHcB=ZVm&7%==2EEYo+gEzwQ|iqVSR3F;vvCe zt1?{38|YFiG)RaK;;PjsR=m2i^#azW9c}cqFQ^!n0e}jutpJ9}><1fO)uPXAA%LAO zCsQ=0jBOnR5dvqhi9UVHA(L-w2 zr66lvL|ZEV0CYW)*^xg#h;1(GlHu!TcYg-6QJn)8AZpSmsjXJxq3|6zZ22M z9Yh6fMw@}2nFWriDhpmwMy(n(DO4v&sO*}g$Yuf%zM~LkGm#(-QNv&U6`g+R{{TXM zn>jyFc#YniV)a9ta?V)w^DbYgQ-K$3+*!D06 zzkN=w447I$b-fCQC@ z)Clc3!ynKtInZ8R$#~~iey6!Yd1q4kt%ZB4JkO9*%BP!D_IY-*XmEnFS%S+*r%xqk zQyzwx1XNXe@a8tcf&Ty?yyn%7OLl5C?+2*DqTszsVircd2To+fH(>zj;w$I5XS!P? z6m5f3rA&!nTAJ&q8K4OcVcmXnuwF7J{BpT>yQ&=-tB=aFxkpbrEyZo=agoy?qqCws zp5*1&f1#la*LNCtp;{JIDMQfFd@R_xVcGe8Izqu-jViP$R;ty_Dul5PI!rHMCI?_X zqlr09CZM(6r7aciYX+>y#H$$TBPT#G59ZE60~~XIflV#`qWx82=iJq?nM04^@|~-< zaR}*Y>sZtD)NO08Sr|AG$YQEKh=)B z01T1eWjmW)R!&<|+KPb807t`Ub}=jm`oM(^#n)!cy-!R@ zOT5&+;>jN3MJgGQfuEfnR5e))0lvC1$r`h(MZJ9>#;g>Ujc%vr`NRx`^=5Z7D#KT^CjmLKR z2UG3Z&Ny94x^;+>ci`o02^sFP@(>lUVhCnu_lZ8!6Ei-#Zca9wZ(o zr%jfgh9r|D%m@VSiS7O~8*t_GXZ2HWhIBVKa`o%{OA6kP8|la$WG@ z2!q?fGpAZUgLzK`;I+Do$U1?{`EM7+;e5#!c0b9re&)+K_LmfPZRd`X0H2K7)(l+7 zn~JjlL_uRFW=J{sW%Q{UVf{%|Q~Ya?OcN7`%#AaWXN|%=Ve=DH1pyEamXn^~4O&1M zKQ@vA@rd1G=_#v zL{cCu?RBEl)IcRE>rpTcnPEAYJ^pCMH{xoP*n*M|N}mu(2XG8P$MlH@g0EDzZ&}IU z@3e@1inA%#suh|mAjPSkpw2e}zo zw@@>7F=(TWDnv!MOGRj#gJu+@RWh#W5>@gT?t)e0m#m&L8>4P z^V)McjvH^TURlWc!PHF6*yGjs?tdl2<2ADRXWQt~vCk;eMQr+dYDKXVTb}DE1^QBv zWDq#sVqw)LDF-8W_`5j1V%xbP^nv+YGRlyQcEtZWwJ!TkR$;+ zLEB)i=ycadq`cIywG6+_R2|^%>UWi8Hn1?D;i*#=(NoA~8otn7=4N6dejbdKP%^_I z*!KE+=5ZSc#Bn$PE<}?t=k>&WqsWtg`M8vm1gaE92Lfd-@%7JXnE5pzodJ}X+=j;c414b* z*fbX4mpvv9l0kz4WDFQ5J+~vqIDT-cUD(d!6?pX&bl;1a8?~$^Rrb2pHkK$2UujpGJm zZM*R$zqe8Gm&ULv-pb(htqwHk)UHRN$$PljseXc(a@uqyXX>WbLf9ZBeX|Pi656Fe z29?4H2eBu(-LW!fJ|OSJRH%kv_~&`XTX>vh-+IMey1RRE8VeQey0<( z8CjSa_h_q$aICbe?DJ0cys~OC`jrAMtwGi-ON^Oi;${o^r?1d8EEUEhU^bHpO+9M>2hNh4PrYa^o4dZ_x8P%ESyRyep_oY@Y z@C<_ck8j=Sw??Nay=BnRI*f};(OQOD3lv=Jqnm1ze6$*IB&xkq9G1z(PDvliJ5101 z8Fx1J)M=mum?NqN1nzMMZ0>fR!FVZDr_U%>s+RGi#zG(bhlO|Q8WO|snE-}cFi?MD z9^g}057rE5kBl#^c0htu`%c3ryRJI`F+L&-I(BsWN=r_4KOYY8I(kP8 zX>#9CE~fnFpFW@ z5ZRcO%G*9M8|t!DJ_wRbVn>ib*meeE2ZC3ZtpY$r-XpjJwokNTJQ8E1wZ=G`?+WGB zYydCA-lmU_izGY!)pMTl=P^FXf$s+sFmWdx$^mf7PIUv|2q*OiX#W6b_(8(TwtCdC zPy*!RG28k0E%YFbSJrgpY!CTLk2QbQ@QIHIFBod4z+hyS!3U;hMoN1`{{W_7 zX%io5{_M$P?Z2Jpr&s0F-L(*N^)ZhFOzeLw_sq$+2Te_4t8J&@VU=JD^D{0a59&re z;wB&@4ypTZNPX77q zIGEX$3Mr`p=5gKz-L{h$*vS(-N9ytWw)O3Tq9JnEx@bc|lt=?Xpi z1lg3lSKL+SF%t`z7#_ zF97908o#w*2rkGL!58PhaPlE*Z&{mofMatQ6Or^W1p5M_S(IH{B~r`sb_5Cg$=@$JlF9BRhL9$4R zMdLpRq0Ak0`i&qMlKW3^2JkYfK=V229;J;IT_H&@V3Y(&oyiJ)y!`0=Q{j0J>Enx;Ovvub~_v|eLI;N&ebAbowwkvI@>9f2P|725Vs zM0A&`fJUgw^=#mL+4}x~g<~)`tyZ;@pQRvyBalteV7>-OyDkl$WfybcCOwG{J1B^U zRyM9C1+q9$kE}o##u)zqI3R*C8O*K+3G0akbn21>E;gLZZT|oWlRIOJwu+a>cWLb` zl}hN@t6_v-2bbmy_EBv2_E%Vmc($d92(M{JD`6$0!*RNL4x8)Iii^7N)2SgRudR%ES?d8e#_lE}ZPXB4Vj(0iJ?A(axTEm1(} zB|#ni@qmqzo{m(t9ECR(Dxw(#fW#1_L>-`IWY1KL>DSd~FzKDIq%WzfLC7&VO0;k} z-P*MgdbgIsv+AWJs>SW^XcD!^8$x6!zpwM(_Kw1(kp9pVYoEr}=N10|4A4fqF-RpU z=MjRXyP3p^o*=ihsZQ$E9223?xdKL)oIh=MM$KH*L!N9C>_P;x5$`eaZQWJ1t#~d-tyHTq0Cl)} zU~3b+U;<({fdFvM>el1?YTD-A7@8$+8T5!LL16U~7D)iA#E@XbhT@I>EnE+maNdjb z^NaHCtm<_Kj;on;mo}~=xYY`|rIko**m8>fcSK91R~C&hiDj{s%*ep_e0$0k+J&># zx2aa9t7A@|sU>JL4O&;Q0GUN!%`>YSMWt8q_jRbH8gy#{?@=kSVp*hu0}hZ5-~u2M zBz(??(*6Ed^#Ro09n}7js-H~y3)6m%qoY{c)47AvZd$a z>nvW4Mh&(?$pKv2s;$%3Hk4koH9JamsUrkn<=zXH!o+%g6#yEU$rviiEy`1~qh6iG zBKLJJ61_pHDboq^7ERiz2dD;35&UGY3;zHf>K6myzgT^C1`RKUp(k$K3(Rqgp$a*^@G--bwZn`;le8iAUH-< z4qaMXtSXW74WegK%~9EGsaxGO8iYmFDs;!D-!U|GfUpTzVmg{44r7L==~}xm`Oj9d zx<}Kd@fK6mqf29!@~a_JBYoF%m_O#i#Z6Y|=-JU+)wY`!MG)lAro;#U{Eq7FMeVi2 z)ml}k0QU8%i7QQriCoDf5*|*Ilid^xy>&vhYND!DI*RGr64d(uptM4q8Qvq0C!gY- zJcsE&{7pWgI&H%FiYoN`>SKp==XXzok))R<|0f92%NMPO6w8!7bb*>4+nx2(+i_+fr6c(RWnqH3z1| z6$v0f4J3`FrzTGxoA}ol&EcIV=+#~|(GEMw`bV$RlH#13q#R#El0Rd3DQDK$yUK?) zOy`Yn3gM2cGd(d>0_CYK2VB1 zdX*@FQk$y=6~Iy{6@bble9B1GCmnR`Z> z7zQ&ENu4XDd#!b{{T+8*BgFv zxQ`9hR&{L$r$zR$xsKCyV@0l}Hd$5e1cVhU*6doP>-9jSdjQ^fO~0jRfm%bkbORy| z0F#p=bDwDKJ)u?VmS9S;8GuOY(in&bx7d()P(F>i&u{6Tmnh&JPU*J4E9l1{&cD-+ z6V;p9&$YvPK<;g*txflC8!P%RY(*vo{D5gZz}H@>B`G*Ji_C25C1kpx59z5jXsH^2 z`Bj9pkU@X}z*zyOrfQTbRIHCd&a!FL^vgkQA#{ixO(Yo+v=|aX`#ky>x6=Ny=5+(8 z{2P_<&a`Gmml@#vLzwMxiY?xDZbur<&KkL@rBP|f%*$zLNROft`?A-%0NVysTfS%L za~7qpr4%$!3J_K5N&f)doR|#6vqt9tfz%b{18;3k(@jc2g=((rQGg7!PPGw@MW8Sv z3myqvA^!j#ZY`6`>~*85oRgMNosNj|9tZU0#CUffr=3wxrKyislX49Wdx{lWewAfv z1Q-}GaJrzS9L$k?y>iZAwGnL?I~q2NTssJlQGhDtzcR$g+kEjSDN{w3*F{YgX&aGI ziyxO-kYw&4ujTrH0pXed0NHD^@Nha~)$XQVnpXOYuf?$Q^4|K87BOqBY#i*#I1&{{ zcx|_~W4npHSn6U#LDb6q(I~I( z%L8y!#%Dw*Xz76mURX#!l4ZRS{{Y3Fum1q$96$Z}{{Z{^qkn5w{{Z6eKj5kUTnDPx z{^)=5r~d$<@!-F%{C>(h4a0XdBNokt^*ht{m0R}|*Io3P4V4F!!IH>?B!pnR{{XOl zCO&q$cP5o|LS$7AWGTQ?F#>0ADdI@t*;<~OAW|~hG@iq`1%^rL+{ACizv+{wcDj|+ zj+u17jWXrieMRON*w-^~Kclw`Qta&9PIbo#O;^`OdD!-_3R0CdzBA+7;|qs0t?n)k z7L6CGNq{;QW)3$o-e+vUSwYk@b$EaegPmQvM4XI!aiTxsi@nrczN&O9lyit$T>*I}!JMA6B{es@f#Rn1`%PO|uYZA1Xf$R;KNKQT4TY-m~dF+%(A% zf}$od9iR>v+^45NQ=-E{ApmXwc_a)Ha%N&)R>sQl?p)Y$B5GIHm@g9`et3pC`LpxY0Z_N=92_uVK2}a1uSD=bC3fc0Eyr&{x3Z&M@{~z`K}Ka%(}dcOl}7&8$(<4 zewI((>c81s1f4SbCdqxdeYQ-=J*6`f@=t}XYFu)lIw%;fuo-)yW4CB1b4!LsG;pCrg;=$c)nD>tV04(&ET#_V~k(}&Te0ZI*cLHT$radGps{_171jnCm%k8Ac zRYQStckI?iXEhrYIcE~nb`K)2T@YonkDF)0EKTAy3|MN*?hw)l86VOI#O4NiOriA7 zL5SZ_`W9G>eMj)e8rE0zQlK3yW<~*x#7@&4@!0No(%z@@j#bS*pL#>mek)sTohs@7 z08#90t9&@nm7Y7h&o2zNHaR*xrY-1$>_%-XFah9xD1(t0U)MC~{wkeSE|qE`piMvq z3E3A=$vwenIV2Onwuh=?w)fF8+(p# z*}L~{xm#OCsDB5h!>lthF$?<|^`6z&_Q^7Q<_-mSHMMQno2{k#nhDr%d@#_Mn3*ryb;a)6k}g|In;c4-s1i+C^%3SPVxX%e*rqqj$kk-YlLIU$6PTXD zA~DZ4Bn4Pg zj~~i+Abx^4(%-HAoFj^1eLnJR4n$kmV~xgAYYT>IcWB_DN~K6SAOdB9jiOp&bCR+G;tgCqC0CZF6Qql%+L-*BOB5bm!pD3B13DTwj-WmeM20?8ym1ANK$82a}<<62b9 zhdm=mEhKC}*e>u-zCDKw_XF0e#n6D!AhBgum%W3Y_ytaI*n!`{m{w0TD>1@Q|CrJ!C$S2XP#do#&PM99yO z6>b%Y`G8VEQ9FS>@Nt>cy`uo08Qaq;rLYdD4H9A^%y28N^>afTD#`ZY{?z_ z2=vK;6>u1jF;8=P0VEuph%$S`8PB`}-bsO?!)MDxL0AAu00SiVBc$*2^BuT+c&7#D zp0((5o7uwPa8?QorB*J?_JE%$RcEa+2NNSPGa?W$0}&DNHEZVsNtrm1W;40TjYf4r z$i#`2YP`KE{{X8PFi0@D_W8E=1L$PwYj}Sex>c1-tJ)Y8_&Igs)~OhX{JXUFWX#WR zSx0XPnU%-q$t`QLOA`6W0&-&zz_BOB;y1|84lV4}ilhNDWq?1<1e3qz&PMUWFU2@q zOsam?QfdJbl35XW_WuA$GGa)AMtgtO@rj5Z9_1eb_35HB2AY$W&;I~7r*pP95F&K5 zCZGumrWG<65i&bRZK6p!a6vf%nc+N6kg5{;)t~F=K9`W2lJNU^@k7=3y{k#4n z`xIjB;DV556}+TK<8fo#m_2gMG{U`~6fgxA`O!i-t ze`6*80NXR(Vmy0KagmypQ&Z+>J8zE0bmR;O-#N*M<8`8o0zXdQQ;lE3I!AMuC9S!r z1=lr1Dnw7Q5RYPL_V46SFg`v#q9%KO{{V@Cj^7C%C;-mcoDZxMn8wZTo+V3kYLSeB zb{YDfsLo@_%;fPmKW^IaFd#pmE_{C-;3vnlemsZqJ)^n*0B^xTe{tL~9r5mG{15m~ zoN*Ym8o(GN!JJ1?+Xg_$pL}csRjy}($|M3H84wV^7>_CP?l4`azhaX7%=YcuziI71 zG&OYY*xzv&^ZIY~_u!PZT8er|(*vmFnA`HkU}IL@$l@l)ADNc0Ss-i!gCQ8~B*FFe zi0?nxJO2P_@rms-+I#%l@gT_&Fh;4w4a$k%wg{hY(Y2JZB|tcmd=A<6$F}hi$99%7 z2_%UK5fB9z_m1j_p2H9^{h)iq6v)r*`2PL=U_6pRlQ|{7ubtvyw*(AvqtFR0q+~(c z?J*Oui5=$>c#vguEkZDnAXG#pa}gsm<32OmW_+e6w~S20N8qL+W+b*X7z6t4J7P8w zI5V7mHE<8%$*UtE$7uaNY!3wc83geM_G!HM0w|d7QcVHzAvuWsd%=EYC$vn&{{U!@ z!S|0GtwsQE6PcMidjUP?+(c}+SQef$w1%*=Ld=@O+71_q-!BxgOnvVFYf zdTTnIZndBokbq-9bC?;Q9|`w9qBxAw zBoI-eJA>o(-g~dP$bio_;91s_?Yn91tf)ErMXDsb$Lrt?!R{1Le1hx@?~$xT%X z43Vp}pXM?KKIg!R;vr5mq6F@9Cq8H2zv~<(ocE@*Fa6Rpv#bN8;fZSdlcr{8`d+0V zR+yOY1>Ljkm?`~x%!vZBx`x{kyr0zlJLk82#>h#Hj6lu|49s>hoRSZ7yC|IFqPa>3 z9fF#AJ^W6n+Lp8Tkb-*xvlu904A1IQ1^^Qg5K})lr*}!nr!m?{kRV5Z0VnGl&s6aR z&6p@9Lohms6SPkL2|YbP`G89BoAU0F;#{)wMfx=wvN8R$wX8u&Ln39C7Lb7xk%*b@ z!6_+%IqY&wsk(q|0KyGGs5ODC&t>=~7CI$#JOZ2)da?$QaIDZypGd#CvL z^(R)w*B`xFosQ31RLIhzY3(E9nKrEnu2~|r*kUFOW<-Hg9$yKj>Oq66KY!9NF#wHN z1c`z;Q(Bvr5|0t{8~}473}+G2r(=PRDw(}T#N+pTXt$8tKrR56WCUqdLWmE?EWeY7 z=ej(*fgq9KBN09$-Xe6bp@{^Mv>n*Y_Bh92d`!-z0>_wO^oJLVYHc$=o4W4L9L$t%@VuzMZ!<=g zrg@dS+;%l8ux?PPVJmb{u!f5%TMt74$n6D9_@3ILqE)9$)uipBs(i+q90maB00IDL zB0DwWy%6G&}0mY})saDRF3QnQ5F6yCGdyzUuuKKYg>`H;bXX=XgN;-}7dDgzB z^mCqTbfX6@&nTI*G_(1}O}$FS&OM8Vbc)73cA92w5}jT0WJ53mB?O<9G8?)} zUd>jrunVMN3rI2s3eMtN83(@wHEWA|npE|Q^u;9g8B}!QLv}idImwVSra|K#I&;;Y zv+!&_mD;nyoa&b}Gx=u-EH-=)Yi@^kLgp<6$X?&sJ#x3M5vccu;s3*50Lfq8%h z%lSP^Y83@CrkrWOIfH^^^qc}@WNzCI0_D$XZCUPg6IH=fb?Gq8B(j|(;i4ut`A8bI zm+4!t+>4WaJpDOv3f)u9=${6=TJC+De0zSfd7kQwRz_TcG-xir2+Yz;rMcIYZT4)r zqIB&ld?2bdlBKO`&Q_&!PPuI zqf2*RT5VZ%#@o`PZO!&I0^=qV6L(JiYFBj>NrYgbh+PU48Np#708ZNp+z;rQ$zG*e zRcl;|6p=I?IWV%Twu)(hAWR5|8gyUi<9n!Fx2T<4+quDi%eY%LIE}muR(TZ-EdA`N z)R~`KXBOpQd0o%92{qSo#dd#AMASXxi3czhDsu}8NK;6)dY-9u3sEUl^2KGBSNgcj zlNbuvHmKAnuc%lnR9b=vbY{{(VIN$~1DxhOfq#q3HF|;mC*5Ja!m=&%ex&gJU(?K< z3yV`7+Qny;@bpsNtCWgKkvr9f#ecVYL`W@G3qD>el0ThJ){uTol&Uo`PNlljE?Gv2 zkh%(UrT>mJR_QPCkLxMn~d#d z^?MpE)9N;6cMiQm$4~jM8_o>< z$+xVP6EUnORSM|P%lpb?9Dfw%N#S+nxYoahsHskBC( z!$YKUg&7Z`5ICN*^>HY(EXq$vedoiai81V^|$PWFPT(^^2Ir4L4$q=jeNqm!x&9OUn? zhy#o+`p)`{!}^l?-RmZhqJ}3-H#to{H^li545U*lRwiv~6t%Oh4{*^Pd&cS2D1(bj z0I^s`Am(}(E){55S}j~Lr%$FMNNs^JzChUT8P6D}K`79sM73oMSTuDWajhz5YVhrkL2iuQ<}cw zu3#1Yit_<4if_%}smvKs)zk`93WybH9H}v#Xfe2K!)k@o7MhJttzp#Ol|xK{6C{A6 zCvN9R`W^UTEOoo6KT3R`tiM!WN}SJ&ao#3Ymaq6Z#qbpVG0nJd9@U(VRm!l&%au=v z@-?Y8F4MAVy+ZT~32UD@5eC03qt0H%s`~c)#@dikT_G93Ai7ji4yjoH>t3P^v~=rh zRw?=yXezFf%3gI-XHK5-M8=>qzS%41)9RP%8}(9+n#>X%V0br!X~H?K=a zTTghBj`wV*)GTF60yNVIQb9n!6$k){1)VLaP)a2!$)`wBkbxX zmXhHKr%89ER=7ez>X5S9aPuhzee$#Yt5f*wzxgKr z0L#H2gQNcd-M{0f{{WMZ9RC1O8Ap5RbBbDG@y+x(XOOOw61Ok0sZi%4yu)JOjIEx> zcuAjj5eZLkooEeLG*khoGZ@phrH^lyCNm~aw+^mp=}N4@;Xu-M>HC5Vwhjv99DY*o z&|fIo=`UP9oVvAAu6Ss1{%c!3c{YRYYuZ-L3w&&>*?4;8Y?}Ivvlb`2Qq=n5OW80? zdE%{ZPkl&LWOHiZvJeWyG^slgr+7Pz7|JUeEmGwvlwvDXT5vvSX6ykHdlB@=o*O^$ zM(Y08JL^{x!sM8=4%Y$YH!TK|G%OnQIM*rF{(7&iZLYCc(esfASBlx-$&+cxpUNlf z_JuljRqNBJ)qP5jNxe^*YEouekVK4}$9>5J^xHllvOKWUwkx5%1t2gg#a&K^A_U5a z2itQ1Fl(t?+g6V<>5g3*tI*8gdUZKh8ns5i7q@oyhJMy>WH5bRns+GIiv}Lwa=gZ8 z#za0_u578V;%d@zwQ9j*x{U*IKf{(IzqAv@cHX^fiq`tqQcEd8A%Hy)3Xb6In3$4I zqb2pV)E!@_52@~}?%BoUSx>dZIPh4=x}$=Rq@H)e?(H5&IOM%#=)V3x8>ItRA8#j))=hmjXjC~fex75H<|U9~ zm62AMg6v;3yrWsKd3|L-Qi@ge37A(7O2ONx5Pz0qq&rYG)eJ~hI%anSH*AO+iZM;%sf?ar3xH<9Sr@X@KWcGcT-8<|o?kJCE^>&hzVSxB{@b1)r) z;*Nh-v9D!QDk-&K%05lYASUnAYQiHFE}MqXW$uf{nJUe+c}O-lA(nGA}xvr};FcqNT#Sf}c+Lk(d_ zMN64g7L+PejMiF3vQtj2#sN6(kETg8`b(sO!hB;S3H>v%0DWU99_ODcE-lDO6{~%v z6|Q{B&5tD!6G&@^;cFNs7!fdvd(VEw+J0P+mr%wh>yLi>{YSKkhEDw@kZf9g@%jjZ zoOT*wBaH*3T<$TwMzF7_2$R@-2N4xr0? zr?hs6nfYA!l853ghLM_NkOoTsZFL#>Op2i3N6Yc~_-8Dt^z5x^x}8Rn zDeOkUNd=@5H_v%9Bypuncsdx#SXPLG6W@OW1jKA&IDWpYADf%g->0lp@!bq;EG{oc zOA_m?G!EkQDreubpz3x?#Esn-!&q%mr7Te@3?sbAi(?fk-?phN?@mrt2qZ*($7!4h z#~ND&O;Ae^Mj{UT8TR-3fq}&5_=0uUk9C`nbZqje+Rv{?q8NDesasL`_qFL`Nw?o3 zE-`8XPEp{sGqRRv-vW&2CD~LyxSffBa+-)1yi!asY|KDU zl+Vjj(lsim(p6AKuNlGbj6mBXIl+aUY0_dC7JskncH&<4`n?^J?g*6pms>&(Y{SV_xG=zX8!IDfIcmgp$ zq{up?X$Z3gX(S&bB#)qMoyH*jV~W>V`PQc{>Kp4;T^!7wI+CP;b&vTq+d89i?Vq-m z`i1PQ&~BB)tJ=d=S_lzQ6cKeQ=$)_u!I{9w1y_CZf;?&Kij^%ifdrD~eaMplpJH*m zgP6eK80!aBQBH-vC&GJ$Z@Fl4j#D4izo&hcS!H!-u$Kp(uE!&EhF<6&(L{rZKZx$G zPOgG7WSRSq)Sdp{#Sd)l35TdPW2FhlobA|*M{E{@rC-G3+MJ5&-|U>s#xPR-M;nxu z6dWCggBc{}2zM#*iGqB@1Qd6V6_qQ%W|B^PkNOfnD=r7P3f`R{L}&oc$|gYb+xefV z#-|IWd>@EpXoA#E=(9WuJ2Iwqr!B@i4}f=PeV4E7RHUhu;K+Cl6- z!ec$;i8#h|zO=L<6_`sl!*Zf=Fbom7AT(f-W2;GeW5X_7D6fM^F-3&3T}l^qD<7@pDIW=K$1y;^QdwyjEWg9EUMCN~@Jfjn6(?HR~aQ4$Li`gRfe z&ZzC3PoE0un4vWS)Xlvv#U@UJFbVv5luH}?naudCr@Ve9W+EfvQ_Pjl-6R=cbw{zx zgWtS+;kat z`T0!#zOO0nP)jK8Q8OL8_lcSLD$8`v6#oE2nZWbfC&-B4GoZ0$9VSA9h#7;C+GKS0 zz%v|Qvaz!gYn7|fAsyP%GLLls0PY5RN8|I_d(6y!Vm}`?9ZVM>rsv4*1E>S_k+$Lw z1X2>FQ9yzugDiaGFh8CEkO<;>Vzc~448m6K-KYB$kNtS<0|gWP;%B#S&-%OkC!`5R zV8S9teM30#GZQ5E0PrJHdS*xvXCojFc!|a#r}ZE#{URlP%qSt4n35^+h=}bFQQAs> zy#D~af9@hCB79{vO)>~=Yu-nQ6FQ+}X_MiRR^WXb+j@|xhYKEMG*#ZbV44&GvCm+rT;G|aq zNgYGR*~EE|GB?QW#A~g>;vgnSq9$1p+wdVDiSASVrf0uz#DCfOp8o&`6-om+GDHXf ze|VGQwhjT1am4FQjAMPVC+Iq)`7!?h3=SdHw>93u?IjE-{_)-;`}X|)@g1Z4N8&#{ zza6GW>UDsVsFNh)Yyq(8J^uiTJgM2Xugk-xqD_aEjDwCVZ|<4q3iq6C~^ z!1KJc_}E9kd207SjeYz8xS1Wn_Qn|SUnI*QkH2D{+rN-__x|ubqGo5e7S`oxS+cpZae zbmBoUe>eOj{{TO+;xkNnpRxX-$Mp7ro-3Mls@Z@Mh!b4N49{W@#C|*X{6R-yp7SJM znTe13-!?UN00G=jc){#{`Mq28$5da2OvyR=55C9zHX{;G0&ZKv(RD3kNlO_5QkAw; zw|+;+NJc^?PG`A!&tjPV(c8IW)L{^D-A+f`j2+M95%0!n+n{YENXeh+{XU)nnN|M) zk??$|i88~2hC!{;P5?jc{{UC|6XHWZw`r5|i0#@U4o75lKp@K<@qyxY{C9IfI}W1M=zz#wRCE!AA-&QTkb#U|#CkA|!r2ie`N+F_Q2ZC3b;0kI6$)LQy}N z2>cYH>5!cy<|8}CdmQ)Nn4O1EVX0bEa}`yDt_V;u^@fw59ltakIo;K6ls^rJcBVsC zuw=lgK}z8J8$F7^&6L!E5|W=GmJk^JJ9)?KUA6EGeMlIA5Hfz`%zd-kbd~q~xm5H3 zL`X)7GvZ_t0rB8xJRuy9g5^e#Xy(+rNbTIcO8$Jv4SIK3O2necRhKyDGUJ7F16`y* z`GtP5@x?2JsJ$c-rVO3uKI{O`k@VuXTFlZ?brMvpsU1Lj{{Z^<{wKgt9v33o#n-6z zzwPzwH0#j3#tEgVKOj8+052h6P_nP+vWuDhqya|;BLGzz%{Xj{gE;}W5+(rmC)y@B zS8+g)3S@#z7?%A^Pl8F_BVxO6(N1+klJx%oiX&?5GQI5jxp8vQ*j>b5UU|RxOAjMH z?MTDdLfn9~>!CgTi9%m4b{9a>rmVq+ft<)AFnXgK$VMZ6D7UnYI`vg)>Fdlh*)&g^TQIRzwklwkYr(zZ z4oW4i%T}8tdVqEy8JXB;EN8r_jpjnb6e<%*y*Uzh=n*(Sf6&e(#V3Q|@vd#o=+)Ap ze-^zffnj!XZzA=RuypM%$XRx!LuuJeEtS}U5;RYX(oC6QMGKtNsY-=kPMUxk0N1cD z1Po^%QT4&%!)V2kUsz>T5<8PKB2N3x(cD2(!u|Cj(0&4+p}wwlBZDH(-Ef|xT(`(< zZCbHvykozTlGZDx=A8uYU1@sso0_Uh?4dS?muUG=vARResnKdcI|mBGc>qu55Hzp6cw2s`cI9Ixs9op$FFW^ z3o|P;E>W{_Rt@o1ytS23vQWXUJYakH{Ny%#&8{lctP-_4%~8ok{{UA|62vZ@iT9Bk z5Pvoj*i_~SMGboDf{q!8QnEC<<1!~DK;Op+4p&Y3J!heOZ(FP!AB1Ueu94^LZt#u1 zLcTQ`oU<2mOzYtqqB^Vp00?^Z z&-gDKdNqXzqDDL;)ce5PceCJmk;9h4`KBwY8#I_*kcpf3wXGj zbq)!SOJsuSb!|$gCjQMz?3zuxdXh^kBZdM}3{D~lFbO-z6?LFAouZo~QzZ710dJH^ zz&|d;#9;Az$@=xb(5|-iyRB8epL)&7zMuK-WzM4gM`zZnk;pjjP5Qj*(478TnSQ1B zh3sAIz)0#v6!!MkTV|?UY=Q>*!sfeJ)wg7Ox)jYjfQ+=w#WV})vkH!-okEl`>M>JH z0SPBG=@d~yv81I)6l;iLDV@P2O5~De0g^$-S9Gzu>x9sMCMxZ`l#cYxg6tHZ;aW^ zsc$5v!t7@Q{<`F156$vw#mW(-+rW4$lqxp5A6$Lb58c`d`3<1aGF{nql9kMvm zxX`OjNFbnHWSvBXf?yu=19P@B6UK`vLY*}SVh%yand$B(e%eG0 zZYDoW`6zS?i!*;&>CNPsnfxn;;?${GVS4ucvv}m|qfJKHw8hvLKkUe0Dk-)d5CPmE zv}zSDl&K8h>TJvcd=n?fcjR_I?ZUVyyqt5 zRJO?`*i#s#p<=5{?r@(d?c*ihIN!(Azx<=i>ZKa5s>ZW409l1UK*a|TS9k~7ubII&;V2HLW)8B-(!+5}@``fc2Ur?vFEr+X`tbn|Q` zM=K+cVCt-Iuf0X4lg7co3WdiovHq~H_JTF|HYG%J4(^d2%DU80!lfrxQ;7vgB1fNM z4~g9?HdP*#<*ii!y*r9-0mvjaJM}?^i4y=A2aN{*021z3SE;{H-6*$9iz^d1Z-8O2 zY%ZJ`?A5A^5Y4-1R1G$PUcpGf$jZ`CxDOcaH1C8tBXUYN)Wf?`xUSXcqn?^&%rK`o z?WvhvwHQB_PZke|dAsvkw&_%wm8rE#b!!tZysaV-v?VeKrZJ>60|!?Mw=Uu}Flljq zSgu2Px-0fNxlAR1pGSw-)|3&;l7*P|l$iJ&b_K+DnfYOQuT<2c$}|~PU`a4=a5ZKp zB1GdjDh)ePZpl$t%m!Ej0R}-LVoCB2{{Zn<^0{W;ChLbWmRO48M6)99y7n@X<0vrm$KY4#I|-C(U-*1C49 zEXE9zz6M8N0N8f$a#U;3+bprOyfoOT*r%zhDbHnZxLWD5+h)I$N5uy#P(b*Uc=nHE z_LA&PR6?T-h!Kg5Z=bvf8g|GNz*WMEhe*;47&#Dg*q`zH{LC!+l}(Kc7&l^_RcaQM ztlJt^6ZUnhMTmmzm|T~s@*^r#{?P*yB2hCEdD2PsCL`Y`v=19^jpIACK;Vxt{JzJ} zk@xKzjg)t3CyqD?yA;nIu5xXIn& z+vwJ}Zu0!e(H1j7RY8iZ=OAT<5)(5dkDpmwT_qpGR!Sf{8oHE_RH#)|0(ztqKKUc^ zvE_F(5pF*eDpY2s1-VkA)2azN+;x^ifdJ1**b}FZ>OZQM`lZ!=u=PhWs(O7_lkwxp zYAH75X{}DaQWCt!UI|s(hh*cuyG;>287T4*hyf6JcP~pzN*2yTwK|OYj$xe6#aUo> zn1TtGnQfk|vydlH0!#>r$i_G9ci%rMIZCdDK0TXEDYorpF%}B2v`L_s1Q@sU{I)#E zpai=_L{D&z-TqlsnN&%UAc79t_Rhrnc+UI<((XhQXfjIR>_F2aFmt!xhMVcR>u&b` zdhuDW1;+LCPd2&NrFNP!VLBHz=SapKMR|1vjhL!yX<1^*CPZL9F2APrv}mX)g&>*g z3=djvr=WJQ{$Ng?2h`5)O2IC4=45Am zyLX7h;xfv0Td7~maDJX~5vQ`xM;c!vR=$;`Xz6S6*r z7@3)Zg|DMlwP_PkPeK9!Qb7i1E#10TVe?N;tO7A@$suq6&Ihm{ne4uC5mG0u{PMP^ zJLCCUi>H4(gY^52aqR6XoO0Gf9HY+*VS7@ww*B_0D$CxbXiCzXLJFG63gR+kSR&FD zT1Yiqy8>n!VW2=D5gpIIcp2#sbqt2jh@8jFe((V87>*X65zd{py{WN{V_-A5{+*-~ z+c7_LZa@;G=F|qUb?jUsDF6=^Lj0K74<8~r-m5)QS0Q?2W_yq~$7vgGU>#gy;+l#f zRF9Z3Fd)Fl4ZcavFl1%}aKP+RwZt&4W2;Urz8(DEA|PP??YI?fO7t9UIACX8XlK(% z!7>ll_YbxfKOLb8vm{6wwlVz2l5%_Q2gz3UlS z7F5xyH}$O96@9E7c(9h;{0#P&>}oJ5X8}*M`1Ksb$xrAN#Zu1W>a(~x+z#MIGn~m2 z!&^Ib(L~h3m{?Lyl6J}L3>GAx8G=+v{Ek=gXtS&;wRZ7M!lVmne<1dDeXGuqm;}?C zFvV7Akdx|+jCUr%3R=%3lM^rpm_4uugYQ37Btbch{%cRtPPjgEWv5L(-3X2)>3>|h82U>)|J!Y2geajShU z=}kPG;&O9|nwHp+GdS*q3=i~cNk?OtTt}UWf{$}$hdts%Lc4U%>`X%Bc*^#O#s~SG zhZZYjN=$$a_wObTi2gtfZckG-_?$MSG*yxj_855-u04&g&fVD&ANxcif6@^%+9UHN zGvX=<3cv!SK_o6FCP&Q^j{9kV7|SU_sb(sm2@C^o=OAsg4Y4@igO?Yi6)8&n0>8<$ zdx%4b3V%V8?cQU&Nw3SwN4M=AralstbyHY@5^#O|K){IZ4h-jsDya(0p!}jt4Y6{>GkOS;9-ILw2axy*O;G`l;yhoRR#DBERe#EH+WI-oP_U)6A2ee7| z+#Usm3WPx<<#JL9~ zGqFtmN+x1^ehNE&BftLuYyFD>iC`qi+doX~JK)JXVkGe@DlDq3Y47}`e<3}_+mhdw zHrltu9EesC?qfaXf4BW&p8fv-?fgjg^89z7&(EH#DG0>vgTHVmw2g)WlNpXBS`<&r z0VEEw2PQ-vx9t*QNq{)(hUd7u!1fpk@sA-PnEwEO+CRVN@*Xo0+B;8q@8!3s$0H5& z9`V_Kr`9G~bfSojtz-ble$hR&j@~i-(T3{3TzpABK2qWB{rew}c<&Sa=U&mR~epZ@Rb9sHxT z$L1rv7ij+gt@-))6BYy%VS_LU{zDUhCw}-Fo;wk?5Cz%_A_jj_{CS)Y<^nu_BQg2! z-|^miNp}5qOP5S60UufR&dfvtKpnOO22E1gS&1+=h;P$id&lNJNHGuet?JA)dx%B^ zJ9inG{h~inpU0Q_zgY4a?f$6#@FIO1faHGcDG?+Sfw;jVsBH#&k-P=0jEo2*!I2{b z&YWkeaU_VuXMQCA0Fp!mt#LiZ1pfdYL{s0&W;+j;@F5mATi2O%y zpL(UfTu*q|M2VT0KENM%9AMFqOk`xh%+7oN01soU`2)w@z^_uqI+Q-a+m+8|h%TB;qxU-7Qc@xS+;k^Aq^|#4cb)3M6JCCVvq<{{Rs^%yLTZ zbeRHonc4)93BbY5WDs=lQe{B^NWt4AWXAD3Z`|%j3rA5pTs!nE=L#Y4bKFV*3)L|o z*pdktp55YqQ7ICR!~Mi3;X0!`PJW&CJ|kiNd<|7Yu`G5H6UZIdNZ)J$2fJ;?e)Ui3 z+NCqSdp#bzKBIkrNT1u2(qPz#S@)44XubQ&L6hL)zD`Mu z0iF}wP*>E+Oo^Dp?H>3Y{$J2%jF;-yNFs|3g-zL3;#=&wD}tnc%N}yU5D9@l&u}>- ziR~Ys$6<3uxMS0ewS%xT4fH+ZQ3E1R6}ze-8#pAHQQ9|&gZ1h55_7?S>AR8Ix5Rpf z$1UyXE;wD@8>f$Us+i~6yH{IIfY?&Mb6|x}vnz25IfR11`-w^XvGU53tOk7!wOcnjhDV5q;*v@jW0GZ=Ox;xFsKbK6T*%`XF^R$o1 zrBpFFrHaQ8s%#VN8T`J2+=j&e0Lu=gN~U8L9M~%j3f)j}1Z)8(+prPa&$Mw@yQQk0 zqN7qC!{?;Mfh~gt4nW4{Sio)>zEar9uYZ4v-nE0jdYo)$=%f_d!D&7UXQ$MIYhO&w5*zr;Dx0db`W>3sGx*3wCK+p{lD}1!?P)eWPR(HP~!a%(y$x zc`qX6_WY)?_VZ|JxogQV_;aY34RSz+g-| z1~@5^ZbgC7p#sv~jBYlhq+5`ZkdR3WNI7ZAFGzR$+xz|n&pFSz&*$9Nbzi|AwOOi@ zr8q#w%BjFq;0}d!O77E;rSa^Z&8(l${nt1i0biYoc8CVon08q^yDBRvr`sSEtyU zkJ94rYK(13#wh!?{AHg{ACRK#YgoCrbRiRgF{sQqKz>4oGj19bI$&1rjzN0kq@xT9tY70Wt9$z}$Z36=>8RE(=L0C9P_BF_Mp zKb*lkh(G#LE8%)}QymW&F6=9NsxAsR)Qa>Q*bO!UT&zFok(IV-Dau=$J)M8}rDVft zy^^|*&dwf4$)$ALjocYZixM0Z*00!#Yj}~C$Eo^w+BgS7E?dwMLM{ES>c|i!T^^t} z>BaMu3nL!8@~)ZVm&jui$zPySo#I)0eV&&m&v~E7`n&v0b_q*f1rsYS8)DTAQYM!l zm66I#1H0d0C?HFK;g~qk%r7-_yE~#PwkDrd&1>Kn3?x)!DNyh(v+Z?XvqMsVmzO8W zN16F1uX0XnLtpMcQHQ1ZIkgF^Ck}d4T_Dj3L1|iN`oYjaU;A&-(mc=I(Y$&Tqau)A zek7^QhVh0_`27MlVG&H*9Fz)AYGsDPLT+VgDC-u>KK(Oe@~!&s!7J(FZb?zGnHr2x z;^<*2D7eqGP~<@cq|p7LPf8>8@x!ZYnw0-;G;5IQRMFTn%M5nQEJt=4b28b|MX7=# z*{1wDV+hAE3pBX{w6!L1qYy8{weTQ9R7*h{5uIT!j(JZOH8tXz9C zbj72#-IX-{h2+n8EWfs-wlG!}o)OMqz?!OE_JJo`f&Qj6>)&lmj2m&Kms>2kf#}h@xh08fyom!R)>(qf80uKqLha=yS z>~7>Oy%5fB?^Q8oH`RFR$v?A9HA9}7FVpcK`_6+Z$L5?yUzx;QYrh%TBjISLfhAK9 zA?VZa{$rt+h7C7;TXW5*Z*9n`txyW$^tnq!-ZASoRgEYrH+oa^?epks>To$yTv z03!kSa~>x-YOYNDo@|e;OJ&KETn45$bXJ_*ec zN2pn1MZ`*ACp~Qhg+WLNTt!m)Ell0dT4K0jE=d;+lb3fsAX_RfqoPN%F+(K|MX!0X zG|!(em_A$=tt$EN52-#cAfv$~W5#}$JNAk6--*hPEtU_+-#e3HNlaKAe_3h0hx+k7 z;XTOkRki!EQ}wqn$$x3AfW1&r971K{9mrhprYVRI9`-P!<(`=zpY&~NJ=WAjxKdUm ztj)Q5Ibe3+@cBPL!vU%DQ+|o((qW$@v#TZDl23aspZuS@xUulXHR&7`aQIN>>(R4- z`=?(wuY5O14w#DGmnmd-%P-P>c1(ZBYsMJWi#%CIGJRsZT2}5_dz~9(thyJBRz*1z zIaT;M=y$&zfAn^Y>^2%y8RPL}I1lh#7m_jfVo+Z5-=rKeJum9)bAEB|^E>LNk9zh~ zMaW!XNN(3P&~@@AD%q*fK}S%D1`?jx^LC(({)9u`_Rn*%aQbKDG^DT+>JH+9{AKZm z{_`vCzdX__CbEy~^Bs?zC6rm4(!;VID7VWsgve~`)SIi}03U7ygAN2J2kA&_2XbGK zgcvd_J~vCd39ce-Gc)yAi+?vKWOu#)-D8Jek#1tG$_(mXCmprx5QH+eHa z8i99;FmK47e=~n&$eH!Ns9|?7R(f*th4%wJw-zRQTy(xjgApYftia353+;>BMPU=S z4mP6d)_>8_HY zg0sJaXZ&2#ZeWuUfUO;An+Ekjl|b4>9xb|CN4-9qJ0=jtr);$=YI zEcwQr+fy1@RRLpj^1c~iu}rlAaV{qAKQv6emsa&%H-QB9sZM0|C*Bw!!8ap6P$1z; zsY+s2hP&j>rd&nQvm1)#dgZ1(a;Zi}Kmj8leKOoDD05YQL3#4|D}g?bEf*#8x1(Xs z+{iSiH(RKNR>>yYl4&yLw6D-N$q;SQC6#dz>Tmb-eedQAznJ7J8Uzq08xzdusXc?s zK~-dKVko{4y566A5G0hreQLG2F8aBDGlLdZjV3&PYn3@#pQdz4#}|q*DLrD&r=zyWh+LFuF59-n%o7-!+zgJM-Lh52YLF1 zl>NGAltN4~E(OS9*m6P9}`>tkRT$J93+{b)smM;Ew z_nm3DTFc-=qwtUsvJO<&VX*4ZCe%`%I@;zzJu8sOIT`BL4N-~g$xML7wy!HIyPn2T z3Mw}5)W0)lsJ%=52q1w>@saM(5BDrJ&Gk!O^;>RTwV$wWN!pi}7JxHhh^)9|>dR+F zj~jARaAEp0w7u7`)>thO*XngCllz(C*1rx?dqHZGfr+f~&@DM@^4HiLVhh?mZtv_7 zA!;DHSW6FJ6`UNxjQP7J!IMS{Ouua~tXI`AY>oJ`Q`502_nV0oi+{DyPrSDSg7aCM z1}O+5dBGGAw2m-h>5OgXG%`6SXl&A`mIjb8tpGUPxB!Ea@o{X?=z>{wvInDpzH`pN zcOn|rBARzHnr|^c7FtJry(kPOln;qKaRc`fZjP-mqji_g2Q}aP1Mn(_*1}j5>AVBa z&E%01Szn8>47aJDjIDRZumZ+O%y(74ZRUt=!%#LL~{Qm290|DdA2D#xEmOR z!v!9&*|Oa6(q_qYXTxUCwzr}O<&SZvek@8|OnzVY*gDA>SJ`$E1%e$q>0=X;1km!f z^YohqCYv(F`j-0D-a#_P_)N{(t|s@b1q2C**Yyq#uiCVtsh5sYOTFzEz(ur@!u(td zk#gu&bxh=BK`7gOm2tqd*UqX~0##{tVvov>Xd$gdDCF7npKYF=wS8?ch+EFIoom(R z0CAcVlmu0P_YPv?bEglf%fF1XEWf+4zVWbGRmy!I$_m1t9MF57I!dlQ*u2lob1%`N zU)moEtJ7S&S`6Ts{|E57P%_lsw)sy6)5eE7|4o6trC@qDgN(9a6hK%X2>6=7FK1Yh zW<-0BeNzu`%#6dnj7%Z9mS+_Mc&6K3NG-5pltE+E>&rgeb3T6Rnqcx0q@EPp`l!z` zwvQib2@n&(#5ZK3yZUeU#f>QHB4G3F{=2X_KGa44iI18)#xkr)1tvFf?X_7dY&yph zmYpGjpeO}ETw-KU0v$=9{$G?VtEP{Qk{@tD=C$RUi2t&+OR?{{ZAZ zI#@Ym8BF2}Zk{f!0fI5MkWDm*O8y_<*PBaaP8uRqI1-=38i{@Mk`Q2ieU)FMxoQnN zKybCFr^Vc?5RANT*R&1#vJEw3FrlD{473K~w|z+?KZ7T;sNlF`X5E|)7`tmLYypKT z*Rim8G2sSJ*y4WjUMyKpb@-B#MO*Hs;AP&zHIIvOf zHeDx=TmjZM>7>!8?mHl1?fLda=|Oe>`N%W)iR8}Fxu549RTniXbEw$hnFNm=>zUaa z*86yF4-kK^%6W}?lXYuJ4lb+>M(}Lioo;oHjQhEd26oV^ZDXdZHM z4K5hk+7~o_wlmk#9p+3+hHM63#|K!Zn_UtH(VB}S6=(qBH2dH*KCBD?PoRxEm_jD` z+?*GWC<{-TKbvHuJw8ejH3iV62 z+6V|voTy!Vxzl-=r zTnkIz*KTXJHPXc~60o(x)f*?Ba}v2J8*xC5B>LOy8`mS|7`3=sm41fI!Fy7%#_QG3 zL$QDbd)brTX=yK8g9Wq8r=L}$neR#@ThfQJ00RQ1t2;IQyFj4%x`0Nz{qC^H;(5pH+2q#G(bSKOiWo@en5}8lXxD0?x_3Txoqhy+;?cw|*yBe2KkjNPb9 zGxfU&-_!fX!cU$egxlwvu6=mAHY6+6!+$4)6k{a$;ak(Pp~JWQ7?r9a>-Pu>+*gDD z0D(bkG7nlZ`JvylHG2ef?=ZsjFOHo4>!@})x%UsyKmE!2Be2xK@%PJnF@tGGM!F;Z zK$PH}4Lh*t< zZI9byN9yW|{M4>DlwEC}_B?CTwK93kr}j0yGA8E*D}^YLtF@;$QtyNJ`z*n)#=u}p z)%(r>ax+UjvS6!NTSnr`SMMN!2XM-}#VQn%L9L9=_}<81*74z)joT!@oP(3_SdYU@ z2JuAg_TZnNUueT840mQ`hVSl@3t%r(7~gy|bt0Q=?vO9U0i+gzSp`9duX8sRnCgWl zA9rkEG8bLBT60`5d^AE8T#V>N=A0E=mh;ulJK7Jg+{K44%djjfS z*)0^6>Pw2O*L!jfIXoHA-*a?^urX^>wJ$pVxeG}oi}+lSfeGQ2TVYS^JA29~ROq#J z^)xoga5yHV^ZcStf{T^;Xuz$hGicGGm~iN zjTUzh=SYo>M@a!EOYiaY3e1Us&3r3kv3RNt#ah#UGsp64_$(cG3B{c7hxHWxqKI)B zdNQzXAk*4bV0{oX)7rM*Gm&d+FJ$QYim5io2o$Xs5HG@thXjMu)4oH7a+`=x08`=+ zN0#A!2>sdJ@rlX}J$+ynl-htLGgDsJKf38GKTwGgx+Lzo%E&1sC3(*2BRHh1Lw6hV zyJ0Kd%Qm!?c(2x3knKzVH`F8+x2elVK9Q@JGD^npnM~0JjA8r{&+ev4PZ4bRa;G&& zIzGVZ@QNPXWlA`&85Wnq((?knMp)VLhL~WEO}QX zi1(9|@sot+KzImTl4qEX3K9V8igL`76(pIP@h<~DGhRTP;Dnn@Jf$3GR|4ljOS8`w z8(i05C1;HiuQpbr?91vue4H1pO1c&JB7UlL9zk>0>|Tan{vU%g>Url_bxeOn~#N*G(aH(X7)Rw)SRPjhHy% zkI~eZrD}^S>_S5lNiQQ2`=&pU+~J=gsP=C9EG3r-m>|I-StE{wVcN5exELYhm z>XuRorb&<=e%sa&J~yxZKo%k|sJ?tk3BT7)@HHhqGcV6q)Vj(6l7-K02=)ynyx3IzPNrjoK&2tgr|B~8BwGE-HW zJ}9U5+|)R`e$hD{tGA9)l1r{@3zz#qs`4QS>~0%Q$O())Ed6?gEH$Zj|F3l5XZKBs z#46PhDbZK@hsaBLzf#3*AfDw+tJL+T)$2^xw$~=HYKqAe7~QeXkAmYwq`XH-grFFn z(&D!LNUxx-SJ7c`xIwQV+CK1Z4WM^seS%XZ;Jao~&b9kB?LR;#HA#)N=+AZf#_Auy z_a9*Q(b?{8)Q-?Uz<$K#>FcI7z$yv-Ml9?6VXO1#q1%o=a(kkbDxz3-_Yupi+l6hS zm4dP2PfBK{ry__e?EIg<^-u487N*Zsk)sEfTadlViN{)tGVf(OUH(yoHxB#9Rd)S`W$C* zYKGYz<*I%Bp2D}Rb&jw>h_ACkHZiKep7>RzXA3bGC$h``0J?<*$Hynx$#!|c6)(z9 z&E8mJWOHqXq^#L>Gv?f?5d0ri)jof%lGgcQrR5h*q5tNQ9&GSZ7d%bqmXW_mWYLT{ z{m!#QQtcNE&u?|wiC89+PJ5;;ByQ>FQgob+@z-LKr7gC zjpQuEUpH;ZDAH&_RK}nrTL<<$x$&K44<_S9ZCH<>)Kby+S^HhP`16ssm8Xx4u5*KS zvCN6#XN+E0Y~&dkFYav;a3QOE{vl@CcOki4hpo0qX_sHK&Fn3crH?f=6aGARF!Mvm zZL-SYDi6Q!H$`mmo`id5h5$=So;m4bJjtby4Ajwki#uO%-(bt`mOJ27)+rq)k55v7 zXC}7-G@Vy`-csw=GMCDRl|{Q*_iZ5DHos!#TLh0G7C=9Tz2Hb{VYAldkA4>lzCq*k zeSLc*Siya@;IRh?w4_+3IFFO7_@uhyCwoteH49qiCIgohKy3`Hvcb0Oc&_-(gZ4=w zlXmeLl-5*ehPE~Zxt{-=v2kL6toGu+Ib+xH%xnh7BBy~h#~aPzeqWo9kd94(6(m?z zxL-%NM%er^OMfiykwi*(N+De06#{8QgpqYQ+3@C2O6;ATs<_ev<`+3i1ZMtr^2Q&7p*t(RAc1 z`dG~-n}1y#+LAd!FZj}xcZj^P_;TPq6$fN$?XB89|G31bl``&`#ImJALV$H9B&0awD9x6&Gh(NDjAhT7e0Vnfz?LB+t5 zh@)(LG0|y2q+S7Tpg^wEmotaC(xm{&$rc}NF~d;UofYo8`X#G|D~XBoa@ICNs9`Xt zNMpf00hCM(C7^^DD27WW6{qYuXirr>cUxVqHX?`H+tLU2DZ2!{jnoXirnv*=owK$Q4N-;uDa-M2dF-(Fi`#_nQ{ElLAK+CHCF+C)L ztmc%zl^-`7#ZnQ1WN=~NP3hBkT*?`}=9_Q#aW@Y@SA@u7eUMVI#p9qNGQb50=rNN$ z{Z7=X%m{Gj$>YB<6=q4w1}mDFOUW*!s;#6i(YHpr!g)NCEpo#FEnf9HvV=IF_?R9~ z?lCY3vAeJVuwp$6qF$X7cii+zX~!zH{tqgT@I+c0_<_L88Wa?EqGsk?u>#m4Er5jrjp0PB!sJt$K8W-SpF_8FRV zi}me;5V_Gjb@!db$dtdOv$OPkk{zzU3#a?71GpA#s;{NpuV?f3P)T?!R(m{1`tjy$ z9xB{-i>u8xH6hQggQf#qq9RO2`tiw$5GHC*8BP8tybJ;mf!A-D-c@Kk&FA{)FH2}u z$*;?#^}ZPP#9dzE>V+3dAXbgnEgn2`{MDfD!;(8sMO#03z&`VMYW9J1H%k$Cra z4YX-fzJ_hnbnW6~=l^*te@J_svS)naS49P&m}R%McxJGF&CIF_Sz)oGp_avOlry*> zMrG96#qW6i@>r0vpkyD9m1G>XMx*SwXGw)Td>9gOqL>_jLSyHI5zW~PJN$O=XYct&P3V$T z*=SkM1gLm{Z9j#zNX(5LHfLuJbA)5f3FUoiDm8%%RzBzg(KsLVp2yuXEHL?~kD(Xk zdLqtTN$*P;su80- zzz#e|PA<{$eu`r(Zfo(lClZk_g1)s#hiWAsg7x&n!&Pl~h!L$GT>t5NThdJ4yjoP& zqLPl?l^c`cE6cEFViJINN>JDJG-aw?nGJIu--KKQJ+m-oe)leM|MjYQPGlMr=V=U@ z(Ce{-su=Ei-T{y|;6vXVwDDV7&s<=ps=B+8ga~RssgdoY_`+kJc7Im$3sn%XqTBF3 z4TZ}rGu)ZyQ~A}9O^v5J;^9vuSCRiS)$dPdC=!%2{fb~rW}8q6y1ZL{>TLD+tzfy= zDZ(U)S+})enPmNPFPR#Yq_+-voxaw%`mCoe1+8*Snpm5DD2C8D(#G$cxwF)bSmUIh zus5^H?VDkP!mRh;@&h&?VG}&lI*&T@q#WcYSCc6-QUFWeWA~2ZM z?cahyw)yzCKGi7~0TnQ|a%vu(S9nMUwz%=cmK;=82J_!PNOcSl0Aggxy%Cwfa=lS~yYhiQ@Ua(r4~9%Tnx?4jQQ&Tcu(`Cv3p#hc>#6 zGq7C)KA9q$#(@V=SyA?&kZB-URwz%q(*bo<;phoPnQ`1fTocz*9lKneSYDSP{_QfGO*B?R{jo176Mrs0@C*2DE?Aoy=2dFS9lDPDoevHZhlM# zH~|zg{X9vbAj%x`puS0#<%g`IKth$miQ`3{o;|*TAL%W!QUGBGWU`uED#lBdBQ}Ah20SvzG zMQ|`Q+Ja35^FFdiiU5R%=csU~6wr~hQ#Yw!w4|@UeBjGGc$p+G{kB{Sdv>qJ&`QC> z?ynVtRAOGyMq>b_z8%g4eoQ*c+dqNs&76|J6HQI zV&BCD8NzP?lNq=Ha7~uz(Z)4osQGNW#?IY8lMNedYcdW004dAZyyf4@aX!Jwfu8_@ zv`hl|MNCIbr}5@gSn8L8Bi0)E%|TV(+Bm|^&1V`^ql~u(dP=?CJ4D&Dh^0IE5um_UYRaO}8i zt2L4$>>^^9TmC{MgYEspACc0HiYi-1<+_iWjBKMWp@u+<9$bzWGxzKT!Iyh3Svm}G z8{alkB3!4?$B^(+zP7vWEB`n07!(W6xwyCJ)Us~yBTR4iKPk&)-ZEwn8qr^w@cWatl*=&S{D_zOJGyeIXPyl zm^Fli-B*oxC!`QPUV^VG_N$)VUhkO`tVW~DC#RMt4MaOX+#U_0d34E&%L&P_Wh$%a zHLn|5RU3}n7Q^lkR!b+Pz2N@)n zXckW(E34YFgT)g5mOq{h?WJyLVWxTt{qE&i9Wf+tY13#6nv2yqWuD>Hr+2Uh5?cdZUK%JY2K@DBt?gJ%N&NsB*1-r)Yss{LBPCHJ{^j8SBBPrA>2 zOV8vf3z6SRSkM;5zM@;H^-Pkw{6+WP5+XC1A~t29EcqdNsH?bXumkQf-GB+AmN@Xx zt|GRU)TFdlj1KGH)3=qRHIv-6KWeU}LZfQDG?!b8k8*-<(ZfBYN#K;Z>0)QmyN}ND zt0E8$0%f0GI|J2yYNUDS8WF=WAlge%LpP2N@C+=Bmpc(%jO>fh=KQ)F@-j395NXYn)l}hd@eqmg9=|+G z&LJYyGqSx5uGx=i*Xj%+@|3Il(LUOL@zOOQf3sYhEyzcz+@yar_18#Blnm>gU$WWE z2UtE4*yNM>I)bm{U8PEI|f_G5}e`n zeJt)YTW4F~DQjybzEO3bC7A2lz#CN@Fy_(ftepP5dE2F%5KOi4;olGR>>GgYG0Ak5 zDj5}G6Av_rh84nwy=s@eP#z2fNc-{mNBqs~<53DJem6ps^$$?Ij_-!s&9{nM1lpQv ztUh^3t?jMvKKkG5pznsEc^#PqYP&Q*giG+<>@}Xqc-fY>;6r}4#6+&WB)@{xGq#Y& zThkIENI7e>A@Qs)XMb#$TfJB2o>qRWvnsM5E{rhl*m&xBRW_umUr()*;a~u4C ziJ~4fQ}Sm+i!Cn%$^+<~NMa2tu^DDDvu$H`7K{RKX6!#eOUN|2IwK~z6Or*4@vE%z z`D1(Av0RU%vV5Y!&fJ$4+ohu_(WItN>1t||`Q`LoRS2OVqe;EQtY?6@MG@Jc6V$Jx zo*S7f%6puC?TMW{IKPG^IsaTQ2;<}7F(<h@&W_(mRN99cdjW9H}ofr45;{I`He%lNl2!CRJV>PafnIR)JZXFh3KP(J7LI%6gz zFCAka$pg9t27^w6vse4Jz$E7oRwrLiDl$BVjGy&}w4b{Dg#xGQ^ORI_Kj8>ReCGQ| z#FWinI^<-nRU}Jl*dTMbUr{Fvzx@iTAiJulB+cTUGwe@#CrOAg9df^db(nNRXl6Vm zJ4tvhLiv}tUxCgdxp%dvq*(}= zWmaZOL^9B(x;wJb0AjYDYWqnG&OoRMrh&=IN(M{6&0P7Y_O(3S?Bz2T2oBBQLm32K zf-&3eN+NOws3->Wj|AfOQIGEt@Uqlq!gFKU%Wq&rwxx$!4-?C^gBn%Gnutj^NmQ#I z(@d&^2?w{GTH5!{8^&{8lvj(=ieF>H6VbzOCF1r_`TPgde^?@K-g*YzwoI`=Lk%chH7Q~#~^_g@w+@SpRiHcl@uhTWG!?c}ZX~sf{zfL)3kl+y+c{x0o z@kUH{70nvImSev4rBlGU^v6mH}&HLBJz-Ul1jO)@D z^-uz`iI_vZS=J)yn`GPDz2bM44g}QiuUqJ?-W<#HK&(l!B7xQQ8#4Eb$N1f0@C`C0 z{`JL&XHhFX2wy41Ck8IMLe)-d1PVAoD*H*^j6#OBBOdnC?SmAqb zD|}l>eXi(ul6R39>3$^2UGh#wUXIVqGv_}l#v64F!WI@#_c;+4AA+BH70!WWFpb0# zM6fVMiWr^++t?=BB+|za)K7a>11z?m&iVTmhNjMtW7sHqNfBcYx@Q#CGVNg-unl^R z->f+w7tMvyWPe=>Qg86ls-F#>-~&mhUh61u7!B+FMVy znUR#hWWYOA!Df$2=CD(oYa)fPTu+}iqg3S+rOJ#WRr-qgM{~Zxl;1?4^bcnjT88bi z$CePN`Np^Sd?S_jcC3Ws`CjR*L~3$OiS)?0KwkpAXu8>8^jVPRw%O z%fTiyHLDx!Yb)wF{^=iL1T{ljxWOw$%PNb}5H?KbKk0yq2H+9whcPs=lfZDKUM$e^__GdL||Q zhgLe0Hc|h4<%IU&s(2+YC6rR-J`G}v&y}2B<+7{QGEPVwvO)DuKc#!A7`)8MxTnP* z{SXCvdM%#Q=YUS@pnck7n`Qtz} zAu8bC&0O@1vgMF8%V{C{+bLOliz-lYW5#R#!SO?Y1Qu3@jXQ3pk-aV#6#G z*}2Uxrz(P>2${WN7L*h&()Tw*luJ9coymq$bRoX}Bw1cOIAU#!QD9Vng{#%kX4uQ< z275u1_jG>-!JC>Tc?_0Jzx=BaYoPtt@_7%Gbqg~Xe|b{)wY6O+5*&1^2q9@wb=XcJ zep33a>_t{j*9CR5jQ&ns|4+oF!Yoipy#_e!*(vRw4LyCRjC9)^vg+RWeVA>k^sB*%=97Ea zYSiN^JKF5IjlZ@TR#uD8*l^ih3-v7@hQN-6WN5V1(p7o2J&*cH!;0~ zqD*FXK=sEInK5ikhQgY_>momt@$!;JJJk=)9^;DX7_r=klOrwW%HD-$k6U`V-sqW8 z-xZ9%N4vMbspb_R=Q5FWzNYIPreg`nrzVoC$28Xb`99D8bmV~mizyon{_P^?1Lh39 z2qEF_0I&^o=0Dvf1EKk-6@^|W{z}n8LEbsob>`{E`{4Mun;>!+Mcr2ufpm0`~}& zhCEk<`E(=aS6;WPjoR-gqAo(N&wSh+_th(#=q_WY#>wr8wb~80@wEy60D5A1KS}4p z6-uaSH=OFD)Mv9*d45lB5l9DsETrkav_xL+En!~uH<I7^fg8322Z?tcd*$FF1O6(Q01;wIOAv+}lmtWg2+}h9i12*3#o{#&rD-g8WJLW(B z1ITf|ILld$OIg@)xD_7HQ*^vmlQj7>nef@7`Kbtuf0_9DNLrXu%S9K#4R;H8&NS|# zFx{}!(5;LKbZ%#Dd{SCkWz^umsHBcT0^JF0i#4MGD-Q~hDQU6gbl(1*bh_PR&xRB% zisZW}=kd@>bSKCQNKln`5GbNaWl}i-9mzH#xMKENbRRoZ6+aj&7Jk3%bN}iN7zZj1 z^3$^ZbZPbxZ|!IkWp6M%SxOUxdFaWNmgjrpn=~1D?xZ4&`p*TYdUupO01=3e+i&Sl zKniyq_*g0f^<1^WwB);K`{XYCKPYGQ^+c|0j(?xHPN(iL?r^y5&xuAmJ{W5?13%=iGGDDZ20gxgSB_!< zv25C+?!J!~0=mK-=AiHVzsZY02p2n~Jor+jtwPJUtkV9u?sg~rd*U@8dc>ODymqQg zICSyaM$;SS6YZ<1gV}Do#^ld+`T2$=BvCwr>S$j?=oseHelJ^W`>fngT#^{%9h>cY zIFov??}kM7Da=JVvpdbu+wtvC^SZfYO}<-w)prL#jy$2qEa&0w4aSFSTt(J5_WsOa zmRY6&N;XcR$_Bd#)*)x0yv$p@Z%N|dOpO=RZJNF(Gd;TcIX$Jbqvc5VqBjo6yr{>A z#Jno;B9*lXVrUjEYY;Qdy<|B8V!CI!>>I5FoiiSe8Nv<6nqJ3N)TNN~DkMMzhBOXP z7zt@j@@vO7xhJlClUqVR6PdE^OjK~!0GazfH<{5GkrG(;uK15gm4Q6di|fEZg{>q7 zo})-uD;Kgmsy;Ea#)?f=his=ofk@ziH6(sgI>fj1N=5b+XM4IS3sza3_r9B|fAw-b z!V60XOMb+?AdHYhAV8oCYy7Rm{y=$Hzy~IEH+9Yp9L!b%ZyJ?hX(?>CwSVEp=n8da z<$8JN^O?Hg=nFRg6>p&$zJk|3P6|57)plDLj=;)YbzrEqg^%V{yQmpI9T4NV)V6k= z2bjjznK5~=CDxrv#a-1LhHx4Q9+9b*TAywwpU?U{qO+kP&Z z=J|HDDi7D?lt@qyUCVP_DxAcLLSlG6UHPGg#20@McE&LhUtMeD)NQJ|{6~P6tUG7# z2k86XXyWi*Xm^^Bq)iyNbY8G`HI)Ya)p>frdHj!j1O-P{LC0REdsd;}$>LA;$o70` z9`E$nH<~s2%1n-S-er|e4faP_KKDPnbJq=evUk4w9_U#`860?i z$0?tiL}kINbz!ZmKHSsx7uXG&7ZbHUZ(_qgaOJf0y%%rDQCH;f4!h5xU#>+H$Yw-a zmgYPl<3DLMsrexn-dMD(p4}~G4Br5cV!B0D_Bfw@)MKa#^goL`n@JGIzx-C7h4FrW zRQGl>=&ED=nUH=tN!0d8fko)oU|EBBT~+lI8+eNdxrJO%dKEz4DA_O=O^>K#Dk<29&?KbChAB%P03*z<04NS1b!!ve*q8@#n%7$m8;FJrliqCSwGzH)_l2(zsZu3;lWd zU+?=1vb%O~D+(iuju8eqtBrCKSY{ukY3CXcHL--dOhtI$;M>G3$Pu@zlRs~r+D#qt zLwWOz>W|kJqy(VRwMgw?8K)kUtLky&ms2zvy>%ZfO~re48E+G^=k^yc2Oj`s=!^m# zlAITS)RWSqiy$HEtLdTIj8kEPvu%+USb~Ew2oEpVaN(Z_Csz%f;;v@z(!cEt2X(cv z`cJQ{a`E`6>E_*{R0}}yP0+}rCp5cnQ9TItI+tc2&#wKPKyQ7Ix{Yt|;iRTx7;0$I zmAu_}vd=qD4#?iggDz0P2zySwk&FPoR*LMl$uKHJ%r@1Gs%)HM^AG6q)XQF|F;pzG zHwBAg{%@t@=z{E@{((6bTZJC|G$G${f<*eB29gfS?i{xhP;D=dsVw;QaDuG-p$!B9u5NvX zZM~7R_>1IoN4cb{u0Q;CG$TFE{Gl`9TXe8+-V6WVwfY-<&$B$jf%noWA5uPCqyb2i zgMe4cCelSD&5@I?kH7&r6Qg#J??D?}HY7em+-(LKkkCXl^& zaUfd8rZ^U4B5yK^`aIJ5k4SrUto6G9{Pa0ZgHqO05aZ@)!PGv``1}}sk+P=tN3$^dx3y02!d%^p7z2iXGy1bmeIw1p+bh z7ev~O;MCJ3{R_49qZIcTT*~QoH#Oc;xnB=gwDh4gYYjWNF3O^ z!LW%(icyK+){giDDH1{Uy5ml{>y?GPb@Q(BOQCm=)-~sBE9{hOA-ro z1|8<=fRLo`Nuw%L;k7ByTHQySqw$W2&F&??>V7g`^lB^ z`NeDFQsSLh7^u-F{dnu?7R4N8^;1@6|CY$*yW7W9PvoPCRt(-OD9I0`J{e`~3EsJ% zGcYldl9x(|9DpM-lM-I2L0IVEn*E!IjxY-3O6MJx;=Z2QL)8K59SB9`^{AT^_DQ8$2V*%dMFdf2+8`_Jd5cs%dyychyaIy00LuCDIfWMk;@KId|C@Q>#1GBc{=2YC@@NXU z+xcI{bz5;SOZqo?w#2O4JAxx%zC8z3y3{T?;E*w)o1|rsSAQS@Gnp*J3xv67IfXB% zj-7AFkEoWUmpCR5p=z)-K>>F>rSODLn}1t)4#SkdqcsTIp^S6r1Jt!<8EM{5&Iht$ zEYM^826gqyM85}S8c@DvSYqW>{#V#}gKi>LIvpIXHq~W$J6*^oY42|Q5n_9a*H$Up zw{n5~{T+b4EvN0;ZC;Mp*OsMWg?;>7)N&f(So3NzWld+9k{I`!?_H^T!orKSLwT$= zZ;n zcKM_C=cXTh9j*Q#Dfd(SD9Kp7F0kJ)qTH}KpnX3M_vp$Q+DShi;cqN{UH|@ADR?n? zA%MJtL-6owrHx!8#F={dl!fv^Bb4ZJlWA=I8*84r{^;dPj2ReMs9|{g+N|0P6K7>(Nn}v9zR%f5iI(fBBZ%$w|NIBl8|@ig$0h1eOx&C^%Tgbu+p{ve(KB zR0V9Jyi<(Te|#%1zYnGKPX(R`b%oOj#q+!`(R!XSlS7$s(-S<_MHF<_Neh5>&&WF~ zdU<#)DSb|``0-Rs5$$QT^z3_;&btU>$<|C$#^TCgCr3d?>Kjb?>u56l?a7A5Luxbg7U;z?#YhO5IV z(Y&xZXvC6})62ErO!f?HGL>yACqPmh{j;%gYkOLc&cNRM<4uE}qK1BrV)gGZW4JTl z#4I3vWSBeSy0NK9+6Th5Nk`P`1l?(%PYj_Tdp8nP@%4mRVzp^^I>#0{d~TrK^DK*f zwr@c&jVY9()CEgZ9I*INpEU-HQ6b)25SI8_X__KglW*l#b#f|m4Wp7JbA_*~yfyW?{{kl-9!M>yT1&(#iM1KduF((FarXJFAbmj- zJwbMyPu}|TGNG4XIWB>c&+|S%2~fVtA)n(r5lA9Qo@)RJ)Du1llk8<}IX)s>bLm{7 z`TP4~0TR>7Z6(GJAgUofW{c#4S7QG_dWYn0LWYrvUV?>)aKH>>chaF+UV|euqgtw^QIxU8upwQ0t z7WN-Fj>c(f39|hT0kCri^BV>?q2zmH%Mg1jZ--HaO?4;4R3B_#g~)eaF;6=ahobu*{G z&Jp}bfB#wLqi9AB*hx_k%Tth|q3dIlMx&~Sc3aLYH2rnp88zf(98<~Gv$Y?w?rZN$ zOH>rOc%Gj~b*q1Am8sDc28aU&q!cxkHB{HUiH6G#kw6cRQuThbq-9mKW;c*cK8A0x zsm4?#ORUl$cLCV*w#|6XaJ7pe8_-%t`c+@kaks{^g!7MKyA`IHHCrF}?M0VdCuzuC z6y$^?5KkDjyH}~71$-^8{d^KFeZY!T-_RuaeTk4R$)Fnnl1DZ9$D5GQ(>JR;0UtZd z)b~V72&ayow-Rz+nadmqDLO`LRBQHufy%{xUZ7Y7d~CrPL7iMeTy1U+>4#D+0?ELl zLj*2$8eZHqqZ;ydI&s-x;W`zK2OEraIFIR9WB;uU+mCiTNLGU2&za2O>Xx&QEhQkQ z11g&YUBBiGd^0EYd02ka%6NGFCfeLH|ZXTYsZh~YpUvhOFZEL zM;irZ2iZ@tT_=^Rwe0n}KWyyYYjh}>hoc7G-%q|x`5=U&qg1K4P939#m$i*q6?iVR z)MMyGGts*&tpqH-#Zo2QluBe%eaH&Aq#08m8`-rL`L!UzZWka~YPu9W=2_i#8SgDZ zsZadH1{I+fJgn9xx((=j(J%>2r3Or_bp8W`_MGEyYK;^z(M|WI(E^x}F;fTJ4MQVI zPheM}zX>%p_tf>CR!lg!Yr9y|;UDV_LVnoe`MwA`gMbftUohNhT6X|DQj`6)gQvM` zLcWngwFaq>Jng8m4p-E|1Xnk(aBMy2$p0jAh@}hAsmSSR@@C>lIPtZ2t?x%`{hpze z>g6hKp*i@qKi6dJ!!{_2@-ANYDJu#)Z}KbG?};Uesq63=>?^DHB^X4;t;Lez3bNbpa1!y>+v2h2%zNbr(i>)#^9|bzo7`!s)WV;C4I_S2@V`6wDJ^mz zwg5-)-)W_}{}` zN8o&s;ChKXaq^4RGL2i+HV4JNADH99+juOktz`J z51^xbQP8Q)GJea5?V3M``yW84%Ns)^+wZ75^XE7GN5AmerhhaOe?_sy+(vJ4YrJt? z??XW95uEf8yB_mbLBH4M?>ctuZ|J*uy|Jm|{+7`Wgbi@ye2A_W5`vd{=tJj?~BTgG! zL&la~>87uo@%3(SK37?tS0gOjQi>FiH(?nz0 z<}`z6skI|aQ)DPrEWG($Q9a_t#9oR3CVC4r)k96KPef{}=VoM(h8A{N)woCz zS9w=fEKy&Dpm#LQ0?ZvDj;Iy5!-=%Q5y1cZ-C+%myXG0X?=`PAf@Jz{(v$leY$v}oKmTyP#^ihZ@`4FFnjov^fv(?jp#NPgL>o0=a^*- z5vE-+qEFRGF_Gvn7~ahnX~`2Lb8`4fhL*Ahu_9JGV$68Ay`Lxh;A79dYi!gi%I6_h zVLF%$UB{+{%T1!Ma@YFcw1;K0tmHklY0!pT`n2KW1^M~!$+Cr}bo>#h9%$jqzfU9l zXOw<8E!JLx%7J-lGKcQyh*8^jV6KRqjo}wPjcuiueJHBN&F(J5 zw<4n(Ew>>zZ)R;ITSXmz?G>F+e>|*=w+Bc&o9tfpWV#!1$NFlQg(#~P8(lDAN+mRJ zqKd}>FysMo4S%ZW55sti;JHF72DpKW>X>^&Oqo)2v9ra-*jGce0 z|A|nVR#DF4s{KGj_n?~UM$%EgT}t+$^|*5S1WrVH4Y!3j-KLl>)7_&%JXeDM(o0AcxqT!DFVaH z)An9Q85sirPVhduOS)IihRaGtVONe$O@{qyL}m!zC3<=lyCOdYVx*#4cnm|#IRtIj zsmzK>9Fdl2YUtwBMImX|ZFD20Rbw`^gyHvQsL3VXzT^yA>_8H~?|4{V9OTmjGym{P0&U>)O0pT9Rhl*Vc>I6pdp8EciLca64mmP;A|IZvHlsxBgDlOOFnQP+^lA z7hpGww1vxqGk@(hm}pU=0b(ireel90axCDsWuH1DXG-Mm)Le~D;Tl5b6Ln;)&fw+w zMX`(i^RH$aCMJ|kZdbfAhJISkVOS;oQIhTtq~3XS`00mF1H}B=-*XO&kCoriKrlX@c=+BkC;cN%~_dYCYe|hyb@~m#4!207en1+A;#F z`NMp$*#Qj^sEV5rm2nDR@PsSkQh854nQ6}O@vOq{ATVjAM^QV7T+lO9qUqPmg;+rxE4FSP zfXGjy#>5Z)9EB3~ZBiTj%*s@rrS*J%PBD8eB)Z|okLnmrv1csUG&`o0M{#Z)iTy+} z+KFhGcu{9L%@LR?LoqJj+Y&sqoI4-<=h|diaW{>i&J1nxf-h*Q3tTYvvFc7Mr4(E0 zMGd8o{`jXGrg*4UQ@x~45-w+--nS0Ju`YK%U8;aNd!2?lzeJ{(X}6Ayhln0Z7c!&X zWzR3dCMG)74Um07=;){q*j~PW0ItHXjUh#z^?&>dYy>}fq_W6bJmRdcseWMq0cd^m zQ0uMtu8x-a%#h}6VcFfge@b@s?r$pzyM4FJpXiV2l%r|+#TvE}`}4uUUFpz3DJ%8) z*Qb+C>BYXa&F90Asx}MfvF}Gsur0)9%nL(wMO15sVZUYtkVjKZT7GKA&Al60r!2Al z(E8Pziji+nR1ESo2{TCnSPpo&zOedZCf_J?7_1Td`$h3&y~p1l8z}P46hik@&DD`! z`~5|6cvs+Yzig7_2tQnnSY5zsh)ez??=c`;p5y9@73FR>b+a$ZFo~0ezX|GR zSRRIH#l!}avD65OE zPn?}-Z^TWWs`g)d-dgRd;=v*{{y9r5#b!~!b$WG^x}@MQ0v^%Dw%g48D)5J8B^KDa z3^_Oa>!-9f_42ut#Ix##>((eI?({a|92cT9O?S?=pd|9J(=1~%;aAhM{ibyf*`ee` zcNeQcNHDaNm7dc)OY1 zr>O&R*A0($HCaAFIt^HhhSIu;k zG~CEC;LKEPth|{wNo!n^*oUTufd}3EW0HkC-d6?M-lVjQjLbH`v zJK!6`#2@LI5AG9bQAPABYxQvL4{8A2QIj{WM)x$2hP;{5(W)l8(j(uliC93vHyu(9 z7Ecr%d}e5V3caF%F5stoV<%?|eI5)p+V%X$pC|$^Os)`cthY-~9=JyK8u%GY$QKCW zr9M}hgk-Va2&75@la6IXToE)%j#Vkf8XNxUivFFGjhPpt`L!1k_I9-_HErzV+t?som!2T2?psJkPrT*~up2JgtsAfXV#97h z2R-{GJw4~>>eP+^mQ&KAQ^N@~ovy&So_YC^I_8p03rWUWrh-y>3~p`WNa$4;_nVUu zZP5H{InUKiaaWSkXT6eZ=ej}OmfR)=76D?{p3k)Hse*5RFA5C2mnB?3KD`0mYLw<) z=tKLkv|q1yMp#V@n(#TAmEQs8<*C^Av~P}rzb>}OAl!`~TpUUzxNY`7RylitQlG@_ zaoj0Y>p}kKO9;o90*2wvlK686tjc9^-=ln5cJNDMh#CQZOw)S6+nD8A`_oiv%l|xN zb@lQC!nLOHynN{DzUaXZ*1M+lR#o`-Wtun8Dbc9}&7mj%0F-yEcrhYOkfT%)hXVh& zl+dq=ffj1_8OpEEqcj8cUZxok@FEPfiswH@4(!T!=Sy>-cU?%HW_6d$wyL^n_}nA&t+Q583R#HBos z3QB%cJGgyA3S2!9q0ke#B=;)stJL3iMMZ|ER%=Pmt8mvYJM?}+L?FWynCQJU)_@l+ zw9|1(ox`badqX}ZR+@}S4=O~RZ+(zZM-q>dzcbAn)MULIy?S8#{_KnY@>f)qSfM#{ zx8YMrs{Rm4N9w&A^l?;ybG6QL7|F1|!i*h81&oN&#a+_0!%9UoQW>a{=H<3@x$<_D zW0a&qr%k%JSDL)9Tf2o>82{X?^{N$?EqI*pz=Cy939``Mn*k!}fP(kHr?Kxn#Uws} zhnKD`r7TL9u8p@~@xPIbN`VF=SD<`FLu75-Ndt%6oR zT$5;rmA;|YXbMGJ9gPAdrzxa{t#y9Lu~|Dh5F;KhtjrDwMp+)N3b#aS22V|~$xkQt zZI`+j0ez4QtruER#sxoLY2WWv85|O;T#8wKWSjLs7~fl{CM59`7gg63?o3=cKia=I z<`?ylH(FrbvK*vNpX!R4`U`|dp|i|I?wn2$3*PLQ=C5FqJddk?bXvVndd3`DKC>t+ z>yUt?0=Sf_2?5>6mvh%*S-Y-I>I%H=NiBhDbS_juVM!K0F|-FG45939 zeQbZ1oJrggn>6?RUFMmJ{AH3Cq`bDJW`Hix`LFd-V!yz-k3_rij)f*jFyal-0n|SI z1WAr==vpVEH2Sy$9du)Z_?YwNE!&B5EnD_uPxSja4(|9{{&Ic!z5>ry zs9yEPtkKT7%!<>>$L=ew%q<|rX=CfyJoI{Js#C|3P0!n-v&5ij1E;Q8o``R&gPa}n zPjXH!Un=vVU}mp3yjYw2PSqPKgasVKi%{SE&vp94hc2VO_a3rscHP_+(H|Q8w!mQ( z_&KtSZKUm@2V;Y8b4U6#^;Gw?(FqbTVy}*p@Vjv@=?wb;8%ljpgQ-KN-j*3nqCaDn zLpX~{kxHz8drI9#mdR%~iM8<$?0?S(?6<3nzv9+Gk0MV>(hvYqi_U}`J1wf!7`zZJ zB}Q4!^)Ij<7QlpbV~fL&AmsbpZs5&sz=@P$K|2(rZm{yaT+n`^e_83Kq^e7*rgiOa zKkbASZJ&1vBC5NL@6DF5wzTmLT57B+*f;@>hO5mu9id+Dh;!^zfA`JIB&oL}cR?)% z6X#Kq$+9+Op-cH&x10vL!YzVx%oqn;{5(bqjzOkmOQ$?W@OVVgzekm)#cLFC+{z;3 zT$CW0Q(eC9M9!ztAY1|QXq(b>J&j5oQb7aRqMPdA2@>w;nYP9D=i5;=&?)QN4d&E8JlWJ{6SJUrgbOj$=P2pUg@y)ax>xmEs0MDDB~)TXG8 zpJo9#!c}iGi<-DrRU`+EAOOcC?N|yVt2XZJNW#i==JN4oW`7HjNMDSkqYWjUXh-#~ zPjy9JMHGb*^->{$Pjrt9M*q^_bN%{Q-po;ymnX7nZpR{~KuN@yFUQxT{~VV#YB=gq z-Y=AS|lQI zPzO{M>vH3B6*!fyiRS2`Ib2e3lVV*Omu??|7Tvqu>v1*8OS%PRebYnHha=9VRPYY@ z!_b)y8fn1mb&hbj;b;??(4H^lb$@&^GeOU?8~XWX6OrPEVfjm6wA3P5L#YPQ_@|gAtG;J144B${t94z?OMj*w|Jo#+(;D%RGarwAZ7B)0g z4Nhx-%)o@wp$wOGilVv#@@G>dC`XGEWTlG2Qd-oecsm9S{Jvj(#MteNir{e%|dnf+2& zWRIEVoM@o;uPJwg!P_3m_ zQ~t^-{?Uy34ndj?eAHh6?pp=u{v@(o-$Prn_8=+H@7mN+H`K4{l?R-fsu%VzKcGm4 zec8G|LVDMwquXjhD;vuvfhj@5i#j3SUE=OoQ)cKtz?&#G;a}40nEma`uZPm;3qo&7 zNCA-?rOA30?LW&s_kZCV%zroNQ#w?F)!Cr2l07{91CBZ@B<&ls>xS58m4*W5G0nGq z|Mk#2a%fY@*cbe4OYyzh_ATIG034-M%(7*YLN_V({f~<+S&QkeCy8i<^$~e=$CKiP zt0`OEuLS@zXqlfrrVA+uE%@0Hg5^KYjr^|MWw*bugDtOYc*E+dHFrAc)?%>^hccyp zCqBc*OkXam+W&kNA4qAP)gtXTVU$Xx{M9%zOj8|n4^RZFk97J62r=eoC>1tHXLZ}| zNV}VRv`;1dX+10FWs@8}?h0Ul<#S+~>q)gQEFteW&j{vj!hDYJo}o2+JusUx_Gmm_ zjEfxVdw0COzpUHu?JH=SrZttkKRew`^^l%E9tYHA(SGQ(PAxqj8P8NJ9-P41S2v$U z?5or?lA+&5{7O}h4?`V`3En1Nb&)BL9P=!tD=mJ(Bbd)x34MnhX{#a6xE3zMW<0yr1e|DcL=SJCml%tl``N=;3 z8`l)CzHeo0CzgePEq6$xcH>InRhf}rX%Ll9vH1b|Bl8`U+3Y`cxIL-m$fPQBe}#%4 zx66eV^Bl2$Rp}HWF?tn3J{*_jEhzwE{S%u9v$LIdXM*Red`!A;KlGMl19Sa0k1Ay6T3(I7aHu;+)EG=O#J*8Upe z){yj~57$JIpw7$p&N9UE5*TKti4k8o=?Vl3kucB`a+ze} zrJeQb#^R;^5(|lFnLZgv$v}Ga!>ShAOn7wWP3=S3V_{MbG16Yae^h^zk&fCx?pQfO zTlMxqzg-mRPK^mc-NS^AUo>~)DWaZ@RTr8s#w#*1UJ=@UTm{NQj3MzR$S%T9P$il=J%czM}m6N_2 zyA-77zsJ>Kk^0r^%SzIvQVl)uPX?Q*U|Cti%0WQ#o0v+HrtZxpXW*~fyn|PoXk1w?Yw>#=}e8L^2(K2eO z&k|q4tq8xB#^&zIAHJTBO3_+Q%6|}?>bB}JBmUPw92_*5bXfb)PGH9c@RIoet>oZP z`^&1f2v1GC;zQe zGjIR1-p`CAUPew|1PZrAx+aut1$L%Fn3qe4)<8}HUW8&hYeI`WK z5imIQu`_7{$V}o;%(^PNjjld5NZ*%*CpJrp{rGLWZ<6;~`Z~RPXGPAE6vsIy5OjXk z4PxW-{VQu$_P|r_O%>ZRHT9>+%fZhtvTrp1tem^XNZMuPB!TmHTbx?tix1mRFMLaF zm6z`XS`=3viJG;%Z!~>|ONBJP_WY7V&h9T6ZZ&${cx%K%he{+}&%2}i zf+Ct$1U00|)a0Rly&{3-9#5bHk#@_iN5)~*QSy-U`i1Yyy+PqL_F6ADlhNs>Kl^2& zBxc*3Tlx##g5)7oU7eygG=*#Dv^FcywHM{=2Ac#N??*$t-&_~}PGh`AB@ZxI_NVGO zKN=I6{BZGZ+Uj0K;eD6!>T&xp+5t=adkT%twFj>Elt!E;1j)yJkMy3Nc56FZz49d) zPL$q>y3Lwq#k_<>4gK6p?2X}FjC^?e=JzjlgMkM3?uP7@??^}$ogHe`<#im^WTFfp z){@`TMPC`8M>+z#TK%P7stK@}wx}l>>_bTEy}A<%Mm(}wr5<7?{;>v3!Km|`;R zgcN@MXy}C!}NccycTc7=zk@mT~bH!F_7Ek zf`K;FAc|IzugH!WL@wXnsJCe&Z+_O-_2zMc`;%#fhz{}Y;MblJa(nvW;zu04uzB8x zg1|Q-(B$&l%0<+eKA%cr<4|5z}D~wi^pFu z7#ev_nm-+9)!iHhlc_cxPda!q=5sh)CV?!5p*?ek?Z*ttnweJVqdPOU6rM zDM#~DEii13t;IRHyeF?M-6WRskCQy`GCZlJ_b0IND-j_dCH*%^FIR+5sRYk_O>yvx zci0Ix4RsZVa2D-ntKcPvz#dL|8=JX^7p{>^hhX?P<iPyIVB zzg3EAdERC5kV(YY>obd;lPIv1VqNZ3NzS=Ne)9`!`JpHT9_6qG-a-c`5OJ5?qvGpT)PyO0GxIZQaSdTEmRTGo4T z0AyL!!GZH0v*AR@csm54B886PMDrK+EPOP8(b_*xS-OfZdoCDTHVrH0$Z)RF4G3!U zN`%&->%OF0<7=0!c;j!r@3Vj(@2h}TW~@%>4(R}&KIRKkz+e#R1yxw1sfn(L?glkA zJis;R^hJ~kuw%?Lu@orhw#>gWuRL|rJ-o8nEwCU<^L~oa+Ek8F6h~^(_te(qSfqeZ{X(lBFA`D>)%0z_oW?c#N?sX@yr8C;a_a=2$_fbu9fD({`<7N z-FaxzKS1+Yq@#z4mZZpCRvG9P%9pFB5P|M_?RCZ$AG~tUd}bC>p*JOgk~=YeO%G1} z?;n5(L75G8xbN7_#X?@dW)O$E^XD2tZ#>9gRQ^YR-otc!th$S+)x#uVv`~N{d3ma8 zIevke6Y6cNZ0j`*^cRx*jq-7!5-laMzIM$!Y-tl3L{e0}pG%5Cz(A8uhXpxRT)f>f zFqJR@b}{TIx`WO-rgTY5+nX}ViaYqFQ_@*ktVkL6%kAOb`Sv$K0j%+skv`Zajyao2 zufi;vFDZ>NMqm-RA`R}(;e5X56%PNhnK5bXM)zkS# z&pk+*dT6gah5^t!isvptpung~5*I+ly?`B#d_`8Q_3lHs9;=*p=UwnC6l5VPEtY|3 zU<^=G`*h_oi(EfMo3qxdr+)BD&0~0TNyRIz%TjU(GU1ImI<5ZV{<9$uGtUI}8rk(6 zLPW&d#h&VG5CW`|8Zfj3p>#s9;n9z+)97xZqJu=7H-ZuTpW$Jd`P8SGSYKg#cTby8 zq1qp|LA9gwO^&#_H7J2l1Y2~>byet(JwK}=+JJRBfMC&Ak2fUWrjvK zE**-#+PxU|Qd5hAUcLxk9j{ZN!AgIjlAz2s42cpRN64+)75U!tOeh`o>I%$6@Y|ou zy$`B+x4CQd0Mk-Q_IJT`aoBDOS$2>}c&~%pcELQ-*;5FU`O2)6GD&>moblWtqIVY4 zN%h9+=Gd}_@;P%;;zdV;h;@)zc4)*@Do8}K3H5mVm9UI{v={pl4OF*wVigKbdpgyg zn?CzwijkJbN^*i`q*9*+Ks*68`Zmo|qq>zmgY`WAn)$r=OT0gNb3jM^Tm;-j8x%2|F(%KrAX&Fcl?c9{XFv zcZ6bOWJa`B*0kjM_PDPj|Jt?~uEeFysrb5_!zRCws?7 z`e@3jyRW)LW2vO1f2wV}V06vmX3c2`Tivxw0KGFy`6Cn^isx0hTUSP;Llo5so(s2q zS+;nvX;;ZX#argH=hOjO@kWydJ!^y^^-N%vr;NyC!toV<(>?Y8`4gg;yH%6{7aIj= zT1x~88rtNeI$_~wCPqym_(Y5wE1v;mWDUcwV^+7Z?fg05I70J!72g0&ZoleIw9e{P z=-A}DL;qiE0;>GiNGM|Mqc44l$Cf~waere|>e=rjk%yW4`(}b5~wd#vwhfO-X-KTUSbA^i-CHknG+&2J<)rkF}~o0dFA z@8ZYd$my`7GAj=-Z~R>D;!4wIV=&l=eB3f@(KiN9s{Iazc@Riw2$mRe$?7D? zY39c_>-OjEIpDukFpFUZ`GeNcKK}=B=bvI1XPJChqlGQ+8yYR|>zgL&CY_`uz()XB zW!q@p7jDux#;IjN`L%}e5W`%qnmBSr&%}i&$Zhjes=Lt2};IvM4gDg0*L&w$+(iT+VTp(wvA;2?U57Lz5f7r6a+Yi>hwPe zDs?@^kca3O6|0%H_p^C5mzCqcp!#4{A?P_Ry^I-4N(vdbP+k#=y{8i=(~^9(u1+)~ zC@);$d1mi#IJLtEweAyAWYs1g??qV9ZPJy~qM65V@8EEwCza3?7{sd?eCo_6S7=w% zlEo|e03L@-XQj>M58+#Uc2F1WtCqw4+<|M6Za*!v68PaH&gpn7qDZLsaQn1C6ch7} zzL`B?O*l#b#MU+&#fSW?DJFZXoUf=drK3BkjtO|w*S*8w|Kx4QXZQQDE^lmTIaDSD zK^Uyc3R$-+)pJyhI?#+n9;I`gCiQs-+asj_{!uYU9wOptFT7d|y1-Z^qVt}yQhHFAY@l|z;E&p4X*xdaT@N`Y@N zsSO~7Hx_QD=(;$@Nz=?g+0k|4ZOfX7KE{uqJ$3#8I<(j79C}wzHWv_!pxM?Zmy6BM zwHkU{{UKLxJ@6Ae=gvu46(x8FZ8F7A9m~65?oxOF z>QDnlf1|gttkm?5mUmTzorrmC=bpn>ap_QZCKY$62Frur|?&;O=RNwQK&O&jq zXut@cjEw?Yr?&TESWks5DE@uHOrL{u_Zk;+!GFb38$`cg+hw`WMK~yl&|IPS$Qez{7L@##bm}l%*=HY5HKpek2jtHdb165bPe((vGN&_~#)>z9 z0O?p=5#`6T7$T@61Jl*f2m4=LmWHulgOhs@0u#>KZN2ku18(!dtp|VNC<#3X zs*b+%Ez=EZin$-Y`^2`zvJ{ZXERQ>zOcibXSb#fHsckNOtCk#9m` z|4s)sSL)tXnLm`vsL$50T*xLcYI|koIPxtPkk5o~P~}BxGA$iA%7xfV)m~2I%3V;+ z(qdfP9XhRnH`AE`*aStWgpWn=>fm_rTk@#EmHB_apHJz1`n6U4uFIG!n)pt+rmhW)sv>6J*ky5Q zQ?`AhrJic=MQ3B;kHb$L4~(N46pe=lVIgja1uhr9L!l(srz)6$@ar{|qP+~s!)mLx zRa)CTA8_!E4>>At8#vZ5N~S?xx##O`Iu}T^+D}@(bmlE|Voxx42%qa+RvO@x+4I1$ zS^z$O5AY9Api8p~dudOTxW1d8>9Mj~GbmFsXTvagck8*2X~%t*Tehx6(^e+l6xgtY zCP^Jl=LF=?K=T){pAO6%Di*B;B?D1Bk9S&Z$fog<*p~XZQk}$DKcUqF5)<%G?4~uz z@p-mp&@JT0@K?2*ECgp5&V z#1ER0v~TYRdKd-Cvt9JMpVMy3X#B}Wr70Jy6)amDE)9A2$lpA)Ks?(p8fdz6Am+2y zu=#Fo``Q7iTH+_COq*`SQ%k7e^+GBNR^?K`NR~t)KOG!&C1PPxb?AwlalE?>h2>`c z?*^Sjw(YWOuXxqY9OikxQZXpJk)PC=&v~l%P4@g(l&!)a#{?4FZ>A&5ty@PFC{&6& zS+v)>tQd2l<<);F1bqP${y0a9(AlVD*Vl46l)dxLfq_!Htw+0k62Jrb72b$Hm{NJ} z+s_*1@zslrs>C>{`)`#zX>=`p$pzVxpaSn_zWYSa&>y#QZTBq){dH;DQ@DsA1YfWS zM;#jEQ`zh0xanGP@!N-sO+ER&sE&-^&-G6)3{&)7o@mQCKOGk|391l%?4visn=H^k zAu~ia@(18z_D&GZ_oI_pVIgS-m)Z~j^K3+RY{f$j#&}52lusUJ_OEh1i{0XQP|L09 z-(Ls23SZqz8K?YD#m0>wlT9 zL{ES1A7B+NMebs2bA6}_3cUGlocqjcy63y|Qp;2qw}*JU3^%1v(?WpeReBBfIj~U`Z=M#*FQBm~V&$`Fdx0d9h$!{H@$@>>;9g93UgO!| z%sQJK2F5}+iDGUEKEG^h-rAU?3D|tZo$+igK;whlhUR2xtC&$vbHkjAJGt6`;^k|G zZV4PZRb^?RA?>bDu_xE@2#_nO@sM=gT3jxZ#T3cveCY0kRbK3oPIP4OM;s=NUgeIQ zS^vo$O4_50Qe0gi?+D=K=pnAg=i?>S!Qu^5THbYZO#bwjvCTv7hHI*9*84XvKE0h9 zFIFqI&3u7*uBC>o&J`nz@eo6<5;tS8f}5u6hyz;jg|;IB40Qyq0R6N0kFD+h0eUy0 z{yz9eruK2IxBUaO^OyYt*buv74d+$ME>1!e3~%xz=U5mTc7qT9283U3d8Lg|{1_ho z!Eu@I8zrK+70laULP;fIf=Qe0O<25g91M;Q^UY;-DjWPv>_^W{2K}Lvyw@`kt$dst z(Y*!Sv8)(4dbe9E-SQ=~$~Btw^J(~HLcl`zJs3z54v9tEpY=rTyzL$EvucX90fs8z zj=HCMLy&&#rfYs8OmDz=X^g6I=jhZB&$$_Ofw8agmE4zSQ+=c#wW#zX>BKVTREH~0 zeykW8f1!@X{pWZZ-%suwce=9qA4TWkPxbf5@r!G3mymV2xK?$qTe4S|d+BCml$pJ^ zTx646!nIwpDqCdljJg>SWo3oTubsX1yT5e z930Xd(m{~jWn0aGIG;U1P$yBgO*O%EVsGHmOXyY)?wE*`nO3+aYjay_RM??|g8G?> zzm5{*zX{oqYfYB1sPe0kYcVR*U+|F3v^QAB zO!b3twyrq>dme$OW`ov%MtII!e)kP5&U$2Eg?|sxOO>kcT8$`i-3gxNEY!fKuyMVj z`IgokVMyASxqBMr#;FN2`d~gU`8@CO9f89mM&A3fu9ud^$VZkZODV?~Yx?+j!GzZ> zTD@H;(~fiT7TC%@quAG!_7^$fu`Mz~{R z^Wm|0ar}-Mn+A$X6Pt_2SE0ft0&wCZXjvBKQ<^T;>h8UEentz>)KJgx@~Ve}LoN1+ z>pefR_cENtEUD4b+GubWgY7Ss@Le|0%G$p65AoC;mDLcU?Yy#VcC`NH%h+_`wcbV{ z-&JAHR?GV>6h6n9Ugmz=Zojg3D(;ZI*Qo!sWpMM11`^b@NzXUf%aVCkOS1j z{rqEp7VMP@f^7viMD@im88n9N@99FdAAU67-+8&erL0!`TBX$>yTa(^E&XVRGovWG zZ?RD8_;luv#CFWLpg`p1+#5eYP~b)29_iEZQt*Q##+MxhP9yfcAnh9zk+MOA;g-UM zk_zK$4L?EY!g4Eggp{@ajWlFZKk=-0)oxYOuEexZS! zBEDOo@2m&qJ?Yhc*JZH(9R1h9dmFHw!>@c+L$Up<&V0mq2)$M}u}ekAe#3OqPtP$- zMx%n2fi;qwnOq3Z&G!(q##bBGhdIOTffQXZSu?xm0i^AzLiP=fy6<_it>2!S7n>%0 zy9oV4|LFB{Xch61ifHf4m;mKvIDFQ&clj_MjgwaJbC+eM3MLZ@Ipte-wXq|cQ{xv! z%bhO?nAm*3@8G64HTk=0p>HV-+_9fbUVIi&mie;6$Q*wE=wq!|ilLW$~ATpPmp5CeW9?JP!BIvp(w zDNw&L>7a!d*0pgYx8i5u6JK&(l)rng$?x;7sIL(xkCF9Cy7O0|Z-*0KvP!G~nuV!K z8Lp!{-Yjdpm%72M`rX$1!7Zc?ai{+5_T|!Rs8sQ|1s|3X!8OmOH51T<>qd-Ld;)*; zITD+4_tZDdN_(SjrA4tsDTd%k>q6?jp$i@?snPP!=eHirc1=DaWr$H$)T5n=@*l<2 zakQ78Ay-KuoxRFG$4>qZ&da+9a39=2cB>BJ$beed*e#7^IO_%SSp1S5_3gLI?eVy` z5{HQLhS6`oDC`OGjk{rAf z@=3pa<>PhY!uj_L`{wM{8iTdAyYI_qNRQ3XQx=LM3&|CO+dG@m8c|)O=?A2A+5UQu za^}yNdrIENM=&(&ilF}-4Z(pK5xqqab$G78p z)i<}BZ;3^hlFE-lA4oPW%^bSuUBl++4UOyeiAyI8*NbN|hj~rH6<|g9cK&NIf8>z#PdrUyo0Cq%aAvPoB{{T97 zKZGCUMrKhXxIxKKp5e_Vj~*{ zPZoeHiqxRE^Q3W28AYNO=^r|?$wzd{Mu4rfwjMmcu-bc^`1G6Ah+%hoqC+r~Ah=U* z=Oe8+1kasvWfvx`e-?pD{)z>_Sbhp=t{t(G!png~6gB%1T?MH@OS~FV!@J?Ykqk@< zB=>ttoVq?-T1|3C?`_@^l;VORAl^T3Mc5ZH7gpb$sky$eRQ@8ua=VdM)PMNWjC}gj zyV%Xy2LLpreUPS)e~wlHbwm4dkN`dl$hk&7S=!-dO&pQdfa$~M9H@s#(3g`~e2gF? z#>lo<*IjiXVF~7kRJ~%5jyrrVN9Uaig&d{#vcHR?#U70KNvAaqW zl2q8_J4EIjUHl@n-akL4VnhE!KkjKfV=ULu?ZvRdK5IYl_JOBoa5Sdy+} zQ1PVijYso%wX~#}T~Q1_E59#fNgyi=KS?Zj-+Z!7CVgwTSiNi=Ok~JKwWkV(rY4S{ z=y#m&b<^Rd->FfN?c61FIf0HL7?zQS^X>>bLHA^<Z3ursdGqz* zbn!fE-L>Bn0cIkvI|0-kqXXl59w0DER)oyx`c}*g)mxB_vYhV zO}TwcG<+7w1z{?3rQi=bC;gUOqv~eO+K% z@7+TosyBJ7KF6o4*S?1npAWD-3rFUi<$dY6ic6B|gFK3xCX_C~ zB`K(A#(JG$ssfnpqd(!p|2gWMelpb=x4(L{>E|*z*4e98sl%~}?HWF%46UdQTxZST zj<^8Wy%NURJm#6kr&%azyr@NU4eM{$t3?1APq7(e#;FjPxOa5p);UzSU}W6gtQ3=Ei3h~Yhb=K8D-zNRSMpH` zMv`_ud`jcXi$CjUj;PqPQO^0#i{{d^M;@~X{Y+ftS2Z|@On=$98v%tO@oC!$$0y(> zmI{et0HGfm%$t`rXT_ROm%^E+HFRG2z+Wf<=(u1|SycUI-oMWS`6t&Eq*Z5scj9N| z1}g|tR&)b2`M$c>$1Y8r&Ac?ljt|^>Kyi*6jHM*UmXDo-#i;cGY>r?ol=ysioKGWi z7SHH8Wek;@&G0ZRsTn*y&ULA*(;lDTJsvJybZZn>5=AL322Wc$2pPclB0r2kfl|@Q zHK3FD!TcHoUCnxq6E&iudcJT6>H}3i>!rx!uJOFRJ>auaJdkoh|4PVi+!?gCL6F^} zOwz*Dbq4N8D?dvZNe{4=%alchz=oP`zR78G+X(smQxpeT_e1+K%y zhXZqKGE5?#Y_Mj*u*u7Akmljl1Nymz_EzR7pGyh54Kv{x$%nIk&!=D zyK7AgyTt3@=kfTpdM!Q;-47D16l(hOP$va{C}oEeK~dxJyXq0LBriv&y$AtxY-*xJ zOT%}zYn5nrAK&MawRi7^MCW_HXe&AIk2=`D$=06_pgjmiUtnzebkO~Je;#_`KSnj4 zMJgQ2zY9(xvQQ}`o+UR1r7GLlefUJ9XnxBBf(E(Njn7q1e>gB?&B1+~C>1|^8A=vG zNp7ulsW2QcdDDTyWf$Uh;g6f_J*_{Ctfi6!fXHemRRSk-{{bu&F5@XmEf{7~mu08w z#atdQdJIj~8u>rpQUX#1;v7@?<8CFt#^YJgp>__`HDucF%zl*`Y1ZmIU(~_a(GG55 zXUV#6ia1hQX(DO&$*5k~yg-Eqe9ynYul)na`uqc&N-$qV{LTM5zMo(45Aa`(bKyS# zPbjI8o0$OozVr(o$Sn4~_JZ|Zy{Ew}pOE@$nW1@Ypb;nRiHYw#1Ir4_0AzElf^_r4ocp0- zs+xKt{j5n)M8Yzv8u|~w$>7iPM{?c6V#vAYtfRnuZK9&YwHSFQ2Qdx5HSn8L5b9BB z;+x=NI?dnDy>ECr7HknCR(fw)LcV|r*V3Cv9`oc zr6gMw34_cpnNLo3kRq-kkYcKEj*oo=qqSDR^M9EkQrzx-MkT5hO#w`}4EI6Iw?x30 zIGOVAOZ6nqB@g?3K(meNjlgIl&8$QFjxj8Ob@eemfAhOldwcGdJC{>|d$i6%n@>mv zW|7L9GCNd6Tb0uDrCnar7vF7FG z;EM~>-xvDxV~oR0WG^8epz`{R7>NHaV1gc@kpv}_khBoyIRP_?yHrMRut!I2k6f@Z zw|kzZjS$r|qZ~>&UVJ|;m4LEdNLMj(Ir=yeQV>Me%3&G$kUGtuX_H() zFq+8D9-oFp2A}#9b$`t?V&WHiADq0ryJAmBCqK6GT0hLr}# znr#NNhfrl5eVc-x3OOsMU=~H~m^QP($LArzUyWZSB_BL+p>|Uf_(C|@7<5kTSmRns zKj!IOXLNOGvdp?KH{PT9-EZV))k<^;Kt{ba{2!pqriyfs?lm`~m*itZTzruUJc;KyN_ot=FctRU8sd zBNGivk3TtC%XHuVFzaK%-~BY{`nxl`wxiyduWT1j6u1wbEL3p*N3y4a?wS%j>0FZdHmt z^-SvuXErS;eDe^*e>Rvm$3Xkn>XqUnSJXtQ=<_nl&fAJQ{_q)c8?D9UQKbrb#dnd< z`J3Olzq-h#s=Xf-5`AwdC@YoWor+2-GYGT(nU*!{rytw)xad z(5Z=p*4hC0R2_mC;%(6mb2>>5a9&RsX>w~b2TtZsl}OxBz4_U$WT1NZ0&@-RbD-|b z9P+jxt!{-=w6<(?fL%Q!S_cuGe5U<(M~g>AI9{kmGUa`Q?gCRMQ4jwCx<^ejz~(<6 zZyG%FwKe~#uf5h|V>bhV0QFZ`P&3WP{6DYX-P*o4Am2sJhB#I-(|x6)#Wu2YCH7x& z1G@%{2U2*_H`9b^zB0pq@~B&Ve0+zW_eetfQ;O+PPEE3a7@>({+_2!CjiSIeYYe z=o`Kz;c(Fx)he$k5;QE(Q`E{~>R!HyznfA=#$5$01C9ZT*`*YETm$^oAkombSL_d` z_==GuxqVk-1*5n(iqvM|at-(5_&qCh2;9nncUp^L;$#2TD#nIO2ky4_C7d-70U8us7)j zd`tWX(3-@Tg5-pti9Zjre@(!51n#JSBVdkcgMr9X;x*|iGIPv-&4+UcBZtq?4zA}8 zPX>i3>UwaEGQx(*|gJu0X*CeVyg)vEwmRCj9 zpTG6}v!D-w!@-EDu!yT-$KFn-)C{ye#BK0OVV5)kCM(iYy%Dogg-Q#dHf)4~JYYgk zV^F?hon$^CwQSO`&sT8|hz>SFqSW>OFCSkF{hWiWDp@D&9p+C2>!?!CQ#}tD>t+yF zz6h8_jRh)f_rB;zl5=EJ)}gF^h3=F$Ho)u-Pp?*!@;Q?b4(5h+;BVYy~jS!z4EIzqWu$8wKNd&DIzOhVA|>xi3d2) zbaJVi874SB?O=#p4E-C#y<2}p(m2OT8aCW}tQLv5iOqe$*E9kRR<)#Pl0%5kR!fM~ zN-^}l&YCqg*o+m#;n=nXS12f15%Zk^`>8nW{|CX*^XnWL?U&-LWYtiB(lt+R&K`#vLTb^Ha|DZm_+w7`U72T)151=JNw3fvE*fMB!YzC2@jpWEfUn zf@q)E$R@)npUBtxCYY$`%AHV?-@E&{tPJR^fS1bx)!^TB$#RVe^K<4|92Mtm(pG(v zF=f2s8+hRxN1p1;oh#X@gPNND-kqLu)@WPnpVW(hxJME0{AcM zBTeptO<-$uk^*jaaAE24DV1wI9s>s<3Ks<4z+}#^?vb0tUJh4UD!S2(&8)P@4769p zgYP_P)>wfxkdJRYi*ZWLO4?S6Aw3CmhtSDIqW4K5iiHs1phu`Bp!AnLp%!MFqPF7? zM(6YJF0ax14V>2&l^@OwEHjmP$I#20)xT>t-(G|b2bZfprG!JaWSLQ>I zbl99`761E}&8yVb-Gxbvnq^X0AW^le?s(+4CviqN#FYymSipdg5+wXsyWW z$n6k%qmuEby@$nLh(h!=tg0CH;6o=$5gHq;9*tbikfSJ>5Tx#T2BVU;+voR+nom*2 zSYY?;DwPZzlIUUo0P*1NhtUu-wui4d+zQ|3jsF8wv?~HBqf}BNar}ET1HF4HoaUzA zWvswul#ZbXAeCqPT?^$H>SCV~a4k9BF zlFgIWys~}sspyxFZ@DtJvn)uwWoh}qh4QTwIHAFWrQVdB@DSZlSO^~#QG-g84{7y5 zJQgC@>}dWNDCa+Vwjy-{9cx_;z@#ql2JJXQ?I#fTJab>)Evxk7boin;q*Wdjrx!N* zDr>gKGyX%I$UeB4hAruohXFpONqc= ztGn0zSa$etEI*`tz=^FA_Mc!l-E=qgrFy;s<9Xxw+`uMBm95!!3I4tU{_)DVw7d78 z`hfOtM+z@RRct13-wh{Y+3fpZrd>}#FatKp3cX650zJcU;bdq{vM>bz|Hoc#H^W7| zf|BwTP%WGZLhUgMUQ)m<;J?4{HsDFv+#eF{To~7Q5%j!zIdW*YV>;7vTV}S7$^OUl zVS9oWra!F;RWm&Jv)RwjnC&Mpo8M^G5h&I2Cn@R!>L)25|lH z<}$CVu1eh3H}=;~p9e-cO{(40i%(gNI2cYu66CUUUt&0GgF?aYG^$@<@;b64`7||= z7s=-z*U5~y3RmAgwz`WXB>ts(3+>?ql1ZajXD{fJ{|*F`mP}JCPTm7rC1ysAQ9o_2 zUX#1jxF7Y!X|hVDVN<#B{nfV3$-3^on{Ul2K3v(tbQZ8Xn#x9rA1h488h{O&f(IpD zP#AWU^n|}bJKB|*{d+0J_~VoJkBfHhV>}E#WJ#t#=)_$%E*35;rVIy_Yx}~(_)D{{ zLOILQIC3123uk**D6$2+m(hXwXvK5itE}8^_hc!dxi-J{{f*bujb~pjs!v zr`@szU&y@}z5;N1&~QN<=J|c{b)IEP3jNYUh8EY0w&GF)_p2VAhvG*m<|P&uBJH}NPHfbaud-d%T<)Xj#l$6Qo*7&ASfJyE-E}u|(gQhZlbM?a zRoh9K;@ejp)`^DL>^nbRh(Hqn33^(|Y`(Lg%ZQVG<9~pIP|s`e(cLT3 zOmKPzt$c93-Kj=;1&zPvYsa6yxop1W*Mu`&ve5B=4BMP`SMe;32_P1`2Iwi{(zHq_1h2&&z!(Q8G9&3~r$le8Xw6I-7Bonmh z*YMAd8JJB$X`uv*=KZ;_XPbvyBDjk*DIbq#UzdeKS7e7wK~(CoIzd)JAZ-#=2R<38 zX7SAiOfuLKc*pI1OK~tC%$YLTg%)9(9AzY_uXfNAwfEK0dk1(E^~ta*j8hkG{`kKy zEtGBNSRM;R4%`VC8e=JJP$!istHFa7#Y!$cI7<~l=!km*-&eKdRMnOyBCcrZ->MRf+eAX(n zacv{)Bh@XnQ9y26SKk(7eKv5LGKiFnXHqardfuDpYc0BBjQwp&|Gg@;Qf;4OX6}6A z{hX}e&;mb*6+wazL4ZG>8qIOn-BP5|!9mM!YwSD@_m5)CfG&8787$Bw^1E7~jt z@2pw1Q)9&T-t>^s>l7UhcGe3{Wr8)08Zi?ea{}9f!y)`CrBRfR2K%}5*XN0+vo^-u zlAbl&0|gO&!-J2-Nmya5AL`3B;tJTI8H_}LoJ#(rR0DbnJb*2v4+s1z;uBsL-`Z0cqV=MMCzvDYI`xMEwz%BX2}clrG%brv|UfK_R7 zXwi$r|Aq&PHD+)@WRk52nvZ0Xw2_U!FvEhYqIukY=k9_nU# zSMs)UyXU0ao4J0}ot>ik_;apI$xYy1w>%qvbP_R?&Z^_#I<$sw0eQM3L>-`TEE+Jr zL#de5`$YTm_6!UvO}1Bai>*4xGYo&Q^Z4AvINN3sILj%1B>96N*d$eJ937hWLpaP- zE0j%05F`a=O-CkC2fJUBt7QI%U2r+#PHVL40x*_bOYQCxyJWk}n&8WK*Y}_8uklNf z(%$#-V{4qa;-c$6%c`{e46EgSrZk5kc}0vYti>VF!P8(7t`C&w{u_Upyw9)t6UhPj z11d&@3)17pQ+)ls%_lNd#!vWHrpHo!I;a8ab?h$m!rPoP^6ssLgWc~^h>;p=O7Za_ z7Rv=I~2E3z2jLr_>q?9@e>NJc?Ir$p!b`c6MyV=PhJGg z+U>y>goEnM*-$0(g7t~#{o2;Qm}~~t$Hng7W-re9*qroBZ?2FGOcuINcGt@|~p6z7F-2;E=P zPB<%K@qymzct);_Nu(XoKWdkI2%H-qEBKK&a`F>W6l17o#keQjQ-o8VHwX0=>@ae- zqv+Z^FEXB2FgLc|NPGK%^7Cij6Rum9{r><*(JJ`$mzm4XY2qTLAEsm3xZ=@rJ;Mse z;9Yov64l~+J?|oG^2qAYm;F3>)23v*v-e`s{hmC${qFhi+>TQVjlNzUO}^t_G|E2# zpWcKv1Q3a}hmQAtvLH)B=Y-asQ-zI+~C9gx#_gu~1s=1z|N2Y!x zYA*EnPW$tZ2ZMa)a7S_wQ2EH@j7+aaIo3&6gk6f?9S!#zp1JyLY&#AAGg+WitS-WHGSg zPHt}sZC}NZ2Kw>-Bul_ZsQr~@QIu?~c@+Oj%B0?5uCAou(bL%d^{@GddOPbIl% z&ptboYC3AL^07JauHzA21ho#O-%qhpB{4cKF_-a1s%H8?iW%@OkJjWr0L@>=1YN1d zNcFp(jbHxL4ttEHS0ViT(z@0xYT$^eG``0i7-pB1cAMPhhByDP)A25q`hWS;D%G?| zmr6}JYU_I4cE0WzZ)$k8gQ3_>Z6!E4Szjt&4L-A)T4_-;gE{6(70YFA+T@TtSl-)c zbL*$$;rsfRX_)X#XPjNAjiL@i$%0P(k$SuuPjb^3C>-je>#oSv+D57<{kl%=bo!II z$mfW>Ey0+2FK=pu5MO zCZ*V=F2a~c!)$D=gqW$NbbK|pUmR0)2U#qL-r8kKirTWVd#sN@oYInji7|_GVfE#A z&!@tHk!DH?Lz9U49wZ^_PjLYjF<%$j=DB;h6s02|5ezeIJTXkVO|vR$d)=Oi_uhg- zghb}A3rV7=9n&47%nvpe8wMnQwI7&t9$m zrqI>Le4L6Cj}nthCe4X_o#@MJ8|ZSB#7!_)L{r{WLi=^QcD*|BQeh zc5xb8^)(}&;oBzIK5TpsA;2xhwWTa_x{VM>C1G85XPWylBqSPZ=c%s8h;QS9F1O2b zBTNB>ZHFU;k7Jm^DitlyehA#=4ZU~@E(PabT}Tszs)k(8@3I(BsLa`rdiR2&Rx&#C zPnF);seWcm0r;-mp(lZ8m64 zPE@#yqM8s<=vBOLS)g(EyWr2ju7-(QCaE$VUtT$A75Yw__D*J;o^>-YoW4Zg`KD81 zpnjOd+#!tRyIKQq=N}^v`+Kxl^OlYdsIg8tynkEVd|sR0s^*YlGo3oHPkCOoc*9^? z5VZuc6c$B3eXxs5?4EXDpt+ZVW^R!iOr;PI)1R~op_$Sw(~5GYK%@erpl&_N$tj{< zvhU=_ex<%NZxMy?-!|Z+6rofe7P|+Jp+wBt=`cM!SXEldPQAIq+k(dU31#w_v4o}XzR0?ydD2EB%k;t*BHZIcX@IVnJ*S8wkIgS zKgwcpq3bPp+sfrOcMv18{%Qzyx?SNRba~@r9}zE%KAsG^3TC3`4Kpickl(luHiWD| zOzR$JrHEZNymU`-%e4QZ4b(*aWmCFrxE^5$(tMlHc6G44!1s;CHXJXexAHzvA%)tT zhxI7`2Z}NC5h?g0Gvs9G?kZyHKzaP6{@x!DTq#?_L@eW!I`G2#0i+X`OyuU;xxLP` zcfQ7T`5WIHk(1h^QY%@F*FiFhQbA=ArD&C}BoV->UBEBX4|b`nHs>+`9n88C1*0HH zkZI7I_GSDaZ>v(VKI_U7i~hlKTk$Gy@*>T(Lr9F?U4W7I^ z6Oa>&AM?sv9eT?EDP5X8$;DCnE9uu`{O)qi=F1m(H{pZy7lx<%rlyRh_mn|!lc|hk z3)S0PBCm-VULnO#ArKW2D+?XR{gk^n!u+ajVph~D@UYD1`22CfSwBvWAF~8FqY54E zQrY+ii2t?9Co&uC(AqpPRO4ORWKL)N_5Gw5HUCN)Z)g-igL2kEv4>6>Ae1mp&6BD3 z6L_JE=Sddzp~|DRaW#EX(hlJsw5H_pgMs#D;=YVZm10;v z6CUM18@yCo)`7TWjK*-$EiVyksXI};7(8xpG;=uCOy~Z{2fSpp}x?d zVcIIb04Jmv`>GHjz#TY6B+J;hGNY#3l8OQa{d|0G{S}Wxmmxg}l#HSChgboFIaN#K zo}6GlSAM$B?Xjetr)5Bnyr~^bKRA_diH(}`rbCj62E7m%0ADx^583M&c^|iVx=T~} zqH4j=G9&e7#9`wUMcP+}F3#Eb01hm&50^^AfdI|gsDwm~ms&TOQvd{h36z{N317`# z749}(sK9b#kep1U}aX+wnjsn&_qq8Sr#js!-tzuDzKeT_?=g6wY=h;44Tq}=L!(3s}JNJ zWcRgAVB)k_hT%ipvh=req)Q>Xq7J|$rkJu;}lDlWum+Mws6R(f1`^&agC%a zXV!yZ@)a?Cf711;7H!YDV=qX-!xsP>N0~07zwVL$8#m5MyvDbQ_a0-?Zb_`=k{Hf3 zP!^me;y5!vmK&EvIOCHT*1x*Xq7|d zRX#`vTlL7ljuW^L_|t*AzH=#gZBBdoa__R2>B(4;d!;2si^oAx%dD?jhJJSKh5b(| zaqmC_fILh?wjKDeZj`9?~7y$I6Ed_;0-W zS(rELb~*7&((7A7vVV6N(8nA^d^BjKIrSikXYK-%qJ~6Yp-c)!nz`I*>Ap6v=SH0v!X$Qlxg#K;V3`4m3?)-j@L`-Iq$=ij} zUJS3elHPr2m?(xHkjt<6U0d>N=W#FQuH<}UIZO&?n*DX$gqyvrj=>pC%5)5e$?avG zs*g`}s%}x&v${f7IyK&{;;zQryqZ2cwg0x zA9Gw!x7}=ewHYg2y^_YBHetF->B7eStXc@EknH>f4U&iW045iBiQQ1pED^ zbC6VoGM>qr#Y7MihNuPjp33;@IDC74&uK5t^Z0Q(pFn-7{GR@BZOlVsWa9vpYyJnj zf|Maan)u8Qk{ui!SMQttXY&5*X2ad_|6Zj2ky_>)wN`%g zDZtr2+B)my>=w{bG%=+bEtfnxAF}3M6Yo}ikA{Rv>e3KW6_6gbtw$0ygSNX4$_?K7 z)U5n!xS3(=5t%gdhW~o&#n@>p|1)=XwY<&4ZgLI-G{>WdTr8~4TCc&jZ2lD0mmQ<^ z`5%bZAui63-a>^_6#{xgdc!`fX%}1>O~h|TJy8Cxuq6(+{o`4p4mm7yUCpE4IdhNddgMBq+p+?A*S+XGGxB$mXw^+pnIO$USm3 zp`@v<8iA1y0|dQ>ZSZq+(55if%hN7TiL)H^G^8oyFUr%)Q9D!fb7Jk?QUrzXyf3^!+%b#{B4l3FZ3taU^=XJ z#YbAX1%I4+fZqm9L#zMuaM1!X=uj{Bb73A6_DX%V2?C; zQxJ}Bf4jm*cb;+INeXWukSb0fH4Qq>B9zrgH6|iz_B<2Q= zH7joqP66U;BQm{Op(Xvi_kBoyENI6h_dz19nN7@UIuA3HTLBrryxOGf^wYE*LCXg=p{fr2kv@eVm4w@#>#eD_e+uUTD5BBdHO#Ff%9(L&@X+>~L z!+iV)^HnS_<|8bF`6xD>#~#tZ2I zlQxxs|FQdne)Oq7b$k18uQBw`8-70cy~z4o#kCPBN1^et+?Qd7k;stTc1>LpiL}>e zTjU<1JmoCEC0=K3eRnBV6}TLcuyjURjef{iWaG72HkW-=jQ2E zTQYCV!t16qBnI{J410;qe`5%6c3a5W5>=!w{C=S-CG`M5Z~%gZy5g8f z%_1MAS1R===eWN_XaD{-D1ROAltK7u*&T|b z_nM#bTMM+r(o6f7o)R4}?$nu?JT57CK-^2VD7p>Jz%Jxi!GvWtmn2-0&u0WeJ|zN8ER5I*7tPmf{0zDz17QmkZ1IR{{T zkF<@BGn|9NQ`D{c6*_Upv#$jl&$iga)y})>H`({vecG0?-P_od8@<*3%IU-?R}l^E z;I@1Y<_HYgGu}U`fHHRa_~L7KRjh_IjU+kP9}@(Of4%wxF8Y5}86_ z$5GXq_@70Riu2jgG2jbucNv)|FYg{JKmak`WSNbR+>ht)jx025F&Sb8cE~u!`cB8e zj>NMKfx+Y&Y14r3Oku6qS`FSnZP`y`@95Az+nME3F+eIZc5fM@9> z4YMXXd&d66F(R`lB$B|uo#Xk9@dqAfFxI7U9PXfSqqR?!*234#&fJ&ao7@uW)an_q zfu|!}(`Dlzbmm~ifiv7Y&zQ9;wFXovsThv2w388nW8OgCebeQq>w>_f=`9igft)D8 zZ0ty!Z@Pu^D{b!ZO3zOyvqy@~EYRM^P0I08Q%0-Ee4dC?B|}VO*r@|q5%~WAt_SsC zq8J%75L?Er<86qMp5turw1;On01Qsx#AEX2KQ?}cC4?9Bx1E^I4@W+L$$K-Ya-9S; zZs_DbA{y%OBOX>O#KFTz1Q=@)@tE&FBD$~WsAqx+fhT#){GwnD$LM$Ca;$Wx(8%u* zj`1=9-Ug$X;5o~BR5ZsWd}Y=1a52KbYSs10S=={-jkn)vcQ!oTGS$eR;%BsY{{Wy@ z(ZBUrE_NdXLH_i46Wsj)lcq66c6A`J8QODLKCd}yy7X90H`k^t?6+~dcHiDAlVs8B?c3%(=D=6CTi zJHgau;<E$c%u@&*maZA5&SBVhVyI zPGp{=^A&{~VrR@x8@g0@1=rpL;C2wW%nh5gZ>xh>6Y3RXsniZH2It&H8#C#s&+s?2 zQ!X|wU)n@YVi0E{GD2|^F&{S|{KY|NkQ94oI;X@o{eWWyb3~?8k~Eo}zvUDC>;WaW z91A$-BJ}d9U5{l}W>rA87AB(hxiC*`lKV@7C$t-Nw>= zrZrCeB=MWqw^|I99I~*C5w1qYNdpH-GC(r~2?vEE&aeI}Z~p+lYySXY{6E%z;6L#H z0QgQTANFti)A~RE07K#U*R4x`2Hh8Gjf$M>IO&y&jGK(q`^w~(rp&rcsf9iO&+U8< zz$PYUBj!<|oRfe)q9YmDd!4W&d0QkJl$mlhApn7@8bJa^cfl~dqrV7`8sNJ87CE0! zw%1;CsF<>?hPbcpsX5^N#a?vD9KfI!8?8WcQOK54TD78v^}f-{l;)7S1mhMt#zC zWL&Dt7}eXQ8P-Ga8Lt?R6|H_mPF$I+M49-X$?DA5@4U?8`D64SQMh4aU5I971c({% zGxeU>2YtakGdWtL$E{y?Gv6jWff8{M!#TzGxt&cP^A{hbrBh2fi>si0R(1lp z`auTtRv46`%%s@<=i}JOBg%WkeA=xt@}v?7`fOxQ@HP-ed>P@LwOWa%L4rh18({_k z(j>%roMv@#7sxH|+2!~*Z0KFpto~eejyLbSMK%dRY(wrrYo4Bgcv0HVT zSG1pxPeQsYYWcpzv>Iv>tVyad5({5qC}a?Kn2!5+-1|;*zhjYusx#go@v4yDF7q2s z(E|`qewoDJiQx0n4t2t?`3TfIAX!yGFy_L2Vss1_YN_%tQk+Kf?r{cvjxm)P8ZH%g&`=xL{Ca zTAI!~S7s38TdelME`$tDy5gA;^cribWw7n`|y*5w)l0l5laVH;6 z-Qe(jx5FIsncY&MYs)Vx)TqHxR*Url*qIQsBfOda0DFZ4pM4qqPAMxa{)KfHdflq1 z@U)y{s+}2z!vzcX+Cv*P!2bXQ#Y}s|qGlk-fL5-TdY0T8tN;uEf;@w(PR9^V(ZA-n z*Zq?Gdu2oMcb^X4(4rV9cl0tl5TdAVpu~VyM0WhT!SoB#PJPn;pPepi!#R#7^^47x z=4$2}bgUB(#qBS(D+@R;0g&~Xc!a?A9`XZJ=RXbei^{aBP_nMztTvU@0Rv&A?qq3_ zKa{n%kn(H3TWO~DlUr5RS*g{=pCpmfkEq!E)6(1WFQ;z0a?ATWnB6zz(6-h<*3fn` zseRc1`1l3>K{b(9f|gMvm9M)S=ILzA_xMG>!u#1iS29v09fZOPdql+ku%Ci{A$DRc*C$*xbyNBP1_=K2?*ss>eJ_alG9pt; zqQHOmH{5qKBYjxUZxFGzIHe569Yu29n25m4GQ9N{iCiV0d4dl=5n$)*)EH=;W*rdG| zpkg~j41U7}KbVil&yT{q?w(%8uB?n{JItN?W-xa58h)?M?Hd&8s|}cj!0#Z)m>uIe z7=y$|CgGHE>DYaon_7k<>G994_h1X}{iE##^Cc583^Sr$E*=%6zXX()t{;aKQ zD@vl4GDQH!*mX!yj6lhElPMtaneE@V#oyWd!V5K(%I0%#TS*wmVydyfapw?q9Ba_3 zPKxx46Obw)gSTU!J-wHOKIcB*dW~}P`37TN4I!GVvbm9xnI%C-pRxp_zwAOYBOYD< z0B9lm@OJ_MRtF%0AdUAKBL;FY-ed;AW~O(La-^6HI}yB(Z{`Mg_>u-hHB3tZo)W(X zcZ2f$LqlMqLIi81>9VtAQqna&Tf+xbuUd*pE=opkB8 zdm9CD`H&L*XZ=MnKikX7JNEs%%+GGo9lj3q!}o*ykGHno4%3K+Eh{KqqcM@({{S*G zyaVDSMDWe@pKc<}c35Fb$kH(%pVTFhF(orS!?gFFiZ12H@f`PUooQO*RBSXj#v-sdyvFU+KtUzqI?*clZ0HI_W{JbNx3;RIJKMazH znedM|fu8-urX%EZ33e{?+qcRI>^$kr_&LOJhth^*7zf0igyMJlulH2-agQAJp384i z-w__k5)SewBu6tdJ^Z`DemgK@@jbglcaivFqcMObdp1DwXAwQ7GlK#+8m(m`1d;9i zPnpR6UHA{$MS90KtmvYpYQ-}4+cA+5pY@9)wD`yNfs*n3m*!?>Bjnkqqfd+Q{Z4H5lEmFYvDKg~kNUHI@1YKf zC?PXZnD_}qf2Jem)hyTzYow}~f<%}eGe2zoB6uCDbn0gTLD+`Xw%`Cv%>KRN8%DZM z!a9d>17@d8IDTl=0`-0VtKQ)c?n4n)qlP?&XSa<03(6zQ{l4m%N zln@R!%S6TtVeah^OH_6w0%1TN{{S-&yum--QoK4Gdl#sbbEeL>PkMc21MB@0wi)jb z81FrUO05<9rXnIgXo>F;J)(QbYskJJtR~*woRa}cFee9Tp8HG;_5yf_^FB$bQSR#@ z#$ag&15WsZJ5Su5TuxZO(cV`}_`ZL{DsY=g6C%B9V~m6WPl&^lbs7%rF%$a?$q$Tn zA(@$vl2YW{^btzGE4e}lP~$~K->P&uZs&FFh#U)Xc$VxQ-i+;vKJEq^>9fu;YIYU>0I3#y-3kXm_x`jp0OC7Hj1OaVPvHJ* zbxvtplx5siKAxZioDE;B0C5pAxwiO@CADTSAmHLbKI0q_@%e{I)?O#|4bTehDDUfUSGaftpO5S$XKacF zJY>6!cbK2dL`-~TPvAav9b(-^Ao|mSci5Qzesx5s2jUK1;g9dhkr7Oh`r{;+$LrJr zwEZ&J;4IOhIURlaez_3?FA6U_l~BocEQ{-WQ&wVqO0LFC!0_ zr(WB%TzpHI@ZAVwtpbF`uujqn1Nw8|rJRbyyDTPS!{q+}{w_WuuTVOruR`qA_kp^6k`^FJd>k!?*K#YhZ77!kTr7ST=G3WZuSS}(Du$K} z4v?d^`3JT>jezTXDNC5vdbYgdm1%yns_jKUN%>h0m_8&EkEW=8l>G>D&Ku9Ja*eK_ z@GMv8RS>_Bnb2&y>YWnXN9SI;%5(xCE-2F^659~2&}HML?<~8pxAlE7S5eWXnAFNn zA`3%b=R^%3_F9+N=+Eb0f}I*Xk^`b}o;7-KV- zam2NJi#t_X>m33S#iH41=Dv~>;%t;irlo4K8;ReXLA#pdcf2&Br_aO(D(vD;b!Xp083xNDMD=O>uSTY%B?#T z?+{n7FxZ+_S0=c$mHvE24L+t3NQnVNd`Hdsea|eclq<3@dV1w??lB(zezAeY6P$9Y z{GrmDkowh%l~V+v8*YB043oHk8XF%2kXGr=rgqiKM_6emN}jb#g<^3`$RRq`#28I; z1+nWf0QUN1ib7%vJ}Fy=S4Bb7sOl&41MMaWIP81rB%Mp>qcyWLO0YSZIP8A8>@f!> zbgJ8=Jd2LzHkOwrtfH(M7nI8Fw*_o0d7O>3LCnlq8DiG60rK9h)I7-m4muUOgPV(=nz^P0kG))iyWE2vJy{W0gy+aBFJtRm%RK_nf9aTA#&e=<*jfy5@yQ*3e# z@YaP{)1gYMbLic@OY8)Z?T-r~3z2Eqt^+U%keQ$CQ3W0(72UY5hMOD$2yLga8QFCH zv+Ov?xt9e;U?9owlbJal^Noo-47!iZhgE9g*jDUV(!Nz@qLA3G2^eV*w)+pnfwMi^ zDuVL;J4_Tu#gy$m2QE~`C;PG9F+O{b8y4V~VqlTKgODOfh!LFk9j6j%(a`QcpUn7Q zDBkKeP0M%aG5b7gOW79PeQG%tApzstJeokbFB9zp-^=q5Od77BLW~vM%o*-SSlsMS zi7h=KsAgi}8I@zK5?3Z5oJkY88QUCP_d0X0kwVWZ;h3$hT&+T^#Tj$yteNCZuoTV= zogat>Gy}v2drbbaUm#op2?WfXK?XL-%#8f09y=)-NSHD={ePelo%6PwhU;(&_V4nk zX(b4>S)x`gM{xG5EIqyq_IH-LVa72nA1ILR{iCrMj|C)#mG9Fr+vPH1PyYZk37#}6 z0+IxhPfyo#v5B1H_2a+C>6?2todyN;VIdX^EJLm>%)p z=A^?rY$s6LKG^RgxDs>U8KSXJ2{7NH_JM&h?te+)&*L35v9605`Dm|cm1|o}XcftT zTC1=?hr~)C&#`$F#1W5jn3x!V_!HHqOhZc?jp61TyJ)$NGN5|y0)oHmf z2Hk-7`s8f`PV!FUfO(CRK+Yp!oco<8KE*`iNicYT{{S#?zx;bA{{TyO{=oQNr2hcB zKd=7)gyBcvkCFcX0pW$|FD0R&qnCo59Ljs0J)T*6xvI_A>ulP4X=PJYG?WR1MTZLD zOkzD^?%(erl6II4(jszv9>@88pq?Jma;X9)Op!g$`?KbMP#^~zb$^ai%wTh7Sk2JZ zuccEXlTjP4Rr=O+{N(H=q@0;6C2&c%1t01$RCYL?fz#PJo z8iKNtOpQYwdEdU6f=m&CiJlkUH)gh$uQlUVyA-0_GPRrM5WOr*iGhe)OTUuq43K(b z`%2%lu{u2r?0I(Yaj6m`?>Vhz18^$_HatCcNQq}t_TWdPSw_DL0h8A-_>H+)tluYbvJVnup`NX1AP6) zmMYamusW2lgZ}{6LH=Gkh=s6o+_a-Zk3UM)surkbo_lqhZ^gP5M9@VPhuBI)B!*D? z2@saSc}o~`+;FXdw1tuHgWFC!oR;2WJTaisauOw^5!E0-1kQVm2Kw#5Gpbm;TaY7^ z?NUZ%hEbA>!8NPKtON{uo8)G0kX%okeoiOk4P0p>K?-_j6EJc>#+dzM z83T!H1PI!ZQvwWR5r`P=-#7yUsGb&n0jpY`wv7a#fEQjMp)r#qA1@Q~m6zkR4{`71 z+rM#%{{TsSMRwFg6S9Lbxa|ixA1>c8B$A%1L;$Px+CiPfnILKGu{)6*G(ApI`Hw*y z73=8Sv&Nr9soAMwzo9K-jMGZOnd5E&VcyXxB>=);LHA^K`8zu5M2RFpAedj~CPq8s zJ)?=&(liv13k=T3u+jzw44KYyFbUub&<-%0JgUkD*_CJ`m!YF%CbPX=n45g8Hfk_p80)cI!hZe@|hm4%0AEH&aAvogwRal)a6_iAYhMapGFS`NglZccF-kEtv+fM!%r(fjDYOipqJXYIZTBQe{7 zPOhS)sN8p)h!Wr8BXCCCSoFBn+nSo2tg~8-(cdOy;L(|qCp|2ed+dRfkT8SWW?V!8 zK0*L910qZ-6=DpI(+l63qgnwbsnM7!|uFC!~pY~P#aMENQRJ3c%!b&6o zL_TNoMir4AsTiuDH7}NRDGVcg$c&La#ugZSIdN}g&iQT4;%l{~dv5i{8}}jC)F2TU zT|`Xz+;OVY^~dx<^zp^A$F5&iir(rbel}3@o}6@_8%vn)aeeDqwQk(Q*VsEL&8K3# zT39fjDpGuh%kkqQc0pih)BI)q7HtaEX?m)~Sge|5LDxC?bjidL2^@MF@8jzBH>dj# zlu@NcNqcL{+Vri%qL9kzNd~Ho1Q4{{nY%2BWVfq7`!RRYClFhx^n2;^pLI`3KE>7C z>W4P|Hx;wqJ2b@jzb)Jbjdk}dj9|_ z<}{#lKbH%sFFpOL{CCelw5`nTttiuCx@|1YB0&+TwZ@9f#7wwCaR8n8&5o)60PM=Y zS^S19(CE(>=`TYlGD!+=7vgm;>wuWZkNxnw_pLby{{Ty8NLXb)QUO2cnzzLenk{Ut zs7pw9YL_gqdRiCJM%-csK52>Kk62f}VN;m%9$|daREMrctyvp&03&TU7|isoZ|ev6 zzx`z`x|$r<>IaeW%^3Hdt=*mO2g4Uxz(ee>VV840?wLgY08H~^K~p8%NMa^_Rymi( zmk7nKZ&ftlC9-(gHts3+?_bqL{&9#2N0Q%cbR(txv5rC6OL8E2}skg)C;A?YBg zkXz=5_*wPamNj~vUFu&8%xj0;a&&6g>IS;hKePc~*8?Fh+@XJ=codY(NW}ToZvHnd zE*|FLlon4?qp%L!!Kg`GZ-Mgr4jdncDPGeQuGCbvR@5m~Bhgm_=N9yUpc*7dS-?7m zM+IIt{u13THA^>n&Zd;8`?b_KXA#Y*;Sv6|cgrReL{EPclY^L#^!`iYk@21T=h9V7 z=Tpw{0t{o%oaSSVQ{e7Ef*Q`wq5lB-Lr>6gR%LtIR@o&Gl_NT$+b}+T87`ss729!Wf$9e9Zpo120_uJiGVsnZ7LM2+QhST(M9BO~uVd-$)9kp2rU-xs@c5ODJPi5IKzbp<7}4NgSD#W@Y6Bpy z%bErDE>mGnS_QqMU9WTwrvO zo&#E3vWV4jM$EYk4!C5I10XsaVlsA~oj~v-9(UHBH1D_JpIe*v6<_cf-9ya84E87)m5IyuBBjh$RLd@*$oAHx4NnIC$1CzZ0YCISJWP2L&&Ch?QA|rW;={_ zRoYKUizRacxm3djN(v@tzntgdE?-qzjrE1Us7}>UsW9>yRn(T?974O20|T>zLX#0mnxhue~4Rjv75t=oeW!BVgVjdK^P!IrkQ&BweAA zo5+Tau~&}K5R9p|LNZXWkrEMx0OpmdFb% zq*J-(mhM~DOPWD7e)V3RLcnE2u0-Wosw*~~N2sE3#qh!D$5sk|;Ez~1Ur}$uiKWD^ zu=rO#tpz1A4C5S*YB{eC#2yNmA2A^UnV652>%W37G`&t=b4K+9wv{?6pzWw;P6&qd z$Y8?;PYx}4#VUj)dTnm-hpftVjYHgMv=4OohU~GEMB0Offl*i}ZWimoj+J8Wg&-a*~ z(eWK*V8H5w1i_DCj`Qq9&zb3bGf7!bV?Fb*AC#QOY3w5P`_mWuOSr`*2l z8JStfZ)Ep}OC|@jKtROzj_^I>^eWk0qzDQN5$tDe#z)nUomo5$Qo7ry)+CXu^^pJ> z`er2fnBe(af=B@Sy<))pLlG-EfPwE372-&yA|`wJ_L!dY_=xN?Kmf}L?EwD(k^ca# zfu0w=*d|QLKb={8?IdHll0xHfK;UuDIJ{MX= z9lydbXC`C5E;9$2bA9V|xZXC7rG*n6Xx5<$RT17#+ji4PD$-YI{IbJV10&uK%0Fk^ zzRtb#iZ=9~s}e*g`AZyXW73&8)!!dUTW)YLyDA3CV&-Q37NvpCIM9 z2y?CPryiSX*{{|vuYaF8Ef@J8OL`qZIgOm_$u*zY@=j#;SK=8OUPLRD>F8Hgc=IK# zrV_xsQ20`f6css*$fiPqZRr_!kTS(sk`GQaECO-^m9E@#YYJ}EZN*Ef;-et+(7hp0 zQ}y)Ks-daTXQay1#afCVS_R9X;ynKV#1oluOxqRu2lV5`_-=4|lAc@DO}!3TtzxR9 zv`et)e*=bznV##6{e2}2#4x`QKrCrl@@f%Ijgr@6g?g?kM1VCFQox^L>SJ&NX;3aX zg>Va8*IG1Qk;O7~tyk0vWpsdxq0jLW$Ou&q)a^g4&+!81QYCk;TO20}-FWrT>F*E3 z!>21g{eqKDjja}p02m1c*35Xs2|<^Uyro9CLiToc^g;>!)v|&{G^-k|SwN91r2(9s z$l|MW&v`{UW|hTut5;H;X0sve2t=pSM)Rt1CtwQE=aO#i7xMjJ`iuIhyO!w`*torJ zNkbs*vA(FUHW6efrWxX6-asd?_Uv7zJ}tei@g{;(N(3^1JvutO3G@*d_|i!Lh#Fv? z3$XZ;;mXlL)`W#H65U$6GadES)a(=xMv^_IOuPLO=y&*Ubp|OvsXAS6++lVfy4>n! zqpMnUi4Tye4{MaRcYz{k3>3`7emnf7-F!juMU_@s?O7=0J#LNRHZ0X! z7|c!nZLf73Yys~G``Kj9dyljXIu?DqOcYE^z^=IXmip@yZrD{>F>6EI}eEN5ZZ zfFMlA2iv7;EEky?w=^oSKCLRvrDtXuLaE=t%%1#FxYzh-{RZ_fFISFM;GBZhZynKVW?62hV0a zMS05TrDIx~HEVJ5)YT@Mv|>nR7$9M$Og494_kY4`qDs^}!ai%pz*{RDmE$hzkHQ%R zDV;pN(ojR?1?(Vf3{UJ6-{EWVP4ecVP+=J!n#mD?IfEGQ5Cm286`KPl=zL6c_ai zzi5&=pT-ozDXn8ongWqcQU|z!s|UAji36xFHZG4)t<6HEivray6}4xmu&VPd1kR#k zPxrW>a!>G>{{R!-hpC#qB-uWoXH-l!tn`Da7qT`XKoQu}#W&wuHXGsF)=9oCp5*}^ zU+b}1d}^hpk1)HlXVRclt5hKijI$c_#v^o`VTAQ8y5;XuTCKIc<8_vIM0a8bNot_e zu1U-WYLLKu!J$^b{{RwC@k;tnPoEm4DV!Rqxf!od6&7 z8)@oUu5vy?i5u{EcLNUr>C#dK4Cf2 zQ`#jy!J^HCtYyXD)f1R4Z{xb*U~&^L#QeFbm#xcMo1M+ITWX4b*;=25d>P#~{JG(f z=bsiD)af_GZC;=L?Qdw+sZq!VS>C$TMj}8zy=Ws30(dcVF1GNlv~o@X)n1Eno?XW} zdChpv4%TNC>Nb{Ek0Il+q7yeN*>-W+&a?jjF$r#IrOp6Xv)v+oGIUORc|)09`qafh zf|{zOLZRudsVmbfOkYR`uNhB`>>I>YnrQ5TijN z29ge=FC>#6O^iSQZvrQZsH=9sksuu6WAqXZ0l_oekve!&?sKK@=xFEl0-kpitA93A zk*}H~eWkJ_*+O{@zavFRx)1^ci8}0$A%g>v2p|{%r>0=RALEbxKAdIcse*O0PG@24 zq6uv6k>pHqe5IYovzuSS&1>Zp?t5+}N-Bz^z*MD_s}zeXvSthogeTrh%kdKgIALb6lCx19RRrzgj+vT~OqMmv2iPn&wMUV0kmodQBqC-Y{!8RVouvfm~Cv{ z7GCYl$Hspo4k@ilUP`}X`t`)AL_wvTAqb z4p|&T0?dc=;xi-q&a~4)v8FP12W;=-++s7lNE~f=#K`Uuh#H9!yJN92;6Thc`E$$s z-*^82+Q0kL^Xa($1OEUI2!9b@_*e8l<;H^a=VM=v?_Acyz4jV5tJ-zzOp$YiW9G53 zRA=@kdANm#Gx=eOekKHmS7lJM+F^`gvm#*PJ@GhD2;q!?7#SglY2RV}dvDvn6npHr z-bbuwS4VPn&RpjPb+xUwrXhyj?C!HlrWx!QDf4gjB1%p^Y_7^BNsYYk?tjk^dyXKe zH4sP}cP4k*W-;SE`h*QaZlY#Zrx;w5AMZWLd1G4{2UlWzmPA2iCnF(ixy-H@Gc)58 z`0o%>b+Soe_wF}S7{pI=*nyBZ)|w`;8*IxQe~b?!BWTDSM^2yeecRnLw_Pu_TR;TW zLqKetIU3Z`N__jwyv5G0XX!B>>NEAn;(PquHl$#}PvtY;&w0q(3%>FRoRy~FtMr}1 zja~Nk1W%v$XQZ3co+s5EP4rXB<{Vp==MDvtuWyxVL9MQ{U9G*6+a|JQTXhJ05(O{> z=n(>$`G$}H)uI7eZM%c#ZsZR5m>d&H#DW#1#?W$bb|P?hm7O_>mL4~#{FjdKerd)n zWNJ}8>kGrOtlX!{FRMDs7UbOVQ+*X`!O6vmkbzuDkwBDqiBg}aATR`GC;5Yq@fi`T zW-6+yQ<6zwbpab^W04pHv4SxNV0aF2$V&HIaDygAr21keN+5(_qz}A*Ou~Go4m-(y zJ4AMmEFfUg2?1bv^F5Anz7Ehs0C-JSNM*!>0!&6vazKNP^Wdlu)b6nJo?FGarwPl& zy$6A;#{U2@T-?+y6<_O0dE<`-O=wB$AdwP(t44 z{pZihNA>L!@=7Fn;3z7vXMMD+;7?F9;~0iH<2NX#lRHUNAcYv*oXj@f3;a^v7iEv|-b`;B4{_`P;p*1u5X1lpXP z(Cx%NqUjy7ofU~TwT+w=q9Xnf$J-R(uhd`eX|Ht5@5=Xge;0l zP+?{S$4vIl=W`v@&kHKMnQ{OwPko7)#B4Sh8{i%neyQMc(Ab^+oW7?i$5zf&M}<{3 z^@>!I$1np+8Bd2(crl5r=L|BleI5P8G6V3Ej9{jw6V(7n+7Huw9Wn?SvQHmSth#Cn zsu9$5u_s0j`SMKUoZ}cRnx9i2-{_J+n$S5;oORkH01qYbv)`^c8pB3J4WKc;4z`( z$hVbNx4HMUr64sER@3EW0V*6b(^aU|14Shz-LujTl{$djogxVkTeOn9^6Q(9VSo5a zRFAH1MG9!dF1FIl(jAj2PMun=6u(6`3XIF8y&?YqhR@JH)1Ob5QH$wz(}gR^Ud>xOJOC1Y%N|4;m*s=C@S`r$(buq@>{+K__)3 zpb-K98u1cf<+?qcg{2Ibtx^akIwU|W74mX4>LX8eKsehjYf{|PT^m%>UX&=JvNO}( zoG0@lv{gVtkpcLHgs20ze$Z1tDeY0+i!&BxGD*o}^~dzjfhUHOBc<@5V@M!>x-}8D z!yWQ5!cEMx*JkfTo-#o!!j4f@M*f%tAy*T)^j0o!5_0>f^rG0 z0W&7nb_xE`i^X=^v-3>>ZY_= zN_i@@zzkq_Y?%A5a6#b@<>dOj+h(rX^!D>IblFK6H#96(sS{E{9qjb^O^VQQ4=K7>+oPKC@#6N4XPs~xAOnP2lOy49-w`SfbQ) zAV1@eO{nbYWuHr_)TFl>*D9q-(^k+RXoI2}Ei^hmq)Z7D?;n7XlP`}c+uAg1qO7ED zqjNrbKm)$hu+Dfy_3lxCTFtDTQd!^uIL!ALQVfy#cqG;26!fog&O_78s~j^j)l7xk z#tZiDRl0VI({ox9!4=YJ`$lH62*FZ;fqec)<$s!TyDO!rqKbn67Saio$qK_s>d4R3 z5gJ<6$@!i2t5Yj*rNI<5Skal+!W?kE_hCS$w{ zzT>zbcrVA4kw^BKh>y=VyXDoa>bIiKsxmzzkq5t-fr2(anCFha%DKf0$^g5bm{vy$ za}zQM?stV;=5&PvzE$+Y!8(vD+T!ROO?22ps%i1aG_r-lWHk<5bQsNddt^u%Q0ur% zJeGftADU6;SGI*!nA8A*Rq)aXA{;{qDy9aNoSir&jawU5A)!GakTmKFT^gGa`GXX8 zO=?x3uo6V9mM7G2)VI>5s+>EoSr$H_Ivpm8s6I+Y{$IyX4i47%hdH<9tDmt=GbWr4 zPiox_LReQ`rg)JVT4?h-K6Pr=x0F_(c~sODDpO)$kx(NsARS3PM5$d&%rvf1d&{fR ztw~v}Xstt9-&--L6&FI%dX)mK6e6IhsJm3wSS=9j+4BN@%N9fw;8`HPysq# zMsf^H;0#CtW@dYd>1FEG#7Z7d)67_$%MM#^v>X)om>%UHr;L6IB7A#H#K}Z?3qfaG z0!=E0K)|HLgZ1=>p3^==M+xdRSqfKCUaeYW74R_=_Ur*7BZb?lSMd!O05x(2$XJbK zD^{Zoo;~7X09_+6QTxP*#1wuz#LsVnTwB)^m^B)ratWrx=_jN?Bi;^@SmZI33{zcL zEKcG+WB&j%w%xZJdGz|}<{duan0lPMsa&e39^xEe-EU67m_J70J)SkOacP9fUE0@V z#aa~7Epaftkf9FCGkGU3t9^3HipxhHr9v14+gkuskjJ-4oCr7^9Z_i8Q2b@lB9dwe zN?_bSzP9|r+?d=866iYfUuV>0B3u{nTqk>3S^=cU1+9gH+fZDqjG78l0 z?{7J^P_+ma$)nStq@@aK7&5bI$q+CMdWuUj1!8!I>Ayqxg-htg;uvJpsIXb`J({Y0 zf=fz7ol=yDRiL*_dskkn!bH14lz+O$xx2e0y+&0mSgxW6yEF{GB$FhQCQcX{{{V&M z)Rk${q;vsJny6K#sDcqu6qN{y0@DiwD2M__6a9VY^YFGfZfMI54ykg$x@J2=(cnYH)A7`$Mk)A(s2@qN{b3>!K*4yKkk6XG_fN$9xY%A&MOtlB zOPQRVgharB=X`CEJaem|>p5*IOvsQRG-q;gGo5(t6U5#ddET!drW~I`Qt>ik9h}<> zRY)r&a-y0FS}8H2n35AU5eMv1GDd9qCCh6qvo4_o=5e{gv5}ZEKAj|eN>f9uEkK}Z zQ4zl1GK>=&A9E7A1JKTiYH9RyGm%-7tm4+#WeXjNZDwNcwkZ@qj8CdR5|}xVA@J?` z$j{23Ii*3^j*vpkZzo8Qa%bNF-~%(o%ad2QD4LnImt_Q)(h2D!Y?YXv_!$84J;u4; zHR0Psg`c6wb1c`eKpI)kvZ|*40A#TzaYr(b-BBuO_Ws^8J~J~PDA#sY)fL!NRdGE* zK#@CQ2=O|x`A8ac^sZlXc}A9(YLzJvL#*lqV{zQagVo;73)kJ~B{iI5-0;)!nb6ut)?j0jH>KnbY0`{)7+< zHL}r_>5okV)2KSBu_QqPNc_?#0~s0Yt*N|A{LRd9^XX`EeigDJB6E7Cq=W7n*CZ~k zt0F^K$z?J?!Z{A}0W;cQgcO@|QC|Wi&fSUrxWr;-0X-U2s<~QKQ&!=D!6baJ1YpPI z1b-3fRzrhX$ir1y4ZoB&$GBOqMTxcK?(zFWCJC}eLlAN?AD_eziGH~Z1L^VGf4Prh z2JPQ42O1lCj)V7X8&}N4fETgq5TIoIrZb%m%#f1T@ z<;#)1X0!`Rr9@)P71wnj7-((O5C)ur3HA$F=l6?3@paHjw8%iz*!41j)K!!c7_LDB z?nJ9$Z`ME1=TAPI`nA%|M2ubTtjC#^jLRtstI;?% zE|lD|@|meEH#U6A>a`1+OHE&x+XhcW5$V-%R4G-JBr=vbIWRFjQ2a;As#)^NJl~Z8 zacg;KYTFv52B%k~y%YIM(^AeBV3i01u?WH3&QXJ7bCdlpbSW6S{1>Xc^AkTeoovIO zsg*}bbQ8U4PCDvdtxPJFG&JjJGTl-Nbef9wtTF;#Wf~Q#idT1rtqX*rOWTk2dXz$j z10jjh2-0%Ji9K9VtCJByRmAFNjjOtv7Y%P2sQRxi@)}>}5h05=+F-q%gkQyoJ+j-;tM+~ctsZqu;jFhVYs*D(b29jlt zoI;u3N0ao@KmnB+S{SedDFSzi>~@LCz&m7(j~3o3L3Q zTN&eBs`e1UlzS)j>;fhw+sC-AuB3yfc~Bu>?o4kRMq+n~fvJr%AQth|LD)nSf=7K= z{#er=)9y{b8<(ra8Z5H*+5RIimT4NqGcpPT6N!Jfv`YlMLSkpXgbc^U%}i?R>4S-x z20N3O_R^=fzY~|zK&B+9lG|rjU;rJ#{L(SgrH==$nr>`za@xt}SQ@$3)w=g%Lzo1s zYtOA+kgUmqpDN9@?O4@kb2H-@+ao%rCwUuYI)o^?$6yanf7i#HhM(^KTa9;A+l>fKeH?qvGHn_o z4jOjqDMZhCpK(bXND+*m=6k?-%s|0yvHt)M`tjUH>A{reV~mU(6$Uen$&SO`Harcw z*T|c_i=B`weS2)Awb=!`*b@XmEMY+xU=yDo6~~l}e%<@T!Acwe3>AZzu|g+@s$4n>81Ywu>NP&_ixw!d_e=i{RbKu^zFwwbGc0% zkF4CO{3$v}+-@OJtX75jYauN3n*v`ycVpIfSSMcDj=DQnKReAN9S4%!g)l_zj(P1r>`joY*Jj|Ke zNR9wkB0fJfYHb>ffv_O&_S}Ah-eMLfX*zq!GERR;0yDgsiTd^{bVjdJU3u_`r=7}4 zjdiQK^p#qTtjh?h>uYNy9yX25{g}WKWXX6*C{BKS3@)6R!6f-O@%+i{j1DyK08jqE z^CxV<*q#i$E9tB0bERPXPEFO0L3&%J3)QqPP0AUT?6g^M*J7{`GPo{@|NuS-cHV1W5KGIImI`wt|JBsU_G zz&%1BV4nx{#K`?K#!d2`r}K>6YcwO7Map{jZiB>KP~6JR{{ZERmRch|(J=vASie~r zM`$?hBRc*OBocBwiGx2tPl&+oNXX&?c~O|N79ipQjC<#}pW+?23V5hp-lLU%G)3Cc zR_(OgysAHK1Tv*AWGl#!kip2vEG7f<21nn#c9Ct15u6j^Fg^B@w{y9{;xk-9=^(+( z{c?QoKl9^^cJnz4+kDq~RNHRCXe(N{8x>@O*o7v{DUlFgf_RLCOw31O{CtzsRVPlR z?YzuKeEHf(xRJz3QI;8EPkhXKgFU^3cxXIHbn137ISw*~BNxaH*;uBp1*SvTEdAX+ zGC~L3eo4vYhQtxI4k#(GQvzir_*s4;B!3b+RQbNs`jhW6Hr(x*gT!>H zm+1#!HxavY8GI&)%8fY*emVr37XDTcX*!}9L6(m<8aqWB?AH2&RfBrbLbN8(<0cpLpAG#2**g zr=@3=(y3yl8%PyaO9`Ek0e?`9^=JSbnu@_GHOYe?fH*1f@{#$nmKu(oEFdd3L}Mdq zhWE&92mHCH6!gcPw$46{w|%^MH9wS>5aYJ>c+Onc(xZ!hu#cpqa@~AB)@#X6e#KMDW8O{NZWbF)805!ai};#WC%M)Y4T3~Tiy=b1vr+@jPlC+euZZuhW8@d zu*$ZqBhIEhF=8rAS8Ci=Ea=T}Re3O6_%cc)ApDO>SZO4T{{WE?+fVmK{{SmTPP%1J za$tS8h#(G-C%nLuka%4Fx_A!9KH+;hQZ{et@v4dDwZB<&hsX>0>dGkgIfzuwS=#<896jGwjd>Q2LjeekocFy*Xgds z*qr_%fZWJK>XuQ}jZ7D|!ZSTR2auYp&O#|FACmKGin9v!s8r?3j7SqH>ed1Qjb!df zUr|3wyjNe1@lHEi2}%|oO02F;hP8u2=ytEg?^lo z(-`o?ocB}u@W+=`Xc~mzsEkMekvoaU-Y^#@ho$;`N+2s0IQ_*40vGzvs9*)95_8&QyB$qHefmyG0OxFWAIskjJPlP@k{E&p03!+sgPxtHMh2~< zeqjbb({Cy2CF_@YUEK`o>$BXcRAKkE?3p4L_{7X2f3&TC^?z7_7?CFQ?A48SkbB z+3$}8a3<>K_;dAhU#J#&Zx74K+~ZuE%Fd=+uO^O#eTN5!vXGLcM2E6gQ(zb-5?_@N zOc34qcRRN9DbuL%1~m{GBM@*1`ws^s@y+?a!uLpBt<`Etlhjqz%w%YTj2Hy$#DY4O z-~L4pUN`qE+PTt(f&JUo_m1Xj>OF8%3sV_3AlV`ULdOWq_{@03K}3rO=C+X9ZX;0H zWI>YvZZLYgeYfJaIS(_&yM06$kUI$OV35Zr?b14mow7f}bE=q;m4B7hRxsGETNrbNGbR&K%@+M#Y6^2*kWh#&VGM1sRC>qCNQr7P z{aDD4ArUE9l+Wv5{??;hIdzR)$eI40{g2l?UoCzauWM3Kf)WPoNFWSI)wlOR&d{+9 z`_Uh!zGKAq^Enos^yKl272O(HCg`yO3sVFBW#LDIlmzK!VdV6Sl16MttCGcy0FjJHdK^n`4h_Q%5Mr9n){be>&#Y0j)y@dzs<>8(Y82h`GX7>3de zgCSeN0u=FByrsBSZXGe!cJM zHtlbyu3?0@X`1NqjE*6ZNipVA;|bWpqW~IrLhnwIu}1`w#FHwk zG(Y0E{vhQ}1iq@ez97*dpH+^V#KcE=FE1XK3q)u8Nkd1!;wF4zAZBiA^xj1Z6CyVg z2mTZE_RIn}VMfE4vV#8r5c5RTqCug{SPuA`69AAg0~?d8f&Q=fm;G_yHZt$juT}36 zcJd7id>YawAmoS54d4QT891Kd;648U5k4_LnI81qsTFPLI+&1GrWk6-T`Yac(>qDo zSKWVTP$D_^HD*(QQ)PLfeU zQOYS_Rhfd*LzY%zNx%r&6030b@$=NRaVW+F)@B$GOU z14xY{Njey0@23j6DYl+1pM^FTZF^}Jc*uOMdIe4 zF;hfo(n^v~atv*dJ+dZt5gaYKCUk$^Vboc&1l5e2y7<^KRoCDp7dIXW7t zeiEBo^4_R(k`6 z+lrq$_%e;fCp7CZsI#EJ(*h!PF(Q4BatS;DHb28pta{C=tPVxMZlz3M%Ps1@jN30V zl42_rcgkWve{_?4d(06)_9-XU-w@fCLd#Whdjbz_Tf zpM96N+GjC>-OAtLMbrbioBnm?JSUE01;ndXa4y``(Ycv3`Jk7%RVh_^2h3f0q;p+u znT5eX7_zwdir$L1rmp3|3S5ArZlX3M4Bk@ASlI{@jEtUnUjN^+f zJ5HT0g;qA*XV`%RZ;+y5BH==TJP;j`#h(cbqH zw6Wd1tCiO)6s9b(i125OdW=~;&41h>Y>^@Jp2hi(i)!17O@bLtL4zMcI<}tT1d;HX>QMC6PSJOOZj5gLtBM{-Hpal1j7#?gZ?G~IV`uIK}8 zzySULWRMbGf2WB_2*ihO)7}PVKfHgVw3NqDiAlgBG9zi*8hystp8o)w8djRqB$$~X zo}h8mHi`Pl@+4`+fAJ&1P6%;rKS&*lOgqUKQ6@|6Y^ls}nM_H?pMgXKjEVTRC&p*x z=i#b@@azC4Nbfp*!2o?sj2|+1r+iK(x;Q7~fF#Ko!9D)~-U0?Oo-t$mOL>^`FQcA? z7Jw)<^}3PCbp}d%k=t#Jqq}Cee&o7tQmmXTNMr%MiaXD5j4e@zKE6771+6ZrfLQ1t zMCsH9=Opa{XN%g>N}sasQ(CO+y606=l9McGQWc(~ERsqW050NAk`%P?k^caOk4dpP z&sZ#S4qn2|!{hjR8Cg3PR_M*I&36W&<^BE9gZpHoC3d}i#%96j$_8M?VEYsYih6>U z)uu#~0GvZj2-oz~;N2=-?n#WZj>|B&8S%_ePO1O;-83cw;NtXC# z{(Xw{xlKBoxHvh~Xw|(#N~3DpvwdU=v76^)4A##zLZyq757eYXxsSju=;>__rHJhk z1c96bm>}Y3yl|?{r>R9MDJnMu}nHtnV zCTCTo2H`nSfj&H9A`iT;&$K}fX$DZU1ZGAyowqoc$86vT;N__5(j<8jVEs1x;Ksso zR29pve;4<{$I2qbr3KzQoJah7!moEB#%~dsceEI7uHVQv%tmLt$pq-Zh$Mh9GGOO? z5#aVSk&N*x)drC#B!Uc_&*_bzN}2xv#GnpK$X(C5RU9ra$7wOws3>))$Vet@A%-$; znG>!?=$KIiz>3Jb_w6I|0-{sPnSfPu}C* zPT{pLB6yoOIM1<`^357jzDvie%#tnEtV7;4sl<1HLzq5Fm>41v4#m*81mFTeFctuY z+IJJaA$QF1sfsgc)?|~2X!?!&Zhx5HgC7JQ6RpJ*`~15vA4a8RDsgVbpXXmBa}JwX z#01D%CL$W)gX}Z?=eJ?YwN_$eBF+JY1Q?i-NG3kSh}bi2DNh;;u>dvav{5fMXPMD)Nvxknk?@S~`&quArEm zSp&4dF_|(K{6sSAE_QTf$fehy)Eeltn2zZdwUl*c?>X{i=D->Z0t+g zOE0;GULjf%U_lcgQXXD&-@k7Ug4RifH34ZdSGfi!24~(!kr|CedWNQ-OvGfw2r=h< z{IK`H($R6dIvM!&>!YVUme#&ixxANc{e~ni$S}l{+S^%Sp&>F%3gAIA*n_v{xGcJ2 zSWaYtoOg}!vDg@n^2C>7I#iH8VLRjPAAI)Mo+uoi&sH@qUenUz{C{q%jBK@ew+ky> zb-=}dhAz;3rS`-`rdjV)91K9rOhlZiQ%aSCELe=k&~1U6X5vOer5`iXBf#Eq z{XOG@7YXa1(}tHE+TncDGnnwMq$Y)WymodgxO44a2zlC7bceR1SL*Oha*H3lM7AWL zeq%+(uuBI*brFJAWSriRL$N~wCM;Vjf8TTir z3&cWdxKV>1*nzx&qsQg=N(Z1bn>YAdQp* z9gO4kIK)VAZ~+m-60I`LjKP6BNZpL=VlYlNhB0Py{ue?mReh7y+hu}+3M-oR)7ZOr zwJHx$fO$Z;MkL}yy9rEuj4%pulO6v6nCvo`?Kf1q?q6Qzc=$bfl{1m zja&7>mmur8IWW%$q7GP@N4_Rtz=yjba|J)Nf7YLx{=duqzpm#S71>rG2_*YwewvT< zALGSWLz1Ujm)laat`+@}76QbSF_8fgAjL^S)&kP@M61iUaPt21<*{N$p4jo28*B&L zCp+hlnI*G>j1jOi?xcOmpDPDY`)8$8?G+tuPDQ&TS;DEKN%rQ$7vm@3`+m6{_|a+3^uQa;9~)|dF+L96 zr11-MYSrcF#9ioXF6P+pVMJ!6`9vJ9Tpi*@dla>DLPAB~%6p7_`6L6|IR60GzS)6; z=ZPXtm>GZx7+L)%?0vd-;@P3Exl@Eq*9OlbtCgc)GdCkW(aq%#ddv%k&$5r->dBvu|){+R)ENRlJtW z0u*dw04xE`gy@tHk;HbGSTJHDOSgIN1UjR7h9*HKPmgnvxsAt6Vr1%*kJT+yZ|YEN z=x5aX+XR9cU)q{(Eu^!7h?ynC$v}9Iae<@3Y-*3~iGo`nYPma)|Zp%)Nn(87R{{Xl4MY=&Yr8g(J zlWfQ$MhnEHl3q&kp1~NV2q3A|7%}gUAjp7wj~hn;)d03<2WXH-P6!hn@(B0dQa6jx zjGi+b(~w}+G}hPDbt9s=P;v;@TA9bI-K!<)i)?8-PUKhf5~)%lh!QaocTB>EB0zw3 zVptgi&wqbDRY0ymswmNDE;A%bNX&Mg!*E7mV6k6{l?%LgZq3DGYEs?K(#787O%z#c zDr(X*+*UnH0;#V;!euJ^wvxbHJ}F#88ItSj%1X;C|yw`?_`$FcghjwgAZap`vo>M7pf z!1j?$t<_gklIK{o(w&AgjVUum>?Dk8s|dh zt*yZd-ax>mLp^z@^(Hm!EnwIm8k5dn7yTygEAq`VQFb27FiK?RK& zW_qT4WXg?pFc5y;&@=8pgFDZd#7s=Y%>3m1^11byz#5H1(LXHCU=ni!5Pta_SZ%0< zW>_Feh8}0U!GaI8o%F;r7LFxKP!+En2o%euFv*iPL`UX)dkV^XcJ29zkNZAW&1fY5 z026kJ-z4BfV-kF33l{VRNG6>yU`J8L`|XIxF(NY}c%X9bknp|gb_Vu9!9UI#c35&2 zB7X%1;=zv?9>WnHJ^Ocvp8ftEZ&3+536s3_Z-E$*5tADp!Aqzr3PqR#;L4Sjli&n7m)JtAP{zu5x#fK{{TJ#nmi+$@fNVFPA=M01?#k@8eG`#6Wxd~ z{Uvh~B-F|Gj^C_*)%d=s3-eB7gSioi5jn{}tWOgsQ2?t?X!>V=@;=fhi7Y-{$m)XU zOejyG!pW}ECL_4SNQZJ<5J=250rabM~86jC))27H( z>5WHYnIb#vF`s?Nowy=V>pROqV|HGvE!5W%iQW)UgG(LQ77U|7?Sew0$F&;{Ed(j zNg#c39f*@Bw~rZV1B_Ix5D`rjz{Wmfx$I_1lZnItAn@ySx2H?S2C=gDOr%P?Um{1| zOr_-u@n7YI1LHH>#9&~1&wq+WNGEsxvS*e6(OiXrL+43vz7_nDsFClmU4 zcJ1?*Rrs6Y>aXg4B~}P3sZ9}Zb}%C_cRh~y9C|X}_Idjnr$o@BLF%yGs@&G7`drV= zsDcE*5Kbh-!k`~j{{TrGV}(|+sl@oEmj}?zQ*B?!ZL6AjcPp>$STTaS@_ga@e zw}_ONVv;Y9eir!eoE~JHyPYjMw60&%rEnM1t4gf&iiBwnj7VI7SMtOiT>k*}iTeup zV(a1S{(JCcN|x6=u7h1xvh@vWH&iKRuci*RO3J-MP8%SN&kf)H%lpFebiS~;X2r&) z&5kqBZQEGjW#B(^$$6bh8kSVKi1Ko&U}sk{{{Y%bIf?KWc}@4?>g+ipg*r`3)mn-O z1}Rb1O!tne&$!QdAaUd#O=3sH6;iD|VughhZ(J*Yzlp0<*QGjzGGgA{jD~b83cyuh z-m1Ef(A>UNUfp}N#nWTEW7N2hTrbFo)WMI{@Ff%8B#4d%Vmo*F&h2l`Db(9?Msc)A z`}aOz4lj>t>s4Nc3>g6Uz&-cg2tUJ~5pG-19&5$g=HqDCeHm1#H3^QhY=yBi9?wz& zSqI@Ai?Bk=%3>$H7sj+%2`(xPCmVx_%*Swh01TgQqL!ip7O*9T%1%K)Z0;ii#Erm| zZFMh?&-~a|Wwf>w%+=iscbf~Izzw5es%XQ1+&~Pczvc!aK5tpv$VX-Xv>Ay9R^z_@ zXT;A2t*r{G9Md68gQP1F<7o%~01VEa{97>k#B_;@XU}ZgiZ&6jl&_x<=wWpZ8=z z&j+bJFSwIc#BvPRwmjN`+0%7!7Wm12uM+GFFUORTh{Q=mK==8n=0FpzY8WvE+>fkl zKu^*m$&xr0w=KA&?dUx=0LCezv~CW^1bHjlc;K^^N5b@p5Ljot35R}3sYi8$QaMw0g@wdGI5FF9mTyCnxX2|nZc*0Q6v4c z0%B$W_nkoh0D;qisR<^4`;+_1mU3(T*FEM5q2y)UzaQ^Czdl{Y1_y0=n^9ufDl!J6 zl73+W-1`~dj8LVe!)PKFVCDvVwtoKrZ8IDb7sjhf0ltwdLojL(GgCXn{zbx2v6z_r zb}63G{{U}p#39D``s4cWSg5Zsgb@pzLHZIS>Pf*JBaYiM(c>!s0AWN7CI$r0{V^l? zk6BOS#zsS1Tm<`-<5W$I9NiIX^<(>m20+ia--_Qlvo-4?0M#&y(!Dx~h%u9rF_WC;2=fR2Ec^7h=I=u~2}aj*>90-G zzsVI50qEqMuf*fnAp#Ke(XF7yu1CxU*hxUi36-;!(_{FG?*Z%9Xe4AHFp36A69X&x ze8}R#@e7MiS;+Yku~ExxhPfyeE7r|IMBtFhSd$%RY{MP&h4k?}W20ONIPOax)9?C+DPf#0|C9lpmg8CAuaDuIY3Y61w}C5i7d zgBu)fIO=+IaGhUS;94JRP2rJEx%QBCKan_(aa&w8e&sad)kxZXg0z!bP*D-^CUvA7 zf=eGc6V(9xt&n0fGl|^{bPObu3TN_nAa@dDJ%KY4-!BN?S?h1%D^Hi+c0#Ra>e#NV z;j#CYPHh3hoglax>uT0rmlOH!a(+=G<2co(i!^n(CKyKJ?}!uLOz*i8u?$!ML`XZ6 zJBayA9FZsVgQ;Zc&neR3n>TIe!-VFP)6@I9u*Sdd3JF=XT*3Cs45nk+xbgum{r>L{Ys0{{UsvFBJtP5d^BdLcnFPQ1d2Cgm}-C&VtYfkOtiuKY8hs;&MnbBY^>9 zB$l5fefEX|KV6Sk%)fm;<@**bP^F@Nl*qcqUb@hWdKuKW-lVEVz3Mv z7oA4ZEKK;rtENS@?f8CV77SC027HV0G4du8i4z+c&(=P(+iY%lnwc^H3UW?H)A|m> zzWyX}dE)&@%k2oRS27YaE**Uwvqcu%nUG?F1@~GGNS*th|C_%OXrtv94n>+ZFctk|^k1eTzP=Tl{?W=#Lz~k&blg29&2qV5@ zY(!_~+kKAv9VL`MpY;dLyelg^7=trc&^9kQ($JE!^Ayz<_x3W}SB}oRN^Bk&I&7%B28?Ijb~PB$Tj znAq?ALE9K2WXAY~nHa!gBilaxzG?n{;s;zKE&0@Re)hv1{FQc2m6(x>XnNZVTLAz> zeT~f@1DFt6yA03berr!kP5>u-OrOwyxrp2heGEzS+aFvvGP0j)^X#zM zdbM?u7*+$tacCkzawI-5CkrX>5k5bkDD6LzhOSt)WX|SCu-Y*v$B+zkkxT}j@dpQO z^Nt|0x_OhcZ%an*b&S0!eX7*l!5NuQtYBD%FoVb#0y7<_@NqCR6W%^#7GzLF!Rj&~ zeTMu001?i|64E&WLBS($uHCWXH|;(6?s815q;Z^kS(>N!d|OQH{iEVVx6CAu*zjn~ znowVn2@-~g_VbSM-gFvQ5G3MrAEeKXyLXNR3Z>s(RzCj#bJG#nXUv_=kB#kzKI#5t z9XB}PO@HyzNh@&+C=V8zUR-5 z(x9~zO7Bt#1pBi`U^XBe6B0b^8;Opc@_gLxE6VU}kH1R+Upm;SQSA8l@uWUW*2&P^lm~l|4bUX)^Mvpa2k}Gbjk38HQM4ly;eE zn&JdTJCt~ri2nel20uPDGf29uwugYCtZogxCNfU6%!`j|7k%y|&Rb#gO? z+Pn<=I@OT`J=RwL0I3XYcZl}Pgpm1n?mjWzXS7F*d^XsfLIgpQGlAOzXXWfp4r9KR ziX;+CW&uzD2W`%+kLKIIOv2GQKS=agFlo`xSx87Xy)$xQk})J*o=WhKFYV*y1_M3j zB6~v(PH-AO-I(%s_8vWn{Gpnm`b9Qu_Gt4aMsc@e?TKIyF8XV`$iJ@MWi*Z#-?nxZ zbS?xUW@Ey`OlSR4Qc(dX{kwOL-!@8$j73_c0x&=VGr7T!u5|YqF^}F1tr85Fm({d1 zWK5h44?E+4Ht*6`AGQo`r&W0)BvmVv(ap-OGvq!!=2Di-erL3PB%(j;nEj+Rtyr#` zg3L$?fsW*2aWXUfI()oCO1BqD09&U0OkM^%z$3`muEtj)033(uML}XK@!mntHV^h& zN~Sa^Ah-|dI&O*==3njG_VSo1n2*H#!&0V5fJ~n{W9)TI;0~;NfC88; zgCqK#Q{QMOfvkNJau!~DZR~DHH2i?Jtsn|SrUq1zNHy|N+6;yRzvul!+qcZBs2~*( zKQTI|+^7H_^=8?}!KJXB$D1IFYKB3P>tR)x5yOb{hkq z)INYyu#r0TsZB^Has)sz-vA8EZ=80-6TxRAr1;paq~}$rRudj4Yt{#4s|fEab515n zVsjEdj^hI(9pYwWTdMUSs=P0AI%Y>=5rd_tm%FT{vC6WatoBJ1Q{}LxcykoD>EJY$=uR;z?Y9{2^jsR ze12d43#X^?6<`9vP)tOVn8pAA6S)E)i99LzpZ7MlHo*;2?NYOV4$q`hQG!W2Sztgp zGXpvq{s{j7M~l$uZeJ_e%L6+o`^8JZh-THx1#GQH^#1@1C_fPaDE&NVyo?WxZ<#Xl z4sU;Q#JU$0sgCHhN2a4POn@iPkPI(QtAA7B--LMwB(`HlrFtKXz0v}+)w=!dK$bv!4OJ%sOe`{UR0IgbzKr+F|f$ycFBgrIoBXgKP;#>4Fk;pn{cW)Cd zL9tC%r$K8}F?Or{v7C|m#AF)~UFJx9PrQtFn3y2B$1SH$nqFFi_?k~ljUqZw>gj<2 zj(_^CGcC~n0B64(*G^#kub9?nu)SKAjj!b?a;BRCJOIQA8IJsUfd2po-6QDF_=+ow zUwBe2J}=acB7`0I^3S?jECOFMBBh)`M1GMG+=gSz_Lz_Nn2Dd1Hnad8 z5E~erM&}_XVVM~1s(N-xwH36GH4R-#0hsP23=np~>`M`jx$%6xBs5O6t;gfpp%D`@ zUFX*a0y7a1B@@`Ev`ok2@&zR-Pf1RinQW1#6Bs1UPo07GiEUn&sKKMC<8Tbl2GD1| zJ+KY{y+Y_NcCe8xnE@6>Ys_D0@(5!)1-J<*?WeuB|((vyPAa!gc zfq)SnP_rp0{CP=FY5A>7geYMFCQ6V*;~VB9-J>Qo6_93^1WjPg+I^7%Q8C;5ejrLUsktJeiIEeL<2|Rg zBV*te9=lzC?y4zK7$c}0_lYof?cnTqG;t1wbheb&6X}}uEJR7KLEp+%_0KQ%pLF%u#TY%WMV zq9#0KXZ@l->j_u`AqayT&-FVQ9fWy<3u@|61F?KsT+C#N*1ojRR@f+yuC zQ0zRx2N;o(Cy&quB!W~8xm;8%a#<5G1?N4@UHiR6F+HXXq#{4+&v=iK#FF1PXxI#5 zI*ew2if85+JY|psA_5&yDhLGoK{Et+U#v%06vB_(Y6g3In*k&QGPnqsA&F@BIsX80 z_4v<80h8?Jc$c5xmp$atR= zS>7y!3=is?;cPiXygIvxTLQInRN_I z5uE2Bff@O{#xct6sa57?rNV}xWl65B&i=B`gf%K1#ED1fTEQ!iataL2puLt zhdIQNG1N)!GFFq%uc}>LU-K4Vty;6a6ebAO9-6EZno6okItNITB}X2EIHoLy&OHlf z$F#<>R+^PsxYyMh`H|p}d&0oN>s5uWDb4E#?G-{mh&xZgnTSwiGY|PLn5f8&$pP#kp4a|s0#)H0M#Qs=Y2Do!Q6N; z92E-jEdT(-V9#&}AYgc59_M~DubgVo>i19)l7|iMF;_{KT-0ZJS@*{!G%CKjqzz~& zpI`MnCb<4)B&K8L)f-Tzf)oYG{{TMmGr!vrz-(m6PzsYIV4UFWa&R&+^1;rkH(hIL z#aIItJ#n@9OQw}-uWJj=jjKPjUT4+=lVgZ>wyPlyR zLF{FKjsdIG^V0!wIqKVNVgNE$3<)M@g(ja1Zl_bQQ=#{pSmq^e#iv+nGpX8WLUR`P zFr=EsJ|#*X%phs^p543r3a+}9gvme*f@BgSvHt)#9?`@tOh>2)CUApdPRlcZ3=ncZ zy1OxBFN;-HMO00W*=e3=>fcK8b1T^9Jadoeub6S$G_0F3 zbZ*jXD^aqmPRP^DO^98U*;2_$fmEzayu5!szE4k>dWTGcNHOG3d`I;-+HyFL@U;-M zA?X=|h$dtV48-?4le>-^j+$t3j-k=d)1>>1OP?IQyUT*v|{6)CWyzbnJha&eM(BJLd}~ zn-%P(e7t>L+T-|H$gqpBhi})s{{X6dS=!b0=?PsRVE+Jkex#rJ1%|>3tOEXENcw;3 zeSrjb#u}Bt_-ae1e6y1~62^%8u2Ro^Kw-U1in6A*$tTTYn!+PnA)+7#J^Z2yBjL8} z13@1t)H9hKrx^1-HYK$1MI~0GSdBzZ1_>$%KV$ar2*Kei{{SOq{{ZHOcmAJ4{{Uh9 zYxt%95B~r-fBF|1N`Kt{0OKF|creiEZq5fyxb9Y-;x%nmsGqmApD>WKY6%IVf-e$y zs9*%jMPi~J@%WD-R?b|KNi*Dtk72j;`c9H~)H<^rC%&j2Ja(ALBo7%8!3SrLAmVV>hJYc8BW+G!Erj{Q!Oc@3;4{(3_lA}1u;(x<>pc%wz-XcDr z&L_B?pujwHVSONQL~|}Ky$vf!;9}%EgK*rPYuuNP z(&QX=qwcgzFzvTSrQhky1Ht!Levu&_;347roX6+CZ<(igQQ$l=0C&JMk^~GGGXtx^ z)5OnCnxKKCf}oAT$Iy^Q$5Ubn5&_j+t%-+Uj?%2ias@gUG~H^*06FJY%t|Y;zXB*s zhqxjYtxyY&kZ@kXpw)`tYpMX%VjM8(gAOAYx5GD`*nVB#Izox58GXGQh&A20e}mCS-tp z%wU3iaT`!zK{zJ|{3j#z)h7l75yZC|;7j7p9E_~e&wX7BgKFKXzOt)ugyqs+Ai*(W z!@R`7NS}D`F*5>DsiZ(@JLBKT{{RRbMC}|#)L4+Q3*ILqIR_iW3?KKk0mBGcNVP)+ zD7@GUlq!yjGnm`n`$I5mWI~L8quyiI9lSihSc!XuivG?OW@f=6<&?7P76Mgstm+XL{TZ$xNWDR0#$(Kf#6M{xp3@>yq`)6o z&waKBJ^pWFJ8=v!C!#^x2=^b(J@)sNC*pMR7r5&4>Q;))s8tOYt_mqo(XUuMCpq>Y zz>|KvPFv#=LuF(zV=M^rEefgtU;s1p8;~dbGx~5dtCuhglQ95682P{S^B9(`yD=PR zR_%>{xvLHZtX1imEB2ByL@9cz*<$Nh4_c;DFhk-kB7&I`hPo?*0AffxObHPZFmu`` z8P5&5RmDmbO3trPs+pMreB|Vx7>SX>Q_eb6j-prF>D4MNte#VP_+^9Y+EI?prMl*O zVjd&qA!T5&u@E6(e0*c}@%R9&ln7!N1#FNo0XcwUAmH_6jDyp~qLD3g(>NfBm=Fd= z-r(Z^n2`g9Q~U`2qBQy~*WP*bh17XGr*iB18k|;^izcDf>L(bkoZef;g;rP-HaQ0! zr=Pz@s#h~&{ub_mnN9F7kbFyIoYR#up`GQ3{-4RW=O z65g#FqAO8iYPd&GwBDGX-BCgUvQttER`p9DVDs#c7+2!rAc%1nilCwhW5$0ZQ8C&_ zx8^$(e}B(!$Ic|mP{BJAB0L@Pa~|PNcFZl=0vW>`#zb!*M*bkT^qAnMlUYYIFyZhh z0#gP?KVJcZ?+lpj;(lX4h>!dp{h+=XZz(Qc=3bHGGbiSQBO~r2NZ?ww#oz_$?HHWr zGC(7~0QbSw+GZUl6qMFULkdj%pqcC`iSIGrw02}ia_tcj6Fs}cOpz;Dpa2_Lz$Az| zz4a)a#CDi0H(`a7E3<;Y4$+Of2^01)+>s?#4#x$nWB|@UF~0~IK_8zY_MuT80W&|Z zv7O=yR#Dr=5A)x*c#hvBU4Q$<)#4d{n9@J!{Kq4Wv=Z7?fI9;}uEs(7l5j?Hc%)$cB7jb0Z!5 zcA1#%@pIhQlC;>c-&cM7nC;+4zji4tRg8LIp5y=kCp%$I@vwuQpgO7y;0{BIbDyDLAe|%w z(&&|7`8U-*em%-Y7DF(@kJe|)`^*$Xe|hka$c&VZrYOQ& zc_95@?UCL$+|214ng*B*zbQS;#P|n%M3BR1j7xXfk3y_=fK>MgVGnRo5{L`9{(Hcg z#21g|JSVh&Y3=fG0ud3uLjHu4)x2i}{Unu#Ocfi+Gqit>-STI)nVtljS%d63Sd>cG zaFAhtRANWC!+!%EoqX^}D5G>Tm1E2c`U0f0EDPmbmXsbC$C6 zn;HpXwRu(8U`Y&&SZb@8N0h~?mH@jPzFR}o zCo_oF*214R_?Fl)Eoo<4)T@wJ@YX6#Rwh6M)z?lZx*X?^b-zOW2&vLelJPDjnvhcK zWad7yMg7a#z?sm>!$8b6cA=^~<+_P$O@L`ysOIG+8Ye1N<5FZYNkAMfS+ zem*fvMZ__l@wf9;#7^WHBnX09v2k)mLk>kY1{BDg36gMPG65OD9bH16weU>tN5wSr z?blmZVx6VRjt|z)B{RH3e59fR5Bih)41QzR9lVD%)+p8tH2^z^0M0b|&%Sek0B}z( zxu})WwH9T1WnE0k1zJHX?S~U0KDvkb(>vQN={6V7Kng*nRC<(zu>>kt zL8*WQ0gXyRv;P3vJ-Tyh{EOoIYE-ILVM>j|>#B=F{?d`QxN@{sp{ZK6g<7VPI(l^~ zH&>=Y)K6%=DCE@np9SOl{C?0{%h1lDcAdpE9Gg^bR!!QD5)$!{Wf!Q2w9J3*(b{CI zb*Y+FYExkkO^UGE8E{9u0UuqvabF@Ki3ikGsD^bGRv*O*ax!Ipk)1`VRKVV^P?Ur_ z%3?c;{{X8Wi17WtF+M~+`+h&(d&~&zHEd%}N&f&3sE8mI-)sis5OjhFlee)Qu{+5B z00KAS+fgb@KV>L}W(GoLJIgzG_90|PoJ@W{6CKO{00*~!ksUB*V+YzZBet!^KUt7? znw=tGvk-TYlOwdw44D`{(H&gHiI(w=o@Ce82!#>Rbl$ zDfNWPghV|2#KwQQ3BWEV`+j>wPk8SI9s5SDk%5`)0sdglV|X6>Lu_bzU?|e+G8u9J zC)ga3zsfu2V~(APWJHZoHP~!^3;oVwN@RVV{6@?G1Agy;y24>_rZ6N2#G*le?;VfCPG`K# z2~0%C`~G9(NbW$vEv^XZ5+ft!^O@d1F7fkFrZpJGol!78g93Jr{%0ePF2DOQht{Qj z9n<N_LN)x*?N6jii4&or~0Zy$TG%Um}%1u zr%-MYf2E(Jy&t9TqO7l~j%PX)I)m0-4Cb$nu_aF4KL)k;594`H_W)T%oeE|e+Z3WD zm2KOa?2=kU{td=pYys(0zFbw2Yk_p}-B6f&_*xUpJ2;T|Nj zL&`!RVqmA(MJFKweGvudGbg65Pf-PPl6HV`xjTIU z&weIGjmaWAU=thU80{GLF&qjx&pNW~+FS0gsZseX*+&|qdhD@`od`L4!c8r(>WTJG z>mETP00Z1YC*VWoA`Tdg?qlivUr*C+HEV;cfDCsBG6D808T#X5G8#LMabMqUy2X!}{P*3xK5fyHuS<-o`v04X6NnD7Z{)7F0Dw&OkV-0dU2*gRsb zWz(oI1#(p8SozN1N%5(sLeI^4S0vu3H4ZCkI+n|Gv+A$LNKlzK7MOpd%(S~hXU2Sd zW(cARC+3$F@}v?;20>T?Fe9tmVo&>96%8u2MrjZPLB>IxW`C4@PDcW*z3rTF$ef#Z z>7I>}w`_n$(*?QD67={SN~(a*c=t45$NgjEMFsLTfU}I8nV;v8{XX+8Fa+XunEH4h zF8=^d`v^L$bvL3uZ7knFK_5CpkTdhK`ZKw4GBLgnj#&nVblM4)UN573M3cx;=ix)8Lb7A zHiE~6G*p^QDQ#-aC{>n{z@{7wnej31L)r)6SBq@+)eTi$Y7eHvCJe}(B*8FZW2Q{R za9ABM3u@H6lcWJ4gV=^92VvwyHsWA)FJ~hMjn1x@+-N3(;^*6~6Q>Lz60;4kJ|in> zcRxu)$do}0{vr#U)u8VyCIcANCjgidJX zTih#Vwe1~C(2~=s4zX3y1FE4Anfs*|cowmqQ9cDQv+RshM~(1F z9k<+V97y_ISxF!;i3251aUk~Z+dJSg8Ijf~={Cnvc$WvX9;Inp*5VqxgU+p5&bQvh zyxY*YlUKge7&U#r)0l6B2}B3WV4}4>GW7z26f(|a$!I?3Z?x&(iuFomJwT*{W+HoS z0Kg-wND_1Byl^SxJzD3S^NAxvo}x4N)wrVART2i}wjkux?9mdL1$r8_%o)RE;JeJg zkH!iXoiM2*N!)E4kPHkF+>aq(h~gIj(kl|dr%|1-V2+vnS108^mG9_I&MqH9xyrdO zGP}nEONr@S!y1cCnbp|WuW!Pe!c^0$6e1P(ET0ttltCBk<}%TPTuTn8NKksGy5>Zf zGl9Q;kbX;pLYPej*_r1|$dDk3oXCTLj?y}^3+#4(^PZRf{#X9S`C`#;{{V9T0HXf@ zmjim${{ZgS^gsEy(+*#S%L|aLd|#$~%VUq}WM@qqO3jwteamu7sc$y1$jSY}`!=`) zu-KxO0KrFy)s56MC_+GQn^Z|B028D@+XZL287lE?snSW(&c|T{W9l|D6O)KAH61jL zZN$2rhs-i~uQB0#iwWB})OkwNp_as4p_yUel;~R-BNz+Fu5t=w0mrSxaolI&0-`X( z1|&hq)Ff??6u}sdm>aaf7452kB_uR~^gH$&4aA?kY2mD?n^&6ZF8%%l zZ-i+$s7x5hlebYMX9SfP_s)}rl&b}GP!y0NKI7&vPJVAV5(>lYaPC3HI*E&5a8s=n zs9C3_7{s2LK1Yyr_{{d1SYCxv zDZ)tpDHu51;&JVq0UAz`r!Ub?B@)Hhrxh$h?dt0UwasdpZX;Djdko}Mr3@0}WBtp@ zWT(H$DK?`DOG$~|0sCbBpl%6?;Ci%~SfheSILB`@fI;nv#4*nmT$91mX3Lw$EIngF zotR~X`DZS?tYwRZ&vNsh7-SH6dHD!Oe$xXbVigJSMkhVDJNT2n$BmH-bcK&%NWh); zC%BD-?qW^W_XD?)msgW1GO9Lng}h>t>_UFUmPRJa0>6Y1lrlynCMU=X%e;2+_$pKd zw8=kmvVNKKKQ?k8NaJ%t03^mhA-wk^sLYioVIamhsNT$d-0O96DE-CM*Hi7ywu;)# z4@Zs0ro%=9^0F?%(uj{Q%)uN-!%^l7Vj#w>mBuIPKZ+zA;#&@&2r;(gZI7#eHvMCy zQAWnt-hmDn?6eyNu2j&c@PM|r?ixx!@nVux%Zv+pQd*P zap16(6I+Fl0f{5DY!ANU`kWA^?lS9*-Q_%29WN@CuTisiUjBlC&9bojdrz0zmN2Kd zJJ(1I*$NUSN9`r^dwONH07;e^GmJ;ReqU(bBpwE=65gSbN$FJrLGu7df;_}dLvHt*zflFOfk`YuN%+0a!7{S{r#}WtGJ9{X>L&DVpTMPywT3EoZr5vBuo3>xHV8@sVCkbU}R(@OnjndOHW$e z3Dhe4$4pFjJH}67Vr0k*pHAiJYhOuBlB&%QFm(1J4}%fp@4;VEx6_tS9k6s4%%5JG z!tE8&x5xg<81M%|(snydwzS#dX|pjbV4Qi##Qd3!3_)f)c!3k%Gy0NF!ZmRnr8~O6 z>k88Vl15tJbsB?$dFHBe%XLd*gdz*@Ld9FRAR>nz~icCZ1nrd?|}m+i>^;d4wjcsd4n}k ze#*7&TDWeO8`W)YOVXnPbj+A}6rls)5DSTj84{0`E@@gd($xlfbGRoz!{f}2)-oHZ zd^^k5>X+hG1FxiYGk+9mfJ|;CLbvIHMqlFt^o`0oFVsE-_5T3SFS^LJ8oXCipSvEb zfH=QUF@DwDZ=N}IePSlUHfUbD-AO0U-SDaJp zL!i7{sTr0RSCWQ$a(V9+`;fAHYnV_0Um?q5dnM0M+9frn$xmp3g8u-sxBK?&RRz^dXFoy+1IdizJ|hd* zmTHvsL<6E=n27$fIn4b*2a6c$82HC_Ne6%Q2?#_->piAMdwxIcp8o**iSOP%^+{p5 z%;3SD_R?fexQ@^>#OBdh6EPz@A3fo|QU2F_iQ;CZ_rJmgP8pDah$-%&DW1S{5#O{F z&wtEMs&bKU~M%pm*C%a`gqFN9%>8#L57fAYNW!*hYUaUyjiw zF(2AH#K(UB03{m?M$#kxyXV{6iFAksj>9vy{z?A;Fl6}}Xu8z}3?iQ?GGBjUmbIyp zG3DK|CS+qJ+sE~TKOWQH#z=>#h!{LEF@U24_MBoqvDn5qAR}p=B!jkoSUaje)xEpL=hatMf!k)F$t?lMAOk68@$`er{IbIatdQiYu?Q=-Jw(XnEn1&`vVk+I7}2s{~U zPx~*Dhbk%aE^4H%qQBoP7u=E|F-_Adrm6LTZ%^K$1ZxI)UG+ zXJB#;iI#ax%1Kb3&=Ea^3O~PjA(@}g;(TZS0CN!&+I~4qi3&g)&fT+!#@_yTM^G#Q z>a9alEJU*@+5#{G83cnknHW5DwbYBXEP1cGy(Tf_uHy#M67wGM1^JllU!LU?F);W_$wqtGkapy1mXTF0Y>Kmgv z9LI-e3p$6>#NFM>^?^h6E7F3qVwiRUh=PSlt!3mf5!wPKV2B$kqRQ@#%K{*kaR)S> zCU9h_$i^eECyR}}-6ylXBH&xEK&miV=`+v0j@5p{)yEZdFjttzwe$HW}A zn(4o`ePX;dPz0bop8ypMv6fK`96^cVzfIlJwT;45OUoLMnyACnq+yv=nypu=X%eSf zveaPujU_^j$E51?AUi=EK}f+7$r{t#r?km;kI6(w;!X#*{ZagUsE`J(#e{hK`|so5 zGI)($TN4^*O#Wt^f^!}7Ir4azLHgU_16{(n_W}B$(cuv>3i64WgNTVJ?Gf0E%3>tE zLI&w3LE21r_WI*}h{5?!1yOZkWPnF({{SgjJ~ke6w-yTF5{O{QqHKNy$YgsqN_<45 zyM#e@p5x=?{=#GOKTFghq^V#qLBwt{eb0bmAjV9UEOU@BcLU$YbLVc;#h2_n>#RXP zcC#M>Fb^o6(;iVUeW7O)`6&L={{U2@zmLh<-f5^AjBVm`h9^GTk+7UX%%H_X4gSmm zM{^K7XF1Pk1kfFjN+ehWH))=W&(*4UFktXHv8gC+F&eh=~wn!puCU^&l2O2X4{hFak5% z^WV3Bnl&<~IorGg+6T=u^bzj|R)M2z>DnTU^&rAqqr=n;TOKhHDY z&fs>Q4K|jkctM>N!JJTvl z2~~60sb&4TXtaDsemQq8sFfrp!_)#m208TNq_MFdX0P$|<@-B0)QuQHaTy8&FbMv$`%e6X+ zXppUkGa^=AvtNr~VrQ{_$N7jUf`($i0H!2&+x?*t>@(mHRaeHV6(ncuV#=Xl$B zm{osss@uAj7shwF7JDkt)xS!w+Eg#KFES=E9z_DzEu@hJNO*XGc8QUJkA$U4aiv%j zKdH_~wtIa{;upG95X_`vVlf*>>pRb!Y_U9P_Vl@5yEkR>Q)D6Z>!K`MtZj%~)k>tK zi6F9)Gbn9rkD|#JD)xGC+P}&4T~*?)_!PR zur6>lZF7_oOo)X_{GjI04{}PFQ4!_%@sj=j0Pub%V#-R%FksBIjC~^=;ieaSq9k439oRbx~jP%=JJ^?3UvWaNGyJPf^C$VgQgczK0%lxAPD%VPx!MTt1zOFS)xK4w ziG{_j-nm;(Djc%b12tc=kAHC!cZmuA0Dj&wMeF@X{?CdX85>fx8T_IMk0T~ec%L$| z2M2-*fjJ@wnIjv{C+q!jsK>%M-x=pR)^oW9+O_s}t5~h}cc5nWs1m_5VhVrPG5)2@ ze&j>>@$Mz_XS%zlQTmxsfdjOCqrvvyO5iu%l_Su~G7o7!F@-zekR(rO0YZI7;cn@^ zxb)7J%yKIK0Lx8kx5`RRV$(f}uI9j$`;rs(>zR_6@a|oRd(EGhS6iA1Q!30PhXO>C z@{PUvm805paEk3TIaLIqG13@8_YsrhE4&{(5xFlU<=HqEI4lB6t=zH5w6q)vRn298 zGAXg^4^@F=b}rLCLSLUK?6F}%5Y<=IK?F*Ti3jRqZp0B8pPPXL=$QbSCU%lX(ClEy zn8?$*s;&P34Kx1$?YI8`(bfL|;e2nceE$F={#bD%pzrwf*F7pmF zI5Wuhuj1uC$Jze?&fmPVlu1w9u>zuGtmYyHCTHS`#;q1Cps{8HrvPmqF((nIvF$tL zwA#ZeFw#NXVia$PfJu^Z_Xln#`2PSb$K;o|&hBpyrN@!UGTFuPITcKm0v$rby=}VT z@QJK0H31!UACv(0?sFjp8(ZC0AhVM>P)q<1a0l1MWAdH;EUG6?K3NcF`jbCzvFWmx&mRl9oi1tmXv3 zKjc7-i5Z_V14rtaor5d^++%VwNSSX3BO8f5jtSJ{W8-q}AI9u#<>Tt!rOB@Id-U;J zpDR%omW@W(Yb_`3=(EHHtAG#8$plYtnu}P~)?gen3D1rD=V5{7dP+i7RGA?4NbQNA z8z15x(ekq8&Z_i3sJxfoW0z+ZK2=1;jYP1m)Vs)Bwb;_CcPsZrii;u|;J7~$cX^4Q zF_NB@}qn z+z_BySj{`W??q)fijxq6@^O@#pgTa!419&fMrLh9ti&DU@2C6^*vMX-suBT+kUhQf zH~oLa$zMjJws@Y^7ll#E$m8{H(bK3WW9*+|!dp<+0r>_5fg;MhM-qMrI{KQ4(l!y? zfrAA8ibv8+s+c(3rr$E-d`o98(lvP5Po^!bYBu-ZTozS3uJDzWh+=lm zL*fCK%G+kAhMyT$zhR1%0O4Q6IkKf$wic11#rtbPvZmT;l~y2^4ov*TK-7~1u=;FHH0;?Vy?Qv|CsJ5& zX|_gd0tjTlq*#k2v-MA4C^4U#{v%5TBiXYdLD-L%e83PRd}oh^v0zAJ1Y=<*JNCi% zJ`8nguJJB%IXm|F#`hel%W|qO`rpX?;@=glJX=DAYu>9?(zZ2H#edA0RUnhD0s8@G z@uqa>46~9>nC?3q92g@5Xr3b$WzL}N3ba5y0E0b4pqP+gt_CxZ#;jfq(!NcsxAH7r zSHiB|S(S7JnwoiI;@-g{pkRm1iAw{r?bk^ZPwX;&Q}bGt5v8<{aBy%(*ZB^EJ4SkU z{5@4Wht&!Kf(Y|7=6{FZ7#v%u@gANIHOximY!|0a?Ip&|@8jKhd9)hpmFgu_h@nI* zFH}h{&9NRO*nNLBr*MF2QK;$zcmV8kr^fPddzs>4ZC;oxbctPEyPW3{Fnqx>d~gWk z+?OuptqwWF0F|L`<8IWns^@iSpuIV$iRz&VB};cuLlPU=m8O1-Ffq6%VmRwR0NsV7cVkqQat%r& za92rEr0Otak=z+ymM4vx)ZqMQs(m!o;N0gLzV{R5+;p%x1|^Y`uce!Mq$0&)gM%1s zM`j_OP-}33#7A#Gl?#fsZEqH>ysb)19SFoQI1Qv8ld<3EWbt*fqe`_GrBFzgQXpUh zy0>Nq2q1z82l!7OVe4`Jv;P2r_tn*2W~y8K)9DMUnkPy2)E913ZdtvNQCcWhlve5o z5ksb<1IS;>aKphrd81aXbLkd0)|~g2y{*b(;3ut3fNHdAFaGsLG^sNf)D#2|RC=l& zjmIawTBVCqh6-hcI@FI_C{LAY1|f(6La{Y93M|?^aP%vzT`~HF>8>AAx=+a{Z}ENV zDYsh9I}3ZM3*s1H(GMc+44Duz0VtU8KL!a8$`3Z=56WwmWG0KtTi0DiomD?64b%fJ z=MW}Npaz@0o!Gimt4)AIhF_GcDA~apm4N`65)Shm9VH_IwHrSX+xIi_l9`Y16CH{p zzv2hCfBSp)iTJEq#zCIo81LRjcJf+he4ZOByg{Z;`ya|g`GR~AK4i>sX3Gxed4R~2n0-=<03ad zp-^@i;^4m1;SB8o+B-MOJfy$gAV0K3{{U$6nVJ6pfBSwA9|u-)um&;yCV#xK+GLIa zY8p(4J;cZ(z>I_Bj7&tHBXur~0e8NAXc6)fgHbjjN4yj)AeRx|A|ea%{?p&{KQhiI z=7Huuy`vsC-x1%96gxsSF(tRyfve6Y0sHMe_#55arqY9gIQ@z{_)L++cJU(YFd|=x zpC2jxN_;2AK5OeJ@j^iKl`#{fl#}w_|iXuM3kMH6`3Lhiy z9y8!FhI>lKeq_A~bsNGOjm>5imWZ=Y7xHZXS$TqsK*t>CQc7vBa zy!kHgu`JY|0~}niMl!v2U4A}TxsBZ}MP*LT+8*w{> zz82)E@kcg1!u1-TBjrQYtwa$O>C<0aStBXZUbdZe-ArU2ps0BEhxBRDj1H3YtAz19 zEYPP{8yhOPBtyn##HbF;kw=u14S^@xW?&*_KYz@Xe^V1ILle};9Ywxl*hw;RnHeL6 zeB$);E@^XeR8vZoUqv814T6XXazaxBJ5=3Wg*@LD)=*C%GVi zM#J3stW*qg%z%U^Loro}lAFk5@>#@a`X&nV$XTB>P0i z_x}J)$o$-)U=)Z1lL3UmgOUbwJ^;>4aiyyoR|2CtmkFU)u+`LIgl&b+scnjk@EPha z8>^*MYPiNngd)t?kd`bdWOPSff^)4wh9T0l7~-RJ^asd4VF{tzxt))G{0x z9*Jy)lCrD`UzL1y$kWxlx3vk*ttp7_CC35WaPGcU*))OKuE_J1!qPD_;~k{R8w9!YFZzQbU!L9j_lWsf{7q1q1OtH3 zWRE8bWP2Xc0Nt9mD9Nk<20LOt&{%lHA7UGZOOJBuPVZ~n$9rNVKuo0~!JpJgaxx2H zckv_b6DJ*}BfJoiO~D$Pp_2n3z$5G=dBNQ8902z9RFS3t`LUB8GbRo@9PDR{&UWb8 za{Ui@LS!VX6U)0Grf2j)go{JJVKNi)?fry&XW?|uNnqy&R8L9!f5?Bd*c@q6iP8gr zWMGj$P-p!9!yH@>NLrDL3o6YR?lPzO07=hy7L3VAE6cG=N9!@)^Z5M;6(n^HEJtkn z3<(1>wv;F;k^z)5I{yIrlObn22_rbb_>c7dd-sBpg8USB{{Y>7d@2wCJ8cuR{{Xr`KW6bc1I)x`@Ib-u92@sja4=FT*bt83G*8dE#$Rts(O_a)20tm*%WUwVg_;UaWNbLUacEy zEpu|=P)twFRE@$ciEi*V&O(yhc@w|J{;9vm`|6wTvX!yZpQK)vXu7I-CW`%79A6PG z0Ml-I+^MC?`nkyJ6BA0JS7R%Tsf;~VyVDt3_e$4RysFdCqySZ>TR!ay^K zzyn`apa&Ei*SY4^o#{$ZMolZFhzqDW2T4MpsMJ`yRC3h9Q(9ou;w08;kW&DKY|S7&C?HvXoirn=R;ZEK#xHhU+WURJMr zQngyd>T(-QkQ6S#XRr_s{e~o-8FH$tVS7d5LIbIY-H%CT*|K9DIz|D2M;Kf6d(U}3 z{{TGX4yRqi$2i`y@ObcyL>0RUFiT8?h-L-1mgh{#_<{gg5F;`_Ec~w9(6png{bMJn zNXA&qc>`|8$6{BV{AJFnb53t?%5>Y(VL<|e@>4M}Bug2X01zTfkOn+tZqiq8bkLPh&`2niO+yurbkJ08$Kp7XqUD^9HHdb@$| zp8g5?0JFKwgTQfi^g6Z3Mfo^ZGCFXZ7T9NrkL%3X-Us$l4eBYC)?<9lQeGX@G9%cF zylYmd3^fo5U#{N8N&uEjxzf^cY+Nq#rj%+}<4hUF3-^)`}k`xj%m=I1jIWj*&GawxBEkxBL zAW0eV^&1}qPG`ifeRW@=;m7vpYaepO$`w{Nv{dOzbgA5efrA((fjhdSWn!=*kIa-z z6v*nebU8Bw9AmV=m;xupexQTEtc=|mfM8}IA85%3{i*hVyjAkFT#c{gla^URXH_?} zmJOSF4GR9KY5xFqC|ptyP!sr{$B+z^4lXW%&_RgYgFBgx#83N2xGXr>TepxzMws7! z%>!ajR0!P8$#s2n+NWfwnrotBZCSKRW1C^pL>3|lA}7iWdwS+(C$vP5_Ah}nxg|uY zlOw24+-W4te|NTtme|5znD3t-VfFg$+lrS{X$lL1TeX>Uo8oco48BRnwevNDJ8Oy+ zsd-w?s=G3xS(BDt%CgW?6jBocGd@i%l}!^&u>`3AMwtzrq=PUpb{oh5R2xB-Dnj=l zf;9*LWOSIu>z>3%7azG_EUBM*uQ$Jkm1$!3@t#ods{JLvgtP5eY|fA>3TL~MD<+TT zC&Z+_0aC`8ELy0L#ubS)=!po_WI z0jH+9G+j@D!)vVGAC;{oF+RYF>^Tvh?dp% z1N7YEvbW``^Io9x>U^@A739rHw_TxVRu{@ne^uBg$wW&EJCTsUoSgRYnVIruXsnc! z(~p)ScEpH-9(D)G1~?A8PN6VnNdEw>=1!MgIU6 zM@`ofa&EV8?^tCj_XtT!BgR2f*pO;}Od(cYsU(149lTB>viHQEA~&BkMJq5d2VpS* zuw>3)<_Bp?_Ag5dTYoP`=P$jGn+(yN-5hHu&0ggc*Yq7_SN(k;*4P_Tiv(-5Dv^jU z5TB4UED*#&k|sgG_TR?C0$@oTYfqY1bb3tTRtAY925@AJz#Wgmu;Y$3&?J8iE2Vk=u;udn37~=fizHYX^BCo*THa)4#t~Qmsp%E{& zZQ*CmEV&19EBj-UU@%Zc5gubDF0do22XG0K5rA`p-Ud#3rwu)NbwDdsSMtZ^2Y5RG zHr#sxK2-{G9yIP>fuqBDKU2BTPUS6{nb~aFbvin~abig3SPW3uw`HDC`mYmfL&PJ< z0K#-Gr(#Gufr0_S$bw8`dE5xpIQpH%Lg5mie8dqDtDg-R+I2w(#;zecJH_}19LJ&4 z?ZMZ?;e&3)K2ynAn=$+@f|S(bx~W+LMa>d#XIeVb;6O6N>@fopB1>bWE;sq0oMJJ# z!26k=7#ga{1ZHH3#P)~_CIo)57~=8PuAA-QaKG~S^Dde4EOgf6`gwY^sVsT53Se&v zHQsdGHJ0rJ$9qD=`z3P{P;(PMCG6&anFCY3SBhu|A)7iu5CFvE zBPIwt6CbZ@09xp#j7&)2q%N=?q*x4ri$+0Kh$SnW8{D~VZHO}cQit4`>} zVeKLiQfa1j{M{sv?@n_D41JggIPb)bP_!M1GJKz~01tR4`tj7^+?R!MrqH9ubv56_ zxhiSTCCGIXvA@UNMwm64Sm~JYN4{79Yd;b4@?;!L&ydyZkd=BtuqBBD014g*9s6(R zW}bz8AfvzxV;CTAC*E>E$o}rkapU_u7q4_VqPYg1A5OT3PuWR#`5mnK-{&jNzf<^( zOVuWeiOYAm=Iz1pF+_s=iVjK#48olv#&m-Pvons^+z|!=j7rZ06)OvdF4ANcg#5C- zS=)2Ib0n=ZGo=)Gm1`8Ma;9_sQiu@g}>i6tH9Q8wi6@k8BwUq z@KRgU>xg448A8iSftipAfJlfFyy6>5)HTZ}PymP}K)~8g^X<6DVZxcrc`jCl-P|sq z*DNda?JF+D{8&?HS8#YY?9V#W4TBX{B3?^$_=yQ71`E~=+2}Ptn?0swNt2T~m>7fK zQOMybHK`Vr8bRrg%sb#o#Epk(IDp;E9wigJn*3Ne&ad-V08yt4I}!HKa1gt691FWn z(0*AXDlif5z$sC8AxkL9BlX~fV#)~+U=|)d`@s93uLRDc@NQG1dmoDH+|5K3KJZk^ zAe;A0K#8|dYLJAb;wN2#)&%L^M^M!DDTF;E zQGL6N41?Nx`52H`a-BtGMSQxrS2rr>UqRccV^srbl9F*jz5tEIG2TH$L`GJxOs0{U^N9QTKd;!C87-U=OB@*x zvF|w(Gv+>oo|^{{xnTlKVJM;^ zN?^rJY9g&7q)C|*+z$TV`au#yJ>-|emS0=kKB5EZe#}wzDe@plY=gFA@_HQLS5jD6 zW#Vj$RTAic?dp!|85Tg7Vu2}-F_Z5-raZg$@A9F{5vkOu(%&rnyY?PLi6Q~-IqE!h z{!vrZHCaxyVi__&KU9OASoagS5_It$)Xtdn{{XE#Gkd0eOXAsl(}-^&mGt=L(!upz zEg#m(jFv}FLa~&cYv?d)3}EBDM}aS=%qm$`TJ&las0fB1De?#)0thmCNcQi=wXCV$ zTN+n1!=nIX;gUL$kccYEKq^6zKG4`tvGh;%qy8B@>sPElK)bsAXXs74XRFITpEs|Y zc{Va7gzLqN4VEuo`Xo{}sR!O!fn@4Z)U{=Hsa>o-C$Z*qMUOE>{orX10HR-QbgG>u zjYA}zKp`N&1efBxd1o-?HGO*~Yq~&^Q%E%+L<*EbjcTB1SEZ@dGyzO4FdgvvaQ^@s zAEf@STB*aj%XeGpv#ZErsq1kbr%|)VtA8aYDkN65a1&LgO#*?0SYSlQ<_Ek7!Y)1$ zr*K2^tF@@D)Gexzu0$LoU{#$bRT9M9finqtwUme&`>{e2R>2$!}*`gN0k0buXFI{ zIHbH!k2E;~ZJ~kP|v{)<%83aTh)&%5%Ol=^f}^I_Ps#S&&l?C44c}IDk<2~f2I}eP%*;n=Sns)AB&&qK< zzETIiTkRq?dY04x^{UbyP5=-$p5%?=>USJtuheh&mi;pHb@%D)^*Wz9<{(D89X{#X z1C;IBwsAg+w;PY|t)6B>A?Iz~-sl%joV+9g7XQl-|MmI5@Z5E#w`m+4kz&&?i{eNumj_4*vkuJpA2o_>wpjB?TDi^iKm*Imd2LVcdvQNxkVb5_G%4AA2`zG zpR+r}4Z2ZD0t}w&9j7biJm$LBuB9n4P$ZD5v6Q7nfG*J>)^Mm$fS{?WjK%QYf#S2A z@P0GEIO=ftR}8V0UXEo7(iLmhp=nJAF#Lb5ZLpXt7o7yl4KFh6GafWclTz~OZWxzf z%2>R~Ajks*K=#|T;ajm!}#D zKq4W+rGCfQL}Dk!b0kp{BtMTLh!~FkGv^cVvPeL4GbS=tAdh_bo}T)!c;wFp>ZLFQ zF*q_JTOF9@0Mt&d{aJ>~jM)Xc09I==NyLy863G1fu!p~4m@d*o{ri8_h>7s{ZMDGG zsz=I(J=$_caVt16KcH={=$Exqr9ei?H_R4%f@Vw|Mm%FIrL2uAqzxHwnHc_N4q`{f zBe?l~dryD&j@_sG{(H~H)T?@u7cD+U-qGLc2j6Zf+{VV?T|ajUB#Z`-NrR+~vERtw zBuSG})I=nsyiedvyTr%0@i8$U+G1qCCExMmFi}6Lv`a`jl=kclWJIja*~Vl{5KjRL zbrQ2RMI~TKQJE(0(!3>!bjA_jL)8Bt+88NF$J>|5EmJxwZ3S4nSdkr5GGGCLe_4M=8hrl% zsBG#d56xAb3g;;__yd990ok5vV91}ErUi;;ln{AGauO){Pv;!H>oQ$YW9lmw3OYb1 z<{{2FSm2OFYQ7PNjl@9IF!OoVY8ytD_xC4w_>Tl4;Q95JP$m=iC zcJn$9*6#3o{XW}M1EzgQKs(=OJVlGI)e~CppW0muY zWy4d@l^sd^yFR(p2I|Un2+1LtUtfv;0B6bRROi18Jv#KxskE~qzNP}uqBU5=?rw8@;K&e@g41pi+*Kc zq1vj_MpbZXHAbkZg8(WDYwFcvl$6v+sZf?>*8YNhSp78hI25jx{a*A#m^Q>L{{Vy6 z<(2rKBVW7By*DxT8mpC@V+17)Q2ydk{l~bkt?(t_W?L%ME}{yGbwMOJAQPaFBmyCV z>gs}Ylv2k$P8;NE>uOIr#?B3_~aBgh)( zSJ!L;I23b!U8dK_=`bYyNmYf9pqLUQ@VRw6>o@4{q8awCbzkbsqdY-XguBz8 zS)HM+ZIJ~tR*gki^08pVNoS6xK`yZ3HUoQdx|z z0R9jD7@t&sT)j=q>!0|k^>W>;`gbQwjQu^LPtou5g zjnLawta4jaA2k72F!+vzGHKXTMgZ=sqLRqc3Mms&usWCkq_je|jx=6RdvbkS<5Hb6 zf+?DT>mUbHEXSuvPBkqsNC{9P)lF;Y503B-9n#*O@$Q@Sdye3A^M`TNR-={2IL#Df z=H+Hp00MwQV117@k7ikPHO_3wnH2n~-c+x2ORZK~tWdL(WB^HEgJA*`b_egnN|sTjQ%78$|7W^;Wo5AR;Mh-_km8tk-p$$k^ac~KparHrR!0raaI7Dbpq)blhQG# zBuI?0EuPWGm)=p#ceO6iEo=35t0HggFjeVGMYWBLH{3tWUuU&~l&LZxeg`r_Q3O`) zK~+EoG{A`1WK3$v{{Rn^#4jhSSXN@hlKGN;xCghK?stI_!LChvnDZnn4!Z5>?D9*h z&YSSg_SJ-tlT}z*5X8v$C9W7f;u2lthqrBoNCk$j>#+X-(BY%p%O&mx@fOll`vIKizY~kM75D0avAaR zD9SZv2LNJc894`Y5Imn7V~JAfEz{ErIg_23jpOtuCSw|AIHUDXmuThTH`eOTi*^*^ zw{oMheN4=00TpXU<2TzXkNKG<3QA%p$IgFf+MPgUOdYaEpQ-ozU`Nfyk66@gNa{6i zpv;WM4CD{faSc~m zeQIW;g9MSJW@Lg!-N$|8z+NPxwX3rFY!C)Gf>^=H!~rAx%P>$h%gWi8%XfO*1^qXB>KVzK!YoWyt(_{$Wp@aO5%2uYuN;{e8xhFQP*}D*`kFwWGUbTL*f(cX?lzc&2`jSHISON0M z?b}iDlhSi044x`%&h+)GsnlahXd(vsWal_O(Va5a9$&(FFGe{&&V8)(RN2(TcI)Um z{XN{(;cNQ+5m(qw`*lo0;u@(7RipDT9ml%5;p!FufME2D&hd$9nT(c_F{V1kD^W}_ zAjoW!)$9n7fg^l-#}sS$n-%JJPZGMC_o;v1f~?{((;G%FC8em5SSG{(l9tKxf*FuG zF3>=?;%&mhxe`e#c7cwWf=)z8+;5#x+*P}y9-TUxWJ~M>9FRulNAZ5L2OQiJ5}mEK zO0;fgGe89u=x@baL!D-|X+gq2k#rqO%K=DKQqCDJmY3(x_deRWtbmntAz%gs9A-?+ zdjd8FXFM>rzo=D10va+{mXoyPu6=;o3@MV`DjB!%uWH3wdHji(&wk_Ml6!2tZ`H7+ zMeNCs*=Gv#@u-%_g1bW35iFV0@nVax3e6j}o*UX&Y}9`!*mkv+0<#XxIboffodhefO>AVsQLzffEDf1enWzvN7jO8kbPGuPeL8ta1A`H?{5Gv0*@aEpUjlRd@hZ`SLzy9$=W3Q31PXMf4p+HS{GL0 z-55BibPZY5bwDH;(*)0AG~!_OHH)R_=Uh0K#($OL)wuT)t)+g>&cs=`JcibVi9}@B z*ED;e>-Qk5T#v%aC-OoeAWWL#fTStw1k9F#3~okn-&h;KpUMSW{Rh;Zok9!|01@2C zjDHG|pCIsX<9ug{V2+O<=Byn3rc=n8fy8O9{WfuH+X0}s(Nqtr+uLnsp-9QOYGkZn zWIRVS-q;2sNmh=Qry~=z2*~dv?=8gZ&0eGmZw)yjB*vbZ(*OWA$va5drusLko++!Z z&v-2hlA~^Y_l`%@GsG4PqfvCsM zdnpY790+dlnHo>6%oepUAD}Lo0A!f#b0TMdq>LUYT)(1v{EtI#m-QQ`oj-pT(z9P{ zn(pzgO1kTHdM6-@l(X3|=r@4{E3QFI3aiX`k(79D_N}OvDgh+v9kb&AZa4m90(SMS z-j1Ctk8*Johv0+&HjnXXfy`IBx=jk907ioTk zKq*#L*_J#Oa}nj`@Or6FLt%^!<}x-h+c*U<%!jK1X6C*JL&$N%U^z(3x4ZE6o z+I*UA_R2Eq!uG%$_H4Kko^_cGf z$CH3~T6(Fw!drTE_+=BvWow!&y_Ijg>J>Rz2E}#(2ty*Ln0gvBDlbx)J-w47EimSF zl0h4s=e$Xg`uz-v;EI%$f%M`=4kiJ?5sc(#>O{nF2h`iIvI4* zs}(P*2t4$?0S+f5Bg~X#1>5%{I3bds)=}C;tFZzXQFqq6&l~Djbbq z7N6!Ik>9omjy)4}b4D!iE)>RjgO83-u5)5kPcpAvOY>}uc7_~}?7qTFnTYS)N`4^b zlt-xS3wR@Z9Q#In=QEwS=>Gr1N0xd*ptb%tF8%6YAOZwokmznV)hw ziIRYy?-Bi{zvd<;JNEwFrX#gZvJFLAHl2u#;CG$i#tg`ao;qzB@u7Ak2+y>3QU`Id zj+uf-9lQH&Ralj@l1qjZ7qmh~XSa+DM~O`K?>)QzW4~_EKd`jG3^9#B&Yh$6-x5cH z1a%%9*wfbFk>AcljOjZmB#cbP2x#e1dh_((^w;$pKoQo?k8pl(Wq|=Yx;V7y?TmQ; z0MaLFiqUdf6FzDu6HI6J^Pc^_Gq>j)!8Cx(~b4syO@ zNTt=K3asnZP^OBEJ?bY^?Y^LPJL(yc$^QVt1E73)%)j*e*AJ+Vr)y;PSyc`j)D1g0 zh+!PKN-Da@oGaV^0JO#ZWxPSc0$#m|}JC zhOfYOptIRtR-seVS~^%O(o_HiR14|E2nxZqZe$ZYR}u7g{vg~?M0#=lC%o~rM16JD z%Q2UJ$iR0HhaLJ_lEaSOq|3P!{z#+spNz-t2Hvq&scOLQtR)5_1Y{ar0bau@fBZ(`RGwk)U^HF z163m+ZTOr#UxK{?K6fp;V65BQeasz=O!#OHLD&hTi!;~0Sd09^J7l~}2Pon{Zl zJ9ieuNq%Akq6_E8BJi(rG-rW>VJ0c9_meHiHTTWn8Rey+Op))}$;-^$eGTWZ3wB@+;qXJ~5MqLj~R zfkj%XG5kcw=6(-X;`^qhQE1Ra4;U;$;sgYz>C=M{S4fj5cGS}khEeJzON-jh=mO0( zWmXa-m1>TKi6m;MtV-aB14~o%$@Hi6_0o-Ovf_O_!wr80ea9i%0N&OaN%@niJ3tsH zY)rvNsbXFGOc2QT_@b{izq(WFTr?165b08X-8mV7KW$QUyXBmlmRZ!MDB9Cwu0uMj z6((ho2kwyLB|^y&9BmH_={~nre%TH7RK%trHNmCe4Ow>pn3(?YG5bLO0Jw}m#6@mD=sExj1Cxe$9>S9GIl(ZQm*ks`Q zckCVFe{cK_^FA}%y!ZUij1R$*sY!e{J98)T2y6Fb}-ryvW-m>;Sbat?1!lV|~CR z6B7|Q_dDbrT{fPZTh8A0jqe*XZL-;!@R9wBf73BN{{ZN}5go>6dkjSUV2eqSQYThp zd;fIRmSLBv2YJH(9fb&H{VBcE~W)O9i`oXXGbtO~#^ zB*(m{*4ct!5KoNE#Kgz;?>+wjRs6@7(XU5aEINF+6Sr*Qdx;=#-$L8I8~B?40G8E5 zM!iu?f$CE=7^4I80vM2RSO9l{umB${{{Uay3H(=-)#6*!)w`b+tfrH6>!STqsaWDx z4OOMEWrDet`7T57J^YWnOb=#i(x!A&dPG8ktkO?!bGL!y5fIJ3BClzGR+Yljrc%8u zqau(TZeWN6{M%wCNnpHV4#z+`AIfgJHh3jEt|v3t?5k{~UFDyN1edu2zaPXbMg)78 zXqlM!dh@PnN93*(fVBW#_!#T}1C9Rx_DhxBvTM4!rl2aOf;6J(R0skZa;g*sLm~z= z$Q&0k{)^~eHKo5-zgD%=-bw*oP30t|!5obGr~JVSnSlQQXz`ElF%#dv&5!Lroh&G$ zPCNekuVIWAY5JHs@A+eI*@r7;pY2{|L6*hk*vm{6({Q0chzlA}i2xbYM$Z2L4GxO( zKQHsITE3(?`dG;WtH^qZq|~)19ie6}xZ;3sc$tZw@-s92(eRtPlSHO*jGu<5RYnff`EWN!}T@=@0m5{Ri-? zlTBf5fq%^+@{Dec=+p% zKJJhn@#G*A_&I|9-TY&>d7mE99}7~-1QJO;e=oTE=eB#mR#9-Yl)j+ZRy`*e>^Gj6 z&JTP{a6wLhIj`BXjk&hq5L=lf^i0|;~W>#}O{pKW-kJGky z)DIONVW;wv5-~nJp4iU*d`}t6{wAGbr^5OvpTh>%qVGH8DvK2=wnp`=ys}f{JutEv zgxs|%Q=T#4wZXx6ISD7W4M{<BnC14$*S96-P%#GDxI2PE<@if;`R6mzbto#d*j zh*Vm=*9s#JnOl24BvrheoT+W?4(APpOhfzEsaM3LG? zuP12Y1>DyK7isBo2~0FW@7+El1PrCX`%(9lFe0t z)Bpk_5MaQFCs8XoCw&}Khh+)}Ed5_D$O_UY6xY zyB#%?HlyuD%~GPu>cse1Y@0=Oh9k2`zh3eFpRJUKQNEB#%$YbNzkoj`{oo!VxhSBM zq>PjD&L&1S9r6xjAC_0+sNGn~;kH*9_aCIk)T2*ok0OC7BO8@7*d-vAtMldqxFK)nI)Ph%tj34iNv1d#2knlEQ*~W zNyrvu?m^dGmx%zzSmTm1d>-817d6>F3pSi-`jRN!)q^We8Xlq;}Z3%lB zczbz& zKr|>#9)HGTA0tBrF4WhyvHRNap-hBWI*@8Fd<6M2SK|i*F%#Z9d`@mDT$ZFrC61iH z?o5IR$J0P+D~Bd__S1~Xf!h*6+l<`Yx4XI2%`2JZUtNm5Ue=xZYx`)v z^~Z|W>SuDSfbA0tBkw=nJfdgH^Jz%XwWCiMMFLM_wlSFZoaSSx1<~~@RUpQvl0hP4 z&Q5dH+uY9*tK>GrCuB4iZW*P1*Ja%zu(cB)*7uAQ{Xh=XL0Axl6I~*T*pPX`< zb4ryxMl~cD{o=mUiItFhOauolJ8{a%=4)^+DZ5CJ6~QV&?W3dvf-$rP!6loUkn;_0 zU9U@#P=0E;A!eR!eAuMV;gwpwcn8jndj$85cN9T*i#&Yiz+_?sz7{VpZL11ujq1); z2nQSX{WI7|Cszj^V|MHn#+DjRoh(5+m{YX-le=#*!4EI39af=&(>rEJ~W&hnFEH$n%q>P zw4G&CZ|UUrfN|Pz0SCTdp1uBQgofZf+;JSUTRzfk%@@LJSFvK;i=i)Sk}OcBe7hKV zluUM>@ep4Leq_p3eJyTvN}&rKhT*?X`8denI=sOa07q(!K+^{V#FMeke2J1ag`(c& z+}lL%Rd*1jmqy;B;Tj;-`Q)MOf|SK=%;diz}20*)lv4k2-zxAEuSm$WUga$5IK{6PyA607fVL7lR{* z9?;*-p;vO-v*}yEWU<#aLB0V5_O|Q<4AfARun0TE$jJ7RpTsp2KnF>YxbAn~ZvOz+ z6QpWa_y(=hNgrCvFZr=v&yywbP3FxO`tRnw6YZmqNXbTQUNxi=kMW!ASyL*c{mMZ{>sM z9y0{!-BQ9ks+D4&T(`kLA`isUIWX7?IRF`>R__Oo!RH)3QY1GtC01_}VAdt`o zFb-lQf>}bKYg4ESRn`NNFaU#y@2m75QG>)!8R=&pZOonB9<1B3Z0J_x74Scc)qNb> z2EvBq=r2OhgRM1i(zNwRrbyyP!UiMeX?loFO4XfoL@wa8NS@IE9n|mj;?Qm=8dQR! znuJRkh(B>Xow^@@6fiV^fG%R&tR8ox`iLtC$mYsCtzuFG+AiNq~BC zcFv;$M&tF#P-ycfrpG}lNR3cG@jw&x$?g{{0PCxDc`l7@y=^*XdzEIrYJ;6gNluec zew#P0Lf;ZrTAu|^tMT4E;9!KKQ<-*t-|dlLjfc2^y43iO0?=ev8e&lx``9p^G99= z#~T>{akE`A-c`;Q#@wf(>kBj(SCVU*ul=NZscIO2G4Yrs{{Uc-kb(VVz9Q#Pw+y2p zqpCLTkDIt1=gh`$UkcoVZ%maSin7tLog*8VoOkykK{xr=fQcco0#aoY_<M~&e0GD|xy>7{cLX1co{RH+o^X${u9C2UP+?RWx zN;|9Z5Oezequyo;B1DQq{in;f_L-j3@Ft_AJV#LQGcru?^w?*plQ6daUP7{vW)FM` z_sGG(9Yh~!CyFbzg};^$naOtidBm(Be=`H$#ygMZKOdg|0PWxP6Vk~nrM;sE$CH>o zV|?)RSCUMqGngmbL>brsB>hB5JXJOcEL+TcrW~(a$;^cRpZ`APM?)4lZ&B(vV2CM5v!ND7;p2 z5g(HBn35mQe+i%LUysarK}aZIZ6ro8exI0cv>mWW&l)w$2#EUQ^_{e)InTSIpq1 zXSZ{ah$xBeACK6JriujWAYdPV9mpg{XqmthfNES|0z%2e9K_GCG2d?a8}MZ5^j@dU z`1!ETzHD2BI!T?DLdAqEW`8q3p5iCp^*QbN?;j*(Rs_I^Iqy4T9&wH5RE<4R(zq#c zNQR1GrP#C#FaQzYWcKoKNNw-vzcb0h%&aYKHTY3qp!q^Zt5DFY3W9I}FmV!Q2BRRK`5dngFRF{GNCw0J#R$lOB786dJwC)>@L1;j&bEgY z5&llKi(FtB(VTv=H5u#F|f{&#$fdqtM<5lAz^Dzbaj{V26 ze|e7M9|n42s01FNzH$t0F&+-Xd6BmVhKJWZa_=!Q5fR=Xi>GcCWR~B&nD5(3CUG(hXAumu)}q?Y zLdQ|T3JH%BIFpEgzuq|OHy(YPEICI9IxjO)$P2){L&LOq ziFrq2iISf4F(VNl_V<|Y^DORUl{1Z5F@fwOwsH2s94{_+B!LmUNx!vs*Gm03F4pp; zBc@419nKH7ar6KKGXe^RPmJDUmX<<_6&^6&{inzzVmyQ+#LKxvPiXO<{mgcl_#@G# zPf7t2Pe{@~MNx>3@&;#a9;Y#7j-pA}U~M_X_uIE)5;U>AV%LwN79Kvn(<~zJml*aa zF4GeozZ3ZHGZ7uT7x($1+o*z|Kp) zPna9eO!?adLIH5nk?ka8F%uvl9jCN^tF%l^%tUvN{iAbCj=yj_7zY^J2LfTEcsSx_ zxzq>%6Bq_}@$)o6{+-N~JMoRQ>cA2AisuV3>=8et$7quN^X3vikJbu*6Z!q7GPHv( zqD;xe0rVhwAE#(DtA&-^pg9ZPW;2`*f4}J&D`affN~SA5P2(~n5K;IU{_=4>yNtv{ zk8k<~jx{Yk54jr)G%buNR0~=p1TME@&I+{8=CrMn z$${~CN|Tx?P?m@yB19Yp)2MDc$&UCW@yWUVi|dC9GKDbR*bRpLi%c-_A^~N~XQFuY zsd4=qRl>TpnWX#n5Elzt$N+dLu3MGbwK=kJ5V#)F69gTn%42a^sDm9Oi4)t%i6&wJ zVsWM*j;<8Z<$|b_w}`+U!OD^#nH%F_o;%rnAHshk(`nPco*J1aR+2_hUZXb@h*Es4 zKpd*Pud$L8$Z0e4F&)2emzq>n0wJX8Dsi71k@}yZ&Z!3}qzMXGf;7h2{vdjWBNGG3 zf)4-=sQn)GPgCx5FsjB|sM9V*KYe8~`&f%$!$A%8flLvXoB;VSmT}@AiIKp+h6Rg3 z_S})Qmcb;R+jV$64gHl-YZ%A?z?~!>-^KSPM|kbPo0D``tej6Kj}KM_?fP{9HSa$H zXGb!)_$I)U8gK)}ezGFRJA=2<$Sw4N1`YW^2#dfJl4KFvyZ=A~Cv26)&ukcD-73_Wa9 zSRp0j5!=dV!~|p33?NhlOHaV?S`%5t5maT%?L2fG{Lp!4eUV8Ep~$Rc2U*&SoI}pyMCN zk|%{1TAgZOhG`fZ&IumV{It&FkB+%k)MdTyLgXD@F~}o6&aIVKN*nUU{^$twl2g{Fd_>Zge8!+&paBxk^o79)sp zy>5AxJ2%M}_>FvZ)4Eqw)!dy39`s2f!;*jrm9g*aU5JJ|$A88UMMnl^NU(=-&bJ3x0&44g)(UJEw{{Xm0xQo0KJ4rx9 z6pZ$ft<_{*afQhTw^mQo>iTUkW_WMumQbrO3jY8(le8S2`v5lwr&GE&(B4mnvn5M1 zF~ZF|cRRYm^Q+mUnTQ)T?b|V~MP02T2oA#}U4yj5$t1hCr(6+CN{q2~fKPazmpA|t zGd=)eZx+jz8g&2wGn0Y@WMm1J+cF8$#jBQU^L*LUr~J?>FtfDM`ypk1>ot8c2VIGg zG_*WB%rsgZ;9^%1-cDA~u}@VQ6R%ptV`k3bhk3}Eo##rH_b1b0z=Z&fO|bf8nV7^b z_~8xX)afg0S6lA~6D*ju*?ZS9_a)}B7=RIiV+pmb_7$2*bl8$iayYQ&b}`_~>7M<<7Jo>|6_#v*a0^Iysp_I6AK zgJ;&y-G8J>l4G>{3(K_rN8nafDw}~o~`<-XJ~+%)TABVlN6)rB&KudZTPi00v1Bylg~p z6H%jHsn8L?!#)7UAY(8@b~_0?MR9Hw)Q(^_HMq_yx2jjB*wk1FE^-%|viseMQ#=_p zE2=_F6a>ckL;z25?4|h&K9fuLgk*(*E881*ZQ6FslDgEg2)>}u5HM2*QIcd}urU}} z^BRnG3rf}W=(P27ZP$u+n4z;n7FX*k!c$$J^9Al$_bD$a36Br%1wW6Gjz-K)Ri@CR zs6m4nCOg6G14aoaNmdf2r83~C5~WGi8Qc4?eDC!in~tV_w<9!M(Sr?)cesou!cA5J zD>h=pIAgbnu*N0^L%_rJ{YGL(b7(;5QUKN%9D9EO*pnZoKn3;ysniCmGk7$V2v+&;F8%db@p4?29xU3{2Q zc{FqC7qDUbZe+QKT;*!4EhrIGoV11_N`5%0cQ6S7ixOiI^@7{rZ8;G!#iSZlfn+UB zus~ym#9;0Y*pPeloN&YJa84`J3S2uK9H{r^<^7W7d1%sQjyj7~sw~DcbXZIqK5mE> zH9dfRV5sdeA@U=B5mXYy20o#jX*r)8&PMs~#5YB)X``_Y&pOP(=|4=!j7iLZPW(b- zbK8u_xq+3&_^O|GIh8G2)Ivt(R8yNLTT?2~S%RG_64irA!4j|t@$cNAV*b#hS)u?D z1`fk%kIe_XgWQq8BJz{Zq$P$)lhQlFh?DIm2GO1@8*!LeoM# zWvAkB)i!2*pfLtT<0IR*pUJjOX)ezycN_jiFk#1T3`?hjlGI$5urD09fh=aCmHfpRS&NOsZ2h zC6;o|)oC?4POuAE4T+GS2pIC2nU39r3VeJ@OXioAO=xC~OO03n_kcaN-0g{xr>^`H zaJB4Ssf|LrfS|#GLCi)>;6cP~JYIKslPf%D(3Xjc;d3lF03#pVyd${&?3IL1~n16@;k=U#fsMU89*>kPUKEEAd@|} z3oSbc(;SXUEb^>m3UNsnu%uvCBOYC)u@WUdG1^8Zd(Us}-abL-$uh(VpE!@}+dMd1 z!W0rj$5af?dxwnY03R}7aaeBffmj;$P4g4qxs;N`3V$&keWE`x{fEcAPxqhi^Wi`O zNg$Kn5A>Cf)cv%LSr?`Rw&So82=Zq!p7MTJEjp;1HqG+a7>AKbNfeK7!N-3t^Zn)` zB4fy4f9;8w`HxCuW^wso81cB1>_m6~o+E3*a9#w%5Fm_fInyqR4q=HGA_xcza#yssjPjTJ?%`+QNg76dW!cy96CV2Twi#=rkL*KDX zPlxT=3-S8w&RF0W!QKgs<_J0SGae#Ng_($xf%1sk`ACtB{V@{O-bKXOpP0APO^mLwBtAXjSrhnzF2gEYMF+dtRg$GPCHB=>mQF8{{T^V{?R`osX|T(l5k8IjfY}0 zvFamYPW%;Ve(6vvkKHJZCo%8bfg4DXBxeu--~Rv*zN_Znu}Y60Jv`_FP{`?xX(5O& z>>`H@GGqIG!c*J7Q2X7tDKEp5v64k$I%J^eqAAIOj5J{w6>VYEzX$s%%k znEr<(PCi~0RlLX%0gqv}OicFQV0!{KD`(59sYsk0 z&u=c_1KegOzxEmQDcDY*eZ2nw)QIhq4G_IlU_q82D9FadoQeGpoC0Fzp0M%-j1gRP zj7*P$dF?)3iUvPokuLuLtVBFVK#LU3QZ=VWD7$!*r z*dL^*5+@=~1cQOV)h`ub0!+%0sF>}OAjjqYu>|lm<(+ruSAr<rx2D=GZ^_|qB~ZzmZTIr>fpBg~lqjy_A8WD?A)n1W%*+?}9! z^F5YWuy%KPvB>THzK2lxoylRxl?>N1fS4ERN+ss0Mr)CX5r#0lVkRKt@gE}mJvM^M z8|$YB$iXHDlb9bVCyBqc){y~mqY@aw!IPx1Q;FC??nDW5x+lPS(yA-(G&RDQoJrdWO;~tT+?ic%y!5T5FN|dzsEXr|!faFnNR4*~+X)1dpkR z+a0ibnHXlmze*Wo&qmC{7-K6Vf3S$#0TRK9?AA}9XT)Sj@_(%Mi0n9EViaI_+vx** zk7)8_>Cn`J1&K4cB0%}pSu}bDvQmKfD5$!S^$iF?i5yy=G z0PbV44jhM77$j|v5YavM}yqq z{Jk>~BGLAofcQ*LtO^Pvxde}tOm_bOSBy`8_ws>}h|*#_{UiEF$@4lvfG0YqOOQw? zgn6dVqkXQ`gb8zLml!y^Y57d03#US&56iija0?c^mq(N9EWcK{1pB>kd$_r zng0NI{Qf({M5dZd<~xZ#(V0JAAns2SrTLhgN6bVq&gMRl44qq@iBnz|ED9BY2)P6% zClDnC=_sGbcJY{oH}>v7+x^6#W=ei-0et;3axwMy@8e(s!F6PmHbGFWHKq!{P^Ewq+A*k1nBPoy z4Zj@q^*wiXB+C-IhrI4cAVxnnGXsu#T%zw4>I2VMs`6Z&Y8pIGEN)b*Up~z@SM*o3 z#`xJ{Sf4Mtzid_@!%+HS24X$|P&qPJc>sej226;ZyiVj0F{)@)`CCeWH#05nN6HRP z%^OD${{T;%FPF2Lm1}yudsR*TQMuMR6=l`Hvv&#{Ra>59M8hkKHPs7D5%Yx+kVHgJ z%_-;=mqJFJfitlD(js>;2i|eU;|f?Zu*e^6fjQgd+A9-q|RtKmOV{FfPh0+9VIrEtzx*WHUaqU{M#d*D4 z9jsQ=t9JF&8%R{SR@RADY5wc6i2!|q_yGYZQKf`W4Q)VTZe8i#rM1dOO!NFY*0#GRWa!+_Rlk52&)1 zCJwB$g>oeTY>zmP9_N9~tJ93B0OCP8IqfEP=~4B}jsdA#2g~e6dqClDq zxBJXQ6vvwPnvPmGMbyLljKxyfo#r(Y~9I?fH7uadsY4vL@^88IJlLeuh^ z%l!Dqi9rkr7PfSiE8-#s@rK#}Iodl+_?cO*YAaCFom!-f#E*8;a3sVS`E%2&J9)m| zm0a#fv2D=3RJ%-YfSme z)X&mDVoGKsz=E|+HOnf1z!In0VozZ);0^bUO(j}{ApwCTg-}k}+{lB0+X6`b6UIyS zyZ->pbAS37{{YyZ8~XKsi-A2$uL=JE$`A76*75_W74coFtyD6d^@R4jOoOE zPA9M&cES9V>Qld=$}O+lZAN7$?6%#pt3Bex8|^jhND)M>iiV2I@G;-WXW%utrQJ~r zQJJz~-I_+&lG}S=cLsQYt1#FooQ)s>^??F$*csk%c%HAx`imJKYnJBZq3oZty=a$4 zr?3J*n_#P*bGJ-jJ2V_*1=>7>qB}`e+}gTR3sHa|z#xD}XpjNOI(F^3k~sMs)nb)1 zsx!RD0O_31c{%-K z$IGz1x8{{84x>v;_CM6j!JKS!6rFb<)c+sH562Oi(b*SgM9SSEoSlr4omF(4tjOM* z)Sb*j?#kYKWn`3f8M!DcD`ZB_-s}6j-~adLeO~YP`}KT2AJ5ZT(W#Wz?{`B2I!38x zF1pAySt_;Go!+?iJzuRao8SF}deEllU!`(H4FkWM$7az8Mjc*bI*!w1yGG+1hIW|3 z{?OND7d_T#{Q=4(1R-e4@05BoR>eP z5sx2Q(lBm55r&5T2PmbvSPAZkUHwvDdIe*yps0!Ep1{AGzFPd{Lb_(W175)nbcFz~nTojXLYC=^Y%fWUbF7fDFK5Ta1!O~}WN=S=k42ziV9uQ=AT>;{9L<2{O{SBz9pQ8C)bBUY~8Q4=>@G=ihU)7L?6+qKFy@~^uV-^n!JkzB%# zGI)=_8~zUP^8WS*b7Qw`R!b5+`U#!)a8zIT24)w}`B>6KHqALl1NfjcMGnja!qJ2GP)F22ff?gy>WaU%9b&y8*-hc+-827s4<_dyl@)OOnF5{Y-?XKiBCHl%ueBiQWrC5lK zr5NfS(`ux&af~vz>5f1xGg$NiX#dbf1v&s$~6XtkTF5 z+MZ!8W7R4jwMMUTT720*T^R;A*uVR@QA^Mx4xQRQX&9Fg$Es`t*2EzF*fo`jNTPv9 z+=H%LN1MW`y24&ztyxI<>;DAUmK;n6D`d)hqTMFWDppI9APo+N_dmX~gowwy$GXZ@ z6m&N^XU8;jVaGK~CeZyydrfQ2N@^}9_^VmR3M#+zZD%rfn%K)@-3EUA_^AK(J7qk+ zcib`(hi7efa?2k!K2SEg!Ldv`rRBI%^ch}|!iL{EKbo$g6zQzTulpp?UGy7!PA_L_ ze%p^;I&Z$SFPVyUTwjssko;{SW1_b>p*Cb+h&|E{cf_REm>-K{u&VoqCs(z#)Imnk zSa!r5zfYQ z?!c{a_bT168XUc4SPyDw>&DW3HvSbMQM>RF!NVcX2lhnmsE>&p3VTW6lN?N^@u3pX zgXp&PIWMXakZpS$t4NFf^5Rnwv=b>-FBTvYN3{u)ezbeznH`yfZ7M{p-e3W}qO6?5 zKw1#~m!j%)>S5R_kMmO&25NsPy?CfkWZ=^>kgLeg{`cxyTt-H3*xqzoV%UEGEaf3} zaW-VTC3ukMlLP_SqY~!Ij*c#neYbmtr(KE3$^6oj@zMMWNC){UH}4KkOUdiNrMxH9 zaA^|(WuTeKwYKyGpHF8CsNXd(QFgH5E_P!j-6qM|u>5dMqY^pNMS z(zVl#=A};@R+%PGLwD@{$jixA_O^*C70)fwAVBOk_sK$Rf{3l-^CLW^4}cWVp@mZ) zOyNjrcZ;0S_6p4lnB1eBhRzMcO+ z0E|{}y4`7uRu-n`7$7DMQ}JNpSPc&lj(jwxrtS>o2CN7xAB2Vvc#hSMAonBv%N`Xd zQ9KhGs5xn>tYJ#mT#@zx7LLVS{#5Q=Aw9;7umR8%84b}>{$$jx{sihi1&VLO_MCYl z)S6v195w@$QL5*S4`taj{RBnnNk`{lkXJ(X1+jLH_2-k8n(V5NgqwaQt|K;CbffA4 z1suh^z2`?m&KM;d`C@@zD?ww!`?`g2{&%d>4Hdy*wLqKu!j${_L*+dC{5|DXCReqT z^PvkliYuIzOE^+ODH_q&c`OR*?8l5f$^`IqAftwhC z4T@@s(~0d-WfNvYNSko*ub|tG1WKeWQT#g`+22&vgn;}_Q5vHYZUBaCYS8lG9F z%pa$fP^HR3qxq@s)6#7S<*OvI#4%+%XFDI^wv^i`;HqPIxJ&_Ax+6^m<*1iM%*YmE zS?d_%Zjc5A0JhkJUm`F3r^MoZtyf=d?Yh%+Px=Xjc4723Q^;a_i`W77nP=kd<%usEN&{YCT*4XLF!{P^n$tj87m3NdpJel)q%bpT&-$GEUy{%jw z3CqWL@l6Dq2xysOZf|j}*+pw<5EYLZTm@hcN0IJ)-1>jHZ1O}}3e>j!Z=stzf&Da1 zOySp;rZbPoiO0qrny&S-Xy3x?^)&8nq-n4WrhE!`t+y zSCpIN*T{PxYx`xy6RP=!T$-HMf}K(r4(dwnC#pAn)2)QUPrECixTnjm- zrKu7Nke5UbW>C(<`i|+yndlm6CSA$|?>jRHL(VWOwhuhZ%>D9~qo-NwjQb=bCg|GV z=SHo1Jo;|0M8lTpaopLi$M>;!02dBAwpGUOc_!kRlf*g@bL)DEXku7NQ6UN|KSQm* z_-VRLyOx^Az3N<&u-}zHUA?RK`>d^^9uYQlqvU=Hklza?$r)N1xu2Tp$V<)FaU<(@ zUKoqJS!TDXS`{@wtK>$a7aic}ouZoR?c97T$5ALNME#`LdrA$|(i;x&kpE*f6DapG z|3&iO@sY7z>{XeTE{XSM56$cT1blXcO5TfuE&%g;P{DoQ>Ycfwqcf6u^?&HFgS_K& z_>0YMLa23!j{SHV4`E{b;0$XT$%}JXD!4N(-`w+VXb3KS>T8y=Lqov3<(B-3>QG!g zD`?%2zO+E2iy#O~s55@tp?F}wnTzh6IAR5mG%3lnQLj#O4q$)p%Lxbu5xwCNz~>`Y zbu@{-FI>VGHY`bQBP+gF=Nw!t{`{co;-6-!xEE!NieQ`22SsvIocag_>459wPEg&BS%fl4vu9D@ zF2~p4SYLLc*R;H)g1;~M$@@}9g(tJ;4c%p-EoJN4aWud=1Zr{}r+FlBkyE^KiKFQH z1UfI^Nz{{oI8*?VPs)#fJ_AgrvTE*GPhM`C&mLIYuF7!se4vO7iQin>&{`GY9XH!l zaw7$OaA?)I_7HdNq4V$?*7TR8$ZZVUY*~;fF0y1)B$`^s0(&(Q8lQ7ZoJ90|^FXho zoGx^tr%RaIGHXQ9yS|(8bgTMny@IZ6-LcXJ4) z=LVOpz0oYy^C^fwl)j^%W;SHQU|-;Pk;&}+J7Jnn?Dj(=rIYsG@m3AXjS8|nkIZ8DrM#S4KEH4II@?wF97e8) z2CC~&)R-cB`yA{Z?UfPX8h~1_(RKj}%^eMx$-I3HMU?_pY(1e?PPJ28Q=&fgCkD(Z z{a!q2D{YBHn~NS>=O@d)aN=wV7%xf-?T!7gmMu2%`Xgz)FNVlUKfq!vZ8SV6TPUMFgd~UC3G>eiiTbg^D{(#|4Pm;6H$m`?2b|m_W5(AG~~|rN#eOkC*1B)7s66 zEiM_E2(HgC(}0PZuk!X?17@=M16|0Yi$Dthlc6(uB2<4?ZE%sJX=R~R28Gh|^%DrU z`2o8WOgjyJPremrtvlS`P}1VYm{a0Yal5}cY%KLSHkB2!#-5A?Z?VCoH9=9bcqU

hQpmLmy|&86dAwNsj{>TzC7Vn~@+YAM ztVqFXK0;g2e*iIYi{GYN@@!I0U(NSI<*c5k&=LTlHSK zE1-0dzI$mmy%^xDTQgE(x#8tht3u z#IG?wyQ*T>I|*Seof5ps)^{_3Z_WPcWm4AEGHn{yX=g4KmycYyOH*|ITC1gBL4QFe zW31H#egc~MZGNWmT$#spM4=()KS3vhN~mp+wtn4-zi@(TZF6!@u!UCP0MGc~XDX%1 zvsV=G>n+khOI1$Xpcel`P6FeFm}xN2Nx3yUKvn)}+={LOjWjBbKI6gdzfD%K!PE#s}z%MN-_HZ$*PN+0AWieiT_dkBotG8^;vL(hgK3iIBbpc%sOUPnSiW|tCgm#9p^a_3{xJubf|2+K`UMFol32; zr#!^l@9l?;l3c)&z`wSl>PaEGlNH<(!3v~-p$gaEpK}v)?cNy(j14kk_D9k8?|g5X z*CTND^N1a8XW!UWr9Tcod~WsPQJFIQ!Ma3ZNrfx@c|jaO0d5;EXNEIJRa?-3r*uhE>#v%%|7-zx4OjY-qiKk zX*tgOu9-sKyYYnl$?rI6jBb9G+c`g7CM)GLQus94$oDFp-q4Drm|I*0fm%4B3cKq@ zli!dVJ0)K4oAfsQCL5KUJ(b3Q3M=F!YyH_r(&Sv59&22Pwc#&gnsp(646VMj9r#So z@qnq)FeoybeKo}XXehBjAoG2bJ=+U^vQM?!c#r4=qd*QMKn2=P|Ju(KSK*i97x432#I>t&bu zL!ri}LRwF9wnmf%vn#O5&Y!2MBBI!mw@9}Z?MAcjC5skMufkLeJ??Bi1mB*OSLz{110!~d z`-n^obe_}elQa&^#siJ?X>f*QHmtD1^t9kzu1l6`rSK9_*M-L8&6l{My5;dX7p-X$ z;jgEHq@`XTs51ZR7VRS9cGFsrKrw$&@AqXPq!YxszkgLxgls8Yfsj z@X}1^1&0zn|MCWXmYLKA^yB%5enigqRNvAvV4M>z^%PwlI=I@7Pmau1ooLk=T5Tf^ zrF&rr++dS+mr!Z#fFBrfrpAvC*N|t8HK7fcT5Xi7|UVnOTEBTsWpgef@b~jAy79k23 znb;eUwR>N%{*WMu1=wG(oVN}JUtQhpMm=@{)Fdt-{{t8@kW;os_>|9OU^OKgIcio$ z%L0=$A<5cH<|5PWftbtRXW@CM@b-?ftFAtM?C9h6rSxL#9f zPA&*EQ6L@mhV~vTsC7ikQ>w6?dAD903GjN;R`&c^=zB0_to}DSjo4d53a?i2UX&K5 z1>P6CftZD&Yp}4G{*X7M=xp9t|E=}gpx8(Ha0`lWffB*G!niXVe3Y=W=XLseF(h(&D)>-A zfPQ9MwSX{GPcB%TA(I7!V}I=X)xgBRBn3kY!rA6J=JSL^RevbIsi(++O=rCejQ2cB zg;xNFz4bJNkuM>aHkFFXao^PoPOzemuFeV50eE)K;@kT#Tb!=v6{2sdbg`((F*{Po zF8!L-hg(9z-U zv&VW_#vVr|PYfO3I-UXXzVX>F|E0Z;Zuv45GSnP+Bz}0S)sJ(?ov23S8w$t<?{LP0yO_6cFM>IExmDXEL}1FIVi07Nce!FLnSFCx-rN~(&!la8fgk)2 zupf4xR+YSXl8pn>09k4k2#Ez+Kld>SvQ~LM*}N2gQ~XD47SX+vuA0pnN{j{QOhM@> zzJk<_-eiL2wF>FvP@{`$n^(=z1m+^RdzS$0bcz$@fPN zImsnzbYQ{UPVk`pR#E^5`LSN`uetFP*SBDG#+VcB113t4(m7FZYZmE;+nTP`DlhY1 z487RZp)3O1JZsM9cnjrUGe&2U{AcMqw>BMoN)XAbH?zjT<_8kW&mqr`>9^ooMspY1 z2%-DaZQ(@@NX<32qapY03*l)c+j2l;s_x4PXmE${N%a2H?c9T*$kCCGn!quJcNMpC z!#L=S|2!o!9Z~k^ZIq~EfFL(Ed!p%w<&`KZY5RofRZ;X5OJe|M#(XW5kU$I>}tdLRRcs13M{vHY6pd z*4y{BC%GW%ad7-#2$=QmtV(j{%OF#Z$m!b14Nw{@OS9z3(-7wC@TI4PAkmwQh(*>M zFCw~6AF0!wgC4YV9QySC#mVhmerN4FN-N)-%r9*RyjgPe29=rg+t)f1S zzB&oG$Aogw5EA>-R+rUO@*kk1IR)z+eE=1#o_?h?JFg*1A~D28b`xFnQ3SE_w8(+38iS@g^pS;nvfVhUKLYzC51t&tA(4!OEYBnmqie|h|9}IZ#R6kI)b&`Ai7VPoX?cNPiQgV!5&;*PfsP$t%*@(yn zmF|-cUgu2~;89t*SKkjPfAHm;Fevcb)6$@{4Wj;g4x* zPP6yPO7o@K{f+hSYu=4JUCEi8>w7x1fu3vnzRwdk29&@<3~ZyJB*7vyyMZ}}P|7|V zl!f<*=KS-ve7@&rYZZ<@I6UxC+2*2IP38FSm*$Y7Y4=KJ(v>D3iJ%PG%~G2e#$yY^ zENpie#1r4cBD8QznMf@boDj656P$wOV>2kt)xDa84E3^uIO^-pYH9FiQF=4la zgmf3KTkAibr~$4yj|ta8m>ldQW_?!V(E;gHYs|e8lv;@_lb}SN{{XjV-P-gx-j6D9 zlG?^&?tb^q<_Qw)y9si7rM=0j#Ub(!%%66^4(5!*>8nR(bY!<2W}RR4e)KoqROBuE zwV-qqgQzf=n6RT*ki|6>s=sg^Rw>0uNH4b^ezId(?5w9J=;3M>japB5?^L#FDgV6<1UEX&v8KIitN+^0heg${98pZL%6V@Q$L?mI(Hc_NF zs?-TSz}jVA`#PFh$q^ji`p`_iiGTDbx^!ivX8u_=*O3x7LLWFY z2TvlbiN;%IzrI2Xt-(90hjIwHy$K#3=PYl;nRL` z$?O=uY7>6>jxd`OysdZfe^3IeGkaC!y-yVhxeI21UZ;E%AJGr}yUNe$FTsU^yjI?p=5 zJ-zE)Dv!q~nWGPA_W0kDO%1%UgpA4BObsG5t24B8vn9n0pK_h%)@80gCSh;k(Wgq& z^lHOOsAZ=xrSS>vO2kH?Jkv{lh>9wzgQNVjCiuJcPa=D5(w;aMD+Z;N+c55-`k9{| z6N91m%Y_8ZGcp3cTMBlDn;c%wG3{Fwz@O6-XecuFxuHHlVS;Z*im${0vVu(X?|A=1Xzle@Lr3ERgAq4K5`-WYz3OqYv1DmS;4a(gv&n8p$| zoNJ%Tmz11+h$rj_l|Pe9(P75YqSy_#Nr^ikv4Xg&e@0^ktSr%x@HZI3*fd|rF8|9U ziK?qM;#NBDe7XiVrGAOfxNInyzA~&^C8n9dL*DeOxlmL`73(j&8kIctEo*6|#kv4- zxaK;>l0Ofytq-Ea^@zD^own1>);S~$T=*iZ>*LqMBiFxQ7U3nejpnGIAyIrjh*FB} z27f;-^{qZb6pn%IavMF1H0YPA%$$Qno}j@9V8)W$l$>&}ph%%6AX*Xsx<6rBuYaPa zP~gmy>FdkKl|GE0!EdB~?cZMgAQi8mutA(gtBE|aC2G=V2PT3ETY^Gg>g_csGfbV- z5$E5tYWG7Y1+vXguGA{}YcHwqsH#im?c9Ug3n2Nd0(K<5epi_U#SN6zy>NDvO0AZD z|IC?D7Vb7$(}z6(be5cj1(VL_(!6XRTYq-IOY<&IIPNZjIaJeutUq?ot~l%G1Qpr@ zsaYE9K(DH4&BtYQc21V_J~_M7=TqWq zu^LjKUcymGGBx!^uObCt2Az+R_B*}Wn02Lepro16s4&5^DRdX0>Z_39BvP@q%zsZY zbx#)Nyy|${+N`QD$ue}=frTtBncb|O94N!ADOMW*^H%N)``GxzSHjx6rl;!C00&$J zN0@qEpTGHJKp1xG`A1=51iCkCb!DOD+e~aq`QDXrSJP0~er1WOv&mD1hQjTGD-+GW zcAOZ!NMgFPLRR-qDEYzuTT|u-dODzf{u`}?T3nLJRbS5Y+~2ubGmfYNZxf-4?jpOLr*^;@}?m8&O~IP%|iV(qaqXr zvD#jnoJkARbCpyib zKPEI->=oPR^_{7*(1gSm=vYq>3xctVxtcRia(vl!qcmB?W&q4|irUWB$Muu3Y3Y?3 zop@;vmXQ+t#D9Q1)|SFS25bly_xGTQ1t!L?snGAl${q!T*i&1PMO3IQ;6wZ&X46GUpV8%AIi^ zC3hXtrt3Nb_@!&)B5McF4$#kuELHHQw`cvnO3GkE7Gzx7nScDPG%b^OOWQ>K7w*8< zI`H&CECj_gUziVDQ2+Vh+ShwVx@D?k($aZyqu|UTJaHoc>pXlQD{ZXh8@P#NN_?ju z`ilK$#jmlSGx`*Y(rm%q5j3TNI{wpl8E`kJh3ZvlGVQ-RszaWMy-wN54L^eR3F@7I z2s;FbC+zdteD#V@=4!65R$1|YsbYoCIzI+HrROhbSk}+W>n=M8()^YE4YVgBDgFAG zvd&)DZW>fV95Ns# zR?eX$6aM+E+m%W~%k}&aHGmWvVQ5be3zcb*`aSjng z$1mw9A(Xs<+g>YOlJbutyrng?JOqqbVal=EJ0}X51Pb-)O}*cw%epTB4F~t2RgjCo ztYhrFS(&doFuntJs&4p1cWY$9Tj|krFVKDbS&fxhUiQ;~WaW>ma2rW7D8V0cLz`U|CBGBIyI=@Dq5!CMC7~d z!k=x;9b%L04dTKmeQiieQ17*9?k8Q_89@?)Ek+XCFIEFnxE&LK0&^=E)6+=S+s_|n zyQfBX_9jfcxn-93ln5m71rT)A!@6l_we(cCivVwTjuUov0pD;rteQI8!4X=e)B8PW z;}7em5zo=5q2boA4w0FX-V&ovBc-_p25LgvUg_uw165c za7+wy%Q=!jZs@@HLR@6HpP%T@9SANiwzIh{`tElMrFPs0SQ~}-nGy`F?O{yJ}vMFzF&yhr-v^1M0v2APrK=L9_>3WNm14M zS{tLqN%R!b0CU=dZgM7298s8r%V)AbG}^WbYOotn{hgXE0ca%LByR(~yBVy*-=uOT zSRo+sVu3Y?N<`~g`%Mv@ykJeHp_i*{!u)4(Yz9ur$JoMf4>d?QWRrToHluSJA5q>jKv+_hhJ-QNV?m>UyO7->)=1kqHAd?%Bg4H;a!|V zCAUXCS%W{IHGbr^MIDY8k-5Sntct}dV90E%Q$pHp>O)S19m2`&#=Z0QWx?XPS7cWP zfDZB_C!W8}VC{ff)BpGTdsgak)G3J`B&H+RGidMLtnfkN6zGf?CMwn*p3NaqutoR} zFmiC2`6-Q4E$JRhfqFcKQqEM3778eKHm_jW?>kXi{O-ke8rQo8M43#9itWu%ro|Bk zM~E5hZajBrO^SuEPNyih1t_nCE-C>7<;}OfH?%7r@M#T;gGKp{lha?$VaZd}4#5UT ziQtnGD4oi@16{ZKNZZ|TJ{qGgn%xeJRMsw?@MXZJt?5g*@4KGZv5L<(jY&7Gqc3f{ zLCeR4+bhZoc6N~B`XfSOZ+Zt84xaBN4LaCq)J<5$3^$kPXUa>XC@8RO!FvE|>45mf zllIl1b`K}Jq^&#m15#eA5&NWmsPK0MtJUNMA8xUoV53eLe_0uNcy6F4mfW|5<)`GQ zDz!~$?Gt7Z%>s}Fa(3zCLYGjLm`j~Li_XxI>vCLasiHQu;vhbqKo+mv#G5y>h0aNS z;NzCUj?&$DI61QpNvR5Y*-=GZSFdI`n(_sBJ~D%Y;k0Dt^(pOBLCV|;Y0H0KOcTMp0dngp*=9c~pX&AL^j!giUrb*zJf(|WdKnh#WhOO#iU_~_ zo$dR9Gm^9`##ZSfD-Knyq}jw@*k|qT-pq&a>&tRsF<@4D_=`ToJ2PSJqa;7@NV)e=OhGw(ZWQilo2fx4JP1`YTqraMNFxKT5 zPuglUY9Ko`k8e$MsftJOL(<)ZbY^fmgY%ohw%LCoge zX8caBKhWxNORR6GbmtkZ)^nN+{K|b~S;@tHa~JQA14dS|2tN(0xyQD;9aEjh8Uf^> z2Cjy6Q^X0S!747rqP)jgu@xz?KxVdwcZeh{*V`92dL#1gD>pE@{u+ipP z(vNF!LBD6levMnI&otfxmtL0|KRf=|UK=^3-DM6F3e+xSG{i^!*vxw+T?TC<6kF+P(+CHlck@FM)g1MzXhlm$Q4#-bmNXP=WkIiU^gK3B42rM(9{{-8T?=u=sR3}?$a;c zs`0VfEanwp#>HK;Ma3>4xRh} zQpw8@VDFV+J&=8tP1%M+u#to;$PGMyZ$JzUO}MBzPfMPBnxVokoFnMVNe`u`;VizF zm7s)p=j}Yv-}H=JC8jvu@-m3I#V6-67Y|&=&V38P z?)j#^wX%WweX4Jis`6eJ%2?K5eBXR_ak49{s8d+x{6^ntOs}4vjXAu~EH~D_7RB-% z8QGR8-}xUvMCn0ES1@KLVOaEu?WSW}-w0*P@7<*JA7C%LMDUz9@v`THt8vHV^2UFF zl(^Fu=QsZYyorbY2Y}x$0_~_-PZ|q3lUBlZOg3 z`47Ngiu+lB)3@i(&98@DAR=2mgBpFUc*XBV0&AzGxx+sQDhA}f7LyxTl>B9C*_!cN z@&WI)J7vAoPiP2-FkN_Zm4?~8o@>gcX3Rb#Fy;bRhMDywjc6M6e#HRQcRwD)-FIoBU%!tZ zn~0_~LB-oZci2C-{+3^dA1B*BcM{|nDKECe!I!1lXPKWRGX+~TKHkvc;%kT0&@gl3 zF$B4V!4Djd)Xc7E#&jT~yeX$?>k+{$^|xF3u)l#3Ws@kkaql4mSxAkeQEN(mVU8_ZpnL-1@JD5}Of(|+T6d{Q-*ccL9(xmW!n4~jO8LYf zcML~TxTCq_2G#Kq+5>OL3q8HYr2ba&xI=}O=y~1S+Z@m4?lO|AWV`-6Y^dfs+Pm{s z>So|wee$C`LskqH$fhV#b`;P}7t*GiA~*qkDEuJbC!LoQ1EIl*%{5QE?i79&<%BNQ zANHSRmqwmNoDdyupQZ7?WRA<-hl2MLR1p&Sjs&A@pzh+tx-ZZ@CX}4*AqfB=fG3w z;@er&QRGUi1V#U*JjmYXNbyjXO_cR<vDqO~8VrzoKq+|Buqn})o=s=g{} zy2MG#xo(wzMT7rYh+drFH^i#aBVA`FS|SWcCB&@dU9iMzV7pvs!1+6qkN-1||EEED zvxY}z6&6E2{)(P13~p34Zr~kxOi+$0v<{yX&A?8bhMTHo&efb(!*4|@L@p7MF=(Ri z%^Y`Q9F)>Rh|j7MQY|=HSJE`F zuTyd*Vt%>a27A@*Mbfa{xaiPczKahd6HwB4b|Eo&c2QvuKxPL-fa`tg58rxs0X=vA z)?obwJNZKI2L=nb3i~|CqKAGJR!ti55Br+}CEn1!Fx?*!(ywa2d01vCII11ZxngLv z11qR%@f;0%75>z1@e`42LD5%yTtt1Wpg;aq$_y$=YnTNMcxW}*hy5i!RjvQUq#DPl zY&mfFlgjGG%Qr0=^m+95EA(o)Md8ymy&X2u`Rh+Rh%6&K{`j7vY-;cF3j6rDDzsZ0 z8Mc(ryZk5S5AQEo@BQofo_t2KnrYP!7h{^Ps<`IDmL!6zkeiH_u3?dPY^kRa)JV6N!FMA)s{q5|t*`|?1LD%SF zs(hHNMY*?iTtxP`s6O(fD#QU9UPR}BdhH+-q9t}lI=b&vn8yI$cc1N$*?UCi_Pn5LP>tP7%S-d9mPq=F8#`RiJzdw(I z?%&0tshKs&6ZFyTZXbc$Nh7Bph>IZ^>o2?iIm*7dc!638dT#_|gR?+W? z)muisrhVuvQqdNULM6r(*=acd7u(Yp_s>H*3ygiKwRb{tR)?m-(6HSDZT6p4(v`xz zG&6N)I*A^dnSDxdr90{KANV+BO@6aKvi|@P$+JFRFkfZ2VG~(~sbU2F14POvu@`>2 z#jQ>t^^GFgih{LkktC1J_@@$U4jU6t!M3c~E+sr4YjFvmOZ386mNvd6;o#e#TLFq% zO_N9@D3b`s2|p2(Tv_j_wj#A1B#t3UedGTZ%*=^6eCqprjUt?S&$iZ1sfXu`L-?(G zKK#G*Km4q}S=;BG1BTbxTI%?@tSCpJ#Sn`CL*x5X5X!6mBHIN+%OFjK`l5OOL!NMt znplpy@ff-Y!iE~X&_i=goO!2(Yu2g($S}f{0;c|7reE1#2Vbi&)w1qG*&{od-UKGn z@1D??S)|EK!>Md(*o1kSPA|lKUp#y_c7u{NkBdXo6zGSNHs$zCZ-v_UY?DJjgAiKH zY~tJ+TpEbE9f7=L`$Mv7O}{!&ok$`b#Y!TaOvO{8)Lp6Oc=T3l&sL*{Bv85PuaSPd z$Ae4)+oBMnhuvbk;S7x>Ys8G6Z2zvfn7Kxyx?@^1XAQr)=jXrx`RbO^&lyN5y!%hcMuU^p!xS##>=zu8Q zMo;kx&b9eRVs_cLV0*qAl>1*s55v%!a5x;Psk10I&M8avI7(bvkE#r=20F#*lG`MH z30i#Em#Mg_1n{-IN&iHu`zr9uB7c@Jd^j1uyL0haMH6m)mYNrj`kG2hr^%o?>fq>l z90@Nn|5#U&tJR2_Qy?bK0;V@V3w!gtb5dzgkhRwiQ;ET9Z~Srwm9(KkfZAg-5C`Tf zSAvuE3~ULBH_g%v0k~Ox!Vv+=fZ^4zW)+Mff2*|k81zOr@f)Ri%cA2XVMS>b^CJ7H z(jkL}#i+_4gB$0g-r8c=jQy|%t1e|AspbZ1uRvPUJxjMRotuQ)0d7dRkY9@ zAxa@jMWRBY!Yf=|%O7OL>RFX{An0{3-3~!`dG+(>dXc5Sy_tF-x*0JF$-+bw1N$BS zaluiLxyv+hfnVZdf}|&_+h&Gwk#d*j4cYa~;t9Atk%DI{f0finA{vCJe6BO38TYsr zKl&UCh|8d+W8P^ba^CZI)MS&*OX0r%v{kK7BmzLwXpyjkGvhKoO=~BWr_0vg&ijQY zt~xewny$#9(xHC0*w3Rvu)qHUOxX)jrswE$=19XBPK5FQUF=*GY*9U|U4qMU@m5m_SV^Vc9olMAG4AXuBN+9F}l#X zCI`;y=m8Ndw=X7_RaDd`97(8PVrps}WzP!6Mxh_cT#us; z(yqB|R$c94ENcZ24}9H{C%|CHfq|Jnc=02yh3|g=*FY%0Z!XrQu>{x7r)xsNwCBPG zx{1Z?GJqf4ykorhc#(hB{71qmqdO=;GQb^y1I$4SH;!cGi+}@`F}s8dCU8TJ+B6=?$X!C~&$wro5Cud3K+c3P@?0 z0vHJW2V)05OnZO`BZ}=J55lTa& zl33IQr0qM-!%T1Ghq?K9XcV9X3`cKn?R|t6C4pZuTk>4twQGyv#LBVOORQl6!$cofqU)aRI;j- zt{X~WX&TY%HIVwjt!YF|#DQ;bNq-oU@~o`7iG@SBov=VLmhZNa#C1^mOv(m7g|!k1 ziHI4JJZ}ah1p}vyZgh($d#Ib)^!EP3)th)-t}4oR8Wsr@wN_Ku8&)hxx>E{+@KhND z0YrRzObSQm)&}3+@WVPMVUTzEfRI&w9;=lBNE>D+a$t<_w_--DWN}*He9w+Qi*lY< z%=O(iE!cas>$1q5X)86u`t?XwUMyX+6d=fdq98*P9x){bOBN?{{c>QOfsq*zzCape zu5KubYetzM=?qEti4qt6;ydC5OAq(k#F07r`HrYq7i=vy@999Y<&tbJ!MwNbHuPp!~n%(834p%&VODQ-k0_7j&RLO z8!VnFbIOZqXA+}jl&Gx$rYJLNfqzxDWP81{T`5isoC+SUk7wL|5&4MmzUB$wr?(9yzzqU-2T9L@JpTab zA+$^BJCSreFnXzv)czI6Zd&1dV%C-yBDRm>f0emqR}z}Sn|1!+WdJHAnGctdFeK(j zv@EKwwM3Q*PDE@8*zEw3Bp(2oh%+TM3KCQR4`>Y=b{ov%VB^W*LFV@Lx}~~0+!y&< z6vIlSmlJTJtwOpL6;d=yOSq*ptB-=w?k@`qpJWKke#cX-MQLewlDU)izF-y@h&%jc1T; zq{Sl0ZHN+7up>1nM#W0Io5K znLjWeny@^KbFXF4xq`N7IjFl!V&}@vZtQlIveWJ6JxS^*$Q4s_^z6usv z2*k-?35d)CiSf9`!0x0Y4JKF;Vh#Zj`po-j+q}mCeLYTph04))sN7p*`1ZNn+DfU^ ztvD7kEWs@{n#VlayJXOx1R9WJAGdPt6Y^r}vCyVFgSZeSw}K~m8x{ixiBU~R2w;4_ zPUI=TgOeGPj==F~{{V^I7XJXu%OCnWKlV4sy=49m>A_!vt^8y4JT&{ACbt?_voZNK z8+=b1(<>lFjBUw!`-^lU#i=wpnZPTsNDJh9N1u=9;)}|(81&XhIZ_wLkZ^H;ao_95 zCVHX4P`CzVNIOQ882@~$p{yW#K&kkU00V~(`2holQzU@2eI1| zG31a4I|&>sy5;rg)YPg=695Af*ufjlpikWQ91-mCt&0I$pIF;EJcC~~pUSelBK_N0 zI$qi^Id*K358aR`6=ACa`)d8*e0xEQ8a&uhcNYPTS<>?*l!%a5PiXHm7@fE*D+E-i z%t*=9PVpViet?4+#Be*YgQN#w-J@Gjw5nRq(`D--aY4qYI^NbXP={J#(kwO13s6H2r%Y6dZI2@-mC|HNgf78(o9n{My!VJGnfaw|duFTx%Sg%6cg_wwPDubA@gs>p zv}sNxObp{W+;1LadED{XoZPEVinqp@GAo=C>)BMNp)S#hqJ*WG`OhH8XM9jqe zS9$Rjr4?YV6`Xd=1sjbyp7;VrKocAdbIoX|Q0@uX08HoX0GQ4(5tux8IA*o{1pfe- zk0RB)?3)Kpsw)uYh%sa8`>z$Bx3me*b76QgedtZWAa^n} z#&PannNr>dD514>owVD(QZo?R-s7^w8b#xsWlgUuruIXc)1NX(KtK`x-yhmkriufS z1ek_{J%}WjAokdMoV4~}Qq-gpyQ?2Nn8ac{k0MAsUbn5`Uf0JerDN<{i%~p^=FJ2( z4BGaaE?TAb=_aaxPlGCy#5HC_zQi#ARvQYdPo=8#f-^0&{IGJrtjFb9A(^WJWr%^1 z0}?PNXg)EYVLVc_7MpsS`{|{gvWiTnbf`f>uB}diQ&~}-r6rQ07A;kL4kV@uAf{*K zvYq`UMMZ3tB%BBljBI=pJ-f-`W~(Y>VKyLn{WpP$_7EqE_U#Q?p1VU*gmFY6YR|H} zNE-~a)Avn#DW3MEWaNekvR<+7vGNYZ@~zFT$f#g6PO><%keQK zBOfbAnZpp3at;Y6NhUPFo&Ny983d7`E8Pl*O1YiqZ_@|KlQY{G)%eW4{y%R$X{HIT z7cEp-*M^%CCbFWQ)K@ruQ>kPORlX0DV>Hc4tDfDY5^dLyx_M5ZB)Fo`k! zpr&RZl2*H~TCb?g&L_x&5jYxv$Q`_noaq+2TvcFNtb>LFVUdgt;~!FR2qmNPGudQx z{{Wdh`t8TCXjKrVEwgH123G$77EDUg$dUp4lE{a@Vex_^(xRvZhLT6IF*)BD_LKVQ zQ)p9qATpMnDzH55Esf58VS(ExkKT92i$4dA=wffHM?-UQgz8AEVR|_e9#dUxP?1)Y z+W?OuUttr8^COb)Biz<50HzP9%qK|6otI9gCU-tCOy^1U=kV0(dW)@_s*2_ zdTdpjO9SW6X!=REeQCt*o6qpY&t7E)L0DEb}aDC7Laf5qzl2??6 z_Vb?IyN_=nA0O_>{_*|)03C{cGN_GBomgjaG6Z|$81~G}VmS0&qY9K!!1;h9V-wm& z03H6I9w6s5Z9WQP?TuqS$PQv;3L~_PetY*W-Mi0z-Qs)q`Jpf-0gPk$3DbcY$j$~J z9ts4A&PEJ-`H23W4kL@#2;@3gx7kAb*=my{SzMXoGAo0RHG#x-WOg4R@{u5)^+e1> z{9Sc#PNKU(1ZNSsF)}A}{zG!gMLUNOr9%(O(grrpGwu2V051$2wH%W?{Xy8f`46&x zB5~qkFhWrhOo)i>9@vikrhnK|6=i*n4DA~k#_^H&@B!1q0MfHKF`349ncjPXJ+Z)F zsk{k1f}2v0Soc{Aq|h-f5Dw4?2R zKZ%1P87U`JmfE=3%R&5enE13O<#D~290C5?f<0s+gJu@A|liMa^I5^S= z^Wwv4X7>k9r9-r^I(uNkF_Lk1&j5zhj(iR7`D|`U%zE#p6R)2B28#mqs9dT z5KyU~7!jH6GP1VHB0Pu zeggaxFVwMRL*OJ5xdUXuL=4QZi@L@nWSKr96Z(Wm$BfkJ06;k~Hy!@oXB`0D6pfMTzzo^lYOyJ1=>XWQfWpL05AQDvSJvq*DF&zL_n=|-ts=~Y~L`V51ei6VOfB=GulzoLrA!Kqjf z*oVVxFbQeyPjN)B?+K8-j$@A^&fnj-f5*aA}0BAXn+9o^q{_zp< z1xl>yWRjo*G9>*r&U_4K5z;VaRo6@t9ZCU#8{|fR@ZvjTj{YAGM+j3Z%LVOOm@Jlo zOv>l>TI59LB$pE&Oo@e%o-*SCK0=h{3nDVH-OM`1m?5&W~0IqZ1U6<8q0 z(BS<<`){7}*sNBQ=E|DiWU+GsAs&8H;}IqJ?FAo#i1L7*)A^3`{?F}c1VAw)NhV1< zXL0`kMoBOS3U6)*C3jF@LEN3pNgEFrn*(` z9bn23b?98ej+O&yJaC`ecf?m5qv7Lc%WKr5%x;FGQr*f|p+p!*Mhe4IJ^=0ybMmOI z;W+(ItXf?aZbuT>)z}a7PZQp^;LRM9iJca4OE2*X!ly{YrGYk9EK-x|WC;A}AGB}S zJx{~GI`~F~W;JTnsauwXApqW>wvbs08Ja|B?6OHbb068)?W&2gx8=VW+!K3iMQOh< z(FEx_DYe+xGhj3CDqXNg6ppFxV$$Ur99yUvH=nHt%~d)$b}h{ewJ1CxvPwDcAP6>? zG^h6?neQ+XUPryLxwfrZg~cbMR+F_iN~-AU=z5ivvW8?s0g?GhI=wAxb#HS@mBk9H zR-I&8i0HwCQ*kKZfI|YXNl+vm#Bf2(vBml4P56d~P6L`hmFPF@_Z0dQR=9D>w&s;( z(P2W@aA4!whIujuB>cQaB0d9Bl?ry~bw93H>(%na>^^BH1b(nd1)JN`uUQ$?6HjpS zzMLJ_L=m(uuE!imb(a&U{79?uT+9{$TSe-1H=3&Fr4^VJ){kh9b^9#0v>UtjNXUqc z_bK=#HHLO1X#;J*`pm$R4nz^T9pYfX05wji+utMY^(+AsCO8=$4=c*!Ir*2fte-eW zvMi{I&3TFuRjMT|*ZW8=wU$iz!$jBE=lcMZ{P}8Xcvl2z*kqB6&a4O(^Q1pS%sUDIwScUl- zp2&y^{6O~n{LiW>)RqK^J7B?y>`9KoVlV=TJYi`P8!@Z_(g@S~x0&|OeT7J z#+_`S+C@Kl0DzL#y7-hXRYuzx5Dqwx_c555k|XNXu9XcoRD}pj2$QS08G|RN!0FD~ zivd!aI1#9WI(&j7c0IpI*hWJT*V2AD)$UuFgMkX@44^6 zP`GaY021T$xd#@jjf9ZUxLmg@X)G1o%}qzHu`2j6%TtZSs|s48R#HBK?GxNb^i{+9 zs#ZZLL4iIG&7SZi_BwD|Bs!AY&Iadw%;(<`9n^q$ndhAW>$7@Yjc(RY1`Y>ujZIp- zrz>G^Q3u}+!6l~F4g89k0iwl>;0O8)@JT~7Z1$GFG-t}p&H`3I^$gnonk z_#@J~{x|;sgn#r-HB&dEo7=cp_BjJ%8QND>b1?-|Mzx;(`<6-V}Zzu?j@D()wyE9nwcq3!6;9&2iQZ~$YOqKmddU)m<#hCGe0)o z;}PJ=A(&RCQIwM{xQvewepowaPJOp!*K>e7b#*4rM%GS!RHa|W*SR(Ag@K=8-zEW! zSOTvB;)?<%J)#9u1u$;y?&({h`AG|cKp{kJ26q#_M%~8)Jw_Ew%=ZidGm`@=Gms#O z?pGR%oNqGTMGQ>k@9JN-kh<#1OKdTbMzBbeYFVLU`;U2t3&+a1n3tE3%#0wtuWwqd zM<}vbl2i@Pa~sAs&d_tlnhUXJ1Qy5x^^#Bg@q;C$XKYrR*awQ;N=~z_sO$E8G?wOo z)`r!aqnw}BE|r=3%0!??_W<#UkAoYU1r0SI7yj@k=wt}Vj{yE=*#xgjk?8{xK_}+! zW^C1&+Il)Uhf)f;h#3v1)`*I|yu@CDpU>=F#G~;fLn9v+T~Q$c zrmWxs8&CX!Gn1JgLP2g6R<~tf!c1h0{{W8UnCu7`B*Ef-%D8q{98QTM#-f{GJq^z6 z7Ac*Gk+JuHp3N_{i0?2_`TUV~neFncbxhPC92IYtB<~oAj2X($Pi*N~UXgcQfeqC# zPv7+eZvD9F>(tslE`%mf7!R z^&BA^u+P`q00}-s`^51by~S2KPf#*D$7~M3k>r@dnT2j8%Nd$9uUo&cB+7KcjV$_5 zVKpRA$wzKg`aNSkgF!wLo>PgKNA>VfP;aN>#=0JN`hCL|9h zJNOYe@J}2ic^E1$y0e_>rJg9ab`VEpHwp@2iDMKY4Pg+3caIs7{kued(A{%=BSJYO z$dGe|?st!)LB<5a$8}d4DH6fS>*3@X}=<)Pdn@w8ly2)5Lgg9Xy z-%LcrisnXqpqDe=dq9upzsf~g*EMU@EY%ol)zn;oB*6g6{d$Z>z`1NgY)@JO#Fu4t!njjXkzYQ0>elra@fG| zB%i60%mpV4AP}NubE$^pfu?uPCo&+IJXs%GBd7e)c-+Sxg#C-PGmGYY#i|f0Zg(=Xsq}8ex z5H)E80H{dT#eh|UNyutvGUwc%ky}>3=NH!Y78df=?rvRa(nJ|7%t#(bc_Kg=9165- ztA<|W`PFw7v$x3Xx~^SuuA2{9=v*>trD~RiPd{Wf&Vvm=saz~Xo9hzyH^V>Z{{UBz za{4^W;(&(g=9)3(#3?i&e6iZP>g+|| z#Cm{?08dmAu>;QIY=4C9#=z!I!_LMB{{T1iF@P`$EpUw_E~4t6X=iDJiTuy!@jc-D zP6#Kzk8c@>{{7}+ekQpw6EXx~#%I0{_^eona}8@StJID>=SkL#b6 zzG`H{X_o9T4~g49r+kJW5x-8H>tBLpq}^8-EO)g~|~V$Nr|PjA!xi@Zn1 z7S-xdVH+4MNzZ-2#$yC9li~>Cds~;J;%5>x?g0dEIq&7MJ{6jwp~h?FGd)%yt>lHm zhXV*0RcfFJ&0fif6@hucc}&d8prU^<*lZLp%Pho<_hpO`MXdgO*b<;GU7MJPQ=DYh{mX%HMPH@8HGwnsz@q823Twf8wnyNVCD?R z0ezqFo#@S5jj3`rV@tili!^m?qOElbzUH*9VZ_HU*)pO@_JzoodGRQb`DoQ%=uf!D zdykj-p5Tozbv~zYLbi&O=(18^m|!&qCqB>yU~C8tk;R{Z{{V)+NNF%6FzH*f-Dhep zhujdD!Fd!LP2K0nJG`KKOrLJ$5!>MuE$UX7Ad!N6ZT;S#Kc@37wR_7dWr4SlAw+^s zequHlkb6gEZTM`lKAgI5#Iy2Im&I~Wn&4r6$RHBw?ti?c%OD|Gj7o$DxeDS)yZKDZ zS`<+<0;FMW7#uf;WcN-4Vfs?-z-B?@NqSULQs(>JA2XXUBCk8xZ?eh)}JvZraq6IdP zvUh)TCaA}Vh@8sQK0^l!4&KIj&zX6CV0+BUD~sUj;Ym_)<57$;$sp!GG)6E<;)l(- zfCPlvU~L9T#BO#G=eNo@XYeq>tHn+6{*VuW40-sJ&*Y>d@ev=7DVd)09iy~F$yaVF zmVGlI5udDT+_Vvx`plk?sMIU#sy=1_m@~FCh@UwV=f4*>)}(8Ff5TX30>Ksdtd2hf zDVgv4NdEwM9?=v1yZm0tWR2vX18=yWwBYX2PZ=Yl0Es(6I}_yN+~=_HKyQ`t(WsUL zHL%2#%)t+HkHIno=lw-96XPQtyGY0T&xw5mFh(E^`;XRRzrTFO4-nKLM6?FdVq!@H zCVx(-5gns|ZnI6b*pMY|ITuw2G z9^Ua7;akw0;@m6XgUz z1KvCKh$-zc5go*fds>%QG%e_Y)h;TfBv3P{+ekR^Ad`$n2@Fd6o2pi}^{ZB3w5ii) zb|>Xk8j1Y}{R!hFy1R9qV&q%fwNQEdkBw^%Z2}{-q`tD-<(o>|*wqDl8UZ5gtEe9m zW0_TQ(h`9E8U37cS|0=Qo=r}ZJCvtR;^ZNPsc}>*RhcSSf=Yr;eY2~=UmJXHfAI%4 z=N9egHLc&RM%!nrM%LVs>L~(Ls*OxA6hS0>+;liwYUr&9Ba z(!jnbY1>FRW=6|eqXxAiY;Ukees`W<@pX;itx?gGBE>s#L>kR#{MQwb$x!5ng~ zeel0BHr6z`GTSh@p!ZjN#qLfiX~wUv+pLOjvTQ9Hqs~_vJJT#?`o=Ke+BhvJvw02knRUUv=?`ukjBe<&3vh)hhlW zDbWk~iaJ9Yr4%Nrh{TfP3^yWUk3oLOJ|wBhDN^N^*Y?#pZC+ZH1zNQnwY}Qa(3H}v z>Qhd#iCSQfO|T27;BTMJEOh&+I6Q8qRu)fBqNpQAjT0R<m%`;P@;^CN}ZfTQ>B!xgYte(zoKC z9=DZ)y~*kZh2=_eneAQT^Ao}wHR17J76>dN%* zpYF-yTYKfM#+^p}?LffeD+?fC;$kzW$%&pW=zkTr_k6bxkMZr^CuNw;-qWd~p}4m& z{W84FO6r$|WgJV}>FmRAl+SpRFOM49lUoYqRI22aA($OKluYFJBoAzGEmy1@pwWV z9zoFF1=8enr`(}nYw{W0`n&six) zLptI(TR+FfmFpanioE#dwt&OnXjYUuoqIG`gGw7dXX9!iu`)s8b~LQjqEv_Jl!(j- z+a`UReZrA0k=msaO+*T21Vb!fh=xB%W4DQ37anWUp0?4}<@o$X_O*PlE>-2+*8z{I z_GMt0%Jgy6XXc7i-ecQUz7k!$=1x14AHq>arBS58T}oR7k~>Ue*tfCFa6(trMd_8m zB!VFQ+o*eF1>Ovh7mVHG{{TwgRLSa>8E+*$>KQzSM!S``>laA!@4>8XUNV~&$&UaA zs0`p>COdi0a`<)4(o?1Yogf{E@h30@?l%7bbYo3%tm?b8pO}_d?U{{-DL zvC;mnbhAF*b_KQ1DvniKFE+jEDSC`40kw-_TOh%w>s(n$oGaLY{-n$ddSHY9a543o@n*84-HGrXY*vmlaW?Az4WbJ4k^tKoED_l6|-(X*qX> zH4;Eo#;~MLL_psYGab6Pt#dAfa^6F`Ey?OemK7FOs1b!TwvB6*DNGfGSk3l{F=Yqj zg;xSt%#4qDi4U66y`@PCm68dQu-|>hZREx`S&k&BX{>I`i3NcKZeZZR&T8>XmrZoQP-)sU*M+q}D{TJ&ROi29-lbY(g5-pQ-yokG0(KqzLh7D36f2q` z=nZCifQ1@k<~oQc>^nr0JP|*M>)-xB+y4OQc*p+$7xF(^&-mN_01y6tH1v6g{n7sb zh<~3PMa;U>o5=TXan7>X+~Hd9Ed0#Gmo0vUoNEXQJ2hEYMU<t7-)YYL$=*#1Ljr-b&7mAMA7zJ<kcsgc;?vbrptUD4)G`3~1_2lzA|y;{VC5%P4rek6+I;^2NfSAoSUGE1 zVw>8oD2zT+V>u>NtlB|Zt&*Du5A`;foRuPmdcc8@B_44lGCjT|7B%UYil(mQF%mKE z1QK)70+0rn&aO17z*8)xWNjV4^@svYcqGbf$XtEA0FIAangKK^nJ zM0@QG1C>KdcK(%nmapkzQdODCuVtTUek{TpKWh7v9;OTzxtN%$L=6tGZJIsUSKmZ9f}B(Uh=9CT8k|F#7Nd;h}uTe?o5KDXNDK(_rx-UVq#g-W|qEwBh#1E37LJE94 zNf*V6d4f|YnuP@n${9!+Ni(FL;}`_M`W-53)4Zf5HF`i3kvTFr#_ z3vr%R>b=0ywN|}zw^3}YatRhsa)bw&LkY@{A_?ykB@-gCE)7O>swNLq?jXiSea7&6 z&l(p}fU=&TCP9f9J8kSopRX0JOa52jjEu_)%noy5p`k^z8*!3UL;$N*wx~|Blzr17~8Q2;@CIo6b=QI87v4J?8@g1>iMuRcz zU%ZWZ^Wy3KO9#S#ui0uRA+_%1Z=SrghuBLPlakLYOBze;@ zf*`9BD|&{aq^^Qc5;ZF~@ehH!kcTUwCXk$+5^GKD=wwsWhOi;3F%p69gGamzv3d6G^IA7eN}XCY4ANoeG)^Nh_#GAb5f#0jLPa^v_hv#tjan+~pcu zI{aE{*D6$2nZDYk8lItowGaEIPm`D0Suz9JkYgX7@?QtLrAECWQCs(!X_+(IQ0fMD zB%bk{GL=wU)S#5oO;AW^1_>GSl4b-PN6OMln6dBV^2Cl~E#1o3@PSoiwL@CCFTeF? z#eOglF)&diAP^mX@y6e^@8tQK|s%Z9=K0#+?cbU?|dCb|zS-&mTRl6nRNJ}IR&Ld1g`5~}Yx=a+vJUsa<(iZ;sbzPo(B zm?u++RmK9WnJ21Yj-OM0#{U3jl>QLqG_E;-H7@Q}okyeddMizJQ>5r#qOC&23Yu6h zLX;X<@let0nmTFA`1a-gPCL=Rt5Tk{Rk81`yy;;o?r_PMXV}R0jmBt*tXz?WCJlrD zJfrr5$oW?^FdgQ9 z)!WKH(=qv;@Lpaky|k%kaak)Siiq_DZSt6yj1qr`P$NBn9}L0NSU4R z4rBrTUHFl%MOEU&FrjcTA^Z7xcK*PBpDx4S^Zlf!w|^ed9~80Fb$29_9lz>4{f7K7 z4I~Mf?FK|+=oSalwsvdP3*b>+#5~hv4o*B~AY^7I_wCvwre+2_dyG%#v`l{A6xOGt zuqz^b>`0vbc8JH9lcrG6WEl3`W_x~OAjjW=hE?ZZT8?MEf{!m?k8df6nU81sW54EN zJHX6;Y5d2)ok&6k^Cn<+pE5+xY<4-*!z*gqfYx%ew}|W~BzK)Xz0U?*OxNC~l8-s? z^9F&$50{8{i39zlqDni=c8T$y>G%9hLB2odzqTj!&e)C{Q_^BK8%L3t@BTkRFlw3< zrzNeXRgM&XZkYfhlOj2ppDxGw82tY8{h~YmVm?hwz&>VtWPOOt?}+RUcqO~HV0(Sz z@0llY-T>mZ{Do-)W=cY2fWSyU=%dC2e`$j9^O!FFa~?DQ0Bm;nN;JfoGl~6Uv;pU_ z>_#|WP`08*^FQO?^o}XI_}+nKI|)-%_z`oVa6-62trc8NJCgP8J(kI(&)@akmK zf+yuC1ip8PBOY_dMCIJpjlorOlK`9?d!2^l z>d%d^rY<$to0*i6OK}b-q7xx#e1Mc^J*0_oFj3t5$DBuanC?DZ=dlUQ8jJ#Fa%N{6 z#^N*lJ)#c$SjR86xmI+mpg4$)UzUDp$LaNBsyHfJ`jm6)vbrr0IdyyhOaf$61Lfx( zgkX$}N4q|46Kra1-Hr|{{SxUGZ0_Z@%=-{XSd_?-??a<&48MKgX7)~Grrqm5AjU1 zA41-a&VU2_TOX!lPjT%scH#oB)U^yGx4{;M)85c%n&oD<1jsA)WJC-#arhtGxKH|L zzki0(=Pzywp;BH1oX5X>{{YrwjhmKQ*KJL3pb{JAQSShIXR^oJh%G; zzZrm9p_3p3=6py5#xh&@K?^)B=A72X$$dH|p1+s240sFL0GyGu9G)6JAA^x|?a@-N z{Jlmfg^-XFhW6}J3^{{=5}EPs5=13E#s)q_*9^*5N`rtG6TbP8oSBIm?=Q)@_dBRH zX{)Hd0*#Cuw8VfrV2>sN3dZ5mMN$P>M`@fB5rG5FK;SaUI@s@k!(+MFPVw0OxZ{sI5p~+zGQmZQ z4fa`dcZRiV9>Tbi{{TR&MgV4iLGmdOp5+2{GD3qASjo(1+DCc&M(aI7)L`xg>pDgw z>Q9d2CkGgw7q6&(VYgcayl;#f)32_xtZYJ@>90UQoX7d4W&-Al;06wTj)s3lWXKWu z*gs_!Jh^$w^Nw##YTMYIRA{k@}4UJI5KvN15@cr++t>R(()jl!860y+=|Rm7oo)HkRm{PoS5P z-AvjebwgLM1XSW!`zMlf+C1u#nigh-BxYK0(3%jyDg+TDOicGM37$No&p&405_~k& zuIcmpZeep$(_O)(>D6~z+AHf(T8de9teR&?jKdT1@ekCi{HKjqZ&^6Zv=1e?*?iM2 z^=Y}wp>cJr*5u?>p^&(}kRp95`eoCzU;!(rGG9+UO#c8s z&PAIJt8jqtI-P}7YS;uTQ)*JQ`zlb7KDqWQXoe-@5DX!5i|1SB9QNL?CA?l&%k|$; zq3cshYJDX}fh^^JEWtfMjpr=4u1)aU*x6bo&oMTfw(hk%ORBY-_p4T_%`~@F07zjN zgFYm1!1D^Y`GWmxaL;& z^;__@?jCME0}W1{M+EEiumQm%EDlE$@1{6ox>qjTx{G$VuFPqZO7cu5Qa$b1l7T5Rc_Y@kJrV661M*u4!eN7M;mJR_1 zL?XeG$BV6BjA~q|W=#V~(-KH0VgkyN2-UFuhk=(xdR0q1YpBiY1&4Co+>LD z0AK?ktaiM{ts1_vI4@CH%MC$8K$utn{pV)PjemVz=+7eHl`QJ?M-=>VVL9BYt!)%K zw;ZEoAnHv`TNOoDtX!_n^Vpj*$s&pG5F~Bo8X}?mGm@m8Ntc=1rFJuj0Cqnp>a|(( zrFtwb#WrY^U^JOrnUOi10lY@?I7~XZ(=JQLFmt)3t|QbV(yuOmHQv1zvT+e()%Q6z zD0|$u9MMq=EHVHp`F}}s`0OB*YDA3+MD9Q$M`5uuk|I11*a}4p`+DlHQlOyEQK`~H zs5L|RPhvGFjF4skCr?!e2dI5m=h*>Y50diNaeQjIwY4=HZ%VFU09*yli}D7td6Z9} zTkSYL@@$b@L>xL5OERDbc8<|L1Po3<9!N6^a}vR6tF)2=f=S8D6RU375H=vsR=RXg zhxJR2^!Fc~TEElXJj_hZN?baY&6ZQTewPc$bf0>|De6r&v5odO$=<`5?K1-cWwjNk z5~`x=YzHO;3}-kX_r@EvOAS<0q)4jHVsNBtLU+!@ez89AqugG$b&cHMw})1wMcWO_ z`QoU-!4C9*a8`=PX~M+!F*{E2 z5+X+eRH#s}I|6razS>M4+xeK`<)J115t``cq#B{1A{aLKC zUhRkIt&0-6O6=2JfsuH4&+J{uTHLz3hNvngAx5IG3CxI-BPSaZ5@fd%P^^v=K~)f3 z%oZ?A&r}JB{UB|`Jx{G0TlyLMoLhoFPpTH|_~L15;7hVIDngtOxVF!N>jXkrv#$B` zHLxZ;XTp99LdL2*u=NzSq7HQpq6i=<>^gM+0DQqD+|{R43TZTqL11QN?oL1AB*^@@ zJWX{M>Pwqj=#L=eyx;hHJyPGfXEhkyO%}4g)`eHLGOiPm4qk?(A(b4~@tFL|Pq`P*YD#!3s2-7$k@o*hxLJ4B>b$k;5W2}iAgB>5fDEVvk}?J}j+o<=B}FxL09XQ8j0vA; z19B%lrbP7F_NZrdN~J2iQoXhEInc0oZs_VDDL|E?X=m4`9$U^)0H!qP}s+`^^$I)XdSj%ZiDyi8?8I44Q z!(bZ%W893Sq-H)TPF-~xNE(@bXoWlADU-J5dx8KXh>D#6Qv|M)84zTH6Ow(f1d-A( zIIMCmRl+!|UB}L)jj|k;?Ee5e{1&~3fVK(kd(5lTwx?>LbO2TY^+6ei z-*ew#JIUa>?@1Bug)rz_>*uD@S2pYWE=7MyF>9}8KUMhzc9l3k9&$2Wq-W&KNl`#X zBY6iP4`_zT@;ajsFuzo{S`nZ^ox~mclY#&nfPbg~=uP80PNAoBEdKzNtzR;g%0XI{ zvn;c@=l2=+CR8A)SwMunQfis~XTOi=8`P?-dV$h$kK_h7+w1&Wm^HGD5;L&wW3dKD zw{x^+c;d~hc5)S=w`#6+tkQAYT}$mzcaVhN}g(g?=<9Tpe8WkzFyupjE=hbZ$ z*&p3DFAF@@P*L-?e%ZP8(yg~Aw0C#9sH*NkEV_mYl1h-h#BNIBBZ{~BE&MlW%qv&s zJlSon?Dd>~N~r5tG{dbgn_CIg+JlG4^xdPGnyjP1xXfHV zlr1^RiYRSF`-jgB($LTN z)h$_8UCIPmx2*|jR)@N}e)wlF1vw90upj4l(UBq)trIA|QlVh?1WYhyCR2>AUSI<8DWy!<4T-(zxQ3EYu?K(w4 zf-wLbneP)kcTa=4+tYDrS;41JqtIeibm~ydgp<`dh&xE?fH>Kb^;tkUP(&vY9l(4@ zB&WQ|%>DpJv=4al{FiB;&y;c}QjApM5B+B+AM=5?9d?^*P|^k@nC}O)lerzt?d~P> zA?pPTL&LfXgd3W*e;yu6L^@#YQ=AXXi1PCHBbJ}*> zw%y)AK4UU6F&lOp;P3wMj+UQ@GPCOpLYAV40rD9cnfy$7NR0j_Bk|)u`_;U+pz7)e0B_^G@41Xb zOq?$Pu!2hbvyX4Q`Tpqbw%i4*`rHR)7A)l4_L)S-kc^1WZ~8HkiCcf#Vj^NcyhnNQ zn69N7Oi1_r4#bgwJZF8lmW|bF3>jIsoSzb7$Rq;AAi2K8QPc1UfMXXR zL_Sd>Fk)T55k2AMV^ULLMbrqL%^&O*S}DHYX77zr5}cY^Wp{P+E& zrhh%AW;=OmDO5spAdHEDzD)V-O#4dY@%4X;E?8+U+K9<-n0+CnU>^CH+Cw+IBaTt7 z74BxYG@K-YtZXz|^QpBY~dr-H`@jBR%AQ5kK3!NcR5#ZqYrw zw_Q$Qw177tLDlXhMrC6jObwY$O-$OIi<|~A3?%K7pSBDT1PI{Ek6K!oCVBa&Kdl|Z z$mIV3Kcsh_!P+~2?j~k?&*$VdD!?Efpu#6nTak!@q1&32En324V!yh@A zp56tDC^q<}YT0xM5SdpP3I33NCJsEK_AdTC=khWAzts>Sv&k$DMmxlKk~jMOC5R`3 z?Ub+0G65n;iQgSN<}wfK8cA66R;p}~SqF@ana7;S&y2(XK~MK6?b;;2yhnc?{{Xaz z$LeQMoG+P>r!$XuAk3-a3#z5+{{VOb2HHV^1bL127#WgbX?*WJEXx;ofjA<2#258N zWi#4Y5g)1dn2GN*`H#Si-U3QSC&nka{{U|&?;i;$b(o0V10?DoW53KKSA zVlsdj?K9%fm)X6|bfpCKcJ2r1u?uQG5K|E zK2JP{P*+aFl1eIiAz?Rc<)+~jI;|(l?5WgJqILONn<-l>u|*?3CE0l;r7AY!*$UGt zjZ`@yepLjiXgPt>4|ozX4+!{{?e8yItzA=As?vkirK%kWu@FIMT~UD89k?9z=YnSP z4=v10+?MSF9NDc>#p?roHH9{#XS=e+PI6LH?5!*rxF%|mnIRvYkKnJ0JsUT&uJqQG zHc>>#M>Qr0ELvm7%Yi3-TypMh%c^q!0ElcYZYkUodMYEcOlgn3TTBrIuqzWFfiOfBkWMf``FT&r+=)3?KD52Drps%3~O14)y0ZX7u_ ztqpNbsR~qes{mLZnn4g{x}PAsPI1bud6gP(OjJ+a(jbPXSLJ$cx`_oC73$T9$-z<& zOLyCatHKZg@>tij}q=TeP01o6FLf5C z_YbeiGI=i?<{R@%1^6^YIqv%=$j!LaZ~VXl$Y#AHILJrGvlUN7@9YmKu|7+2*5$USlEvkF7f&JB~E95G})NxLI@{L;xcr^ z@3{v!0FJH}R_0vJrCL>skZ9>@aIvI@?hXtBVovcBJWO#uW6>Uy=Xx`^u7@Yr<~&-) zeh(Ybx8akLOGypi3Zf)7V0(&Hqhu^vObE(Gd!J_fx1CWc^zLP#`nZ)$Z6b)XZ@r!*j;{8R+;VH!4;C!0j9WIS+ zY;1B)EpZyv>{lz<>#*uI^c>)D-oCJ{pJuI)7o~*?m}$2B%HEx2CKMrVmt*LYw2_4q zs~8YRbr1}GYexQu;p&#RtymD%kgBlilHu~ISg2BSgWC_mC!zgXvC0o)SFG7Qw}5n~ z3XGK-xK`K4Fmn=$rn04toE4dR@)#2`>0_)i!I-Izn2!)B^GkLyDrLVWRt)4G-UsLj z*d1v}Xwe$3eMF}}yws$S#{rC=ylOqVbctxUe~G!#vFpc3Ic~9W+BPWXV&vICyGs1$ zUJt)00e81DN(ai)Tpq?I4j@wY5u5*6;JsM_SQpF3QTHX;mK;#cHwa zGPst@1de3GyqOlrlvrq_*((^q_MYA2^8v9PM@oxZ%ln37dRmj2!x%$7lq~rIO3C(_ zSJ#Dco;CG<(hf2=`6o%aR@~pbgk#pemGb<+rERe_moJjH?$1wTxd9}(XYU>1kpqV+ zl3tHJ9`I9rrf@_ zBb7Udj4Q<;&@G7WHTqWci3U|w<{#g~>a%GP3743YAmi$56DI)GwX^$h%= zcrh~~A~wjJaNp8Oh=!V$41=k$C%KJE0R2M~kO(7zrxxm$NPdbO+@0IEcQbi*SyZD- znveOYa~-K5t6Slq#M-fHy>N81MAu**3m)JQP*d=#b=BNfkqaKCsgQp!vB)EQ%=Hj> zTX$ylm1!ZUDpP%6tPBszb^{v$r|$r_FGb!RHu{a_eNRVNIG-EM-L8c3Z5p##gSKivNS?`nVj8;oe9 z{0IL4lpp8DMsTb7x?6j~o<#fMW!1=OQ?)J9M6t&L%GX9K`1tumopT=_&*Qw5K=g}( zwBte$OGL{@h#iDRNg_{->g3c(Bdvh&00ZxcneiSBZWh4Y>s(@%nt1&yngX3fuS9Lw zQYtME_8mB7q8~KgWJ7&{FjCl)@$cXE8l6UF>t?P~+#>PTdq#MxUS1IVDFW50+Fgk*!6IRtP0clvRfQYB6z00c%OZrGoEc{4l! zc@G5JcP`6%xb9_au&#vKY+UuCwcD2FxGLSzX|~!#fZtsFOd==O<}(vN74rwB=@+Xf zO6~@zQ6x^saoN4GgT}RzjXG~uPOhR9c02tf`;EP_q>|y;`8L;{i@C1~M`-TQ&Sar0 zunZ*EClrheCK=}Ff%5S&{{VjTKPol()g`xTR`giZl1V?6e9klHCmR4KioI*s-Ai?6 zNf5)>hBGI8kW6eRj)e?OVP@4n8%$`v#!jxrw#_Q^m6JLGTMqcTzg<5CQrxAfG4>}=1b*1s%omvFarS_;z0*JQKSiw-Jo$?(#tA9I!iA4 zWJqBkhKU~ehydZZCF$q%;}?hJa^6eJ*6yO~rTE!g0|ylsG)CZ(r1iGM$ScJv{npRC zQ#?)>QTrcw`E0U#8a5PGrD{)f#FnI@T?sQSCIF4Q9gdQ~aMIkdK%{`@00IsSK-+x( z08g7GdUxHKPg^Fo{^8uV%Cy{JT-7Wnzgi1ErAC;sYgKcImiZ8b$8iYkBjg3|Iklxw zt!jpFyXH0+Cnk2sZ<(T373(d%K}w)yVmpkE!xCmrJAF(YH8%9;jOSO@*uSwhbvEl? zODWOyPIVhCA#%&u<+H)3F&@|fkcg6ifPoqKvhSMNB`Fc9fiboW5AQ}lz4aSv_+6k4 zfFwYWHu87OW(n^SH{uHBa_}qWXz1%_QTu9-vQ$@TwyxWGq|sSm30#Ro88hq&nJ*6+ z7%2H&^_H}-h$OT(Y+x|oAe?RvdzDb$sH@VVbe;3J=^%af8Qk$%!B#J_t!2Ka7=!ygH9 zPlxEOB&EtZRke~W8ev!WG_<)oj2C8=)J(CKm=nkIfAMd`ci$TR)P5xRw(g^~PIJub z*41@gn@t*(R9fgMnslj9M0Nwsnev-J=bb0fB7Cv>I!ZL(D6+stB{@UbqD+WT>33+nIA~ro6hh^eCV@ z*>NuT7%`KQMq+a_9DAdba%&HU5b{1>byodvDbY23VuGc=qzq6#lq*O9h&u2lNd$3= z{b9RW?s8W6*KVuq6zkkd?k1?yU4I;lR;SZ7;zfeCwp7n6GR8`KBuB*`#2mdWUcGuX z1y`%9BdDPmh5?F*hycMdG9nG$t@UcuC|X>gOG|M^ofO#W)z(9RScWEh5v4%ECxj=a znjDUYQnj^mESjw-+_x|q%E)wAH8>k20(KP%V%1?Xh5^IK4h?e?=fwGwx9=LXmS!sC z;1Jj;w|Iz*`P?@_<=TSNG>cPEic$#-um;&A5w!J3ZBP{YoBA#KoX+ak539%a?k#-J zj-V62RfQ8`z{h2$ML{Jq3V49|iAfN%%%jdGC+95ZT&qP>UfeIGP|QSWW0>9u9gYdy zK?EMZ$a%uvQA#!3i<(LUm>^UDg#-;!$It5^5L-plR= zNjqXB@Ycq#4W>GdKqnAA^v>$CNNk<=;QACr)g^{if149Rob z^`GzhjFG^U#7swS(=*;mBlEg8Li_LEzilo3e_goC0B2$1a~b;{;BTL4;6106dZGf5 zSC$LOhbcKPgOvy)o$7q@PFeAL}*!dq(BP-`` zXyYc;mL!qnkYnF$Vt)ILhZjs9oH3S!khenI9$v}s1QFgKCJ_T7Q!_C=`vAm#L37$- z1FKmzN3qABX&@Xy5gqv2ovUh6P9{eA5#D=u?oZqmWB6wfyorQZtilAx^w>imn2*PP z!{RXq5k0@uQ~itmGvxv~H8n(;jU>!Wf=ulT+B@zhc$&Qx6v@4N{{V8DtFEFoK_~)&=1vYV-(!Ka z5(^DaELEum1WZXJs6FI;4&aH~rg*+rsbM6jip01tA(6Ic=}BVYDZ9G-jVDla+)9un!jMG3An;Y)<2>s-5oyNM zWlHqyB{r&P<~f5@Y7A9P{B3HU4@F|CYZ6WLIY#9`HIo75^Lu|vo0_46fCw5`^f7}B zumCVO1~!s+;RVIRR<3H+U3CK;4(B8SOvwPrpC9gkL1a^`eI~t$$WN)ULe_FtGJ!S6f_h{_WAoEJ-EYG6lEVcr`^ZZqHO zXN+uT{{WSG=ZkW+Ext4~_ z?)hci8_E?z)6^2>SE)$%`iaQGoQxC47UXu%ZkJY8O;plnHLM-M6&-cOaErN8#28(I z$kPBZ=jiY1A8DwjAdo~$PCzq&^8VMD{vGR!3h!}JlPbAa^(G*@In)=^ z5vEQu4E|k4>Rk-(bHZx!Jo&j_Yb!3T;chV+Y#k#%ajoVaQ1qr#?iB%(5JTl4BQhn( zKhkI5OAdYT?dQcjy2hbv^2z#8F$PD!BTh=bM8DwSw1QK?VW>|O&Ad|j0Kk@YMY5wxMii%^bLXp@FfCK@V z$=+a&ICzcC8rcKM`3^;0{Y#thth}uRu~h8SV|`6BdsND@W7=g%3Gz_-yTp`Fh}Ebm z3UrVT0mueU(SXu16YbRG{^@o`AS;Y`3=~Mmc_{(X zjg`%*Ml7ULn8`i2nC+eQPkqSZRIPVKY0zqbQzS@YJ4Oc6;~ICs!6a{mg`?BHo$4u@ z_v0&&CcUazRnElVBU9QDFMTyKPVL`VkKA~A{h{4w++t7RzP<1yBQQcRqM_5g$y@+Q z10D9s6U3(NmY`69)krxwK3X%m5(IgII~=gC^jnv4?L2;?=W{L%%{Gu(v}|zvK)ahY zzmJbQQ=d_B_RG^`39zh~1z?yMGANIhJ05Rfs-pFp-l!19oj0hiVhi*s(xl>8?grB> zir4I~P0%k+lEN@l88exkfZMc?Rh%$DiEM4 z7Ny^~DgkCk&d637>#ZF^r0S=|bhvK|#&pg;#jBQbtLQdFZCG8^du&1;Kpnl_!Gi8<0rKu9 z(Wt64w*9~$%wxeg$84VB2;qHhWB7}gDKwd+0Ft6H=O+XMor&@|VZNWbaXnJ!Jj;xA zLyPh>>DsQ`?vqau$X)!3I#uYv1Md=Xk{#-2lcmDvzFaH21{{YwUtgBClWNN9IHOkrUMP$;m2HfREqg*&E zxl12WF){22pJW&qNyqCECv&Spux46_%cRKqbcw?p;$sma0f^#^mo(}S^vGpkG~^9L zjLAOR;9wDu0b)irpDW`heSSWytB$LA@7b~zYcguXCb}#G;gdd@WSVV{c8JXOiHV8$ z6H^D}`GFtxypU&U`<=LzU221*bu4ahbK*>iC;tE`0EX;na?9M_$YU>7mM)tYPU|=G zQ0fI`CUUCE3%<22s`jtKB%{0u%oIw>eq&C9Ihf4m50A`z1CJZbEP+5XCP>Kb_3nMM zrhFJA#a__m8Fy@h5qjnN1oD2U1UlV&3yE36&}3BD(FpD&G1=oj$pV>)`J;nW^%DR= zUc!CJ6CnQp53re2#vL^fW83;|k>4l2Fh;JP=Lvr^WUK9Hp``TLx1n==18SwL^rW11 zMxc7l7r$?9V(;J$7=j5I^7uW?Dl1M1S5gX*-#tF_1FQV9co5NA%*xC}Nypa!obUSY zu>kS+JVuCX=%-7Y)-*Dzb?Y@XqSeOfSQ#+SCZc=5$oWGOP$0Sfq5gYM%5^?VNZi#` zkcaag_`;LyFmOQZM2-^IkCPBR^nsurtP zKW%31l2}t_(l8^o;AF|)<;FLX(f8qE?r`X7OQH~44pWW?*>LOG4~i8 zMx&qg7nEsYCdMsHeltZ@Gc`Prnzn6jR%fy%qlLf=%zFO-3L#t?+XWLbNf~-B2}1lLy3iAw_FaucD|`C+I}ap7{`V_s9W-l$FAD zZnECKMva{BfI-2Y3A3edLbW%ptZ9mX)nId=5hQ)KTyGkL!Mt)_rsDIRL!*nsk2{ZWJiB=q z8|bP-uc+MMh{~^RATtkj7MH4C(w0h~$b0rOD$V(&aLTU$Y64_VPhxi&`U#HOn2(9C zta-I-W}|2*rL{jwonWxes-sD$I~c~I8czPhVZF+2Q{(RBD6>y;AYycdpCkb?3vZ{eCu0gGTHBGiUss9Pb-M^Fm3VxSej15I4Cpdo_Rb^$;d z*ep|$bnv_LEm*&wUbSUKTkcD2Ciqr~3Uw?GalX&EAT}Hv3{+cRyHd>%7BFxIN$msx;)B_n(t1&tPWq%#HC55GH+f#CnXzNjKYrQh6j+HbtRH|a7 zN?MggGwM4dRLxhZr5Qit+S&@_yx#*?P0fMPuu-?~OZ3AA9Ujv7$%URf=d+Rugk2~R-qr2-YKMji6+4?mvSpIhCE; z*T1Dk=?AM@RVVdZ)2ije)Fv1e)ByW|AQs^X>z+>2tvdefR}E{a;9VYWL`u+y!l6O8XejSomxO}dkw%jCv5GQ%*cu2{bzE@%T49AO!=Q< zoDUeD+kk|qy2bkKC9yT4JNPbp$ei{r?EJ(Oc+B_j;{_j)iS66snzV&M2Lw!g{Q1V) z4*j6DHBbqeAebG&AJ0DcjEwMG;QWnNB`%l3o+Qsu$K%V_`0Ow}{N&F00xd&d@Yq_(an`eIC( z{6};0IFj$%ziE!+9i}Jsp55cTe!o88t|=OriFo^+>`4PWVc)EIOvmalQQCZFKRu`M z-}UW3i1@;lcxWIQ0ydu*J+(mEfAK=b%2|opXMU;o`Ia`?H}Na1Wp9gv+hQ7-d%sW; zPysE2GEpYqI4JXoF7f&86CL6vXZDeaBvi)nwgHa$$&74pCVHcdS#+F$M}j}DOu_aW zd+|!q;AdJ1cB(qiHHGVFK!hZZ@Dbt?GT9Hn#Do`r#1uzwFPoZ+iBq(1A4tYz_s07T z_|x#z+aprvzkgxlfBiUwUrVq3%z~;P_8BReDJkz0A`&t&*n7l&N5AGgWT*2#GFdyc zM{Jz?MB~olb}hdftq`EWBkil(_?B=ybnFXpc*^NU(?K3p?SuaSs!?DS?J1F%6BrVa zh!G#wd}qtbJI8L{wpyyFfHij{bdTyY9z2L)CrqN1I0T5ulLAQJtAEqiW7e~w1 zQD&ZxoWx9(q{u^Oyg`5MIPjj+GcaAFw`rM@n2(TpBmUDfK0kV(h+0~s zF8#n*$T1`SBieU5aiC@-n81$z0AnAd`^N2ltJC|Gnz35W#FT?cN`dnq+s1r*$p?%~ zOp(m^L{Dk&Gy3gHqHEZR7#o=HojLjp;{SP3W9^k<3Cc%k_i0u*hh7XiafAs!4mi088s=>)7K{LFKwmnh!}{N{6zK{iH`pO-hWXP2FVf*bxy)| z$GP$DCp>7%QFXIv0Am`7AodwQey^lwj-K{)w3QX)64f;cPO^riiW^iWWq>o}bNQdv z3}^O;{P+1I(~_o;k|4O-w{FmVmPdjPo<5}1^s%UL#uSsbFf`=J^9DCFvru_AAu5s! zRdI%Ydn*F&00R<3`~E+_=eOcKd(VFJ5g#X}%2*vEtF{bd$=vtEe>h~<7%myL0!c>4 z%bt;sc#k7?ff0%fV!|T1?cJtnyV#+CPECD3G7fkH{m`&uTie9RD=KoQm`EE#1bGV>D=cV z%r8IZ{Nf7QQm6^l`_@w>RN+*>-w~1~NX7>qYB}E?BO`JK(s=7;3Nmje8^r;uzA+Ij79DLJY)vVd7Tz&A(`ao7;-lb!pSFHvXGYb2j>G zQ(8k21^_tsA2tf+kJL{?*I!?i^xJPbpCaL0*Nx}p-bH3vVlUL#&WbdyNwn8RcJQsJ zX&`JIroWwAhY1To=ju*Ra-8pyYKMATuTFpkKy?rH%6)4K%_n0sWdq5)sH*VP6 z`<5P2O8)AVD#-N6xT8*z5cKMyb4;kyii$`Yfn*U33XMLPKA8E1UU$Puj@+hOl$lbt zx|)wVPNd~7i2nc{6$nvnHtZ~pKS8ry4e?_pdq>Yv{431t?>Rkt4qYv`}-SruDk z+ypL!prNW+3Qi^@4U3YF(=p-a3(7TU)T^SSl@HP*@|Yt@3JeJHW`9%esn({)O1glc zoRADeA3FkIALl$XxV*Pl7mV2AIX4&nScEfN6S-QIs7;6XN!8DDQctQ&fr4GNw4|9N zATco#45)MJ>1+5_aMa**G6LF$XHn`A=1;W82U+tbw_Qwiuw6P@NFGRyM__wzjDB9D z`NrAPtBpsNa%$Ncnu%_|4$RT=#^na1NlhKoF!G8gAT#L(CL{a#7nFz1==1v2!%|wc z7L8FAD60=nq;`IgU}`%RBp5Oz@XDV$q+psV^?Mj3q#2M%SSLN8`%EpHlB zaRLCCIT#<8a?L2-)2=#96XzXAw&o*n%sq%CjZ`-9?w#`wv*X%lYldqktxP&n0^3+8 zHH_N3MA~L5uywqqe69-Qm{TF;+6&_%mbKAz)bz<*cg`?kG5LJ=$vjC`into53>X{( zIlw=>```m3D;+iaTt<}dPDf5TFC^ZMIhuGD^Q>FWshewIu?6=uYh6>C1tA9uTKfy6 zhxZTedb-(BQA`?)!b7pgQ2>A<2m{re38+(lsFvRIu#;mY<^m-$**RuTsM2T}q{wVz&b18T_(y$xag6Q`Ap6MfH

A1=4a^)GX-o$`)PgOQ5#uZnutNABl5hwd~mpHN3g zHS0|K+ew1D%}jS0kdFTV45dW9Fvg23&Q!$XCRFZZV>p~|sa)C7s3@1XsybtGNDwm~ z0h~cIJ1hX#7xf>c8C-LfTF2vjuZeDQ4nLf#%Ha2ubVh508z+!cxhrj58(z4JQ(O&X zLIL>h!YwO$Gzs+!6Qsja8J^Ra5Lt3W;v~kVg6iS4u%|1YmIVI*lnKvq9f_WnL!i2S z6XkqE2P3IpQoU*DUCY&$?2ci@_*>tj4A23Sorcm=>{iQ=l-FCrSsd*;MP^@ z`s!W1J7O~668QopLbwFvl1vQC9vq>qO4^M|R8s1pqIIozG{Xi=!7tQuNS6fdq{uuq zU+^QLzx|jFv;P3_Ey(`>_9w?5jIsX!;_Lqa@_+shh2M&+{{Y&XU;bDB0K##Sw7CAB z4#Vx=v~Oigr*botPGa-4;=IC~ghY^%S~4nu?lI(35h2{ZdII9mSIZqo<&Qn2{Jn>j z1ek-z8?vgj`kl@UX$C-pp1>R!{$ozUMVzYct#Zx0%{x}B-&6EPSQJ@CmvHq1E0n={ z06W7Y!}0+I9lz?Lsb(o^IzWv_6V(|o0V8Pq$2-9!E9!OiMvN&VW4=502LNt&?>!|I zaw?ZIo%|O4djcj2L=@IBQqaIrC2Vhiz{F3uAYyz7yyvt;gr=wtvLZ9I#P^?UV?U=2 zXt64F5>BI^Xb>aXa!1tuSTS*KSC7P;8g=f+AlA~YB~rfoDcz*~fUZcaMO{_=K*9&c ze_|LP2Y!D&hO0@cD#VOrM%$*vj<#^Zebgdni zSv&@nDpp9|YGg&F5E#KIyclsSTR>nt2+Z~jS+bBz1_1ZeOlKJe4m^zg2O1QHI$>1C zGuVvJT;@;8H)tnF;Iq?SUsr}@UPm42cMs|v{d?95?w3(@?qpe~lX+im>a>QkTnMvC z12!qz>}q)oiAcdiK4pSX*$zaW(g$EjBi!TL7~@KkKv#`3A(Ru<7*jh9fnayqL~yFw z;G4Poyt{s%HhoqXnwLbi?6*5UgDka;Dh($^dg8DlR7!>w6a7i@X-SY5VvWv_>ZL)( zp$2CJ z>FO%pa(MdME6~#1mNinEZSCsMu~pvpI!YySFlF%CcC@G}fT5fLAVeX?F$MwLFx#+( z(oJYuV6K{Iz?}mMb|44`ZsGvk0gwTAs$Wk#Ssb3%7(EKt8H^Qd>t4pzb;~zomMp1j z8kK&^pK8cv$Fe|!!^MDI5=M(HEM+>S|Dy^mu*W7KiWMqH| z@Gu7yu9I<2E{=4vUPLeK>1SlpVeUF2M@vwu$*0Rs-s*6H_8 zdQZQ`inN(oI2kDATMSmnz+4C`Pp^z)X{0e`SY4=6!xM@fSblwwGR=wIHoY zTsdGA7d?U;G>I`g#wTtfK8vt2vk|6qC+#v%)9bM*XY$wcbu>}T-mP`!jjAd-C{}tY z*O#0*G#XXFTMrKw*JEVk+1#X}XUFA!wIN=RT1V(i#(NBz-^Miek;HYVs}Q0wlLvVd z1Hm0Yh$0VT3dG3jcT4SRw*!w<(>qn6HX4k~R=iRpO+#RxZmOkBl}yRmvD}Bqgn#Q7 z#2m+!R;6s!SwphoRzWiWuRu`0^2nc<_=Wd?Mu4x8Y1>6GW(YC4j1ir_xQHW55~Y6k zd1h48uIN#|f)`-)E!w0YRnkCSw5fy(BNzdnHIWaL&x|j>a*FjfyQByR{VGnAIga2& zlAxF*#(PQCP^(&~V9E-W)C8{68{-+C^X4|GxBz+V^uU?&VZQy zyuZR!Ad&X4;hr(rqxP8}oip}t%CEjF=bXQhSy>ya?qOXhQKo`eF^zQ&(N3Cz?!k^W zJwWh_pL}&^@P{PkocEnw(6i+{veu=|&DFzbT5C;p4J3TU+o4oRE#GjnT!b_D?iNoH z;_Nzgv#vfORHrJ6f1kiphev}tfDkj6slCH z%At)$b(ZZ-3csomSg>%k>7pkJX;MutWytgBJ+pgS&VG6`t26CnX1Uk~T(2IM68)(L z_?nV}NP1+uF_LA$J~Fcdo7Q8{bO;mGq%+9^1_787cO)@U$LR<9ICEBCgbl?;rphX_ z)VO^jh%T$3mnyC56=hZ=q!mzPf=u33J(@e5nb_Qyj|B_&9>5FrV%^*Fl&amDEV61l z6ziexT~6ZwF?LOCfG}A6q;qanZFONzkpR1?C|F2FoS1E5Mj+=uuO7d!=J!-hoy}If zqW+$VqFgVeK~!xs){eUp0Ejt|RaEOQF{^KhH7s@NduVkLc6F4BYtfR`xSO?sf;C>i zaEw^?GDFt3Elrk`#QfzKoTaQQy%OKKht)qYVJ#qdJ(my;dltBI*S@-`Qmw*MYEN0k z8;WERQr|!{^$wL+oHBuvnG?ox^tU%FH?BM2!CHdbQGJybFvyjT%m0UB}?^XmZ+0**>K}$9W9X?*vb50&vYI zjWfl|r-ix4I)g^RDYP_-j)L5fLuV?>y9ru3C}fpO0-*6Aio~)q5;Bi6=3L6LXCPuJ z$iXIXN%}~i8OFnop~BuB1D{l?grDJGQtlQ`UJA6jVug;Ak1(5 zdnnJ1){mrHJP%N~-tigMXL8Ea7$w!xtOwo%r2BYU0GNgWOnCP010gX4HE^z~>j91k z_YsZ6Ph$ehJJc+%_`eM zy>36;mTymZSd$H4GHV~$e1>}jkjzQI#K`_T{6|{c(+$xB%ylnuKW@^q<^kg4VR(j` zg2)J90Gt!p48Sr@2tG&>6`8C-MVX&sg`m)pT=;j0iJuF?=k)V0M#A8J*VJn z8y@%vc>2tKx!*mCxRxVOI3GjrzMmW9dq@QF+`YyjOqHC2s~AZXly(sr{QhQR)@2hv zto{qQ{+*^`3t)t1bMzn6S^i@_(Z*D>5C%`WVsXw=Y0f&}AfB<$2w#hUjOIKU1OwQo@e})u%oqAgw9n0zTk{-t1Vo=3DantO@$DeA zl;}N1(yB;NEOrnxj2^^ov}|~^&k(6yaQlm`TxYt_30$rILVbh0%t&~YcbV`0ulJFU z82O;8;|c}|i5^H9={X)cu@FY1!zwmv)7%or^v_|Ba%BBU)5YqBePz@_*pMRugHa6s z08%`A!Ay)qb{Q_kB0c{AGv0q+&&XP<7sYNl(;N5C>bd+aHUPwe(YzBlG9Z!M zNHf?(2I9e5sKB&*MW9l$f}bBaGKuyHcKpw8$LIE$?H#;(6c%YzP@ofz#E%WCY-uV#T2htI?H$(<`aq0%NP+f`DW33C+px?}X`cQ5 z5I$!*vc!SlDH%RDfPS5bV8b;!ROFS%X`b7`>NAW-nS&vC{gs7TnOrn`c)^#7yG%iT z%9%2s^-TFp2Y>J0CO;B?3Sb40NfWD2eB(HX`u&Fx5lyyJ857kXc+@w>!x1O)uQ55Y zQi(&2U@}BeFhe0RGvginMj|8i{C+<_?c@7O00saeak;?ylb+KXdq`d)sYEE$SW6N# z=NRrVPrleWi9B|Jszq5I#0)qw6f8sv24Y}fcX6{{X@sKgg@`r$ydn_SP!oBuEaXP?#JA+f(1i+M<}0ufdZu z+9o0*e0mop<+MdwDOpWI%r+1Nk@Ib~aCh)bjyaz?zNu5yY9|3nB=#U3w-Wvt~0a%DuF*PNjLP zaX@4(W_BgDKAYDk{DGgj(z6=N^5;~pD%jr&Oah9m+F7PG4yOdJl>`$4N)iac1El_c z%PslWEEQ`z)O%VA6%METLkA?&?p~^b3XSv)GBQV^t5tFI`1*NbzS3$@Ol;U@MAdt& z2#eWT{y&z%;pS|AY}uuXOctF+iY8u*~{E^$tU?O{&F_Oja5sgF*bYR{&V zoDgNYW*`Dgtd0!lHa3_1u+^z-x8}y1UDBsTjV3K^Pt*!3pk|Y&NmV)tg)$(ooFDNv z=rj3;OtL!L!ZY*nIiCUZ$=nLAvG<#}c~jwOS}sd0D#|alCwY`wJ?gnt)PA#9e~f-U zt#QlwwHvgp`-`Hql+zkmG3wUQ^CNAj%mIR8c$fBVN{2D!_tx*~t#BIYsNAe65AqErsO;;AJ+Zbu+A~#I@70qYSLI(~AfYxmMEJ zbgIb2M3be1VfbU1T_p=z_BRznRon*YT}=cLkUFFg**gd#IO*KCmLgKEY{^NjYEfh8 zu~m?YeL+rhtRNad026_bb!{Is)znWAM?R-+g>>n`%MR9YXlGo#Yn0C|(x2+*P&%RZ zs?LX-tVFqj1v4ho@q=gjdrWri*t|b;c}y`z0!e^K+IvTd^FDASadf(?LFtekU7`eh z*!o0(VkBe92Z);5m0OV?eV%pUm3?`+;$vx+;=3hMXH|YukrtG|!O4G2e51dRfUaa` zbt*8@Ol67QNDJHiL6rg-N8uR3${`{n!EYuAB8nP|juz`pcq~LL#E|;I?HM1}y#6LcrMGgW{U#`a8MKfD_}k{h?ciz4ak|m;NSH88 zkux7%AjIt?_#h9K-J7c?jirt}4cD&hvc;SY^w>lasgIiUJ++-A%Y`eCFBKTVMtekl zBgD!kr=(RwLkt)iex1DdGZQ(>8Z6$J(M`i@B)AxmKIS$N5vojx)5k8wjr&+RH%AXL zwP~unq*${>sIXciK4V?ZNv%y5oDX;*lkxIg6v=sfVwLV_B|TL7Mit1@B%F@NZ_~8o z0%=`TM=H%ZT~YuB1Q2(?+p+i9aXGJ2mj~jUcT17+t}?D~bHT--gohccIdKJMY^qL4 zR{@NlTaV0=uP+|aJ|7A7*1nQYrswjBCm70;f=6I+GBtH+Wp8vHv_k7M5J->=c9SAT zJ@XO?q4Azo!_;??#rYLF+E3&|X3K?Fta4AbYKYk9-M=t|Q)tZ9LWo6X_>+N=8G$9Q zDbZyP`bxGpk-kTH?X+*a4h*MR=rY!s0kiKQ$UUI!NnzOFMo<^Ju=WNEPjsm&86)A-s2Ox=n%NUU{ zgNzUB(sIw`)g8|`eP{INqnxvlZt`lkUx=L4V+OE#0xsrgTst#CbWHr^Y-ETk%!FAl z1DS#EDAF-dsXM;AI)KziU`Emj$9xVQzOemEbP~T7 z+x&1Av#ni#E}qU(NqhfBfJN~KecRV(DbWv zuTj|`R8oqw5wj5{cMt*oBf^`8x)K)lpc{dyi7T+kQJC%R011@Hzq|TV{{ZJ!>L>pI zOm0*E0I@zS{C_Y10PT7I0Ocuvp8>_!{{R(J{wMzc&BQ&tYR#K9{@sn!vd;xsGPcAa zbJ%-K59nmjh&lGa`s1|6;(mHpiY!NHoc(?HtM2gXnTY#r&Q14QEwZ=WqfVg~f+fBD zXYQUvHiSe+uuq&1VHt_<+vHU!({Mv5&NtqF>u3yj#kgN z$Fu#S6{%3rHWApaX4#*+hCmufJbQ3cY)w}KU4PIq4lbUk5e-dgBgvDKHlfsey%ROr=>c4cy%uH z*{aqGN~vv0R0U~ zh?xb!EuJwS-edFP%7wB%mrz&$OhGbpyzkyHHtmiu%|&EsaB~pZEf|s5cRP5*Yb8D@ zuhafP(~d#Y&Qr;4@NBvj;fulI_a>lg>Sm~JSJBh6K!(-a=e$%d%CO@|x)(5$A1@+n zTGt9qGD%^Abpjf5I3(>7Nb}!M5jAq!X)psyf(a5L2!jk{z>%g5$<-ow?)1;D@2`D& z3dC-W4lSO#H!bK>m-0QR&QGr8hm@2P#q~xJwV- z+j3w?j*%nG;H(wx=!6=pfb7FaBmhQs9YTK5whZ+5_qk^S-n*Ht((XrBSvf5wLvhN*$B4kF~Vkx3% zR@49kP(lGer1YOB-?`H*)0U@@Rh;U&dGz@PMU4dLWgWHWT+XzmK~=YrWc+IC)q4b* z<;2U-zYt&)ND)%Gk080WthFCBn97wpfS4Lg0Sr6Hh#=qw3Oy?djaav-rgEoL4Ywm{ z?j)!JefYR+eLnS7YYw6ECREtz^|}vvZ|G^#jQ;?AU+dVe+LXz*LGBq8)oT(dl3OAV zy!iZ^@pd$=+TTMP%4BI!1dvy8AZ9?sE+@cl9{&Jga|NI-^lG3~F{$$aHUxLvbrCQ( z21w(dfpTj(9FI|W?V)F0#v5p`X{m3gG_R#mR<*VbXkuX0u?q$a7kG!Ckp5s(RlN^1 zs#9PwAQ_DALG!Y^Q&z^(v2qrrMW(^(QqumJpHK-3_0C-70_?7~ zowz8b4?=AEzTv*H3oeU2m@pv18EtZ(IWvTesLo=3d*0!l$kmpDKwN-2;E5e3AY?2{ zbd_B#`8~;L)2VdrO4_2^)pe%zDrr$ru%)Ou51@r`?iEc!bW?&IwIHKTU6EG(WI__+ zmClT5+^cU^DBNGPWpt^FQIgXtg#u>dXo)P`s3=f;Sk(fMRGAr&3=*WvW7Cicfu=zc zL{63M{G*jKt1U&KDJTh~vZ+wYg_R{P7MnheFY6M@LLpcP&JE*nlby^fYNXXp;*#L1 zimo&{y04Yjg`cjo-BxWbO2PYKt@4;7fPw{l;a{~&rLnlDR*MRCUfeAq8cF$%qZl$w z;Q7?Vi65eW=(Wj@J@{vta{G&K#!*ga8u6+yuB$;7h*71PWl<@C>70TgKwT(*qlHqt zhtK=!ZdlJfO8Ob~>L2nN_d#LVk1HB-l3fbg&zkm>(4+5taB2{6s57SWYV~h9ZKHD) zYIWE$)2H3!!nhwS9Atw%uw;7c;}>34&MYmtH494C>2F4rW^J_vS7?7wRQ2ACK9d2ETvy{^17c=zhS*vox>;BXoO9~ zm`n{j;=3qf?hr^%{*eR@F(?DWj`QCRRiZ ziH(5466ikv4x1$GY3 zVT0JE9In+ zq%0OEz&MiHNR#e0L>(gysQ&;%ol_1$)BNn~mtQ)#_Fa{1{jzcnqR6fU$sgEa(!eY% zdrWw?3-BcnH9lu*)T%QmB}U}pPq(+z&UPGJIc~jU`ji1^D+9p3AFZVrCO%Q6IA=AR8aBPk+z~ zrUQ*iG|7TM*!yGKupOo(aSzu@XJz zK-}avVs;QX)GnKHsplAoY3YSxKrjh` z9v5-_b3WPXTg?i=m0e3|)1M+q&UGpJ0Wc;=g$(sK=H8}ia}vld{;=-@*kUB62ekJh zOia&ci4yM}<~#TKvS<|EOtIL=Ajh;|_|LFj{2GCzGR^}7gP(Z*nLA_nma@$EDUEe- zTNaY@SoesRtp5PHj6_Fi5$)gcKkq%DW3+EiDnQjimiE?V~LjLR2Vv!Pt{tQYcf7D|U-XdaVNTLQm z5k3C^Fh3!vf}8+lfyt95GnoW$1J7~~1|>EU>p5*840ij7@^Lu_%ScUys6r4*^hrnz zcl^hBi2VwGbMlCe@iXHQ`TRsj!A&_9#+!~^#e zmYkEvV$1B-`%zND6Xgcq-Xy$xcU%mN$COF_r3Ko2Vq|CW5L%+ij>O4jKp5OMVCPS_ap2GC5_nqqk0}09%R#Ln{bhtO$rY4IOcaN2$%My@ zc+3FDZsQ-^hgx*XfMAlwGBR>AmL`73u>c>HJ^fmA$x_)W42Fn1ZyH8M(#`b88JqQ; z*LdMr7O^p`gElYKVwg!X^P^M4w3vu@AtRsku6#+65Kw)fhvlo4*@kagwCScur-&zwlT+!PpzI$V|H}x-D_NJH%vT>*(D`2 z1~UicJ^Rc|OwVBNJ~KVwM(fIyEzKfHVrn51wk5XA5C$M^oX#nCj8=tRw3>A_OagjN z*fIR~*ump0`07&aItq8{4Y%r+4`Lu0^7eL#?LX+B82p(9mxAk zeNLY+26*Q6H2}e9R>C}YBYwxZ?ca@~;oK?xNcw-{)@atv)8w2szPbV_nycB-&&rQ) z*}etEZET9a)3H64qG&?^WJ+LHa%$E2M>wx{ND)cuYtgGAv#w*LgFA>A?nJSbZY=9t zEv?Hisp+TyjY}XwAnZG2h!fK~j}-Ii=1*BTT_ValL4KMJm{&74ux-^`MY`;)!Y7S| zcu2@Te7wd>yo?Wi_`J@-_mCR&Xj1%3(`9iZI3c@0U_@*TiNQQqEcr`6IHIj_aaP%? zrlnF9P;C*_jgMeAGC&*zdN<5@2hs0aDQs*|m*LHiR-_$k7DAnO3lnDS)~-Oy5+F+Z zimPZ##Y>3t?cBJ}KIIjv+se^pW^HQvbge=IL@^yCXRPf%loDH_Tw2)Dr&8LTz+_Sw z4NIsvILML-1&Le}AbX#0f8o{jQ`66=J{>tqRVAvn6%|+rN}YxwwA&y4jM$bHJEiHX zG*WMM0z$%kBxAich5dUfbnPw;d1~5ZjZPo`0Cqv$bb(76qqb~kP#{%x+U+yJjnR$N(B?F`i`&t*5-L_tO!|Nz40P`1qhLV<7+y0YuRI4e{GaLr zq(h#p96kE>&XVa$T#?D;nabKDiUbj9r%qV%WwUjHmONCkP>+wxtFqkuC23yrg*vru zEqx00s*jZ0p{Df$oN9=Si~_60u7GQQ^^o;@XJ%Ifn9v{t=JlrHV*lTw;63SO#9 zt1#(+W*JXOm<=aKrN-uK@*XD~SAO2VAl296HWvHprlez2tb<*u+5QXghOrB_71?P~ zB!f!W$O#~io`UDxl8-U{l>BHbQ(Bi-e3r1acS>(gon=i*ut6Z{TrgY^NRuOv8gkxq za^D-G+FCy>0CR-tW4+0yE;_1cYlhLRl8uHtEG1Q6ercpTQ~r&9WNn|FEDOJ9$t zEHs#>-mjrrz9oM`34f-x$i_+&7YAaV|0L*gNz`mEeo?JsPgo)XU>dIzPvpIA zpU;nf0Ymu8=&wnfnq-nhMyQzVV^7pc(%6or=kXMomZ4yEe7)e9kk~sxh+=lZCyMS5 zH?xn5IGEgZm%~P}SUR-OH*91Z1Z%pmgFM|(Z?r}46q@EhOoz00l9rwF(+Gf_MW!ZX zMr42bzB3cSl&%UWrU@m0oaAq@)wn;TgA6c~$=2um?`HltIMlC4kKmSmY|!J`_p-3F zK;l@N7U`6mqRLl3ZPAxB$%Q38w8X&|b6M^UF{p$PLJfiQGDsvu$snDTVDY)3s}c!Ka$(i^y@2zW+0dZOdX>p2mrzJfUz>C(Y6-#y+Yx}lMIeB2%K&<0}+nX zo;dlZA<5-D?@yEQ?jvK0qhk_^ZK|iL{XGz=_&ZuwMzGgKoAzb4o}{cHM&C-}0AhXv z%-*d^p@&Mqf&JhKG9-`*F`c)PI3D8Wtww}0vSg9~?YGSsF$81|_$oH8yPS#kaI;VW zvZ`*mO9wwbUu38$hc_<>pd?reh?hQOB%ps$?cdAd!CK~#l_a0b0s+Yo24n8M`^jdG zir&2XDDS)l$mtsZ22YXpp5mE_z-)5s<}!8kG&1Zt`C(|HlHs+y+-Et4Xqy%gOEwH2 zl&pIc2$XpI-ls9WCrM{2L^QJxXp938AQO$mM-kPlbx`Q^%9R~mQ6zSewm>jLNfE(+ zhPvL`E%BXYSDi}iutl=kD9!WCkQgP_vd|{`*I)q6a9!Er6CK7oi@P3Z)oGeF*y=$b zzjPUiB$)>X$P2hK#o69f#VX2XF&*N2`-9$g&PfVwx5uqh;~XbLDlv6A9U5Di`bdx< zZiZahDbKa5RYbbhT%aLZY{3e2@MquoH_WuCFPufQn z%5=4fVn$3yaX(YyGmY>vID)v1Q`4zujmA^kV6SH=W~#&6H%bP}*$){tH2q@5uudU9 z9!v4=QS)`Gq)QN|#~5lv7}8vO0|(o`=zq(KXCUEcv^B=J9JBsn%(mK=m7Zx;v`u+! zy81~CH~|q~;v=@M%KapQq-2f9dBz9w7>(dIjEqPjN-)Zm1Ojs!unhi_ z?U10$b~AX^{XRv1mg(8iDmpdxIIlrdO_}48cjVp~~LLwBYe@iJ+neK$G6u z1_jeeY|ipVBjsM>`@T0hF6=I<^rQ3;K{Jg+2w0xeCuz=gaAWA_J+8MxsI&ws6Zyq4E5yGu8Xajq+ObNMRx^oq@^xYCuUJu1%D zyfW7%u1~pYk%|4LB<5qe0(m91BwDiCfFzLECT2<)>hB=rV>865bnQ^65V0iZRP0B9 zHpF8G8*n<(9Zg_og_cz_10}%(b|{!1B@c{A&v2Q7k1vx^uJ+)q zG16dTVfNKMhI+mH@iNm|jp|fxyonz+d;Fj;coQcCX^1fP=lPg{Bm_oHnN-A=>ERusB4go8cJ)q|k8g3~fAiQ(bcXrTnd#KTgWoyM zJb{BgKEsOM{{SEBF5dl{csf0>GVKi;^f7ARuR3^E5-DHY+DH>>8>T^I&@*Jg7=%oJ za~k0(Qze`My9te5qZu*})h9hblI>YrwRl~J)oUij`NU7;_bx|X7pI% zxP_I)!~s;apJu=&)e=#{-%g=6Rbot6By3?Ay~Q8A&wtfL)yBS{pp0oC%;T`cOyfRD znA5?c*U;&xa)wSLZxb<{v-OzbThqsKx3kWT`*zfe_8ETT<*nO z`laR+zVnV}>FC?z8rO8s{JD23rIlBwaD674?6@#~HH-!FwgEvLl!13CkDU7Q$a==J zBrq@>$7aSmNZw=_89y@3Y0vqUwG+$MyEU`P6p9AZAOZoBR56JgPBAgW{{R})=lpY# z4=dx@8OO&qwi9EQB~%I4R;ptuU7|De#Za(`H7>0&FevTi+IttocNXQ{s%QWcu#j3k zvoayQ{#-yjeW*A!Dlyj3K<%+HpW!=8d*gly`kb?lb3RyA^YZVNU6uQNcA8Nc6wxzg z&Vz|^P?P}y0v6-$mUt?EBl_PzW#0;3d}r}*EBJE8j=GokDz)4Q^lcQ6Ub8kz>8C2E zb5W3DNgTKKqx(bUf3u&1z99I!nbt(MfB0W9-CDFj9*5OA^Y+Iun@x$>FsC;9b{?f(G7KMh&@C&)Q}Bjun| zx8?S9t*NXe0ZA21wnG*myBrU41h?}Q`$7Gp^Do+u#ysERuZS4exaOS7z0y+^DnwjX zB@t<8QmaasM5c3CX(aTKpd-ja&4Ryb7ET31Qvo2jeW(4g_gY>8^eo{r!%2dpyeukFyxn**s-MTqk|1T}&bS2V16s93nAE_e;wg*})v;wtr~32j$UGKWwa zmI#vD6(7YixB$+eOr1n_zEfmY-V&`Ppyk%4wE$93y6M-din^%!ZmebMfk~F9q*N_3 z!u=jYcZln%@XCy}8E&-&*HoQlSz;AX2Edqhun1C3y`&+Gaeb31m{wkuVxYEr9Iz;LUYQqBR? zNsS^WavEcd@aty_p~P+0sjG>9Q&PxZ@uz-1rTrwX>{(oVtaWKJGbUgH2;7i2i3qEC#Tqu_m|87*YI$>0rCA3_{q;Rm z4I!ih>L~?4_P_&#ACK}({O-CgzN!^lGiHu&wz8+#nl1N?`5^lN%px7^^I$+4!HyW8 zT!C@Vi|PLW`AXwS0x2;7OqS{~xEYLPPUFerp?m{yWIjnED-&hG+tyWulPUr+66N)qRY@Qc zF&)0x5PpZilM;HIwJJc#1&BJ2X~qWQZov044+ig{{-)Q?>Nh3AP4#Q7C#)R!j3J@Tj+g?gDY+yY?Ye_`=I+B{3Jl#k=I!AR5U(=^z^z$BUa z;y+&T?Tm47x2fu?>S*{Mc-@cW z^DMpC!(e=4_ha&Gfk`jcaoSy1@!>LGxJPkh6nCGD=vAjzS~Npx5Z&-&zI%;~Mq&n~ z6$NT_Nd<@m9At< z+GaevL=s+UVyHDB1z5_31jJ0pfN?o8quYy@CFXvjtv7QYDAaa26WauxUoT<|My@nh zg!0O}l*6^7hu1GU#!u|eF$kDc$KaVI-aoWNc8QYj_!6%&fsxql{{THkMtpv|@p+|N zN~)0n!h1@8OpLU<6nMnLA%D3?c#)3aGu(d@J^n!Y z^neV(z#Y4vtj;68JdPeS8mp=iKmnWk!?^7@+j3;gW_YUdtjkMdrokN3A9vfaq%0W@ z@#W(a5ZIpO0orCJdl!s$pPCmt0Ayn`xa~3hOqhvTm4KDV0YEtL2-##o{WSybWkSd+ znJ(zB)cCFlT)NZ)1F$Y)Vio@V&4b)$w2$}kp54A~9g(S+jfpTxKG7e1?fvMK(bAf( zkR_f5W&p(d5kEpZ>IN2BoUXJbAiD~fXnBWz<1lN9?FT=Sh#~&~s4*U6`Hs=vejNUd zN^2$~GO7d-5+k&jFek=v+%KzJ(I!p=2m*7NCtx@2^~~wwy?2z$6*iQ9%PW`xQf3w9 z6EpjEF_?(&Gd=r}Ob_S3_Z*U+GFO{pq)xm4t5isz?=^s zdEO(1b>5ndW82jx)ui`9JmN|&@k@|wa^YSW01HW7GtTEZ56{f2+B zNq#5Cz(`D<2Id(^=;u)CCzc|d>&naGF;`2K%0J)^vLpO(5HC;$jgl~yvW36gf+ zCU(IY`J0{AdZxCmL<5`!i629rzPCV!^V_DD>OeqtlQ zNQ`DA=kYNy-Y4VkTBe<3a|2NTlhgk6b$?&@umbzX6)DsTXHpOpPs~7)4%_2za}Wmz z*AxE$m)@rNNU#-Q#{$4MCQU0d@i2&zD434I$nGmR1D`*{Pf4@V{*to=2LS!BXSeg6 zgiNgsiB@qt7#r*f86pmm_wn9%*#7{au5oSLkCbt3t8}vSHBy~=S%F!`@KCMk)N|in zsaQW+X0>8NfLGwvl&l~OhfZfomo5~XQerC#iq1?RazRohL68CIoc{pxgRN|rwNa9y znv|4EDOTtWh64~(l32;r*i7*O)r}qBrhd2{{!grUhauLdeYva%Jd_hmK{^nqgE)&3 zFsoq%Q7}?WSdZX^Yfcj5?tk`K!BO^en}S^w!Gj zV?_)=K)DuN#RM{Z0Y+xOU*ad{r4K9Re--jay0M{7n>9A5v<8q=ft?-|jIxkO%tC=J zOZdB&-uyMqDb%HZR<%7e{c6a~qM{m|Bq?SDgu|Y|V4Q`J{{RL~Z_7AmTCH)rdWAW( zxq31XO}k2lT^f9XgKM%}_pV6@En29!4wM2x~{QIr9hPus$`PX zsu9uy4y`aXZa@MIGjVgW<{E_+m8)-3kgP!iRJ^MRDtcLASxJ87h?rkq+Vre-3x>Ou zY11F&DO%IT*s7%qt=Pe$kxp5OIxcN z?MKt*?N`*bt8u3$l@QHNwJJlYScTIBaJY4chGaw(iISf?if?*nCSAE^0hu=Rm*a{J)*@{&i)`sd{u;o3#s9Z=H&RSQ9fL zi~y_v@Ie7;KeH}h&psCBcD5f7H{v;&zVxlGGz#dH&;=JXS*i!8NUoAg2q#HxBo#h) zbusHF>wl~pd6$B~ZSs07tm2gYz4ckEYCAZc{{Ss0t_4v2g(k!DvFuQK{Qd>-EwQ)d zuTI@shosO{9hjJP#=tR|Ajr&YSK1tto7A$ssP{Eetq^INm1*^|jWY|rq8b5MSQ{A_ z*j}D!Y$JnG;!JSn>LE|wmg#>Xp{Z?F>BUIBnPts7d6_abE6iqtTU@|aGn@ZfiVom|$mHyRvA1tBqEw{%@Z&s(`MZJZg zxAbi7DBIGl_=HOK)Q}ptENYV1EHzX=GZVlofpU(e(dRi#COKY9VG0 z#E=Oa-z`2QrF4{mE)~%LQ|vIK5Iji1_mWiI)NI}UMWa<58s8niT~=z`oZdESs0p%> zFTV={zfv1;oj#j30wtut5t8gaB&xSEq|6W+bwL0Er(8(L5D!q;436+2lrv6XX;md8 z(;#5L&UO$_QT}l;Gs6A7$oY*fM{2vU^ir!yTw0-A#@(wwZbCA+miZ>b35*gDFZvMg zBm?0S@JiY%5JD>MKPd#pB0HTMz*_vejF!-Qs<91tSgkPX)#W5iu=ROFK}^JX z#D~MIFH|zKFhve$)Gz@t{qZatNI zXR?6|WXwz9Vh88?GDu6m&FUi$-P8b(2T{l6!6VKNM#t$KE9Mkx`t@9zY#qH)P(%cU!9 zx0~)|IBMs@sXg8>))~x773?s9DIU=te8`RP? z2Gi^S+i}ukefv!AE&l+=;(y|e{{Ye*{{Y2&gYfrW{{YYb051Y-PapV`{{SZuTPaph z+qY`Qy_(@HrAl(aNo?0eiDO(|$bfRmZ2p{y%rJw-dqj-<_OhvBU`$5y^uZm#Be#Cg zctR?=fTAEwf7jc`>&FJY(#m}ePAP9hR#ae(7HKZa8ch!iRn-ps1&6^wkB@Izd3bn7 zt4^T9t1aG%7PT)`gubGPfVN!Ncr8&zjdh3YGYP6iCSo=!!vtSRRrHAFq+Q z+)f6=fojxujXf|y$DclT_a96W%XX$syqDFs14(t7`p^DCo2yAItC2(ttIT+s<|DYJ zqD?Z7+^5PuExEHU!TW5J zjalRSs-IPYt!EP5jKF-P-1>FxE4eKYmz8sqgD{zonp{+KOIsjev`%y6M*ZS3*sQ{x%ejqPGf}Dh&Ey6c z#;kxadn$?FtMdmm`Y`%@^4~Vvy=K*2Z9%*fN|J@myQOKC?CID^wAB?X%mffuj_H9h zSjx&IQC{;3i%e5WMNBB2z>Gk^7~k}gHeD|2)T-Kyb+OeL-)sopCSzgP$l@QVeLlvG zwCUn%RIhe&(WF)}URQHyXIrqvh3yr><}#Qq16iDxsRbmq_gobGn!&Fg(d&e2h@|S+ z#xf4mJDr4Mr&NUZU%Aq(qo`vsGGpI8fN0yvmNxWzp^;|S`J)PPweazudk!@+ zwR07&LSJemms7dzC4-dDFz+!PyLJ^fOtnJ>0bFJd2=Asbu@l^r+-<>}n$4?MTJ=gb zB%MK{^o$tWX`D+5Vz*K1Cfm7pAmPs7I~nb(ouB-fa4f6Z{C>tpf{x8y<>^2jdVg+e zk+c?m)?tEv&mb>9nX%-xd4jK_Nva*#A`%y{)<(x9fdW9^2T^?Ot`#+9lD3u#3nMXk03IoqrY}GZg}2RBiO!Rs zatwWt#ma#L*j5AJKIhDTRdWz05K9klIa+~1Qdv@5qdti#T|B--a{>!?+iOq2fL8o#rf zHU9uh%u0Q#tV{?q36T|5Ldez>DuQ&$a0a6qn7|W*13H44*e_M&jORAbS=p*nrXSL! zLD|I>dV-ZO3o_Fv=qfXn^q#5FCAHM0$v5La7wxXnqIalPwrXuf-dQ|_VNfhWn;&sD zMns8lNfpT@c7;&EUsS9Y14=Pj7?sA7U=iI+0fv>5Cyuwt(V0@p;mh68HQJlFC)UVu zOsS~dK`N+yU>aH;DZiw|1UZKmt<0|39hS1SYA%bbUMR(DBI=D1vt<0=*HfdhF!VuDT)S_BHe% z-77$;lU&G{VGV;C7drmUyXAE(ndyU8)+*!@wKhUCA;Hd^Y%vlW-aM0;HMq27}m31{T6fTnLb<_Z_MNX2VZRd>a>Q+2Z>MtA9W9in@l+V7Az)LK; zEQ8oK$~JJoxp}satFME#BuQhU?4w14~^8~z`TukHcio4b;?qv#v$jFLY+EYcfPQ9=@ zi69Nc$J5>n24~8-qtUJC*3wlIpor840~3M?@C+EpIq1^Pj&%ih)%MmJ3!eq8$?=#b zxIB;92j93E6nOX#i3xecf3*Ck-B1hw(gyiy15hWIyZgWNos+J8bJk6=MeL{EPu6)BY55e7Va z`;tFMh{-U`lDw*%F~Qr<@eKQP&ODjwr}DZheNto+{Y@wX39aMg!59-0iR=VLF*7CO zK3)F+S(%CXr6-Rv3<4)(u{aqq`9k>2OG4x`7TQDu{vXmXV;)FVXHaeNEmiB}0auu5 zG0B)DgMTt5`b3Px97l-u`~?1Ew|?IntJAGSQ>XzTPOXZ--+3p<=>}ne#}KG`RYTJ0 zGb}clGXxK4%!xVv>o=Vt`hXdzlNWh1w~Fo8BL4uH7$K1E9sd9&G1_~aK}X_0J^P=^ z{pT+liCF-hz`^kwMq_Y%9YT1y^1f^cRIJ-zOMJ1N@i-XTBLI*KJiJS)xjOXMcCD#f zRRC;UvI4*y40*sz_7ePm(}>GE{;)m%DmKL!^%W{iZP;(`Gm$X|69s+-T#=7snTd!mF5l}J^6`lL_W6bNl#n2a`eVSz z-|BPNPf2v)n^O^nnfZ=983V|`J)$R$pZrkPRhX-ptX?Y>*R-f{A9VS5m=ga0d0+ac z#y?mXk@AS`^H!%`r9~MNk`Ir|*)V>-_=1e$jayA5)utG(Y2^ICZSsxh9Y0BqD*HUr zn=FCGg6$go$kQ+}GJslF_%6^#+sC(Ik(Qt5%lnK(L`VIz{io!gf+Uhm z4fY2Raz2Bxl6(G8tJ|3|CP~2{K!Mx1E4DH~_5-U|#SWwa8x9&?5?`nbmLVfO;zWdJ zyv$E|?J+U7A{$o9e{Hd;MQKF6e*3v;DI~Xtkk_evqhmaZo;Z)^)X|ulN5}1C=6Y5G+ z`73LP@%5-;2$u&F2HO+si1Hzch#%FBD>4`Yq=Si`+xWoqw9dymY1jPV6md<=4&BM# zGKYVcybxp^Kq9}??n`-#d+o82BBn07EB@+A{FxFf{^Dcb@Ww$&{w5>+UxyTt5Gp8u z43h^ZzTn9}F4@M}idDsDs7txjGx@#56Nx#T`){`yNA(BDS1Gd|=zvUl8w_S(tQHD9VX*zf)02V!#s7>NGxB(^N| ze)G2UCYlsBUgI%R7bhOVpVPPy!FvnG^9qjAU)W@#I}83;7&xb<0Pc5UOhE1!OvHeD zf_PPMG^^BR2%zD&3OATj1N+25JI(>(y>o!m;aC`q1gV<<)jH;tl$gX>x_^QYWQ2e$ zP82JZnR@56lpJ^Ya<_Vhy)dFQx|6uT?Y8GJ0em-oG}J)rby>=VRd!H%NR3T6%%8aT zU?+wvpc}lKi*;j<>t23(avOGYwwGEU8D~Dg#Fu8Y%u>5nBBV6K*es6AV1!9X*ghk( zOU_X2Lu;X;ssLbyMjDAHc&D!sjrxc@B;{8WXN_@K18+w;A6gUA& z(q+_JP(lPAf_5WrD8C)^9!~UVP`s?kdq<%yX@E_2EXI=(AZb&u21tTH3N(s7lX(Gj z)9YmvE8VA3<>qI*KBaunu&rm}J^Gg=6|Avz+b5ECiFJs=*-8!u3RY4)%l6gCDcajt zyfpoKRESb?mbrigo=h>{4SgxPCOY<()Z3^>SVr*yqz0I2ibe~mM zYSv$j(=VKQw4ySXD{&dJ7ZB0=clRqB3!6*Z>q-KPaBt z=eRssJ~X=Jm-TFIY2B?`Z%V(ZXwu70wvkb4F~|%6R1C3|fTl?+)p)dgr)N1-b~I&n zEjAa3i#J@LsMwOSOwa8x`B(8xH6Idl#nReWnR5`OwpcHTs8PLe6+8P<4{niuub zy)kk%t1TaKPHm-!>6jKW7U_I;FF7z{XS^49?=$faGPJ2}Q%hDZWN9E`M&f({IfgI} zb{r_bvLdKgAwiNLDKqwxK%KD~M}l5|hwf=zCxxAQ4y^mNAhYxm1_y9Fz$X#3brJD7 zwZ5EeCjKW-^?0q#Ke_vAv{veGOdA3hp1LVEd!p>f6h)P?20uOH@;?oyMymh~Hfm(a z)C3=-sN2YCj>mo&SUv5HVN#3LP((tAScn9jC4JnwuV_z*QX$ViOBf*`SF+N|h)1(DvWJSaUI!1hCVBkm?&a8pMON*PT z)Ef4!M|B554JZBJ08#>x7%*eB9aUrGyxUovOm^wL7`XYWsc!XI^IB1O%>LHfYm$#e zD4$wpS=^l?EPxS-kDG>2r$sN$2bp9D$*zzptupXc{H zUib!gOZjaMp?{bw>`s;xTF?d8&ur7NLal!*8m~Le39RMMh)_ajxtKvmkDqUhEvT(< z!P$%H7*#DYAjix%_V)_Y3hs?(s*CBnHi4avJQIiqu`B?HnIQ39Jp4M;Yw9TUOOHOh z9bF5{wui|1zzkiL#L5Z_w`r36K>lE+3Jqm@I>}ZAH`2ZInPCDWe9rwq2;Ad|g(|K< z2xR9O#O^RBeWjv*G)luzM<&M|y=+~dR~^oyVr=PFqg!Ut)HV05_9bwbOe8sj2CxY- z81LdE`uuHgN$FDs)u6#3lZng{7Dj*C?HR`e(xy>bom7&|5PQZBK#b#JC&J9AkNo?{ z{{Z&n^1uE7^MCl$cw0NtnRKhJ~h&8xFs?HhJ(2>s86A->wt zUaH|C0-2yF_QUKDm!yc~d>0bmBJJPTfB`T7lRhNzLOniA;Ag++-~8U+n>v21@(jVP zQ&R1k^y+1uS^EkepSr(s3qK&QFCl?>%LS}iM30D1gv?Kfis_^b<&5XIu{-|&F2-kq zE!YspumXLeW<(F^jkx&X97CS<^O$R8bjv!U6~^%P`917S>veOVdlw3*+PL|$=c#t? zD4FD{HQf^E@fCzjNc_+ZK*w$SV>!tBc!}xL|C$NnSy&YA}zJ~dwUg$(pZ*%Bb4oM7ZRxt6Y2*gIrLi zrBzy*fEjSrBLudi6SlyExdY{U&llx6_&B-Tg3W3dLie!7q%D-woR1M~1kmC+d>5D= z1nqG?O_LjkvfB@BH!A-Cf3=$oPo^{JFWMr?huxN;Ktms?5kvrVD0lp%b^|+PVCG2k zzIfScAi!uVmQn^bIP7r-Gu0E}WBBcQJiV8-bT+f|sPSqQ*{M@H&1%#HS#Xrgbkkzr zu(Lw4#W!H5-7?7}z>&d94rxfy2;oUnjDk#!`$p64w*LT?jS7iW5W`MeOMYpu?hj8fZ*dy5@Z$=fBFa`yR!=0xFqn3I>XOR`>o?3a*x-jXP$k?* z%)T+?)vqhixVJ4DX-%!2`8j7%LvVt0YZ z_YCXqenpa3P`5b#611(={{Y3+9JXoc<^d`!SyH8Hs>F#p!%xMB;Y6wT_%{M4u#b) z3xBhT#&*X7ek;|EtMzBCD}M;)oT7(Ec=)*aHl`Ry8CNgDuMjP-p=zg`ROl(z5w`t;r1&|%kc-)R;u-zS$cGgs%X-Rq{K1AgRvl%aY*ER?An&UDdBiSwfG3GqE>CmyKMuoB>g+QwUn&U#mKmY-- z8|x za0><}pjUN$ArwVGNYw#ZS_qDo224(H7DEy9`TLMixaIZO-n~;=&ZRXeZ!GjGAuCxb z7OJ{MBz9FA*Ih<5*n%}upkB>IdzY&w`>W<-=!-(d`!;)5vf6a4D?sz9SMp-ysEX3Y zNRgD05^Z{$V3%r`3?77Q)snoO}w!#4aYQuDszZdHh<4QKMK6RK#g5fFw@w3avr*0Gn7 z*|6&@Gx<8nmt>Bn~-VR2{U{r;jTY1PtuoUpTJC)lG6 zWi%^OWh%8|o2X3LJ2I%B51dSL`ZraXYt;*IScq3Tzv12t;&zQnP6Gare#^hKE_pvN zTie9uR)?59*DtL}DpCA3sa6SZSzUzHr_`*<9T;ItK?G-^(%aohx0k7Nh_!eb`YgX{ z^mbYumMrXyxImfMiWP4n)0j$hO%ylntoCc?!Y*OSR?gMxBe~ar#0Xi&2n!J;jLzp! zfvNXD_?MO1Bl7B%5$=YSr$JLz6p0N2Jx7&=@uZ1~<1@aXxL%HK&0bALtj>KM+jp%> zYm+J{KKs;2NrMZuFH#}3*w&1k2sD{8LI=aXDzYm_c4yW}W0Md@`f^N?Oyq(@NhxXg zADyVFQre<=6wMl5nzRN7siEMLAmbww6369rzMFL|KBakYrn1dY+6M8g5Ea{97pM}+ zk|xP{tf7)fA?zY0ui*hZ~U8U<0`rwA_r`TS)c58j~#g_JlK`MUoq_u6CT3He03@Kn%FKCcenMzY~ zR0c2!%+7^@7(+RNqJ>XqQC$VnGG`@@FqNn@F#vOKMnI9U~7?Z<&|4V z)F5Pt!PGs0BiMte>CoPnbq0qcrC8N%WK-(~95jepne0g;D4FjwAL*X(aoe!@49NVe zT~TB(OqmDo-zE2){Jo6tJalRmC4p0u+c^6HEDVi?HaHv|thB9FAW<#?VoTIx?uP)3 z`F^JoN@&mH^E29HXY(*#OF~4LjL&&JQ{_S11_=KEW+9m2fI2}v{=M^z=XmXio&mhm zj1}~;!CvCp1KDO!B!3F~6XV~;d`(BEQkX{_ z=1xXGo@9&(JHYDFC~A~kIBk#r06C^}v~Q1pZZF(RuSb~rwX`fcy@im(HW)!&vmfd< z!9)o8&v_v@j^iDp%45nRYbteQs0VxxKg9-OM3Duy?SaLD$DNQ8o3TQ_nqU$q$vv|q zA7BWc8U8c%3x{b^t2P=IC4=i4?wKM!-bD}|K_tH2<|Jgpw{f4(!5+z}&f*LJGJMW8 zN7hHvJXy+CB~jx}brQr!`1kJKCk@3!I{l+eylR1;hxCcQ!d#d~=eR_a ziNH_$Ob>YP5fc$VCH~K*-!7mdx`CbZ-|eJ(%<1Fd&WPTk6X&aM0)JRBwl{|CA73?? z5`k<4n0SGy7wvI#20z6=Q{#bE6 zzI{+orLYX(>?VDu>L5m29ltelZ1x-+>~T!)AD-AzePb>f#8875M~uKKT(+`K_^%B z8HtcQ_X=WoNcyqK`!(tNhI-;mPDzmq0U4P!r@&$$Bge?gyiCvFjJtd;lC)~qsDC-Z z7~4Orn2d0~;-M&2Ne8jUMD36xzmu7rqar%@%88|6BE^}JvJ{ay?mfJg3@8Yh0`cz= zKi!2fAFRy$dqR^k%8v#&Y@XjK#BJVjrWsjvO5hWNj7$s=e)6JcF}}kih0}-h!n)qB zSIO(H+{gvY`heMwvT=9?YgZ+ZAwRR?Jq}-CYP&?i$a!12WtpZ0S0R{2oMnGD#Z+LO z`bZu`4l5ka?Wz$V4*0-z_5&Nebbsm{e zAOZy7@6(;c@l(9*YNLl`d$lJ^GVN`AiKin%>MRxU{wVecMgG)T5i^uDWGW?d4DS#D z^7VB?@f@FDgFxVTd%44suzaRbrobE zC}10;f((v5G1A1X&AydUYV|SG(`_}V>!4iXs}0YCl@Mu3Sek3J3~DP@Xbs?TTCz13 zm=oc4=H{EL2M~mUrzNzzZkI4iuEgEJ?_AP^gj%=&fC zsa~HZ;*$2}?{WS8<=3!`-O!p>*swSQzB=8JqE zcgotGb5io9Nw0bg59cv#4m*sil8% zjZMP;0Q}b)@wo(bxSbb_%2Flv7{Xgyo+t zPB!h1nDXS&<+_#h^cTxCl@=)tSGpBZzhh0%f|8(RHT;IvIaT&~8|nzSrhahlSI$!mG1U8iX2*T`;ej=fpBlC1K7-nF*0A$k@dnue=JrloMTcP3PI zqxm!!&uUv-)mw$SrUh516;;4i45dULoPr3_Bmv|{%f~JFXW?s)h{P5iEm0m-_YAuO+(_ZNKeQt*bj#aREo7>_qQ_xxvgxncyFbz8?6q zm448^9lf!>SNLx0ms};)xUMP#c#lqjrE^Sx)ao*mRWfBx`3RSMj!{9nyITA)S-fo^ z6B;X9Wp|xw7n=4B9D&w4!TOV`{ zZPu#knl(o^+;X`J)rJbP8m3IJ#$euTktr$cUlm((3iiwxBT9p|=fttbJ>$#`h98*K zZl|^+u0&H=FeCzG3==X>kszKrHF*yp;v9l`PanCDe|AZB5}gIL=N6KFqYzDxA`~V| z-V-Gs)QMsDi17Kd^INL5H9Cz%^vOE?0DfW<9pk+B?E+JyT7^JNZ@=`x0wZyfOda3{T;C7h+f>%UUS)=y z!I?UQU=GHAS?P|o!5wQ>H}hLozaiqzi}5S06I#m_1jVk>?Y)sFQ^|se8JHveWC&n< z8qmIah3FI*+>J0YPDT!VWR3A7g8u*&P>(~?fB+6gAOj$XB$7`3B1A}@t|WSA)s5=& z$uaY^aQ5=9t7guMFY)9rQl8Dz?RjKIK(}TD3>t(IGwcs3WDI~o6)p`bOf(~EFbP>8 znViJN*Z_GmFtSs=dKR^9y(DT*?4^hV^$F>miQIL2XK+ly-{Zx?IVN23Z7u!FtL#LP&IDZ%3oA#>e>E67(JeV8#dgvDt|SNOw0?=m1o8CQvU$Fr^KI+{_6g}{dWS69myHWufn!% z=jUd%cHeZ=gO@f?xitVbJlwY{#vln`BQYlh9in6Bt=Ir?LwP>l20r`$08^%p=-uE6 z#>IsB0kQWP?gwcpPGzewDm)YttgTt~uKxheKVT3hX%naUpIpiJhmVQDkgbwY9xyF! z$Ml`fduRTRVrPtuW5(UFoc;IMdCbDVD^EJAFDAx;_t3D);1uD*b3V#h`vHo5EDFR? z+6iV7P%tJXGe05Qxz6xM=ziZ^pQu@>(pLqEfDeJ&Grzypcj}S^125Q%wUNI{%Zi;v zSF+C~c*Uhj{{S04;_@X>G?$bSz#KMaWa4A?!2yIY$87D|b~7K$05KE9twul^%n9w> zo%8v{2dd@_Gs(eTohWWQepB;7pM_ zM16n~V@wH|-eIH;KBUE^AVT!cR($yr8Q%xUCL*ty;JHnP64Wh9>PuwN`Dh9DADjB@h9&^mZ{h~uO~~#$jan2?T)G6N0U`ZD#BI1B zK;IcVN#I)ComGKI#g@*mssg!C31~W& z0LGO#-#y^xAmd3b9aQAl7@GVUx>_2z6_^z9u~A~)&051;)aAp|7Yd{jESUikP_a_G z#0O~eAQFv6sFaeMJw|OR6lyX|Lo% zv$}=B11C#6cAb2?+AV=~yt;>}xV9-K*-a|YZY0uh50%QTtj{@28vD%0jmF#HK6}il zifPNqdYOkJmm=9N%#acgdcmlyH*S1p~J7Z|8)K~NN- zZ5PkCu~}JBT8Ol%1*c3X3ZfNM0u=Vi5Hs5ncNA`S@dXZFU@KhLT6L&j`Hx?HmQ?(&7jDtyw3 zCh*k?DAS9aICX`qFV^JF1IQfk|FI4o@s89u2 zn%kdeZF$>u>}tNI(o~&FYWmQ@f(I0X_HXr830?xp6*BS$a3%ua1)P zjXOK)(;_YER&7K|l@!~sU_?O$TM9se?fGN=kw56?OOjmw0PPPVv|DHNsWttrl`&4G z!Te9FTbK}EG@LL{;k`;4WCkEpZaYdQT4cj5%%Y=MRVikSR<&-1s3L1vwwa37Wfk;X zA8~b6Yeb)}YSK21faz6gdZ0uGW6;IFBP`BFp%8>9Voo47oh{|wS#M94)JjuQrL{VB zKM*O=b}4eM6;wg=tA#ZkUZq5&tGm*o&KX1;wS#ue>$H%bJzk>CiKxge6N=fOFLG*% z*oZdRu$Xl+Fl}G@d(Q6CR6e=S)LhxRDhBmBL$T6z6FS=@s3W8Tb`c^^7vI@(u=u9l zm0Q0_sM9XgRvMEB0TzJMrD_n>1;s|vfInDv&`YnSnHQ7P`&8@w<;=%Y)X8gEY1(;m zDRs>S=YmWLbT1)g?uzYO7R5|JoE*q`%b8YgThFaXrp@bAYIP6F04w0oaw} z^_Sqw+En=mCJS1nKl>nBv}vM>Rh>j3r~-xz4j1M!dqLo}(tQ06S;jw+VSV$#sp!$EfpD}U)nc0S-lSP*$yllepGi!|t>YfRhzSBJ55@eW*0iWaJvA1I&>@#9 zCnuu7g`ZgiHeU#{bnYCa!N_bZQ})Id2+Aw>KM3wHTn?tBa2Xj?ih+sj)2FMX z7@0k`gC{(9?n74E_TNP_rPzo95E`&G=nFoVmyxoncpDbuB~v6Sw~MbR~X`O z{kVGpEA|s&L}DU9%!Xv6zw9O4r|}*8c*T0=-IT(J#O5SKeGK?h)40`;QsGMA02qK( zvEC=n+ZiW+w29)=s)XJ^O4Vu*Q82$Th(9uk?;bt8M1LK-6qJO-_U+zCf@#z+aqN5f zJ-ls!o$*t_bD|r2wy93$i|bclOPOAT_}gK548%l! z^Z8u&?+{TECFTA90AHTr5}Rp)W@IowOn4{E zals@Ll3;fpCNaOajpJ?^+LT$Fmm>;_RE#}g+rlDd2#P|Y<_+GGp72B-F){moJbX$O zX4)n*ydUZG$v;84;BtfL21F0mWw!e2`$tlr)&5j8k#El2DU0(*^P4wRoA}3a=W0oyK+tX@vm&H^=vY>dG##^TkO# zM$;Q0k_l&NfbK^p5fKw2#!QS4d5-fl;}iJtCsZw%sF5acJd^AIm;hkv&P!v3mlc!( zpv&NCGGrXdCIm--2WUNBnmYP%b4>Pbsdi#lCrL(q$u^_OsJBSTwO+Q<1BE@=5Ne^*bs#*S=dCkT0slMPfFP!rScuFcNIcg!>x;$nq(M zwCoTQPCQ4q!q%Fx`azuIu|1D$;}fuhr|X9_0@6m*SP38*0N~1!1oj^`1jv{dTnp`A z4y&hkHZzLF$}<{zIf^!VYz0VFiyC6?>%?E16kv780YVO+Armj11$7$&p^GV_F0|yi zP^7|$P+AX2?q>nIc;b~U-1DcWMco>u=ww{_VEU`5pOkNsHq(;AN#oT&(^5GvIr?o; zKT{uJbU34>x|z0f?^Jblx9u35@L9vGW`H~ZE3(7yGJGjeYpIj#qk+~bMxmwyscO|= zFe3!B1!39%?n#_rNaMEKCrSehN`mB^>@qTOp8`e(>&991UnfT&>L;muPN|6tv}s=Z zDAsU_qHoUAEWKbClcPmUYPxW{6|NYa|*V+!H3~)O4OoRb&6G$ z3N!;c3!ZY|>HswHZ#s^Bj?dLTx3}Z!Ok+>k_Qm`YV~DF<0Lc0cG;SFe2%E{JutA<9@%x!<|~>eF);H*4dZ{@AB#l|`CLA{2TF~IrwSb-S+?rLeYO6S>8WhT(csF{Z5)l>rOFjOLA@2I3B?*Ysu$(#{d z{xqv&@OQyn^7ieunq01^^(50TR;@oxRbUbfoaZ}W3E+WSn)6-o}e`tu~sf>5viL|(oXp;5F!q)`kxqGgfy9vsUK_E25JUkH07=hC$)969GO)homn^NNP&xoXRXS=I zI(oao5=N2+XXYzAt*DM|sW68O>w_pD5%x2Ucz$KsE0aPPWg;XGTVDrWK2NF z{{W-r{LadaWDRc$R!D#~V06ePd+h{cv`fLwtSoAUG#^9dj0KW*Q-~rngCq3?Xphgp z!%W>QTw|cv7=vRocK&lM&qevF7(M?l362?!$NG3=KVt!PsU%j&$bql9aLo1KA zNF&F{d*&mG`(4(yPs56p6stZ#0~pQ#1P-l+bm3c3)ia0F4Q@YedL1q09FIwC@}&CD z(ezVz!o8zY*Jt8hLUbE17L<;@%nUZvEelyh8czCboPjeR@e?2CFC66Y2CgJ+dxwM&Zt zZG$e|k;d#Z~v6(ykUvxOyX;*3mRW-$#TlvML#RBWyvkXmNw230J*)KN@W=NDlM0qZ& z%PJIO1h`#50VEJH<|Z*Jd>uwl8r@P1SZJDr4KllknV6hJ?*{{m*5@?O&$~i~Jl@^S z%(I_|OZWj()jF~wOeHyxKX;zEi3GHy#C!KBIhdc7shWwj!uh0$?_JZiisAe+(vAa2})r-;yZTnFKs14 z_|QfsXA)+6dvD)`q>@QHwn^?hc^&qgd&d=>JbH0_Ek8E31}eEFHr8)j7vi}ofCDIc z?k+|m84*5H5HS(>KbtxR64QWV{C>Z$A938qqOCerivc#@FSIeFT1XmdAc!(;lpFio zbiu?W@S!gLaLE4vw4+LB$_`JQ;&Jbvc>`|q3$nW}%475~?;p>7@dGpcq!<)J`?M+z zdsW^meni{*^xL5GkGZH%^~qO=nI8jZvQ-h@MkloVh13QhVmp)X_rM<0j84rWRcY&N zMqqc&(dPs0J$OjW5i>_91%vt%c+H zENNh6fFQ<_ai1q3pYhy`k;eMx6o-=Cx9)a3%t7!ow;wh$?g`fl+=W?-msrxFs+0wE z=8zmo*vDk~uifpR)u@>D_JV_m8IPJ32N5TPe}JvUXF5 zI*B`DOl>*eZkR04R`n^fGLxjJP%{zWc!|z?NCE+C)?Y|NuUk2|{1+2GiKTk>b!=GE zkX@hSP*2j;*siZ7(pp~-l-2NTiI56LTLvR$(bE|=GMu2A0&f3nEL>sdn!zvUB zWZ&IaN%#_0b4jqYRfB*sxPUZ)B0D!}sjov@NZ)939RVfVP*bjPTP#FfYAeI* zjg)ruZm#Q7N^BJrxh`K!1czY4Siv;`k~Sm>(!pisH&&d!(XJ{iEko&nrDa=3&UPRK z!6mZ=KxiY4!s!o170^1_{7)5{OuNbV?K6<_AoL;kEl_wyVw9-x6r|{yKHiwPqU1-5zyyXIAtgRIOuLuccemfT&41Q=IwIR~S-E9y%jfaz>-bd3VGD z&bKtHO)hzDjnEZt6%`eh?uu1iOQiy8`8tN>mKiPo05kGr>|f5ke@BUP9Blg>ZxfPP zub)Eo35Bf|`0pfrg{WOw(*_vZ3Z*}0(voe40i662y(&hd+}e*?jX-1^l_%x_x7-<> zfX2g3lbhSx)a7)N_R!y2t!sMEXew0*%|Ygk%@tNtu8}g-&Il3!#}S(?4slp;F01(N zt45RdcJryOKJL(BrUN~~bwzbmzowNc^)_Y!QnxBAM>X1DKNP(scB~ZzB_mR(j>=_w zk|I`D-uj5znA=b->C}3?NIRJTkvKa! zCe~HDJGs+WsF`y|M=`oNHJhikn`1oRvwG!=3j4O&saCwYD`bk@ywUp!dZxjgAIRy{ zMXOmV8dnswI#6vOlpxJPEjS=ST$zEW5=9^Oeaz@FsJyoM zDymu8dX(9eQLL}l^+8HgC19&H&V|7$Q^NFZ->GWQ=`Tj1img%OnNo zS$T|N?-|;4W1nTl{j*es)vB|)uysI@OWJ8fOe`3&=$9ltH)+-xb!;I>n1iOI$P!HI zU?rwF^1Yo~j#tg>fGt4Lxk_$DXEiD&x=dNAE7J79dfu4V6zH=pG*Y?YHtIh1zYw>u zq`Ne-EP-S42>WXhW`)xgCY`q2u5vT>{p^IPiiAQ*ge^G%DB>Rxa=^U78n#&*)TTfh zPNS$!!?+PRGCOhlY5NcOn&0A{RZf)})l%ZptA$#bYP9Jy3Y7FTkf4o0gBbu3saszL z9H7s^=>A#f*|RztWm!?JqH^obH9BopVx;V={`V@l*R4*Xlr3sMq{3T*XXQqPmo%;M zO})#w{58D>VgX}AkMrxoq z%&RnlOA>6!PV)2{6BD%Y(mp;^#H~|Iln&+ zl_|ODC!huBoj~cy5>6qqL0vuRtMIum99!GS%H5e)@GQ)yMeEPh21w0H`gT0dq*0oW zwe~VzKVcF+voK`G$~9|RmyC%?S4cjfsKiPTxeC89C%g^29V1nejon(*`pAl_O<6*a z(uO8`$TRtHBT*!kEH|gw`rmO@gXz@GtyK!nxGb91nv&r)K(1<5)ZV_Von2&J~x1rG_dk%)Q7ex>As) z{$NM3_Y`;bWB^h_X(e?Tk=zd19>RNUc)jvYYfAEyg5IL8lr8y`#E~H82+0B<<84i& zoNGrja;P)jCISLP_B0`LVUFvyEiS7j-f{z%3@fZ>z{{Trw zrU*Vi>;Au}--+B3FgyPMfj{;2j{F06xce7#?rPhzNuMBUVz4kLy#D}3J%m1SGQyde zp5r}+JH(H2oMt1ue?B!XjSVoc5mB!7sHZPaFG97II-ARWNR_4@b7i0*i+BhsK^A3_0z z*`^JC%>pUwxy0>GjWl>Uicz z(EEW&B~FQgm4%NrI$&aaX1I*^l37fdM`A#*MU>k8Auy)6Z9>;KgyfK`27e?WOp-QO%*o5MeQc$yJWu^sw1X3p*q=2WfME3!L zm?7GyPl7`XncKk9p!VGD;z4OCQIGi>R>0k8g4*!5Y0 z0*`SIfO{II4|rHe#D{r_BN(3nxGjMr9)Iil@m1$r=%Z50pJoy@;8rZI#@Jk`pNJ#@ z7jY*40HC}F!BP|3`%l6x)U!lL-(Y4BeWZEMu$_oE4pm4QZOG5lJH{vejEsKzkm}}e z+qq3wR}#N}X@|q){3c+B!aEcf=6r~xe3>&pw{Mm^Y9SPN9u6{hpO_!lKQB;{sxIpg z2@$c*Nu7qQL75S-5%z5F4e;IHqRlt=L*FW?{{S#41w5ijQqb=LaJU}h2iiQ!A_$^p zK3jQJr7E?ldVqlg8;Oa`4Ue%Wc>_?>Rpz%u|K0F9$Q$G3jRi$5yhc7mfc zDL~uEh^_Bpy>)Cu4X$G>x6jNFkzqtln4fg`tyR{j86&(szJ%!lDZnupSjU4ClZiOk z4Lnn=U(~oIw|%focRHKDkPc@eVDPSTIkG^>uBO`?sf9z?16fP3xvNib~F@jfjD~f%lWND&ly~ey*W`A~RjU z(h^`8A8M_F?F?GMeaHZNipkBDlgqHLxQP(&*ujm&j?z7+aR5eGPTO&#)2tZPWtm}< zA_r{#u^*_wYi&)twsYx$-nP8koN}c-YVk61KFZy#R}x%Djd_%)nibS6TzU3C+z7fi zCIRhE<%*$M>_*-oN!u~8-c3HOg{iysT@_lLYK{rq7AIatT6XF=fB|oreO!NrhTTq2 z&@8SFG_x$~>}Q6fpAr>;*YgA)>$+7#xa*{3at2AJ~z#*)j5YV zsL&LE#RN8shE z^h;bFCh(+5)Rj zNYucQ4WnP-OgFg~(;pVEj$Vr#OB;+-T)hJ~LELJ9;cIBWNgi#bnyJgPL>%?bgYS}% z&oz|T`QcBD?$VdUeBo|XRHWR_t;nLvG?aS2xzB+^j z8b+iav>tD9@fDXSx2aRqyX6)X?CM>zb=F#7(LTu+2l}P! zW>%FIvaZi1#<^t;LV|Bq)2Uw65)j9+l|svEoojU*@ik2N0W5@U59t2@>}Q%AkA}Hk znJZSxow?R^xD?4M4MIZ_K~hP<)jDQE@mDl|q`;HU9vp z2lsT|s?3$w^F+e+kRy&9zo}jc)eF`5PPOU_x|>&~?QJhq=+32Q2W7dBm6Pk8s>OvUGu)raiTeA!fFPb@gP1fk%r+`DS+YW77^{xH6m?gsEP$|8|HHKznO`dHb zFCd7B@h(q&$~lKCsalnLDzptcnq0aO*Q!2ZBw&9JX^Hg{pgRfGJLq9BU zs1u3AeSzXvi*b9}+Pao9HVZn#7*5rzd>Nx!BEvkY*SvDXvc)2Zm8E_nvVtfN$7zM2 z+@qIb+FP?p2Tt%XKm)jAnFNU8jkPP9l}$!`8DNBw2S}Vq{vBT|%N?dT>>ftmeQrX$ zOzJusHjGtXbL1QRKY9KNc_hq86)_-$dRafTsoP>G+5p40mvw((K5bP?HkoB)0P1V zji&&%0BE9E<}e}+Bu`TdPFG+km#=uV z+{+4PWX1$Wo$=EiWP%7{f;Tsmg;%G27G<&av?B6TRW#pbJ>Pf<@M9%_P{u6ehuA&h zCS*ng3%goW$s$2qVD<;h0NT4r?;1!PC#^vhYBCcfn9KttLBN1wNr~>H=2rgzg~znI zjjzp6GFadryjcB+kx?IqY4AzcMkQtTzf z6XsFnD~|p@uVeXOq46u1W~hZh3MGD>K|tCF$JYcy24tMhr9x`ANu9E3MSOmnx59jB65$~Kx9A}C>Z5}gP z?uyF3j+Tg@wNJbjkx6Nv0y@>L_Lcttss(H8aT0U;_wn!YT0bEAp5Nm?;g5Og`-yc^ z%#A|icJ5KA#IP<88?#lnROX@phAz}kt%p+N2ddlw`wb^77L3f7!RmbN_5T2`@bR$s zErI?g`k(st;yn1B-OL_G)HBHj@me}mat)bDLn&bv(7weCg;i|D-%&&|YP80|p&~15 zre-JRYE;#wa3i;I+wZruc<#i~D5f<_H*JP8V305dNijW!4ys zMxm3IzqW+QDMMMFaFx1$a}^K>9F!E#ko?UwEM_%lBhEhlJP)rE69CS@PB8+1<=cI+ zjtHGD;#ypzkloGpU3W@<^W>zN5 zmvpF5qRXxHmLPzmyAzxRlfL5;Ks+kGxu;5yoZ)-*!DtQNo{BJ{Vz-woOF z+e1d&ui|uRdy|r7dWqf=CroTWB@PD_TD9MpCZfKa;b{$0%p~c9J#F3rol!aIpZdb- zFIjroz;1JIr7h9IzD88g$!!k!0UJ}k00rr>Y;byxHt_TdNq-YDDpbj9MUrq zXqNd*%q^=j%JP5}gTW(KXxD4}yo&by_>IIZdmWAln?K4SVvP{fh^ zOrdfH17$B+H#w(9I!V(mjP&<`a^9SE6Pf7eV_ewde7?Rf7PrjyG={#_=h6N~@D=MC zc5D%87)zEA-8Qiupu>J;n_6^5HxA_LC*>zd=wB_c-VVY6={B2s_ZJsWrmZqtNjcJ# zz$$jt01m`>&Uje5Mf8EyKc~N_Jq?bn)4IJg;nu9%v4_hm^2J0cW=ID~XpF5!<#F3s9L;qfk4sAZ{hnQwsbeaZ?(34F zpj9A_vR`14FzYXGl|;x4{?R^?(yuKX*2bW-9-D>@_;6>`kIZs2w`C-jh>oTl(S=Tb zOPm6W}dEi5m~6zG<<LbinQF53=ZN< zf&q+`fFzSWC!6al$}4$F>YW-QvMp*%#i>(prXGW5Mz=s2S%47{lfVUd88x`)3hG{k zuk!w<8raLjLm71b+L5UCE%?o zv?pLvBxPHxd?~T!O90ci6ha8pypuBS1+wG!6K3ZX#a3pM zE2~2zj`5oHy{Hu_`qjy6e@q5Y)DNiP`hcE0di#`dPL{o-Iq(F6Au?it?q}lTjesWvrM13G@%be-u>a2~NuI z)28B{nspZi>N)Q?>0C*PATA^_{%Zc*zhRcY3sU~y_;ssZTK@oOQ)}x6FI3oC(x3)` zRcRrs>Q`gxX$qlruvgQe(@v%~C{cZKpe`NDpv?+w_dPJ6U)&?_9&Clz7+Rhv1-;G` z`z*K0CQIaB6xoVSmn5^x=z3>KA}|Ob$(9gIMBp|E;?@3+oZ^**Wwy9#cH+IYXSb_E z)KYqjQl?9SDX*l(^pmgCQ0xYx++Mm{ogBME@41%Y6GIUfU&WrI4Eo8vWmQ7R**DEY z%Bh|bxCJO|F{LvAJ;=UQKeVjqC1yem$e{9|6@4&Don1gl1Gs^-9P#OHXUw^sB(t;R zlzj?Um&SqSav7D7hI&YiCF@mY1Q~!fWd>JJ=<)1Z7`fw?4AINCQl)B(XCanZEaug6 zDB?bxIiI7Yl zEuR+rE6si^{hi(2bGy&MUi>s@R^>F3$tbzI6{#~{rD3OCKTA_p3hbq@1860(x9Ssy zVrJ1j3YF(CTkdtS+^Utq6bngKtXiY!r!Q?<)vGR9on>D3*0t9`GFb1o&Ny2o-{jy^Eyq^ zrFE_Q%XLXZ_UdQcd#|Ks4!EG%T_(1oKSVFvDowRG3*NO1X?1n@l~n|Ef?$6U5MyE_ zcY`yHpvI{_rHQ>c7? zp8Y@P;HqYFEc)KXm%0)pmoThVOo%Ea$~gx3x?m;`v>*v&CGaboLe$tYfK-wj0Ejt` z(0fEjd7hnZE)OoMjYywDqk&eMzyOnqiefcPh%ydxK316M&sb$*iq@W?@tH)v=bVb9 zVnCtdG9(5^1p>wd!2bZG`+f+jH)LFPMt$~tXxACSsP{`=68=D;xITJ*vf(>*{qiooHNay z`0S{e>|Pzp2fUPa@h``J?c4hg%A$LcL~JqkmKzw|agt1NIjEjz#9-&f4`bu?oI&5s zHds@ob1s%7d#+7~CEQ?6T17Cj8UFyJAnYBxM3nyke}qv<)vQSe0QT>;azq0@B*sSr zJ4q%tGo9jKMsehTPrSh#A2?oIu1aDKCPZLXNtyh^AlPzveNJ`8Md^~0(r@wL% zugxANezW7Z`2D!nB*5-Y+4^G>^_=^6Z20=-vevq)M8OZ60fw^`4`s|iGG-pT5OLqP zX^EfA{{X!F6q3da;{b0P{XX-btZ-^5PyqVRljn}!_b8my%(3hfM8J!UP_vjH)ct-! zByzbk_DhNf@gBb85%YRzSdkKNGuv-`k3QdAYgVA70dD=VKi9X~JKzw=7u1Ja8J&}D zRc?5YiwI>)SVYLo!p=OTi^yhXC%0gUp8cjKW8=#UUgU`&Y$G3}k@p^CfH+THs4_&E zB$+WMFeJ!0@<3gu^KpS)Q|e8fUANeq3-^nh1Pn4ZGta#FpXL|&F2&eHOZ!a3Pk8xa zwzgGL&((+n8$`|uoxGm%f4rutTAG{W6YO9BJ;X@d#(rQIP|^gc$@DDiqt#_l*oz93 zJY&}e?=H-bDS{>kZ~1Wv@`e|9?=c@NoUH=YNXb(J!0*_Z>N;XO_Q*A^YD7305OR0k zHqYu$(s$!b`YCdznGR0l?%e)DfG5y#?MN`|35>7_V+bUgT?RrPGSd#z4w>?^v#`gJ z{S1IiF#iA}Ak2cQxqUl2<<_7G$WK(n;Piq77}|8q`5EKqs^3nOHtJhf9==uFnFf_` zAuRX0L5_rqVRK|G0TGZC6s@qs?zl0Jni9?cN!E4>BpK{BGmi{A%FbQZrRbv6ER|^K zB^cmdiG8AkP6^NOSg!y-m6_FGScbNHy zXQZGq6=O3X@0_2PB*5H3z#d_xac^}`LNEp+NP#iA9f%XLAQ?NKmWjkNKKhuQW?f!l z=+(|_w9nVI`!5N;-JSYR3=O@0}&T?0WvP3@S&w%&}foXbTKjW0S05X3=&}Y zUi{~u)%*?1Q+CwPE}?QGQNhZcTSz;IjUl?Xr~DlHyv+W0%{n=yQwF;0@tV6?8n;ny z-iIZ%R{KufSXF?L%)6IH#%nA*B2}V>N)}54u%PFEw60P3d)s^4Ms}%rQl&SdpsJ#+ zI&6(0m@>P1fk4V=A+<`Dla})fhqso}sY%0XhH^EQAd#fTA~dstXDSF)oBF%sR`}o5 zUkBAM1ys=36adv}0HTnPSAnwqm0gNI6LP9fi%zes zDb%LR0ER|hphSd>u_v_72x$KR_J-c>&zsb`aBX>gI+aCqeL|o?I?xEyP`D_lcQ6lX zic|WP>CVD>cXLjVD;@Os+APq^({pW4AhH#!vdhH;D2h2p0#2`G2=80%QcSMVX!!d} z;(Px9Q;}D;wJOe0$kUZD1k(qxAZk}KOc?A}&Gi00vF1FJ%N6EE+UC7AG}h9X zsLf3uJ#|+BfMGevf>hi75!|b7-6qYp)Z)^q$F;93lC4CVwyN8qwR;DTYF$%aNxY2) zrpq|}H!eMlYi#IyEY|TK&mW8I>DA}_!5P1KQne-%G7WVV3aKgyH1+AJhy+NL>2FqY zi_3Ay?dvMl^wx!Hlm#k%FHW5N+P+w(x}-=F0mDk|e;BcqZ#wEo`qQnM>5;B>;>7e68KmRj#&;TBTJ8uEk<$Dv=t5HVeijW+1_y zD4h1SYW%$4mlSo^YH3ZBjdaL5KmnAoVoa67uB{ftFnG10ubap! zGAmG>VWTNd^>wz=ZLMuhidOP8&9zvYk3kw_*azZDIr-{4Y6;D`J#<&6_@$>&Y zn0Jo<062FswyAG*RYJ8rPYk0Bt~CRRSz4Gm#@G|b>2K_B;>tYFk<;Zj6lGA84~f5MU$P!~@O{J4{2Sd{%PLl+k_r$?up$(Q znfV~8-eZf6k2be(+KAHN0CKt-VSolCAFSd`NIY{>=8cxr%$(%$)cIAitIxXoYkS^e zB9fE;08z<*QIX%oK%k}vw0U{UoAN8GSq7bIYAngETfq^j*+?);#Hp1acP&%im z5_>@#^ym15LF@GwU&#k}@F~@PdW*i1Ly} zZ|l=KYCA{D0WfAah(A;M07R8Y!ue`wEmIEzPCyeTCJRiC*!CoGPj@BaBdoy7)x+0R zUI&e8L+Z2R5Fc|kjf{gsV7!t&;6Jqb<_1K5GB(Z>F)sa50XT{9M}yq>!P|zE?dn4M zwH&tkSVWJ@p4jXP`x0l5yH#p#@@c(`$#-}*Wx26k#HX$-qQVwl*JfQ&3?ZZ3z+~l! znEwD#`S%}$+t;|YK-6`k2xTC|5j$z=TzBgK0B5%c(pr?ty)sitfB_5If%iT zm5UA)jh~@=X12EXIFh644Xs;NHZGvr3!giWwA(h^;)v}h1Sv$6z>Gll7%ushYrqA0 zs>(E$s7tu>)Wn$a01p9#9s}k^wZQoSBobRzFn`!F+{ggtLgR>%>7K79$fsV#*5~^y zJ65%Xfwsie@$+3Zr^sc7*dGru{Zu|4!8z^meKylNgDA;^fw&kQ{^ozWum=n6?W(R7 zNs+nEaVBAN(s7Nwpj;W#zM}B{JI3^#$L#+A19H^gtG>HQa;vO-;nXdmCFZRIk(rwB z7-0cKL`28WtKEL+lqOEVk08X69y^`lBQkYTX;KTNfsG;rgMtZxK4TB#fj?vt`{C&#*0B$#}A+Zx;?Zi}{d5ad9_7I#z&*QiHehY7Lc45^>V)-Fw5sX0_ z$9D16^KbLTUkvrH zJ-<#5{>rxh0OKluFaCwXd&qUuz{3* zpoS$EXjbwzdDc*z3|J2k{LlSqm>^qo^*`5*xVUJIt8jKEoWe~JF`J)avMyIiPoJCX z&vvMmGbaGwQIyOS{;(7C0G`Jl$B(#xzf0~eiC5LRL{*rEVSqET05-)^)6{~3!(3Tz4WRr~}&d?`r;kM%{%7;-CA1`A+<$;XPXy1wr-fdM| zxZGZbwQY+U`=gOhQo*E(i8xQao6l{Cgi)-8GD>C`Eb)ae%1l08c^|>N!v6r$VxNZv zl1)IvNSv4_JCXz>!OltGYn%Ei#tAX`XLgJM;I;>kI;K}lIw4n!aK`FbjzPpRG;y#O z%Xs6i3pTlh?mdTct*!mI{i{^ZX4JH)V=lTZ)vcZ6#fx0uSo%ko{5i`xKP;fOl&Sj9 zlTgJ=iilJOsPerk#!9F#HYAcbtk$@#T!v~*q@PU7C{a<#0b+Wz0X;=j`8m@0^6k!Z z)Xoi=)h;Q)v2rTit4_BLo@;$L*-Ff~X|a2#yrQCNze4ybnu;;90RaRHSbxfcj@Yo} zRLUB{pH)IgWPg!B0Ab=g=R4qm9Y9+(8+Y_-)OU4bP?ZIV#*s~P0Fnf}4-*(_ms`H3 zb~-!J&NqavoON5Tsn*kY@(n&6ordcwW841#dl8pacPeM#Qf-ohsU>FPs#OII!tV`{ zj`D>|TakHjOjK&i=~7!tE<%_v)oMh!><*C`(ho^1n#f&US9*15St=Dion|EJ1Wu(g zNIS6A0!|E($Df|5bffA+iFHe-{ayNnPHO9+UhN>olcx_txNB48d1b! z$19Y^O`@h2sEeM}+HC#!_yiqtp{Y!{x)rcn6%1(-2#Lu2u&ooMF%yn0cE--VyVA21 znzGW-C^XZlUrw1CVppNg;*Jg224zWqz%OUt8(Pu z=)v)4$2HTP^ZsW;l5&Q&M&kn&q#jig%@~dsZuJCbyK5MfUSAyk5s7W zUAUSPX15KbkD#IA8(LR0HZ!Z%W&6BCTEh~Z6=hdJqKjDw z6R#Q;5KOGbkuShJfp;ERR;{(|9Zbt^S$#^_3e@Bl1OnYkJ3;fDjycsYiYPxN=5&_c zhyKu8QFr_$Ry57SB_=d39I~)L=3hX`(;7_AA658O4jDUE@^#0k9QQAfU5vW53^a^s z)#JxSSW3*e$*=dNilSB6SE{yi+S0Bd)EGqH6P{aVc~0uem}5otTvZYS2AMiy81(>H zkvj+@tCxH`SK=>?xqW*MVzoI3DCU-mqMIXI+$G&Bkyf3+tcogAqAH@gr~^{2Lk0tc zA8u_UsK05)>@GNWBejmV4u%=9ZqwD!8dcM3seQdP+~eD9D;{cNEXY+BE*@)gPh`4J zi7YhI9aN5^JtT$vpnylpG{&Qo`GB4~1+A5*!`!ZwesB`+n(~_~ik_vl${MRujTU;k zT}v~y#kvAfhH5naWpy!Q-|8ndvCcYat!EOV=nj<&_wB%Cay%pdVs98Pwv^^vZ*8m1TLa+9Vh14M^i3;&b}4+Q{nH2 z{{Uz-E$^yRy{AD%%`$6S+f;OFH7T>N`ir0mYJr_rr=;p~r;92&MQM|1ml{?X=qui!Pbz2+BJuS%~e_>5HbblyGE1h=h3D?K`9 zlc%X!m{auBp(Ey6XDGm>TTaH$1XC|%U!cnsS;X-;^J?dv>U6go#x88~O;p>ppp>cA zbqE+K5M$;a?jR9^m2*=@4<^;u%^5Y)I@%#BY$iS*!dfdCtBcG=3rm*O*)_ z#l7XV{{XBFQ%b#CdKY4fQFLlmi>*p;r1Y#n05Dj3%$M4st$XbAannS8>LoT4dEd%c za~q{vA6>7lRG@FnX|_el5K?MaGVd&&vw6%@oAR|PWTiQ{)D!^0NEy>ABtRj%v1!R~ z2EQb~_!r|!wzW=^Q<^mnR@VB$Ri{)tF#Y3K_UEs6A?BMVhX3IsY6u2HwzU$;*mab>VfJo3IwoFbFcVTO8lf8 z^n%-+!Q<2%thhF}x0khU=-MR;)Gd*kDOXdbvwC7n5m1@+6YPw2j+x+f-rSa>W?cTj85zsdax- zLOnGMs<074w^k*Df!T(gk5m%3t6zz4Z#gYrgQ?teE7p3Id3_t3*DqLAx}x?8!|`- zEsW%5Bf66$7e%>8H*Zclt6?n~^egTOXw$7|#0%|>lBqFJG|R->5MaoOm=cLOj{YAN z-}07vNY>Pg6;*jvkO@g}(SQf#kR;+xbwA`h$!;kyW-RMfsS+gYMv{8AGEAR110kP| zk9A`^Vx>eLf-2cKm|~y=0dpiX?TLeAOm>JOk0=?4@8chsnqAe6R3&OD5__+Gzyuub zaUwU7CyRYLr$~b=JM0f>BTtR9{{Rlm)Yol2DDCeeDH9`zR1%^_JBj}Qyu=rN{L(k#g~+mOh%oCE+mQSM zQ!(bY2X7Wfc@gmM*k%k)3M2C|-@nb?lCe_}7~6d2b`#%W0Oz>lWni-=d_?~Ml)y8# z`v?bD0#Coap8FNAEp@?W6Ag5j9K*_3;gxv^3rEgkAb&~xOm>dmQfP>6Ad@0jBxLxB znTdn-#~*sjC0Oa-CNq!sSoXx==P-3yo5=IcO0Vpgnf*k2fXS{3AiO33gZC2f@89xK z-aEucjwA^r5hpk?{v;mRAFm&J)Xp_+?>_y%uFcwbICTF2kV=>zbkI6W85F}L0dl#4 z?O22Y1*<6UGID-lKc4aUkp`tMv!p`c&Pg-8U=9BOPSXT&ve!g@$9T`bup|f*n3&Gf z!M!}U3b3(->j_+T22>+!C-EgA>@8F@j`jUwXCG`s<^6(_-ldcYA{2A$`On%s;Cig^c!zF^KY+AKFK} z`8cxW%}|C^I-u?dk^cYz`fg9S&ZW0Ehyg6p2^lAUw2$J?({Uy+>*{l_iL*L1YTf&% z6bvm3vpyomK|_9~%Q~3F7u;kZONz*~(KoAI!ee|6D15eF~93;BE zpr{p|0Xu;{@CK}sartIs>_-`YyU6VWh_5}O$4Rw-=q0ec%g4I>yLMR-f2u#%GunIp zCLGe_H_T4DEPw&jdPjYhOzq=6_alVWd4)KIc?3jGz;0(dc9Lh7GHR~M zD`5KX>0MF@?WXf{Osx4uTWZwaqn##61Rt0j3=@fs z;y2Zjc(3z2)--DV9+|4(X;91t7#&TY5DrZD0P(ZEAkIc}>e{cKaQjRpTL7^p=&ckP zYwkQ^D z$tOv`B74N)f@HZjv-K`g69(6>J%KWJ)pP!1)TdSPn1)$Un;^wOWepfu*9E{euuxJe zDF6Toli&K_us)5kmgqFw$D=9NddRh>FoqYwiy z5GD_C5vT*&bHua{|a0@yO+I7o1 zTpF-secZoDtznvG0RqTd4!aWdk^})GR=J5SqyPm!Jyp%#^2*BM)hjJ#2+dOmp<*>5 zCT9l(h|{E#GPV9$PvEX|WklNY3Kq1j`Hfp^g{MhrD7~a&rCOm1mJcD;q10VWp~AAx ze}&cET)vk(lGdjOr(0i^)Z`p;*?ponaV%{)R4(SS!lI>{^lA%j>C0u`#3gPjlNKBw zJbCdo5mVwjhaFVH+cGzD!kG1x_rr+Pbds=g29wy?d2KI->shVcItf&@3rmpNO(rKO zPQ_hYNC!@$7i}aihoe@h<#+q z3|3e}n<+W@=s$+JrEXvFk1lF=2&jic8f58)snV`YusWJaB11HQA~>r2KVwewnq1Jn zpzTfV>W50GtPM4hsXkz3RO%{V?Y>sS^rP193)P+}ftRIN^S5}IJ$M&2w%7>Hp_&=7 zZ%0>Z$gs94@{UTMwauXJ+L(lDRD0^n;5{k(H2C_4s#cd(6Y%v=FR0ZQT1P_$)ugd@ zG9auRbpQ(Z>-{PJ0MXs~y|sUgeiWkhxs9_W)zyWq25NUV4N>ZoZ*Ol(Et1yK-CnU| zWUCI;*}7wGI;o?=_|GxpI={j4{lvx0 zbn_ZjXVaU=w|KP$1L^Xb(y2Q(c={QizLADGX)jVL)j3ZxYm{Hb0FFLqj5HY3Kmd{k z>=8l6(=Garme%xX(z~{D_O<9OT1)ekYN{EjtLhiaHEl4i(jY zA*I!f`UM;^#>M6*ACYWPsK!Se+QhuddYNyrSuIs)_o7-}GG1roC$Bx_JfoZ2IIB{f zTE3YYDr`tHqa&oqA#ieF>f$w4&4T3e$D3dEHMx!9d1a0HPcfjuJ*Daosuf+gGVy%H>#D4A-wL^AKHvT+=?wHZXilynp*h{fyMRs&6@9 z3UbXxr7u7fJw%7=9W#Kgb@qZ`FZ_?<3p#auHmT}TmJ=gTg&+I5Uk3q*9`I$Y;QPHf z<&-YxaQwk;X}-fTb2J%Z=Dk|%j|rOv23{(-ee5V=3%ndeGsZt}8+>8#r!x3|n7uoT zT8T`s30N+qFgrYCOvxok!0o4puU^*V)T>J}dSVoeM4V*pjELLzS9wE`bF}twF*T1o)fu;j;u9|CHS07}9qPf_id9qb( zH6(xYnTl7K7|b@A1f0{`;O5-BadAqt z+iK`k2(YBu-8yO>__7@f=>1XOecMC6

OEq{EF+Ii%jEEhH5)Mwl_u#5jR-kWHiY_2Vs6YD8p2OK`-_7zy?{XZ=djCaA29rQ-v z>VcBNmIBmJPL0{{Wu@ zXdlEs;6MErguH9r=@oYn)QeZT>m@b~hhoz{@%~ zr}uR#kOPShlOXRJmi?o#e&uBHC32m9OJ_=rOL4f$`IW4Qts_wLt}3r(`L09KgC!8L3kDd( z`A5wP^$@E9PB#AlILwXvK$si|&~h#84GMt7fn#C*VN)zXy9ATc0Fk_U|E zz9ZO9{1dpxPx)Yuo@2h8?sX@3Z;EN^;X9b~n*1k%@UAynkGn&d@`J9c7j51fPL@%s z*{Z34Rd`Ue*kjr`55hF;D6L!j7BvcjwMacU`Hs-jKQjneIVKH%nBTjq3O7cInxH?X zq3Szs5M_w&uZ)4VA+rYlMn?zb}Mn>>pv={k%ISTJJ|b5m#o&7*{^J1();CLl^9Ug)eK~p3P1oT1wh=(7?L!IjvpSV zbDpmC=jtarqV%@7sc?)=FU2o&-VhhHc;+_^%v#vWncYL=Ic$Pcc_YE|qsOwL*DWft zF#)QjX5E09X}z0QwWiN-NY_`>CF+%b%d4?a!)T2nK_W=>s#CVMdfuB0hc&$#niP`M zz&dG@yaR&62O@Zv`i9*4jN|+D`2PS%_-9HumrimsrwebXJhy*e6N~44u~nY+F!N}^ zs*U4v^Qf(NZ)iB(>~zV50T*|5cB+)^C?7!*k=I?N(lOQyu+&Ke!Rj5sfEBe~UtE!L z^{QRFf2uJpJw|G5SZ>$=3>ec44EomUZ5)i6l<1cA-fjL|!^7IL(6^Pt@#(ibP9rNi z^V?u8Iz`t&niX1Va%Ss*#1Zn7@ux7SXUqLGBwgGnR$^o#x=U%;>VR~a5#C4ybx+v; z0Kz}}LGdNKT$Mbs&auC0bm5I>p-^b2SGI<88To)>B%U%mL3is#8}7|=YH2Mu15gH4 z_p5xFEa|#pVW@QYvRyywQ2P{RYo|Vov)8=O-zeH00&W##$ zt*TMdpSU#XIya+Im;tD=DgE1^`H_k;%oR{duYU(>#y*}^s@3YHnHp)+S=)(COD4Ms zwoWC9%tcPYB(s>7yT6Z;mQv6bl}5F=r*S1pRShFJQzS-7kUPh)5Jv>_3KV(2DZJ(N zP(3@VJBwy8HGkSEJuOS~L#U{9ojoTyPa1zoS{$2->P;vEv#o-yib9HL$Y#i zt)w-qna90XFAQ5-ocMNom{BY;g8AJj)oY6iRguUwX)DOgYDwFF8iw2N62Ff;@~18M zTjSHq?(A4vo^4;20O!`HSt?a}8f#ThtjzHU;4Ba#nAiX=W^ZvGS&Hhsj~Z+36^7o` z3WX(-cXI65`0O6IVD(=o> zvDi9!)Ren6FWAoA)PmZ~X6Uq27IQBxdulT9u~Ak=fdzYT=_a2K^FP5=OD0f^)imkT z^C>3ggK|bwE*;iwSMm;J{#DMs#JP=_&S^lpxLKo;8#ljY*ejWgfcD#b+_zx}6-*Uq z0iO4jQnKd*oV`XQRxXs}l_Dok$^=1ylOAycZg+cM;%|n&BmIwZ>kCd+d8Ms8u3J#R zSEkSFJKIxe6zZjVWQI*CO&YG+rKvleOy+#oWz_TOitul3rtN&AP`T}sGg6#&B0Q`n z5OV9P6v3oOf$gvj5G^^kF}>!rt67%`M&*=JrB0-hcMAXm1jg7P6ODuKgFk0K2!7A` zcOd6n!q7j&9E+AcEB+R&uB4Yr)aur+Ld%2!T*@oaLc{mWw1E-VR2Q>fGa%&&FJ>y;NFm#RjY zAOa@>a0pqCF{Ej{d{=n?026#8WAW$1*7q0IZDCTSdo)rjy)~=*vZqeH0i4u#P*l@H z3$;F!O-vm-mTW$3k9whX0HZ)$zXB>5 zJM8WX5i3U|xUMP-aa@{JK4fVMSdrVjX_X`~p3->nukBLXmfdm#&;I}cOx@d-t4`=6 zHCjrbu?bPxLV(Q;^)v8-50pKNMM+%Cs>7@bcI=dva8T~*cRpGnI*U8JZ!oJUo@;e&s zuR}*>=O5%~yH8sdLY&QF&03LS(J?b+f-RQ=07%=d#bHv5(h3UI^%x)}nWPwy4CF_F zGnSs4ufyD;>dTy5qa|wp0E?bIrvvO@Fx^U_#d@#C_~lLQ zdKtR>#_DTM?pAg9og3G5t%B-Yy|APOTMouB?Qi~4h3EQ{1d${c#nwED@2CpyK#5c& zOU|r=7=W;TV*?Oimaw-nxvR~&y`5?mZa=f=nu_6(8Iql6m99x{z)#I7lJKQN>ag;ogsX#(5)6b9&vQd$5zE{6j%G4x%+x{Yf6gBrJ5G0HX&DYP(x(xuCSHL z{`W$a3xJa3+`h?@p{ZJsC*}ajBprx>1GyQ4zH_Ixh7C+ZPVpxm2stAJ<2k{^aN*SA zmpLLuwFJ*F&+k6L)*Tj}$DAW0Hn7V8Tboj?b8a=Dm?e1?3yKDi560J5w)QI(fi6_y z$pkFKWP#NY<^%&7E!Yk)ck6FgmOrUVoH5R>t~bE;UJbo{RM}#o=}}dsQ8Aet_8IDU zh(I)!Ebvqr8JOSsC4r%D6~Z{GQPhK|1c`zmb!0#c2hB4Iye7531#6lDy%!Z0Lx@1@ zhF}8%z?mT9$8dNR^gFG(yi<*7atrC?O40st`8oD`TGz=F;Zl$kP1kRN}B(`zZK{?4HOl<&5%U_Ew6<|Y5kU=M@nq%ob zET+fRIF}T&tw&FZXg;0IN>(XX)(o50Ri1LQw;8&hZK9a79hWic6xOjaGG0*;@L&G` z6&$?Ii@M%am6rO(pbCk^#KyyNLi}N0P!B*XtdO-e0DMUeC+;JD@x@sB&c0K(Qw3Eb zB>4DpEe%UnD+MZf-ozw|4TdbVBFig+Ao4hY5;cI&VG5V*s=lIH@|XdL(SiudvlamC75LPs*!AwQ{qrDGl5U8`Ki3 zv}pRXROvXmPQq@baMEItBvt?fApRDAWmd$HZy6kFEeX=hA{9=@ag(H;%f66*_O;}& zw_z|?nt?{~fG6z}ld#)@v~`P-PZ{94+-Xj2&Q+D4a-Dk^TCwl5p;FYMLW1l1RIt&g zbS!<%s?vf!93|(P8hO| z`Gbhb9iWMr;-{C_uCiKHJ6w;AS*=HyMXDJ!*4jC(&)v>L-8JH>e(OogzPk1hacC+P zVo*<+L+8P@FWJY#ls!30j>^z|MYoC?)`% zQ}}b0Qe`x(=>`g@t)vJ8zA#De^pU_-dA;j_uC-FH3=|zJ2OzW z03fN7K^YC4odnN(og>k$Rom6|D#;`ja!%0zgSzzs3W?Q;P&$+za|z`cwKwc)KGXI1 z{rVPdIn772bBg(kQkeT%(#F_88(}q&HNOVQV})Quy5fyB^$@PDQWQi?L6AV40ym$V zN#0bSS&R$}f^?EMJDKn&8ILCihX?6rfm+Ywc^r+ah_T8lBHqD}*;bseD9F{GHrZRpysrB0wr^y$nT*|9>x-otYwrA%FPL^YHy0d8Xsv2Yty(c)9gR}_lPi~Vl#qZ$iytez6{vg~@{6i96(xZiRgVIWM zF%DSiI0hk!XewoPV5*XCJe;+E5iht@Z5ZCGXzCJ~Wkk^THqnx+y)|sjiMW{-*cQps z5m`!}p{SCuL}DQA_t@?+nFsS}&ui4SO1_m6#;wq%7NGspAYrKw@QpaupmhQaL0%$y z$%|iSf!;SN{YEa0Q>}W$nJ(r{kG2}DUudUMAtX`cpSE3=GxEi%0%PRNCJvU5{3mQj z-zFH1qGyP?Rcf5Nva+qcN;fo$)}R1p52xnUyrv^CXY+5yC~%7$w}Be;_FddY?T`-ix6m<*|uttm6BR##6PZY+41{+WKHb(`c)$W0HA3s}@ zg`2#WkEwQ&|365_2*aIgPK|=;VrHmEh^Fh5IPx0B$6tn zNFt_~8l7Wu1B3o6oPLEX{UX|3ICZu2oNYQon4_(qS{Zh@X;QC_NnfU|JCN-fjeCO4 zy;vzwsSqz8IeyRih1&D3TTabZqNRILZ%(Mx?CVqoa@t6$-<+wDsX){T2afgeUnRB_ zuP*8_r&_HFtybYopeBoIMh7aa6e%DWMlK|RI0ALQmt{{D)6YSitr!=Voptk|gjj6> zUVu#smR{D-JCgaT>I{(|cvzbzV9=BIpFXs%-wzAQ)`+(z*!uNQX&OrWE?&(@zTibpjg3+m5Q>>J#(sZD16e)VHpe;r~uqvFDWrC!rRtNk&vMR^ZhF*|W zMXpxBJ#5fO9qWNMHeEKB+Jp_QLY7#~l_e{#!`5?B%aE2Nyu3xutuBu|zw`mEO&?Q` z8G=&_A*7KS#F8Quo!CL+xmUR4+^dt&q0BB(?Jh29(zIu&i|f`E!zrdjDImch0(6Kh z2@|qBCsz|ch}f-a{)60kS~=eL)K;*+S4HVwnwOks4U6>F-2uaig&Y#Pujx!VKLw}L+Ev!5p+pZ2xX7k5C`51=dS=}1G z{y%3F@>5j1 ztuprYDh+2-FQ`Kq%JlS^zyN10$LB}wclKRn@JGh{xAu+6tm!8zuW>@poAUYw-qzNJ zeJWIWbr1zcq_ms&)hY&6uH9_SRSy7ev~)*`^$NrKugrJDfb9hvn7e$srytsSBK{_H ze+PR;uB~c?rjAaLYNPEJ_Yb(1LLCJnL^ZQDI<;K?zT-XJ-c^?a&AAJ z>&(sNHF&*Rn?s#m(yLaoH!Ci!+bXC^1B&`6n=*cI=Eb#$V_7NIrWnzH5Voe?Mv&%Yb)2XG#4OHo538P? zX7Nory*1(d!>2r%7jxDa`v$$1FPpY&OqIcvAn*vsA|-#31CnJ!nQ|Xga!i> zrb+XiM6QhW=UthYw)uYz(!cy>m`(jPCi5~as0IzHb;4WWcFk;t$Q_cHASe~cL6Lb& zwMQ3K=- zYs)#OBfP6&e@?X~qU<7|Z7d}9As`58)gvR^&aQ#VP~EDn7A$tqtRP2GCS+g({{ShG z_sTgGU(#f2t(kS|m?LXzS3zme$|N>dBV*KFqNIoz@M8$%SLN(J)7VugS7EObK!Rg) z`e2chjO@8n_2BgDX(V_AVo2Za4EFtJh=AjEbolQZ<$64liE;f+jV!uxnaQc@Yw0qp zOj0ABf+LkkDBbO@KB=sKxfG0#!$mZqC4!YFO0bOioP9)t{_8-~3lB*d*#R>eqa)k{ z00{b?*y69F$1GQ?caXg&v7x8BG;o_%-E6ArrC30vUt!?FCIaB0!xC%meoSOczzDA6 z)MpW*0>CUH}c@xbg5Kb zELaeyD*zt@X_9t5V<7hKG#g{f!?$80iE5CFq`c`0ut?YgAE5902v`-_1P6d5^+6GmcO8yS{{T060dKto z=mc|ZjUGiAZ>z^EF7~!>5Tf6>+tZj~cg6=@G}S_Xs?fiz@&KzO4Nt7(z&l6m{{Z$~ zaq$ZtVa!8Qu%L@m(F6C_E>HTQ3^XZ>yAK*j1ED`K=2WfKR8$)35(^n3n4Gf#lk5hP zcMX~0?zMx~h43%)J#L3_6y~ZbWAILzW$&-;?OGX>BarnQkkQUzF;pgH)N<4=zuu z>RzB)w2kD}ytV2V8f6KuZp|9!YLP5Oq@woP?&!6>L}XHhYAqw00k#3i3Zwu)jZmyX z-&YeQC9btr`igTm+a-^cXy9mV?7*6y$L{{WEnqm^;C z+sNrRM>yz*5$U%AtOATlW8|aIK20K1Z4bXetgzJx)=z>fX53rVWRaLaldDRoh1A4= z8bqBS$vsEA?{7^(Hn#TFQjIn={aWqUUab86gb;fL)3K6EaIyVA@@}ka^^>MNZ>5{b z;<%WXY;*0_Y~3vWnoQK8IHk!sRj2x`X|A=qRc|jUM5j}dSII?M@F^=*%b{Am3bfLg zhNwUj<=AN;!Gfh?NN$!Yqo^yG6$K|+z?Blrl>TA_^pBY2#0^1-86ip1>hbOFtbI6r zO?7{W^e=OV{YCZPs+~IH8*1cSvfFZa{{RK#Th^~zzt4G{A+W{hu&X|`YzmfcQJi(t zWpY##4Qe)m-F_yu3J+GLeR9uFcKKPU66AoSmemGMph_$$;v0&zt|(enu7cGm8mK7_ zwklu-g&iZL>Jijt80SpCQF-V1jd4y%!oHMwCssNy%DKN)A3aIX>vuD+{FXi!5YE+Q zynZE1ie}j3wdvKwwly-3MQ$x_R$~Vv67#WMcYbKs)Om3f(yKi&QH4lNE~s2HzOZGL zm?)>E8Q~>B)g237To&h;dNpc_0tE#MB#=h1$_M2x04(KT7S?u!>aQ7JP&%Yp9AFj5 zHZt?VdW+7K-?&FB(XZ$m*HceUpoj#-npcz zx0R`|dSbbaaiz3@P6#Rv4kcig^at$En^5=@;4XXdSHu?yWm4h~EUGcxGsHi?OPa806q`vOGWFn#{_{`MN)R2nhVi{U~?w(~o zLqfF~DL6I15%Sv$UQnozhR#l{7KJ$X#CJzb6bnP zTg{s8?dUl^jVhk0t4-;aT4tN+QmjHTbuh*uxK;S=eFrL2E=|!;oS9fkzTUc2pDC{n zAz!t&{{RD6B;k@mLV=l+51fMiz&#^KEu53@FhGd!B=5P#85I$T+lMy_Aa(|^4Dzf-@<70VCk=tDI z_p_mWQ5Ky#K;I8m@x2yG0&X4Yh6P($cItr&znZ)UD^reR@g04uHJ6*qb$_1Sr40{L z?CA}>^zELbRi>UVTD7MFbPYn5SB13-bPE`ZXC=iu_aBtVBIv$Jp!fO779HKFkoa2xT^1F@I+#7 z?B!HdtZNJ8HMN(Syvud+ad(csWA@jdVk~)+Z&5|MVkHcbzAvz02B9JO&DSeD>V?-X zp;nb3pfWLm>PQAPdzl&3JAsY{_m1uO}Wz5wY#;f_Uf%t>WejFMx&_< z>C>fIsZ}i!X{3_Clts9H4<*zso}Dj7?-!}d7k2#SGeXsDTi3YP-QLIh$}rKZXG25< z(^MD=Vyu2j9tMvWoX}Uj`=w-)rY`3dh-*Ubo{LcG7m zj=rkm$zIixV8XlvG+c`fSn}Rh{Ho&4t6kh&RTS=Kl@n0Wr%3$%y3HkZSIdw>nSKe~Yw7nG z#>C~bO^2b%akR3vwRh>XZ({9O=1j4ygt+U6bX(8NRSOcJh)^=jve_ILykzFKLOqQ3>{iHfez64cMg6LBS5Jz$ z@4|I$>VIank1nliSz3iE71rv53yN0s0{U$9Y9J0!iKun31odUVS+rl%exUH)tmW6u zV~QJmRdC|3b+i*w(DZyee-E@>Lg>e8c1$XBWWDkME*aP=IyjcQ{+ z>R|DXk0I0Ke0l85U2a=KcXI5t$cKw|Mrt;- z5s73l%lSzB!SR?}WRPb&ao1`!wVfI~!mUyk-$fP_^y#XEX$tMqpa2G^l1MPb1b*76$#p}3&)AO@|IAB+As<}8+W zHdN^)TWa-<)4S9S(f(9 zGt<=-5=C>+>nfvOCrLaqd?A=v==Ux90mmt8`QCNKTg{1<5>9I|WR=BYvL#sx)l65c zeo+<`Gb|C|_umgw4_2LpHmhBhDTNxkjzmEUNK!_TKEz1V(yTe3FrbuIoKstBtbj6( z#9*;vNJ37_jUxwCgTs-G`pe8+4E+aPJw&@=t)oA68kO|{Y-2>}H4LwQ%Fw0FaF0aY zuNRUcY@a@HePY7kn~EhGixksQN|IFWL5LCtz~VtT99hNU`fT*-hD>1yEfWO(V~Cym z;g9AHbpjEcoSe+$Hyo7n(6eQxt7R^R!mJaQVq})vu`?B$=j>P)xt=rn9@J@)!WLo( z3!D#W5fUc{+hHjyUehg6>ebUnPJ&K1J+zqzU^0Hjb$N~d025sVZt@&0tj3jU++%uh zD6LCe{kVl7dAfnft{}$TL3@S>{QTfZVoZysa;hOLZK{A>wUuaQ1W3SLyupyd2@#Ct z1wnAY(~El6iCU9E1cq=R>~j(UpCD}9CcIOv9Kjtw*Wm2eq?)@Nqcd&Rc?MeT*@xiP zPv8BOpI@mnrBy#?pqY;X0)dG^lsDy}Fo-e;1$&bwM3~jxhD1yPW=5#ZKTf4rATsq3 z2*44ga$pD}KT1%?4 zqJ&X!Eh1nSR>B$hc%)29&cqQvS&Vms;s^u{*FjC!NG!w=4FV6GK`>(xk8y*+&yDp} z+=rEN130vD9wKuZM;Q<+k zX$CutLG75+gD`!fI<^5RfPtEz02g*?bC?ql2$lnE4CV`QY3a9JviQC1=jwEC1CZlt zbn?d#eqOPyR!vW;-8%c(`WHf-gzBq8x+IVveV*7DfVQAq3Fs-Zu`t<{$Rlj;oc53i z?+Y!)D$rWWqLo^dqM%?vFjSae(oBsaa0E8fCyAa9*B(bxrnuX4fyjT(THR6P!J09H zZ}{yBBZ8_4t8bAlp`V>m2?_NxK7b4CsFz~on9O`BDX~KOS_2tkxPTWkyou~F)J(`c zWYVGQJ-aw^Dbryf2q&mFaC>87B;Y{ccTcQ5&}#98akuPc?{QvXuc&P@$Zxu)YgMjU zt9Z4lVX9kw6+ZI2jy;2L)Gu6_e3T220G}U}2*@+lrcOp!ouChzcA}KYQ@c`}64Is! zk=Wp1Om^HAmg*eyygR4KKJ}$zb8C-s#b@!&TibWDvw1z_Ty=|@-ZlRKiqh~V!`AyV z3#v?!t~JV5G8;WAU4lT&jO1i@@>juQ{2_p7X*9Q_HX4Se(y^&JVoFvL ze=65TE*6zIcLP@OGr1TOh{JC<15jQM9*|PiStypKNG+&>*&q@IAdKO?{{XwNoh0d5 zFAG5!JEh02Dl)0oPas+?XENogD4O6*h+)vFYAi4jXbQbjmCtKKBiye%lu$uUE1-fv zZI=ggvjRcHk&zNVhl_KXwJxjFx}g_TskLoph}9;T%4a2)eJTV`azs7ccP^EihsJmP zm9==(vc}5lCdewwDqxC`-0QgqS@Td8ax6Kamvs;)YDS?)KwVLl1N5UQ$r{(bn27qAbTu;P2G14JtfkOh zYBg8O7i2dSyNDW7R=wiQZAaJcQ!Mnpr6_1bt!rJU8%K`wkA{yg*HIL*%$+)pnKA?e zncKknPaQ^=#A(l%gs7tBDb%cO%LFb2>v}^9!B0~VUDUzgrDa;Sv_;V;-X>OA`{GC65LIflS%RXC1Y}RmC)#^?{LuieI4_r8 z&_}Ay%Tct+Q4GW&)2NBm6hKlyDoK_mCya3D%?-^CxNXK3;AGg#zsK&s8MR%dS9>;j zE!eY|V5o&KY}b)eSu){B*cY1!4ch8=8gTC28(VMzfyGirGLC^-yvktDzxg$MSC zqnz1WP+LF5BE8%C!J+KWrAYePp;`ke*|HKdBaI>4;=@YsPWq9=axv;x&+3O8jjUa8 z{!=kccJ+9_Nsg9OJ-w`Uw8uNpCsiJ#3K9hXA1+@SH9lkT&oNt;V#`9Irn~9|4xGe@ zeIURPnUS{@{{Vt{jV@Eq`F)kjej=sPQKBviB|7w~1sWSl5*C$EH4?|vF9J?a4o~={ z5mv3zE5@S#0KR;-iiL+@uD-5?gsGKVb&i!Zt=0lyR;ejlq?shoo@@9PrAvQ_ z`7fY4((ctM@1C#ss2r4Q+Q)~bvz}2VF+}hILncG*lx)E(zDkXD}eIr5jfC82S zOo>x7Bmh8O6CbGy`Wbyaxu?h+J9e__qNvac-nFjd5~`E4qhO^)Ht&^H`PE3h44DZN zV4R=N(=Us;jlC{id&(M(H@O_LlClzgIciEvoDd|Ebg4MWFlBr@Q{j)p*S}jPe>TY7|P6qLUT`wt^I~$O3eL2vgtDzBZWsKV-i!FIsdpIu;8nV%B!)nQ@BDQ(TXjuGnR?4F(w_4(#sivT% zqBPUf-9Q7Sm??k>jX)0f$T{WKBtJyPH)-spcbIL{x zreyuL+4E~Zhdv#&-c*fQ$~|gssxI)HP@$7pQ!`}K}a(wJwl`nF~(bU=b)IJe_L0SV%Zz`nYwq7%QA=2 zWk#yfbBk&`#{1BuN+|UU7O2Lc0fGQ(M3zws%LhqegU6SDY~Kgd=YI|JL1oJ- z);BiBok2~h#WJ+WqgI>M0cg3nN?x-ni3HRUr0MwPT{6wK?n%b22rR%lVBvh3EeO2wuM{zNRiHyHqqE(xXu-Xg;+!q5#$E!DTX_ zjA3sT>5oVgi;>%X1E`r)f`jy+7SRYR5ZgQz#N7sx91 z(w5W}3{RZmkj%M-W&SF{ujDv45s%{r)bvY7}D3RDC- zf?&Y}>i~o!3fl^cScR)<^lqr*_iPkCn36tV>OX{o6TU>5Iva`pf79jM8yl=VM~ih| zDU8(1;oNE4xVT+MtxmMwR_6@fHl1o(PejPK3fsoQ`#=Gjj2hl&@h|MllKf9my?$G2 z_YSTDr4(YCwHQ0LnIO+bPBiv;W$_jFD=5pVjS|vMvI@*BkPvkUk|4-EfZ&NsV)^99 z@>IsAtMC1h+2jheRh-p3(|&9!$SiHZ5}WLjB8h0s5lP6w_^ru=f%%Nx8e zVcAyEqe@meC^~fX>`8~S(2H7)`cJoh zI3oTg#$K#!t6BN_pA3#OYMhB#C^-m?Op{hyxh|aC>ks{{VuRfBSW}{{ZwvfAurrUbBA){{W4@ z%ZYy&f9}8VHw)hg>1QeGA2fdwsnk4CBn~N1e(;s-~lc2=pOg65qR1cgd#q!R)FI<&Ac1M;pITRZ%ppxt^EeKzyXs{K~= zbE;aL8<6vQTt0>VJHjx^;pOrEA;$EtI)WLf6vSdIS9wD0rJEa%%gQSC*G;ca)N@d3 zW{1|Mi?^iOgw#@szz{diu%If3VJywR1C+S<=r%1lbqkW7f^6!TXJFKc`UM+idMo z`YBblk(2?cde`* zFeliY@V|8b0GV|+8?0PImUM=*c6s&twzm#BCB$5@drvnzO36nphF*nQ%E?uT)NNZ< zODuGeO_EM(1L2mJ*Hr44v`(hX^uY zX*y&`Xw+bli6f0)=pRQp{{R|KBl@8Fhvp9F9cS7apXKbyV>D>-?AIKcbnQd3<{YIKMI0PPIS^n-!taH`?A)v3Ch z+^*@T?Q$(~u1iUW_ifg*OJj~(r0}DVw%w|$7gE`ium*-Z!F9fK>Rhs=vpw#H^{X?6 zs^Aq(M^phgVm~yKlZO1X584OET+8-}czJKb)o4`Ye5R!$meq}GDz_z*@cykj;q=uY zQZ6YtBp@D`1iqku)n1*7%R^JEw2EAdm+DfCnpyhp%KK`T*-94ccRt>|P|gbD=uJFy zE@7;aX?j#=T==T98+vqoK&?=_`gkCO0vMb@+olKGj_3Z2e75QSDY<3k;-w1I2o_zF(gpJ^!Y?d8vf-Z z4L{UK41YX^{lhQ-LCHHABftLuUHP&|Qz}1A`7IZrLAB!wqzUm8+z2PDe#GYr%XBvV zcjDtWmKw3B$+d+c%pH9dcWn~Tc2v)7U6Qd_woH+P&8DoK33)HqsDL(o?sHM z;HlSK*@}R>InI!CBEUG!QgfO$3ej|x8iNHANCI^{_|!&abBF+O9lhR)=~AkJYiKnk zHq_PBGLc@YolsV#L(;L8)FlW5gk!0j?7`(3oRmW4E%${TMb)^)yhD2)e~Y{s^=T6* z&RMpsy>YC8a9eGMN2Nk07QRV&ae2!tQf-+UjVOXlf=MF)Ff#{Z5t$=`YVv+nV#Aaz zWxchwbz1p-VvWw5s%Ym2)cxoZhE!uw(s;De-W*Em;W~8mb7PllXYz&E!SFD&yD84S zTMpf=CCY&AK5Qo#!ONHh&5j0}_wiqa!zn9v z^RjkYujlo7mSwAae*&hj@gkKQBg?9CnWfgW?*w)f^HY#-g$4yejN0-M7Zja5VQ*lDmlibl9!ugBTAg$J~ zPMWD19-wts1ywVv#yMU!o*up6xnB#ZyfK#ZGqY^m&#$Re6;n#4G%xC2+t<19_7jARI7*sd(tWw9kn-A2PA#w<6kw-!b?{ zm^UxKs{_`f>Q$v~^)1NKwW(C9X@gJ-i73Hz{LP!cR*JW+fv>0cI`)}HIR1=iUeckp zQ4EEw<`t_|b=vug>0gR%sBKo2n%q<0%$)NS*UFKlh_$Qg4I}`Gs!J6)Cs%Sv*k&h> zF!E$eKiXZfYATD(tt#5qU*77j%|)rw6v&!Nnh2EXa?14pDpW210H#@Y`5y`CUn;`W z^zXkV3!_PC9~IoB-$P;l0B+-v0S2a9?VlqOU$R5Le~xS}(d}tDDpAllDh6f<0F39v zV{;Hk7h2c$1N%dh-i9=2RjWnWlocYA6b`$V8^J0JsW3_6^*t-)8(l)Ko#Ut)Jg%qm zHD5*HrB;}#n4bhNOWEk4Z@re`}oWqwVIk2V!bp}VL{nh~Iyx{rmImr^( z{9{_3N-0S{4P6B@aZaw?Yn4*1CD~d6gBX%nMwgc(>Kl>s(XoR7fqR4UiI zV>bSU!dKeWODU2{x!X*CSMBRpf|j%T%klZ1U+{gED)jDscSoknQP8jfO5}`!3cO4Y zmM{#S4(6W`(yeZks~R+}Y66JPcBzHcQGq&`0Uabs&vL1C;cQfQTVz#99k-8 zw6d7W`dT#WwQRLYu30U%j+CV9J#V7QU>+gsYmxhqf$=ZGyo2IkDNr7lFs9B0S}W5~ zpZnk>w3QG;Dcy>(Iz-lA5Lxnu6)p*CYzqGLx+DStG{Z!U;Ft~l*q#n`x~s%`t;dy* zxsf{wPz|M(DwY##cSJkEY3)@~gI|xOHG9iy);6S#Lcma(O=rb9&%ylX;pHs0`iq0K zIY0zyMiLp0;sZp-ehjU3$?vK>-07m#MMPpk(@KnGPzLnIlrSO4>D~yQGc)ST=w7!z z;=8uGTb|?It?J}?7*=mijcpS<1zyEv{{SNu?orXZM40^cv{_mbOL@F|$P%tq@j17o zwl{#LhmTd)NmD&UMj8)dK7g6v`_5y`s$VV5T@|@<(MnXR)A^BTrtrAaj?*Kk6BEW9 zZ1GM-)2xi%70dYRbF9+pZiV{_)g=p+PP?k*=~7;rSPqbLTE-j)np#q3uHaEWIu(`8 zD)cGTqJ2eaha0qRyo!>v{{WQ9ysawQ8Cg|;VQ8sn{9mV@_!Wp&Nog!(WCRpzm;tcRVsO!<2pH|# z0$_#nq|cZfVXKk`g_+-CJ%&ik_WngrdEVx=4x^kGR~pBcWzqa z3$1?@ORtv}t*+B7*@iB={mC&QG(<<3v83oKhH5(1Q>vfwN*N;%V1WX6`Jjz0piorF zAiOHZ;tZC|W3lbuFh>@7(Bc>C@XH5AILa7zglJjK3kG$alW;uF-Dqi7qejEMRj6J^ zxYj=6IPoC)GqonDx`0$w`D0KLC);hH!5bL>J8l`A=|l1mvJC2j6C|jU8^&i~2n|0& zmEeED{uHh*H?_8?a&AWXp=}nFu(h??zE&7$UI+2e6tC{JCq0=yVh@WYY9PO9>C~rI zfb`o6>=zhjD{_QwAG(Mud|#)9x*jKj2Tbv`b0?tHVDUIt1=^OiPOfb zRW&w|BstXB5xfxrjBExsBYB<#d0jQRZ8U71oA+t*E-QN~y>h>Nva`igX~kOACpFUB zX}?sFSz-jGnM{+EBt&2}4J(G!M*eb0By`Av0hu_0U=UYPUTPT{RSZcId*@{Ud5uOl zoDK#&<1-s#pDC$dKEkqeI8CoiDV3AkZ#0(Rd{#q)#V`YIy9g555fzW(89x`*L@3QC zQVl{j5udose^S^~Au2DZkaBhdyzCC~m?B4=ADb8gM9Bvn z0nE9btszp%wMea2i4@2Mi7Y@G^zAT6QZtAk0L{5Q9TTsQpTEelJvKCIuZQa5TcUNV z>SD6gjvlD#2elFsfSm1WwbN@tg{kpMRWl7cBbG{Fpk>nxt+c_)<}iJ+0!)(}SH2&z zPe?fD$<&ll~RH|IiT&vGw$=}-ztG5h~9n~=hn+-b#Hd$RQ~{3M7RhObN@Bu15eU0hLb z@eF6_mQOFgLw8K{vVQx2UIX}6NPX6}R)rpahSFM8@J&=_7b2j=TGNG%M+!tn7V|Vq zx=&M5)TYfQFiF?}6DRbE&*nToD6rp;BVS)h3uO-v}#sBUzzjILad8qsLe&%u*I!&Ga%rV zajWZki+xd1NDA;(fz)zf50Y`)BRDD`w_dTTSC^XgUX{VGZfXmPD75LGqo#-~&cUha z0hNGVK-K@NVNQ(i`sMU~e|wJFW39t8EJ zLWB0mL)(5yR;$|80IE@}uUAObt2hTqh?A#gk~SPHH7sn&m)#~>y0-;hI6W;Qz2US?C$5yi(X5j zlx%DjN{pvWisf}xiglU{@DbPchjnpn$uFtfE{97KQeIbBC26{^%oF!a#&Q82TSpn? zUj$m(^WH;yQnS@ccGk42GZAS}~NsNqbxoM^8y7X^{-ZISha&#^=tOZHv+&yTqcE3nOOF73r?Mn8KstAMCI>~@AC$t!o z%A^*eN}CsnXit|hHm-WxGu7hnzExeClSi(~hZy?jAYu5K+7;o7#FYBZ@C1glX| zm>9~9;Yo=g!k#Su0J3@?4DwUXEp2F3N;mgSeDuOTnr6GTMyIP#3aZAUfi5;kS^~(dC-D&HhtwkTLBV^OdgTCdb;+3f2*OMv~ELxiKv#C$KgIP9VuO3YE)L zR&8h&DS|+TUdOoyA5G-!y-KRkr(4&zUqX|%luc5_hU&)!W@ZRG{MaLgo9Qo%R^e1^ z*5SKYx480oRvmV6-b*%a4%T}1H8Qi?7FeN#at!6^)oDt_gzRgT6xJyfya{#93wqbK zwQ5pjx%BH&rEyJ&{5w9Hnpv_D3v5U*z>yQc-xcz@Ey($itx~NPx3V{@PMd$ObVaE2 z(G-tVfEX%Fk_b{hZXmkP!E-X~?Blkr8zqISV^b$TTPXPxY2|B+v@>eDo|cAlPSwG; zDN5+*rm7T_DF!~ENc(H{cXR&$wLi1VOM106UGlc{8J45HMl?VMnW*D(gfW>`Y|M#Y z$o~M2xvf8dKW+DXz+2U!WASB@7jx*^QuXRM4O9w76%w^Fkkk!GVo>Rytge304F^IMTUR zQ8?E}`irmAhw?s|@x2as7#6Y?8ML#swHuo>)mgVNmfA1`Lq$h7a#SV(n?A5oCck9- zYfhEd;hoe}xMefD7K0cr`k)+@llij{M;$ub(iI9-AytFv@S-FEg8*y@(-DXm<3ajA z&N{``AEq9l0Ixt$zwoP^3!wd9%IP)gIk~)Vl=1S%>nBa^T}~-CPohK4^AiQ7oYtG2F6bl;-67p!Rfk_ff^Wgdf(G6%!`7`I6Ui( zY1XFu7cf`$)^jy!ShZ?ait5m#buMMrjcZreU&8(s>s2)M)BV{j5V1xf+FPZDmCo`Y z2pkJitqMHX(`u%TN{Q}kJQlpwV@nmcb`O>QeO2kLpl54GmBlk7sZpiAJ?BRotlxCI}g2F{VJ60Eq+^0A*Muw?muon^tHJ zJ(?IT3Yto|$Xk;T*C^LCG@u(w%DrXfS;EpsnFO%BVqm<7@%Q>Q*;cx|<{YY_r>NAe zN{J`a4BClAVK4#PQ2P)DtG_MyfguH0Xv9cbiVu;z&LeKad_i;Gvtj1(-MUyidbZa5 z+J4;0KjZchaV+-TDK)fJd=$v;ZSasalqFFi9wp=MefvD;T%PWw3#&qibp@;lWgtie zLh8&9J7mr@-Ysq|@7YyqC@emA=^nt&!ee!llhXik!7cdcYEthp6-kJRMnzsfDgD7v<)X}ZFOl`+4K$ezSO9x_C>LR6zz z(_pN+mA&e6hL*F-c7% z)@xyF8xy#MHH^UrQwGu_Bx3=aAWG$kC4dvgG*b<#0+act19+aG3W+4f;$lp2_hEGl zq`cH0nbKafYw{i`(-_BX?n7&p?ehKur&)9?DRZ5{^)+#`G%@O05UWz&N?{e+KXU?q zXhbZj)Ta7P3XBzhEl|Z&446hLNdvIcA`eI)u<|L?sLf89tEh$~g21B@1%P506)5Te zh6EUr0|(Ggs1K+euAw`k+z+Td9?{6dxZT}urrxrz@?22SnbA6#H1iuJskxU_TN`6r zOjS)OQr-6IKVe?}9p_gy747NMp;2im4OXjY)Bq6?I>-z_fFOu78n_1TjhiX2b4Iz> z2#S-gduj(%b^}+}zC*F4*B#`>7aG>%7FAug_|&EYW0BVw zQ?ck!s17w#cBYU;W!alc^_h8#ABk^iTUqG0tf_Z)6#}I^DzO^sGf5}_KQ5Uk5L-NH z%c)bTSkW}mC{k6b>gOu+`?_u3;&?e^K0f z>ci`E>np5!yWC@mY3bs0`!=4XVdPsWPu+(R*9wv<%j2{z9cEYI96vp6WttHi`Zg>- zNkLwXTTrBKt7$;`%#e!C;-jbl9Z~{-!I*+cWQAIK*!r#MRkr2)zK8D_`rNPXlo**K z+o_nGPuGRd=;P_fZ>m;R)!m}e%iz{;Xg?PdR=pYIJpS!pQ>u>b)g%V59HmyOQ*$jO zEWcz$O*yO$uKvaQBlsfMo!5Ve`Q>D`r5yL0cNKD z(7qwB$mu64v8Pt%{_~(#wnlZgrpHUVXjeLjCrcISP-aOwb-SJNtgaNURF2AY?Pqny4jE5kT5Q#%&aZNHY(=Y@Fia=bTcc)RKj&TZ3nABi5%) zu!g>ll3;2;xm{dyFN5vxz96XdOPUF7by~f3&_Y(Vqza!_qehpif7cN!r`&(TQFj)YikMZZXRbT8 zLx=MYZnUT+ONmQ*LSC3~Isx7@|CBQWplr{0gQIDrWg$42D z)(ec63v8z;q$#$-{VbQF!T44gf{6FA?_4D-d_EmM*eM40Kb4D`9;TkHFj&SQ_aylH#Cz?> zW~0m=lJx*d(g2(SL_|c9u>wIDl3>dz&}~R7vs?KJ^&A$smxVSe=-IzkmYut19?GkJ zbyU$63EHd3XuC81kJ%2mERwW*A4-i1m54IUOaME|)uz$ zw9#j@p>@oj>f_a>sb;TjMwK?G7_CxkHl$NQgN-GsK9Ni*n&&;ekE5iGE(3Dc6y4!; z0lTzOhkH5Iacn*b&ufjfm{*3n2HkoEcc52LYBe>LSp@C0 zZIOZrB<=x%o?G}qZEk5jsJ^G`8=AEmo74?boindW^p4ZnA9)t7Xf&uUS@OelrnfBh z6g4z;0;NX>UgxC=0F7Pr?~V504~cns<@JHKxu;r%R1DQfP;*p-wqj39I~7s@Jwr<5 z^oshRYO{M?ws`&@BH-OW**4zV#mX9bb|`We>BU|&=T~unp_(CLy_sL!*;SKSR)A0x z5g!TNidAXavbUwF@m3H(RRqoadcd>lAC;8ng{d1f-R2 z7oZ6gLb3u@Gty*`EabSm+}ns?ZP!aw95Z-sERE4M7Odzz_A-3O*xs)BT&EbPKZQw3`HzXQ#}=lA$OQt$Fl^EE8HKWVFE zFtn;k;L)pmJ0-M?LSzGOVq6BAgc@cnAmk6!lcO-*1@rqv6skTn4v zmnRyGNhV2-Ik&|hw)^kFS#!^T`3E2hl<(HQf*-$X6Zp@V_ zlmXAoNCG%iy7SC;b2@#Q$|`a-7ckW2IvkyRde#vUafQ*dXD*GUv299gR=Jjnib4Tp zUAa;gC%CnyDfyjR7I(BQ?HOA_e(}~=8ii+hDgh)wI-(=6UOM-|zYLFwxmP|M&z0As zVOGZ0(%QJV(^HvJ(6=>KqUhFZQ?6XSS!Y_h6r2?qv-K;KR_gB_wTyP%(Z}s(#&%11 z$X#mw(9`byr@S4Dy%#azOs zR+F2daVFpuXW08~6R-f$hn2=x5o(SlcFAk;h%k?t4x`XvZX`7NzJuhE+tc)t!34OKh^Gs7;0`Y=Bp^`?fz2QnjpYX;VY5NKGa* zvCaSnLmX!-oDv9XFTr&=+wvNBHQM&QOADHm?bhKchNulal=E=aTH&Rb(sGsT^^Xfj2#|VBGu1r{iT*(%5;y5CfWA z)~D}ag;%60^o+?6(h87c+DYn=FnF@^`!0XZ=vpnslT&IpwQd|%xW>I;)2NPwu>+`t zVpeewNn0la`hw@2+niygdkxFTtDKU$s-tH{R{Za4?mqoctbl#nNp6H48q&(q_YGnK z86rZWh4rr_2CWIHSi4jTb+KZ8i$mf@7Hpvu5Y1uBuHi^bK;8}%wj@ElJ&kRg<2%jbTbZYx(7 z^5pRCYu=H4sv|T_g4+v)O4zj_)s*=t4tvi23*;Q?x*L0Up-?#$9)JNVNy})Fsfi*F z-bOgRd>zWZG59{8F*xS|`>tE9r669tON1~^Q*kMxtqxIXTKc3-nu+?P)`a6^n(tH{5T-K(Kn_{yuf z1?z5Jx)8Y}n%PnVy=1`MH7I&-CFQo9;>_2yu3eFHvuSu@K^Tx^#u`bH5ypp{^9ol+ zno%?Ag14Mw!sWGQBy6nIa{ib#Pj}^s9or(ds7`&_=e7Mvvb%jM0jj%C)0f z#eIyyMXNA~azf;Y{q=RpFbL+lh!^FHeo@Tcp=Y+OPT$rvy=@KYmKluilatavE(s-e ze~WEsa~9sS8( zE^g&r6-#t8Y*j{R>+75HSU$J0i6k%BE(AnCsz?c{@=KCVX#k*tDg&7iB3y#q;v~m> zGjorRe`~MsmCdc9a%SF6s3sXQFC)LVQ$dkHFHE67 znW{qVrZW=~NE0Iih8&;v#mc$$g{L&Lx4&vP2~}#;txwYYJJ1JRI*89I!PJ1cfJaf2 zs>{LpL&7INkT;N7xsl1=VGY{Ud{-JV3DWb+r)?;%q?k!DWCWRofi@&DNO%iIl)xae{ zVWpQzBVbhVUFDoRM~bYmZE?E%&gs*?ev|&rYR?z1?TIj3+exFKwjpP2wZvoXiCx6W z$JiTNa(Y*xY9(4a4pdZi6#xbZDoHXwgpT-dN#ZL$ea*R{Vk%y8<3_5}sQ|SbxJ=Yn z1O))b<_Q4fR0WpL1V=Nx>^ZBkpH_`E-F1#JXGX2GCZplj-EO2hBGZY^YT4)gIT;u@ zYQ(&v=3fZC%7wKB4rGI{i82UPBg{TKM*LfAd{8NGmag)&!jq_p zAP5Eza%2^!@&5p#L*y1J^3Em7xpto&UCr#xSk}?bn$=Nltn#BY*h6Nxp=0)*it6++ zrRD1)86c=#d^kC6R7=~~1b?e#i43?BLY#vqsL!#2$`9=+XxBD%D7WLiF;SW^wWUjv^c#)a$5SDn(xlLpL-bdiSDesnQjkZK{~Sh zq^y)0NS!euF(d~xu%Sk+r5iyjR)?uki4CZ^DkH=!90>+a4Dc1L_0YM!=Qs6D!K+zm zTH7uq78sDli~{FQdl|?9PK@XAo@1BNzD1Cp9_@T3XxZepw6bp*jcS~#ojRHOSS~*$ zC~It@Za6Zll8u1!W{8`tq)MyF?#;RcR;f)_Tj^7&N$ddBbm(zH0EVFj*GLDZIGklZMqH1P5Hk|GuV6>t za!R?^X;Rg+E5WqVP*R?*NF-vg-*&ikbM5C`*aQORpmxua3h(t^|^`ke=~ zsm`J@Td7xU1G<^QcWirIdacq_R=cA{4poO&P!w4lfKsx^b&sVX@G5Fp^%XdbE&{d8;_$w^q zwxG_=_H}J1f6VUZ)tH3h?V6^|-04aV0X2XJmvc*G{?=VI$yX{01?rjV0akJ_PFNW; z$4uonS55G%hIOQCRI1eKy+DY_xJrghQ!c`4BoL=7r%-09a88|L{{TI|VJ;i_S2f!7 z7G;a#uu8$UM@lk+7VBf42AbAAz^W)`my--g$LB)jJg^D{pnzg9Ry+*I5~TLX!6SYg z@*9h)HWUq9RjPov5a0pQbg+TcA|R4?#yE7ontzGA^)u2wHts_ix{Kr=eO}397c230 zZau9`_L5eUr;T(WVL(YUbm>-Kl8zfbKRKW6MF6s|NmNzSMG(_h2+kS6Be`ygG2D|e zfyJt=()t#T9MNi5DO=Ph{Xz*@fhE+F=jMWDH1NfLhz`!4k6iwrFsxkFv7o&LS&YxH zhgut`V~WcZg;oo;E?BN_V@1{BV(>Fpl)!;nNy?65{){k>C>*2-6=6E z=`6Mo5GN84gC`OUamPLYxaIuAo?CM|cJ&22Yc{hBR*+Fn=}7(IFCn1;D9T!9V`Ij& zbXSV<94?vk!yR?(qV|6bu|q!|`e!M*RToKFtJ!`Xv$Bg7Y`%+ZyOS-ezE1R0l}K7P zM~QihS&KW0l?JhOboFV4EHnb<7f6FO0h16X<dh6hzj zwHZBH(|{0DEEFsP4x`i~iw9iT%&s+`mvzIUaG=zi&D61TEVsaZ$w^L0DnEuR&Nwu| zxsEvUfg@Q#{9o|LsMj@WvVo+spcM2IO=RmQ^8inrcbT3L-P&7TSM(O8N`advS;P*{ezj!t?r?OsLXVnw>R`Wiu2B!O2mrv81KEkLc- zf~c{8`r`*>=C^1qjmwa!ZBe^;@j}YBc(%{(zm5>)G6Ll zYDlm(iqs*+Fi6N{B$8yzc#aMII^YhRt<3zKy-M3y4rB9fEljh;YVqsF+d7(7ss)y& zXE}=Sf=S8!wS^(@5+c|~%Wf-wXrfT0^_J&eUr{K-BS-gGV1N#N$>XTKN;a_E(5bgV z=*SH^H5tMYPAW%9EgGgwV94p=lIhMHIL|i1r=fbX)5&XBisw~FUZfVaC+AZ)Dd0}2 z=XNN4g2A>_VM!K2V!87jxy5#)Z&}from49NMxA6M0_zL_B05O?+K$r;j%h{j`B|!z z61ANbM5tJuL3Ej9WG#eJS%^QF2oR*ieMRvGKZ$XjT5+pKUMQUnJtWsklY z8ZlvZt=O?;>vfuQ2%P$bA|JYpy*$ToV^i3hcLHCsD^1WidCtlH5%46 z24Z{0@f?0N{{W{4DXGh7a&L+)9+FnKzPYieXwhR;S;?(Q>Qrf>>eVQUYB2EbLl!c* zRo31c!a0{sdY{Dk7Z$yRM)h}H!#ccwewwQ?jQzP^4S>6MjlS)jZ#zr4iOYM$7dEN3 zNSIv@;z~3q-`&zQ-ho<#(m0nPr?d?-83B8#QZvQB_8ZQ58sMBs_|HwBU1WgxCDOgH62;e^g+b=g}m;hmq-4q?{x2!)OE_7*M{%p(z(ZI zp=Qk7u0dsEV!nnr(1g^vQ;}v`t@bXKe=RpM>IE=e%Wkh#s`?esGXgZRVWb7@HYZk+ zWf-i$!7sZ7bOVdR=5{G<2yN{6SG@q11K^!DRNxu(HALBZ9+wb4wYI}tcomrWoW;Q zuX43(3tHr)r(dNtI#G*jHA*uxX=2VYCSlWpd9O1zDa(q}4`)!qqA3+As>8bVgBn9B z?iqx^k*kJd=~L^O_ItSh0PV{E0P=7CTuc4E zQ^@}SiNDW+E$t3v)xV&|ajR(#ll1$eT*F9z$uG%@+|!otvu@okCvRMB-9TxxTSnrF zE?rl;&fS)IxhI}-KG@J&)heM>r%lF@4Gh&-D;9__l2m{UFR@d?8g|qvb1BzSrAwN4 zW+iD+O)}(k_WI{(B#t{jR{sD-*}qA@O}!b^*y>(yP`a1ZD^pN-%_L>j;k;Wi-6_#k ztf#uM-_)y#;}NmQ+P@7g0y(EOQ!7{sD%}!kTv5;&bs{#CQDP;kXkwyCIur>2iIXLQ zqja}*t`QgabPkGSn35PP9f;UWL_jB`o-*Zn6uNJ@$TIdZc<)d$IR>sajoqCr25D+H zEi}nH6T&Q_=1TLFRYk0=Q3*BMyB_n25;-_)Gc7Wz8&T*X8%&Kw%*NR<^L(L3NCScq zS=g$9^B2gN=?l_M6c9EsmS&EqX>NdZH-~YqWz^Zq+^3wat5K8F%$`QQD*h`}k=NcU}!qv#)xO{E7ykCIv z>i429rjABlY=)mN-)hvGI@h<|W{Ln>g_0KeS#)k+%Bl03BBjlG(M1h9YK#F!-%*go zn^)5qGD48L_TbppRHH(YQ@W^?YF2Gb?q^7rEJ6Tg5mZST7!hVq(04=mJNTbB>n|(n zqB8E^GUj`Wy-#Onkz&@Z=n9(mQmZk8i%--IYBCUVs>lkJ%(<(4ONBz((E`mkG0#Md?2E&YV3p zY$MO&9!>Ct&pKPnik_=m^7>Cvz>-ZhsLkohjRk`fBn2cf0f;QClg%F1`pVVHrAzhn z)Wqm^{o64tbpUjg!6F8trMkG*eSR;K(~g-~rJ8}`9A}MWMs77aHY&Zk8Feesve-px z)U4Z4HID$%nFUF%=4XTZO9)Zj{6&9deaozsZ#Ex^&>;5oVv0vj}yqRl&vC0R}|xwxcbNLWh2_)DDMTEHJJz( z-eiIuFiTv)xMSy^dA~Kj<_xv2orpnH5H_G9pe?B?F)*WrqH zuGdXYwbkD;yIb3*aIGGlK|+Y8lEzR50|T(&IO6nbB8$SaX4^jTtnf0JPzwZ;O~osd zi=Hbfb0p>=UM1q?$dDeSsELUFzJ5{MN!vRO`01n~mX%#vvSV?%*qD+D8%*~NEym{4 zto6RN5}g!2UJ{yXB*Y)EVJLNhI&TrJ%0F;}z;6 z7M1#sS|A1pN#1kv(eSBpcm|nMN(NxbJ-g@pBzNP@Kj|4>lKdTa)C-!7T|FeO1wC}s zv(jP-P++c>R{4)%sk!Dj*Y&WvdB(Q)9aA@%@!oKG6m9Z4m+kVMI-1J18SJx3`T6@- z{`QirjV%PJ$fO}Q6#LBrUHnz0XLWCJUZ1N?Ya*ILGPTE~RmP^t6j+uVzgu)I%d^P>gA|mpwyuA1eIq?mfr329>Tb{QfBIPCdr+ z^sl0<>FKJK))?|!{ER6le-~!$oQl? zMmk_>!uPf_v^f_}aX-oNV^Zdy`P4LO-Km#m*1c?7!Eknym{|6ztNlDq!g)3KN{P89IOxbFdjA>zQ(EPluP8bDFj4a}|3e zO*H^CP0BUBYSdVQ-kT{^u7jo6&;khQ0q$kp%ByLv;_bx-^=lU>X_b8^Kn%k}&mZuQcWlyv~UX=7VZRAVX$kkcAje!0hBSLTDNAy9tP`2~Vq zTyy&XDAeXw_P&){Z330WOCjl#NuiaRRa&6X%u7fr)2vnWogJ|`E@9EJXEzsN5V>N0 z?wBk2OPf~EP-6-v$vG=R$MRAgiD&JY_(hVGqft^1^~{ICVTI~ioAsBN4n-FrM*kb zW9q8B9wea|Tk>pn~)nfba)}dnHvfhq1c7kTD%GT=G zT-q=a2z@iGL<_FONvPlm%mY%})-~!>L~RNHIbzG)43Q!SPjNjup=bD%o9=O8Z^*5= zirUx|Yx`;g(6Titvzm3}5~iaG4jN>F!9?j%pm44uZ|}A_$1{rx;$`ZQ;&~6&zI3YG zSE9ztA{>TPiX;oYZj+N~VDp0n91o3zT#B@ggQHZC^C0RIzeF74s7dn>+P5?3Rz8Yy zeq0u%{Xj{(s!*zeRQ?%H%Id-O2L!wV6iDFpJevnxA6qT;_@Qwp(aO}f#~nhxrk+c) zP&Ms+@f@<^rmGg>iSV#)S;GLx;z=dJYjh<;RcXT#&Cp2_BzX}4{{Xe9#+RIyr!}>? zsZX8VTwmLB3J+LOxCGl#txv1c^hqhHUP9(ECRC0jdV({hmrHvOsXUUh_}?F)pRHSc z@6n@5f|q1dcA?a;cDE`rNQIBTUqagI+V(EY3RmI1t@T-_L|Ze8jSksGNc`CC+vOmg zz#K97a_5uTb6#OW=akr6)xWP=w=KA|pjNHRg{3s=4JB>rcLIYYGH?)*&BBv&sl36g zSZDGX+N%0fo1jvs{k@DuGu4)-gi4D>lrm<GHhOfOfFdD!RSP;yi&N9|+_|bX0jXV}xn=$)Z=0z@Z}P}igQmYw9H%aw3_BRO zJc=(^si4}rIj^U-T11M?VB01yh7c?VIsCoaCd9Zc-}Q(CX)<~t`Y$mCE;Z4EY5cA{J1sGC;6o1dMez#Q0od&V79DiF1dji1&jV{tl?U6D&n8w>+@TlX7>)aHEUm0 z)L&MoQ%z@0C(^(@F0kr#2mlTWyeE)aTE{$f=rxTE zO~679p2jOKDz1e~{{VcJOP~NcTGsVxiglGKs9@xxs0*cgsK(Mv1}1kfXP0xXL21dk z(fFE`-Q9M2UxWNTWfc&5rAvcXrYI{JrgV)UX*!CA2M#+NTaWRoy7tw{QJrr?ZssnB zBi*f|S05YQR;#zMGOHEpv}xCcT(Lqlom)UF91_>)NdP5kwsr0vl7uXppVR?#g#~4? zA%vWXQ@n!&@yw2gnAE%ciZW9=Id@L=>8iTv3bizv_OiBS z+S65YM_y`uuBb^;(DbqqCzr2JRW%h+jZl-2(b%h#f_yjTC%kdf{u-rk%z0fZwYh82 zsYmgYDO0?m>C_cV6{%j5g;o?Q89E7-5W+!V){Y_9dPkr9S0-JXA7@muGppIaOyAP4 zQss*F4Me-CKfIQ_tX-hiRRj>NnZV&TkIL6GI%9&D2*KfD%(Az$ zmdv{M`0kUFj?$-hPcpUJ@PusG1-dRI&TKIQonZp(<8oPXLX^V+{m;d;ZK_>Sw<=o6 z0@p3pMxmulDKJ^KQ>lys00)W(KllUU9%t~?jSgf$t98w2R-;%qr%t$_;Jv*t>VTD| z2`&zj!%VR!P{T(4M@qS`B3@%vlaq3zQLcM%7Ey4~e#32sL|J^+Rn}OsDplD(w+gPb zO0sSx`teHAhs3;=jecgUMz*SzDOOXGlcY&12^v5pNrK7%B15ao{!o_n(%y-wU+{>j zM55?mO(3Mz(ixhXm@b4vfjWo;C=Zls?DBrKV{=Tr+Scmunhn4{{RxWoTg1W z^b?tnnR(o|LyqXW(;=3Wnic9`<-WBjbwyl#i!xn-zbECj~*T#+4Hd~9ohi;1_ zRBL*M#=7#mpH3o^yV8GI)Rw9hXRZ1J9e3q*De)emGWMWZ%-r5yR~XAG%xdoFw=RXd z>|V9T938nEBBxp<`(bhEg|E^-6kpt+xw=9~VgOJhVifOzfr%ny%$YnW-rSZhyj%zHJU}4WXl93HyPp6E=!Sft@*d?xt5ib8JvF& zH>j;z85gd-G+w?!+idqX-bhQqS^%?#=G!t{wX(mIiq%$?=i>!2TBR(4Lb30t9LXdf z0K|fzo;rO_L(X~qoyWteS*=p2xvN}IXf+;_7Nl(e(+c4W5CB$Ssu@8%J^X*CI9vTn zsf}8TIQD-e*LBSt?FWAtq@Ki>Y1czDhIA)5>`}K}E82Z&iP5bU*69-;mdlIQp~|aY zB{o8~=Bgzia9gEiOpcad&9MMR0Fl!rx8mcurF^W_kvbI{S zAnMMlNQ!JEh2fa`c*ibB^-s;YPYvVrxb9B@;X2#Zs=c(e0*!1s_3`#Iso6Cclj9QM zj1`C`{q@SEF$5B^^54dM&i3i3*QnK6omgexaKqACw{KJN6zTydN};HuMv%;cHFx5b^+C#=v-Q2xml~r!w-MuG za=cKJv1pW3>keIJIvh*;;dh;@H&D?ZaQH=Vi$1Yo`z7T|d+~i2qxV zs2L!FT&rY_vKYZPR#omk4(8mx>Yt|71#w=TGEEb0>PxXuk?At5x*yES$4-zKeb;_H zrObJcOZax1_PGwHFzXLa?r18zEZJhR*=1VRGHO>!+3s60DO>EA6Y{VpQOlGy(ml`i zk!*QCE5Es^P@0sj>(i)?B~q(VYIJa33awhS$<78rE(V@u$hnmZhUNUC!6{nV)2g=g z5dfo7zo%H5wG3%`l}!qDhy;-3R5{SZ@v*#1I?aAMtnCh2iGPk^ww9%wt)yswobrHm zuCkg3lGM3vBqNmq+j!oJm8n}@szTzN6Qbxeu1M>vm+|FsYMsmSq1T^ zvdCB%WhO?TNEiT5hp60ExmFFKjWW+v22^6FNno$#^<$<6O9>iU{{W`?RNu)uLCk2_ zC0H$NoaxZx9_VK04(cr?)Z}?rO2D#$oq@gKqTKoSx=O8&N2$ReRgQm3rQasz_VsD% zdUR`4O=Z9pEvkB`N`fH@(~Qi_-~h{W<+pWg`NcC@<4QGpbl21n$#hDhB9N`5)dpI>Hd@Ph_k2b? z(qk5{+0Vutx0Bv}JDM6rttl%~v8`?^Eqzo0Z4aKhg}-(;1_rYlm4EGT;r@T|9fcZv zk!lNmVRE!9+$GIQCWX);b(n&b^Qe$SLlZNlua4o`qTg>c)SMnedM`yX3zV^DACw@cLN2lGCbz$*buabm~)b zHR_}SGwKCcsE8%F=j!$%cX~z2tLS~|uW_9kmvd@t@&;;IubFv<1ze~sLZgi7S_w_$ zn`^KokLciG_tzo_rjGg-BY-ysg%87K^h*Cf}|M|1Wz6^ z{{T-9h5rB-`!x94ooi)BvAVVAyuIWBR@EMvs3}{>4E1T6>8V*+PEN#eko=YpE0NXK zWt@Y|&b<5$)!#=kJ2vvK%Id&4us-`M#;pc7{%>nC#aFVbfUJXtPv^RSXq={k*Rtj- zOa-{665|9Yay177Nmn{b=Mr65QU|a!Z=jsZ^(VXF=+yky4d(Ra(J-U{CJ@6%EpGNF+}c&Vl_l z{X=~t4-f0lPHtp%Qh3hh&k^eX07^5Pr=Q6+xNc>mu3Nk#dYz?L!&?4>t2>W9kRx1BmB+9Zc&l4t~;+v1CZzF z&;hjOUB@D+tk-6PJQfd>l-6WN%lN5Q&XKsex2bPQ9+4r?YxJuQny9P_ogzV24F*Y^ z1zX9eCB<8sXwF46D@EHHoE^vwvqm88gZ<^Bt-nyXx!g0O-B;7;rw-)(80A|UoNt!z zV`yJaHXUmx;hBG9JM5Kx7OLshLo3~VVmfRV)_jx@7?FF;tlrT;2Cbvi(n~?60L(JL z2_G;34M1SYP|htVKdE(YSW>u3Xla2G%q5x(6@s!NL7W+rz%x4`+5c{W!S(X)kTGYab~s`nmEIXsrzm*{B7(iycdrCD@K#q{0k`B>)Sh25*I zxu-C!463h8^qJCCO8#Xe#sr=-POZ(o$R^gMN)@T9`m3lgj5f6}rBiSqOaf*=ReGP+ zC)3|iKAU-m9QwBDCjft4dT%YMkH5v~XYr0*&oARV)jM1w`n)o8d4C%iRi|TX+AO`O zm$N<0s@VXlT3p)HwRXAd zrU;l+sM0mgkkSI`$pps=w>bKy>EG24O4G-=r%f!ESa=?VC-mEwXJJsON0DG>u+gks zlMfv=D1~^cpZoJ#tOb&L&5LBMz#|9tg&XT(LZt&v;)DuSX%ZHq8TCCx(}DmU+!L9> z*P3fu-Rc@uHPWhcP4YpIN7g1Je|V5Vl@;{P(DLilD0LH%(CN<%-{tsO7@UTMXWPc% znNF|icC~B*`@g^AQ1BO9_7K?2LAoXsl@q=}bSqsW2Rig1T0r#4R)d_*48ZD@fI%vF znz|}d)lQbGpcg2z0T4(NkO?3U9mH?OiGSR8^yU8m*)#ff{{ZQJfBjbYv->iq{nank zzw&U4@%MlItgrt7$}j#-3cWh|co3`nmNn&FOUWfba@DpR8I@#cU;4^m(Sv zUDjq~!j)P}6N7Q(hlXPy{_B!!+xwVRQ>1OZuQCw1Ic;{;GClFOtYg?yosk|&rX%;U+ zjW0%>THp4X`kzv~s#8i#^eNOI!&vA*&8tbNtSe|_Vd$cpi;8q;Xi6^i3Xnm1jCzXU z+gm}?A{fGwp^F0_)$c&Qhx!xrC&0Rm!g)TX=HEHe&Bf_gQLF+qIlo)2uwM=3_`i0- z$;9bBwR;B-_{v3D)n!&|l#P@VRdWP;O7B3sHNw@ZwW*af{{RnB2@P#ho08P3pj{+X z4CS|NdWRZV*S&kjTB)Xz(Q{u<$h8QCq)8v^KqF9q7prAnF-Ph9sXTv#bsD~BHsJgS zE;5df8LxkqyG7Ycy`+hCTfB!Ns=MoJUC=u8a{LORBHOElaFxj!eamhsirQ)**6P^m z1UiW%ZeT2&ff0~&jZ#%wm8G?sO01>?Oc zr*rAkxrC@z4MbXlaS<(yO9<^Cx#KRPtBrKSy0V%o%uOX~e<>(%8AF{!D4Y|r1<=l0 z%y@SW;k`fT=hI&p=X;!y%*55u<2$^Onz{&%Lp8J0x78j+dr?{zbMWg~R=wUQjCNZ> zmU8QRi5R|GgOlxCZ&`Qq%khXaiIpL1@wSfQlWb9UgG%ERMym} z8etV!2AHNg8Yj8b(W~Z#95h@Lqnt~o+#bF!ALJFsRyAyHb1ehQVE+I;$TW1?-F=2P z$ZM$t6I;<%Og`NwQoTi)A^@zsi{c059F^?q^0&83+QOi|CKBL~8lyU}3d+J(m;`{J zOiOX6CgknRejuxTc~+g(t+l;Uw&_qHp$QaKVVTvDg0Yr@MrJ`(H>!Nn-k(&kI986; zY)pJqmmuBTP0KDpQO|f>rCGU%H&kG-;S^tF!G_A<3s4J&xEDA0o`);uyue;lqKj8o z`r1Gys%#8O3~CXQut~_+22b%9HsLH{^sx**)dR+@0 zD#Az|M}q$VGKycxRpxwB?tShn$g-)ZZogiq9ty*!dW~(0N3*FU83SxC5<@){-^mvN za<*D%)%F3=@2zPuuW3*nEDFFmU_|6cU_swIfTQN&-?F=|ZOQE^(dSg2*Pimr>a|*v z(0Y#Fgl21M@VZvJGbz&HiVzf*VnAHe=X_Dzf0uF{T@0A7k+V+rrb{z^^+?gmucH+O zv@Tt%P0yq)g~MzmEvm*097njI)xfvrdS8g6U1=Z{YE!1q>8*1vp|b&ok)&!7J+!F(zq-IIXp3wU zl`he`mYA|D*Oph7jhVImF_%dQjE|WJm`ceGm;iSlHU#+d0QexB?oS<+pok+-#`!V0 z{XY29r)XKEw#!rEnzmY?ujD3a1^V353m<4`H2_?d2ulUD%pRdBYMm*A^A@2{`eYJB z35)?5-f=s1M}EhNsg)qH8-Er(qih0C&8xKcCyT#CY!CcKRo~{d>y_5DM@r_UnicB* z0N&edFk3UzXlPZN9+4dgsb@iAFeL_4W5&tw>s0>Hv{2-AkQhWn84zP9{Gv!FjwAbO zv9=!qd`Ep!(w%pqZ%(~1Y5LEr*QSR}JxL==5?4?~nKD8S8(KIcTUVyH^)|M)BzZp^ zR~@K)lZfuMcTbP(=r?~N@cpybSRds~Gxk?e_a6g#{cmCv(&!g9V7*GGE3I@XFb`Ne z;e@vMJv$iV!2GkDo9ljS%WcJvE~K2g?)L7wTfG*2H7L?sG9{j+LY-mNTtO-n0>wfZ zoa-Y6R};iISm^50zsNQEhqKIcZys)zrfE5Ks!=Nqk6lYKb(EQQm@>50wC6bH$krq6 zR5KNQDAm+cbu5t0DZewxr7d*SNph{5qc84yl zO+N=k(5qZr9Zc-ufT$@s3`~G@#}`g7$%~NQCdAy*PE_Y))Y+`6%*PzR zdYrdZ4Lkls(Rlq`yOw7!%+GntcJY0ES zAh+ZnvulO;s8i?H_7=40JKF{ot5>$H4JwrCI+9IFNR=O20Fgj$zbKWez>QS(F2X#Q ziH=^X*=~;^;T+s>w5!=rX4uJC23vf!Dj%v`=Hoy9f~Y#hgau0PmYY~-(kkiZT(Oy0 zI7DUCyAq=qA!ZsumhFNd>Y~=>Jig*+2LLw2S8uf4x+)g+jW{HpYwaaS0SD~Ld} z8iQO2j{^9omQ%FmAt*b(mLb^4)Xa4m`Jie#be6{5S!DKC?7N=7#XqzA+Lb}Fta(M{ zk*7|yRj{c-*{D!Ob|4DK=|$A87_=S9R!f;R%csbx;`G|J+8FtJyL?j!HBQbban4BK zYBX^nwWm$G`Zu(nr<zjN>$d{MeU2~yObE~QpZaJkIW$tU;T2XbULcXm^G*d;U#29Bxh6^j1-%>DKZ>U_8 z`HK~~eeL|-KSx(e9N?u#Y4la-u#e+Vr?K?x1k$Ky;MhN7nx=@966!7JUsYOe9*drW z06Khu1-2LjnCxfrmCowh^M8W5Rkh^`{I>U%y~SU_)K*IVlGJ5WP&q8ASTa?8BLodP zNa0E7SZr109Lt7Nn_u(#7jJE5*J3VxrCF<4)25(L^xD9gsu5&Y5DjzzgcSCNmP`ZG z5zt&Q^Qei5Bn-x^L}U`t$0g>J!R5BKnaR0I27(Yv2BeJ&dy?H?OGNiLE5W;#(b%n{ zpUQas{{Ut2QE|)FPN{mUW!hCLcC5Tz?XT&jU`ck(^9KZ9)9_JtCQ2=v zB*+0sg(5`dgn{*#;jb^IDsl^ZOWH<*(<0)XWOX{K>SIWq;lddTLjwnRmMvoPJO*C0 z7k_}ExkYz}FFLYpy-cZ)m|dL;$aPi|lOELz##ksjulrioP{pK*waj@kCY>k(sRe66 z00v@0nC~;5t%E?y@knFwJ)bkUdnUNKvAU6PuE7J05$TpJ1BW_fI*DO`BZ0s9LX7gj zxyJRQXE$2KyLVMhzI2zDq$N(oK84kKrs&vNuh$a!V^kH^*|0<9JqQAw8l6DO!s82o zPjGfIs}dr2;=y&#E5F0_D_mRt0{+Y>wDsvlM7dWb0cD<&t^V=JVx}4ke;*e~5L|yd z{GYd($fm8Tc52vO#lh?-oy@D&7`Q4ms@0tv8114rB$6_)7&RG|>YV|VQkIftO9SmN zPE5{6e#L-`lXLspXr}(XUU73>LlgiaBD+i!g*^$JNrk`%NMXJHC84X%DPhvmxyU@R zVfHpJO~HpIEhsKs7QK*5$WXK`8UFxD3lei7L*i`tH4{iv1vEe;0+3|l1P=WzldBt; zAaT;HxwXBmu(_{J-irpBR_D_-LDF={0g6;D39yd-I6Lv4kX4O+)o&|7)0^850}`39 zE{@2zHFlR4+htNMhO}2<{-}K`b)+1IEsFB_1%6~HY7EM*trXRW>mc_6A`WsG<^fqP z!voDZ#roU+4JcXKJu_DJ^0*Fd>4z%_n3j1OBw@a!DRZaXhmUa1NucSq#)of7=N1B~ zo$Wq9YTXqLdigwwm7P&zWoeLJ;Y+O6(v?$K2(}Gc(&lvTmBm_tDg)4~0z+(6u_j44 z>_!aqvr)b=<-ZZ$^DA1My4t(^J#3WeR=TeBDA2oL%ZcJ2ObC**<^8TDo42D>f|Qxu%XFQ33IeKu zk_7bk_;t5%jmO&9FVKDwLB_mS_JdYqa3lZ@`)w`&huV!hmxt6deWzKc7zQ5rPvie|p} zkV>*0PO=W|R--U#4~w2lxo)z8d_0GUeRb zpQTQ(EU|kkblPd00ChR5Eazdz9!dl40?y9Eb#k84Mb0yQqt#?-eSJo%{MIzMCg($ zr*8GVnnROyoL=o?;jaETtTK+m>shqglrNv&UCUxa?&{QE+S$Rg_Y5mUur$j~YL^Dq zliG%GD$=U#fwC|abto~CNCq_sGBojK=06hI^2*C*E<-g`srY-w1yDV@xiuc7g&Sxj zu{9X4NYr1BE?I$v$M-X{KK+_8ZmlY5Wol8fK}$~5S4~|{B1*M#*JXdlD?t+)wLH>N zLs(c%yM8v_(v1eIR2inB+b7d3fU?XTh&c_NIV8a29|U|y%>FFpS62^bPL0CXxTQ_1 zu%%AnX#W7Jt(J;q=`uM5m@$~sAn?X(a=cAmS(nBUdh?&h_~kp>u=4v$)LCAduFhg~ zQXVyIQBs!_b`VUf_LdJnNl|2mLb|-Bl2f~_XH(Uw%xVjo>{q7*>C=IQTxkS@f=I~W ztNzh%Xi~De=T_-P)04Au?psYkabCSZX|-ln51Q2_*_#cew1g$XtepF)v~+0(v{<58 z9A@KFos^(*Y~I+9;iptqA~TGrU)c52rvFU`qKr)Ym9fvW*dn)CCTtU;&@vu4vDR8%TZd6jL< zsL+#i|v_0=Ip;0PTYa5D@>gqQxrjV;vDHH@140TT6bk|Pw zbi0r1H!DWHoW0C_3t@Np1YXAFe1A($vb2+GZogf|=*T*i=lJ-ibPZHe*(R)d?d|8{ z`42C;^{E<-Z|XYwlNB18lAxTHk_dw<2Lb1g3iEDfY0Lg4_|~N>JG(JJLsQjBs|xkpooGZA)2yO>VbIaZ8l5rR<7o~BY?(%@zyWut@=8@1YVErB zpq*yD!PkWvHOUu&gyM81y(SX3O%H-tC zz||N(u;NJB=wrwy59{Tt@D+wz5>C*UP&=Azcl&aEhlVwfgI89>qikOK++ z98U2NET4>{cm*R~jl+Ugs;eM%1*EB_u>g@BAg%^P#}8-Fb|#+*`eWtYR_B+g;_|Me zbu*B1x*8g`YS=dM&AiGyX9BJ@V2Q6%ld0y8o9I+zS6?A5TaB)k8l=3z&2KFEpFQT3 z_1c;?ys>P3H3>?!E3|5t4^)qp+t#a6id104+odIzDm8D-s#TylMLR^eHJafo`qWqV zABIOOp{oEG5lcf<)E%{w6@mm_5GP!#$$qMNd82H%Zsp=xP3O7Jn$_E%Z726N?I1}o z=>)Bnf)*t%zusbBa=eOxvfI~{t*R`&U5Qe8+L1ut-nNU=y%rG;z4j$_2 z#)UeRu4~mMg<3b1*qV!?=}OL%A{`EswjZ{ZJQuOKJw5*bsQ#5!W}1@TIb)Y~L6)O%u0hFuWMSRDpwXZ))vWA(<*A}MyeFFDpL$~UM|9<7z-L>JIsu6dFJ1=J5kDLa~9<~szpsTs?dE(DV;1SgEB}u zi%OC8o)k#V%C9 zfeH?1WlNLRZdq!uwAosr48*pzkkV#9UHr!#KlXofOPX>XU(TE>8?>nCnqZk64TlcG} zk#kA6EzON8$VP=Rr7=E)dz8UjY;4Unt4fU(3p9qFzJxjxNh=^ik|%A8L~(fdbCuoR za$9?fmsKuJ*f&MdhMJXxg<(X60t}>dki>y7O1FHzpz;o#@aE*ut%HMeCf6$~RJQ2f z$Ux>&Nd1@HrB_0hyxVNtT~-i*JxX6ANbe#7$d`#h~j;Qs)PIZe^kMuk6!n(B>B zHA?;(?ywUoSP=xamKZoP3FFDX=>bOj;~$6l#eudr)vx{uuVqr-O+8g_C@pO@DQb#p zA5jK0zz1^yvKy6S&YNwewrwuQYZmKWrO0%6>s@gKM5V^AFb;yN))l$WS+(`7Ko6Bt zSe$(QZRggO+~?u7sH$opwx?dzq7_S1)D2B6U_PM1fMPbDu0I6-0ASx0U3@wFSglID zTh^(}Z+Txkw5v{?dZbgj^c}5|wJ@TkxmI6It7&Co!kXwML=>CMG&vUv)Zj_Ay_>zg zlzg9Gj`HO1>~U*3TcwrF8*2QWSmfB;mya>_?D4DgC5H8_iJ+TI89=7t1AJsmZheT=XbIzaeolZ~FPNQ)Q+?d7XRlkwMx163)k+q~VkEfig zbDN=5p@76%K~TQH7PtcCcJ-|uodiWzr%)Qm12IP$Sw%$>9TYJP0Xm*H1aZ|Zd3`#z z*9mm&wOmrILbDY#*ej_t_L*U*G;Ja@l?8^qZu+F;-9O>oCF@SME=IQ%!Q@r2`nS{F z8kq&ibTIDu@%wu*gMmIET&?ups|qIJn@)sP`8KAD0_66Fd^tR?wwNl+ArnO{*fXV8n%%Qlav4x`0*?RhUNr4iV6< zgMCx|9_(psW$^BX_3xt`ksLpmYVhmd!Q*mh&TP=cx6E)TJ(Bz$PW<#!yQgMuRxJj~ zS0nc9XUfgBr7GgERyOr1f^vFldDKpQNsSJm6@pknA&N`NmiKCiUaHon%Avwp7JlBhJ&NrFUpQ&zx>|LaK7=f#@?pvO>0OspiZF;HAxk;AT zidO|BXP&@@)R?S7!h~8`*0-#TT9s;2`pd$W*r8odAT*|-jW9?uQFT|lrEpQ)y&8)L zNXSxUOv1KT=2Ci0fvN*COT#Jk05TuLc>REd{{R`Ir>&rv&y5yg>PsDJrg*vNb!#;2 z*D)2442(W}b%Pg^>3zUvuWw};);zpcz)$jI#BXmTztwtpXIKUYAT zk)qVSafI+|LcoB-1e+^YyK2Bxa>13=Boxq0kfikjq!9@7{?KBCdiCqt+%P0PBATCA zflo*UpsopJ8nQDc6?bd5{lgt+{{Y)L$iMhEv;P3bzaReqVh`i1{{RU80Lj3=+q6IL zsDH%Y`FL~o^YVEYL;AbYeofT7Tmz@yb5^%fuG5OOw|N#e2M<>N07`M|Nm;Kf4s*-( zD^qrI#_(IuQ|bsN%T~fo%&1bown0SEHLQtxW~i8`LYP%60p1jA$I1cKrAnCIr5mah zR+p((r6p=Wo{(Oyik?W&iby(tH&)$M?tpy(a9)vpCVfnGbL+>bZseMMB4`a(HmS{*n9=^rr$tj;+f`TB zsY=sUe6w$qhlIZH24qFmsq-?!@7giP9ebgKP98YGdFY(sB|dnKL*aI4Ji;=op22c z6x7fLO#?p6YLt)bD97byxocvzRdy)Hax|A9u#k6yjZzeY1Q1$RtwkJBdZ8MuC=hZ% z$dEx&4#bS?7?4jD-Z?#3>fQbaQ>2qUV&XzYB8n|C}K%=0RRd}W+kuz15gqP#}ibiN`ZQSfEn7Y zLRvw@mdWp#I-p>P5>Ke!hVUQwW5;u4)auyUxTQJtgFdRKj|48g%N{pfC4T0MNtKUE z-S^0oy%oU<0Vpq5d?WBJ8XTstFz4ooxvVY9C08gYA^H7Egb}Uc4v-)$LBQkAyzAl% z?sM@yUR7o<=<@qys?>i4>I&C4H9=Z_qXhve50q$v1L`tla6S5m>tXbp5_t}uA#yr( z8*53?HWi->a8{LP*ePVBZ&I6VA9&P6N)1s45)myXzaD%~@O7ssPFqzd*ST5%sOnhico<7_Y|H6TJo<*YSp#o1 zMy{N(%aykZ_0UUQvt!IK?KeqnI!Rf;vCmUJ7+tZuTP5cZac`iOBcyFM)0b}mlhXYwCC$^2x6^` z8&)){r#g1RovXB}wR$(J)&TH3()pNFeE100IjU0!V|= z`L(aZw|u_Z*PDd?(zn-6(zTK;F;z`9qF0z{K=tkP8K+FT!-xbfp~c6kcsabTq|oHI zsfEDKL+Ed3FR1-VZW!PRx45VJMIo-33Kwj&G~&UV`k!@ z4|0lG2t7g+&)AdslzngmW8g1|`LFED;%j@ZP|x@Z)wI{;wQeZba?8}Ys1nk@{NO;;?|9>mOf%uE>! zG6;Th`>H>Ol_*0g8iBzC9M98uiNV;B$LY5q=Sk1GeLKnwwJ6tGRcKX4ko4Fo)(9e0 zPTL)c1dTE|?=)44*DQbD*93)?vYnjkYcbVBY1f@%6gD={7Hv#PuejDwmsYIx7S+l@(h##YckZo$ zCUzR6s2G{!@~Wkto^ZESFTFHd8vd6nr|PXnbZT!>yPy&hjWp8==SV7BI`~@o_eWjF z^|_I8-f*1vlTLzb-?R2KSzmSvkX5eRf16Yxv1k1$ZN!(*uV;V=xy_e&Fq}5WRT3&i|Zs~x<7L83v^y?v2 zR${dRlm$T~5KJ8)?p1S>*yp_a9U4K;wKc0@^PWMaB~MT&Xj#tAQ*Yw2Yp?u%%_~Jw z4to3)cIwmw)d$otZ%X)5y4U3>6~xu1!HEN?$zuSRkj)3r_h61W*NNty@iSbQ|vA?5M?4#1C>rlCq(rY+6 zfG6_PCPRK`5P0dllbukneRuFj!}bqhcg{W+=6vjWPf;j(^lxj}k5GmtYE`7Q!B))Z z&_IU2v#7fGJwY7q6TjobKa>l2wQS?quM|gmXxh69rr>qBpmJgNK-MD4!4NbTUbyxu zuf?{_Y3bZ@`oLLTDPD{@W)p=Phy)Xu9rq?hi}rQPYP^ys zw_Lqzx>Q=Bw3@_CHC5NBxh@+E!nfC4(&Cw0xUJ5wXI7U|_4$G~e${z1VO3ieD|xcn zw$0U#t(CFrFacG?q{g+xVgRPjot3vNqyYx+)MsWVFn{-AbFn5doMeqmH!|e>%i~^m zSKy3unh7}a3f_$xJ5g-A5$7R29U?Owa34dD>^n8 zOxXH%0?-Xa)oE4Hu=0I)s?{=7-?k?-1mZ$8{AZz3M5%J5D14F|a@W!9RMNn&7ILp!gx^WQ zeZr`WeWM^0YG7QI6~OMIA@N%wl0q{!URr+`T%)ZvN=;FT=`y2G)1;luWF1m6qw`j4 zUq3CQLZPQt=?(R@0ZS~S)omdF0knt&nV1Kp<~Ulp4o4)bYYUlUUeVXU%hk`A)uN(} zoU**?(PuA2N7e-4vr#S;wMR=;N?M?%86q)MyQg}Utp!vSX;E;?I3x*~j+Pk%rbbAN zo;sy2PLz3jRJ^vOZuivciZ<0;s#T|?pHhnxAO$}%v2`+J0UR2+PfxaW`8^wXdUtU7 z6?&HP#{PbKe}`QSTG4g3%8SScIrnVJo2fxi>R{ki$^5Ki$}$j}QQjDtK-c~bco5lu+y z7FJg%!ns=II#w_&Mx8@_RG1xodiO2Ox_T-B@U904Hj79W;Zg)my&H$TrV(X8Q2~IN z9vEPp?E~9@IcF>7mzQ;`a+hi~X;vPhUL#UgLUNQ8DORVX03mQOCO9AQPI1n6FIvOn zjM%kEU!QLp_O(j%hcdc5cFerct7(+;9?sELC zgU!<97xgqoY}mfOH2V}&Ld*L-NYCUAusVh`6t*g>X}p@1`w;PUa$Jpe<(=#P@X0d_ zAj+NdncENsO!WvHUAaiGy0BZDduX>ZDT}=z)b!37=$?ZoZNZ7$O4~9zff!hGzg*{t zq;cA;%2{_R`)qC2HxR!~w*hwLx~?xL&`RL};!UI$JdQ@e%Ps7!E$ddOKM_`s3Mfg`}ivB!U16rx<~zOGE^eo>@<4 zrxmn01?=m*ukA83^)Hm7y47Wt-FYq0uUF8pyHOxz8UtJG?2*L1X?LbRE&M%uj%9Cg zUaJ~Z-Gw2m2&W_gfIFxpF^$d%;>7&Mw7O6bse1KVir@YQjoGNRoJXK*E|gV>XosZ6DY1#-hfkAoI-N$w+2plzxW%4S zq~mfS;#o;hCbhr5jQb8wo9&%#Hno+XR^UDppphwiCBzFG?LKEj($Q|={UidBBuQxj zM@m``|_Xz^K%1>)y4tySiz2S~Ybps!ZylDV5aPVpR0V1Zf~Yf)`B# zlyh!LnacMLrPiK4hE9cOR64rrF?*VJY7LL$b@l|nEH>+53{3W@sbq)3EpBOZ1_Hae zT4h*M0D#0al`eFdAHxP@g@6feZOr~5_@(c87a{md`!|(XH3t6Kd*+I?)VNPV&ef5s zTv98j`k@0;anxc8*79~X_=Y`}WZ{x4bu5{$hhm&pR)-tP(V0cgPprpjPwV@}LnRJp%y zO7t6=7-ge>UV+umAQhYH3ilML09v(RuUWG^k^kOw${{vsa#vrYE-n+qQ_erBC3)C%OVpp3vr;_4<{R@Ju2nd z{EH2gx#uIyp~m>^*h6%)@zxh?JCHWAvZ1vRc7=*{XoSOl^aD!un@Fx^=so^3=1F16 zt*vNN)mr4zr=(&L09uirn1FYZaim9AVEvAAmiUL`ABVY}dz2pz*wDV_m-Q+g1+G06 z{>nzC+MTG?sP#QWU;qlCq>@y5fzw_J7IMGiA2QqEoBEB;ns}8n(^1jKbcatw_G}vk zt6W_~QKgMi3@zGoP9%vYO!-W*eHfaXQ7g%CtA%`6sR7UQe}?}&mZa=IA=vQ<=ldvuJuCF ze;wo)<6ic;do`@k#2YyF%A0eSQD|FlUS)gCtJ}hfBtU>JyoSBS8dNz|{tY_%iFYt* z<|zr&xSb|MU}_uj;~(11pDDihsC-k)Y}%cVG3Bdi%sF?aO}!hnr%mr`Bu}hftGK3& zs9AcL08K{6p1t+IpN5WZ80hEhTGizCQ;40)Do)*-yWELY+i*#8xrZk6il|($vaTXM ztSykBl6%d0Re`1&iYnB@sZtN+Eh2I?5iy>r00}%X_*a(l>inj)s6`I zaYlhukyk0eXT=6y}jD{YP#Z~$kgIjNs$mSJvq`rAjcf;@vS)> zO6Ql~Rikxt%q=fU8lH>#R}RJP$*BufssaI~0hG##!qzKut!_)3W2Dz#p{D*twZ*A% z8t%cJs|an8er}Gwb$aiwTA^dAP1Pe=rFtApZNyay;~y*Ev<`1p!Rd~sUrm^uHRBdXl5y^vP)AL%`3tPjtXP`N&a0=U_R|?#VQ$^9)okc4Ut0CbLQph> zS|{YIW)&~$++16|t^Lv)ySN0J=}xYZC80$|Kzc(l(~<{Hj+=F7b8hx_R`>KPa_0yr z(x_FcFXEH;WHQ9lB>wDLAbN1~iVIJMpEjEWPEa&N;n%+Z7FTZEdW=`jzf< zpTm+WYF0(4s0}y_!Lc&0!+q7hqK`y6Ry>1k57!6VjO^4g-)=h9^_-ejQ&w)Ut-0+B zA{A25^C_yhIFpY*o>=DG!i5fh%<5R(+*YD4Dpjjcxv3-qYAO>8pfDl$v?JiZ7tooYSN=`_;8+p2qL9aT>6X#s)Q>xrZ9wvjPM`Id#X^w^PUR%Xcy->{%;Ss=sF8QdZWa=0=wM$uGHzR3@P{Lo2%-SW)@6 zG6n$|orxVQ)A@Cfu;td)=}w(GR4A=9T6Fy+00`-)3IHu9ZlYw9C7WG1Ul`NN&bw$D z%LJOC$f#Om(5oh`N=f5frl7FfVw);T%B_|KPgx8_?GtRj!_B4OwyN~0(=DLW2*YJ& zL#mkuK4`(1ff*A#Oj4cUdtD%xUz~t+IO?Wk$jJgkZ6XYH80=zVzH@m#ZmW4w?C}gT zW?ubd*U5!v+t1^=)O7SOsV%>C*{Y#xy{j`~O3rw>NHxURReGv5y&BXKhoouOtU?A3 zSx796ObtRAjOh$O$8vPseipA<-piL$zP(B$J6B0jUh9~xY8+E+%RU9G`NHGNoK7hoS+PfNjbA|NQpvA*nySdNIkwwj)qrNMqYZ_2-*NuEdkd5(Og(C| zR;DOkmq)KwwXNj}?2TK~R9b~zny76cpwd9qxbYsI+}z z`YE`mREcCIYHbcBG4l>G)0)tJewQGvV#O=B^LF$~8%(3n35zT-Y7RWRtQ8`*B>Tkq zOi$)!K7PFX8-Guk{3*$5vduf5Q)sVAiIqQxRCr~?h7vUQG60ZU`J4FX@!tsk*?ufG zHLfkH+5B0}ZY{0~u{B-!g=?xSQj@_Vs#RG84v@;yp|}LUlWcK2I(wX2_*$MjwwdJ3 zG;xZ`D>o|u%4UrQqgJhoORrRrdvu}K+%ikJN??F;(|L37?>o6$N{Td9b!24>jvh%? zP!udcf=*yWXOG^G*f+%%{{RzwJMi6l)=xt7m~x6`DHCwtx~0|Xdes!ub}n3$S3a2< zLKr+{4PU7qru56MU3tLb)N1u#p7mmB^8O9T`hEkunjETX1&r-#d}^)2oN0bV3m9DM zk7s?w4F}SV5t_FyT3^JMKN41Z+pC9Q)t;|PtO_S=1cE_fAjxSs7(^U<5Boely{j`C zDM@rXR2oXC5gLrCP+^Jbl5?p{@zv``)qhVq@6tRDmUUB)T;g3{$qrMWYlze1nml(- zp2C7IVd{o!5ImAGH!B?HT1C`)tL#^*P?;Yn zGGLMb)x-&9Uf!^%l@+Nrf*d}ysDtJJbgG!#OGc=c>D6M``jF@kSbm2(ZNxsIGbrHw zNA#aM#vcIcW-k+^hs|lXj;0TaS?1K&K^HD(Y}ZY6?N>!a4Gm~x-S*WaAQW3xvq~*k zxVNaLpaET?r9YV|02pf1CYY%1#M1$_8ier`WT{TQcc5F-cMPEFI$8v{4Vane3Smqc zfOr*kXM3T2MEY^zU1RAVP)DYHV6m9)57!=$;cmF_IWD}NM{Zibl~%&g$k`*M$!aG_ zZ%b)iZOUq0w^}jQxVL*nxwotmvr4)$%hZxo)J8c~Vj+b-kjhY&&<_(@Ez+wSlvXUp ztp+nGujg1gRhdX3Ur1(_hy{zGRdaawEBPF*Nlkq~&$D%wD_of7WNgBMvS zv0Yl!(OS&Fx~LhJ!ij*!qi9~bbgDWr8}=btKvE8+e$05P}ePoHz{3(R@nP(GEqGtD}?(8_45S7(T=f5z_P6hm#;Za&>VL2-MO}=)v1*_v{N-$Q<(-9KqezOT||&b3d3Bb7fNe& zT~_9!02-qOQ@Xh-(K46>ZxBe5+%~`blD=X80Je|nqyGS<5r6T&%P;=ZQvU$$b=Us@ zAiw$egZmSg{{ZoPzyAP~q5lBUxMaSj@Va1phpW2!g-2_G^$Q<|@9$%zhKXk3=U1Zc zCmAypf=47~Yh7NS^cAqnF;i0D0!s~`NyK;DYHMQETAdQu zsX?nHGGwqdFx!zQ0tc}m9l|~JHO`mSUXXGQ3CYzM-8|_vW^*n&+Qt=d45c?>DEsmq{16 z*B964DYtH_voPq@ucF<~R{6KL%UJyhir&^+am+b_K;Jd{+SadHiK#-}0b-@gieVO& z6H-~FDWy7e=_pV{s)+z68r@vc<#dv(QrA|kP>Lz)`H~pwY$(eZk(MSH!z&RwTlJyU zKcjAt_3P=wh~V-*sP*4>s`_0O={AV^mC{a*@LnsCM-ujr4#R=5Z*#7_yZ>P?qQ|cc(>i1YNc@{o$ zJu{nyU?(9wgQ$0A^y+f0TGm+>iL_3-M!7sI(zkiqwT8SVrDvC6{hR*J`KL3fN0=Ii zm)NykRBozM8nn<=6%DmEI;yQO%QZBxC5D0kW5+l6&*Qp0qO}`pR37f-sXwJ-MZH$9 zFQsX-DuTM^G=K^zk{s#LvwUBFEi=fu4XqwITFq;cs+l@gNngijuqy#wt5|D&#k~gF zDDc+LLLt5a~&}-HL0=TAfpb37MB%Jo+nR5Dx8Z8>7O$|k9QW~E~mvf-OGTw5&yuPb5y1Ccxlzlp{ zZw1)R>R%Ye=b6f=y)61!Rkj`aP?7(5a_ikXcc=wQ3hn4DuG?oS4!po0JAIKhAppo^}jggoWid# zRh>=M>SfW<*VV2zDO<0G=T=<^g)PtoHa<00x2%HVy-*5L2~AzF2vVq8Gtzs@3`{f- zeAVzZ^>2guoo+)`(%@6(+|KuyEpV{3Zyu+qUP*T}=oJB}y+%SE%&0IpT$*{8x{J=W zvpO+TQ;X(R#Z|1m4`^QV)utIw-O5eIMXdMf=!n@&+&Rq}Z;Ep;PPjtJC|ZtRZ%U^z zDOu{fvr{pz_)-H&05QlA2n2RsJz$i%y?Qp)Jw>N+MvAemTT3dZ&Jl`+W&)YhAU3Fx z+fh@bToa$x>8ByeXm8WGXG=$yCXw|n*B;G(9K8J5eVDmot-E1Y0;Nj1{{U-DQ*TzX zLfcbJAz*ddyuR9nTD2%tYKYIJv6^fLP$EQwlC3@jn3k#heerb;TYt(0-8(ZvvL%fQ zi)HH;xeo0@l)|V|?9~%q12F)GBZNP$wQvJcw$~8<04+TSyUDBT=V0n=a{7UBH_lnN zqs+AHF0Iz=F_!1;S7Y9RNt6^B91k`4!JI;k?Y7|oEmsPr^w$*6L6V|B2p_ryDbV1Wo&jIX(&qSccA^0?W61Xr-Ueb0N!8;M$GF92 zWe>9urZ)#-BgcG6nUX{W;``FMobGcr`wL_}sHGdmI@TbzE-oZtK_@ddL}M%T+PNv7q3 zkxq*{8jns`CN(iY8k(%2CQCb*`4X6N)d{Cszdhl%wrXDK<9N>2hKTNKFBMvr)YtKD zM(I-B{$>9FFDThUpRr1swj#NoR(f$qQ@#R@kY^f9VBl|!UF7GF%)VXDsE>-N3RW#j z<+)clp=DB~!rahWRcw7L<4&1DUWH99uF-0{6H`mMDY!_?jE0zE9K$*tN!8G*pvj1X`C=LGBlw!%;^nJp%k658SQTm9)Ov44(xs(D zsZi6B3(}}im8t;7vPKAMXED>_-7n^RFPY)#j{BF#M}ltg;{{{fufK~}m9MR}o=T}2 zG_D=h1var-D{IEp)tH@RxLW)Zc|q)G+lLjj(>lPARk}e|>I7+#7#IhsxS{-0@c#fg z=Qke^@=N+R7AsGiQs=z8nYk6{-E#Yi`t7Z4s~2SzsMAnuDimbtrqEQYh&T%{xrIKY z=j|?{_3Gr|?Qvw$QydDz^12)#d!=Bti9%ze=5%Pg+jiJhF7|aIrpt2?x+%rKwF}4)m*N) zAqgtbD;`UGZ_RIQ9fZBLuCY#%o}~cb00AU}89J&arW2?;An8dfhQCl2 zIZqShoiFFtA3>TNYm~b_EQ6(va&s&;p_R0tMidrl15yB)`*ug{F(gnVCoxA_^pwmY zAk>l|k(k){2f*9ftIFzfdX>4&K3PraCorb>X3W!73iW?@!ADVOs6Zt3fB-WkhM0;o z`F9Yd%4nIo?!_;+FSW7umlk$TDJ_IB**Jx<=n?z%}TqWQqJ+ITv_Q9sx4rxPHDQXM}ev`59ml>c&6gr*Qb+KmD+wzMrMy2 zv71pas{D&0O;$2>btSMPpG4l4o@=ZWF|6_`td_N*@XHHIl(~g!p_uxoR%DIK49v_Q zCk+vR7Q@cICbYceiAJ!d^tpSjYBgF5TU8XzD%4T8)KQfxqLPJGO9Kk4U7@G5mVB#2 zYK@L?x|ZqG?CqCeYh$BkMzJ{Nyx5ymO1hNF$3`p(4QUqyCP+o~b*ERRx+o|akbwb6 z1cMT?4grxqBTf*Vk!JoEtjzEyLFW-k_+6p|ogi0GDx%LT~}WjxIWQ$}T9Z z+}`3lR&A`1Hs4y?<1+cN4`M=mK3CSs(y(ii1Q}Hu zjN)*7Z9Sti95t_U=+(Uh7WTC1RjRbAu9u}om}2f}uSy|?WF&PgFa-4RCVf7mpKErG zV%4c(_xS$+P4Vw<%T2YnW(Myo+*&jg)SYUAMPBKpqPpanCH4nG<|@}h*hasU#7L2m zxH39m<^~%F90ALpnDbZV-k0KNThUv-ZFX&GwMtd(bc!~XPf>Ib?%}Ec(pgv>WU(g4 z7yCOGZAMJ*P}FijXxCjfy5An+v{zFXiY=(jlKx+7eX(&+BC@fXwz&LNPQNSHSqm$q8ON@S=2A>709-Z&Q3O6>T`S6r zVVEi}QI!W!@w>6f#80`vE1I0v`r5aqwdqx$t4tboA)8HLt7}D|Hm_X|s>BFoF_xV* zZ!S%niLiN3ZnqqDES^)eEk$y>nSpy)Ka5)2b>Zc%$P$v5`LUjDRfvk@P%RHMtN3GV z8A8>90Lau72{0Q1RJNbu$T15AZcp&#_c^oX)oAGEHmh3Fi9+bC>XPv%OCThLViqJ2 zbHQn%Mf8zj6oY}p3*>F;7KdKKlqjY9^}=z-Nmxp)VHKR145$y>ZQ#c zH7S~b3a}K=J8MXNT1eDb4nE)17Um#hMm;CVYS;V#k=<*>d~3miDzF=SitfbPAI_ARPBH3js0Mo&n^X;`^KOj#thv z9^;ff%bUcYt{#nlQk*ELj6*Fj;#ExRQe|}+;upP_mk?fUpusV9;T)(o}}* zH#{v%N++A%bIxN&m)+F#g&H;O=%SQI`I?dt=8@J*Hk~X<*eN4{XE5ZvzntE!!!njL z%ym~&5A1U;nl&tDQK4f}j|j5GF;%@8wUpZo)~_X0)L(4hYrBE$jZH|cR=9D73d9*` z(#%)6k=_ZJJ3|jv;+xu4rOP>OTUrlVrAqbMoarv#yfoRBR-04=>1B5Sh)0qNyuB6Z zwpT{+pOeXRY;tZ_xU75qLgKuFg0dIN@=U7fD`Tho0ESmvc>T+4%%z)AEOqe8nFNUH7FyNXn)D~jHCO^di(Oaaa?qz_XZH~;Ry@VG%8qO(3N-1WwIWrjBt<~$ zmQ$T7Fh|TV1a5e)bKXhGej>9!5LLdut5){z-#vIJB=e)Vm{)cM=p>E(1a1#wQTZ@vn2r2rhg5!HO=9HQ3QZMlK3_*<)ZY80NK6skHBX)CW%gr?3ix-Z36mW4HF~w~gZ8G(W#Q|QaU(ZnZruf<@CD~lC8c=4A%Xfi$J%yhu_->Q6y($%L zEn1Ztbm0}+nsk7f644+Q0C~wgX!rF^*t4s3OVg`Po8201Ht1Sfp+a)o7N@i?`U*t&wox{VUQ`rqs62wLtGD2=TfI``IT+nM(B!2@r5E$`Zw)n})DgQz+OB8n z6T6t`>R^}JwqGeaX7z!>q~b(*?S-3LTGe~TU3UVZW2ntVWIE}-7ZoMJ82O|y2ac6% zUX>^OFr%$0yCTX1p#*>iL>(eJbinbD+spnR{GOjR!Qpvary=vn^=oHW(&R2`bM&>2 z&U_VZeD{MFPK?uSKe@iEEy=!Crn>81d<>-dou%7b)~w_R4$DmH)Bq<+gEJ%EH<_5? z(Q{&tMOj*5WUr87I|Gy8whm%P8$X+voVlxcFGXyMH!JQSPR<=?9v;9?8{|`Ugp#)p zR*hxrMBJ{XrxvqX%T6{)otA)l*6UG&HC0zeL#};9{o+6n3pyAb!x&;?ie=@)*<7Kd zjtxRlfWozECtW#cgRDv@$tS*&O7ZdaU#rEk^Oh*)N5;8tJ;&xX@3lv=fYaSp?JJyO zd}_zpSjxD*L!6=y6n+IP1%BTN{3W^P)h*tu+KT+L&aDGflC_&b8g(5?t{9q30RTjd z20S!wcX9oZ%sHKERBh|r+>4t_g? z@S@`4?wo0@bd?&Hr;M4V6@i=*WE}9im1=y`l{I>VElRHR)hZF{s@0{cQObuU80sPn zD(Mll+b1WgEIv zYQBUmfx*Xlx!O@bcZXu`%@-$idB=PcU8uONx-~`Vmn}g)MC0SWT$C@U-l0G$QB7;4 z4NQYis6xi&O0g!MrUb&tUOjvMhJUplYgggKvTDd{S#wruZ(jbQg`-~Kp5nrot*(hH zLZPcnE9zQI5Hmt?hl9r$pER7rxB5I7~6DiW*!k$^##ADH$2A^!ke{{TzBRv+a)pmf6n zUjwUrT^vV$7pdywD&z0IO*TBSwUcpa)Q#>5rm(uVYv(gBOCI%bt8YjO;8$hLEArW(=%OkW*G{EoxfPYA;yTq=4zBWc6!14CVki0Z1kR z1n|FoZRT+fuk>G`-%=^MwsY=#lw50DV=kmv&bg(>=j^r1PcoG-jH?ve?6U%Lrnw~0 zbP>YO;8(QL>V-SC&?=AJIwqYeSM!(v>Ma_FQ7e%EYL^r=wI!rOrnNX%>e8gK z1TL=IVrB_!_^rn>FR?iMPO{QVM=Lb zY|14x7Sk@r20)(#i~>l=h>~J>lBSh~cLECu(<3Kw8n)hMB=?Sz)N#IR(k5|?erMC0 zDdj$8u--Gtg61sm!yL^D*z=ha$>j+}t}HjO*Crx?F!jD-KV_$*dDgQ*yVkm_{v!E3_2Sy;`+t z3W}nkW!0b^%8{tDjFXnz2Qk2l%xb8rl?4_y^(rDceyCDRv?%$+1S35#W-_<*lacio z{4l<r4Rlj{{Y6H{S%7btDpW4{{YdC z{!TLAt3OpAQGH>KAN2XddQY#%zNYqe`q!J){A*b-Ys=yFj?F$N#cZbfy}9-BxN+m# zCizVoR*yXja6bKY*yp#mU)nIVD7zO>SrkKRV4>AWQYA{%00I~;OqD!b?I@+YRlKWO znX_|DTb6J_)gg8?kMc~OEyQ|l)to(Q>JCA} z`S(x0*54cCv|OR~&0DGom)N^qi@K%Ob((;1a7t<1B^4UA4yJCY>L~h3wyl^lGUj2V z$>=;5D6%yxkHrdz&h;r3G7UPTsZY?3%)s~gVV!=Pb+76S^7JfyGVop<)84RkPb*KS zqs@660`_+f<8=7tIMn2Lw9rL2cJ!;tqZ+%Kq*gnY?}H6Vu~@8?kaK$hMvb#NyCqub z>Uv;`IbCccr5mUz4J;%8NGiIxN(ha6a%xkiWmF{UtnOF>0RR9n*uWc%jxyKiC0w&h z>f?*!*xXkoe4jM zkRm$9{{TOYYCo1)#jYHH8KeAw&KpIRFP-~0Yx@@43!e*S(~$*Kttzf1YQe74WUP66 zLzLI#Ysh(B&CUHP9Ja!U9W@%>+Nlw(S%6krNKlB<+K2=LJb9Yqo%4GOORK9ZYfG+u zb66#v$fT7kB}Gb%riw6sN}{kJwx66Kc#f1G>+kAKUrfCz<{8-ammIH=%Q%|zFH!Ot z(G|p$e<3xj(wkKn*4$-!?RrH4_R4& zBabXR6OQwJPN(AY0tbfp@)cTqH8E&RPfOo;$t;LLF2z}30-4&aIfcylq(l$K?)ok( zT-55RF8Yp7s6%AP$dV@q0tEE&>8ls~Bl50V>h(h#OKN&mw5>Wql`6bm2)-b308}eJv|!r7}UmC4bA~jddb)U7z;rCss7Dqd_QulZEZro zE#~y9+*{k(QlVi^*3(_P8aJHE;hMc_roUNrIsiIJA%v9r@zYLUqrmgIuNK=Vseg&# z==B!|Dy3|0Uz<79TKfFL%^D1K_Gjo?IJbX2#sx@B2iGL0kzDy{N>%B#dPvm3{oAw< z0RRAXY6=IO5hG9-E6siu{iIpjd_l~973OW|)?4b<);Beny`9w~(zaBpR8>;+6eJ1%C-Fy6|cNxSd2VK=P^+}54{6lWVAh)%dLb?%OL2Z(Bs*H)))_pnB zxWv;01nXbo8!ATPlG+q+TT8VtAi;^8VBidARzlBDV)&2tZOgtZ_;Q^qTGVOVbDmpv z_IEl-{?F%j%EpbYT8ixSY6^;l*)XY8s(`_V7jB{OY_4~OhskrV>U9^6@=Rv#<*$?T z91fZA%v;y_4wzI`Y!znJy=}B>Ev#*OLdg>ICQ6}zFabPa9jlO!i(BH)z}H|i%>BV5#jY*h zjYn4!!M=bG%1kVS0DPeQvVXh#2k^H)PIJjDIfVudO6uC3SvhJ&kTFJrdeoSxgi5rG z8RPqi;|*?6m3f0%)~L4<)pA54#b(sIr&)HaLY4yaO?1l}rCux*jOxB?Pm9SjA6u|n ziW4F#D;bypPA6z2;C#TzGuw~3)q`L1j(Uqyt5!UtnS@FJdevF%tF<(@nI5~<(@g^s zo~tN%Li|2;D)F)5db@7e*1t=MA~fsP>EBZB?PuJCbq#C2?mx!$B6KlaoWW_L-RlSM zF-@{RRdbT+jP}cB)jdY4O$X4bWkJ)@OB}2d8xH3(6@M&Ew>GKqJ+~?rsmiNeUke7g z=C(eK)tfc5fcJGNAbNr7;;*J4f)wn9X`OB!CxeU0`Iif+ouh%x=%U4(yZe^B+V-0^ zYw~s1{irUsDuqG3Tf zA~D6Q<4@W3K6L*8313{@tqZS*>s2=u?WmwPK%k~*SEfxiG>NCKm<;NO#6Xu`I$1XQ zTbrrV`Qc%wYi~l*wr^|bsn#wiRkpgfGAKsTC>LI~ey8wQcKsDoyBDqMR|J((Z^-ZY zXEwZ}c>KiGWkq_FRh-li126>SDL8B?sE}*kj#&t3-uAo6HO14glX;K=L zZNQSEwe2agY(B*=z$&YCQ>9gaWMnEwOi3`HsOnHdHk<<)F~>K*wB~kP^2*}poh#bh zTi;r}gS+~ou)9l|RTT`aN|e^43YBR{(&1YY5wKI|Pd#$GSlPrG@5a%!)x&?oKrt*+iX>BRj%C)H>TPxsVK2sHfd&7CKxb} zney8^cD{zMQ5D(NwI+nGL5hrxRf>`sRNimY4o${#bTVSM0Ctdi`& ztN6=To5>mLD>j{MG9t_fO9J)Gc@?F_t0Iq1x>G)hRRMu;O32lj2Vu5lTjdo?3m#cV zm^U=0Db+ziEDbTF2U-l5hAR~oS8Ll!wTpEweT^%hxFKscb~gbRp1Iuzrfw;g^>kLli5iq;Pk4|H;Cbo< z9xoiOtpingrI^u1wY^(*EX9wh@(^A?W?2EWg`V*XsfPMZa9F+q_AMJKIkmN>?oaQ?udikGJc{S(h!oHvO2JlJwsqWTr5m)6WS+@Io)cv^qRd& zh_z`{L~lso5G6qyz*u5iP%|Z zuuO>vXWXF`Vbz*b1tc%8u2#C2MH<$bVi&wdqD*gz{!!H&3-DLNS3I)L_m|eYt8;Bo zHR1}34MthsrL&nD8k~jDq!|G9My@SLK>m_In2o|2}X6~n62NPbdbXw75K&n8si3vqscYxDQcT` zJ+&;UMWt`E8#Ks;fV+ASwA;WceIjjtm^8b526;J zYc{!PC=As#c>)@J?A4GZ+#fQ9q-G=Fw)bsLq1IWL>|}@o=pY^QC%3m5=aekDjjyJQ zMx9lormHjR(WQeu2~jU<^C*8QAx@~35^~*qJ(Vdp^5$nkY2yh(;~49hU-#E;P*+ou z#aprKv9p-wp!84K_ir@8r{j8j&r1SGApTs4@s6NR&<*#Q;(=b!P-Pa?fTXn=q?7Bt zP|2aGRA&jL>B$rw82Vtss5^*Vvh?n0BQF4*o|_kTR!UVXr*o6Vf3TicX0cHe+xt{R z)S<;tC0!o4w$`npKOEchDm5EBfDDs^pUP>VO=XgjKd7{p z9%Y+RXFXo|(@|R9qpadX0YB{In34gAoJiY@Tiu(>dX%}X1-zzNVwGAUlA5LfD$+0p zvdgDG#lHim@=jwz9>rQ$)uqP&0Faw{w7glDwEHG;X`v!Mi%V-$ z6@s4JQTbn4scocnC4r&5^nJcn5Hb(2Qal8iq!qwj2C;!+EKii^$BIpWLlRM3UcGw4 zThdh!Ac&AhNg80cxY{J@z~dHw4$n7*UO&iCMvKYUQ%E{SG;D1232Q3 zU(9=x`bY#|AIwVyU1TcXUJhQ}9KMxmOrN}}F>C{GymT)Y5Q^WDcCto>GAQ@`!cszF%giPOYfOto{FpkZ+xtZ zww85V1#qhBsvK!>UFLtDm>=v~+}_g8$t|ng*Q;kng=%B_Go4yJNV~YG zX@a7)N);ZLsZa`nbh_d=&pt`klxlJU;uprnRPtHn_XeaMgs_xT4P&*{$#PN&?oH?(-mFfCD|R#Cl`;hb3TPto0R@df8F7e~?c2&bx9OqF(75e)_i$$nPT@_TwM zO}%BN?)1{ERbaIov8g{Pr$}LShM6QOSyp&D77gjfW_~U;yd0ZTo1t3QD0VgN`!wmb zv7QO*5QjarpXytP%`ti}t}@VXvYZ7Hdx3DOR-cPb+UiI#~m1E|5$VXK$C zuaMl=Yu;-@t=ro(*IPkTNU2tTcCQKh%4&B)RFT{>YZJXd(BnN#<=khE@vVDY(?Zvy z7k7$hx}6czycEwdd#rvxr5B{C80u*o?KxRP{{Rhh4tNNs$yUb3`)Zsdb54sCRixF0I+d%Vq?T%=<$yVplEk)Z=4nR^;9Gp9q^acnMAnOZhdh+O z`^=QsVM;7our@EP^-%d5RaOgA()zZZ&(sNC&;6f#5nJJoPSq|EKeQ{Y5nYODsER~| z4LJ%}nTZ`z-zxFve~$U>ert73-pZY1Hce2}dMw2%RB)j|fW`qu>eHw}!2yo}u7mN+ zzO8k1HXT<-SBCH#c5WL^Y1Lk*Aze?or3JWXv zHejT=74JTK+V5|u(xkt;1Q2y#gMp?9V+2UsRtaF;dY3LKPGI_kmkeV*#x;FMz6di4 z`~3uKWAmOWtC|?h<6BvFE#+&`p}lfID}}UVuVo$TpcLdZzUiRd3e#kky2&QN%|omL zmbd24qk4+t)C^3im}n6s#F8UE;s%zfsi`U0fgQjwbnXYXS|Cj6GBoWc8^qqgwLF5O z*`>uvH8QpE4<|i03ZEy?oL*Urt;nV-pzP!AMLM)AXwj>5>%RLCM!D_s-EDnAO$uvG zfvCmj)oImhPxp0c(~<|NOv|rcRzL(Y@l@v4)$b|2B_V24cdEL;1iEzqqy;kFvL{yN z0b+j@Zr--%Kguh)*f!KT)AxzFLdmYaOIMGpa>4{vU1nHt%IhXHnK{^0DB9Rx{7*~a`-)eUuX)ER zOU_J%irOtpdg?CeRP@;it3~S4eJnJP(a=B+PqD}mXN6_#UZF>j-Ok(4rL%n~fVArw z783k>;Px2%LrRIfj9G;KmYd5fO@(D}bF_{#75F6O-sS5*U1wy4a~DxfR@ z!vM7bA-5ZeIOE!12Y$pkaI&|w=C-BzspZ!9_LmEw)2dFTuT7Cj>!+zT43t+!GKN-{ z#k4mn=yGeAie8m{`KQSODXm#|tm0GGub*!6ZCnb9R;5F;Rz|D9JJwU_-XXSMb7!&g z--zx0BjvoZ`lUyzZ0bF`qPse+&02EnReD*2X(33-IXYm5;a|1y+5Mlx3(YJj)I778 zUQt!4UZG0HjY@L%>q?;37L`I;B$#FdO8kol)tw8qxke_{_TQJzT;)Qnr7L48(`%3~ zms4rej>e;v$8+-wM9Crii2jQI0HQZEw5jm#HRaa&Jz7_*UZpy2=Soyp0t3ka{!mN_ zgQ`Iv&L{r>WV7MUVQcYMz}(KgMtfU}8`~N!RVfov=AWTe*u5jy0-O<`HpL(U84Kml zLXvxetkwSj0?Ha_J#*VmZCiAll1<0x@z*O}j7t`%bJO=#82}2qRjK@ssD3Q4qjkz| zD-=;yf~!(yT&Tbx5iDeh%xNHz84c$z^iSuU$Cm!mIZwmJ{{1;`JU4>s-qNkQnw1il zsZH9A*)#w|);bj{R3@590bZ6k@?O9452fz9^~ZtECF56P65;m#23oI?2a0^|WPXd+^aC>{9ZC3K#)|;!+#rTv}%0K@#E=F(_UN(*V}Rj8h_wXo4F18O?7hLsM2V^Ycn%g$dzfog2(Ze-%! ztCw>_VOd;Rt*jA&Pg2bz%43yI!B?a>GTREUNVe6@d7nYM)DlasZIzswdJ2+8m6!m| zK{@Y@K!12|s>-!dhswx6QbR-s+yexI8QAF(O3cR=7xepQi}AMTXV^ThIy=;^Sir7# z6^+T$TE$iCaurvLNT$8J+3@5yPJ)Hkfocn-WDubVn$_vjsZ(64NCQxkbl_?$u+lJO zFXd2LcqMs*5)i2*fPj&JH4}lDfvE2U?>sa7ujzxUY5HZ)`ikj)9Lnb%3%ANN5RO5S zqU)Teg{KcxvNX}j>sM8Me5;gpE8NYxyK>uEmjy&*mZZ!|Q&x==>Ctx6rOOpoI$c%J zkR+9QY#kh{Q>jgZ232ir4O-~eMuW1gXF*YEG^4u4B2_>nOKCb|w2%rAx$o5vQNE

k~G1!Z}Iz)S%aO(Y_uJdv|bi#`n!=I z)vgw;0ZdcrQLH!-QFL`clm$YFQTxduWuib`ZS1Jp*7fyQN2VUDC~zsbB1$rVLDE#1 z@?a7GL!cdQ>0kJn{Uze`GfNlvE@P8NM;4YRH^9($HC9#Hm4b43?>p1)Lfh%lCkt^k z>{uK66-C*XE;WXkBB*m3k4BBB;hmcUO+clp(uM^LqaYiVB4A0<0N`tTj$L^7Oz0^w z)-*JukOXR3lFR|rWk?b-(ItKQrs}WNm)2iUIPVea2N*67Mcitx-q#da;%{`jlXWkM z6?IzV+*grZ$g{o06{%h6ZQeLsh-lO$HLjx5hgoSAEo$7Q8&PO#Js`rBNA)$+O-@>v z)1reeSFh80lK{9_3AGATZ4}m_TTY_#07)oxl{FtuqQj~PFjU|eW58qn=0B=`{FBmO z{ttLx{=oSM_9fT<01SUY{{Yc=-|>n60D3>;{{ZlONATbA2jqDF09Bn(`k>@^XtBw9 z)3=F0Ykl&qzXOC(yschQjmGZSs1)qp7@%f^O;;!qX!w64<`*A@VjTL6#+Ri8)HK`CmYEdt8lX`+mq%KFSB<%) zqG_Q?sngK)+}##1X;Y?()aopEXJQBhk)X(F1C1v7B>I2jT^-;*@f%MYLZ4MRXCBD6 z%{eb2?7SRXR^4je_C5|}Qcb6(<(55zwJKvao|7AHT~#l~tNb7ErOn$b*i#;@W^dlr zQk2$+P+YVmhzt%)mjD8}El%dV?tr@d6=$h`Pfc1hxow`2b8%c#Ivh1kN|fmxB(oln zBnDu*o}ORSEZnTSmTlG0us5{{ZFcSJAU& zyjH6-C3W_m;ae-7d3j#)rJa|i#_8FE3XN8)Q7c4hMOQ7Rq6N~xLlUZ?bnod$ z{6Bc7C&ZlXD)?F9UYzJw(X;N;Y1U=rC6q{)Q^W2qisf{fY@s$D_&>r{aq^A(c=$fU z;17j4cQ>^|`1_mXwyjIE3e<~-Qhszs65t5{=`6FWi;wIZ*ilMp(YrBrZk55w^ejH71hqF^wO(&E@w<~`DY_L9Dj)K;^6O{ zswfPqOw8mCsTZjwHJs_xr=NPcTdoA!-QKD8a6U)O>QSd}TB${9img+?Jy^t&H4s4~ z8$p0u^cJ`5{{WZ%)V>hp*E(%2x%HK;J6k?yewQ_Emvo-ihNO3@!&N$uMVT8` zrxePvIXrh}akMzgx_?eNB3`FxZgK9BVQBK~&Q)JGi=xVL`i<4S@`M%I&RE*f!>UwC zEf%D~i8F%he505x-4uQ3E|rphFf)XbK@rs2N}Z%M9zQ?7ZC|r1zlZO6hc38xrOqm= zRplJVnp10PT)ve{jVAWa*5Bnd0+C3&RW1l+kc`V}oUFc|^1eGqlFd9GEn!j3c#HLG zOz7h1=)X04{Wj9KS?qIY=RZx7xNVmYA#mw6AC%SY7Og}9TjPetUw@%Q>!^3 z!7xg}I5Jl6`yl3D5#0O~w=Ho}_S(0pYkyLuQ%}{aV^QrBnNgzPJr=n1ssp8E$Tto1 zjt6|DbHI9$%3QnrV%A?Ixu?rGCLc86YS6{Tw*3tz@)N^?S;e$A)rDzU#LrR2TE6Cs z?F;BMz9;2ZwX4@q%v~viNs!_((V1;Tcx)J%F&td^uP@}+mUih{ol6$0N|)pLrM>5& zRjN@+OG1_1R$VI2i4G`vXT$&+l)K-`29Z()$&IMY2#(x8j>76)9X(q!kz{)GAbIKb8Q1X9U0>FU%$z z(fhiz_zO@iFJwcs&Y?Ifl_LtNa`eGX(mL(K_>!tP(8>o z$E~d)os^KE(`7RPRN;s-jY@C`fF$D@oz98!YO1xuak(w~)HApAv=PX%ZFIiv3p-eP zUuz_@>Sj_QARA+Px!1D67v$%|NY(=OurGEi>e)~Z;XzKKNC#!f00Kew829b@wfktd z<~+CJKa21A4T@g#J3Dl`<-u>2ss1L~pn8<;>?*@6)1ywZfCDYXcGR*4hgrJ8$96F2 zbk}Tl^j*jGbNLiy&{g(mQNpv8udAEU63f!$bDwTyrG|GM=g*wko)JQK$ z*_GX4b;$g^6)i3)$+(sVZztxlsHAdEHT)X9)I(bW$O*!zci6vROGWbQ$B%)JfmJ~5+Dl-IQ8cGcSUg)u_S>4O!RLaAaP43pW0 zFwGOk7xksI)23xo zyHwPW{RxqOQ9J{kleu3Z@y=+~m+ zvO8VUDlSDt3Ln&TF3xpmEM{;D-}H^l4;3Y8Gaj4Onl3?BU`VEkdX%NMy=Z;M)=R88 ziEP5oJxPL@DUq%DknX#pC>FBmq53$r$&dVQBIw3 zV380ugQv3*B#ey7m}4kkSliljJKC0Hy|o&btz)ICR!T9d*n}a8YNU@%H5bznstMrH zg4|7>Lt8%(VZ3XDMZ8)%*M{h~dzITkCud=?Tko^0>9xdSy-I^2bo~;e7Q|1*9N(GI zLQqr&qYyAj5Icy71o;cFFgA3YmzMLIG-~rcX-`GZJaxONY(+9`>|BPlv=spLJt}BK z(W%u)!kti05_tJeiUX_Zut012Kx<=zR1e*?}vY2c!tYDvtCPK{yOwCC(kx#*mU&;=kLXUqEOm>no z1neR>ilR`XSk{e4Axn}pgSj12BYnv1NQvXLj#`SPW*b_Ro2*y#X)tamBvuP4cmS9x z-JpnYuR$&cs7b4GWI_P83Zi`b_mcp`k>YdP5jjhA)vn2>PIPLLRt7|X7=|!B=RM$p zI1+DYU((aOM%LZ6WaP;g8y9LasD5gaU{td{%RewSDGyh9Y#9DbaYzDKwljzu7#ZFV z)M^r^$UH0NF82PFDiqL#XsA?fS!ISvA*Flj1d{|xjw-hF?pFT*mYl$af*1IQF>R`A zV1n)4h#u}A)ZuSYtKPEkHq6hL!e>uRo%0{a`@j(dfhTNrE?So@Eu;Xd!DIr*aRhaO zc9YsCJ8@FlL(b8~V(?{!99Up1Yo-4Hp-og=1`AS`u1L;{+9Ihj?z4wTv*QYNj+#d* zW;#rE+uZ%ua6i1N(JQHDs3bO`Ff{|bFwO>WNr{fF_)@v@c@>Mv&CWX{%>s&4)|!rc z#QemYP+&?dQ&R>`&WhdaZnIRmi6zH}!;ZW`8ANCMVh9H zbkM0*l~!ePOGL?NA9*LJj*+MzK|etmTNHTz09$&Ald+~+PO*OzhG=5;t4UZ-Hjc)N zn#HZ&>Zz@*T8T9(6V9))f*l14DB$$OEU;6hn?SZ3+uJz{I!J@ zM--azTc)Y25ln!r*dUEqjLxVUHfL4Y{BRk6I)&Pn)q1eaJqlPSU0HPIYv`k)vPV3T-9oB65m)f zO-ZXz0`4AYq<{{U1+__7VZv3_jU`ps$lBI<@RuTNr;hcbhgYRSrOWWS3+d-IeL|AY zrUv}V_3P)$E3fd-x!9}^OyyO*TY4UuSPB)UNF_#e$I55CiN^c48t0F5XMN6W(w$4y zd%Bfsatp0yrM(*LDktFGiEV67vHQi?5Ki1DTx*V#xelK9BE#aFw6WI8%6A^Rrz=kF zyG3kormMcrJcQJmuF<4kwb*B4S$L2Iitgh@ka7yUC>7CY>rOpiwn;49*E4sF0xc1%mv8nsPpI%gxQk-E+8B3eW0xUOH5{>J0JM@I#^(d_h=U)RM$r?)di3d7 z+bBRuXI45%)=td^7%r&*Za!HfRPgF_o1;A|;5qqQeCqEWgq+Cx@oPuwx`*+ zoAoRhT&+@#1)A2YskAD1R!~VKlVYe7uZMZXYL9S@V$D|Fh*+3rjUbunB0(B~Bm*5% z4+nfx%DJ-go?TOyUX@$=16_JasMv#{y5&P_WjzS0K{FaaDzY5Mo%EB7Y~q!Uc2t)c z$mDP17#P3D9aP%|pmXweO?DVr1zf)5tXw%&=Ctehm4S4<$1~@b)ve7Ut87%Vy2&MA zVSZv*oSB6u0stiO&8+zsHMuu$ZEs%gs#o+WUe$7GI7f%MG=b8LOlH~ z&Nu8HC7g$b{{SK6)*Ht1xb;flwm#zmS6XJdFiK5SlG|^wBto!LP7Biz1M2R3@iSQR zN_CA*R~0BN6Nn(`6#9THN!0*MNq{4D=i`;wRJ-Oc=`AXD?rSs3>Cz1D6wI*#1_3(_ z_|l$`^s9yRlj-jZjeXjc%I+mvyrS)@=9>@X?5`)cZv7kUilA4jUZZ-8sypq`L15tp zu;9LV6PbKN%`JW?zoO{nZb-PGjnz)xl||HwAAKr1K_*KAU`YnSb?~jH#a|3^-guXk zQr6jPSywdMLArWX4r!q*;4A9G7Seyx|8ItVG8-r}W7w-+^?gR?b40l+iV7r4MNkvRi1rpd`| zDe|seZD(jH)Td1pHBe5oB$27H)Fc8(j1e0Z)Lw+~JswAs#4+`$Uc%Ksc#|7V~NxRd}s1BfU270sW?qoAk(vej}!G{?s7Rhs7 zD?>v}YBJ+PDe1MGt4CosDnkuQd7}$@)~rggYW|5N9YR1k%z#D?^Abn^o(r`Pbx!9% z0jVIUi=6dA{uzPVGwq#QJ%_nVS0;_EdsVAs-nEld!%a-Lo0L4twTvUPa@I|<`x*$p zT6FSPcrN1K(AE}Y=~sXXA!y_$*SM`lg6pqE9^+jyJx2UGhh;$7B-0>NU_)qJsStRs z^J*0hwR(!PP^o2ReJZik%s_(5tEiBz@{GV~OWLbv6FEmBM>NXf_cFLD=X%8qxuKt8 za|`&*s)UU=d^*Ba{1U6ZB7erV-K(sNYFVJ2)J43qqswYr+}EL2?ulweRQirEheJQr?i%?pAFYp^*VxkyTg{)WFjWLh;u5NBTm1U17;q-<7bb zUzT19G?Jo$R-iSII2AMp0+RwM{_{zIDuNV_D7srYE9REClQuLLZxa_wBMNCE)`}_F zyL7Joj`oVxTGgC@tlIVTomfrGHC3%>q3Nw;?7t5DFK5avEh*Srr6g+9?k^2eR8`SW z(=}FDfc+(uF)Et6LV|d#{?a}#=e+xtU30E;X!q5xi7pjYD;}LiXww}f(jj0d-V7E} zBl@q?Kl0xN%#>7hq%17!S)n(ku1)w@Qf6c3-GOUeF<6+dU2|J2GtJCXLOq_={{Tgm z^!c|st!-IN3V=~$MyBgmK=l@?vTD*b2xT!bgVj8KAO8UMMEPx{*TNqSa|*WXZ*4D@ zwF1JeI`*{d>!9@OQ>mhvVwLd9tO4r5RDKa%HJyfikKMMCU0I4d6<&31)gu@fWjmlJ z6>I@WfO#e2vXz?!qudYNT*8uE+1=1#hqJ6;1!mMKmQoCWA%MUnoJ$OG`CI!v<}Jy- zEckDm(vM!#;vQL1O-6L0^qS=ove#N)yfms&YMPy06oSgjqf}$J_?=*J+I&ZXZt}gJ zYg1a`?F%Q7XXWbEyqrWFvSX8RR4TqwqcK8JXV59X>Y7#-7wpIvnJ+DA{wGRQKC!4( zsgMXKSf>Gr9a@-esDKM#DLj8hKd|n9%z2+Dxu9*V-M!YMp8Y!X4~25OTwjfI)8#O19;I+C5QwLl%BhaW8KKKpY8h#S znzA&JaS3%@C;%{ZR>7y{wNy~Cr&rQnR;%hq3~DK)QDI#AKr76cWzr;o#DZJD*wv+Z zUab(sJsnc4x;3zJI!<*31PM7NNFzF9dVTd3(+*4Y@l)wnuN*=x*`qp1!|D|}fnSq! z>hXc>aqTKEQk|etMq4<#)PY}Zb3RjHkw5bgTTs2UuWZ)0DBh_q%UYk!O}7P(0Z>Vi zJ7XLa)^#nbC1j}zx&fr>DkIVo7!pVT_&QHio|)k3^pDkk0$)iTUf_L0<@%f`c;8Si zZftKOm@gWaAmLk|{cfXnc+2|^#&9($smbWYDQ;_lTLbO_3x6c#wJB4jb4I0OOw=i) zgao*OP9h6TfJhT6>kz@!bh$e2sMgRLqN^ZgPzOQQt{f5-La<-Bkd;SBqz9+-DrzX(p4!0?@1bb-Y8O7(v zSIilGS0e5WU2O4J806M04eHk5C~NvGSz&y#a&u_3kdz_AifvUvXspVx;bal1dQP1w z(#J?ptk}6rdN*}cQd*Ush^(xlL#dRgs97aR5KQSUxNMQaQ*Y{@t9%!#eNy^RqGlsex{`n2We`*{BVC*+2t(&RgQe?HQ(xi13QTwBq+>~2F_lGV7`&0ce6$=a{A zZg4Cv>ZCcUwIZE2rV$8iyjLWYYJq@3_;;o!TRx;TSl_`=sX~>?vWj%+2CYRkL(@=Y z6L_qqNz_WWay0edKkhI3vi|_*m*`*r07yQifB4t)Z~dd@U;HwE_uu~j^6>)Di#Y)u~z}{Fj7PV(}~PSPn!AHFuhNT`O|Wm zRThN{s=U69(y1v_?W&tJ0W1!at88Zjh4<>`t9Llp9j(K#ZPdTYxqU$585J7wCmxoZ z4O;GAOF)Lza5k6Q*aNR+1~QXkqsc{A{igoX`Bgqu%{f;tw53w&81c}LIH3FAQtF$-vK|uJhztoL-7YR7k8JnMa$ZyQTvN_Pf?RzOECv98FZH<6df|3 zNgWZR#rP?0^2{67b@1IoD@}GPOwzR5x#tXa)q81MlQu|8!HW?FSgzCuiz|66Dpat591z`c?UWri3F^ zr$N=w0(C2eS0JpMMEK)vKAappd!;<{TO&V0nQ~o>zB|XNxoNYXR-HnnPg_=@sZ(J} ztrl#xS8XsWD5*MBaA`R%U;dd_?7Z{gUPQGi+<#}-Rk^-20SdM5orS$Bs@j6qS}H0P zoj^XKPNmYb_6P8ddmBo%DpjPkD&AFkl?6qh#@5c$lq`^oGO6iz7ywdgP zt6S^$R%x;<%{tuES+#B1BYE7-iwTO&S^~koV4L>`HW(|AnXb=BfshZAEh;&+q+A4G zq%;{AFtIq>+CbDn#4qWW?3`0PhnLb6{W8L!dULC%)M5w<42>mD2eu4xd+FCb*HSKV zM&AbUJuN%SyT`6_OW1kab}wg5WVZhR+d5T{#hjuCkW;TO(?T0b;Sn;?w){8FYueXZ zMyVFg(BaQOk@E~rq1$=Hgo$R~wqMzgDEQOiTiW#O6}k5=xBN#hy|=WZbAP6X?-3imSVH~GgV#%n8E zQ}2XXjx|A5z-`LEj-g!cmVzMKkcQtct>w+d5^6PI6rcUyr>7ttO(cmv;$n2MU(U`! z&MNX>N0;*&9KyrZ=2R_e)#er=(tuixRZ`xiA{AEVt0+{b6DmLivvFuP0_RDd-of()?An8blO5NC>S>|f&h?oaV`8VO-( zWpT;t-EzyOKZt%|PK6uwr*Uv@80$Gle1oVKU4l_F4fVrb5QpE~@#a@@ywwo5~ zv_9YNBBe#ww$W22)QnYwTV^bf-7qy2%yJ8wDM{Tz-6)TU(edvL6OnXmZS*3Jv-hmmqCY9Y>r zMYQMRP_cNbfDx$d*SzZ5jjczsq<<511rY=dQU;*MumHr3OnzR(tP4`hk=DJsTArfO zs+4Y)nLeyrD2jE`N};4`0PavRX8;$~(qC^q$>imRd}SrcRmFP4#kk)c(a~#x&%ep| zgE4Gx*46jBJ~;G!K$zl{@ipaMX;YY3)#gg$NJS` zOKDNUGso)R+~b`80H$uEK|HHJ`QA;3Qpwa_ingou#0-hJvhHfqILYfmPn zM%yTbm;mUQ;o%87cwvmu3-H5{>K-#V5|V{k-6 zcV1Ig_u?Cyu2j!Pm6tH&S5~Qh$~4ifSkiiED9>-sd0o9imBm_1 zpim}}pw{&s8mrW3*QHf*bL!FDqlDV6h0M3tzsoFhj!SD}0^CKL7sBWL=e?zGMYW5; z4LeksT8I!V5UB})B^;GwnYAanb4dvO+a{n3YtBz>5&$`33Ec77xh zX+xLUEtS9pTqQH>B|RM>nRGf-cM~iL0LHAj=oOI6=XZF9r9P6k*f$ zOH8R236<=WsWJjFBm1^{K#6>$x@mVZaM7o<2|2(smhY1>r{!$jl}=de(ihQHmLsXb z6xtRsAWR7mz{mvgqH$r_dq`euH2&GSS4P}cm&{YXc2*@TrpKavLccoV^N0DZvEf{m z1V1(O!61@nxDqz@p5wGZ?c7#EmblDWGTxp^2i;fhyrjhB((aGzbbu;Jbs?i zU@g$B^gsChbs@tjN(kx<0~QcybK1T?bUZc!_xW|1d`W6$ z#Z=)L7Uin3cO{A_vi(alE<)dYnKqQ-@%GF}d~i9?G(|B@)sA&JkEnoH^o-;-NG2mL z=vq?sCGQofel;sys(NJJJt;KSp)AuzK*B&%fU6+!4^yfYxRxb8FPo5YQ8cI~Z&S$o zOGRLoB-U7ZtZxUjbg49`{shz_thwpZ$L5iYog@%D>QG0VA@AmM z1++nRE!?!5K4oS!0s-}mgked4gupPNWG7jZ1ai6VhL30;o2mtv+xmmpjQ+Ej?Y9c~ zh2uzirb^)fm;l7U2RSB56FX|+*_(G#){BLfB)uS#81|@<40$k7J`yaJ3nD#iQw6L5 zf_0Gv50}k?onAq}pB==@jYEw>qd8OGrvf&pyS8?N9*P2EdoE(6bTwzpTpuDv^jLdrS|@nd}4}>@%s+ zC%BUbFQ&hyeQr<5G`fq+^wb#t9EJp)KyEw%-!O6jox(0{b!p3KQ%j13 zHA>XMR95M?cF-91%Ltf>8JRP!`dsojb^~r@UE_S8k8##yJCWWzyIJvxpRbk{xc!V( zS+QM(pMP4*>#>ndNRy!gsc*8N6}Hpi>W-{yzon$AstClG)Kr=Md;V4iQchKQ$`nzh zLfYn+GQOvAUd$SbY1B5gKX{DZnF8Ud9Y}=gB!YJZ*S%!y}QUhJ2rg>a7hc6gb|0gYKzP)z^yZu1>98@?(VfW`#PgUE4@tS2A?(EI`c2 z@r=om7TsUt`cKO(e+^|(s9aGiNAVz2{2kS|?v~#2l!HyKY61ar5bBjy0J5sPJlm1) za%rUgQ(KT&vw^K+iREc|5s=R6O&P1b!SYsk9nu4!`<__oYV6&vQ zbe57Kbq@`aTS0BW4J4d*9t;p0j!*G7HMi#W)w!k32eiAXbw=9xY^bQO$}gEcHoLYk z?OxsjfEc|n&xvR%yWQs6THQXUi2G_`t-ee%mZpsiLz2n<#j~Yh8%; zM3~kXKRBO^?&_NKsZghoAcfybgNXojNk7622oN=Jx6r9d?ANI5ZK%^qbn8_8%T=z* zCBa~)_k*qhnKOV$XzqyNk58-WUb(w+mD}crHmk0!1!e8Jr8Dm3)#Ag_4G+0hF=(z@ z1>U`@6^%_}Wy>#6fdfjfkmhQWz;GV z9Al&rl1CXx6LTYuwwm^r29;fBBh%7h*j}dHlb705pxxrNRIUYC(BcqMlUS?xhvVnI zxhF5={I8MO`eK-?N`Vg0CEV&dh>R5@d18BlU^kubt8)uF_k6mbPv2aINf7k4Bxu$7 z_um~|gvSIgrp}0uD%DS0b*cvajC(3iOZZ5G6a3ag(tpo#XKQ+iRMS<6ViL5AU8$Bf zQBj}sEs){=02h2<*Bt2EARmLYQq}4gf|bQmiA9;AlwrjCz zR=D*IRbNoED&;^dvPO2~-dX;ahQ=*UQAQe>D9DQOw7z&rL`nUJLFFG5-K&XF1LmBO05ba?^Q;d!-c65Ia zYpnokjGmc5y}P!u05ILOgY=RW?8(~I*y)@%F$u)~059jWVj%LA^3m9#a)#v^<9da0 z%=EKmcL?=v*0;sB7nD}t8Q0137Y~GQ6P9y|{!^QKeixCT~#cB#=~S zQ>+t)Bddj1*3~Pd(kE5ARB;LF7KQ3ko%%`D1u09bIiEy1R%Qnk;FRy<*wE#d$!*fZ(x`%{eydtlc44`1{&j~cb(~vrhqUCa(i;SF zet-DJ%AKD!<_l|038zu&sZv1;QW*7j|_TliW}!=B+wrogiqfWGm^shHN(q=S+Eik5|5(`a#mI&S`zN zHF&ob$r!Nn-j=VoL(`?AEEBqW}g??h|jG6%W<1pO(kjdtgHcTpr>1w>9}Aj zX{Z-TkP5RMNQXfHY@moM5L5xjzv^559;@El(^g7~)oL+NtxrWcFfzbt8k-+BqDg|y zTK@nLzKS#H=dOJt4gQ<+4x012=)mNbc4C7xaCyQrZf0n|Irok0E zs`dg}C*rd7a@Rpt(xNlHDz^0wP%D~Ffq?;LqB918-9&;57T&_{mHi5atweB8ih4R& z0uGi6VT#NU#0;r`vm8FZTRkHBb^6KSeGuv&3+ZO33Fnv`Lx=NAdb!*v_@;}oG&Glq z$+0Mj6@oVA3sY7qP8mTN63I|>&!}7$&Mp^0y2H!A$_3Rw z^B%ovX2%KT{UYivK4bGVZRhRNw@(`1733L!v|7?#Yc;MzN-vW><;xbdteU~f2iAlX zU<86pkKQH=Fa*Qv8g!uwtcu)9)v46~09cgGMy3&+kP=idWdl%xBmywRlZXeTh4kH_ z*N!L1n}OE)e55#{IecrsnCWdRD<5wT_Q#93`mX2ZS2nVC5PM85%l zuRaZh#J;4f^#i0FTIW+ZF8=l%rr)p3j;(#&4+yblmL-$6%bY7#os_z@iYFgoXGv2P zNGw@@j=c7gt#MGXV(TtL3WzEgg``6x9Gr$dGl0QD@K-E)?`*4C0@UhSi-9XSp3vaT zf($40ChUQ_P}-5{{W^|hyMT%;!R`CU;ak7{$2_6DE|Q5QvU#km-+DM{Sf~E zi8l`VKIo+$m}T)xHu(i^k#!;buVqw9>Up2k4eflpvc#;ZRZ*D&2n&!}czmCiW#hX;61@fK1+C3nbU-1^(Z9Giv@|p; zCJB}ltG6blYMy4L0LqXOa0E2!_Vo=tAxfhFS5!*{!p<;23MX$999a!QO7t8q2+R^A z0y2C1NibxNDIACCldT;0h4bE^@cuQu$T&9|xm|7KYRaDcUagxB%g;#D^#g5%i*&4C znw602Ffrj}ouDL_wWYv%E-1RwYRaCX4O)8Omed#mh{24KqfDJr+GnMGLe!_K>*{LO z1{AWae8H9@P=IF|j3YrjdS~dhD#z1qD0%DE(A~<)s*Md}(+w2n9@Jm!3} zl=B@GFRhY?Fu7{9Qnf{<1#|`O!wiD7gYSjy^f%RDKenZ&RHM!FSCpd2OXstkvq$rFU4Pj+u5! zO1f$^NT%9BK*`3aB8G1@)2+g~MPEBc{{SP~udS`AT5rx_)~qdosrS~HO}Na^W3TMk zSe{!oB{NuC=y#33;9f_0@K-73zqB8VoA7ijXk1j}_EnhbsadGh)2x?6Ow_8vm_^gj z9KnTVs&il3P15q~Plj%)-5yoRnK>1MR#0wQw=%h;Zf-3pB#M-4%{n&J>OQrmDrzKx zygvO!<6L{CT{q6Fl~H%Ke%kGvq||zmU+;M(v|;5?l-spbGS0hQV)lvMSWGRl)S5t8 zKR-NvHu(Dg0OH<#&bh}ks?07b)~rQCtJ7*gvuP$|A}FOqD>3cfN2xhSEUU`7CnV(@ zxsj5UDi%#}aP?`h^-We{Bug>%05OQn4L&%zEx}J$x|@2eE|=PK%hbm=KVGHdi&eyH zS52K7{%C_syU9wx>69OtPmz0sK2cRJd16&lH0lpg3Bf4DMo19@u^@r+-rvs_9cl1wQwuTG(-Ge5)+NZ59stbj&Hka+Zvqj2;Ks!vXsCO%+rajHC! z0U~Eibm$*Rxh}@G4F3QH%Js2%4O*njH2B{_^0{V(G`D!YPG=ymEtS-`IpWJ_r3R$1 zUrIohUOze)B<8f9hvC`Mqah0oP+?dJ3Jy=O4Fj|iLjHaK0HiPM*0rBMJp6J-xv;;f zN~f*;uJkC2+bdAun_<;RrA7$MqcSUMW^R_9JDI#K_`JraPW4BJ^;4PN%-MF^7soE< zmME`04^NL-dDu+dnkm`6jvk9T&A*>W!|k(GDr?QHZLTd;tDjXEfDcgghGdc;!yQ4V zX&7iN;|@)C@txPi6{%nIRX1M@@~e7NWIaW>t9w?hT6NZ~Ha@4ODkY%DrxdlR1e3r6 zPLNmS+^;3vGlyz$o)o^(eX1r%MoOk*P-NZx!Qudps*+h&w9Pt!&Hfx0}bf?w1$E z%3PsUvSB(~5TVhxqg@T1CzrK#pjTNz40KBod`gs3vQ?oUzCh2UjWW0x6S#m#1f9tK zA5nh5`KLRn%ek$GI9l~-SY19HwJiPE%-09X@ssYl{1)}RR^{APt)4ZzTuzz;T9wLqJl{GFr3~gb#kVi&K(^hK0Hl}XpcWj3*ZVFt zFlKUegZ{8!7*mPIl0ZGd%E`b-jJtJMP8r*E7k}gf(Qpm0s$Iurg-t!#58MsBkH5&U>zQ@ITAE1pC%*!mpUveNuTn7S^c{Gyp{ zYEA&?1V}lF0Owa1YF5YME?;e8$kx2WnAlWXiim(!^&ZI;h22r9Ts3g7tOsLLNNF_3 z9Al7<5yIO;iCUvEsM{Q?jsE~Cp4%37IWSVSznJTZlB6{0hGhmEZKO0Vh2>LfZ+rBXnhEMNxFCn9=`g&;|d!p&jLu6a$puxiys&5gB~Yif0UT7fR!+*Eqn zB^mUqH8M4^>M$gZ313gVhmM}l2hGzYVRD`?XGWOm(uiF0>N~u9PZIWhtCk#^DLUhR zES^;+5~uh|x*^_H)sG*y78hQnCkBG>0g2-4(kMAt8EO*od2o1+i{jl;+jBI`g z=KS);<+;Bnzvb3-2P+k2`HLYjJ&UX@j(5vr9^4SJM_0fA+Y>bE~HFf@42 z8lgO^m1J@UZBH`svA3rFuD}wKXteFByO<(0CFJ9#3Vz!Hi=QKt%KBR zWQMNLRl0H#B0wbOcY`7hl)nYqb9OL(#@+fh^VpUhWH7dEtz`xP{{XqH3$RlVhc$m} zm{~`0h%%#JP-+)75r{H%k2_CtHBWpW%tXv^Z7X`5?~*Omxg;U1Tt2A+lSmr@AC_=r zNElGs(ZVd&+A)u)1Al*}1oQigYKGycQd82v)JD`vW4Nazl42*c_`u@Bx6;j8)tXp~{!+D@ zHE+jQ$50s$F}8y z<*I^oRHhg~44Mcq8No10#_)ci#}U!fgw3Qanbx%&+MjW1nKe`FN{S}*XIX+kGUxz# zCTuBRa*p7&dC7o*R;x1-qGCeF{)5l;z>X}uy3UQLBIdKw)5K5J&{mTr(*Mz~r&g3cy4Qcth=B^G-Zd{ETz=oqs&%QE7TccV9rR5qhXQ%0L|V6nH(y-_5pSC=E{75A44>0Jwia^w_`k|h-I_1HjW+)t!C$&{^|qU0de((CR1e)XGcwftBj8d zrsX1iC#J?Xj`6-b4MdD&_nC>ETtA~movk|FibFP#q6t;;p+xs8NGHEcNYioPT^A0c z-&OgsVSIk3wpw)Vq;3|4rd|7l(69DtZFFaDG|*L6`d+`dq3wo@6h!=W$@x`IVNy@} zlyE8-5v1k;f;E0=X|%xt9082* z@YT%5;@orX@%Z0yrHjXT&mRhS&0U^OysWz%OQ?KZ2AfWGb3!V$+AvB0ajA{-g+Rpg z+j}Zjrii6YsjVo8%lsOs8|pnaJ+I(sbTTGnbFDD$J(i#Q8scxe0%=$^R6SMWz{cnt(7^&D%0b2 z-!$`1t67A?cEgR5XQGtX)e^ZK^mN8n+?aze`Q~T{$nf5k4))9Hm&; z;(UPUwMV6$er51>XWLzQ_E$(|3C~DKw$!R4J$7J3l_M-?STYVLJId_SgFGy1buTGv zyt$`LRz|1Hl8_+RyrzG3b40lwnH!K1NvYI8iC_kYDs9y9$L)?)6fr}l&KrvpQxSUB zrRgiF%Yt)z+8ZKGhuH9xq!bYE56~8%5Y)L;+)4nWBqJyay?n!gtyErh8`#*5vQ>j(oqI^Oqn$@ytT)Lm6gxtz9;%Uh^lTF1 zVQVD?f@xEl&@zXzO{!`Q7|ZcA-d!k%q4%PcvZms%XCU%s@R zs#(dOeW4Z~l8$|>p!`+wC5I`tEpbq_TE|5yDy6EqV;X=Hr9=SCNCI%I0{|5&H9500 z+S3;lXr8K!6skc0k*I=9LZz*Ma^-^#kLa2P&`AQ(f;}D>Le*A0Ft}?TOC=`x2=0~Qch^DHCdgdqIlR$p=LSU;tMT?ZJ-m+?J5*BCtTk&N(D|fWAu0>j? zK=i(9GGGt^AOLb^W)A90HFniGk1*!XeatC!kj+Mm;ChXc)g}p0pp`H-7(bPi`lb4h z;9UmkoeJDpdKB#7;91DR!$t9}wZ&O!6IPjxe9~H3jizjQzT#%Xg(k8!BE&n#ky!HT zw>292;-h+EpnCNmDR5MRH4N%uf`cG)WlCWSws0%hp4<{p3fIAVH?GUyL3+kNXnq5?6s#{AXDsiPyr&fsqV@PfF zK@*81jl1c;7j73C!&Y`Z{Qm$7l<8?#xIlWW-@xP=OS_&xi9%37sQMcy{2UY=f=SG>C zKEe}QJdB%aVP81~T25)-+#!OMsJm(;6AVNsjkdu%z|O9ljgUloXknFsR-K4D?sxDA zKEnq~$I`s9oVs&Twzho=thYg#SvxWJHXWeib4@DJG}^T~pqiB_nuwYJsHM_AQPcrK3YH8FjOpL1`6Te-ifQ((tS8)n8iQeJDs?bHhz zwrW|Vmw(?pY8h;bV){_B9=T4H4ryn1%kC~%QM#rctx6PG6eU2nND!47EljC8mUU?{ zAaTgPAgM;)f#+$;$1&!-vgX^I(6p<0RUVLJTN}FiRhpZsTh%40>3V8Os0d(7RI|A+ zC7^M}dCgviO5vIJ3RDunSJ)(`$<=Y+8a-^t25X z3k_(Df!VN4NmeXO@OtEfhmmiWW?XH1`&$>d{*N8XwSDb=CNp$tF}oOrY00~dVh|@{ zq2@X1JB%p+5{oroPvo4w`sT*i+?*<0ekqv&O3Vdbh8m!fKoUw|ND(|Y__v!=b5EJ_ zK~m1DnrmL7Qc7l`0HNNZCZ(yRNgkt#)03;Dt#_XBI@%pP!$m2*f|q-iHm zcdX2wk3rOWmt1LV!&p1C0|rp0ivg~Y85aq6?54q(NHAlxL1_n=36aNNPk7HPh0hIx&)2w91y$a0dcjVZkqR z808#gdK}QWr#`>{(OoN;Tf93vLpACGZM>ZcX6GMa1$9Z=$!n=H$GLVY!LS5*2gi0z zN1NArZp%)xC1M&$E4+CAj^J{{TaOjd^{a?Ee7aAQD`JrI_;}$n_s|-m{_$+%xV!;5PGQe zf_12Qq(p)QlyjzIEdiB;@%pJ@%2D`>5j@-%aN>(+F@ zSrr49Sof=mKOY z9x!AW+F8NE>lf4?6XQHfhw}Um6+kcW4PF#{n_5+JzBQF*EO^$=*6w^<=^#ueL_)2VA`LH#Mxfl!Z^I96|&qixK0fjYR>;?9A0>w2zh({iRKNheZ) ztL75YcV+NTPAzfos&1ojES|7*yOZ%voqnh4@_YAqW|tK(zom=L#=_U5TAJ*Vs$SvS zw3cOaC9EQKH~rO5Zi1P@rUT-K<~8JiSzHESg+!BrGEzVUb|j5J9g8%JYFvt{$jV$S zs2wVB!b2T|9FV;sw3!?&oet}^4gS65m(-OD+;a<^b354EI+__rt5=TiUsYJEyT8YT zhMl^(+JR=~x!0_wR-t6?OoA(0vZR+6?^dd%LoZQ=J7#rf0I26G*XlV*w7Jhg-LHR(2&Xb8Db}#-L{SyBG^Yi@?fA9~f{{Z9H!+!}^_^N;MRsR6b zg&vnX{{Xpf_|yEj&)%DTA#?6q(CrSk;U|w${{S@K=bL;lg6MJz?&NLh?+3>CdPDd{3FBZ9+ zy-kj-^LoE;Uy<^*J@3qG?e}B!>B|^6O1~uAOd~U0*4vmRjWlLZDJ=R+Qi_EQP;@IX z0hoDXmKY&%!us?mSFcjtiKz9fvn^&MMFogdA}2`z4L_j-4huaG)=m#d@j2Dq%ACo( zql(gd6-p~-E{&@h%0bi5iCR%H57u%q0a8HgwaPnqk-U0;#6K7}_64F+qV>^i=@+Ds zh#;serDIHg0{p`SkRb#QsB(|Vxl*HohCLB$<4 zs%j@eCJvY(&W`Z?Jxe`s>IR^NeLb$N>SWrgHloJXjXWw;^T`i##zw`PHmhi87`kS( z0RpRgBe|XP_9phGmMo^OZq>&&$V~wIm&m13+Vzsg{N}&uHSz6@*TZ+#)>ky?SlL{D7*JHz zPUiH-;vKZWEkc-86RAPgM^kA|@$g5%wSFMye6G8xt6J2AeqVD)=rw|i&&GO^Tk$l2 zN@$FsS5rA@jt+lTbVo0U^6o{*0X1~d6IU8EEKphl8xok)!m3&H;`_9$hOKQTo`Y(A z=+Xo(7@2E9Uy{EQ^I8_vSMc>@-2>`V>ecF0b|sch5Ft*Geqv@b$Gtus<^H9Wy|u-# z7gpMhM|GDDgH~HJ8;qzpiQ~ow1jHvY;ylsD);Z7!0I%PiZi;n_)Zb)jqZF#bH{jrbdsE}@7E zBnj#tnsnsj1^@(iJacc_6_pNq$oY+{{LnNwAi zQYT3TU#)zG+0NvBGVZdmyjGg>WN1ta27kk{ciurl2UY6b4 zFy&C2>yITAtoj!)TdS*DH7NRsg?hjMEH_L@BuC~^i3UqZA-^kmEn6RlekeTanmz05 zYc5ZAx16enDzbJSwJLtThF~f+TB^R8SX$abu=&>FJy)@{#5jjh6UXfqLM`N{l{Xfa zT5p1Eau2c5r=gX0EJX^q*4tBBha3|qsNA}R$*jQ`QCRdUAy|8gbee zArOBv<#g|DDb}VLYABUKgQd4Q1_J!T2q(;*FQ2kMjcfcR%|)H%<)YeL*7PeSB&|{E zUy5Fx6{lSTQ*>RK*y$Uts5*mP8^3L|c?DEe-H@$TolA_b&C|!UEZ3{cQRia!65kKw zG-*|)QZ+GaEbI}CTxsA=a(|CYw97S-Tb5~v@rsIF5rZ$e&4)5ar&gA;3K#cEOBFr92UJSx zY^y|K4<1hVGo5n&d(F9z!(6Jytr{Gg;+|z;dwqPA-i120cRE0}t5T2!ZVgsSD$s4| z%~q(-!h_Tv4Zx~(Hou)Qh*}HkpHK>P+2x2e7fLt&@b#!83eMBo;xWlfksfYU`n+#D_&pmIZHluLOs8*A z!K_kkf`wMGTSs}KEBnce?i9pDpl6kKt(P#jvsy-*16|gkrc|>)$x>h*rc!WBf?_1I zF0c4&inS;UhF2zN-JuT3jZsjwH6#I9m0F4cEDIXZ!kNkGeTOjV-y-GtjOE-rKZ+wp zwJi473mLeaTa%8A>@L2M1$?q*Z6r4KNhM9MO{!FwfDruZe+t}P+}b58${J{TG}4O4 z&bkdY8v+jbI*2~_nBrY3t{OtNA(^JR;u};&?p&055-$&c#X%vu?)D-ft<|TaKu@#T=X1x{?>r zqF^%ZpsiSt0%kU=nkiqBej3`3qid8k3X;twgLQ^5x{pi){p~t}w<-ORa_jp`E^AMi z{7Flk{84kuYFpZ^3firyeinsWY85G|>UQ8)lc z;NXQAmNWAb!f^s*jrjE+#@wIdPI2%T!rbSP^Gn;x7Tma6d`rnwLKCW$DlL6L(|vWS zP`G-nRRYxMxAhE6DKnhX&C$itcRa#J55=X$`mz3HOjM=z8C^+r8ezvYh^y2GRu|Ou zJ0~KnjXNv5a`nh@8psO2v$3(ZxT{Tv;u(WcRWedhwJb=TM5q{3j1iH?mAP%-#drSz z6H(=yzWV2xkHvgHL8}!ajn~8U>N&eoToCPU!IHgsC0RmeN^#_4>cVwHn`QD2K@^#< ztB0LTY*8};Vls)fm91iYN*eYpVxnJS^h5&YOR&K6Pl#NtdRHY%wLqZe3MdDth>bge zl0bsDIC!jpyuK_W?F6p~5dnfhGoo>^8LYg#M6 zVs0$M9{p)_k%y|h*_GOs(+HT4xvOPKanX>T%IzdS7cdV@)dx^vxdurC#taQz!IO|i zM-^^o%4$)(Qq-oJ0%=yKAQfiVkg!-B8R}6ZAOiZ)#C9_;)yuRO4?Kz1E<)LA^7Lfa z;KMP~uPMEtb{|=PX;Qr8Qwu-}v`VJ&0|n_em4KjR#!1@%<#0Ph&i&viOWJhrYNJ}y zsC1hmPV}raLiQh4q@fm^v`^>9*S|<6bxwU^;<=7eyi|MKK)v+qh z>IkTe12RMAIG)qTR(4}A*xL0`lVzrFUf4|vQlf=awQ^FXJZmz8$I;37{GEjRC016O zmj+26H5RT2NnkJ-e@UM|?H>6O8Sf;sHrFln2_?yXeioGn^a`bVH{8Oa8bmb;R>r8NU#XW>cT-F$gffwm z3iUL%F4Pn$SxD0m)Y;CiwG1s)ak%Y5(A?L$w?(M2nWD<0a-MB5NDiZv)L43;#28$B zJ2b%y@}m_%H`9z6`;vQV$8Htb^ILmsk6PlbYDTRadbJi9>8W)loh2X~r}YsznFQnR2zTD7XuH!xU=^kZ5Q#Yt*&I%BI=oOEiA zvBq1aX~4>r)mU^6?Ai+k=hZc+;Q)(3bia~`B?>G-6?mIUYXEhqH|}Uxu;oM&Gi_L~ zT?TawXeYLiOi2?55HfVh2ZnD$l-bdvSk35KNT;R@}L z*QOQ_mAU}vTYaFgwBvvSXculbYGRYs>v)KOBY z1VA`w2YrbE$8F3v3956RU(4z}L~mL|q7_l0k~DQT#4ze%zQ7$|sW1G*E91E|xW*<< zEeyRxR?~2Dnn!RHn=Ia|%5<&~z)+0$Eb>2-?Epdd6(Ser2lK!3Z-hBlDz-{=(^=6B zNd%W8Bye{ngdUxNEwU{R%>oD%(;NMJAy_A;mg`Yp4N@XHJs= znR(jk?<>{Q(Qh2qyc3aX{j5xUjao|nA(qitIUDjOjDe#%@>gIkuG4HW%YU`)DP@U% zIo*C+%PiigRBuT@C`7vua2ZM52@twT)0}{KvGTu%?e4157gp5O?%tWWxOF4d^p-94 z5=fdr1Wh^$;3zTz)2?+Jr&-ulXxGTU$g^rLt!)k6dAFZmXOP&CMq;w4)e(xh6zvQ) z`9T$>SI3xI%&OF;x{`=hI*Q;)!VQdo0U~7fApEj?LCWvBLJO+*G$>qL z+|;FOW{1^kcC^5n)gNA=RfkA$2t8~d2;pVhNHvWs_y?mbSn3A|POXGZkK@nqkR>>LkH3a1LW^K!RYvCo4s@<##vN5L?ox z>gymdGYux85D722EvR-IPXoKWz1AkZZ0ffeZSimIr4Z+_M9CU?z{Ro$?V2k0kraE? z9h4@x?cA^m(L*jmjKPS`;&%jnv-Bmp9i4?MWmJGm9+FgpsQy!|v4}mJJ>=@rQsd4E zRZ8}%BIhDX4Pi|HT3c4x5Fm%0HAFN@7`RANtd8vjl$%VNP*omvep;+keKeLjr%G=C zdqCTzkFgEG-o0C3QjH@8wCf5^T5vmpK;CCe`*82{^XYq$a)JAfVWpv)z)uq|dAkV5 zv6;25+aoqlGbfK(s+}BG(LqYC+0)LMq-+^jxyELc_BXzwJ=d%i5!#zGFg-v_2#rKZ zIq&3PRQ@4awHa;bThuoXPhOoGR0$wGO_>`)_GNmpf@%C;?s2PTU0`B4f^8>te+w{GGc%ZVR=Tq&Xu5-j6gc7EVwK( zG=V6&F{G|$AO%$uk@B8tXK(7SGOtd(LM15>Es;|p+g9%ft5&H348tB2&2FUf?xi7B zi(6X1OtGt5vqst7cDDZjCq;W}VAIdhu^!a%*KUc=i4qct>6|DT@PYuxW*z^9i}D(2>$?!{v)#w zQs0-Q6!eQNHC968rUzgN$OI@6-wXo3_-B#b*RM(M>QtdZ<%-a1tWuc@rHLv@ClCe> z;xy!NedPUA;e8I`8DUnB^LbaXKJxCG2Pxsn=B!Zn0Sw$dh95lEsg-wi8Th z6f@@^HXOq8y-T|FRVC#GO+cv2G|41nV?U$=!Ovfmy>~S0O+hryfeAFukf7v~J3)i7 z&RZ}#_sBW#SMFi+PGqOQ}kg&yGMISDgyphw9(Ya-1 zwsZ{601ARN0N8;&wKGh69Xz?YjU&zPDXSXy30|=30)AitRg%Ci3m^7LVS~q}{{W{< z8#Q=-uOGFEPHMf}gNj%^D(8t!_w&|UI3rgw@!PMfr*^x_h=Lpi_b6?uOmdSSYr*~_ zxVLMob*3r`im3}vC4&--H5~vUFiZmkMr82olYS=F>hAS!sSib|UDJ|DR%52HsH#C| z4JZi6-a1DbONsZIXiW-v?q02~IhfiNU2jAM2daU3F$(UVruI~6ZRNjHPCfQ&+p4P4 zw)nWzR8DbWrh%^cqv|?T1$8?;H7n`>vPy(?6wF}A69K)rSdHKgyREakdu*Hnyr((8$odnV<~Xz`dnb{SR(y z*8tbyHG6C1w4+QCPP(C-l2|+qDrHo%yg@ox2+0$`JoSBa^=mmOlCUr!lO*{OumnfV zAmL{ZPmI5MrHEf2#-Pqq)4rtd=kedeD3v15uGp05EWKI=JMQ%U=Hgm)<41TUuSk zPG4_UNqW?c%_{1e@IqFml@Wx5Y11kQJW1TFc^u=K1|`YnnEKq2xtkn=VU>o*@@5(Y z8#Xj2?*lTFR-xfgG3zN-Tn1E7F=I-QSosZFcXn14wE8u@32I0G0C!n>T}t$;B9cjy z(oBe!4L3dfeLHg1s#B+0!jh%3*dkS@TbEZfnvD^oLHt|SXZ1D7^wfrESzP0wX1=xe~ zDUd`60gBx^fl_%V8s(Z)(P5FhlW$;QujI*rP1lX*Aw=jc(x9_bzq&M72n^xg@Emk571F;$v9pCImw^ z40i+`J>~v~9M7L6&o|}v6|SzW`E~7u(|T1K#vPYSn7 zKA_*d)DAuL$+@w5wy!2#J|wWybdTi`j_9vR<<|p-<72fvs7r>CpMHZPO3!; zFjp|Os+50n!FATtO^K3BpD+(|gN#6uApYFav$bn$PGder2o6U;b9Qh|0Ue^Y&Ytj~25^e&ZukDXgx@x17>d zx`64oT&A%|k$cK2(xTN7gP5HwsLJUi6%Y$JB1q42G7-3QO61hi>_VkLG?N4aGG}ZN zJ+~i`Pg|Mv_4iym2b@pW4hEiVabBKf;#zG;V5z)vg)0HArKgdd?y7vZEALI{Ix6+- z&}DE?D5a2{*wm%#s2`AuRm2bhk~*7=AO>hhMBTB&ZTOs zPCB(TTS+Lgn6P#f4-i#esT6M*y-IfgxgewKm_0Jebew6OIzs9o1C9=DcN8=<(Rzvo z2xKoAMd~_?$b~FQ$&3NRozD3PaISNzzNGaBg>eoirN%T_bXMo8TB-3URskazI1B~^0Wj-E+Q1mMK&RlR4ao~tGkWE7Z6JwSuhJoDy%&1 zwC}})R!V^?W{MprLkRq$WD$(!Htp3XNpt+i{{Zg}Y5xGEtN#FvzJ2)1zx&GHr7!%P zN$XMn0K8Asf8^ol`hV(A7y4w}$sM$R_$uc z@Y?rm?cU3&oTk{m4W@WI=rfMGx|9kv=^(7a|M^1PqMgc!BFT(xwO0pF%pDnMbPJskuA6?)NC< zd}mUOKF1``u`m=E2|2VV(cA=2)U(;+}omhi|Cq)>;|Dy zM|lNQ0XU3~HR;<_Tcux8xVI>YxJ;KMf*{I(0V4qHV~Q;qtHU~5TZ{3F#Mbbh zKW~OGuqrzCZC&h~hN&rC%L*yEda(H0)zD3UIzm#3m9Z`IvD;kGYkE}PqLyV!T4qFm zqEwQkLvH}a`TW&ihCU(Xe5!}VoWGhHNp*8h=b2IlKM)0XDN3D1U1eywA!!Iu=0{E& ziyzTGYU%Jh+=A_Q>{vdbb52vEou=CE(9|$!X5?6YpRslQXZC*DG?KGCuSzaxUn`~OH6&^Z98hhGZJo_3a@10 zV16s-H0kpbN}|DAbzNlX>jfc6A|aT>kTbp@4m~MjS*g~mq(Q1!wghV44oQF~t3BX& z@%eKrJCF5~tu!)fs^v>vM!o5odqSOndMZkrOwIypiyaM;vno+b{8+Gs8sq0~_$3*+ zEuvH~!6PC-0l(T9?>_$kmjjN)@I6&w=u;ByRgR-00w=cP1bZF#0LHw*?Ll{x2n8EY z@9k5EFz!JC#VdcZsjERE(Xgf{+!_h)9Fi?K+I)lLI*f?F0FZSH3fz%9S)J42}qS5A?M!S~F4eWPfS+y~Ann!<`vWh`|SVmF_aS|kw7h&a4Eoycp6=U6Eu+V^+tO@pO3X;?~V;8m&Skzp7(MbJ_4{cw$AQ7D#=N*IPZ zfyu)F*kB_I3L-&)#GkV(Ux;~SQ)|txE7csc;u|VvfOgogIzQsnQF2*Z&IPE zDYJr_PJ#~#_e$$V`*{K1rWT6L4QmdeYl%lYty?! zuoK0^h<=?`3RuRGL3#A=g>5YE?uuTxIx6Z_W>FyOg)T!y{{W7DCHT*m z*#6Ne{4>j`THSK5h^gKww4qb+7Z!ZljVkSY9+djGRP}1Txzt02)Ss1jnApwf7f!hv zvFCychMhV)EMk& zTBYWqJ1V(Tum_-wxiMb&K*trOEmvs`eR5)&TD6O`- znOk+~YuQF(s6|^D@6OE+ZCcv7inhxx4{29x)`z4W2O*DvZqmOmYL)#mw9Bw9rVy<{ zq5u;!w@gTYFhghKdrLa2|Tn~qs)PU^zqy-Q-$^;fPJ)2yuFfJCVaxz`NI5*Z-= zr?WZdQ#d`%t-2X}d#1YlP=i;G<;9$fdKPpWeWvynQl08nqM-_bUbSKZILZbL3Yp?H+>aBHbqi5@nYlRm{Z8qZ9_8FmM%|-H z4O{$OA&GrjDXn35=Iu(vdH(=*=!%}{YaBpF0m$ics~%&`uI%dpMEVuQ1|$ZjaFYT~ z&ZH7%U|@F(>0`|KkHP-{0MW0=Zf-e^e}=v&yX9Q+mA<;ob4Xr~#n5CSSrUU*3hD<_ z0x%Q?4{{E79Yg8QAL2WDTiRE0dG=ksom#anX*Ty9!Qmt@HcqG?@^-T?S8btV)g|PE zXgSDC!PXBc{5A61-qtk~v0~LK2wD9T!FEGGcmukM;DGs>GH~y zY#-EH%GOr3Z#B!4TBR_}Q&R!7vcQD{GD(gz;}>_3@%~Lbn+;Yn^l)@*sg*E`@~y zI+tzeAfiZP%m_(@(F_Kq3;{jFNZNhpsjx`Ek;fC}+>-i-GOUYNr&7H-{{SqZp1r`; z1j>*g$S_oC&|V?b23pjxhfw3XPreK!Q@*UKwi;I2qPHP-Y$YkGz@b$XAG-{XER6h*rYkEySwuK=@KtU%hCrk|H zW<|T5=#IX}2;(DTDEQoL-R8S#7iU`jO*|=S)-1)^H>#C=dd?bo+SH-SE9*enbHS~e zp!4DSOIE2@45c+`Q~9Se8Ih%+K?m!d5Ba|^dwy3|BYERLk0s7yLA&2jZuk!RP+t_<~+*w+LP0xSe2>FZcinlscR~W4oF4C z^(JM70_N$`qC0Qt=nQ4J8**(PFJIGI>fmN@tK)d)kuZtuSRTZ zp_XoH*!7*F)m0O4p>^WIag-~qGd@x#ve%UI>t1hY_u)SeRn%0*6cQ!0DEy}z!Q9~V z;`g6?O@GUIwSGruPNh2Ooe00u-||85~l%*C@&8J=!hf<;PC; z?DLPStLC^H!s0%YdVFYLuYOb}A22oitV?E}dE%I1pik zvel{7&I4qHl({CwDtx~_^?ST~kgABzrm5(St5C!)yiE)MbwayBlZMVM-B~(FhNivJiR5??4CcD$X+lsmz)f6mjZJ}a~H7ZbE>|eBZoxJq=nhCW{ za0q#d<{v8<51ju1;;v}bFP@DG%U0C&f*MKH^DZ@fqyT&EAPjNQzh+mppW$m7-xE`* zQ<>Roq_kL%OgbK#tU>Ci2g|GF*y=DY7~VXww0^0w8V%NG!ov@~WwMSNC;&N`XB(Wf@gg zM6^L^CjpEM5+*}%!Sv_o(|cWGXQ}*-I6Zl0GpQCWk+q-=v1$SbLg12W*IQyl)jq}t ztuB`+fMdm1GWf&+rrwZo9ZynzQvyN=Dn>ydv6zi8c(JD;<~3oI1S%>OHCa$tsKn{{T<^Rd|*TR@SU0^4-Zbb=}Ch1*ccDV65Ku!qOE(PQd$X&$0%J zTUm4zX@~(qYg+S~-IiKrpmCukA3_2^4630*uvHl|lO{EJ zti-vOttt&mP=bL`A@cKO@QZqBPQ6-mQl&u8>6YF)oQ41?hyptaB)|8b!Bi6m@E69524@(RrGvESwyQ-02uFVwrww#cd8V>WCnJC7u&wj%Q@S3#d>uPknsH#E)W3w2Ygh&QDr%c02@(!?Y>KEzg>+e>`%+x_Xv}N6!HtA+M(0gg5 zP(Q6RS^PYV0@-JT>L%B ztUejL3(VQMg*P)4=@(y9^0Oe+&!FV0s~OvP;u?Cb>zAXZ_A~FH`m}`KPwuK6N*7LZ zY7Zc?z^VL&QU|ruT!pMN2KOlLQS+Zz*;l4RU**S9Bdb#x)dwdhawo}QF~ygibGmmZ zqRAZ*0YY_U04J$Jsy4JV-b(vr zS0b2_RIXB}P%)?iMlq{#I;I4IB#0*%9A3Ff0cOEm3bmBVH10>r4m!liJ)<)Oaio7s zR%{m}pf-w*UQZr!TThf{9Rz46N)_Uo77?dLz)oH2F0oPe93xh!krRpoxi6_#TbsT? zUe7l)8rn1f+LQncED%cq1AbVH6OqLmp4#%ROLfccZ0=C=tDB=n=H9cVMX64xhhW;2 zr9jw?B$>f5b$VRre$M<`71+YN9jW88iRdykCgE>4QAL5h8_4uDo zc*j>T*5A14M^X6RzNM<32mB8j(^uQyQrR9vszt^veWPA8XV&RmiT21u7bv0lO82*E zS4tjZN~$4J%37s8X#;t6U#K>008sqUNniK7~Xy6qNr6x&7-~Sle@I=Kh7f zwY`YFK$aoGC9}D%2_;bKLr@qJpppTCS)Sw+DO%>CyR(s_rJIe@ZX-*nTpK4a;moni zv2RqRy@J31tamK(zefH0Yh!L?*%s($ki#z_wXRV7=I*_>Eai7pZR*@}>zX{i?(VEo zr$)-MJ)l;sOSaM#~n@OiWqaayET&sCAn10=^ zto|jSMYYrXC0j)cixx_?Udhs-A?RuvML@ZYP(ct5l{_*@%Dq~J3`%U!zsIlf@Uf2d zas=Y`%v*He)u1bD<}_);p*0eO!j89s;v@7|&oebgviO?uFKk}VnR32uYj1N|sI_Wq zU35J+p`;&Dg++%15tbwz=u79sv~4-ZCb#DNx0xl!E9Uk$Y%Kjnx2{7;W~f_96-K6} z;g}WDB$+M7Ir8K@c^31c$UoU}=A_*4WP-uYRHW_rE?IO*UQyg&(z#GbC?mkXWc~`d zt6$;1P-|97m4CDCne?DqY{r(UQJo}%Dm4g#S&W`Pnt$7$EM8OlQ23VA*SeC{18GOn_QitECUtN6#^oqgf_%cpvk zc&{|AI~(^|uSVi3ThPkctm;qHq2@N}GHSNBg=CuT(vf1pt%+ct1aDNZu8u)sN`tf$ zyb{L|h*KB_K@7jY{{RBisaxTGS8sDsYb`5ldLE)Ut4)@j6+`J(lCCu=8oS1+?A^>C z@doEScbM`GzHP6aV(&fS{Ie#%6Q`YDU@^Dp-=_SsqkV^&V(n*ML#&AbCFfHA0H(@R zV97Styy~qws4iYWLJ$>G`rw88mYJE`i$S45o}wsnn>DaO%K-b+c>i*B%% z#EW|T=H8vT(5lYP0i?hxy0B@;ojP=^!(y)eU4h$_Wkh5><+eo_pOSyZcv6lheZ=vL8i%2l~yq=us=1}7rSei|}b2fmgA9pIIT_-*ct{ga) z3bmMHQlzO+6@^6M1a~~jyVRP`Qn5I8b}r32i!&M2?oUxCOS3Uz#K7T>icE!#Flr4% zF!|t!AZ@7QW4z91R|fB^DagT? z#@&o+44$KZ?k4{L4?q0t>F@rJ#lQNi^6~wV$o~L^{{Z4|^5T*Bo^$+v_}BjcCmv+^ z6DL?WtYzcgtC}?NwdYbhD<%3KwHmVWvjMT9r|A;TG_87NGLSeHxC%$*a;0Ne(Hqr`&Ug^y4cN8g5pvjZj^;$Tn(LAu7RbS4y304$~^;=S>HXae83yoTj?pS=OSc zuTe!hY|u5JQ7D9j1c0ECC8RbW%Wx+$vP$i3T>zqKRc0g@6;+WpBUFSk1c}_le6J49 z7dq79{{T$YFLJQs`#m|1H?7I|?Z#TeS0&K9M@MH_dwi0Yp!2M+J;7eZ4^4+?YZx|H z_CQi#b30qgF0#cbs}bqvQ3MhvX&PcApD?F|iCfTqWWtxK_0Xj=f+(y6QnCnXBiaPU zbH#g(P`iiIzNT=_L-yG2aPvM)>e%`TbBr1}u3iQPFo;)@CZstD}d>Q*uPIJjp zrE6b{tT{(6t4Gj!nrU*%o}E!mMCk`F>MF?BI>v%{hWa|!v&uST^0B17TdL(38DMXv zTz^qjwAGcXUPVJ%y=iChU!gv*#8p`{v9{rmk6q+QZOce-`gJi%Wy%MHkdO%)rbO&^ z5;$ATDO97)sohq!t7~U;i%d#2MSAqBU&ZQ2DppzwF>Sb+P%k+ZR=X5sLCaXs( zxoU1}>25r=9bDS@xuB}{bnLb~s3N^A>UFcJ=Yh7}YeMD>`5%|(SNdGs+T4E8)0Ee0 zhP9V2wy0av=>767P!OqXv3aZ=Vx3iRtS67ruh^}l*nA&N!-t^f(xf0GXrzHPFdNG! zNr=&z^FcyCQ02?%ajSnc`xcTL3lop39ytvi@tkQsksA`W*03?M(t;o79?)+SKYG|q}}%NKmlYOVe1B3mOPeD(ujb35i+_~zXiu>{heIY zugEB~F*OvLafyK2$7zfaB!M^tm+K$1@uI7Il$$+B4Gzyu_$FR$|4@#v_CWU;6)LURl?byeh2_rj_ z-Jxndb|+#a`H%n^9dKiFpS+R9;*MTx!;a~OXQ%9aDwRUT2S{t#SY9`QO{L9MnF{@- z-Ouj3z$fb{`7(g&V*nq`nH!Ai9(}>b%eMkoXHQLmD-I6i6Fy{sGXo>M&IsX->5DXJ zbKXZojTbY&qsS@p1>MKuoTi5v$DDXB?axWJ^FV7h`=s|xE<(XaNcuPSJ=K0V|W z3SFf&U=`R%I<&-UFfdG=@jo+2!j2pJbLDn-p9Wvwa~c-6AB4EIsZH*vn!=NMjuDYT zBveyUDbzSwZ9oDK4kt*eCnufpwlTc>KTDHT=asExbq3~Lnw2moj$v%k&~Vjd*|Ehf z+N&1tJXvD-=w{V=WIoLNt(?-zjVl|gjYHPdnp6e=ZSsx5iJ38ie}p%L>ho@Ac9-0< zO}XXVwvCO|o!u05N4cq9tt!^4&QXoU0RV_J!c=0jDmbg|Y+s}HoLi0Q^W4+ooYSnF zTHd|RCr_DO{!5qDj&P%^pLG^&>gd{?0u?HiP?M$wa++32n20RvHQk{=0x6}ZSp!UI z2m=ZB3`b6WUgw6~g2L2u9%)+jxpnRLGqB`5iyBc?_a4mZ_E{26`$i~`pS0y@X5_OvPq&~*yYOGK~ zgh(z@%R;4kWWyl<5OBoE5z@E^V+4$nAnD`NU$h%X<{u2Mrl(GoTXkaXNAZ*>=}Kf& zWHah0XhdKZH5ME2-EzJ-T(6{AR5N(i<2mO34)Pe&*lz85 zyn)ST22NKesi&5Ds0QWD%y?a9F*Ek`sp&e>p{lkG(Y%;b><|b5+b%0w^J}|WwJvi_ zK+s%BZ~$NEuSl80 z5*Xq1=ZVVthjRwb*DJkyjr7_1I!M;HO*%7pAnj(Zrhby-{{DENz_-w`wRkr#W?1rg z(Pep8@xR24Y0A>Z+M?Bakjna4JJJeMPN3j}ei`JJ+|KWqQ?jROZc?({ zIw~a^)oM$S=mL<{dW)4Rnn5iD;_uXaTXh|R26C}sUjL<1qlI;uArkx)Vl&9l>XAv z&8%y!xKI6_{=%VuaP;YdF51GeIw%l}SJE04X(NVZ3R^QRG1LwX&U89qxym|&&GwSK zWpXd!=pJfq(`DM4S=szz$|zB>F=CF!J&U&Q6Ijn($5%DYf)#x0wJTg)8`^17irb-I zkpKi}5*za|4J#VENstJCH2f9uXTzM6z+LvT% z%|bJp*SNIiEpbeGC2G}ruDA?c+!Md%Imd0pXOC`v%IaW(Gbd|;KOYRz*%cV_bwXZt0=Nz)q?$1?1;8v+? zQkSYwX%wonb&?lOIel_yVsWZiT3lFd)HP{vt?{67D6eB$x*ux?zZjNa*YT0GZ#K`4Ox@iz!)5lEA;EU6xhn-GB6R z2FRI-`B_V5uW?t>oj#|kz%GzsbjCuMAPjY6Y%qAd>(fq~Rb>O^3{R6P00Lx!azXn@ zQ#sSEE@hR}`?ffY8)H7>+p9s05u)0lEu`6WSf2NTn#r|h^pPFA{YhhCKDzn8i*oJ zI-mgr$SJ=u=TzFlk}wLiZ)n_rl{&RiN{k7RbyF-!(slko2m^vc_n|M-w_>w*sB85X zFA-d8B@3ZnO*~tVnJABS!pW7%#x%AgT>O9DCjpmCh`75 zo29@xu4RVRwU?X5D`{b?Q$|G=>4VkgIXrg4oKvQYs;z#N%FuL_p2lSF`)`_CQ}s(N z)iAo45(`sOep02Rx8@9?fZG7DX=~ilwWK8tT57>4xE*z=A_1;Ou(b<8B#0!CRPDt7 z03X%qUrulGRdAiEnt9t2gbn0Yvo@;M$10YtRi|ZA$0se?6%CXg>z>;n(m80?2%m>g z_}7+Gw5vw;h+C@505XC_Fm*}j(q?w)81It2gW~UozBK0dR@JFfAKBC@txIZj>uF}K zUs9tsmegLFE|(e5Sc3q=B;5Z1ug<@$PPK0XBa=6J;xV-@=IUqY>GAr_4HfFzzPGeS z)7M?>TWyJV0bDqr_aLI>oXYyG-OJZjU&4(>kwyTdw5}iljo3$HA_?QroRgQ(<)05( z+P$@E*DY+TcD|xjrj=^b(^i@eg1{38Jw-{71h7+^b-R!6Mp}73GeVl(qjIM8Eh@CF zUUjuJ_YCrjQ9dIEvadI)=9v;&lIae{0RZsiau*j_{1t)H`>Os&La zPwcm0xnLFM<|gAp_GgCHL3kzHL>TCZBOC^{+A3Uotciie^K^+W;VrdL_~m2Iph zho`Nyy;pYX)@NbgIPNh!DF|DdJ2;YB4Cn4kURZJGs9Z^(wTo%ZRyjf7eFYH0DPp( z6B9l*0#ru|Cnx%>>Sn%vw*vnF>wbprl)B9c&#Q4RzJ|<<&`g!KQVe|bR68DNSKNwW zf(?&id=&v_&)e4BUHHd(K(chfFJu(WQ; z7JFAebb{{x00IjP{h!(*Suq5DQxIP%7Z-Om&^61uRZ5W$APQhh0=sTw5MyQ;JQTDQ zSgImZQ?`Uff=L;_2Ot3>8S|WduypZAz2=7Mx1@@}1wgK3F=s~797K@8 zS4j#R5(^14sy#61s;E(*i^kbK)10uFFc#6KfJh`b|sc9SfwZYg-LbyOg3E~f1{!VUFp*)68T)6P6Gm>J3%=k zzQ^+t4938nEg0mjM!mY+RTY#=7JwKt7*RhgAP^vdw#1$X9LI@SQs+Y2%~f+$lTLDK zCbKFnHfq%gV-g=>i&@V1GOgWcv8!k7+%*K_%!(u<)k%}p*v$LxLB{)xd-XhPYP3pv zY%!}qBOn!-nFNu8usDONcsq2Lb83@j&m67+Qo#DOEM_qpmFTRIXm6(QS*%m0i-R9- z=g8MgR67Q3Qs5`&um1pLmVA!?0FkXrx&pm(P6`rArBC1u3N=97;GB1kJazj>zqq#e zuBEMS#8h>-nhXpxmHfznvu9-sst$TUCAZJ%cItV3UNNriRpf3|FWKBflH{HN6jC$FNYuD&SiaBd3{Q}y5{>uI$@%s zT3mkd-P;8X8B@4fGlx5D4?EVr$dO70xzzg)=6CiN@RTq)hI zUaEw{PJ)W7>_CTa&8u?$ZC0DKQ~X6Tn%7o^YJxg+#1#-Ccat;2cDAQp))EMCbpl{E zh$KiLoX9^c>3~f1=xutmE%ECcX>R_7U3G9>8*b9lr;Qc%?XmnfeU>ul{p!K@v?8;t zEmDl_%2W=s~G@ zwp(kfl>7%fx42fSlxP4gYSlE<1|^!IP?X#&c{)ii!P}5>jjaAj#ddXXxF(k$!%~)H z^E(>_Whu=srtX_8rOk2b zy~Vv_lrrG2Nr;1_02w#z$n&b)nw7=)p+@rWlyd6!+`hK6S-E{GbuHG`_Ja)(O;j(e zf(2I8@j900%QLC+8`^wx_>5az7k@z)Z)fIc*Pr7roYk{uMy83#=riq?qIa5Wf7scl zSz{_fsaa6lIUhIXc62RoX>$vo?Q0qa(ARp5!sd#^dX(ox)F_~_lM>)CBydIV!#t|` zfVHX0Ev=}v)7;jzt4UFngPbD;3_`O?&JMtsjyO2z^_nzitpQf_?q3|85V>ri$b%Mqf1u|HMCd}Vk9MIZ2$&O=^yM*nZGmm50uraYBg$9 zDNvfkGcwanD*mAPh(ruQi6&|Fh7c_GG6HR3lHh{CXjcMiPPhrE zmUHEK_^EwfOM6HGIuVN=kfT|K1w>~H4FXt55Dyez^lQ2{f3<&wS*um6N~2xdQ=vkY zJG%95KMn&>6syUp)SkJ5Nm>pALzP@$`liD>zf?Uh-F|B~^z8F~52vF`J8fLbj>jF< zznWS+lD(zY6zP!y!cNDBJkhmFjV0B?OESg%F*m(ZzAoVCi-qSnoD%H*@ssYnu2=E-0s zb{d*kf=J;J`s(^|&FH5xO!02FV zIwZw57K>;xkxG_xu31vrtvbzFVRV3`#*(FFN{pZOhMtl{f(d4PvhX#brBc?QW*Doa zfl{0Nq!Y0QRg4ikMCw|nR6eyh*D>g?Pr7Nwcz+hRm1n2Xoz$*12X8riYpBY1pn8+T zEDE$NWo+J=v|N)JFT07a?^~lnudVZbwQ+Sp2nqYfVtULSE7S%ha>GD4P#098l+!H^ zT$HYw;cL2G8I*eaZI9=|M-KF-oxm6rHGhXYs?u-0$ z>+;HO$s3h&@2D-?sun`As$DJ{2Pj&W_i7v{rULqu#-mw%MIMb!083Nq+z~h>Oa*G! zHRn_*v9o#%po(<`&;IOcC0?1v1TIwm*_!tcb`{{Zn8-~6pV{)fVa!FsE?Z!?Z@^u;!F4IR;DR=p29J2dMuu8>AmJm_7D zadDIq(F#`li5~cBOUfh0gK5lKs){C`%+n*NsGL9$MnE$LOaU|3WvNM(osCmbl|-x< zV>@m{neMcs-p+<#0ZBN#$^pw$%&rx(fs-&c9!b-Vs$1x9S9nXC#WOlv z$azhTY=*@0pynG~I)gKL1*E-U{FNM(RjHRzb(|N{FFR9NfCL~%Kj%g6E7591RHsWy zRaT8n>GXoL9tb3i1`;GIM?&7#r6X}}YBfr%su=ZA6zSDE)Br({z?nJxq-Iqd`Yh_t zGv$3P>hB2WdV1EjFgI-O?pf1kzUJkQLrYTa9DA)#klrjMmVek&mN&XyGHZ`Sn_c_? z@oDAuwQMU`z01_NW9q8Q0>DDHrY(acnZSqu4Lno6F!(!`d_nO)C+0Q1DplxN)S++d zC#g!_l0j;!3@RF=lEeT@D@2Bc&~Kw|N1K-lcPcZDtPcw5a|pcUND?mX@WNERtZV3W)mT$v9GXa{OMdnQ%Ifg}cGS(&#Qre& zKH7)I*7ufmDpa)P{H3Z26sjM>-B+tnrCP>}gcX*eGE#E*rwC&qJtOq@kFOu2+yy92 zXVkaEF>E@nK4q7coVC-;&&s!plVim;E>le*ea=OB3MZA1a%&RK8}srb_Sw(Z;t$#< z#y2LLhNaFgEyt-;R;V3rD?6*y5(P0(kxr5wDgI($Ve{x3(hfy&RTa5Dkz|YcDq1v}pq`CU1wY#CAb(XEN(wH?a*`rr0`1fmR z*-kzx5~$A-{b2Z|%|>boN}<$SU{?SjQGq*yCNwmWCj*PcGRjl}Vpdc@)K21Y{vcvz zXOA%d08q4N8>-v&%t_pXZ8^neS!ONMu2)L4;__NQx!8(jJ|SR*rC>;meAyyBllEZM z4=Jum2AHW7lp$6DSccBsOvVmB`j+bd00vxoA>RZ!&;;d4FiiI_34^8wi3NHqp+h>> zP4S^yQf{@(T3Xo2QGvF);^c@nB{=Lc?pOWs@o0=h6iIDHzy9w|l6MRWH293?Iq|NtTs>Ae}_)C8INu_W%$>aBtd|?tU2OuT)ny^emm(ZK*A5imB>T zPux&x6&M!PP#mo_cbq|edR>LbI2Mmc=AwAR`L?Gkr&7hmbu#ZNqou~@j8!_f>6>Hj za zb3S`a*SJnhoT9aBRa-G{RWGUD!K6r?Ri4@D1!-sn>Hsvzt@4_CR5@QaqMjSFzu@|# z!n%-~d^eJgUo5;GPp@Z$D(%S8u~n2bZvB)Fa+o!Lg$Kf1$WXK39GTVZ3Mpkpav6>z z69k=Dff4~avyJ$>(W%S%Z#U$;;rX?dUTe?35axWA-qQZf>e*6WK-BrgQmsOrP-)Wj z{Yugs@h&w}LRgLyL~)&7JJky#zg-HWUsi+LQ>xcdd}{r+$A*nAL3b_euAhBq25Rfl zu-Q-)n%Z7W5>NvUW9d_@RSW4PfTf7r_dkEmy@D)wJ*#n2nb1(YqO0oZkPe{B0DwtQ5BkoWNHBQt z-#z4(ciisct?k86z*qQc-pbyLEuS)OXJ^e;{{a5cDiNs%r%cocG_GTD7sOB->+^Fzl?xTJ@6) zI}vti*QEZ9S7K6>u$2o3MnP=sNLe#6jW&8MaaWq!*1o4!r7KDlZR*lwG}%o$H5G1U zR0CG!^t!qAKkV966QrIdx_!y*^S)4_;{0y!6yiK*?)3F3ODJXKqU8&#+M!=+%z?72 zBGT`3X5On{XDPJNPQyZj!nrEve5Q{tqH0yRdz+f4N&-x&JwzOxu&M~mk|R!TR4bdi zYILuvQ>H1a9dy!5tFPhJy)&a%sZ5FptY2Q^s7UYqL&q1X#WT5jwd;nS^_rSoYOaM^ zwo5q_^dF!OpGP4L8V^fFHN{s68;`74O~LyU22Res)G4WxbZs$yQ=8ZRv}etCi}Ne8uQyS=ghhCSP&|nOwc* zh!W@*e-PSQ-Bf>A)tY)m6e^+W>tit)0eno!!2H!43ktsv{AbH3j$O*FxhFN{HScbD z(4|f4derXy1f@>tR!M3#5|5$^`c(x(@|OTE&O>i!gK-V4o=i3CbgP$Hr_6Zuy7l0A z)>h{?t+SDLV%Lg$(cI)7?PLPdQR=U4s_}-b0-^B@O7|=)dJL&>fUips#Sa8cIS)5$k-#u!reQNad=vpYpT{TiP;Y0}>VBbx-TyR}`y3Chcvu!G(xm-d- z&{l<)jb|YQYAVQAum)lY%2>~MDG*U=O|hx!5c%ea!AgMTu%Ra3EM>gh0aY}0xnXmw$EtRA#lVaZK}t#An-KU{!XA#(joLvi$?=-%>Q&yN{=lnWtBV z;XmE94Uk4PwOBfgec)e;?O4{coOUUpK0d;(&;b<46mMk3Ylj3VA+}<4g?JgC1Y@gg z3{Mp+K zF(h{H)T=u5tv1D1`n?a)5D6guX_6$IPOdKZy52OjwRY@c=hRM}TGlx~8?l_^ z&5d$WRg)TX3vq|8Yz zxH0Yl&<|5Aw%oJgE2^fQN+!;YqXjh^2B-=Ks(symGcp6Dfh1}HmL8&Yn@d*P8(%&z zYEX-r$lZ09J$^9X=~{}Y5vu1dyCn?*6{@)bDKcw`kBTZ>%{qdLj|!-y!JPhSF@e~U zW&zI@%Pvr9Eic4YEUNNAA|F)9IS_CNA#zD12nU7uv~_u2jY=B&Y-%K|-h5bdBB)6y zoYyU0*;?d9ap15B0qq~5cJhRIuXdK~_%iQKBh*veh?$s!nAnAd*8ZI&qfMC>3}B(!1x$uiI*N!6T0$WS#CDfA1d zTmcy!K)b~p>bcyfE`^7|G*-L%YMV8f_JA;IA}icQqFV)k%JJwhpE8e(ZTWQ?MAW8^ zOT2>0CQBAM-`PG`nH~V`bxnXpWg$eEZNR{diGzsD?sg-GQ|WW1jOwlKW5+q(`dPua z)m<*4-@9ZbdQG>le6i`#$&(Lc<;I#x#=32Tiq@$&!WK|%v{~}is;lZSU3$c+fK4@} zIUmGoFI069!}76EE8YYIQlwq=DH5>(*@g%J0yG?T=?V^z3XTSR{{Uj%S2$+|!i(yw z#Wgt<-a)r=)0<-TD?(72NS}7su8QncW>EbGn zl+r_D$fTgt4E4YiOwYvEbhK1wRRpRPKv;qS(qxcYByBqY2?GR^N~#c{$k(Wa-FMiH zw;N-;k_e70JP(+04n5PWU&|S4a|-;gE~gr*34E1qGJ~&wX7e9hWQj5bW|fzCvf`Kp z5DimSA2+gOx1hAEhpNo)O(U#`3}&Mea;5~nTN4r|dh_Z^06<2S)Bw~$(q;@y$bpR{ zK?ZoH^(!+kSFF)T4*c7})4#_lrHi!%PLg%zx~7l0U#TXuo!CvI<}iLu7=V}*T*j4J zm*S-+s`S-#odk+oG)e$N5!}Gjxjj5euGHuWU}|Y3`7APGRK^E@4!{yX;^oxNp72DY zPml5YCtr7pa;kWiL9&9oZ#P@h`LXI@!^*U|zKaX;26_x;Vmyc7R~BBPkx_usOvXb9 zGP{obfRV8xM*}tM)27w5>QPOeO7x*sB}wWg3&j5bcOo&TS3aIPYgZnI30tmZ=*C)E zaZVN$i#6diD_5?#RK8j2ut#BO?PZ`)$Zw3v5|d0Ee4rnHc#Hw=a$~$_ibpE9YPW4! zhtA}XFb=XoX#*Jt$U1?64XOC2jyimDl}gn<=B9L78vZFpYr{KQ+RMB=e5>T0wAUy- zD^9(u2XSj!<^o6N%S{ThfHgz{XL4Y9BeNWs+7IQtwI-ofiB%~~8f;VMmZSubK$8u& zz{wec&(Pe%r)Jv<{{U*|oGE`~ED-U1w^kOs0vl@bnC$xBdvYurbhR(Ftfy5=lOKaJ zJu{_-54qBFi8$Li?X)t$<6E{BUY%qrS!w{tMv%fkF>DFWM0~P#h6MFjE&`ZHn<=UH z^QE=wh=~d~quP012Fm>x!v8Sx86<4KW4^ zBTA)KJ&CZVq|-6_$%oHa{=s>Y-CNz$6sgzzIMQhr{KbirKbee@<&4{{ z9P4$TXOZ!)PeAjA?F#Hg`afjIZar#-t;vhfQji~(&-9V_QjazLnWs7QidW^gPgbP` z3a?t;vI`)vU9Yaiq-+71HCHCnkki`|U;y*0_Us zdm?Kkg0dgWSEEk;h2jb{1$$6hiKYqDXwcNuzCmH7WRnUa36|s3k8GTZqFVZmHhjNJ3!3f;PsITG!8Fmacyw<7-XiRJq5(!cHr`>TRx2s7-rXjwX`f(`!Be zSp}h1L2I@T1)|Aw>Xb_=)EYrrt*lOGR!*4{4Zd*(U~glj`T%-|5=aMfBlU^WNuM(Z zk2K#-`Z3f00H|KR^fyw7*KQ9|Ly~1@kGC@Qy#7}ZiPM!ZAr@A~uDO##@nOMb&Pmp7 zfGs)qEfkH9GFE3xnm(Jc)1hk8$4o&6BXRB`V}`u8_*V631wbgCnJ`#5)$)Ki}uU=>eGF&G1OUvV3|eMGCP!}+qbIQ8y7$2W6pTqt&Re11WV zR^G??ly)Lz+xY+M5dp^x~MMU&!=$A2xNj-<-_*z zXN>9c9laWSIPX%ZHz-<0`Sn}Cfk*`fHO^fxx2d{pd z&c~P9hH7TBySkCB6%f9c7%@zUDpEltG(wI#dF2-O6)r9**^Xm!mX$(WD-lk$r66t* zjXzMTR|zDxvcxZ@W)--p@J>gYnankDp&JR*wQnB&Zo!Ruj>)kUm6>lf)e@vt)6Hd) zT?NWUO(!Uaka*H`+qwS$C#ij9YjJaPPMw`Je_o3@QCKKeM^*)3qf2R@&4?4L%byo> zmpAnfEVTxzs@tHWOjfCi^$S#{PQ4NzAc#6;K_mrNzN@CKPG?I^^slD3CC#aP+p3t( z(|*k|G-&;wx4la=(1We&2xqX6Clla72lLur=(ElR&%;+dtlV4FsmkkArB0Q-My!c$ z$JC|jOlU2rKW?>QegTT01N=A-^5;^KnQsFmGJ)3;<-6~-si z!C~q!DWEc_*CB1LYh`r;?4VgB%0aCYVF|GaeXQ2D5n#k5QTBvBs{k=1vN#AoPjh+; zTEgrV1U0J+!Rdh@pd=D>h}33A9cQP&U$XvXSC;&7@V#no#8~sIdd8YmdIY-Eb`46t ztxE!{24!HH2@g)A2Ap*3KUOn(EAqEF+#KWmn+P_sN87c3XA$nR*w|Z{u;rW z@*`2%EYG)E+LV#FNkGcI&JGwR;-X5B@DkiXPU_rqMHUO0k+fV?LiJm{C{{Y!7 zMdv>X)VNEUm96SrQd&y&g=0pjrVZ5tQyRdk=vB39(qO?QIJb2F0IYml>gO8^9}4gA zY3WZGr;#ILk+g~q3X<9o~TsiD>@*7U-o43LBYBSXT0Je&LNiijhtKM&Ec#BHZ zs0*x>Sqg9&^vVe+2yARyF@wia)A%I}DTjLRPLjzItn#3^5*1oBwCWKsKQ|cd_5FjD zo!0x!;@wUTv=23>U#J=UsrJ{MdrK{u^(rei)e1hEA-PR35EiQ?QEW;{;4kv_^Y4i6 zYBh8Qu&U0whg6aQ+^cE;sU$GnjC`OVR@Mfi(QjUh(joma2rQJ@HUJGK2@FI|Gnj@` zxcx1x-H6bspI-8jX1#S~f+U%vUQ~Z~Kr4vNg0x<<8!{qJh=Xu0WIPVDNnzEQNnJ?J z2IYS+S?~2MBzyEtP_#4{XgW!P2_iu-W*6AZY)lRjidLWHKp$?Z`zfc~jD9eIj9+{cf#ozw`$~5|s{{YO6gn#tL=O6z7ZGIj7rEC8H zd2{~&`1Ah&``l~)0J0o|`~Lv^q5lBrJbBE@zsx#!N!vIpinT88*|SrS+0L-5^fYo> z>Pynge8S3{s8(fZLk4VuG+GhuK7834=E;@xuw9a-3`fn_gSY_p!G@XP%}o`8I@DHp zlByU&uPj+x`A0UaeJW_oSrCIDLMFP5=ss8#OKp1;k^M>aC!cjIpX*e2Ak)sG z=%a$4Wga*DT@1`xIGIO3ZuV>23R7m)hxP?J%PPQxNO+ZA_Kn3-bxl!%J8uJyEAt$RBTh*gQDwH}l!+|MX-O(+05B+mbpl8Pj?zdN#GNZoJ>#~m;4dRG&Xocz46O}@Xako{ z%+t%Y6^lbA7;c&_SSYy16z=t7H?f`%zIm~$jGrw*;x&2Mm*hg16HJules9d+(J)u})-F*5^a*b>`9^T=w zY#Fjjl{&;a2fEZV9hdfLb3yrSK!Rb$u`vXb2PEK#-gi)5j**8 zl4M{277j^hg(>|#^s{H1?eT71PmSmEysU0QiaIIg&8vZrtG8xK?Sw472i;-{r|8r-ARsB?w7j(S?kQtJGaXjzt-7hbXqXSjdzthnb##`K;!SF z)#}yiFC7j=$?s6F$81#6SM!*ov&HkSS<r4NE{R?ecm#l;$yr52?$s`Q?ordu{O z12u69#Z+nws6m)_Qf+jn7lm=X%>4cX$hw8E);#>{e7lFz(8l9jg9j>ww!bPe8W%0& zH#%QN&6U?K! zZCd8%`zQMVIk76fy4^cE`piooR9wTYGF$h3LK*-)E?=>GsM>vJt5-DdFF%EGK1;a0 z^gE7jZePabC~8k;?W8l+$QfMBf`aC~deuoDq`8wG3;BJix3oHC!?|GKY^O*h4w)u= zWKIM_bc>4oo0eUC63*4AU)^8d^1BLDP+U@=ZtWEUl-H~9iea7liEzly7Yv~F0+P6Q zQJISVK3qQFe<<4AS4UYq-oAb|s3TAoy)~rQsEw~&-M9w~DrqTD_Jq&GwY0A>A+o`g z<=gC0>ZV-EQf1XrD1xUHtJkxYAfj$cui3=x28^_!UawE~d0V4ve*NVX<03?yJeEf9&9rg)o)O}bUM8{ut6d<4mvs1&5lh%&k@!r*VyHpXL&e$%Raf-wW&uZ4&KheTbT;> z2A-(bPan?7a#w2~?5#SBT9av`r1MXORqIjal!}H8NWB#xE33kwMFUV^a>wN?!Mb3E z^Y6s`{=X}wZ)|Hb@@`wq)Ge)M9;GI^yU?sEI5H_cM|J{3CK+)m8|jrT%zVzP@ip-k zU)|~-My_vXX=rFyPH5rrZbO}|XPVNi+R3?mxb~IlsaEIQIKrbBSXN6^kwx(}rOg{V zT55s4D#1p905woB?|^5~Xi+B*8v(~z_cS6b0kWq23W zt?fdb)ydDPv)!i<9N{hpsMfdChoT8G0X<`}hHj__5(#h2bH{)9hL z+qZF3TFRw8CaRWGB}kFf4n|`n1NAu6AQe89#VWv+Fh)v~20n)dawG5CiS853HhO>j zh5k2ovwr^oA4Zet+^6>e;h|sbiPBz5ax_Io*v~&?1%GL<*55zd4Q!7@dV!%-yd(m| zgTAFDpiTi^t>zbq+M+Xn#txwxvk*2B1H&=zup&rW=NaXfxUzYceqF0MG&v?se0?l= zFk|nzFB@4^$fa{CHQ6^PG$6s0#=$ZUB4mCjqV#FkU@h4loNDR~Qguow4GKXlV^aj) zSSew|wP~86RzOh9bFloTRv_R41dh=J9Y$Waj~wJ&!90_Lnr>j!;&^p*_0FIpFtp~g zCvz#nP%~=v897_ZC+;gKVYA?33?}s~O4Uk$u&E;-FaWBz${3Q&P5@JZ+#aB*yCV8c zzO5iBhW_kgLXjDa?a=_VhbP7BPF-y-OUpRVB+<+2MlDV~lX}=Lq8%qdv}m@i74`+y zW@xeKn@UhxS$pN?={PNknK0xC7+LiRVe%3uNDx5$zcAGdM)X-KFhpfXrrMbfWDGXN ziETiTJ@YUEj<_dBZF2rO$@M0|=jdQ+<>lllN;fj~+fnUcL_owEYXNqjPk&p|(0ssn z&&6*|s%xj|`bLvs%t=$WlBC2PUDim#fu)?rWe$iy*7DkW03~lO(vNz|5 zH#%3oei}!+w)v2A2d9{|`yk45wIx&(~P= z8WoGdc49s~OF;J)^{WOB7$c1Z)}rNl^uR?Utg`?CMNXo@cn0fBP_Ib{P-(WFto+Hb z1dm3^GyGLRu8O)xg2+ZrHz7rM(So2>nVn`? zZk%&XVjaSjI zz^Q4brNDVB%ptJ4twx@vENOJFxMCQH5ssBINHN|^4Mx>8h>nRRR&%INgSm+8Gbg<8 zd)mp;=iCcppJpKq<#p=qZFK_hjhLwxc>-eoUR5`PnFO5NfddcPe^Z0D6^nO%NoA z0H|^f@q)cWi9ZS&qtww=Y0E-l7GwGQ#R$? zT+n+``?{fejk+1O;0z9bkj!G1%EmH2HkgYN9w8;IJ8-KELV$HN8G_{MJu+}+NYwxX zvEyphpItghC7sCE6VkKPUJZ&Jm3C z9;+(UXy2xmyp2V71QR27xY9R6BF=*J^5}6rNHEe1l4WhmQoAm?eJAEo(nO4adw2#KH-n})=RCjSbhN6}DN=Wn6(VF)OHRfd&`VJ{7JT)T2-eWwJ+>Z0VlPW{{WF^+tt6H zU7-!s8pN`K)YhM~v^N@XqVPB>?Nw@xkN^-4Hq#sVkO0Sh_Z(h%e=nzR%WDmFsx#G8 zqp&oUMy4C|s4b)@)!KaU58+!iINv1N#=S+Hgey2Ll2N#{-)^K(6+_c%m@h0Gs9Zr% zK@D`Dk9ItR~Rcd;}TT>@UEtOco+6Jg2&SQ>Q@vp+4`%`;sO6H?b zt!!qC(?(&T$%b%Xw*C7I>V++<-%kKMN0DT;T3WG-c5AU4+AMXYlvN6lrq$MY*l5!2CJ7MW)E&KxOqqAPg|BRf2{Do{~7xNRw`&H8FIxja5Zfpf&85PqVhWb01my)r3QOr zX>(4UdNe6$MzE`$Ec!v{Q%}1Qup$Q@wD@+h<`ir>hcc&1rQf3Nmtyg2rADX%)JjN< zQI?Aa>Rmug_S8*Mkq_S%!qnpzSE)cGucr1@kVb}h%B~Hobk|)9lqRLOxz!507f7F) zKq%sT!Ip?6sj@*pta1CrC0}5Q%*EG*PrJZ6>MxP$)X{(RiwQfl?^8hS;_Z`n;PG_*v@qdyV5%!jS5`&dA*E#$}E@jDT zTP(GEQui8jt4jRjM2H}f^Zk|vnJ80g75#F623-(4&(sX~0B6eJaNTf>>q@0On*Bg- zg+Z^Zy>3;sR%$aX?7fpY+;I@qSt@l=sJ!~&X^*#v>8j!b%Z0R?d_Q)z{To_UD^Uc6 z6m985xB<)5sucQUk)#f-Qb)|tl@E&O(RON;ZRjy-Y)yNI)U=U0%+niz1+^Wt$U)K> ziQv53h-7 zjf(c`v`iLVH4qJCl*O*glD)!XtimhBI3iUlFqK$yms+L+{G%-KB zDi|*uIog;OH#chFQ5~H{XAVY#T}J)uIssqpi3*@nP=;@}J@y|?@huE}jNd~JUWyhU zVdi#qYjV2^m-kMViOlI%3ZNZQ3TZ2nX}K-aEDkow09nbNhPLWTX$!}m0; zi`7~*t*Yv>hzK^(bXP-_=JOvi>26%?#}lNZGM^oLVrt>c?`dU#hgj zR}ZXnTF;N(Sw=Tj(g7S?JLuHdUTml8kC@@Qm?3`NBXx=p^6wj zWHSC^f2WVbJj{F%%df6&t9rT}tDtqb3wLcv+Ldc+f#=Rgu7}ER~s8}-|(Fuw7%}RRdR8=e2Nr42pB!wi1*pc%I zF*5~ac>I6(9jPxpp8a=A{8=#SPR17Z zBBiMEpcb+yR?xn+@*UzumspFeQbba^sP$N!tKfbGL(~ieuqJ+aW8$P*SGQ0~%~oX~ z5D1eQfI$L80Lowy6Q_^p_x%{}Y%2T%ZEtGIq&iirMKY9Y)}=^P0q0W^pfy0!!f^mX zkh_r@#(q4tS09-Kx zQl)oZlNLvbLb76D#-Jxz5+@@tV9xw{POhp;sH!m-T}Z4JU`Q=1szjOWb_53EaU4Ti zKMRuE)1xe{#?W@F>K03FCyzPuHd485S%{@GCIp)g5dxW9`I{t}@*34DRH?3^EJz(p za|Z!IJyK4T9E%x2xSx=9;)v zxR(7|yrX-I;?{33;Ci@@7)Mg4J^t{$g9Bn}@?5e|wfR81t6OWC{6Cl4{wbYQ4_GKP z(ceb5!=k%1Dj|?Hc4MfpBxyy=udeOx7M&GG4jEnK6ocbW0>ck)fmu)uDf0{{Z3B^vTb9OTvDjevs|)&O4i*LyYG0{s*tL z)gF#VC2oFhe^DyrYcy;o#tn-ezCtyH)!$4TX&soL0{ZKh>9?a-)vZugruB@2r2tq8 ztTn-tjZq|sfEd(W%daj-u9wB0sDevelopment Team

- +
-

Misty Cracraft
(MIRI)

+

Rachel Cooper
(NIRISS)

- +
-

Brad Sappington
(MESA)

+

Maria Pena-Guerrero
(NIRSpec)

@@ -178,35 +178,41 @@

Development Team


- +
-

Mike Engesser
(MIRI)

+

Misty Cracraft
(MIRI)

- +
-

Ben Sunnquist
(NIRCam)

+

Brad Sappington
(MESA)

+
+
+
+ +
+

Brian York
(MESA)

- +
-

Rachel Cooper
(NIRISS)

+

Melanie Clarke
(MESA, NIRSpec)

- +
-

Maria Pena-Guerrero
(NIRSpec)

+

Mike Engesser
(MIRI)

- +
-

Brian York
(MESA)

+

Ben Sunnquist
(NIRCam)

diff --git a/jwql/website/apps/jwql/templates/home.html b/jwql/website/apps/jwql/templates/home.html index 9c16c55af..0e4ed8a3a 100644 --- a/jwql/website/apps/jwql/templates/home.html +++ b/jwql/website/apps/jwql/templates/home.html @@ -24,9 +24,9 @@

The JWST Quicklook Application

- Logo link to {{ inst }}
{{ inst }}
@@ -41,9 +41,6 @@

The JWST Quicklook Application


The JWST Quicklook Application (JWQL) is a database-driven web application and automation framework for use by the JWST instrument teams to monitor the health and stability of the JWST instruments. Visit our about page to learn more about the project.

- - The JWQL application is currently under heavy development. The 1.0 release is expected in the fall of 2021. -

Find a JWST Proposal or File

From dd4063567b56749578aca05397757511cd460d13 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Mon, 13 Feb 2023 14:14:55 -0500 Subject: [PATCH 048/449] Removed test directory override so that dark monitor can see its actual output directory --- jwql/instrument_monitors/common_monitors/dark_monitor.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index 2377d5908..17160450b 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -245,11 +245,6 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no mean_slope_dir = os.path.join(get_config()['outputs'], 'dark_monitor', 'mean_slope_images') - - # FOR TESTING - mean_slope_dir = '/Volumes/jwst_ins/jwql/temp_dark_mon' - - ensure_dir_exists(mean_slope_dir) output_filename = os.path.join(mean_slope_dir, output_filename) logging.info("Name of mean slope image: {}".format(output_filename)) @@ -764,7 +759,8 @@ def process(self, file_list): # Add new noisy pixels to the database logging.info('\tFound {} new noisy pixels'.format(len(new_noisy_pixels[0]))) self.add_bad_pix(new_noisy_pixels, 'noisy', file_list, mean_slope_file, baseline_file, min_time, mid_time, max_time) - + + logging.info("Creating Mean Slope Image {}".format(slope_image)) # Create png file of mean slope image. Add bad pixels only for full frame apertures self.create_mean_slope_figure(slope_image, len(slope_files), hotxy=new_hot_pix, deadxy=new_dead_pix, noisyxy=new_noisy_pixels, baseline_file=baseline_file) From 6e393fcfd7eb3285a896c29507a1d756f28abea6 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 13 Feb 2023 17:09:46 -0500 Subject: [PATCH 049/449] Add instrument viewed status API --- jwql/tests/test_api_views.py | 1 + jwql/tests/test_data_containers.py | 85 ++++++++- jwql/website/apps/jwql/api_views.py | 36 ++++ jwql/website/apps/jwql/data_containers.py | 161 +++++++++++++++--- .../apps/jwql/templates/api_landing.html | 4 +- jwql/website/apps/jwql/urls.py | 23 ++- 6 files changed, 274 insertions(+), 36 deletions(-) diff --git a/jwql/tests/test_api_views.py b/jwql/tests/test_api_views.py index 8c472a469..964a3333f 100644 --- a/jwql/tests/test_api_views.py +++ b/jwql/tests/test_api_views.py @@ -40,6 +40,7 @@ # Instrument-specific URLs for instrument in JWST_INSTRUMENT_NAMES: urls.append('api/{}/proposals/'.format(instrument)) # instrument_proposals + urls.append('api/{}/viewed/'.format(instrument)) # instrument_viewed # Proposal-specific URLs proposals = ['2640', '02733', '1541', '02589'] diff --git a/jwql/tests/test_data_containers.py b/jwql/tests/test_data_containers.py index 0807e78d8..e53cab5dc 100644 --- a/jwql/tests/test_data_containers.py +++ b/jwql/tests/test_data_containers.py @@ -7,6 +7,10 @@ ------- - Matthew Bourque + - Mees Fix + - Bryan Hilbert + - Bradley Sappington + - Melanie Clarke Use --- @@ -69,8 +73,8 @@ def test_get_filenames_by_instrument(): @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') def test_get_filenames_by_proposal(): """Tests the ``get_filenames_by_proposal`` function.""" - - filenames = data_containers.get_filenames_by_proposal('1068') + pid = '2589' + filenames = data_containers.get_filenames_by_proposal(pid) assert isinstance(filenames, list) assert len(filenames) > 0 @@ -78,12 +82,63 @@ def test_get_filenames_by_proposal(): @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') def test_get_filenames_by_rootname(): """Tests the ``get_filenames_by_rootname`` function.""" - - filenames = data_containers.get_filenames_by_rootname('jw01068001001_02102_00001_nrcb1') + rname = 'jw02589006001_04101_00001-seg002_nrs2' + filenames = data_containers.get_filenames_by_rootname(rname) assert isinstance(filenames, list) assert len(filenames) > 0 +@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') +@pytest.mark.parametrize('pid,rname,success', + [('2589', None, True), + (None, 'jw02589006001_04101_00001-seg002_nrs2', True), + ('2589', 'jw02589006001_04101_00001-seg002_nrs2', True), + (None, None, False)]) +def test_get_filesystem_filenames(pid, rname, success): + """Tests the ``get_filesystem_filenames`` function.""" + filenames = data_containers.get_filesystem_filenames( + proposal=pid, rootname=rname) + assert isinstance(filenames, list) + if not success: + assert len(filenames) == 0 + else: + assert len(filenames) > 0 + + # check specific file_types + fits_files = [f for f in filenames if f.endswith('.fits')] + assert len(fits_files) < len(filenames) + + fits_filenames = data_containers.get_filesystem_filenames( + proposal=pid, rootname=rname, file_types=['fits']) + assert isinstance(fits_filenames, list) + assert len(fits_filenames) > 0 + assert len(fits_filenames) == len(fits_files) + + +@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') +def test_get_filesystem_filenames_options(): + """Tests the ``get_filesystem_filenames`` function.""" + pid = '2589' + + # basenames only + filenames = data_containers.get_filesystem_filenames( + proposal=pid, full_path=False, file_types=['fits']) + assert not os.path.isfile(filenames[0]) + + # full path + filenames = data_containers.get_filesystem_filenames( + proposal=pid, full_path=True, file_types=['fits']) + assert os.path.isfile(filenames[0]) + + # sorted + sorted_filenames = data_containers.get_filesystem_filenames( + proposal=pid, sort_names=True, file_types=['fits']) + unsorted_filenames = data_containers.get_filesystem_filenames( + proposal=pid, sort_names=False, file_types=['fits']) + assert sorted_filenames != unsorted_filenames + assert sorted_filenames == sorted(unsorted_filenames) + + @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') def test_get_header_info(): """Tests the ``get_header_info`` function.""" @@ -115,6 +170,28 @@ def test_get_instrument_proposals(): assert len(proposals) > 0 +@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') +@pytest.mark.parametrize('keys', [None, [], + ['proposal', 'obsnum', 'other', + 'prop_id', 'obsstart']]) +def test_get_instrument_viewed(keys): + """Tests the ``get_instrument_viewed`` function.""" + + viewed = data_containers.get_instrument_viewed('nirspec', keys=keys) + assert isinstance(viewed, list) + assert len(viewed) > 0 + first_file = viewed[0] + assert first_file['root_name'] != '' + assert isinstance(first_file['viewed'], bool) + if keys is not None: + assert len(first_file) == 2 + len(keys) + for key in keys: + assert key in first_file + else: + # only root name and viewed by default + assert len(first_file) == 2 + + @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') def test_get_preview_images_by_proposal(): """Tests the ``get_preview_images_by_proposal`` function.""" diff --git a/jwql/website/apps/jwql/api_views.py b/jwql/website/apps/jwql/api_views.py index a0ef8b30b..719c624f7 100644 --- a/jwql/website/apps/jwql/api_views.py +++ b/jwql/website/apps/jwql/api_views.py @@ -50,6 +50,7 @@ from .data_containers import get_filenames_by_proposal from .data_containers import get_filenames_by_rootname from .data_containers import get_instrument_proposals +from .data_containers import get_instrument_viewed from .data_containers import get_preview_images_by_proposal from .data_containers import get_preview_images_by_rootname from .data_containers import get_thumbnails_by_proposal @@ -137,6 +138,41 @@ def instrument_proposals(request, inst): return JsonResponse({'proposals': proposals}, json_dumps_params={'indent': 2}) +def instrument_viewed(request, inst): + """Return a table of viewed information for the given instrument. + + 'Viewed' indicates whether an observation is new or has been reviewed + for QA. In addition to 'filename', and 'viewed', observation + descriptors included in the table are currently 'proposal', 'obsnum', + 'number_of_files', 'exptypes', 'obsstart', 'obsend'. + + Parameters + ---------- + request : HttpRequest object + Incoming request from the webpage. + inst : str + The JWST instrument of interest. + + Returns + ------- + JsonResponse object + Outgoing response sent to the webpage + """ + # TODO: define more useful keys by instrument in config + # currently, optional keys are just the values available + # in local models + optional_keys = ['proposal', 'obsnum', 'number_of_files', + 'exptypes', 'obsstart', 'obsend'] + full_keys = ['root_name', 'viewed'] + optional_keys + + # get all observation viewed status from file info model + # and join with observation descriptors + viewed = get_instrument_viewed(inst, keys=optional_keys) + return JsonResponse({'instrument': inst, + 'keys': full_keys, + 'viewed': viewed}, json_dumps_params={'indent': 2}) + + def preview_images_by_proposal(request, proposal): """Return a list of available preview images in the filesystem for the given ``proposal``. diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index c15fad920..1928729a7 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -13,6 +13,8 @@ - Teagan King - Bryan Hilbert - Maria Pena-Guerrero + - Bradley Sappington + - Melanie Clarke Use --- @@ -532,7 +534,7 @@ def get_filenames_by_instrument(instrument, proposal, observation_id=None, restr filenames in this file will be used rather than calling mask_query_filenames_by_instrument. This can save a significant amount of time when the number of files is large. query_response : dict - Dictionary with "data" key ontaining a list of filenames. This is assumed to + Dictionary with "data" key containing a list of filenames. This is assumed to essentially be the returned value from a call to mast_query_filenames_by_instrument. If this is provided, the call to that function is skipped, which can save a significant amount of time. @@ -598,7 +600,7 @@ def mast_query_filenames_by_instrument(instrument, proposal_id, observation_id=N Proposal ID number to use to filter the results observation_id : str Observation ID number to use to filter the results. If None, all files for the ``proposal_id`` are - retreived + retrieved other_columns : list List of other columns to return from the MAST query @@ -625,6 +627,79 @@ def mast_query_filenames_by_instrument(instrument, proposal_id, observation_id=N return result +def get_filesystem_filenames(proposal=None, rootname=None, + file_types=None, full_path=False, + sort_names=True): + """Return a list of filenames on the filesystem. + + One of proposal or rootname must be specified. If both are + specified, only proposal is used. + + Parameters + ---------- + proposal : str, optional + The one- to five-digit proposal number (e.g. ``88600``). + rootname : str, optional + The rootname of interest (e.g. + ``jw86600008001_02101_00007_guider2``). + file_types : list of str, optional + If provided, only matching file extension types will be + returned (e.g. ['fits', 'jpg']). + full_path : bool, optional + If set, the full path to the file will be returned instead + of the basename. + sort_names : bool, optional + If set, the returned files are sorted. + + Returns + ------- + filenames : list + A list of filenames associated with the given ``rootname``. + """ + if proposal is not None: + proposal_string = '{:05d}'.format(int(proposal)) + filenames = glob.glob( + os.path.join(FILESYSTEM_DIR, 'public', + 'jw{}'.format(proposal_string), '*/*')) + filenames.extend(glob.glob( + os.path.join(FILESYSTEM_DIR, 'proprietary', + 'jw{}'.format(proposal_string), '*/*'))) + elif rootname is not None: + proposal_dir = rootname[0:7] + observation_dir = rootname.split('_')[0] + filenames = glob.glob( + os.path.join(FILESYSTEM_DIR, 'public', proposal_dir, + observation_dir, '{}*'.format(rootname))) + filenames.extend(glob.glob( + os.path.join(FILESYSTEM_DIR, 'proprietary', proposal_dir, + observation_dir, '{}*'.format(rootname)))) + else: + logging.warning("Must provide either proposal or rootname; " + "no files returned.") + filenames = [] + + # check suffix and file type + good_filenames = [] + for filename in filenames: + split_file = os.path.splitext(filename) + + # certain suffixes are always ignored + test_suffix = split_file[0].split('_')[-1] + if test_suffix not in IGNORED_SUFFIXES: + + # check against additional file type requirement + test_type = split_file[-1].lstrip('.') + if file_types is None or test_type in file_types: + if full_path: + good_filenames.append(filename) + else: + good_filenames.append(os.path.basename(filename)) + + if sort_names: + good_filenames.sort() + return good_filenames + + def get_filenames_by_proposal(proposal): """Return a list of filenames that are available in the filesystem for the given ``proposal``. @@ -639,16 +714,7 @@ def get_filenames_by_proposal(proposal): filenames : list A list of filenames associated with the given ``proposal``. """ - - proposal_string = '{:05d}'.format(int(proposal)) - filenames = glob.glob(os.path.join(FILESYSTEM_DIR, 'public', 'jw{}'.format(proposal_string), '*/*')) - filenames.extend(glob.glob(os.path.join(FILESYSTEM_DIR, 'proprietary', 'jw{}'.format(proposal_string), '*/*'))) - - # Certain suffixes are always ignored - filenames = [filename for filename in filenames if os.path.splitext(filename)[0].split('_')[-1] not in IGNORED_SUFFIXES] - filenames = sorted([os.path.basename(filename) for filename in filenames]) - - return filenames + return get_filesystem_filenames(proposal=proposal) def get_filenames_by_rootname(rootname): @@ -666,18 +732,7 @@ def get_filenames_by_rootname(rootname): filenames : list A list of filenames associated with the given ``rootname``. """ - - proposal_dir = rootname[0:7] - observation_dir = rootname.split('_')[0] - - filenames = glob.glob(os.path.join(FILESYSTEM_DIR, 'public', proposal_dir, observation_dir, '{}*'.format(rootname))) - filenames.extend(glob.glob(os.path.join(FILESYSTEM_DIR, 'proprietary', proposal_dir, observation_dir, '{}*'.format(rootname)))) - - # Certain suffixes are always ignored - filenames = [filename for filename in filenames if os.path.splitext(filename)[0].split('_')[-1] not in IGNORED_SUFFIXES] - filenames = sorted([os.path.basename(filename) for filename in filenames]) - - return filenames + return get_filesystem_filenames(rootname=rootname) def get_header_info(filename, filetype): @@ -869,6 +924,64 @@ def get_instrument_proposals(instrument): return inst_proposals +def get_instrument_viewed(instrument, keys=None): + """Return a table of viewed information for the given instrument. + + Parameters + ---------- + instrument : str + Name of the JWST instrument. + keys : list of str, optional + Additional FITS key names for information to return. + + Returns + ------- + viewed : list + List of viewed information by observation for the given instrument. + """ + # standardize input + inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[instrument] + if keys is None: + keys = [] + + # configure some special keys to avoid table name conflicts + special_keys = {'proposal': 'root', + 'obsnum': 'observation', + 'prop_id': 'proposal'} + + # get files by instrument from local model + root_file_info = RootFileInfo.objects.filter(instrument=inst) + + viewed = [] + for root_file in root_file_info: + # for now, report viewed by root name only. + # if specific files are needed, use get_filesystem_files + result = {'root_name': root_file.root_name, + 'viewed': root_file.viewed} + for key in keys: + try: + # override root file default if needed + if key in special_keys and special_keys[key] == 'observation': + result[key] = getattr(root_file.obsnum, key) + elif key in special_keys and special_keys[key] == 'proposal': + result[key] = getattr(root_file.obsnum.proposal, key) + else: + # try the root file table + result[key] = getattr(root_file, key) + except AttributeError: + try: + # try the observation table + result[key] = getattr(root_file.obsnum, key) + except AttributeError: + try: + # try the proposal table + result[key] = getattr(root_file.obsnum.proposal, key) + except AttributeError: + result[key] = '' + viewed.append(result) + return viewed + + def get_preview_images_by_proposal(proposal): """Return a list of preview images available in the filesystem for the given ``proposal``. diff --git a/jwql/website/apps/jwql/templates/api_landing.html b/jwql/website/apps/jwql/templates/api_landing.html index f803cec4f..bfb6b27bb 100644 --- a/jwql/website/apps/jwql/templates/api_landing.html +++ b/jwql/website/apps/jwql/templates/api_landing.html @@ -51,9 +51,11 @@

List of Available Services

  • Filenames by Rootname (https://jwql.stsci.edu/api/<rootname>/filenames/)
  • Preview Images by Rootname (https://jwql.stsci.edu/api/<rootname>/preview_images/)
  • Thumbnails by Rootname (https://jwql.stsci.edu/api/<rootname>/thumbnails/)
    • +
  • Viewed Status by Instrument (https://jwql.stsci.edu/api/<instrument>/viewed/)
    • +

    Where <instrument>, <rootname>, <proposal> are the values for the instrument name, rootname and proposal ID respectivly

    -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/jwql/website/apps/jwql/urls.py b/jwql/website/apps/jwql/urls.py index 739e60f17..2c6dad092 100644 --- a/jwql/website/apps/jwql/urls.py +++ b/jwql/website/apps/jwql/urls.py @@ -105,11 +105,20 @@ # REST API views path('api/proposals/', api_views.all_proposals, name='all_proposals'), - re_path(r'^api/(?P({}))/proposals/$'.format(instruments), api_views.instrument_proposals, name='instrument_proposals'), - re_path(r'^api/(?P[\d]{1,5})/filenames/$', api_views.filenames_by_proposal, name='filenames_by_proposal'), - re_path(r'^api/(?P[\d]{1,5})/preview_images/$', api_views.preview_images_by_proposal, name='preview_images_by_proposal'), - re_path(r'^api/(?P[\d]{1,5})/thumbnails/$', api_views.thumbnails_by_proposal, name='preview_images_by_proposal'), - re_path(r'^api/(?P[\w]+)/filenames/$', api_views.filenames_by_rootname, name='filenames_by_rootname'), - re_path(r'^api/(?P[\w]+)/preview_images/$', api_views.preview_images_by_rootname, name='preview_images_by_rootname'), - re_path(r'^api/(?P[\w]+)/thumbnails/$', api_views.thumbnail_by_rootname, name='thumbnail_by_rootname'), + re_path(r'^api/(?P({}))/proposals/$'.format(instruments), + api_views.instrument_proposals, name='instrument_proposals'), + re_path(r'^api/(?P[\d]{1,5})/filenames/$', + api_views.filenames_by_proposal, name='filenames_by_proposal'), + re_path(r'^api/(?P[\d]{1,5})/preview_images/$', + api_views.preview_images_by_proposal, name='preview_images_by_proposal'), + re_path(r'^api/(?P[\d]{1,5})/thumbnails/$', + api_views.thumbnails_by_proposal, name='preview_images_by_proposal'), + re_path(r'^api/(?P[\w]+)/filenames/$', + api_views.filenames_by_rootname, name='filenames_by_rootname'), + re_path(r'^api/(?P[\w]+)/preview_images/$', + api_views.preview_images_by_rootname, name='preview_images_by_rootname'), + re_path(r'^api/(?P[\w]+)/thumbnails/$', + api_views.thumbnail_by_rootname, name='thumbnail_by_rootname'), + re_path(r'^api/(?P({}))/viewed/$'.format(instruments), + api_views.instrument_viewed, name='instrument_viewed'), ] From 2b64b5e3fdf4867688e96fd099258ff0d28e037c Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Tue, 14 Feb 2023 09:03:50 -0500 Subject: [PATCH 050/449] Fixing making sure that all of the jump file outputs are actually sent --- .../common_monitors/bad_pixel_monitor.py | 14 +++++++++++- jwql/shared_tasks/run_pipeline.py | 1 + jwql/shared_tasks/shared_tasks.py | 22 +++++++++---------- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py index 06197c6c5..3d48f1fcb 100755 --- a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py +++ b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py @@ -943,11 +943,23 @@ def process(self, illuminated_raw_files, illuminated_slope_files, flat_file_coun if hasattr(self, 'nints') and self.nints > 1: local_ramp_file = local_ramp_file.replace("0_ramp_fit", "1_ramp_fit") if not os.path.isfile(local_uncal_file.replace("uncal", "jump")): + logging.info("\t\t\tJump file not found") dark_jump_files[index] = None + else: + dark_jump_files[index] = local_uncal_file.replace("uncal", "jump") if not os.path.isfile(local_uncal_file.replace("uncal", "fitopt")): + logging.info("\t\t\tFitopt file not found") dark_fitopt_files[index] = None + else: + dark_fitopt_files[index] = local_uncal_file.replace("uncal", "fitopt") if not os.path.isfile(local_ramp_file): - dark_slope_files[index] = None + if os.path.isfile(local_uncal_file.replace("uncal", "rateints")): + dark_slope_files[index] = local_uncal_file.replace("uncal", "rateints") + else: + logging.info("\t\t\tRate file not found") + dark_slope_files[index] = None + else: + dark_slope_files[index] = local_ramp_file index += 1 index = 0 diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 8dc7ade34..851f1559d 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -215,6 +215,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T with open(status_file, "a+") as status_f: status_f.write("{}\n".format(jump_output)) status_f.write("{}\n".format(pipe_output)) + status_f.write("{}\n".format(pipe_output.replace("0_ramp", "1_ramp"))) status_f.write("{}\n".format(fitopt_output)) status_f.write("SUCCEEDED") # Done. diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index 3732b14cc..fc1003d71 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -406,20 +406,20 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save raise ValueError("Pipeline Failed") files = {"jump_output": None, "pipe_output": None, "fitopt_output": None} - for line in status[-4:-1]: + for line in status[-5:-1]: file = line.strip() logging.info("Copying output file {}".format(file)) if not os.path.isfile(os.path.join(cal_dir, file)): - logging.error("ERROR: {} not found".format(file)) - raise FileNotFoundError(file) - copy_files([os.path.join(cal_dir, file)], output_dir) - set_permissions(os.path.join(output_dir, file)) - if "jump" in file: - files["jump_output"] = os.path.join(output_dir, file) - if "ramp" in file: - files["pipe_output"] = os.path.join(output_dir, file) - if "fitopt" in file: - files["fitopt_output"] = os.path.join(output_dir, file) + logging.error("WARNING: {} not found".format(file)) + else: + copy_files([os.path.join(cal_dir, file)], output_dir) + set_permissions(os.path.join(output_dir, file)) + if "jump" in file: + files["jump_output"] = os.path.join(output_dir, file) + if "ramp" in file: + files["pipe_output"] = os.path.join(output_dir, file) + if "fitopt" in file: + files["fitopt_output"] = os.path.join(output_dir, file) logging.info("Removing local files.") files_to_remove = glob(os.path.join(cal_dir, short_name+"*")) From 14784be234e5153c229fd48a5edbbf09b276a9e9 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Tue, 14 Feb 2023 09:22:52 -0500 Subject: [PATCH 051/449] Updating dictionary key passing --- jwql/website/apps/jwql/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 218a86d9f..cb9a9c66c 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -399,7 +399,7 @@ def archived_proposals_ajax(request, inst): thumb_obs_time.append(max(proposal_obs_times)) # Add category type to list based on proposal number - cat_types.append(proposals_by_category[proposal_num]) + cat_types.append(proposals_by_category[int(proposal_num)]) thumbnails_dict['proposals'] = proposal_nums thumbnails_dict['thumbnail_paths'] = thumbnail_paths From 72f8b3ef282c244acb08ba06ad8bdbf5f48bc93e Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Tue, 14 Feb 2023 09:53:33 -0500 Subject: [PATCH 052/449] Fix variable name typo --- jwql/website/apps/jwql/static/js/jwql.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index 04c9388ef..f388f32f8 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -627,7 +627,7 @@ function update_archive_page(inst, base_url) { cat_type = data.thumbnails.cat_type[i]; // Build div content - content = '
    '; + content = '
    '; content += ''; From 054b62d70a3dbd61ae02820cf40d18bc55188bad Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Tue, 14 Feb 2023 09:59:38 -0500 Subject: [PATCH 053/449] Fixing other typos --- jwql/website/apps/jwql/static/js/jwql.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index f388f32f8..0eefcd628 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -624,10 +624,10 @@ function update_archive_page(inst, base_url) { viewed = data.thumbnails.viewed[i]; exp_types = data.thumbnails.exp_types[i]; obs_time = data.thumbnails.obs_time[i]; - cat_type = data.thumbnails.cat_type[i]; + cat_type = data.thumbnails.cat_types[i]; // Build div content - content = '
    '; + content = '
    '; content += ''; From f0cb9b06203cbbce93dd10ef659ba737f0e8064a Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Tue, 14 Feb 2023 10:07:08 -0500 Subject: [PATCH 054/449] Updating dropdown filter name in js --- jwql/website/apps/jwql/static/js/jwql.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index 0eefcd628..b70544eb2 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -627,7 +627,7 @@ function update_archive_page(inst, base_url) { cat_type = data.thumbnails.cat_types[i]; // Build div content - content = '
    '; + content = '
    '; content += ''; From cf8f737d25cebb02cc248a5511df5767057c78bd Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Tue, 14 Feb 2023 11:16:17 -0500 Subject: [PATCH 055/449] Add new/viewed/all looks API options --- jwql/tests/test_api_views.py | 12 +++- jwql/tests/test_data_containers.py | 14 ++-- jwql/website/apps/jwql/api_views.py | 65 +++++++++++++++++-- jwql/website/apps/jwql/data_containers.py | 49 +++++++------- .../apps/jwql/templates/api_landing.html | 4 +- jwql/website/apps/jwql/urls.py | 4 ++ 6 files changed, 108 insertions(+), 40 deletions(-) diff --git a/jwql/tests/test_api_views.py b/jwql/tests/test_api_views.py index 964a3333f..40567a64d 100644 --- a/jwql/tests/test_api_views.py +++ b/jwql/tests/test_api_views.py @@ -7,6 +7,7 @@ - Matthew Bourque - Bryan Hilbert + - Melanie Clarke Use --- @@ -40,7 +41,9 @@ # Instrument-specific URLs for instrument in JWST_INSTRUMENT_NAMES: urls.append('api/{}/proposals/'.format(instrument)) # instrument_proposals + urls.append('api/{}/looks/'.format(instrument)) # instrument_looks urls.append('api/{}/viewed/'.format(instrument)) # instrument_viewed + urls.append('api/{}/new/'.format(instrument)) # instrument_new # Proposal-specific URLs proposals = ['2640', '02733', '1541', '02589'] @@ -90,7 +93,12 @@ def test_api_views(url): try: data = json.loads(url.read().decode()) - assert len(data[data_type]) > 0 - except (http.client.IncompleteRead) as e: + + # viewed data depends on local database contents + # so may return an empty result + if data_type != 'viewed': + assert len(data[data_type]) > 0 + + except http.client.IncompleteRead as e: data = e.partial assert len(data) > 0 diff --git a/jwql/tests/test_data_containers.py b/jwql/tests/test_data_containers.py index e53cab5dc..eb9bece3b 100644 --- a/jwql/tests/test_data_containers.py +++ b/jwql/tests/test_data_containers.py @@ -174,13 +174,13 @@ def test_get_instrument_proposals(): @pytest.mark.parametrize('keys', [None, [], ['proposal', 'obsnum', 'other', 'prop_id', 'obsstart']]) -def test_get_instrument_viewed(keys): - """Tests the ``get_instrument_viewed`` function.""" +def test_get_instrument_looks(keys): + """Tests the ``get_instrument_looks`` function.""" - viewed = data_containers.get_instrument_viewed('nirspec', keys=keys) - assert isinstance(viewed, list) - assert len(viewed) > 0 - first_file = viewed[0] + looks = data_containers.get_instrument_looks('nirspec', keys=keys) + assert isinstance(looks, list) + assert len(looks) > 0 + first_file = looks[0] assert first_file['root_name'] != '' assert isinstance(first_file['viewed'], bool) if keys is not None: @@ -188,7 +188,7 @@ def test_get_instrument_viewed(keys): for key in keys: assert key in first_file else: - # only root name and viewed by default + # only root name and looks by default assert len(first_file) == 2 diff --git a/jwql/website/apps/jwql/api_views.py b/jwql/website/apps/jwql/api_views.py index 719c624f7..4a7fa7ac1 100644 --- a/jwql/website/apps/jwql/api_views.py +++ b/jwql/website/apps/jwql/api_views.py @@ -50,7 +50,7 @@ from .data_containers import get_filenames_by_proposal from .data_containers import get_filenames_by_rootname from .data_containers import get_instrument_proposals -from .data_containers import get_instrument_viewed +from .data_containers import get_instrument_looks from .data_containers import get_preview_images_by_proposal from .data_containers import get_preview_images_by_rootname from .data_containers import get_thumbnails_by_proposal @@ -138,8 +138,8 @@ def instrument_proposals(request, inst): return JsonResponse({'proposals': proposals}, json_dumps_params={'indent': 2}) -def instrument_viewed(request, inst): - """Return a table of viewed information for the given instrument. +def instrument_looks(request, inst, viewed=None): + """Return a table of looks information for the given instrument. 'Viewed' indicates whether an observation is new or has been reviewed for QA. In addition to 'filename', and 'viewed', observation @@ -152,6 +152,10 @@ def instrument_viewed(request, inst): Incoming request from the webpage. inst : str The JWST instrument of interest. + viewed : bool, optional + If set to None, all viewed values are returned. If set to + True, only viewed data is returned. If set to False, only + new data is returned. Returns ------- @@ -165,12 +169,61 @@ def instrument_viewed(request, inst): 'exptypes', 'obsstart', 'obsend'] full_keys = ['root_name', 'viewed'] + optional_keys - # get all observation viewed status from file info model + # get all observation looks from file info model # and join with observation descriptors - viewed = get_instrument_viewed(inst, keys=optional_keys) + looks = get_instrument_looks(inst, keys=optional_keys, viewed=viewed) + + # return results by api key + if viewed is None: + key = 'looks' + elif viewed: + key = 'viewed' + else: + key = 'new' + return JsonResponse({'instrument': inst, 'keys': full_keys, - 'viewed': viewed}, json_dumps_params={'indent': 2}) + key: looks}, json_dumps_params={'indent': 2}) + + +def instrument_viewed(request, inst): + """Return a table of information on viewed data for the given instrument. + + Calls `instrument_looks` with viewed=True. + + Parameters + ---------- + request : HttpRequest object + Incoming request from the webpage. + inst : str + The JWST instrument of interest. + + Returns + ------- + JsonResponse object + Outgoing response sent to the webpage + """ + return instrument_looks(request, inst, viewed=True) + + +def instrument_new(request, inst): + """Return a table of information on new data for the given instrument. + + Calls `instrument_looks` with viewed=False. + + Parameters + ---------- + request : HttpRequest object + Incoming request from the webpage. + inst : str + The JWST instrument of interest. + + Returns + ------- + JsonResponse object + Outgoing response sent to the webpage + """ + return instrument_looks(request, inst, viewed=False) def preview_images_by_proposal(request, proposal): diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 1928729a7..e110745d1 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -924,8 +924,8 @@ def get_instrument_proposals(instrument): return inst_proposals -def get_instrument_viewed(instrument, keys=None): - """Return a table of viewed information for the given instrument. +def get_instrument_looks(instrument, keys=None, viewed=None): + """Return a table of looks information for the given instrument. Parameters ---------- @@ -933,26 +933,30 @@ def get_instrument_viewed(instrument, keys=None): Name of the JWST instrument. keys : list of str, optional Additional FITS key names for information to return. + viewed : bool, optional + If set to None, all viewed values are returned. If set to + True, only viewed data is returned. If set to False, only + new data is returned. Returns ------- - viewed : list - List of viewed information by observation for the given instrument. + looks : list + List of looks information by observation for the given instrument. """ # standardize input inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[instrument] if keys is None: keys = [] - # configure some special keys to avoid table name conflicts - special_keys = {'proposal': 'root', - 'obsnum': 'observation', - 'prop_id': 'proposal'} - # get files by instrument from local model - root_file_info = RootFileInfo.objects.filter(instrument=inst) + if viewed is None: + root_file_info = RootFileInfo.objects.filter(instrument=inst) + elif viewed: + root_file_info = RootFileInfo.objects.filter(instrument=inst, viewed=True) + else: + root_file_info = RootFileInfo.objects.filter(instrument=inst, viewed=False) - viewed = [] + looks = [] for root_file in root_file_info: # for now, report viewed by root name only. # if specific files are needed, use get_filesystem_files @@ -960,26 +964,23 @@ def get_instrument_viewed(instrument, keys=None): 'viewed': root_file.viewed} for key in keys: try: - # override root file default if needed - if key in special_keys and special_keys[key] == 'observation': - result[key] = getattr(root_file.obsnum, key) - elif key in special_keys and special_keys[key] == 'proposal': - result[key] = getattr(root_file.obsnum.proposal, key) - else: - # try the root file table - result[key] = getattr(root_file, key) + # try the root file table + value = getattr(root_file, key) except AttributeError: try: # try the observation table - result[key] = getattr(root_file.obsnum, key) + value = getattr(root_file.obsnum, key) except AttributeError: try: # try the proposal table - result[key] = getattr(root_file.obsnum.proposal, key) + value = getattr(root_file.obsnum.proposal, key) except AttributeError: - result[key] = '' - viewed.append(result) - return viewed + value = '' + if type(value) not in [str, float, int]: + value = str(value) + result[key] = value + looks.append(result) + return looks def get_preview_images_by_proposal(proposal): diff --git a/jwql/website/apps/jwql/templates/api_landing.html b/jwql/website/apps/jwql/templates/api_landing.html index bfb6b27bb..fdd992b49 100644 --- a/jwql/website/apps/jwql/templates/api_landing.html +++ b/jwql/website/apps/jwql/templates/api_landing.html @@ -51,7 +51,9 @@

    List of Available Services

  • Filenames by Rootname (https://jwql.stsci.edu/api/<rootname>/filenames/)
  • Preview Images by Rootname (https://jwql.stsci.edu/api/<rootname>/preview_images/)
  • Thumbnails by Rootname (https://jwql.stsci.edu/api/<rootname>/thumbnails/)
    • -
  • Viewed Status by Instrument (https://jwql.stsci.edu/api/<instrument>/viewed/)
    • +
  • Look Status by Instrument (https://jwql.stsci.edu/api/<instrument>/looks/)
    • +
  • Viewed Data by Instrument (https://jwql.stsci.edu/api/<instrument>/viewed/)
    • +
  • New Data by Instrument (https://jwql.stsci.edu/api/<instrument>/new/)

    • Where <instrument>, <rootname>, <proposal> are the values for the instrument name, rootname and proposal ID respectivly

    diff --git a/jwql/website/apps/jwql/urls.py b/jwql/website/apps/jwql/urls.py index 2c6dad092..569afe764 100644 --- a/jwql/website/apps/jwql/urls.py +++ b/jwql/website/apps/jwql/urls.py @@ -119,6 +119,10 @@ api_views.preview_images_by_rootname, name='preview_images_by_rootname'), re_path(r'^api/(?P[\w]+)/thumbnails/$', api_views.thumbnail_by_rootname, name='thumbnail_by_rootname'), + re_path(r'^api/(?P({}))/looks/$'.format(instruments), + api_views.instrument_looks, name='instrument_looks'), re_path(r'^api/(?P({}))/viewed/$'.format(instruments), api_views.instrument_viewed, name='instrument_viewed'), + re_path(r'^api/(?P({}))/new/$'.format(instruments), + api_views.instrument_new, name='instrument_new'), ] From dcbf48cc48d6b451cd5d582d4c437a5c6e8eac5b Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 14 Feb 2023 14:16:17 -0500 Subject: [PATCH 056/449] Add specifics about plots to the documentation --- .../common_monitors/dark_monitor.py | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index 2377d5908..fa4252d81 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -17,12 +17,12 @@ is entered into the ``DarkCurrent`` database table. The mean slope image is then normalized by an existing baseline slope -image. New hot pixels are identified as those with normalized signal -rates above a ``hot_threshold`` value. Similarly, pixels with -normalized signal rates below a ``dead_threshold`` are flagged as new -dead pixels. +image, from the previous run of the monitor. New hot pixels are identified +as those with normalized signal rates above a ``hot_threshold`` value. +Similarly, pixels with normalized signal rates below a ``dead_threshold`` +are flagged as new dead pixels. -The standard deviation slope image is normalized by a baseline +The standard deviation slope image is also normalized by a baseline (historical) standard deviation image. Pixels with normalized values above a noise threshold are flagged as newly noisy pixels. @@ -38,6 +38,26 @@ The histogram itself as well as the best-fit Gaussian and double Gaussian parameters are saved to the DarkDarkCurrent database table. +Currently, there are three outputs from the dark monitor that are shown +in the JWQL web app. First, the dark current histogram is plotted, along +with a corresponding cumulative distribution function (CDF). The Gaussian +fits are not currently shown. + +Secondly, a trending plot of the mean dark current versus time is shown, +where the mean value is the sigma-clipped mean across the detector in +the mean slope image. Error bars on the plot show the sigma-clipped +standard deviation across the detector. + +Finally, the mean slope image is shown. Any new potential hot, dead, and +noisy pixels that were identified are also shown on the mean slope image, +in order to give an idea of where these pixels are located on the detector. +To keep the image from becoming too busy, this is only done if the number +of potential new bad pixels is under 1000. If more pixels that this are +identified, that number is reported in the plot, but the pixels are not +marked on the image. + + + Author ------ From 94838f5e0f51860e4a47d297b2289c7fb4f623bc Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Wed, 15 Feb 2023 09:40:20 -0500 Subject: [PATCH 057/449] Adding test for get_proposal_by_category --- jwql/tests/test_data_containers.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/jwql/tests/test_data_containers.py b/jwql/tests/test_data_containers.py index 0807e78d8..3fd014145 100644 --- a/jwql/tests/test_data_containers.py +++ b/jwql/tests/test_data_containers.py @@ -132,6 +132,13 @@ def test_get_preview_images_by_rootname(): assert isinstance(preview_images, list) assert len(preview_images) > 0 +@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') +def test_get_proposals_by_category(): + """Tests the ``get_proposals_by_category`` function.""" + + proposals_by_category = data_containers.get_proposals_by_category('fgs') + assert isinstance(proposals_by_category, dict) + assert len(proposals_by_category) > 0 @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') def test_get_proposal_info(): From d2ea9a94dd367a00d1d32983748647feb14192ac Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Wed, 15 Feb 2023 11:30:58 -0500 Subject: [PATCH 058/449] Update Models for Filter Fields --- jwql/website/apps/jwql/admin.py | 16 +++++++--- jwql/website/apps/jwql/models.py | 53 ++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 4 deletions(-) diff --git a/jwql/website/apps/jwql/admin.py b/jwql/website/apps/jwql/admin.py index 0020f83d6..caa7bc1f5 100644 --- a/jwql/website/apps/jwql/admin.py +++ b/jwql/website/apps/jwql/admin.py @@ -1,7 +1,5 @@ """Customizes the ``jwql`` web app administrative page. -** CURRENTLY NOT IN USE ** - Used to customize django's admin interface, and how the data contained in specific models is portrayed. @@ -21,7 +19,7 @@ from django.contrib import admin -from .models import Archive, Observation, Proposal, RootFileInfo +from .models import Archive, Observation, Proposal, RootFileInfo, Anomalies @admin.register(Archive) @@ -43,5 +41,15 @@ class ObservationAdmin(admin.ModelAdmin): @admin.register(RootFileInfo) class RootFileInfoAdmin(admin.ModelAdmin): - list_display = ('root_name', 'obsnum', 'proposal', 'instrument', 'viewed') + list_display = ('root_name', 'obsnum', 'proposal', 'instrument', 'viewed', 'filter', 'aperature', 'detector', 'read_patt_num', 'read_patt', 'grating', 'subarray', 'pupil') list_filter = ('viewed', 'instrument', 'proposal', 'obsnum') + + +@admin.register(Anomalies) +class AnomaliesAdmin(admin.ModelAdmin): + list_display = ('root_file_info', 'cosmic_ray_shower', 'diffraction_spike', 'excessive_saturation', 'guidestar_failure', 'persistence', 'crosstalk', 'data_transfer_error', + 'ghost', 'snowball', 'column_pull_up', 'column_pull_down', 'dominant_msa_leakage', 'dragons_breath', 'mrs_glow', 'mrs_zipper', 'internal_reflection', 'optical_short', 'row_pull_up', + 'row_pull_down', 'lrs_contamination', 'tree_rings', 'scattered_light', 'claws', 'wisps', 'tilt_event', 'light_saber', 'other') + list_filter = ('cosmic_ray_shower', 'diffraction_spike', 'excessive_saturation', 'guidestar_failure', 'persistence', 'crosstalk', 'data_transfer_error', + 'ghost', 'snowball', 'column_pull_up', 'column_pull_down', 'dominant_msa_leakage', 'dragons_breath', 'mrs_glow', 'mrs_zipper', 'internal_reflection', 'optical_short', 'row_pull_up', + 'row_pull_down', 'lrs_contamination', 'tree_rings', 'scattered_light', 'claws', 'wisps', 'tilt_event', 'light_saber', 'other', 'root_file_info') diff --git a/jwql/website/apps/jwql/models.py b/jwql/website/apps/jwql/models.py index 0ba91f75d..d1aa6a4d1 100644 --- a/jwql/website/apps/jwql/models.py +++ b/jwql/website/apps/jwql/models.py @@ -60,6 +60,7 @@ class Proposal(models.Model): prop_id = models.CharField(max_length=5, help_text="5-digit proposal ID string") thumbnail_path = models.CharField(max_length=100, help_text='Path to the proposal thumbnail', default='') archive = models.ForeignKey(Archive, blank=False, null=False, on_delete=models.CASCADE) + cat_type = models.CharField(max_length=10, help_text="Category Type", default='') # Metadata class Meta: @@ -100,6 +101,14 @@ class RootFileInfo(models.Model): proposal = models.CharField(max_length=5, help_text="5-digit proposal ID string") root_name = models.TextField(primary_key=True, max_length=300) viewed = models.BooleanField(default=False) + filter = models.CharField(max_length=7, help_text="Instrument name", default='') + aperature = models.CharField(max_length=40, help_text="Aperature", default='') + detector = models.CharField(max_length=40, help_text="Detector", default='') + read_patt_num = models.IntegerField(help_text='Read Pattern Number', default=0) + read_patt = models.CharField(max_length=40, help_text="Read Pattern", default='') + grating = models.CharField(max_length=40, help_text="Grating", default='') + subarray = models.CharField(max_length=40, help_text="Subarray", default='') + pupil = models.CharField(max_length=40, help_text="Pupil", default='') # Metadata class Meta: @@ -108,3 +117,47 @@ class Meta: def __str__(self): """String for representing the RootFileInfo object (in Admin site etc.).""" return self.root_name + + +class Anomalies(models.Model): + """ All Potentially Anomalies that can be associated with a RootFile Info """ + # Note: Using one to one relationship. Cann access Anomalies by 'rootfileinfo_object.anomalies' + root_file_info = models.OneToOneField( + RootFileInfo, + on_delete=models.CASCADE, + primary_key=True, + ) + cosmic_ray_shower = models.BooleanField(default=False) + diffraction_spike = models.BooleanField(default=False) + excessive_saturation = models.BooleanField(default=False) + guidestar_failure = models.BooleanField(default=False) + persistence = models.BooleanField(default=False) + crosstalk = models.BooleanField(default=False) + data_transfer_error = models.BooleanField(default=False) + ghost = models.BooleanField(default=False) + snowball = models.BooleanField(default=False) + column_pull_up = models.BooleanField(default=False) + column_pull_down = models.BooleanField(default=False) + dominant_msa_leakage = models.BooleanField(default=False) + dragons_breath = models.BooleanField(default=False) + mrs_glow = models.BooleanField(default=False) + mrs_zipper = models.BooleanField(default=False) + internal_reflection = models.BooleanField(default=False) + optical_short = models.BooleanField(default=False) + row_pull_up = models.BooleanField(default=False) + row_pull_down = models.BooleanField(default=False) + lrs_contamination = models.BooleanField(default=False) + tree_rings = models.BooleanField(default=False) + scattered_light = models.BooleanField(default=False) + claws = models.BooleanField(default=False) + wisps = models.BooleanField(default=False) + tilt_event = models.BooleanField(default=False) + light_saber = models.BooleanField(default=False) + other = models.BooleanField(default=False) + + class Meta: + ordering = ['-root_file_info'] + + def __str__(self): + """Container for all anomalies associated with each RootFileInfo object """ + return self.root_file_info.root_name From a56824fa64c82d53b83863209599cef08379e31b Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Wed, 15 Feb 2023 11:43:31 -0500 Subject: [PATCH 059/449] Add CSV report download to archive page --- jwql/tests/test_api_views.py | 4 +- jwql/website/apps/jwql/api_views.py | 71 +++++-------------- jwql/website/apps/jwql/data_containers.py | 2 +- jwql/website/apps/jwql/static/js/jwql.js | 27 +++++++ jwql/website/apps/jwql/templates/archive.html | 7 +- jwql/website/apps/jwql/urls.py | 12 ++-- jwql/website/apps/jwql/views.py | 51 +++++++++++++ 7 files changed, 111 insertions(+), 63 deletions(-) diff --git a/jwql/tests/test_api_views.py b/jwql/tests/test_api_views.py index 40567a64d..e3a2d1ca1 100644 --- a/jwql/tests/test_api_views.py +++ b/jwql/tests/test_api_views.py @@ -42,8 +42,8 @@ for instrument in JWST_INSTRUMENT_NAMES: urls.append('api/{}/proposals/'.format(instrument)) # instrument_proposals urls.append('api/{}/looks/'.format(instrument)) # instrument_looks - urls.append('api/{}/viewed/'.format(instrument)) # instrument_viewed - urls.append('api/{}/new/'.format(instrument)) # instrument_new + urls.append('api/{}/looks/viewed/'.format(instrument)) # instrument_viewed + urls.append('api/{}/looks/new/'.format(instrument)) # instrument_new # Proposal-specific URLs proposals = ['2640', '02733', '1541', '02589'] diff --git a/jwql/website/apps/jwql/api_views.py b/jwql/website/apps/jwql/api_views.py index 4a7fa7ac1..76d9e6d9d 100644 --- a/jwql/website/apps/jwql/api_views.py +++ b/jwql/website/apps/jwql/api_views.py @@ -27,6 +27,7 @@ - Matthew Bourque - Teagan King + - Melanie Clarke Use --- @@ -138,7 +139,7 @@ def instrument_proposals(request, inst): return JsonResponse({'proposals': proposals}, json_dumps_params={'indent': 2}) -def instrument_looks(request, inst, viewed=None): +def instrument_looks(request, inst, status=None): """Return a table of looks information for the given instrument. 'Viewed' indicates whether an observation is new or has been reviewed @@ -152,15 +153,15 @@ def instrument_looks(request, inst, viewed=None): Incoming request from the webpage. inst : str The JWST instrument of interest. - viewed : bool, optional + status : str, optional If set to None, all viewed values are returned. If set to - True, only viewed data is returned. If set to False, only + 'viewed', only viewed data is returned. If set to 'new', only new data is returned. Returns ------- - JsonResponse object - Outgoing response sent to the webpage + JsonResponse + Outgoing response sent to the webpage, depending on return_type. """ # TODO: define more useful keys by instrument in config # currently, optional keys are just the values available @@ -171,59 +172,19 @@ def instrument_looks(request, inst, viewed=None): # get all observation looks from file info model # and join with observation descriptors + viewed = str(status) == 'viewed' looks = get_instrument_looks(inst, keys=optional_keys, viewed=viewed) # return results by api key - if viewed is None: - key = 'looks' - elif viewed: - key = 'viewed' - else: - key = 'new' - - return JsonResponse({'instrument': inst, - 'keys': full_keys, - key: looks}, json_dumps_params={'indent': 2}) - - -def instrument_viewed(request, inst): - """Return a table of information on viewed data for the given instrument. - - Calls `instrument_looks` with viewed=True. - - Parameters - ---------- - request : HttpRequest object - Incoming request from the webpage. - inst : str - The JWST instrument of interest. - - Returns - ------- - JsonResponse object - Outgoing response sent to the webpage - """ - return instrument_looks(request, inst, viewed=True) - - -def instrument_new(request, inst): - """Return a table of information on new data for the given instrument. - - Calls `instrument_looks` with viewed=False. - - Parameters - ---------- - request : HttpRequest object - Incoming request from the webpage. - inst : str - The JWST instrument of interest. - - Returns - ------- - JsonResponse object - Outgoing response sent to the webpage - """ - return instrument_looks(request, inst, viewed=False) + if status is None: + status = 'looks' + + response = JsonResponse({'instrument': inst, + 'keys': full_keys, + 'type': status, + status: looks}, + json_dumps_params={'indent': 2}) + return response def preview_images_by_proposal(request, proposal): diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index e110745d1..efd2707f8 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -944,7 +944,7 @@ def get_instrument_looks(instrument, keys=None, viewed=None): List of looks information by observation for the given instrument. """ # standardize input - inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[instrument] + inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[instrument.lower()] if keys is None: keys = [] diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index 3440a0eb7..435a20a55 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -597,6 +597,33 @@ function toggle_viewed(file_root, base_url) { }); } +/** + * Download filtered data report + * @param {String} inst - The instrument in use + * @param {String} base_url - The base URL for gathering data from the AJAX view. + */ +function download_report(inst, base_url) { + var elem = document.getElementById('download_report_button'); + elem.disabled = true; + + // todo: include all filters + // current look filter + var look_status = document.getElementById('look_dropdownMenuButton') + if (look_status != null) { + look = look_status.innerText.toLowerCase(); + } else { + look = 'all'; + } + if (look.includes('all')) { + report_url = '/' + inst + '/report/'; + } else { + report_url = '/' + inst + '/report/' + look + '/'; + } + + // redirect to download content + window.location = base_url + report_url; + elem.disabled = false; +} /** * Updates various compnents on the archive page diff --git a/jwql/website/apps/jwql/templates/archive.html b/jwql/website/apps/jwql/templates/archive.html index c6ff05f5b..bf8164dc7 100644 --- a/jwql/website/apps/jwql/templates/archive.html +++ b/jwql/website/apps/jwql/templates/archive.html @@ -45,6 +45,11 @@

    Archived {{ inst }} Images

    + +
    + Download as:
    + +
    @@ -73,4 +78,4 @@

    Archived {{ inst }} Images

    -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/jwql/website/apps/jwql/urls.py b/jwql/website/apps/jwql/urls.py index 569afe764..43fc5de58 100644 --- a/jwql/website/apps/jwql/urls.py +++ b/jwql/website/apps/jwql/urls.py @@ -85,6 +85,12 @@ re_path(r'^(?P({}))/archive/$'.format(instruments), views.archived_proposals, name='archive'), re_path(r'^(?P({}))/archive_date_range/$'.format(instruments), views.archive_date_range, name='archive_date_range'), re_path(r'^(?P({}))/unlooked/$'.format(instruments), views.unlooked_images, name='unlooked'), + + re_path(r'^(?P({}))/report/$'.format(instruments), + views.download_report, name='download_report'), + re_path(r'^(?P({}))/report/(?P(viewed|new))/$'.format(instruments), + views.download_report, name='download_report_by_status'), + re_path(r'^(?P({}))/(?P[\w-]+)/$'.format(instruments), views.view_image, name='view_image'), re_path(r'^(?P({}))/(?P.+)_(?P.+)/explore_image/'.format(instruments), views.explore_image, name='explore_image'), re_path(r'^(?P({}))/(?P.+)_(?P.+)/header/'.format(instruments), views.view_header, name='view_header'), @@ -121,8 +127,6 @@ api_views.thumbnail_by_rootname, name='thumbnail_by_rootname'), re_path(r'^api/(?P({}))/looks/$'.format(instruments), api_views.instrument_looks, name='instrument_looks'), - re_path(r'^api/(?P({}))/viewed/$'.format(instruments), - api_views.instrument_viewed, name='instrument_viewed'), - re_path(r'^api/(?P({}))/new/$'.format(instruments), - api_views.instrument_new, name='instrument_new'), + re_path(r'^api/(?P({}))/looks/(?P(viewed|new))/$'.format(instruments), + api_views.instrument_looks, name='instrument_looks_by_status'), ] diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 8781b6f4d..c01b24d63 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -44,6 +44,7 @@ from collections import defaultdict from copy import deepcopy import csv +import datetime import glob import logging import os @@ -70,6 +71,7 @@ from .data_containers import get_explorer_extension_names from .data_containers import get_header_info from .data_containers import get_image_info +from .data_containers import get_instrument_looks from .data_containers import get_thumbnails_all_instruments from .data_containers import random_404_page from .data_containers import text_scrape @@ -565,6 +567,55 @@ def dashboard(request): return render(request, template, context) +def download_report(request, inst, status='all'): + """Download data report by look status. + + Parameters + ---------- + request : HttpRequest object + Incoming request from the webpage. + inst : str + The JWST instrument of interest. + status : str, optional + If set to None or 'all', all viewed values are returned. If set to + 'viewed', only viewed data is returned. If set to 'new', only + new data is returned. + + Returns + ------- + response : HttpResponse object + Outgoing response sent to the webpage + """ + # TODO: define more useful keys by instrument in config + # currently, optional keys are just the values available + # in local models + optional_keys = ['proposal', 'obsnum', 'number_of_files', + 'exptypes', 'obsstart', 'obsend'] + full_keys = ['root_name', 'viewed'] + optional_keys + + # get all observation looks from file info model + # and join with observation descriptors + if status == 'viewed': + viewed = True + elif status == 'new': + viewed = False + else: + viewed = None + looks = get_instrument_looks(inst, keys=optional_keys, viewed=viewed) + + today = datetime.datetime.now().strftime('%Y%m%d') + filename = f'{inst}_{status}_{today}.csv' + response = HttpResponse(content_type='text/csv') + response['Content-Disposition'] = f'attachment; filename="{filename}"' + + writer = csv.writer(response) + writer.writerow(full_keys) + for row in looks: + writer.writerow(row.values()) + + return response + + def engineering_database(request): """Generate the EDB page. From 8a6243e7648fdf1f3d8771dccd89641819532db0 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Wed, 15 Feb 2023 12:27:04 -0500 Subject: [PATCH 060/449] spelling --- jwql/website/apps/jwql/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/models.py b/jwql/website/apps/jwql/models.py index d1aa6a4d1..06d91b696 100644 --- a/jwql/website/apps/jwql/models.py +++ b/jwql/website/apps/jwql/models.py @@ -120,7 +120,7 @@ def __str__(self): class Anomalies(models.Model): - """ All Potentially Anomalies that can be associated with a RootFile Info """ + """ All Potential Anomalies that can be associated with a RootFileInfo """ # Note: Using one to one relationship. Cann access Anomalies by 'rootfileinfo_object.anomalies' root_file_info = models.OneToOneField( RootFileInfo, From 2aa27ad846c1d366a9e7d5d0156db0063942472d Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Wed, 15 Feb 2023 13:03:37 -0500 Subject: [PATCH 061/449] Test new/viewed looks --- jwql/tests/test_data_containers.py | 42 ++++++++++++++++++------------ 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/jwql/tests/test_data_containers.py b/jwql/tests/test_data_containers.py index eb9bece3b..0a5c8df3f 100644 --- a/jwql/tests/test_data_containers.py +++ b/jwql/tests/test_data_containers.py @@ -171,25 +171,35 @@ def test_get_instrument_proposals(): @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') -@pytest.mark.parametrize('keys', [None, [], - ['proposal', 'obsnum', 'other', - 'prop_id', 'obsstart']]) -def test_get_instrument_looks(keys): +@pytest.mark.parametrize('keys,viewed', + [(None, True), (None, False), (None, None), + ([], True), ([], False), ([], None), + (['proposal', 'obsnum', 'other', + 'prop_id', 'obsstart'], True), + (['proposal', 'obsnum', 'other', + 'prop_id', 'obsstart'], False), + (['proposal', 'obsnum', 'other', + 'prop_id', 'obsstart'], None)]) +def test_get_instrument_looks(keys, viewed): """Tests the ``get_instrument_looks`` function.""" - looks = data_containers.get_instrument_looks('nirspec', keys=keys) + looks = data_containers.get_instrument_looks( + 'nirspec', keys=keys, viewed=viewed) assert isinstance(looks, list) - assert len(looks) > 0 - first_file = looks[0] - assert first_file['root_name'] != '' - assert isinstance(first_file['viewed'], bool) - if keys is not None: - assert len(first_file) == 2 + len(keys) - for key in keys: - assert key in first_file - else: - # only root name and looks by default - assert len(first_file) == 2 + + # viewed depends on local database, so may or may not have results + if not viewed: + assert len(looks) > 0 + first_file = looks[0] + assert first_file['root_name'] != '' + assert isinstance(first_file['viewed'], bool) + if keys is not None: + assert len(first_file) == 2 + len(keys) + for key in keys: + assert key in first_file + else: + # only root name and looks by default + assert len(first_file) == 2 @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') From 87b9b9585734ca31e787ed5969948b9300efa46a Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Thu, 16 Feb 2023 09:03:28 -0500 Subject: [PATCH 062/449] Splitting the file's name rather than the file's name+path --- jwql/instrument_monitors/common_monitors/dark_monitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index 17160450b..28c525e81 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -312,7 +312,7 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no # Collect information about the file this image was compared against if baseline_file is not None: - base_parts = baseline_file.split('_') + base_parts = os.path.basename(baseline_file).split('_') # Get the starting and ending time from the filename. base_start = Time(float(base_parts[3]), format='mjd').tt.datetime From 5519fd6fa3dc7cf2dd9799af51bcd949abe74fa4 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Thu, 16 Feb 2023 11:28:15 -0500 Subject: [PATCH 063/449] Test fixes for floating point equality --- jwql/tests/test_edb.py | 4 +++- jwql/tests/test_edb_telemetry_monitor.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/jwql/tests/test_edb.py b/jwql/tests/test_edb.py index 010329950..a7da24be5 100644 --- a/jwql/tests/test_edb.py +++ b/jwql/tests/test_edb.py @@ -262,7 +262,9 @@ def test_multiplication(): 'AllPoints': 1}]} prod = mnemonic1 * mnemonic2 - assert all(prod.data["euvalues"].data == np.array([75.0, 75.0, 165.0, 171.0, 171.0, 171.0, 171.0, 26.333333333333336, 24.0])) + assert np.allclose(prod.data["euvalues"].data, + np.array([75.0, 75.0, 165.0, 171.0, 171.0, 171.0, + 171.0, 26.333333333333336, 24.0])) assert all(prod.data["dates"].data == mnemonic1.data["dates"][1:]) assert all(prod.blocks == [0, 2, 7, 9]) assert prod.info['unit'] == 'W' diff --git a/jwql/tests/test_edb_telemetry_monitor.py b/jwql/tests/test_edb_telemetry_monitor.py index 9430f812a..289b44a09 100644 --- a/jwql/tests/test_edb_telemetry_monitor.py +++ b/jwql/tests/test_edb_telemetry_monitor.py @@ -178,8 +178,8 @@ def test_find_all_changes(): inst.query_results[dependency[0]["name"]] = EdbMnemonic("CURRENT", start_time, end_time, current_data, meta, info) vals = inst.find_all_changes(temperature, dependency) - assert vals.mean == [359.07] - assert vals.median == [360.0] + assert np.isclose(vals.mean[0], 359.07) + assert np.isclose(vals.median[0], 360.0) assert np.isclose(vals.stdev[0], 6.9818407314976785) From c589c0d21b3aec352e80b22a32400faf7a69cb25 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Thu, 16 Feb 2023 12:27:11 -0500 Subject: [PATCH 064/449] Including backoff if unable to allocate memory --- jwql/shared_tasks/run_pipeline.py | 16 +++-- jwql/shared_tasks/shared_tasks.py | 112 ++++++++++++++++++++---------- 2 files changed, 86 insertions(+), 42 deletions(-) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 851f1559d..1b1e39583 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -33,7 +33,7 @@ from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path -def run_pipe(input_file, short_name, work_directory, instrument, outputs): +def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_cores='all'): """Run the steps of ``calwebb_detector1`` on the input file, saving the result of each step as a separate output file, then return the name-and-path of the file as reduced in the reduction directory. @@ -62,7 +62,7 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs): status_f.write("Running step {}\n".format(step_name)) kwargs = {} if step_name in ['jump', 'rate']: - kwargs = {'maximum_cores': 'all'} + kwargs = {'maximum_cores': max_cores} if steps[step_name]: output_file_name = short_name + "_{}.fits".format(step_name) output_file = os.path.join(work_directory, output_file_name) @@ -116,7 +116,7 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs): # Done. -def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=True, save_fitopt=True): +def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=True, save_fitopt=True, max_cores='all'): """Call ``calwebb_detector1`` on the provided file, running all steps up to the ``ramp_fit`` step, and save the result. Optionally run the ``ramp_fit`` step and save the resulting slope file as well. @@ -170,7 +170,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T model.jump.save_results = True model.jump.output_dir = work_directory - model.jump.maximum_cores = 'all' + model.jump.maximum_cores = max_cores jump_output = uncal_file.replace('uncal', 'jump') # Check to see if the jump version of the requested file is already @@ -179,7 +179,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T if ramp_fit: model.ramp_fit.save_results = True - model.ramp_fit.maximum_cores = 'all' + model.ramp_fit.maximum_cores = max_cores # model.save_results = True model.output_dir = work_directory # pipe_output = os.path.join(output_dir, input_file_only.replace('uncal', 'rate')) @@ -232,6 +232,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T pipe_help = 'Pipeline type to run (valid values are "jump" and "cal")' out_help = 'Comma-separated list of output extensions (for cal only, otherwise just "all")' name_help = 'Input file name with no path or extensions' + cores_help = 'Maximum cores to use (default "all")' parser = argparse.ArgumentParser(description='Run local calibration') parser.add_argument('pipe', metavar='PIPE', type=str, help=pipe_help) parser.add_argument('outputs', metavar='OUTPUTS', type=str, help=out_help) @@ -239,6 +240,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T parser.add_argument('instrument', metavar='INSTRUMENT', type=str, help=ins_help) parser.add_argument('input_file', metavar='FILE', type=str, help=file_help) parser.add_argument('short_name', metavar='NAME', type=str, help=name_help) + parser.add_argument('max_cores', metavar='CORES', type=str, help=cores_help, default='all') with open("/internal/data1/outputs/ops/calibrated_data/general_status.txt", "a+") as status_file: status_file.write("Created argument parser at {}\n".format(time.ctime())) @@ -281,12 +283,12 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T if pipe_type == 'jump': with open(status_file, 'a+') as out_file: out_file.write("Running jump pipeline.\n") - run_save_jump(input_file, short_name, working_path, instrument, ramp_fit=True, save_fitopt=True) + run_save_jump(input_file, short_name, working_path, instrument, ramp_fit=True, save_fitopt=True, max_cores=args.max_cores) elif pipe_type == 'cal': with open(status_file, 'a+') as out_file: out_file.write("Running cal pipeline.\n") outputs = outputs.split(",") - run_pipe(input_file, short_name, working_path, instrument, outputs) + run_pipe(input_file, short_name, working_path, instrument, outputs, max_cores=args.max_cores) except Exception as e: with open(status_file, 'a+') as out_file: out_file.write("Exception when starting pipeline.\n") diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index fc1003d71..bddcd18ff 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -224,6 +224,29 @@ def collect_after_task(**kwargs): gc.collect() +def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_file, cores='all'): + cmd = "{} {} {} '{}' {} {} {} {}" + cmd = cmd.format(name, cmd, outputs, cal_dir, ins, in_file, short_name, cores) + logging.info("Running {}".format(cmd)) + process = Popen(cmd, shell=True, executable="/bin/bash", stderr=PIPE) + with process.stderr: + log_subprocess_output(process.stderr) + result = process.wait() + logging.info("Subprocess result was {}".format(result)) + + if not os.path.isfile(res_file): + logging.error("Result file was not created.") + with open(os.path.join(cal_dir, "general_status.txt")) as status_file: + status = status_file.readlines() + for line in status: + logging.error(line.strip()) + return status + + with open(res, 'r') as inf: + status = inf.readlines() + return status + + @celery_app.task(name='jwql.shared_tasks.shared_tasks.run_calwebb_detector1') def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, step_args={}): """Run the steps of ``calwebb_detector1`` on the input file, saving the result of each @@ -275,33 +298,41 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, outputs = ",".join(ext_or_exts) result_file = os.path.join(cal_dir, short_name+"_status.txt") calibrated_files = ["{}_{}.fits".format(short_name, ext) for ext in ext_or_exts] - - msg = "Running {} cal {} {} {} {} {}" - logging.info(msg.format(cmd_name, outputs, cal_dir, instrument, input_file, short_name)) - - cmd = "{} cal {} '{}' {} {} {}" - cmd = cmd.format(cmd_name, outputs, cal_dir, instrument, input_file, short_name) - process = Popen(cmd, shell=True, executable="/bin/bash", stderr=PIPE) - with process.stderr: - log_subprocess_output(process.stderr) - result = process.wait() - logging.info("Subprocess result was {}".format(result)) - if not os.path.isfile(result_file): - logging.error("Result file was not created.") - with open(os.path.join(cal_dir, "general_status.txt")) as status_file: - for line in status_file.readlines(): - logging.error(line.strip()) - - with open(result_file, 'r') as inf: - status = inf.readlines() + cores = 'all' + status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, input_file, + short_name, res_file, cores) if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") else: logging.error("Pipeline subprocess failed.") + core_fail = False for line in status: + if "[Errno 12] Cannot allocate memory" in line: + core_fail = True logging.error("\t{}".format(line.strip())) + if core_fail: + cores = "10" + status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, + input_file, short_name, res_file, cores) + if status[-1].strip() == "SUCCEEDED": + logging.info("Subprocess reports successful finish.") + else: + logging.error("Pipeline subprocess failed.") + core_fail = False + for line in status: + if "[Errno 12] Cannot allocate memory" in line: + core_fail = True + logging.error("\t{}".format(line.strip())) + if core_fail: + cores = "1" + status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, + input_file, short_name, res_file, cores) + if status[-1].strip() == "SUCCEEDED": + logging.info("Subprocess reports successful finish.") + else: + logging.error("Pipeline subprocess failed.") raise ValueError("Pipeline Failed") for file in calibrated_files: @@ -380,29 +411,40 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save cmd_name = os.path.join(os.path.dirname(__file__), "run_pipeline.py") result_file = os.path.join(cal_dir, short_name+"_status.txt") - cmd = "{} jump all '{}' {} {} {}" - cmd = cmd.format(cmd_name, cal_dir, instrument, input_file, short_name) - process = Popen(cmd, shell=True, executable="/bin/bash", stderr=PIPE) - with process.stderr: - log_subprocess_output(process.stderr) - result = process.wait() - logging.info("Subprocess result was {}".format(result)) - - if not os.path.isfile(result_file): - logging.error("Result file was not created.") - with open(os.path.join(cal_dir, "general_status.txt")) as status_file: - for line in status_file.readlines(): - logging.error(line.strip()) + cores = 'all' + status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, input_file, + short_name, res_file, cores) - with open(result_file, 'r') as inf: - status = inf.readlines() - if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") else: logging.error("Pipeline subprocess failed.") + core_fail = False for line in status: + if "[Errno 12] Cannot allocate memory" in line: + core_fail = True logging.error("\t{}".format(line.strip())) + if core_fail: + cores = "10" + status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, + input_file, short_name, res_file, cores) + if status[-1].strip() == "SUCCEEDED": + logging.info("Subprocess reports successful finish.") + else: + logging.error("Pipeline subprocess failed.") + core_fail = False + for line in status: + if "[Errno 12] Cannot allocate memory" in line: + core_fail = True + logging.error("\t{}".format(line.strip())) + if core_fail: + cores = "1" + status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, + input_file, short_name, res_file, cores) + if status[-1].strip() == "SUCCEEDED": + logging.info("Subprocess reports successful finish.") + else: + logging.error("Pipeline subprocess failed.") raise ValueError("Pipeline Failed") files = {"jump_output": None, "pipe_output": None, "fitopt_output": None} From 49c973f6db26b49374c2e1fffa6b8b1c76ad43e8 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Thu, 16 Feb 2023 12:51:53 -0500 Subject: [PATCH 065/449] Minor bug fixes for array access/definition errors --- jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py | 2 +- jwql/website/apps/jwql/views.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py index 7f286c5be..1956c2e20 100755 --- a/jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py @@ -624,7 +624,7 @@ def stats_data_to_lists(self): self._stdev = np.array([e.stdev for e in self.db.stats_data]) self._obs_mid_time = np.array([e.obs_mid_time for e in self.db.stats_data]) self._stats_mean_dark_image_files = np.array([e.mean_dark_image_file for e in self.db.stats_data]) - self._stats_numfiles = np.array([e.source_files for e in self.db.stats_data]) + self._stats_numfiles = np.array([len(e.source_files) for e in self.db.stats_data]) class DarkTrendPlot(): diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 8781b6f4d..ab84ab796 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -1151,7 +1151,10 @@ def view_image(request, inst, file_root, rewrite=False): suff_arr = np.array(image_info['suffixes']) files_arr = np.array(image_info['all_files']) splits = np.array([ele.split('_')[-1] for ele in image_info['suffixes']]) - idxs = np.where(splits == poss_suffix)[0] + if splits.size > 0: + idxs = np.where(splits == poss_suffix)[0] + else: + idxs = [] if len(idxs) > 0: suff_entries = list(suff_arr[idxs]) file_entries = list(files_arr[idxs]) From 44f88a8b15f5ebf672af736c710c8db5c4b34f70 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Thu, 16 Feb 2023 12:52:31 -0500 Subject: [PATCH 066/449] Update environments for python 3.8-3.10 --- environment_python_3_10.yml | 72 +++++++++++++++++++++++++++++++++++++ environment_python_3_8.yml | 67 +++++++++++++++++----------------- environment_python_3_9.yml | 65 ++++++++++++++++----------------- setup.py | 12 +++---- 4 files changed, 144 insertions(+), 72 deletions(-) create mode 100644 environment_python_3_10.yml diff --git a/environment_python_3_10.yml b/environment_python_3_10.yml new file mode 100644 index 000000000..61a42909b --- /dev/null +++ b/environment_python_3_10.yml @@ -0,0 +1,72 @@ +# This file describes a conda environment that can be to install jwql +# +# Run the following command to set up this environment: +# $ conda env create -f environment_python_3_10.yml +# +# The environment name can be overridden with the following command: +# $ conda env create -n -f environment_python_3_10.yml +# +# Run the following command to activate the environment: +# $ source activate jwql-3.10 +# +# To deactivate the environment run the following command: +# $ source deactivate +# +# To remove the environment entirely, run the following command: +# $ conda env remove -n jwql-3.10 + +name: jwql-3.10 + +channels: + - conda-forge + - defaults + +dependencies: + - astropy=5.2.1 + - bokeh=2.4.3 + - beautifulsoup4=4.11.2 + - celery=5.2.7 + - codecov=2.1.12 + - cryptography=39.0.1 + - django=3.1.8 + - flake8=6.0.0 + - inflection=0.5.1 + - ipython=8.10.0 + - jinja2=3.1.2 + - jsonschema=4.17.3 + - matplotlib=3.7.0 + - nodejs=18.12.1 + - numpy=1.24.2 + - numpydoc=1.5.0 + - pandas=1.5.3 + - pip=23.0 + - postgresql=15.2 + - psycopg2=2.9.3 + - pytest=7.2.1 + - pytest-cov=4.0.0 + - python=3.10.9 + - pyyaml=6.0 + - redis + - scipy=1.9.3 + - setuptools=67.3.1 + - sphinx=6.1.3 + - sphinx_rtd_theme=1.2.0 + - sqlalchemy=1.4.46 + - twine=4.0.2 + - wtforms=3.0.1 + + - pip: + - astroquery==0.4.6 + - bandit==1.7.4 + - jwst==1.9.4 + - pysiaf==0.19.1 + - pysqlite3==0.5.0 + - pyvo==1.4 + - redis==4.5.1 + - selenium==4.8.0 + - stsci_rtd_theme==1.0.0 + - vine==5.0.0 + - git+https://github.com/spacetelescope/jwst_reffiles + + # Current package + - -e . diff --git a/environment_python_3_8.yml b/environment_python_3_8.yml index 865e13eeb..5d5b69947 100644 --- a/environment_python_3_8.yml +++ b/environment_python_3_8.yml @@ -18,55 +18,54 @@ name: jwql-3.8 channels: + - conda-forge - defaults dependencies: - - astropy=5.1 + - astropy=5.2.1 - bokeh=2.4.3 - - beautifulsoup4=4.11.1 - - celery=4.4.0 - - codecov=2.1.11 - - cryptography=38.0.1 - - django=4.1 - - flake8=4.0.1 + - beautifulsoup4=4.11.2 + - celery=5.2.7 + - codecov=2.1.12 + - cryptography=39.0.1 + - django=3.1.8 + - flake8=6.0.0 - inflection=0.5.1 - - ipython=8.4.0 + - ipython=8.10.0 - jinja2=3.1.2 - - jsonschema=4.16.0 - - matplotlib=3.5.3 - - nodejs=10.13.0 - - numpy=1.23.3 - - numpydoc=1.4.0 - - pandas=1.4.4 - - pip=22.2.2 - - postgresql=12.2 - - psycopg2=2.8.6 - - pytest=7.1.2 - - pytest-cov=3.0.0 - - python=3.8.13 + - jsonschema=4.17.3 + - matplotlib=3.7.0 + - nodejs=18.12.1 + - numpy=1.24.2 + - numpydoc=1.5.0 + - pandas=1.5.3 + - pip=23.0 + - postgresql=15.2 + - psycopg2=2.9.3 + - pytest=7.2.1 + - pytest-cov=4.0.0 + - python=3.8.16 - pyyaml=6.0 - redis - - scipy=1.9.1 - - setuptools=65.5.0 - - sphinx=5.0.2 - - sphinx_rtd_theme=0.4.3 - - sqlalchemy=1.4.39 - - twine=3.7.1 - - vine=1.3.0 - - wtforms=2.3.3 + - scipy=1.9.3 + - setuptools=67.3.1 + - sphinx=6.1.3 + - sphinx_rtd_theme=1.2.0 + - sqlalchemy=1.4.46 + - twine=4.0.2 + - wtforms=3.0.1 - pip: - astroquery==0.4.6 - bandit==1.7.4 - - jwst==1.8.2 - - pysiaf==0.18.0 - - pysqlite3==0.4.7 + - jwst==1.9.4 + - pysiaf==0.19.1 + - pysqlite3==0.5.0 - pyvo==1.4 - - redis + - redis==4.5.1 - selenium==4.8.0 - - sqlalchemy==1.4.39 - stsci_rtd_theme==1.0.0 - - stcal==1.2.1 + - vine==5.0.0 - git+https://github.com/spacetelescope/jwst_reffiles # Current package diff --git a/environment_python_3_9.yml b/environment_python_3_9.yml index 46fc7d758..7e6a97f00 100644 --- a/environment_python_3_9.yml +++ b/environment_python_3_9.yml @@ -18,53 +18,54 @@ name: jwql-3.9 channels: + - conda-forge - defaults dependencies: - - astropy=5.1 + - astropy=5.2.1 - bokeh=2.4.3 - - beautifulsoup4=4.11.1 - - celery=4.4.0 - - codecov=2.1.11 - - cryptography=38.0.1 - - django=4.1 - - flake8=4.0.1 + - beautifulsoup4=4.11.2 + - celery=5.2.7 + - codecov=2.1.12 + - cryptography=39.0.1 + - django=3.1.8 + - flake8=6.0.0 - inflection=0.5.1 - - ipython=8.4.0 + - ipython=8.10.0 - jinja2=3.1.2 - - jsonschema=4.16.0 - - matplotlib=3.5.3 - - nodejs=10.13.0 - - numpy=1.23.3 - - numpydoc=1.4.0 - - pandas=1.4.4 - - pip=22.2.2 - - postgresql=12.2 - - psycopg2=2.8.6 - - pytest=7.1.2 - - pytest-cov=3.0.0 - - python=3.9.13 + - jsonschema=4.17.3 + - matplotlib=3.7.0 + - nodejs=18.12.1 + - numpy=1.24.2 + - numpydoc=1.5.0 + - pandas=1.5.3 + - pip=23.0 + - postgresql=15.2 + - psycopg2=2.9.3 + - pytest=7.2.1 + - pytest-cov=4.0.0 + - python=3.9.16 - pyyaml=6.0 - redis - - scipy=1.9.1 - - setuptools=65.5.0 - - sphinx=5.0.2 - - sphinx_rtd_theme=0.4.3 - - sqlalchemy=1.4.39 - - twine=3.7.1 - - wtforms=2.3.3 + - scipy=1.9.3 + - setuptools=67.3.1 + - sphinx=6.1.3 + - sphinx_rtd_theme=1.2.0 + - sqlalchemy=1.4.46 + - twine=4.0.2 + - wtforms=3.0.1 - pip: - astroquery==0.4.6 - bandit==1.7.4 - - jwst==1.8.2 - - pysiaf==0.18.0 - - pysqlite3==0.4.7 + - jwst==1.9.4 + - pysiaf==0.19.1 + - pysqlite3==0.5.0 - pyvo==1.4 - - redis + - redis==4.5.1 - selenium==4.8.0 - stsci_rtd_theme==1.0.0 - - vine==1.3.0 + - vine==5.0.0 - git+https://github.com/spacetelescope/jwst_reffiles # Current package diff --git a/setup.py b/setup.py index 1f8220c54..e9e046e76 100644 --- a/setup.py +++ b/setup.py @@ -11,15 +11,15 @@ DESCRIPTION = 'The James Webb Space Telescope Quicklook Project' REQUIRES = [ - 'asdf>=2.3.3', - 'astropy>=3.2.1', - 'astroquery>=0.3.9', + 'asdf', + 'astropy', + 'astroquery', 'bandit', - 'bokeh==2.4.3', + 'bokeh<3', 'codecov', 'crds', 'cryptography', - 'django<=3.1.7', + 'django<3.2', 'flake8', 'inflection', 'ipython', @@ -40,7 +40,7 @@ 'scipy', 'sphinx', 'sphinx_rtd_theme', - 'sqlalchemy', + 'sqlalchemy<2', 'stdatamodels', 'stsci_rtd_theme', 'twine', From a5ada6da6b94673a57b4627902d81e8554c2128c Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Thu, 16 Feb 2023 13:41:42 -0500 Subject: [PATCH 067/449] Fixed typo in variable name --- jwql/shared_tasks/shared_tasks.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index bddcd18ff..4b1ed9ef5 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -301,7 +301,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, cores = 'all' status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, input_file, - short_name, res_file, cores) + short_name, result_file, cores) if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") @@ -315,7 +315,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, if core_fail: cores = "10" status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, - input_file, short_name, res_file, cores) + input_file, short_name, result_file, cores) if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") else: @@ -328,7 +328,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, if core_fail: cores = "1" status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, - input_file, short_name, res_file, cores) + input_file, short_name, result_file, cores) if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") else: @@ -413,7 +413,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save cores = 'all' status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, input_file, - short_name, res_file, cores) + short_name, result_file, cores) if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") @@ -427,7 +427,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save if core_fail: cores = "10" status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, - input_file, short_name, res_file, cores) + input_file, short_name, result_file, cores) if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") else: @@ -440,7 +440,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save if core_fail: cores = "1" status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, - input_file, short_name, res_file, cores) + input_file, short_name, result_file, cores) if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") else: From f2a299c08fc56e7ef29cecb2a39080106c86b7b0 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Thu, 16 Feb 2023 14:40:29 -0500 Subject: [PATCH 068/449] Update pip requirements for consistency --- .readthedocs.yaml | 4 +-- environment_python_3_10.yml | 1 + environment_python_3_8.yml | 1 + environment_python_3_9.yml | 1 + requirements.txt | 55 +++++++++++++++++++++---------------- rtd_requirements.txt | 18 ++++++------ 6 files changed, 45 insertions(+), 35 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 16da97253..069f15a23 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -15,7 +15,7 @@ build: python: "3.9" jobs: post_install: - - pip install sqlalchemy==1.4.43 + - pip install sqlalchemy==1.4.46 # Build documentation in the docs/ directory with Sphinx sphinx: @@ -29,4 +29,4 @@ python: install: - requirements: rtd_requirements.txt - method: pip - path: . \ No newline at end of file + path: . diff --git a/environment_python_3_10.yml b/environment_python_3_10.yml index 61a42909b..9aca9d74b 100644 --- a/environment_python_3_10.yml +++ b/environment_python_3_10.yml @@ -64,6 +64,7 @@ dependencies: - pyvo==1.4 - redis==4.5.1 - selenium==4.8.0 + - stdatamodels==0.4.5 - stsci_rtd_theme==1.0.0 - vine==5.0.0 - git+https://github.com/spacetelescope/jwst_reffiles diff --git a/environment_python_3_8.yml b/environment_python_3_8.yml index 5d5b69947..dbe97c537 100644 --- a/environment_python_3_8.yml +++ b/environment_python_3_8.yml @@ -64,6 +64,7 @@ dependencies: - pyvo==1.4 - redis==4.5.1 - selenium==4.8.0 + - stdatamodels==0.4.5 - stsci_rtd_theme==1.0.0 - vine==5.0.0 - git+https://github.com/spacetelescope/jwst_reffiles diff --git a/environment_python_3_9.yml b/environment_python_3_9.yml index 7e6a97f00..fc0d2a424 100644 --- a/environment_python_3_9.yml +++ b/environment_python_3_9.yml @@ -64,6 +64,7 @@ dependencies: - pyvo==1.4 - redis==4.5.1 - selenium==4.8.0 + - stdatamodels==0.4.5 - stsci_rtd_theme==1.0.0 - vine==5.0.0 - git+https://github.com/spacetelescope/jwst_reffiles diff --git a/requirements.txt b/requirements.txt index 31ee766fb..6ff5103ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,33 +1,40 @@ -astropy==5.1 +astropy==5.2.1 astroquery==0.4.6 bandit==1.7.4 -beautifulsoup4==4.11.1 +beautifulsoup4==4.11.2 bokeh==2.4.3 -codecov==2.1.11 -cryptography==38.0.1 -django==4.1 -flake8==4.0.1 +celery==5.2.7 +codecov==2.1.12 +cryptography==39.0.1 +django==3.1.8 +flake8==6.0.0 inflection==0.5.1 -ipython==8.4.0 +ipython==8.10.0 jinja2==3.1.2 -jsonschema==4.16.0 -jwst==1.8.2 -matplotlib==3.5.3 +jsonschema==4.17.3 +jwst==1.9.4 +matplotlib==3.7.0 nodejs==0.1.1 -numpy==1.23.3 -numpydoc==1.4 -pandas==1.4.4 -psycopg2-binary==2.8.6 -pysiaf==0.18.0 -pytest==7.1.2 -pytest-cov==3.0.0 +numpy==1.24.2 +numpydoc==1.5.0 +pandas==1.5.3 +psycopg2-binary==2.9.3 +pysiaf==0.19.1 +pysqlite3==0.5.0 +pytest==7.2.1 +pytest-cov==4.0.0 pyvo==1.4 -scipy==1.9.1 -sphinx==5.0.2 -sphinx_rtd_theme==0.4.3 -sqlalchemy==1.4.39 -stdatamodels==0.4.3 +pyyaml==6.0 +redis==4.5.1 +scipy==1.9.3 +selenium==4.8.0 +setuptools==67.3.1 +sphinx==6.1.3 +sphinx_rtd_theme==1.2.0 +sqlalchemy==1.4.46 +stdatamodels==0.4.5 stsci_rtd_theme==1.0.0 -twine==3.7.1 -wtforms==2.3.3 +twine==4.0.2 +vine==5.0.0 +wtforms==3.0.1 git+https://github.com/spacetelescope/jwst_reffiles#egg=jwst_reffiles diff --git a/rtd_requirements.txt b/rtd_requirements.txt index 24dd18554..32f6a2b2e 100644 --- a/rtd_requirements.txt +++ b/rtd_requirements.txt @@ -1,15 +1,15 @@ sphinx_automodapi==0.14.1 bokeh==2.4.3 -celery==4.4.0 -cython==0.29.28 -django==3.1.7 +celery==5.2.7 +cython==0.29.33 +django==3.1.8 docutils==0.18.1 -flake8==4.0.1 -jwst==1.4.3 -pygments==2.11.2 -pytest==7.0.1 -redis +flake8==6.0.0 +jwst==1.9.4 +pygments==2.14.0 +pytest==7.2.1 +redis==4.5.1 sphinx>=2 -stsci_rtd_theme==0.0.2 +stsci_rtd_theme==1.0.0 tomli==2.0.1 git+https://github.com/spacetelescope/jwst_reffiles From 5beb8db95bb7c5ed4e0b26c1864b51b4d4d85b39 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Thu, 16 Feb 2023 14:40:48 -0500 Subject: [PATCH 069/449] Clean up documentation build errors --- docs/source/utils.rst | 8 +------- jwql/edb/engineering_database.py | 4 ++-- jwql/instrument_monitors/common_monitors/dark_monitor.py | 6 +++--- jwql/website/apps/jwql/bokeh_containers.py | 4 ++-- jwql/website/apps/jwql/models.py | 4 ++++ 5 files changed, 12 insertions(+), 14 deletions(-) diff --git a/docs/source/utils.rst b/docs/source/utils.rst index 856caed01..3956b7afc 100644 --- a/docs/source/utils.rst +++ b/docs/source/utils.rst @@ -2,12 +2,6 @@ utils ***** -anomaly_query_config.py ------------------------ -.. automodule:: jwql.utils.anomaly_query_config - :members: - :undoc-members: - calculations.py --------------- .. automodule:: jwql.utils.calculations @@ -90,4 +84,4 @@ utils.py -------- .. automodule:: jwql.utils.utils :members: - :undoc-members: \ No newline at end of file + :undoc-members: diff --git a/jwql/edb/engineering_database.py b/jwql/edb/engineering_database.py index 7ca5ba44a..fea43aafa 100644 --- a/jwql/edb/engineering_database.py +++ b/jwql/edb/engineering_database.py @@ -1014,8 +1014,8 @@ def plot_data_plus_devs(self, use_median=False, show_plot=False, savefig=False, may be unexpected. Also add a plot of the mean value over time and in a second figure, a plot of the devaition from the mean. - Paramters - --------- + Parameters + ---------- use_median : bool If True, plot the median rather than the mean, as well as the deviation from the median rather than from the mean diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index 2377d5908..70ce2372d 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -574,8 +574,8 @@ def noise_check(self, new_noise_image, baseline_noise_image, threshold=1.5): def overplot_bad_pix(self, pix_type, coords, values): """Add a scatter plot of potential new bad pixels to the plot - Paramters - --------- + Parameters + ---------- pix_type : str Type of bad pixel. "hot", "dead", or "noisy" @@ -1176,7 +1176,7 @@ def stats_by_amp(self, image, amps): hist, bin_edges = np.histogram(image[indexes[0], indexes[1]], bins='auto', range=(lower_bound, upper_bound)) - # If the number of bins is smaller than the number of paramters + # If the number of bins is smaller than the number of parameters # to be fit, then we need to increase the number of bins if len(bin_edges) < 7: logging.info('\tToo few histogram bins in initial fit. Forcing 10 bins.') diff --git a/jwql/website/apps/jwql/bokeh_containers.py b/jwql/website/apps/jwql/bokeh_containers.py index 87d93a650..9e686ae4c 100644 --- a/jwql/website/apps/jwql/bokeh_containers.py +++ b/jwql/website/apps/jwql/bokeh_containers.py @@ -521,8 +521,8 @@ def standard_monitor_plot_layout(instrument, plots): aperture list. This function assumes that there are plots for all full frame apertures present. - Paramters - --------- + Parameters + ---------- instrument : str Name of the instrument that the plots are for diff --git a/jwql/website/apps/jwql/models.py b/jwql/website/apps/jwql/models.py index 0ba91f75d..d831cc666 100644 --- a/jwql/website/apps/jwql/models.py +++ b/jwql/website/apps/jwql/models.py @@ -47,6 +47,7 @@ class Archive(models.Model): # … # Metadata class Meta: + app_label = 'jwql' ordering = ['instrument'] def __str__(self): @@ -63,6 +64,7 @@ class Proposal(models.Model): # Metadata class Meta: + app_label = 'jwql' ordering = ['-prop_id'] unique_together = ('prop_id', 'archive') models.UniqueConstraint(fields=['prop_id', 'archive'], name='unique_instrument_proposal') @@ -85,6 +87,7 @@ class Observation(models.Model): # … # Metadata class Meta: + app_label = 'jwql' ordering = ['-obsnum'] models.UniqueConstraint(fields=['proposal', 'obsnum'], name='unique_proposal_obsnum') @@ -103,6 +106,7 @@ class RootFileInfo(models.Model): # Metadata class Meta: + app_label = 'jwql' ordering = ['-root_name'] def __str__(self): From 4c0c33bfe78112803a32dded811edea3aca98288 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Thu, 16 Feb 2023 14:57:48 -0500 Subject: [PATCH 070/449] Another typo fix --- jwql/shared_tasks/run_pipeline.py | 2 +- jwql/shared_tasks/shared_tasks.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 1b1e39583..83294ed06 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -240,7 +240,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T parser.add_argument('instrument', metavar='INSTRUMENT', type=str, help=ins_help) parser.add_argument('input_file', metavar='FILE', type=str, help=file_help) parser.add_argument('short_name', metavar='NAME', type=str, help=name_help) - parser.add_argument('max_cores', metavar='CORES', type=str, help=cores_help, default='all') + parser.add_argument('max_cores', metavar='CORES', type=str, help=cores_help) with open("/internal/data1/outputs/ops/calibrated_data/general_status.txt", "a+") as status_file: status_file.write("Created argument parser at {}\n".format(time.ctime())) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index 4b1ed9ef5..3212836cc 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -225,10 +225,10 @@ def collect_after_task(**kwargs): def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_file, cores='all'): - cmd = "{} {} {} '{}' {} {} {} {}" - cmd = cmd.format(name, cmd, outputs, cal_dir, ins, in_file, short_name, cores) - logging.info("Running {}".format(cmd)) - process = Popen(cmd, shell=True, executable="/bin/bash", stderr=PIPE) + command = "{} {} {} '{}' {} {} {} {}" + command = command.format(name, cmd, outputs, cal_dir, ins, in_file, short_name, cores) + logging.info("Running {}".format(command)) + process = Popen(command, shell=True, executable="/bin/bash", stderr=PIPE) with process.stderr: log_subprocess_output(process.stderr) result = process.wait() From e427b9739143e8de9fbdc721648b78c3c5636e4f Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Thu, 16 Feb 2023 16:25:48 -0500 Subject: [PATCH 071/449] Yet another typo --- jwql/shared_tasks/shared_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index 3212836cc..19ccd58a7 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -242,7 +242,7 @@ def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_fi logging.error(line.strip()) return status - with open(res, 'r') as inf: + with open(res_file, 'r') as inf: status = inf.readlines() return status From 365d1f516e56849ef5feba86aea396a3b09461bc Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Thu, 16 Feb 2023 18:15:00 -0500 Subject: [PATCH 072/449] Used the proper signalling for number of cores --- jwql/shared_tasks/shared_tasks.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index 19ccd58a7..189b0d1be 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -224,7 +224,7 @@ def collect_after_task(**kwargs): gc.collect() -def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_file, cores='all'): +def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_file, cores): command = "{} {} {} '{}' {} {} {} {}" command = command.format(name, cmd, outputs, cal_dir, ins, in_file, short_name, cores) logging.info("Running {}".format(command)) @@ -313,7 +313,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, core_fail = True logging.error("\t{}".format(line.strip())) if core_fail: - cores = "10" + cores = "half" status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, input_file, short_name, result_file, cores) if status[-1].strip() == "SUCCEEDED": @@ -326,7 +326,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, core_fail = True logging.error("\t{}".format(line.strip())) if core_fail: - cores = "1" + cores = "none" status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, input_file, short_name, result_file, cores) if status[-1].strip() == "SUCCEEDED": @@ -425,7 +425,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save core_fail = True logging.error("\t{}".format(line.strip())) if core_fail: - cores = "10" + cores = "half" status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, input_file, short_name, result_file, cores) if status[-1].strip() == "SUCCEEDED": @@ -438,7 +438,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save core_fail = True logging.error("\t{}".format(line.strip())) if core_fail: - cores = "1" + cores = "none" status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, input_file, short_name, result_file, cores) if status[-1].strip() == "SUCCEEDED": From be7b141a21593cbaefbb4e15e022444b1b817808 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Thu, 16 Feb 2023 22:02:59 -0500 Subject: [PATCH 073/449] Forgot to not raise an exception if a backoff worked. --- jwql/shared_tasks/shared_tasks.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index 189b0d1be..93de31247 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -314,10 +314,12 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, logging.error("\t{}".format(line.strip())) if core_fail: cores = "half" + managed = False status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, input_file, short_name, result_file, cores) if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") + managed = True else: logging.error("Pipeline subprocess failed.") core_fail = False @@ -331,9 +333,11 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, input_file, short_name, result_file, cores) if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") + managed = True else: logging.error("Pipeline subprocess failed.") - raise ValueError("Pipeline Failed") + if not managed: + raise ValueError("Pipeline Failed") for file in calibrated_files: if not os.path.isfile(os.path.join(cal_dir, file)): @@ -419,6 +423,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save logging.info("Subprocess reports successful finish.") else: logging.error("Pipeline subprocess failed.") + managed = False core_fail = False for line in status: if "[Errno 12] Cannot allocate memory" in line: @@ -430,6 +435,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save input_file, short_name, result_file, cores) if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") + managed = True else: logging.error("Pipeline subprocess failed.") core_fail = False @@ -443,9 +449,11 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save input_file, short_name, result_file, cores) if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") + managed = True else: logging.error("Pipeline subprocess failed.") - raise ValueError("Pipeline Failed") + if not managed: + raise ValueError("Pipeline Failed") files = {"jump_output": None, "pipe_output": None, "fitopt_output": None} for line in status[-5:-1]: From 0a6b819d0bea1a370fed4145163fd8e3aa626e01 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 17 Feb 2023 10:24:31 -0500 Subject: [PATCH 074/449] Add report key configuration by instrument --- jwql/tests/test_data_containers.py | 25 +++++++---- jwql/utils/constants.py | 12 ++++++ jwql/website/apps/jwql/api_views.py | 18 ++++---- jwql/website/apps/jwql/data_containers.py | 52 +++++++++++++++-------- jwql/website/apps/jwql/views.py | 11 +---- 5 files changed, 74 insertions(+), 44 deletions(-) diff --git a/jwql/tests/test_data_containers.py b/jwql/tests/test_data_containers.py index 0a5c8df3f..6c1fa297d 100644 --- a/jwql/tests/test_data_containers.py +++ b/jwql/tests/test_data_containers.py @@ -183,23 +183,30 @@ def test_get_instrument_proposals(): def test_get_instrument_looks(keys, viewed): """Tests the ``get_instrument_looks`` function.""" - looks = data_containers.get_instrument_looks( - 'nirspec', keys=keys, viewed=viewed) + return_keys, looks = data_containers.get_instrument_looks( + 'nirspec', additional_keys=keys, viewed=viewed) + assert isinstance(return_keys, list) assert isinstance(looks, list) + # returned keys always contains at least root name + assert len(return_keys) > 1 + assert 'root_name' in return_keys + assert 'viewed' in return_keys + + # they may also contain some keys from the instrument + # and any additional keys specified + if keys is not None: + assert len(return_keys) >= 1 + len(keys) + # viewed depends on local database, so may or may not have results if not viewed: assert len(looks) > 0 first_file = looks[0] assert first_file['root_name'] != '' assert isinstance(first_file['viewed'], bool) - if keys is not None: - assert len(first_file) == 2 + len(keys) - for key in keys: - assert key in first_file - else: - # only root name and looks by default - assert len(first_file) == 2 + assert len(first_file) == len(return_keys) + for key in return_keys: + assert key in first_file @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index ada1dc312..b286fb64e 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -418,6 +418,7 @@ # The complete name will have "_{instrument.lower}.txt" added to the end of this. PREVIEW_IMAGE_LISTFILE = 'preview_image_inventory' + # Keep keys defined via class as they are used many places with potential mispellings class QUERY_CONFIG_KEYS: ANOMALIES = "ANOMALIES" @@ -468,6 +469,17 @@ class QUERY_CONFIG_KEYS: 'nirspec': ['NRS', 'NRSRAPID', 'NRSIRS2RAPID', 'NRSRAPIDD2', 'NRSRAPIDD6']} + +REPORT_KEYS_PER_INSTRUMENT = {'fgs': ['proposal', 'obsnum', 'number_of_files', + 'exptypes', 'obsstart', 'obsend'], + 'miri': ['proposal', 'obsnum', 'number_of_files', + 'exptypes', 'obsstart', 'obsend'], + 'nircam': ['proposal', 'obsnum', 'number_of_files', + 'exptypes', 'obsstart', 'obsend'], + 'niriss': ['proposal', 'obsnum', 'number_of_files', + 'exptypes', 'obsstart', 'obsend'], + 'nirspec': ['exptypes', 'viewed']} + SUBARRAYS_ONE_OR_FOUR_AMPS = ['SUBGRISMSTRIPE64', 'SUBGRISMSTRIPE128', 'SUBGRISMSTRIPE256'] # Filename suffixes that need to include the association value in the suffix in diff --git a/jwql/website/apps/jwql/api_views.py b/jwql/website/apps/jwql/api_views.py index 76d9e6d9d..7ae0dced1 100644 --- a/jwql/website/apps/jwql/api_views.py +++ b/jwql/website/apps/jwql/api_views.py @@ -163,24 +163,24 @@ def instrument_looks(request, inst, status=None): JsonResponse Outgoing response sent to the webpage, depending on return_type. """ - # TODO: define more useful keys by instrument in config - # currently, optional keys are just the values available - # in local models - optional_keys = ['proposal', 'obsnum', 'number_of_files', - 'exptypes', 'obsstart', 'obsend'] - full_keys = ['root_name', 'viewed'] + optional_keys + # parse desired look status + if status == 'viewed': + viewed = True + elif status == 'new': + viewed = False + else: + viewed = None # get all observation looks from file info model # and join with observation descriptors - viewed = str(status) == 'viewed' - looks = get_instrument_looks(inst, keys=optional_keys, viewed=viewed) + keys, looks = get_instrument_looks(inst, viewed=viewed) # return results by api key if status is None: status = 'looks' response = JsonResponse({'instrument': inst, - 'keys': full_keys, + 'keys': keys, 'type': status, status: looks}, json_dumps_params={'indent': 2}) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index efd2707f8..57fd26c8e 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -51,9 +51,11 @@ from jwql.database.database_interface import load_connection from jwql.edb.engineering_database import get_mnemonic, get_mnemonic_info, mnemonic_inventory from jwql.utils.utils import check_config_for_key, ensure_dir_exists, filesystem_path, filename_parser, get_config -from jwql.utils.constants import MAST_QUERY_LIMIT, MONITORS, PREVIEW_IMAGE_LISTFILE, THUMBNAIL_LISTFILE, THUMBNAIL_FILTER_LOOK -from jwql.utils.constants import IGNORED_SUFFIXES, INSTRUMENT_SERVICE_MATCH, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_INSTRUMENT_NAMES -from jwql.utils.constants import SUFFIXES_TO_ADD_ASSOCIATION, SUFFIXES_WITH_AVERAGED_INTS, QUERY_CONFIG_KEYS, QUERY_CONFIG_TEMPLATE +from jwql.utils.constants import MAST_QUERY_LIMIT, MONITORS, THUMBNAIL_LISTFILE, THUMBNAIL_FILTER_LOOK +from jwql.utils.constants import IGNORED_SUFFIXES, INSTRUMENT_SERVICE_MATCH +from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_INSTRUMENT_NAMES +from jwql.utils.constants import REPORT_KEYS_PER_INSTRUMENT +from jwql.utils.constants import SUFFIXES_TO_ADD_ASSOCIATION, SUFFIXES_WITH_AVERAGED_INTS, QUERY_CONFIG_KEYS from jwql.utils.credentials import get_mast_token from jwql.utils.utils import get_rootnames_for_instrument_proposal from .forms import InstrumentAnomalySubmitForm @@ -924,31 +926,44 @@ def get_instrument_proposals(instrument): return inst_proposals -def get_instrument_looks(instrument, keys=None, viewed=None): +def get_instrument_looks(instrument, viewed=None, additional_keys=None): """Return a table of looks information for the given instrument. Parameters ---------- instrument : str Name of the JWST instrument. - keys : list of str, optional - Additional FITS key names for information to return. viewed : bool, optional If set to None, all viewed values are returned. If set to True, only viewed data is returned. If set to False, only new data is returned. + additional_keys : list of str, optional + Additional model attribute names for information to return. Returns ------- - looks : list - List of looks information by observation for the given instrument. + keys : list of str + Report values returned for the given instrument. + looks : list of dict + List of looks information by root file for the given instrument. """ # standardize input inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[instrument.lower()] - if keys is None: - keys = [] - # get files by instrument from local model + # required keys + keys = ['root_name'] + + # optional keys by instrument + keys += REPORT_KEYS_PER_INSTRUMENT[inst.lower()] + + # add any additional keys + key_set = set(keys) + if additional_keys is not None: + for key in additional_keys: + if key not in key_set: + keys.append(key) + + # get file info by instrument from local model if viewed is None: root_file_info = RootFileInfo.objects.filter(instrument=inst) elif viewed: @@ -958,10 +973,9 @@ def get_instrument_looks(instrument, keys=None, viewed=None): looks = [] for root_file in root_file_info: - # for now, report viewed by root name only. + # for now, report info by root name only. # if specific files are needed, use get_filesystem_files - result = {'root_name': root_file.root_name, - 'viewed': root_file.viewed} + result = dict() for key in keys: try: # try the root file table @@ -976,11 +990,15 @@ def get_instrument_looks(instrument, keys=None, viewed=None): value = getattr(root_file.obsnum.proposal, key) except AttributeError: value = '' - if type(value) not in [str, float, int]: + + # make sure value can be serialized + if type(value) not in [str, float, int, bool]: value = str(value) + result[key] = value looks.append(result) - return looks + + return keys, looks def get_preview_images_by_proposal(proposal): @@ -1116,7 +1134,7 @@ def get_thumbnails_all_instruments(parameters): Parameters ---------- - parameters: dict of type QUERY_CONFIG_TEMPLATE + parameters: dict A dictionary containing keys of QUERY_CONFIG_KEYS, some of which are dictionaries: diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index c01b24d63..6f02e2677 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -586,13 +586,6 @@ def download_report(request, inst, status='all'): response : HttpResponse object Outgoing response sent to the webpage """ - # TODO: define more useful keys by instrument in config - # currently, optional keys are just the values available - # in local models - optional_keys = ['proposal', 'obsnum', 'number_of_files', - 'exptypes', 'obsstart', 'obsend'] - full_keys = ['root_name', 'viewed'] + optional_keys - # get all observation looks from file info model # and join with observation descriptors if status == 'viewed': @@ -601,7 +594,7 @@ def download_report(request, inst, status='all'): viewed = False else: viewed = None - looks = get_instrument_looks(inst, keys=optional_keys, viewed=viewed) + keys, looks = get_instrument_looks(inst, viewed=viewed) today = datetime.datetime.now().strftime('%Y%m%d') filename = f'{inst}_{status}_{today}.csv' @@ -609,7 +602,7 @@ def download_report(request, inst, status='all'): response['Content-Disposition'] = f'attachment; filename="{filename}"' writer = csv.writer(response) - writer.writerow(full_keys) + writer.writerow(keys) for row in looks: writer.writerow(row.values()) From 31b22dd579a21524dfdc0e6cc7240a2aa21fd9c4 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Fri, 17 Feb 2023 12:13:27 -0500 Subject: [PATCH 075/449] Declared a variable at the wrong indentation. Fixed now. --- jwql/shared_tasks/shared_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index 93de31247..507d9ffee 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -306,6 +306,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") else: + managed = False logging.error("Pipeline subprocess failed.") core_fail = False for line in status: @@ -314,7 +315,6 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, logging.error("\t{}".format(line.strip())) if core_fail: cores = "half" - managed = False status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, input_file, short_name, result_file, cores) if status[-1].strip() == "SUCCEEDED": From c39bd1143bebb0c819ab7a3ad16822b8c719e326 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 17 Feb 2023 16:29:24 -0500 Subject: [PATCH 076/449] Create html files with plots ahed of time, and update view --- .../common_monitors/bad_pixel_monitor.py | 77 +++++- .../monitor_pages/monitor_bad_pixel_bokeh.py | 238 +++++++++++++++++- jwql/website/apps/jwql/monitor_views.py | 27 +- 3 files changed, 329 insertions(+), 13 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py index 427623715..4c10e9a06 100755 --- a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py +++ b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py @@ -432,6 +432,66 @@ def add_bad_pix(self, coordinates, pixel_type, files, obs_start_time, obs_mid_ti 'entry_date': datetime.datetime.now()} self.pixel_table.__table__.insert().execute(entry) + def create_badpix_plot(self): + """Create a Bokeh figure of a single bad pixel figure. For the given + detector/aperture and bad pixel type load an image and display. Then scatter markers + + """ + + def create_plot_layout(self): + """Creat a bokeh figure to hold an image of the detector with + bad pixel locations marked on the map. Save the figure as a + json file, so that it can be loaded into the bad pixel results + html page, and displayed + + There should be one tab for each detector. On each tab, there + are plots showing the locations of new bad pixels from darks, + and new bad pixels from flats. Then there is a separate plot + for each flavor of bad pixel that shows the number of these + bad pixels found versus time. + + BUT: nircam has no lamp, so we will never had any bad pix from + flats. + What other entries would be missing from other instruments? + """ + do we want to create one json file for each detector/aperture? + or like the edb monitor, one json file that holds all the + plots? The latter case is probably easier. With multiple json + files, we would need a custom html file that loads each json file + individually. This means we would need a different html file for + each instrument. + + + # example of creating a custom grid layout + sliders = column(amp, freq, phase, offset) + + layout([ + [bollinger], + [sliders, plot], + [p1, p2, p3], + ]) + + + + + + + + + # Wrap the plots in a Panel + panel_list.append(Panel(child=grid, title=key)) + + + + # Save the tabbed plot to a json file - this is from EDB monitor + item_text = json.dumps(json_item(tabbed, "tabbed_edb_plot")) + basename = f'edb_{self.instrument}_tabbed_plots.json' + output_file = os.path.join(self.plot_output_dir, basename) + with open(output_file, 'w') as outfile: + outfile.write(item_text) + logging.info(f'JSON file with tabbed plots saved to {output_file}') + + def filter_query_results(self, results, datatype): """Filter MAST query results. For input flats, keep only those with the most common filter/pupil/grating combination. For both @@ -821,7 +881,7 @@ def process(self, illuminated_raw_files, illuminated_slope_files, flat_file_coun # Get observation time for all files illuminated_obstimes.append(instrument_properties.get_obstime(uncal_file)) - + index = 0 while index < len(illuminated_raw_files): if illuminated_slope_files[index] is None or illuminated_slope_files[index] == 'None': @@ -893,7 +953,7 @@ def process(self, illuminated_raw_files, illuminated_slope_files, flat_file_coun if not os.path.isfile(local_ramp_file): dark_slope_files[index] = None index += 1 - + index = 0 while index < len(dark_raw_files): if dark_jump_files[index] is None or dark_fitopt_files[index] is None or dark_slope_files[index] is None: @@ -908,7 +968,7 @@ def process(self, illuminated_raw_files, illuminated_slope_files, flat_file_coun min_dark_time = min(dark_obstimes) max_dark_time = max(dark_obstimes) mid_dark_time = instrument_properties.mean_time(dark_obstimes) - + # Check whether there are still enough files left to meet the threshold if illuminated_slope_files is None: flat_length = 0 @@ -1004,6 +1064,10 @@ def process(self, illuminated_raw_files, illuminated_slope_files, flat_file_coun elif bad_type in badpix_types_from_darks: self.add_bad_pix(bad_location_list, bad_type, dark_slope_files, min_dark_time, mid_dark_time, max_dark_time, baseline_file) + + here: create_badpix_plot() + + else: raise ValueError("Unrecognized type of bad pixel: {}. Cannot update database table.".format(bad_type)) @@ -1038,6 +1102,7 @@ def run(self): self.query_end = Time.now().mjd # Loop over all instruments + updated_instruments = [] for instrument in JWST_INSTRUMENT_NAMES: self.instrument = instrument @@ -1181,6 +1246,7 @@ def run(self): # Run the bad pixel monitor if run_flats or run_darks: self.process(flat_uncal_files, flat_rate_files, flat_file_count_threshold, dark_uncal_files, dark_rate_files, dark_file_count_threshold) + updated_instruments.append(self.instrument) # Update the query history if dark_uncal_files is None: @@ -1208,6 +1274,11 @@ def run(self): self.query_table.__table__.insert().execute(new_entry) logging.info('\tUpdated the query history table') + # Update the figures to be shown in the web app. Only update figures + # for instruments where the monitor ran + for instrument in updated_instruments: + BadPixelPlots(instrument) + logging.info('Bad Pixel Monitor completed successfully.') diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py index 0be5d85a0..aa424e459 100755 --- a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py @@ -20,6 +20,11 @@ import os from astropy.io import fits +from bokeh.embed import components, file_html +from bokeh.layouts import layout +from bokeh.models import ColumnDataSource, Panel, Tabs, Text +from bokeh.plotting import figure +from bokeh.resources import CDN import datetime import numpy as np @@ -31,12 +36,237 @@ from jwql.database.database_interface import FGSBadPixelQueryHistory, FGSBadPixelStats from jwql.utils.constants import BAD_PIXEL_TYPES, DARKS_BAD_PIXEL_TYPES, FLATS_BAD_PIXEL_TYPES, JWST_INSTRUMENT_NAMES_MIXEDCASE from jwql.utils.utils import filesystem_path -from jwql.bokeh_templating import BokehTemplate SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -class BadPixelMonitor(BokehTemplate): +""" +class BadPixMonitorData(): + def __init__(self): + pass + + +class BadPixFigure(): + def __init__(self): + pass + +class BadPixPlots(): + #top-level class + + # Get the data from the database + m = BadPixMonitorData() + + + # Create the figures + for badpix_type in badpix_types: + p = BadPixFigure() +""" + +class BadPixelPlots(): + """Class for creating the bad pixel plots and figures to be displayed in the web app + """ + def __init__(self, instrument): + self.instrument = instrument.lower() + #self.apertures = self.get_inst_apers() + self.apertures = ['aper1', 'aper2', 'aper3'] + self.run() + + def modify_bokeh_saved_html(self): + """Given an html string produced by Bokeh when saving bad pixel monitor plots, + make tweaks such that the page follows the general JWQL page formatting. + """ + lines = self.html.split('\n') + + # List of lines that Bokeh likes to save in the file, but we don't want + lines_to_remove = ["", + '', + ' ', + ''] + + # Our Django-related lines that need to be at the top of the file + newlines = ['{% extends "base.html" %}\n', "\n", + "{% block preamble %}\n", "\n", + f"{JWST_INSTRUMENT_NAMES_MIXEDCASE[self.instrument]} Bad Pixel Monitor- JWQL\n", "\n", + "{% endblock %}\n", "\n", + "{% block content %}\n", "\n", + '
    \n', "\n", + f"

    {JWST_INSTRUMENT_NAMES_MIXEDCASE[self.instrument]} Bad Pixel Monitor

    \n", + "
    \n", + ] + + # More lines that we want to have in the html file, at the bottom + endlines = ["\n", + f"

    {JWST_INSTRUMENT_NAMES_MIXEDCASE[self.instrument]} Bad Pixel Stats Table

    \n", + "
    \n", "\n", + """     Bad Pixel Stats\n""", "\n", + "
    \n", "\n", + "{% endblock %}" + ] + + for line in lines: + if line not in lines_to_remove: + newlines.append(line + '\n') + newlines = newlines + endlines + + self.html = "".join(newlines) + + + def run(self): + aperture_panels = [] + for aperture in self.apertures: + all_plots = {} + all_plots['new_dark'] = NewBadPixPlot('darks').plot + all_plots['new_flat'] = NewBadPixPlot('flats').plot + all_plots['bad_types'] = {} + for badtype in ['badtype1', 'badtype2', 'badtype3']: + all_plots['bad_types'][badtype] = BadPixTypePlot(badtype).plot + plot_layout = badpix_monitor_plot_layout(all_plots) + + # Create a tab for each type of plot + aperture_panels.append(Panel(child=plot_layout, title=aperture)) + + # Build tabs + tabs = Tabs(tabs=aperture_panels) + + # Return tab HTML and JavaScript to web app + script, div = components(tabs) + + # Insert into our html template and save + template_file = '/Users/hilbert/python_repos/jwql/jwql/website/apps/jwql/templates/bad_pixel_monitor_savefile_basic.html' + temp_vars = {'inst': self.instrument, 'plot_script': script, 'plot_div':div} + self.html = file_html(tabs, CDN, f'{self.instrument} bad pix monitor', template_file, temp_vars) + + # Modify the html such that our Django-related lines are kept in place, + # which will allow the page to keep the same formatting and styling as + # the other web app pages + self.modify_bokeh_saved_html() + + # Save html file + outdir = os.path.dirname(template_file) + #outfile = 'test_badpix_saved_file.html' + outfile = f'{self.instrument}_bad_pix_plots.html' + outfile = os.path.join(outdir, outfile) + with open(outfile, "w") as file: + file.write(self.html) + + +class BadPixelData(): + """Retrieve bad pixel monitor data from the database + """ + def __init__(self): + pass + + +class NewBadPixPlot(): + """Create a plot showing the location of newly discovered bad pixels + """ + def __init__(self, data_type): + self.plot = figure(title=data_type, tools='') + self.plot.x_range.start = 0 + self.plot.x_range.end = 1 + self.plot.y_range.start = 0 + self.plot.y_range.end = 1 + + source = ColumnDataSource(data=dict(x=[0.5], y=[0.5], text=['No data'])) + glyph = Text(x="x", y="y", text="text", angle=0., text_color="navy", text_font_size={'value':'20px'}) + self.plot.add_glyph(source, glyph) + + + +class BadPixTypePlot(): + """Create a plot showing the location of a certain type of bad pixel + """ + def __init__(self, badpix_type): + self.plot = figure(title=badpix_type, tools='') + self.plot.x_range.start = 0 + self.plot.x_range.end = 1 + self.plot.y_range.start = 0 + self.plot.y_range.end = 1 + + source = ColumnDataSource(data=dict(x=[0.5], y=[0.5], text=['No data'])) + glyph = Text(x="x", y="y", text="text", angle=0., text_color="red", text_font_size={'value':'20px'}) + self.plot.add_glyph(source, glyph) + + + + + + + +def badpix_monitor_plot_layout(plots): + """Arrange a set of plots into a bokeh layout. Generate nested lists for + the plot layout for a given aperture. Contents of tabs should be similar + for all apertures of a given instrument. Keys of the input plots will + control the exact layout. + + Paramters + --------- + plots : dict + Dictionary containing a set of plots for an aperture. + Possible keys are 'new_flat' and 'new_dark', which contain the figures + showing new bad pixels derived from flats and darks, respectively. + The third key is 'bad_types', which should contain a dictionary. The + keys of this dictionary are bad pixel types (e.g. 'dead'). Each of + these contains the Bokeh figure showing the locations of new bad + pixels of that type. + + Returns + ------- + plot_layout : bokeh.layouts.layout + """ + # First the plots showing all bad pixel types derived from a given type of + # input (darks or flats). If both plots are present, show them side by side. + # Some instruments will only have one of these two (e.g. NIRCam has no + # internal lamps, and so will not have flats). In that case, show the single + # exsiting plot by itself in the top row. + if 'new_dark' in plots and 'new_flat' in plots: + new_list = [[plots['new_dark'], plots['new_flat']]] + elif 'new_dark' in plots: + new_list = [plots['new_dark']] + elif 'new_flat' in plots: + new_list = [plots['new_flat']] + + # Next create a list of plots where each plot shows one flavor of bad pixel + plots_per_row = 2 + num_bad_types = len(plots['bad_types']) + first_col = np.arange(0, num_bad_types, plots_per_row) + + badtype_lists = [] + keys = list(plots['bad_types']) + for i, key in enumerate(keys): + if i % plots_per_row == 0: + sublist = keys[i: i + plots_per_row] + rowplots = [] + for subkey in sublist: + rowplots.append(plots['bad_types'][subkey]) + badtype_lists.append(rowplots) + + # Combine full frame and subarray aperture lists + full_list = new_list + badtype_lists + + # Now create a layout that holds the lists + plot_layout = layout(full_list) + + return plot_layout + + + + + + + + + + + + + + +"""OLD CODE BELOW HERE""" +"""CAN BE DELETED""" + + +class BadPixelMonitor(): # Combine instrument and aperture into a single property because we # do not want to invoke the setter unless both are updated @@ -380,6 +610,10 @@ def _update_badpix_v_time(self): self.refs['{}_history_figure'.format(bad_pixel_type.lower())].title.align = "center" self.refs['{}_history_figure'.format(bad_pixel_type.lower())].title.text_font_size = "20px" + + + + # Uncomment the line below when testing via the command line: # bokeh serve --show monitor_badpixel_bokeh.py # BadPixelMonitor() diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py index 74f2f4abb..bc2b7025f 100644 --- a/jwql/website/apps/jwql/monitor_views.py +++ b/jwql/website/apps/jwql/monitor_views.py @@ -64,19 +64,30 @@ def bad_pixel_monitor(request, inst): """ # Ensure the instrument is correctly capitalized - inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()] + #inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()] - tabs_components = bokeh_containers.bad_pixel_monitor_tabs(inst) + # Locate the html file for the instrument + #html_file = os.path.join(CONFIG["outputs"], "bad_pixel_monitor", f"{inst.lower()}_bad_pix_plots.html") + html_file = f"{inst.lower()}_bad_pix_plots.html" - template = "bad_pixel_monitor.html" + # Read in the html file + #with open(html_file, "r") as obj: + # html = obj.read() - context = { - 'inst': inst, - 'tabs_components': tabs_components, - } + #tabs_components = bokeh_containers.bad_pixel_monitor_tabs(inst) + + #template = "bad_pixel_monitor.html" + + #context = { + # 'inst': inst, + # 'tabs_components': tabs_components, + #} # Return a HTTP response with the template and dictionary of variables - return render(request, template, context) + #return render(request, template, context) + + #return HttpResponse(html) + return render(request, html_file) def bias_monitor(request, inst): From 29d22374bab89799ada2582c0af1abefa72fabfa Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 17 Feb 2023 16:39:39 -0500 Subject: [PATCH 077/449] Respect current filter/sort settings for downloaded report --- jwql/tests/test_data_containers.py | 36 +++++++++++++++------ jwql/utils/constants.py | 12 +++---- jwql/website/apps/jwql/api_views.py | 10 +----- jwql/website/apps/jwql/data_containers.py | 39 +++++++++++++++++------ jwql/website/apps/jwql/static/js/jwql.js | 34 +++++++++++--------- jwql/website/apps/jwql/urls.py | 7 +--- jwql/website/apps/jwql/views.py | 21 +++++------- 7 files changed, 90 insertions(+), 69 deletions(-) diff --git a/jwql/tests/test_data_containers.py b/jwql/tests/test_data_containers.py index 6c1fa297d..157a7d5a9 100644 --- a/jwql/tests/test_data_containers.py +++ b/jwql/tests/test_data_containers.py @@ -171,20 +171,30 @@ def test_get_instrument_proposals(): @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') -@pytest.mark.parametrize('keys,viewed', - [(None, True), (None, False), (None, None), - ([], True), ([], False), ([], None), +@pytest.mark.parametrize('keys,viewed,sort_as,exp_type,cat_type', + [(None, None, None, None, None), + (None, 'viewed', None, None, None), + (None, 'new', None, None, None), + (None, None, None, 'NRS_MSATA', None), + # (None, None, None, None, 'CAL'), # cat_type not implemented yet + (['obsstart'], False, 'ascending', None, None), + (['obsstart'], False, 'descending', None, None), + (['obsstart'], False, 'recent', None, None), + ([], True, None, None, None), + ([], False, None, None, None), + ([], None, None, None, None), (['proposal', 'obsnum', 'other', - 'prop_id', 'obsstart'], True), + 'prop_id', 'obsstart'], True, None, None, None), (['proposal', 'obsnum', 'other', - 'prop_id', 'obsstart'], False), + 'prop_id', 'obsstart'], False, None, None, None), (['proposal', 'obsnum', 'other', - 'prop_id', 'obsstart'], None)]) -def test_get_instrument_looks(keys, viewed): + 'prop_id', 'obsstart'], None, None, None, None)]) +def test_get_instrument_looks(keys, viewed, sort_as, exp_type, cat_type): """Tests the ``get_instrument_looks`` function.""" return_keys, looks = data_containers.get_instrument_looks( - 'nirspec', additional_keys=keys, viewed=viewed) + 'nirspec', additional_keys=keys, look=viewed, sort_as=sort_as, + exp_type=exp_type, cat_type=cat_type) assert isinstance(return_keys, list) assert isinstance(looks, list) @@ -199,7 +209,7 @@ def test_get_instrument_looks(keys, viewed): assert len(return_keys) >= 1 + len(keys) # viewed depends on local database, so may or may not have results - if not viewed: + if not viewed == 'viewed': assert len(looks) > 0 first_file = looks[0] assert first_file['root_name'] != '' @@ -208,6 +218,14 @@ def test_get_instrument_looks(keys, viewed): for key in return_keys: assert key in first_file + last_file = looks[-1] + if sort_as == 'ascending': + assert last_file['root_name'] > first_file['root_name'] + elif sort_as == 'recent': + assert last_file['obsstart'] < first_file['obsstart'] + else: + assert last_file['root_name'] < first_file['root_name'] + @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') def test_get_preview_images_by_proposal(): diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index b286fb64e..1e01f7b22 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -470,14 +470,10 @@ class QUERY_CONFIG_KEYS: 'NRSRAPIDD2', 'NRSRAPIDD6']} -REPORT_KEYS_PER_INSTRUMENT = {'fgs': ['proposal', 'obsnum', 'number_of_files', - 'exptypes', 'obsstart', 'obsend'], - 'miri': ['proposal', 'obsnum', 'number_of_files', - 'exptypes', 'obsstart', 'obsend'], - 'nircam': ['proposal', 'obsnum', 'number_of_files', - 'exptypes', 'obsstart', 'obsend'], - 'niriss': ['proposal', 'obsnum', 'number_of_files', - 'exptypes', 'obsstart', 'obsend'], +REPORT_KEYS_PER_INSTRUMENT = {'fgs': ['proposal', 'obsnum', 'exptypes', 'obsstart', 'obsend'], + 'miri': ['proposal', 'obsnum', 'exptypes', 'obsstart', 'obsend'], + 'nircam': ['proposal', 'obsnum', 'exptypes', 'obsstart', 'obsend'], + 'niriss': ['proposal', 'obsnum', 'exptypes', 'obsstart', 'obsend'], 'nirspec': ['exptypes', 'viewed']} SUBARRAYS_ONE_OR_FOUR_AMPS = ['SUBGRISMSTRIPE64', 'SUBGRISMSTRIPE128', 'SUBGRISMSTRIPE256'] diff --git a/jwql/website/apps/jwql/api_views.py b/jwql/website/apps/jwql/api_views.py index 7ae0dced1..aeb32371f 100644 --- a/jwql/website/apps/jwql/api_views.py +++ b/jwql/website/apps/jwql/api_views.py @@ -163,17 +163,9 @@ def instrument_looks(request, inst, status=None): JsonResponse Outgoing response sent to the webpage, depending on return_type. """ - # parse desired look status - if status == 'viewed': - viewed = True - elif status == 'new': - viewed = False - else: - viewed = None - # get all observation looks from file info model # and join with observation descriptors - keys, looks = get_instrument_looks(inst, viewed=viewed) + keys, looks = get_instrument_looks(inst, look=status) # return results by api key if status is None: diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 57fd26c8e..f160d7b21 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -926,17 +926,27 @@ def get_instrument_proposals(instrument): return inst_proposals -def get_instrument_looks(instrument, viewed=None, additional_keys=None): +def get_instrument_looks(instrument, sort_as=None, + look=None, exp_type=None, cat_type=None, + additional_keys=None): """Return a table of looks information for the given instrument. Parameters ---------- instrument : str Name of the JWST instrument. - viewed : bool, optional + sort_as : {'ascending', 'descending', 'recent'} + Sorting method for output table. Ascending and descending + options refer to root file name; recent sorts by observation + start. + look : {'new', 'viewed'}, optional If set to None, all viewed values are returned. If set to - True, only viewed data is returned. If set to False, only + 'viewed', only viewed data is returned. If set to 'new', only new data is returned. + exp_type : str, optional + Set to filter by exposure type. + cat_type : str, optional + Set to filter by proposal category. additional_keys : list of str, optional Additional model attribute names for information to return. @@ -963,13 +973,24 @@ def get_instrument_looks(instrument, viewed=None, additional_keys=None): if key not in key_set: keys.append(key) + # get desired filters + filter_kwargs = dict() + if look is not None: + filter_kwargs['viewed'] = (look == 'viewed') + if exp_type is not None: + filter_kwargs['obsnum__exptypes__contains'] = exp_type + if cat_type is not None: + filter_kwargs['obsnum__proposal__cat_type__contains'] = cat_type + # get file info by instrument from local model - if viewed is None: - root_file_info = RootFileInfo.objects.filter(instrument=inst) - elif viewed: - root_file_info = RootFileInfo.objects.filter(instrument=inst, viewed=True) - else: - root_file_info = RootFileInfo.objects.filter(instrument=inst, viewed=False) + root_file_info = RootFileInfo.objects.filter(instrument=inst, **filter_kwargs) + + # descending by root file is default; + # for other options, sort as desired + if sort_as == 'ascending': + root_file_info = root_file_info.order_by('root_name') + elif sort_as == 'recent': + root_file_info = root_file_info.order_by('-obsnum__obsstart') looks = [] for root_file in root_file_info: diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index 435a20a55..5d93d5d86 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -606,27 +606,31 @@ function download_report(inst, base_url) { var elem = document.getElementById('download_report_button'); elem.disabled = true; - // todo: include all filters - // current look filter - var look_status = document.getElementById('look_dropdownMenuButton') - if (look_status != null) { - look = look_status.innerText.toLowerCase(); - } else { - look = 'all'; - } - if (look.includes('all')) { - report_url = '/' + inst + '/report/'; - } else { - report_url = '/' + inst + '/report/' + look + '/'; - } + // Get sort value + var sort_option = document.getElementById('sort_dropdownMenuButton').innerText; + var options = '?sort_as=' + sort_option.toLowerCase(); + + // Get all filter values + var filter_div = document.getElementById('thumbnail-filter'); + var filters = filter_div.getElementsByClassName('dropdown-toggle'); + + for (var i=0; i < filters.length; i++) { + var name = filters[i].id.split('_dropdownMenuButton')[0]; + var status = filters[i].innerText.toLowerCase(); + if (!status.includes('all')) { + options += '&' + name + '=' + status; + }; + }; + var report_url = '/' + inst + '/report' + options; + console.log('Redirecting to: ' + report_url); - // redirect to download content + // Redirect to download content window.location = base_url + report_url; elem.disabled = false; } /** - * Updates various compnents on the archive page + * Updates various components on the archive page * @param {String} inst - The instrument of interest (e.g. "FGS") * @param {String} base_url - The base URL for gathering data from the AJAX view. */ diff --git a/jwql/website/apps/jwql/urls.py b/jwql/website/apps/jwql/urls.py index 43fc5de58..e03974b90 100644 --- a/jwql/website/apps/jwql/urls.py +++ b/jwql/website/apps/jwql/urls.py @@ -85,12 +85,7 @@ re_path(r'^(?P({}))/archive/$'.format(instruments), views.archived_proposals, name='archive'), re_path(r'^(?P({}))/archive_date_range/$'.format(instruments), views.archive_date_range, name='archive_date_range'), re_path(r'^(?P({}))/unlooked/$'.format(instruments), views.unlooked_images, name='unlooked'), - - re_path(r'^(?P({}))/report/$'.format(instruments), - views.download_report, name='download_report'), - re_path(r'^(?P({}))/report/(?P(viewed|new))/$'.format(instruments), - views.download_report, name='download_report_by_status'), - + re_path(r'^(?P({}))/report/$'.format(instruments), views.download_report, name='download_report'), re_path(r'^(?P({}))/(?P[\w-]+)/$'.format(instruments), views.view_image, name='view_image'), re_path(r'^(?P({}))/(?P.+)_(?P.+)/explore_image/'.format(instruments), views.explore_image, name='explore_image'), re_path(r'^(?P({}))/(?P.+)_(?P.+)/header/'.format(instruments), views.view_header, name='view_header'), diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 6f02e2677..cc0739075 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -567,7 +567,7 @@ def dashboard(request): return render(request, template, context) -def download_report(request, inst, status='all'): +def download_report(request, inst): """Download data report by look status. Parameters @@ -576,28 +576,23 @@ def download_report(request, inst, status='all'): Incoming request from the webpage. inst : str The JWST instrument of interest. - status : str, optional - If set to None or 'all', all viewed values are returned. If set to - 'viewed', only viewed data is returned. If set to 'new', only - new data is returned. Returns ------- response : HttpResponse object Outgoing response sent to the webpage """ + # check for filter criteria passed in request + kwargs = dict() + for filter_name in ['look', 'exp_type', 'cat_type', 'sort_as']: + kwargs[filter_name] = request.GET.get(filter_name) + # get all observation looks from file info model # and join with observation descriptors - if status == 'viewed': - viewed = True - elif status == 'new': - viewed = False - else: - viewed = None - keys, looks = get_instrument_looks(inst, viewed=viewed) + keys, looks = get_instrument_looks(inst, **kwargs) today = datetime.datetime.now().strftime('%Y%m%d') - filename = f'{inst}_{status}_{today}.csv' + filename = f'{inst}_report_{today}.csv' response = HttpResponse(content_type='text/csv') response['Content-Disposition'] = f'attachment; filename="{filename}"' From 06dc9be8d0d990b0d2ed1e48bdaac0fa0e9bbc8e Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Sat, 18 Feb 2023 13:18:03 -0500 Subject: [PATCH 078/449] Adding new logging to figure out where the time is going in new hot/dead pixels --- jwql/instrument_monitors/common_monitors/dark_monitor.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index 28c525e81..b13151d9c 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -730,14 +730,19 @@ def process(self, file_list): baseline_mean, baseline_stdev = self.read_baseline_slope_image(baseline_file) # Check the hot/dead pixel population for changes + logging.info("\tFinding new hot/dead pixels") new_hot_pix, new_dead_pix = self.find_hot_dead_pixels(slope_image, baseline_mean) # Shift the coordinates to be in full frame coordinate system + logging.info("\tShifting hot pixels to full frame") new_hot_pix = self.shift_to_full_frame(new_hot_pix) + logging.info("\tShifting dead pixels to full frame") new_dead_pix = self.shift_to_full_frame(new_dead_pix) # Exclude hot and dead pixels found previously + logging.info("\tExcluding previously-known hot pixels") new_hot_pix = self.exclude_existing_badpix(new_hot_pix, 'hot') + logging.info("\tExcluding previously-known dead pixels") new_dead_pix = self.exclude_existing_badpix(new_dead_pix, 'dead') # Add new hot and dead pixels to the database From db2e907c6692d7e1f554ff57b681c257d1c84e24 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 20 Feb 2023 11:12:47 -0500 Subject: [PATCH 079/449] Default sort text to descending --- jwql/website/apps/jwql/templates/archive.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/templates/archive.html b/jwql/website/apps/jwql/templates/archive.html index bf8164dc7..a089c64ba 100644 --- a/jwql/website/apps/jwql/templates/archive.html +++ b/jwql/website/apps/jwql/templates/archive.html @@ -37,7 +37,7 @@

    Archived {{ inst }} Images

    Sort by:
    - +
    Ascending Descending From 7226d2c8022c490f47c7f2e128735220ec1273c3 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 20 Feb 2023 11:13:13 -0500 Subject: [PATCH 080/449] Case insensitive match --- jwql/website/apps/jwql/data_containers.py | 6 ++++-- jwql/website/apps/jwql/views.py | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index f160d7b21..ab5636c6f 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -978,9 +978,11 @@ def get_instrument_looks(instrument, sort_as=None, if look is not None: filter_kwargs['viewed'] = (look == 'viewed') if exp_type is not None: - filter_kwargs['obsnum__exptypes__contains'] = exp_type + filter_kwargs['obsnum__exptypes__icontains'] = exp_type if cat_type is not None: - filter_kwargs['obsnum__proposal__cat_type__contains'] = cat_type + # filter_kwargs['obsnum__proposal__cat_type__iexact'] = cat_type + # not yet implemented in proposal table + pass # get file info by instrument from local model root_file_info = RootFileInfo.objects.filter(instrument=inst, **filter_kwargs) diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index cc0739075..aa9dc593e 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -16,7 +16,8 @@ - Mees Fix - Bryan Hilbert - Maria Pena-Guerrero - - Bryan Hilbert + - Bradley Sappington + - Melanie Clarke Use @@ -592,7 +593,7 @@ def download_report(request, inst): keys, looks = get_instrument_looks(inst, **kwargs) today = datetime.datetime.now().strftime('%Y%m%d') - filename = f'{inst}_report_{today}.csv' + filename = f'{inst.lower()}_report_{today}.csv' response = HttpResponse(content_type='text/csv') response['Content-Disposition'] = f'attachment; filename="{filename}"' From fbd34488db0f549831bae067f2ebec4549bed317 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 20 Feb 2023 12:14:33 -0500 Subject: [PATCH 081/449] Add viewed to report for all instruments --- jwql/utils/constants.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index 1e01f7b22..c0c5c7deb 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -470,10 +470,14 @@ class QUERY_CONFIG_KEYS: 'NRSRAPIDD2', 'NRSRAPIDD6']} -REPORT_KEYS_PER_INSTRUMENT = {'fgs': ['proposal', 'obsnum', 'exptypes', 'obsstart', 'obsend'], - 'miri': ['proposal', 'obsnum', 'exptypes', 'obsstart', 'obsend'], - 'nircam': ['proposal', 'obsnum', 'exptypes', 'obsstart', 'obsend'], - 'niriss': ['proposal', 'obsnum', 'exptypes', 'obsstart', 'obsend'], +REPORT_KEYS_PER_INSTRUMENT = {'fgs': ['proposal', 'obsnum', 'exptypes', + 'obsstart', 'obsend', 'viewed'], + 'miri': ['proposal', 'obsnum', 'exptypes', + 'obsstart', 'obsend', 'viewed'], + 'nircam': ['proposal', 'obsnum', 'exptypes', + 'obsstart', 'obsend', 'viewed'], + 'niriss': ['proposal', 'obsnum', 'exptypes', + 'obsstart', 'obsend', 'viewed'], 'nirspec': ['exptypes', 'viewed']} SUBARRAYS_ONE_OR_FOUR_AMPS = ['SUBGRISMSTRIPE64', 'SUBGRISMSTRIPE128', 'SUBGRISMSTRIPE256'] From 4fa928fc5fe9920628f516650e7885757c20a30d Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 20 Feb 2023 12:28:29 -0500 Subject: [PATCH 082/449] Fix url for looks api --- jwql/website/apps/jwql/templates/api_landing.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jwql/website/apps/jwql/templates/api_landing.html b/jwql/website/apps/jwql/templates/api_landing.html index fdd992b49..424360e36 100644 --- a/jwql/website/apps/jwql/templates/api_landing.html +++ b/jwql/website/apps/jwql/templates/api_landing.html @@ -52,8 +52,8 @@

    List of Available Services

  • Preview Images by Rootname (https://jwql.stsci.edu/api/<rootname>/preview_images/)
  • Thumbnails by Rootname (https://jwql.stsci.edu/api/<rootname>/thumbnails/)
  • Look Status by Instrument (https://jwql.stsci.edu/api/<instrument>/looks/)
    • -
  • Viewed Data by Instrument (https://jwql.stsci.edu/api/<instrument>/viewed/)
    • -
  • New Data by Instrument (https://jwql.stsci.edu/api/<instrument>/new/)
    • +
  • Viewed Data by Instrument (https://jwql.stsci.edu/api/<instrument>/looks/viewed/)
    • +
  • New Data by Instrument (https://jwql.stsci.edu/api/<instrument>/looks/new/)

    • Where <instrument>, <rootname>, <proposal> are the values for the instrument name, rootname and proposal ID respectivly

    From e74d019154f370bb9e34f8f37eafe3bf0e76b0e4 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Tue, 21 Feb 2023 10:22:53 -0500 Subject: [PATCH 083/449] Fix typo --- jwql/website/apps/jwql/templates/api_landing.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/templates/api_landing.html b/jwql/website/apps/jwql/templates/api_landing.html index 424360e36..5218ce734 100644 --- a/jwql/website/apps/jwql/templates/api_landing.html +++ b/jwql/website/apps/jwql/templates/api_landing.html @@ -56,7 +56,7 @@

    List of Available Services

  • New Data by Instrument (https://jwql.stsci.edu/api/<instrument>/looks/new/)
    • -

    Where <instrument>, <rootname>, <proposal> are the values for the instrument name, rootname and proposal ID respectivly

    +

    where <instrument>, <rootname>, <proposal> are the values for the instrument name, rootname and proposal ID respectively.

    From 88bbfe7d78ea32766eaa9d662d9cf067ecfc6d78 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Tue, 21 Feb 2023 16:45:11 -0500 Subject: [PATCH 084/449] Catch root file / file list mismatch --- jwql/website/apps/jwql/templates/view_image.html | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html index be4d4b36a..4041d7241 100644 --- a/jwql/website/apps/jwql/templates/view_image.html +++ b/jwql/website/apps/jwql/templates/view_image.html @@ -63,13 +63,15 @@

    {{ file_root }}

    - {% set index = file_root_list.index(file_root) %} - {% if index != 0 %} - < Previous - {% endif %} - - {% if index != file_root_list|length - 1 %} - Next > + {% if file_root in file_root_list %} + {% set index = file_root_list.index(file_root) %} + {% if index != 0 %} + < Previous + {% endif %} + + {% if index != file_root_list|length - 1 %} + Next > + {% endif %} {% endif %}
    From 9f92ab1f1ad3161ba5aae496e4173712b60f66b5 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Wed, 22 Feb 2023 17:27:28 -0500 Subject: [PATCH 085/449] spelling fix and add exp_type --- jwql/website/apps/jwql/admin.py | 2 +- jwql/website/apps/jwql/models.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/jwql/website/apps/jwql/admin.py b/jwql/website/apps/jwql/admin.py index caa7bc1f5..f3c6ef854 100644 --- a/jwql/website/apps/jwql/admin.py +++ b/jwql/website/apps/jwql/admin.py @@ -41,7 +41,7 @@ class ObservationAdmin(admin.ModelAdmin): @admin.register(RootFileInfo) class RootFileInfoAdmin(admin.ModelAdmin): - list_display = ('root_name', 'obsnum', 'proposal', 'instrument', 'viewed', 'filter', 'aperature', 'detector', 'read_patt_num', 'read_patt', 'grating', 'subarray', 'pupil') + list_display = ('root_name', 'obsnum', 'proposal', 'instrument', 'viewed', 'filter', 'aperture', 'detector', 'read_patt_num', 'read_patt', 'grating', 'subarray', 'pupil', 'exp_type') list_filter = ('viewed', 'instrument', 'proposal', 'obsnum') diff --git a/jwql/website/apps/jwql/models.py b/jwql/website/apps/jwql/models.py index 06d91b696..8a4c5bbd2 100644 --- a/jwql/website/apps/jwql/models.py +++ b/jwql/website/apps/jwql/models.py @@ -102,13 +102,14 @@ class RootFileInfo(models.Model): root_name = models.TextField(primary_key=True, max_length=300) viewed = models.BooleanField(default=False) filter = models.CharField(max_length=7, help_text="Instrument name", default='') - aperature = models.CharField(max_length=40, help_text="Aperature", default='') + aperture = models.CharField(max_length=40, help_text="Aperture", default='') detector = models.CharField(max_length=40, help_text="Detector", default='') read_patt_num = models.IntegerField(help_text='Read Pattern Number', default=0) read_patt = models.CharField(max_length=40, help_text="Read Pattern", default='') grating = models.CharField(max_length=40, help_text="Grating", default='') subarray = models.CharField(max_length=40, help_text="Subarray", default='') pupil = models.CharField(max_length=40, help_text="Pupil", default='') + exp_type = models.CharField(max_length=40, help_text="Exposure Type", default='') # Metadata class Meta: From b35114142d8fa3dbffd773a0c64c8dcb0aafcb91 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 13 Feb 2023 17:09:46 -0500 Subject: [PATCH 086/449] Add instrument viewed status API --- jwql/tests/test_api_views.py | 1 + jwql/tests/test_data_containers.py | 85 ++++++++- jwql/website/apps/jwql/api_views.py | 36 ++++ jwql/website/apps/jwql/data_containers.py | 161 +++++++++++++++--- .../apps/jwql/templates/api_landing.html | 4 +- jwql/website/apps/jwql/urls.py | 23 ++- 6 files changed, 274 insertions(+), 36 deletions(-) diff --git a/jwql/tests/test_api_views.py b/jwql/tests/test_api_views.py index 8c472a469..964a3333f 100644 --- a/jwql/tests/test_api_views.py +++ b/jwql/tests/test_api_views.py @@ -40,6 +40,7 @@ # Instrument-specific URLs for instrument in JWST_INSTRUMENT_NAMES: urls.append('api/{}/proposals/'.format(instrument)) # instrument_proposals + urls.append('api/{}/viewed/'.format(instrument)) # instrument_viewed # Proposal-specific URLs proposals = ['2640', '02733', '1541', '02589'] diff --git a/jwql/tests/test_data_containers.py b/jwql/tests/test_data_containers.py index 0807e78d8..e53cab5dc 100644 --- a/jwql/tests/test_data_containers.py +++ b/jwql/tests/test_data_containers.py @@ -7,6 +7,10 @@ ------- - Matthew Bourque + - Mees Fix + - Bryan Hilbert + - Bradley Sappington + - Melanie Clarke Use --- @@ -69,8 +73,8 @@ def test_get_filenames_by_instrument(): @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') def test_get_filenames_by_proposal(): """Tests the ``get_filenames_by_proposal`` function.""" - - filenames = data_containers.get_filenames_by_proposal('1068') + pid = '2589' + filenames = data_containers.get_filenames_by_proposal(pid) assert isinstance(filenames, list) assert len(filenames) > 0 @@ -78,12 +82,63 @@ def test_get_filenames_by_proposal(): @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') def test_get_filenames_by_rootname(): """Tests the ``get_filenames_by_rootname`` function.""" - - filenames = data_containers.get_filenames_by_rootname('jw01068001001_02102_00001_nrcb1') + rname = 'jw02589006001_04101_00001-seg002_nrs2' + filenames = data_containers.get_filenames_by_rootname(rname) assert isinstance(filenames, list) assert len(filenames) > 0 +@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') +@pytest.mark.parametrize('pid,rname,success', + [('2589', None, True), + (None, 'jw02589006001_04101_00001-seg002_nrs2', True), + ('2589', 'jw02589006001_04101_00001-seg002_nrs2', True), + (None, None, False)]) +def test_get_filesystem_filenames(pid, rname, success): + """Tests the ``get_filesystem_filenames`` function.""" + filenames = data_containers.get_filesystem_filenames( + proposal=pid, rootname=rname) + assert isinstance(filenames, list) + if not success: + assert len(filenames) == 0 + else: + assert len(filenames) > 0 + + # check specific file_types + fits_files = [f for f in filenames if f.endswith('.fits')] + assert len(fits_files) < len(filenames) + + fits_filenames = data_containers.get_filesystem_filenames( + proposal=pid, rootname=rname, file_types=['fits']) + assert isinstance(fits_filenames, list) + assert len(fits_filenames) > 0 + assert len(fits_filenames) == len(fits_files) + + +@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') +def test_get_filesystem_filenames_options(): + """Tests the ``get_filesystem_filenames`` function.""" + pid = '2589' + + # basenames only + filenames = data_containers.get_filesystem_filenames( + proposal=pid, full_path=False, file_types=['fits']) + assert not os.path.isfile(filenames[0]) + + # full path + filenames = data_containers.get_filesystem_filenames( + proposal=pid, full_path=True, file_types=['fits']) + assert os.path.isfile(filenames[0]) + + # sorted + sorted_filenames = data_containers.get_filesystem_filenames( + proposal=pid, sort_names=True, file_types=['fits']) + unsorted_filenames = data_containers.get_filesystem_filenames( + proposal=pid, sort_names=False, file_types=['fits']) + assert sorted_filenames != unsorted_filenames + assert sorted_filenames == sorted(unsorted_filenames) + + @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') def test_get_header_info(): """Tests the ``get_header_info`` function.""" @@ -115,6 +170,28 @@ def test_get_instrument_proposals(): assert len(proposals) > 0 +@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') +@pytest.mark.parametrize('keys', [None, [], + ['proposal', 'obsnum', 'other', + 'prop_id', 'obsstart']]) +def test_get_instrument_viewed(keys): + """Tests the ``get_instrument_viewed`` function.""" + + viewed = data_containers.get_instrument_viewed('nirspec', keys=keys) + assert isinstance(viewed, list) + assert len(viewed) > 0 + first_file = viewed[0] + assert first_file['root_name'] != '' + assert isinstance(first_file['viewed'], bool) + if keys is not None: + assert len(first_file) == 2 + len(keys) + for key in keys: + assert key in first_file + else: + # only root name and viewed by default + assert len(first_file) == 2 + + @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') def test_get_preview_images_by_proposal(): """Tests the ``get_preview_images_by_proposal`` function.""" diff --git a/jwql/website/apps/jwql/api_views.py b/jwql/website/apps/jwql/api_views.py index a0ef8b30b..719c624f7 100644 --- a/jwql/website/apps/jwql/api_views.py +++ b/jwql/website/apps/jwql/api_views.py @@ -50,6 +50,7 @@ from .data_containers import get_filenames_by_proposal from .data_containers import get_filenames_by_rootname from .data_containers import get_instrument_proposals +from .data_containers import get_instrument_viewed from .data_containers import get_preview_images_by_proposal from .data_containers import get_preview_images_by_rootname from .data_containers import get_thumbnails_by_proposal @@ -137,6 +138,41 @@ def instrument_proposals(request, inst): return JsonResponse({'proposals': proposals}, json_dumps_params={'indent': 2}) +def instrument_viewed(request, inst): + """Return a table of viewed information for the given instrument. + + 'Viewed' indicates whether an observation is new or has been reviewed + for QA. In addition to 'filename', and 'viewed', observation + descriptors included in the table are currently 'proposal', 'obsnum', + 'number_of_files', 'exptypes', 'obsstart', 'obsend'. + + Parameters + ---------- + request : HttpRequest object + Incoming request from the webpage. + inst : str + The JWST instrument of interest. + + Returns + ------- + JsonResponse object + Outgoing response sent to the webpage + """ + # TODO: define more useful keys by instrument in config + # currently, optional keys are just the values available + # in local models + optional_keys = ['proposal', 'obsnum', 'number_of_files', + 'exptypes', 'obsstart', 'obsend'] + full_keys = ['root_name', 'viewed'] + optional_keys + + # get all observation viewed status from file info model + # and join with observation descriptors + viewed = get_instrument_viewed(inst, keys=optional_keys) + return JsonResponse({'instrument': inst, + 'keys': full_keys, + 'viewed': viewed}, json_dumps_params={'indent': 2}) + + def preview_images_by_proposal(request, proposal): """Return a list of available preview images in the filesystem for the given ``proposal``. diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index c15fad920..1928729a7 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -13,6 +13,8 @@ - Teagan King - Bryan Hilbert - Maria Pena-Guerrero + - Bradley Sappington + - Melanie Clarke Use --- @@ -532,7 +534,7 @@ def get_filenames_by_instrument(instrument, proposal, observation_id=None, restr filenames in this file will be used rather than calling mask_query_filenames_by_instrument. This can save a significant amount of time when the number of files is large. query_response : dict - Dictionary with "data" key ontaining a list of filenames. This is assumed to + Dictionary with "data" key containing a list of filenames. This is assumed to essentially be the returned value from a call to mast_query_filenames_by_instrument. If this is provided, the call to that function is skipped, which can save a significant amount of time. @@ -598,7 +600,7 @@ def mast_query_filenames_by_instrument(instrument, proposal_id, observation_id=N Proposal ID number to use to filter the results observation_id : str Observation ID number to use to filter the results. If None, all files for the ``proposal_id`` are - retreived + retrieved other_columns : list List of other columns to return from the MAST query @@ -625,6 +627,79 @@ def mast_query_filenames_by_instrument(instrument, proposal_id, observation_id=N return result +def get_filesystem_filenames(proposal=None, rootname=None, + file_types=None, full_path=False, + sort_names=True): + """Return a list of filenames on the filesystem. + + One of proposal or rootname must be specified. If both are + specified, only proposal is used. + + Parameters + ---------- + proposal : str, optional + The one- to five-digit proposal number (e.g. ``88600``). + rootname : str, optional + The rootname of interest (e.g. + ``jw86600008001_02101_00007_guider2``). + file_types : list of str, optional + If provided, only matching file extension types will be + returned (e.g. ['fits', 'jpg']). + full_path : bool, optional + If set, the full path to the file will be returned instead + of the basename. + sort_names : bool, optional + If set, the returned files are sorted. + + Returns + ------- + filenames : list + A list of filenames associated with the given ``rootname``. + """ + if proposal is not None: + proposal_string = '{:05d}'.format(int(proposal)) + filenames = glob.glob( + os.path.join(FILESYSTEM_DIR, 'public', + 'jw{}'.format(proposal_string), '*/*')) + filenames.extend(glob.glob( + os.path.join(FILESYSTEM_DIR, 'proprietary', + 'jw{}'.format(proposal_string), '*/*'))) + elif rootname is not None: + proposal_dir = rootname[0:7] + observation_dir = rootname.split('_')[0] + filenames = glob.glob( + os.path.join(FILESYSTEM_DIR, 'public', proposal_dir, + observation_dir, '{}*'.format(rootname))) + filenames.extend(glob.glob( + os.path.join(FILESYSTEM_DIR, 'proprietary', proposal_dir, + observation_dir, '{}*'.format(rootname)))) + else: + logging.warning("Must provide either proposal or rootname; " + "no files returned.") + filenames = [] + + # check suffix and file type + good_filenames = [] + for filename in filenames: + split_file = os.path.splitext(filename) + + # certain suffixes are always ignored + test_suffix = split_file[0].split('_')[-1] + if test_suffix not in IGNORED_SUFFIXES: + + # check against additional file type requirement + test_type = split_file[-1].lstrip('.') + if file_types is None or test_type in file_types: + if full_path: + good_filenames.append(filename) + else: + good_filenames.append(os.path.basename(filename)) + + if sort_names: + good_filenames.sort() + return good_filenames + + def get_filenames_by_proposal(proposal): """Return a list of filenames that are available in the filesystem for the given ``proposal``. @@ -639,16 +714,7 @@ def get_filenames_by_proposal(proposal): filenames : list A list of filenames associated with the given ``proposal``. """ - - proposal_string = '{:05d}'.format(int(proposal)) - filenames = glob.glob(os.path.join(FILESYSTEM_DIR, 'public', 'jw{}'.format(proposal_string), '*/*')) - filenames.extend(glob.glob(os.path.join(FILESYSTEM_DIR, 'proprietary', 'jw{}'.format(proposal_string), '*/*'))) - - # Certain suffixes are always ignored - filenames = [filename for filename in filenames if os.path.splitext(filename)[0].split('_')[-1] not in IGNORED_SUFFIXES] - filenames = sorted([os.path.basename(filename) for filename in filenames]) - - return filenames + return get_filesystem_filenames(proposal=proposal) def get_filenames_by_rootname(rootname): @@ -666,18 +732,7 @@ def get_filenames_by_rootname(rootname): filenames : list A list of filenames associated with the given ``rootname``. """ - - proposal_dir = rootname[0:7] - observation_dir = rootname.split('_')[0] - - filenames = glob.glob(os.path.join(FILESYSTEM_DIR, 'public', proposal_dir, observation_dir, '{}*'.format(rootname))) - filenames.extend(glob.glob(os.path.join(FILESYSTEM_DIR, 'proprietary', proposal_dir, observation_dir, '{}*'.format(rootname)))) - - # Certain suffixes are always ignored - filenames = [filename for filename in filenames if os.path.splitext(filename)[0].split('_')[-1] not in IGNORED_SUFFIXES] - filenames = sorted([os.path.basename(filename) for filename in filenames]) - - return filenames + return get_filesystem_filenames(rootname=rootname) def get_header_info(filename, filetype): @@ -869,6 +924,64 @@ def get_instrument_proposals(instrument): return inst_proposals +def get_instrument_viewed(instrument, keys=None): + """Return a table of viewed information for the given instrument. + + Parameters + ---------- + instrument : str + Name of the JWST instrument. + keys : list of str, optional + Additional FITS key names for information to return. + + Returns + ------- + viewed : list + List of viewed information by observation for the given instrument. + """ + # standardize input + inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[instrument] + if keys is None: + keys = [] + + # configure some special keys to avoid table name conflicts + special_keys = {'proposal': 'root', + 'obsnum': 'observation', + 'prop_id': 'proposal'} + + # get files by instrument from local model + root_file_info = RootFileInfo.objects.filter(instrument=inst) + + viewed = [] + for root_file in root_file_info: + # for now, report viewed by root name only. + # if specific files are needed, use get_filesystem_files + result = {'root_name': root_file.root_name, + 'viewed': root_file.viewed} + for key in keys: + try: + # override root file default if needed + if key in special_keys and special_keys[key] == 'observation': + result[key] = getattr(root_file.obsnum, key) + elif key in special_keys and special_keys[key] == 'proposal': + result[key] = getattr(root_file.obsnum.proposal, key) + else: + # try the root file table + result[key] = getattr(root_file, key) + except AttributeError: + try: + # try the observation table + result[key] = getattr(root_file.obsnum, key) + except AttributeError: + try: + # try the proposal table + result[key] = getattr(root_file.obsnum.proposal, key) + except AttributeError: + result[key] = '' + viewed.append(result) + return viewed + + def get_preview_images_by_proposal(proposal): """Return a list of preview images available in the filesystem for the given ``proposal``. diff --git a/jwql/website/apps/jwql/templates/api_landing.html b/jwql/website/apps/jwql/templates/api_landing.html index f803cec4f..bfb6b27bb 100644 --- a/jwql/website/apps/jwql/templates/api_landing.html +++ b/jwql/website/apps/jwql/templates/api_landing.html @@ -51,9 +51,11 @@

    List of Available Services

  • Filenames by Rootname (https://jwql.stsci.edu/api/<rootname>/filenames/)
  • Preview Images by Rootname (https://jwql.stsci.edu/api/<rootname>/preview_images/)
  • Thumbnails by Rootname (https://jwql.stsci.edu/api/<rootname>/thumbnails/)
    • +
  • Viewed Status by Instrument (https://jwql.stsci.edu/api/<instrument>/viewed/)
    • +

    Where <instrument>, <rootname>, <proposal> are the values for the instrument name, rootname and proposal ID respectivly

    -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/jwql/website/apps/jwql/urls.py b/jwql/website/apps/jwql/urls.py index 739e60f17..2c6dad092 100644 --- a/jwql/website/apps/jwql/urls.py +++ b/jwql/website/apps/jwql/urls.py @@ -105,11 +105,20 @@ # REST API views path('api/proposals/', api_views.all_proposals, name='all_proposals'), - re_path(r'^api/(?P({}))/proposals/$'.format(instruments), api_views.instrument_proposals, name='instrument_proposals'), - re_path(r'^api/(?P[\d]{1,5})/filenames/$', api_views.filenames_by_proposal, name='filenames_by_proposal'), - re_path(r'^api/(?P[\d]{1,5})/preview_images/$', api_views.preview_images_by_proposal, name='preview_images_by_proposal'), - re_path(r'^api/(?P[\d]{1,5})/thumbnails/$', api_views.thumbnails_by_proposal, name='preview_images_by_proposal'), - re_path(r'^api/(?P[\w]+)/filenames/$', api_views.filenames_by_rootname, name='filenames_by_rootname'), - re_path(r'^api/(?P[\w]+)/preview_images/$', api_views.preview_images_by_rootname, name='preview_images_by_rootname'), - re_path(r'^api/(?P[\w]+)/thumbnails/$', api_views.thumbnail_by_rootname, name='thumbnail_by_rootname'), + re_path(r'^api/(?P({}))/proposals/$'.format(instruments), + api_views.instrument_proposals, name='instrument_proposals'), + re_path(r'^api/(?P[\d]{1,5})/filenames/$', + api_views.filenames_by_proposal, name='filenames_by_proposal'), + re_path(r'^api/(?P[\d]{1,5})/preview_images/$', + api_views.preview_images_by_proposal, name='preview_images_by_proposal'), + re_path(r'^api/(?P[\d]{1,5})/thumbnails/$', + api_views.thumbnails_by_proposal, name='preview_images_by_proposal'), + re_path(r'^api/(?P[\w]+)/filenames/$', + api_views.filenames_by_rootname, name='filenames_by_rootname'), + re_path(r'^api/(?P[\w]+)/preview_images/$', + api_views.preview_images_by_rootname, name='preview_images_by_rootname'), + re_path(r'^api/(?P[\w]+)/thumbnails/$', + api_views.thumbnail_by_rootname, name='thumbnail_by_rootname'), + re_path(r'^api/(?P({}))/viewed/$'.format(instruments), + api_views.instrument_viewed, name='instrument_viewed'), ] From 3a7d30971c47f2a099fb5dad7fec2b05f5db8c53 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Tue, 14 Feb 2023 11:16:17 -0500 Subject: [PATCH 087/449] Add new/viewed/all looks API options --- jwql/tests/test_api_views.py | 12 +++- jwql/tests/test_data_containers.py | 14 ++-- jwql/website/apps/jwql/api_views.py | 65 +++++++++++++++++-- jwql/website/apps/jwql/data_containers.py | 49 +++++++------- .../apps/jwql/templates/api_landing.html | 4 +- jwql/website/apps/jwql/urls.py | 4 ++ 6 files changed, 108 insertions(+), 40 deletions(-) diff --git a/jwql/tests/test_api_views.py b/jwql/tests/test_api_views.py index 964a3333f..40567a64d 100644 --- a/jwql/tests/test_api_views.py +++ b/jwql/tests/test_api_views.py @@ -7,6 +7,7 @@ - Matthew Bourque - Bryan Hilbert + - Melanie Clarke Use --- @@ -40,7 +41,9 @@ # Instrument-specific URLs for instrument in JWST_INSTRUMENT_NAMES: urls.append('api/{}/proposals/'.format(instrument)) # instrument_proposals + urls.append('api/{}/looks/'.format(instrument)) # instrument_looks urls.append('api/{}/viewed/'.format(instrument)) # instrument_viewed + urls.append('api/{}/new/'.format(instrument)) # instrument_new # Proposal-specific URLs proposals = ['2640', '02733', '1541', '02589'] @@ -90,7 +93,12 @@ def test_api_views(url): try: data = json.loads(url.read().decode()) - assert len(data[data_type]) > 0 - except (http.client.IncompleteRead) as e: + + # viewed data depends on local database contents + # so may return an empty result + if data_type != 'viewed': + assert len(data[data_type]) > 0 + + except http.client.IncompleteRead as e: data = e.partial assert len(data) > 0 diff --git a/jwql/tests/test_data_containers.py b/jwql/tests/test_data_containers.py index e53cab5dc..eb9bece3b 100644 --- a/jwql/tests/test_data_containers.py +++ b/jwql/tests/test_data_containers.py @@ -174,13 +174,13 @@ def test_get_instrument_proposals(): @pytest.mark.parametrize('keys', [None, [], ['proposal', 'obsnum', 'other', 'prop_id', 'obsstart']]) -def test_get_instrument_viewed(keys): - """Tests the ``get_instrument_viewed`` function.""" +def test_get_instrument_looks(keys): + """Tests the ``get_instrument_looks`` function.""" - viewed = data_containers.get_instrument_viewed('nirspec', keys=keys) - assert isinstance(viewed, list) - assert len(viewed) > 0 - first_file = viewed[0] + looks = data_containers.get_instrument_looks('nirspec', keys=keys) + assert isinstance(looks, list) + assert len(looks) > 0 + first_file = looks[0] assert first_file['root_name'] != '' assert isinstance(first_file['viewed'], bool) if keys is not None: @@ -188,7 +188,7 @@ def test_get_instrument_viewed(keys): for key in keys: assert key in first_file else: - # only root name and viewed by default + # only root name and looks by default assert len(first_file) == 2 diff --git a/jwql/website/apps/jwql/api_views.py b/jwql/website/apps/jwql/api_views.py index 719c624f7..4a7fa7ac1 100644 --- a/jwql/website/apps/jwql/api_views.py +++ b/jwql/website/apps/jwql/api_views.py @@ -50,7 +50,7 @@ from .data_containers import get_filenames_by_proposal from .data_containers import get_filenames_by_rootname from .data_containers import get_instrument_proposals -from .data_containers import get_instrument_viewed +from .data_containers import get_instrument_looks from .data_containers import get_preview_images_by_proposal from .data_containers import get_preview_images_by_rootname from .data_containers import get_thumbnails_by_proposal @@ -138,8 +138,8 @@ def instrument_proposals(request, inst): return JsonResponse({'proposals': proposals}, json_dumps_params={'indent': 2}) -def instrument_viewed(request, inst): - """Return a table of viewed information for the given instrument. +def instrument_looks(request, inst, viewed=None): + """Return a table of looks information for the given instrument. 'Viewed' indicates whether an observation is new or has been reviewed for QA. In addition to 'filename', and 'viewed', observation @@ -152,6 +152,10 @@ def instrument_viewed(request, inst): Incoming request from the webpage. inst : str The JWST instrument of interest. + viewed : bool, optional + If set to None, all viewed values are returned. If set to + True, only viewed data is returned. If set to False, only + new data is returned. Returns ------- @@ -165,12 +169,61 @@ def instrument_viewed(request, inst): 'exptypes', 'obsstart', 'obsend'] full_keys = ['root_name', 'viewed'] + optional_keys - # get all observation viewed status from file info model + # get all observation looks from file info model # and join with observation descriptors - viewed = get_instrument_viewed(inst, keys=optional_keys) + looks = get_instrument_looks(inst, keys=optional_keys, viewed=viewed) + + # return results by api key + if viewed is None: + key = 'looks' + elif viewed: + key = 'viewed' + else: + key = 'new' + return JsonResponse({'instrument': inst, 'keys': full_keys, - 'viewed': viewed}, json_dumps_params={'indent': 2}) + key: looks}, json_dumps_params={'indent': 2}) + + +def instrument_viewed(request, inst): + """Return a table of information on viewed data for the given instrument. + + Calls `instrument_looks` with viewed=True. + + Parameters + ---------- + request : HttpRequest object + Incoming request from the webpage. + inst : str + The JWST instrument of interest. + + Returns + ------- + JsonResponse object + Outgoing response sent to the webpage + """ + return instrument_looks(request, inst, viewed=True) + + +def instrument_new(request, inst): + """Return a table of information on new data for the given instrument. + + Calls `instrument_looks` with viewed=False. + + Parameters + ---------- + request : HttpRequest object + Incoming request from the webpage. + inst : str + The JWST instrument of interest. + + Returns + ------- + JsonResponse object + Outgoing response sent to the webpage + """ + return instrument_looks(request, inst, viewed=False) def preview_images_by_proposal(request, proposal): diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 1928729a7..e110745d1 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -924,8 +924,8 @@ def get_instrument_proposals(instrument): return inst_proposals -def get_instrument_viewed(instrument, keys=None): - """Return a table of viewed information for the given instrument. +def get_instrument_looks(instrument, keys=None, viewed=None): + """Return a table of looks information for the given instrument. Parameters ---------- @@ -933,26 +933,30 @@ def get_instrument_viewed(instrument, keys=None): Name of the JWST instrument. keys : list of str, optional Additional FITS key names for information to return. + viewed : bool, optional + If set to None, all viewed values are returned. If set to + True, only viewed data is returned. If set to False, only + new data is returned. Returns ------- - viewed : list - List of viewed information by observation for the given instrument. + looks : list + List of looks information by observation for the given instrument. """ # standardize input inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[instrument] if keys is None: keys = [] - # configure some special keys to avoid table name conflicts - special_keys = {'proposal': 'root', - 'obsnum': 'observation', - 'prop_id': 'proposal'} - # get files by instrument from local model - root_file_info = RootFileInfo.objects.filter(instrument=inst) + if viewed is None: + root_file_info = RootFileInfo.objects.filter(instrument=inst) + elif viewed: + root_file_info = RootFileInfo.objects.filter(instrument=inst, viewed=True) + else: + root_file_info = RootFileInfo.objects.filter(instrument=inst, viewed=False) - viewed = [] + looks = [] for root_file in root_file_info: # for now, report viewed by root name only. # if specific files are needed, use get_filesystem_files @@ -960,26 +964,23 @@ def get_instrument_viewed(instrument, keys=None): 'viewed': root_file.viewed} for key in keys: try: - # override root file default if needed - if key in special_keys and special_keys[key] == 'observation': - result[key] = getattr(root_file.obsnum, key) - elif key in special_keys and special_keys[key] == 'proposal': - result[key] = getattr(root_file.obsnum.proposal, key) - else: - # try the root file table - result[key] = getattr(root_file, key) + # try the root file table + value = getattr(root_file, key) except AttributeError: try: # try the observation table - result[key] = getattr(root_file.obsnum, key) + value = getattr(root_file.obsnum, key) except AttributeError: try: # try the proposal table - result[key] = getattr(root_file.obsnum.proposal, key) + value = getattr(root_file.obsnum.proposal, key) except AttributeError: - result[key] = '' - viewed.append(result) - return viewed + value = '' + if type(value) not in [str, float, int]: + value = str(value) + result[key] = value + looks.append(result) + return looks def get_preview_images_by_proposal(proposal): diff --git a/jwql/website/apps/jwql/templates/api_landing.html b/jwql/website/apps/jwql/templates/api_landing.html index bfb6b27bb..fdd992b49 100644 --- a/jwql/website/apps/jwql/templates/api_landing.html +++ b/jwql/website/apps/jwql/templates/api_landing.html @@ -51,7 +51,9 @@

    List of Available Services

  • Filenames by Rootname (https://jwql.stsci.edu/api/<rootname>/filenames/)
  • Preview Images by Rootname (https://jwql.stsci.edu/api/<rootname>/preview_images/)
  • Thumbnails by Rootname (https://jwql.stsci.edu/api/<rootname>/thumbnails/)
    • -
  • Viewed Status by Instrument (https://jwql.stsci.edu/api/<instrument>/viewed/)
    • +
  • Look Status by Instrument (https://jwql.stsci.edu/api/<instrument>/looks/)
    • +
  • Viewed Data by Instrument (https://jwql.stsci.edu/api/<instrument>/viewed/)
    • +
  • New Data by Instrument (https://jwql.stsci.edu/api/<instrument>/new/)

    • Where <instrument>, <rootname>, <proposal> are the values for the instrument name, rootname and proposal ID respectivly

    diff --git a/jwql/website/apps/jwql/urls.py b/jwql/website/apps/jwql/urls.py index 2c6dad092..569afe764 100644 --- a/jwql/website/apps/jwql/urls.py +++ b/jwql/website/apps/jwql/urls.py @@ -119,6 +119,10 @@ api_views.preview_images_by_rootname, name='preview_images_by_rootname'), re_path(r'^api/(?P[\w]+)/thumbnails/$', api_views.thumbnail_by_rootname, name='thumbnail_by_rootname'), + re_path(r'^api/(?P({}))/looks/$'.format(instruments), + api_views.instrument_looks, name='instrument_looks'), re_path(r'^api/(?P({}))/viewed/$'.format(instruments), api_views.instrument_viewed, name='instrument_viewed'), + re_path(r'^api/(?P({}))/new/$'.format(instruments), + api_views.instrument_new, name='instrument_new'), ] From 8119940d01a99c923738e649069ccadfedcaf48f Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Wed, 15 Feb 2023 11:43:31 -0500 Subject: [PATCH 088/449] Add CSV report download to archive page --- jwql/tests/test_api_views.py | 4 +- jwql/website/apps/jwql/api_views.py | 71 +++++-------------- jwql/website/apps/jwql/data_containers.py | 2 +- jwql/website/apps/jwql/static/js/jwql.js | 27 +++++++ jwql/website/apps/jwql/templates/archive.html | 7 +- jwql/website/apps/jwql/urls.py | 12 ++-- jwql/website/apps/jwql/views.py | 51 +++++++++++++ 7 files changed, 111 insertions(+), 63 deletions(-) diff --git a/jwql/tests/test_api_views.py b/jwql/tests/test_api_views.py index 40567a64d..e3a2d1ca1 100644 --- a/jwql/tests/test_api_views.py +++ b/jwql/tests/test_api_views.py @@ -42,8 +42,8 @@ for instrument in JWST_INSTRUMENT_NAMES: urls.append('api/{}/proposals/'.format(instrument)) # instrument_proposals urls.append('api/{}/looks/'.format(instrument)) # instrument_looks - urls.append('api/{}/viewed/'.format(instrument)) # instrument_viewed - urls.append('api/{}/new/'.format(instrument)) # instrument_new + urls.append('api/{}/looks/viewed/'.format(instrument)) # instrument_viewed + urls.append('api/{}/looks/new/'.format(instrument)) # instrument_new # Proposal-specific URLs proposals = ['2640', '02733', '1541', '02589'] diff --git a/jwql/website/apps/jwql/api_views.py b/jwql/website/apps/jwql/api_views.py index 4a7fa7ac1..76d9e6d9d 100644 --- a/jwql/website/apps/jwql/api_views.py +++ b/jwql/website/apps/jwql/api_views.py @@ -27,6 +27,7 @@ - Matthew Bourque - Teagan King + - Melanie Clarke Use --- @@ -138,7 +139,7 @@ def instrument_proposals(request, inst): return JsonResponse({'proposals': proposals}, json_dumps_params={'indent': 2}) -def instrument_looks(request, inst, viewed=None): +def instrument_looks(request, inst, status=None): """Return a table of looks information for the given instrument. 'Viewed' indicates whether an observation is new or has been reviewed @@ -152,15 +153,15 @@ def instrument_looks(request, inst, viewed=None): Incoming request from the webpage. inst : str The JWST instrument of interest. - viewed : bool, optional + status : str, optional If set to None, all viewed values are returned. If set to - True, only viewed data is returned. If set to False, only + 'viewed', only viewed data is returned. If set to 'new', only new data is returned. Returns ------- - JsonResponse object - Outgoing response sent to the webpage + JsonResponse + Outgoing response sent to the webpage, depending on return_type. """ # TODO: define more useful keys by instrument in config # currently, optional keys are just the values available @@ -171,59 +172,19 @@ def instrument_looks(request, inst, viewed=None): # get all observation looks from file info model # and join with observation descriptors + viewed = str(status) == 'viewed' looks = get_instrument_looks(inst, keys=optional_keys, viewed=viewed) # return results by api key - if viewed is None: - key = 'looks' - elif viewed: - key = 'viewed' - else: - key = 'new' - - return JsonResponse({'instrument': inst, - 'keys': full_keys, - key: looks}, json_dumps_params={'indent': 2}) - - -def instrument_viewed(request, inst): - """Return a table of information on viewed data for the given instrument. - - Calls `instrument_looks` with viewed=True. - - Parameters - ---------- - request : HttpRequest object - Incoming request from the webpage. - inst : str - The JWST instrument of interest. - - Returns - ------- - JsonResponse object - Outgoing response sent to the webpage - """ - return instrument_looks(request, inst, viewed=True) - - -def instrument_new(request, inst): - """Return a table of information on new data for the given instrument. - - Calls `instrument_looks` with viewed=False. - - Parameters - ---------- - request : HttpRequest object - Incoming request from the webpage. - inst : str - The JWST instrument of interest. - - Returns - ------- - JsonResponse object - Outgoing response sent to the webpage - """ - return instrument_looks(request, inst, viewed=False) + if status is None: + status = 'looks' + + response = JsonResponse({'instrument': inst, + 'keys': full_keys, + 'type': status, + status: looks}, + json_dumps_params={'indent': 2}) + return response def preview_images_by_proposal(request, proposal): diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index e110745d1..efd2707f8 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -944,7 +944,7 @@ def get_instrument_looks(instrument, keys=None, viewed=None): List of looks information by observation for the given instrument. """ # standardize input - inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[instrument] + inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[instrument.lower()] if keys is None: keys = [] diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index 3440a0eb7..435a20a55 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -597,6 +597,33 @@ function toggle_viewed(file_root, base_url) { }); } +/** + * Download filtered data report + * @param {String} inst - The instrument in use + * @param {String} base_url - The base URL for gathering data from the AJAX view. + */ +function download_report(inst, base_url) { + var elem = document.getElementById('download_report_button'); + elem.disabled = true; + + // todo: include all filters + // current look filter + var look_status = document.getElementById('look_dropdownMenuButton') + if (look_status != null) { + look = look_status.innerText.toLowerCase(); + } else { + look = 'all'; + } + if (look.includes('all')) { + report_url = '/' + inst + '/report/'; + } else { + report_url = '/' + inst + '/report/' + look + '/'; + } + + // redirect to download content + window.location = base_url + report_url; + elem.disabled = false; +} /** * Updates various compnents on the archive page diff --git a/jwql/website/apps/jwql/templates/archive.html b/jwql/website/apps/jwql/templates/archive.html index c6ff05f5b..bf8164dc7 100644 --- a/jwql/website/apps/jwql/templates/archive.html +++ b/jwql/website/apps/jwql/templates/archive.html @@ -45,6 +45,11 @@

    Archived {{ inst }} Images

    + +
    + Download as:
    + +
    @@ -73,4 +78,4 @@

    Archived {{ inst }} Images

    -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/jwql/website/apps/jwql/urls.py b/jwql/website/apps/jwql/urls.py index 569afe764..43fc5de58 100644 --- a/jwql/website/apps/jwql/urls.py +++ b/jwql/website/apps/jwql/urls.py @@ -85,6 +85,12 @@ re_path(r'^(?P({}))/archive/$'.format(instruments), views.archived_proposals, name='archive'), re_path(r'^(?P({}))/archive_date_range/$'.format(instruments), views.archive_date_range, name='archive_date_range'), re_path(r'^(?P({}))/unlooked/$'.format(instruments), views.unlooked_images, name='unlooked'), + + re_path(r'^(?P({}))/report/$'.format(instruments), + views.download_report, name='download_report'), + re_path(r'^(?P({}))/report/(?P(viewed|new))/$'.format(instruments), + views.download_report, name='download_report_by_status'), + re_path(r'^(?P({}))/(?P[\w-]+)/$'.format(instruments), views.view_image, name='view_image'), re_path(r'^(?P({}))/(?P.+)_(?P.+)/explore_image/'.format(instruments), views.explore_image, name='explore_image'), re_path(r'^(?P({}))/(?P.+)_(?P.+)/header/'.format(instruments), views.view_header, name='view_header'), @@ -121,8 +127,6 @@ api_views.thumbnail_by_rootname, name='thumbnail_by_rootname'), re_path(r'^api/(?P({}))/looks/$'.format(instruments), api_views.instrument_looks, name='instrument_looks'), - re_path(r'^api/(?P({}))/viewed/$'.format(instruments), - api_views.instrument_viewed, name='instrument_viewed'), - re_path(r'^api/(?P({}))/new/$'.format(instruments), - api_views.instrument_new, name='instrument_new'), + re_path(r'^api/(?P({}))/looks/(?P(viewed|new))/$'.format(instruments), + api_views.instrument_looks, name='instrument_looks_by_status'), ] diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index ab84ab796..41d51098b 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -44,6 +44,7 @@ from collections import defaultdict from copy import deepcopy import csv +import datetime import glob import logging import os @@ -70,6 +71,7 @@ from .data_containers import get_explorer_extension_names from .data_containers import get_header_info from .data_containers import get_image_info +from .data_containers import get_instrument_looks from .data_containers import get_thumbnails_all_instruments from .data_containers import random_404_page from .data_containers import text_scrape @@ -565,6 +567,55 @@ def dashboard(request): return render(request, template, context) +def download_report(request, inst, status='all'): + """Download data report by look status. + + Parameters + ---------- + request : HttpRequest object + Incoming request from the webpage. + inst : str + The JWST instrument of interest. + status : str, optional + If set to None or 'all', all viewed values are returned. If set to + 'viewed', only viewed data is returned. If set to 'new', only + new data is returned. + + Returns + ------- + response : HttpResponse object + Outgoing response sent to the webpage + """ + # TODO: define more useful keys by instrument in config + # currently, optional keys are just the values available + # in local models + optional_keys = ['proposal', 'obsnum', 'number_of_files', + 'exptypes', 'obsstart', 'obsend'] + full_keys = ['root_name', 'viewed'] + optional_keys + + # get all observation looks from file info model + # and join with observation descriptors + if status == 'viewed': + viewed = True + elif status == 'new': + viewed = False + else: + viewed = None + looks = get_instrument_looks(inst, keys=optional_keys, viewed=viewed) + + today = datetime.datetime.now().strftime('%Y%m%d') + filename = f'{inst}_{status}_{today}.csv' + response = HttpResponse(content_type='text/csv') + response['Content-Disposition'] = f'attachment; filename="{filename}"' + + writer = csv.writer(response) + writer.writerow(full_keys) + for row in looks: + writer.writerow(row.values()) + + return response + + def engineering_database(request): """Generate the EDB page. From c0aa03ec25f8211594f898f7093af64aa19529fb Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Wed, 15 Feb 2023 13:03:37 -0500 Subject: [PATCH 089/449] Test new/viewed looks --- jwql/tests/test_data_containers.py | 42 ++++++++++++++++++------------ 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/jwql/tests/test_data_containers.py b/jwql/tests/test_data_containers.py index eb9bece3b..0a5c8df3f 100644 --- a/jwql/tests/test_data_containers.py +++ b/jwql/tests/test_data_containers.py @@ -171,25 +171,35 @@ def test_get_instrument_proposals(): @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') -@pytest.mark.parametrize('keys', [None, [], - ['proposal', 'obsnum', 'other', - 'prop_id', 'obsstart']]) -def test_get_instrument_looks(keys): +@pytest.mark.parametrize('keys,viewed', + [(None, True), (None, False), (None, None), + ([], True), ([], False), ([], None), + (['proposal', 'obsnum', 'other', + 'prop_id', 'obsstart'], True), + (['proposal', 'obsnum', 'other', + 'prop_id', 'obsstart'], False), + (['proposal', 'obsnum', 'other', + 'prop_id', 'obsstart'], None)]) +def test_get_instrument_looks(keys, viewed): """Tests the ``get_instrument_looks`` function.""" - looks = data_containers.get_instrument_looks('nirspec', keys=keys) + looks = data_containers.get_instrument_looks( + 'nirspec', keys=keys, viewed=viewed) assert isinstance(looks, list) - assert len(looks) > 0 - first_file = looks[0] - assert first_file['root_name'] != '' - assert isinstance(first_file['viewed'], bool) - if keys is not None: - assert len(first_file) == 2 + len(keys) - for key in keys: - assert key in first_file - else: - # only root name and looks by default - assert len(first_file) == 2 + + # viewed depends on local database, so may or may not have results + if not viewed: + assert len(looks) > 0 + first_file = looks[0] + assert first_file['root_name'] != '' + assert isinstance(first_file['viewed'], bool) + if keys is not None: + assert len(first_file) == 2 + len(keys) + for key in keys: + assert key in first_file + else: + # only root name and looks by default + assert len(first_file) == 2 @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') From 7c3709b69ffac096e881f2fc307c73ca07ceb1f0 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 17 Feb 2023 10:24:31 -0500 Subject: [PATCH 090/449] Add report key configuration by instrument --- jwql/tests/test_data_containers.py | 25 +++++++---- jwql/utils/constants.py | 12 ++++++ jwql/website/apps/jwql/api_views.py | 18 ++++---- jwql/website/apps/jwql/data_containers.py | 52 +++++++++++++++-------- jwql/website/apps/jwql/views.py | 11 +---- 5 files changed, 74 insertions(+), 44 deletions(-) diff --git a/jwql/tests/test_data_containers.py b/jwql/tests/test_data_containers.py index 0a5c8df3f..6c1fa297d 100644 --- a/jwql/tests/test_data_containers.py +++ b/jwql/tests/test_data_containers.py @@ -183,23 +183,30 @@ def test_get_instrument_proposals(): def test_get_instrument_looks(keys, viewed): """Tests the ``get_instrument_looks`` function.""" - looks = data_containers.get_instrument_looks( - 'nirspec', keys=keys, viewed=viewed) + return_keys, looks = data_containers.get_instrument_looks( + 'nirspec', additional_keys=keys, viewed=viewed) + assert isinstance(return_keys, list) assert isinstance(looks, list) + # returned keys always contains at least root name + assert len(return_keys) > 1 + assert 'root_name' in return_keys + assert 'viewed' in return_keys + + # they may also contain some keys from the instrument + # and any additional keys specified + if keys is not None: + assert len(return_keys) >= 1 + len(keys) + # viewed depends on local database, so may or may not have results if not viewed: assert len(looks) > 0 first_file = looks[0] assert first_file['root_name'] != '' assert isinstance(first_file['viewed'], bool) - if keys is not None: - assert len(first_file) == 2 + len(keys) - for key in keys: - assert key in first_file - else: - # only root name and looks by default - assert len(first_file) == 2 + assert len(first_file) == len(return_keys) + for key in return_keys: + assert key in first_file @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index ada1dc312..b286fb64e 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -418,6 +418,7 @@ # The complete name will have "_{instrument.lower}.txt" added to the end of this. PREVIEW_IMAGE_LISTFILE = 'preview_image_inventory' + # Keep keys defined via class as they are used many places with potential mispellings class QUERY_CONFIG_KEYS: ANOMALIES = "ANOMALIES" @@ -468,6 +469,17 @@ class QUERY_CONFIG_KEYS: 'nirspec': ['NRS', 'NRSRAPID', 'NRSIRS2RAPID', 'NRSRAPIDD2', 'NRSRAPIDD6']} + +REPORT_KEYS_PER_INSTRUMENT = {'fgs': ['proposal', 'obsnum', 'number_of_files', + 'exptypes', 'obsstart', 'obsend'], + 'miri': ['proposal', 'obsnum', 'number_of_files', + 'exptypes', 'obsstart', 'obsend'], + 'nircam': ['proposal', 'obsnum', 'number_of_files', + 'exptypes', 'obsstart', 'obsend'], + 'niriss': ['proposal', 'obsnum', 'number_of_files', + 'exptypes', 'obsstart', 'obsend'], + 'nirspec': ['exptypes', 'viewed']} + SUBARRAYS_ONE_OR_FOUR_AMPS = ['SUBGRISMSTRIPE64', 'SUBGRISMSTRIPE128', 'SUBGRISMSTRIPE256'] # Filename suffixes that need to include the association value in the suffix in diff --git a/jwql/website/apps/jwql/api_views.py b/jwql/website/apps/jwql/api_views.py index 76d9e6d9d..7ae0dced1 100644 --- a/jwql/website/apps/jwql/api_views.py +++ b/jwql/website/apps/jwql/api_views.py @@ -163,24 +163,24 @@ def instrument_looks(request, inst, status=None): JsonResponse Outgoing response sent to the webpage, depending on return_type. """ - # TODO: define more useful keys by instrument in config - # currently, optional keys are just the values available - # in local models - optional_keys = ['proposal', 'obsnum', 'number_of_files', - 'exptypes', 'obsstart', 'obsend'] - full_keys = ['root_name', 'viewed'] + optional_keys + # parse desired look status + if status == 'viewed': + viewed = True + elif status == 'new': + viewed = False + else: + viewed = None # get all observation looks from file info model # and join with observation descriptors - viewed = str(status) == 'viewed' - looks = get_instrument_looks(inst, keys=optional_keys, viewed=viewed) + keys, looks = get_instrument_looks(inst, viewed=viewed) # return results by api key if status is None: status = 'looks' response = JsonResponse({'instrument': inst, - 'keys': full_keys, + 'keys': keys, 'type': status, status: looks}, json_dumps_params={'indent': 2}) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index efd2707f8..57fd26c8e 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -51,9 +51,11 @@ from jwql.database.database_interface import load_connection from jwql.edb.engineering_database import get_mnemonic, get_mnemonic_info, mnemonic_inventory from jwql.utils.utils import check_config_for_key, ensure_dir_exists, filesystem_path, filename_parser, get_config -from jwql.utils.constants import MAST_QUERY_LIMIT, MONITORS, PREVIEW_IMAGE_LISTFILE, THUMBNAIL_LISTFILE, THUMBNAIL_FILTER_LOOK -from jwql.utils.constants import IGNORED_SUFFIXES, INSTRUMENT_SERVICE_MATCH, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_INSTRUMENT_NAMES -from jwql.utils.constants import SUFFIXES_TO_ADD_ASSOCIATION, SUFFIXES_WITH_AVERAGED_INTS, QUERY_CONFIG_KEYS, QUERY_CONFIG_TEMPLATE +from jwql.utils.constants import MAST_QUERY_LIMIT, MONITORS, THUMBNAIL_LISTFILE, THUMBNAIL_FILTER_LOOK +from jwql.utils.constants import IGNORED_SUFFIXES, INSTRUMENT_SERVICE_MATCH +from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_INSTRUMENT_NAMES +from jwql.utils.constants import REPORT_KEYS_PER_INSTRUMENT +from jwql.utils.constants import SUFFIXES_TO_ADD_ASSOCIATION, SUFFIXES_WITH_AVERAGED_INTS, QUERY_CONFIG_KEYS from jwql.utils.credentials import get_mast_token from jwql.utils.utils import get_rootnames_for_instrument_proposal from .forms import InstrumentAnomalySubmitForm @@ -924,31 +926,44 @@ def get_instrument_proposals(instrument): return inst_proposals -def get_instrument_looks(instrument, keys=None, viewed=None): +def get_instrument_looks(instrument, viewed=None, additional_keys=None): """Return a table of looks information for the given instrument. Parameters ---------- instrument : str Name of the JWST instrument. - keys : list of str, optional - Additional FITS key names for information to return. viewed : bool, optional If set to None, all viewed values are returned. If set to True, only viewed data is returned. If set to False, only new data is returned. + additional_keys : list of str, optional + Additional model attribute names for information to return. Returns ------- - looks : list - List of looks information by observation for the given instrument. + keys : list of str + Report values returned for the given instrument. + looks : list of dict + List of looks information by root file for the given instrument. """ # standardize input inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[instrument.lower()] - if keys is None: - keys = [] - # get files by instrument from local model + # required keys + keys = ['root_name'] + + # optional keys by instrument + keys += REPORT_KEYS_PER_INSTRUMENT[inst.lower()] + + # add any additional keys + key_set = set(keys) + if additional_keys is not None: + for key in additional_keys: + if key not in key_set: + keys.append(key) + + # get file info by instrument from local model if viewed is None: root_file_info = RootFileInfo.objects.filter(instrument=inst) elif viewed: @@ -958,10 +973,9 @@ def get_instrument_looks(instrument, keys=None, viewed=None): looks = [] for root_file in root_file_info: - # for now, report viewed by root name only. + # for now, report info by root name only. # if specific files are needed, use get_filesystem_files - result = {'root_name': root_file.root_name, - 'viewed': root_file.viewed} + result = dict() for key in keys: try: # try the root file table @@ -976,11 +990,15 @@ def get_instrument_looks(instrument, keys=None, viewed=None): value = getattr(root_file.obsnum.proposal, key) except AttributeError: value = '' - if type(value) not in [str, float, int]: + + # make sure value can be serialized + if type(value) not in [str, float, int, bool]: value = str(value) + result[key] = value looks.append(result) - return looks + + return keys, looks def get_preview_images_by_proposal(proposal): @@ -1116,7 +1134,7 @@ def get_thumbnails_all_instruments(parameters): Parameters ---------- - parameters: dict of type QUERY_CONFIG_TEMPLATE + parameters: dict A dictionary containing keys of QUERY_CONFIG_KEYS, some of which are dictionaries: diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 41d51098b..d2fd30915 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -586,13 +586,6 @@ def download_report(request, inst, status='all'): response : HttpResponse object Outgoing response sent to the webpage """ - # TODO: define more useful keys by instrument in config - # currently, optional keys are just the values available - # in local models - optional_keys = ['proposal', 'obsnum', 'number_of_files', - 'exptypes', 'obsstart', 'obsend'] - full_keys = ['root_name', 'viewed'] + optional_keys - # get all observation looks from file info model # and join with observation descriptors if status == 'viewed': @@ -601,7 +594,7 @@ def download_report(request, inst, status='all'): viewed = False else: viewed = None - looks = get_instrument_looks(inst, keys=optional_keys, viewed=viewed) + keys, looks = get_instrument_looks(inst, viewed=viewed) today = datetime.datetime.now().strftime('%Y%m%d') filename = f'{inst}_{status}_{today}.csv' @@ -609,7 +602,7 @@ def download_report(request, inst, status='all'): response['Content-Disposition'] = f'attachment; filename="{filename}"' writer = csv.writer(response) - writer.writerow(full_keys) + writer.writerow(keys) for row in looks: writer.writerow(row.values()) From 6e9360c980fe1ab0e6726bd547f3f998fe026185 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 17 Feb 2023 16:39:39 -0500 Subject: [PATCH 091/449] Respect current filter/sort settings for downloaded report --- jwql/tests/test_data_containers.py | 36 +++++++++++++++------ jwql/utils/constants.py | 12 +++---- jwql/website/apps/jwql/api_views.py | 10 +----- jwql/website/apps/jwql/data_containers.py | 39 +++++++++++++++++------ jwql/website/apps/jwql/static/js/jwql.js | 34 +++++++++++--------- jwql/website/apps/jwql/urls.py | 7 +--- jwql/website/apps/jwql/views.py | 21 +++++------- 7 files changed, 90 insertions(+), 69 deletions(-) diff --git a/jwql/tests/test_data_containers.py b/jwql/tests/test_data_containers.py index 6c1fa297d..157a7d5a9 100644 --- a/jwql/tests/test_data_containers.py +++ b/jwql/tests/test_data_containers.py @@ -171,20 +171,30 @@ def test_get_instrument_proposals(): @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') -@pytest.mark.parametrize('keys,viewed', - [(None, True), (None, False), (None, None), - ([], True), ([], False), ([], None), +@pytest.mark.parametrize('keys,viewed,sort_as,exp_type,cat_type', + [(None, None, None, None, None), + (None, 'viewed', None, None, None), + (None, 'new', None, None, None), + (None, None, None, 'NRS_MSATA', None), + # (None, None, None, None, 'CAL'), # cat_type not implemented yet + (['obsstart'], False, 'ascending', None, None), + (['obsstart'], False, 'descending', None, None), + (['obsstart'], False, 'recent', None, None), + ([], True, None, None, None), + ([], False, None, None, None), + ([], None, None, None, None), (['proposal', 'obsnum', 'other', - 'prop_id', 'obsstart'], True), + 'prop_id', 'obsstart'], True, None, None, None), (['proposal', 'obsnum', 'other', - 'prop_id', 'obsstart'], False), + 'prop_id', 'obsstart'], False, None, None, None), (['proposal', 'obsnum', 'other', - 'prop_id', 'obsstart'], None)]) -def test_get_instrument_looks(keys, viewed): + 'prop_id', 'obsstart'], None, None, None, None)]) +def test_get_instrument_looks(keys, viewed, sort_as, exp_type, cat_type): """Tests the ``get_instrument_looks`` function.""" return_keys, looks = data_containers.get_instrument_looks( - 'nirspec', additional_keys=keys, viewed=viewed) + 'nirspec', additional_keys=keys, look=viewed, sort_as=sort_as, + exp_type=exp_type, cat_type=cat_type) assert isinstance(return_keys, list) assert isinstance(looks, list) @@ -199,7 +209,7 @@ def test_get_instrument_looks(keys, viewed): assert len(return_keys) >= 1 + len(keys) # viewed depends on local database, so may or may not have results - if not viewed: + if not viewed == 'viewed': assert len(looks) > 0 first_file = looks[0] assert first_file['root_name'] != '' @@ -208,6 +218,14 @@ def test_get_instrument_looks(keys, viewed): for key in return_keys: assert key in first_file + last_file = looks[-1] + if sort_as == 'ascending': + assert last_file['root_name'] > first_file['root_name'] + elif sort_as == 'recent': + assert last_file['obsstart'] < first_file['obsstart'] + else: + assert last_file['root_name'] < first_file['root_name'] + @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') def test_get_preview_images_by_proposal(): diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index b286fb64e..1e01f7b22 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -470,14 +470,10 @@ class QUERY_CONFIG_KEYS: 'NRSRAPIDD2', 'NRSRAPIDD6']} -REPORT_KEYS_PER_INSTRUMENT = {'fgs': ['proposal', 'obsnum', 'number_of_files', - 'exptypes', 'obsstart', 'obsend'], - 'miri': ['proposal', 'obsnum', 'number_of_files', - 'exptypes', 'obsstart', 'obsend'], - 'nircam': ['proposal', 'obsnum', 'number_of_files', - 'exptypes', 'obsstart', 'obsend'], - 'niriss': ['proposal', 'obsnum', 'number_of_files', - 'exptypes', 'obsstart', 'obsend'], +REPORT_KEYS_PER_INSTRUMENT = {'fgs': ['proposal', 'obsnum', 'exptypes', 'obsstart', 'obsend'], + 'miri': ['proposal', 'obsnum', 'exptypes', 'obsstart', 'obsend'], + 'nircam': ['proposal', 'obsnum', 'exptypes', 'obsstart', 'obsend'], + 'niriss': ['proposal', 'obsnum', 'exptypes', 'obsstart', 'obsend'], 'nirspec': ['exptypes', 'viewed']} SUBARRAYS_ONE_OR_FOUR_AMPS = ['SUBGRISMSTRIPE64', 'SUBGRISMSTRIPE128', 'SUBGRISMSTRIPE256'] diff --git a/jwql/website/apps/jwql/api_views.py b/jwql/website/apps/jwql/api_views.py index 7ae0dced1..aeb32371f 100644 --- a/jwql/website/apps/jwql/api_views.py +++ b/jwql/website/apps/jwql/api_views.py @@ -163,17 +163,9 @@ def instrument_looks(request, inst, status=None): JsonResponse Outgoing response sent to the webpage, depending on return_type. """ - # parse desired look status - if status == 'viewed': - viewed = True - elif status == 'new': - viewed = False - else: - viewed = None - # get all observation looks from file info model # and join with observation descriptors - keys, looks = get_instrument_looks(inst, viewed=viewed) + keys, looks = get_instrument_looks(inst, look=status) # return results by api key if status is None: diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 57fd26c8e..f160d7b21 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -926,17 +926,27 @@ def get_instrument_proposals(instrument): return inst_proposals -def get_instrument_looks(instrument, viewed=None, additional_keys=None): +def get_instrument_looks(instrument, sort_as=None, + look=None, exp_type=None, cat_type=None, + additional_keys=None): """Return a table of looks information for the given instrument. Parameters ---------- instrument : str Name of the JWST instrument. - viewed : bool, optional + sort_as : {'ascending', 'descending', 'recent'} + Sorting method for output table. Ascending and descending + options refer to root file name; recent sorts by observation + start. + look : {'new', 'viewed'}, optional If set to None, all viewed values are returned. If set to - True, only viewed data is returned. If set to False, only + 'viewed', only viewed data is returned. If set to 'new', only new data is returned. + exp_type : str, optional + Set to filter by exposure type. + cat_type : str, optional + Set to filter by proposal category. additional_keys : list of str, optional Additional model attribute names for information to return. @@ -963,13 +973,24 @@ def get_instrument_looks(instrument, viewed=None, additional_keys=None): if key not in key_set: keys.append(key) + # get desired filters + filter_kwargs = dict() + if look is not None: + filter_kwargs['viewed'] = (look == 'viewed') + if exp_type is not None: + filter_kwargs['obsnum__exptypes__contains'] = exp_type + if cat_type is not None: + filter_kwargs['obsnum__proposal__cat_type__contains'] = cat_type + # get file info by instrument from local model - if viewed is None: - root_file_info = RootFileInfo.objects.filter(instrument=inst) - elif viewed: - root_file_info = RootFileInfo.objects.filter(instrument=inst, viewed=True) - else: - root_file_info = RootFileInfo.objects.filter(instrument=inst, viewed=False) + root_file_info = RootFileInfo.objects.filter(instrument=inst, **filter_kwargs) + + # descending by root file is default; + # for other options, sort as desired + if sort_as == 'ascending': + root_file_info = root_file_info.order_by('root_name') + elif sort_as == 'recent': + root_file_info = root_file_info.order_by('-obsnum__obsstart') looks = [] for root_file in root_file_info: diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index 435a20a55..5d93d5d86 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -606,27 +606,31 @@ function download_report(inst, base_url) { var elem = document.getElementById('download_report_button'); elem.disabled = true; - // todo: include all filters - // current look filter - var look_status = document.getElementById('look_dropdownMenuButton') - if (look_status != null) { - look = look_status.innerText.toLowerCase(); - } else { - look = 'all'; - } - if (look.includes('all')) { - report_url = '/' + inst + '/report/'; - } else { - report_url = '/' + inst + '/report/' + look + '/'; - } + // Get sort value + var sort_option = document.getElementById('sort_dropdownMenuButton').innerText; + var options = '?sort_as=' + sort_option.toLowerCase(); + + // Get all filter values + var filter_div = document.getElementById('thumbnail-filter'); + var filters = filter_div.getElementsByClassName('dropdown-toggle'); + + for (var i=0; i < filters.length; i++) { + var name = filters[i].id.split('_dropdownMenuButton')[0]; + var status = filters[i].innerText.toLowerCase(); + if (!status.includes('all')) { + options += '&' + name + '=' + status; + }; + }; + var report_url = '/' + inst + '/report' + options; + console.log('Redirecting to: ' + report_url); - // redirect to download content + // Redirect to download content window.location = base_url + report_url; elem.disabled = false; } /** - * Updates various compnents on the archive page + * Updates various components on the archive page * @param {String} inst - The instrument of interest (e.g. "FGS") * @param {String} base_url - The base URL for gathering data from the AJAX view. */ diff --git a/jwql/website/apps/jwql/urls.py b/jwql/website/apps/jwql/urls.py index 43fc5de58..e03974b90 100644 --- a/jwql/website/apps/jwql/urls.py +++ b/jwql/website/apps/jwql/urls.py @@ -85,12 +85,7 @@ re_path(r'^(?P({}))/archive/$'.format(instruments), views.archived_proposals, name='archive'), re_path(r'^(?P({}))/archive_date_range/$'.format(instruments), views.archive_date_range, name='archive_date_range'), re_path(r'^(?P({}))/unlooked/$'.format(instruments), views.unlooked_images, name='unlooked'), - - re_path(r'^(?P({}))/report/$'.format(instruments), - views.download_report, name='download_report'), - re_path(r'^(?P({}))/report/(?P(viewed|new))/$'.format(instruments), - views.download_report, name='download_report_by_status'), - + re_path(r'^(?P({}))/report/$'.format(instruments), views.download_report, name='download_report'), re_path(r'^(?P({}))/(?P[\w-]+)/$'.format(instruments), views.view_image, name='view_image'), re_path(r'^(?P({}))/(?P.+)_(?P.+)/explore_image/'.format(instruments), views.explore_image, name='explore_image'), re_path(r'^(?P({}))/(?P.+)_(?P.+)/header/'.format(instruments), views.view_header, name='view_header'), diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index d2fd30915..cd49cf080 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -567,7 +567,7 @@ def dashboard(request): return render(request, template, context) -def download_report(request, inst, status='all'): +def download_report(request, inst): """Download data report by look status. Parameters @@ -576,28 +576,23 @@ def download_report(request, inst, status='all'): Incoming request from the webpage. inst : str The JWST instrument of interest. - status : str, optional - If set to None or 'all', all viewed values are returned. If set to - 'viewed', only viewed data is returned. If set to 'new', only - new data is returned. Returns ------- response : HttpResponse object Outgoing response sent to the webpage """ + # check for filter criteria passed in request + kwargs = dict() + for filter_name in ['look', 'exp_type', 'cat_type', 'sort_as']: + kwargs[filter_name] = request.GET.get(filter_name) + # get all observation looks from file info model # and join with observation descriptors - if status == 'viewed': - viewed = True - elif status == 'new': - viewed = False - else: - viewed = None - keys, looks = get_instrument_looks(inst, viewed=viewed) + keys, looks = get_instrument_looks(inst, **kwargs) today = datetime.datetime.now().strftime('%Y%m%d') - filename = f'{inst}_{status}_{today}.csv' + filename = f'{inst}_report_{today}.csv' response = HttpResponse(content_type='text/csv') response['Content-Disposition'] = f'attachment; filename="{filename}"' From 459babcabdaa4123ba7b9c913fbcdd714b9d64a9 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 20 Feb 2023 11:12:47 -0500 Subject: [PATCH 092/449] Default sort text to descending --- jwql/website/apps/jwql/templates/archive.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/templates/archive.html b/jwql/website/apps/jwql/templates/archive.html index bf8164dc7..a089c64ba 100644 --- a/jwql/website/apps/jwql/templates/archive.html +++ b/jwql/website/apps/jwql/templates/archive.html @@ -37,7 +37,7 @@

    Archived {{ inst }} Images

    Sort by:
    - +
    Ascending Descending From 478a06f1458d5f326655670771480999102840d9 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 20 Feb 2023 11:13:13 -0500 Subject: [PATCH 093/449] Case insensitive match --- jwql/website/apps/jwql/data_containers.py | 6 ++++-- jwql/website/apps/jwql/views.py | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index f160d7b21..ab5636c6f 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -978,9 +978,11 @@ def get_instrument_looks(instrument, sort_as=None, if look is not None: filter_kwargs['viewed'] = (look == 'viewed') if exp_type is not None: - filter_kwargs['obsnum__exptypes__contains'] = exp_type + filter_kwargs['obsnum__exptypes__icontains'] = exp_type if cat_type is not None: - filter_kwargs['obsnum__proposal__cat_type__contains'] = cat_type + # filter_kwargs['obsnum__proposal__cat_type__iexact'] = cat_type + # not yet implemented in proposal table + pass # get file info by instrument from local model root_file_info = RootFileInfo.objects.filter(instrument=inst, **filter_kwargs) diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index cd49cf080..8066d86ab 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -16,7 +16,8 @@ - Mees Fix - Bryan Hilbert - Maria Pena-Guerrero - - Bryan Hilbert + - Bradley Sappington + - Melanie Clarke Use @@ -592,7 +593,7 @@ def download_report(request, inst): keys, looks = get_instrument_looks(inst, **kwargs) today = datetime.datetime.now().strftime('%Y%m%d') - filename = f'{inst}_report_{today}.csv' + filename = f'{inst.lower()}_report_{today}.csv' response = HttpResponse(content_type='text/csv') response['Content-Disposition'] = f'attachment; filename="{filename}"' From 4edc22dfe2aacefd742f3c279050b9033d826947 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 20 Feb 2023 12:14:33 -0500 Subject: [PATCH 094/449] Add viewed to report for all instruments --- jwql/utils/constants.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index 1e01f7b22..c0c5c7deb 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -470,10 +470,14 @@ class QUERY_CONFIG_KEYS: 'NRSRAPIDD2', 'NRSRAPIDD6']} -REPORT_KEYS_PER_INSTRUMENT = {'fgs': ['proposal', 'obsnum', 'exptypes', 'obsstart', 'obsend'], - 'miri': ['proposal', 'obsnum', 'exptypes', 'obsstart', 'obsend'], - 'nircam': ['proposal', 'obsnum', 'exptypes', 'obsstart', 'obsend'], - 'niriss': ['proposal', 'obsnum', 'exptypes', 'obsstart', 'obsend'], +REPORT_KEYS_PER_INSTRUMENT = {'fgs': ['proposal', 'obsnum', 'exptypes', + 'obsstart', 'obsend', 'viewed'], + 'miri': ['proposal', 'obsnum', 'exptypes', + 'obsstart', 'obsend', 'viewed'], + 'nircam': ['proposal', 'obsnum', 'exptypes', + 'obsstart', 'obsend', 'viewed'], + 'niriss': ['proposal', 'obsnum', 'exptypes', + 'obsstart', 'obsend', 'viewed'], 'nirspec': ['exptypes', 'viewed']} SUBARRAYS_ONE_OR_FOUR_AMPS = ['SUBGRISMSTRIPE64', 'SUBGRISMSTRIPE128', 'SUBGRISMSTRIPE256'] From beda76d490bc5f15ea8f68f463131fb9e6a5612f Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 20 Feb 2023 12:28:29 -0500 Subject: [PATCH 095/449] Fix url for looks api --- jwql/website/apps/jwql/templates/api_landing.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jwql/website/apps/jwql/templates/api_landing.html b/jwql/website/apps/jwql/templates/api_landing.html index fdd992b49..424360e36 100644 --- a/jwql/website/apps/jwql/templates/api_landing.html +++ b/jwql/website/apps/jwql/templates/api_landing.html @@ -52,8 +52,8 @@

    List of Available Services

  • Preview Images by Rootname (https://jwql.stsci.edu/api/<rootname>/preview_images/)
  • Thumbnails by Rootname (https://jwql.stsci.edu/api/<rootname>/thumbnails/)
  • Look Status by Instrument (https://jwql.stsci.edu/api/<instrument>/looks/)
    • -
  • Viewed Data by Instrument (https://jwql.stsci.edu/api/<instrument>/viewed/)
    • -
  • New Data by Instrument (https://jwql.stsci.edu/api/<instrument>/new/)
    • +
  • Viewed Data by Instrument (https://jwql.stsci.edu/api/<instrument>/looks/viewed/)
    • +
  • New Data by Instrument (https://jwql.stsci.edu/api/<instrument>/looks/new/)

    • Where <instrument>, <rootname>, <proposal> are the values for the instrument name, rootname and proposal ID respectivly

    From 45c589ed560f02704e870bfa35f1f62358755b4b Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Tue, 21 Feb 2023 10:22:53 -0500 Subject: [PATCH 096/449] Fix typo --- jwql/website/apps/jwql/templates/api_landing.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/templates/api_landing.html b/jwql/website/apps/jwql/templates/api_landing.html index 424360e36..5218ce734 100644 --- a/jwql/website/apps/jwql/templates/api_landing.html +++ b/jwql/website/apps/jwql/templates/api_landing.html @@ -56,7 +56,7 @@

    List of Available Services

  • New Data by Instrument (https://jwql.stsci.edu/api/<instrument>/looks/new/)
    • -

    Where <instrument>, <rootname>, <proposal> are the values for the instrument name, rootname and proposal ID respectivly

    +

    where <instrument>, <rootname>, <proposal> are the values for the instrument name, rootname and proposal ID respectively.

    From 2b63f7a9a3f54fbe59ba69057ca25995b2a31ecf Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Tue, 21 Feb 2023 16:45:11 -0500 Subject: [PATCH 097/449] Catch root file / file list mismatch --- jwql/website/apps/jwql/templates/view_image.html | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html index be4d4b36a..4041d7241 100644 --- a/jwql/website/apps/jwql/templates/view_image.html +++ b/jwql/website/apps/jwql/templates/view_image.html @@ -63,13 +63,15 @@

    {{ file_root }}

    - {% set index = file_root_list.index(file_root) %} - {% if index != 0 %} - < Previous - {% endif %} - - {% if index != file_root_list|length - 1 %} - Next > + {% if file_root in file_root_list %} + {% set index = file_root_list.index(file_root) %} + {% if index != 0 %} + < Previous + {% endif %} + + {% if index != file_root_list|length - 1 %} + Next > + {% endif %} {% endif %}
    From bd985c48504bb1d9cf5cbaa45e8b7e4d5f934f4e Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 23 Feb 2023 16:44:05 -0500 Subject: [PATCH 098/449] Tweaks to work with real plots in the jinja template --- jwql/database/database_interface.py | 22 + .../common_monitors/bad_pixel_monitor.py | 4 +- jwql/utils/utils.py | 81 ++++ .../monitor_pages/monitor_bad_pixel_bokeh.py | 419 +++++++++++++++--- .../jwql/monitor_pages/monitor_dark_bokeh.py | 15 +- jwql/website/apps/jwql/monitor_views.py | 25 +- 6 files changed, 469 insertions(+), 97 deletions(-) diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py index e216081c2..541e8d0d4 100644 --- a/jwql/database/database_interface.py +++ b/jwql/database/database_interface.py @@ -374,6 +374,28 @@ def get_monitor_table_constraints(data_dict, table_name): return data_dict +def get_unique_values_per_column(table, column_name): + """Return a list of the unique values from a particular column in the + given table. + + Parameters + ---------- + table : sqlalchemy.orm.decl_api.DeclarativeMeta + SQL table to be searched. (e.g. table = eval('NIRCamDarkPixelStats')) + + column_name : str + Column name within the table to query + + Returns + ------- + distinct_colvals : list + List of unique values in the given column + """ + colvals = session.query(eval(f'table.{column_name}')).distinct() + distinct_colvals = [eval(f'x.{column_name}') for x in colvals] + return sorted(distinct_colvals) + + def monitor_orm_factory(class_name): """Create a ``SQLAlchemy`` ORM Class for a ``jwql`` instrument monitor. diff --git a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py index 4c10e9a06..6094798ae 100755 --- a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py +++ b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py @@ -109,7 +109,7 @@ from jwql.utils.logging_functions import log_info, log_fail from jwql.utils.mast_utils import mast_query from jwql.utils.permissions import set_permissions -from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path +from jwql.utils.utils import copy_files, create_png_from_fits, ensure_dir_exists, get_config, filesystem_path THRESHOLDS_FILE = os.path.join(os.path.split(__file__)[0], 'bad_pixel_file_thresholds.txt') @@ -1061,9 +1061,11 @@ def process(self, illuminated_raw_files, illuminated_slope_files, flat_file_coun if bad_type in badpix_types_from_flats: self.add_bad_pix(bad_location_list, bad_type, illuminated_slope_files, min_illum_time, mid_illum_time, max_illum_time, baseline_file) + flat_png = create_png_from_fits(illuminated_slope_files[0], self.output_dir) elif bad_type in badpix_types_from_darks: self.add_bad_pix(bad_location_list, bad_type, dark_slope_files, min_dark_time, mid_dark_time, max_dark_time, baseline_file) + dark_png = create_png_from_fits(dark_slope_files[0], self.output_dir) here: create_badpix_plot() diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index 49fe6cc03..1a967b1a3 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -38,6 +38,14 @@ import http import jsonschema +from astropy.io import fits +from astropy.stats import sigma_clipped_stats +from bokeh.io import export_png +from bokeh.models import LinearColorMapper, LogColorMapper +from bokeh.plotting import figure +import numpy as np +from PIL import Image + from jwql.utils import permissions from jwql.utils.constants import FILE_AC_CAR_ID_LEN, FILE_AC_O_ID_LEN, FILE_ACT_LEN, \ FILE_DATETIME_LEN, FILE_EPOCH_LEN, FILE_GUIDESTAR_ATTMPT_LEN_MIN, \ @@ -117,6 +125,49 @@ def _validate_config(config_file_dict): ) +def create_png_from_fits(filename, outdir): + """Create and save a png file of the provided file. The file + will be saved with the same filename as the input file, but + with fits replaced by png + + Parameters + ---------- + filename : str + Fits file to be opened and saved as a png + + outdir : str + Output directory to save the png file to + + Returns + ------- + png_file : str + Name of the saved png file + """ + if os.path.isfile(filename): + image = fits.getdata(filename) + ny, nx = image.shape + img_mn, img_med, img_dev = sigma_clipped_stats(image[4: ny - 4, 4: nx - 4]) + + plot = figure(tools='') + plot.x_range.range_padding = plot.y_range.range_padding = 0 + + # Create the color mapper that will be used to scale the image + #mapper = LinearColorMapper(palette='Viridis256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev)) + mapper = LogColorMapper(palette='Viridis256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev)) + + # Plot image + imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, + color_mapper=mapper, level="image") + + # Save the plot in a png + output_filename = os.path.join(outdir, os.path.basename(filename)) + export_png(plot, filename=output_filename) + permissions.set_permissions(output_filename) + return output_filename + else: + return None + + def get_config(): """Return a dictionary that holds the contents of the ``jwql`` config file. @@ -627,6 +678,36 @@ def query_unformat(string): return unsplit_string +def read_png(filename): + """Open the given png file and return as a 3D numpy array + + Parameters + ---------- + filename : str + png file to be opened + + Returns + ------- + data : numpy.ndarray + 3D array representation of the data in the png file + """ + if os.path.isfile(filename): + rgba_img = Image.open(filename).convert('RGBA') + xdim, ydim = rgba_img.size + + # Create an array representation for the image `img`, and an 8-bit "4 + # layer/RGBA" version of it `view`. + img = np.empty((ydim, xdim), dtype=np.uint32) + view = img.view(dtype=np.uint8).reshape((ydim, xdim, 4)) + + # Copy the RGBA image into view, flipping it so it comes right-side up + # with a lower-left origin + view[:,:,:] = np.flipud(np.asarray(rgba_img)) + else: + view = None + return view + + def grouper(iterable, chunksize): """ Take a list of items (iterable), and group it into chunks of chunksize, with the diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py index aa424e459..60ac5f591 100755 --- a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py @@ -20,57 +20,51 @@ import os from astropy.io import fits +from astropy.stats import sigma_clipped_stats +from astropy.time import Time from bokeh.embed import components, file_html from bokeh.layouts import layout -from bokeh.models import ColumnDataSource, Panel, Tabs, Text +from bokeh.models import ColumnDataSource, DatetimeTickFormatter, HoverTool, Legend, LinearColorMapper, Panel, Tabs, Text from bokeh.plotting import figure from bokeh.resources import CDN import datetime import numpy as np +from sqlalchemy import and_, func -from jwql.database.database_interface import session +from jwql.database.database_interface import get_unique_values_per_column, session from jwql.database.database_interface import NIRCamBadPixelQueryHistory, NIRCamBadPixelStats from jwql.database.database_interface import NIRISSBadPixelQueryHistory, NIRISSBadPixelStats from jwql.database.database_interface import MIRIBadPixelQueryHistory, MIRIBadPixelStats from jwql.database.database_interface import NIRSpecBadPixelQueryHistory, NIRSpecBadPixelStats from jwql.database.database_interface import FGSBadPixelQueryHistory, FGSBadPixelStats from jwql.utils.constants import BAD_PIXEL_TYPES, DARKS_BAD_PIXEL_TYPES, FLATS_BAD_PIXEL_TYPES, JWST_INSTRUMENT_NAMES_MIXEDCASE -from jwql.utils.utils import filesystem_path +from jwql.utils.utils import filesystem_path, get_config, read_png SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +OUTPUT_DIR = get_config()['outputs'] -""" -class BadPixMonitorData(): - def __init__(self): - pass - - -class BadPixFigure(): - def __init__(self): - pass - -class BadPixPlots(): - #top-level class - - # Get the data from the database - m = BadPixMonitorData() - - - # Create the figures - for badpix_type in badpix_types: - p = BadPixFigure() -""" - class BadPixelPlots(): """Class for creating the bad pixel plots and figures to be displayed in the web app """ def __init__(self, instrument): self.instrument = instrument.lower() + + # Get the relevant database tables + self.identify_tables() + #self.apertures = self.get_inst_apers() - self.apertures = ['aper1', 'aper2', 'aper3'] + #self.apertures = ['aper1', 'aper2', 'aper3'] + self.detectors = get_unique_values_per_column(self.pixel_table, 'detector') self.run() + def identify_tables(self): + """Determine which database tables as associated with + a given instrument""" + mixed_case_name = JWST_INSTRUMENT_NAMES_MIXEDCASE[self.instrument.lower()] + self.query_table = eval('{}BadPixelQueryHistory'.format(mixed_case_name)) + self.pixel_table = eval('{}BadPixelStats'.format(mixed_case_name)) + def modify_bokeh_saved_html(self): """Given an html string produced by Bokeh when saving bad pixel monitor plots, make tweaks such that the page follows the general JWQL page formatting. @@ -112,21 +106,36 @@ def modify_bokeh_saved_html(self): def run(self): - aperture_panels = [] - for aperture in self.apertures: + + # Right now, the aperture name in the query history table is used as the title of the + # bad pixel plots. The name associated with entries in the bad pixel stats table is the + # detector name. Maybe we should switch to use this. + detector_panels = [] + for detector in self.detectors: + + # Get data from the database + data = BadPixelData(self.pixel_table, self.instrument, detector) + + # Create plots of the location of new bad pixels all_plots = {} - all_plots['new_dark'] = NewBadPixPlot('darks').plot - all_plots['new_flat'] = NewBadPixPlot('flats').plot - all_plots['bad_types'] = {} - for badtype in ['badtype1', 'badtype2', 'badtype3']: - all_plots['bad_types'][badtype] = BadPixTypePlot(badtype).plot + #all_plots['new_dark'] = NewBadPixPlot(detector, data.num_files, data.new_bad_pix, data.background_file, 'darks', + # data.baseline_file, data.obs_start_time, data.obs_end_time).plot + #all_plots['new_flat'] = NewBadPixPlot(detector, data.num_files, data.new_bad_pix, data.background_file, 'flats', + # data.baseline_file, data.obs_start_time, data.obs_end_time).plot + all_plots['new_pix'] = {} + all_plots['trending'] = {} + for badtype in data.badtypes: + all_plots['new_pix'][badtype] = NewBadPixPlot(detector, badtype, data.num_files[badtype], data.new_bad_pix[badtype], + data.background_file[badtype], data.baseline_file[badtype], + data.obs_start_time[badtype], data.obs_end_time[badtype]).plot + all_plots['trending'][badtype] = BadPixTrendPlot(detector, badtype, data.trending_data[badtype]).plot plot_layout = badpix_monitor_plot_layout(all_plots) # Create a tab for each type of plot - aperture_panels.append(Panel(child=plot_layout, title=aperture)) + detector_panels.append(Panel(child=plot_layout, title=detector)) # Build tabs - tabs = Tabs(tabs=aperture_panels) + tabs = Tabs(tabs=detector_panels) # Return tab HTML and JavaScript to web app script, div = components(tabs) @@ -136,6 +145,10 @@ def run(self): temp_vars = {'inst': self.instrument, 'plot_script': script, 'plot_div':div} self.html = file_html(tabs, CDN, f'{self.instrument} bad pix monitor', template_file, temp_vars) + with open(template_file, 'w') as f: + f.writelines(self.html) + + # Modify the html such that our Django-related lines are kept in place, # which will allow the page to keep the same formatting and styling as # the other web app pages @@ -153,45 +166,320 @@ def run(self): class BadPixelData(): """Retrieve bad pixel monitor data from the database """ - def __init__(self): - pass + def __init__(self, pixel_table, instrument, detector): + self.pixel_table = pixel_table + self.instrument = instrument + self.detector = detector + self.trending_data = {} + self.new_bad_pix = {} + self.background_file = {} + self.obs_start_time = {} + self.obs_end_time = {} + self.num_files = {} + self.baseline_file = {} + + #self.identify_tables() + + # Get data for the plot of new bad pixels + self.get_most_recent_entry() + + # Get data for the trending plots + self.badtypes = get_unique_values_per_column(self.pixel_table, 'type') + for badtype in self.badtypes: + self.get_trending_data(badtype) + + + def get_most_recent_entry(self): + """Get all nedded data from the database tables. + Parameters + ---------- + detector : str + Name of detector for which data are retrieved (e.g. NRCA1) + """ + # For the given detector, get the latest entry for each bad pixel type + subq = (session + .query(self.pixel_table.type, func.max(self.pixel_table.entry_date).label("max_created")) + .filter(self.pixel_table.detector == self.detector) + .group_by(self.pixel_table.type) + .subquery() + ) + + query = (session.query(self.pixel_table) + .join(subq, self.pixel_table.entry_date == subq.c.max_created) + ) + + latest_entries_by_type = query.all() + session.close() + + for row in latest_entries_by_type: + self.new_bad_pix[row.type] = (row.x_coord, row.y_coord) + self.background_file[row.type] = row.source_files[0] + self.obs_start_time[row.type] = row.obs_start_time + #self.obs_mid_time[row.type] = row.obs_mid_time + self.obs_end_time[row.type] = row.obs_end_time + self.num_files[row.type] = len(row.source_files) + self.baseline_file[row.type] = row.baseline_file + + def get_trending_data(self, badpix_type): + """ + """ + # The MIRI imaging detector does not line up with the full frame aperture. Fix that here + if self.detector == 'MIRIM': + self.detector = 'MIRIMAGE' + + # NIRCam LW detectors use 'LONG' rather than 5 in the pixel_table + if '5' in self.detector: + self.detector = self.detector.replace('5', 'LONG') + + # Query database for all data in the table with a matching detector and bad pixel type + all_entries_by_type = session.query(self.pixel_table.type, self.pixel_table.detector, func.array_length(self.pixel_table.x_coord, 1), + self.pixel_table.obs_mid_time) \ + .filter(and_(self.pixel_table.detector == self.detector, self.pixel_table.type == badpix_type)) \ + .all() + + # Organize the results + num_pix = [] + times = [] + for i, row in enumerate(all_entries_by_type): + if i == 0: + badtype = row[0] + detector = row[1] + num_pix.append(row[2]) + times.append(row[3]) + self.trending_data[badpix_type] = (detector, num_pix, times) + + # For the given detector, get the latest entry for each bad pixel type, and + # return the bad pixel type, detector, and mean dark image file + #subq = (session + # .query(self.pixel_table.type, func.max(self.pixel_table.entry_date).label("max_created")) + # .filter(self.pixel_table.detector == self.detector & self.pixle_table.type == self.type) + # .group_by(self.pixel_table.type) + # .subquery() + # ) + + #query = (session.query(self.pixel_table.type, self.pixel_table.detector, self.pixel_table.x_coord.length, self.pixel_table.obs_mid_time) + # .join(subq, self.pixel_table.entry_date == subq.c.max_created) + # ) + + #self.most_recent_data = query.all() + session.close() class NewBadPixPlot(): """Create a plot showing the location of newly discovered bad pixels + + Parameters + ---------- """ - def __init__(self, data_type): - self.plot = figure(title=data_type, tools='') - self.plot.x_range.start = 0 - self.plot.x_range.end = 1 - self.plot.y_range.start = 0 - self.plot.y_range.end = 1 + def __init__(self, detector_name, badpix_type, nfiles, coords, background_file, baseline_file, obs_start_time, obs_end_time): + self.detector = detector_name + self.badpix_type = badpix_type + self.num_files = nfiles + self.coords = coords + self.background_file = background_file + self.baseline_file = baseline_file + self.obs_start_time = obs_start_time + self.obs_end_time = obs_end_time + #if self.data_type == 'darks': + # self.badpix_types = DARKS_BAD_PIXEL_TYPES + #elif self.data_type == 'flats': + # self.badpix_types = FLATS_BAD_PIXEL_TYPES + #else: + # raise ValueError("Unrecognized data type. Should be 'flats' or 'darks'") + + # If no background file is given, we fall back to plotting the bad pixels + # on top of an empty image. In that case, we need to know how large the + # detector is, just to create an image of the right size. + if 'MIRI' in self.detector.upper(): + self._detlen = 1024 + else: + self._detlen = 2048 - source = ColumnDataSource(data=dict(x=[0.5], y=[0.5], text=['No data'])) - glyph = Text(x="x", y="y", text="text", angle=0., text_color="navy", text_font_size={'value':'20px'}) - self.plot.add_glyph(source, glyph) + self.create_plot() + def create_plot(self): + """Create the plot by showing background image, and marking the locations + of new bad pixels on top + """ + # Use the first background image you come across for the given bad pixel types + #background_file = None + #start_time = None + #end_time = None + #baseline_file = None + #for bad_type in self.badpix_types: + # if bad_type in self.background_files: + # background_file = self.background_files[bad_type] + # start_time = self.obs_start_time[bad_type] + # end_time = self.obs_end_time[bad_type] + # baseline_file = self.baseline_file[bad_type] + # break + + # Check to see if all of the most recent entries are comparing to the same + # baseline file and have the same obs times. If not, we'll still plot all + + + + # Read in the data, or create an empty array + png_file = self.background_file.replace('.fits', '.png') + full_path_background_file = os.path.join(OUTPUT_DIR, 'bad_pixel_monitor/', png_file) + if os.path.isfile(full_path_background_file): + image = read_png(full_path_background_file) + else: + print(f'Background_file {full_path_background_file} is not a valid file') + #image = np.zeros((self._detlen, self._detlen)) + image = None + #title_text = f'{self.detector}: New bad pix from {self.data_type}. {self.num_files} files.' -class BadPixTypePlot(): - """Create a plot showing the location of a certain type of bad pixel - """ - def __init__(self, badpix_type): - self.plot = figure(title=badpix_type, tools='') - self.plot.x_range.start = 0 - self.plot.x_range.end = 1 - self.plot.y_range.start = 0 - self.plot.y_range.end = 1 + #start_time = Time(float(self.obs_start_time), format='mjd').tt.datetime.strftime("%m/%d/%Y") + #end_time = Time(float(self.obs_end_time), format='mjd').tt.datetime.strftime("%m/%d/%Y") + + start_time = self.obs_start_time.strftime("%m/%d/%Y") + end_time = self.obs_end_time.strftime("%m/%d/%Y") + + title_text = f'{self.detector}: New {self.badpix_type} pix: from {self.num_files} files. {start_time} to {end_time}' + + #ny, nx = image.shape + #img_mn, img_med, img_dev = sigma_clipped_stats(image[4: ny - 4, 4: nx - 4]) + + # Create figure + self.plot = figure(title=title_text, tools='pan,box_zoom,reset,wheel_zoom,save', + x_axis_label="Pixel Number", y_axis_label="Pixel Number",) + self.plot.x_range.range_padding = self.plot.y_range.range_padding = 0 + + # Create the color mapper that will be used to scale the image + #mapper = LinearColorMapper(palette='Viridis256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev)) + + # Plot image + if image is not None: + imgplot = self.plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, color_mapper=mapper, level="image") + else: + # If the background image is not present, manually set the x and y range + self.plot.x_range.start = 0 + self.plot.x_range.end = self._detlen + self.plot.x_range.start = 0 + self.plot.x_range.end = self._detlen - source = ColumnDataSource(data=dict(x=[0.5], y=[0.5], text=['No data'])) - glyph = Text(x="x", y="y", text="text", angle=0., text_color="red", text_font_size={'value':'20px'}) - self.plot.add_glyph(source, glyph) + legend_title = f'Compared to baseline file {os.path.basename(self.baseline_file)}' + # Overplot locations of bad pixels for all bad pixel types + plot_legend = self.overplot_bad_pix() + legend = Legend(items=[plot_legend], + location="center", + orientation='vertical', + title = legend_title) + + self.plot.add_layout(legend, 'below') + + + + + def overplot_bad_pix(self): + """Add a scatter plot of potential new bad pixels to the plot + + Returns + ------- + legend_item : tup + Tuple of legend text and associated plot. Will be converted into + a LegendItem and added to the plot legend + """ + numpix = len(self.coords[0]) + #######TEST - if too many points, cut them way down + if numpix > 2: + self.coords = (self.coords[0][0:2], self.coords[1][0:2]) + numpix = 2 + #########TEST - remove before merging + source = ColumnDataSource(data=dict(pixels_x=self.coords[0], + pixels_y=self.coords[1], + values=[self.badpix_type] * numpix + ) + ) + + # Overplot the bad pixel locations + badpixplots = self.plot.circle(x='pixels_x', y='pixels_y', source=source, color='blue') + + # Create hover tools for the bad pixel types + hover_tool = HoverTool(tooltips=[(f'{self.badpix_type} (x, y):', '(@pixels_x, @pixels_y)'), + ], + renderers=[badpixplots]) + # Add tool to plot + self.plot.tools.append(hover_tool) + + # Add to the legend + text = f"{numpix} potential new {self.badpix_type} pix compared to baseline" + + # Create a tuple to be added to the plot legend + legend_items = (text, [badpixplots]) + return legend_items + + + +class BadPixTrendPlot(): + """Create a plot showing the location of a certain type of bad pixel + """ + def __init__(self, detector_name, badpix_type, entry): + self.detector = detector_name + self.badpix_type = badpix_type + self.detector, self.num_pix, self.time = entry + self.create_plot() + + def create_plot(self): + """Takes the data, places it in a ColumnDataSource, and creates the figure + """ + # This plot will eventually be saved to an html file by Bokeh. However, when + # we place the saved html lines into our jinja template files, we cannot have + # datetime formatted data in the hover tool. This is because the saved Bokeh + # html will contain lines such as "time{%d %m %Y}". But jinja sees this and + # interprets the {%d as an html tag, so when you try to load the page, it + # crashes when it finds a bunch of "d" tags that are unclosed. To get around + # this, we'll create a list of string representations of the datetime values + # here, and place these in the columndatasource to be used with the hover tool + string_times = [e.strftime('%d %b %Y %H:%M') for e in self.time] + + # Create a ColumnDataSource for the main amp to use + source = ColumnDataSource(data=dict(num_pix=self.num_pix, + time=self.time, + string_time=string_times, + value=[self.badpix_type] * len(self.num_pix) + ) + ) + + self.plot = figure(title=f'{self.detector}: New {self.badpix_type} Pixels', tools='pan,box_zoom,reset,wheel_zoom,save', + background_fill_color="#fafafa") + + # Plot the "main" amp data along with error bars + self.plot.scatter(x='time', y='num_pix', fill_color="navy", alpha=0.75, source=source) + + hover_tool = HoverTool(tooltips=[('# Pixels:', '@num_pix'), + ('Date:', '@string_time') + ]) + self.plot.tools.append(hover_tool) + + # Make the x axis tick labels look nice + self.plot.xaxis.formatter = DatetimeTickFormatter(microseconds=["%d %b %H:%M:%S.%3N"], + seconds=["%d %b %H:%M:%S.%3N"], + hours=["%d %b %H:%M"], + days=["%d %b %H:%M"], + months=["%d %b %Y %H:%M"], + years=["%d %b %Y"] + ) + self.plot.xaxis.major_label_orientation = np.pi / 4 + + # Set x range + time_pad = (max(self.time) - min(self.time)) * 0.05 + if time_pad == datetime.timedelta(seconds=0): + time_pad = datetime.timedelta(days=1) + self.plot.x_range.start = min(self.time) - time_pad + self.plot.x_range.end = max(self.time) + time_pad + self.plot.grid.grid_line_color="white" + self.plot.xaxis.axis_label = 'Date' + self.plot.yaxis.axis_label = f'Number of {self.badpix_type} pixels' + def badpix_monitor_plot_layout(plots): """Arrange a set of plots into a bokeh layout. Generate nested lists for @@ -213,6 +501,8 @@ def badpix_monitor_plot_layout(plots): Returns ------- plot_layout : bokeh.layouts.layout + """ + """ # First the plots showing all bad pixel types derived from a given type of # input (darks or flats). If both plots are present, show them side by side. @@ -228,7 +518,7 @@ def badpix_monitor_plot_layout(plots): # Next create a list of plots where each plot shows one flavor of bad pixel plots_per_row = 2 - num_bad_types = len(plots['bad_types']) + num_bad_types = len(plots['trending']) first_col = np.arange(0, num_bad_types, plots_per_row) badtype_lists = [] @@ -243,9 +533,18 @@ def badpix_monitor_plot_layout(plots): # Combine full frame and subarray aperture lists full_list = new_list + badtype_lists + """ + + + + # Create a list of plots where each plot shows one flavor of bad pixel + all_plots = [] + for badtype in plots["trending"]: + rowplots = [plots["new_pix"][badtype], plots["trending"][badtype]] + all_plots.append(rowplots) # Now create a layout that holds the lists - plot_layout = layout(full_list) + plot_layout = layout(all_plots) return plot_layout diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py index 7f286c5be..0b915760f 100755 --- a/jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py @@ -38,7 +38,7 @@ from jwql.database.database_interface import FGSDarkPixelStats, FGSDarkDarkCurrent from jwql.utils.constants import FULL_FRAME_APERTURES from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE -from jwql.utils.utils import get_config +from jwql.utils.utils import get_config, read_png SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) OUTPUTS_DIR = get_config()['outputs'] @@ -235,18 +235,7 @@ def create_plot(self): """ if self.dark_image_picture is not None: if os.path.isfile(self.dark_image_picture): - - rgba_img = Image.open(self.dark_image_picture).convert('RGBA') - xdim, ydim = rgba_img.size - - # Create an array representation for the image `img`, and an 8-bit "4 - # layer/RGBA" version of it `view`. - img = np.empty((ydim, xdim), dtype=np.uint32) - view = img.view(dtype=np.uint8).reshape((ydim, xdim, 4)) - - # Copy the RGBA image into view, flipping it so it comes right-side up - # with a lower-left origin - view[:,:,:] = np.flipud(np.asarray(rgba_img)) + view = read_png(self.dark_image_picture) # Display the 32-bit RGBA image dim = max(xdim, ydim) diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py index bc2b7025f..e31a65c5d 100644 --- a/jwql/website/apps/jwql/monitor_views.py +++ b/jwql/website/apps/jwql/monitor_views.py @@ -62,31 +62,10 @@ def bad_pixel_monitor(request, inst): HttpResponse object Outgoing response sent to the webpage """ - - # Ensure the instrument is correctly capitalized - #inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()] - # Locate the html file for the instrument - #html_file = os.path.join(CONFIG["outputs"], "bad_pixel_monitor", f"{inst.lower()}_bad_pix_plots.html") - html_file = f"{inst.lower()}_bad_pix_plots.html" - - # Read in the html file - #with open(html_file, "r") as obj: - # html = obj.read() - - #tabs_components = bokeh_containers.bad_pixel_monitor_tabs(inst) - - #template = "bad_pixel_monitor.html" - - #context = { - # 'inst': inst, - # 'tabs_components': tabs_components, - #} - - # Return a HTTP response with the template and dictionary of variables - #return render(request, template, context) + #html_file = f"{inst.lower()}_bad_pix_plots.html" + html_file = f"bad_pixel_monitor_test_junk.html" - #return HttpResponse(html) return render(request, html_file) From 547830e3fdc678b4b81203f8854a69efdfdcb678 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 24 Feb 2023 13:13:23 -0500 Subject: [PATCH 099/449] Start work to show only png if too many points --- jwql/utils/constants.py | 6 + .../monitor_pages/monitor_bad_pixel_bokeh.py | 104 ++++++++++++------ jwql/website/apps/jwql/monitor_views.py | 4 +- 3 files changed, 76 insertions(+), 38 deletions(-) diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index ada1dc312..66cae1454 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -143,6 +143,12 @@ DARKS_BAD_PIXEL_TYPES = ['HOT', 'RC', 'OTHER_BAD_PIXEL', 'TELEGRAPH'] FLATS_BAD_PIXEL_TYPES = ['DEAD', 'OPEN', 'ADJ_OPEN', 'LOW_QE'] +# The maximum number of bad pixels allowed on a bad pixel monitor plot. If there +# are more than this number of bad pixels identified for a particular type of +# bad pixel, then the figure is saved as a png rather than an interactive plot, +# in order to reduce the amount of data sent to the browser. +BAD_PIXEL_MONITOR_MAX_POINTS_TO_PLOT = 15000 + # Possible exposure types for dark current data DARK_EXP_TYPES = {'nircam': ['NRC_DARK'], 'niriss': ['NIS_DARK'], diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py index 60ac5f591..0b32cd568 100755 --- a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py @@ -23,6 +23,8 @@ from astropy.stats import sigma_clipped_stats from astropy.time import Time from bokeh.embed import components, file_html +from bokeh.io import export_png +from bokeh.io.export import get_screenshot_as_png from bokeh.layouts import layout from bokeh.models import ColumnDataSource, DatetimeTickFormatter, HoverTool, Legend, LinearColorMapper, Panel, Tabs, Text from bokeh.plotting import figure @@ -37,7 +39,9 @@ from jwql.database.database_interface import MIRIBadPixelQueryHistory, MIRIBadPixelStats from jwql.database.database_interface import NIRSpecBadPixelQueryHistory, NIRSpecBadPixelStats from jwql.database.database_interface import FGSBadPixelQueryHistory, FGSBadPixelStats -from jwql.utils.constants import BAD_PIXEL_TYPES, DARKS_BAD_PIXEL_TYPES, FLATS_BAD_PIXEL_TYPES, JWST_INSTRUMENT_NAMES_MIXEDCASE +from jwql.utils.constants import BAD_PIXEL_MONITOR_MAX_POINTS_TO_PLOT, BAD_PIXEL_TYPES, DARKS_BAD_PIXEL_TYPES +from jwql.utils.constants import FLATS_BAD_PIXEL_TYPES, JWST_INSTRUMENT_NAMES_MIXEDCASE +from jwql.utils.permissions import set_permissions from jwql.utils.utils import filesystem_path, get_config, read_png SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -299,27 +303,10 @@ def __init__(self, detector_name, badpix_type, nfiles, coords, background_file, def create_plot(self): """Create the plot by showing background image, and marking the locations - of new bad pixels on top + of new bad pixels on top. We load a png file of the background image rather + than the original fits file in order to reduce the amount of data in the + final html file. """ - - # Use the first background image you come across for the given bad pixel types - #background_file = None - #start_time = None - #end_time = None - #baseline_file = None - #for bad_type in self.badpix_types: - # if bad_type in self.background_files: - # background_file = self.background_files[bad_type] - # start_time = self.obs_start_time[bad_type] - # end_time = self.obs_end_time[bad_type] - # baseline_file = self.baseline_file[bad_type] - # break - - # Check to see if all of the most recent entries are comparing to the same - # baseline file and have the same obs times. If not, we'll still plot all - - - # Read in the data, or create an empty array png_file = self.background_file.replace('.fits', '.png') full_path_background_file = os.path.join(OUTPUT_DIR, 'bad_pixel_monitor/', png_file) @@ -343,8 +330,16 @@ def create_plot(self): #img_mn, img_med, img_dev = sigma_clipped_stats(image[4: ny - 4, 4: nx - 4]) # Create figure - self.plot = figure(title=title_text, tools='pan,box_zoom,reset,wheel_zoom,save', - x_axis_label="Pixel Number", y_axis_label="Pixel Number",) + # If there are "too many" points then we are going to save the plot as + # a png rather than send all the data to the browser. In that case, we + # don't want to add any tools to the figure + if len(self.coords[0]) <= BAD_PIXEL_MONITOR_MAX_POINTS_TO_PLOT: + tools = 'pan,box_zoom,reset,wheel_zoom,save' + else: + tools = '' + + self.plot = figure(title=title_text, tools=tools, + x_axis_label="Pixel Number", y_axis_label="Pixel Number") self.plot.x_range.range_padding = self.plot.y_range.range_padding = 0 # Create the color mapper that will be used to scale the image @@ -352,6 +347,7 @@ def create_plot(self): # Plot image if image is not None: + mapper, nx, ny are not defined yet imgplot = self.plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, color_mapper=mapper, level="image") else: # If the background image is not present, manually set the x and y range @@ -372,8 +368,15 @@ def create_plot(self): self.plot.add_layout(legend, 'below') - - + # If there are "too many" points, we have already omitted all of the bokeh tools. + # Now we export as a png and place that into the figure, as a way of reducing the + # amount of data sent to the browser. This png will be saved and immediately read + # back in. + #if 1 < 0: + if len(self.coords[0]) > BAD_PIXEL_MONITOR_MAX_POINTS_TO_PLOT: + output_filename = full_path_background_file.replace('.png', f'_{self.badpix_type}_pix.png') + self.switch_to_png(output_filename) + print(f'Switching to png for {self.detector}, {self.badpix_type}, {len(self.coords[0])}') def overplot_bad_pix(self): """Add a scatter plot of potential new bad pixels to the plot @@ -387,9 +390,9 @@ def overplot_bad_pix(self): numpix = len(self.coords[0]) #######TEST - if too many points, cut them way down - if numpix > 2: - self.coords = (self.coords[0][0:2], self.coords[1][0:2]) - numpix = 2 + #if numpix > 2: + # self.coords = (self.coords[0][0:2], self.coords[1][0:2]) + # numpix = 2 #########TEST - remove before merging @@ -401,14 +404,19 @@ def overplot_bad_pix(self): ) # Overplot the bad pixel locations - badpixplots = self.plot.circle(x='pixels_x', y='pixels_y', source=source, color='blue') - - # Create hover tools for the bad pixel types - hover_tool = HoverTool(tooltips=[(f'{self.badpix_type} (x, y):', '(@pixels_x, @pixels_y)'), - ], - renderers=[badpixplots]) - # Add tool to plot - self.plot.tools.append(hover_tool) + badpixplots = self.plot.circle(x='pixels_x', y='pixels_y', source=source, color='blue', + fill_alpha=0.75, line_alpha=0.75, radius=0.5) + + # Create hover tool for the bad pixel type + # If there are "too many" points then we are going to save the plot as + # a png rather than send all the data to the browser. In that case, we + # don't need a hover tool + if numpix <= BAD_PIXEL_MONITOR_MAX_POINTS_TO_PLOT: + hover_tool = HoverTool(tooltips=[(f'{self.badpix_type} (x, y):', '(@pixels_x, @pixels_y)'), + ], + renderers=[badpixplots]) + # Add tool to plot + self.plot.tools.append(hover_tool) # Add to the legend text = f"{numpix} potential new {self.badpix_type} pix compared to baseline" @@ -417,6 +425,30 @@ def overplot_bad_pix(self): legend_items = (text, [badpixplots]) return legend_items + def switch_to_png(self, filename): + """Convert the current Bokeh figure from a figure containing circles to a png + representation. + + Parameters + ---------- + filename : str + Name of file to save the current figure as a png into + """ + # Save the figure as a png + #fig_array = get_screenshot_as_png(self.plot) + + export_png(self.plot, filename=filename) + set_permissions(filename) + + # Read in the png and insert into a replacement figure + fig_array = read_png(filename) + + ydim, xdim, _ = fig_array.shape + dim = max(xdim, ydim) + self.plot = figure(x_range=(0, xdim), y_range=(0, ydim), tools='pan,box_zoom,reset,wheel_zoom,save') + self.plot.image_rgba(image=[fig_array], x=0, y=0, dw=xdim, dh=ydim) + self.plot.xaxis.visible = False + self.plot.yaxis.visible = False class BadPixTrendPlot(): diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py index e31a65c5d..822d58f0f 100644 --- a/jwql/website/apps/jwql/monitor_views.py +++ b/jwql/website/apps/jwql/monitor_views.py @@ -63,8 +63,8 @@ def bad_pixel_monitor(request, inst): Outgoing response sent to the webpage """ # Locate the html file for the instrument - #html_file = f"{inst.lower()}_bad_pix_plots.html" - html_file = f"bad_pixel_monitor_test_junk.html" + html_file = f"{inst.lower()}_bad_pix_plots.html" + #html_file = f"bad_pixel_monitor_test_junk.html" return render(request, html_file) From a29f2219158f2ec42794bb0ee7589a99fd2ca123 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 24 Feb 2023 16:46:03 -0500 Subject: [PATCH 100/449] Initial work on grouping by exposure --- jwql/website/apps/jwql/data_containers.py | 39 +++++-- jwql/website/apps/jwql/static/js/jwql.js | 105 ++++++++++++++---- .../jwql/templates/thumbnails_per_obs.html | 23 ++-- jwql/website/apps/jwql/urls.py | 1 + jwql/website/apps/jwql/views.py | 35 ++++-- 5 files changed, 148 insertions(+), 55 deletions(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 2ccac9f93..b21483666 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -1550,6 +1550,8 @@ def thumbnails_ajax(inst, proposal, obs_num=None): data_dict : dict Dictionary of data needed for the ``thumbnails`` template """ + # TODO - tap call in get_rootnames sometimes fails. + # can eliminate, using the existing filenames instead? # generate the list of all obs of the proposal here, so that the list can be # properly packaged up and sent to the js scripts. but to do this, we need to call @@ -1571,14 +1573,15 @@ def thumbnails_ajax(inst, proposal, obs_num=None): # Get the available files for the instrument filenames, columns = get_filenames_by_instrument(inst, proposal, observation_id=obs_num, other_columns=['expstart', 'exp_type']) + # Get set of unique rootnames rootnames = set(['_'.join(f.split('/')[-1].split('_')[:-1]) for f in filenames]) # Initialize dictionary that will contain all needed data - data_dict = {} - data_dict['inst'] = inst - data_dict['file_data'] = {} - exp_types = [] + data_dict = {'inst': inst, + 'file_data': dict()} + exp_types = set() + exp_groups = set() # Gather data for each rootname, and construct a list of all observations # in the proposal @@ -1591,6 +1594,10 @@ def thumbnails_ajax(inst, proposal, obs_num=None): # The detector keyword is expected in thumbnails_query_ajax() for generating filterable dropdown menus if 'detector' not in filename_dict.keys(): filename_dict['detector'] = 'Unknown' + filename_dict['group_root'] = rootname + else: + group_root = re.sub(rf"_{filename_dict['detector']}$", '', rootname) + filename_dict['group_root'] = group_root # Weed out file types that are not supported by generate_preview_images if 'stage_3' in filename_dict['filename_type']: @@ -1605,14 +1612,19 @@ def thumbnails_ajax(inst, proposal, obs_num=None): 'parallel_seq_id': rootname[16], 'program_id': rootname[2:7], 'visit': rootname[10:13], - 'visit_group': rootname[14:16]} + 'visit_group': rootname[14:16], + 'group_root': rootname[:26]} + + # todo: the following comprehensions loop over all file names + # several times - should be combined, and maybe make the + # filename loop primary # Get list of available filenames and exposure start times. All files with a given # rootname will have the same exposure start time, so just keep the first. available_files = [item for item in filenames if rootname in item] exp_start = [expstart for fname, expstart in zip(filenames, columns['expstart']) if rootname in fname][0] exp_type = [exp_type for fname, exp_type in zip(filenames, columns['exp_type']) if rootname in fname][0] - exp_types.append(exp_type) + exp_types.add(exp_type) # Viewed is stored by rootname in the Model db. Save it with the data_dict # THUMBNAIL_FILTER_LOOK is boolean accessed according to a viewed flag @@ -1620,15 +1632,17 @@ def thumbnails_ajax(inst, proposal, obs_num=None): root_file_info = RootFileInfo.objects.get(root_name=rootname) viewed = THUMBNAIL_FILTER_LOOK[root_file_info.viewed] except RootFileInfo.DoesNotExist: - viewed = THUMBNAIL_FILTER_LOOK[0] + # Add to list of all exposure groups + exp_groups.add(filename_dict['group_root']) + # Add data to dictionary data_dict['file_data'][rootname] = {} data_dict['file_data'][rootname]['filename_dict'] = filename_dict data_dict['file_data'][rootname]['available_files'] = available_files - data_dict['file_data'][rootname]["viewed"] = viewed - data_dict['file_data'][rootname]["exp_type"] = exp_type + data_dict['file_data'][rootname]['viewed'] = viewed + data_dict['file_data'][rootname]['exp_type'] = exp_type data_dict['file_data'][rootname]['thumbnail'] = get_thumbnail_by_rootname(rootname) try: @@ -1653,12 +1667,12 @@ def thumbnails_ajax(inst, proposal, obs_num=None): if proposal is not None: dropdown_menus = {'detector': sorted(detectors), 'look': THUMBNAIL_FILTER_LOOK, - 'exp_type': sorted(set(exp_types))} + 'exp_type': sorted(exp_types)} else: dropdown_menus = {'detector': sorted(detectors), 'proposal': sorted(proposals), 'look': THUMBNAIL_FILTER_LOOK, - 'exp_type': sorted(set(exp_types))} + 'exp_type': sorted(exp_types)} data_dict['tools'] = MONITORS data_dict['dropdown_menus'] = dropdown_menus @@ -1670,8 +1684,9 @@ def thumbnails_ajax(inst, proposal, obs_num=None): data_dict['file_data'] = sorted_file_data - # Add list of observation numbers + # Add list of observation numbers and group roots data_dict['obs_list'] = obs_list + data_dict['exp_groups'] = sorted(exp_groups) return data_dict diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index f0362afcf..4ecc468d4 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -364,6 +364,40 @@ function get_number_or_none(element_id) { return limit; } + +/** + * Group thumbnail display by exposure or file, save group type in session + * @param {String} group_type - The group type + * @param {String} base_url - The base URL for gathering data from the AJAX view. + */ +function group_by_thumbnails(group_type, dropdown_keys, num_fileids, base_url) { + + // Update dropdown menu text and update thumbnails for current setting + //document.getElementById('group_dropdownMenuButton').innerHTML = group_type; + show_only('group', group_type, dropdown_keys, num_fileids, 'thumbnail', base_url); + + // Group the thumbnails accordingly. + + // TODO: actually group thumbnails + var thumbs = $('div#thumbnail-array>div'); + if (group_type == 'Exposure') { + console.log('Group by exposure'); + } else { + console.log('Group by file'); + } + + $.ajax({ + url: base_url + '/ajax/image_group/', + data: { + 'group_type': group_type + }, + error : function(response) { + console.log("session image group update failed"); + } + }); +}; + + /** * If an image is not found, replace with temporary image sized to thumbnail */ @@ -438,7 +472,7 @@ function search() { * @param {Integer} num_fileids - The number of files that are available to display * @param {String} thumbnail_class - The class name of the thumbnails that will be filtered. */ -function show_only(filter_type, value, dropdown_keys, num_fileids, thumbnail_class, find_substring, base_url) { +function show_only(filter_type, value, dropdown_keys, num_fileids, thumbnail_class, base_url) { // Get all filter options from {{dropdown_menus}} variable var all_filters = dropdown_keys.split(','); @@ -446,6 +480,13 @@ function show_only(filter_type, value, dropdown_keys, num_fileids, thumbnail_cla // Update dropdown menu text document.getElementById(filter_type + '_dropdownMenuButton').innerHTML = value; + // Check for grouping setting for special handling + var group_option = document.getElementById('group_dropdownMenuButton') + var group = false; + if (group_option != null) { + group = (group_option.innerText == 'Exposure'); + } + // Determine the current value for each filter var filter_values = []; for (j = 0; j < all_filters.length; j++) { @@ -453,29 +494,34 @@ function show_only(filter_type, value, dropdown_keys, num_fileids, thumbnail_cla filter_values.push(filter_value); } - // Find all thumbnail elements var thumbnails = document.getElementsByClassName(thumbnail_class); // Determine whether or not to display each thumbnail var num_thumbnails_displayed = 0; var list_of_rootnames = ""; + var groups_shown = new Set(); for (i = 0; i < thumbnails.length; i++) { // Evaluate if the thumbnail meets all filter criteria var criteria = []; for (j = 0; j < all_filters.length; j++) { - var filter_attribute = thumbnails[i].getAttribute(all_filters[j]) - var criterion = (filter_values[j].indexOf('All '+ all_filters[j] + 's') >=0) - || (filter_attribute == filter_values[j]) - || (find_substring && filter_attribute.includes(filter_values[j])); + var filter_attribute = thumbnails[i].getAttribute(all_filters[j]); + var criterion = (filter_values[j].indexOf('All '+ all_filters[j] + 's') >=0) + || (filter_attribute.includes(filter_values[j])); criteria.push(criterion); }; - // Only display if all filter criteria are met + // If data are grouped, check if a thumbnail for the group has already been displayed + if (group && groups_shown.has(thumbnails[i].getAttribute('group_root'))) { + criteria.push(false); + } + + // Only display if all criteria are met if (criteria.every(function(r){return r})) { thumbnails[i].style.display = "inline-block"; num_thumbnails_displayed++; list_of_rootnames = list_of_rootnames + thumbnails[i].getAttribute("file_root") + '=' + thumbnails[i].getAttribute("exp_start") + ','; + if (group) { groups_shown.add(thumbnails[i].getAttribute('group_root')); } } else { thumbnails[i].style.display = "none"; } @@ -820,13 +866,6 @@ function update_wata_page(base_url) { filter_options = Array.from(new Set(data.dropdown_menus[filter_type])); num_rootnames = num_items; dropdown_key_list = Object.keys(data.dropdown_menus); - - if (filter_type == "exp_type") { - // Any filters where there may be a list as a string for the attribute to filter on - find_substring = true; - } else { - find_substring = false; - } // Build div content content += '
    '; @@ -834,10 +873,10 @@ function update_wata_page(base_url) { content += '
    '; content += ''; content += '
    '; - content += 'All ' + filter_type + 's'; + content += 'All ' + filter_type + 's'; for (var j = 0; j < filter_options.length; j++) { - content += '' + filter_options[j] + ''; + content += '' + filter_options[j] + ''; }; content += '
    '; @@ -848,6 +887,26 @@ function update_wata_page(base_url) { $("#thumbnail-filter")[0].innerHTML = content; }; +/** + * Updates the group-by-exposure div + * @param {Object} data - The data returned by the update_thumbnails_page AJAX method + */ +function update_group_options(data, base_url, num_items, thumbnail_class) { + + var dropdown_key_list = Object.keys(data.dropdown_menus); + + // Build div content + content = 'Group by:
    ' + content += ''; + content += '
    '; + content += 'Exposure'; + content += 'File'; + content += '
    '; + // Add the content to the div + $("#group-by-exposure")[0].innerHTML = content; +}; + + /** * Change the header extension displayed @@ -945,10 +1004,10 @@ function update_thumbnail_array(data) { // Build div content if (data.inst!="all") { - content = '
    '; + content = '
    '; content += ''; } else { - content = '
    '; + content = '
    '; content += ''; } content += ''; @@ -1002,9 +1061,11 @@ function submit_date_range_form(inst, base_url) { update_show_count(num_thumbnails, 'activities'); update_thumbnail_array(data); update_filter_options(data, base_url, num_thumbnails, 'thumbnail'); + // Do initial sort to match sort button display update_sort_options(data, base_url); sort_by_thumbnails(data.thumbnail_sort, base_url); + // Replace loading screen with the proposal array div document.getElementById("loading").style.display = "none"; document.getElementById("thumbnail-array").style.display = "block"; @@ -1035,11 +1096,11 @@ function submit_date_range_form(inst, base_url) { * @param {String} inst - The instrument of interest (e.g. "FGS") * @param {String} proposal - The proposal number of interest (e.g. "88660") * @param {String} observation - The observation number within the proposal (e.g. "001") - * @param {List} observation_list - List of all observations in this proposal * @param {String} base_url - The base URL for gathering data from the AJAX view. * @param {String} sort - Sort method string saved in session data image_sort + * @param {String} group - Group method string saved in session data image_group */ -function update_thumbnails_per_observation_page(inst, proposal, observation, observation_list, base_url, sort) { +function update_thumbnails_per_observation_page(inst, proposal, observation, base_url, sort, group) { $.ajax({ url: base_url + '/ajax/' + inst + '/archive/' + proposal + '/obs' + observation + '/', success: function(data){ @@ -1049,9 +1110,11 @@ function update_thumbnails_per_observation_page(inst, proposal, observation, obs update_thumbnail_array(data); update_obs_options(data, inst, proposal, observation); update_filter_options(data, base_url, num_thumbnails, 'thumbnail'); + update_group_options(data, base_url, num_thumbnails, 'thumbnail'); update_sort_options(data, base_url); - // Do initial sort to match sort button display + // Do initial sort and group to match sort button display + group_by_thumbnails(group, Object.keys(data.dropdown_menus).toString(), num_thumbnails, base_url); sort_by_thumbnails(sort, base_url); // Replace loading screen with the proposal array div diff --git a/jwql/website/apps/jwql/templates/thumbnails_per_obs.html b/jwql/website/apps/jwql/templates/thumbnails_per_obs.html index 1476fba74..d2aa8fe56 100644 --- a/jwql/website/apps/jwql/templates/thumbnails_per_obs.html +++ b/jwql/website/apps/jwql/templates/thumbnails_per_obs.html @@ -27,9 +27,10 @@

    {{ inst }} Images

    -
    +
    -
    +
    +
    @@ -75,18 +76,20 @@

    Proposal Information for {{ prop }}

    - {% set index = obs_list.index(obs) %} - {% if index != 0 %} - < Previous - {% endif %} - - {% if obs_list.index(obs) != obs_list|length - 1 %} - Next > + {% if obs in obs_list %} + {% set index = obs_list.index(obs) %} + {% if index != 0 %} + < Previous + {% endif %} + + {% if obs_list.index(obs) != obs_list|length - 1 %} + Next > + {% endif %} {% endif %}
    - + {% endblock %} diff --git a/jwql/website/apps/jwql/urls.py b/jwql/website/apps/jwql/urls.py index e03974b90..2d492ee0c 100644 --- a/jwql/website/apps/jwql/urls.py +++ b/jwql/website/apps/jwql/urls.py @@ -99,6 +99,7 @@ re_path(r'^ajax/(?P({}))/archive/(?P[\d]{{1,5}})/obs(?P[\d]{{1,3}})/$'.format(instruments), views.archive_thumbnails_ajax, name='archive_thumb_ajax'), re_path(r'^ajax/viewed/(?P.+)/$', views.toggle_viewed_ajax, name='toggle_viewed_ajax'), re_path(r'^ajax/(?P({}))/archive_date_range/start_date_(?P.+)/stop_date_(?P.+)/$'.format(instruments), views.archive_date_range_ajax, name='archive_date_range_ajax'), + re_path(r'^ajax/image_group/$', views.save_image_group_ajax, name='save_image_group_ajax'), re_path(r'^ajax/image_sort/$', views.save_image_sort_ajax, name='save_image_sort_ajax'), re_path(r'^ajax/navigate_filter/$', views.save_page_navigation_data_ajax, name='save_page_navigation_data_ajax'), re_path('ajax/nirspec/msata/', monitor_views.msata_monitoring_ajax, name='msata_ajax'), diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 29a62b893..624b24daf 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -42,7 +42,6 @@ placed in the ``jwql`` directory. """ -from collections import defaultdict from copy import deepcopy import csv import datetime @@ -449,6 +448,7 @@ def archive_thumbnails_ajax(request, inst, proposal, observation=None): data = thumbnails_ajax(inst, proposal, obs_num=observation) data['thumbnail_sort'] = request.session.get("image_sort", "Ascending") + data['thumbnail_group'] = request.session.get("image_group", "Exposure") save_page_navigation_data(request, data) return JsonResponse(data, json_dumps_params={'indent': 2}) @@ -492,6 +492,7 @@ def archive_thumbnails_per_observation(request, inst, proposal, observation): obs_list = sorted(list(set(all_obs))) sort_type = request.session.get('image_sort', 'Ascending') + group_type = request.session.get('image_group', 'Exposure') template = 'thumbnails_per_obs.html' context = {'base_url': get_base_url(), 'inst': inst, @@ -499,7 +500,8 @@ def archive_thumbnails_per_observation(request, inst, proposal, observation): 'obs_list': obs_list, 'prop': proposal, 'prop_meta': proposal_meta, - 'sort': sort_type} + 'sort': sort_type, + 'group': group_type} return render(request, template, context) @@ -843,6 +845,7 @@ def log_view(request): return render(request, template, context) + def not_found(request, *kwargs): """Generate a ``not_found`` page @@ -1096,6 +1099,24 @@ def explore_image_ajax(request, inst, file_root, filetype, scaling="log", low_li return JsonResponse(context, json_dumps_params={'indent': 2}) +def save_image_group_ajax(request): + """Save the latest selected group type in the session.""" + image_group = request.GET['group_type'] + request.session['image_group'] = image_group + context = {'item': request.session['image_group']} + return JsonResponse(context, json_dumps_params={'indent': 2}) + + +def save_image_sort_ajax(request): + + # a string of the form " 'rootname1'='expstart1', 'rootname2'='expstart2', ..." + image_sort = request.GET['sort_type'] + + request.session['image_sort'] = image_sort + context = {'item': request.session['image_sort']} + return JsonResponse(context, json_dumps_params={'indent': 2}) + + def save_page_navigation_data(request, data): """ It saves the data from the current page in the session so that the user can navigate to the next or @@ -1142,16 +1163,6 @@ def toggle_viewed_ajax(request, file_root): return JsonResponse(context, json_dumps_params={'indent': 2}) -def save_image_sort_ajax(request): - - # a string of the form " 'rootname1'='expstart1', 'rootname2'='expstart2', ..." - image_sort = request.GET['sort_type'] - - request.session['image_sort'] = image_sort - context = {'item': request.session['image_sort']} - return JsonResponse(context, json_dumps_params={'indent': 2}) - - def view_image(request, inst, file_root, rewrite=False): """Generate the image view page From d85dbe6330f3706bae803212d9dd2f747590640e Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 24 Feb 2023 16:48:57 -0500 Subject: [PATCH 101/449] Plots are looking good. Still need to speed up loading --- .../monitor_pages/monitor_bad_pixel_bokeh.py | 97 ++++++++++++------- jwql/website/apps/jwql/monitor_views.py | 9 +- 2 files changed, 69 insertions(+), 37 deletions(-) diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py index 0b32cd568..25eda85ba 100755 --- a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py @@ -40,7 +40,7 @@ from jwql.database.database_interface import NIRSpecBadPixelQueryHistory, NIRSpecBadPixelStats from jwql.database.database_interface import FGSBadPixelQueryHistory, FGSBadPixelStats from jwql.utils.constants import BAD_PIXEL_MONITOR_MAX_POINTS_TO_PLOT, BAD_PIXEL_TYPES, DARKS_BAD_PIXEL_TYPES -from jwql.utils.constants import FLATS_BAD_PIXEL_TYPES, JWST_INSTRUMENT_NAMES_MIXEDCASE +from jwql.utils.constants import DETECTOR_PER_INSTRUMENT, FLATS_BAD_PIXEL_TYPES, JWST_INSTRUMENT_NAMES_MIXEDCASE from jwql.utils.permissions import set_permissions from jwql.utils.utils import filesystem_path, get_config, read_png @@ -57,9 +57,12 @@ def __init__(self, instrument): # Get the relevant database tables self.identify_tables() - #self.apertures = self.get_inst_apers() - #self.apertures = ['aper1', 'aper2', 'aper3'] - self.detectors = get_unique_values_per_column(self.pixel_table, 'detector') + #self.detectors = get_unique_values_per_column(self.pixel_table, 'detector') + + self.detectors = sorted(DETECTOR_PER_INSTRUMENT[self.instrument]) + if self.instrument == 'miri': + self.detectors = ['MIRIMAGE'] + self.run() def identify_tables(self): @@ -82,6 +85,7 @@ def modify_bokeh_saved_html(self): ''] # Our Django-related lines that need to be at the top of the file + hstring = """href="{{'/jwqldb/%s_bad_pixel_stats'%inst.lower()}}" name=test_link class="btn btn-primary my-2" type="submit">Go to JWQLDB page""" newlines = ['{% extends "base.html" %}\n', "\n", "{% block preamble %}\n", "\n", f"{JWST_INSTRUMENT_NAMES_MIXEDCASE[self.instrument]} Bad Pixel Monitor- JWQL\n", "\n", @@ -90,13 +94,12 @@ def modify_bokeh_saved_html(self): '
    \n', "\n", f"

    {JWST_INSTRUMENT_NAMES_MIXEDCASE[self.instrument]} Bad Pixel Monitor

    \n", "
    \n", + f" View or Download {JWST_INSTRUMENT_NAMES_MIXEDCASE[self.instrument]} Bad Pixel Stats Table: \n" ] # More lines that we want to have in the html file, at the bottom endlines = ["\n", - f"

    {JWST_INSTRUMENT_NAMES_MIXEDCASE[self.instrument]} Bad Pixel Stats Table

    \n", - "
    \n", "\n", - """     Bad Pixel Stats\n""", "\n", "
    \n", "\n", "{% endblock %}" ] @@ -182,13 +185,17 @@ def __init__(self, pixel_table, instrument, detector): self.num_files = {} self.baseline_file = {} - #self.identify_tables() + # Get a list of the bad pixel types present in the database + self.badtypes = get_unique_values_per_column(self.pixel_table, 'type') + + # If the database is empty, return a generic entry showing that fact + if len(self.badtypes) == 0: + self.badtypes = ['BAD'] # Get data for the plot of new bad pixels self.get_most_recent_entry() # Get data for the trending plots - self.badtypes = get_unique_values_per_column(self.pixel_table, 'type') for badtype in self.badtypes: self.get_trending_data(badtype) @@ -215,26 +222,27 @@ def get_most_recent_entry(self): latest_entries_by_type = query.all() session.close() + # Organize the results for row in latest_entries_by_type: self.new_bad_pix[row.type] = (row.x_coord, row.y_coord) self.background_file[row.type] = row.source_files[0] self.obs_start_time[row.type] = row.obs_start_time - #self.obs_mid_time[row.type] = row.obs_mid_time self.obs_end_time[row.type] = row.obs_end_time self.num_files[row.type] = len(row.source_files) self.baseline_file[row.type] = row.baseline_file + # If no data is retrieved from the database at all, add a dummy generic entry + if len(self.new_bad_pix.keys()) == 0: + self.new_bad_pix[self.badtypes[0]] = ([], []) + self.background_file[self.badtypes[0]] = '' + self.obs_start_time[self.badtypes[0]] = datetime.datetime.today() + self.obs_end_time[self.badtypes[0]] = datetime.datetime.today() + self.num_files[self.badtypes[0]] = 0 + self.baseline_file[self.badtypes[0]] = '' + def get_trending_data(self, badpix_type): """ """ - # The MIRI imaging detector does not line up with the full frame aperture. Fix that here - if self.detector == 'MIRIM': - self.detector = 'MIRIMAGE' - - # NIRCam LW detectors use 'LONG' rather than 5 in the pixel_table - if '5' in self.detector: - self.detector = self.detector.replace('5', 'LONG') - # Query database for all data in the table with a matching detector and bad pixel type all_entries_by_type = session.query(self.pixel_table.type, self.pixel_table.detector, func.array_length(self.pixel_table.x_coord, 1), self.pixel_table.obs_mid_time) \ @@ -250,6 +258,15 @@ def get_trending_data(self, badpix_type): detector = row[1] num_pix.append(row[2]) times.append(row[3]) + + # If there was no data in the database, create an empty entry + if len(num_pix) == 0: + badtype = badpix_type + detector = self.detector + num_pix = [0] + times = [datetime.datetime.today()] + + # Add results to self.trending_data self.trending_data[badpix_type] = (detector, num_pix, times) # For the given detector, get the latest entry for each bad pixel type, and @@ -347,7 +364,7 @@ def create_plot(self): # Plot image if image is not None: - mapper, nx, ny are not defined yet + print('mapper, nx, ny are not defined yet') imgplot = self.plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, color_mapper=mapper, level="image") else: # If the background image is not present, manually set the x and y range @@ -358,7 +375,7 @@ def create_plot(self): legend_title = f'Compared to baseline file {os.path.basename(self.baseline_file)}' - # Overplot locations of bad pixels for all bad pixel types + # Overplot locations of bad pixels for the bad pixel type plot_legend = self.overplot_bad_pix() legend = Legend(items=[plot_legend], @@ -372,8 +389,8 @@ def create_plot(self): # Now we export as a png and place that into the figure, as a way of reducing the # amount of data sent to the browser. This png will be saved and immediately read # back in. - #if 1 < 0: - if len(self.coords[0]) > BAD_PIXEL_MONITOR_MAX_POINTS_TO_PLOT: + #if len(self.coords[0]) > BAD_PIXEL_MONITOR_MAX_POINTS_TO_PLOT: + if 1 < 0: output_filename = full_path_background_file.replace('.png', f'_{self.badpix_type}_pix.png') self.switch_to_png(output_filename) print(f'Switching to png for {self.detector}, {self.badpix_type}, {len(self.coords[0])}') @@ -389,19 +406,31 @@ def overplot_bad_pix(self): """ numpix = len(self.coords[0]) - #######TEST - if too many points, cut them way down - #if numpix > 2: - # self.coords = (self.coords[0][0:2], self.coords[1][0:2]) - # numpix = 2 - #########TEST - remove before merging - + print(f'making new badpix plot for {self.badpix_type}. numpix is {numpix}') - - source = ColumnDataSource(data=dict(pixels_x=self.coords[0], - pixels_y=self.coords[1], - values=[self.badpix_type] * numpix - ) - ) + # If there are no new bad pixels, insert a fake one, in order to + # get the plot to be made + if numpix > 0: + source = ColumnDataSource(data=dict(pixels_x=self.coords[0], + pixels_y=self.coords[1], + values=[self.badpix_type] * numpix + ) + ) + else: + txt_source = ColumnDataSource(data=dict(x=[self._detlen / 10], y=[self._detlen / 2], + text=[f'No new {self.badpix_type} pixels found'])) + glyph = Text(x="x", y="y", text="text", angle=0., text_color="navy", text_font_size={'value':'20px'}) + self.plot.add_glyph(txt_source, glyph) + fakex = np.array([0, self._detlen, self._detlen, 0]) + fakey = np.array([0, 0, self._detlen, self._detlen]) + fakex = [int(e) for e in fakex] + fakey = [int(e) for e in fakey] + print(f'Found no new badpix: {self.badpix_type}') + source = ColumnDataSource(data=dict(pixels_x=fakex, + pixels_y=fakey, + values=['N/A'] * len(fakex) + ) + ) # Overplot the bad pixel locations badpixplots = self.plot.circle(x='pixels_x', y='pixels_y', source=source, color='blue', diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py index 822d58f0f..fb54ed495 100644 --- a/jwql/website/apps/jwql/monitor_views.py +++ b/jwql/website/apps/jwql/monitor_views.py @@ -63,10 +63,13 @@ def bad_pixel_monitor(request, inst): Outgoing response sent to the webpage """ # Locate the html file for the instrument - html_file = f"{inst.lower()}_bad_pix_plots.html" - #html_file = f"bad_pixel_monitor_test_junk.html" + template = f"{inst.lower()}_bad_pix_plots.html" - return render(request, html_file) + context = { + 'inst': inst, + } + + return render(request, template, context) def bias_monitor(request, inst): From e79d9f019e8fb6729f12fed87a8a11d1f02e2e45 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 24 Feb 2023 17:26:38 -0500 Subject: [PATCH 102/449] Make sure look status check is case-insensitive --- jwql/tests/test_data_containers.py | 18 ++++++++++-------- jwql/website/apps/jwql/data_containers.py | 2 +- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/jwql/tests/test_data_containers.py b/jwql/tests/test_data_containers.py index 157a7d5a9..98d84a06d 100644 --- a/jwql/tests/test_data_containers.py +++ b/jwql/tests/test_data_containers.py @@ -174,19 +174,21 @@ def test_get_instrument_proposals(): @pytest.mark.parametrize('keys,viewed,sort_as,exp_type,cat_type', [(None, None, None, None, None), (None, 'viewed', None, None, None), + (None, 'Viewed', None, None, None), (None, 'new', None, None, None), + (None, 'New', None, None, None), (None, None, None, 'NRS_MSATA', None), # (None, None, None, None, 'CAL'), # cat_type not implemented yet - (['obsstart'], False, 'ascending', None, None), - (['obsstart'], False, 'descending', None, None), - (['obsstart'], False, 'recent', None, None), - ([], True, None, None, None), - ([], False, None, None, None), + (['obsstart'], 'new', 'ascending', None, None), + (['obsstart'], 'new', 'descending', None, None), + (['obsstart'], 'new', 'recent', None, None), + ([], 'viewed', None, None, None), + ([], 'new', None, None, None), ([], None, None, None, None), (['proposal', 'obsnum', 'other', - 'prop_id', 'obsstart'], True, None, None, None), + 'prop_id', 'obsstart'], 'viewed', None, None, None), (['proposal', 'obsnum', 'other', - 'prop_id', 'obsstart'], False, None, None, None), + 'prop_id', 'obsstart'], 'new', None, None, None), (['proposal', 'obsnum', 'other', 'prop_id', 'obsstart'], None, None, None, None)]) def test_get_instrument_looks(keys, viewed, sort_as, exp_type, cat_type): @@ -209,7 +211,7 @@ def test_get_instrument_looks(keys, viewed, sort_as, exp_type, cat_type): assert len(return_keys) >= 1 + len(keys) # viewed depends on local database, so may or may not have results - if not viewed == 'viewed': + if not str(viewed).lower() == 'viewed': assert len(looks) > 0 first_file = looks[0] assert first_file['root_name'] != '' diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index ab5636c6f..c4db30833 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -976,7 +976,7 @@ def get_instrument_looks(instrument, sort_as=None, # get desired filters filter_kwargs = dict() if look is not None: - filter_kwargs['viewed'] = (look == 'viewed') + filter_kwargs['viewed'] = (str(look).lower() == 'viewed') if exp_type is not None: filter_kwargs['obsnum__exptypes__icontains'] = exp_type if cat_type is not None: From a7da36d35d5f472d238f231e7653c864b8f46c55 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 24 Feb 2023 17:30:01 -0500 Subject: [PATCH 103/449] no need to write initial html file --- .../monitor_pages/monitor_bad_pixel_bokeh.py | 62 +++---------------- 1 file changed, 10 insertions(+), 52 deletions(-) diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py index 25eda85ba..3781f3ddf 100755 --- a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py @@ -125,10 +125,6 @@ def run(self): # Create plots of the location of new bad pixels all_plots = {} - #all_plots['new_dark'] = NewBadPixPlot(detector, data.num_files, data.new_bad_pix, data.background_file, 'darks', - # data.baseline_file, data.obs_start_time, data.obs_end_time).plot - #all_plots['new_flat'] = NewBadPixPlot(detector, data.num_files, data.new_bad_pix, data.background_file, 'flats', - # data.baseline_file, data.obs_start_time, data.obs_end_time).plot all_plots['new_pix'] = {} all_plots['trending'] = {} for badtype in data.badtypes: @@ -148,14 +144,11 @@ def run(self): script, div = components(tabs) # Insert into our html template and save - template_file = '/Users/hilbert/python_repos/jwql/jwql/website/apps/jwql/templates/bad_pixel_monitor_savefile_basic.html' + template_dir = os.path.join(os.path.dirname(__file__), '../templates') + template_file = os.path.join(template_dir, 'bad_pixel_monitor_savefile_basic.html') temp_vars = {'inst': self.instrument, 'plot_script': script, 'plot_div':div} self.html = file_html(tabs, CDN, f'{self.instrument} bad pix monitor', template_file, temp_vars) - with open(template_file, 'w') as f: - f.writelines(self.html) - - # Modify the html such that our Django-related lines are kept in place, # which will allow the page to keep the same formatting and styling as # the other web app pages @@ -163,7 +156,6 @@ def run(self): # Save html file outdir = os.path.dirname(template_file) - #outfile = 'test_badpix_saved_file.html' outfile = f'{self.instrument}_bad_pix_plots.html' outfile = os.path.join(outdir, outfile) with open(outfile, "w") as file: @@ -551,53 +543,19 @@ def badpix_monitor_plot_layout(plots): Paramters --------- plots : dict - Dictionary containing a set of plots for an aperture. - Possible keys are 'new_flat' and 'new_dark', which contain the figures - showing new bad pixels derived from flats and darks, respectively. - The third key is 'bad_types', which should contain a dictionary. The - keys of this dictionary are bad pixel types (e.g. 'dead'). Each of - these contains the Bokeh figure showing the locations of new bad - pixels of that type. + Nested dictionary containing a set of plots for an aperture. + Required keys are 'new_pix' and 'trending'. Each of these contain a + dictionary where the keys are the types of bad pixels, and the values + are the Bokeh figures. 'new_pix' and 'trending' should contain the + same set of keys. 'new_pix' contains the figures showing new bad pixel + locations, while 'trending' contains the figures showign the number of + bad pixels with time. Returns ------- plot_layout : bokeh.layouts.layout + Layout containing all of the input figures """ - - """ - # First the plots showing all bad pixel types derived from a given type of - # input (darks or flats). If both plots are present, show them side by side. - # Some instruments will only have one of these two (e.g. NIRCam has no - # internal lamps, and so will not have flats). In that case, show the single - # exsiting plot by itself in the top row. - if 'new_dark' in plots and 'new_flat' in plots: - new_list = [[plots['new_dark'], plots['new_flat']]] - elif 'new_dark' in plots: - new_list = [plots['new_dark']] - elif 'new_flat' in plots: - new_list = [plots['new_flat']] - - # Next create a list of plots where each plot shows one flavor of bad pixel - plots_per_row = 2 - num_bad_types = len(plots['trending']) - first_col = np.arange(0, num_bad_types, plots_per_row) - - badtype_lists = [] - keys = list(plots['bad_types']) - for i, key in enumerate(keys): - if i % plots_per_row == 0: - sublist = keys[i: i + plots_per_row] - rowplots = [] - for subkey in sublist: - rowplots.append(plots['bad_types'][subkey]) - badtype_lists.append(rowplots) - - # Combine full frame and subarray aperture lists - full_list = new_list + badtype_lists - """ - - - # Create a list of plots where each plot shows one flavor of bad pixel all_plots = [] for badtype in plots["trending"]: From 15e025dd72bca02ffbf1b5d11e70ff794501290d Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Mon, 27 Feb 2023 14:04:17 -0500 Subject: [PATCH 104/449] add cat_type to admin page --- jwql/website/apps/jwql/admin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jwql/website/apps/jwql/admin.py b/jwql/website/apps/jwql/admin.py index f3c6ef854..668373880 100644 --- a/jwql/website/apps/jwql/admin.py +++ b/jwql/website/apps/jwql/admin.py @@ -29,8 +29,8 @@ class ArchiveAdmin(admin.ModelAdmin): @admin.register(Proposal) class ProposalAdmin(admin.ModelAdmin): - list_display = ('archive', 'prop_id') - list_filter = ('archive',) + list_display = ('archive', 'prop_id', 'cat_type') + list_filter = ('archive', 'cat_type') @admin.register(Observation) From e94f8e391cd4896b021dba6c4040ec3675cf4ba3 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Mon, 27 Feb 2023 14:14:27 -0500 Subject: [PATCH 105/449] Added more dark logging, and don't check against existing pixels if there are no new pixels --- .../common_monitors/dark_monitor.py | 14 ++- jwql/shared_tasks/celery_cheatsheet.md | 11 ++ jwql/tests/test_redis_celery.py | 109 ++++++++++++++++++ 3 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 jwql/tests/test_redis_celery.py diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index b13151d9c..8558fce1f 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -381,15 +381,23 @@ def exclude_existing_badpix(self, badpix, pixel_type): new_pixels_y : list List of y coordinates of new bad pixels """ + + if len(badpix[0]) == 0: + logging.warning("Now new {} pixels found.".format(pixel_type)) + return ([], []) + + logging.info("Excluding {} existing bad pixels {}".format(len(badpix[0]), pixel_type)) if pixel_type not in ['hot', 'dead', 'noisy']: raise ValueError('Unrecognized bad pixel type: {}'.format(pixel_type)) + logging.info("\tRunning database query") db_entries = session.query(self.pixel_table) \ .filter(self.pixel_table.type == pixel_type) \ .filter(self.pixel_table.detector == self.detector) \ .all() + logging.info("\tCreating list of found pixels") already_found = [] if len(db_entries) != 0: for _row in db_entries: @@ -398,6 +406,7 @@ def exclude_existing_badpix(self, badpix, pixel_type): for x, y in zip(x_coords, y_coords): already_found.append((x, y)) + logging.info("\tChecking pixels against found list") # Check to see if each pixel already appears in the database for # the given bad pixel type new_pixels_x = [] @@ -747,18 +756,21 @@ def process(self, file_list): # Add new hot and dead pixels to the database logging.info('\tFound {} new hot pixels'.format(len(new_hot_pix[0]))) - logging.info('\tFound {} new dead pixels'.format(len(new_dead_pix[0]))) self.add_bad_pix(new_hot_pix, 'hot', file_list, mean_slope_file, baseline_file, min_time, mid_time, max_time) + logging.info('\tFound {} new dead pixels'.format(len(new_dead_pix[0]))) self.add_bad_pix(new_dead_pix, 'dead', file_list, mean_slope_file, baseline_file, min_time, mid_time, max_time) # Check for any pixels that are significantly more noisy than # in the baseline stdev image + logging.info("\tChecking for noisy pixels") new_noisy_pixels = self.noise_check(stdev_image, baseline_stdev) # Shift coordinates to be in full_frame coordinate system + logging.info("\tShifting noisy pixels to full frame") new_noisy_pixels = self.shift_to_full_frame(new_noisy_pixels) # Exclude previously found noisy pixels + logging.info("\tExcluding existing bad pixels from noisy pixels") new_noisy_pixels = self.exclude_existing_badpix(new_noisy_pixels, 'noisy') # Add new noisy pixels to the database diff --git a/jwql/shared_tasks/celery_cheatsheet.md b/jwql/shared_tasks/celery_cheatsheet.md index 5398fa4cf..32469403c 100644 --- a/jwql/shared_tasks/celery_cheatsheet.md +++ b/jwql/shared_tasks/celery_cheatsheet.md @@ -207,3 +207,14 @@ whether you want the function to return whether or not you have the lock into th variable (`blocking=False`). In the case where `blocking=False`, if `have_lock=False` then the lock is already in use, and you must **not** execute any code that requires the lock. If your code has no way to proceed without the lock, then you should use `blocking=True`. + +## Testing Celery and Redis + +- Create and activate your test environment +- Make sure that in the ``config.json`` file, + - ``redis_host`` is set to localhost + - ``test_data`` is set appropriately + - ``transfer_dir`` is set to the test or dev directory as appropriate +- Start redis with ``redis-server &`` +- Start celery with ``celery -A shared_tasks worker -D -E -ldebug -Ofair -c1 --max-tasks-per-child=1 --prefetch-multiplier 1`` +- Run ``jwql/tests/test_redis_celery.py`` diff --git a/jwql/tests/test_redis_celery.py b/jwql/tests/test_redis_celery.py new file mode 100644 index 000000000..1e61394b7 --- /dev/null +++ b/jwql/tests/test_redis_celery.py @@ -0,0 +1,109 @@ +#! /usr/bin/env python + +""" +Tests for the redis/celery server infrastructure. + +Authors +------- + + - Brian York + +Use +--- + + In order to run these tests, you need the following: + + - A running redis server (separate from the production server on pljwql2) + - A running celery worker communicating with that redis server + - A config.json file providing the redis URL, and pointing to the JWQL testing + files. + + These tests are intended to be run from the command line, because I haven't yet + figured out a way to actually set up the entire environment in pytest: + :: + + python test_redis_celery.py +""" + +from astropy.io import ascii, fits +from collections import defaultdict +from collections import OrderedDict +from copy import deepcopy +import datetime +import logging +import numpy as np +import os +from pathlib import Path +from pysiaf import Siaf +import pytest +from tempfile import TemporaryDirectory + +from jwql.instrument_monitors import pipeline_tools +from jwql.shared_tasks.shared_tasks import only_one, run_pipeline, run_parallel_pipeline +from jwql.utils import crds_tools, instrument_properties, monitor_utils +from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE +from jwql.utils.constants import FLAT_EXP_TYPES, DARK_EXP_TYPES +from jwql.utils.logging_functions import log_info, log_fail +from jwql.utils.permissions import set_permissions +from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path + + +def get_instrument(file_name): + if 'miri' in file_name: + return 'miri' + elif 'nircam' in file_name: + return 'nircam' + elif 'nrc' in file_name: + return 'nircam' + elif 'niriss' in file_name: + return 'niriss' + elif 'nis' in file_name: + return 'niriss' + elif 'nirspec' in file_name: + return 'nirspec' + elif 'nrs' in file_name: + return 'nirspec' + elif 'guider' in file_name: + return 'fgs' + return 'unknown' + + +if __name__ == "__main__": + config = get_config() + p = Path(config['test_data']) + + # Test the standard pipeline task + print("Testing cal pipeline") + with TemporaryDirectory() as working_dir: + print("Running in {}".format(working_dir)) + for file in p.rglob("*uncal.fits"): + file_name = os.path.basename(file) + if "gs-" in file_name: + print("\tSkipping guide star file {}".format(file_name)) + continue + print("\tCopying {}".format(file)) + copy_files([file], working_dir) + cal_file = os.path.join(working_dir, file_name) + print("\t\tCalibrating {}".format(cal_file)) + instrument = get_instrument(file_name) + outputs = run_pipeline(cal_file, "uncal", "all", instrument) + print("\t\tDone {}".format(file)) + + # Test the jump pipeline task + print("Testing jump pipeline") + with TemporaryDirectory() as working_dir: + print("Running in {}".format(working_dir)) + for file in p.rglob("*uncal.fits"): + file_name = os.path.basename(file) + if "gs-" in file_name: + print("\tSkipping guide star file {}".format(file_name)) + continue + print("\tCopying {}".format(file)) + copy_files([file], working_dir) + cal_file = os.path.join(working_dir, file_name) + print("\t\tCalibrating {}".format(cal_file)) + instrument = get_instrument(file_name) + outputs = run_pipeline(cal_file, "uncal", "all", instrument, jump_pipe=True) + print("\t\tDone {}".format(file)) + + print("Done test") From b1311ebf3403d83fa5d3eed194e3d9db571dde93 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 27 Feb 2023 17:18:02 -0500 Subject: [PATCH 106/449] Plots all look good. Do we want a background image? --- jwql/utils/utils.py | 131 ++-- .../apps/jwql/monitor_pages/__init__.py | 1 - .../monitor_pages/monitor_bad_pixel_bokeh.py | 705 +++++++----------- 3 files changed, 341 insertions(+), 496 deletions(-) diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index 1a967b1a3..dd6dc07da 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -126,46 +126,85 @@ def _validate_config(config_file_dict): def create_png_from_fits(filename, outdir): - """Create and save a png file of the provided file. The file - will be saved with the same filename as the input file, but - with fits replaced by png - - Parameters - ---------- - filename : str - Fits file to be opened and saved as a png - - outdir : str - Output directory to save the png file to - - Returns - ------- - png_file : str - Name of the saved png file - """ - if os.path.isfile(filename): - image = fits.getdata(filename) - ny, nx = image.shape - img_mn, img_med, img_dev = sigma_clipped_stats(image[4: ny - 4, 4: nx - 4]) - - plot = figure(tools='') - plot.x_range.range_padding = plot.y_range.range_padding = 0 - - # Create the color mapper that will be used to scale the image - #mapper = LinearColorMapper(palette='Viridis256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev)) - mapper = LogColorMapper(palette='Viridis256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev)) - - # Plot image - imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, - color_mapper=mapper, level="image") - - # Save the plot in a png - output_filename = os.path.join(outdir, os.path.basename(filename)) - export_png(plot, filename=output_filename) - permissions.set_permissions(output_filename) - return output_filename - else: - return None + """Create and save a png file of the provided file. The file + will be saved with the same filename as the input file, but + with fits replaced by png + + Parameters + ---------- + filename : str + Fits file to be opened and saved as a png + + outdir : str + Output directory to save the png file to + + Returns + ------- + png_file : str + Name of the saved png file + """ + if os.path.isfile(filename): + image = fits.getdata(filename) + ny, nx = image.shape + img_mn, img_med, img_dev = sigma_clipped_stats(image[4: ny - 4, 4: nx - 4]) + + plot = figure(tools='') + plot.x_range.range_padding = plot.y_range.range_padding = 0 + plot.toolbar.logo = None + plot.toolbar_location = None + plot.min_border = 0 + plot.xgrid.visible = False + plot.ygrid.visible = False + + # Create the color mapper that will be used to scale the image + #mapper = LogColorMapper(palette='Viridis256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev)) + mapper = LogColorMapper(palette='Greys256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev)) + + # Plot image + imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, + color_mapper=mapper, level="image") + + # Turn off the axes, in order to make embedding in another figure easier + plot.xaxis.visible = False + plot.yaxis.visible = False + + # Save the plot in a png + output_filename = os.path.join(outdir, os.path.basename(filename).replace('fits','png')) + export_png(plot, filename=output_filename) + permissions.set_permissions(output_filename) + return output_filename + else: + return None + + +def screenshot_as_png(): + from bokeh.io.export import get_screenshot_as_png + if os.path.isfile(filename): + image = fits.getdata(filename) + ny, nx = image.shape + img_mn, img_med, img_dev = sigma_clipped_stats(image[4: ny - 4, 4: nx - 4]) + + plot = figure(tools='') + plot.x_range.range_padding = plot.y_range.range_padding = 0 + + # Create the color mapper that will be used to scale the image + #mapper = LinearColorMapper(palette='Viridis256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev)) + mapper = LogColorMapper(palette='Viridis256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev)) + + # Plot image + imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, + color_mapper=mapper, level="image") + + # Turn off the axes, in order to make embedding in another figure easier + plot.xaxis.visible = False + plot.yaxis.visible = False + + # Save the plot in a png + screenshot = get_screenshot_as_png(plot, height=600, width=600) + return screenshot + else: + return None + def get_config(): @@ -695,9 +734,14 @@ def read_png(filename): rgba_img = Image.open(filename).convert('RGBA') xdim, ydim = rgba_img.size - # Create an array representation for the image `img`, and an 8-bit "4 - # layer/RGBA" version of it `view`. + # Create an array representation for the image, filled with + # dummy data to begin with img = np.empty((ydim, xdim), dtype=np.uint32) + + # Create a layer/RGBA" version with a set of 4, 8-bit layers. + # We will work with the data using 'view', and our changes + # will propagate back into the 2D 'img' version, which is + # what we will end up returning. view = img.view(dtype=np.uint8).reshape((ydim, xdim, 4)) # Copy the RGBA image into view, flipping it so it comes right-side up @@ -705,7 +749,8 @@ def read_png(filename): view[:,:,:] = np.flipud(np.asarray(rgba_img)) else: view = None - return view + # Return the 2D version + return img def grouper(iterable, chunksize): diff --git a/jwql/website/apps/jwql/monitor_pages/__init__.py b/jwql/website/apps/jwql/monitor_pages/__init__.py index 4728d5cd4..b5ec24cef 100644 --- a/jwql/website/apps/jwql/monitor_pages/__init__.py +++ b/jwql/website/apps/jwql/monitor_pages/__init__.py @@ -1,4 +1,3 @@ -from .monitor_bad_pixel_bokeh import BadPixelMonitor from .monitor_bias_bokeh import BiasMonitor from .monitor_cosmic_rays_bokeh import CosmicRayMonitor from .monitor_filesystem_bokeh import MonitorFilesystem diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py index 3781f3ddf..cbbf07b4b 100755 --- a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py @@ -8,13 +8,12 @@ Use --- - This module can be used from the command line as such: + This module can be used from the command line like this: :: from jwql.website.apps.jwql import monitor_pages - monitor_template = monitor_pages.BadPixelMonitor('NIRCam', 'NRCA3_FULL') - script, div = monitor_template.embed("bad_pixel_time_figure") + monitor_pages.BadPixelMonitor('nircam') """ import os @@ -23,10 +22,10 @@ from astropy.stats import sigma_clipped_stats from astropy.time import Time from bokeh.embed import components, file_html -from bokeh.io import export_png +from bokeh.io import export_png, show from bokeh.io.export import get_screenshot_as_png from bokeh.layouts import layout -from bokeh.models import ColumnDataSource, DatetimeTickFormatter, HoverTool, Legend, LinearColorMapper, Panel, Tabs, Text +from bokeh.models import ColumnDataSource, DatetimeTickFormatter, HoverTool, Legend, LinearColorMapper, Panel, Tabs, Text, Title from bokeh.plotting import figure from bokeh.resources import CDN import datetime @@ -49,7 +48,26 @@ class BadPixelPlots(): - """Class for creating the bad pixel plots and figures to be displayed in the web app + """Class for creating the bad pixel monitor plots and figures to be displayed + in the web app + + Attributes + ---------- + instrument : str + Name of instrument (e.g. 'nircam') + + detectors : list + List of detectors corresponding to ```instrument```. One tab will be created + for each detector. + + pixel_table : sqlalchemy table + Table containing bad pixel information for each detector + + query_table : sqlalchemy table + Table containing history of bad pixel monitor runs and files used + + _html : str + HTML for the bad pixel monitor page """ def __init__(self, instrument): self.instrument = instrument.lower() @@ -76,7 +94,7 @@ def modify_bokeh_saved_html(self): """Given an html string produced by Bokeh when saving bad pixel monitor plots, make tweaks such that the page follows the general JWQL page formatting. """ - lines = self.html.split('\n') + lines = self._html.split('\n') # List of lines that Bokeh likes to save in the file, but we don't want lines_to_remove = ["", @@ -94,7 +112,7 @@ def modify_bokeh_saved_html(self): '
    \n', "\n", f"

    {JWST_INSTRUMENT_NAMES_MIXEDCASE[self.instrument]} Bad Pixel Monitor

    \n", "
    \n", - f" View or Download {JWST_INSTRUMENT_NAMES_MIXEDCASE[self.instrument]} Bad Pixel Stats Table: View or Download {JWST_INSTRUMENT_NAMES_MIXEDCASE[self.instrument]} Bad Pixel Stats Table:      \n" ] @@ -109,7 +127,7 @@ def modify_bokeh_saved_html(self): newlines.append(line + '\n') newlines = newlines + endlines - self.html = "".join(newlines) + self._html = "".join(newlines) def run(self): @@ -147,7 +165,7 @@ def run(self): template_dir = os.path.join(os.path.dirname(__file__), '../templates') template_file = os.path.join(template_dir, 'bad_pixel_monitor_savefile_basic.html') temp_vars = {'inst': self.instrument, 'plot_script': script, 'plot_div':div} - self.html = file_html(tabs, CDN, f'{self.instrument} bad pix monitor', template_file, temp_vars) + self._html = file_html(tabs, CDN, f'{self.instrument} bad pix monitor', template_file, temp_vars) # Modify the html such that our Django-related lines are kept in place, # which will allow the page to keep the same formatting and styling as @@ -159,11 +177,67 @@ def run(self): outfile = f'{self.instrument}_bad_pix_plots.html' outfile = os.path.join(outdir, outfile) with open(outfile, "w") as file: - file.write(self.html) + file.write(self._html) class BadPixelData(): - """Retrieve bad pixel monitor data from the database + """Class to retrieve and store bad pixel monitor data from the database + + Parameters + ---------- + pixel_table : sqlalchemy table + Table containing bad pixel information for each detector + + instrument : str + Instrument name, e.g. 'nircam' + + detector : str + Detector name, e.g. 'NRCA1' + + Atributes + --------- + background_file : str + Name of one file used to find the current selection of bad pixels + + badtypes : list + List of bad pixel types present in ```pixel_table``` + + baseline_file : str + Name of file containing a previous collection of bad pixels, to be + compared against the new collection of bad pixels + + detector : str + Detector name, e.g. 'NRCA1' + + instrument : str + Instrument name, e.g. 'nircam' + + new_bad_pix : dict + Keys are the types of bad pixels (e.g. 'dead'). The value for each key + is a 2-tuple. The first element is a list of x coordinates, and the second + element is a list of y coordinates, corresponding to the locations of that + type of bad pixel. + + num_files : dict + Keys are the types of bad pixels (e.g. 'dead'). The value of each is the number + of files used when searching for that type of bad pixel. + + obs_end_time : dict + Keys are the types of bad pixels (e.g. 'dead'). The value of each is the ending + time (datetime instance) of the observations used to find the bad pixels. + + obs_start_time : dict + Keys are the types of bad pixels (e.g. 'dead'). The value of each is the starting + time (datetime instance) of the observations used to find the bad pixels. + + pixel_table : sqlalchemy table + Table containing bad pixel information for each detector + + trending_data : dict + Keys are the types of bad pixels (e.g. 'dead'). The value of each is a 3-tuple of + data to be used to create the trending plot. The first element is the detector name, + the second is a list of the number of bad pixels, and the third is a list of the + datetimes associated with the bad pixel numbers. """ def __init__(self, pixel_table, instrument, detector): self.pixel_table = pixel_table @@ -194,10 +268,6 @@ def __init__(self, pixel_table, instrument, detector): def get_most_recent_entry(self): """Get all nedded data from the database tables. - Parameters - ---------- - detector : str - Name of detector for which data are retrieved (e.g. NRCA1) """ # For the given detector, get the latest entry for each bad pixel type subq = (session @@ -233,7 +303,12 @@ def get_most_recent_entry(self): self.baseline_file[self.badtypes[0]] = '' def get_trending_data(self, badpix_type): - """ + """Retrieve and organize the data needed to produce the trending plot. + + Parameters + ---------- + badpix_type : str + The type of bad pixel to query for, e.g. 'dead' """ # Query database for all data in the table with a matching detector and bad pixel type all_entries_by_type = session.query(self.pixel_table.type, self.pixel_table.detector, func.array_length(self.pixel_table.x_coord, 1), @@ -260,29 +335,79 @@ def get_trending_data(self, badpix_type): # Add results to self.trending_data self.trending_data[badpix_type] = (detector, num_pix, times) - - # For the given detector, get the latest entry for each bad pixel type, and - # return the bad pixel type, detector, and mean dark image file - #subq = (session - # .query(self.pixel_table.type, func.max(self.pixel_table.entry_date).label("max_created")) - # .filter(self.pixel_table.detector == self.detector & self.pixle_table.type == self.type) - # .group_by(self.pixel_table.type) - # .subquery() - # ) - - #query = (session.query(self.pixel_table.type, self.pixel_table.detector, self.pixel_table.x_coord.length, self.pixel_table.obs_mid_time) - # .join(subq, self.pixel_table.entry_date == subq.c.max_created) - # ) - - #self.most_recent_data = query.all() session.close() class NewBadPixPlot(): - """Create a plot showing the location of newly discovered bad pixels + """Class to create a plot showing the location of newly discovered bad pixels of a certain type Parameters ---------- + detector_name : str + Name of detector, e.g. NRCA1 + + badpix_type : str + Type of bad pixel, e.g. 'dead' + + nfiles : int + Number of files used to find the bad pixels + + coords : tuple + 2-tuple. The first element is a list of x coordinates, and the second + element is a list of y coordinates, corresponding to the locations of that + type of bad pixel. + + background_file : str + Name of one of the files used to find the bad pixels + + baseline_file : str + Name of file containing previously identified bad pixels, which were compared to the + new collection of bad pixels + + obs_start_time : datetime.datetime + Datetime of the beginning of the observations used in the search for the bad pixels + + obs_end_time : datetime.datetime + Datetime of the ending of the observations used in the search for the bad pixels + + Attributes + ---------- + background_file : str + Name of one of the files used to find the bad pixels + + badpix_type : str + Type of bad pixel, e.g. 'dead' + + baseline_file : str + Name of file containing previously identified bad pixels, which were compared to the + new collection of bad pixels + + coords : tuple + 2-tuple. The first element is a list of x coordinates, and the second + element is a list of y coordinates, corresponding to the locations of that + type of bad pixel. + + detector : str + Name of detector, e.g. NRCA1 + + num_files : int + Number of files used to find the bad pixels + + obs_start_time : datetime.datetime + Datetime of the beginning of the observations used in the search for the bad pixels + + obs_end_time : datetime.datetime + Datetime of the ending of the observations used in the search for the bad pixels + + plot : Bokeh.plotting.figure + Figure showing the location of the bad pixels on the detector + + _detlen : int + Number of pixels in one row or column of ```detector``` + + _use_png : bool + Whether or not to create the Bokeh figure using circle glyphs of all bad pixels, or to + save the plot of bad pixels as a png and load that (in order to reduce data volume.) """ def __init__(self, detector_name, badpix_type, nfiles, coords, background_file, baseline_file, obs_start_time, obs_end_time): self.detector = detector_name @@ -293,12 +418,6 @@ def __init__(self, detector_name, badpix_type, nfiles, coords, background_file, self.baseline_file = baseline_file self.obs_start_time = obs_start_time self.obs_end_time = obs_end_time - #if self.data_type == 'darks': - # self.badpix_types = DARKS_BAD_PIXEL_TYPES - #elif self.data_type == 'flats': - # self.badpix_types = FLATS_BAD_PIXEL_TYPES - #else: - # raise ValueError("Unrecognized data type. Should be 'flats' or 'darks'") # If no background file is given, we fall back to plotting the bad pixels # on top of an empty image. In that case, we need to know how large the @@ -308,6 +427,12 @@ def __init__(self, detector_name, badpix_type, nfiles, coords, background_file, else: self._detlen = 2048 + # If there are "too many" points then we are going to save the plot as + # a png rather than send all the data to the browser.\ + self._use_png = False + if len(self.coords[0]) > BAD_PIXEL_MONITOR_MAX_POINTS_TO_PLOT: + self._use_png = True + self.create_plot() def create_plot(self): @@ -319,45 +444,53 @@ def create_plot(self): # Read in the data, or create an empty array png_file = self.background_file.replace('.fits', '.png') full_path_background_file = os.path.join(OUTPUT_DIR, 'bad_pixel_monitor/', png_file) + + print('Looking for: ', full_path_background_file) + if os.path.isfile(full_path_background_file): image = read_png(full_path_background_file) else: - print(f'Background_file {full_path_background_file} is not a valid file') - #image = np.zeros((self._detlen, self._detlen)) image = None - #title_text = f'{self.detector}: New bad pix from {self.data_type}. {self.num_files} files.' - - #start_time = Time(float(self.obs_start_time), format='mjd').tt.datetime.strftime("%m/%d/%Y") - #end_time = Time(float(self.obs_end_time), format='mjd').tt.datetime.strftime("%m/%d/%Y") start_time = self.obs_start_time.strftime("%m/%d/%Y") end_time = self.obs_end_time.strftime("%m/%d/%Y") title_text = f'{self.detector}: New {self.badpix_type} pix: from {self.num_files} files. {start_time} to {end_time}' - #ny, nx = image.shape - #img_mn, img_med, img_dev = sigma_clipped_stats(image[4: ny - 4, 4: nx - 4]) - # Create figure # If there are "too many" points then we are going to save the plot as # a png rather than send all the data to the browser. In that case, we # don't want to add any tools to the figure - if len(self.coords[0]) <= BAD_PIXEL_MONITOR_MAX_POINTS_TO_PLOT: + if not self._use_png: tools = 'pan,box_zoom,reset,wheel_zoom,save' + self.plot = figure(title=title_text, tools=tools, + x_axis_label="Pixel Number", y_axis_label="Pixel Number") else: - tools = '' + self.plot = figure(tools='') #, x_axis_label="Pixel Number", y_axis_label="Pixel Number") + self.plot.toolbar.logo = None + self.plot.toolbar_location = None + self.plot.min_border = 0 + self.plot.xaxis.visible = False + self.plot.yaxis.visible = False - self.plot = figure(title=title_text, tools=tools, - x_axis_label="Pixel Number", y_axis_label="Pixel Number") self.plot.x_range.range_padding = self.plot.y_range.range_padding = 0 - # Create the color mapper that will be used to scale the image - #mapper = LinearColorMapper(palette='Viridis256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev)) - # Plot image if image is not None: print('mapper, nx, ny are not defined yet') - imgplot = self.plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, color_mapper=mapper, level="image") + ny, nx = image.shape + #imgplot = self.plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, color_mapper=mapper, level="image") + + + + # Shift the figure title slightly right in this case to get it + # to align with the axes + #self.plot = figure(title=title, x_range=(0, self._detlen), y_range=(0, self._detlen), width=xdim, height=ydim*, + # tools='pan,box_zoom,reset,wheel_zoom,save', x_axis_label="Pixel Number", y_axis_label="Pixel Number") + self.plot.image_rgba(image=[image], x=0, y=0, dw=self._detlen, dh=self._detlen) + + + else: # If the background image is not present, manually set the x and y range self.plot.x_range.start = 0 @@ -370,6 +503,16 @@ def create_plot(self): # Overplot locations of bad pixels for the bad pixel type plot_legend = self.overplot_bad_pix() + # If there are "too many" points, we have already omitted all of the bokeh tools. + # Now we export as a png and place that into the figure, as a way of reducing the + # amount of data sent to the browser. This png will be saved and immediately read + # back in. + if self._use_png: + output_filename = full_path_background_file.replace('.png', f'_{self.badpix_type}_pix.png') + self.switch_to_png(output_filename, title_text) + print(f'Switching to png for {self.detector}, {self.badpix_type}, {len(self.coords[0])}') + + # Create and add legend to the figure legend = Legend(items=[plot_legend], location="center", orientation='vertical', @@ -377,15 +520,6 @@ def create_plot(self): self.plot.add_layout(legend, 'below') - # If there are "too many" points, we have already omitted all of the bokeh tools. - # Now we export as a png and place that into the figure, as a way of reducing the - # amount of data sent to the browser. This png will be saved and immediately read - # back in. - #if len(self.coords[0]) > BAD_PIXEL_MONITOR_MAX_POINTS_TO_PLOT: - if 1 < 0: - output_filename = full_path_background_file.replace('.png', f'_{self.badpix_type}_pix.png') - self.switch_to_png(output_filename) - print(f'Switching to png for {self.detector}, {self.badpix_type}, {len(self.coords[0])}') def overplot_bad_pix(self): """Add a scatter plot of potential new bad pixels to the plot @@ -398,10 +532,6 @@ def overplot_bad_pix(self): """ numpix = len(self.coords[0]) - print(f'making new badpix plot for {self.badpix_type}. numpix is {numpix}') - - # If there are no new bad pixels, insert a fake one, in order to - # get the plot to be made if numpix > 0: source = ColumnDataSource(data=dict(pixels_x=self.coords[0], pixels_y=self.coords[1], @@ -409,15 +539,17 @@ def overplot_bad_pix(self): ) ) else: + # If there are no new bad pixels, write text within the figure mentioning that txt_source = ColumnDataSource(data=dict(x=[self._detlen / 10], y=[self._detlen / 2], text=[f'No new {self.badpix_type} pixels found'])) glyph = Text(x="x", y="y", text="text", angle=0., text_color="navy", text_font_size={'value':'20px'}) self.plot.add_glyph(txt_source, glyph) + + # Insert a fake one, in order to get the plot to be made fakex = np.array([0, self._detlen, self._detlen, 0]) fakey = np.array([0, 0, self._detlen, self._detlen]) fakex = [int(e) for e in fakex] fakey = [int(e) for e in fakey] - print(f'Found no new badpix: {self.badpix_type}') source = ColumnDataSource(data=dict(pixels_x=fakex, pixels_y=fakey, values=['N/A'] * len(fakex) @@ -425,14 +557,14 @@ def overplot_bad_pix(self): ) # Overplot the bad pixel locations - badpixplots = self.plot.circle(x='pixels_x', y='pixels_y', source=source, color='blue', + badpixplots = self.plot.circle(x='pixels_x', y='pixels_y', source=source, color='#EC04FF', fill_alpha=0.75, line_alpha=0.75, radius=0.5) # Create hover tool for the bad pixel type # If there are "too many" points then we are going to save the plot as # a png rather than send all the data to the browser. In that case, we # don't need a hover tool - if numpix <= BAD_PIXEL_MONITOR_MAX_POINTS_TO_PLOT: + if not self._use_png: hover_tool = HoverTool(tooltips=[(f'{self.badpix_type} (x, y):', '(@pixels_x, @pixels_y)'), ], renderers=[badpixplots]) @@ -446,34 +578,73 @@ def overplot_bad_pix(self): legend_items = (text, [badpixplots]) return legend_items - def switch_to_png(self, filename): - """Convert the current Bokeh figure from a figure containing circles to a png + def switch_to_png(self, filename, title): + """Convert the current Bokeh figure from a figure containing circle glyphs to a png representation. Parameters ---------- filename : str Name of file to save the current figure as a png into + + title : str + Title to add to the Figure """ # Save the figure as a png - #fig_array = get_screenshot_as_png(self.plot) + print('Saving plot as png in: ', filename) export_png(self.plot, filename=filename) set_permissions(filename) # Read in the png and insert into a replacement figure fig_array = read_png(filename) + ydim, xdim = fig_array.shape - ydim, xdim, _ = fig_array.shape - dim = max(xdim, ydim) - self.plot = figure(x_range=(0, xdim), y_range=(0, ydim), tools='pan,box_zoom,reset,wheel_zoom,save') - self.plot.image_rgba(image=[fig_array], x=0, y=0, dw=xdim, dh=ydim) - self.plot.xaxis.visible = False - self.plot.yaxis.visible = False + # Shift the figure title slightly right in this case to get it + # to align with the axes + self.plot = figure(title=title, x_range=(0, self._detlen), y_range=(0, self._detlen), width=xdim, height=ydim, + tools='pan,box_zoom,reset,wheel_zoom,save', x_axis_label="Pixel Number", y_axis_label="Pixel Number") + self.plot.image_rgba(image=[fig_array], x=0, y=0, dw=self._detlen, dh=self._detlen) + #self.plot.xaxis.visible = False + #self.plot.yaxis.visible = False + + # Now that the data from the png is in the figure, delete the png + #os.remove(filename) class BadPixTrendPlot(): - """Create a plot showing the location of a certain type of bad pixel + """Class to create a plot of the number of bad pixels of a certain type versus time + + Parameters + ---------- + detector_name : str + Name of the detector, e.g. 'NRCA1' + + badpix_type : str + Type of bad pixel, e.g. 'dead' + + entry : tup + 3-tuple of the data to be plotted. (BadPixelData.trending_data for a certain type + of bad pixel). The first element is the detector name, the second is a list of + the number of bad pixels, and the third is a list of the datetimes associated + with the bad pixel numbers. + + Attributes + ---------- + detector : str + Name of the detector, e.g. 'NRCA1' + + badpix_type : str + Type of bad pixel, e.g. 'dead' + + num_pix : list + List of the number of bad pixels found for a list of times + + plot : Bokeh.plotting.figure + Bokeh figure showing a plot of the number of bad pixels versus time + + time : list + List of datetimes associated with ```num_pix``` """ def __init__(self, detector_name, badpix_type, entry): self.detector = detector_name @@ -505,7 +676,6 @@ def create_plot(self): self.plot = figure(title=f'{self.detector}: New {self.badpix_type} Pixels', tools='pan,box_zoom,reset,wheel_zoom,save', background_fill_color="#fafafa") - # Plot the "main" amp data along with error bars self.plot.scatter(x='time', y='num_pix', fill_color="navy", alpha=0.75, source=source) hover_tool = HoverTool(tooltips=[('# Pixels:', '@num_pix'), @@ -566,372 +736,3 @@ def badpix_monitor_plot_layout(plots): plot_layout = layout(all_plots) return plot_layout - - - - - - - - - - - - - - -"""OLD CODE BELOW HERE""" -"""CAN BE DELETED""" - - -class BadPixelMonitor(): - - # Combine instrument and aperture into a single property because we - # do not want to invoke the setter unless both are updated - @property - def aperture_info(self): - return (self._instrument, self._aperture) - - @aperture_info.setter - def aperture_info(self, info): - self._instrument, self._aperture = info - self.pre_init() - self.post_init() - - def bad_pixel_history(self, bad_pixel_type): - """Use the database to construct information on the total number - of a given type of bad pixels over time - - Parameters - ---------- - bad_pixel_type : str - The flavor of bad pixel (e.g. 'hot') - - Returns - ------- - num_bad_pixels : numpy.ndarray - 1D array of the number of bad pixels - - dates : datetime.datetime - 1D array of dates/times corresponding to num_bad_pixels - """ - # Find all the rows corresponding to the requested type of bad pixel - rows = [row for row in self.bad_pixel_table if row.type == bad_pixel_type] - - # Extract the dates and number of bad pixels from each entry - dates = [row.obs_mid_time for row in rows] - num = [len(row.coordinates[0]) for row in rows] - - # If there are no valid entres in the database, return None - if len(dates) == 0: - return None, None - - # Sort by date to make sure everything is in chronological order - chrono = np.argsort(dates) - dates = dates[chrono] - num = num[chrono] - - # Sum the number of bad pixels found from the earliest entry up to - # each new entry - num_bad_pixels = [np.sum(num[0:i]) for i in range(1, len(num) + 1)] - - return num_bad_pixels, dates - - def _badpix_image(self): - """Update bokeh objects with sample image data.""" - - # Open the mean dark current file and get the data - with fits.open(self.image_file) as hdulist: - data = hdulist[1].data - - # Grab only one frame - ndims = len(data.shape) - if ndims == 4: - data = data[0, -1, :, :] - elif ndims == 3: - data = data[-1, :, :] - elif ndims == 2: - pass - else: - raise ValueError('Unrecognized number of dimensions in data file: {}'.format(ndims)) - - # Update the plot with the data and boundaries - y_size, x_size = data.shape - self.refs["bkgd_image"].data['image'] = [data] - self.refs["stamp_xr"].end = x_size - self.refs["stamp_yr"].end = y_size - self.refs["bkgd_source"].data['dw'] = [x_size] - self.refs["bkgd_source"].data['dh'] = [y_size] - - # Set the image color scale - self.refs["log_mapper"].high = 0 - self.refs["log_mapper"].low = -.2 - - # Add a title - self.refs['badpix_map_figure'].title.text = '{}: New Bad Pixels'.format(self._aperture) - self.refs['badpix_map_figure'].title.align = "center" - self.refs['badpix_map_figure'].title.text_font_size = "20px" - - def most_recent_coords(self, bad_pixel_type): - """Return the coordinates of the bad pixels in the most recent - database entry for the given bad pixel type - - Parameters - ---------- - bad_pixel_type : str - The flavor of bad pixel (e.g. 'hot') - - Returns - ------- - coords : tup - Tuple containing a list of x coordinates and a list of y - coordinates - """ - # Find all the rows corresponding to the requested type of bad pixel - rows = [row for row in self.bad_pixel_table if row.type == bad_pixel_type] - - # Extract dates, number of bad pixels, and files used from each entry - dates = [row.obs_mid_time for row in rows] - coords = [row.coordinates for row in rows] - files = [row.source_files[0] for row in rows] - - # If there are no valid entres in the database, return None - if len(dates) == 0: - return None, None - - # Sort by date to make sure everything is in chronological order - chrono = np.argsort(dates) - dates = dates[chrono] - coords = coords[chrono] - files = files[chrono] - - # Keep track of the latest timestamp - self.last_timestamp = dates[-1].isoformat() - - # Grab the name of one of the files used when these bad pixels - # were identified. We'll use this as an image on top of which - # the bad pixels will be noted. Note that these should be - # slope files - self.image_file = filesystem_path(files[-1]) - - # Return the list of coordinates for the most recent entry - return coords[-1] - - def pre_init(self): - # Start with default values for instrument and aperture because - # BokehTemplate's __init__ method does not allow input arguments - try: - dummy_instrument = self._instrument - dummy_aperture = self._aperture - except AttributeError: - self._instrument = 'NIRCam' - self._aperture = 'NRCA1_FULL' - - self._embed = True - - # Fix aperture/detector name discrepency - if self._aperture in ['NRCA5_FULL', 'NRCB5_FULL']: - self.detector = '{}LONG'.format(self._aperture[0:4]) - else: - self.detector = self._aperture.split('_')[0] - - # App design - self.format_string = None - self.interface_file = os.path.join(SCRIPT_DIR, "yaml", "badpixel_monitor_interface.yaml") - - # Load data tables - self.load_data() - self.get_history_data() - # For development, while the database tables are empty - # self.load_dummy_data() - - # Get dates and coordinates of the most recent entries - self.most_recent_data() - - # This shows that for e.g. NRCA2_FULL, the data are what we expect, - # but somehow the plot is not showing it!!!!!!!! - # if self._aperture != 'NRCA1_FULL': - # raise ValueError(self._aperture, self.latest_bad_from_dark_type, self.latest_bad_from_dark_x, self.latest_bad_from_dark_y) - - def post_init(self): - self._update_badpix_v_time() - self._update_badpix_loc_plot() - - def get_history_data(self): - """Extract data on the history of bad pixel numbers from the - database query result - """ - self.bad_history = {} - self.bad_latest = {} - for bad_pixel_type in BAD_PIXEL_TYPES: - matching_rows = [row for row in self.bad_pixel_table if row.type == bad_pixel_type] - if len(matching_rows) != 0: - real_data = True - times = [row.obs_mid_time for row in matching_rows] - num = np.array([len(row.x_coord) for row in matching_rows]) - - latest_row = times.index(max(times)) - self.bad_latest[bad_pixel_type] = (max(times), matching_rows[latest_row].x_coord, matching_rows[latest_row].y_coord) - - # If there are no records of a certain type of bad pixel, then - # fall back to a default date and 0 bad pixels. Remember that - # these plots are always showing the number of NEW bad pixels - # that are not included in the current reference file. - else: - real_data = False - - times = [datetime.datetime(2021, 10, 31), datetime.datetime(2021, 11, 1)] - badpix_x = [1000, 999] - badpix_y = [1000, 999] - num = np.array([0, 0]) - self.bad_latest[bad_pixel_type] = (max(times), badpix_x, badpix_y) - - hover_values = np.array([datetime.datetime.strftime(t, "%d-%b-%Y") for t in times]) - self.bad_history[bad_pixel_type] = (times, num, hover_values) - - # if real_data: - # raise ValueError(bad_pixel_type, self.bad_history[bad_pixel_type]) - - def identify_tables(self): - """Determine which database tables as associated with - a given instrument""" - mixed_case_name = JWST_INSTRUMENT_NAMES_MIXEDCASE[self._instrument.lower()] - self.query_table = eval('{}BadPixelQueryHistory'.format(mixed_case_name)) - self.pixel_table = eval('{}BadPixelStats'.format(mixed_case_name)) - - def load_data(self): - """Query the database tables to get data""" - - # Determine which database tables are needed based on instrument - self.identify_tables() - - # Query database for all data with a matching aperture - self.bad_pixel_table = session.query(self.pixel_table) \ - .filter(self.pixel_table.detector == self.detector) \ - .all() - - session.close() - - def load_dummy_data(self): - """Create dummy data for Bokeh plot development""" - import datetime - - # Populate a dictionary with the number of bad pixels vs time for - # each type of bad pixel. We can't get the full list of bad pixel - # types from the database itself, because if there is a type of bad - # pixel with no found instances, then it won't appear in the database - # Also populate a dictionary containing the locations of all of the - # bad pixels found in the most recent search - self.bad_history = {} - self.bad_latest = {} - for i, bad_pixel_type in enumerate(BAD_PIXEL_TYPES): - - # Comment out while waiting for populated database tables - # num, times = self.bad_pixel_history(bad_pixel_type) - delta = 10 * i - - # Placeholders while we wait for a populated database - days = np.arange(1, 11) - times = np.array([datetime.datetime(2020, 8, day, 12, 0, 0) for day in days]) - num = np.arange(10) - hover_values = np.array([datetime.datetime.strftime(t, "%d-%b-%Y") for t in times]) - - self.bad_history[bad_pixel_type] = (times, num, hover_values) - self.bad_latest[bad_pixel_type] = (datetime.datetime(1999, 12, 31), [500 + delta, 501 + delta, 502 + delta], [4, 4, 4]) - - def most_recent_data(self): - """Get the bad pixel type and coordinates associated with the most - recent run of the monitor. Note that the most recent date can be - different for dark current data vs flat field data - """ - self.latest_bad_from_dark_type = [] - self.latest_bad_from_dark_x = [] - self.latest_bad_from_dark_y = [] - dark_times = [self.bad_latest[bad_pixel_type][0] for bad_pixel_type in DARKS_BAD_PIXEL_TYPES] - if len(dark_times) > 0: - self.most_recent_dark_date = max(dark_times) - else: - self.most_recent_dark_date = datetime.datetime(1999, 10, 31) - - for bad_pixel_type in DARKS_BAD_PIXEL_TYPES: - if self.bad_latest[bad_pixel_type][0] == self.most_recent_dark_date: - self.latest_bad_from_dark_type.extend([bad_pixel_type] * len(self.bad_latest[bad_pixel_type][1])) - self.latest_bad_from_dark_x.extend(self.bad_latest[bad_pixel_type][1]) - self.latest_bad_from_dark_y.extend(self.bad_latest[bad_pixel_type][2]) - - self.latest_bad_from_dark_type = np.array(self.latest_bad_from_dark_type) - self.latest_bad_from_dark_x = np.array(self.latest_bad_from_dark_x) - self.latest_bad_from_dark_y = np.array(self.latest_bad_from_dark_y) - - self.latest_bad_from_flat_type = [] - self.latest_bad_from_flat_x = [] - self.latest_bad_from_flat_y = [] - - self.latest_bad_from_flat = [[], [], []] - flat_times = [self.bad_latest[bad_pixel_type][0] for bad_pixel_type in FLATS_BAD_PIXEL_TYPES] - if len(flat_times) > 1: - self.most_recent_flat_date = max(flat_times) - else: - self.most_recent_flat_date = datetime.datetime(1999, 10, 31) - for bad_pixel_type in FLATS_BAD_PIXEL_TYPES: - if self.bad_latest[bad_pixel_type][0] == self.most_recent_flat_date: - self.latest_bad_from_flat_type.extend([bad_pixel_type] * len(self.bad_latest[bad_pixel_type][1])) - self.latest_bad_from_flat_x.extend(self.bad_latest[bad_pixel_type][1]) - self.latest_bad_from_flat_y.extend(self.bad_latest[bad_pixel_type][2]) - - self.latest_bad_from_flat_type = np.array(self.latest_bad_from_flat_type) - self.latest_bad_from_flat_x = np.array(self.latest_bad_from_flat_x) - self.latest_bad_from_flat_y = np.array(self.latest_bad_from_flat_y) - - def _update_badpix_loc_plot(self): - """Update the plot properties for the plots showing the locations - of new bad pixels""" - if 'MIR' in self._aperture: - self.refs['dark_position_xrange'].end = 1024 - self.refs['dark_position_yrange'].end = 1024 - self.refs['flat_position_xrange'].end = 1024 - self.refs['flat_position_yrange'].end = 1024 - - dark_date = self.most_recent_dark_date.strftime('%d-%b-%Y %H:%m') - self.refs['dark_position_figure'].title.text = '{} New Bad Pixels (darks). Obs Time: {}'.format(self._aperture, dark_date) - self.refs['dark_position_figure'].title.align = "center" - self.refs['dark_position_figure'].title.text_font_size = "15px" - - flat_date = self.most_recent_flat_date.strftime('%d-%b-%Y %H:%m') - self.refs['flat_position_figure'].title.text = '{} New Bad Pixels (flats). Obs Time: {}'.format(self._aperture, flat_date) - self.refs['flat_position_figure'].title.align = "center" - self.refs['flat_position_figure'].title.text_font_size = "15px" - - def _update_badpix_v_time(self): - """Update the plot properties for the plots of the number of bad - pixels versus time - """ - for bad_pixel_type in BAD_PIXEL_TYPES: - bad_pixel_type_lc = bad_pixel_type.lower() - - # Define y ranges of bad pixel v. time plot - buffer_size = 0.05 * (max(self.bad_history[bad_pixel_type][1]) - min(self.bad_history[bad_pixel_type][1])) - if buffer_size == 0: - buffer_size = 1 - self.refs['{}_history_yrange'.format(bad_pixel_type_lc)].start = min(self.bad_history[bad_pixel_type][1]) - buffer_size - self.refs['{}_history_yrange'.format(bad_pixel_type_lc)].end = max(self.bad_history[bad_pixel_type][1]) + buffer_size - - # Define x range of bad_pixel v. time plot - horizontal_half_buffer = (max(self.bad_history[bad_pixel_type][0]) - min(self.bad_history[bad_pixel_type][0])) * 0.05 - if horizontal_half_buffer == 0: - horizontal_half_buffer = 1. # day - self.refs['{}_history_xrange'.format(bad_pixel_type_lc)].start = min(self.bad_history[bad_pixel_type][0]) - horizontal_half_buffer - self.refs['{}_history_xrange'.format(bad_pixel_type_lc)].end = max(self.bad_history[bad_pixel_type][0]) + horizontal_half_buffer - - # Add a title - self.refs['{}_history_figure'.format(bad_pixel_type.lower())].title.text = '{}: {} pixels'.format(self._aperture, bad_pixel_type) - self.refs['{}_history_figure'.format(bad_pixel_type.lower())].title.align = "center" - self.refs['{}_history_figure'.format(bad_pixel_type.lower())].title.text_font_size = "20px" - - - - - -# Uncomment the line below when testing via the command line: -# bokeh serve --show monitor_badpixel_bokeh.py -# BadPixelMonitor() From a6a84abcb142c5ba91f86ed4992faf0b69ad30e9 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 28 Feb 2023 10:01:17 -0500 Subject: [PATCH 107/449] Make cases with few bad pixels more visible --- .../jwql/monitor_pages/monitor_bad_pixel_bokeh.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py index cbbf07b4b..cf28a0221 100755 --- a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py @@ -477,7 +477,6 @@ def create_plot(self): # Plot image if image is not None: - print('mapper, nx, ny are not defined yet') ny, nx = image.shape #imgplot = self.plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, color_mapper=mapper, level="image") @@ -487,7 +486,7 @@ def create_plot(self): # to align with the axes #self.plot = figure(title=title, x_range=(0, self._detlen), y_range=(0, self._detlen), width=xdim, height=ydim*, # tools='pan,box_zoom,reset,wheel_zoom,save', x_axis_label="Pixel Number", y_axis_label="Pixel Number") - self.plot.image_rgba(image=[image], x=0, y=0, dw=self._detlen, dh=self._detlen) + self.plot.image_rgba(image=[image], x=0, y=0, dw=self._detlen, dh=self._detlen, alpha=0.5) @@ -557,8 +556,15 @@ def overplot_bad_pix(self): ) # Overplot the bad pixel locations - badpixplots = self.plot.circle(x='pixels_x', y='pixels_y', source=source, color='#EC04FF', - fill_alpha=0.75, line_alpha=0.75, radius=0.5) + # If we have very few bad pixels to plot, increase the size of the circles, in order to make + # it easier to find them on the plot + radius = 0.5 + if len(self.coords[0]) < 50: + radius = 1.0 + pink = '#EC04FF' + green = '#07FF1F' + badpixplots = self.plot.circle(x='pixels_x', y='pixels_y', source=source, fill_color=pink, line_color=pink, + fill_alpha=1.0, line_alpha=1.0, radius=radius) # Create hover tool for the bad pixel type # If there are "too many" points then we are going to save the plot as From fc78cb4990800279e9c64f58ebb83668c14348cc Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 28 Feb 2023 11:13:57 -0500 Subject: [PATCH 108/449] Note line that should be uncommented later --- .../apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py index cf28a0221..72fec3eef 100755 --- a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py @@ -606,15 +606,13 @@ def switch_to_png(self, filename, title): fig_array = read_png(filename) ydim, xdim = fig_array.shape - # Shift the figure title slightly right in this case to get it - # to align with the axes + # Create the figure self.plot = figure(title=title, x_range=(0, self._detlen), y_range=(0, self._detlen), width=xdim, height=ydim, tools='pan,box_zoom,reset,wheel_zoom,save', x_axis_label="Pixel Number", y_axis_label="Pixel Number") self.plot.image_rgba(image=[fig_array], x=0, y=0, dw=self._detlen, dh=self._detlen) - #self.plot.xaxis.visible = False - #self.plot.yaxis.visible = False # Now that the data from the png is in the figure, delete the png + # UNCOMMENT THE LINE BELOW FOR PRODUCTION #os.remove(filename) From 3bb41e2b14bb832579375fbbf3085e463810cf21 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Tue, 28 Feb 2023 11:18:07 -0500 Subject: [PATCH 109/449] Hopefully speedup of checking whether new bad pixels are overlaps --- .../common_monitors/dark_monitor.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index 8558fce1f..f92bf1779 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -405,6 +405,8 @@ def exclude_existing_badpix(self, badpix, pixel_type): y_coords = _row.y_coord for x, y in zip(x_coords, y_coords): already_found.append((x, y)) + found_x = np.array([x[0] for x in already_found]) + found_y = np.array([x[1] for x in already_found]) logging.info("\tChecking pixels against found list") # Check to see if each pixel already appears in the database for @@ -412,10 +414,15 @@ def exclude_existing_badpix(self, badpix, pixel_type): new_pixels_x = [] new_pixels_y = [] for x, y in zip(badpix[0], badpix[1]): - pixel = (x, y) - if pixel not in already_found: + ind_x = np.where(found_x == x) + ind_y = np.where(found_y == y) + if len(np.intersect1d(ind_x[0], ind_y[0])) > 0: new_pixels_x.append(x) new_pixels_y.append(y) +# pixel = (x, y) +# if pixel not in already_found: +# new_pixels_x.append(x) +# new_pixels_y.append(y) session.close() return (new_pixels_x, new_pixels_y) From 322efb8de146d57a5ba84e496d28e73f8e567765 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 28 Feb 2023 11:37:03 -0500 Subject: [PATCH 110/449] Clean up code in badpix monitor. Update plots at the end --- .../common_monitors/bad_pixel_monitor.py | 77 ++----------------- 1 file changed, 6 insertions(+), 71 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py index 6094798ae..89e217382 100755 --- a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py +++ b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py @@ -104,8 +104,8 @@ from jwql.instrument_monitors import pipeline_tools from jwql.shared_tasks.shared_tasks import only_one, run_pipeline, run_parallel_pipeline from jwql.utils import crds_tools, instrument_properties, monitor_utils +from jwql.utils.constants import DARKS_BAD_PIXEL_TYPES, DARK_EXP_TYPES, FLATS_BAD_PIXEL_TYPES, FLAT_EXP_TYPES, from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE -from jwql.utils.constants import FLAT_EXP_TYPES, DARK_EXP_TYPES from jwql.utils.logging_functions import log_info, log_fail from jwql.utils.mast_utils import mast_query from jwql.utils.permissions import set_permissions @@ -432,66 +432,6 @@ def add_bad_pix(self, coordinates, pixel_type, files, obs_start_time, obs_mid_ti 'entry_date': datetime.datetime.now()} self.pixel_table.__table__.insert().execute(entry) - def create_badpix_plot(self): - """Create a Bokeh figure of a single bad pixel figure. For the given - detector/aperture and bad pixel type load an image and display. Then scatter markers - - """ - - def create_plot_layout(self): - """Creat a bokeh figure to hold an image of the detector with - bad pixel locations marked on the map. Save the figure as a - json file, so that it can be loaded into the bad pixel results - html page, and displayed - - There should be one tab for each detector. On each tab, there - are plots showing the locations of new bad pixels from darks, - and new bad pixels from flats. Then there is a separate plot - for each flavor of bad pixel that shows the number of these - bad pixels found versus time. - - BUT: nircam has no lamp, so we will never had any bad pix from - flats. - What other entries would be missing from other instruments? - """ - do we want to create one json file for each detector/aperture? - or like the edb monitor, one json file that holds all the - plots? The latter case is probably easier. With multiple json - files, we would need a custom html file that loads each json file - individually. This means we would need a different html file for - each instrument. - - - # example of creating a custom grid layout - sliders = column(amp, freq, phase, offset) - - layout([ - [bollinger], - [sliders, plot], - [p1, p2, p3], - ]) - - - - - - - - - # Wrap the plots in a Panel - panel_list.append(Panel(child=grid, title=key)) - - - - # Save the tabbed plot to a json file - this is from EDB monitor - item_text = json.dumps(json_item(tabbed, "tabbed_edb_plot")) - basename = f'edb_{self.instrument}_tabbed_plots.json' - output_file = os.path.join(self.plot_output_dir, basename) - with open(output_file, 'w') as outfile: - outfile.write(item_text) - logging.info(f'JSON file with tabbed plots saved to {output_file}') - - def filter_query_results(self, results, datatype): """Filter MAST query results. For input flats, keep only those with the most common filter/pupil/grating combination. For both @@ -837,11 +777,9 @@ def process(self, illuminated_raw_files, illuminated_slope_files, flat_file_coun # Illuminated files - run entirety of calwebb_detector1 for uncal # files where corresponding rate file is 'None' badpix_types = [] - badpix_types_from_flats = ['DEAD', 'LOW_QE', 'OPEN', 'ADJ_OPEN'] - badpix_types_from_darks = ['HOT', 'RC', 'OTHER_BAD_PIXEL', 'TELEGRAPH'] illuminated_obstimes = [] if illuminated_raw_files: - badpix_types.extend(badpix_types_from_flats) + badpix_types.extend(FLATS_BAD_PIXEL_TYPES) out_exts = defaultdict(lambda: ['jump', '0_ramp_fit']) in_files = [] for uncal_file, rate_file in zip(illuminated_raw_files, illuminated_slope_files): @@ -903,7 +841,7 @@ def process(self, illuminated_raw_files, illuminated_slope_files, flat_file_coun dark_obstimes = [] if dark_raw_files: index = 0 - badpix_types.extend(badpix_types_from_darks) + badpix_types.extend(DARKS_BAD_PIXEL_TYPES) # In this case we need to run the pipeline on all input files, # even if the rate file is present, because we also need the jump # and fitops files, which are not saved by default @@ -1058,18 +996,14 @@ def process(self, illuminated_raw_files, illuminated_slope_files, flat_file_coun # Add new hot and dead pixels to the database logging.info('\tFound {} new {} pixels'.format(len(bad_location_list[0]), bad_type)) - if bad_type in badpix_types_from_flats: + if bad_type in FLATS_BAD_PIXEL_TYPES: self.add_bad_pix(bad_location_list, bad_type, illuminated_slope_files, min_illum_time, mid_illum_time, max_illum_time, baseline_file) flat_png = create_png_from_fits(illuminated_slope_files[0], self.output_dir) - elif bad_type in badpix_types_from_darks: + elif bad_type in DARKS_BAD_PIXEL_TYPES: self.add_bad_pix(bad_location_list, bad_type, dark_slope_files, min_dark_time, mid_dark_time, max_dark_time, baseline_file) dark_png = create_png_from_fits(dark_slope_files[0], self.output_dir) - - here: create_badpix_plot() - - else: raise ValueError("Unrecognized type of bad pixel: {}. Cannot update database table.".format(bad_type)) @@ -1281,6 +1215,7 @@ def run(self): for instrument in updated_instruments: BadPixelPlots(instrument) + logging.info(f'Updating web pages for: {updated_instruments}') logging.info('Bad Pixel Monitor completed successfully.') From 7adfa411fb1a39e4dc8d9dbb92915e829ef6d07c Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 28 Feb 2023 11:44:31 -0500 Subject: [PATCH 111/449] typo --- jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py index 89e217382..1922c41f9 100755 --- a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py +++ b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py @@ -104,7 +104,7 @@ from jwql.instrument_monitors import pipeline_tools from jwql.shared_tasks.shared_tasks import only_one, run_pipeline, run_parallel_pipeline from jwql.utils import crds_tools, instrument_properties, monitor_utils -from jwql.utils.constants import DARKS_BAD_PIXEL_TYPES, DARK_EXP_TYPES, FLATS_BAD_PIXEL_TYPES, FLAT_EXP_TYPES, +from jwql.utils.constants import DARKS_BAD_PIXEL_TYPES, DARK_EXP_TYPES, FLATS_BAD_PIXEL_TYPES, FLAT_EXP_TYPES from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE from jwql.utils.logging_functions import log_info, log_fail from jwql.utils.mast_utils import mast_query From 4c075865ba863ddd4ac67c71e771216c703d01e3 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 27 Feb 2023 16:01:58 -0500 Subject: [PATCH 112/449] Refactor filtering arguments; add exposure group display handling --- jwql/website/apps/jwql/static/css/jwql.css | 18 +++- jwql/website/apps/jwql/static/js/jwql.js | 116 ++++++++++++--------- 2 files changed, 84 insertions(+), 50 deletions(-) diff --git a/jwql/website/apps/jwql/static/css/jwql.css b/jwql/website/apps/jwql/static/css/jwql.css index eaa74e690..56f86b757 100644 --- a/jwql/website/apps/jwql/static/css/jwql.css +++ b/jwql/website/apps/jwql/static/css/jwql.css @@ -526,7 +526,6 @@ text-align: left; color: white; z-index: 2; - font-size: 0.75rem; } .thumbnail-staff { @@ -537,7 +536,21 @@ display: inline-block; margin: 0.1rem; } - + + /*Format thumbnail groups when active*/ + .thumbnail-group { + display: inline; + font-size: 0.75rem; + } + .thumbnail-group-active { + display: block; + width: 90%; + height: 90%; + border: 1px solid #2d353c; + box-shadow: 5px 5px #c85108, 10px 10px #bec4d4; + position: absolute; + font-size: 0.65rem; + } /*Format the version identifier text in bottom corner*/ #version-div { @@ -581,4 +594,3 @@ padding-left:10px; line-height:25px; } - \ No newline at end of file diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index 4ecc468d4..15ad326b6 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -6,6 +6,7 @@ * @author Brad Sappington * @author Bryan Hilbert * @author Maria Pena-Guerrero + * @author Melanie Clarke */ /** @@ -175,8 +176,8 @@ function determine_filetype_for_thumbnail(thumbnail_dir, thumb_filename, i, file var img = document.getElementById('thumbnail'+i); if (thumb_filename != 'none') { var jpg_path = thumbnail_dir + file_root.slice(0,7) + '/' + thumb_filename; - img.src = jpg_path - }; + img.src = jpg_path; + } }; @@ -370,20 +371,34 @@ function get_number_or_none(element_id) { * @param {String} group_type - The group type * @param {String} base_url - The base URL for gathering data from the AJAX view. */ -function group_by_thumbnails(group_type, dropdown_keys, num_fileids, base_url) { +function group_by_thumbnails(group_type, base_url) { // Update dropdown menu text and update thumbnails for current setting - //document.getElementById('group_dropdownMenuButton').innerHTML = group_type; - show_only('group', group_type, dropdown_keys, num_fileids, 'thumbnail', base_url); + show_only('group', group_type, base_url); - // Group the thumbnails accordingly. + // Group divs to update display style + var group_divs = document.getElementsByClassName("thumbnail-group"); + // Thumbnail links to update to group or image pages + var thumbnail_links = document.getElementsByClassName("thumbnail-link"); + // Show count total and type to update + var img_total = document.getElementById('img_total'); + var img_type = document.getElementById('img_type'); + var group_by = document.getElementById('group_by') - // TODO: actually group thumbnails - var thumbs = $('div#thumbnail-array>div'); if (group_type == 'Exposure') { - console.log('Group by exposure'); + img_total.innerText = group_by.getAttribute('data-ngroup'); + img_type.innerText = 'groups'; + for (var i = 0; i < group_divs.length; i++) { + group_divs[i].classList.add('thumbnail-group-active'); + thumbnail_links[i].href = thumbnail_links[i].getAttribute('data-group-href'); + } } else { - console.log('Group by file'); + img_total.innerText = group_by.getAttribute('data-nfile'); + img_type.innerText = 'activities'; + for (var i = 0; i < group_divs.length; i++) { + group_divs[i].classList.remove('thumbnail-group-active'); + thumbnail_links[i].href = thumbnail_links[i].getAttribute('data-image-href'); + } } $.ajax({ @@ -430,20 +445,18 @@ function search() { // Find all proposal elements var proposals = document.getElementsByClassName("proposal"); - var n_proposals = document.getElementsByClassName("proposal").length; // Determine the current search value var search_value = document.getElementById("search_box").value; // Determine whether or not to display each thumbnail var num_proposals_displayed = 0; - for (i = 0; i < proposals.length; i++) { + for (var i = 0; i < proposals.length; i++) { // Evaluate if the proposal number matches the search var j = i + 1 var prop_name = document.getElementById("proposal" + j).getAttribute('proposal') var prop_num = Number(prop_name) - if (prop_name.startsWith(search_value) || prop_num.toString().startsWith(search_value)) { proposals[i].style.display = "inline-block"; num_proposals_displayed++; @@ -460,7 +473,7 @@ function search() { }; // Update the count of how many images are being shown - document.getElementById('img_show_count').innerHTML = 'Showing ' + num_proposals_displayed + '/' + n_proposals + ' proposals'; + document.getElementById('img_shown').innerText = num_proposals_displayed; }; @@ -468,11 +481,13 @@ function search() { * Limit the displayed thumbnails based on filter criteria * @param {String} filter_type - The filter type. * @param {Integer} value - The filter value - * @param {List} dropdown_keys - A list of dropdown menu keys - * @param {Integer} num_fileids - The number of files that are available to display - * @param {String} thumbnail_class - The class name of the thumbnails that will be filtered. + * @param {String} base_url - The base URL for gathering data from the AJAX view. */ -function show_only(filter_type, value, dropdown_keys, num_fileids, thumbnail_class, base_url) { +function show_only(filter_type, value, base_url) { + + var filter_div = document.getElementById('filter_by'); + var dropdown_keys = filter_div.getAttribute('data-dropdown-key-list'); + var thumbnail_class = filter_div.getAttribute('data-thumbnail-class'); // Get all filter options from {{dropdown_menus}} variable var all_filters = dropdown_keys.split(','); @@ -501,7 +516,7 @@ function show_only(filter_type, value, dropdown_keys, num_fileids, thumbnail_cla var num_thumbnails_displayed = 0; var list_of_rootnames = ""; var groups_shown = new Set(); - for (i = 0; i < thumbnails.length; i++) { + for (var i = 0; i < thumbnails.length; i++) { // Evaluate if the thumbnail meets all filter criteria var criteria = []; for (j = 0; j < all_filters.length; j++) { @@ -536,7 +551,7 @@ function show_only(filter_type, value, dropdown_keys, num_fileids, thumbnail_cla } // Update the count of how many images are being shown - document.getElementById('img_show_count').innerHTML = 'Showing ' + num_thumbnails_displayed + '/' + num_fileids + ' activities' + document.getElementById('img_shown').innerText = num_thumbnails_displayed; if (num_thumbnails_displayed) { // remove trailing ','. list_of_rootnames = list_of_rootnames.slice(0, -1); @@ -688,7 +703,7 @@ function update_archive_page(inst, base_url) { // Update the number of proposals displayed num_proposals = data.thumbnails.proposals.length; update_show_count(num_proposals, 'proposals') - update_filter_options(data, base_url, num_proposals, 'proposal'); + update_filter_options(data, base_url, 'proposal'); // Add content to the proposal array div for (var i = 0; i < data.thumbnails.proposals.length; i++) { @@ -854,18 +869,19 @@ function update_wata_page(base_url) { /** * Updates the thumbnail-filter div with filter options * @param {Object} data - The data returned by the update_thumbnails_page AJAX method - * @param {Integer} num_items - The total number of items that will be filtered upon + * @param {String} base_url - The base URL for gathering data from the AJAX view. * @param {String} thumbnail_class - the class name of the thumbnails that will be filtered */ - function update_filter_options(data, base_url, num_items, thumbnail_class) { - content = 'Filter by:' + function update_filter_options(data, base_url, thumbnail_class) { + var dropdown_key_list = Object.keys(data.dropdown_menus); + content = '
    Filter by:
    '; for (var i = 0; i < Object.keys(data.dropdown_menus).length; i++) { // Parse out useful variables filter_type = Object.keys(data.dropdown_menus)[i]; filter_options = Array.from(new Set(data.dropdown_menus[filter_type])); - num_rootnames = num_items; - dropdown_key_list = Object.keys(data.dropdown_menus); // Build div content content += '
    '; @@ -873,10 +889,10 @@ function update_wata_page(base_url) { content += '
    '; content += ''; content += '
    '; - content += 'All ' + filter_type + 's'; + content += 'All ' + filter_type + 's'; for (var j = 0; j < filter_options.length; j++) { - content += '' + filter_options[j] + ''; + content += '' + filter_options[j] + ''; }; content += '
    '; @@ -890,17 +906,18 @@ function update_wata_page(base_url) { /** * Updates the group-by-exposure div * @param {Object} data - The data returned by the update_thumbnails_page AJAX method + * @param {String} base_url - The base URL for gathering data from the AJAX view. */ -function update_group_options(data, base_url, num_items, thumbnail_class) { - - var dropdown_key_list = Object.keys(data.dropdown_menus); +function update_group_options(data, base_url) { // Build div content - content = 'Group by:
    ' + content = '
    Group by:
    '; content += ''; content += '
    '; - content += 'Exposure'; - content += 'File'; + content += 'Exposure'; + content += 'File'; content += '
    '; // Add the content to the div $("#group-by-exposure")[0].innerHTML = content; @@ -959,7 +976,7 @@ function update_obs_options(data, inst, prop, observation) { * @param {String} type - The type of the count (e.g. "activities") */ function update_show_count(count, type) { - content = 'Showing ' + count + '/' + count + ' ' + type; + content = 'Showing
    ' + count + '
    /
    ' + count + '
    ' + type + '
    '; content += ''; content += 'i'; $("#img_show_count")[0].innerHTML = content; @@ -968,6 +985,7 @@ function update_show_count(count, type) { /** * Updates the thumbnail-sort div with sorting options * @param {Object} data - The data returned by the update_thumbnails_page AJAX method + * @param {String} base_url - The base URL for gathering data from the AJAX view. */ function update_sort_options(data, base_url) { @@ -992,7 +1010,7 @@ function update_sort_options(data, base_url) { */ function update_thumbnail_array(data) { - // Add content to the thumbail array div + // Add content to the thumbnail array div for (var i = 0; i < Object.keys(data.file_data).length; i++) { // Parse out useful variables @@ -1003,13 +1021,17 @@ function update_thumbnail_array(data) { filename_dict = file.filename_dict; // Build div content - if (data.inst!="all") { - content = '
    '; - content += ''; + var instrument; + if (data.inst != "all") { + instrument = data.inst; } else { - content = '
    '; - content += ''; + instrument = filename_dict.instrument; } + content = '
    '; + content += '
    ' + content += ''; content += ''; content += '
    '; content += '
    '; @@ -1018,7 +1040,7 @@ function update_thumbnail_array(data) { content += 'Visit: ' + filename_dict.visit + '
    '; content += 'Detector: ' + filename_dict.detector + '
    '; content += 'Exp_Start: ' + file.expstart_iso + '
    '; - content += '
    '; + content += '
    '; // Add the content to the div $("#thumbnail-array")[0].innerHTML += content; @@ -1060,7 +1082,7 @@ function submit_date_range_form(inst, base_url) { if (num_thumbnails > 0) { update_show_count(num_thumbnails, 'activities'); update_thumbnail_array(data); - update_filter_options(data, base_url, num_thumbnails, 'thumbnail'); + update_filter_options(data, base_url, 'thumbnail'); // Do initial sort to match sort button display update_sort_options(data, base_url); @@ -1109,12 +1131,12 @@ function update_thumbnails_per_observation_page(inst, proposal, observation, bas update_show_count(num_thumbnails, 'activities'); update_thumbnail_array(data); update_obs_options(data, inst, proposal, observation); - update_filter_options(data, base_url, num_thumbnails, 'thumbnail'); - update_group_options(data, base_url, num_thumbnails, 'thumbnail'); + update_filter_options(data, base_url, 'thumbnail'); + update_group_options(data, base_url); update_sort_options(data, base_url); // Do initial sort and group to match sort button display - group_by_thumbnails(group, Object.keys(data.dropdown_menus).toString(), num_thumbnails, base_url); + group_by_thumbnails(group, base_url); sort_by_thumbnails(sort, base_url); // Replace loading screen with the proposal array div @@ -1136,7 +1158,7 @@ function update_thumbnails_query_page(base_url, sort) { num_thumbnails = Object.keys(data.file_data).length; update_show_count(num_thumbnails, 'activities'); update_thumbnail_array(data); - update_filter_options(data, base_url, num_thumbnails, 'thumbnail'); + update_filter_options(data, base_url, 'thumbnail'); update_sort_options(data, base_url); // Do initial sort to match sort button display From 54ad1e943d02b599009a73194fb04cf0d154d48a Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Tue, 28 Feb 2023 11:48:44 -0500 Subject: [PATCH 113/449] Javascript lint fixes --- jwql/website/apps/jwql/static/js/jwql.js | 242 +++++++++++------------ 1 file changed, 121 insertions(+), 121 deletions(-) diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index 15ad326b6..81820d563 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -30,19 +30,19 @@ document.getElementById(type).checked = true; // Clean the input parameters - var num_ints = num_ints.replace(/'/g, '"'); - var num_ints = num_ints.replace(/'/g, '"'); - var num_ints = JSON.parse(num_ints); + num_ints = num_ints.replace(/'/g, '"'); + num_ints = num_ints.replace(/'/g, '"'); + num_ints = JSON.parse(num_ints); // Get the available integration jpg numbers - var available_ints = available_ints.replace(/'/g, '"'); - var available_ints = available_ints.replace(/'/g, '"'); - var available_ints = JSON.parse(available_ints)[type]; + available_ints = available_ints.replace(/'/g, '"'); + available_ints = available_ints.replace(/'/g, '"'); + available_ints = JSON.parse(available_ints)[type]; // Get the total number of integrations - var total_ints = total_ints.replace(/'/g, '"'); - var total_ints = total_ints.replace(/'/g, '"'); - var total_ints = JSON.parse(total_ints); + total_ints = total_ints.replace(/'/g, '"'); + total_ints = total_ints.replace(/'/g, '"'); + total_ints = JSON.parse(total_ints); // Propogate the text fields showing the filename and APT parameters var fits_filename = file_root + '_' + type; @@ -83,7 +83,7 @@ // Disable the "left" button, since this will be showing integ0 document.getElementById("int_before").disabled = true; -}; +} /** @@ -103,19 +103,20 @@ function change_int(file_root, num_ints, available_ints, method, direction = 'ri // Figure out the current image and integration var suffix = document.getElementById("jpg_filename").innerHTML.split('_'); var integration = Number(suffix[suffix.length - 1].replace('.jpg','').replace('integ','')) - var suffix = suffix[suffix.length - 2]; + suffix = suffix[suffix.length - 2]; var program = file_root.slice(0,7); // Find the total number of integrations for the current image - var num_ints = num_ints.replace(/'/g, '"'); - var num_ints = JSON.parse(num_ints)[suffix]; + num_ints = num_ints.replace(/'/g, '"'); + num_ints = JSON.parse(num_ints)[suffix]; // Get the available integration jpg numbers and the current integration index - var available_ints = available_ints.replace(/'/g, '"'); - var available_ints = JSON.parse(available_ints)[suffix]; + available_ints = available_ints.replace(/'/g, '"'); + available_ints = JSON.parse(available_ints)[suffix]; var current_index = available_ints.indexOf(integration); // Get the desired integration value + var new_integration; switch (method) { case "button": if ((integration == num_ints - 1 && direction == 'right')|| @@ -160,7 +161,7 @@ function change_int(file_root, num_ints, available_ints, method, direction = 'ri // Update the slider values document.getElementById("slider_range").value = new_integration + 1 document.getElementById("slider_val").innerHTML = new_integration + 1 -}; +} /** @@ -179,7 +180,7 @@ function determine_filetype_for_thumbnail(thumbnail_dir, thumb_filename, i, file img.src = jpg_path; } -}; +} /** @@ -192,6 +193,7 @@ function determine_page_title(instrument, proposal) { var url = document.URL; var url_split = url.split('/'); var url_title = url_split[url_split.length - 2]; + var final_title; if (url_title == 'archive') { final_title = 'Archived ' + instrument + ' Images: Proposal ' + proposal } else if (url_title == 'unlooked') { @@ -203,9 +205,9 @@ function determine_page_title(instrument, proposal) { document.getElementById('title').innerHTML = final_title; if (document.title != final_title) { document.title = final_title; - }; - }; -}; + } + } +} /** * Determine whether the page is archive or unlooked @@ -218,7 +220,7 @@ function determine_page_title_obs(instrument, proposal, observation) { var url = document.URL; var url_split = url.split('/'); var url_title = url_split[url_split.length - 3]; - var url_end = url_split[url_split.length - 1]; + var final_title; if (url_title == 'archive') { final_title = 'Archived ' + instrument + ' Images: Proposal ' + proposal + ', Observation ' + observation } else if (url_title == 'unlooked') { @@ -232,9 +234,9 @@ function determine_page_title_obs(instrument, proposal, observation) { document.getElementById('title').innerHTML = final_title; if (document.title != final_title) { document.title = final_title; - }; - }; -}; + } + } +} /** * adds/removes disabled_section class and clears value @@ -259,16 +261,16 @@ function determine_page_title_obs(instrument, proposal, observation) { function explore_image_update_enable_options(integrations, groups) { // Check nr of integrations and groups of currently selected extension - ext_name = get_radio_button_value("extension"); + var ext_name = get_radio_button_value("extension"); // Clean the input parameters and get our integrations/groups for this extension var calc_difference = false; - var integrations = integrations.replace(/'/g, '"'); - var integrations = integrations.replace(/'/g, '"'); - var integrations = JSON.parse(integrations)[ext_name]; - var groups = groups.replace(/'/g, '"'); - var groups = groups.replace(/'/g, '"'); - var groups = JSON.parse(groups)[ext_name]; + integrations = integrations.replace(/'/g, '"'); + integrations = integrations.replace(/'/g, '"'); + integrations = JSON.parse(integrations)[ext_name]; + groups = groups.replace(/'/g, '"'); + groups = groups.replace(/'/g, '"'); + groups = JSON.parse(groups)[ext_name]; // Zero base our calculations integrations -= 1 @@ -345,7 +347,7 @@ function getCookie(name) { function get_radio_button_value(element_name) { var element = document.getElementsByName(element_name); - for(i = 0; i < element.length; i++) { + for(var i = 0; i < element.length; i++) { if(element[i].checked) { return element[i].value; } @@ -388,14 +390,14 @@ function group_by_thumbnails(group_type, base_url) { if (group_type == 'Exposure') { img_total.innerText = group_by.getAttribute('data-ngroup'); img_type.innerText = 'groups'; - for (var i = 0; i < group_divs.length; i++) { + for (let i = 0; i < group_divs.length; i++) { group_divs[i].classList.add('thumbnail-group-active'); thumbnail_links[i].href = thumbnail_links[i].getAttribute('data-group-href'); } } else { img_total.innerText = group_by.getAttribute('data-nfile'); img_type.innerText = 'activities'; - for (var i = 0; i < group_divs.length; i++) { + for (let i = 0; i < group_divs.length; i++) { group_divs[i].classList.remove('thumbnail-group-active'); thumbnail_links[i].href = thumbnail_links[i].getAttribute('data-image-href'); } @@ -410,7 +412,7 @@ function group_by_thumbnails(group_type, base_url) { console.log("session image group update failed"); } }); -}; +} /** @@ -426,18 +428,6 @@ function image_error(image, makeThumbnail=false) { } -/** - * Parse a JSON string containing a Bokeh plot - * @param {String} element - json-formatted string - */ -function parse_plot_json(element) { - // Determine if the URL is 'archive' or 'unlooked' - var formatted = Object; - formatted = JSON.parse(element) -}; - - - /** * Perform a search of images and display the resulting thumbnails */ @@ -463,18 +453,18 @@ function search() { } else { proposals[i].style.display = "none"; } - }; + } // If there are no proposals to display, tell the user if (num_proposals_displayed == 0) { document.getElementById('no_proposals_msg').style.display = 'inline-block'; } else { document.getElementById('no_proposals_msg').style.display = 'none'; - }; + } // Update the count of how many images are being shown document.getElementById('img_shown').innerText = num_proposals_displayed; -}; +} /** @@ -504,7 +494,7 @@ function show_only(filter_type, value, base_url) { // Determine the current value for each filter var filter_values = []; - for (j = 0; j < all_filters.length; j++) { + for (var j = 0; j < all_filters.length; j++) { var filter_value = document.getElementById(all_filters[j] + '_dropdownMenuButton').innerHTML; filter_values.push(filter_value); } @@ -524,7 +514,7 @@ function show_only(filter_type, value, base_url) { var criterion = (filter_values[j].indexOf('All '+ all_filters[j] + 's') >=0) || (filter_attribute.includes(filter_values[j])); criteria.push(criterion); - }; + } // If data are grouped, check if a thumbnail for the group has already been displayed if (group && groups_shown.has(thumbnails[i].getAttribute('group_root'))) { @@ -540,14 +530,14 @@ function show_only(filter_type, value, base_url) { } else { thumbnails[i].style.display = "none"; } - }; + } if (document.getElementById('no_thumbnails_msg') != null) { // If there are no thumbnails to display, tell the user if (num_thumbnails_displayed == 0) { document.getElementById('no_thumbnails_msg').style.display = 'inline-block'; } else { document.getElementById('no_thumbnails_msg').style.display = 'none'; - }; + } } // Update the count of how many images are being shown @@ -568,7 +558,7 @@ function show_only(filter_type, value, base_url) { } }); } -}; +} /** @@ -589,7 +579,7 @@ function sort_by_proposals(sort_type) { // Sort by the most recent Observation Start tinysort(props, {order:'desc', attr:'obs_time'}); } -}; +} /** @@ -625,7 +615,7 @@ function sort_by_thumbnails(sort_type, base_url) { console.log("session image sort update failed"); } }); -}; +} /** @@ -680,8 +670,8 @@ function download_report(inst, base_url) { var status = filters[i].innerText.toLowerCase(); if (!status.includes('all')) { options += '&' + name + '=' + status; - }; - }; + } + } var report_url = '/' + inst + '/report' + options; console.log('Redirecting to: ' + report_url); @@ -701,7 +691,7 @@ function update_archive_page(inst, base_url) { success: function(data){ // Update the number of proposals displayed - num_proposals = data.thumbnails.proposals.length; + var num_proposals = data.thumbnails.proposals.length; update_show_count(num_proposals, 'proposals') update_filter_options(data, base_url, 'proposal'); @@ -709,17 +699,17 @@ function update_archive_page(inst, base_url) { for (var i = 0; i < data.thumbnails.proposals.length; i++) { // Parse out useful variables - prop = data.thumbnails.proposals[i]; - min_obsnum = data.min_obsnum[i]; - thumb = data.thumbnails.thumbnail_paths[i]; - n = data.thumbnails.num_files[i]; - viewed = data.thumbnails.viewed[i]; - exp_types = data.thumbnails.exp_types[i]; - obs_time = data.thumbnails.obs_time[i]; - cat_type = data.thumbnails.cat_types[i]; + var prop = data.thumbnails.proposals[i]; + var min_obsnum = data.min_obsnum[i]; + var thumb = data.thumbnails.thumbnail_paths[i]; + var n = data.thumbnails.num_files[i]; + var viewed = data.thumbnails.viewed[i]; + var exp_types = data.thumbnails.exp_types[i]; + var obs_time = data.thumbnails.obs_time[i]; + var cat_type = data.thumbnails.cat_types[i]; // Build div content - content = '
    '; + var content = '
    '; content += ''; @@ -735,9 +725,9 @@ function update_archive_page(inst, base_url) { // Replace loading screen with the proposal array div document.getElementById("loading").style.display = "none"; document.getElementById("proposal-array").style.display = "block"; - }; + } }}); -}; +} /** @@ -751,7 +741,7 @@ function update_msata_page(base_url) { success: function(data){ // Build div content - content = data["div"]; + var content = data["div"]; content += data["script"]; /* Add the content to the div @@ -769,7 +759,7 @@ function update_msata_page(base_url) { document.getElementById('msata_fail').style.display = "inline-block"; } }); -}; +} /** @@ -783,7 +773,7 @@ function update_wata_page(base_url) { success: function(data){ // Build div content - content = data["div"]; + var content = data["div"]; content += data["script"]; /* Add the content to the div @@ -801,7 +791,7 @@ function update_wata_page(base_url) { document.getElementById('wata_fail').style.display = "inline-block"; } }); -}; +} /** @@ -815,21 +805,23 @@ function update_wata_page(base_url) { function update_explore_image_page(inst, file_root, filetype, base_url, do_opt_args=false) { /* if they exist set up the optional parameters before the ajax call*/ - optional_params = ""; + var optional_params = ""; if(do_opt_args) { // Reset loading document.getElementById("loading").style.display = "inline-block"; document.getElementById("explore_image").style.display = "none"; document.getElementById("explore_image_fail").style.display = "none"; - calc_difference = document.getElementById("calcDifference").checked; + var calc_difference = document.getElementById("calcDifference").checked; // Get the arguments to update - scaling = get_radio_button_value("scaling"); - low_lim = get_number_or_none("low_lim"); - high_lim = get_number_or_none("high_lim"); - ext_name = get_radio_button_value("extension"); - int1_nr = get_number_or_none("integration1"); - grp1_nr = get_number_or_none("group1"); + var scaling = get_radio_button_value("scaling"); + var low_lim = get_number_or_none("low_lim"); + var high_lim = get_number_or_none("high_lim"); + var ext_name = get_radio_button_value("extension"); + var int1_nr = get_number_or_none("integration1"); + var grp1_nr = get_number_or_none("group1"); + var int2_nr; + var grp2_nr; if (calc_difference) { int2_nr = get_number_or_none("integration2"); grp2_nr = get_number_or_none("group2"); @@ -845,7 +837,7 @@ function update_wata_page(base_url) { success: function(data){ // Build div content - content = data["div"]; + var content = data["div"]; content += data["script"]; /* Add the content to the div @@ -863,7 +855,7 @@ function update_wata_page(base_url) { document.getElementById('explore_image_fail').style.display = "inline-block"; } }); -}; +} /** @@ -874,14 +866,14 @@ function update_wata_page(base_url) { */ function update_filter_options(data, base_url, thumbnail_class) { var dropdown_key_list = Object.keys(data.dropdown_menus); - content = '
    Filter by:
    '; + var content = '
    Filter by:
    '; for (var i = 0; i < Object.keys(data.dropdown_menus).length; i++) { // Parse out useful variables - filter_type = Object.keys(data.dropdown_menus)[i]; - filter_options = Array.from(new Set(data.dropdown_menus[filter_type])); + var filter_type = Object.keys(data.dropdown_menus)[i]; + var filter_options = Array.from(new Set(data.dropdown_menus[filter_type])); // Build div content content += '    '; content += '
    '; - }; + } // Add the content to the div $("#thumbnail-filter")[0].innerHTML = content; -}; +} /** * Updates the group-by-exposure div @@ -911,7 +903,7 @@ function update_wata_page(base_url) { function update_group_options(data, base_url) { // Build div content - content = '
    Group by:
    '; content += ''; @@ -921,7 +913,7 @@ function update_group_options(data, base_url) { content += '
    '; // Add the content to the div $("#group-by-exposure")[0].innerHTML = content; -}; +} @@ -938,7 +930,7 @@ function update_header_display(extension, num_extensions) { var header_table = document.getElementById("header-table-extension" + i); header_name.style.display = 'none'; header_table.style.display = 'none'; - }; + } // Display the header selected var header_name_to_show = document.getElementById("header-display-name-extension" + extension); @@ -946,7 +938,7 @@ function update_header_display(extension, num_extensions) { header_name_to_show.style.display = 'inline'; header_table_to_show.style.display = 'inline'; -}; +} /** * Updates the obs-list div with observation number options @@ -957,7 +949,7 @@ function update_header_display(extension, num_extensions) { */ function update_obs_options(data, inst, prop, observation) { // Build div content - content = 'Available observations:'; + var content = 'Available observations:'; content += '
    '; content += ''; content += '
    '; @@ -968,7 +960,7 @@ function update_obs_options(data, inst, prop, observation) { // Add the content to the div $("#obs-list")[0].innerHTML = content; -}; +} /** * Updates the img_show_count component @@ -976,11 +968,11 @@ function update_obs_options(data, inst, prop, observation) { * @param {String} type - The type of the count (e.g. "activities") */ function update_show_count(count, type) { - content = 'Showing
    ' + count + '
    /
    ' + count + '
    ' + type + '
    '; + var content = 'Showing
    ' + count + '
    /
    ' + count + '
    ' + type + '
    '; content += ''; content += 'i'; $("#img_show_count")[0].innerHTML = content; -}; +} /** * Updates the thumbnail-sort div with sorting options @@ -990,7 +982,7 @@ function update_show_count(count, type) { function update_sort_options(data, base_url) { // Build div content - content = 'Sort by:'; + var content = 'Sort by:'; content += '
    '; content += ''; content += '
    '; @@ -1002,7 +994,7 @@ function update_sort_options(data, base_url) { // Add the content to the div $("#thumbnail-sort")[0].innerHTML = content; -}; +} /** * Updates the thumbnail-array div with interactive images of thumbnails @@ -1014,11 +1006,11 @@ function update_thumbnail_array(data) { for (var i = 0; i < Object.keys(data.file_data).length; i++) { // Parse out useful variables - rootname = Object.keys(data.file_data)[i]; - file = data.file_data[rootname]; - viewed = file.viewed; - exp_type = file.exp_type; - filename_dict = file.filename_dict; + var rootname = Object.keys(data.file_data)[i]; + var file = data.file_data[rootname]; + var viewed = file.viewed; + var exp_type = file.exp_type; + var filename_dict = file.filename_dict; // Build div content var instrument; @@ -1027,7 +1019,10 @@ function update_thumbnail_array(data) { } else { instrument = filename_dict.instrument; } - content = '
    '; + var content = '
    '; content += '
    ' content += ' 1000) { show_thumbs = false; @@ -1110,11 +1110,11 @@ function submit_date_range_form(inst, base_url) { } }); } -}; +} /** - * Updates various compnents on the thumbnails page + * Updates various components on the thumbnails page * @param {String} inst - The instrument of interest (e.g. "FGS") * @param {String} proposal - The proposal number of interest (e.g. "88660") * @param {String} observation - The observation number within the proposal (e.g. "001") @@ -1127,7 +1127,7 @@ function update_thumbnails_per_observation_page(inst, proposal, observation, bas url: base_url + '/ajax/' + inst + '/archive/' + proposal + '/obs' + observation + '/', success: function(data){ // Perform various updates to divs - num_thumbnails = Object.keys(data.file_data).length; + var num_thumbnails = Object.keys(data.file_data).length; update_show_count(num_thumbnails, 'activities'); update_thumbnail_array(data); update_obs_options(data, inst, proposal, observation); @@ -1143,7 +1143,7 @@ function update_thumbnails_per_observation_page(inst, proposal, observation, bas document.getElementById("loading").style.display = "none"; document.getElementById("thumbnail-array").style.display = "block"; }}); -}; +} /** * Updates various components on the thumbnails anomaly query page @@ -1155,7 +1155,7 @@ function update_thumbnails_query_page(base_url, sort) { url: base_url + '/ajax/query_submit/', success: function(data){ // Perform various updates to divs - num_thumbnails = Object.keys(data.file_data).length; + var num_thumbnails = Object.keys(data.file_data).length; update_show_count(num_thumbnails, 'activities'); update_thumbnail_array(data); update_filter_options(data, base_url, 'thumbnail'); @@ -1168,7 +1168,7 @@ function update_thumbnails_query_page(base_url, sort) { document.getElementById("loading").style.display = "none"; document.getElementById("thumbnail-array").style.display = "block"; }}); -}; +} function update_viewed_button(viewed) { var elem = document.getElementById("viewed"); @@ -1190,4 +1190,4 @@ function version_url(version_string) { a_line += version_string; a_line += '">JWQL v' + version_string + ''; return a_line; -}; +} From a6597b3908e69f1b13838066b1dd5618a5787f9d Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Tue, 28 Feb 2023 12:37:17 -0500 Subject: [PATCH 114/449] Updated dark monitor to look for local rate file --- .../common_monitors/dark_monitor.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index f92bf1779..7feed5a17 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -386,7 +386,7 @@ def exclude_existing_badpix(self, badpix, pixel_type): logging.warning("Now new {} pixels found.".format(pixel_type)) return ([], []) - logging.info("Excluding {} existing bad pixels {}".format(len(badpix[0]), pixel_type)) + logging.info("Checking {} potential new {} pixels".format(len(badpix[0]), pixel_type)) if pixel_type not in ['hot', 'dead', 'noisy']: raise ValueError('Unrecognized bad pixel type: {}'.format(pixel_type)) @@ -397,7 +397,7 @@ def exclude_existing_badpix(self, badpix, pixel_type): .filter(self.pixel_table.detector == self.detector) \ .all() - logging.info("\tCreating list of found pixels") + logging.info("\tCreating list of {} existing {} pixels".format(len(db_entries, pixel_type))) already_found = [] if len(db_entries) != 0: for _row in db_entries: @@ -679,10 +679,12 @@ def process(self, file_list): logging.info('\tWorking on file: {}'.format(filename)) rate_file = filename.replace("dark", "rate") + rate_file_name = os.path.basename(rate_file) + local_rate_file = os.path.join(self.data_dir, rate_file_name) - if os.path.isfile(rate_file): - logging.info("\t\tFile {} exists, skipping pipeline".format(rate_file)) - slope_files.append(filename) + if os.path.isfile(local_rate_file): + logging.info("\t\tFile {} exists, skipping pipeline".format(local_rate_file)) + slope_files.append(local_rate_file) else: logging.info("\t\tAdding {} to calibration set".format(filename)) pipeline_files.append(filename) From 92a47de944ecc532e86e75e779a7e3b290501bd3 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 28 Feb 2023 13:35:48 -0500 Subject: [PATCH 115/449] Create archive page asynchronously --- .../apps/jwql/archive_database_update.py | 2 + jwql/website/apps/jwql/data_containers.py | 107 +++++++++++++++++- jwql/website/apps/jwql/views.py | 101 +++-------------- 3 files changed, 121 insertions(+), 89 deletions(-) diff --git a/jwql/website/apps/jwql/archive_database_update.py b/jwql/website/apps/jwql/archive_database_update.py index ebfdf275a..19e9185d1 100755 --- a/jwql/website/apps/jwql/archive_database_update.py +++ b/jwql/website/apps/jwql/archive_database_update.py @@ -44,6 +44,7 @@ from jwql.utils.monitor_utils import initialize_instrument_monitor # noqa from jwql.utils.constants import MAST_QUERY_LIMIT # noqa from jwql.utils.utils import filename_parser, filesystem_path, get_config # noqa +from jwql.website.apps.jwql.data_containers import create_archived_proposals_context # noqa from jwql.website.apps.jwql.data_containers import get_instrument_proposals, get_filenames_by_instrument # noqa from jwql.website.apps.jwql.data_containers import get_proposal_info, mast_query_filenames_by_instrument # noqa @@ -313,3 +314,4 @@ def update_database_table(instrument, prop, obs, thumbnail, obsfiles, types, sta instruments = ['nircam', 'miri', 'nirspec', 'niriss', 'fgs'] for instrument in instruments: get_updates(instrument) + create_archived_proposals_context(instrument) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index c15fad920..b8a559442 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -27,6 +27,7 @@ import copy from collections import OrderedDict import glob +import json from operator import getitem import os import re @@ -53,6 +54,7 @@ from jwql.utils.constants import IGNORED_SUFFIXES, INSTRUMENT_SERVICE_MATCH, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_INSTRUMENT_NAMES from jwql.utils.constants import SUFFIXES_TO_ADD_ASSOCIATION, SUFFIXES_WITH_AVERAGED_INTS, QUERY_CONFIG_KEYS, QUERY_CONFIG_TEMPLATE from jwql.utils.credentials import get_mast_token +from jwql.utils.permissions import set_permissions from jwql.utils.utils import get_rootnames_for_instrument_proposal from .forms import InstrumentAnomalySubmitForm from astroquery.mast import Mast @@ -79,7 +81,7 @@ setup() from .forms import MnemonicSearchForm, MnemonicQueryForm, MnemonicExplorationForm - from jwql.website.apps.jwql.models import RootFileInfo + from jwql.website.apps.jwql.models import Observation, Proposal, RootFileInfo check_config_for_key('auth_mast') configs = get_config() auth_mast = configs['auth_mast'] @@ -93,6 +95,7 @@ FILESYSTEM_DIR = configs['filesystem'] PREVIEW_IMAGE_FILESYSTEM = configs['preview_image_filesystem'] THUMBNAIL_FILESYSTEM = configs['thumbnail_filesystem'] + OUTPUT_DIR = configs['outputs'] PACKAGE_DIR = os.path.dirname(__location__.split('website')[0]) REPO_DIR = os.path.split(PACKAGE_DIR)[0] @@ -147,6 +150,100 @@ def build_table(tablename): return table_meta_data +def create_archived_proposals_context(inst): + """Generate and save a json file containing the information needed + to create an instrument's archive page. + + Parameters + ---------- + inst : str + Name of JWST instrument + """ + # Ensure the instrument is correctly capitalized + inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()] + + # Get a list of Observation entries for the given instrument + all_entries = Observation.objects.filter(proposal__archive__instrument=inst) + + # Get a list of proposal numbers. + prop_objects = Proposal.objects.filter(archive__instrument=inst) + proposal_nums = [entry.prop_id for entry in prop_objects] + + # Put proposals into descending order + proposal_nums.sort(reverse=True) + + # Total number of proposals for the instrument + num_proposals = len(proposal_nums) + + thumbnail_paths = [] + min_obsnums = [] + total_files = [] + proposal_viewed = [] + proposal_exp_types = [] + thumb_exp_types = [] + proposal_obs_times = [] + thumb_obs_time = [] + + # Get a set of all exposure types used in the observations associated with this proposal + exp_types = [exposure_type for observation in all_entries for exposure_type in observation.exptypes.split(',')] + exp_types = sorted(list(set(exp_types))) + + # The naming conventions for dropdown_menus are tightly coupled with the code, this should be changed down the line. + dropdown_menus = {'look': THUMBNAIL_FILTER_LOOK, + 'exp_type': exp_types} + thumbnails_dict = {} + + for proposal_num in proposal_nums: + # For each proposal number, get all entries + prop_entries = all_entries.filter(proposal__prop_id=proposal_num) + + # All entries will have the same thumbnail_path, so just grab the first + thumbnail_paths.append(prop_entries[0].proposal.thumbnail_path) + + # Extract the observation numbers from each entry and find the minimum + prop_obsnums = [entry.obsnum for entry in prop_entries] + min_obsnums.append(min(prop_obsnums)) + + # Sum the file count from all observations to get the total file count for + # the proposal + prop_filecount = [entry.number_of_files for entry in prop_entries] + total_files.append(sum(prop_filecount)) + + # In order to know if a proposal contains all observations that are entirely viewed, check for at least one existing viewed=False in RootFileInfo + unviewed_root_file_infos = RootFileInfo.objects.filter(instrument=inst, proposal=proposal_num, viewed=False) + proposal_viewed.append("Viewed" if unviewed_root_file_infos.count() == 0 else "New") + + # Store comma separated list of exp_types associated with each proposal + proposal_exp_types = [exposure_type for observation in prop_entries for exposure_type in observation.exptypes.split(',')] + proposal_exp_types = list(set(proposal_exp_types)) + thumb_exp_types.append(','.join(proposal_exp_types)) + + # Get Most recent observation start time + proposal_obs_times = [observation.obsstart for observation in prop_entries] + thumb_obs_time.append(max(proposal_obs_times)) + + thumbnails_dict['proposals'] = proposal_nums + thumbnails_dict['thumbnail_paths'] = thumbnail_paths + thumbnails_dict['num_files'] = total_files + thumbnails_dict['viewed'] = proposal_viewed + thumbnails_dict['exp_types'] = thumb_exp_types + thumbnails_dict['obs_time'] = thumb_obs_time + + context = {'inst': inst, + 'num_proposals': num_proposals, + 'min_obsnum': min_obsnums, + 'thumbnails': thumbnails_dict, + 'dropdown_menus': dropdown_menus} + + json_object = json.dumps(context, indent = 4) + + # Writing to json file + outfilename = os.path.join(OUTPUT_DIR, 'archive_page', f'{inst}_archive_context.json') + with open(outfilename, "w") as outfile: + outfile.write(json_object) + set_permissions(outfilename) + + def get_acknowledgements(): """Returns a list of individuals who are acknowledged on the ``about`` page. @@ -1004,7 +1101,7 @@ def get_thumbnails_all_instruments(parameters): ---------- parameters: dict of type QUERY_CONFIG_TEMPLATE A dictionary containing keys of QUERY_CONFIG_KEYS, some of which are dictionaries: - + Returns ------- @@ -1057,7 +1154,7 @@ def get_thumbnails_all_instruments(parameters): # Get list of all thumbnails thumb_inventory = os.path.join(f"{THUMBNAIL_FILESYSTEM}", f"{THUMBNAIL_LISTFILE}_{inst.lower()}.txt") thumbnail_inst_list = retrieve_filelist(thumb_inventory) - + # Get subset of thumbnail images that match the filenames thumbnails_inst_subset = [os.path.basename(item) for item in thumbnail_inst_list if os.path.basename(item).split('_integ')[0] in inst_filenames] @@ -1152,8 +1249,8 @@ def get_thumbnails_by_proposal(proposal): def get_thumbnail_by_rootname(rootname): """Return the most appropriate existing thumbnail basename available in the filesystem for the given ``rootname``. - We generate thumbnails only for 'rate' and 'dark' files. - Check if these files exist in the thumbnail filesystem. + We generate thumbnails only for 'rate' and 'dark' files. + Check if these files exist in the thumbnail filesystem. In the case where neither rate nor dark thumbnails are present, revert to 'none' Parameters diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index ab84ab796..bc72306e0 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -44,6 +44,7 @@ from collections import defaultdict from copy import deepcopy import csv +import json import glob import logging import os @@ -181,12 +182,12 @@ def api_landing(request): def save_page_navigation_data_ajax(request): """ Takes a bracketless string of rootnames and expstarts, and saves it as a session dictionary - + Parameters ---------- request: HttpRequest object Incoming request from the webpage - + Returns ------- @@ -202,7 +203,7 @@ def save_page_navigation_data_ajax(request): request.session['navigation_data'] = rootname_expstarts context = {'item': request.session['navigation_data']} return JsonResponse(context, json_dumps_params={'indent': 2}) - + def archive_date_range(request, inst): """Generate the page for date range images @@ -268,7 +269,7 @@ def archive_date_range_ajax(request, inst, start_date, stop_date): end_after_start = Observation.objects.filter(proposal__archive__instrument=inst, obsend__gte=inclusive_start_time.mjd) all_entries_ending_in_range = end_after_start.filter(obsend__lt=exclusive_stop_time.mjd) - # Get a queryset of all observations SPANNING (starting before and ending after) our date range. Bump our window out a few days to catch hypothetical + # Get a queryset of all observations SPANNING (starting before and ending after) our date range. Bump our window out a few days to catch hypothetical # observations that last over 24 hours. The larger the window the more time the query takes so keeping it tight. two_days_before_start_time = Time(start_date) - (2 * u.day) two_days_after_end_time = Time(stop_date) + (3 * u.day) @@ -278,13 +279,13 @@ def archive_date_range_ajax(request, inst, start_date, stop_date): obs_beginning = [observation for observation in all_entries_beginning_in_range] obs_ending = [observation for observation in all_entries_ending_in_range] obs_spanning = [observation for observation in all_entries_spanning_range] - + # Create a single list of all pertinent observations all_observations = list(set(obs_beginning + obs_ending + obs_spanning)) # Get all thumbnails that occurred within the time frame for these observations data = thumbnails_date_range_ajax(inst, all_observations, inclusive_start_time.mjd, exclusive_stop_time.mjd) data['thumbnail_sort'] = request.session.get("image_sort", "Recent") - + # Create Dictionary of Rootnames with expstart save_page_navigation_data(request, data) return JsonResponse(data, json_dumps_params={'indent': 2}) @@ -331,81 +332,13 @@ def archived_proposals_ajax(request, inst): JsonResponse object Outgoing response sent to the webpage """ - # Ensure the instrument is correctly capitalized - inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()] - - # Get a list of Observation entries for the given instrument - all_entries = Observation.objects.filter(proposal__archive__instrument=inst) - - # Get a list of proposal numbers. - prop_objects = Proposal.objects.filter(archive__instrument=inst) - proposal_nums = [entry.prop_id for entry in prop_objects] - - # Put proposals into descending order - proposal_nums.sort(reverse=True) - - # Total number of proposals for the instrument - num_proposals = len(proposal_nums) - - thumbnail_paths = [] - min_obsnums = [] - total_files = [] - proposal_viewed = [] - proposal_exp_types = [] - thumb_exp_types = [] - proposal_obs_times = [] - thumb_obs_time = [] - - # Get a set of all exposure types used in the observations associated with this proposal - exp_types = [exposure_type for observation in all_entries for exposure_type in observation.exptypes.split(',')] - exp_types = sorted(list(set(exp_types))) - - # The naming conventions for dropdown_menus are tightly coupled with the code, this should be changed down the line. - dropdown_menus = {'look': THUMBNAIL_FILTER_LOOK, - 'exp_type': exp_types} - thumbnails_dict = {} - - for proposal_num in proposal_nums: - # For each proposal number, get all entries - prop_entries = all_entries.filter(proposal__prop_id=proposal_num) - - # All entries will have the same thumbnail_path, so just grab the first - thumbnail_paths.append(prop_entries[0].proposal.thumbnail_path) - - # Extract the observation numbers from each entry and find the minimum - prop_obsnums = [entry.obsnum for entry in prop_entries] - min_obsnums.append(min(prop_obsnums)) - - # Sum the file count from all observations to get the total file count for - # the proposal - prop_filecount = [entry.number_of_files for entry in prop_entries] - total_files.append(sum(prop_filecount)) - - # In order to know if a proposal contains all observations that are entirely viewed, check for at least one existing viewed=False in RootFileInfo - unviewed_root_file_infos = RootFileInfo.objects.filter(instrument=inst, proposal=proposal_num, viewed=False) - proposal_viewed.append("Viewed" if unviewed_root_file_infos.count() == 0 else "New") - - # Store comma separated list of exp_types associated with each proposal - proposal_exp_types = [exposure_type for observation in prop_entries for exposure_type in observation.exptypes.split(',')] - proposal_exp_types = list(set(proposal_exp_types)) - thumb_exp_types.append(','.join(proposal_exp_types)) - - # Get Most recent observation start time - proposal_obs_times = [observation.obsstart for observation in prop_entries] - thumb_obs_time.append(max(proposal_obs_times)) - - thumbnails_dict['proposals'] = proposal_nums - thumbnails_dict['thumbnail_paths'] = thumbnail_paths - thumbnails_dict['num_files'] = total_files - thumbnails_dict['viewed'] = proposal_viewed - thumbnails_dict['exp_types'] = thumb_exp_types - thumbnails_dict['obs_time'] = thumb_obs_time + # Read in the json file created by data_containers.create_archived_proposals_context + # as use as the context + output_dir = get_config()['outputs'] + context_file = os.path.join(output_dir, 'archive_page', f'{inst}_archive_context.json') - context = {'inst': inst, - 'num_proposals': num_proposals, - 'min_obsnum': min_obsnums, - 'thumbnails': thumbnails_dict, - 'dropdown_menus': dropdown_menus} + with open(context_file, 'r') as obj: + context = json.load(obj) return JsonResponse(context, json_dumps_params={'indent': 2}) @@ -518,7 +451,7 @@ def archive_thumbnails_query_ajax(request): # when parameters only contains nirspec as instrument, thumbnails still end up being all niriss data thumbnails = get_thumbnails_all_instruments(parameters) - + data = thumbnails_query_ajax(thumbnails) data['thumbnail_sort'] = request.session.get("image_sort", "Ascending") save_page_navigation_data(request, data) @@ -827,7 +760,7 @@ def query_submit(request): HttpResponse object Outgoing response sent to the webpage """ - + template = 'query_submit.html' sort_type = request.session.get('image_sort', 'Ascending') context = {'inst': '', @@ -1048,9 +981,9 @@ def explore_image_ajax(request, inst, file_root, filetype, scaling="log", low_li def save_page_navigation_data(request, data): """ It saves the data from the current page in the session so that the user can navigate to the next or - previous page. Our current sort options are Ascending/Descending, and Recent/Oldest so we need to + previous page. Our current sort options are Ascending/Descending, and Recent/Oldest so we need to preserve 'rootname' and 'expstart'. - + Parameters ---------- request: HttpRequest object From 37b1b1f2d2d1ee4542eb5f0f481f24efa3f7d2a1 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Tue, 28 Feb 2023 13:46:08 -0500 Subject: [PATCH 116/449] Typo --- jwql/instrument_monitors/common_monitors/dark_monitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index 7feed5a17..6b58f7a35 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -397,7 +397,7 @@ def exclude_existing_badpix(self, badpix, pixel_type): .filter(self.pixel_table.detector == self.detector) \ .all() - logging.info("\tCreating list of {} existing {} pixels".format(len(db_entries, pixel_type))) + logging.info("\tCreating list of {} existing {} pixels".format(len(db_entries), pixel_type)) already_found = [] if len(db_entries) != 0: for _row in db_entries: From 7d8292ea6d725560dde6d6bccb28e0a147fbfe00 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 28 Feb 2023 14:34:23 -0500 Subject: [PATCH 117/449] Doc typo --- jwql/website/apps/jwql/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index bc72306e0..e739ac956 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -333,7 +333,7 @@ def archived_proposals_ajax(request, inst): Outgoing response sent to the webpage """ # Read in the json file created by data_containers.create_archived_proposals_context - # as use as the context + # and use as the context output_dir = get_config()['outputs'] context_file = os.path.join(output_dir, 'archive_page', f'{inst}_archive_context.json') From bfeb71dbf39acb9193b2103f0b1f224274adfed7 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Tue, 28 Feb 2023 16:34:44 -0500 Subject: [PATCH 118/449] Clean up invalid HTML --- jwql/website/apps/jwql/static/css/jwql.css | 2 +- .../apps/jwql/static/img/default_thumb.png | Bin 0 -> 30110 bytes jwql/website/apps/jwql/static/js/jwql.js | 62 ++++++++++-------- jwql/website/apps/jwql/templates/archive.html | 6 +- .../jwql/templates/archive_date_range.html | 4 +- jwql/website/apps/jwql/templates/base.html | 6 +- .../apps/jwql/templates/query_submit.html | 3 +- .../jwql/templates/thumbnails_per_obs.html | 2 +- .../apps/jwql/templates/view_image.html | 2 + 9 files changed, 49 insertions(+), 38 deletions(-) create mode 100644 jwql/website/apps/jwql/static/img/default_thumb.png diff --git a/jwql/website/apps/jwql/static/css/jwql.css b/jwql/website/apps/jwql/static/css/jwql.css index 56f86b757..6e841a95f 100644 --- a/jwql/website/apps/jwql/static/css/jwql.css +++ b/jwql/website/apps/jwql/static/css/jwql.css @@ -334,7 +334,7 @@ } /* Get rid of padding around GitHub logo */ - #github-link { + #github-link, #github-link-collapsed { padding-bottom: 0px; padding-top: 0px; } diff --git a/jwql/website/apps/jwql/static/img/default_thumb.png b/jwql/website/apps/jwql/static/img/default_thumb.png new file mode 100644 index 0000000000000000000000000000000000000000..cdb71f6008e291e1fd70608d6d72b7e05f2f88e4 GIT binary patch literal 30110 zcmcHg1yoh-!uF5S-6&nsN`rJPI;24a1nKS$i3QRvjnW|@NH({T7|}(DF!T9c!XT>a47h-#Ca~Cb?G`X)hE6l;8GWZrpARg6{=S zvw1cL=X_A%>If;~nQ^sHX$*CkW3kb5n2HKU6jb3c+~7%d5NX5oXiQ|~X2?Z%@fmxs*ZxirQ3y}?SRim2 zk6e<1^=DMH+7ao8cSd|&P9Zs($RT@P;pa34T^wkUnp1Z8s*ze(lDj{MA1r^(06^t6DC^g31PA8d1P5b8)ny)pkoao z!s|T=c>?d^SH`SKK~Y-C@r#4xBS$@hUVH4zCB%^FiN)~)2?J7r>agMC=GI^r^ z-o;Nu^Cbx-M;sx?;Bv6cz-mWo5L!DzU;qw%rA2@O4Jw+@E!oXd=2s>XrvMT&crhn6 zPAR$sGf_#e7{u-+)$pqvBQO1cvQ*qintS)OWJ5~v!mu7ne6u&zpy`dx_|G1n9Xg+l z=-;PNdzX=RURz*aLiV!Hr7pZHvG1n|^nxk7W~vgW1oxOH;I?!yC&Jy0dJDX~+(lEq zls|E-s`hfpja{OKhGvyLPefMXRr#i3xaJoEDH-v6Zlp}0Oo&-sjk&FKC9^a-Em^)Q z-&~Veg9$fs7zH^$XOz8|7k3kCbv?a@YcDc)ayt6q$zFyWa7X2I%kQ^oM{+cpi`72O zST3zkzv;9c0=GzD<5aYbXljIGx8$1T;`Iy6>wuEXi=7T2knSNt#@z#?21>zXNY4*5r zYW{>bHDYDOQ0(UB;^M<(>F%$3Rz_qSI&_UO5p!-#?M$oEx!s7U-;wKuFxbtsT64s3 zh2l$W5ZAD*eT3=-g2sq5kPz&l8zAvL`S4yQO_KRZ5}^kk4xvitw=pR%IeG&3@Cj+$ zMqEXyxL)*R0pf*Y?*ePGnqguX$vGOq7pv zyEI*xU7zkU*CmZ6wIuQP_xD@$XZ3USBdiPd?{quQA;`o;_-5rW^-JQof$~*jHsL!p)@_F-~pYWzw?M(<29V;akSbrid z=o{6~X;MB-+Mpwo@-@H*;Ca=un!opz~asiB_ctd_94&u5R2 z3$A?!N~PWxVP6u9YLqqu2Hi7Nn`N7o-n1T=%)g!=Xg)lb-MZdAK7Vn(b5_3XH&V2V zy1ltmG3+?pW*I;}McPH8#5HB+G?U{PJb`wIKZpN`4kx-l;`_Uaz`i=T!s3j}a9S?PhffQGKq>e3=SE`4#yMzi*YXm8Fy! z3iXtQl|kvc=?rPVcuIL%ELFHH(=0z$@Hz4xHgHyv*m8B|^{mu0OeYwBuy2~EoW@!#XM=o|%#1vn`qDQqaH1a<^u1qNO1oO}iQOu4*sA<}D$5_|zi+qs)4klObw|=2-Q&al$ks{!_(JkwbKiL(`7isjS;>Z1 z4HXAe2PT)VFAZ=ptu4z7L9EceR zUGQOu3pHPLcyyT%s@mVTkF*cc{K^T=!IroT`t`2no$b3@sX58QunaVY;Lxygv~L}J zCa%-tdr;1}$`8i&%g>gbm&w0V%TCFJ(|zpCGwtjtC-dPS;npCT#zW4VQZLOrkkOM< zjTIx1ijIga3HK%t7V?m_^PymKFqu1O^gM5Vh&m1YP1-GR`X2{R}v~i(X--~zCqOOt~dmj})qzhkOdaIni8uSqc zS^J+Nyz47U#99bOwEt`;c+ozkht4{w*Ds+F7Lfw0Qe>r0Gm=3S|+6p_6arr$+Oo;4SIJzOux)*G%;b zg2}!+HhM~Wo7+dDmP1I{g5!gBj)J?c7h8fAuF9LFo8;_`hQ)_Yy8@7w*~2j5dEs4- zxB7Np*St%bY?UUD2mDi24bZ>3Y&^T&39*fzY%XWgZ}XOB6q>P~Z{s3^P0_lckG<@kl|X~U7DC+aCGstg5}+yzTON2G-X z!qHnMxUolYR!6?Fq)eduJ`7q=O+Vh1s*b;G(qm6Jo8@~SC2B}Z6MTE?{?Qfk`S=K2 zcp(DbA-WoI>avjAtHex}Pa$YS@OpV|7HgIPV9hnlLFyG~6S9Vu_L1u?hpd;cE|zwy z0WhOTG1pPBP*#R}1}vk&A;S~EAp%SAz%Lv;865J%G926sc=CTQYrr%A-39>;F3cJZ z>F+kLflt`ad*BZ!{r3|wAruZ3_>Tbi3(i6K*VgcEIf(x(1K+_(Xi6z40H2zsF6QPA zu2zn2-SC(UzzTFHd0kgHI3i$~hyh+fgZ>B(4q?*zrH-49vXYRgqdmK^nWKq0yO+Ha zY(H=!UP8d4y}6q)jhDThgR792DDA@*LclU?H3u!t!zOOFqO>~7YBW-gF6K0R?40bJ zv|<=EG&CYEW)?yk(z1Vx1OJKATDiG732|_EdU~>Z@~}I)SaNU)3JP*?a&vHVvjJPM zxq3Ue8GErgxYGUE$-nj^ZSHF7V(sK+?dU)Q+pn>Sqr00ZEiLRo|Ni-Np5|WG{~XD| z_3vc?7svtohJ%ZpljDCX=4Ng2zZ8Rg^QYLub^SS<2yACUYSv!nb~@75_Q0(MPECx5 zpIhYNF#p##|D5Qbl3K3jE>e#6KtVULf1cIf!vFc@e;0f>r0#zn^7(%r@;|=$kCL!U z5YjMrb+mJb-9jw~Yd10AO#i3V|6NM=pJHNM&w*n-6#CD#|6M}oe=PBzYyZ21s*5#n zAB-e6@Ae`but)zNRpC!PdsqupG%*Yjj(=BLF$|MtuIF%Y;&2Mm5-+{r z_cQI^&MK9enSQbJaN+0OO)gY9ru?V_RP2l);-f~fjakKy$x4meo~aq5=(R6kc*O;k+2Vq4wn!||R&rEUcO-I;>M zinthg3>j)SyjOn$97JV3U$F;7s8e@32|Wi%<{oQ9A^cFXzQHm|buz4?X3WN~L<_W6 zv{Q0CBKAM@46B#s$eR<6@g5y%O3g`ClXd0 zc1^2q9yGPx8q!Dph}_@W5Q;}7)WD{NZRmR5zc|bOVejpwu4-SC_+Yn2l#3PIPsPyg zpDpHIE_)pH#*Y=Bv*3fYj&3fU9UQs`vq5V6ZCPFTy(Ra&(Ee=gYnE!1I+Xn`jety1 zxcgyR0p5x4qZmQ_N6e!tN6ObD-6xTjOVT+W27^m<&EKqF`(3*G|5Ur3ye9*%50yg3 z69$lb5MNS5h%Bp}GpXhZDH-~+Pu+&| zzQG1N#`%avS8rCr)lf+X4fOQ|nNTR*>#J|-R(4Dc?Pu+hK7LQSvYo;s@HCjmiBbrR z6=rR7f3p}IPS?BMW5_C=mIMMF^OFh%Y%rj{=!rfq6Jq>Y5X+h`Pej+bcHG3AUr2;V zvD1>shhm92(+3%4(jBC6JI(VNWX`0<>&ezphx-09Tx5Hm7ShmN@zLYqGL1y?l6THvwQH5-Ke`C0co0NvP zK9q*AaAKP9Ii~+)RjMl`_n?qBW(H{%F6He^<)?k?DY~wdC#HKk>CT_ZkLSK9(c)YW z6GRg_;}&Ih`7ugJtPjTB*<}C~a`MWR6%RK8JufJA@uSn(D=57?(U-7+rLnHMD$WF< zYK?bKwl(7X8|{BM-v3_W-S3+*pR#T)Y1e6P6ipW%+s^OF&3xig$@-3E;TkwQ>SKxE z=g<_M^`TveG6uF*o%1)_rO)3wu>As`4`28Xq67*kGTHSs_QQ*uPwN{+)_Nt&NZtPY z{^*0iu;l1ng=g@xYw35km$Q{yISnd~RI5x4mL78srZ@>2)EcfUTcNCZ}D2(cA zqBPfoKdtPV$5G5vmB89(8_>Q@`Eald)(+wPB!o)2pEPyYU%g#Y=yiS`NGv~^ zx}NMst;unfzejZR<(c`0x~@zrMDex4m)~_SH+#wIb@t|(UDPM?_(kLK-hPp{ z`aNFhw+Y0?nP6RK+>0OjjbGBEbp+!u5k)Gj!^ z+3%?Yh31ZQECYOB?H&87-^WJc%ExR3fZFB6)7{EBi%L#SlYqL>z?ERS>bPf|mjtRf zx@2v-65M1;Br+OH46}Xu6Rv@Ck10icEVSesman;Z%kN5$}(KTDntNE8rsS!rC`Dq#sE;tXV7f-6nkkT%lX|a8u%c1c z5`otwi+e!O?CAFtx=>p*ec~AOn5#wU&arc~OZ(4t=@R0<`bM8eGSlD{8hN;{Eg!;V zTtUZr@EmgmZ!Wgp@DxE5Fe(a6|#o zQr)v4u86%^?1YCXcKX^DoMHOrGqi}r77{_UJ|qm`9sXi;;vl}?`(aqy9--VD_cw>c zrpis|64mz>`+=Oj(-)MYAaCcqhSLFd(~oHd7a&K1VHdDz^CYwgU13E)(`I@@q-Z-L%+BX6i2RyGz zD{@+{5s542O}g$pFw|J0#M}tuCHBsll_7PABAbxLM#Y(OLT!5 zxHaH!B~T{C_YwTSpbHeA#P9c}vE34i`Ha%9(AvRK!PM{f;Yj+)a8d7v`bT@w(uMvy zi2Vvrnnh$&aTk$Y&GBEAa!_i}ce*)2Pdz8wX(yV5cQvcK`unLDl1@XN z;xj*XNJCA5Ho3~2alTG?-iD;Is=A;C?LwLH36bihzA7E)n%%}VF5tIYNt<*GEt?m4B( zmIlJe!RxqG#|Yp%(IM2HPAh68ql|usPn!9&wfWN5u369}=mnLpSy@1ty|l*zzda@Vd#WL8*U_YQQ*QEyR1tCG%Cr5J%|*}fh=A&eT> ztb!uRoz=z#Kv!is$Dd}dC;d%{b?L!DGlrp+Y{0k}tm!4m9$z5)Y!?*MrURFH&41nb zMgn{gOOzp*ZbaU$p*p+LUD6aX)%}TeKT#a=Vux&Ko&fhfm9J*}OoFe*(tKU4fC*4> zcERW*`{ZE8{wXQn;a5P#sW_VK>NZ+HvM`CFU&Q>FrU9Dclh(_smp7bATRGo(pG#u%H{cvsze6CeZEOI*_#I_=oSU6F%Pw=gxtG~QnmD6o^aUOj#)K#oW375Lom`k4^f>RU`G zGtX<;w0MaonY80;{da@Z@Bbx*?4XftYvZy9d8zL2+q-?v;Ys`HmT7hPJ|-r&WZyU> z@ZOKY^vknt#1Y>X{MFyt+MB%|)iV3kpqb)wpyur1ug9Uovp*&WuT}(5NgQ zS$(Z^EN;+!jFm~f zJN`$i(owggwJlc=l|Lw$4U``+1*4ku+c8j{t+msueNR#E%3XF)lxl48gLQGsB)7t@ zaN8H%+ga5W-E&aUtC+TS$|5)hp%jp+zkj#k!W3$GQ;t5buClT&JLzNRx6N`w6TIzQ z)e%GSxO~$I?y9am`5!=7>A}?*`P*`)Ok0pT5WD)lVzi(c88|8|SH?~gYDYj-`Q0#l z=a(XFz{KwY#&`Xf7m5PW=sx+SjKK|bZ%C%*>7>3N&|VA$?A2{QqM7$0LLKEcb@#{O zNOUT`+h>_FeZ!0s5;3N6Pu{sUe8mDra6=AFm3}0E;^dWA|GwL=R{e3oG=&jM8CmzQ zu~CJf&cUE&K(^%V;rh6xRgpBFN;%ydJxQ;;=+$$J%q9fiscu+7@TnxCjT6YaPb|y3 zJ=@cUF_Z2D2u(|rQuF%vYGc^-4jB9 znH+3{@(yWRku@TFk1+F|sI?HO{Jh$%ctDJSlHm-!jf&Y3+Aj3X!Q^XS?aMRtclmN7 zMl7S(RnjW)PyaqI#HZkmiq>~3nA^0y7_KGJPX@9)zC@998Q`LF!Jqld3pH{D*9yxi zk}W+`H!gy|lok(kd8oWylC=59VU#9OC#Ytvy^G|N$#Ei}bdcv>ce(5DyjK<3IALBMIdiCj(m`w_NL&L#nu( zh>S1#76J9*%!8x|ryoDjy3UYxrPW&XgMfQ4L+~ZT;ozxGy=qoCk4k8(X3^)FJ8y4O zBw*C0E}IS70mrnRY-&T^R2x(nDaro&ptk2Xw+wVVz9t8?4vT2cOZ4s3W-2H&*=F;k zVo$FF7FLxh49j!&AY!Bwe=6`P75Gv}vV$lxxO~(r#}2PFPALOd2zP0d;294^>~1!X z@>2nlzy^Pvpg0vb?`U>Phx9j_amtnE`yeVl1&KS{3M*ht0i8~2_lmk9MtL9HMhBZs zbW&k6Hi}6dU;b zlN7?!BtG}$gA7WPN=BaIbMA;(Op7zfAK`+oIh;n7lFM7LnOpYDF>_AU7C8Db#^B9K z%#9&J(J_-F9yw&B74k&Hf31QYzQ@XxbJmgTC;rY!Q;CaJ#*CeLCU8rr{AMEstWE*su)`z@>UL}{tG#r-;;tRa}d^0N>%dcYmQm75v z$E5b67km_dlTG&t5QfZToCwk(xWFL5E}*bPlU?VKT2O*3ye{ui41mQmp$UY{`1v4m z_2iU(-{Gc2%MARZb*^Wpqu2J@=|>*}|F#|;I|e!bkhX@fgBc24F#EYaHvT*1$1a2u2Rm_g&?DSnMr90Rh0HF~D`kZ|QC3P2^o*WMkqg97j z=~$HFry6h!rmqCcdDz_)D0#_pdisn`y7ZIOHKX7M%MJ=+I;P)wEK2N%^$~QySPbn< zxh-~io}ejgsBP7FP&7d@tGt!xU+#l+!%O#kw)>r{Ju7V(c;WT$1qz0Taf4*yUPZld z;NUfZY$5uLFV*~CKt|PwORqm$XN3uR!+zTG1^HzydObYrc`fqzXR-^lD&)bGERRO^ zbI>i40ac8x0$Yaw{-x85wRpCInp)GTsWEvG@Vt1T6qsF7CV3Dp#=1^|7QbJHgqPt8 z@bxuQ@q6-*@0@1w?DZwRs-4%)Bxg$LpDn)3vCdA;!r9glUmfnl6V!Rt?Yc&M0GUF? zJT8jwIMe(~69$SEuv%OhA3PiJGWE$R8y@~Habr&1KiyZLwOT~sDZx}=(l+P%CrMUMXS zsC2Kd79k83uae;O`I1#rL|&?^ItddzkGi@pqKc}@H|OvB2(qqf)pS`){$h(GSIC<; zdtH@jr+(WT$5#qd!hiJ)Fy;bkhLq7t;62gU8Cx-(XM)Z7RTKP{S9AtX90aFS>{aOR zRqmcy5n?0NCjgL1PBh$ijLVdUpB=kjAw5p)$qrn7b6r#4&i~zzqlwg6+t~6FVRG6>Rv8$QjQdtC|B4!f zSN15Z#aTBwR7=e*C>R{s3sK9);d@Nk`6b)-6JvS7V)dn&|(~gT|7Zz?!*I!X*ya50gn{ts-HREo*6fYy3dUJxC|UTJu+k_?7NUn#i#^ z4zovbl&xtNQL& z-WUMujcA9BvJ+xm5>7Vz4E>Pje`U*kscL*_@yp!Q`b`(j#an?VL@*>n=s@T%g!Pfl z+NM-GBok#0goNA=>y1+ZNEO{f?Get_PyPOy3hbz_Emn@?j@s&47iqt1#goW&R35d+ zVaw?KYJl~ClkBb8je|zC3>}7JM9iLoHFI9c4_SY5xFZegQy6Dg_Pz zGG6{Pm#U{sitOC|7V`D)xor>TY8}1N8U2$Te^v%l*bY*BUNP~e`gzf}e$?DIJb6~~ z*#m5_K+*x!fK|?#@drDi;_45U33HZJW~8|gJIJ`N#$I{TcXQ>o<%BM0iC?fF@{C50 z>dPutb_ePimE$8o6}2tAyia6+xfvW+Ff8Nsp4Tl?RtwQQNXO4?CBTEp<&uZguz=hZ4N>pl=Ib$_TsVp((Bg@<8MY1f zjPW)NpH4Eow|uJZu99%0%waP;OxdKs{iAGNwVayv%`C>66cyhahw5~s;sV%=T4V(o zdgJp7H#+RM7+SkP^85=x(pbwyoNS7`!ItkqJsoB%jmvs_XD|1Eav8t=|F>M`1D^|D zVHaZN_FNG8c(1jjx!6_>gzFyd$OxkiyfO^yAhM;3j~nO_XxHON1ml;JW3{h*e74G! z9j(4kQwHYhtEf;t8tP5^x1_<|<4k{&)s}+#5-}?sHX+#OZ-mi({%}rTeRi-3eBPQR zlu-(G_^VmF*-z%v!ceUAI|lb*TCW7#b9sL|D)TW}Cd>_oVfLaRh~&@Q=EuBZmhT6t z$&DgiWxv2+IH2Q2p*IdsqKWdtYuYdHJgVg?{nJ==@0}LJ<1l)+GHW8*Yg1>g;!OM~ zUh)p&im_PvmRMvc4PTq~PSsqbnoc+T!)1D46YVt)PvpBM;n!AUj8jxiVaR^m1 z35Uqap}%n(u&?(kCUv$HeX1D){XRN8E4FMq)HZS3S_jM|&2j6xve$$0hK&z)D{b(w zQpO)|vgL4l`Ez3n|KULV*Jy-4eo~4Vf^&+>RJ1ckzMZIC;%sb0kc}^7RE$jS7rL?8 z_94X81?CkbUclNS|J5z}mdNPb)?-zg0V9FO9Hk`lp|L|Gbf#=EwVMG}^?OqP}kVK1lcT z$J5d{((ECIEeO2sejFeaDEMMP>A4W^Fc^XeLlcUy@x=cvZ(gOiP&Mbjz7^_17-zkFs_u9z}#E;2sJPwN^z)LJFW^ zZ>o8r69iYX*PS2HEBexOH~l4(aV0W!g^QUJT@YWB#!XV7HTni2uX2*TAxVo7%hgkt zC;ozYQcDwrCJOWhhNuat0CMIa=Y>@_-hV*M6qE1M<=1=rX8qOJj<@M9s2?-_-Iaq2KHwMRadPzL8CtbSpDhZ3T)d3?F8o8sP{II)G0F3|r0G0QIg zw*wJDe@ILSx=zruT+1e#iAf!+|Q zsO>%|7c{bOGk(Pmc%@wSpPTS%#G(-D<$u{1ZASpY#ja@9$ZjB~CwfE^{7H@{_XeT)nI4Ts)L{T6r_`2RylWilxu?Z{%a%38XXEz> z{Lzxk-SkZUWZ>hJ6Vy~RK)O_Xtk+k|64%hoy5VQ1Q&GK??`^?y%CLh|j?MZFqcMYg zKk;u{veqe-7q6>+a#c#a0wW#` z!gMe18y+D*ly%@G#LRi zaXjX#_8Gq@9HYGeVKc(NDT=#1U;m4PR^mkso+RjNZzkf>!C$kh_!Rmp#BIT@m0b!3=cH8%7wIA;ql@{R?I|cGp#tZn@7R zBy&{bc0JXG?VnA$L{}+tKZuts$r&0Eats4yZBrwA-$d)m6fwNbxO`QU@g?-K2dYAyD=q+2 ztc$0U@*WoZ;k0e92>%cHl5M94=$DOu+e*y?9JRgnb(T`MA96imnV(63>d@@pLdH|G z!Yj8vHamF^;vjk@e#p%<-fc9J9Rfw;B{2;oa3aTS8WC%fXrVm^M6@No6eTc?>mIR(dH~Bf>_#rrqgk1a5(-;f3b|T zUO?tI4(c)5MwoWt-g?9T&LREeiR@clgJUcRErD}3livPd9D^-bI5IO28b(S6*2#t7 z>jFxw&GMq%i)24&aYc!*Tr=peqZGTn9&p?t;uZz7lpf#N)BPXP1%&e`_49oU&QdTsKDZd+^?j<^<4?&onmGZQY{Q_w!zHY|euyZ-d@e=EClgYH~ z*udKUKHrX0L%QPq`Y6C=>1RAl!%WWasjkvqyaJF+72GQuNIaQ^g7%;~^aq`!Melo$ zzG3TC266_|!r4M{;^u+_n8_6QZ#EMFvZ&pjPLe~@O*YBKZ50+~a8;9tbi|4MZvSa?hlvw1ICtMP4M<;rOu?zj# z1%r9#PNW$h8~KQ2HRN8s;G}=E^GDo($i5be>b77BEAu}lv4v3@_t*bWn!N{Qe?=on z1|Iluvgr=_)3|`YG4mdU{|tFVeD+q59BG9$vyyc68S2rPrEB2Rkrgc>IoanVbJVXJt5HS(#HIOw;mqDq*lv1m%Oa&|2j%Lu2Zv*8JX5@lRl;pBBQylcUs+ zZADXK;0m{Nje;4F+4p`>mRgZ@D4uA|n)GSUsn9JBP03(fojI}HzrfiLkd|<5U@WG@ zBd^T4LAMxA`1+awFq#^?=}3u2@Xz5(ep4KwUZGh@WzL)L{%=0h!9PCJskYpCMTFqZ zK5hh}Zlf9A=?RANK|`LKxK}@?SnTI_sHZA730sgV@>`(i<7c|=$u>fe>CdJaow)i4 z3(Jh1j5|^{cN2aEw2eA{_;?!MxIS8NcK6rEOZB52HIE_gE<99<%+JMmKka+QXGy>$ zTg5%Y3xek`x%LKXv+PUr^Nr+5A)M#1Flh~WF8>`-_}GfDk);hJwem$qt(M5CFyGJZIgk49>>)d^gBJ2T)*8MGe5Cqfac41!{vo zu$npoUXx4a6Y28}2K+wzNXO!<1%LD~E@MYG(F~r0WiC`kr^_EdeSZ%CR#TsP@|VRL zfi$A(%RvspA81o?Qu~gE9aIh^0fteoa*-Y#d?S9kHzD@amo#NOX5Z?&&}tcjP`D;W)Hf;%+Ew z_=?BAUHbYI5fC=QN_KqW(}1uMw{K;nSTOpH={8qV-{B!};9PGH#Um(qj>+}vG?6&W zAk%OnqQvH2VEpt;{?0k5I}wr(%T;t4!i#c}*4vN-KkVnkVE(T-iF$r zZfLna;gK}hyp`k$Cp!{1J~n=B>i#oPQ;y* zl%=5oW4nhmi~|g}PqvpVj`BNmrV|W4tHU%-V;kB7tbu8Y{~gv~hL1)EdcJ}9>+&q; zr{RL~L*OmMx&10Ge`tvYdUkqbJ{tSw1XQFpFWI}eoZ)w)Kz`=SvE{2uLV-%yOOQcS)jJMq2f*0#>cWJaUq(}r;O6_`1vo!zNmhw*7>y+WoTp~8_tCg65pVnR8qQyy zH^eC2<5MILI(C}qJBp1#J!pYY9Y?t_zcmD(dVMI+I)@7~j?Cnyw0Vw|^9je0n|Fni zOpsi2C2-K~pd)5y^?5Eg^_ea-4UFW)8MFv;2Apm1-O!Vad#aYqIjhJ!W%LiK>Q3pqzs!oUy#q`0#BA) znna8}6;uVMe!>Gz%tf>^;5Op6Nb=T{3jGC55UPx()nyFqGd*R_=M&Q9^bGP4L%a(A zdPq~3*RzQ=a&3yJ{8Rcsebtiv`G+! zq2yB%bIG9gZ_u2n@iX8E634IME30aEn=zguA)6fyK;Biw9y zyOLe}nvVqG41)f?)xapxHjL5CjJjz`h;CvkkHXjD4-2?`mX9IUsuy+n{&0`uwrHL| zCcUu+o_58F=i+Fmmb0hhLMrjKJ;fg_!)Qf#ryR!o5?F$3Wf}+XA6)J&^%-oYxNKEe))| zye0GKan6}v_q95UPX$lzW_QkxN7+=wV`qU5 zo@Q+;pRb$Xi#P;t<3TRIdhse4pIu2;?Xxf|(GDTD#X3zFy*TG>^e?Vt0U-xpy&SAf zRbq1+*=FFP(DPI1lJJ=?+Ro1zM0Lh>#$lLzXJl zcG~FJHP;JZq?sk@t2*Yt;8X|R1Gw@@(T8L#Cn{ia?(}umJ+cVUBe@_Xd}`ArYuuJ|CPGw(UX$%DPZCTR&hB8Arq<>T6e) z=AKFq>d&WfJ+&npl5;z{kHTea9E6osA3@wrc&Hqo5U-Oe&%jbLWQH>j*QHzr+sEbg ze~}p}(i=9s5+DjPxP~VLqircn3p{a6V zMV=}i1uXu_^CfQVzEXM>x3!C4Di7ifQz^~qAg7}lUfDUvBDlKhEh&C{(k_Fz?_CcW znGLGbUCvx<1w?&@XM4|=ve1~X;wG+b!TP4KknT(R7i3_Xd>A$5FK3CnO>0Ipbm*jR zdx7D=s5$qL^9_zO5raghs7roOTC`om80VS7;us!%&?$i0;g{~DA_Jq6uE)U}zp73j zVz}=Ch>W}g54~^XIUZmoMR}vk-hEypk7CX3&k)3yo2^|GyA!;MQVKm-Tz#q464&{f zd3o;I6jiVCucv8r#0X9;dyNb(4-(GbP=dKhYm3t;$K_^zzCvz|OU!LZID&0hKbUm+ zPf2~n507Q5fYD=D`5dI12kTwmwfRuyMx*LV0q+Fzs;Qh{`+JM39}E;QVFZea4%U8J zN`#Lup`HkE(FkXc`~9iYzGfFvg}zaCx5@@+$R7cs$0GmsQ&r2Kcpmu21n z>2_;L5%7|{1_#A{*EJLaqGn6ap=;Rds&+Zw zQcK(+fq#I^yZr`|4K#OM8+9Dz_*9hG&~bvOv!WYs4v86ERl|s@l6TsEy7)h>pdn9b z_hRTGA36qJXfT448q{~dFrK%V*DCB&(3FuD%Gwiki!VI;OGMvxQQYdcIdu^<`mi8R z=vTy4>t`~l6(^t`kA!9JaCy2gT74UVr?Z65aWuKe^VUct`a%M(r)2My5O1@x@QWnS z=C+MEx7ZJcou^R?6Pn|T_1t%IPOx-Ll=HW1wcFW#a>5Ncsy&yhB9 z#xPgrBu_w!kZ<2uc&F8xol*p<&}b7X|L3^TU?3f1d2I^CU+O^ezwgQg?(8`zOl{Rs z0ZF=_n#+Z@lZ6s^)8byL^%{E$Gb0}FHLvR&l4DJJRd}5@X!2 z{OFl2vyM6JnQN>ix)XrDM3QCm$Zy7Bb9M{n{}bx+YrvDmR1Y?_@(C7+%?~AlNw~@ceO-tT3JG^4V1N3OM%rK3b`cDkGm~aJVU>4$A_prG%E( z;N|qi4|JG7KL_On4Zk$oDuOAD{zX4kTqu7( zWT)T87S>Y|qK;o(75R+`clyweE?#f0!N`302iSQ~fkE(@mLJK#t)QEcVih4ojB~$v zRwBYZF(U7S^M1+v&x=W^wY-fL+HEg9{B>n#Nt!O&NB~@J3bT`zmK`CKIUGN( z9sBabtWlfUZ&_2gUUgdW{|WT|c_9gWme%la{oPWlL()RJ=GUi0VC1u$u?{3n-?#c| zqo~dE@pqA>8=EK@t!sO7&>zS4GXGC|=i$}Vny+C%1f@noFQOnt1nEsmsG`zEM5UJq zqSBjygeEoApuka@AjO6h0gp&gq$q?)Xd+d_0MZ2{kc7K;kTWjNxpVHBHFxITS^NWg zWoKu7`}^Mad42&JdNdqaeeGz2K<(GkJco$^ewFUcRM9aWV2?!G8U?kvR&amz2q~~l zh)ed%pmFiS!D89V%sQEAw>2W#13M_AKN#zcNBY2*mdl!!crR&Yvv1m<%E*B(F2>|^ z53rPFcL!(H(*(}-Qa|105c=rqn0~B@CMvJDox4X5VjYP*BWhra~0)LWCa8(a7bz2^gG(*x`!&t~P+Oc(lMI|6ENSfv5nOAuR&tV{4E@_i|m z^#a!z#{ER{Zo5d7DyTnekNkgbkIwg-7+M|`H3oGgB2PM*d>OZN$=$ZF_+`aFs3) z|Ibt$Zu2E+lBg%?7$jp`A(!c?ME$+!fX=8iqxHlUrYCG?6Fy#?o+@ax$7v3=m~oi) z2HeUi+tZGgjDI6qm1J;+d~ny%1|?iZpCjKf-u0KZGSj@J7*CPw+8$tA1UFf|b}unI zr&eisno#~WLs+g=eFm$E*>CIOx39vO5eY^N2oH=Rw$fO&oq49Dbr)PZ@Rri~_ z%tJa(kRUZWC@lsEQ!;|uOJ7A63Nd=zEPjFw4HC#5g@N{%X9CErn5KwcB7`BAaD1fU zn#Zbhj?+xR&lnK|vBNc)2A+Md=hNk(>T&o5fXd$s2hx<3BWF78WNS5y662+x@2fGi~p$Wl-UXP&0#sVj?9 zn6dDD0*lfof(*#=wmUCSzG&eAVbfB@o=vA%a8;ux&G|;k>~~_cE;8-WP;OIPn^@6# zCW*=xrCwiF!rlrurupk0(%>I^NY+~(QdYJghc%!fEm2E60^aD=JY$LEF7T@*pjU_b z%R1SHx6)>lFy5n&e1i7PRjH7{uTseh<$+OS z;HyT7&iOiChbS3_T|+6R!OFr`Z3=rAX-U&wUPuMHsu70_!EShouCko24Yv~aS$*-Y zoN3$(>PyaQ*M&Y7(b=rPb*1w09ECAY6ca(u3v95UbPbyLuM!B|-rU|?zFSNV5$0UU z(K?w=(UlRF%a3_5?+af!?L8qRyAYimv!VLl)jvcW0Q^m7SXW@}kRxkLW5Cmi(5r2* zMRZPvElLg0QgRlRTTXRPD$I(SlI|2epln+^kX~FQ9%yN7j`;9qnZMC=il89gyz&_c zNJYxu0>mP>AF7#u(UioSLBsvZ*4jg>#mAB)ncIEp&|tjV70ZyjltMLi-?x5lyYJ45dD`FGs+#C{P=Up4(F`MrA}nA>;$j z0$`Z%`k^-x)}A4`*+su+SR_1E&E;MgOasmf?)8m)Ae5vTIX3QvhtflYU|+YS>RrN) z$1@e>o%&}aOLn~#O@*^$s467MxaOmFdn>x&WJ8l$H<(TzaScWv$bu;}u;P#x$g9aj zXAoEuaZ2dyOUzq2IOCMmupxE6oFv^Pqz_;5K3Z11g}!_|{x|f+kMwk;UzfdYBzHCa z;Fb7wTk$d0>}35zB`I&#jmIU~8{i~El7|H*=Xw6_B8+f9&693yRe-0f+d)6i!}=SC zAoZ!gHhWh$?J9`AWp^hgNNH9sxAV}CPSQjtVyrZl{>h8I?M>__>p0}{BpP}Wt5mP1 z)K!XHzN-dwL*3qW78e0pJbdf`3u|Wx&sX_ssZs&Z_A=!+eSX0+)lb1kUt=7eSOQXdB^;6G?_k z-Jt^rUZ0YJrPg!$g0spR;pvQgl24En)?K}$bv8t?_jowhRmV>Qn-0kJ9OOITwH6*- zzX$>_IKkbABM$)rQWINz&=n1<(cN9k6Ava`+I0#E6%K61*(p6JD5b1=jy?vzYw7#4 zgzT_gOs^fV6Hc}Q9ct@WWc)=u8!=sWuK65e{t&@9ehYrF_NLx^=QvEhe^%xM7<9_% zCdePNVIoe`LJbzDJT(pZC;fm4G>?DD;qh2ct>DKCwp6oC733bjvXXfOg$%!?v)z^)A?b z?E)dlu$ip0=Ll+@GzYbj(_ejm<6S7Ls%}UMX(hZ6F}-y)0NZ*v;)c(e;J=IL-UG!G z=eF0KTG(kXWp3}%;enymjjfto+{!apuhH`IAlQVb@Fb9e_s;A|RP^DZ5rkych%B#@b_KvWy z3Z+N$gsS7S?$w-~$&w{!Ht7eY>GD`ncYtvguPxv3TL}>0_S*7SpXI8C6%?wjxLX~+ zlRcM`=+8B(^qyjUP~%Vur?R$L*_0!~I1`{Xp`Ra_p; z9UJ!#;iAm<{J}l)xHWqM^mCnu14R1pv;gCBM5^|Xfc8U)J-LHzhntuJUN{H{mMJ%! zJtrS`yQBz36P-ot%WrJ0bAJ4MNV9;HO1-#Pe!semDY=8$$GB}zYe|}{jvGwUBif3+ zYqfrsjzX3`Vjle7od)E4HpQ^Ei&6fFU$}Iz>~as>+DxK}3(=o0Oz5v&%dN%hrrRh= z_d2bMty4FG(SZ_n{REL2AUQ_$#ls}P$sAkqG;OK1<3CBBo=_kxJ}4NyDD!$-J^K8r z(}hbY4RDW+KQ>W|=vbgq8)E1Ke=hu<+Hx~_i~h-`Bu>ahA;3Laawl}8#(z@Kepz&m zc?@m%JR{w%yYv+39o=uwgnCvQZp>nHBI;Y6CP`{W=1(`29^PO<1LW3|tr-?T+DYTS zRgb_b$dopksFgsoYuz%6`ligM(of7u;J62W=Mv9s3h2n29b_Q1jn?0epcag zFmuJDB@IL6oLj!$|3x`I>Z#>D#+Ma&qdh+IG`-o}(LRBRRz52yAHIQBRswJL5w}Q{ zUNUR!{u`ew@EvgJn*FyHKY!*;uPpyXZ<+=Ek9gDh|5|U_V06cu{$p?YJJNy;RY#4+ zki#<}yGnPUve~>*FMPCNRagpYc?fTViL9cqP#IXN)Lrk&ei*wdH+06N6R*|hLvfW_ zjwws*^tkTLt4VjH$ZV|Ou|0;mQ2_1;j{X_?w!9@7P3VU3(30U?b$y}758{2SZcER> z(;nemTt7&Anj|D42J~LY*6PU+vk*pZz?nWN1FVkb7r!GcN3u0;HF}68pOe7eOMSD2 z=2GmnQShm4kfHK9X&JE*xbnaWmkzH?NV&WjFFlQr0lSmE`$^3slNEN!3^Dy@KcZM)5&{l3B^di_r&)|UHr>?7zUT?4kUwwq?EawLNSl=&@~ z`=W&g^C><9snrq4Wkuy4B zA#A*jtt`&Va|Hn>(l3a_JwoQ-Qg*TfSgdn#ZiXf*2Wa{S*z3~v$ggJWoUa-)6f9z3 z?=oHdjafmV**?qph9JQnUE#Rb6+(iVLD~|Ijg&3K0drn+>Y_q!F}b7c+Y%8L!Q%gp zkE^M+RtvT9DCRTaBqvJ!vNI&zyqh!lgsa*r@UzN6SrLsJbyyFpvwMP{h>YYOm?k5+ zN^#N;hoYwm`D7%wI*@(0L&`57jIhw%h^&5Oi)E|swd1HN3s2RE=5#;ueL*jS*SHRV zD}f-m;@}plI+=yUn?!}s>wqeeta2opUHMyp3BtKB&uFv#V`bgf58O}dha6P3>k3X2 zB|Tb1BTpN6eJon!a6gRjh+Rm(!Prwd5xzv73fM&TAh-eu(!=o3ai?9A-n&u?96m7^ zzAj86s=;r0%Mp<^P^DzxQP2C{0)YR*#p(b~05pcI93g3)aR<lfq*9!`yH&ReZ6wM+8NQ(_f< zdA>@J2%L9wu8n6|;rKrphdyptNR44At2z+sB~k=Y8C{8sMK*n=u(aX55tJ-Ia4{6e2bb@<*FAghumNf63MFVgV=2rl(E z$naH58hDPy^d!D5Ub_@6`5bx4A$0&yqti-3BWR-YGQoV1TKG2CF!9nwh@mCs;jcf< zcj2%O9BMj;ntCt^rEUSXZsP`D1Xp0<@75wD%c!eF5jGQm89n!znYY$2Vc)k8hczCF zuVV~#Xv+HXr93HK@TVp|YatvnaJkS$VL=8|M5yRU`9L1B1>x>viv^|?{RZKVs~65S zq<>gwDNPN(-NyPOq5@Ad5hf3KSz%CkMgJYc{eoy>eD@@}c|`L#-ccjt=FDgBw)#ay z{iCTL8208ks70zM&N|qHo7Rirn+5cXWt@$g-YLvh8zY!V0;s%=Z%XDpbK_6gPbvA6 z`Wx9FZZ*QRoV9HuLJ!V1=6o4N7b`jD{eL&zW!KWl-i}}zeoRoUDUdd82AUC# zZ-HKO{ph@np6Iw{FUFVKoG}d*9ARq%Lby}Ehj4$yQT*UKzbLYsGYyCuN3PXUR_Nq|KcRBi0HAkcRv9+$K+7-4KXHOdI zwtzi!O(h5C0y>uHXNSoS5$VA193ri4hiI}}^(TkubIY}kJgkV1labi20Yv8;Xq Q|5WItu941TExYjl0@oxy1^@s6 literal 0 HcmV?d00001 diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index 81820d563..a714cf486 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -444,7 +444,7 @@ function search() { for (var i = 0; i < proposals.length; i++) { // Evaluate if the proposal number matches the search var j = i + 1 - var prop_name = document.getElementById("proposal" + j).getAttribute('proposal') + var prop_name = document.getElementById("proposal" + j).getAttribute('data-proposal') var prop_num = Number(prop_name) if (prop_name.startsWith(search_value) || prop_num.toString().startsWith(search_value)) { @@ -510,14 +510,14 @@ function show_only(filter_type, value, base_url) { // Evaluate if the thumbnail meets all filter criteria var criteria = []; for (j = 0; j < all_filters.length; j++) { - var filter_attribute = thumbnails[i].getAttribute(all_filters[j]); + var filter_attribute = thumbnails[i].getAttribute('data-' + all_filters[j]); var criterion = (filter_values[j].indexOf('All '+ all_filters[j] + 's') >=0) || (filter_attribute.includes(filter_values[j])); criteria.push(criterion); } // If data are grouped, check if a thumbnail for the group has already been displayed - if (group && groups_shown.has(thumbnails[i].getAttribute('group_root'))) { + if (group && groups_shown.has(thumbnails[i].getAttribute('data-group_root'))) { criteria.push(false); } @@ -525,8 +525,10 @@ function show_only(filter_type, value, base_url) { if (criteria.every(function(r){return r})) { thumbnails[i].style.display = "inline-block"; num_thumbnails_displayed++; - list_of_rootnames = list_of_rootnames + thumbnails[i].getAttribute("file_root") + '=' + thumbnails[i].getAttribute("exp_start") + ','; - if (group) { groups_shown.add(thumbnails[i].getAttribute('group_root')); } + list_of_rootnames = list_of_rootnames + + thumbnails[i].getAttribute("data-file_root") + + '=' + thumbnails[i].getAttribute("data-exp_start") + ','; + if (group) { groups_shown.add(thumbnails[i].getAttribute('data-group_root')); } } else { thumbnails[i].style.display = "none"; } @@ -577,7 +579,7 @@ function sort_by_proposals(sort_type) { tinysort(props, {order:'desc'}); } else if (sort_type == 'Recent') { // Sort by the most recent Observation Start - tinysort(props, {order:'desc', attr:'obs_time'}); + tinysort(props, {order:'desc', attr:'data-obs_time'}); } } @@ -597,14 +599,14 @@ function sort_by_thumbnails(sort_type, base_url) { var thumbs = $('div#thumbnail-array>div') if (sort_type == 'Descending') { - tinysort(thumbs, {attr:'file_root', order:'desc'}); + tinysort(thumbs, {attr:'data-file_root', order:'desc'}); } else if (sort_type == 'Recent') { - tinysort(thumbs, {attr:'exp_start', order:'desc'}, {attr:'file_root', order:'asc'}); + tinysort(thumbs, {attr:'data-exp_start', order:'desc'}, {attr:'data-file_root', order:'asc'}); } else if (sort_type == 'Oldest') { - tinysort(thumbs, {attr:'exp_start', order:'asc'}, {attr:'file_root', order:'asc'}); + tinysort(thumbs, {attr:'data-exp_start', order:'asc'}, {attr:'data-file_root', order:'asc'}); } else { // Default to 'Ascending' - tinysort(thumbs, {attr:'file_root', order:'asc'}); + tinysort(thumbs, {attr:'data-file_root', order:'asc'}); } $.ajax({ url: base_url + '/ajax/image_sort/', @@ -709,8 +711,11 @@ function update_archive_page(inst, base_url) { var cat_type = data.thumbnails.cat_types[i]; // Build div content - var content = '
    '; - content += ''; + content += ''; content += '' content += ''; content += '
    '; @@ -866,9 +871,9 @@ function update_wata_page(base_url) { */ function update_filter_options(data, base_url, thumbnail_class) { var dropdown_key_list = Object.keys(data.dropdown_menus); - var content = '
    Filter by:
    '; + var content = '
    Filter by:
    '; for (var i = 0; i < Object.keys(data.dropdown_menus).length; i++) { // Parse out useful variables @@ -903,9 +908,9 @@ function update_wata_page(base_url) { function update_group_options(data, base_url) { // Build div content - var content = '
    Group by:
    '; + var content = '
    Group by:
    '; content += ''; content += '
    '; content += 'Exposure'; @@ -1019,15 +1024,20 @@ function update_thumbnail_array(data) { } else { instrument = filename_dict.instrument; } - var content = '
    '; + var content = '
    '; content += '
    ' - content += ''; - content += ''; + content += ''; + content += '' + + // Make sure thumbnail img always has a src and alt + content += ''; content += '
    '; content += '
    '; content += 'Proposal: ' + filename_dict.program_id + '
    '; diff --git a/jwql/website/apps/jwql/templates/archive.html b/jwql/website/apps/jwql/templates/archive.html index c3fa5dad1..da93c43cb 100644 --- a/jwql/website/apps/jwql/templates/archive.html +++ b/jwql/website/apps/jwql/templates/archive.html @@ -17,7 +17,7 @@

    Archived {{ inst }} Images

    -

    +
    @@ -35,14 +35,14 @@

    Archived {{ inst }} Images

    Sort by:
    - +
    Ascending Descending Recent
    - +
    diff --git a/jwql/website/apps/jwql/templates/archive_date_range.html b/jwql/website/apps/jwql/templates/archive_date_range.html index c44c162e3..d5cb37ace 100644 --- a/jwql/website/apps/jwql/templates/archive_date_range.html +++ b/jwql/website/apps/jwql/templates/archive_date_range.html @@ -12,7 +12,7 @@

    Archive Date Range {{ inst }} Images

    -

    +
    @@ -55,4 +55,4 @@

    Archive Date Range {{ inst }} Images

    No data match the date range.
    -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/jwql/website/apps/jwql/templates/base.html b/jwql/website/apps/jwql/templates/base.html index e444fc446..a06f7ffe8 100644 --- a/jwql/website/apps/jwql/templates/base.html +++ b/jwql/website/apps/jwql/templates/base.html @@ -47,7 +47,7 @@ title="JWQL Logo"> JWQL {% if inst != "" %} - {{ inst }} + {{ inst }} {% endif %} @@ -133,7 +133,7 @@
  • - @@ -208,7 +208,7 @@ alt="JWST Logo">
    -

    +

    diff --git a/jwql/website/apps/jwql/templates/query_submit.html b/jwql/website/apps/jwql/templates/query_submit.html index ea256a432..4ff6206ef 100644 --- a/jwql/website/apps/jwql/templates/query_submit.html +++ b/jwql/website/apps/jwql/templates/query_submit.html @@ -19,7 +19,7 @@

    Images of Queried Instruments

    -

    +
    @@ -58,4 +58,3 @@

    Images of Queried Instruments

    {% endblock %} - diff --git a/jwql/website/apps/jwql/templates/thumbnails_per_obs.html b/jwql/website/apps/jwql/templates/thumbnails_per_obs.html index d2aa8fe56..84e0bc540 100644 --- a/jwql/website/apps/jwql/templates/thumbnails_per_obs.html +++ b/jwql/website/apps/jwql/templates/thumbnails_per_obs.html @@ -22,7 +22,7 @@

    {{ inst }} Images

    -

    +
    diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html index 4041d7241..4a1992aae 100644 --- a/jwql/website/apps/jwql/templates/view_image.html +++ b/jwql/website/apps/jwql/templates/view_image.html @@ -101,9 +101,11 @@
    Submit Anomaly
    {% for field in form %} +
      {% for subwidget in field.subwidgets %}
    • {{subwidget}}
      • {% endfor %} +
    {% endfor %} From ef9e50a7372263ff591bebc04e916f86a54a6270 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Tue, 28 Feb 2023 18:47:27 -0500 Subject: [PATCH 119/449] Hopefully this is the final set of log formatting --- .../common_monitors/dark_monitor.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index 6b58f7a35..0ae442fe8 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -383,21 +383,20 @@ def exclude_existing_badpix(self, badpix, pixel_type): """ if len(badpix[0]) == 0: - logging.warning("Now new {} pixels found.".format(pixel_type)) + logging.warning("\tNo new {} pixels to check.".format(pixel_type)) return ([], []) - logging.info("Checking {} potential new {} pixels".format(len(badpix[0]), pixel_type)) + logging.info("\tChecking {} potential new {} pixels".format(len(badpix[0]), pixel_type)) if pixel_type not in ['hot', 'dead', 'noisy']: raise ValueError('Unrecognized bad pixel type: {}'.format(pixel_type)) - logging.info("\tRunning database query") + logging.info("\t\tRunning database query") db_entries = session.query(self.pixel_table) \ .filter(self.pixel_table.type == pixel_type) \ .filter(self.pixel_table.detector == self.detector) \ .all() - logging.info("\tCreating list of {} existing {} pixels".format(len(db_entries), pixel_type)) already_found = [] if len(db_entries) != 0: for _row in db_entries: @@ -408,7 +407,8 @@ def exclude_existing_badpix(self, badpix, pixel_type): found_x = np.array([x[0] for x in already_found]) found_y = np.array([x[1] for x in already_found]) - logging.info("\tChecking pixels against found list") + msg = "\t\tChecking pixels against list of {} existing {} pixels" + logging.info(msg.format(len(found_x), pixel_type)) # Check to see if each pixel already appears in the database for # the given bad pixel type new_pixels_x = [] @@ -419,6 +419,8 @@ def exclude_existing_badpix(self, badpix, pixel_type): if len(np.intersect1d(ind_x[0], ind_y[0])) > 0: new_pixels_x.append(x) new_pixels_y.append(y) + + logging.info("\t\tKeeping {} {} pixels".format(len(new_pixels_x), pixel_type)) # pixel = (x, y) # if pixel not in already_found: # new_pixels_x.append(x) From 02f0890cebd60b5005dd089edc06e253218ce2d0 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Wed, 1 Mar 2023 10:46:44 -0500 Subject: [PATCH 120/449] Fixed a bug --- jwql/instrument_monitors/common_monitors/dark_monitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index 0ae442fe8..d0c8206bb 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -416,7 +416,7 @@ def exclude_existing_badpix(self, badpix, pixel_type): for x, y in zip(badpix[0], badpix[1]): ind_x = np.where(found_x == x) ind_y = np.where(found_y == y) - if len(np.intersect1d(ind_x[0], ind_y[0])) > 0: + if len(np.intersect1d(ind_x[0], ind_y[0])) == 0: new_pixels_x.append(x) new_pixels_y.append(y) From 110c38ab5cf58da69ed62ac576067861910f8f4c Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Wed, 1 Mar 2023 10:57:26 -0500 Subject: [PATCH 121/449] Remove old rewrite parameter; start on view_exposure --- jwql/tests/test_data_containers.py | 2 +- jwql/website/apps/jwql/data_containers.py | 71 +++++++-- jwql/website/apps/jwql/urls.py | 1 + jwql/website/apps/jwql/views.py | 175 +++++++++++++--------- 4 files changed, 171 insertions(+), 78 deletions(-) diff --git a/jwql/tests/test_data_containers.py b/jwql/tests/test_data_containers.py index 1e30d9e3c..a417c124e 100644 --- a/jwql/tests/test_data_containers.py +++ b/jwql/tests/test_data_containers.py @@ -152,7 +152,7 @@ def test_get_header_info(): def test_get_image_info(): """Tests the ``get_image_info`` function.""" - image_info = data_containers.get_image_info('jw01068001001_02102_00001_nrcb1', False) + image_info = data_containers.get_image_info('jw01068001001_02102_00001_nrcb1') assert isinstance(image_info, dict) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 917df76ee..4ff0ec93c 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -52,7 +52,7 @@ from jwql.edb.engineering_database import get_mnemonic, get_mnemonic_info, mnemonic_inventory from jwql.utils.utils import check_config_for_key, ensure_dir_exists, filesystem_path, filename_parser, get_config from jwql.utils.constants import MAST_QUERY_LIMIT, MONITORS, THUMBNAIL_LISTFILE, THUMBNAIL_FILTER_LOOK -from jwql.utils.constants import IGNORED_SUFFIXES, INSTRUMENT_SERVICE_MATCH +from jwql.utils.constants import EXPOSURE_PAGE_SUFFIX_ORDER, IGNORED_SUFFIXES, INSTRUMENT_SERVICE_MATCH from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_INSTRUMENT_NAMES from jwql.utils.constants import REPORT_KEYS_PER_INSTRUMENT from jwql.utils.constants import SUFFIXES_TO_ADD_ASSOCIATION, SUFFIXES_WITH_AVERAGED_INTS, QUERY_CONFIG_KEYS @@ -201,6 +201,63 @@ def get_all_proposals(): return proposals +def get_available_suffixes(all_suffixes, return_untracked=True): + """ + Put available suffixes in a consistent order. + + Any suffixes not recognized are returned at the end of the suffix + list, in random order. + + Parameters + ---------- + all_suffixes : list of str + List of all data product suffixes found for a given file root. + return_untracked : bool, optional + If set, a set of untracked suffixes is also returned, for + logging or diagnostic purposes. + + Returns + ------- + suffixes : list of str + All available unique suffixes in standard order. + untracked_suffixes : set of str, optional + Any suffixes that were not recognized. + """ + # Check if any of the + # suffixes are not in the list that specifies order. + suffixes = [] + untracked_suffixes = set(all_suffixes) + for poss_suffix in EXPOSURE_PAGE_SUFFIX_ORDER: + if 'crf' not in poss_suffix: + if (poss_suffix in all_suffixes + and poss_suffix not in suffixes): + suffixes.append(poss_suffix) + untracked_suffixes.remove(poss_suffix) + else: + # EXPOSURE_PAGE_SUFFIX_ORDER contains crf and crfints, + # but the actual suffixes in the data will be e.g. o001_crf, + # and there may be more than one crf file in the list of suffixes. + # So in this case, we strip the e.g. o001 from the + # suffixes and check which list elements match. + for image_suffix in all_suffixes: + if (image_suffix.endswith(poss_suffix) + and image_suffix not in suffixes): + suffixes.append(image_suffix) + untracked_suffixes.remove(image_suffix) + + # If the data contain any suffixes that are not in the list + # that specifies the order to use, add them to the end of the + # suffixes list. Their order will be random since they are not in + # EXPOSURE_PAGE_SUFFIX_ORDER. + if len(untracked_suffixes) > 0: + suffixes.extend(untracked_suffixes) + + if return_untracked: + return suffixes, untracked_suffixes + else: + return suffixes + + def get_current_flagged_anomalies(rootname, instrument): """Return a list of currently flagged anomalies for the given ``rootname`` @@ -800,7 +857,7 @@ def get_header_info(filename, filetype): return header_info -def get_image_info(file_root, rewrite): +def get_image_info(file_root): """Build and return a dictionary containing information for a given ``file_root``. @@ -809,9 +866,6 @@ def get_image_info(file_root, rewrite): file_root : str The rootname of the file of interest (e.g. ``jw86600008001_02101_00007_guider2``). - rewrite : bool - ``True`` if the corresponding JPEG needs to be rewritten, - ``False`` if not. Returns ------- @@ -859,11 +913,6 @@ def get_image_info(file_root, rewrite): jpg_filename = os.path.basename(os.path.splitext(filename)[0] + '_integ0.jpg') jpg_filepath = os.path.join(jpg_dir, jpg_filename) - # Check that a jpg does not already exist. If it does (and rewrite=False), - # just call the existing jpg file - if os.path.exists(jpg_filepath) and not rewrite: - pass - # Record how many integrations have been saved as preview images per filetype jpgs = glob.glob(os.path.join(prev_img_filesys, proposal_dir, '{}_{}_integ*.jpg'.format(file_root, suffix))) image_info['num_ints'][suffix] = len(jpgs) @@ -1075,6 +1124,7 @@ def get_preview_images_by_rootname(rootname): return preview_images + def get_proposals_by_category(instrument): """Return a dictionary of program numbers based on category type Parameters @@ -1102,6 +1152,7 @@ def get_proposals_by_category(instrument): return proposals_by_category + def get_proposal_info(filepaths): """Builds and returns a dictionary containing various information about the proposal(s) that correspond to the given ``filepaths``. diff --git a/jwql/website/apps/jwql/urls.py b/jwql/website/apps/jwql/urls.py index 2d492ee0c..29801dfd8 100644 --- a/jwql/website/apps/jwql/urls.py +++ b/jwql/website/apps/jwql/urls.py @@ -90,6 +90,7 @@ re_path(r'^(?P({}))/(?P.+)_(?P.+)/explore_image/'.format(instruments), views.explore_image, name='explore_image'), re_path(r'^(?P({}))/(?P.+)_(?P.+)/header/'.format(instruments), views.view_header, name='view_header'), re_path(r'^(?P({}))/archive/(?P[\d]{{1,5}})/obs(?P[\d]{{1,3}})/$'.format(instruments), views.archive_thumbnails_per_observation, name='archive_thumb_per_obs'), + re_path(r'^(?P({}))/exposure/(?P[\w-]+)/$'.format(instruments), views.view_exposure, name='view_exposure'), # AJAX views re_path('ajax/query_submit/', views.archive_thumbnails_query_ajax, name='archive_thumb_query_ajax'), diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 624b24daf..d6f5e9b7b 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -60,11 +60,12 @@ from jwql.database.database_interface import load_connection from jwql.utils import monitor_utils from jwql.utils.interactive_preview_image import InteractivePreviewImg -from jwql.utils.constants import EXPOSURE_PAGE_SUFFIX_ORDER, JWST_INSTRUMENT_NAMES_MIXEDCASE, URL_DICT, THUMBNAIL_FILTER_LOOK, QUERY_CONFIG_TEMPLATE, QUERY_CONFIG_KEYS +from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE, URL_DICT, THUMBNAIL_FILTER_LOOK, QUERY_CONFIG_TEMPLATE, QUERY_CONFIG_KEYS from jwql.utils.utils import filename_parser, get_base_url, get_config, get_rootnames_for_instrument_proposal, query_unformat from .data_containers import build_table from .data_containers import get_acknowledgements +from .data_containers import get_available_suffixes from .data_containers import get_anomaly_form from .data_containers import get_dashboard_components from .data_containers import get_edb_components @@ -946,7 +947,7 @@ def view_header(request, inst, filename, filetype): return render(request, template, context) -def explore_image(request, inst, file_root, filetype, rewrite=False): +def explore_image(request, inst, file_root, filetype): """Generate the header view page Parameters @@ -959,8 +960,6 @@ def explore_image(request, inst, file_root, filetype, rewrite=False): FITS file_root of selected image in filesystem filetype : str Type of file (e.g. ``uncal``) - rewrite : bool, optional - Regenerate if bokeh image already exists? Returns ------- @@ -977,7 +976,7 @@ def explore_image(request, inst, file_root, filetype, rewrite=False): fits_file = file_root + '_' + filetype + '.fits' # Get image info containing all paths to fits files - image_info_list = get_image_info(file_root, rewrite) + image_info_list = get_image_info(file_root) # Find index of our fits file fits_index = next(ix for ix, fits_path in enumerate(image_info_list['all_files']) if fits_file in fits_path) # get full path of fits file to open and extract extension info @@ -1015,7 +1014,7 @@ def explore_image(request, inst, file_root, filetype, rewrite=False): return render(request, template, context) -def explore_image_ajax(request, inst, file_root, filetype, scaling="log", low_lim=None, high_lim=None, ext_name="SCI", int1_nr=None, grp1_nr=None, int2_nr=None, grp2_nr=None, rewrite=False): +def explore_image_ajax(request, inst, file_root, filetype, scaling="log", low_lim=None, high_lim=None, ext_name="SCI", int1_nr=None, grp1_nr=None, int2_nr=None, grp2_nr=None): """Generate the page listing all archived images in the database for a certain proposal @@ -1037,8 +1036,6 @@ def explore_image_ajax(request, inst, file_root, filetype, scaling="log", low_li Signal value to use as the upper limit of the displayed image. If "None", it will be calculated using the ZScale function ext_name : str Extension to implement in interactive preview image ("SCI", "DQ", "GROUPDQ", "PIXELDQ", "ERR"...) - rewrite : bool, optional - Regenerate if bokeh image already exists? Returns ------- @@ -1049,7 +1046,7 @@ def explore_image_ajax(request, inst, file_root, filetype, scaling="log", low_li inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()] # Get image info containing all paths to fits files - image_info_list = get_image_info(file_root, rewrite) + image_info_list = get_image_info(file_root) # Save fits file name to use for bokeh image fits_file = file_root + '_' + filetype + '.fits' @@ -1163,7 +1160,87 @@ def toggle_viewed_ajax(request, file_root): return JsonResponse(context, json_dumps_params={'indent': 2}) -def view_image(request, inst, file_root, rewrite=False): +def view_exposure(request, inst, group_root): + """Generate the exposure view page. + + Parameters + ---------- + request : HttpRequest object + Incoming request from the webpage. + inst : str + Name of JWST instrument. + group_root : str + Exposure group, matching file root names up to but not + including the detector. + + Returns + ------- + HttpResponse object + Outgoing response sent to the webpage + """ + + # Ensure the instrument is correctly capitalized + inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()] + + template = 'view_image.html' + image_info = get_image_info(group_root) + + # TODO: get file roots/detectors from image info + # and check available ints handling + print(image_info) + + # Get available suffixes in a consistent order. + suffixes = get_available_suffixes(image_info['suffixes'], + return_untracked=False) + + # Get the anomaly submission form + form = get_anomaly_form(request, inst, group_root) + + prop_id = group_root[2:7] + + # if we get to this page without any navigation data (i.e. direct link), + # just use the group_root with no expstart time + # navigate_data is dict of format rootname:expstart + navigation_data = request.session.get('navigation_data', {group_root: 0}) + + # For time based sorting options, sort to "Recent" first to create sorting consistency when times are the same. + # This is consistent with how Tinysort is utilized in jwql.js->sort_by_thumbnails + sort_type = request.session.get('image_sort', 'Ascending') + if sort_type in ['Descending']: + group_root_list = sorted(navigation_data, reverse=True) + elif sort_type in ['Recent']: + navigation_data = dict(sorted(navigation_data.items())) + navigation_data = dict(sorted(navigation_data.items(), key=operator.itemgetter(1), reverse=True)) + group_root_list = list(navigation_data.keys()) + elif sort_type in ['Oldest']: + navigation_data = dict(sorted(navigation_data.items())) + navigation_data = dict(sorted(navigation_data.items(), key=operator.itemgetter(1))) + group_root_list = list(navigation_data.keys()) + else: + group_root_list = sorted(navigation_data) + + # Get our current views RootFileInfo model and send our "viewed/new" information + root_file_info = RootFileInfo.objects.filter(root_name__startswith=group_root) + viewed = all([rf.viewed for rf in root_file_info]) + + # Build the context + context = {'base_url': get_base_url(), + 'file_root_list': group_root_list, + 'inst': inst, + 'prop_id': prop_id, + 'obsnum': group_root[7:10], + 'file_root': group_root, + 'suffixes': suffixes, + 'num_ints': image_info['num_ints'], + 'available_ints': image_info['available_ints'], + 'total_ints': image_info['total_ints'], + 'form': form, + 'marked_viewed': viewed} + + return render(request, template, context) + + +def view_image(request, inst, file_root): """Generate the image view page Parameters @@ -1174,8 +1251,6 @@ def view_image(request, inst, file_root, rewrite=False): Name of JWST instrument file_root : str FITS filename of selected image in filesystem - rewrite : bool, optional - Regenerate the jpg preview of `file` if it already exists? Returns ------- @@ -1187,80 +1262,48 @@ def view_image(request, inst, file_root, rewrite=False): inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()] template = 'view_image.html' - image_info = get_image_info(file_root, rewrite) + image_info = get_image_info(file_root) # Put suffixes in a consistent order. Check if any of the # suffixes are not in the list that specifies order. - # Reorder the list of filenames to match the reordered list - # of suffixes. - suffixes = [] - all_files = [] - untracked_suffixes = deepcopy(image_info['suffixes']) - untracked_files = deepcopy(image_info['all_files']) - for poss_suffix in EXPOSURE_PAGE_SUFFIX_ORDER: - if 'crf' not in poss_suffix: - if poss_suffix in image_info['suffixes']: - suffixes.append(poss_suffix) - loc = image_info['suffixes'].index(poss_suffix) - all_files.append(image_info['all_files'][loc]) - untracked_suffixes.remove(poss_suffix) - untracked_files.remove(image_info['all_files'][loc]) - else: - # EXPOSURE_PAGE_SUFFIX_ORDER contains crf and crfints, but the actual suffixes - # in the data will be e.g. o001_crf, and there may be more than one crf file - # in the list of suffixes. So in this case, we strip the e.g. o001 from the - # suffixes and check which list elements match. - suff_arr = np.array(image_info['suffixes']) - files_arr = np.array(image_info['all_files']) - splits = np.array([ele.split('_')[-1] for ele in image_info['suffixes']]) - if splits.size > 0: - idxs = np.where(splits == poss_suffix)[0] - else: - idxs = [] - if len(idxs) > 0: - suff_entries = list(suff_arr[idxs]) - file_entries = list(files_arr[idxs]) - suffixes.extend(suff_entries) - all_files.extend(file_entries) - - untracked_splits = np.array([ele.split('_')[-1] for ele in untracked_suffixes]) - untracked_idxs = np.where(untracked_splits == poss_suffix)[0] - untracked_suffixes = list(np.delete(untracked_suffixes, untracked_idxs)) - untracked_files = list(np.delete(untracked_files, untracked_idxs)) - - # If the data contain any suffixes that are not in the list that specifies the order - # to use, make a note in the log (so that they can be added to EXPOSURE_PAGE_SUFFIX_ORDER) - # later. Then add them to the end of the suffixes list. Their order will be random since - # they are not in EXPOSURE_PAGE_SUFFIX_ORDER. + suffixes, untracked_suffixes = get_available_suffixes( + image_info['suffixes'], return_untracked=True) + if len(untracked_suffixes) > 0: module = os.path.basename(__file__).strip('.py') - start_time, log_file = monitor_utils.initialize_instrument_monitor(module) - logging.warning((f'In view_image(), for {inst}, {file_root}, the following suffixes are present in the data, ' - f'but not in EXPOSURE_PAGE_SUFFIX_ORDER in constants.py: {untracked_suffixes} ' - 'Please add them, so that they will appear in a consistent order on the webpage.')) - suffixes.extend(untracked_suffixes) - all_files.extend(untracked_files) + monitor_utils.initialize_instrument_monitor(module) + logging.warning((f'In view_image(), for {inst}, {file_root}, ' + f'the following suffixes are present in the data, ' + f'but not in EXPOSURE_PAGE_SUFFIX_ORDER in ' + f'constants.py: {untracked_suffixes} ' + 'Please add them, so that they will appear in a ' + 'consistent order on the webpage.')) form = get_anomaly_form(request, inst, file_root) prop_id = file_root[2:7] - # if we get to this page without any navigation data (i.e. direct link), just use the file_root with no expstart time + # if we get to this page without any navigation data (i.e. direct link), + # just use the file_root with no expstart time # navigate_data is dict of format rootname:expstart navigation_data = request.session.get('navigation_data', {file_root: 0}) - # For time based sorting options, sort to "Recent" first to create sorting consistency when times are the same. - # This is consistent with how Tinysort is utilized in jwql.js->sort_by_thumbnails + # For time based sorting options, sort to "Recent" first to create + # sorting consistency when times are the same. + # This is consistent with how Tinysort is utilized in + # jwql.js->sort_by_thumbnails sort_type = request.session.get('image_sort', 'Ascending') if sort_type in ['Descending']: file_root_list = sorted(navigation_data, reverse=True) elif sort_type in ['Recent']: navigation_data = dict(sorted(navigation_data.items())) - navigation_data = dict(sorted(navigation_data.items(), key=operator.itemgetter(1), reverse=True)) + navigation_data = dict(sorted(navigation_data.items(), + key=operator.itemgetter(1), reverse=True)) file_root_list = list(navigation_data.keys()) elif sort_type in ['Oldest']: navigation_data = dict(sorted(navigation_data.items())) - navigation_data = dict(sorted(navigation_data.items(), key=operator.itemgetter(1))) + navigation_data = dict(sorted(navigation_data.items(), + key=operator.itemgetter(1))) file_root_list = list(navigation_data.keys()) else: file_root_list = sorted(navigation_data) @@ -1275,8 +1318,6 @@ def view_image(request, inst, file_root, rewrite=False): 'prop_id': prop_id, 'obsnum': file_root[7:10], 'file_root': file_root, - 'jpg_files': image_info['all_jpegs'], - 'fits_files': all_files, 'suffixes': suffixes, 'num_ints': image_info['num_ints'], 'available_ints': image_info['available_ints'], From 6b2fb2bf95f2bc8de39ea847a6dc439c44924284 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Wed, 1 Mar 2023 13:09:18 -0500 Subject: [PATCH 122/449] Add handling for time series stage 2c files --- jwql/tests/test_utils.py | 16 ++++++++++++++++ jwql/utils/utils.py | 19 ++++++++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/jwql/tests/test_utils.py b/jwql/tests/test_utils.py index 7aa3c2307..624841573 100644 --- a/jwql/tests/test_utils.py +++ b/jwql/tests/test_utils.py @@ -194,6 +194,22 @@ 'visit': '001', 'visit_group': '02'}), + # Test full time series filename for stage 2c + ('jw00733003001_02101_00002-seg001_nrs1_o001_crfints.fits', + {'ac_id': 'o001', + 'activity': '01', + 'detector': 'nrs1', + 'exposure_id': '00002', + 'filename_type': 'time_series_2c', + 'instrument': 'nirspec', + 'observation': '003', + 'parallel_seq_id': '1', + 'program_id': '00733', + 'segment': '001', + 'suffix': 'crfints', + 'visit': '001', + 'visit_group': '02'}), + # Test root time series filename ('jw00733003001_02101_00002-seg001_nrs1', {'activity': '01', diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index 49fe6cc03..11d3c7fa1 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -371,7 +371,22 @@ def filename_parser(filename): r"(?P\w{" + f"{FILE_ACT_LEN}" + "})"\ r"_(?P\d+)"\ r"-seg(?P\d{" + f"{FILE_SEG_LEN}" + "})"\ - r"_(?P\w+)" + r"_(?P((?!_)[\w])+)" + + # Time series filenames for stage 2c + # e.g. "jw00733003001_02101_00002-seg001_nrs1_o001_crfints.fits" + time_series_2c = \ + r"jw" \ + r"(?P\d{" + f"{FILE_PROG_ID_LEN}" + "})"\ + r"(?P\d{" + f"{FILE_OBS_LEN}" + "})"\ + r"(?P\d{" + f"{FILE_VISIT_LEN}" + "})"\ + r"_(?P\d{" + f"{FILE_VISIT_GRP_LEN}" + "})"\ + r"(?P\d{" + f"{FILE_PARALLEL_SEQ_ID_LEN}" + "})"\ + r"(?P\w{" + f"{FILE_ACT_LEN}" + "})"\ + r"_(?P\d+)"\ + r"-seg(?P\d{" + f"{FILE_SEG_LEN}" + "})"\ + r"_(?P((?!_)[\w])+)"\ + r"_(?P(o\d{" + f"{FILE_AC_O_ID_LEN}" + r"}|(c|a|r)\d{" + f"{FILE_AC_CAR_ID_LEN}" + "}))" # Guider filenames # e.g. "jw00729011001_gs-id_1_image_cal.fits" or @@ -405,6 +420,7 @@ def filename_parser(filename): stage_3_target_id_epoch, stage_3_source_id_epoch, time_series, + time_series_2c, guider, guider_segment] @@ -417,6 +433,7 @@ def filename_parser(filename): 'stage_3_target_id_epoch', 'stage_3_source_id_epoch', 'time_series', + 'time_series_2c', 'guider', 'guider_segment' ] From f97e20607b803d76ac7f61d38c1f9d38d41d22fa Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Wed, 1 Mar 2023 14:02:12 -0500 Subject: [PATCH 123/449] Catch missing obs in navigation data --- .../apps/jwql/templates/thumbnails_per_obs.html | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/jwql/website/apps/jwql/templates/thumbnails_per_obs.html b/jwql/website/apps/jwql/templates/thumbnails_per_obs.html index 1476fba74..a17882b66 100644 --- a/jwql/website/apps/jwql/templates/thumbnails_per_obs.html +++ b/jwql/website/apps/jwql/templates/thumbnails_per_obs.html @@ -75,13 +75,15 @@

    Proposal Information for {{ prop }}

    - {% set index = obs_list.index(obs) %} - {% if index != 0 %} - < Previous - {% endif %} - - {% if obs_list.index(obs) != obs_list|length - 1 %} - Next > + {% if obs in obs_list %} + {% set index = obs_list.index(obs) %} + {% if index != 0 %} + < Previous + {% endif %} + + {% if obs_list.index(obs) != obs_list|length - 1 %} + Next > + {% endif %} {% endif %}
    From 551b75a6c541b9fcb3f19dce45d1fe9f60717467 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Wed, 1 Mar 2023 16:40:11 -0500 Subject: [PATCH 124/449] First pass at exposure page with all detector images --- jwql/website/apps/jwql/data_containers.py | 24 ++- jwql/website/apps/jwql/static/js/jwql.js | 105 ++++++++++++- .../apps/jwql/templates/view_exposure.html | 142 ++++++++++++++++++ .../apps/jwql/templates/view_image.html | 2 +- jwql/website/apps/jwql/views.py | 14 +- 5 files changed, 267 insertions(+), 20 deletions(-) create mode 100644 jwql/website/apps/jwql/templates/view_exposure.html diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 4ff0ec93c..2320b1d2c 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -881,15 +881,22 @@ def get_image_info(file_root): image_info['num_ints'] = {} image_info['available_ints'] = {} image_info['total_ints'] = {} + image_info['detectors'] = set() - # Find all of the matching files + # Find all the matching files proposal_dir = file_root[:7] observation_dir = file_root[:13] - filenames = glob.glob(os.path.join(FILESYSTEM_DIR, 'public', proposal_dir, observation_dir, '{}*.fits'.format(file_root))) - filenames.extend(glob.glob(os.path.join(FILESYSTEM_DIR, 'proprietary', proposal_dir, observation_dir, '{}*.fits'.format(file_root)))) + filenames = glob.glob( + os.path.join(FILESYSTEM_DIR, 'public', proposal_dir, + observation_dir, '{}*.fits'.format(file_root))) + filenames.extend(glob.glob( + os.path.join(FILESYSTEM_DIR, 'proprietary', proposal_dir, + observation_dir, '{}*.fits'.format(file_root)))) # Certain suffixes are always ignored - filenames = [filename for filename in filenames if os.path.splitext(filename)[0].split('_')[-1] not in IGNORED_SUFFIXES] + filenames = [filename for filename in filenames + if os.path.splitext(filename)[0].split('_')[-1] + not in IGNORED_SUFFIXES] image_info['all_files'] = filenames # Determine the jpg directory @@ -898,8 +905,10 @@ def get_image_info(file_root): for filename in image_info['all_files']: + parsed_fn = filename_parser(filename) + # Get suffix information - suffix = filename_parser(filename)['suffix'] + suffix = parsed_fn['suffix'] # For crf or crfints suffixes, we need to also include the association value # in the suffix, so that preview images can be found later. @@ -927,6 +936,9 @@ def get_image_info(file_root): else: image_info['total_ints'][suffix] = 1 + # Record the detector used + image_info['detectors'].add(parsed_fn.get('detector', 'Unknown')) + return image_info @@ -1666,7 +1678,7 @@ def thumbnails_ajax(inst, proposal, obs_num=None): 'visit_group': rootname[14:16], 'group_root': rootname[:26]} - # todo: the following comprehensions loop over all file names + # TODO: the following comprehensions loop over all file names # several times - should be combined, and maybe make the # filename loop primary diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index a714cf486..cb343142d 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -9,6 +9,87 @@ * @author Melanie Clarke */ + + /** + * Change the filetype for all displayed images + * @param {String} type - The image type (e.g. "rate", "uncal", etc.) + * @param {String} group_root - The rootname of the file group + * @param {Dict} num_ints - A dictionary whose keys are suffix types and whose + * values are the number of integrations with an associated + * preview image for that suffix + * @param {Dict} available_ints - A dictionary whose keys are suffix types and whose + * values are the integration numbers of the available + * jpgs for that suffix + * @param {Dict} total_ints - A dictionary whose keys are suffix types and whose + * values are the total number of integrations for that + * filetype. + * @param {String} inst - The instrument for the given file + */ + function change_all_filetypes(type, group_root, num_ints, available_ints, total_ints, inst, detectors) { + + // Change the radio button to check the right filetype + document.getElementById(type).checked = true; + + // Clean the input parameters + num_ints = num_ints.replace(/'/g, '"'); + num_ints = num_ints.replace(/'/g, '"'); + num_ints = JSON.parse(num_ints); + + // Get the available integration jpg numbers + available_ints = available_ints.replace(/'/g, '"'); + available_ints = available_ints.replace(/'/g, '"'); + available_ints = JSON.parse(available_ints)[type]; + + // Get the total number of integrations + total_ints = total_ints.replace(/'/g, '"'); + total_ints = total_ints.replace(/'/g, '"'); + total_ints = JSON.parse(total_ints); + + // Update the APT parameters + document.getElementById("proposal").innerHTML = group_root.slice(2,7); + document.getElementById("obs_id").innerHTML = group_root.slice(7,10); + document.getElementById("visit_id").innerHTML = group_root.slice(10,13); + + var detector_list = detectors.split(','); + for (let i = 0; i < detector_list.length; i++) { + var detector = detector_list[i]; + + // Update the filename lists + var fits_filename = group_root + '_' + detector + '_' + type + '.fits'; + var filename_option = document.getElementById(detector + "_filename"); + filename_option.value = inst + '/' + group_root + '_' + detector + '_' + type; + filename_option.textContent = fits_filename; + + // Show the appropriate image + var img = document.getElementById("image_viewer_" + detector); + var jpg_filepath = '/static/preview_images/' + group_root.slice(0,7) + + '/' + group_root + '_' + detector + '_' + type + '_integ0.jpg'; + img.src = jpg_filepath; + img.alt = jpg_filepath; + // if previous image had error, remove error sizing + img.classList.remove("thumbnail"); + } + + // Reset the slider values + document.getElementById("slider_range").value = 1; + document.getElementById("slider_range").max = num_ints[type]; + document.getElementById("slider_val").innerHTML = 1; + document.getElementById("total_ints").innerHTML = total_ints[type]; + + // Update the integration changing buttons + if (num_ints[type] > 1) { + document.getElementById("int_after").disabled = false; + } else { + document.getElementById("int_after").disabled = true; + } + // Disable the "left" button, since this will be showing integ0 + document.getElementById("int_before").disabled = true; + + // Update the view/explore links for the new file type + update_view_explore_link(); +} + + /** * Change the filetype of the displayed image * @param {String} type - The image type (e.g. "rate", "uncal", etc.) @@ -44,7 +125,7 @@ total_ints = total_ints.replace(/'/g, '"'); total_ints = JSON.parse(total_ints); - // Propogate the text fields showing the filename and APT parameters + // Propagate the text fields showing the filename and APT parameters var fits_filename = file_root + '_' + type; document.getElementById("jpg_filename").innerHTML = file_root + '_' + type + '_integ0.jpg'; document.getElementById("fits_filename").innerHTML = fits_filename + '.fits'; @@ -74,9 +155,7 @@ document.getElementById("int_after").disabled = true; } - // Update the image download and header links - // document.getElementById("download_fits").href = '/static/filesystem/' + file_root.slice(0,7) + '/' + fits_filename + '.fits'; - // document.getElementById("download_jpg").href = jpg_filepath; + // Update the image exploration and header links document.getElementById("view_header").href = '/' + inst + '/' + file_root + '_' + type + '/header/'; document.getElementById("explore_image").href = '/' + inst + '/' + file_root + '_' + type + '/explore_image/'; @@ -973,7 +1052,7 @@ function update_obs_options(data, inst, prop, observation) { * @param {String} type - The type of the count (e.g. "activities") */ function update_show_count(count, type) { - var content = 'Showing
    ' + count + '
    /
    ' + count + '
    ' + type + '
    '; + var content = 'Showing ' + count + ' / ' + count + ' ' + type + ''; content += ''; content += 'i'; $("#img_show_count")[0].innerHTML = content; @@ -1201,3 +1280,19 @@ function version_url(version_string) { a_line += '">JWQL v' + version_string + ''; return a_line; } + + +/** + * Construct the URL for viewing/exploring a selected image on the exposure page + */ +function update_view_explore_link() { + var types = ['header', 'explore_image']; + for (var i = 0; i < types.length; i++) { + var type = types[i]; + button = document.getElementById(type); + selected = document.getElementById('fits_file_select'); + href = '/' + selected.value + '/' + type + '/'; + console.log(href); + button.href = href; + } +} diff --git a/jwql/website/apps/jwql/templates/view_exposure.html b/jwql/website/apps/jwql/templates/view_exposure.html new file mode 100644 index 000000000..cb1facb53 --- /dev/null +++ b/jwql/website/apps/jwql/templates/view_exposure.html @@ -0,0 +1,142 @@ +{% extends "base.html" %} + +{% block preamble %} + + View {{ inst }} Exposure - JWQL + +{% endblock %} + +{% block content %} + +
    + +

    Exposure {{ group_root }}

    + + +
    +
    Proposal:
    +
    Observation:
    +
    Visit:
    + +
    + + + View File Type: + + i +
    +
    + {% for suffix in suffixes %} + {{ suffix }}      + {% endfor %} +
    +
    + + +
    + View Header + Explore Image +
    for
    + +
    + +
    + +
    + {% for detector in detectors %} + + + {{ group_root }}_{{ detector }}_cal_integ0.jpg + + + {% endfor %} + +
    + + + +

    Integration: /

    +
    + +
    + + {% if group_root in group_root_list %} + {% set index = group_root_list.index(group_root) %} + {% if index != 0 %} + < Previous + {% endif %} + + {% if index != group_root_list|length - 1 %} + Next > + {% endif %} + {% endif %} + +
    +
    + + +
    + +
    +
    Submit Anomaly
    +
    + + {% if form.errors %} +
    + {% for field in form %} + {% for error in field.errors %} +
    + {{ error|escape }} +
    + {% endfor %} + {% endfor %} +
    + {% endif %} + + + {{ csrf_input }} + + + {% for field in form %} +
      + {% for subwidget in field.subwidgets %} +
    • {{subwidget}}
      • + {% endfor %} +
    + {% endfor %} + +
    +
    +
    +
    + + {% if marked_viewed %} + + {% else %} + + {% endif %} + + {% if 'rate' in suffixes %} + + {% elif 'dark' in suffixes %} + + {% elif 'uncal' in suffixes %} + + {% elif suffixes|length == 1 %} + + {% else %} + Unable to show image for: {{suffixes}} + {% endif %} + +
    + +{% endblock %} diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html index 4a1992aae..836f05594 100644 --- a/jwql/website/apps/jwql/templates/view_image.html +++ b/jwql/website/apps/jwql/templates/view_image.html @@ -28,7 +28,7 @@

    {{ file_root }}

    i
    -
    + {% for suffix in suffixes %} {{ suffix }}      {% endfor %} diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index d6f5e9b7b..e90895e0f 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -1182,12 +1182,11 @@ def view_exposure(request, inst, group_root): # Ensure the instrument is correctly capitalized inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()] - template = 'view_image.html' + template = 'view_exposure.html' image_info = get_image_info(group_root) - # TODO: get file roots/detectors from image info - # and check available ints handling - print(image_info) + # Get the proposal id from the group root name + prop_id = group_root[2:7] # Get available suffixes in a consistent order. suffixes = get_available_suffixes(image_info['suffixes'], @@ -1196,8 +1195,6 @@ def view_exposure(request, inst, group_root): # Get the anomaly submission form form = get_anomaly_form(request, inst, group_root) - prop_id = group_root[2:7] - # if we get to this page without any navigation data (i.e. direct link), # just use the group_root with no expstart time # navigate_data is dict of format rootname:expstart @@ -1225,15 +1222,16 @@ def view_exposure(request, inst, group_root): # Build the context context = {'base_url': get_base_url(), - 'file_root_list': group_root_list, + 'group_root_list': group_root_list, 'inst': inst, 'prop_id': prop_id, 'obsnum': group_root[7:10], - 'file_root': group_root, + 'group_root': group_root, 'suffixes': suffixes, 'num_ints': image_info['num_ints'], 'available_ints': image_info['available_ints'], 'total_ints': image_info['total_ints'], + 'detectors': sorted(image_info['detectors']), 'form': form, 'marked_viewed': viewed} From f0f7411f6da31ebef019c76209266473acda8070 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Wed, 1 Mar 2023 18:25:40 -0500 Subject: [PATCH 125/449] Add handling for group viewed/new --- jwql/website/apps/jwql/static/js/jwql.js | 68 +++++++++++++------ .../apps/jwql/templates/view_exposure.html | 2 +- jwql/website/apps/jwql/urls.py | 2 + jwql/website/apps/jwql/views.py | 34 ++++++++++ 4 files changed, 86 insertions(+), 20 deletions(-) diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index cb343142d..52fa4d285 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -55,10 +55,9 @@ var detector = detector_list[i]; // Update the filename lists - var fits_filename = group_root + '_' + detector + '_' + type + '.fits'; var filename_option = document.getElementById(detector + "_filename"); filename_option.value = inst + '/' + group_root + '_' + detector + '_' + type; - filename_option.textContent = fits_filename; + filename_option.textContent = group_root + '_' + detector + '_' + type; // Show the appropriate image var img = document.getElementById("image_viewer_" + detector); @@ -707,7 +706,8 @@ function sort_by_thumbnails(sort_type, base_url) { * @param {String} base_url - The base URL for gathering data from the AJAX view. */ function toggle_viewed(file_root, base_url) { - // Toggle the button immediately so user insn't confused (ajax result will confirm choice or fix on failure) + // Toggle the button immediately so user isn't confused + // (ajax result will confirm choice or fix on failure) var elem = document.getElementById("viewed"); update_viewed_button(elem.value == "New" ? true : false); elem.disabled=true; @@ -729,6 +729,38 @@ function toggle_viewed(file_root, base_url) { }); } + +/** + * Set the viewed status for a group of files. + * Ajax call to update RootFileInfo model with toggled value + * + * @param {String} group_root - The rootname of the exposure group + * @param {String} base_url - The base URL for gathering data from the AJAX view. + */ +function toggle_viewed_group(group_root, base_url) { + // Toggle the button immediately so user isn't confused + var elem = document.getElementById("viewed"); + update_viewed_button(elem.value == "New" ? true : false); + elem.disabled=true; + + // Ajax Call to update RootFileInfo model with "viewed" info + $.ajax({ + url: base_url + '/ajax/viewed_group/' + group_root + '/' + elem.value, + success: function(data){ + // Update button with actual value (paranoia update, should not yield visible change) + update_viewed_button(data["marked_viewed"]); + elem.disabled=false; + }, + error : function(response) { + // If update fails put button back to original state + update_viewed_button(elem.value == "New" ? false : true); + elem.disabled=false; + + } + }); +} + + /** * Download filtered data report * @param {String} inst - The instrument in use @@ -1259,6 +1291,20 @@ function update_thumbnails_query_page(base_url, sort) { }}); } + +/** + * Construct the URL for viewing/exploring a selected image on the exposure page + */ +function update_view_explore_link() { + var types = ['header', 'explore_image']; + for (var i = 0; i < types.length; i++) { + var type = types[i]; + var selected = document.getElementById('fits_file_select').value; + document.getElementById(type).href = '/' + selected + '/' + type + '/'; + } +} + + function update_viewed_button(viewed) { var elem = document.getElementById("viewed"); if (viewed) { @@ -1280,19 +1326,3 @@ function version_url(version_string) { a_line += '">JWQL v' + version_string + ''; return a_line; } - - -/** - * Construct the URL for viewing/exploring a selected image on the exposure page - */ -function update_view_explore_link() { - var types = ['header', 'explore_image']; - for (var i = 0; i < types.length; i++) { - var type = types[i]; - button = document.getElementById(type); - selected = document.getElementById('fits_file_select'); - href = '/' + selected.value + '/' + type + '/'; - console.log(href); - button.href = href; - } -} diff --git a/jwql/website/apps/jwql/templates/view_exposure.html b/jwql/website/apps/jwql/templates/view_exposure.html index cb1facb53..ac516ff73 100644 --- a/jwql/website/apps/jwql/templates/view_exposure.html +++ b/jwql/website/apps/jwql/templates/view_exposure.html @@ -17,7 +17,7 @@

    Exposure {{ group_root }}

    Proposal:
    Observation:
    Visit:
    - +
    • diff --git a/jwql/website/apps/jwql/urls.py b/jwql/website/apps/jwql/urls.py index 29801dfd8..4a6227047 100644 --- a/jwql/website/apps/jwql/urls.py +++ b/jwql/website/apps/jwql/urls.py @@ -99,6 +99,8 @@ re_path(r'^ajax/(?P({}))/(?P.+)_(?P.+)/explore_image/scaling_(?P.+)/low_(?P.+)/high_(?P.+)/ext_(?P.+)/int1_(?P.+)/grp1_(?P.+)/int2_(?P.+)/grp2_(?P.+)/$'.format(instruments), views.explore_image_ajax, name='explore_image_ajax'), re_path(r'^ajax/(?P({}))/archive/(?P[\d]{{1,5}})/obs(?P[\d]{{1,3}})/$'.format(instruments), views.archive_thumbnails_ajax, name='archive_thumb_ajax'), re_path(r'^ajax/viewed/(?P.+)/$', views.toggle_viewed_ajax, name='toggle_viewed_ajax'), + re_path(r'^ajax/viewed_group/(?P.+)/(?P(viewed|new|Viewed|New))/$', + views.set_viewed_ajax, name='set_viewed_ajax'), re_path(r'^ajax/(?P({}))/archive_date_range/start_date_(?P.+)/stop_date_(?P.+)/$'.format(instruments), views.archive_date_range_ajax, name='archive_date_range_ajax'), re_path(r'^ajax/image_group/$', views.save_image_group_ajax, name='save_image_group_ajax'), re_path(r'^ajax/image_sort/$', views.save_image_sort_ajax, name='save_image_sort_ajax'), diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index e90895e0f..f5db29128 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -1136,6 +1136,40 @@ def save_page_navigation_data(request, data): return +def set_viewed_ajax(request, group_root, status): + """Update the model's "viewed" field for a group of files + + Parameters + ---------- + request : HttpRequest object + Incoming request from the webpage + group_root : str + Group root name, matching filename roots up to + but not including the detector. + status : {'new', 'viewed'} + Value to set: 'new' for viewed=False, 'viewed' for viewed=True. + + Returns + ------- + JsonResponse object + Outgoing response sent to the webpage + """ + viewed = (str(status).strip().lower() == 'viewed') + + root_file_info = RootFileInfo.objects.filter( + root_name__startswith=group_root) + for root_file in root_file_info: + root_file.viewed = viewed + root_file.save() + + # check actual status as set + marked_viewed = all([rf.viewed for rf in root_file_info]) + + # Build the context + context = {'marked_viewed': marked_viewed} + return JsonResponse(context, json_dumps_params={'indent': 2}) + + def toggle_viewed_ajax(request, file_root): """Update the model's "mark_viewed" field and save in the database From bcfe886e331c1f4b4d43be064eb281bf2fe95c94 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Thu, 2 Mar 2023 10:53:31 -0500 Subject: [PATCH 126/449] Fixing missing import --- jwql/instrument_monitors/common_monitors/dark_monitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index d0c8206bb..e7c139b62 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -83,7 +83,7 @@ from jwql.jwql_monitors import monitor_mast from jwql.shared_tasks.shared_tasks import only_one, run_pipeline, run_parallel_pipeline from jwql.utils import calculations, instrument_properties, monitor_utils -from jwql.utils.constants import ASIC_TEMPLATES, DARK_MONITOR_MAX_BADPOINTS_TO_PLOT, JWST_INSTRUMENT_NAMES +from jwql.utils.constants import ASIC_TEMPLATES, DARK_MONITOR_MAX_BADPOINTS_TO_PLOT, JWST_INSTRUMENT_NAMES, FULL_FRAME_APERTURES from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_DATAPRODUCTS, RAPID_READPATTERNS from jwql.utils.logging_functions import log_info, log_fail from jwql.utils.permissions import set_permissions From 35440d60edd12155f6586648364ab1aff044eb5e Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Thu, 2 Mar 2023 11:11:29 -0500 Subject: [PATCH 127/449] Re-ordered adding things a bit --- .../common_monitors/dark_monitor.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index e7c139b62..c7fa67bf9 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -756,18 +756,20 @@ def process(self, file_list): # Shift the coordinates to be in full frame coordinate system logging.info("\tShifting hot pixels to full frame") new_hot_pix = self.shift_to_full_frame(new_hot_pix) - logging.info("\tShifting dead pixels to full frame") - new_dead_pix = self.shift_to_full_frame(new_dead_pix) # Exclude hot and dead pixels found previously logging.info("\tExcluding previously-known hot pixels") new_hot_pix = self.exclude_existing_badpix(new_hot_pix, 'hot') - logging.info("\tExcluding previously-known dead pixels") - new_dead_pix = self.exclude_existing_badpix(new_dead_pix, 'dead') # Add new hot and dead pixels to the database logging.info('\tFound {} new hot pixels'.format(len(new_hot_pix[0]))) self.add_bad_pix(new_hot_pix, 'hot', file_list, mean_slope_file, baseline_file, min_time, mid_time, max_time) + + # Same thing for dead pixels + logging.info("\tShifting dead pixels to full frame") + new_dead_pix = self.shift_to_full_frame(new_dead_pix) + logging.info("\tExcluding previously-known dead pixels") + new_dead_pix = self.exclude_existing_badpix(new_dead_pix, 'dead') logging.info('\tFound {} new dead pixels'.format(len(new_dead_pix[0]))) self.add_bad_pix(new_dead_pix, 'dead', file_list, mean_slope_file, baseline_file, min_time, mid_time, max_time) From a9fd219b773b5bee0d6cac8a2b580f830c530d59 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Thu, 2 Mar 2023 14:47:58 -0500 Subject: [PATCH 128/449] Integration slider working for image and exposure pages --- jwql/website/apps/jwql/data_containers.py | 22 +++++--- jwql/website/apps/jwql/static/js/jwql.js | 55 ++++++++++++------- .../apps/jwql/templates/view_exposure.html | 7 ++- .../apps/jwql/templates/view_image.html | 4 +- 4 files changed, 56 insertions(+), 32 deletions(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 2320b1d2c..fbd901d99 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -923,16 +923,22 @@ def get_image_info(file_root): jpg_filepath = os.path.join(jpg_dir, jpg_filename) # Record how many integrations have been saved as preview images per filetype - jpgs = glob.glob(os.path.join(prev_img_filesys, proposal_dir, '{}_{}_integ*.jpg'.format(file_root, suffix))) - image_info['num_ints'][suffix] = len(jpgs) - image_info['available_ints'][suffix] = sorted([int(jpg.split('_')[-1].replace('.jpg', '').replace('integ', '')) for jpg in jpgs]) + jpgs = glob.glob(os.path.join(prev_img_filesys, proposal_dir, '{}*_{}_integ*.jpg'.format(file_root, suffix))) + image_info['available_ints'][suffix] = sorted(set([int(jpg.split('_')[-1].replace('.jpg', '').replace('integ', '')) for jpg in jpgs])) + image_info['num_ints'][suffix] = len(image_info['available_ints'][suffix]) image_info['all_jpegs'].append(jpg_filepath) - # Record how many integrations exist per filetype. crf needs to be treated - # separately because the suffix includes the association number, which can't - # be predicted for a given program. - if ((suffix not in SUFFIXES_WITH_AVERAGED_INTS) and (suffix[-3:] != 'crf')): - image_info['total_ints'][suffix] = fits.getheader(filename)['NINTS'] + # Record how many integrations exist per filetype. + if suffix not in SUFFIXES_WITH_AVERAGED_INTS: + # time series segments need special handling + header = fits.getheader(filename) + nint = header['NINTS'] + if 'time_series' in parsed_fn['filename_type']: + intstart = header.get('INTSTART', 1) + intend = header.get('INTEND', nint) + image_info['total_ints'][suffix] = intend - intstart + 1 + else: + image_info['total_ints'][suffix] = nint else: image_info['total_ints'][suffix] = 1 diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index 52fa4d285..0885bbf94 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -30,6 +30,9 @@ // Change the radio button to check the right filetype document.getElementById(type).checked = true; + // Store the currently displayed suffix + document.getElementById("view_file_type").setAttribute('data-current-suffix', type); + // Clean the input parameters num_ints = num_ints.replace(/'/g, '"'); num_ints = num_ints.replace(/'/g, '"'); @@ -109,6 +112,9 @@ // Change the radio button to check the right filetype document.getElementById(type).checked = true; + // Store the currently displayed suffix + document.getElementById("view_file_type").setAttribute('data-current-suffix', type); + // Clean the input parameters num_ints = num_ints.replace(/'/g, '"'); num_ints = num_ints.replace(/'/g, '"'); @@ -176,12 +182,11 @@ * @param {String} direction - The direction to switch to, either "left" (decrease) or "right" (increase). * Only relevant if method is "button". */ -function change_int(file_root, num_ints, available_ints, method, direction = 'right') { +function change_int(file_root, num_ints, available_ints, method, direction='right') { // Figure out the current image and integration - var suffix = document.getElementById("jpg_filename").innerHTML.split('_'); - var integration = Number(suffix[suffix.length - 1].replace('.jpg','').replace('integ','')) - suffix = suffix[suffix.length - 2]; + var suffix = document.getElementById("view_file_type").getAttribute('data-current-suffix'); + var integration = Number(document.getElementById("slider_val").innerText) - 1; var program = file_root.slice(0,7); // Find the total number of integrations for the current image @@ -195,19 +200,23 @@ function change_int(file_root, num_ints, available_ints, method, direction = 'ri // Get the desired integration value var new_integration; + var new_value; switch (method) { case "button": if ((integration == num_ints - 1 && direction == 'right')|| (integration == 0 && direction == 'left')) { return; } else if (direction == 'right') { - new_integration = available_ints[current_index + 1] + new_value = current_index + 1; + new_integration = available_ints[new_value]; } else if (direction == 'left') { - new_integration = available_ints[current_index - 1] + new_value = current_index - 1; + new_integration = available_ints[new_value]; } break; case "slider": - new_integration = available_ints[document.getElementById("slider_range").value - 1]; + new_value = document.getElementById("slider_range").value - 1; + new_integration = available_ints[new_value]; break; } @@ -223,22 +232,30 @@ function change_int(file_root, num_ints, available_ints, method, direction = 'ri document.getElementById("int_before").disabled = false; } - // Update the JPG filename - var jpg_filename = file_root + '_' + suffix + '_integ' + new_integration + '.jpg' - var jpg_filepath = '/static/preview_images/' + program + '/' + jpg_filename - document.getElementById("jpg_filename").innerHTML = jpg_filename; + var img_viewers = document.getElementsByClassName("image_preview_viewer"); + for (let i = 0; i < img_viewers.length; i++) { + var img = img_viewers[i]; - // Show the appropriate image - var img = document.getElementById("image_viewer") - img.src = jpg_filepath; - img.alt = jpg_filepath; + var jpg_filename; + var detector = img.getAttribute('data-detector'); + if (detector != null) { + // exposure view + jpg_filename = file_root + '_' + detector + '_' + suffix + '_integ' + new_integration + '.jpg'; + } else { + // image view + jpg_filename = file_root + '_' + suffix + '_integ' + new_integration + '.jpg'; + document.getElementById("jpg_filename").innerHTML = jpg_filename; + } - // Update the jpg download link - // document.getElementById("download_jpg").href = jpg_filepath; + // Show the appropriate image + var jpg_filepath = '/static/preview_images/' + program + '/' + jpg_filename; + img.src = jpg_filepath; + img.alt = jpg_filepath; + } // Update the slider values - document.getElementById("slider_range").value = new_integration + 1 - document.getElementById("slider_val").innerHTML = new_integration + 1 + document.getElementById("slider_range").value = new_value + 1; + document.getElementById("slider_val").innerHTML = new_integration + 1; } diff --git a/jwql/website/apps/jwql/templates/view_exposure.html b/jwql/website/apps/jwql/templates/view_exposure.html index ac516ff73..0359944fb 100644 --- a/jwql/website/apps/jwql/templates/view_exposure.html +++ b/jwql/website/apps/jwql/templates/view_exposure.html @@ -21,7 +21,7 @@

    Exposure {{ group_root }}

    - View File Type: + View File Type: i
    @@ -51,11 +51,12 @@

    Exposure {{ group_root }}

    {% for detector in detectors %} - {{ group_root }}_{{ detector }}_cal_integ0.jpg + title="Preview image for {{ group_root }}, detector {{ detector }}" + data-detector="{{ detector }}"> {% endfor %} diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html index 836f05594..72aef74b1 100644 --- a/jwql/website/apps/jwql/templates/view_image.html +++ b/jwql/website/apps/jwql/templates/view_image.html @@ -24,7 +24,7 @@

    {{ file_root }}

    JPG Filename:

    - View File Type: + View File Type: i
    @@ -47,7 +47,7 @@

    {{ file_root }}

    - {{ file_root }}_cal_integ0.jpg Date: Thu, 2 Mar 2023 15:59:08 -0500 Subject: [PATCH 129/449] adding archive_database_update 'update' data --- .../apps/jwql/archive_database_update.py | 81 +++++++++++++++---- jwql/website/apps/jwql/data_containers.py | 44 ++++++++++ 2 files changed, 109 insertions(+), 16 deletions(-) diff --git a/jwql/website/apps/jwql/archive_database_update.py b/jwql/website/apps/jwql/archive_database_update.py index ebfdf275a..80940ef38 100755 --- a/jwql/website/apps/jwql/archive_database_update.py +++ b/jwql/website/apps/jwql/archive_database_update.py @@ -19,6 +19,13 @@ instrument = 'nircam' get_updates(insturument) + this module can be run from command line to add new elements to the django database + $ python archive_database_update.py + + This module can use the 'update' argument to update every rootfileinfo data model with the most complete information from MAST + $ python archive_database_update.py update + + Dependencies ------------ The user must have a configuration file named ``config.json`` @@ -27,6 +34,7 @@ import logging import os +import sys import numpy as np import django @@ -45,18 +53,21 @@ from jwql.utils.constants import MAST_QUERY_LIMIT # noqa from jwql.utils.utils import filename_parser, filesystem_path, get_config # noqa from jwql.website.apps.jwql.data_containers import get_instrument_proposals, get_filenames_by_instrument # noqa -from jwql.website.apps.jwql.data_containers import get_proposal_info, mast_query_filenames_by_instrument # noqa +from jwql.website.apps.jwql.data_containers import get_proposal_info, mast_query_filenames_by_instrument, mast_query_by_rootname # noqa FILESYSTEM = get_config()['filesystem'] - @log_info @log_fail -def get_updates(inst): +def get_updates(update_database, inst): """Generate the page listing all archived proposals in the database Parameters ---------- + update_database : bool + true: run updates on existing rootfilename entries + false: only create new entries + inst : str Name of JWST instrument """ @@ -73,13 +84,19 @@ def get_updates(inst): # Get lists of all public and proprietary files for the program logging.info(f'Working on proposal {proposal}') filenames_public, metadata_public, filenames_proprietary, metadata_proprietary = get_all_possible_filenames_for_proposal(inst, proposal) - # Find the location in the filesystem for all files logging.info('Getting all filenames and locating in the filesystem') filepaths_public = files_in_filesystem(filenames_public, 'public') filepaths_proprietary = files_in_filesystem(filenames_proprietary, 'proprietary') filenames = filepaths_public + filepaths_proprietary + # There is one and only one category for the proposal, so just take the first. + proposal_category = '' + if len(metadata_public['category']): + proposal_category = metadata_public['category'][0] + elif len(metadata_proprietary['category']): + proposal_category = metadata_proprietary['category'][0] + # Get set of unique rootnames all_rootnames = set(['_'.join(f.split('/')[-1].split('_')[:-1]) for f in filenames]) rootnames = [] @@ -126,8 +143,8 @@ def get_updates(inst): # Update the appropriate database table logging.info(f'Updating database for {inst}, Proposal {proposal}, Observation {obsnum}') - update_database_table(inst, proposal, obsnum, proposal_info['thumbnail_paths'][0], obsfiles, - exp_types, starting_date, latest_date) + update_database_table(update_database, inst, proposal, obsnum, proposal_info['thumbnail_paths'][0], obsfiles, + exp_types, starting_date, latest_date, proposal_category) def get_all_possible_filenames_for_proposal(instrument, proposal_num): @@ -158,7 +175,7 @@ def get_all_possible_filenames_for_proposal(instrument, proposal_num): e.g. 'exptime', and values are lists of the value for each filename. e.g. ['59867.6, 59867.601'] """ filename_query = mast_query_filenames_by_instrument(instrument, proposal_num, - other_columns=['exp_type', 'observtn', 'expstart', 'expend']) + other_columns=['exp_type', 'observtn', 'expstart', 'expend', 'category']) # Check the number of files returned by the MAST query. MAST has a limit of 50,000 rows in # the returned result. If we hit that limit, then we are most likely not getting back all of @@ -169,10 +186,10 @@ def get_all_possible_filenames_for_proposal(instrument, proposal_num): public, public_meta = get_filenames_by_instrument(instrument, proposal_num, restriction='public', query_response=filename_query, - other_columns=['exp_type', 'observtn', 'expstart', 'expend']) + other_columns=['exp_type', 'observtn', 'expstart', 'expend', 'category']) proprietary, proprietary_meta = get_filenames_by_instrument(instrument, proposal_num, restriction='proprietary', query_response=filename_query, - other_columns=['exp_type', 'observtn', 'expstart', 'expend']) + other_columns=['exp_type', 'observtn', 'expstart', 'expend', 'category']) return public, public_meta, proprietary, proprietary_meta @@ -206,11 +223,15 @@ def files_in_filesystem(files, permission_type): return filenames -def update_database_table(instrument, prop, obs, thumbnail, obsfiles, types, startdate, enddate): +def update_database_table(update, instrument, prop, obs, thumbnail, obsfiles, types, startdate, enddate, proposal_category): """Update the database tables that contain info about proposals and observations, via Django models. Parameters ---------- + update : bool + true: run updates on existing rootfilename entries + false: only create new entries + instrument : str Instrument name @@ -234,6 +255,9 @@ def update_database_table(instrument, prop, obs, thumbnail, obsfiles, types, sta enddate : float Date of the ending of the observation in MJD + + proposal_category : str + category name """ # Check to see if the required Archive entry exists, and create it if it doesn't @@ -252,7 +276,8 @@ def update_database_table(instrument, prop, obs, thumbnail, obsfiles, types, sta # Update the proposal instance with the thumbnail path prop_instance.thumbnail_path = thumbnail - prop_instance.save(update_fields=['thumbnail_path']) + prop_instance.cat_type = proposal_category + prop_instance.save(update_fields=['thumbnail_path', 'cat_type']) # Now that the Archive and Proposal instances are sorted, get or create the # Observation instance @@ -292,11 +317,29 @@ def update_database_table(instrument, prop, obs, thumbnail, obsfiles, types, sta # Get all unsaved root names in the Observation to store in the database nr_files_created = 0 for file in obsfiles: + defaults_dict = mast_query_by_rootname(instrument, file) try: - root_file_info_instance, rfi_created = RootFileInfo.objects.get_or_create(root_name=file, - instrument=instrument, - obsnum=obs_instance, - proposal=prop) + defaults = dict(filter=defaults_dict.get('filter', ''), + detector=defaults_dict.get('detector', ''), + exp_type=defaults_dict.get('exp_type', ''), + read_patt=defaults_dict.get('readpatt', ''), + grating=defaults_dict.get('grating', ''), + read_patt_num=defaults_dict.get('patt_num', 0), + aperture=defaults_dict.get('apername', ''), + subarray=defaults_dict.get('subarray', ''), + pupil=defaults_dict.get('pupil', '')) + if update: + root_file_info_instance, rfi_created = RootFileInfo.objects.update_or_create(root_name=file, + instrument=instrument, + obsnum=obs_instance, + proposal=prop, + defaults=defaults) + else: + root_file_info_instance, rfi_created = RootFileInfo.objects.get_or_create(root_name=file, + instrument=instrument, + obsnum=obs_instance, + proposal=prop, + defaults=defaults) if rfi_created: nr_files_created += 1 root_file_info_instance.save() @@ -307,9 +350,15 @@ def update_database_table(instrument, prop, obs, thumbnail, obsfiles, types, sta if __name__ == '__main__': + + update_database = False + for i in range(1, len(sys.argv)): + if sys.argv[i].lower() == 'update': + update_database = True + module = os.path.basename(__file__).strip('.py') start_time, log_file = initialize_instrument_monitor(module) instruments = ['nircam', 'miri', 'nirspec', 'niriss', 'fgs'] for instrument in instruments: - get_updates(instrument) + get_updates(update_database, instrument) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index c15fad920..f896024ad 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -586,6 +586,50 @@ def get_filenames_by_instrument(instrument, proposal, observation_id=None, restr return filenames +def mast_query_by_rootname(instrument, rootname): + """Query MAST for all columns given an instrument and rootname. Return the json + response from MAST. + + Parameters + ---------- + instrument : str + The instrument of interest (e.g. `FGS`). + rootname : str + The Rootname of Interest + + Returns + ------- + result : dict + Dictionary of rootname data + """ + + query_filters = [] + if '-seg' in rootname: + root_split = rootname.split('-') + file_set_name = root_split[0] + detector = root_split[-1] + else: + root_split = rootname.split('_') + file_set_name = '_'.join(root_split[:-1]) + detector = root_split[-1] + + service = INSTRUMENT_SERVICE_MATCH[instrument] + + query_filters.append({'paramName': 'fileSetName', 'values': [file_set_name]}) + query_filters.append({'paramName': 'detector', 'values': [detector.upper()]}) + params = {'columns': '*', + 'filters': query_filters} + response = Mast.service_request_async(service, params) + result = response[0].json() + retval = {} + if result['data'] == []: + print("WARNING: no data for {}".format(rootname)) + else: + retval = result['data'][0] + return retval + + + def mast_query_filenames_by_instrument(instrument, proposal_id, observation_id=None, other_columns=None): """Query MAST for filenames for the given instrument. Return the json response from MAST. From e7146842a12e5c9397430c3c595cd026c56e280c Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Thu, 2 Mar 2023 16:39:17 -0500 Subject: [PATCH 130/449] comment update --- jwql/website/apps/jwql/data_containers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index f896024ad..190bdd719 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -587,8 +587,7 @@ def get_filenames_by_instrument(instrument, proposal, observation_id=None, restr def mast_query_by_rootname(instrument, rootname): - """Query MAST for all columns given an instrument and rootname. Return the json - response from MAST. + """Query MAST for all columns given an instrument and rootname. Return the dict of the 'data' column Parameters ---------- From 502d1d8681d21b91f901187a94a463eb36510e7e Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Thu, 2 Mar 2023 17:10:58 -0500 Subject: [PATCH 131/449] fix detector parsing --- jwql/website/apps/jwql/data_containers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 190bdd719..312f3a3cf 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -606,6 +606,7 @@ def mast_query_by_rootname(instrument, rootname): if '-seg' in rootname: root_split = rootname.split('-') file_set_name = root_split[0] + root_split = rootname.split('_') detector = root_split[-1] else: root_split = rootname.split('_') From b0d94be05a17c78d4fd9d221dfe3236a0e9cb58c Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Fri, 3 Mar 2023 10:21:09 -0500 Subject: [PATCH 132/449] update models allow NULL --- jwql/website/apps/jwql/admin.py | 8 ++++---- .../apps/jwql/archive_database_update.py | 9 ++++++--- jwql/website/apps/jwql/models.py | 19 ++++++++++--------- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/jwql/website/apps/jwql/admin.py b/jwql/website/apps/jwql/admin.py index 668373880..15e08899a 100644 --- a/jwql/website/apps/jwql/admin.py +++ b/jwql/website/apps/jwql/admin.py @@ -29,8 +29,8 @@ class ArchiveAdmin(admin.ModelAdmin): @admin.register(Proposal) class ProposalAdmin(admin.ModelAdmin): - list_display = ('archive', 'prop_id', 'cat_type') - list_filter = ('archive', 'cat_type') + list_display = ('archive', 'prop_id', 'category') + list_filter = ('archive', 'category') @admin.register(Observation) @@ -41,8 +41,8 @@ class ObservationAdmin(admin.ModelAdmin): @admin.register(RootFileInfo) class RootFileInfoAdmin(admin.ModelAdmin): - list_display = ('root_name', 'obsnum', 'proposal', 'instrument', 'viewed', 'filter', 'aperture', 'detector', 'read_patt_num', 'read_patt', 'grating', 'subarray', 'pupil', 'exp_type') - list_filter = ('viewed', 'instrument', 'proposal', 'obsnum') + list_display = ('root_name', 'obsnum', 'proposal', 'instrument', 'viewed', 'filter', 'aperture', 'detector', 'read_patt_num', 'read_patt', 'grating', 'subarray', 'pupil', 'exp_type', 'expstart') + list_filter = ('viewed', 'instrument', 'proposal') @admin.register(Anomalies) diff --git a/jwql/website/apps/jwql/archive_database_update.py b/jwql/website/apps/jwql/archive_database_update.py index 80940ef38..ead2ab4a7 100755 --- a/jwql/website/apps/jwql/archive_database_update.py +++ b/jwql/website/apps/jwql/archive_database_update.py @@ -8,6 +8,7 @@ ------- - Bryan Hilbert + - Bradley Sappington Use --- @@ -276,8 +277,8 @@ def update_database_table(update, instrument, prop, obs, thumbnail, obsfiles, ty # Update the proposal instance with the thumbnail path prop_instance.thumbnail_path = thumbnail - prop_instance.cat_type = proposal_category - prop_instance.save(update_fields=['thumbnail_path', 'cat_type']) + prop_instance.category = proposal_category + prop_instance.save(update_fields=['thumbnail_path', 'category']) # Now that the Archive and Proposal instances are sorted, get or create the # Observation instance @@ -319,6 +320,7 @@ def update_database_table(update, instrument, prop, obs, thumbnail, obsfiles, ty for file in obsfiles: defaults_dict = mast_query_by_rootname(instrument, file) try: + # NOTE: If implementing a new field, add to this dict and run script with 'update' flag defaults = dict(filter=defaults_dict.get('filter', ''), detector=defaults_dict.get('detector', ''), exp_type=defaults_dict.get('exp_type', ''), @@ -327,7 +329,8 @@ def update_database_table(update, instrument, prop, obs, thumbnail, obsfiles, ty read_patt_num=defaults_dict.get('patt_num', 0), aperture=defaults_dict.get('apername', ''), subarray=defaults_dict.get('subarray', ''), - pupil=defaults_dict.get('pupil', '')) + pupil=defaults_dict.get('pupil', ''), + expstart=defaults_dict.get('expstart', 0.0)) if update: root_file_info_instance, rfi_created = RootFileInfo.objects.update_or_create(root_name=file, instrument=instrument, diff --git a/jwql/website/apps/jwql/models.py b/jwql/website/apps/jwql/models.py index 0197044fc..72cda5d31 100644 --- a/jwql/website/apps/jwql/models.py +++ b/jwql/website/apps/jwql/models.py @@ -61,7 +61,7 @@ class Proposal(models.Model): prop_id = models.CharField(max_length=5, help_text="5-digit proposal ID string") thumbnail_path = models.CharField(max_length=100, help_text='Path to the proposal thumbnail', default='') archive = models.ForeignKey(Archive, blank=False, null=False, on_delete=models.CASCADE) - cat_type = models.CharField(max_length=10, help_text="Category Type", default='') + category = models.CharField(max_length=10, help_text="Category Type", default='') # Metadata class Meta: @@ -104,15 +104,16 @@ class RootFileInfo(models.Model): proposal = models.CharField(max_length=5, help_text="5-digit proposal ID string") root_name = models.TextField(primary_key=True, max_length=300) viewed = models.BooleanField(default=False) - filter = models.CharField(max_length=7, help_text="Instrument name", default='') - aperture = models.CharField(max_length=40, help_text="Aperture", default='') - detector = models.CharField(max_length=40, help_text="Detector", default='') + filter = models.CharField(max_length=7, help_text="Instrument name", default='', null=True, blank=True) + aperture = models.CharField(max_length=40, help_text="Aperture", default='', null=True, blank=True) + detector = models.CharField(max_length=40, help_text="Detector", default='', null=True, blank=True) read_patt_num = models.IntegerField(help_text='Read Pattern Number', default=0) - read_patt = models.CharField(max_length=40, help_text="Read Pattern", default='') - grating = models.CharField(max_length=40, help_text="Grating", default='') - subarray = models.CharField(max_length=40, help_text="Subarray", default='') - pupil = models.CharField(max_length=40, help_text="Pupil", default='') - exp_type = models.CharField(max_length=40, help_text="Exposure Type", default='') + read_patt = models.CharField(max_length=40, help_text="Read Pattern", default='', null=True, blank=True) + grating = models.CharField(max_length=40, help_text="Grating", default='', null=True, blank=True) + subarray = models.CharField(max_length=40, help_text="Subarray", default='', null=True, blank=True) + pupil = models.CharField(max_length=40, help_text="Pupil", default='', null=True, blank=True) + exp_type = models.CharField(max_length=40, help_text="Exposure Type", default='', null=True, blank=True) + expstart = models.FloatField(help_text='Exposure Start Time', default=0.0) # Metadata class Meta: From 17d4a2bbece51e7dba31b4cab888c3053149ed9b Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 3 Mar 2023 14:13:37 -0500 Subject: [PATCH 133/449] Add exposure grouping to archive query and date range; fix navigation for groups --- jwql/tests/test_utils.py | 112 +++++++++++--- jwql/utils/utils.py | 18 ++- jwql/website/apps/jwql/data_containers.py | 143 ++++++++++++------ jwql/website/apps/jwql/static/js/jwql.js | 29 ++-- .../jwql/templates/archive_date_range.html | 5 +- .../apps/jwql/templates/query_submit.html | 5 +- .../apps/jwql/templates/view_exposure.html | 6 +- .../apps/jwql/templates/view_image.html | 2 - jwql/website/apps/jwql/views.py | 92 +++++++---- 9 files changed, 291 insertions(+), 121 deletions(-) diff --git a/jwql/tests/test_utils.py b/jwql/tests/test_utils.py index 624841573..c2f6795c1 100644 --- a/jwql/tests/test_utils.py +++ b/jwql/tests/test_utils.py @@ -43,7 +43,9 @@ 'program_id': '90002', 'suffix': 'rateints', 'visit': '001', - 'visit_group': '02'}), + 'visit_group': '02', + 'file_root': 'jw90002001001_02102_00001_nis', + 'group_root': 'jw90002001001_02102_00001'}), # Test full stage 1 and 2 filename ('jw00327001001_02101_00002_nrca1_rate.fits', @@ -57,7 +59,9 @@ 'program_id': '00327', 'suffix': 'rate', 'visit': '001', - 'visit_group': '02'}), + 'visit_group': '02', + 'file_root': 'jw00327001001_02101_00002_nrca1', + 'group_root': 'jw00327001001_02101_00002'}), # Test root stage 1 and 2 filename ('jw00327001001_02101_00002_nrca1', @@ -70,7 +74,9 @@ 'parallel_seq_id': '1', 'program_id': '00327', 'visit': '001', - 'visit_group': '02'}), + 'visit_group': '02', + 'file_root': 'jw00327001001_02101_00002_nrca1', + 'group_root': 'jw00327001001_02101_00002'}), # Test stage 2 MSA metadata filename ('jw01118008001_01_msa.fits', @@ -78,7 +84,10 @@ 'instrument': 'nirspec', 'observation': '008', 'program_id': '01118', - 'visit': '001'}), + 'visit': '001', + 'detector': 'Unknown', + 'file_root': 'jw01118008001_01_msa', + 'group_root': 'jw01118008001_01_msa'}), # Test full stage 2c filename ('jw94015002002_02108_00001_mirimage_o002_crf.fits', @@ -93,7 +102,9 @@ 'program_id': '94015', 'suffix': 'crf', 'visit': '002', - 'visit_group': '02'}), + 'visit_group': '02', + 'file_root': 'jw94015002002_02108_00001_mirimage', + 'group_root': 'jw94015002002_02108_00001'}), # Test root stage 2c filename ('jw90001001003_02101_00001_nis_o001', @@ -107,7 +118,9 @@ 'parallel_seq_id': '1', 'program_id': '90001', 'visit': '003', - 'visit_group': '02'}), + 'visit_group': '02', + 'file_root': 'jw90001001003_02101_00001_nis', + 'group_root': 'jw90001001003_02101_00001'}), # Test full stage 3 filename with target_id ('jw80600-o009_t001_miri_f1130w_i2d.fits', @@ -117,7 +130,10 @@ 'optical_elements': 'f1130w', 'program_id': '80600', 'suffix': 'i2d', - 'target_id': 't001'}), + 'target_id': 't001', + 'detector': 'Unknown', + 'file_root': 'jw80600-o009_t001_miri_f1130w', + 'group_root': 'jw80600-o009_t001_miri_f1130w'}), # Test full stage 3 filename with target_id and different ac_id ('jw80600-c0001_t001_miri_f1130w_i2d.fits', @@ -127,7 +143,10 @@ 'optical_elements': 'f1130w', 'program_id': '80600', 'suffix': 'i2d', - 'target_id': 't001'}), + 'target_id': 't001', + 'detector': 'Unknown', + 'file_root': 'jw80600-c0001_t001_miri_f1130w', + 'group_root': 'jw80600-c0001_t001_miri_f1130w'}), # Test full stage 3 filename with source_id ('jw80600-o009_s00001_miri_f1130w_i2d.fits', @@ -137,7 +156,10 @@ 'optical_elements': 'f1130w', 'program_id': '80600', 'source_id': 's00001', - 'suffix': 'i2d'}), + 'suffix': 'i2d', + 'detector': 'Unknown', + 'file_root': 'jw80600-o009_s00001_miri_f1130w', + 'group_root': 'jw80600-o009_s00001_miri_f1130w'}), # Test stage 3 filename with target_id and epoch ('jw80600-o009_t001-epoch1_miri_f1130w_i2d.fits', @@ -148,7 +170,10 @@ 'optical_elements': 'f1130w', 'program_id': '80600', 'suffix': 'i2d', - 'target_id': 't001'}), + 'target_id': 't001', + 'detector': 'Unknown', + 'file_root': 'jw80600-o009_t001-epoch1_miri_f1130w', + 'group_root': 'jw80600-o009_t001-epoch1_miri_f1130w'}), # Test stage 3 filename with source_id and epoch ('jw80600-o009_s00001-epoch1_miri_f1130w_i2d.fits', @@ -159,7 +184,10 @@ 'optical_elements': 'f1130w', 'program_id': '80600', 'source_id': 's00001', - 'suffix': 'i2d'}), + 'suffix': 'i2d', + 'detector': 'Unknown', + 'file_root': 'jw80600-o009_s00001-epoch1_miri_f1130w', + 'group_root': 'jw80600-o009_s00001-epoch1_miri_f1130w'}), # Test root stage 3 filename with target_id ('jw80600-o009_t001_miri_f1130w', @@ -168,7 +196,10 @@ 'instrument': 'miri', 'optical_elements': 'f1130w', 'program_id': '80600', - 'target_id': 't001'}), + 'target_id': 't001', + 'detector': 'Unknown', + 'file_root': 'jw80600-o009_t001_miri_f1130w', + 'group_root': 'jw80600-o009_t001_miri_f1130w'}), # Test root stage 3 filename with source_id ('jw80600-o009_s00001_miri_f1130w', @@ -177,7 +208,10 @@ 'instrument': 'miri', 'optical_elements': 'f1130w', 'program_id': '80600', - 'source_id': 's00001'}), + 'source_id': 's00001', + 'detector': 'Unknown', + 'file_root': 'jw80600-o009_s00001_miri_f1130w', + 'group_root': 'jw80600-o009_s00001_miri_f1130w'}), # Test full time series filename ('jw00733003001_02101_00002-seg001_nrs1_rate.fits', @@ -192,7 +226,9 @@ 'segment': '001', 'suffix': 'rate', 'visit': '001', - 'visit_group': '02'}), + 'visit_group': '02', + 'file_root': 'jw00733003001_02101_00002-seg001_nrs1', + 'group_root': 'jw00733003001_02101_00002-seg001'}), # Test full time series filename for stage 2c ('jw00733003001_02101_00002-seg001_nrs1_o001_crfints.fits', @@ -208,7 +244,9 @@ 'segment': '001', 'suffix': 'crfints', 'visit': '001', - 'visit_group': '02'}), + 'visit_group': '02', + 'file_root': 'jw00733003001_02101_00002-seg001_nrs1', + 'group_root': 'jw00733003001_02101_00002-seg001'}), # Test root time series filename ('jw00733003001_02101_00002-seg001_nrs1', @@ -222,7 +260,9 @@ 'program_id': '00733', 'segment': '001', 'visit': '001', - 'visit_group': '02'}), + 'visit_group': '02', + 'file_root': 'jw00733003001_02101_00002-seg001_nrs1', + 'group_root': 'jw00733003001_02101_00002-seg001'}), # Test full guider ID filename ('jw00729011001_gs-id_1_image_cal.fits', @@ -234,7 +274,10 @@ 'observation': '011', 'program_id': '00729', 'suffix': 'image_cal', - 'visit': '001'}), + 'visit': '001', + 'detector': 'Unknown', + 'file_root': 'jw00729011001_gs-id_1', + 'group_root': 'jw00729011001_gs-id_1'}), # Test full guider ID filename with 2-digit attempts ('jw00729011001_gs-id_12_image_cal.fits', @@ -246,7 +289,10 @@ 'observation': '011', 'program_id': '00729', 'suffix': 'image_cal', - 'visit': '001'}), + 'visit': '001', + 'detector': 'Unknown', + 'file_root': 'jw00729011001_gs-id_12', + 'group_root': 'jw00729011001_gs-id_12'}), # Test root guider ID filename ('jw00327001001_gs-id_2', @@ -257,7 +303,10 @@ 'instrument': 'fgs', 'observation': '001', 'program_id': '00327', - 'visit': '001'}), + 'visit': '001', + 'detector': 'Unknown', + 'file_root': 'jw00327001001_gs-id_2', + 'group_root': 'jw00327001001_gs-id_2'}), # Test root guider ID filename with 2-digit attempts ('jw00327001001_gs-id_12', @@ -268,7 +317,10 @@ 'instrument': 'fgs', 'observation': '001', 'program_id': '00327', - 'visit': '001'}), + 'visit': '001', + 'detector': 'Unknown', + 'file_root': 'jw00327001001_gs-id_12', + 'group_root': 'jw00327001001_gs-id_12'}), # Test full guider non-ID filename ('jw86600048001_gs-fg_2016018175411_stream.fits', @@ -280,7 +332,10 @@ 'observation': '048', 'program_id': '86600', 'suffix': 'stream', - 'visit': '001'}), + 'visit': '001', + 'detector': 'Unknown', + 'file_root': 'jw86600048001_gs-fg_2016018175411', + 'group_root': 'jw86600048001_gs-fg_2016018175411'}), # Test root guider non-ID filename ('jw00729011001_gs-acq2_2019155024808', @@ -291,7 +346,10 @@ 'instrument': 'fgs', 'observation': '011', 'program_id': '00729', - 'visit': '001'}), + 'visit': '001', + 'detector': 'Unknown', + 'file_root': 'jw00729011001_gs-acq2_2019155024808', + 'group_root': 'jw00729011001_gs-acq2_2019155024808'}), # Test segmented guider file ('jw01118005001_gs-fg_2022150070312-seg002_uncal.fits', @@ -304,7 +362,10 @@ 'program_id': '01118', 'segment': '002', 'suffix': 'uncal', - 'visit': '001'}), + 'visit': '001', + 'detector': 'Unknown', + 'file_root': 'jw01118005001_gs-fg_2022150070312-seg002', + 'group_root': 'jw01118005001_gs-fg_2022150070312-seg002'}), # Test msa file ('jw02560013001_01_msa.fits', @@ -312,7 +373,10 @@ 'observation': '013', 'visit': '001', 'filename_type': 'stage_2_msa', - 'instrument': 'nirspec'}) + 'instrument': 'nirspec', + 'detector': 'Unknown', + 'file_root': 'jw02560013001_01_msa', + 'group_root': 'jw02560013001_01_msa'}) ] diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index 11d3c7fa1..43f8ec580 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -279,7 +279,12 @@ def filename_parser(filename): """ filename = os.path.basename(filename) - file_root_name = (len(filename.split('.')) < 2) + split_filename = filename.split('.') + file_root_name = (len(split_filename) < 2) + if file_root_name: + root_name = filename + else: + root_name = split_filename[0] # Stage 1 and 2 filenames # e.g. "jw80500012009_01101_00012_nrcalong_uncal.fits" @@ -474,6 +479,17 @@ def filename_parser(filename): elif name_match == 'stage_2_msa': filename_dict['instrument'] = 'nirspec' + # Also add detector, root name, and group root name + root_name = re.sub(rf"_{filename_dict.get('suffix', '')}$", '', root_name) + root_name = re.sub(rf"_{filename_dict.get('ac_id', '')}$", '', root_name) + filename_dict['file_root'] = root_name + if 'detector' not in filename_dict.keys(): + filename_dict['detector'] = 'Unknown' + filename_dict['group_root'] = root_name + else: + group_root = re.sub(rf"_{filename_dict['detector']}$", '', root_name) + filename_dict['group_root'] = group_root + # Raise error if unable to parse the filename except AttributeError: jdox_url = 'https://jwst-docs.stsci.edu/understanding-jwst-data-files/jwst-data-file-naming-conventions' diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index fbd901d99..41741e07e 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -151,6 +151,77 @@ def build_table(tablename): return table_meta_data +def filter_root_files(instrument=None, proposal=None, obsnum=None, sort_as=None, + look=None, exp_type=None, cat_type=None, detector=None): + """Retrieve and filter root file table entries. + + Parameters + ---------- + instrument : str, optional + Name of the JWST instrument. + proposal : str, optional + Proposal to match. + obsnum : str, optional + Observation number to match. + sort_as : {'ascending', 'descending', 'recent', 'oldest'}, optional + Sorting method for output table. Ascending and descending + options refer to root file name; recent and oldest sort by observation + start. + look : {'new', 'viewed'}, optional + If set to None, all viewed values are returned. If set to + 'viewed', only viewed data is returned. If set to 'new', only + new data is returned. + exp_type : str, optional + Set to filter by exposure type. + cat_type : str, optional + Set to filter by proposal category. + detector : str, optional + Set to filter by detector name. + + Returns + ------- + root_file_info : QuerySet + List of RootFileInfo entries matching input criteria. + """ + # standardize input + + # TODO: update when more fields are available + # get desired filters + filter_kwargs = dict() + if instrument is not None and str(instrument).strip().lower() != 'all': + inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[instrument.lower()] + filter_kwargs['instrument'] = inst + if proposal is not None and str(proposal).strip().lower() != 'all': + filter_kwargs['proposal'] = proposal.lstrip('0') + if obsnum is not None and str(obsnum).strip().lower() != 'all': + filter_kwargs['obsnum__obsnum'] = obsnum + if look is not None and str(look).strip().lower() != 'all': + filter_kwargs['viewed'] = (str(look).lower() == 'viewed') + if exp_type is not None and str(exp_type).strip().lower() != 'all': + filter_kwargs['obsnum__exptypes__icontains'] = exp_type + if cat_type is not None and str(cat_type).strip().lower() != 'all': + # filter_kwargs['obsnum__proposal__cat_type__iexact'] = cat_type + # not yet implemented in proposal table + pass + if detector is not None and str(detector).strip().lower() != 'all': + filter_kwargs['root_name__icontains'] = detector + + # get file info by instrument from local model + root_file_info = RootFileInfo.objects.filter(**filter_kwargs) + + # descending by root file is default; + # for other options, sort as desired + sort_as = str(sort_as).strip().lower() + if sort_as == 'ascending': + root_file_info = root_file_info.order_by('root_name') + elif sort_as == 'recent': + root_file_info = root_file_info.order_by('-obsnum__obsstart', 'root_name') + elif sort_as == 'oldest': + root_file_info = root_file_info.order_by('obsnum__obsstart', 'root_name') + + return root_file_info + + def get_acknowledgements(): """Returns a list of individuals who are acknowledged on the ``about`` page. @@ -1040,26 +1111,10 @@ def get_instrument_looks(instrument, sort_as=None, if key not in key_set: keys.append(key) - # get desired filters - filter_kwargs = dict() - if look is not None: - filter_kwargs['viewed'] = (str(look).lower() == 'viewed') - if exp_type is not None: - filter_kwargs['obsnum__exptypes__icontains'] = exp_type - if cat_type is not None: - # filter_kwargs['obsnum__proposal__cat_type__iexact'] = cat_type - # not yet implemented in proposal table - pass - - # get file info by instrument from local model - root_file_info = RootFileInfo.objects.filter(instrument=inst, **filter_kwargs) - - # descending by root file is default; - # for other options, sort as desired - if sort_as == 'ascending': - root_file_info = root_file_info.order_by('root_name') - elif sort_as == 'recent': - root_file_info = root_file_info.order_by('-obsnum__obsstart') + # get filtered file info + root_file_info = filter_root_files( + instrument=instrument, sort_as=sort_as, look=look, + exp_type=exp_type, cat_type=cat_type) looks = [] for root_file in root_file_info: @@ -1660,14 +1715,6 @@ def thumbnails_ajax(inst, proposal, obs_num=None): try: filename_dict = filename_parser(rootname) - # The detector keyword is expected in thumbnails_query_ajax() for generating filterable dropdown menus - if 'detector' not in filename_dict.keys(): - filename_dict['detector'] = 'Unknown' - filename_dict['group_root'] = rootname - else: - group_root = re.sub(rf"_{filename_dict['detector']}$", '', rootname) - filename_dict['group_root'] = group_root - # Weed out file types that are not supported by generate_preview_images if 'stage_3' in filename_dict['filename_type']: continue @@ -1781,10 +1828,11 @@ def thumbnails_date_range_ajax(inst, observations, inclusive_start_time_mjd, exc Dictionary of data needed for the ``thumbnails`` template """ - data_dict = {} - data_dict['inst'] = inst - data_dict['file_data'] = {} - exp_types = [] + data_dict = {'inst': inst, + 'file_data': dict()} + exp_types = set() + exp_groups = set() + # Get the available files for the instrument for observation in observations: obs_num = observation.obsnum @@ -1798,10 +1846,6 @@ def thumbnails_date_range_ajax(inst, observations, inclusive_start_time_mjd, exc try: filename_dict = filename_parser(rootname) - # The detector keyword is expected in thumbnails_query_ajax() for generating filterable dropdown menus - if 'detector' not in filename_dict.keys(): - filename_dict['detector'] = 'Unknown' - # Weed out file types that are not supported by generate_preview_images if 'stage_3' in filename_dict['filename_type']: continue @@ -1815,7 +1859,8 @@ def thumbnails_date_range_ajax(inst, observations, inclusive_start_time_mjd, exc 'parallel_seq_id': rootname[16], 'program_id': rootname[2:7], 'visit': rootname[10:13], - 'visit_group': rootname[14:16]} + 'visit_group': rootname[14:16], + 'group_root': rootname[:26]} # Get list of available filenames and exposure start times. All files with a given # rootname will have the same exposure start time, so just keep the first. @@ -1823,16 +1868,18 @@ def thumbnails_date_range_ajax(inst, observations, inclusive_start_time_mjd, exc exp_start = [expstart for fname, expstart in zip(filenames, columns['expstart']) if rootname in fname][0] if exp_start >= inclusive_start_time_mjd and exp_start < exclusive_stop_time_mjd: exp_type = [exp_type for fname, exp_type in zip(filenames, columns['exp_type']) if rootname in fname][0] - exp_types.append(exp_type) + exp_types.add(exp_type) # Viewed is stored by rootname in the Model db. Save it with the data_dict # THUMBNAIL_FILTER_LOOK is boolean accessed according to a viewed flag try: root_file_info = RootFileInfo.objects.get(root_name=rootname) viewed = THUMBNAIL_FILTER_LOOK[root_file_info.viewed] except RootFileInfo.DoesNotExist: - viewed = THUMBNAIL_FILTER_LOOK[0] + # Add to list of all exposure groups + exp_groups.add(filename_dict['group_root']) + # Add data to dictionary data_dict['file_data'][rootname] = {} data_dict['file_data'][rootname]['filename_dict'] = filename_dict @@ -1868,12 +1915,13 @@ def thumbnails_date_range_ajax(inst, observations, inclusive_start_time_mjd, exc data_dict['tools'] = MONITORS data_dict['dropdown_menus'] = dropdown_menus - # Order dictionary by descending expstart time. sorted_file_data = OrderedDict(sorted(data_dict['file_data'].items(), key=lambda x: getitem(x[1], 'expstart'), reverse=True)) data_dict['file_data'] = sorted_file_data + data_dict['exp_groups'] = sorted(exp_groups) + return data_dict @@ -1894,15 +1942,16 @@ def thumbnails_query_ajax(rootnames, expstarts=None): Dictionary of data needed for the ``thumbnails`` template """ # Initialize dictionary that will contain all needed data - data_dict = {} - # dummy variable for view_image when thumbnail is selected - data_dict['inst'] = "all" - data_dict['file_data'] = {} + data_dict = {'inst': 'all', + 'file_data': dict()} + exp_groups = set() + # Gather data for each rootname for rootname in rootnames: # fit expected format for get_filenames_by_rootname() + split_name = rootname.split("_") try: - rootname = rootname.split("_")[0] + '_' + rootname.split("_")[1] + '_' + rootname.split("_")[2] + '_' + rootname.split("_")[3] + rootname = split_name[0] + '_' + split_name[1] + '_' + split_name[2] + '_' + split_name[3] except IndexError: continue @@ -1912,6 +1961,9 @@ def thumbnails_query_ajax(rootnames, expstarts=None): except ValueError: continue + # Add to list of all exposure groups + exp_groups.add(filename_dict['group_root']) + # Get list of available filenames available_files = get_filenames_by_rootname(rootname) @@ -1960,5 +2012,6 @@ def thumbnails_query_ajax(rootnames, expstarts=None): data_dict['tools'] = MONITORS data_dict['dropdown_menus'] = dropdown_menus + data_dict['exp_groups'] = sorted(exp_groups) return data_dict diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index 0885bbf94..516387230 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -612,18 +612,25 @@ function show_only(filter_type, value, base_url) { } // If data are grouped, check if a thumbnail for the group has already been displayed + var show_group = true; if (group && groups_shown.has(thumbnails[i].getAttribute('data-group_root'))) { - criteria.push(false); + show_group = false; } // Only display if all criteria are met if (criteria.every(function(r){return r})) { - thumbnails[i].style.display = "inline-block"; - num_thumbnails_displayed++; + // if group has already been shown, do not show thumbnail, + // but do store the file root for navigation + if (show_group) { + thumbnails[i].style.display = "inline-block"; + num_thumbnails_displayed++; + if (group) { groups_shown.add(thumbnails[i].getAttribute('data-group_root')); } + } else { + thumbnails[i].style.display = "none"; + } list_of_rootnames = list_of_rootnames + thumbnails[i].getAttribute("data-file_root") + '=' + thumbnails[i].getAttribute("data-exp_start") + ','; - if (group) { groups_shown.add(thumbnails[i].getAttribute('data-group_root')); } } else { thumbnails[i].style.display = "none"; } @@ -1188,7 +1195,7 @@ function update_thumbnail_array(data) { * @param {String} inst - The instrument of interest (e.g. "FGS") * @param {String} base_url - The base URL for gathering data from the AJAX view. */ -function submit_date_range_form(inst, base_url) { +function submit_date_range_form(inst, base_url, group) { var start_date = document.getElementById("start_date_range").value; var stop_date = document.getElementById("stop_date_range").value; @@ -1221,9 +1228,11 @@ function submit_date_range_form(inst, base_url) { update_show_count(num_thumbnails, 'activities'); update_thumbnail_array(data); update_filter_options(data, base_url, 'thumbnail'); - - // Do initial sort to match sort button display + update_group_options(data, base_url); update_sort_options(data, base_url); + + // Do initial sort and group to match sort button display + group_by_thumbnails(group, base_url); sort_by_thumbnails(data.thumbnail_sort, base_url); // Replace loading screen with the proposal array div @@ -1288,7 +1297,7 @@ function update_thumbnails_per_observation_page(inst, proposal, observation, bas * @param {String} base_url - The base URL for gathering data from the AJAX view. * @param {String} sort - Sort method string saved in session data image_sort */ -function update_thumbnails_query_page(base_url, sort) { +function update_thumbnails_query_page(base_url, sort, group) { $.ajax({ url: base_url + '/ajax/query_submit/', success: function(data){ @@ -1297,9 +1306,11 @@ function update_thumbnails_query_page(base_url, sort) { update_show_count(num_thumbnails, 'activities'); update_thumbnail_array(data); update_filter_options(data, base_url, 'thumbnail'); + update_group_options(data, base_url); update_sort_options(data, base_url); - // Do initial sort to match sort button display + // Do initial sort and group to match sort button display + group_by_thumbnails(group, base_url); sort_by_thumbnails(sort, base_url); // Replace loading screen with the proposal array div diff --git a/jwql/website/apps/jwql/templates/archive_date_range.html b/jwql/website/apps/jwql/templates/archive_date_range.html index d5cb37ace..15b7ec4af 100644 --- a/jwql/website/apps/jwql/templates/archive_date_range.html +++ b/jwql/website/apps/jwql/templates/archive_date_range.html @@ -18,7 +18,8 @@

    Archive Date Range {{ inst }} Images

    -
    +
    +
    @@ -27,7 +28,7 @@

    Archive Date Range {{ inst }} Images

    Date Range:
    - +
    diff --git a/jwql/website/apps/jwql/templates/query_submit.html b/jwql/website/apps/jwql/templates/query_submit.html index 4ff6206ef..20d1fccbc 100644 --- a/jwql/website/apps/jwql/templates/query_submit.html +++ b/jwql/website/apps/jwql/templates/query_submit.html @@ -25,7 +25,8 @@

    Images of Queried Instruments

    -
    +
    +
    @@ -49,7 +50,7 @@

    Images of Queried Instruments

    - + diff --git a/jwql/website/apps/jwql/templates/view_exposure.html b/jwql/website/apps/jwql/templates/view_exposure.html index 0359944fb..834bc335f 100644 --- a/jwql/website/apps/jwql/templates/view_exposure.html +++ b/jwql/website/apps/jwql/templates/view_exposure.html @@ -52,9 +52,7 @@

    Exposure {{ group_root }}

    {{ group_root }}_{{ detector }}_cal_integ0.jpg @@ -73,11 +71,11 @@

    Exposure {{ group_root }}

    {% if group_root in group_root_list %} {% set index = group_root_list.index(group_root) %} {% if index != 0 %} - < Previous + < Previous {% endif %} {% if index != group_root_list|length - 1 %} - Next > + Next > {% endif %} {% endif %}     diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html index 72aef74b1..6afbdd403 100644 --- a/jwql/website/apps/jwql/templates/view_image.html +++ b/jwql/website/apps/jwql/templates/view_image.html @@ -48,9 +48,7 @@

    {{ file_root }}

    {{ file_root }}_cal_integ0.jpg diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index f5db29128..acfc0b417 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -54,7 +54,6 @@ from bokeh.embed import components from django.http import HttpResponse, JsonResponse from django.shortcuts import redirect, render -import numpy as np import socket from jwql.database.database_interface import load_connection @@ -229,9 +228,11 @@ def archive_date_range(request, inst): template = 'archive_date_range.html' sort_type = request.session.get('image_sort', 'Recent') + group_type = request.session.get('image_group', 'Exposure') context = {'inst': inst, 'base_url': get_base_url(), - 'sort': sort_type} + 'sort': sort_type, + 'group': group_type} return render(request, template, context) @@ -261,23 +262,37 @@ def archive_date_range_ajax(request, inst, start_date, stop_date): # Calculate start date/time in MJD format to begin our range inclusive_start_time = Time(start_date) - # Add a minute to the stop time and mark it 'exclusive' for code clarity, doing this to get all seconds selected minute + + # Add a minute to the stop time and mark it 'exclusive' for code clarity, + # doing this to get all seconds selected minute exclusive_stop_time = Time(stop_date) + (1 * u.minute) # Get a queryset of all observations STARTING WITHIN our date range - begin_after_start = Observation.objects.filter(proposal__archive__instrument=inst, obsstart__gte=inclusive_start_time.mjd) - all_entries_beginning_in_range = begin_after_start.filter(obsstart__lt=exclusive_stop_time.mjd) + begin_after_start = Observation.objects.filter( + proposal__archive__instrument=inst, + obsstart__gte=inclusive_start_time.mjd) + all_entries_beginning_in_range = begin_after_start.filter( + obsstart__lt=exclusive_stop_time.mjd) # Get a queryset of all observations ENDING WITHIN our date range - end_after_start = Observation.objects.filter(proposal__archive__instrument=inst, obsend__gte=inclusive_start_time.mjd) - all_entries_ending_in_range = end_after_start.filter(obsend__lt=exclusive_stop_time.mjd) - - # Get a queryset of all observations SPANNING (starting before and ending after) our date range. Bump our window out a few days to catch hypothetical - # observations that last over 24 hours. The larger the window the more time the query takes so keeping it tight. + end_after_start = Observation.objects.filter( + proposal__archive__instrument=inst, + obsend__gte=inclusive_start_time.mjd) + all_entries_ending_in_range = end_after_start.filter( + obsend__lt=exclusive_stop_time.mjd) + + # Get a queryset of all observations SPANNING + # (starting before and ending after) our date range. + # Bump our window out a few days to catch hypothetical + # observations that last over 24 hours. + # The larger the window the more time the query takes so keeping it tight. two_days_before_start_time = Time(start_date) - (2 * u.day) two_days_after_end_time = Time(stop_date) + (3 * u.day) - end_after_stop = Observation.objects.filter(proposal__archive__instrument=inst, obsend__gte=two_days_after_end_time.mjd) - all_entries_spanning_range = end_after_stop.filter(obsstart__lt=two_days_before_start_time.mjd) + end_after_stop = Observation.objects.filter( + proposal__archive__instrument=inst, + obsend__gte=two_days_after_end_time.mjd) + all_entries_spanning_range = end_after_stop.filter( + obsstart__lt=two_days_before_start_time.mjd) obs_beginning = [observation for observation in all_entries_beginning_in_range] obs_ending = [observation for observation in all_entries_ending_in_range] @@ -286,8 +301,10 @@ def archive_date_range_ajax(request, inst, start_date, stop_date): # Create a single list of all pertinent observations all_observations = list(set(obs_beginning + obs_ending + obs_spanning)) # Get all thumbnails that occurred within the time frame for these observations - data = thumbnails_date_range_ajax(inst, all_observations, inclusive_start_time.mjd, exclusive_stop_time.mjd) + data = thumbnails_date_range_ajax( + inst, all_observations, inclusive_start_time.mjd, exclusive_stop_time.mjd) data['thumbnail_sort'] = request.session.get("image_sort", "Recent") + data['thumbnail_group'] = request.session.get("image_group", "Exposure") # Create Dictionary of Rootnames with expstart save_page_navigation_data(request, data) @@ -425,8 +442,7 @@ def archived_proposals_ajax(request, inst): def archive_thumbnails_ajax(request, inst, proposal, observation=None): - """Generate the page listing all archived images in the database - for a certain proposal + """Generate the page listing archived images by proposal. Parameters ---------- @@ -508,8 +524,7 @@ def archive_thumbnails_per_observation(request, inst, proposal, observation): def archive_thumbnails_query_ajax(request): - """Generate the page listing all archived images in the database - for a certain proposal + """Generate the page listing archived images by query parameters. Parameters ---------- @@ -533,11 +548,14 @@ def archive_thumbnails_query_ajax(request): instrument = JWST_INSTRUMENT_NAMES_MIXEDCASE[instrument.lower()] instruments_list.append(instrument) - # when parameters only contains nirspec as instrument, thumbnails still end up being all niriss data + # when parameters only contains nirspec as instrument, + # thumbnails still end up being all niriss data thumbnails = get_thumbnails_all_instruments(parameters) data = thumbnails_query_ajax(thumbnails) data['thumbnail_sort'] = request.session.get("image_sort", "Ascending") + data['thumbnail_group'] = request.session.get("image_group", "Exposure") + save_page_navigation_data(request, data) return JsonResponse(data, json_dumps_params={'indent': 2}) @@ -885,9 +903,11 @@ def query_submit(request): template = 'query_submit.html' sort_type = request.session.get('image_sort', 'Ascending') + group_type = request.session.get('image_group', 'Exposure') context = {'inst': '', 'base_url': get_base_url(), - 'sort': sort_type + 'sort': sort_type, + 'group': group_type } return render(request, template, context) @@ -1116,9 +1136,10 @@ def save_image_sort_ajax(request): def save_page_navigation_data(request, data): """ - It saves the data from the current page in the session so that the user can navigate to the next or - previous page. Our current sort options are Ascending/Descending, and Recent/Oldest so we need to - preserve 'rootname' and 'expstart'. + Save the data from the current page in the session. + + Enables navigating to the next or previous page. Current sort options + are Ascending/Descending, and Recent/Oldest. Parameters ---------- @@ -1219,8 +1240,9 @@ def view_exposure(request, inst, group_root): template = 'view_exposure.html' image_info = get_image_info(group_root) - # Get the proposal id from the group root name + # Get the proposal id and obsnum from the group root name prop_id = group_root[2:7] + obsnum = group_root[7:10] # Get available suffixes in a consistent order. suffixes = get_available_suffixes(image_info['suffixes'], @@ -1229,26 +1251,32 @@ def view_exposure(request, inst, group_root): # Get the anomaly submission form form = get_anomaly_form(request, inst, group_root) - # if we get to this page without any navigation data (i.e. direct link), - # just use the group_root with no expstart time - # navigate_data is dict of format rootname:expstart - navigation_data = request.session.get('navigation_data', {group_root: 0}) + # if we get to this page without any navigation data, + # previous/next buttons will be hidden + navigation_data = request.session.get('navigation_data', {}) # For time based sorting options, sort to "Recent" first to create sorting consistency when times are the same. # This is consistent with how Tinysort is utilized in jwql.js->sort_by_thumbnails sort_type = request.session.get('image_sort', 'Ascending') if sort_type in ['Descending']: - group_root_list = sorted(navigation_data, reverse=True) + matching_rootfiles = sorted(navigation_data, reverse=True) elif sort_type in ['Recent']: navigation_data = dict(sorted(navigation_data.items())) navigation_data = dict(sorted(navigation_data.items(), key=operator.itemgetter(1), reverse=True)) - group_root_list = list(navigation_data.keys()) + matching_rootfiles = list(navigation_data.keys()) elif sort_type in ['Oldest']: navigation_data = dict(sorted(navigation_data.items())) navigation_data = dict(sorted(navigation_data.items(), key=operator.itemgetter(1))) - group_root_list = list(navigation_data.keys()) + matching_rootfiles = list(navigation_data.keys()) else: - group_root_list = sorted(navigation_data) + matching_rootfiles = sorted(navigation_data) + + # pick out group names from matching root files + group_root_list = [] + for rootname in matching_rootfiles: + other_group_root = filename_parser(rootname)['group_root'] + if other_group_root not in group_root_list: + group_root_list.append(other_group_root) # Get our current views RootFileInfo model and send our "viewed/new" information root_file_info = RootFileInfo.objects.filter(root_name__startswith=group_root) @@ -1259,7 +1287,7 @@ def view_exposure(request, inst, group_root): 'group_root_list': group_root_list, 'inst': inst, 'prop_id': prop_id, - 'obsnum': group_root[7:10], + 'obsnum': obsnum, 'group_root': group_root, 'suffixes': suffixes, 'num_ints': image_info['num_ints'], From ed3b99363abe5345558deb1754a66a02c18b60b8 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 3 Mar 2023 17:11:46 -0500 Subject: [PATCH 134/449] Allow empty anomaly form submission --- jwql/website/apps/jwql/forms.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jwql/website/apps/jwql/forms.py b/jwql/website/apps/jwql/forms.py index b1d3276cc..cbf8436d1 100644 --- a/jwql/website/apps/jwql/forms.py +++ b/jwql/website/apps/jwql/forms.py @@ -184,7 +184,9 @@ class InstrumentAnomalySubmitForm(forms.Form): def __init__(self, *args, **kwargs): instrument = kwargs.pop('instrument') super(InstrumentAnomalySubmitForm, self).__init__(*args, **kwargs) - self.fields['anomaly_choices'] = forms.MultipleChoiceField(choices=ANOMALY_CHOICES_PER_INSTRUMENT[instrument], widget=forms.CheckboxSelectMultiple()) + self.fields['anomaly_choices'] = forms.MultipleChoiceField( + choices=ANOMALY_CHOICES_PER_INSTRUMENT[instrument], + widget=forms.CheckboxSelectMultiple(), required=False) self.instrument = instrument def update_anomaly_table(self, rootname, user, anomaly_choices): @@ -202,7 +204,6 @@ def update_anomaly_table(self, rootname, user, anomaly_choices): A list of anomalies that are to be flagged (e.g. ``['snowball', 'crosstalk']``) """ - data_dict = {} data_dict['rootname'] = rootname data_dict['flag_date'] = datetime.datetime.now() From 91804be4ed1bdf3644a8f12c062842f7b53d994b Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 3 Mar 2023 17:15:21 -0500 Subject: [PATCH 135/449] Allow group anomaly report and submission --- jwql/website/apps/jwql/data_containers.py | 38 +++++++++++++---------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 41741e07e..4967d591d 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -51,7 +51,7 @@ from jwql.database.database_interface import load_connection from jwql.edb.engineering_database import get_mnemonic, get_mnemonic_info, mnemonic_inventory from jwql.utils.utils import check_config_for_key, ensure_dir_exists, filesystem_path, filename_parser, get_config -from jwql.utils.constants import MAST_QUERY_LIMIT, MONITORS, THUMBNAIL_LISTFILE, THUMBNAIL_FILTER_LOOK +from jwql.utils.constants import ANOMALIES_PER_INSTRUMENT, MAST_QUERY_LIMIT, MONITORS, THUMBNAIL_LISTFILE, THUMBNAIL_FILTER_LOOK from jwql.utils.constants import EXPOSURE_PAGE_SUFFIX_ORDER, IGNORED_SUFFIXES, INSTRUMENT_SERVICE_MATCH from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_INSTRUMENT_NAMES from jwql.utils.constants import REPORT_KEYS_PER_INSTRUMENT @@ -329,7 +329,7 @@ def get_available_suffixes(all_suffixes, return_untracked=True): return suffixes -def get_current_flagged_anomalies(rootname, instrument): +def get_current_flagged_anomalies(rootname, instrument, n_match=1): """Return a list of currently flagged anomalies for the given ``rootname`` @@ -345,18 +345,16 @@ def get_current_flagged_anomalies(rootname, instrument): A list of currently flagged anomalies for the given ``rootname`` (e.g. ``['snowball', 'crosstalk']``) """ + table = getattr(di, '{}Anomaly'.format(JWST_INSTRUMENT_NAMES_MIXEDCASE[instrument.lower()])) + query = di.session.query(table).filter(table.rootname.startswith(rootname)).order_by(table.flag_date.desc()) - table_dict = {} - table_dict[instrument.lower()] = getattr(di, '{}Anomaly'.format(JWST_INSTRUMENT_NAMES_MIXEDCASE[instrument.lower()])) - - table = table_dict[instrument.lower()] - query = di.session.query(table).filter(table.rootname == rootname).order_by(table.flag_date.desc()).limit(1) - - all_records = query.data_frame + all_records = query.data_frame.groupby('rootname').first() + current_anomalies = [] if not all_records.empty: - current_anomalies = [col for col, val in np.sum(all_records, axis=0).items() if val] - else: - current_anomalies = [] + for col, val in all_records.sum(axis=0, numeric_only=True).items(): + # ignore rootname, flag_date, etc + if val == n_match and col in ANOMALIES_PER_INSTRUMENT: + current_anomalies.append(col) return current_anomalies @@ -371,24 +369,30 @@ def get_anomaly_form(request, inst, file_root): inst : str Name of JWST instrument file_root : str - FITS filename of selected image in filesystem + FITS filename of selected image in filesystem. May be a + file or group root name. Returns ------- InstrumentAnomalySubmitForm object form object to be sent with context to template """ + # Check for group root name + file_root_info = RootFileInfo.objects.filter(root_name__startswith=file_root) + file_roots = [rf.root_name for rf in file_root_info] + # Determine current flagged anomalies - current_anomalies = get_current_flagged_anomalies(file_root, inst) + current_anomalies = get_current_flagged_anomalies(file_root, inst, n_match=len(file_roots)) # Create a form instance form = InstrumentAnomalySubmitForm(request.POST or None, instrument=inst.lower(), initial={'anomaly_choices': current_anomalies}) # If this is a POST request and the form is filled out, process the form data - if request.method == 'POST' and 'anomaly_choices' in dict(request.POST): - anomaly_choices = dict(request.POST)['anomaly_choices'] + if request.method == 'POST': + anomaly_choices = dict(request.POST).get('anomaly_choices', []) if form.is_valid(): - form.update_anomaly_table(file_root, 'unknown', anomaly_choices) + for file_root in file_roots: + form.update_anomaly_table(file_root, 'unknown', anomaly_choices) messages.success(request, "Anomaly submitted successfully") else: messages.error(request, "Failed to submit anomaly") From f0d442d2c41273b3c44d8105f7d4ce4e4f8e811d Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 3 Mar 2023 17:38:07 -0500 Subject: [PATCH 136/449] Clarify group buttons on exposure page --- jwql/website/apps/jwql/static/js/jwql.js | 23 +++++++++++++------ .../apps/jwql/templates/view_exposure.html | 6 ++--- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index 516387230..41faf3bbf 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -764,20 +764,21 @@ function toggle_viewed(file_root, base_url) { function toggle_viewed_group(group_root, base_url) { // Toggle the button immediately so user isn't confused var elem = document.getElementById("viewed"); - update_viewed_button(elem.value == "New" ? true : false); + var to_viewed = elem.value.includes('New'); + update_viewed_button(to_viewed, true); elem.disabled=true; // Ajax Call to update RootFileInfo model with "viewed" info $.ajax({ - url: base_url + '/ajax/viewed_group/' + group_root + '/' + elem.value, + url: base_url + '/ajax/viewed_group/' + group_root + '/' + (to_viewed ? 'viewed' : 'new'), success: function(data){ // Update button with actual value (paranoia update, should not yield visible change) - update_viewed_button(data["marked_viewed"]); + update_viewed_button(data["marked_viewed"], true); elem.disabled=false; }, error : function(response) { // If update fails put button back to original state - update_viewed_button(elem.value == "New" ? false : true); + update_viewed_button(!to_viewed, true); elem.disabled=false; } @@ -1333,14 +1334,22 @@ function update_view_explore_link() { } -function update_viewed_button(viewed) { +function update_viewed_button(viewed, group=false) { var elem = document.getElementById("viewed"); if (viewed) { elem.classList.add("btn-outline-primary") - elem.value = "Viewed"; + if (group) { + elem.value = "Viewed Group"; + } else { + elem.value = "Viewed"; + } } else { elem.classList.remove("btn-outline-primary") - elem.value = "New"; + if (group) { + elem.value = "New Group"; + } else { + elem.value = "New"; + } } } diff --git a/jwql/website/apps/jwql/templates/view_exposure.html b/jwql/website/apps/jwql/templates/view_exposure.html index 834bc335f..0fa2dab98 100644 --- a/jwql/website/apps/jwql/templates/view_exposure.html +++ b/jwql/website/apps/jwql/templates/view_exposure.html @@ -86,7 +86,7 @@

    Exposure {{ group_root }}

    -
    Submit Anomaly
    +
    Submit Anomaly for Group
    {% if form.errors %} @@ -119,9 +119,9 @@
    Submit Anomaly
    {% if marked_viewed %} - + {% else %} - + {% endif %} {% if 'rate' in suffixes %} From 874cf7471c7ac0c99e0847ee8388ba9a53042986 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 6 Mar 2023 10:46:54 -0500 Subject: [PATCH 137/449] Don't overwrite anomalies selected for individual files with group form --- jwql/website/apps/jwql/data_containers.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 4967d591d..55b3da000 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -341,7 +341,7 @@ def get_current_flagged_anomalies(rootname, instrument, n_match=1): Returns ------- - current_anomalies : list + current_anomalies : list of str A list of currently flagged anomalies for the given ``rootname`` (e.g. ``['snowball', 'crosstalk']``) """ @@ -392,7 +392,17 @@ def get_anomaly_form(request, inst, file_root): anomaly_choices = dict(request.POST).get('anomaly_choices', []) if form.is_valid(): for file_root in file_roots: - form.update_anomaly_table(file_root, 'unknown', anomaly_choices) + # for a group form submit, add any individual anomalies + # not in the original group set + if len(file_roots) > 1: + file_current = get_current_flagged_anomalies(file_root, inst) + choices = anomaly_choices.copy() + for choice in file_current: + if choice not in current_anomalies: + choices.append(choice) + else: + choices = anomaly_choices + form.update_anomaly_table(file_root, 'unknown', choices) messages.success(request, "Anomaly submitted successfully") else: messages.error(request, "Failed to submit anomaly") From 293ce6b0d9b876e23545cc64dcce5578f091116a Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Mon, 6 Mar 2023 13:58:01 -0500 Subject: [PATCH 138/449] add unit test for data containers --- jwql/tests/test_data_containers.py | 45 ++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/jwql/tests/test_data_containers.py b/jwql/tests/test_data_containers.py index 0807e78d8..c0e472865 100644 --- a/jwql/tests/test_data_containers.py +++ b/jwql/tests/test_data_containers.py @@ -170,6 +170,51 @@ def test_get_thumbnail_by_rootname(): assert preview_images == 'none' +def test_mast_query_by_rootname(): + """Tests the ``mast_query_by_rootname`` function.""" + instrument = 'NIRCam' + rootname1 = 'jw02767002001_02103_00005_nrcb4' + dict_stuff = data_containers.mast_query_by_rootname(instrument, rootname1) + defaults = dict(filter=dict_stuff.get('filter', ''), + detector=dict_stuff.get('detector', ''), + exp_type=dict_stuff.get('exp_type', ''), + read_pat=dict_stuff.get('readpatt', ''), + grating=dict_stuff.get('grating', ''), + patt_num=dict_stuff.get('patt_num', 0), + aperture=dict_stuff.get('apername', ''), + subarray=dict_stuff.get('subarray', ''), + pupil=dict_stuff.get('pupil', '')) + assert isinstance(defaults, dict) + + rootname2 = 'jw02084001001_04103_00001-seg003_nrca3' + dict_stuff = data_containers.mast_query_by_rootname(instrument, rootname2) + defaults = dict(filter=dict_stuff.get('filter', ''), + detector=dict_stuff.get('detector', ''), + exp_type=dict_stuff.get('exp_type', ''), + read_pat=dict_stuff.get('readpatt', ''), + grating=dict_stuff.get('grating', ''), + patt_num=dict_stuff.get('patt_num', 0), + aperture=dict_stuff.get('apername', ''), + subarray=dict_stuff.get('subarray', ''), + pupil=dict_stuff.get('pupil', '')) + assert isinstance(defaults, dict) + + instrument2 = 'FGS' + rootname3 = 'jw01029003001_06201_00001_guider2' + dict_stuff = data_containers.mast_query_by_rootname(instrument2, rootname3) + defaults = dict(filter=dict_stuff.get('filter', ''), + detector=dict_stuff.get('detector', ''), + exp_type=dict_stuff.get('exp_type', ''), + read_pat=dict_stuff.get('readpatt', ''), + grating=dict_stuff.get('grating', ''), + patt_num=dict_stuff.get('patt_num', 0), + aperture=dict_stuff.get('apername', ''), + subarray=dict_stuff.get('subarray', ''), + pupil=dict_stuff.get('pupil', '')) + assert isinstance(defaults, dict) + + + @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') def test_thumbnails_ajax(): """Tests the ``get_thumbnails_ajax`` function.""" From d11a6d3902b784ab48e6692740664622f6c9a92e Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Tue, 7 Mar 2023 09:46:30 -0500 Subject: [PATCH 139/449] Use new django model fields for filter/sort --- jwql/tests/test_data_containers.py | 14 ++++++------- jwql/utils/constants.py | 23 +++++++++++++-------- jwql/website/apps/jwql/api_views.py | 4 ++-- jwql/website/apps/jwql/data_containers.py | 25 +++++++++++------------ 4 files changed, 35 insertions(+), 31 deletions(-) diff --git a/jwql/tests/test_data_containers.py b/jwql/tests/test_data_containers.py index a417c124e..5d4b39ec5 100644 --- a/jwql/tests/test_data_containers.py +++ b/jwql/tests/test_data_containers.py @@ -179,18 +179,18 @@ def test_get_instrument_proposals(): (None, 'New', None, None, None), (None, None, None, 'NRS_MSATA', None), # (None, None, None, None, 'CAL'), # cat_type not implemented yet - (['obsstart'], 'new', 'ascending', None, None), - (['obsstart'], 'new', 'descending', None, None), - (['obsstart'], 'new', 'recent', None, None), + (['expstart'], 'new', 'ascending', None, None), + (['expstart'], 'new', 'descending', None, None), + (['expstart'], 'new', 'recent', None, None), ([], 'viewed', None, None, None), ([], 'new', None, None, None), ([], None, None, None, None), (['proposal', 'obsnum', 'other', - 'prop_id', 'obsstart'], 'viewed', None, None, None), + 'prop_id', 'expstart'], 'viewed', None, None, None), (['proposal', 'obsnum', 'other', - 'prop_id', 'obsstart'], 'new', None, None, None), + 'prop_id', 'expstart'], 'new', None, None, None), (['proposal', 'obsnum', 'other', - 'prop_id', 'obsstart'], None, None, None, None)]) + 'prop_id', 'expstart'], None, None, None, None)]) def test_get_instrument_looks(keys, viewed, sort_as, exp_type, cat_type): """Tests the ``get_instrument_looks`` function.""" @@ -224,7 +224,7 @@ def test_get_instrument_looks(keys, viewed, sort_as, exp_type, cat_type): if sort_as == 'ascending': assert last_file['root_name'] > first_file['root_name'] elif sort_as == 'recent': - assert last_file['obsstart'] < first_file['obsstart'] + assert last_file['expstart'] < first_file['expstart'] else: assert last_file['root_name'] < first_file['root_name'] diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index c0c5c7deb..befd5148f 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -470,15 +470,20 @@ class QUERY_CONFIG_KEYS: 'NRSRAPIDD2', 'NRSRAPIDD6']} -REPORT_KEYS_PER_INSTRUMENT = {'fgs': ['proposal', 'obsnum', 'exptypes', - 'obsstart', 'obsend', 'viewed'], - 'miri': ['proposal', 'obsnum', 'exptypes', - 'obsstart', 'obsend', 'viewed'], - 'nircam': ['proposal', 'obsnum', 'exptypes', - 'obsstart', 'obsend', 'viewed'], - 'niriss': ['proposal', 'obsnum', 'exptypes', - 'obsstart', 'obsend', 'viewed'], - 'nirspec': ['exptypes', 'viewed']} +REPORT_KEYS_PER_INSTRUMENT = {'fgs': ['proposal', 'obsnum', 'exp_type', + 'expstart', 'filter', 'aperture', + 'detector', 'subarray', 'viewed'], + 'miri': ['proposal', 'obsnum', 'exp_type', + 'expstart', 'filter', 'aperture', + 'detector', 'subarray', 'viewed'], + 'nircam': ['proposal', 'obsnum', 'exp_type', + 'expstart', 'filter', 'aperture', + 'detector', 'subarray', 'viewed'], + 'niriss': ['proposal', 'obsnum', 'exp_type', + 'expstart', 'filter', 'aperture', + 'detector', 'subarray', 'viewed'], + 'nirspec': ['exp_type', 'filter', 'grating', + 'read_patt_num', 'viewed']} SUBARRAYS_ONE_OR_FOUR_AMPS = ['SUBGRISMSTRIPE64', 'SUBGRISMSTRIPE128', 'SUBGRISMSTRIPE256'] diff --git a/jwql/website/apps/jwql/api_views.py b/jwql/website/apps/jwql/api_views.py index aeb32371f..e5303e7a0 100644 --- a/jwql/website/apps/jwql/api_views.py +++ b/jwql/website/apps/jwql/api_views.py @@ -144,8 +144,8 @@ def instrument_looks(request, inst, status=None): 'Viewed' indicates whether an observation is new or has been reviewed for QA. In addition to 'filename', and 'viewed', observation - descriptors included in the table are currently 'proposal', 'obsnum', - 'number_of_files', 'exptypes', 'obsstart', 'obsend'. + descriptors from the Django models may be added to the table. Keys + are specified by instrument in the REPORT_KEYS_PER_INSTRUMENT constant. Parameters ---------- diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 04bf50457..f98ead0f1 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -168,8 +168,8 @@ def filter_root_files(instrument=None, proposal=None, obsnum=None, sort_as=None, Observation number to match. sort_as : {'ascending', 'descending', 'recent', 'oldest'}, optional Sorting method for output table. Ascending and descending - options refer to root file name; recent and oldest sort by observation - start. + options refer to root file name; recent and oldest sort by exposure + start time. look : {'new', 'viewed'}, optional If set to None, all viewed values are returned. If set to 'viewed', only viewed data is returned. If set to 'new', only @@ -188,7 +188,6 @@ def filter_root_files(instrument=None, proposal=None, obsnum=None, sort_as=None, """ # standardize input - # TODO: update when more fields are available # get desired filters filter_kwargs = dict() if instrument is not None and str(instrument).strip().lower() != 'all': @@ -201,13 +200,11 @@ def filter_root_files(instrument=None, proposal=None, obsnum=None, sort_as=None, if look is not None and str(look).strip().lower() != 'all': filter_kwargs['viewed'] = (str(look).lower() == 'viewed') if exp_type is not None and str(exp_type).strip().lower() != 'all': - filter_kwargs['obsnum__exptypes__icontains'] = exp_type + filter_kwargs['exp_type__iexact'] = exp_type if cat_type is not None and str(cat_type).strip().lower() != 'all': - # filter_kwargs['obsnum__proposal__cat_type__iexact'] = cat_type - # not yet implemented in proposal table - pass + filter_kwargs['obsnum__proposal__category__iexact'] = cat_type if detector is not None and str(detector).strip().lower() != 'all': - filter_kwargs['root_name__icontains'] = detector + filter_kwargs['detector__iexact'] = detector # get file info by instrument from local model root_file_info = RootFileInfo.objects.filter(**filter_kwargs) @@ -218,9 +215,9 @@ def filter_root_files(instrument=None, proposal=None, obsnum=None, sort_as=None, if sort_as == 'ascending': root_file_info = root_file_info.order_by('root_name') elif sort_as == 'recent': - root_file_info = root_file_info.order_by('-obsnum__obsstart', 'root_name') + root_file_info = root_file_info.order_by('-expstart', 'root_name') elif sort_as == 'oldest': - root_file_info = root_file_info.order_by('obsnum__obsstart', 'root_name') + root_file_info = root_file_info.order_by('expstart', 'root_name') return root_file_info @@ -262,11 +259,11 @@ def create_archived_proposals_context(inst): # Get a set of all exposure types used in the observations associated with this proposal exp_types = [exposure_type for observation in all_entries for exposure_type in observation.exptypes.split(',')] - exp_types = sorted(list(set(exp_types))) + exp_types = sorted(set(exp_types)) # Get all proposals based on category type proposals_by_category = get_proposals_by_category(inst) - unique_cat_types = list(set(proposals_by_category.values())) + unique_cat_types = sorted(set(proposals_by_category.values())) # The naming conventions for dropdown_menus are tightly coupled with the code, this should be changed down the line. dropdown_menus = {'look': THUMBNAIL_FILTER_LOOK, @@ -290,7 +287,9 @@ def create_archived_proposals_context(inst): prop_filecount = [entry.number_of_files for entry in prop_entries] total_files.append(sum(prop_filecount)) - # In order to know if a proposal contains all observations that are entirely viewed, check for at least one existing viewed=False in RootFileInfo + # In order to know if a proposal contains all observations that + # are entirely viewed, check for at least one existing + # viewed=False in RootFileInfo unviewed_root_file_infos = RootFileInfo.objects.filter(instrument=inst, proposal=proposal_num, viewed=False) proposal_viewed.append("Viewed" if unviewed_root_file_infos.count() == 0 else "New") From 3b1d6a19b6cc309ba052df0f03da033dcf1b7845 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Tue, 7 Mar 2023 10:26:26 -0500 Subject: [PATCH 140/449] Dont crash whole script on MAST connection Issue --- jwql/website/apps/jwql/data_containers.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 53c2faa88..3375e45cb 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -716,8 +716,13 @@ def mast_query_by_rootname(instrument, rootname): query_filters.append({'paramName': 'detector', 'values': [detector.upper()]}) params = {'columns': '*', 'filters': query_filters} - response = Mast.service_request_async(service, params) - result = response[0].json() + try: + response = Mast.service_request_async(service, params) + result = response[0].json() + except Exception as e: + logging.error("Mast.service_request_async- {} - {}".format(file_set_name, e)) + result['data'] = [] + retval = {} if result['data'] == []: print("WARNING: no data for {}".format(rootname)) From 8e41d7d94aa6a69344b3cee592f8a7e636ff1460 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Tue, 7 Mar 2023 10:33:44 -0500 Subject: [PATCH 141/449] Tidy up repeated loops --- jwql/website/apps/jwql/data_containers.py | 32 ++++++++++++++--------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index f98ead0f1..a62577e5e 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -1838,9 +1838,6 @@ def thumbnails_ajax(inst, proposal, obs_num=None): data_dict : dict Dictionary of data needed for the ``thumbnails`` template """ - # TODO - tap call in get_rootnames sometimes fails. - # can eliminate, using the existing filenames instead? - # generate the list of all obs of the proposal here, so that the list can be # properly packaged up and sent to the js scripts. but to do this, we need to call # get_rootnames_for_instrument_proposal, which is largely repeating the work done by @@ -1895,15 +1892,17 @@ def thumbnails_ajax(inst, proposal, obs_num=None): 'visit_group': rootname[14:16], 'group_root': rootname[:26]} - # TODO: the following comprehensions loop over all file names - # several times - should be combined, and maybe make the - # filename loop primary - # Get list of available filenames and exposure start times. All files with a given # rootname will have the same exposure start time, so just keep the first. - available_files = [item for item in filenames if rootname in item] - exp_start = [expstart for fname, expstart in zip(filenames, columns['expstart']) if rootname in fname][0] - exp_type = [exp_type for fname, exp_type in zip(filenames, columns['exp_type']) if rootname in fname][0] + available_files = [] + exp_start = None + exp_type = None + for i, item in enumerate(filenames): + if rootname in item: + available_files.append(item) + if exp_start is None: + exp_start = columns['expstart'][i] + exp_type = columns['exp_type'][i] exp_types.add(exp_type) # Viewed is stored by rootname in the Model db. Save it with the data_dict @@ -2028,10 +2027,17 @@ def thumbnails_date_range_ajax(inst, observations, inclusive_start_time_mjd, exc # Get list of available filenames and exposure start times. All files with a given # rootname will have the same exposure start time, so just keep the first. - available_files = [item for item in filenames if rootname in item] - exp_start = [expstart for fname, expstart in zip(filenames, columns['expstart']) if rootname in fname][0] + available_files = [] + exp_start = None + exp_type = None + for i, item in enumerate(filenames): + if rootname in item: + available_files.append(item) + if exp_start is None: + exp_start = columns['expstart'][i] + exp_type = columns['exp_type'][i] + if exp_start >= inclusive_start_time_mjd and exp_start < exclusive_stop_time_mjd: - exp_type = [exp_type for fname, exp_type in zip(filenames, columns['exp_type']) if rootname in fname][0] exp_types.add(exp_type) # Viewed is stored by rootname in the Model db. Save it with the data_dict # THUMBNAIL_FILTER_LOOK is boolean accessed according to a viewed flag From 57e9dfdc7a417423f60d1a4b1b4c189cb37570fe Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Tue, 7 Mar 2023 14:05:05 -0500 Subject: [PATCH 142/449] Detector layout for exposure page --- .../apps/jwql/templates/view_exposure.html | 119 +++++++++++++----- 1 file changed, 90 insertions(+), 29 deletions(-) diff --git a/jwql/website/apps/jwql/templates/view_exposure.html b/jwql/website/apps/jwql/templates/view_exposure.html index 0fa2dab98..1fe5d9278 100644 --- a/jwql/website/apps/jwql/templates/view_exposure.html +++ b/jwql/website/apps/jwql/templates/view_exposure.html @@ -18,7 +18,23 @@

    Exposure {{ group_root }}

    Observation:
    Visit:
    -
    + + +
    + + {% if group_root in group_root_list %} + {% set index = group_root_list.index(group_root) %} + {% if index != 0 %} + < Previous + {% endif %} + + {% if index != group_root_list|length - 1 %} + Next > + {% endif %} + {% endif %} + +
    +
    View File Type: @@ -46,42 +62,87 @@

    Exposure {{ group_root }}

    - -
    - {% for detector in detectors %} - - - - - - {% endfor %} - +
    +

    Integration: /

    - -
    - - {% if group_root in group_root_list %} - {% set index = group_root_list.index(group_root) %} - {% if index != 0 %} - < Previous - {% endif %} - - {% if index != group_root_list|length - 1 %} - Next > - {% endif %} - {% endif %} - -
    +
    + {% if inst == 'NIRCam' %} +
    + + {% for detector1 in ['nrca2', 'nrca4', 'nrcb3', 'nrcb1'] %} + {% if detector1 in detectors %} +
    + + + + + +
    + {% endif %} + {% endfor %} +
    +
    + + {% for detector2 in ['nrca1', 'nrca3', 'nrcb4', 'nrcb2'] %} + {% if detector2 in detectors %} +
    + + + + + +
    + {% endif %} + {% endfor %} +
    +
    + + {% for detector3 in ['nrcalong', 'nrcblong'] %} + {% if detector3 in detectors %} +
    + + + + + +
    + {% endif %} + {% endfor %} +
    + {% else %} +
    + + {% for detector in detectors %} +
    + + + + + +
    + {% endfor %} +
    + {% endif %} +
    From 15815c00cd31dc5f328b68c85140096369615da2 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Wed, 8 Mar 2023 13:46:40 -0500 Subject: [PATCH 143/449] only mast_query_by_rootname on update/create --- .../apps/jwql/archive_database_update.py | 48 +++++++++---------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/jwql/website/apps/jwql/archive_database_update.py b/jwql/website/apps/jwql/archive_database_update.py index cee2140fa..37ca6712c 100755 --- a/jwql/website/apps/jwql/archive_database_update.py +++ b/jwql/website/apps/jwql/archive_database_update.py @@ -319,34 +319,32 @@ def update_database_table(update, instrument, prop, obs, thumbnail, obsfiles, ty # Get all unsaved root names in the Observation to store in the database nr_files_created = 0 for file in obsfiles: - defaults_dict = mast_query_by_rootname(instrument, file) try: - # NOTE: If implementing a new field, add to this dict and run script with 'update' flag - defaults = dict(filter=defaults_dict.get('filter', ''), - detector=defaults_dict.get('detector', ''), - exp_type=defaults_dict.get('exp_type', ''), - read_patt=defaults_dict.get('readpatt', ''), - grating=defaults_dict.get('grating', ''), - read_patt_num=defaults_dict.get('patt_num', 0), - aperture=defaults_dict.get('apername', ''), - subarray=defaults_dict.get('subarray', ''), - pupil=defaults_dict.get('pupil', ''), - expstart=defaults_dict.get('expstart', 0.0)) - if update: - root_file_info_instance, rfi_created = RootFileInfo.objects.update_or_create(root_name=file, - instrument=instrument, - obsnum=obs_instance, - proposal=prop, - defaults=defaults) - else: - root_file_info_instance, rfi_created = RootFileInfo.objects.get_or_create(root_name=file, - instrument=instrument, - obsnum=obs_instance, - proposal=prop, - defaults=defaults) + root_file_info_instance, rfi_created = RootFileInfo.objects.get_or_create(root_name=file, + instrument=instrument, + obsnum=obs_instance, + proposal=prop) + if update or rfi_created: + # Updating defaults only on update or creation to prevent call to mast_query_by_rootname on every file name. + defaults_dict = mast_query_by_rootname(instrument, file) + + # NOTE: If implementing a new field, add to this dict and run script with 'update' flag + defaults = dict(filter=defaults_dict.get('filter', ''), + detector=defaults_dict.get('detector', ''), + exp_type=defaults_dict.get('exp_type', ''), + read_patt=defaults_dict.get('readpatt', ''), + grating=defaults_dict.get('grating', ''), + read_patt_num=defaults_dict.get('patt_num', 0), + aperture=defaults_dict.get('apername', ''), + subarray=defaults_dict.get('subarray', ''), + pupil=defaults_dict.get('pupil', ''), + expstart=defaults_dict.get('expstart', 0.0)) + + for key, value in defaults.items(): + setattr(root_file_info_instance, key, value) + root_file_info_instance.save() if rfi_created: nr_files_created += 1 - root_file_info_instance.save() except Exception as e: logging.warning(f'\tError {e} was raised') logging.warning(f'\tError with root_name: {file} inst: {instrument} obsnum: {obs_instance} proposal: {prop}') From 48d060556f07d8bdd8b12a0d8c95003ec151286c Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Wed, 8 Mar 2023 15:31:44 -0500 Subject: [PATCH 144/449] reduce logging --- jwql/website/apps/jwql/archive_database_update.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/jwql/website/apps/jwql/archive_database_update.py b/jwql/website/apps/jwql/archive_database_update.py index 37ca6712c..4862396f9 100755 --- a/jwql/website/apps/jwql/archive_database_update.py +++ b/jwql/website/apps/jwql/archive_database_update.py @@ -84,10 +84,8 @@ def get_updates(update_database, inst): # Get list of all files for the given instrument for proposal in all_proposals: # Get lists of all public and proprietary files for the program - logging.info(f'Working on proposal {proposal}') filenames_public, metadata_public, filenames_proprietary, metadata_proprietary = get_all_possible_filenames_for_proposal(inst, proposal) # Find the location in the filesystem for all files - logging.info('Getting all filenames and locating in the filesystem') filepaths_public = files_in_filesystem(filenames_public, 'public') filepaths_proprietary = files_in_filesystem(filenames_proprietary, 'proprietary') filenames = filepaths_public + filepaths_proprietary @@ -109,7 +107,6 @@ def get_updates(update_database, inst): if 'stage_3' not in filename_dict['filename_type']: rootnames.append(rootname) - logging.info(f'Identified {len(rootnames)} files for the proposal.') if len(filenames) > 0: # Gather information about the proposals for the given instrument @@ -144,7 +141,6 @@ def get_updates(update_database, inst): obsfiles = [f for f in rootnames if propobs in f] # Update the appropriate database table - logging.info(f'Updating database for {inst}, Proposal {proposal}, Observation {obsnum}') update_database_table(update_database, inst, proposal, obsnum, proposal_info['thumbnail_paths'][0], obsfiles, exp_types, starting_date, latest_date, proposal_category) @@ -266,15 +262,11 @@ def update_database_table(update, instrument, prop, obs, thumbnail, obsfiles, ty archive_instance, archive_created = Archive.objects.get_or_create(instrument=instrument) if archive_created: logging.info(f'No existing entries for Archive: {instrument}. Creating.') - else: - logging.info('Existing Archive entry found.') # Check to see if the required Proposal entry exists, and create it if it doesn't prop_instance, prop_created = Proposal.objects.get_or_create(prop_id=prop, archive=archive_instance) if prop_created: logging.info(f'No existing entries for Proposal: {prop}. Creating.') - else: - logging.info('Existing Proposal entry found.') # Update the proposal instance with the thumbnail path prop_instance.thumbnail_path = thumbnail @@ -303,9 +295,6 @@ def update_database_table(update, instrument, prop, obs, thumbnail, obsfiles, ty 'Updating number of files, start/end dates, and exp_type list')) obs_instance.exptypes = ','.join(types) else: - logging.info(f'Existing Observation entry found, containing exptypes: {obs_instance.exptypes}.') - logging.info((f'Found existing entry for instrument {instrument}, Proposal {prop}, Observation {obs}, ' - f'containing exptypes: {obs_instance.exptypes} Updating number of files, start/end dates, and exp_type list')) if obs_instance.exptypes == '': obs_instance.exptypes = ','.join(types) else: @@ -348,7 +337,8 @@ def update_database_table(update, instrument, prop, obs, thumbnail, obsfiles, ty except Exception as e: logging.warning(f'\tError {e} was raised') logging.warning(f'\tError with root_name: {file} inst: {instrument} obsnum: {obs_instance} proposal: {prop}') - logging.info(f'Created {nr_files_created} root_file_info entries for: {instrument} - proposal:{prop} - obs:{obs}') + if nr_files_created > 0: + logging.info(f'Created {nr_files_created} root_file_info entries for: {instrument} - proposal:{prop} - obs:{obs}') if __name__ == '__main__': From d59cb044729a3baa96459f99e77c9cb421ae32e9 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Wed, 8 Mar 2023 16:12:11 -0500 Subject: [PATCH 145/449] only allow one instance at a time --- .../apps/jwql/archive_database_update.py | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/jwql/website/apps/jwql/archive_database_update.py b/jwql/website/apps/jwql/archive_database_update.py index 4862396f9..c172a81af 100755 --- a/jwql/website/apps/jwql/archive_database_update.py +++ b/jwql/website/apps/jwql/archive_database_update.py @@ -40,6 +40,8 @@ import numpy as np import django +from jwql.utils.protect_module import lock_module + # These lines are needed in order to use the Django models in a standalone # script (as opposed to code run as a result of a webpage request). If these # lines are not run, the script will crash when attempting to import the @@ -341,13 +343,16 @@ def update_database_table(update, instrument, prop, obs, thumbnail, obsfiles, ty logging.info(f'Created {nr_files_created} root_file_info entries for: {instrument} - proposal:{prop} - obs:{obs}') -if __name__ == '__main__': - update_database = False - for i in range(1, len(sys.argv)): - if sys.argv[i].lower() == 'update': - update_database = True +@lock_module +def protected_code(update_database): + """Protected code ensures only 1 instance of module will run at any given time + Parameters + ---------- + update_database : bool + If True, any existing rootfileinfo models are overwritten + """ module = os.path.basename(__file__).strip('.py') start_time, log_file = initialize_instrument_monitor(module) @@ -355,3 +360,12 @@ def update_database_table(update, instrument, prop, obs, thumbnail, obsfiles, ty for instrument in instruments: get_updates(update_database, instrument) create_archived_proposals_context(instrument) + + +if __name__ == '__main__': + + update_database = False + for i in range(1, len(sys.argv)): + if sys.argv[i].lower() == 'update': + update_database = True + protected_code(update_database) From 8e70353fe83889025d6f7ff053c2718d31f48e0e Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 9 Mar 2023 08:57:14 -0500 Subject: [PATCH 146/449] make vo searches robust against updated inst names --- jwql/utils/utils.py | 4 ++-- jwql/website/apps/jwql/data_containers.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index 49fe6cc03..7643403a5 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -561,8 +561,8 @@ def get_rootnames_for_instrument_proposal(instrument, proposal): rootnames : list List of rootnames for the given instrument and proposal number """ - tap_service = vo.dal.TAPService("http://vao.stsci.edu/caomtap/tapservice.aspx") - tap_results = tap_service.search(f"select observationID from dbo.CaomObservation where collection='JWST' and maxLevel=2 and insName like '{instrument.lower()}' and prpID='{int(proposal)}'") + tap_service = vo.dal.TAPService("https://vao.stsci.edu/caomtap/tapservice.aspx") + tap_results = tap_service.search(f"select observationID from dbo.CaomObservation where collection='JWST' and maxLevel=2 and insName like '{instrument.lower()}%' and prpID='{int(proposal)}'") prop_table = tap_results.to_table() rootnames = prop_table['observationID'].data return rootnames.compressed() diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index b8a559442..aa6833ca5 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -958,8 +958,8 @@ def get_instrument_proposals(instrument): inst_proposals : list List of proposals for the given instrument """ - tap_service = vo.dal.TAPService("http://vao.stsci.edu/caomtap/tapservice.aspx") - tap_results = tap_service.search(f"select distinct proposal_id from dbo.ObsPointing where obs_collection='JWST' and calib_level>0 and instrument_name like '{instrument.lower()}'") + tap_service = vo.dal.TAPService("https://vao.stsci.edu/caomtap/tapservice.aspx") + tap_results = tap_service.search(f"select distinct proposal_id from dbo.ObsPointing where obs_collection='JWST' and calib_level>0 and instrument_name like '{instrument.lower()}%'") prop_table = tap_results.to_table() proposals = prop_table['proposal_id'].data inst_proposals = sorted(proposals.compressed(), reverse=True) @@ -1086,7 +1086,7 @@ def get_rootnames_for_proposal(proposal): rootnames : list List of rootnames for the given instrument and proposal number """ - tap_service = vo.dal.TAPService("http://vao.stsci.edu/caomtap/tapservice.aspx") + tap_service = vo.dal.TAPService("https://vao.stsci.edu/caomtap/tapservice.aspx") tap_results = tap_service.search(f"select observationID from dbo.CaomObservation where collection='JWST' and maxLevel=2 and prpID='{int(proposal)}'") prop_table = tap_results.to_table() rootnames = prop_table['observationID'].data From 60c46729a38e06842f5e4d7d28a3e8f745d8e57b Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Thu, 9 Mar 2023 11:52:25 -0500 Subject: [PATCH 147/449] Remove accidental migration commit --- .../apps/jwql/migrations/0001_initial.py | 116 ------------------ jwql/website/apps/jwql/migrations/__init__.py | 0 2 files changed, 116 deletions(-) delete mode 100644 jwql/website/apps/jwql/migrations/0001_initial.py delete mode 100644 jwql/website/apps/jwql/migrations/__init__.py diff --git a/jwql/website/apps/jwql/migrations/0001_initial.py b/jwql/website/apps/jwql/migrations/0001_initial.py deleted file mode 100644 index 079b24fe5..000000000 --- a/jwql/website/apps/jwql/migrations/0001_initial.py +++ /dev/null @@ -1,116 +0,0 @@ -# Generated by Django 3.1.8 on 2023-03-06 16:05 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ] - - operations = [ - migrations.CreateModel( - name='Archive', - fields=[ - ('instrument', models.CharField(help_text='Instrument name', max_length=7, primary_key=True, serialize=False)), - ], - options={ - 'ordering': ['instrument'], - }, - ), - migrations.CreateModel( - name='Observation', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('obsnum', models.CharField(help_text='Observation number, as a 3 digit string', max_length=3)), - ('number_of_files', models.IntegerField(default=0, help_text='Number of files in the proposal')), - ('obsstart', models.FloatField(default=0.0, help_text='Time of the beginning of the observation in MJD')), - ('obsend', models.FloatField(default=0.0, help_text='Time of the end of the observation in MJD')), - ('exptypes', models.CharField(default='', help_text='Comma-separated list of exposure types', max_length=100)), - ], - options={ - 'ordering': ['-obsnum'], - }, - ), - migrations.CreateModel( - name='RootFileInfo', - fields=[ - ('instrument', models.CharField(help_text='Instrument name', max_length=7)), - ('proposal', models.CharField(help_text='5-digit proposal ID string', max_length=5)), - ('root_name', models.TextField(max_length=300, primary_key=True, serialize=False)), - ('viewed', models.BooleanField(default=False)), - ('filter', models.CharField(blank=True, default='', help_text='Instrument name', max_length=7, null=True)), - ('aperture', models.CharField(blank=True, default='', help_text='Aperture', max_length=40, null=True)), - ('detector', models.CharField(blank=True, default='', help_text='Detector', max_length=40, null=True)), - ('read_patt_num', models.IntegerField(default=0, help_text='Read Pattern Number')), - ('read_patt', models.CharField(blank=True, default='', help_text='Read Pattern', max_length=40, null=True)), - ('grating', models.CharField(blank=True, default='', help_text='Grating', max_length=40, null=True)), - ('subarray', models.CharField(blank=True, default='', help_text='Subarray', max_length=40, null=True)), - ('pupil', models.CharField(blank=True, default='', help_text='Pupil', max_length=40, null=True)), - ('exp_type', models.CharField(blank=True, default='', help_text='Exposure Type', max_length=40, null=True)), - ('expstart', models.FloatField(default=0.0, help_text='Exposure Start Time')), - ('obsnum', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jwql.observation')), - ], - options={ - 'ordering': ['-root_name'], - }, - ), - migrations.CreateModel( - name='Anomalies', - fields=[ - ('root_file_info', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='jwql.rootfileinfo')), - ('cosmic_ray_shower', models.BooleanField(default=False)), - ('diffraction_spike', models.BooleanField(default=False)), - ('excessive_saturation', models.BooleanField(default=False)), - ('guidestar_failure', models.BooleanField(default=False)), - ('persistence', models.BooleanField(default=False)), - ('crosstalk', models.BooleanField(default=False)), - ('data_transfer_error', models.BooleanField(default=False)), - ('ghost', models.BooleanField(default=False)), - ('snowball', models.BooleanField(default=False)), - ('column_pull_up', models.BooleanField(default=False)), - ('column_pull_down', models.BooleanField(default=False)), - ('dominant_msa_leakage', models.BooleanField(default=False)), - ('dragons_breath', models.BooleanField(default=False)), - ('mrs_glow', models.BooleanField(default=False)), - ('mrs_zipper', models.BooleanField(default=False)), - ('internal_reflection', models.BooleanField(default=False)), - ('optical_short', models.BooleanField(default=False)), - ('row_pull_up', models.BooleanField(default=False)), - ('row_pull_down', models.BooleanField(default=False)), - ('lrs_contamination', models.BooleanField(default=False)), - ('tree_rings', models.BooleanField(default=False)), - ('scattered_light', models.BooleanField(default=False)), - ('claws', models.BooleanField(default=False)), - ('wisps', models.BooleanField(default=False)), - ('tilt_event', models.BooleanField(default=False)), - ('light_saber', models.BooleanField(default=False)), - ('other', models.BooleanField(default=False)), - ], - options={ - 'ordering': ['-root_file_info'], - }, - ), - migrations.CreateModel( - name='Proposal', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('prop_id', models.CharField(help_text='5-digit proposal ID string', max_length=5)), - ('thumbnail_path', models.CharField(default='', help_text='Path to the proposal thumbnail', max_length=100)), - ('category', models.CharField(default='', help_text='Category Type', max_length=10)), - ('archive', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jwql.archive')), - ], - options={ - 'ordering': ['-prop_id'], - 'unique_together': {('prop_id', 'archive')}, - }, - ), - migrations.AddField( - model_name='observation', - name='proposal', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jwql.proposal'), - ), - ] diff --git a/jwql/website/apps/jwql/migrations/__init__.py b/jwql/website/apps/jwql/migrations/__init__.py deleted file mode 100644 index e69de29bb..000000000 From b9ae336aa28cac686c91ab39a69c07f2d7c66915 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 9 Mar 2023 13:28:16 -0500 Subject: [PATCH 148/449] Set NaN pixels to zero --- jwql/utils/preview_image.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/jwql/utils/preview_image.py b/jwql/utils/preview_image.py index 5e173bab6..4ba1de914 100755 --- a/jwql/utils/preview_image.py +++ b/jwql/utils/preview_image.py @@ -576,6 +576,11 @@ def make_image(self, max_img_size=8.0, create_thumbnail=False): # Find signal limits for the display minval, maxval = self.find_limits(frame) + # Set NaN values to zero, so that those pixels + # do not appear as big white splotches in the jpgs + # after matplotlib downsamples/averages + frame = nan_to_zero(frame) + # Create preview image matplotlib object indir, infile = os.path.split(self.file) suffix = '_integ{}.{}'.format(i, self.output_format) @@ -757,3 +762,10 @@ def expand_for_i2d(array, xdim, ydim): return new_array_x else: return array + +def nan_to_zero(image): + """Set any pixels with a value of NaN to zero + """ + nan = np.isnan(image) + image[nan] = 0 + return image From 41840c84667c36da182a4e69a4f30b6b550200d2 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 9 Mar 2023 15:19:38 -0500 Subject: [PATCH 149/449] Add missing docstring --- jwql/utils/preview_image.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/jwql/utils/preview_image.py b/jwql/utils/preview_image.py index 4ba1de914..f86c10272 100755 --- a/jwql/utils/preview_image.py +++ b/jwql/utils/preview_image.py @@ -765,6 +765,16 @@ def expand_for_i2d(array, xdim, ydim): def nan_to_zero(image): """Set any pixels with a value of NaN to zero + + Parameters + ---------- + image : numpy.ndarray + Array from which NaNs will be removed + + Returns + ------- + image : numpy.ndarray + Input array with NaNs changed to zero """ nan = np.isnan(image) image[nan] = 0 From d2fdc53e3a2284aadf7e15ef93151504d70ac456 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Thu, 9 Mar 2023 17:19:03 -0500 Subject: [PATCH 150/449] Add pytest-mock to requirements --- environment_python_3_10.yml | 1 + environment_python_3_8.yml | 1 + environment_python_3_9.yml | 1 + requirements.txt | 1 + setup.py | 1 + 5 files changed, 5 insertions(+) diff --git a/environment_python_3_10.yml b/environment_python_3_10.yml index 9aca9d74b..080015f98 100644 --- a/environment_python_3_10.yml +++ b/environment_python_3_10.yml @@ -44,6 +44,7 @@ dependencies: - psycopg2=2.9.3 - pytest=7.2.1 - pytest-cov=4.0.0 + - pytest-mock=3.10.0 - python=3.10.9 - pyyaml=6.0 - redis diff --git a/environment_python_3_8.yml b/environment_python_3_8.yml index dbe97c537..10f1a8f2c 100644 --- a/environment_python_3_8.yml +++ b/environment_python_3_8.yml @@ -44,6 +44,7 @@ dependencies: - psycopg2=2.9.3 - pytest=7.2.1 - pytest-cov=4.0.0 + - pytest-mock=3.10.0 - python=3.8.16 - pyyaml=6.0 - redis diff --git a/environment_python_3_9.yml b/environment_python_3_9.yml index fc0d2a424..4c7236d3a 100644 --- a/environment_python_3_9.yml +++ b/environment_python_3_9.yml @@ -44,6 +44,7 @@ dependencies: - psycopg2=2.9.3 - pytest=7.2.1 - pytest-cov=4.0.0 + - pytest-mock=3.10.0 - python=3.9.16 - pyyaml=6.0 - redis diff --git a/requirements.txt b/requirements.txt index 6ff5103ff..799bdbf27 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,6 +23,7 @@ pysiaf==0.19.1 pysqlite3==0.5.0 pytest==7.2.1 pytest-cov==4.0.0 +pytest-mock==3.10.0 pyvo==1.4 pyyaml==6.0 redis==4.5.1 diff --git a/setup.py b/setup.py index e9e046e76..d3d820a23 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,7 @@ 'pysiaf', 'pytest', 'pytest-cov', + 'pytest-mock', 'pyvo', 'scipy', 'sphinx', From 1643c92278f87ed29232003c19ed8dfc19aef786 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Thu, 9 Mar 2023 17:25:10 -0500 Subject: [PATCH 151/449] Unit tests for data containers --- jwql/tests/resources.py | 46 +++++++ jwql/tests/test_data_containers.py | 147 +++++++++++++++++++--- jwql/website/apps/jwql/bokeh_dashboard.py | 3 + jwql/website/apps/jwql/data_containers.py | 15 ++- 4 files changed, 192 insertions(+), 19 deletions(-) create mode 100644 jwql/tests/resources.py diff --git a/jwql/tests/resources.py b/jwql/tests/resources.py new file mode 100644 index 000000000..7d8ba0794 --- /dev/null +++ b/jwql/tests/resources.py @@ -0,0 +1,46 @@ +"""Resources for unit tests. + +Authors +------- + + - Melanie Clarke + +Use +--- + + These structures can be imported in unit tests and used to + mock various software functionality. +""" + +import pandas as pd + + +class MockAnomalyQuery(object): + """Mock a SQLAlchemy query on an anomaly table.""" + def __init__(self, group_record=False): + records = [{'rootname': 'jw02589006001_04101_00001-seg001_nrs1', + 'persistence': True, 'crosstalk': True, + 'ghost': False}] + if group_record: + records.append( + {'rootname': 'jw02589006001_04101_00001-seg001_nrs2', + 'persistence': True, 'crosstalk': False, 'ghost': False}) + self.data_frame = pd.DataFrame(records) + + def filter(self, filter_val): + return self + + def order_by(self, sort_val): + return self + + +class MockSessionGroupAnomaly(object): + """Mock a SQLAlchemy session for an anomaly query on a group.""" + def query(self, table): + return MockAnomalyQuery(group_record=True) + + +class MockSessionFileAnomaly(object): + """Mock a SQLAlchemy session for an anomaly query on a file.""" + def query(self, table): + return MockAnomalyQuery(group_record=False) diff --git a/jwql/tests/test_data_containers.py b/jwql/tests/test_data_containers.py index 9fd09ad4e..b7e8b63b0 100644 --- a/jwql/tests/test_data_containers.py +++ b/jwql/tests/test_data_containers.py @@ -24,18 +24,91 @@ """ import glob +import json import os +import numpy as np +import pandas as pd import pytest # Skip testing this module if on Github Actions ON_GITHUB_ACTIONS = '/home/runner' in os.path.expanduser('~') or '/Users/runner' in os.path.expanduser('~') from jwql.website.apps.jwql import data_containers +from jwql.tests.resources import MockAnomalyQuery, MockSessionFileAnomaly, MockSessionGroupAnomaly if not ON_GITHUB_ACTIONS: from jwql.utils.utils import get_config +def test_build_table(): + tab = data_containers.build_table('filesystem_general') + assert isinstance(tab, pd.DataFrame) + assert len(tab['date']) > 0 + + +@pytest.mark.parametrize('filter_keys', + [{'instrument': 'NIRSpec', 'proposal': '2589', + 'obsnum': '006', 'look': 'All'}, + {'instrument': 'NIRCam', 'detector': 'NRCBLONG', + 'proposal': '2733', 'obsnum': '001'}, + {'instrument': 'MIRI', 'exp_type': 'MIR_IMAGE', + 'proposal': '1524', 'obsnum': '015'}, + {'instrument': 'FGS', 'cat_type': 'COM', + 'proposal': '1155'} + ]) +def test_filter_root_files(filter_keys): + rfi = data_containers.filter_root_files(**filter_keys) + assert len(rfi) > 0 + assert len(rfi) < 100 + + for key, value in filter_keys.items(): + if str(value).strip().lower() == 'all': + continue + if key == 'cat_type': + rf_test = [rf.obsnum.proposal.category == value for rf in rfi] + else: + rf_test = [str(getattr(rf, key)) == str(value) for rf in rfi] + assert all(rf_test) + + +def test_filter_root_files_sorting(): + filter_keys = {'instrument': 'NIRSpec', 'proposal': '2589', + 'obsnum': '006'} + + rfi = data_containers.filter_root_files(**filter_keys, sort_as='Ascending') + assert len(rfi) > 3 + for i, rf in enumerate(rfi[1:]): + assert rf.root_name > rfi[i].root_name + + rfi = data_containers.filter_root_files(**filter_keys, sort_as='Descending') + for i, rf in enumerate(rfi[1:]): + assert rf.root_name < rfi[i].root_name + + rfi = data_containers.filter_root_files(**filter_keys, sort_as='Recent') + for i, rf in enumerate(rfi[1:]): + assert rf.expstart <= rfi[i].expstart + + rfi = data_containers.filter_root_files(**filter_keys, sort_as='Oldest') + for i, rf in enumerate(rfi[1:]): + assert rf.expstart >= rfi[i].expstart + + +def test_create_archived_proposals_context(tmp_path, mocker): + # write to a temporary directory + mocker.patch.object(data_containers, 'OUTPUT_DIR', str(tmp_path)) + archive_dir = tmp_path / 'archive_page' + os.mkdir(archive_dir) + + data_containers.create_archived_proposals_context('nirspec') + context_file = str(archive_dir / 'NIRSpec_archive_context.json') + assert os.path.isfile(context_file) + + with open(context_file, 'r') as obj: + context = json.load(obj) + assert context['inst'] == 'NIRSpec' + assert context['num_proposals'] > 0 + + def test_get_acknowledgements(): """Tests the ``get_acknowledgements`` function.""" @@ -53,18 +126,64 @@ def test_get_all_proposals(): assert len(proposals) > 0 -@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') +@pytest.mark.parametrize('untracked,input_suffixes,expected', + [(True, [], ([], set())), + (True, ['rate', 'uncal', 'bad'], + (['uncal', 'rate', 'bad'], {'bad'})), + (False, ['rate', 'uncal', 'bad'], + ['uncal', 'rate', 'bad']), + (True, ['rate', 'uncal', 'bad', + 'o006_crfints', 'o001_crf'], + (['uncal', 'rate', 'o001_crf', + 'o006_crfints', 'bad'], {'bad'})), + (False, ['rate', 'uncal', 'bad', + 'o006_crfints', 'o001_crf'], + ['uncal', 'rate', 'o001_crf', + 'o006_crfints', 'bad']), + ]) +def test_get_available_suffixes(untracked, input_suffixes, expected): + result = data_containers.get_available_suffixes( + input_suffixes, return_untracked=untracked) + assert result == expected + + +def test_get_current_flagged_anomalies(mocker): + # get a sample query group with 2 files + rootname = 'jw02589006001_04101_00001-seg001' + instrument = 'NIRSpec' + + # mock a single shared anomaly type + mocker.patch.object(data_containers.di, 'session', MockSessionGroupAnomaly()) + + result = data_containers.get_current_flagged_anomalies( + rootname, instrument, n_match=2) + assert result == ['persistence'] + + # get a sample query for 1 file + rootname = 'jw02589006001_04101_00001-seg001_nrs1' + + # mock two anomalies for this file + mocker.patch.object(data_containers.di, 'session', MockSessionFileAnomaly()) + + result = data_containers.get_current_flagged_anomalies( + rootname, instrument, n_match=1) + assert result == ['persistence', 'crosstalk'] + + def test_get_expstart(): """Tests the ``get_expstart`` function.""" - - expstart = data_containers.get_expstart('NIRCam', 'jw01068001001_02102_00001_nrcb1') + expstart = data_containers.get_expstart('NIRCam', + 'jw01068001001_02102_00001_nrcb1') assert isinstance(expstart, float) + # if mast query failed, it will return 0 + # otherwise, it should have a known value for this file + assert np.isclose(expstart, 59714, atol=1) + -@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') def test_get_filenames_by_instrument(): """Tests the ``get_filenames_by_instrument`` function.""" - + # queries MAST; should not need central storage filepaths = data_containers.get_filenames_by_instrument('NIRCam', '1068') assert isinstance(filepaths, list) assert len(filepaths) > 0 @@ -148,10 +267,10 @@ def test_get_header_info(): assert len(header) > 0 -@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') def test_get_image_info(): """Tests the ``get_image_info`` function.""" - + # requires filesystem for full info, but this basic test + # should pass without central storage image_info = data_containers.get_image_info('jw01068001001_02102_00001_nrcb1') assert isinstance(image_info, dict) @@ -161,16 +280,14 @@ def test_get_image_info(): assert key in image_info -@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') def test_get_instrument_proposals(): """Tests the ``get_instrument_proposals`` function.""" - + # queries MAST, no need for central storage proposals = data_containers.get_instrument_proposals('Fgs') assert isinstance(proposals, list) assert len(proposals) > 0 -@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') @pytest.mark.parametrize('keys,viewed,sort_as,exp_type,cat_type', [(None, None, None, None, None), (None, 'viewed', None, None, None), @@ -246,18 +363,18 @@ def test_get_preview_images_by_rootname(): assert isinstance(preview_images, list) assert len(preview_images) > 0 -@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') + def test_get_proposals_by_category(): """Tests the ``get_proposals_by_category`` function.""" - + # MAST query, no need for central storage proposals_by_category = data_containers.get_proposals_by_category('fgs') assert isinstance(proposals_by_category, dict) assert len(proposals_by_category) > 0 -@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') + def test_get_proposal_info(): """Tests the ``get_proposal_info`` function.""" - + # requires filesystem for full results, but basic test should pass without filepaths = glob.glob(os.path.join(get_config()['filesystem'], 'jw01068', '*.fits')) proposal_info = data_containers.get_proposal_info(filepaths) @@ -271,7 +388,6 @@ def test_get_proposal_info(): @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') def test_get_thumbnails_by_proposal(): """Tests the ``get_thumbnails_by_proposal`` function.""" - preview_images = data_containers.get_thumbnails_by_proposal('01033') assert isinstance(preview_images, list) assert len(preview_images) > 0 @@ -335,7 +451,6 @@ def test_mast_query_by_rootname(): assert isinstance(defaults, dict) - @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') def test_thumbnails_ajax(): """Tests the ``get_thumbnails_ajax`` function.""" diff --git a/jwql/website/apps/jwql/bokeh_dashboard.py b/jwql/website/apps/jwql/bokeh_dashboard.py index 2cd0ba04a..7970fa760 100644 --- a/jwql/website/apps/jwql/bokeh_dashboard.py +++ b/jwql/website/apps/jwql/bokeh_dashboard.py @@ -135,12 +135,15 @@ def disable_scientific_notation(figure): pass +""" +# Currently unused; preserved for reference when moving to bokeh 3 def treemap(df, col, x, y, dx, dy, *, N=100): sub_df = df.nlargest(N, col) normed = normalize_sizes(sub_df[col], dx, dy) blocks = squarify(normed, x, y, dx, dy) blocks_df = pd.DataFrame.from_dict(blocks).set_index(sub_df.index) return sub_df.join(blocks_df, how='left').reset_index() +""" class GeneralDashboard: diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 940d16167..6a22c83ca 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -73,7 +73,7 @@ # Determine if the code is being run as part of a Readthedocs build ON_READTHEDOCS = False -if 'READTHEDOCS' in os.environ: +if 'READTHEDOCS' in os.environ: # pragma: no cover ON_READTHEDOCS = os.environ['READTHEDOCS'] if not ON_GITHUB_ACTIONS and not ON_READTHEDOCS: @@ -764,7 +764,9 @@ def get_expstart(instrument, rootname): return expstart -def get_filenames_by_instrument(instrument, proposal, observation_id=None, restriction='all', query_file=None, query_response=None, other_columns=None): +def get_filenames_by_instrument(instrument, proposal, observation_id=None, + restriction='all', query_file=None, + query_response=None, other_columns=None): """Returns a list of filenames that match the given ``instrument``. Parameters @@ -800,7 +802,9 @@ def get_filenames_by_instrument(instrument, proposal, observation_id=None, restr e.g. 'exptime', and values are lists of the value for each filename. e.g. ['59867.6, 59867.601'] """ if not query_file and not query_response: - result = mast_query_filenames_by_instrument(instrument, proposal, observation_id=observation_id, other_columns=other_columns) + result = mast_query_filenames_by_instrument( + instrument, proposal, observation_id=observation_id, + other_columns=other_columns) elif query_response: result = query_response @@ -1770,6 +1774,11 @@ def text_scrape(prop_id): program_meta['phase_two'] = BeautifulSoup(program_meta['phase_two'], 'html.parser') links = html.findAll('a') + + # check for server timeout: may return empty html + if len(links) == 0: + return program_meta + proposal_type = links[0].contents[0] program_meta['prop_type'] = proposal_type From bfe0ca87b04ec1dc04a4c29b109ef153295dadb5 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Thu, 9 Mar 2023 18:09:20 -0500 Subject: [PATCH 152/449] Performance fix for report download --- jwql/tests/test_data_containers.py | 15 ++++++++------- jwql/utils/constants.py | 8 ++++---- jwql/website/apps/jwql/data_containers.py | 18 ++++-------------- 3 files changed, 16 insertions(+), 25 deletions(-) diff --git a/jwql/tests/test_data_containers.py b/jwql/tests/test_data_containers.py index b7e8b63b0..671d28a0d 100644 --- a/jwql/tests/test_data_containers.py +++ b/jwql/tests/test_data_containers.py @@ -64,10 +64,11 @@ def test_filter_root_files(filter_keys): for key, value in filter_keys.items(): if str(value).strip().lower() == 'all': continue - if key == 'cat_type': - rf_test = [rf.obsnum.proposal.category == value for rf in rfi] + elif key in ['cat_type', 'obsnum']: + # values returned are foreign keys + continue else: - rf_test = [str(getattr(rf, key)) == str(value) for rf in rfi] + rf_test = [str(rf[key]) == str(value) for rf in rfi] assert all(rf_test) @@ -78,19 +79,19 @@ def test_filter_root_files_sorting(): rfi = data_containers.filter_root_files(**filter_keys, sort_as='Ascending') assert len(rfi) > 3 for i, rf in enumerate(rfi[1:]): - assert rf.root_name > rfi[i].root_name + assert rf['root_name'] > rfi[i]['root_name'] rfi = data_containers.filter_root_files(**filter_keys, sort_as='Descending') for i, rf in enumerate(rfi[1:]): - assert rf.root_name < rfi[i].root_name + assert rf['root_name'] < rfi[i]['root_name'] rfi = data_containers.filter_root_files(**filter_keys, sort_as='Recent') for i, rf in enumerate(rfi[1:]): - assert rf.expstart <= rfi[i].expstart + assert rf['expstart'] <= rfi[i]['expstart'] rfi = data_containers.filter_root_files(**filter_keys, sort_as='Oldest') for i, rf in enumerate(rfi[1:]): - assert rf.expstart >= rfi[i].expstart + assert rf['expstart'] >= rfi[i]['expstart'] def test_create_archived_proposals_context(tmp_path, mocker): diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index befd5148f..f1f2cea04 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -470,16 +470,16 @@ class QUERY_CONFIG_KEYS: 'NRSRAPIDD2', 'NRSRAPIDD6']} -REPORT_KEYS_PER_INSTRUMENT = {'fgs': ['proposal', 'obsnum', 'exp_type', +REPORT_KEYS_PER_INSTRUMENT = {'fgs': ['proposal', 'exp_type', 'expstart', 'filter', 'aperture', 'detector', 'subarray', 'viewed'], - 'miri': ['proposal', 'obsnum', 'exp_type', + 'miri': ['proposal', 'exp_type', 'expstart', 'filter', 'aperture', 'detector', 'subarray', 'viewed'], - 'nircam': ['proposal', 'obsnum', 'exp_type', + 'nircam': ['proposal', 'exp_type', 'expstart', 'filter', 'aperture', 'detector', 'subarray', 'viewed'], - 'niriss': ['proposal', 'obsnum', 'exp_type', + 'niriss': ['proposal', 'exp_type', 'expstart', 'filter', 'aperture', 'detector', 'subarray', 'viewed'], 'nirspec': ['exp_type', 'filter', 'grating', diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 6a22c83ca..e96bd283a 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -219,7 +219,7 @@ def filter_root_files(instrument=None, proposal=None, obsnum=None, sort_as=None, elif sort_as == 'oldest': root_file_info = root_file_info.order_by('expstart', 'root_name') - return root_file_info + return root_file_info.values() def create_archived_proposals_context(inst): @@ -1291,23 +1291,13 @@ def get_instrument_looks(instrument, sort_as=None, looks = [] for root_file in root_file_info: - # for now, report info by root name only. - # if specific files are needed, use get_filesystem_files result = dict() for key in keys: try: # try the root file table - value = getattr(root_file, key) - except AttributeError: - try: - # try the observation table - value = getattr(root_file.obsnum, key) - except AttributeError: - try: - # try the proposal table - value = getattr(root_file.obsnum.proposal, key) - except AttributeError: - value = '' + value = root_file[key] + except KeyError: + value = '' # make sure value can be serialized if type(value) not in [str, float, int, bool]: From b403eda416240b679638e3553c9895d4cdec79eb Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 10 Mar 2023 09:27:01 -0500 Subject: [PATCH 153/449] Better fix for unavailable proposal page --- jwql/website/apps/jwql/data_containers.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index e96bd283a..c1e7ba5b3 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -1747,6 +1747,7 @@ def text_scrape(prop_id): url = 'http://www.stsci.edu/cgi-bin/get-proposal-info?id=' + str(prop_id) + '&submit=Go&observatory=JWST' html = BeautifulSoup(requests.get(url).text, 'lxml') not_available = "not available via this interface" in html.text + not_available |= "temporarily unable" in html.text program_meta = {} program_meta['prop_id'] = prop_id @@ -1765,10 +1766,6 @@ def text_scrape(prop_id): links = html.findAll('a') - # check for server timeout: may return empty html - if len(links) == 0: - return program_meta - proposal_type = links[0].contents[0] program_meta['prop_type'] = proposal_type From 5e4fe81f964e96cb58049ec8be65d4a639e5f8ea Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 10 Mar 2023 09:52:28 -0500 Subject: [PATCH 154/449] Add pupil for NIRCam and NIRISS reports --- jwql/utils/constants.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index f1f2cea04..28e9226dd 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -476,11 +476,11 @@ class QUERY_CONFIG_KEYS: 'miri': ['proposal', 'exp_type', 'expstart', 'filter', 'aperture', 'detector', 'subarray', 'viewed'], - 'nircam': ['proposal', 'exp_type', - 'expstart', 'filter', 'aperture', + 'nircam': ['proposal', 'exp_type', 'expstart', + 'filter', 'pupil', 'aperture', 'detector', 'subarray', 'viewed'], - 'niriss': ['proposal', 'exp_type', - 'expstart', 'filter', 'aperture', + 'niriss': ['proposal', 'exp_type', 'expstart', + 'filter', 'pupil', 'aperture', 'detector', 'subarray', 'viewed'], 'nirspec': ['exp_type', 'filter', 'grating', 'read_patt_num', 'viewed']} From f1448ea3695d0ce6e45ea87602732ec254b972c8 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 10 Mar 2023 14:00:54 -0500 Subject: [PATCH 155/449] Add proposal search filter for report download --- jwql/website/apps/jwql/data_containers.py | 10 ++++++---- jwql/website/apps/jwql/static/js/jwql.js | 6 ++++++ jwql/website/apps/jwql/views.py | 2 +- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index c1e7ba5b3..73a0cece1 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -163,7 +163,7 @@ def filter_root_files(instrument=None, proposal=None, obsnum=None, sort_as=None, instrument : str, optional Name of the JWST instrument. proposal : str, optional - Proposal to match. + Proposal to match. Used as a 'starts with' filter. obsnum : str, optional Observation number to match. sort_as : {'ascending', 'descending', 'recent', 'oldest'}, optional @@ -194,7 +194,7 @@ def filter_root_files(instrument=None, proposal=None, obsnum=None, sort_as=None, inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[instrument.lower()] filter_kwargs['instrument'] = inst if proposal is not None and str(proposal).strip().lower() != 'all': - filter_kwargs['proposal'] = proposal.lstrip('0') + filter_kwargs['proposal__startswith'] = proposal.lstrip('0') if obsnum is not None and str(obsnum).strip().lower() != 'all': filter_kwargs['obsnum__obsnum'] = obsnum if look is not None and str(look).strip().lower() != 'all': @@ -1237,7 +1237,7 @@ def get_instrument_proposals(instrument): return inst_proposals -def get_instrument_looks(instrument, sort_as=None, +def get_instrument_looks(instrument, sort_as=None, proposal=None, look=None, exp_type=None, cat_type=None, additional_keys=None): """Return a table of looks information for the given instrument. @@ -1250,6 +1250,8 @@ def get_instrument_looks(instrument, sort_as=None, Sorting method for output table. Ascending and descending options refer to root file name; recent sorts by observation start. + proposal : str, optional + Proposal to match. Used as a 'starts with' filter. look : {'new', 'viewed'}, optional If set to None, all viewed values are returned. If set to 'viewed', only viewed data is returned. If set to 'new', only @@ -1287,7 +1289,7 @@ def get_instrument_looks(instrument, sort_as=None, # get filtered file info root_file_info = filter_root_files( instrument=instrument, sort_as=sort_as, look=look, - exp_type=exp_type, cat_type=cat_type) + exp_type=exp_type, cat_type=cat_type, proposal=proposal) looks = [] for root_file in root_file_info: diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index 41faf3bbf..c4e54dca6 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -799,6 +799,12 @@ function download_report(inst, base_url) { var sort_option = document.getElementById('sort_dropdownMenuButton').innerText; var options = '?sort_as=' + sort_option.toLowerCase(); + // Get search value - use as proposal.startswith + var search_value = document.getElementById("search_box").value; + if (search_value != '') { + options += '&proposal=' + search_value; + } + // Get all filter values var filter_div = document.getElementById('thumbnail-filter'); var filters = filter_div.getElementsByClassName('dropdown-toggle'); diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index f52982db5..9f68c4964 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -539,7 +539,7 @@ def download_report(request, inst): """ # check for filter criteria passed in request kwargs = dict() - for filter_name in ['look', 'exp_type', 'cat_type', 'sort_as']: + for filter_name in ['look', 'exp_type', 'cat_type', 'proposal', 'sort_as']: kwargs[filter_name] = request.GET.get(filter_name) # get all observation looks from file info model From c8d968b7fea285a4c19f3ee9eac1ca3e1272b85d Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Tue, 7 Mar 2023 09:46:30 -0500 Subject: [PATCH 156/449] Use new django model fields for filter/sort --- jwql/tests/test_data_containers.py | 14 ++--- jwql/utils/constants.py | 23 +++++--- jwql/website/apps/jwql/api_views.py | 4 +- jwql/website/apps/jwql/data_containers.py | 72 ++++++++++++++++++++++- 4 files changed, 94 insertions(+), 19 deletions(-) diff --git a/jwql/tests/test_data_containers.py b/jwql/tests/test_data_containers.py index dad5dc589..385559f50 100644 --- a/jwql/tests/test_data_containers.py +++ b/jwql/tests/test_data_containers.py @@ -179,18 +179,18 @@ def test_get_instrument_proposals(): (None, 'New', None, None, None), (None, None, None, 'NRS_MSATA', None), # (None, None, None, None, 'CAL'), # cat_type not implemented yet - (['obsstart'], 'new', 'ascending', None, None), - (['obsstart'], 'new', 'descending', None, None), - (['obsstart'], 'new', 'recent', None, None), + (['expstart'], 'new', 'ascending', None, None), + (['expstart'], 'new', 'descending', None, None), + (['expstart'], 'new', 'recent', None, None), ([], 'viewed', None, None, None), ([], 'new', None, None, None), ([], None, None, None, None), (['proposal', 'obsnum', 'other', - 'prop_id', 'obsstart'], 'viewed', None, None, None), + 'prop_id', 'expstart'], 'viewed', None, None, None), (['proposal', 'obsnum', 'other', - 'prop_id', 'obsstart'], 'new', None, None, None), + 'prop_id', 'expstart'], 'new', None, None, None), (['proposal', 'obsnum', 'other', - 'prop_id', 'obsstart'], None, None, None, None)]) + 'prop_id', 'expstart'], None, None, None, None)]) def test_get_instrument_looks(keys, viewed, sort_as, exp_type, cat_type): """Tests the ``get_instrument_looks`` function.""" @@ -224,7 +224,7 @@ def test_get_instrument_looks(keys, viewed, sort_as, exp_type, cat_type): if sort_as == 'ascending': assert last_file['root_name'] > first_file['root_name'] elif sort_as == 'recent': - assert last_file['obsstart'] < first_file['obsstart'] + assert last_file['expstart'] < first_file['expstart'] else: assert last_file['root_name'] < first_file['root_name'] diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index c0c5c7deb..befd5148f 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -470,15 +470,20 @@ class QUERY_CONFIG_KEYS: 'NRSRAPIDD2', 'NRSRAPIDD6']} -REPORT_KEYS_PER_INSTRUMENT = {'fgs': ['proposal', 'obsnum', 'exptypes', - 'obsstart', 'obsend', 'viewed'], - 'miri': ['proposal', 'obsnum', 'exptypes', - 'obsstart', 'obsend', 'viewed'], - 'nircam': ['proposal', 'obsnum', 'exptypes', - 'obsstart', 'obsend', 'viewed'], - 'niriss': ['proposal', 'obsnum', 'exptypes', - 'obsstart', 'obsend', 'viewed'], - 'nirspec': ['exptypes', 'viewed']} +REPORT_KEYS_PER_INSTRUMENT = {'fgs': ['proposal', 'obsnum', 'exp_type', + 'expstart', 'filter', 'aperture', + 'detector', 'subarray', 'viewed'], + 'miri': ['proposal', 'obsnum', 'exp_type', + 'expstart', 'filter', 'aperture', + 'detector', 'subarray', 'viewed'], + 'nircam': ['proposal', 'obsnum', 'exp_type', + 'expstart', 'filter', 'aperture', + 'detector', 'subarray', 'viewed'], + 'niriss': ['proposal', 'obsnum', 'exp_type', + 'expstart', 'filter', 'aperture', + 'detector', 'subarray', 'viewed'], + 'nirspec': ['exp_type', 'filter', 'grating', + 'read_patt_num', 'viewed']} SUBARRAYS_ONE_OR_FOUR_AMPS = ['SUBGRISMSTRIPE64', 'SUBGRISMSTRIPE128', 'SUBGRISMSTRIPE256'] diff --git a/jwql/website/apps/jwql/api_views.py b/jwql/website/apps/jwql/api_views.py index aeb32371f..e5303e7a0 100644 --- a/jwql/website/apps/jwql/api_views.py +++ b/jwql/website/apps/jwql/api_views.py @@ -144,8 +144,8 @@ def instrument_looks(request, inst, status=None): 'Viewed' indicates whether an observation is new or has been reviewed for QA. In addition to 'filename', and 'viewed', observation - descriptors included in the table are currently 'proposal', 'obsnum', - 'number_of_files', 'exptypes', 'obsstart', 'obsend'. + descriptors from the Django models may be added to the table. Keys + are specified by instrument in the REPORT_KEYS_PER_INSTRUMENT constant. Parameters ---------- diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index cda837548..7bf3e8c12 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -154,6 +154,74 @@ def build_table(tablename): return table_meta_data +def filter_root_files(instrument=None, proposal=None, obsnum=None, sort_as=None, + look=None, exp_type=None, cat_type=None, detector=None): + """Retrieve and filter root file table entries. + + Parameters + ---------- + instrument : str, optional + Name of the JWST instrument. + proposal : str, optional + Proposal to match. + obsnum : str, optional + Observation number to match. + sort_as : {'ascending', 'descending', 'recent', 'oldest'}, optional + Sorting method for output table. Ascending and descending + options refer to root file name; recent and oldest sort by exposure + start time. + look : {'new', 'viewed'}, optional + If set to None, all viewed values are returned. If set to + 'viewed', only viewed data is returned. If set to 'new', only + new data is returned. + exp_type : str, optional + Set to filter by exposure type. + cat_type : str, optional + Set to filter by proposal category. + detector : str, optional + Set to filter by detector name. + + Returns + ------- + root_file_info : QuerySet + List of RootFileInfo entries matching input criteria. + """ + # standardize input + + # get desired filters + filter_kwargs = dict() + if instrument is not None and str(instrument).strip().lower() != 'all': + inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[instrument.lower()] + filter_kwargs['instrument'] = inst + if proposal is not None and str(proposal).strip().lower() != 'all': + filter_kwargs['proposal'] = proposal.lstrip('0') + if obsnum is not None and str(obsnum).strip().lower() != 'all': + filter_kwargs['obsnum__obsnum'] = obsnum + if look is not None and str(look).strip().lower() != 'all': + filter_kwargs['viewed'] = (str(look).lower() == 'viewed') + if exp_type is not None and str(exp_type).strip().lower() != 'all': + filter_kwargs['exp_type__iexact'] = exp_type + if cat_type is not None and str(cat_type).strip().lower() != 'all': + filter_kwargs['obsnum__proposal__category__iexact'] = cat_type + if detector is not None and str(detector).strip().lower() != 'all': + filter_kwargs['detector__iexact'] = detector + + # get file info by instrument from local model + root_file_info = RootFileInfo.objects.filter(**filter_kwargs) + + # descending by root file is default; + # for other options, sort as desired + sort_as = str(sort_as).strip().lower() + if sort_as == 'ascending': + root_file_info = root_file_info.order_by('root_name') + elif sort_as == 'recent': + root_file_info = root_file_info.order_by('-expstart', 'root_name') + elif sort_as == 'oldest': + root_file_info = root_file_info.order_by('expstart', 'root_name') + + return root_file_info + + def create_archived_proposals_context(inst): """Generate and save a json file containing the information needed to create an instrument's archive page. @@ -219,7 +287,9 @@ def create_archived_proposals_context(inst): prop_filecount = [entry.number_of_files for entry in prop_entries] total_files.append(sum(prop_filecount)) - # In order to know if a proposal contains all observations that are entirely viewed, check for at least one existing viewed=False in RootFileInfo + # In order to know if a proposal contains all observations that + # are entirely viewed, check for at least one existing + # viewed=False in RootFileInfo unviewed_root_file_infos = RootFileInfo.objects.filter(instrument=inst, proposal=proposal_num, viewed=False) proposal_viewed.append("Viewed" if unviewed_root_file_infos.count() == 0 else "New") From b591d65eef5a2e08f18b5652fe5f33dc2a4e6cf7 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Thu, 9 Mar 2023 18:09:20 -0500 Subject: [PATCH 157/449] Performance fix for report download --- jwql/tests/test_data_containers.py | 65 +++++++++++++++++++++++ jwql/utils/constants.py | 8 +-- jwql/website/apps/jwql/data_containers.py | 18 ++----- 3 files changed, 73 insertions(+), 18 deletions(-) diff --git a/jwql/tests/test_data_containers.py b/jwql/tests/test_data_containers.py index 385559f50..0a3784b7f 100644 --- a/jwql/tests/test_data_containers.py +++ b/jwql/tests/test_data_containers.py @@ -24,6 +24,7 @@ """ import glob +import json import os import pytest @@ -36,6 +37,70 @@ from jwql.utils.utils import get_config +@pytest.mark.parametrize('filter_keys', + [{'instrument': 'NIRSpec', 'proposal': '2589', + 'obsnum': '006', 'look': 'All'}, + {'instrument': 'NIRCam', 'detector': 'NRCBLONG', + 'proposal': '2733', 'obsnum': '001'}, + {'instrument': 'MIRI', 'exp_type': 'MIR_IMAGE', + 'proposal': '1524', 'obsnum': '015'}, + {'instrument': 'FGS', 'cat_type': 'COM', + 'proposal': '1155'} + ]) +def test_filter_root_files(filter_keys): + rfi = data_containers.filter_root_files(**filter_keys) + assert len(rfi) > 0 + assert len(rfi) < 100 + + for key, value in filter_keys.items(): + if str(value).strip().lower() == 'all': + continue + elif key in ['cat_type', 'obsnum']: + # values returned are foreign keys + continue + else: + rf_test = [str(rf[key]) == str(value) for rf in rfi] + assert all(rf_test) + + +def test_filter_root_files_sorting(): + filter_keys = {'instrument': 'NIRSpec', 'proposal': '2589', + 'obsnum': '006'} + + rfi = data_containers.filter_root_files(**filter_keys, sort_as='Ascending') + assert len(rfi) > 3 + for i, rf in enumerate(rfi[1:]): + assert rf['root_name'] > rfi[i]['root_name'] + + rfi = data_containers.filter_root_files(**filter_keys, sort_as='Descending') + for i, rf in enumerate(rfi[1:]): + assert rf['root_name'] < rfi[i]['root_name'] + + rfi = data_containers.filter_root_files(**filter_keys, sort_as='Recent') + for i, rf in enumerate(rfi[1:]): + assert rf['expstart'] <= rfi[i]['expstart'] + + rfi = data_containers.filter_root_files(**filter_keys, sort_as='Oldest') + for i, rf in enumerate(rfi[1:]): + assert rf['expstart'] >= rfi[i]['expstart'] + + +def test_create_archived_proposals_context(tmp_path, mocker): + # write to a temporary directory + mocker.patch.object(data_containers, 'OUTPUT_DIR', str(tmp_path)) + archive_dir = tmp_path / 'archive_page' + os.mkdir(archive_dir) + + data_containers.create_archived_proposals_context('nirspec') + context_file = str(archive_dir / 'NIRSpec_archive_context.json') + assert os.path.isfile(context_file) + + with open(context_file, 'r') as obj: + context = json.load(obj) + assert context['inst'] == 'NIRSpec' + assert context['num_proposals'] > 0 + + def test_get_acknowledgements(): """Tests the ``get_acknowledgements`` function.""" diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index befd5148f..f1f2cea04 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -470,16 +470,16 @@ class QUERY_CONFIG_KEYS: 'NRSRAPIDD2', 'NRSRAPIDD6']} -REPORT_KEYS_PER_INSTRUMENT = {'fgs': ['proposal', 'obsnum', 'exp_type', +REPORT_KEYS_PER_INSTRUMENT = {'fgs': ['proposal', 'exp_type', 'expstart', 'filter', 'aperture', 'detector', 'subarray', 'viewed'], - 'miri': ['proposal', 'obsnum', 'exp_type', + 'miri': ['proposal', 'exp_type', 'expstart', 'filter', 'aperture', 'detector', 'subarray', 'viewed'], - 'nircam': ['proposal', 'obsnum', 'exp_type', + 'nircam': ['proposal', 'exp_type', 'expstart', 'filter', 'aperture', 'detector', 'subarray', 'viewed'], - 'niriss': ['proposal', 'obsnum', 'exp_type', + 'niriss': ['proposal', 'exp_type', 'expstart', 'filter', 'aperture', 'detector', 'subarray', 'viewed'], 'nirspec': ['exp_type', 'filter', 'grating', diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 7bf3e8c12..e745b230d 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -219,7 +219,7 @@ def filter_root_files(instrument=None, proposal=None, obsnum=None, sort_as=None, elif sort_as == 'oldest': root_file_info = root_file_info.order_by('expstart', 'root_name') - return root_file_info + return root_file_info.values() def create_archived_proposals_context(inst): @@ -1222,23 +1222,13 @@ def get_instrument_looks(instrument, sort_as=None, looks = [] for root_file in root_file_info: - # for now, report info by root name only. - # if specific files are needed, use get_filesystem_files result = dict() for key in keys: try: # try the root file table - value = getattr(root_file, key) - except AttributeError: - try: - # try the observation table - value = getattr(root_file.obsnum, key) - except AttributeError: - try: - # try the proposal table - value = getattr(root_file.obsnum.proposal, key) - except AttributeError: - value = '' + value = root_file[key] + except KeyError: + value = '' # make sure value can be serialized if type(value) not in [str, float, int, bool]: From 99b9f8e12fab06e4c8788e8a4a9374f1f14d384b Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 10 Mar 2023 14:00:54 -0500 Subject: [PATCH 158/449] Add proposal search filter for report download --- jwql/website/apps/jwql/data_containers.py | 32 +++++++---------------- jwql/website/apps/jwql/static/js/jwql.js | 6 +++++ jwql/website/apps/jwql/views.py | 2 +- 3 files changed, 16 insertions(+), 24 deletions(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index e745b230d..e0cedf7cd 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -163,7 +163,7 @@ def filter_root_files(instrument=None, proposal=None, obsnum=None, sort_as=None, instrument : str, optional Name of the JWST instrument. proposal : str, optional - Proposal to match. + Proposal to match. Used as a 'starts with' filter. obsnum : str, optional Observation number to match. sort_as : {'ascending', 'descending', 'recent', 'oldest'}, optional @@ -194,7 +194,7 @@ def filter_root_files(instrument=None, proposal=None, obsnum=None, sort_as=None, inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[instrument.lower()] filter_kwargs['instrument'] = inst if proposal is not None and str(proposal).strip().lower() != 'all': - filter_kwargs['proposal'] = proposal.lstrip('0') + filter_kwargs['proposal__startswith'] = proposal.lstrip('0') if obsnum is not None and str(obsnum).strip().lower() != 'all': filter_kwargs['obsnum__obsnum'] = obsnum if look is not None and str(look).strip().lower() != 'all': @@ -1152,7 +1152,7 @@ def get_instrument_proposals(instrument): return inst_proposals -def get_instrument_looks(instrument, sort_as=None, +def get_instrument_looks(instrument, sort_as=None, proposal=None, look=None, exp_type=None, cat_type=None, additional_keys=None): """Return a table of looks information for the given instrument. @@ -1165,6 +1165,8 @@ def get_instrument_looks(instrument, sort_as=None, Sorting method for output table. Ascending and descending options refer to root file name; recent sorts by observation start. + proposal : str, optional + Proposal to match. Used as a 'starts with' filter. look : {'new', 'viewed'}, optional If set to None, all viewed values are returned. If set to 'viewed', only viewed data is returned. If set to 'new', only @@ -1199,26 +1201,10 @@ def get_instrument_looks(instrument, sort_as=None, if key not in key_set: keys.append(key) - # get desired filters - filter_kwargs = dict() - if look is not None: - filter_kwargs['viewed'] = (str(look).lower() == 'viewed') - if exp_type is not None: - filter_kwargs['obsnum__exptypes__icontains'] = exp_type - if cat_type is not None: - # filter_kwargs['obsnum__proposal__cat_type__iexact'] = cat_type - # not yet implemented in proposal table - pass - - # get file info by instrument from local model - root_file_info = RootFileInfo.objects.filter(instrument=inst, **filter_kwargs) - - # descending by root file is default; - # for other options, sort as desired - if sort_as == 'ascending': - root_file_info = root_file_info.order_by('root_name') - elif sort_as == 'recent': - root_file_info = root_file_info.order_by('-obsnum__obsstart') + # get filtered file info + root_file_info = filter_root_files( + instrument=instrument, sort_as=sort_as, look=look, + exp_type=exp_type, cat_type=cat_type, proposal=proposal) looks = [] for root_file in root_file_info: diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index f0362afcf..612a5cee0 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -610,6 +610,12 @@ function download_report(inst, base_url) { var sort_option = document.getElementById('sort_dropdownMenuButton').innerText; var options = '?sort_as=' + sort_option.toLowerCase(); + // Get search value - use as proposal.startswith + var search_value = document.getElementById("search_box").value; + if (search_value != '') { + options += '&proposal=' + search_value; + } + // Get all filter values var filter_div = document.getElementById('thumbnail-filter'); var filters = filter_div.getElementsByClassName('dropdown-toggle'); diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 7c78e502e..5f7293f2c 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -518,7 +518,7 @@ def download_report(request, inst): """ # check for filter criteria passed in request kwargs = dict() - for filter_name in ['look', 'exp_type', 'cat_type', 'sort_as']: + for filter_name in ['look', 'exp_type', 'cat_type', 'proposal', 'sort_as']: kwargs[filter_name] = request.GET.get(filter_name) # get all observation looks from file info model From 0f587539c494f954bc59cd5751537eee74f2cb63 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 10 Mar 2023 09:52:28 -0500 Subject: [PATCH 159/449] Add pupil for NIRCam and NIRISS reports --- jwql/utils/constants.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index f1f2cea04..28e9226dd 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -476,11 +476,11 @@ class QUERY_CONFIG_KEYS: 'miri': ['proposal', 'exp_type', 'expstart', 'filter', 'aperture', 'detector', 'subarray', 'viewed'], - 'nircam': ['proposal', 'exp_type', - 'expstart', 'filter', 'aperture', + 'nircam': ['proposal', 'exp_type', 'expstart', + 'filter', 'pupil', 'aperture', 'detector', 'subarray', 'viewed'], - 'niriss': ['proposal', 'exp_type', - 'expstart', 'filter', 'aperture', + 'niriss': ['proposal', 'exp_type', 'expstart', + 'filter', 'pupil', 'aperture', 'detector', 'subarray', 'viewed'], 'nirspec': ['exp_type', 'filter', 'grating', 'read_patt_num', 'viewed']} From 778299735fa2ca37f7e7c14e9e9413c4ea0f73f7 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 10 Mar 2023 15:23:43 -0500 Subject: [PATCH 160/449] Fix merging error --- jwql/tests/test_data_containers.py | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/jwql/tests/test_data_containers.py b/jwql/tests/test_data_containers.py index 0a3784b7f..ec5e7aff8 100644 --- a/jwql/tests/test_data_containers.py +++ b/jwql/tests/test_data_containers.py @@ -37,6 +37,7 @@ from jwql.utils.utils import get_config +@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') @pytest.mark.parametrize('filter_keys', [{'instrument': 'NIRSpec', 'proposal': '2589', 'obsnum': '006', 'look': 'All'}, @@ -63,6 +64,7 @@ def test_filter_root_files(filter_keys): assert all(rf_test) +@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') def test_filter_root_files_sorting(): filter_keys = {'instrument': 'NIRSpec', 'proposal': '2589', 'obsnum': '006'} @@ -85,22 +87,6 @@ def test_filter_root_files_sorting(): assert rf['expstart'] >= rfi[i]['expstart'] -def test_create_archived_proposals_context(tmp_path, mocker): - # write to a temporary directory - mocker.patch.object(data_containers, 'OUTPUT_DIR', str(tmp_path)) - archive_dir = tmp_path / 'archive_page' - os.mkdir(archive_dir) - - data_containers.create_archived_proposals_context('nirspec') - context_file = str(archive_dir / 'NIRSpec_archive_context.json') - assert os.path.isfile(context_file) - - with open(context_file, 'r') as obj: - context = json.load(obj) - assert context['inst'] == 'NIRSpec' - assert context['num_proposals'] > 0 - - def test_get_acknowledgements(): """Tests the ``get_acknowledgements`` function.""" From fe2046fb5177217e7def2745ac951c7b66250529 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 10 Mar 2023 15:40:12 -0500 Subject: [PATCH 161/449] Restore skips for functionality not available in CI --- jwql/tests/test_data_containers.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/jwql/tests/test_data_containers.py b/jwql/tests/test_data_containers.py index 671d28a0d..c2ca97456 100644 --- a/jwql/tests/test_data_containers.py +++ b/jwql/tests/test_data_containers.py @@ -34,7 +34,7 @@ # Skip testing this module if on Github Actions ON_GITHUB_ACTIONS = '/home/runner' in os.path.expanduser('~') or '/Users/runner' in os.path.expanduser('~') from jwql.website.apps.jwql import data_containers -from jwql.tests.resources import MockAnomalyQuery, MockSessionFileAnomaly, MockSessionGroupAnomaly +from jwql.tests.resources import MockSessionFileAnomaly, MockSessionGroupAnomaly if not ON_GITHUB_ACTIONS: from jwql.utils.utils import get_config @@ -46,6 +46,7 @@ def test_build_table(): assert len(tab['date']) > 0 +@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to django models.') @pytest.mark.parametrize('filter_keys', [{'instrument': 'NIRSpec', 'proposal': '2589', 'obsnum': '006', 'look': 'All'}, @@ -72,6 +73,7 @@ def test_filter_root_files(filter_keys): assert all(rf_test) +@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to django models.') def test_filter_root_files_sorting(): filter_keys = {'instrument': 'NIRSpec', 'proposal': '2589', 'obsnum': '006'} @@ -94,6 +96,7 @@ def test_filter_root_files_sorting(): assert rf['expstart'] >= rfi[i]['expstart'] +@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') def test_create_archived_proposals_context(tmp_path, mocker): # write to a temporary directory mocker.patch.object(data_containers, 'OUTPUT_DIR', str(tmp_path)) @@ -268,10 +271,9 @@ def test_get_header_info(): assert len(header) > 0 +@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') def test_get_image_info(): """Tests the ``get_image_info`` function.""" - # requires filesystem for full info, but this basic test - # should pass without central storage image_info = data_containers.get_image_info('jw01068001001_02102_00001_nrcb1') assert isinstance(image_info, dict) @@ -289,6 +291,7 @@ def test_get_instrument_proposals(): assert len(proposals) > 0 +@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to django models.') @pytest.mark.parametrize('keys,viewed,sort_as,exp_type,cat_type', [(None, None, None, None, None), (None, 'viewed', None, None, None), @@ -373,9 +376,9 @@ def test_get_proposals_by_category(): assert len(proposals_by_category) > 0 +@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') def test_get_proposal_info(): """Tests the ``get_proposal_info`` function.""" - # requires filesystem for full results, but basic test should pass without filepaths = glob.glob(os.path.join(get_config()['filesystem'], 'jw01068', '*.fits')) proposal_info = data_containers.get_proposal_info(filepaths) From ca7429e6582f3ea9a48131b8dbe3d02b55b0e274 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 10 Mar 2023 16:44:34 -0500 Subject: [PATCH 162/449] Unit tests for data containers --- jwql/tests/resources.py | 29 ++++ jwql/tests/test_data_containers.py | 153 ++++++++++++++++++++-- jwql/website/apps/jwql/data_containers.py | 9 +- 3 files changed, 175 insertions(+), 16 deletions(-) diff --git a/jwql/tests/resources.py b/jwql/tests/resources.py index 7d8ba0794..260f1130f 100644 --- a/jwql/tests/resources.py +++ b/jwql/tests/resources.py @@ -44,3 +44,32 @@ class MockSessionFileAnomaly(object): """Mock a SQLAlchemy session for an anomaly query on a file.""" def query(self, table): return MockAnomalyQuery(group_record=False) + + +class MockMessages(object): + """Mock Django messages for requests.""" + def __init__(self): + self.messages = [] + + def add(self, level, message, extra_tags): + self.messages.append(message) + + +class MockGetRequest(object): + """Mock a Django HTTP GET request.""" + def __init__(self): + self.method = 'GET' + self.session = dict() + self.GET = dict() + self.POST = dict() + self._messages = MockMessages() + + +class MockPostRequest(object): + """Mock a Django HTTP POST request.""" + def __init__(self): + self.method = 'POST' + self.session = dict() + self.GET = dict() + self.POST = dict() + self._messages = MockMessages() diff --git a/jwql/tests/test_data_containers.py b/jwql/tests/test_data_containers.py index c2ca97456..9a172fe0b 100644 --- a/jwql/tests/test_data_containers.py +++ b/jwql/tests/test_data_containers.py @@ -34,7 +34,10 @@ # Skip testing this module if on Github Actions ON_GITHUB_ACTIONS = '/home/runner' in os.path.expanduser('~') or '/Users/runner' in os.path.expanduser('~') from jwql.website.apps.jwql import data_containers -from jwql.tests.resources import MockSessionFileAnomaly, MockSessionGroupAnomaly +from jwql.tests.resources import ( + MockSessionFileAnomaly, MockSessionGroupAnomaly, + MockGetRequest, MockPostRequest) +from jwql.utils import constants if not ON_GITHUB_ACTIONS: from jwql.utils.utils import get_config @@ -136,12 +139,14 @@ def test_get_all_proposals(): (['uncal', 'rate', 'bad'], {'bad'})), (False, ['rate', 'uncal', 'bad'], ['uncal', 'rate', 'bad']), - (True, ['rate', 'uncal', 'bad', - 'o006_crfints', 'o001_crf'], + (True, + ['rate', 'uncal', 'bad', + 'o006_crfints', 'o001_crf'], (['uncal', 'rate', 'o001_crf', 'o006_crfints', 'bad'], {'bad'})), - (False, ['rate', 'uncal', 'bad', - 'o006_crfints', 'o001_crf'], + (False, + ['rate', 'uncal', 'bad', + 'o006_crfints', 'o001_crf'], ['uncal', 'rate', 'o001_crf', 'o006_crfints', 'bad']), ]) @@ -174,15 +179,143 @@ def test_get_current_flagged_anomalies(mocker): assert result == ['persistence', 'crosstalk'] -def test_get_expstart(): +def test_get_anomaly_form_get(mocker): + request = MockGetRequest() + inst = 'NIRSpec' + file_root = 'jw02589006001_04101_00001-seg001_nrs1' + + # mock two anomalies for this file + mocker.patch.object(data_containers.di, 'session', + MockSessionFileAnomaly()) + + form = data_containers.get_anomaly_form(request, inst, file_root) + + # form should contain all anomaly options and two should be checked + html = str(form) + for anomaly in constants.ANOMALY_CHOICES_NIRSPEC: + assert anomaly[1] in html + assert html.count('checked') == 2 + + +def test_get_anomaly_form_post(mocker): + request = MockPostRequest() + inst = 'NIRSpec' + file_root = 'jw02589006001_04101_00001-seg001_nrs1' + + # mock two anomalies for this file + mocker.patch.object(data_containers.di, 'session', + MockSessionFileAnomaly()) + + # post a different selection: others are deselected + request.POST['anomaly_choices'] = ['optical_short'] + + # mock form validity and update functions + mocker.patch.object(data_containers.InstrumentAnomalySubmitForm, + 'is_valid', return_value=True) + update_mock = mocker.patch.object( + data_containers.InstrumentAnomalySubmitForm, 'update_anomaly_table') + + form = data_containers.get_anomaly_form(request, inst, file_root) + + # form should contain all anomaly options and only + # the chosen one should be checked + html = str(form) + for anomaly in constants.ANOMALY_CHOICES_NIRSPEC: + assert anomaly[1] in html + assert html.count('checked') == 1 + + # message should indicate success, update should have been called + assert 'Anomaly submitted successfully' in request._messages.messages + assert update_mock.call_count == 1 + + # mock invalid form + mocker.patch.object(data_containers.InstrumentAnomalySubmitForm, + 'is_valid', return_value=False) + data_containers.get_anomaly_form(request, inst, file_root) + + # messages indicate failure, update is not called again + assert 'Failed to submit anomaly' in request._messages.messages + assert update_mock.call_count == 1 + + +def test_get_anomaly_form_post_group(mocker): + request = MockPostRequest() + inst = 'NIRSpec' + file_root = 'jw02589006001_04101_00001-seg001' + + # mock anomalies for the group + mocker.patch.object(data_containers.di, 'session', + MockSessionGroupAnomaly()) + + # post a different selection: others are deselected, + # unless they belong only to the file, not to the group + # as a whole + request.POST['anomaly_choices'] = ['optical_short'] + + # mock form validity and update functions + mocker.patch.object(data_containers.InstrumentAnomalySubmitForm, + 'is_valid', return_value=True) + update_mock = mocker.patch.object( + data_containers.InstrumentAnomalySubmitForm, 'update_anomaly_table') + + form = data_containers.get_anomaly_form(request, inst, file_root) + + # form should contain all anomaly options and only + # the chosen one should be checked + html = str(form) + for anomaly in constants.ANOMALY_CHOICES_NIRSPEC: + assert anomaly[1] in html + assert html.count('checked') == 1 + + # message should indicate success, update should have been + # called for both files + assert 'Anomaly submitted successfully' in request._messages.messages + assert update_mock.call_count == 2 + + # mock invalid form + mocker.patch.object(data_containers.InstrumentAnomalySubmitForm, + 'is_valid', return_value=False) + data_containers.get_anomaly_form(request, inst, file_root) + + # messages indicate failure, update is not called again + assert 'Failed to submit anomaly' in request._messages.messages + assert update_mock.call_count == 2 + + +def test_get_dashboard_components(): + request = MockPostRequest() + + # empty POST + dash = data_containers.get_dashboard_components(request) + assert dash.delta_t is None + + # POST contains time delta + request.POST['time_delta_value'] = True + + # all time = None + request.POST['timedelta'] = 'All Time' + dash = data_containers.get_dashboard_components(request) + assert dash.delta_t is None + + # specific value + request.POST['timedelta'] = '1 Day' + dash = data_containers.get_dashboard_components(request) + assert dash.delta_t == pd.DateOffset(days=1) + + +@pytest.mark.parametrize('inst,fileroot,value', + [('NIRCam', + 'jw01068001001_02102_00001_nrcb1', 59714), + ('NIRSpec', + 'jw02589006001_04101_00001-seg002_nrs2', 59777), + ('NIRSpec', 'bad_filename', 0)]) +def test_get_expstart(inst, fileroot, value): """Tests the ``get_expstart`` function.""" - expstart = data_containers.get_expstart('NIRCam', - 'jw01068001001_02102_00001_nrcb1') - assert isinstance(expstart, float) + expstart = data_containers.get_expstart(inst, fileroot) # if mast query failed, it will return 0 # otherwise, it should have a known value for this file - assert np.isclose(expstart, 59714, atol=1) + assert np.isclose(expstart, value, atol=1) def test_get_filenames_by_instrument(): diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 73a0cece1..3e67be80a 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -517,15 +517,12 @@ def get_anomaly_form(request, inst, file_root): def get_dashboard_components(request): - """Build and return dictionaries containing components and html - needed for the dashboard. + """Build and return a Dashboard class. Returns ------- - dashboard_components : dict - A dictionary containing components needed for the dashboard. - dashboard_html : dict - A dictionary containing full HTML needed for the dashboard. + dashboard_components : GeneralDashboard + The dashboard. """ from jwql.website.apps.jwql.bokeh_dashboard import GeneralDashboard From 3ce73595316c201e40784a84f2b0cc70bfa8189b Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 13 Mar 2023 11:48:40 -0400 Subject: [PATCH 163/449] Catch exception for bad filename --- jwql/website/apps/jwql/views.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 9f68c4964..4dae2a6a3 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -1196,7 +1196,10 @@ def view_exposure(request, inst, group_root): # pick out group names from matching root files group_root_list = [] for rootname in matching_rootfiles: - other_group_root = filename_parser(rootname)['group_root'] + try: + other_group_root = filename_parser(rootname)['group_root'] + except ValueError: + continue if other_group_root not in group_root_list: group_root_list.append(other_group_root) From 9e14e5f5e243bafbbd7742ab102fa7faf332cdb2 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 13 Mar 2023 11:52:46 -0400 Subject: [PATCH 164/449] Catch missing files of specific types in exposure page --- jwql/website/apps/jwql/static/css/jwql.css | 8 +++- jwql/website/apps/jwql/static/js/jwql.js | 40 +++++++++++++++++-- .../apps/jwql/templates/view_exposure.html | 2 +- 3 files changed, 44 insertions(+), 6 deletions(-) diff --git a/jwql/website/apps/jwql/static/css/jwql.css b/jwql/website/apps/jwql/static/css/jwql.css index 6e841a95f..8431c82f3 100644 --- a/jwql/website/apps/jwql/static/css/jwql.css +++ b/jwql/website/apps/jwql/static/css/jwql.css @@ -71,7 +71,13 @@ display: block; } } - + + /* Make disabled buttons opaque and unclickable */ + .disabled_button { + pointer-events: none; + opacity: 0.65; + } + /* Make disabled dropdown items grey and unclickable */ .disabled-dropdown { color: #bec4d4 !important; diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index c4e54dca6..9767d6d33 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -68,8 +68,10 @@ '/' + group_root + '_' + detector + '_' + type + '_integ0.jpg'; img.src = jpg_filepath; img.alt = jpg_filepath; - // if previous image had error, remove error sizing - img.classList.remove("thumbnail"); + + // if previous image/file had error, un-hide it + img.style.display = "inline-block"; + filename_option.disabled = false; } // Reset the slider values @@ -510,6 +512,26 @@ function group_by_thumbnails(group_type, base_url) { } +/** + * If an image is not found, replace with temporary image sized to thumbnail + */ +function hide_file(image) { + // Hide the image + image.src = "/static/img/imagenotfound.png"; + image.style.display = "none"; + + // Disable the associated filename + var detector = image.getAttribute('data-detector'); + var filename_option = document.getElementById(detector + "_filename"); + filename_option.disabled = true; + + // Update the view/explore links if needed + update_view_explore_link(); + + return true; +} + + /** * If an image is not found, replace with temporary image sized to thumbnail */ @@ -1334,8 +1356,18 @@ function update_view_explore_link() { var types = ['header', 'explore_image']; for (var i = 0; i < types.length; i++) { var type = types[i]; - var selected = document.getElementById('fits_file_select').value; - document.getElementById(type).href = '/' + selected + '/' + type + '/'; + var file_selected = document.getElementById('fits_file_select'); + var link_button = document.getElementById(type); + + // Disable the button if the file isn't available + if (file_selected.options[file_selected.selectedIndex].disabled) { + link_button.href = '#'; + link_button.classList.add('disabled_button'); + } else { + // Update the link to the current setting + link_button.href = '/' + file_selected.value + '/' + type + '/'; + link_button.classList.remove('disabled_button'); + } } } diff --git a/jwql/website/apps/jwql/templates/view_exposure.html b/jwql/website/apps/jwql/templates/view_exposure.html index 1fe5d9278..e14682b77 100644 --- a/jwql/website/apps/jwql/templates/view_exposure.html +++ b/jwql/website/apps/jwql/templates/view_exposure.html @@ -132,7 +132,7 @@

    Exposure {{ group_root }}

    From d90ec449e7205cbc658128d0a13fd15c7cfb8508 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 13 Mar 2023 13:46:38 -0400 Subject: [PATCH 165/449] Also hide the detector division if file isn't present --- jwql/website/apps/jwql/static/js/jwql.js | 9 ++++++--- jwql/website/apps/jwql/templates/view_exposure.html | 8 ++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index 9767d6d33..7637d8b97 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -72,6 +72,7 @@ // if previous image/file had error, un-hide it img.style.display = "inline-block"; filename_option.disabled = false; + document.getElementById(detector + "_view").style.display = "inline-block"; } // Reset the slider values @@ -520,10 +521,12 @@ function hide_file(image) { image.src = "/static/img/imagenotfound.png"; image.style.display = "none"; - // Disable the associated filename + // Also hide the div it's in var detector = image.getAttribute('data-detector'); - var filename_option = document.getElementById(detector + "_filename"); - filename_option.disabled = true; + document.getElementById(detector + "_view").style.display = "none"; + + // Disable the associated filename + document.getElementById(detector + "_filename").disabled = true; // Update the view/explore links if needed update_view_explore_link(); diff --git a/jwql/website/apps/jwql/templates/view_exposure.html b/jwql/website/apps/jwql/templates/view_exposure.html index e14682b77..fc283b86e 100644 --- a/jwql/website/apps/jwql/templates/view_exposure.html +++ b/jwql/website/apps/jwql/templates/view_exposure.html @@ -77,7 +77,7 @@

    Exposure {{ group_root }}

    {% for detector1 in ['nrca2', 'nrca4', 'nrcb3', 'nrcb1'] %} {% if detector1 in detectors %} -
    +
    Exposure {{ group_root }} {% for detector2 in ['nrca1', 'nrca3', 'nrcb4', 'nrcb2'] %} {% if detector2 in detectors %} -
    +
    Exposure {{ group_root }} {% for detector3 in ['nrcalong', 'nrcblong'] %} {% if detector3 in detectors %} -
    +
    Exposure {{ group_root }}
    {% for detector in detectors %} -
    +
    Date: Mon, 13 Mar 2023 15:17:54 -0400 Subject: [PATCH 166/449] Fix int slider for cube data --- jwql/website/apps/jwql/data_containers.py | 7 ++++++- jwql/website/apps/jwql/static/js/jwql.js | 6 ++++++ jwql/website/apps/jwql/templates/view_exposure.html | 8 ++++---- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 3e67be80a..e8d3050f5 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -1171,13 +1171,18 @@ def get_image_info(file_root): # Record how many integrations exist per filetype. if suffix not in SUFFIXES_WITH_AVERAGED_INTS: - # time series segments need special handling header = fits.getheader(filename) nint = header['NINTS'] if 'time_series' in parsed_fn['filename_type']: + # time series segments need special handling intstart = header.get('INTSTART', 1) intend = header.get('INTEND', nint) image_info['total_ints'][suffix] = intend - intstart + 1 + elif image_info['num_ints'][suffix] > nint: + # so do data cubes: + # get max ints from data shape in first extension + sci_header = fits.getheader(filename, ext=1) + image_info['total_ints'][suffix] = sci_header.get('NAXIS3', nint) else: image_info['total_ints'][suffix] = nint else: diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index 7637d8b97..ae9cb6113 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -244,10 +244,16 @@ function change_int(file_root, num_ints, available_ints, method, direction='righ if (detector != null) { // exposure view jpg_filename = file_root + '_' + detector + '_' + suffix + '_integ' + new_integration + '.jpg'; + // if previous image/file had error, un-hide it + img.style.display = "inline-block"; + document.getElementById(detector + "_view").style.display = "inline-block"; + document.getElementById(detector + "_filename").disabled = false; } else { // image view jpg_filename = file_root + '_' + suffix + '_integ' + new_integration + '.jpg'; document.getElementById("jpg_filename").innerHTML = jpg_filename; + // if previous image had error, remove error sizing + img.classList.remove("thumbnail"); } // Show the appropriate image diff --git a/jwql/website/apps/jwql/templates/view_exposure.html b/jwql/website/apps/jwql/templates/view_exposure.html index fc283b86e..31563c6be 100644 --- a/jwql/website/apps/jwql/templates/view_exposure.html +++ b/jwql/website/apps/jwql/templates/view_exposure.html @@ -81,7 +81,7 @@

    Exposure {{ group_root }}

     @@ -98,7 +98,7 @@

    Exposure {{ group_root }}

    @@ -115,7 +115,7 @@

    Exposure {{ group_root }}

    @@ -132,7 +132,7 @@

    Exposure {{ group_root }}

    From 4a18e5aa2e51625e73090a90a36af9145556798c Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 13 Mar 2023 16:51:53 -0400 Subject: [PATCH 167/449] Show/hide viewers without flashing --- jwql/website/apps/jwql/static/js/jwql.js | 62 +++++++++++++------ .../apps/jwql/templates/view_exposure.html | 4 -- 2 files changed, 43 insertions(+), 23 deletions(-) diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index ae9cb6113..9d9a264b8 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -69,10 +69,8 @@ img.src = jpg_filepath; img.alt = jpg_filepath; - // if previous image/file had error, un-hide it - img.style.display = "inline-block"; - filename_option.disabled = false; - document.getElementById(detector + "_view").style.display = "inline-block"; + // Show/hide the viewer as appropriate + show_viewer(detector, jpg_filepath); } // Reset the slider values @@ -244,10 +242,6 @@ function change_int(file_root, num_ints, available_ints, method, direction='righ if (detector != null) { // exposure view jpg_filename = file_root + '_' + detector + '_' + suffix + '_integ' + new_integration + '.jpg'; - // if previous image/file had error, un-hide it - img.style.display = "inline-block"; - document.getElementById(detector + "_view").style.display = "inline-block"; - document.getElementById(detector + "_filename").disabled = false; } else { // image view jpg_filename = file_root + '_' + suffix + '_integ' + new_integration + '.jpg'; @@ -260,6 +254,11 @@ function change_int(file_root, num_ints, available_ints, method, direction='righ var jpg_filepath = '/static/preview_images/' + program + '/' + jpg_filename; img.src = jpg_filepath; img.alt = jpg_filepath; + + // Show/hide the viewer as appropriate for the image + if (detector != null) { + show_viewer(detector, jpg_filepath); + } } // Update the slider values @@ -520,27 +519,52 @@ function group_by_thumbnails(group_type, base_url) { /** - * If an image is not found, replace with temporary image sized to thumbnail + * Hide an image viewer */ -function hide_file(image) { - // Hide the image - image.src = "/static/img/imagenotfound.png"; - image.style.display = "none"; +function hide_file(detector) { + var img = document.getElementById("image_viewer_" + detector); + var div = document.getElementById(detector + "_view"); + var filename = document.getElementById(detector + "_filename"); - // Also hide the div it's in - var detector = image.getAttribute('data-detector'); - document.getElementById(detector + "_view").style.display = "none"; + // Hide the image and div + img.style.display = "none"; + div.style.display = "none"; // Disable the associated filename - document.getElementById(detector + "_filename").disabled = true; + filename.disabled = true; - // Update the view/explore links if needed + // Update the view/explore link as needed update_view_explore_link(); +} - return true; +/** + * Show an image viewer + */ +function unhide_file(detector) { + var img = document.getElementById("image_viewer_" + detector); + var div = document.getElementById(detector + "_view"); + var filename = document.getElementById(detector + "_filename"); + + // Show the image and div + img.style.display = "inline-block"; + div.style.display = "inline-block"; + + // Enable the associated filename + filename.disabled = false; + + // Update the view/explore link as needed + update_view_explore_link(); } +/** + * Check for a detector image and show or hide its viewer accordingly. + */ +function show_viewer(detector, jpg_filepath) { + $.get(jpg_filepath, function() {unhide_file(detector);}) + .fail(function() {hide_file(detector)}); +} + /** * If an image is not found, replace with temporary image sized to thumbnail */ diff --git a/jwql/website/apps/jwql/templates/view_exposure.html b/jwql/website/apps/jwql/templates/view_exposure.html index 31563c6be..e7926d922 100644 --- a/jwql/website/apps/jwql/templates/view_exposure.html +++ b/jwql/website/apps/jwql/templates/view_exposure.html @@ -81,7 +81,6 @@

    Exposure {{ group_root }}

    @@ -98,7 +97,6 @@

    Exposure {{ group_root }}

    @@ -115,7 +113,6 @@

    Exposure {{ group_root }}

    @@ -132,7 +129,6 @@

    Exposure {{ group_root }}

    From 326bacb217064fc34db33c41a68b7f5ee4adaee8 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 13 Mar 2023 17:02:35 -0400 Subject: [PATCH 168/449] Check for highest total ints in cubes --- jwql/website/apps/jwql/data_containers.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index e8d3050f5..3ca26324f 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -1182,7 +1182,15 @@ def get_image_info(file_root): # so do data cubes: # get max ints from data shape in first extension sci_header = fits.getheader(filename, ext=1) - image_info['total_ints'][suffix] = sci_header.get('NAXIS3', nint) + n_frame = sci_header.get('NAXIS3', nint) + + # for groups with multiple cubes (e.g. miri with ifu + # short and long), make sure we keep the highest total + if 'suffix' in image_info['total_ints']: + if n_frame > image_info['total_ints'][suffix]: + image_info['total_ints'][suffix] = n_frame + else: + image_info['total_ints'][suffix] = n_frame else: image_info['total_ints'][suffix] = nint else: From 4377eff8b05d4d845be737871492fdf14f512e04 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 13 Mar 2023 17:33:28 -0400 Subject: [PATCH 169/449] Skip tests in CI --- jwql/tests/test_data_containers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jwql/tests/test_data_containers.py b/jwql/tests/test_data_containers.py index 9a172fe0b..749ead24e 100644 --- a/jwql/tests/test_data_containers.py +++ b/jwql/tests/test_data_containers.py @@ -43,6 +43,7 @@ from jwql.utils.utils import get_config +@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to django models.') def test_build_table(): tab = data_containers.build_table('filesystem_general') assert isinstance(tab, pd.DataFrame) @@ -179,6 +180,7 @@ def test_get_current_flagged_anomalies(mocker): assert result == ['persistence', 'crosstalk'] +@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to django models.') def test_get_anomaly_form_get(mocker): request = MockGetRequest() inst = 'NIRSpec' @@ -197,6 +199,7 @@ def test_get_anomaly_form_get(mocker): assert html.count('checked') == 2 +@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to django models.') def test_get_anomaly_form_post(mocker): request = MockPostRequest() inst = 'NIRSpec' @@ -238,6 +241,7 @@ def test_get_anomaly_form_post(mocker): assert update_mock.call_count == 1 +@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to django models.') def test_get_anomaly_form_post_group(mocker): request = MockPostRequest() inst = 'NIRSpec' From e6f4a3bbe4652f85c533689199391aaf5b0eda55 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 14 Mar 2023 09:41:09 -0400 Subject: [PATCH 170/449] remove unused function. Clean up print statements --- jwql/utils/utils.py | 30 ------------------- .../monitor_pages/monitor_bad_pixel_bokeh.py | 9 +----- 2 files changed, 1 insertion(+), 38 deletions(-) diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index dd6dc07da..bbff87489 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -177,36 +177,6 @@ def create_png_from_fits(filename, outdir): return None -def screenshot_as_png(): - from bokeh.io.export import get_screenshot_as_png - if os.path.isfile(filename): - image = fits.getdata(filename) - ny, nx = image.shape - img_mn, img_med, img_dev = sigma_clipped_stats(image[4: ny - 4, 4: nx - 4]) - - plot = figure(tools='') - plot.x_range.range_padding = plot.y_range.range_padding = 0 - - # Create the color mapper that will be used to scale the image - #mapper = LinearColorMapper(palette='Viridis256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev)) - mapper = LogColorMapper(palette='Viridis256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev)) - - # Plot image - imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, - color_mapper=mapper, level="image") - - # Turn off the axes, in order to make embedding in another figure easier - plot.xaxis.visible = False - plot.yaxis.visible = False - - # Save the plot in a png - screenshot = get_screenshot_as_png(plot, height=600, width=600) - return screenshot - else: - return None - - - def get_config(): """Return a dictionary that holds the contents of the ``jwql`` config file. diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py index 72fec3eef..e75d3ea83 100755 --- a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py @@ -23,7 +23,6 @@ from astropy.time import Time from bokeh.embed import components, file_html from bokeh.io import export_png, show -from bokeh.io.export import get_screenshot_as_png from bokeh.layouts import layout from bokeh.models import ColumnDataSource, DatetimeTickFormatter, HoverTool, Legend, LinearColorMapper, Panel, Tabs, Text, Title from bokeh.plotting import figure @@ -445,8 +444,6 @@ def create_plot(self): png_file = self.background_file.replace('.fits', '.png') full_path_background_file = os.path.join(OUTPUT_DIR, 'bad_pixel_monitor/', png_file) - print('Looking for: ', full_path_background_file) - if os.path.isfile(full_path_background_file): image = read_png(full_path_background_file) else: @@ -509,7 +506,6 @@ def create_plot(self): if self._use_png: output_filename = full_path_background_file.replace('.png', f'_{self.badpix_type}_pix.png') self.switch_to_png(output_filename, title_text) - print(f'Switching to png for {self.detector}, {self.badpix_type}, {len(self.coords[0])}') # Create and add legend to the figure legend = Legend(items=[plot_legend], @@ -597,8 +593,6 @@ def switch_to_png(self, filename, title): Title to add to the Figure """ # Save the figure as a png - print('Saving plot as png in: ', filename) - export_png(self.plot, filename=filename) set_permissions(filename) @@ -612,8 +606,7 @@ def switch_to_png(self, filename, title): self.plot.image_rgba(image=[fig_array], x=0, y=0, dw=self._detlen, dh=self._detlen) # Now that the data from the png is in the figure, delete the png - # UNCOMMENT THE LINE BELOW FOR PRODUCTION - #os.remove(filename) + os.remove(filename) class BadPixTrendPlot(): From 0af2e51de3dee2bd0573a5b4624c0eed7e9af2f9 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 14 Mar 2023 09:43:04 -0400 Subject: [PATCH 171/449] Remove bokeh templating yaml file --- .../yaml/badpixel_monitor_interface.yaml | 417 ------------------ 1 file changed, 417 deletions(-) delete mode 100755 jwql/website/apps/jwql/monitor_pages/yaml/badpixel_monitor_interface.yaml diff --git a/jwql/website/apps/jwql/monitor_pages/yaml/badpixel_monitor_interface.yaml b/jwql/website/apps/jwql/monitor_pages/yaml/badpixel_monitor_interface.yaml deleted file mode 100755 index 82ebff748..000000000 --- a/jwql/website/apps/jwql/monitor_pages/yaml/badpixel_monitor_interface.yaml +++ /dev/null @@ -1,417 +0,0 @@ -# YAML file defining bokeh figures for the bad pixel monitor -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Dead Pixels v. Time Figure -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -- !ColumnDataSource: &dead_history_source - ref: "dead_history_source" - data: - time: !self.bad_history['DEAD'][0] - dead_pixels: !self.bad_history['DEAD'][1] - time_labels: !self.bad_history['DEAD'][2] - -- !Range1d: &dead_history_xrange - ref: "dead_history_xrange" - start: 0 - end: 1 - bounds: 'auto' #!!python/tuple [0, 1] - -- !Range1d: &dead_history_yrange - ref: "dead_history_yrange" - start: 0 - end: 1 - bounds: 'auto' # !!python/tuple [-1, 1] - -- !Figure: &dead_history_figure - ref: "dead_history_figure" - x_axis_label: "Date" - x_axis_type: "datetime" - y_axis_label: "Number of DEAD pixels" - x_range: *dead_history_xrange - y_range: *dead_history_yrange - tools: 'hover, zoom_in, zoom_out, box_zoom, pan, reset, save' - tooltips: "@time_labels Dead pix: @dead_pixels" - elements: - - {'kind': 'circle', 'x': 'time', 'y': 'dead_pixels', line_width: 5, 'source': *dead_history_source} - -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Hot Pixels v. Time Figure -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -- !ColumnDataSource: &hot_history_source - ref: "hot_history_source" - data: - time: !self.bad_history['HOT'][0] - hot_pixels: !self.bad_history['HOT'][1] - time_labels: !self.bad_history['HOT'][2] - -- !Range1d: &hot_history_xrange - ref: "hot_history_xrange" - start: 0 - end: 1 - bounds: 'auto' #!!python/tuple [0, 1] - -- !Range1d: &hot_history_yrange - ref: "hot_history_yrange" - start: 0 - end: 1 - bounds: 'auto' #!!python/tuple [-1, 1] - -- !Figure: &hot_history_figure - ref: "hot_history_figure" - x_axis_label: "Date" - x_axis_type: "datetime" - y_axis_label: "Number of HOT pixels" - x_range: *hot_history_xrange - y_range: *hot_history_yrange - tools: 'hover, zoom_in, zoom_out, box_zoom, pan, reset, save' - tooltips: "@time_labels Hot pix: @hot_pixels" - elements: - - {'kind': 'circle', 'x': 'time', 'y': 'hot_pixels', line_width: 5, 'source': *hot_history_source} - -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Low QE Pixels v. Time Figure -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -- !ColumnDataSource: &low_qe_history_source - ref: "low_qe_history_source" - data: - time: !self.bad_history['LOW_QE'][0] - low_qe_pixels: !self.bad_history['LOW_QE'][1] - time_labels: !self.bad_history['LOW_QE'][2] - -- !Range1d: &low_qe_history_xrange - ref: "low_qe_history_xrange" - start: 0 - end: 1 - bounds: 'auto' #!!python/tuple [0, 1] - -- !Range1d: &low_qe_history_yrange - ref: "low_qe_history_yrange" - start: 0 - end: 1 - bounds: 'auto' #!!python/tuple [-1, 1] - -- !Figure: &low_qe_history_figure - ref: "low_qe_history_figure" - x_axis_label: "Date" - x_axis_type: "datetime" - y_axis_label: "Number of LOW QE pixels" - x_range: *low_qe_history_xrange - y_range: *low_qe_history_yrange - tools: 'hover, zoom_in, zoom_out, box_zoom, pan, reset, save' - tooltips: "@time_labels Low QE pix: @low_qe_pixels" - elements: - - {'kind': 'circle', 'x': 'time', 'y': 'low_qe_pixels', line_width: 5, 'source': *low_qe_history_source} - -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Open Pixels v. Time Figure -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -- !ColumnDataSource: &open_history_source - ref: "open_history_source" - data: - time: !self.bad_history['OPEN'][0] - open_pixels: !self.bad_history['OPEN'][1] - time_labels: !self.bad_history['OPEN'][2] - -- !Range1d: &open_history_xrange - ref: "open_history_xrange" - start: 0 - end: 1 - bounds: 'auto' #!!python/tuple [0, 1] - -- !Range1d: &open_history_yrange - ref: "open_history_yrange" - start: 0 - end: 1 - bounds: 'auto' #!!python/tuple [-1, 1] - -- !Figure: &open_history_figure - ref: "open_history_figure" - x_axis_label: "Date" - x_axis_type: "datetime" - y_axis_label: "Number of OPEN pixels" - x_range: *open_history_xrange - y_range: *open_history_yrange - tools: 'hover, zoom_in, zoom_out, box_zoom, pan, reset, save' - tooltips: "@time_labels Open pix: @open_pixels" - elements: - - {'kind': 'circle', 'x': 'time', 'y': 'open_pixels', line_width: 5, 'source': *open_history_source} - -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Open Adj Pixels v. Time Figure -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -- !ColumnDataSource: &adj_open_history_source - ref: "adj_open_history_source" - data: - time: !self.bad_history['ADJ_OPEN'][0] - adj_open_pixels: !self.bad_history['ADJ_OPEN'][1] - time_labels: !self.bad_history['ADJ_OPEN'][2] - -- !Range1d: &adj_open_history_xrange - ref: "adj_open_history_xrange" - start: 0 - end: 1 - bounds: 'auto' #!!python/tuple [0, 1] - -- !Range1d: &adj_open_history_yrange - ref: "adj_open_history_yrange" - start: 0 - end: 1 - bounds: 'auto' #!!python/tuple [-1, 1] - -- !Figure: &adj_open_history_figure - ref: "adj_open_history_figure" - x_axis_label: "Date" - x_axis_type: "datetime" - y_axis_label: "Number of OPEN ADJACENT pixels" - x_range: *adj_open_history_xrange - y_range: *adj_open_history_yrange - tools: 'hover, zoom_in, zoom_out, box_zoom, pan, reset, save' - tooltips: "@time_labels Adj open pix: @adj_open_pixels" - elements: - - {'kind': 'circle', 'x': 'time', 'y': 'adj_open_pixels', line_width: 5, 'source': *adj_open_history_source} - -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# RC Pixels v. Time Figure -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -- !ColumnDataSource: &rc_history_source - ref: "rc_history_source" - data: - time: !self.bad_history['RC'][0] - rc_pixels: !self.bad_history['RC'][1] - time_labels: !self.bad_history['RC'][2] - -- !Range1d: &rc_history_xrange - ref: "rc_history_xrange" - start: 0 - end: 1 - bounds: 'auto' #!!python/tuple [0, 1] - -- !Range1d: &rc_history_yrange - ref: "rc_history_yrange" - start: 0 - end: 1 - bounds: 'auto' #!!python/tuple [-1, 1] - -- !Figure: &rc_history_figure - ref: "rc_history_figure" - x_axis_label: "Date" - x_axis_type: "datetime" - y_axis_label: "Number of RC pixels" - x_range: *rc_history_xrange - y_range: *rc_history_yrange - tools: 'hover, zoom_in, zoom_out, box_zoom, pan, reset, save' - tooltips: "@time_labels RC pix: @rc_pixels" - elements: - - {'kind': 'circle', 'x': 'time', 'y': 'rc_pixels', line_width: 5, 'source': *rc_history_source} - - -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Other Bad Pixels v. Time Figure -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -- !ColumnDataSource: &other_bad_pixel_history_source - ref: "other_bad_pixel_history_source" - data: - time: !self.bad_history['OTHER_BAD_PIXEL'][0] - other_bad_pixel_pixels: !self.bad_history['OTHER_BAD_PIXEL'][1] - time_labels: !self.bad_history['OTHER_BAD_PIXEL'][2] - -- !Range1d: &other_bad_pixel_history_xrange - ref: "other_bad_pixel_history_xrange" - start: 0 - end: 1 - bounds: 'auto' #!!python/tuple [0, 1] - -- !Range1d: &other_bad_pixel_history_yrange - ref: "other_bad_pixel_history_yrange" - start: 0 - end: 1 - bounds: 'auto' #!!python/tuple [-1, 1] - -- !Figure: &other_bad_pixel_history_figure - ref: "other_bad_pixel_history_figure" - x_axis_label: "Date" - x_axis_type: "datetime" - y_axis_label: "Number of OTHER BAD pixels" - x_range: *other_bad_pixel_history_xrange - y_range: *other_bad_pixel_history_yrange - tools: 'hover, zoom_in, zoom_out, box_zoom, pan, reset, save' - tooltips: '@time_labels Other bad pix: @other_bad_pixel_pixels' - elements: - - {'kind': 'circle', 'x': 'time', 'y': 'other_bad_pixel_pixels', line_width: 5, 'source': *other_bad_pixel_history_source} - - -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Telegraph Pixels v. Time Figure -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -- !ColumnDataSource: &telegraph_history_source - ref: "telegraph_history_source" - data: - time: !self.bad_history['TELEGRAPH'][0] - telegraph_pixels: !self.bad_history['TELEGRAPH'][1] - time_labels: !self.bad_history['TELEGRAPH'][2] - -- !Range1d: &telegraph_history_xrange - ref: "telegraph_history_xrange" - start: 0 - end: 1 - bounds: 'auto' #!!python/tuple [0, 1] - -- !Range1d: &telegraph_history_yrange - ref: "telegraph_history_yrange" - start: 0 - end: 1 - bounds: 'auto' #!!python/tuple [-1, 1] - -- !Figure: &telegraph_history_figure - ref: "telegraph_history_figure" - x_axis_label: "Date" - x_axis_type: "datetime" - y_axis_label: "Number of TELEGRAPH pixels" - x_range: *telegraph_history_xrange - y_range: *telegraph_history_yrange - tools: 'hover, zoom_in, zoom_out, box_zoom, pan, reset, save' - tooltips: "@time_labels Telegraph pix: @telegraph_pixels" - elements: - - {'kind': 'circle', 'x': 'time', 'y': 'telegraph_pixels', line_width: 5, 'source': *telegraph_history_source} - - -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Show positions of bad pixels found in darks -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -- !ColumnDataSource: &dark_position_source - ref: "dark_position_source" - data: - bad_type: !self.latest_bad_from_dark_type - x_coord: !self.latest_bad_from_dark_x - y_coord: !self.latest_bad_from_dark_y - -- !Range1d: &dark_position_xrange - ref: "dark_position_xrange" - start: 0 - end: 2047 - -- !Range1d: &dark_position_yrange - ref: "dark_position_yrange" - start: 0 - end: 2047 - -- !CDSView: &figure_hot_view - source: *dark_position_source - filters: - - !GroupFilter: - column_name: "bad_type" - group: "HOT" - -- !CDSView: &figure_rc_view - source: *dark_position_source - filters: - - !GroupFilter: - column_name: "bad_type" - group: "RC" - -- !CDSView: &figure_other_view - source: *dark_position_source - filters: - - !GroupFilter: - column_name: "bad_type" - group: "OTHER_BAD_PIXEL" - -- !CDSView: &figure_telegraph_view - source: *dark_position_source - filters: - - !GroupFilter: - column_name: "bad_type" - group: "TELEGRAPH" - -- !Figure: &dark_position_figure - ref: "dark_position_figure" - x_axis_label: "X (pixel)" - y_axis_label: "Y (pixel)" - x_range: *dark_position_xrange - y_range: *dark_position_yrange - tools: 'hover, zoom_in, zoom_out, box_zoom, pan, reset, save' - tooltips: "(x,y): (@x_coord, @y_coord) Type: @bad_type" - elements: - - {'kind': 'circle', 'x': 'x_coord', 'y': 'y_coord', 'source': *dark_position_source, 'color': 'red', 'alpha': 0.5, 'legend': 'Hot', 'size': 6, 'view': *figure_hot_view} - - {'kind': 'circle', 'x': 'x_coord', 'y': 'y_coord', 'source': *dark_position_source, 'color': 'blue', 'alpha': 0.5, 'legend': 'RC', 'size': 6, 'view': *figure_rc_view} - - {'kind': 'circle', 'x': 'x_coord', 'y': 'y_coord', 'source': *dark_position_source, 'color': 'green', 'alpha': 0.5, 'legend': 'Other', 'size': 6, 'view': *figure_other_view} - - {'kind': 'circle', 'x': 'x_coord', 'y': 'y_coord', 'source': *dark_position_source, 'color': 'black', 'alpha': 0.5, 'legend': 'Telegraph', 'size': 6, 'view': *figure_telegraph_view} - -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Show positions of bad pixels found in flats -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -- !ColumnDataSource: &flat_position_source - ref: "flat_position_source" - data: - bad_type: !self.latest_bad_from_flat_type - x_coord: !self.latest_bad_from_flat_x - y_coord: !self.latest_bad_from_flat_y - -- !Range1d: &flat_position_xrange - ref: "flat_position_xrange" - start: 0 - end: 2047 - -- !Range1d: &flat_position_yrange - ref: "flat_position_yrange" - start: 0 - end: 2047 - -- !CDSView: &figure_dead_view - source: *flat_position_source - filters: - - !GroupFilter: - column_name: "bad_type" - group: "DEAD" - -- !CDSView: &figure_open_view - source: *flat_position_source - filters: - - !GroupFilter: - column_name: "bad_type" - group: "OPEN" - -- !CDSView: &figure_adj_open_view - source: *flat_position_source - filters: - - !GroupFilter: - column_name: "bad_type" - group: "ADJ_OPEN" - -- !CDSView: &figure_low_qe_view - source: *flat_position_source - filters: - - !GroupFilter: - column_name: "bad_type" - group: "LOW_QE" - -- !Figure: &flat_position_figure - ref: "flat_position_figure" - x_axis_label: "X (pixel)" - y_axis_label: "Y (pixel)" - x_range: *flat_position_xrange - y_range: *flat_position_yrange - tools: 'hover, zoom_in, zoom_out, box_zoom, pan, reset, save' - tooltips: "(x, y): (@x_coord, @y_coord) Type: @bad_type" - elements: - - {'kind': 'circle', 'x': 'x_coord', 'y': 'y_coord', 'source': *flat_position_source, 'color': 'red', 'alpha': 0.5, 'legend': 'Dead', 'size': 6, 'view': *figure_dead_view} - - {'kind': 'circle', 'x': 'x_coord', 'y': 'y_coord', 'source': *flat_position_source, 'color': 'blue', 'alpha': 0.5, 'legend': 'Open', 'size': 6, 'view': *figure_open_view} - - {'kind': 'circle', 'x': 'x_coord', 'y': 'y_coord', 'source': *flat_position_source, 'color': 'green', 'alpha': 0.5, 'legend': 'Adj Open', 'size': 6, 'view': *figure_adj_open_view} - - {'kind': 'circle', 'x': 'x_coord', 'y': 'y_coord', 'source': *flat_position_source, 'color': 'black', 'alpha': 0.5, 'legend': 'Low QE', 'size': 6, 'view': *figure_low_qe_view} - - -#- !Document: -# - !row: -# - *dark_position_figure -# - *flat_position_figure -# - !row: -# - *dead_history_figure -# - *hot_history_figure -# - !row: -# - *low_qe_history_figure -# - *rc_history_figure -# - !row: -# - *open_history_figure -# - *adj_open_history_figure -# - !row: -# - *telegraph_history_figure -# - *other_bad_pixel_history_figure \ No newline at end of file From 4870fc6e13974f966505e15e27bfd04f8c2ffb42 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Tue, 14 Mar 2023 09:48:56 -0400 Subject: [PATCH 172/449] creating anomaly_db_transfer --- jwql/website/apps/jwql/admin.py | 4 +- jwql/website/apps/jwql/anomaly_db_transfer.py | 96 +++++++++++++++++++ jwql/website/apps/jwql/models.py | 2 + 3 files changed, 100 insertions(+), 2 deletions(-) create mode 100755 jwql/website/apps/jwql/anomaly_db_transfer.py diff --git a/jwql/website/apps/jwql/admin.py b/jwql/website/apps/jwql/admin.py index 15e08899a..f9525cef7 100644 --- a/jwql/website/apps/jwql/admin.py +++ b/jwql/website/apps/jwql/admin.py @@ -47,9 +47,9 @@ class RootFileInfoAdmin(admin.ModelAdmin): @admin.register(Anomalies) class AnomaliesAdmin(admin.ModelAdmin): - list_display = ('root_file_info', 'cosmic_ray_shower', 'diffraction_spike', 'excessive_saturation', 'guidestar_failure', 'persistence', 'crosstalk', 'data_transfer_error', + list_display = ('root_file_info', 'flag_date', 'user', 'cosmic_ray_shower', 'diffraction_spike', 'excessive_saturation', 'guidestar_failure', 'persistence', 'crosstalk', 'data_transfer_error', 'ghost', 'snowball', 'column_pull_up', 'column_pull_down', 'dominant_msa_leakage', 'dragons_breath', 'mrs_glow', 'mrs_zipper', 'internal_reflection', 'optical_short', 'row_pull_up', 'row_pull_down', 'lrs_contamination', 'tree_rings', 'scattered_light', 'claws', 'wisps', 'tilt_event', 'light_saber', 'other') - list_filter = ('cosmic_ray_shower', 'diffraction_spike', 'excessive_saturation', 'guidestar_failure', 'persistence', 'crosstalk', 'data_transfer_error', + list_filter = ('flag_date', 'user', 'cosmic_ray_shower', 'diffraction_spike', 'excessive_saturation', 'guidestar_failure', 'persistence', 'crosstalk', 'data_transfer_error', 'ghost', 'snowball', 'column_pull_up', 'column_pull_down', 'dominant_msa_leakage', 'dragons_breath', 'mrs_glow', 'mrs_zipper', 'internal_reflection', 'optical_short', 'row_pull_up', 'row_pull_down', 'lrs_contamination', 'tree_rings', 'scattered_light', 'claws', 'wisps', 'tilt_event', 'light_saber', 'other', 'root_file_info') diff --git a/jwql/website/apps/jwql/anomaly_db_transfer.py b/jwql/website/apps/jwql/anomaly_db_transfer.py new file mode 100755 index 000000000..3cff18749 --- /dev/null +++ b/jwql/website/apps/jwql/anomaly_db_transfer.py @@ -0,0 +1,96 @@ +#! /usr/bin/env python + +"""Script to transfer postgres anomaly data to django models + +Authors +------- + + - Bradley Sappington + +Use +--- + + This module is called as follows: + :: + $ python anomaly_db_transfer.py + + +Dependencies +------------ + The user must have a configuration file named ``config.json`` + placed in the ``jwql`` directory. +""" + +import logging +import django +import os +import datetime + +# These lines are needed in order to use the Django models in a standalone +# script (as opposed to code run as a result of a webpage request). If these +# lines are not run, the script will crash when attempting to import the +# Django models in the line below. +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jwql.website.jwql_proj.settings") +django.setup() +from django.conf import settings +from django.core.exceptions import ObjectDoesNotExist +from django.utils.timezone import make_aware + +from jwql.database import database_interface as di +from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE +from jwql.utils.logging_functions import log_info, log_fail +from jwql.utils.monitor_utils import initialize_instrument_monitor +from jwql.utils.utils import filesystem_path, get_config +from jwql.website.apps.jwql.models import RootFileInfo, Anomalies + + + + + +@log_info +@log_fail +def transfer_anomalies(): + + instruments = ['nircam', 'miri', 'nirspec', 'niriss', 'fgs'] + for instrument in instruments: + # Get the anomalies for this instrument + table = getattr(di, '{}Anomaly'.format(JWST_INSTRUMENT_NAMES_MIXEDCASE[instrument.lower()])) + query = di.session.query(table) + logging.info('anomalies for {} are: {}'.format(instrument, table.columns)) + table_keys = ['id', 'root_file_info', 'flag_date', 'user'] + table_keys += table.columns + table_keys = list(map(lambda x: x.lower(), table_keys)) + updated = 0 + + anomaly_dict = {} + rows = query.statement.execute().fetchall() + for rowx, row in enumerate(rows): + anomaly_dict = {} + for ix, value in enumerate(row): + if (table_keys[ix] == 'flag_date'): + #avoid warning for native DateTime when timezone is active + value = make_aware(value) + anomaly_dict[table_keys[ix]] = value + + root_file_info_name = anomaly_dict['root_file_info'] + try: + root_file_info_instance = RootFileInfo.objects.get(root_name=root_file_info_name) + except ObjectDoesNotExist: + logging.info('No root_file_info for {}'.format(root_file_info_name)) + continue + + try: + del(anomaly_dict['id']) + del(anomaly_dict['root_file_info']) + anomalies, anomaly_created = Anomalies.objects.get_or_create(root_file_info=root_file_info_instance, + defaults=anomaly_dict) + if anomaly_created: + updated += 1 + except Exception as e: + logging.warning('Failed to create {} with exception {}'.format(root_file_info_name, e)) + logging.info('Transferred {} anomalies for {}'.format(updated, instrument)) + +if __name__ == '__main__': + module = os.path.basename(__file__).strip('.py') + start_time, log_file = initialize_instrument_monitor(module) + transfer_anomalies() diff --git a/jwql/website/apps/jwql/models.py b/jwql/website/apps/jwql/models.py index 72cda5d31..ee639afd7 100644 --- a/jwql/website/apps/jwql/models.py +++ b/jwql/website/apps/jwql/models.py @@ -133,6 +133,8 @@ class Anomalies(models.Model): on_delete=models.CASCADE, primary_key=True, ) + flag_date = models.DateTimeField(help_text="flag date", null=True, blank=True) + user = models.CharField(max_length=50, help_text="user", default='', null=True, blank=True) cosmic_ray_shower = models.BooleanField(default=False) diffraction_spike = models.BooleanField(default=False) excessive_saturation = models.BooleanField(default=False) From f12fa1739ec825987095455924ee8a6927a4bc7a Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 14 Mar 2023 10:58:00 -0400 Subject: [PATCH 173/449] fix data going into filesystem database --- jwql/jwql_monitors/monitor_filesystem.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py index afdd603ff..6940619cd 100755 --- a/jwql/jwql_monitors/monitor_filesystem.py +++ b/jwql/jwql_monitors/monitor_filesystem.py @@ -649,11 +649,13 @@ def update_characteristics_database(char_info): # Add data to filesystem_instrument table for instrument in ['nircam', 'niriss', 'nirspec', 'miri']: + filter_list = [e[0] for e in obs['nircam']] + value_list= [e[1] for e in obs['nircam']] new_record = {} new_record['date'] = now new_record['instrument'] = instrument - new_record['filter_pupil'] = list(char_info[instrument].keys()) - new_record['obs_per_filter_pupil'] = list(char_info[instrument].values()) + new_record['filter_pupil'] = filter_list + new_record['obs_per_filter_pupil'] = value_list engine.execute(FilesystemCharacteristics.__table__.insert(), new_record) session.commit() From aa53a267b42976d755e78b6456d881fc501cc2d8 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 14 Mar 2023 11:30:35 -0400 Subject: [PATCH 174/449] Fix sorting of log table --- jwql/website/apps/jwql/bokeh_dashboard.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/jwql/website/apps/jwql/bokeh_dashboard.py b/jwql/website/apps/jwql/bokeh_dashboard.py index 2cd0ba04a..eb28a952e 100644 --- a/jwql/website/apps/jwql/bokeh_dashboard.py +++ b/jwql/website/apps/jwql/bokeh_dashboard.py @@ -301,13 +301,14 @@ def dashboard_monitor_tracking(self): if not pd.isnull(self.delta_t): data = data[(data['start_time'] >= self.date - self.delta_t) & (data['start_time'] <= self.date)] + # Sort the data by start_time before translating into strings + data.sort_values(by='start_time', ascending=False, inplace=True) + + # Now translate times to strings data['start_time'] = data['start_time'].map(lambda x: x.strftime('%m-%d-%Y %H:%M:%S')) data['end_time'] = data['end_time'].map(lambda x: x.strftime('%m-%d-%Y %H:%M:%S')) - # data = data.drop(columns='affected_tables') - table_values = data.sort_values(by='start_time', ascending=False).values - table_columns = data.columns.values - return table_columns, table_values + return data.columns.values, data.values def make_panel(self, x_value, top, instrument, title, x_axis_label): """Make tab panel for tablulated figure. From 8bb2e5d289dfa4cdad29bccf2d672eeecd696a6a Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Tue, 14 Mar 2023 11:57:04 -0400 Subject: [PATCH 175/449] Not using explicit directory paths --- jwql/shared_tasks/run_pipeline.py | 11 +++++++---- jwql/shared_tasks/shared_tasks.py | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 83294ed06..9a196ecdd 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -222,7 +222,10 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T if __name__ == '__main__': - with open("/internal/data1/outputs/ops/calibrated_data/general_status.txt", "w") as status_file: + status_dir = os.path.join(get_config()['outputs'], 'calibrated_data') + general_status_file = os.path.join(status_dir, "general_status.txt") + + with open(general_status_file, "w") as status_file: status_file.write("Started at {}\n".format(time.ctime())) status_file.write("\targv={}\n".format(sys.argv)) @@ -242,18 +245,18 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T parser.add_argument('short_name', metavar='NAME', type=str, help=name_help) parser.add_argument('max_cores', metavar='CORES', type=str, help=cores_help) - with open("/internal/data1/outputs/ops/calibrated_data/general_status.txt", "a+") as status_file: + with open(general_status_file, "a+") as status_file: status_file.write("Created argument parser at {}\n".format(time.ctime())) try: args = parser.parse_args() except Exception as e: - with open("/internal/data1/outputs/ops/calibrated_data/general_status.txt", "a+") as status_file: + with open(general_status_file, "a+") as status_file: status_file.write("Error parsing arguments.\n") status_file.write("{}".format(e)) raise e - with open("/internal/data1/outputs/ops/calibrated_data/general_status.txt", "a+") as status_file: + with open(general_status_file, "a+") as status_file: status_file.write("Finished parsing args at {}\n".format(time.ctime())) input_file = args.input_file diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index 507d9ffee..8811c00de 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -148,7 +148,7 @@ def some_function(some_arguments): celery_app.conf.update(worker_prefetch_multiplier=1) celery_app.conf.update(task_acks_late=True) celery_app.conf.update(worker_concurrency=1) -celery_app.conf.broker_transport_options = {'visibility_timeout': 86400} +celery_app.conf.broker_transport_options = {'visibility_timeout': 14400} def only_one(function=None, key="", timeout=None): From 795a1deb809ebcc73b2501d7d45a6c9d23b9dac2 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Tue, 14 Mar 2023 12:29:00 -0400 Subject: [PATCH 176/449] change to update_or_create for duplicate entires --- jwql/website/apps/jwql/anomaly_db_transfer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jwql/website/apps/jwql/anomaly_db_transfer.py b/jwql/website/apps/jwql/anomaly_db_transfer.py index 3cff18749..68778e281 100755 --- a/jwql/website/apps/jwql/anomaly_db_transfer.py +++ b/jwql/website/apps/jwql/anomaly_db_transfer.py @@ -82,8 +82,8 @@ def transfer_anomalies(): try: del(anomaly_dict['id']) del(anomaly_dict['root_file_info']) - anomalies, anomaly_created = Anomalies.objects.get_or_create(root_file_info=root_file_info_instance, - defaults=anomaly_dict) + anomalies, anomaly_created = Anomalies.objects.update_or_create(root_file_info=root_file_info_instance, + defaults=anomaly_dict) if anomaly_created: updated += 1 except Exception as e: From bb416fe1ae9e087fba638f85c69a310ff8361b55 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Tue, 14 Mar 2023 14:45:04 -0400 Subject: [PATCH 177/449] Modified test script to do both pipelines at the file level, rather than one then the other, and added exception catching --- jwql/tests/test_redis_celery.py | 69 +++++++++++++++++---------------- 1 file changed, 36 insertions(+), 33 deletions(-) diff --git a/jwql/tests/test_redis_celery.py b/jwql/tests/test_redis_celery.py index 1e61394b7..38e939b77 100644 --- a/jwql/tests/test_redis_celery.py +++ b/jwql/tests/test_redis_celery.py @@ -71,39 +71,42 @@ def get_instrument(file_name): if __name__ == "__main__": config = get_config() p = Path(config['test_data']) + + for file in p.rglob("*uncal.fits"): + print("Testing cal pipeline") + with TemporaryDirectory() as working_dir: + try: + print("Running in {}".format(working_dir)) + file_name = os.path.basename(file) + if "gs-" in file_name: + print("\tSkipping guide star file {}".format(file_name)) + continue + print("\tCopying {}".format(file)) + copy_files([file], working_dir) + cal_file = os.path.join(working_dir, file_name) + print("\t\tCalibrating {}".format(cal_file)) + instrument = get_instrument(file_name) + outputs = run_pipeline(cal_file, "uncal", "all", instrument) + print("\t\tDone {}".format(file)) + except Exception as e: + print("ERROR: {}".format(e)) - # Test the standard pipeline task - print("Testing cal pipeline") - with TemporaryDirectory() as working_dir: - print("Running in {}".format(working_dir)) - for file in p.rglob("*uncal.fits"): - file_name = os.path.basename(file) - if "gs-" in file_name: - print("\tSkipping guide star file {}".format(file_name)) - continue - print("\tCopying {}".format(file)) - copy_files([file], working_dir) - cal_file = os.path.join(working_dir, file_name) - print("\t\tCalibrating {}".format(cal_file)) - instrument = get_instrument(file_name) - outputs = run_pipeline(cal_file, "uncal", "all", instrument) - print("\t\tDone {}".format(file)) - - # Test the jump pipeline task - print("Testing jump pipeline") - with TemporaryDirectory() as working_dir: - print("Running in {}".format(working_dir)) - for file in p.rglob("*uncal.fits"): - file_name = os.path.basename(file) - if "gs-" in file_name: - print("\tSkipping guide star file {}".format(file_name)) - continue - print("\tCopying {}".format(file)) - copy_files([file], working_dir) - cal_file = os.path.join(working_dir, file_name) - print("\t\tCalibrating {}".format(cal_file)) - instrument = get_instrument(file_name) - outputs = run_pipeline(cal_file, "uncal", "all", instrument, jump_pipe=True) - print("\t\tDone {}".format(file)) + print("Testing jump pipeline") + with TemporaryDirectory() as working_dir: + try: + print("Running in {}".format(working_dir)) + file_name = os.path.basename(file) + if "gs-" in file_name: + print("\tSkipping guide star file {}".format(file_name)) + continue + print("\tCopying {}".format(file)) + copy_files([file], working_dir) + cal_file = os.path.join(working_dir, file_name) + print("\t\tCalibrating {}".format(cal_file)) + instrument = get_instrument(file_name) + outputs = run_pipeline(cal_file, "uncal", "all", instrument, jump_pipe=True) + print("\t\tDone {}".format(file)) + except Exception as e: + print("ERROR: {}".format(e)) print("Done test") From de82176d1ee65a51e0db899853e05fe3e3ffc561 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Tue, 14 Mar 2023 15:19:13 -0400 Subject: [PATCH 178/449] When output extensions is 'all', actually write that out instead of looking for an _all.fits file --- jwql/shared_tasks/shared_tasks.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index 8811c00de..7fb0fa8bd 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -297,6 +297,9 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, cmd_name = os.path.join(current_dir, "run_pipeline.py") outputs = ",".join(ext_or_exts) result_file = os.path.join(cal_dir, short_name+"_status.txt") + if ext_or_exts == "all": + ext_or_exts = ["dq_init", "saturation", "superbias", "refpix", "linearity", + "persistence", "dark_current", "jump", "rate"] calibrated_files = ["{}_{}.fits".format(short_name, ext) for ext in ext_or_exts] cores = 'all' From 61ef988f717a4124367128b5574771a8b6957aca Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Tue, 14 Mar 2023 15:30:50 -0400 Subject: [PATCH 179/449] Trying once again with the 'all' output --- jwql/shared_tasks/shared_tasks.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index 7fb0fa8bd..fd8b17ff2 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -298,9 +298,15 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, outputs = ",".join(ext_or_exts) result_file = os.path.join(cal_dir, short_name+"_status.txt") if ext_or_exts == "all": - ext_or_exts = ["dq_init", "saturation", "superbias", "refpix", "linearity", - "persistence", "dark_current", "jump", "rate"] - calibrated_files = ["{}_{}.fits".format(short_name, ext) for ext in ext_or_exts] + logging.info("All outputs requested") + out_exts = ["dq_init", "saturation", "superbias", "refpix", "linearity", + "persistence", "dark_current", "jump", "rate", "group_scale", + "firstframe", "lastframe", "rscd"] + calibrated_files = ["{}_{}.fits".format(short_name, ext) for ext in out_exts] + logging.info("Requesting {}".format(calibrated_files)) + else: + calibrated_files = ["{}_{}.fits".format(short_name, ext) for ext in ext_or_exts] + logging.info("Requesting {}".format(calibrated_files)) cores = 'all' status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, input_file, @@ -343,6 +349,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, raise ValueError("Pipeline Failed") for file in calibrated_files: + logging.info("Checking for output {}".format(file)) if not os.path.isfile(os.path.join(cal_dir, file)): logging.error("ERROR: {} not found".format(file)) raise FileNotFoundError(file) From e2fb9e6cb7a9db816532e9952bfbe10656640da3 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Tue, 14 Mar 2023 15:35:04 -0400 Subject: [PATCH 180/449] Remembering how to check whether a value is in a list --- jwql/shared_tasks/shared_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index fd8b17ff2..750b979d7 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -297,7 +297,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, cmd_name = os.path.join(current_dir, "run_pipeline.py") outputs = ",".join(ext_or_exts) result_file = os.path.join(cal_dir, short_name+"_status.txt") - if ext_or_exts == "all": + if "all" in ext_or_exts: logging.info("All outputs requested") out_exts = ["dq_init", "saturation", "superbias", "refpix", "linearity", "persistence", "dark_current", "jump", "rate", "group_scale", From 77023dc18f9a598e6bf3e6f6e6bb87a3d0864c30 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Tue, 14 Mar 2023 15:39:40 -0400 Subject: [PATCH 181/449] Asking for the proper set in an 'all' situation --- jwql/shared_tasks/shared_tasks.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index 750b979d7..516738ed0 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -300,8 +300,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, if "all" in ext_or_exts: logging.info("All outputs requested") out_exts = ["dq_init", "saturation", "superbias", "refpix", "linearity", - "persistence", "dark_current", "jump", "rate", "group_scale", - "firstframe", "lastframe", "rscd"] + "persistence", "dark_current", "jump", "rate"] calibrated_files = ["{}_{}.fits".format(short_name, ext) for ext in out_exts] logging.info("Requesting {}".format(calibrated_files)) else: From 1dad10dde3d87b84d72ed9022086142dd147249e Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Wed, 15 Mar 2023 11:00:02 -0400 Subject: [PATCH 182/449] Update docstring --- jwql/website/apps/jwql/data_containers.py | 28 ++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 3ca26324f..59d715bcf 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -439,11 +439,37 @@ def get_current_flagged_anomalies(rootname, instrument, n_match=1): """Return a list of currently flagged anomalies for the given ``rootname`` + This function may be used to retrieve the current anomalies + for single files or sets of files in an exposure group. Group + anomalies are determined by comparing current anomaly settings + within the group to the ``n_match`` parameter. + + For single files, ``n_match`` should be set to 1: any anomaly present for + the file is a current anomaly. For groups of files, ``n_match`` should be + set to the total number of files in the group. In this case, an + anomaly is returned as a current anomaly for the group only if it + is marked for every file in the group, i.e. there are ``n_match`` + True values for that anomaly. + + Note that the anomaly database may store multiple entries for + a rootfile, but only the last entry is considered current. Also + note that not every rootfile has an entry in the database: it only + appears if an anomaly has been marked for it at some point. The + ``n_match`` parameter is required as an argument for groups because + there is no way to determine from the anomaly table how many files + belong to a group. + Parameters ---------- rootname : str The rootname of interest (e.g. - ``jw86600008001_02101_00001_guider2/``) + ``jw86600008001_02101_00001_guider2/``). May be a file + rootname or an exposure group root. + instrument : str + The instrument corresponding to the file. + n_match : int + The number of root files that must match in order to report + an anomaly as current. Returns ------- From 363807cb59309de2b41584bc385158e324707506 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Wed, 15 Mar 2023 11:00:30 -0400 Subject: [PATCH 183/449] Add app name to Meta for Anomalies model for doc support --- jwql/website/apps/jwql/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jwql/website/apps/jwql/models.py b/jwql/website/apps/jwql/models.py index 72cda5d31..497360894 100644 --- a/jwql/website/apps/jwql/models.py +++ b/jwql/website/apps/jwql/models.py @@ -162,6 +162,7 @@ class Anomalies(models.Model): other = models.BooleanField(default=False) class Meta: + app_label = 'jwql' ordering = ['-root_file_info'] def __str__(self): From 65db3fae29c474970845f3a97bb956dd6198d48b Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Wed, 15 Mar 2023 12:30:49 -0400 Subject: [PATCH 184/449] Update docstrings --- jwql/website/apps/jwql/data_containers.py | 7 ++++--- jwql/website/apps/jwql/views.py | 24 ++++++++++++++++++++++- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 59d715bcf..09a7c4ea5 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -462,9 +462,10 @@ def get_current_flagged_anomalies(rootname, instrument, n_match=1): Parameters ---------- rootname : str - The rootname of interest (e.g. - ``jw86600008001_02101_00001_guider2/``). May be a file - rootname or an exposure group root. + The rootname of interest. May be a file rootname + (e.g. ``jw86600008001_02101_00001_guider2``) or an exposure + group root (e.g. ``jw86600008001_02101_00001``). The rootname + is matched as a starts-with filter on the anomaly database. instrument : str The instrument corresponding to the file. n_match : int diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 4dae2a6a3..6ac214bfa 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -1039,7 +1039,18 @@ def explore_image_ajax(request, inst, file_root, filetype, scaling="log", low_li def save_image_group_ajax(request): - """Save the latest selected group type in the session.""" + """Save the latest selected group type in the session. + + Parameters + ---------- + request : HttpRequest + The incoming request. + + Returns + ------- + JsonResponse + Object containing the group value as set in the session (key: 'item'). + """ image_group = request.GET['group_type'] request.session['image_group'] = image_group context = {'item': request.session['image_group']} @@ -1047,7 +1058,18 @@ def save_image_group_ajax(request): def save_image_sort_ajax(request): + """Save the latest selected sort type in the session. + Parameters + ---------- + request : HttpRequest + The incoming request. + + Returns + ------- + JsonResponse + Object containing the sort value as set in the session (key: 'item'). + """ # a string of the form " 'rootname1'='expstart1', 'rootname2'='expstart2', ..." image_sort = request.GET['sort_type'] From 320c5c8ea1eb8d6527cb80a1a0d716305c9b5606 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Wed, 15 Mar 2023 15:06:26 -0400 Subject: [PATCH 185/449] add docstring --- jwql/website/apps/jwql/anomaly_db_transfer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/anomaly_db_transfer.py b/jwql/website/apps/jwql/anomaly_db_transfer.py index 68778e281..e16e26ff3 100755 --- a/jwql/website/apps/jwql/anomaly_db_transfer.py +++ b/jwql/website/apps/jwql/anomaly_db_transfer.py @@ -50,7 +50,9 @@ @log_info @log_fail def transfer_anomalies(): - + """Update the the Django anomalies model with all information in the existing postgres database. + + """ instruments = ['nircam', 'miri', 'nirspec', 'niriss', 'fgs'] for instrument in instruments: # Get the anomalies for this instrument From db362dbe8d1498e7dd673fc8f71ddcdfedd1b10b Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 16 Mar 2023 09:49:45 -0400 Subject: [PATCH 186/449] Make database entries robust against unknown file suffixes --- jwql/jwql_monitors/monitor_filesystem.py | 21 ++++++++-- jwql/website/apps/jwql/bokeh_dashboard.py | 50 +++++++++++++++++++++++ 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py index 6940619cd..9b7fbe5d7 100755 --- a/jwql/jwql_monitors/monitor_filesystem.py +++ b/jwql/jwql_monitors/monitor_filesystem.py @@ -46,6 +46,7 @@ from bokeh.palettes import Category20_20 as palette from bokeh.plotting import figure, output_file, save import numpy as np +from sqlalchemy.exc import DataError from jwql.database.database_interface import engine from jwql.database.database_interface import session @@ -218,9 +219,11 @@ def get_area_stats(central_storage_dict): """ logging.info('Gathering stats for central storage area') - arealist = ['logs', 'outputs', 'test', 'preview_images', 'thumbnails', 'all'] + #arealist = ['logs', 'outputs', 'test', 'preview_images', 'thumbnails', 'all'] + arealist = ['outputs'] counteddirs = [] + skipped_files = [] sums = 0 # to be used to count 'all' for area in arealist: @@ -263,6 +266,8 @@ def get_area_stats(central_storage_dict): if exists: filesize = os.path.getsize(file_path) sums += filesize + else: + skipped_files.append(filename) use = sums / (1024 ** 4) else: for dirpath, _, files in os.walk(fullpath): @@ -274,10 +279,12 @@ def get_area_stats(central_storage_dict): filesize = os.path.getsize(file_path) used += filesize sums += filesize + else: + skipped_files.append(filename) use = used / (1024 ** 4) central_storage_dict[area]['used'] = use - return central_storage_dict + return central_storage_dict, skipped_files def get_observation_characteristics(): @@ -690,8 +697,14 @@ def update_database(general_results_dict, instrument_results_dict, central_stora new_record['filetype'] = filetype new_record['count'] = instrument_results_dict[instrument][filetype]['count'] new_record['size'] = instrument_results_dict[instrument][filetype]['size'] - engine.execute(FilesystemInstrument.__table__.insert(), new_record) - session.commit() + + # Protect against updated enum options that have not been propagated to + # the table definition + try: + engine.execute(FilesystemInstrument.__table__.insert(), new_record) + session.commit() + except DataError as e: + logging.error(e) # Add data to central_storage table arealist = ['logs', 'outputs', 'test', 'preview_images', 'thumbnails', 'all'] diff --git a/jwql/website/apps/jwql/bokeh_dashboard.py b/jwql/website/apps/jwql/bokeh_dashboard.py index eb28a952e..74849e495 100644 --- a/jwql/website/apps/jwql/bokeh_dashboard.py +++ b/jwql/website/apps/jwql/bokeh_dashboard.py @@ -152,6 +152,56 @@ def __init__(self, delta_t=None): now = dt.now() self.date = pd.Timestamp('{}-{}-{}'.format(now.year, now.month, now.day)) + def dashboard_central_store_data_volume(self): + """Create trending plot of data volume for various areas on central store + + Returns + ------- + tabs : bokeh.models.widgets.widget.Widget + A figure with tabs for each central store area + """ + # Plot total data volume and available disk space versus time + results = session.query(CentralStore.date, CentralStore.size, CentralStore.available).all() + + # Initialize plot + dates, total_sizes, availables = zip(*results) + plot = figure(tools='pan,box_zoom,wheel_zoom,reset,save', + x_axis_type='datetime', + title='Central Store stats', + x_axis_label='Date', + y_axis_label='Size TB') + + plot.line(dates, total_sizes, legend='Total size', line_color='red') + plot.circle(dates, total_sizes, color='red') + plot.line(dates, availables, legend='Free', line_color='blue') + plot.circle(dates, availables, color='blue') + + # This part of the plot should cycle through areas and plot area used values vs. date + arealist = ['logs', 'outputs', 'test', 'preview_images', 'thumbnails', 'all'] + colors = itertools.cycle(palette) + for area, color in zip(arealist, colors): + + # Query for used sizes + results = session.query(CentralStore.date, CentralStore.used).filter(CentralStore.area == area) + + # Group by date + if results: + results_dict = defaultdict(int) + for date, value in results: + results_dict[date] += value + + # Parse results so they can be easily plotted + dates = list(results_dict.keys()) + values = list(results_dict.values()) + + # Plot the results + plot.line(dates, values, legend='{} files'.format(area), line_color=color) + plot.circle(dates, values, color=color) + + session.close() + return plot + + def dashboard_filetype_bar_chart(self): """Build bar chart of files based off of type From c37539a0667375bc39a6d8fc1bce7ae407fb4a6e Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Thu, 16 Mar 2023 17:02:19 -0400 Subject: [PATCH 187/449] Refactor repeated code into functions --- jwql/website/apps/jwql/static/js/jwql.js | 141 +++++++++++------- .../apps/jwql/templates/view_exposure.html | 6 +- .../apps/jwql/templates/view_image.html | 6 +- 3 files changed, 93 insertions(+), 60 deletions(-) diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index 9d9a264b8..79f2715c7 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -34,24 +34,22 @@ document.getElementById("view_file_type").setAttribute('data-current-suffix', type); // Clean the input parameters - num_ints = num_ints.replace(/'/g, '"'); - num_ints = num_ints.replace(/'/g, '"'); + num_ints = clean_input_parameter(num_ints); num_ints = JSON.parse(num_ints); // Get the available integration jpg numbers - available_ints = available_ints.replace(/'/g, '"'); - available_ints = available_ints.replace(/'/g, '"'); + available_ints = clean_input_parameter(available_ints); available_ints = JSON.parse(available_ints)[type]; // Get the total number of integrations - total_ints = total_ints.replace(/'/g, '"'); - total_ints = total_ints.replace(/'/g, '"'); + total_ints = clean_input_parameter(total_ints); total_ints = JSON.parse(total_ints); // Update the APT parameters - document.getElementById("proposal").innerHTML = group_root.slice(2,7); - document.getElementById("obs_id").innerHTML = group_root.slice(7,10); - document.getElementById("visit_id").innerHTML = group_root.slice(10,13); + var parsed_name = parse_filename(group_root); + document.getElementById("proposal").innerHTML = parsed_name.proposal; + document.getElementById("obs_id").innerHTML = parsed_name.obs_id; + document.getElementById("visit_id").innerHTML = parsed_name.visit_id; var detector_list = detectors.split(','); for (let i = 0; i < detector_list.length; i++) { @@ -64,7 +62,7 @@ // Show the appropriate image var img = document.getElementById("image_viewer_" + detector); - var jpg_filepath = '/static/preview_images/' + group_root.slice(0,7) + + var jpg_filepath = '/static/preview_images/' + parsed_name.program + '/' + group_root + '_' + detector + '_' + type + '_integ0.jpg'; img.src = jpg_filepath; img.alt = jpg_filepath; @@ -74,19 +72,7 @@ } // Reset the slider values - document.getElementById("slider_range").value = 1; - document.getElementById("slider_range").max = num_ints[type]; - document.getElementById("slider_val").innerHTML = 1; - document.getElementById("total_ints").innerHTML = total_ints[type]; - - // Update the integration changing buttons - if (num_ints[type] > 1) { - document.getElementById("int_after").disabled = false; - } else { - document.getElementById("int_after").disabled = true; - } - // Disable the "left" button, since this will be showing integ0 - document.getElementById("int_before").disabled = true; + reset_integration_slider(num_ints[type], total_ints[type]) // Update the view/explore links for the new file type update_view_explore_link(); @@ -117,57 +103,42 @@ document.getElementById("view_file_type").setAttribute('data-current-suffix', type); // Clean the input parameters - num_ints = num_ints.replace(/'/g, '"'); - num_ints = num_ints.replace(/'/g, '"'); + num_ints = clean_input_parameter(num_ints); num_ints = JSON.parse(num_ints); // Get the available integration jpg numbers - available_ints = available_ints.replace(/'/g, '"'); - available_ints = available_ints.replace(/'/g, '"'); + available_ints = clean_input_parameter(available_ints); available_ints = JSON.parse(available_ints)[type]; // Get the total number of integrations - total_ints = total_ints.replace(/'/g, '"'); - total_ints = total_ints.replace(/'/g, '"'); + total_ints = clean_input_parameter(total_ints); total_ints = JSON.parse(total_ints); // Propagate the text fields showing the filename and APT parameters var fits_filename = file_root + '_' + type; + var parsed_name = parse_filename(file_root); + document.getElementById("jpg_filename").innerHTML = file_root + '_' + type + '_integ0.jpg'; document.getElementById("fits_filename").innerHTML = fits_filename + '.fits'; - document.getElementById("proposal").innerHTML = file_root.slice(2,7); - document.getElementById("obs_id").innerHTML = file_root.slice(7,10); - document.getElementById("visit_id").innerHTML = file_root.slice(10,13); + document.getElementById("proposal").innerHTML = parsed_name.proposal; + document.getElementById("obs_id").innerHTML = parsed_name.obs_id; + document.getElementById("visit_id").innerHTML = parsed_name.visit_id; document.getElementById("detector").innerHTML = file_root.split('_')[3]; // Show the appropriate image var img = document.getElementById("image_viewer"); - var jpg_filepath = '/static/preview_images/' + file_root.slice(0,7) + '/' + file_root + '_' + type + '_integ0.jpg'; + var jpg_filepath = '/static/preview_images/' + parsed_name.program + '/' + file_root + '_' + type + '_integ0.jpg'; img.src = jpg_filepath; img.alt = jpg_filepath; // if previous image had error, remove error sizing img.classList.remove("thumbnail"); // Reset the slider values - document.getElementById("slider_range").value = 1; - document.getElementById("slider_range").max = num_ints[type]; - document.getElementById("slider_val").innerHTML = 1; - document.getElementById("total_ints").innerHTML = total_ints[type]; - - // Update the integration changing buttons - if (num_ints[type] > 1) { - document.getElementById("int_after").disabled = false; - } else { - document.getElementById("int_after").disabled = true; - } + reset_integration_slider(num_ints[type], total_ints[type]) // Update the image exploration and header links document.getElementById("view_header").href = '/' + inst + '/' + file_root + '_' + type + '/header/'; document.getElementById("explore_image").href = '/' + inst + '/' + file_root + '_' + type + '/explore_image/'; - - // Disable the "left" button, since this will be showing integ0 - document.getElementById("int_before").disabled = true; - } @@ -183,12 +154,12 @@ * @param {String} direction - The direction to switch to, either "left" (decrease) or "right" (increase). * Only relevant if method is "button". */ -function change_int(file_root, num_ints, available_ints, method, direction='right') { +function change_integration(file_root, num_ints, available_ints, method, direction='right') { // Figure out the current image and integration var suffix = document.getElementById("view_file_type").getAttribute('data-current-suffix'); var integration = Number(document.getElementById("slider_val").innerText) - 1; - var program = file_root.slice(0,7); + var program = parse_filename(file_root).program; // Find the total number of integrations for the current image num_ints = num_ints.replace(/'/g, '"'); @@ -267,6 +238,18 @@ function change_int(file_root, num_ints, available_ints, method, direction='righ } +/** + * Clean garbage characters in input dictionary parameters passed as strings. + * @param {String} param_value - The parameter value to clean + * @returns {String} cleaned - The cleaned parameter value + */ +function clean_input_parameter(param_value) { + param_value = param_value.replace(/'/g, '"'); + param_value = param_value.replace(/'/g, '"'); + return param_value +} + + /** * Determine what filetype to use for a thumbnail * @param {String} thumbnail_dir - The path to the thumbnail directory @@ -279,7 +262,7 @@ function determine_filetype_for_thumbnail(thumbnail_dir, thumb_filename, i, file // Update the thumbnail filename var img = document.getElementById('thumbnail'+i); if (thumb_filename != 'none') { - var jpg_path = thumbnail_dir + file_root.slice(0,7) + '/' + thumb_filename; + var jpg_path = thumbnail_dir + parse_filename(file_root).program + '/' + thumb_filename; img.src = jpg_path; } @@ -556,6 +539,55 @@ function unhide_file(detector) { update_view_explore_link(); } +/** + * Parse observation information from a JWST file name. + * @param {String} filename - The file or group root name to parse + * @returns {Object} parsed - Dictionary containing 'proposal', 'obs_id', 'visit_id', 'program' + */ +function parse_filename(root_name) { + // eg. for root_name jw02589006001_04101_00001-seg001_nrs1: + // program = jw02589 + // proposal = 02589 + // obs_id = 006 + // visit_id = 001 + + // used for preview directories + var program = root_name.slice(0,7); + + // used for observation description fields + var proposal = root_name.slice(2, 7); + var obs_id = root_name.slice(7, 10); + var visit_id = root_name.slice(10, 13); + + const parsed_name = {program: program, proposal: proposal, + obs_id: obs_id, visit_id: visit_id}; + return parsed_name; +} + + +/** + * Reset the integration slider for a new file + * @param {Int} num_integration - The number of integration images available + * @param {Int} total_integration - The total number of integrations to display + */ +function reset_integration_slider(num_integration, total_integration) { + // Reset the slider values + document.getElementById("slider_range").value = 1; + document.getElementById("slider_range").max = num_integration; + document.getElementById("slider_val").innerHTML = 1; + document.getElementById("total_ints").innerHTML = total_integration; + + // Update the integration changing buttons + if (num_integration > 1) { + document.getElementById("int_after").disabled = false; + } else { + document.getElementById("int_after").disabled = true; + } + + // Disable the "left" button, since this will be showing integ0 + document.getElementById("int_before").disabled = true; +} + /** * Check for a detector image and show or hide its viewer accordingly. @@ -933,7 +965,7 @@ function update_archive_page(inst, base_url) { /** - * Updates various compnents on the MSATA page + * Updates various components on the MSATA page * @param {String} inst - The instrument of interest (e.g. "FGS") * @param {String} base_url - The base URL for gathering data from the AJAX view. */ @@ -965,7 +997,7 @@ function update_msata_page(base_url) { /** - * Updates various compnents on the WATA page + * Updates various components on the WATA page * @param {String} inst - The instrument of interest (e.g. "FGS") * @param {String} base_url - The base URL for gathering data from the AJAX view. */ @@ -997,7 +1029,7 @@ function update_wata_page(base_url) { /** - * Updates various compnents on the thumbnails page + * Updates various components on the thumbnails page * @param {String} inst - The instrument of interest (e.g. "FGS") * @param {String} file_root - The rootname of the file forresponding tot he instrument (e.g. "JW01473015001_04101_00001_MIRIMAGE") * @param {String} filetype - The type to be viewed (e.g. "cal" or "rate"). @@ -1142,6 +1174,7 @@ function update_header_display(extension, num_extensions) { } + /** * Updates the obs-list div with observation number options * @param {Object} data - The data returned by the update_thumbnails_page AJAX method diff --git a/jwql/website/apps/jwql/templates/view_exposure.html b/jwql/website/apps/jwql/templates/view_exposure.html index e7926d922..f086f48a1 100644 --- a/jwql/website/apps/jwql/templates/view_exposure.html +++ b/jwql/website/apps/jwql/templates/view_exposure.html @@ -65,9 +65,9 @@

    Exposure {{ group_root }}

    - - - + + +

    Integration: /

    diff --git a/jwql/website/apps/jwql/templates/view_image.html b/jwql/website/apps/jwql/templates/view_image.html index 6afbdd403..4355eb878 100644 --- a/jwql/website/apps/jwql/templates/view_image.html +++ b/jwql/website/apps/jwql/templates/view_image.html @@ -53,9 +53,9 @@

    {{ file_root }}
    - - - + + +

    Integration: /

    From 4758c7d1ad35f38d347585df9dfa538b9059bc36 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Thu, 16 Mar 2023 17:34:53 -0400 Subject: [PATCH 188/449] Add fallback not-found images in case no previews are available --- jwql/website/apps/jwql/static/js/jwql.js | 11 +++++ .../apps/jwql/templates/view_exposure.html | 40 +++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index 79f2715c7..ad9027323 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -503,6 +503,7 @@ function group_by_thumbnails(group_type, base_url) { /** * Hide an image viewer + * @param {String} detector - The detector name for the image viewer */ function hide_file(detector) { var img = document.getElementById("image_viewer_" + detector); @@ -522,6 +523,7 @@ function hide_file(detector) { /** * Show an image viewer + * @param {String} detector - The detector name for the image viewer */ function unhide_file(detector) { var img = document.getElementById("image_viewer_" + detector); @@ -532,6 +534,13 @@ function unhide_file(detector) { img.style.display = "inline-block"; div.style.display = "inline-block"; + // Hide the fallback image and div + // These are never re-displayed: if any image loads for the detector, + // they will not show up. This is intended to cover the case where FITS files + // exist for the exposure, but no preview images have been generated yet. + document.getElementById("fallback_image_viewer_" + detector).style.display = "none"; + document.getElementById(detector + "_view_fallback").style.display = "none"; + // Enable the associated filename filename.disabled = false; @@ -591,6 +600,8 @@ function reset_integration_slider(num_integration, total_integration) { /** * Check for a detector image and show or hide its viewer accordingly. + * @param {String} detector - The detector name + * @param {String} jpg_filepath - The image to show */ function show_viewer(detector, jpg_filepath) { $.get(jpg_filepath, function() {unhide_file(detector);}) diff --git a/jwql/website/apps/jwql/templates/view_exposure.html b/jwql/website/apps/jwql/templates/view_exposure.html index f086f48a1..92e1d8d57 100644 --- a/jwql/website/apps/jwql/templates/view_exposure.html +++ b/jwql/website/apps/jwql/templates/view_exposure.html @@ -86,6 +86,16 @@

    Exposure {{ group_root }}
    +
    + + + Preview image not found + + +
    {% endif %} {% endfor %}
    @@ -102,6 +112,16 @@

    Exposure {{ group_root }}

    +
    + + + Preview image not found + + +
    {% endif %} {% endfor %}
    @@ -118,6 +138,16 @@

    Exposure {{ group_root }}

    +
    + + + Preview image not found + + +
    {% endif %} {% endfor %}
    @@ -134,6 +164,16 @@

    Exposure {{ group_root }}

    +
    + + + Preview image not found + + +
    {% endfor %}
    {% endif %} From 03eddf5370418158b876039e81bda5feb875ed17 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Thu, 16 Mar 2023 17:42:44 -0400 Subject: [PATCH 189/449] Fix typo in fallback image title --- jwql/website/apps/jwql/templates/view_exposure.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jwql/website/apps/jwql/templates/view_exposure.html b/jwql/website/apps/jwql/templates/view_exposure.html index 92e1d8d57..f63322713 100644 --- a/jwql/website/apps/jwql/templates/view_exposure.html +++ b/jwql/website/apps/jwql/templates/view_exposure.html @@ -90,7 +90,7 @@

    Exposure {{ group_root }}

    Preview image not found @@ -116,7 +116,7 @@

    Exposure {{ group_root }}

    Preview image not found @@ -142,7 +142,7 @@

    Exposure {{ group_root }}

    Preview image not found @@ -168,7 +168,7 @@

    Exposure {{ group_root }}

    Preview image not found From 8993d697c759471660c965d29ad8a172ce82b3f7 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Thu, 16 Mar 2023 17:58:03 -0400 Subject: [PATCH 190/449] Fix extra-long proposal thumbnails --- jwql/website/apps/jwql/static/js/jwql.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index ad9027323..335bd1227 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -957,7 +957,9 @@ function update_archive_page(inst, base_url) { content += ''; content += '' - content += ''; + content += ''; content += '
    '; content += '
    '; content += '

    ' + prop + '

    '; From fa205e652b33f1f3700ee0baf911347470a1d42e Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 17 Mar 2023 09:56:35 -0400 Subject: [PATCH 191/449] Fix width for fallback thumbnails --- jwql/website/apps/jwql/templates/view_exposure.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jwql/website/apps/jwql/templates/view_exposure.html b/jwql/website/apps/jwql/templates/view_exposure.html index f63322713..a96c12153 100644 --- a/jwql/website/apps/jwql/templates/view_exposure.html +++ b/jwql/website/apps/jwql/templates/view_exposure.html @@ -91,7 +91,7 @@

    Exposure {{ group_root }}

     Preview image not found @@ -117,7 +117,7 @@

    Exposure {{ group_root }}

    Preview image not found @@ -143,7 +143,7 @@

    Exposure {{ group_root }}

    Preview image not found @@ -169,7 +169,7 @@

    Exposure {{ group_root }}

    Preview image not found From df1ca6bf5c984da93a1da738e4c313981fc01915 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 17 Mar 2023 10:12:51 -0400 Subject: [PATCH 192/449] Keep header/explore view available in case with all missing previews --- jwql/website/apps/jwql/static/js/jwql.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index 335bd1227..43de8ddab 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -514,8 +514,12 @@ function hide_file(detector) { img.style.display = "none"; div.style.display = "none"; - // Disable the associated filename - filename.disabled = true; + // Disable the associated filename unless there + // are no previews available at all + var fallback_shown = document.getElementById(detector + "_view_fallback"); + if (fallback_shown.style.display == "none") { + filename.disabled = true; + } // Update the view/explore link as needed update_view_explore_link(); From d9be4b564bcf456eb83d94ad5c0d1875071337ca Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 17 Mar 2023 12:17:04 -0400 Subject: [PATCH 193/449] Tweak plots. Ready for testing on server --- jwql/jwql_monitors/monitor_filesystem.py | 19 +- jwql/website/apps/jwql/bokeh_dashboard.py | 203 ++++++++++++++++++---- jwql/website/apps/jwql/views.py | 6 +- 3 files changed, 188 insertions(+), 40 deletions(-) diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py index 9b7fbe5d7..39d38af24 100755 --- a/jwql/jwql_monitors/monitor_filesystem.py +++ b/jwql/jwql_monitors/monitor_filesystem.py @@ -68,7 +68,9 @@ PUBLIC_FILESYSTEM = os.path.join(FILESYSTEM, 'public') CENTRAL = SETTINGS['jwql_dir'] OUTPUTS = SETTINGS['outputs'] - +PREVIEW_IMAGES = SETTINGS['preview_image_filesystem'] +THUMBNAILS = SETTINGS['thumbnail_filesystem'] +LOGS = SETTINGS['log_dir'] def files_per_filter(): """Querying MAST (rather than looping through the filesystem), determine how @@ -219,23 +221,24 @@ def get_area_stats(central_storage_dict): """ logging.info('Gathering stats for central storage area') + areas = {'outputs': OUTPUTS, + 'logs': LOGS, + 'preview_images': PREVIEW_IMAGES, + 'thumbnails': THUMBNAILS, + 'all': CENTRAL} + #arealist = ['logs', 'outputs', 'test', 'preview_images', 'thumbnails', 'all'] - arealist = ['outputs'] counteddirs = [] - skipped_files = [] sums = 0 # to be used to count 'all' - for area in arealist: + for area in areas: used = 0 # initialize area in dictionary if area not in central_storage_dict: central_storage_dict[area] = {} - if area == 'all': - fullpath = CENTRAL - else: - fullpath = os.path.join(CENTRAL, area) + fullpath = areas[area] logging.info('\tSearching directory {}'.format(fullpath)) counteddirs.append(fullpath) diff --git a/jwql/website/apps/jwql/bokeh_dashboard.py b/jwql/website/apps/jwql/bokeh_dashboard.py index 74849e495..43ba11614 100644 --- a/jwql/website/apps/jwql/bokeh_dashboard.py +++ b/jwql/website/apps/jwql/bokeh_dashboard.py @@ -32,6 +32,7 @@ from datetime import datetime as dt from math import pi from operator import itemgetter +import os from bokeh.layouts import column from bokeh.models import Axis, ColumnDataSource, DatetimeTickFormatter, OpenURL, TapTool @@ -43,6 +44,7 @@ from sqlalchemy import func, and_ import jwql.database.database_interface as di +from jwql.database.database_interface import CentralStore from jwql.utils.constants import ANOMALY_CHOICES_PER_INSTRUMENT, FILTERS_PER_INSTRUMENT from jwql.utils.utils import get_base_url, get_config from jwql.website.apps.jwql.data_containers import build_table @@ -152,54 +154,192 @@ def __init__(self, delta_t=None): now = dt.now() self.date = pd.Timestamp('{}-{}-{}'.format(now.year, now.month, now.day)) - def dashboard_central_store_data_volume(self): - """Create trending plot of data volume for various areas on central store + def dashboard_disk_usage(self): + """Create trending plot of data volume for various disks. Here we are plotting + the results of a "df -hk" call for top-level directories. The results (i.e. on + central store) may contain contributions from non-JWQL files, since JWQL can + share disks with other projects. These plots are useful for tracking disk usage + and requesting more disk space if needed. Returns ------- tabs : bokeh.models.widgets.widget.Widget - A figure with tabs for each central store area + Set of tabs containing plots of the used and available disk space """ + # There are two main disks that we want to show usage for. The central store + # area, and the disk that is internal to the server. Use the logs entry to + # get the central store information, and the preview_image entry to get + # server disk information. + config = get_config() + + log_data = di.session.query(CentralStore.date, CentralStore.size, CentralStore.available) \ + .filter(CentralStore.area == 'logs') \ + .all() + + # Convert to dataframe + log_data = pd.DataFrame(log_data) + + preview_data = di.session.query(CentralStore.date, CentralStore.size, CentralStore.available) \ + .filter(CentralStore.area == 'preview_images') \ + .all() + + # Convert to dataframe + preview_data = pd.DataFrame(preview_data) + + # If the user is requesting a certain time range, cut down the entries + if not pd.isnull(self.delta_t): + log_data = log_data[(log_data['date'] >= self.date - self.delta_t) & (log_data['date'] <= self.date)] + preview_data = preview_data[(preview_data['date'] >= self.date - self.delta_t) & (preview_data['date'] <= self.date)] + + log_results = {'dirname': os.path.abspath(os.path.join(config['log_dir'], '../')), + 'results': log_data, + 'shortname': 'Central Store' + } + + preview_results = {'dirname': os.path.abspath(os.path.join(config['preview_image_filesystem'], '../')), + 'results': preview_data, + 'shortname': 'Server' + } + # Plot total data volume and available disk space versus time - results = session.query(CentralStore.date, CentralStore.size, CentralStore.available).all() + plots = {} + tabs = [] + for data in [preview_results, log_results]: + + # Calculate the size of the data + data['results']['used'] = data['results']['size'] - data['results']['available'] + source = ColumnDataSource(data['results']) + + # Initialize plot + plots[data['shortname']] = figure(tools='pan,hover,box_zoom,wheel_zoom,reset,save', + x_axis_type='datetime', + title=f"Available & Used Storage on {data['shortname']}", + x_axis_label='Date', + y_axis_label='Size TB') + + plots[data['shortname']].line(x='date', y='available', source=source, legend_label='Available', line_color='red',line_width=3) + plots[data['shortname']].circle(x='date', y='available', source=source,color='red', size=10) + plots[data['shortname']].line(x='date', y='used', source=source, legend_label='Used', line_color='blue', line_width=3) + plots[data['shortname']].circle(x='date', y='used', source=source,color='blue', size=10) + + plots[data['shortname']].xaxis.formatter = DatetimeTickFormatter(hours=["%d %B %Y"], + days=["%d %B %Y"], + months=["%d %B %Y"], + years=["%d %B %Y"], + ) + plots[data['shortname']].xaxis.major_label_orientation = pi / 4 + plots[data['shortname']].legend.location = 'top_left' + + hover_tool = HoverTool(tooltips=[('Available:', '@available'), + ('Used:', '@used'), + ('Date:', '@time{%d %b %Y}') + ]) + hover_tool.formatters = {'@time': 'datetime'} + plots[data['shortname']].tools.append(hover_tool) + tabs.append(Panel(child=plots[data['shortname']], title=f"{data['shortname']} Storage")) + + tabs = Tabs(tabs=tabs) + + session.close() + return tabs + + + def dashboard_central_store_data_volume(self): + """Create trending plot of data volume for various JWQL-related areas on disk. + These plots show data volumes calculated by walking over subdirectories/files in + the JWQL-specific directories. So these plots may not include the total used + disk volume, in the cases where JWQL is sharing a disk with other projects. These + plots are useful for monitoring the total volume of e.g. our preview images. + Returns + ------- + tabs : bokeh.models.widgets.widget.Widget + A figure with tabs for each central store area + """ # Initialize plot - dates, total_sizes, availables = zip(*results) plot = figure(tools='pan,box_zoom,wheel_zoom,reset,save', x_axis_type='datetime', - title='Central Store stats', + title='JWQL directory size', x_axis_label='Date', y_axis_label='Size TB') - plot.line(dates, total_sizes, legend='Total size', line_color='red') - plot.circle(dates, total_sizes, color='red') - plot.line(dates, availables, legend='Free', line_color='blue') - plot.circle(dates, availables, color='blue') - # This part of the plot should cycle through areas and plot area used values vs. date - arealist = ['logs', 'outputs', 'test', 'preview_images', 'thumbnails', 'all'] - colors = itertools.cycle(palette) + #arealist = ['logs', 'outputs', 'test', 'preview_images', 'thumbnails', 'all'] + arealist = ['logs', 'outputs', 'preview_images', 'thumbnails'] + colors = ['black', 'blue', 'red', 'green'] for area, color in zip(arealist, colors): # Query for used sizes - results = session.query(CentralStore.date, CentralStore.used).filter(CentralStore.area == area) + results = di.session.query(CentralStore.date, CentralStore.used).filter(CentralStore.area == area).all() - # Group by date if results: - results_dict = defaultdict(int) - for date, value in results: - results_dict[date] += value + # Convert to dataframe + results = pd.DataFrame(results) - # Parse results so they can be easily plotted - dates = list(results_dict.keys()) - values = list(results_dict.values()) + if not pd.isnull(self.delta_t): + results = results[(results['date'] >= self.date - self.delta_t) & (results['date'] <= self.date)] # Plot the results - plot.line(dates, values, legend='{} files'.format(area), line_color=color) - plot.circle(dates, values, color=color) + source = ColumnDataSource(results) + plot.line(x='date', y='used', source=source, line_color=color, legend_label=area, line_width=3) + plot.circle(x='date', y='used', source=source, color=color, size=10) + + hover_tool = HoverTool(tooltips=[('Used:', '@used'), + ('Date:', '@time{%d %b %Y}') + ]) + hover_tool.formatters = {'@time': 'datetime'} + plot.tools.append(hover_tool) + + plot.xaxis.formatter = DatetimeTickFormatter(hours=["%d %B %Y"], + days=["%d %B %Y"], + months=["%d %B %Y"], + years=["%d %B %Y"], + ) + plot.xaxis.major_label_orientation = pi / 4 + plot.legend.location = 'top_left' - session.close() - return plot + # Put the "all" plot in a separate figure because it will be larger than all the pieces, which would + # throw off the y range if it were in a single plot + cen_store_plot = figure(tools='pan,box_zoom,wheel_zoom,reset,save', + x_axis_type='datetime', + title='JWQL central store directory, total data volume', + x_axis_label='Date', + y_axis_label='Size TB') + + cen_store_results = di.session.query(CentralStore.date, CentralStore.used).filter(CentralStore.area == 'all').all() + + # Group by date + if cen_store_results: + + # Convert to dataframe + cen_store_results = pd.DataFrame(cen_store_results) + + if not pd.isnull(self.delta_t): + cen_store_results = cen_store_results[(cen_store_results['date'] >= self.date - self.delta_t) & (cen_store_results['date'] <= self.date)] + + # Group by date + cen_store_source = ColumnDataSource(cen_store_results) + + # Plot the results + legend_str = 'File volume' + cen_store_plot.line(x='date', y='used', source=cen_store_source, legend_label=legend_str, line_color='blue', line_width=3) + cen_store_plot.circle(x='date', y='used', source=cen_store_source, color='blue', size=10) + cen_store_plot.xaxis.formatter = DatetimeTickFormatter(hours=["%d %B %Y"], + days=["%d %B %Y"], + months=["%d %B %Y"], + years=["%d %B %Y"], + ) + cen_store_plot.xaxis.major_label_orientation = pi / 4 + cen_store_plot.legend.location = 'top_left' + + hover_tool = HoverTool(tooltips=[('Used:', '@used'), + ('Date:', '@time{%d %b %Y}') + ]) + hover_tool.formatters = {'@time': 'datetime'} + cen_store_plot.tools.append(hover_tool) + + di.session.close() + return plot, cen_store_plot def dashboard_filetype_bar_chart(self): @@ -217,9 +357,10 @@ def dashboard_filetype_bar_chart(self): # Keep only the rows containing the most recent timestamp data = data[data['date'] == data['date'].max()] + date_string = data['date'].max().strftime("%d %b %Y") # Set title and figures list to make panels - title = 'Files per Filetype by Instrument' + title = f'Files per Filetype by Instrument {date_string}' figures = [] # For unique instrument values, loop through data @@ -242,12 +383,12 @@ def dashboard_instrument_pie_chart(self): plot : bokeh.plotting.figure Pie chart figure """ - # Replace with jwql.website.apps.jwql.data_containers.build_table data = build_table('filesystem_instrument') # Keep only the rows containing the most recent timestamp data = data[data['date'] == data['date'].max()] + date_string = data['date'].max().strftime("%d %b %Y") try: file_counts = {'nircam': data[data.instrument == 'nircam']['count'].sum(), @@ -265,12 +406,12 @@ def dashboard_instrument_pie_chart(self): data = pd.Series(file_counts).reset_index(name='value').rename(columns={'index': 'instrument'}) data['angle'] = data['value'] / data['value'].sum() * 2 * pi data['color'] = ['#F8B195', '#F67280', '#C06C84', '#6C5B7B', '#355C7D'] - plot = figure(title="Number of Files Per Instrument", toolbar_location=None, + plot = figure(title=f"Number of Files Per Instrument {date_string}", toolbar_location=None, tools="hover,tap", tooltips="@instrument: @value", x_range=(-0.5, 1.0)) plot.wedge(x=0, y=1, radius=0.4, start_angle=cumsum('angle', include_zero=True), end_angle=cumsum('angle'), - line_color="white", color='color', legend='instrument', source=data) + line_color="white", color='color', legend_label='instrument', source=data) url = "{}/@instrument".format(get_base_url()) taptool = plot.select(type=TapTool) @@ -308,8 +449,8 @@ def dashboard_files_per_day(self): # Show date and used and available storage together p2 = figure(title="Available & Used Storage", tools="reset,hover,box_zoom,wheel_zoom", tooltips="@datestr: @total_file_count", plot_width=1700, x_axis_label='Date', y_axis_label='Storage Space [Terabytes?]') - p2.line(x='date', y='available', source=source, color='#F8B195', line_dash='dashed', line_width=3, legend='Available Storage') - p2.line(x='date', y='used', source=source, color='#355C7D', line_dash='dashed', line_width=3, legend='Used Storage') + p2.line(x='date', y='available', source=source, color='#F8B195', line_dash='dashed', line_width=3, legend_label='Available Storage') + p2.line(x='date', y='used', source=source, color='#355C7D', line_dash='dashed', line_width=3, legend_label='Used Storage') p2.scatter(x='date', y='available', source=source, color='#C85108', size=10) p2.scatter(x='date', y='used', source=source, color='#C85108', size=10) disable_scientific_notation(p2) diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index e739ac956..453e3e9e8 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -477,12 +477,16 @@ def dashboard(request): db = get_dashboard_components(request) pie_graph = db.dashboard_instrument_pie_chart() files_graph = db.dashboard_files_per_day() + useage_graph = db.dashboard_disk_usage() + directories_usage_graph, central_store_usage_graph = db.dashboard_central_store_data_volume() filetype_bar = db.dashboard_filetype_bar_chart() table_columns, table_values = db.dashboard_monitor_tracking() grating_plot = db.dashboard_exposure_count_by_filter() anomaly_plot = db.dashboard_anomaly_per_instrument() - plot = layout([[files_graph], [pie_graph, filetype_bar], + plot = layout([[files_graph, usage_graph], + [directories_usage_graph, central_store_usage_graph], + [pie_graph, filetype_bar], [grating_plot, anomaly_plot]], sizing_mode='stretch_width') script, div = components(plot) From 854cb343667e229304d1c874820bea3cd6ef51fd Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 17 Mar 2023 14:06:35 -0400 Subject: [PATCH 194/449] set permissions based on the account name --- jwql/utils/permissions.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/jwql/utils/permissions.py b/jwql/utils/permissions.py index 5c6c3c33e..d3031f5e7 100644 --- a/jwql/utils/permissions.py +++ b/jwql/utils/permissions.py @@ -82,7 +82,10 @@ DEFAULT_GROUP = 'jwql_dev' # set the default mode for DEFAULT_OWNER -DEFAULT_MODE = stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP # equivalent to '?rwxr-x---' +DEFAULT_MODE = stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH # equivalent to '?rwxr-xr--' +LOCAL_MODE = stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH # equivalent to '?rwxrwxr--' +DEFAULT_MODES = {DEFAULT_OWNER: DEFAULT_MODE, + 'other': LOCAL_MODE} def get_group_string(pathname): @@ -153,6 +156,8 @@ def has_permissions(pathname, owner=DEFAULT_OWNER, mode=DEFAULT_MODE, group=DEFA elif os.path.isdir(pathname): mode = mode | stat.S_IFDIR + # This will be False in all cases when running on the servers, since the DEFAULT_OWNER + # does not match the full owner names if (get_owner_string(pathname) != owner) or (file_statinfo.st_mode != mode)\ or (groupinfo.gr_name != group): return False @@ -160,7 +165,7 @@ def has_permissions(pathname, owner=DEFAULT_OWNER, mode=DEFAULT_MODE, group=DEFA return True -def set_permissions(pathname, owner=DEFAULT_OWNER, mode=DEFAULT_MODE, group=DEFAULT_GROUP, verbose=False): +def set_permissions(pathname, owner=DEFAULT_OWNER, mode=DEFAULT_MODES, group=DEFAULT_GROUP, verbose=False): """Set mode and group of the file/directory identfied by ``pathname``, if and only if it is owned by ``owner``. @@ -170,23 +175,34 @@ def set_permissions(pathname, owner=DEFAULT_OWNER, mode=DEFAULT_MODE, group=DEFA Directory or file to be inspected owner : str String representation of the owner - mode : int - Integer representation of the permission mode, compatible with - ``os.stat`` output + mode : dict + Dictionary of integer representation of the permission mode, compatible with + ``os.stat`` output. Keys are (with a default of DEFAULT_OWNER), + and 'other', so we can differentiate between files created by the server + accounts and those created by local users group : str String representation of the group verbose : bool Boolean indicating whether verbose output is requested """ + # When using the defaults, if the file is owned by one of the server + # accounts, then set the permissions using the + if owner in get_owner_string(pathname): + mode_to_use = mode[owner] + else: + mode_to_use = mode['other'] + if verbose: print('\nBefore:') show_permissions(pathname) if not has_permissions(pathname): - if get_owner_string(pathname) == owner: - os.chmod(pathname, mode) + try: + os.chmod(pathname, mode_to_use) # change group but not owner os.chown(pathname, -1, grp.getgrnam(group).gr_gid) + except PermissionError: + pass if verbose: print('After:') From d375377989e70e59284406516c9c50b58f56359d Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 17 Mar 2023 14:16:59 -0400 Subject: [PATCH 195/449] Fix docstring --- jwql/utils/permissions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jwql/utils/permissions.py b/jwql/utils/permissions.py index d3031f5e7..6a6a748d8 100644 --- a/jwql/utils/permissions.py +++ b/jwql/utils/permissions.py @@ -186,7 +186,8 @@ def set_permissions(pathname, owner=DEFAULT_OWNER, mode=DEFAULT_MODES, group=DEF Boolean indicating whether verbose output is requested """ # When using the defaults, if the file is owned by one of the server - # accounts, then set the permissions using the + # accounts, then set the permissions using the DEFAULT_OWNER's default + # permissions. Otherwise fall back to the 'other' default permissions. if owner in get_owner_string(pathname): mode_to_use = mode[owner] else: From 25bab3421b5aa361fe3eb64c192bb5a72874fc54 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 17 Mar 2023 14:39:07 -0400 Subject: [PATCH 196/449] only make executable when working on a directory --- jwql/utils/permissions.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/jwql/utils/permissions.py b/jwql/utils/permissions.py index 6a6a748d8..b2eed28d7 100644 --- a/jwql/utils/permissions.py +++ b/jwql/utils/permissions.py @@ -82,8 +82,8 @@ DEFAULT_GROUP = 'jwql_dev' # set the default mode for DEFAULT_OWNER -DEFAULT_MODE = stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH # equivalent to '?rwxr-xr--' -LOCAL_MODE = stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH # equivalent to '?rwxrwxr--' +DEFAULT_MODE = stat.S_IREAD | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH # equivalent to '?rw-r--r--' +LOCAL_MODE = stat.S_IREAD | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH # equivalent to '?rw-rw-r--' DEFAULT_MODES = {DEFAULT_OWNER: DEFAULT_MODE, 'other': LOCAL_MODE} @@ -193,6 +193,11 @@ def set_permissions(pathname, owner=DEFAULT_OWNER, mode=DEFAULT_MODES, group=DEF else: mode_to_use = mode['other'] + # Default permissions are for a file. If pathname is a directory, then + # make it owner and group executable as well + if os.path.isdir(pathname): + mode_to_use = mode_to_use | stat.S_IXUSR | stat.S_IXGRP + if verbose: print('\nBefore:') show_permissions(pathname) From 6483c2794ce739eb03ff89822b74572e9a1cebf8 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 17 Mar 2023 14:48:03 -0400 Subject: [PATCH 197/449] Make directories world executable --- jwql/utils/permissions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/utils/permissions.py b/jwql/utils/permissions.py index b2eed28d7..5d47d9c88 100644 --- a/jwql/utils/permissions.py +++ b/jwql/utils/permissions.py @@ -196,7 +196,7 @@ def set_permissions(pathname, owner=DEFAULT_OWNER, mode=DEFAULT_MODES, group=DEF # Default permissions are for a file. If pathname is a directory, then # make it owner and group executable as well if os.path.isdir(pathname): - mode_to_use = mode_to_use | stat.S_IXUSR | stat.S_IXGRP + mode_to_use = mode_to_use | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH if verbose: print('\nBefore:') From 7eaf3251c82b5b4725bbbeb2ade81c88e0568462 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 17 Mar 2023 15:36:47 -0400 Subject: [PATCH 198/449] Fix failing tests --- jwql/tests/test_permissions.py | 2 +- jwql/utils/permissions.py | 59 ++++++++++++++++++++++++++-------- 2 files changed, 47 insertions(+), 14 deletions(-) diff --git a/jwql/tests/test_permissions.py b/jwql/tests/test_permissions.py index 5ce11fe73..dd8878294 100755 --- a/jwql/tests/test_permissions.py +++ b/jwql/tests/test_permissions.py @@ -51,7 +51,7 @@ def test_directory(test_dir=TEST_DIRECTORY): yield test_dir if os.path.isdir(test_dir): - os.remove(test_dir) + os.rmdir(test_dir) def test_directory_permissions(test_directory): diff --git a/jwql/utils/permissions.py b/jwql/utils/permissions.py index 5d47d9c88..3d998b640 100644 --- a/jwql/utils/permissions.py +++ b/jwql/utils/permissions.py @@ -88,6 +88,45 @@ 'other': LOCAL_MODE} +def find_mode_to_use(pathname, owner, mode): + """Select the appropriate mode to use for the input pathname, + depending on who the owner is, as well as whether the pathname + is a file or directory. + + Parameters + ---------- + pathname : str + Directory or file to be inspected + owner : str + String representation of the owner + mode : dict + Dictionary of integer representation of the permission mode, compatible with + ``os.stat`` output. Keys are (with a default of DEFAULT_OWNER), + and 'other', so we can differentiate between files created by the server + accounts and those created by local users + + Returns + ------- + mode_value : int + Integer representation of the permission mode, compatible + with ``os.stat`` output + """ + if owner in get_owner_string(pathname): + if DEFAULT_OWNER in owner: + mode_value = mode[owner] + else: + mode_value = mode['other'] + else: + mode_value = mode['other'] + + # Default permissions are for a file. If pathname is a directory, then + # make it owner and group executable as well + if os.path.isdir(pathname): + mode_value = mode_value | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH + + return mode_value + + def get_group_string(pathname): """Return the group of ``pathname`` in string representation. @@ -126,7 +165,7 @@ def get_owner_string(pathname): return owner_name -def has_permissions(pathname, owner=DEFAULT_OWNER, mode=DEFAULT_MODE, group=DEFAULT_GROUP): +def has_permissions(pathname, owner=DEFAULT_OWNER, mode=DEFAULT_MODES, group=DEFAULT_GROUP): """Return boolean indicating whether ``pathname`` has the specified owner, permission, and group scheme. @@ -150,15 +189,17 @@ def has_permissions(pathname, owner=DEFAULT_OWNER, mode=DEFAULT_MODE, group=DEFA file_statinfo = os.stat(pathname) groupinfo = grp.getgrgid(file_statinfo.st_gid) + mode_to_use = find_mode_to_use(pathname, owner, mode) + # complement mode depending on whether input is file or directory if os.path.isfile(pathname): - mode = mode | stat.S_IFREG + mode_to_use = mode_to_use | stat.S_IFREG elif os.path.isdir(pathname): - mode = mode | stat.S_IFDIR + mode_to_use = mode_to_use | stat.S_IFDIR # This will be False in all cases when running on the servers, since the DEFAULT_OWNER # does not match the full owner names - if (get_owner_string(pathname) != owner) or (file_statinfo.st_mode != mode)\ + if (get_owner_string(pathname) != owner) or (file_statinfo.st_mode != mode_to_use)\ or (groupinfo.gr_name != group): return False @@ -188,15 +229,7 @@ def set_permissions(pathname, owner=DEFAULT_OWNER, mode=DEFAULT_MODES, group=DEF # When using the defaults, if the file is owned by one of the server # accounts, then set the permissions using the DEFAULT_OWNER's default # permissions. Otherwise fall back to the 'other' default permissions. - if owner in get_owner_string(pathname): - mode_to_use = mode[owner] - else: - mode_to_use = mode['other'] - - # Default permissions are for a file. If pathname is a directory, then - # make it owner and group executable as well - if os.path.isdir(pathname): - mode_to_use = mode_to_use | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH + mode_to_use = find_mode_to_use(pathname, owner, mode) if verbose: print('\nBefore:') From 15e0b00c2d275d19866bb2354c7e708688494a35 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 20 Mar 2023 10:59:11 -0400 Subject: [PATCH 199/449] use default owner to get mode --- jwql/utils/permissions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/utils/permissions.py b/jwql/utils/permissions.py index 3d998b640..b3117e006 100644 --- a/jwql/utils/permissions.py +++ b/jwql/utils/permissions.py @@ -113,7 +113,7 @@ def find_mode_to_use(pathname, owner, mode): """ if owner in get_owner_string(pathname): if DEFAULT_OWNER in owner: - mode_value = mode[owner] + mode_value = mode[DEFAULT_OWNER] else: mode_value = mode['other'] else: From c963c3e68e40b1340b48f19c85f7ade311734e5a Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 20 Mar 2023 14:37:09 -0400 Subject: [PATCH 200/449] define xdim, ydim --- jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py index 893b64936..25fb1e9dd 100755 --- a/jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py @@ -238,6 +238,7 @@ def create_plot(self): view = read_png(self.dark_image_picture) # Display the 32-bit RGBA image + ydim, xdim = view.shape dim = max(xdim, ydim) self.plot = figure(x_range=(0, xdim), y_range=(0, ydim), tools='pan,box_zoom,reset,wheel_zoom,save') self.plot.image_rgba(image=[img], x=0, y=0, dw=xdim, dh=ydim) From 979dd818c5fdbfb4baa4972c8ce090e36502bf3f Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 20 Mar 2023 14:49:00 -0400 Subject: [PATCH 201/449] Fix variable name --- jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py index 25fb1e9dd..47a2ef26d 100755 --- a/jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py @@ -241,7 +241,7 @@ def create_plot(self): ydim, xdim = view.shape dim = max(xdim, ydim) self.plot = figure(x_range=(0, xdim), y_range=(0, ydim), tools='pan,box_zoom,reset,wheel_zoom,save') - self.plot.image_rgba(image=[img], x=0, y=0, dw=xdim, dh=ydim) + self.plot.image_rgba(image=[view], x=0, y=0, dw=xdim, dh=ydim) self.plot.xaxis.visible = False self.plot.yaxis.visible = False From 994ff3cfdfd7ee2124ee8174301672457e055624 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 20 Mar 2023 15:11:16 -0400 Subject: [PATCH 202/449] Remove unused skipped_files --- jwql/jwql_monitors/monitor_filesystem.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py index 39d38af24..255822d91 100755 --- a/jwql/jwql_monitors/monitor_filesystem.py +++ b/jwql/jwql_monitors/monitor_filesystem.py @@ -269,8 +269,6 @@ def get_area_stats(central_storage_dict): if exists: filesize = os.path.getsize(file_path) sums += filesize - else: - skipped_files.append(filename) use = sums / (1024 ** 4) else: for dirpath, _, files in os.walk(fullpath): @@ -282,12 +280,10 @@ def get_area_stats(central_storage_dict): filesize = os.path.getsize(file_path) used += filesize sums += filesize - else: - skipped_files.append(filename) use = used / (1024 ** 4) central_storage_dict[area]['used'] = use - return central_storage_dict, skipped_files + return central_storage_dict def get_observation_characteristics(): From 942421bb6024f475796c620079e5fbfa8758dd15 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Mon, 20 Mar 2023 16:32:11 -0400 Subject: [PATCH 203/449] add --fill_empty flag to archive_database_update --- .../apps/jwql/archive_database_update.py | 329 +++++++++++++----- 1 file changed, 242 insertions(+), 87 deletions(-) diff --git a/jwql/website/apps/jwql/archive_database_update.py b/jwql/website/apps/jwql/archive_database_update.py index c172a81af..dedccee98 100755 --- a/jwql/website/apps/jwql/archive_database_update.py +++ b/jwql/website/apps/jwql/archive_database_update.py @@ -20,11 +20,15 @@ instrument = 'nircam' get_updates(insturument) - this module can be run from command line to add new elements to the django database + can be run from command line to add new elements to the django database $ python archive_database_update.py - This module can use the 'update' argument to update every rootfileinfo data model with the most complete information from MAST - $ python archive_database_update.py update + Use the '--fill_empty' argument to provide a model and field. Updates ALL fields for any model with empty/null/0 specified field + $ python archive_database_update.py --fill_empty root_file_info Filter + + Use the 'update' argument to update every rootfileinfo data model with the most complete information from MAST + $ python archive_database_update.py --update + WARNING: THIS WILL TAKE A LONG TIME Dependencies @@ -35,11 +39,12 @@ import logging import os -import sys +import argparse import numpy as np import django +from django.apps import apps from jwql.utils.protect_module import lock_module # These lines are needed in order to use the Django models in a standalone @@ -61,9 +66,10 @@ FILESYSTEM = get_config()['filesystem'] + @log_info @log_fail -def get_updates(update_database, inst): +def get_updates(update_database): """Generate the page listing all archived proposals in the database Parameters @@ -71,80 +77,84 @@ def get_updates(update_database, inst): update_database : bool true: run updates on existing rootfilename entries false: only create new entries - + inst : str Name of JWST instrument """ - logging.info(f'Updating database for {inst} archive page.') - - # Ensure the instrument is correctly capitalized - inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()] - - # Dictionary to hold summary information for all proposals - all_proposals = get_instrument_proposals(inst) - - # Get list of all files for the given instrument - for proposal in all_proposals: - # Get lists of all public and proprietary files for the program - filenames_public, metadata_public, filenames_proprietary, metadata_proprietary = get_all_possible_filenames_for_proposal(inst, proposal) - # Find the location in the filesystem for all files - filepaths_public = files_in_filesystem(filenames_public, 'public') - filepaths_proprietary = files_in_filesystem(filenames_proprietary, 'proprietary') - filenames = filepaths_public + filepaths_proprietary - - # There is one and only one category for the proposal, so just take the first. - proposal_category = '' - if len(metadata_public['category']): - proposal_category = metadata_public['category'][0] - elif len(metadata_proprietary['category']): - proposal_category = metadata_proprietary['category'][0] - - # Get set of unique rootnames - all_rootnames = set(['_'.join(f.split('/')[-1].split('_')[:-1]) for f in filenames]) - rootnames = [] - for rootname in all_rootnames: - filename_dict = filename_parser(rootname) - - # Weed out file types that are not supported by generate_preview_images - if 'stage_3' not in filename_dict['filename_type']: - rootnames.append(rootname) - - if len(filenames) > 0: - - # Gather information about the proposals for the given instrument - proposal_info = get_proposal_info(filenames) - - # Each observation number in each proposal can have a list of exp_types (e.g. NRC_TACQ, NRC_IMAGE) - for obsnum in set(proposal_info['observation_nums']): - # Find the public entries for the observation and get the associated exp_types - public_obs = np.array(metadata_public['observtn']) - match_pub = public_obs == int(obsnum) - public_exptypes = np.array(metadata_public['exp_type']) - exp_types = list(set(public_exptypes[match_pub])) - - # Find the proprietary entries for the observation, get the associated exp_types, and - # combine with the public values - prop_obs = np.array(metadata_proprietary['observtn']) - match_prop = prop_obs == int(obsnum) - prop_exptypes = np.array(metadata_proprietary['exp_type']) - exp_types = list(set(exp_types + list(set(prop_exptypes[match_prop])))) - - # Find the starting and ending dates for the observation - all_start_dates = np.array(metadata_public['expstart'])[match_pub] - all_start_dates = np.append(all_start_dates, np.array(metadata_proprietary['expstart'])[match_prop]) - - starting_date = np.min(all_start_dates) - all_end_dates = np.array(metadata_public['expend'])[match_pub] - all_end_dates = np.append(all_end_dates, np.array(metadata_proprietary['expend'])[match_prop]) - latest_date = np.max(all_end_dates) - - # Get the number of files in the observation - propobs = f'jw{int(proposal):05}{obsnum}' - obsfiles = [f for f in rootnames if propobs in f] - - # Update the appropriate database table - update_database_table(update_database, inst, proposal, obsnum, proposal_info['thumbnail_paths'][0], obsfiles, - exp_types, starting_date, latest_date, proposal_category) + instruments = ['nircam', 'miri', 'nirspec', 'niriss', 'fgs'] + for inst in instruments: + logging.info(f'Updating database for {inst} archive page.') + + # Ensure the instrument is correctly capitalized + inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()] + + # Dictionary to hold summary information for all proposals + all_proposals = get_instrument_proposals(inst) + + # Get list of all files for the given instrument + for proposal in all_proposals: + # Get lists of all public and proprietary files for the program + filenames_public, metadata_public, filenames_proprietary, metadata_proprietary = get_all_possible_filenames_for_proposal(inst, proposal) + # Find the location in the filesystem for all files + filepaths_public = files_in_filesystem(filenames_public, 'public') + filepaths_proprietary = files_in_filesystem(filenames_proprietary, 'proprietary') + filenames = filepaths_public + filepaths_proprietary + + # There is one and only one category for the proposal, so just take the first. + proposal_category = '' + if len(metadata_public['category']): + proposal_category = metadata_public['category'][0] + elif len(metadata_proprietary['category']): + proposal_category = metadata_proprietary['category'][0] + + # Get set of unique rootnames + all_rootnames = set(['_'.join(f.split('/')[-1].split('_')[:-1]) for f in filenames]) + rootnames = [] + for rootname in all_rootnames: + filename_dict = filename_parser(rootname) + + # Weed out file types that are not supported by generate_preview_images + if 'stage_3' not in filename_dict['filename_type']: + rootnames.append(rootname) + + if len(filenames) > 0: + + # Gather information about the proposals for the given instrument + proposal_info = get_proposal_info(filenames) + + # Each observation number in each proposal can have a list of exp_types (e.g. NRC_TACQ, NRC_IMAGE) + for obsnum in set(proposal_info['observation_nums']): + # Find the public entries for the observation and get the associated exp_types + public_obs = np.array(metadata_public['observtn']) + match_pub = public_obs == int(obsnum) + public_exptypes = np.array(metadata_public['exp_type']) + exp_types = list(set(public_exptypes[match_pub])) + + # Find the proprietary entries for the observation, get the associated exp_types, and + # combine with the public values + prop_obs = np.array(metadata_proprietary['observtn']) + match_prop = prop_obs == int(obsnum) + prop_exptypes = np.array(metadata_proprietary['exp_type']) + exp_types = list(set(exp_types + list(set(prop_exptypes[match_prop])))) + + # Find the starting and ending dates for the observation + all_start_dates = np.array(metadata_public['expstart'])[match_pub] + all_start_dates = np.append(all_start_dates, np.array(metadata_proprietary['expstart'])[match_prop]) + + starting_date = np.min(all_start_dates) + all_end_dates = np.array(metadata_public['expend'])[match_pub] + all_end_dates = np.append(all_end_dates, np.array(metadata_proprietary['expend'])[match_prop]) + latest_date = np.max(all_end_dates) + + # Get the number of files in the observation + propobs = f'jw{int(proposal):05}{obsnum}' + obsfiles = [f for f in rootnames if propobs in f] + + # Update the appropriate database table + update_database_table(update_database, inst, proposal, obsnum, proposal_info['thumbnail_paths'][0], obsfiles, + exp_types, starting_date, latest_date, proposal_category) + + create_archived_proposals_context(inst) def get_all_possible_filenames_for_proposal(instrument, proposal_num): @@ -319,7 +329,6 @@ def update_database_table(update, instrument, prop, obs, thumbnail, obsfiles, ty # Updating defaults only on update or creation to prevent call to mast_query_by_rootname on every file name. defaults_dict = mast_query_by_rootname(instrument, file) - # NOTE: If implementing a new field, add to this dict and run script with 'update' flag defaults = dict(filter=defaults_dict.get('filter', ''), detector=defaults_dict.get('detector', ''), exp_type=defaults_dict.get('exp_type', ''), @@ -343,9 +352,133 @@ def update_database_table(update, instrument, prop, obs, thumbnail, obsfiles, ty logging.info(f'Created {nr_files_created} root_file_info entries for: {instrument} - proposal:{prop} - obs:{obs}') +@log_fail +def fill_empty_model(model_name, model_field): + '''`fill_empty_model` takes a model name and a model field as input, and then updates all the models in + the database that have a null, empty, or zero value for that field. + + Parameters + ---------- + model_name + the name of the model to be updated + model_field + the name of the field in the model that is empty + + ''' + + model_field_null = model_field + "__isnull" + model_field_empty = model_field + "__exact" + + model = apps.get_model('jwql', model_name) + null_models = empty_models = zero_models = model.objects.none() + + # filter(field__isnull=True) + try: + null_models = model.objects.filter(**{model_field_null: True}) + except ValueError: + pass + + # filter(field__exact='') + try: + empty_models = model.objects.filter(**{model_field_empty: ''}) + except ValueError: + pass + + # filter(field=0) + try: + zero_models = model.objects.filter(**{model_field: 0}) + except ValueError: + pass + + model_set = null_models | empty_models | zero_models + if model_set.exists(): + logging.info(f'{model_set.count()} models to be updated') + if model_name == 'proposal': + fill_empty_proposals(model_set) + elif model_name == 'rootfileinfo': + fill_empty_rootfileinfo(model_set) + else: + logging.warning(f'Filling {model_name} model is not currently implemented') + print(f'Filling {model_name} model is not currently implemented') + + +def fill_empty_proposals(proposal_set): + '''It takes a list of proposal querysets, finds the thumbnail and category for each proposal, and saves + the proposal + + Parameters + ---------- + proposal_set : a queryset of Proposal objects + + ''' + + saved_proposals = 0 + for proposal_mod in proposal_set: + + filenames_public, metadata_public, filenames_proprietary, metadata_proprietary = get_all_possible_filenames_for_proposal(proposal_mod.archive.instrument, proposal_mod.prop_id) + # Find the location in the filesystem for all files + filepaths_public = files_in_filesystem(filenames_public, 'public') + filepaths_proprietary = files_in_filesystem(filenames_proprietary, 'proprietary') + filenames = filepaths_public + filepaths_proprietary + proposal_info = get_proposal_info(filenames) + + # There is one and only one category for the proposal, so just take the first. + proposal_category = '' + if len(metadata_public['category']): + proposal_category = metadata_public['category'][0] + elif len(metadata_proprietary['category']): + proposal_category = metadata_proprietary['category'][0] + + proposal_mod.thumbnail_path = proposal_info['thumbnail_paths'][0] + proposal_mod.category = proposal_category + try: + proposal_mod.save(update_fields=['thumbnail_path', 'category']) + saved_proposals += 1 + except Exception as e: + logging.warning(f'\tCould not save proposal {proposal_mod.prop_id}') + logging.warning(f'\tError {e} was raised') + + logging.info(f'\tSaved {saved_proposals} proposals') + + +def fill_empty_rootfileinfo(rootfileinfo_set): + '''Takes a queryset of RootFileInfo objects and fills in the empty fields with values + from the MAST database + + Parameters + ---------- + rootfileinfo_set : a queryset of RootFileInfo objects + + ''' + + saved_rootfileinfos = 0 + for rootfileinfo_mod in rootfileinfo_set: + defaults_dict = mast_query_by_rootname(rootfileinfo_mod.instrument, rootfileinfo_mod.root_name) + + defaults = dict(filter=defaults_dict.get('filter', ''), + detector=defaults_dict.get('detector', ''), + exp_type=defaults_dict.get('exp_type', ''), + read_patt=defaults_dict.get('readpatt', ''), + grating=defaults_dict.get('grating', ''), + read_patt_num=defaults_dict.get('patt_num', 0), + aperture=defaults_dict.get('apername', ''), + subarray=defaults_dict.get('subarray', ''), + pupil=defaults_dict.get('pupil', ''), + expstart=defaults_dict.get('expstart', 0.0)) + + for key, value in defaults.items(): + setattr(rootfileinfo_mod, key, value) + try: + rootfileinfo_mod.save() + saved_rootfileinfos += 1 + except Exception as e: + logging.warning(f'\tCould not save rootfileinfo {rootfileinfo_mod.root_name}') + logging.warning(f'\tError {e} was raised') + logging.info(f'\tSaved {saved_rootfileinfos} Root File Infos') + @lock_module -def protected_code(update_database): +def protected_code(update_database, fill_empty_list): """Protected code ensures only 1 instance of module will run at any given time Parameters @@ -356,16 +489,38 @@ def protected_code(update_database): module = os.path.basename(__file__).strip('.py') start_time, log_file = initialize_instrument_monitor(module) - instruments = ['nircam', 'miri', 'nirspec', 'niriss', 'fgs'] - for instrument in instruments: - get_updates(update_database, instrument) - create_archived_proposals_context(instrument) + if fill_empty_list: + fill_empty_model(fill_empty_list[0], fill_empty_list[1]) + else: + get_updates(update_database) if __name__ == '__main__': - update_database = False - for i in range(1, len(sys.argv)): - if sys.argv[i].lower() == 'update': - update_database = True - protected_code(update_database) + models_list = ['archive', 'observation', 'proposal', 'rootfileinfo'] + proposal_fields = ['category'] + # Initialize parser + msg = "Used to update Django Model Database from header information" + parser = argparse.ArgumentParser(description=msg) + + # Adding optional argument + parser.add_argument("--update", action='store_true', help="Update Entire Model Database") + parser.add_argument("--fill_empty", nargs=2, help="enter 2 arguments-> model_name, model_field") + + args = parser.parse_args() + continue_script = True + if args.fill_empty: + continue_script = False + args.fill_empty = list(map(lambda x: x.lower(), args.fill_empty)) + if args.fill_empty[0] not in models_list: + print('model_name incorrect, try: {}'.format(models_list)) + else: + model = apps.get_model('jwql', args.fill_empty[0]) + fields = [field.name for field in model._meta.get_fields()] + if args.fill_empty[1] not in fields: + print('Invalid field entered for model type {}, try one of the following: {}'.format(args.fill_empty[0], fields)) + else: + continue_script = True + + if continue_script: + protected_code(args.update, args.fill_empty) From bd1ce38037647a4ce876a2605dfb271cb384db18 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 20 Mar 2023 20:46:16 -0400 Subject: [PATCH 204/449] Make subdir lists consistent --- jwql/jwql_monitors/monitor_filesystem.py | 11 ++++------- jwql/utils/constants.py | 3 +++ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py index 255822d91..7d33f4058 100755 --- a/jwql/jwql_monitors/monitor_filesystem.py +++ b/jwql/jwql_monitors/monitor_filesystem.py @@ -55,7 +55,8 @@ from jwql.database.database_interface import CentralStore from jwql.utils.logging_functions import log_info, log_fail from jwql.utils.permissions import set_permissions -from jwql.utils.constants import FILE_SUFFIX_TYPES, FILTERS_PER_INSTRUMENT, INSTRUMENT_SERVICE_MATCH, JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_INSTRUMENT_NAMES_MIXEDCASE +from jwql.utils.constants import FILESYSYEM_MONITOR_SUBDIRS, FILE_SUFFIX_TYPES, FILTERS_PER_INSTRUMENT, INSTRUMENT_SERVICE_MATCH +from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_INSTRUMENT_NAMES_MIXEDCASE from jwql.utils.utils import filename_parser from jwql.utils.utils import get_config from jwql.utils.monitor_utils import initialize_instrument_monitor, update_monitor_table @@ -227,7 +228,6 @@ def get_area_stats(central_storage_dict): 'thumbnails': THUMBNAILS, 'all': CENTRAL} - #arealist = ['logs', 'outputs', 'test', 'preview_images', 'thumbnails', 'all'] counteddirs = [] sums = 0 # to be used to count 'all' @@ -522,8 +522,6 @@ def plot_central_store_dirs(): # Plot system stats vs. date results = session.query(CentralStore.date, CentralStore.size, CentralStore.available).all() - arealist = ['logs', 'outputs', 'test', 'preview_images', 'thumbnails', 'all'] - # Initialize plot dates, total_sizes, availables = zip(*results) plot = figure( @@ -540,7 +538,7 @@ def plot_central_store_dirs(): plot.circle(dates, availables, color='blue') # This part of the plot should cycle through areas and plot area used values vs. date - for area, color in zip(arealist, colors): + for area, color in zip(FILESYSYEM_MONITOR_SUBDIRS, colors): # Query for used sizes results = session.query(CentralStore.date, CentralStore.used).filter(CentralStore.area == area) @@ -706,8 +704,7 @@ def update_database(general_results_dict, instrument_results_dict, central_stora logging.error(e) # Add data to central_storage table - arealist = ['logs', 'outputs', 'test', 'preview_images', 'thumbnails', 'all'] - for area in arealist: + for area in FILESYSYEM_MONITOR_SUBDIRS: new_record = {} new_record['date'] = central_storage_dict['date'] new_record['area'] = area diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index ada1dc312..df64f69ba 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -228,6 +228,9 @@ 'nirspec': ['NRS_AUTOFLAT', 'NRS_LAMP'], 'fgs': ['FGS_INTFLAT']} +# output subdirectories to keep track of via the filesytem monitor +FILESYSYEM_MONITOR_SUBDIRS = ['logs', 'outputs', 'preview_images', 'thumbnails', 'all'] + FILTERS_PER_INSTRUMENT = {'fgs': [], 'miri': ['F1000W', 'F1130W', 'F1280W', 'OPAQUE', 'F2300C', 'F560W', 'P750L', 'F1500W', 'F2550W', 'F770W', 'FLENS', 'FND', 'F2100W', 'F1800W', From 0e32ee1190da9b3fb44367e94fd66cc987a718dd Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 21 Mar 2023 09:19:46 -0400 Subject: [PATCH 205/449] fix incorrect var name --- jwql/jwql_monitors/monitor_filesystem.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py index 7d33f4058..e153c7983 100755 --- a/jwql/jwql_monitors/monitor_filesystem.py +++ b/jwql/jwql_monitors/monitor_filesystem.py @@ -653,8 +653,8 @@ def update_characteristics_database(char_info): # Add data to filesystem_instrument table for instrument in ['nircam', 'niriss', 'nirspec', 'miri']: - filter_list = [e[0] for e in obs['nircam']] - value_list= [e[1] for e in obs['nircam']] + filter_list = [e[0] for e in char_info['nircam']] + value_list= [e[1] for e in char_info['nircam']] new_record = {} new_record['date'] = now new_record['instrument'] = instrument From cd7fd560af63d1f905225751fdfb359e5c303765 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Tue, 21 Mar 2023 11:02:07 -0400 Subject: [PATCH 206/449] fix example and log for rootfileinfo --- jwql/website/apps/jwql/archive_database_update.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jwql/website/apps/jwql/archive_database_update.py b/jwql/website/apps/jwql/archive_database_update.py index dedccee98..dd469dc59 100755 --- a/jwql/website/apps/jwql/archive_database_update.py +++ b/jwql/website/apps/jwql/archive_database_update.py @@ -24,7 +24,7 @@ $ python archive_database_update.py Use the '--fill_empty' argument to provide a model and field. Updates ALL fields for any model with empty/null/0 specified field - $ python archive_database_update.py --fill_empty root_file_info Filter + $ python archive_database_update.py --fill_empty rootfileinfo Filter Use the 'update' argument to update every rootfileinfo data model with the most complete information from MAST $ python archive_database_update.py --update @@ -349,7 +349,7 @@ def update_database_table(update, instrument, prop, obs, thumbnail, obsfiles, ty logging.warning(f'\tError {e} was raised') logging.warning(f'\tError with root_name: {file} inst: {instrument} obsnum: {obs_instance} proposal: {prop}') if nr_files_created > 0: - logging.info(f'Created {nr_files_created} root_file_info entries for: {instrument} - proposal:{prop} - obs:{obs}') + logging.info(f'Created {nr_files_created} rootfileinfo entries for: {instrument} - proposal:{prop} - obs:{obs}') @log_fail From ba19e04e24afed78c7ea6d7ec6050c49515598c7 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Tue, 21 Mar 2023 11:54:51 -0400 Subject: [PATCH 207/449] update docstring example --- jwql/website/apps/jwql/archive_database_update.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/archive_database_update.py b/jwql/website/apps/jwql/archive_database_update.py index dd469dc59..071df24a8 100755 --- a/jwql/website/apps/jwql/archive_database_update.py +++ b/jwql/website/apps/jwql/archive_database_update.py @@ -24,7 +24,10 @@ $ python archive_database_update.py Use the '--fill_empty' argument to provide a model and field. Updates ALL fields for any model with empty/null/0 specified field - $ python archive_database_update.py --fill_empty rootfileinfo Filter + $ python archive_database_update.py --fill_empty rootfileinfo expstart + WARNING: Not all fields will be populated by all model objects. This will result in updates that may not be necessary. + While this will not disturb the data, it has the potential to increase run time. + Select the field that is most pertient to the models you need updated minimize run time Use the 'update' argument to update every rootfileinfo data model with the most complete information from MAST $ python archive_database_update.py --update From fc13047ac4d11fc1f0fb26500ae7c71d028006db Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 21 Mar 2023 12:25:42 -0400 Subject: [PATCH 208/449] missing import --- jwql/jwql_monitors/monitor_filesystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py index e153c7983..9956104a8 100755 --- a/jwql/jwql_monitors/monitor_filesystem.py +++ b/jwql/jwql_monitors/monitor_filesystem.py @@ -50,7 +50,7 @@ from jwql.database.database_interface import engine from jwql.database.database_interface import session -from jwql.database.database_interface import FilesystemGeneral +from jwql.database.database_interface import FilesystemCharacteristics, FilesystemGeneral from jwql.database.database_interface import FilesystemInstrument from jwql.database.database_interface import CentralStore from jwql.utils.logging_functions import log_info, log_fail From 2e02f82ac41cffb3b794102109e15a9b0eb00158 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 21 Mar 2023 15:24:05 -0400 Subject: [PATCH 209/449] Fix table ordering. Resize plots --- jwql/website/apps/jwql/bokeh_dashboard.py | 15 +++++++++------ jwql/website/apps/jwql/templates/dashboard.html | 8 ++++---- jwql/website/apps/jwql/views.py | 2 +- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/jwql/website/apps/jwql/bokeh_dashboard.py b/jwql/website/apps/jwql/bokeh_dashboard.py index 43ba11614..8a58d4ad9 100644 --- a/jwql/website/apps/jwql/bokeh_dashboard.py +++ b/jwql/website/apps/jwql/bokeh_dashboard.py @@ -35,7 +35,7 @@ import os from bokeh.layouts import column -from bokeh.models import Axis, ColumnDataSource, DatetimeTickFormatter, OpenURL, TapTool +from bokeh.models import Axis, ColumnDataSource, DatetimeTickFormatter, HoverTool, OpenURL, TapTool from bokeh.models.widgets import Panel, Tabs from bokeh.plotting import figure from bokeh.transform import cumsum @@ -212,6 +212,7 @@ def dashboard_disk_usage(self): # Initialize plot plots[data['shortname']] = figure(tools='pan,hover,box_zoom,wheel_zoom,reset,save', + plot_width=800, x_axis_type='datetime', title=f"Available & Used Storage on {data['shortname']}", x_axis_label='Date', @@ -240,7 +241,7 @@ def dashboard_disk_usage(self): tabs = Tabs(tabs=tabs) - session.close() + di.session.close() return tabs @@ -258,6 +259,7 @@ def dashboard_central_store_data_volume(self): """ # Initialize plot plot = figure(tools='pan,box_zoom,wheel_zoom,reset,save', + plot_width=800, x_axis_type='datetime', title='JWQL directory size', x_axis_label='Date', @@ -301,6 +303,7 @@ def dashboard_central_store_data_volume(self): # Put the "all" plot in a separate figure because it will be larger than all the pieces, which would # throw off the y range if it were in a single plot cen_store_plot = figure(tools='pan,box_zoom,wheel_zoom,reset,save', + plot_width=800, x_axis_type='datetime', title='JWQL central store directory, total data volume', x_axis_label='Date', @@ -439,7 +442,7 @@ def dashboard_files_per_day(self): date_times = [pd.to_datetime(datetime).date() for datetime in source['date'].values] source['datestr'] = [date_time.strftime("%Y-%m-%d") for date_time in date_times] - p1 = figure(title="Number of Files in Filesystem", tools="reset,hover,box_zoom,wheel_zoom", tooltips="@datestr: @total_file_count", plot_width=1700, x_axis_label='Date', y_axis_label='Number of Files Added') + p1 = figure(title="Number of Files in Filesystem", tools="reset,hover,box_zoom,wheel_zoom", tooltips="@datestr: @total_file_count", plot_width=800, x_axis_label='Date', y_axis_label='Number of Files Added') p1.line(x='date', y='total_file_count', source=source, color='#6C5B7B', line_dash='dashed', line_width=3) p1.scatter(x='date', y='total_file_count', source=source, color='#C85108', size=10) disable_scientific_notation(p1) @@ -448,7 +451,7 @@ def dashboard_files_per_day(self): # Create separate tooltip for storage plot. # Show date and used and available storage together - p2 = figure(title="Available & Used Storage", tools="reset,hover,box_zoom,wheel_zoom", tooltips="@datestr: @total_file_count", plot_width=1700, x_axis_label='Date', y_axis_label='Storage Space [Terabytes?]') + p2 = figure(title="Available & Used Storage", tools="reset,hover,box_zoom,wheel_zoom", tooltips="@datestr: @total_file_count", plot_width=800, x_axis_label='Date', y_axis_label='Storage Space [Terabytes?]') p2.line(x='date', y='available', source=source, color='#F8B195', line_dash='dashed', line_width=3, legend_label='Available Storage') p2.line(x='date', y='used', source=source, color='#355C7D', line_dash='dashed', line_width=3, legend_label='Used Storage') p2.scatter(x='date', y='available', source=source, color='#C85108', size=10) @@ -552,8 +555,8 @@ def dashboard_exposure_count_by_filter(self): # This is a loop over instruments for i in range(len(data)): instrument = data.loc[i]['instrument'] - filterpupil = data.loc[i]['filter_pupil'] - num_obs = data.loc[i]['obs_per_filter_pupil'] + filterpupil = np.array(data.loc[i]['filter_pupil']) + num_obs = np.array(data.loc[i]['obs_per_filter_pupil']) # Sort by num_obs in order to make the plot more readable idx = np.argsort(num_obs) diff --git a/jwql/website/apps/jwql/templates/dashboard.html b/jwql/website/apps/jwql/templates/dashboard.html index e81c8c7ec..d03ed7283 100644 --- a/jwql/website/apps/jwql/templates/dashboard.html +++ b/jwql/website/apps/jwql/templates/dashboard.html @@ -13,7 +13,7 @@

    JWQL Dashboard

    - The JWQL dashboard provides users with a "high level" status of our systems and tools. Here we provide some analytics and + The JWQL dashboard provides users with a "high level" status of our systems and tools. Here we provide some analytics and statistics for the JWQL filesystem as well as the status of the instrument monitors. The figures below are interactive and some provide more detailed information if you hover over them and will take you to other landing pages within the application if clicked.

    @@ -25,7 +25,7 @@

    Select Time Delta

    {% for dt in time_deltas %} {% endfor %} - + @@ -34,7 +34,7 @@

    Select Time Delta

    {{ div | safe }} {{ script | safe }} - +

    JWQL Monitor Status

    @@ -87,7 +87,7 @@

    JWQL Monitor Status

    +
    + + Return To Query Form + - - + + {% endblock %} From a1323aa1f6e8f05844ab6f227e801d346ae68aaf Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Mon, 10 Apr 2023 16:29:17 -0400 Subject: [PATCH 273/449] initial hide of elements to avoid 'peek' effect --- jwql/website/apps/jwql/templates/jwql_query.html | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/jwql/website/apps/jwql/templates/jwql_query.html b/jwql/website/apps/jwql/templates/jwql_query.html index feac6ba96..47e74eafb 100644 --- a/jwql/website/apps/jwql/templates/jwql_query.html +++ b/jwql/website/apps/jwql/templates/jwql_query.html @@ -87,7 +87,7 @@

    Dynamic Query Form

    -
    +
    FGS Anomalies
    @@ -150,7 +150,7 @@

    Dynamic Query Form

    -
    +
    MIRI Anomalies
    @@ -224,7 +224,7 @@

    Dynamic Query Form

    -
    +
    NIRCam Anomalies
    @@ -311,7 +311,7 @@

    Dynamic Query Form

    -
    +
    NIRISS Anomalies
    @@ -398,7 +398,7 @@

    Dynamic Query Form

    -
    +
    NIRSpec Anomalies
    From 26d496512516d31770d23affec9136ad1df3ff79 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Tue, 11 Apr 2023 09:01:23 -0400 Subject: [PATCH 274/449] fix get_rootnames_from_query return val bug --- jwql/website/apps/jwql/data_containers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 4d41f828d..43b1ff2ca 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -1549,7 +1549,8 @@ def get_rootnames_from_query(parameters): filtered_rootnames.extend(anomaly_rootnames) else: - filtered_rootnames.extend(current_ins_rootfileinfos) + rootnames = [name[0] for name in current_ins_rootfileinfos.values_list("root_name")] + filtered_rootnames.extend(rootnames) return list(set(filtered_rootnames)) # >>> END CODE PRE DB MIGRATION HERE <<< From 96508c5e1fa91ce3afae58107ab91e14531f4f8f Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Tue, 11 Apr 2023 09:02:43 -0400 Subject: [PATCH 275/449] remove hide code, affects back navigation --- jwql/website/apps/jwql/templates/jwql_query.html | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/jwql/website/apps/jwql/templates/jwql_query.html b/jwql/website/apps/jwql/templates/jwql_query.html index 47e74eafb..feac6ba96 100644 --- a/jwql/website/apps/jwql/templates/jwql_query.html +++ b/jwql/website/apps/jwql/templates/jwql_query.html @@ -87,7 +87,7 @@

    Dynamic Query Form

    -
    +
    FGS Anomalies
    @@ -150,7 +150,7 @@

    Dynamic Query Form

    -
    +
    MIRI Anomalies
    @@ -224,7 +224,7 @@

    Dynamic Query Form

    -
    +
    NIRCam Anomalies
    @@ -311,7 +311,7 @@

    Dynamic Query Form

    -
    +
    NIRISS Anomalies
    @@ -398,7 +398,7 @@

    Dynamic Query Form

    -
    +
    NIRSpec Anomalies
    From 082ba19598b3365ea62c050c0f3e2405d79c7935 Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Thu, 13 Apr 2023 13:17:20 -0400 Subject: [PATCH 276/449] remove usages of deprecated `codecov` package the `codecov` package is now deprecated and has been yanked from PyPI --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 45f081f50..6d30c3958 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,6 @@ 'astroquery', 'bandit', 'bokeh<3', - 'codecov', 'crds', 'cryptography', 'django', From 5042f6c9dd3e77483f04b5c12248366a0c21a552 Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Thu, 13 Apr 2023 13:18:11 -0400 Subject: [PATCH 277/449] remove `codecov` --- environment_python_3_9.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/environment_python_3_9.yml b/environment_python_3_9.yml index 9918f0ca3..2aadf5067 100644 --- a/environment_python_3_9.yml +++ b/environment_python_3_9.yml @@ -26,7 +26,6 @@ dependencies: - bokeh=2.4.3 - beautifulsoup4=4.11.2 - celery=5.2.7 - - codecov=2.1.12 - cryptography=39.0.1 - django=4.1.7 - flake8=6.0.0 From d6af4d59f25b24a070ae56a69460ff4afcfb54f1 Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Thu, 13 Apr 2023 13:18:28 -0400 Subject: [PATCH 278/449] remove `codecov` --- environment_python_3_8.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/environment_python_3_8.yml b/environment_python_3_8.yml index 2a31ef0ba..df3d3a173 100644 --- a/environment_python_3_8.yml +++ b/environment_python_3_8.yml @@ -26,7 +26,6 @@ dependencies: - bokeh=2.4.3 - beautifulsoup4=4.11.2 - celery=5.2.7 - - codecov=2.1.12 - cryptography=39.0.1 - django=4.1.7 - flake8=6.0.0 From b804ebdfc5dc6e09fb7ec2618bbdfe20328e609e Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Thu, 13 Apr 2023 13:18:43 -0400 Subject: [PATCH 279/449] remove `codecov` --- environment_python_3_10.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/environment_python_3_10.yml b/environment_python_3_10.yml index 22bad9d5c..68a1fa077 100644 --- a/environment_python_3_10.yml +++ b/environment_python_3_10.yml @@ -26,7 +26,6 @@ dependencies: - bokeh=2.4.3 - beautifulsoup4=4.11.2 - celery=5.2.7 - - codecov=2.1.12 - cryptography=39.0.1 - django=4.1.7 - flake8=6.0.0 From 216d94947439774492e9f1eb33bd78da1f6ea31a Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Thu, 13 Apr 2023 13:20:13 -0400 Subject: [PATCH 280/449] remove usages of deprecated `codecov` package the `codecov` package is now deprecated and has been yanked from PyPI --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 66e027461..d4bc70951 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,6 @@ bandit==1.7.4 beautifulsoup4==4.11.2 bokeh==2.4.3 celery==5.2.7 -codecov==2.1.12 cryptography==39.0.1 django==4.1.7 flake8==6.0.0 From 7f92cc8ec920699c2416a3e0384d6e512162f2ea Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 13 Apr 2023 14:41:24 -0400 Subject: [PATCH 281/449] Pass filter img counts as list --- jwql/jwql_monitors/monitor_filesystem.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py index d8f101ead..b63d2e182 100755 --- a/jwql/jwql_monitors/monitor_filesystem.py +++ b/jwql/jwql_monitors/monitor_filesystem.py @@ -674,11 +674,13 @@ def update_characteristics_database(char_info): # Add data to filesystem_instrument table for instrument in ['nircam', 'niriss', 'nirspec', 'miri']: + optics = [e[0] for e in char_info[instrument]] + values = [e[1] for e in char_info[instrument]] new_record = {} new_record['date'] = now new_record['instrument'] = instrument - new_record['filter_pupil'] = list(char_info[instrument].keys()) - new_record['obs_per_filter_pupil'] = list(char_info[instrument].values()) + new_record['filter_pupil'] = optics + new_record['obs_per_filter_pupil'] = values with engine.begin() as connection: connection.execute( FilesystemCharacteristics.__table__.insert(), new_record) From d24f572cfc3a7f994eeeb2ccf28b110f7a92cb15 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 13 Apr 2023 14:42:48 -0400 Subject: [PATCH 282/449] update docstring --- jwql/jwql_monitors/monitor_filesystem.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py index b63d2e182..33fa0ced7 100755 --- a/jwql/jwql_monitors/monitor_filesystem.py +++ b/jwql/jwql_monitors/monitor_filesystem.py @@ -666,8 +666,10 @@ def update_characteristics_database(char_info): Parameters ---------- char_info : dict - A nested dictionary of characteristic information. Top level keys are - instrument names, and second level keys are filter/pupil strings. + A dictionary of characteristic information. Keys are + instrument names, and values are lists of tuples. Each tuple is + composed of a filter/pupil string and a count for the number of observations + using that filter/pupil. """ logging.info('\tUpdating the characteristics database') now = datetime.datetime.now() From 2f06306135469117c23b0a2d147350b8806d55d5 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 13 Apr 2023 14:44:23 -0400 Subject: [PATCH 283/449] update another docstring --- jwql/jwql_monitors/monitor_filesystem.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py index 33fa0ced7..2bf9ef705 100755 --- a/jwql/jwql_monitors/monitor_filesystem.py +++ b/jwql/jwql_monitors/monitor_filesystem.py @@ -294,9 +294,9 @@ def get_observation_characteristics(): Returns ------- n_obs : dict - Nested dictionary with instrument names as the top level keys, and - filter/pupil values as the second level keys. Values are the number of - observations that use the filter/pupil value. + Dictionary with instrument names as the top level keys, and lists of 2-tuples + as values. Each tuple contains filter/pupil string and the number of + observations that use that filter/pupil. """ n_obs = {} for instrument in ['nircam', 'niriss', 'nirspec', 'miri']: # Skip FGS here. It has no filters From f1d833a8b6a5c98926f9a0c78d1ca521e5886e97 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 14 Apr 2023 09:45:02 -0400 Subject: [PATCH 284/449] Move thumbnail img updates outside initial html creation loop --- jwql/website/apps/jwql/static/js/jwql.js | 58 +++++++++++++++--------- 1 file changed, 37 insertions(+), 21 deletions(-) diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index 43de8ddab..f4166d475 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -250,25 +250,6 @@ function clean_input_parameter(param_value) { } -/** - * Determine what filetype to use for a thumbnail - * @param {String} thumbnail_dir - The path to the thumbnail directory - * @param {List} suffixes - A list of available suffixes for the file of interest - * @param {Integer} i - The index of the thumbnail - * @param {String} file_root - The rootname of the file corresponding to the thumbnail - */ -function determine_filetype_for_thumbnail(thumbnail_dir, thumb_filename, i, file_root) { - - // Update the thumbnail filename - var img = document.getElementById('thumbnail'+i); - if (thumb_filename != 'none') { - var jpg_path = thumbnail_dir + parse_filename(file_root).program + '/' + thumb_filename; - img.src = jpg_path; - } - -} - - /** * Determine whether the page is archive or unlooked * @param {String} instrument - The instrument of interest @@ -552,6 +533,33 @@ function unhide_file(detector) { update_view_explore_link(); } + +/** + * Insert thumbnail images inside existing HTML img tags + * @param {List} updates - A list of updates to make, as [thumbnail_id, jpg_path]. + */ +function insert_thumbnail_images(updates) { + // Update the thumbnail image source + for (var i = 0; i < updates.length; i++) { + var thumb_id = updates[i][0]; + var jpg_path = updates[i][1]; + set_thumbnail_image_source(thumb_id, jpg_path); + } +} + + +/** + * Check for a thumbnail image and add it to an img if it exists + * @param {Integer} thumb_id - The ID number for the thumbnail img + * @param {String} jpg_filepath - The image to show + */ +function set_thumbnail_image_source(thumb_id, jpg_path) { + $.get(jpg_path, function() { + var img = document.getElementById('thumbnail' + thumb_id); + img.src = jpg_path;}) +} + + /** * Parse observation information from a JWST file name. * @param {String} filename - The file or group root name to parse @@ -1255,6 +1263,8 @@ function update_sort_options(data, base_url) { function update_thumbnail_array(data) { // Add content to the thumbnail array div + var thumbnail_content = ""; + var image_updates = []; for (var i = 0; i < Object.keys(data.file_data).length; i++) { // Parse out useful variables @@ -1295,11 +1305,17 @@ function update_thumbnail_array(data) { content += '
    '; // Add the content to the div - $("#thumbnail-array")[0].innerHTML += content; + thumbnail_content += content; // Add the appropriate image to the thumbnail - determine_filetype_for_thumbnail('/static/thumbnails/' , file.thumbnail, i, rootname); + if (file.thumbnail != 'none') { + var jpg_path = '/static/thumbnails/' + parse_filename(rootname).program + + '/' + file.thumbnail; + image_updates.push([i, jpg_path]); + } } + $("#thumbnail-array")[0].innerHTML = thumbnail_content; + insert_thumbnail_images(image_updates); } /** From 2437d2c9ee38f22bf4bf594f79a8b7f92e7d2edd Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Fri, 14 Apr 2023 09:57:50 -0400 Subject: [PATCH 285/449] remove MAST call in thumbnails_query_ajax --- jwql/website/apps/jwql/data_containers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 43b1ff2ca..7b32f55d9 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -2159,7 +2159,8 @@ def thumbnails_query_ajax(rootnames): data_dict['file_data'][rootname]['inst'] = JWST_INSTRUMENT_NAMES_MIXEDCASE[filename_parser(rootname)['instrument']] data_dict['file_data'][rootname]['filename_dict'] = filename_dict data_dict['file_data'][rootname]['available_files'] = available_files - exp_start = get_expstart(data_dict['file_data'][rootname]['inst'], rootname) + root_file_info = RootFileInfo.objects.get(root_name=rootname) + exp_start = root_file_info.expstart data_dict['file_data'][rootname]['expstart'] = exp_start data_dict['file_data'][rootname]['expstart_iso'] = Time(exp_start, format='mjd').iso.split('.')[0] data_dict['file_data'][rootname]['suffixes'] = [] From ad219c88fca88ea5526916b31e3fac7087c538f2 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Fri, 14 Apr 2023 10:44:52 -0400 Subject: [PATCH 286/449] Fix MIRI subarrays --- jwql/utils/constants.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index 432344cea..7dbc62e01 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -501,8 +501,8 @@ class QUERY_CONFIG_KEYS: 'niriss': ['FULL', 'SUBSTRIP96', 'SUBSTRIP256', 'SUB80', 'SUB64', 'SUB128', 'SUB256', 'WFSS64R', 'WFSS128R', 'WFSS64C', 'WFSS128C', 'SUBAMPCAL', 'SUBTAAMI', 'SUBTASOSS'], 'nirspec': [], - 'miri': ['MIRIM_BRIGHTSKY', 'MIRIM_FULL', 'MIRIM_MASK1065', 'MIRIM_MASK1140', - 'MIRIM_MASK1550', 'MIRIM_MASKLYOT', 'MIRIM_SLITLESSPRISM', 'MIRIM_SUB64', 'MIRIM_SUB128', 'MIRIM_SUB256'], + 'miri': ['BRIGHTSKY', 'FULL', 'MASK1065', 'MASK1140', 'MASK1550', 'MASKLYOT', 'SLITLESSPRISM', + 'SUB64', 'SUB128', 'SUB256'], 'fgs': []} # Filename suffixes that need to include the association value in the suffix in From 9a020a06d0c72409a2a0f8b8ab53aaf41a916c31 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 14 Apr 2023 12:33:21 -0400 Subject: [PATCH 287/449] Fix misspelled constant name --- jwql/jwql_monitors/monitor_filesystem.py | 6 +++--- jwql/utils/constants.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py index 2bf9ef705..95eddceb0 100755 --- a/jwql/jwql_monitors/monitor_filesystem.py +++ b/jwql/jwql_monitors/monitor_filesystem.py @@ -56,7 +56,7 @@ from jwql.database.database_interface import CentralStore from jwql.utils.logging_functions import log_info, log_fail from jwql.utils.permissions import set_permissions -from jwql.utils.constants import FILESYSYEM_MONITOR_SUBDIRS, FILE_SUFFIX_TYPES, FILTERS_PER_INSTRUMENT, INSTRUMENT_SERVICE_MATCH +from jwql.utils.constants import FILESYSTEM_MONITOR_SUBDIRS, FILE_SUFFIX_TYPES, FILTERS_PER_INSTRUMENT, INSTRUMENT_SERVICE_MATCH from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_INSTRUMENT_NAMES_MIXEDCASE from jwql.utils.utils import filename_parser from jwql.utils.utils import get_config @@ -540,7 +540,7 @@ def plot_central_store_dirs(): plot.circle(dates, availables, color='blue') # This part of the plot should cycle through areas and plot area used values vs. date - for area, color in zip(FILESYSYEM_MONITOR_SUBDIRS, colors): + for area, color in zip(FILESYSTEM_MONITOR_SUBDIRS, colors): # Query for used sizes results = session.query(CentralStore.date, CentralStore.used).filter(CentralStore.area == area) @@ -648,7 +648,7 @@ def update_central_store_database(central_storage_dict): central_storage_dict : dict A dictionary for the ``central_storage`` database table """ - for area in FILESYSYEM_MONITOR_SUBDIRS: + for area in FILESYSTEM_MONITOR_SUBDIRS: new_record = {} new_record['date'] = central_storage_dict['date'] new_record['area'] = area diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index 63e40ffa6..8d8cbab73 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -235,7 +235,7 @@ 'fgs': ['FGS_INTFLAT']} # output subdirectories to keep track of via the filesytem monitor -FILESYSYEM_MONITOR_SUBDIRS = ['logs', 'outputs', 'preview_images', 'thumbnails', 'all'] +FILESYSTEM_MONITOR_SUBDIRS = ['logs', 'outputs', 'preview_images', 'thumbnails', 'all'] FILTERS_PER_INSTRUMENT = {'fgs': [], 'miri': ['F1000W', 'F1130W', 'F1280W', 'OPAQUE', 'F2300C', 'F560W', 'P750L', From bba5f08bcc8e1055dbcf5c0c19d640f2bebba095 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 14 Apr 2023 12:47:31 -0400 Subject: [PATCH 288/449] Remove unused bokeh templating file --- jwql/jwql_monitors/monitor_filesystem.py | 1 + jwql/website/apps/jwql/bokeh_dashboard.py | 1 + .../monitor_pages/monitor_filesystem_bokeh.py | 225 ------------------ 3 files changed, 2 insertions(+), 225 deletions(-) delete mode 100644 jwql/website/apps/jwql/monitor_pages/monitor_filesystem_bokeh.py diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py index 95eddceb0..0d6158d51 100755 --- a/jwql/jwql_monitors/monitor_filesystem.py +++ b/jwql/jwql_monitors/monitor_filesystem.py @@ -11,6 +11,7 @@ - Misty Cracraft - Sara Ogaz - Matthew Bourque + - Bryan Hilbert Use --- diff --git a/jwql/website/apps/jwql/bokeh_dashboard.py b/jwql/website/apps/jwql/bokeh_dashboard.py index 47a3933a3..2c282e274 100644 --- a/jwql/website/apps/jwql/bokeh_dashboard.py +++ b/jwql/website/apps/jwql/bokeh_dashboard.py @@ -11,6 +11,7 @@ ------- - Mees B. Fix + - Bryan Hilbert Use --- diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_filesystem_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_filesystem_bokeh.py deleted file mode 100644 index fa208db79..000000000 --- a/jwql/website/apps/jwql/monitor_pages/monitor_filesystem_bokeh.py +++ /dev/null @@ -1,225 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Created on Wed Jan 16 14:09:18 2019 - -@author: gkanarek -""" - -import json -import os - -from astropy.table import Table, vstack -from astropy.time import Time - -from jwql.bokeh_templating import BokehTemplate -from jwql.utils.utils import get_config - -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) - -FIG_FORMATS = """ -Figure: - tools: 'pan,box_zoom,reset,wheel_zoom,save' - x_axis_type: 'datetime' - x_axis_label: 'Date' - sizing_mode: 'stretch_both' -Line: - line_width: 2 -""" - - -class MonitorFilesystem(BokehTemplate): - - def pre_init(self): - self._embed = True - - # App design - self.format_string = FIG_FORMATS - self.interface_file = os.path.join(SCRIPT_DIR, "yaml", "monitor_filesystem_interface.yaml") - - # Get path, directories and files in system and count files in all directories - self.settings = get_config() - self.filesystem = self.settings['filesystem'] - self.outputs_dir = os.path.join(self.settings['outputs'], - 'monitor_filesystem') - - self.allowed_types = ['fits_files', 'uncal', 'cal', 'rate', 'rateints', - 'i2d', 'nrc', 'nrs', 'nis', 'mir', 'gui'] - - # Load any existing data - self.initial_load() - - self.types_k = ['circle', 'diamond', 'square', 'triangle', - 'asterisk'] + ['x'] * 6 - self.types_y = ['fits', 'uncal', 'cal', 'rate', 'rateint', - 'i2d', 'nrc', 'nrs', 'nis', 'mir', 'fgs'] - self.types_c = ['black', 'red', 'blue', 'green', 'orange', 'purple', - 'midnightblue', 'springgreen', 'darkcyan', - 'dodgerblue', 'darkred'] - self.types_l = ['Total FITS files', 'Uncalibrated FITS files', - 'Calibrated FITS files', 'Rate FITS files', - 'Rateints FITS files', 'I2D FITS files', - 'NIRCam FITS files', 'NIRSpec FITS files', - 'NIRISS FITS files', 'MIRI FITS files', - 'FGS FITS files'] - - def post_init(self): - self.update_plots(full=True) - - def initial_load(self): - statsfile = os.path.join(self.outputs_dir, 'statsfile.json') - filebytype = os.path.join(self.outputs_dir, 'filesbytype.json') - sizebytype = os.path.join(self.outputs_dir, 'sizebytype.json') - - self.statistics = Table(names=['timestamp', 'file_count', 'total', - 'available', 'used', 'percent_used'], - dtype=[Time, int, int, int, int, float]) - self.statistics['percent_used'].format = "%.1f" - if os.path.exists(statsfile): - with open(statsfile) as f: - stats = json.load(f) - times, fc, tot, avail, used, perc = zip(*stats) - self.statistics['timestamp'] = Time(times) - self.statistics['file_count'] = map(int, fc) - self.statistics['total'] = map(int, tot) - self.statistics['available'] = map(int, avail) - self.statistics['used'] = map(int, used) - self.statistics['percent_used'] = map(float, perc) - - self.ftypes = Table(names=['timestamp'] + self.allowed_types, - dtype=[Time] + [int] * 11) - if os.path.exists(filebytype): - with open(filebytype) as f: - fbytype = json.load(f) - times, *ftypes = zip(*fbytype) - self.ftypes['timestamp'] = Time(times) - for c, colname in enumerate(self.allowed_types): - self.ftypes[colname] = map(int, ftypes[c]) - - self.stypes = Table(names=['timestamp'] + self.allowed_types, - dtype=[Time] + [float] * 11) - if os.path.exists(sizebytype): - with open(sizebytype) as f: - sbytype = json.load(f) - times, *stypes = zip(*sbytype) - self.stypes['timestamp'] = Time(times) - for c, colname in enumerate(self.allowed_types): - self.stypes[colname] = map(int, stypes[c]) - - def update_plots(self, full=False): - - if full: - # Initialize each ColumnDataSource so that we can use stream() later - self.refs['source_filecount'].data = { - 'dates': self.statistics['timestamp'].datetime64, - 'filecount': self.statistics['file_count'].data} - - self.refs['source_stats'].data = { - 'dates': self.statistics['timestamp'].datetime64, - 'systemsize': self.statistics['total'].data.astype(float) / (1024.**3), - 'freesize': self.statistics['available'].data.astype(float) / (1024.**3), - 'usedsize': self.statistics['used'].data.astype(float) / (1024.**3)} - - ftype_dict = {'dates': self.ftypes['timestamp'].datetime64} - ftype_dict.update({x: self.ftypes[y].data for x, y in zip(self.types_y, - self.allowed_types)}) - self.refs['source_files'].data = ftype_dict - - stype_dict = {'dates': self.stypes['timestamp'].datetime64} - stype_dict.update({x: self.stypes[y].data for x, y in zip(self.types_y, - self.allowed_types)}) - self.refs['source_sizes'].data = stype_dict - else: - new_stats, new_files, new_sizes = self.read_new_data() - if new_stats: - self.refs['source_filecount'].stream({ - 'dates': new_stats['timestamp'].datetime64, - 'filecount': new_stats['file_count'].data}) - self.refs['source_stats'].stream({ - 'dates': new_stats['timestamp'].datetime64, - 'systemsize': new_stats['total'].data, - 'freesize': new_stats['available'].data, - 'usedsize': new_stats['used'].data}) - if new_files: - ftype_dict = {'dates': new_files['timestamp'].datetime64} - ftype_dict.update({x: new_files[y].data for x, y in zip(self.types_y, - self.allowed_types)}) - self.refs['source_files'].stream(ftype_dict) - - if new_sizes: - stype_dict = {'dates': new_sizes['timestamp'].datetime64} - stype_dict.update({x: new_sizes[y].data / (1024.**3) - for x, y in zip(self.types_y, self.allowed_types)}) - self.refs['source_sizes'].data = stype_dict - - if not self.statistics: - self.latest_timestamp = Time(0., format='unix') - else: - self.latest_timestamp = self.statistics['timestamp'].max() - - def read_new_data(self): - """ - Algorithm: - 1. Read in the json files (this step will be replaced when we move - away from json) into tables. - 2. Create new tables from all rows which have been added since the - last timestamp in the current tables. - 3. Concatenate the new tables with a vertical join. - 4. Return the new tables so they can be streamed to the plots. - """ - statsfile = os.path.join(self.outputs_dir, 'statsfile.json') - filebytype = os.path.join(self.outputs_dir, 'filesbytype.json') - sizebytype = os.path.join(self.outputs_dir, 'sizebytype.json') - - # Have any of the files been modified since the last timestamp? - stats_modtime = Time(os.stat(statsfile).st_mtime, format='unix') - files_modtime = Time(os.stat(filebytype).st_mtime, format='unix') - sizes_modtime = Time(os.stat(sizebytype).st_mtime, format='unix') - - new_stats = Table(names=self.statistics.colnames, - dtype=self.statistics.dtype) - new_files = Table(names=self.ftypes.colnames, - dtype=self.ftypes.dtype) - new_sizes = Table(names=self.stypes.colnames, - dtype=self.stypes.dtype) - - if stats_modtime > self.latest_timestamp: - with open(statsfile) as f: - stats = json.load(f) - times, fc, tot, avail, used, perc = zip(*stats) - times = Time(times) - new_rows = times > self.latest_timestamp - new_stats['timestamp'] = times[new_rows] - new_stats['file_count'] = map(int, fc[new_rows]) - new_stats['total'] = map(int, tot[new_rows]) - new_stats['available'] = map(int, avail[new_rows]) - new_stats['used'] = map(int, used[new_rows]) - new_stats['percent_used'] = map(float, perc[new_rows]) - - self.statistics = vstack([self.statistics, new_stats]) - - if files_modtime > self.latest_timestamp: - with open(filebytype) as f: - fbytype = json.load(f) - times, *ftypes = zip(*fbytype) - times = Time(times) - new_rows = times > self.latest_timestamp - new_files['timestamp'] = times[new_rows] - for c, colname in enumerate(self.allowed_types): - new_files[colname] = map(int, ftypes[c][new_rows]) - - self.ftypes = vstack([self.ftypes, new_files]) - - if sizes_modtime > self.latest_timestamp: - with open(sizebytype) as f: - sbytype = json.load(f) - times, *stypes = zip(*sbytype) - times = Time(times) - new_rows = times > self.latest_timestamp - new_sizes['timestamp'] = times[new_rows] - for c, colname in enumerate(self.allowed_types): - new_sizes[colname] = map(int, stypes[c][new_rows]) - - self.stypes = vstack([self.stypes, new_sizes]) - - return new_stats, new_files, new_sizes From 2eb6da7d1e205c43d9fb325340ff08f7f13a5064 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 14 Apr 2023 13:17:10 -0400 Subject: [PATCH 289/449] Remove old unused plotting functions --- jwql/jwql_monitors/monitor_filesystem.py | 230 ----------------------- 1 file changed, 230 deletions(-) diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py index 0d6158d51..2c1f4d379 100755 --- a/jwql/jwql_monitors/monitor_filesystem.py +++ b/jwql/jwql_monitors/monitor_filesystem.py @@ -410,236 +410,6 @@ def monitor_filesystem(): update_characteristics_database(characteristics) update_central_store_database(central_storage_dict) - # Create the plots - #plot_filesystem_stats() - - -def plot_by_filetype(plot_type, instrument): - """Plot ``count`` or ``size`` by filetype versus date for the given - instrument, or all instruments. - - Parameters - ---------- - plot_type : str - Which data to plot. Either ``count`` or ``size``. - instrument : str - The instrument to plot for. Can be a valid JWST instrument or - ``all`` to plot across all instruments. - - Returns - ------- - plot : bokeh.plotting.figure.Figure object - ``bokeh`` plot of total file counts versus date - """ - - # Determine plot title - if instrument == 'all': - title = 'Total File {} by Type'.format(plot_type.capitalize()) - else: - instrument_title = JWST_INSTRUMENT_NAMES_MIXEDCASE[instrument] - title = '{} Total File {} by Type'.format(instrument_title, plot_type.capitalize()) - - if plot_type == 'count': - ytitle = 'Counts' - else: - ytitle = 'Size (TB)' - - # Initialize plot - plot = figure( - tools='pan,box_zoom,wheel_zoom,reset,save', - x_axis_type='datetime', - title=title, - x_axis_label='Date', - y_axis_label=ytitle) - colors = itertools.cycle(palette) - - for filetype, color in zip(FILE_SUFFIX_TYPES, colors): - - # Query for counts - results = session.query(FilesystemInstrument.date, getattr(FilesystemInstrument, plot_type))\ - .filter(FilesystemInstrument.filetype == filetype) - - if instrument == 'all': - results = results.all() - else: - results = results.filter(FilesystemInstrument.instrument == instrument).all() - - # Group by date - if results: - results_dict = defaultdict(int) - for date, value in results: - results_dict[date] += value - - # Parse results so they can be easily plotted - dates = list(results_dict.keys()) - values = list(results_dict.values()) - - # Plot the results - plot.line(dates, values, legend='{} files'.format(filetype), line_color=color) - plot.circle(dates, values, color=color) - - session.close() - - return plot - - -def plot_filesystem_size(): - """Plot filesystem sizes (size, used, available) versus date - - Returns - ------- - plot : bokeh.plotting.figure.Figure object - ``bokeh`` plot of total file counts versus date - """ - - # Plot system stats vs. date - results = session.query(FilesystemGeneral.date, FilesystemGeneral.total_file_size, - FilesystemGeneral.used, FilesystemGeneral.available).all() - dates, total_sizes, useds, availables = zip(*results) - plot = figure( - tools='pan,box_zoom,wheel_zoom,reset,save', - x_axis_type='datetime', - title='System stats', - x_axis_label='Date', - y_axis_label='Size TB') - plot.line(dates, total_sizes, legend='Total size', line_color='red') - plot.circle(dates, total_sizes, color='red') - plot.line(dates, useds, legend='Used bytes', line_color='green') - plot.circle(dates, useds, color='green') - plot.line(dates, availables, legend='Free bytes', line_color='blue') - plot.circle(dates, availables, color='blue') - - session.close() - return plot - - -def plot_central_store_dirs(): - """Plot central store sizes (size, used, available) versus date - - Returns - ------- - plot : bokeh.plotting.figure.Figure object - ``bokeh`` plot of total directory size versus date - """ - - # Plot system stats vs. date - results = session.query(CentralStore.date, CentralStore.size, CentralStore.available).all() - - # Initialize plot - dates, total_sizes, availables = zip(*results) - plot = figure( - tools='pan,box_zoom,wheel_zoom,reset,save', - x_axis_type='datetime', - title='Central Store stats', - x_axis_label='Date', - y_axis_label='Size TB') - colors = itertools.cycle(palette) - - plot.line(dates, total_sizes, legend='Total size', line_color='red') - plot.circle(dates, total_sizes, color='red') - plot.line(dates, availables, legend='Free', line_color='blue') - plot.circle(dates, availables, color='blue') - - # This part of the plot should cycle through areas and plot area used values vs. date - for area, color in zip(FILESYSTEM_MONITOR_SUBDIRS, colors): - - # Query for used sizes - results = session.query(CentralStore.date, CentralStore.used).filter(CentralStore.area == area) - - # Group by date - if results: - results_dict = defaultdict(int) - for date, value in results: - results_dict[date] += value - - # Parse results so they can be easily plotted - dates = list(results_dict.keys()) - values = list(results_dict.values()) - - # Plot the results - plot.line(dates, values, legend='{} files'.format(area), line_color=color) - plot.circle(dates, values, color=color) - - session.close() - - return plot - - -def plot_filesystem_stats(): - """ - Plot various filesystem statistics using ``bokeh`` and save them to - the output directory. - """ - logging.info('Creating results plots') - - p1 = plot_total_file_counts() - p2 = plot_filesystem_size() - p3 = plot_by_filetype('count', 'all') - p4 = plot_by_filetype('size', 'all') - p5 = plot_central_store_dirs() - plot_list = [p1, p2, p3, p4, p5] - - for instrument in JWST_INSTRUMENT_NAMES: - plot_list.append(plot_by_filetype('count', instrument)) - plot_list.append(plot_by_filetype('size', instrument)) - - # Create a layout with a grid pattern - grid_chunks = [plot_list[i:i + 2] for i in range(0, len(plot_list), 2)] - grid = gridplot(grid_chunks) - - # Save all of the plots in one file - outputs_dir = os.path.join(OUTPUTS, 'monitor_filesystem') - outfile = os.path.join(outputs_dir, 'filesystem_monitor.html') - output_file(outfile) - save(grid) - set_permissions(outfile) - logging.info('\tSaved plot of all statistics to {}'.format(outfile)) - - # Save each plot's components - for plot in plot_list: - plot_name = plot.title.text.lower().replace(' ', '_') - plot.sizing_mode = 'stretch_both' - script, div = components(plot) - - div_outfile = os.path.join(outputs_dir, "{}_component.html".format(plot_name)) - with open(div_outfile, 'w') as f: - f.write(div) - f.close() - set_permissions(div_outfile) - - script_outfile = os.path.join(outputs_dir, "{}_component.js".format(plot_name)) - with open(script_outfile, 'w') as f: - f.write(script) - f.close() - set_permissions(script_outfile) - - logging.info('\tSaved components files: {}_component.html and {}_component.js'.format(plot_name, plot_name)) - - -def plot_total_file_counts(): - """Plot total file counts versus date - - Returns - ------- - plot : bokeh.plotting.figure.Figure object - ``bokeh`` plot of total file counts versus date - """ - - # Total file counts vs. date - results = session.query(FilesystemGeneral.date, FilesystemGeneral.total_file_count).all() - dates, file_counts = zip(*results) - plot = figure( - tools='pan,box_zoom,reset,wheel_zoom,save', - x_axis_type='datetime', - title="Total File Counts", - x_axis_label='Date', - y_axis_label='Count') - plot.line(dates, file_counts, line_width=2, line_color='blue') - plot.circle(dates, file_counts, color='blue') - - session.close() - - return plot def update_central_store_database(central_storage_dict): """Updates the ``CentralStore`` database table with info on disk space From 8b3072cbda154910a4aacaf2e8dc028bbae7831d Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 17 Apr 2023 10:16:28 -0400 Subject: [PATCH 290/449] Remove now-incorrect import statement --- jwql/website/apps/jwql/monitor_pages/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jwql/website/apps/jwql/monitor_pages/__init__.py b/jwql/website/apps/jwql/monitor_pages/__init__.py index 136bd10d6..251fe3f60 100644 --- a/jwql/website/apps/jwql/monitor_pages/__init__.py +++ b/jwql/website/apps/jwql/monitor_pages/__init__.py @@ -1,4 +1,3 @@ from .monitor_cosmic_rays_bokeh import CosmicRayMonitor -from .monitor_filesystem_bokeh import MonitorFilesystem from .monitor_mast_bokeh import MastMonitor from .monitor_readnoise_bokeh import ReadnoiseMonitor From 99924c4faf9a5d089149e2bc863545be4bb26963 Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Mon, 17 Apr 2023 10:40:38 -0400 Subject: [PATCH 291/449] move build configuration from `setup.py` to `pyproject.toml` --- pyproject.toml | 69 ++++++++++++++++++++++++++++++++++++++++++++++---- setup.py | 58 ------------------------------------------ 2 files changed, 64 insertions(+), 63 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cffa0df75..e9e48654e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,65 @@ -# pyproject.toml +[project] +name = "jwql" +version = "1.1.1" +description = "The James Webb Space Telescope Quicklook Project" +authors = [ + { name = "Matthew Bourque" }, + { name = "Lauren Chambers" }, + { name = "Misty Cracraft" }, + { name = "Mike Engesser" }, + { name = "Mees Fix" }, + { name = "Joe Filippazzo" }, + { name = "Bryan Hilbert" }, +] +keywords = ["astronomy", "python"] +classifiers = ["Programming Language :: Python"] +dependencies = [ + "asdf", + "astropy", + "astroquery", + "bandit", + "bokeh<3", + "crds", + "cryptography", + "django", + "flake8", + "inflection", + "ipython", + "jinja2", + "jsonschema", + "jwst", + "jwst_reffiles", + "matplotlib", + "nodejs", + "numpy", + "numpydoc", + "pandas", + "psycopg2", + "pysiaf", + "pytest", + "pytest-cov", + "pytest-mock", + "pyvo", + "scipy", + "sphinx", + "sphinx_rtd_theme", + "sqlalchemy", + "stdatamodels", + "stsci_rtd_theme", + "twine", + "wtforms", +] + +[project.license] +file = "LICENSE" +content-type = "text/plain" + [build-system] -requires = ["setuptools<=65.5.0", - "numpy", - "wheel", - "setuptools_scm"] \ No newline at end of file +requires = ["setuptools>=61.2,<=65.5.0", "numpy", "wheel", "setuptools_scm"] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +namespaces = false diff --git a/setup.py b/setup.py index 6d30c3958..2101b2a04 100644 --- a/setup.py +++ b/setup.py @@ -1,64 +1,6 @@ import numpy as np from setuptools import setup -from setuptools import find_packages - -VERSION = '1.1.1' - -AUTHORS = 'Matthew Bourque, Lauren Chambers, Misty Cracraft, Mike Engesser, Mees Fix, Joe Filippazzo, Bryan Hilbert, ' -AUTHORS += 'Graham Kanarek, Teagan King, Catherine Martlin, Shannon Osborne, Maria Pena-Guerrero, Johannes Sahlmann, ' -AUTHORS += 'Ben Sunnquist, Brian York' - -DESCRIPTION = 'The James Webb Space Telescope Quicklook Project' - -REQUIRES = [ - 'asdf', - 'astropy', - 'astroquery', - 'bandit', - 'bokeh<3', - 'crds', - 'cryptography', - 'django', - 'flake8', - 'inflection', - 'ipython', - 'jinja2', - 'jsonschema', - 'jwst', - 'jwst_reffiles', - 'matplotlib', - 'nodejs', - 'numpy', - 'numpydoc', - 'pandas', - 'psycopg2', - 'pysiaf', - 'pytest', - 'pytest-cov', - 'pytest-mock', - 'pyvo', - 'scipy', - 'sphinx', - 'sphinx_rtd_theme', - 'sqlalchemy', - 'stdatamodels', - 'stsci_rtd_theme', - 'twine', - 'wtforms' -] setup( - name='jwql', - version=VERSION, - description=DESCRIPTION, - url='https://github.com/spacetelescope/jwql.git', - author=AUTHORS, - author_email='jwql@stsci.edu', - license='BSD', - keywords=['astronomy', 'python'], - classifiers=['Programming Language :: Python'], - packages=find_packages(), - install_requires=REQUIRES, - include_package_data=True, include_dirs=[np.get_include()], ) From fcc01bcc00bb41ca9ad3218f45b55996f835048e Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Mon, 17 Apr 2023 10:51:51 -0400 Subject: [PATCH 292/449] replace references to setup.py --- .github/workflows/ci.yml | 2 +- MANIFEST.in | 2 +- README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 88f823818..c23a8a201 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,7 +56,7 @@ jobs: echo " " echo "Installing jwql package" echo " " - python setup.py develop + pip install -e . echo " " echo "Testing package installation" diff --git a/MANIFEST.in b/MANIFEST.in index 478c9f4d7..20b6bed98 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,7 +4,7 @@ include LICENSE include CHANGES.rst include environment_python_3_7.yml include environment_python_3_8.yml -include setup.py +include pyproject.toml include requirements.txt include rtd_requirements.txt include jwql/example_config.json diff --git a/README.md b/README.md index c3f041f11..addbefc19 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ It is highly suggested that contributors have a working installation of `anacond - [Miniconda](https://conda.io/miniconda.html) - [Anaconda](https://www.continuum.io/downloads) -Requirements for contributing to the `jwql` package will be included in the `jwql` `conda` environment, which is included in our installation instructions below. Further package requirements will be provided for `jwql` by a `setup.py` script included in the repository. +Requirements for contributing to the `jwql` package will be included in the `jwql` `conda` environment, which is included in our installation instructions below. Further package requirements will be provided for `jwql` by a `pyproject.toml` file included in the repository. ### Clone the `jwql` repo From 5a2d5c03053d92dab38013bc530b44bc3c98a83f Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Mon, 17 Apr 2023 10:52:02 -0400 Subject: [PATCH 293/449] read `pyproject.toml` file --- jwql/utils/logging_functions.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/jwql/utils/logging_functions.py b/jwql/utils/logging_functions.py index 3cbadd35f..af2e98fe7 100644 --- a/jwql/utils/logging_functions.py +++ b/jwql/utils/logging_functions.py @@ -65,6 +65,11 @@ def my_main_function(): import time import traceback +if sys.version_info < (3, 11): + import tomli as tomllib + else: + import tomllib + from functools import wraps from jwql.utils.permissions import set_permissions @@ -218,15 +223,10 @@ def wrapped(*args, **kwargs): logging.info('Running as PID {}'.format(os.getpid())) # Read in setup.py file to build list of required modules - with open(get_config()['setup_file']) as f: - data = f.readlines() - - for i, line in enumerate(data): - if 'REQUIRES = [' in line: - begin = i + 1 - elif 'setup(' in line: - end = i - 2 - required_modules = data[begin:end] + with open("pyproject.toml", "rb") as f: + data = tomllib.load(f) + + required_modules = data['project']['dependencies'] # Clean up the module list module_list = [item.strip().replace("'", "").replace(",", "").split("=")[0].split(">")[0].split("<")[0] for item in required_modules] From 2e0a50322469c164ce2224071f5bb3c4cc4ba713 Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Mon, 17 Apr 2023 10:53:53 -0400 Subject: [PATCH 294/449] use `setuptools_scm` to retrieve version number from Git tag --- pyproject.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e9e48654e..d21dadcf9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,5 @@ [project] name = "jwql" -version = "1.1.1" description = "The James Webb Space Telescope Quicklook Project" authors = [ { name = "Matthew Bourque" }, @@ -49,6 +48,7 @@ dependencies = [ "twine", "wtforms", ] +dynamic = ["version"] [project.license] file = "LICENSE" @@ -63,3 +63,5 @@ include-package-data = true [tool.setuptools.packages.find] namespaces = false + +[tool.setuptools_scm] From 04b17c49a8b95c7125d7bbd51270b7b053e06ee1 Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Mon, 17 Apr 2023 11:00:24 -0400 Subject: [PATCH 295/449] add README to metadata --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index d21dadcf9..d10f5c10c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,7 @@ [project] name = "jwql" description = "The James Webb Space Telescope Quicklook Project" +readme = "README.md" authors = [ { name = "Matthew Bourque" }, { name = "Lauren Chambers" }, From 299d25548381ca8a88d7e3132d2d8da495a60c89 Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Mon, 17 Apr 2023 11:05:33 -0400 Subject: [PATCH 296/449] move optional dependencies into `optional-dependencies` --- .github/workflows/ci.yml | 2 +- pyproject.toml | 22 ++++++++++++---------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c23a8a201..5accc9e3f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,7 +56,7 @@ jobs: echo " " echo "Installing jwql package" echo " " - pip install -e . + pip install -e .[test] echo " " echo "Testing package installation" diff --git a/pyproject.toml b/pyproject.toml index d10f5c10c..738db37a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,14 +17,11 @@ dependencies = [ "asdf", "astropy", "astroquery", - "bandit", "bokeh<3", "crds", "cryptography", "django", - "flake8", "inflection", - "ipython", "jinja2", "jsonschema", "jwst", @@ -36,21 +33,26 @@ dependencies = [ "pandas", "psycopg2", "pysiaf", - "pytest", - "pytest-cov", - "pytest-mock", "pyvo", "scipy", - "sphinx", - "sphinx_rtd_theme", "sqlalchemy", "stdatamodels", - "stsci_rtd_theme", - "twine", "wtforms", ] dynamic = ["version"] +[project.optional-dependencies] +test = [ + "pytest", + "pytest-cov", + "pytest-mock", +] +docs = [ + "sphinx", + "sphinx_rtd_theme", + "stsci_rtd_theme", +] + [project.license] file = "LICENSE" content-type = "text/plain" From 54ebc8dfbfe5f58c94f2de6f827fc52d644d5703 Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Mon, 17 Apr 2023 10:35:39 -0400 Subject: [PATCH 297/449] drop support for Python 3.8 --- .github/workflows/ci.yml | 2 +- README.md | 4 +- environment_python_3_8.yml | 73 ------------------------------- jwql/tests/test_pipeline_tools.py | 2 - 4 files changed, 3 insertions(+), 78 deletions(-) delete mode 100644 environment_python_3_8.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 88f823818..8a21b84ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: max-parallel: 5 matrix: os: [ubuntu-latest, macos-latest] - python-version: [3.8, 3.9] + python-version: [3.9, "3.10"] steps: - uses: actions/checkout@v3 diff --git a/README.md b/README.md index c3f041f11..32ab7708f 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Getting `jwql` up and running on your own computer requires four steps, detailed ### Prerequisites -It is highly suggested that contributors have a working installation of `anaconda` or `miniconda` for Python 3.8. Downloads and installation instructions are available here: +It is highly suggested that contributors have a working installation of `anaconda` or `miniconda` for Python 3.9+. Downloads and installation instructions are available here: - [Miniconda](https://conda.io/miniconda.html) - [Anaconda](https://www.continuum.io/downloads) @@ -83,7 +83,7 @@ source activate base/root **Note:** If you have added a step activating conda to your default terminal/shell (e.g. the `.bashrc`, `.zshrc`, or `.profile` file) then you don't need to do the above step. -Lastly, create the `jwql` environment via one of the `environment.yml` files (currently `environment_python_3_8.yml`, for python 3.8, and `environment_python_3.9.yml`, for python 3.9, are supported by `jwql`): +Lastly, create the `jwql` environment via one of the `environment.yml` files (currently `environment_python_3_9.yml`, for python 3.9, and `environment_python_3.10.yml`, for python 3.10, are supported by `jwql`): ``` conda env create -f environment_python_3_8.yml diff --git a/environment_python_3_8.yml b/environment_python_3_8.yml deleted file mode 100644 index df3d3a173..000000000 --- a/environment_python_3_8.yml +++ /dev/null @@ -1,73 +0,0 @@ -# This file describes a conda environment that can be to install jwql -# -# Run the following command to set up this environment: -# $ conda env create -f environment_python_3_8.yml -# -# The environment name can be overridden with the following command: -# $ conda env create -n -f environment_python_3_8.yml -# -# Run the following command to activate the environment: -# $ source activate jwql-3.8 -# -# To deactivate the environment run the following command: -# $ source deactivate -# -# To remove the environment entirely, run the following command: -# $ conda env remove -n jwql-3.8 - -name: jwql-3.8 - -channels: - - conda-forge - - defaults - -dependencies: - - astropy=5.2.1 - - bokeh=2.4.3 - - beautifulsoup4=4.11.2 - - celery=5.2.7 - - cryptography=39.0.1 - - django=4.1.7 - - flake8=6.0.0 - - inflection=0.5.1 - - ipython=8.10.0 - - jinja2=3.1.2 - - jsonschema=4.17.3 - - matplotlib=3.7.0 - - nodejs=18.12.1 - - numpy=1.24.2 - - numpydoc=1.5.0 - - pandas=2.0.0 - - pip=23.0 - - postgresql=15.2 - - psycopg2=2.9.3 - - pytest=7.2.1 - - pytest-cov=4.0.0 - - pytest-mock=3.10.0 - - python=3.8.16 - - pyyaml=6.0 - - redis - - scipy=1.9.3 - - setuptools=67.3.1 - - sphinx=6.1.3 - - sphinx_rtd_theme=1.2.0 - - sqlalchemy=2.0.8 - - twine=4.0.2 - - wtforms=3.0.1 - - - pip: - - astroquery==0.4.6 - - bandit==1.7.4 - - jwst==1.10.0 - - pysiaf==0.19.1 - - pysqlite3==0.5.0 - - pyvo==1.4 - - redis==4.5.1 - - selenium==4.8.0 - - stdatamodels==1.3.1 - - stsci_rtd_theme==1.0.0 - - vine==5.0.0 - - git+https://github.com/spacetelescope/jwst_reffiles - - # Current package - - -e . diff --git a/jwql/tests/test_pipeline_tools.py b/jwql/tests/test_pipeline_tools.py index d28b19aea..841cdb6d4 100644 --- a/jwql/tests/test_pipeline_tools.py +++ b/jwql/tests/test_pipeline_tools.py @@ -67,7 +67,6 @@ def test_completed_pipeline_steps(): assert completed_steps == true_completed -@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Drizzle import issue with python 3.8') def test_get_pipeline_steps(): """Test that the proper pipeline steps are returned for an instrument @@ -139,7 +138,6 @@ def test_image_stack(): assert exptimes == [[10.5], [10.5], [10.5]] -@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Drizzle import issue with python 3.8') def test_steps_to_run(): """Test that the dictionaries for steps required and steps completed are correctly combined to create a dictionary of pipeline steps to From f26a122419fa754466da2d8fffd3556a8e6878c3 Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Tue, 18 Apr 2023 07:45:37 -0400 Subject: [PATCH 298/449] run all jobs to completion --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a21b84ca..060c23a73 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: max-parallel: 5 + fail-fast: false matrix: os: [ubuntu-latest, macos-latest] python-version: [3.9, "3.10"] From ca455d230ff53563256f213092d4f518ae457e5a Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Tue, 18 Apr 2023 07:57:33 -0400 Subject: [PATCH 299/449] remove 3.10 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 060c23a73..065a7654b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest] - python-version: [3.9, "3.10"] + python-version: [3.9] steps: - uses: actions/checkout@v3 From 7f39ed03c6c8099c0ef3ad5436299e372891004d Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Mon, 17 Apr 2023 10:02:32 -0400 Subject: [PATCH 300/449] use micromamba to build environment and split job --- .github/workflows/ci.yml | 84 ++++++++++++++-------------------------- 1 file changed, 30 insertions(+), 54 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 88f823818..e87a0ae83 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,10 +2,24 @@ name: JWQL CI on: [push, pull_request] -jobs: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true - JWQL-CI: +defaults: + run: + shell: bash -l {0} +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + - run: pip install bandit + - run: bandit ./jwql/ -c .bandit + + test: name: Python - ${{ matrix.python-version }} runs-on: ${{ matrix.os }} strategy: @@ -17,63 +31,25 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: conda-incubator/setup-miniconda@v2 - with: - activate-environment: jwql-${{ matrix.python-version }} + - run: | + echo "major=$(echo ${{ matrix.python-version }} | head -c 1)" >> $GITHUB_OUTPUT + echo "minor=$(echo ${{ matrix.python-version }} | tail -c 2)" >> $GITHUB_OUTPUT + id: python - - name: Bandit Check - uses: jpetrucciani/bandit-check@master + - uses: mamba-org/provision-with-micromamba@v15 with: - path: "./jwql/" - bandit_flags: "-c .bandit" - continue-on-error: false - if: runner.os == 'Linux' + environment-file: ./environment_python_${{ steps.python.outputs.major }}_${{ steps.python.outputs.minor }}.yml + environment-name: jwql-${{ matrix.python-version }} - - name: Start Redis - uses: supercharge/redis-github-action@1.4.0 + - run: pip install -e . + + - run: conda env export + + - uses: supercharge/redis-github-action@1.4.0 with: redis-version: 5.0 if: runner.os == 'Linux' - - name: Build jwql conda environment and run tests - shell: bash -l {0} - run: | - - echo "Setting useful environment variables:" - echo " " - export PYTHONVERSION=${{ matrix.python-version }} - export MAJOR="$(echo $PYTHONVERSION | head -c 1)" - export MINOR="$(echo $PYTHONVERSION | tail -c 2)" - echo "PYTHONVERSION: $PYTHONVERSION" - - echo " " - echo "Installing jwql conda environment" - echo " " - cd $RUNNER_WORKSPACE/ - cd jwql/ - $CONDA/bin/conda env update -f environment_python_${MAJOR}_${MINOR}.yml -n jwql-${{ matrix.python-version }} - - echo " " - echo "Installing jwql package" - echo " " - python setup.py develop - - echo " " - echo "Testing package installation" - echo " " - python -c "import jwql; print('Version: ' + jwql.__version__); print('Path: ' + jwql.__path__[0])" - - # echo " " - # echo "Reinstall numpy to fix numpy.core.multiarray error" - # echo " " - pip install -U numpy - - echo " " - echo "The conda environment being used:" - echo " " - $CONDA/bin/conda env export + - run: python -c "import jwql; print('Version ' + jwql.__version__); print('Path ' + jwql.__path__[0])" - echo " " - echo "Running pytests" - echo " " - pytest jwql/tests/ + - run: pytest jwql/tests/ From 08dbba8bf929599d9c0368e02c3a896516107d92 Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Mon, 17 Apr 2023 10:31:14 -0400 Subject: [PATCH 301/449] cache environment --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e87a0ae83..55150f2ce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,8 +25,8 @@ jobs: strategy: max-parallel: 5 matrix: - os: [ubuntu-latest, macos-latest] - python-version: [3.8, 3.9] + os: [ ubuntu-latest, macos-latest ] + python-version: [ 3.8, 3.9 ] steps: - uses: actions/checkout@v3 @@ -40,6 +40,8 @@ jobs: with: environment-file: ./environment_python_${{ steps.python.outputs.major }}_${{ steps.python.outputs.minor }}.yml environment-name: jwql-${{ matrix.python-version }} + cache-env: true + cache-downloads: true - run: pip install -e . From a3ad305feb7c7fcd466f6196a09a69431501443c Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Mon, 17 Apr 2023 10:56:36 -0400 Subject: [PATCH 302/449] set job name --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 55150f2ce..660f4207f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: - run: bandit ./jwql/ -c .bandit test: - name: Python - ${{ matrix.python-version }} + name: test (Python ${{ matrix.python-version }}, ${{ matrix.os }}) runs-on: ${{ matrix.os }} strategy: max-parallel: 5 From 350a57548d19680183f1c09927aec5f2a7df1a23 Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Tue, 18 Apr 2023 07:59:06 -0400 Subject: [PATCH 303/449] do not fail fast on error --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 660f4207f..5f062ce01 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: max-parallel: 5 + fail-fast: false matrix: os: [ ubuntu-latest, macos-latest ] python-version: [ 3.8, 3.9 ] From e320a3e76eec09db896f805e3d44b6b9a240a4e2 Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Tue, 18 Apr 2023 09:01:36 -0400 Subject: [PATCH 304/449] drop explicit environment name --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6af15f405..bd648fe32 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,6 @@ jobs: - uses: mamba-org/provision-with-micromamba@v15 with: environment-file: ./environment_python_${{ steps.python.outputs.major }}_${{ steps.python.outputs.minor }}.yml - environment-name: jwql-${{ matrix.python-version }} cache-env: true cache-downloads: true From 48f3af8c94d808ffe76b923261a7c5853907bdbb Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Tue, 18 Apr 2023 09:43:23 -0400 Subject: [PATCH 305/449] also use pytest configuration --- pyproject.toml | 8 ++++++-- pytest.ini | 3 --- 2 files changed, 6 insertions(+), 5 deletions(-) delete mode 100644 pytest.ini diff --git a/pyproject.toml b/pyproject.toml index 738db37a1..70b8ad6f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,13 +25,13 @@ dependencies = [ "jinja2", "jsonschema", "jwst", - "jwst_reffiles", + "jwst_reffiles @ git+https://github.com/spacetelescope/jwst_reffiles.git@master", "matplotlib", "nodejs", "numpy", "numpydoc", "pandas", - "psycopg2", + "psycopg2-binary", "pysiaf", "pyvo", "scipy", @@ -68,3 +68,7 @@ include-package-data = true namespaces = false [tool.setuptools_scm] + +[tool.pytest] +junit_family = "xunit2" +addopts = "--ignore=jwql/website/apps/jwql/static" diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 16d36fb6b..000000000 --- a/pytest.ini +++ /dev/null @@ -1,3 +0,0 @@ -[pytest] -junit_family = xunit2 -addopts = --ignore=jwql/website/apps/jwql/static From dee2b329ab7c9ce0bfac8e09bfe0e11c20a4354e Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Tue, 18 Apr 2023 09:50:13 -0400 Subject: [PATCH 306/449] fix import indentation --- jwql/utils/logging_functions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jwql/utils/logging_functions.py b/jwql/utils/logging_functions.py index af2e98fe7..ab4d93095 100644 --- a/jwql/utils/logging_functions.py +++ b/jwql/utils/logging_functions.py @@ -66,9 +66,9 @@ def my_main_function(): import traceback if sys.version_info < (3, 11): - import tomli as tomllib - else: - import tomllib + import tomli as tomllib +else: + import tomllib from functools import wraps From 81e782ed3441c0200b3df871aa9ff859ac2dba52 Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Tue, 18 Apr 2023 09:57:49 -0400 Subject: [PATCH 307/449] restore dependency for egg build --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 70b8ad6f9..d941dbc56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ dependencies = [ "jinja2", "jsonschema", "jwst", - "jwst_reffiles @ git+https://github.com/spacetelescope/jwst_reffiles.git@master", + "jwst_reffiles", "matplotlib", "nodejs", "numpy", From 7df644e728883e62f1a406a33d3ef123c2474329 Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Tue, 18 Apr 2023 10:24:41 -0400 Subject: [PATCH 308/449] use latest Python instead of system --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd648fe32..be16d2e8e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,6 +16,8 @@ jobs: steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 + with: + python-version: "3.x" - run: pip install bandit - run: bandit ./jwql/ -c .bandit From 258b9a87d95426ab3a9ae246c3a5523dfbec1308 Mon Sep 17 00:00:00 2001 From: Zach Burnett Date: Tue, 18 Apr 2023 10:42:24 -0400 Subject: [PATCH 309/449] use test extra --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be16d2e8e..9c8792715 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,7 @@ jobs: cache-env: true cache-downloads: true - - run: pip install -e . + - run: pip install -e .[test] - run: conda env export From 2236cd6ab0d00d950eb452bc34bae50ff26fd27a Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 18 Apr 2023 15:34:05 -0400 Subject: [PATCH 310/449] fix database typo --- jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py index e601127fd..d57a83c4b 100755 --- a/jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py @@ -447,7 +447,7 @@ def __init__(self, instrument): self.db = DarkMonitorData(self.instrument) # Now we need to loop over the available apertures and create plots for each - self.available_apertures = get_unique_values_per_column(self.db, 'aperture') + self.available_apertures = get_unique_values_per_column(self.db.stats_table, 'aperture') # Require entries for all full frame apertures. If there are no data for a # particular full frame entry, then produce an empty plot, in order to @@ -651,8 +651,10 @@ def create_plot(self): # for each amp, we can get away with using use_amp=1 at the moment. if '5' in self.mean_dark: use_amp = '5' + legend_label = 'Full aperture' else: use_amp = '1' + legend_label = 'Amp 1' # If there are trending data for multiple amps, then we can plot each if len(self.mean_dark) > 1: @@ -682,7 +684,7 @@ def create_plot(self): background_fill_color="#fafafa") # Plot the "main" amp data along with error bars - self.plot.scatter(x='time', y='mean_dark', fill_color="navy", alpha=0.75, source=source) + self.plot.scatter(x='time', y='mean_dark', fill_color="navy", alpha=0.75, source=source, legend_label=legend_label) self.plot.add_layout(Whisker(source=source, base="time", upper="error_upper", lower="error_lower", line_color='navy')) hover_tool = HoverTool(tooltips=[('Dark rate:', '@mean_dark'), ('Date:', '@time{%d %b %Y}') From 17037b034d211244b40eed2fbc0690529c6012d2 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Wed, 19 Apr 2023 12:39:03 -0400 Subject: [PATCH 311/449] Remove bokeh templating files for mast monitor --- .../apps/jwql/monitor_pages/__init__.py | 1 - .../jwql/monitor_pages/monitor_mast_bokeh.py | 92 ------------------- .../yaml/monitor_mast_interface.yaml | 64 ------------- 3 files changed, 157 deletions(-) delete mode 100644 jwql/website/apps/jwql/monitor_pages/monitor_mast_bokeh.py delete mode 100644 jwql/website/apps/jwql/monitor_pages/yaml/monitor_mast_interface.yaml diff --git a/jwql/website/apps/jwql/monitor_pages/__init__.py b/jwql/website/apps/jwql/monitor_pages/__init__.py index 251fe3f60..1405523c0 100644 --- a/jwql/website/apps/jwql/monitor_pages/__init__.py +++ b/jwql/website/apps/jwql/monitor_pages/__init__.py @@ -1,3 +1,2 @@ from .monitor_cosmic_rays_bokeh import CosmicRayMonitor -from .monitor_mast_bokeh import MastMonitor from .monitor_readnoise_bokeh import ReadnoiseMonitor diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_mast_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_mast_bokeh.py deleted file mode 100644 index 4276276dd..000000000 --- a/jwql/website/apps/jwql/monitor_pages/monitor_mast_bokeh.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Created on Tue Feb 5 15:19:20 2019 - -@author: gkanarek -""" - -import os - -from astropy.time import Time -import pandas as pd - -from jwql.bokeh_templating import BokehTemplate -from jwql.utils.utils import get_config - -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) - - -class MastMonitor(BokehTemplate): - - def pre_init(self): - self._embed = True - - # App design - self.format_string = None - self.interface_file = os.path.join(SCRIPT_DIR, 'yaml', "monitor_mast_interface.yaml") - - self.settings = get_config() - self.output_dir = self.settings['outputs'] - - self.read_new_data() - - self.cache_time = Time(0., format='unix') - - self.jwst_bar_colors = self.caom_bar_colors = 3 - self.jwst_datacols = [] - self.caom_datacols = [] - - def post_init(self): - self.update_plots() - - def read_new_data(self): - """ - Placeholder to read what are currently Pandas dataframe dumps. Replace - this when we have a new database infrastructure. - """ - - jwst_filepath = os.path.join(self.outputs_dir, 'database_monitor_jwst.json') - caom_filepath = os.path.join(self.outputs_dir, 'database_monitor_caom.json') - - jwst_modtime = Time(os.stat(jwst_filepath).st_mtime, format='unix') - caom_modtime = Time(os.stat(caom_filepath).st_mtime, format='unix') - - if jwst_modtime >= self.cache_time: - self.jwst_df = pd.read_json(jwst_filepath, orient='records') - if caom_modtime >= self.cache_time: - self.caom_df = pd.read_json(caom_filepath, orient='records') - - self.cache_time = Time.now() - - def update_plots(self): - """ - Update the various sources and variables for the MAST monitor bar charts. - """ - - self.read_new_data() - - jwst_groups = list(self.jwst_df['instrument']) - caom_groups = list(self.caom_df['instrument']) - - self.jwst_datacols = [col for col in list(self.jwst_df.columns) if col != 'instrument'] - self.caom_datacols = [col for col in list(self.caom_df.columns) if col != 'instrument'] - - jwst_data = {'groups': jwst_groups} - caom_data = {'groups': caom_groups} - - for col in self.jwst_datacols: - jwst_data.update({col: list(self.jwst_df[col])}) - for col in self.caom_datacols: - caom_data.update({col: list(self.caom_df[col])}) - - self.jwst_bar_colors = max(3, len(self.jwst_datacols)) - self.caom_bar_colors = max(3, len(self.caom_datacols)) - - jwst_x = [(group, datacol) for group in jwst_groups for datacol in self.jwst_datacols] - jwst_counts = sum(zip(*[jwst_data[col] for col in self.jwst_datacols]), ()) - caom_x = [(group, datacol) for group in caom_groups for datacol in self.caom_datacols] - caom_counts = sum(zip(*[caom_data[col] for col in self.caom_datacols]), ()) - - self.refs['jwst_source'].data = {'x': jwst_x, 'counts': jwst_counts} - self.refs['caom_source'].data = {'x': caom_x, 'counts': caom_counts} diff --git a/jwql/website/apps/jwql/monitor_pages/yaml/monitor_mast_interface.yaml b/jwql/website/apps/jwql/monitor_pages/yaml/monitor_mast_interface.yaml deleted file mode 100644 index 810984eaa..000000000 --- a/jwql/website/apps/jwql/monitor_pages/yaml/monitor_mast_interface.yaml +++ /dev/null @@ -1,64 +0,0 @@ -# JWST figure -- !ColumnDataSource: &jwst_source - ref: "jwst_source" - data: - x: [] - counts: [] -- !HoverTool: &jwst_hover - ref: "jwst_hover" - tooltips: [('count', '@counts')] -- !FactorRange: &jwst_xrange - ref: "jwst_xrange" - factors: !self.jwst_groups -- !factor_cmap: &jwst_cmap - ref: "jwst_cmap" - arg: 'x' - palette: !Category20c[!self.jwst_bar_colors ] - factors: !self.jwst_datacols - start: 1 - end: 2 -- !Figure: &jwst_figure - ref: "jwst_figure" - x_range: *jwst_xrange - plot_height: 250 - tools: [*jwst_hover ] - elements: - - kind: 'vbar' - x: 'x' - top: 'counts' - width: 0.9 - source: *jwst_source - line_color: 'white' - fill_color: *jwst_cmap -# CAOM figure -- !ColumnDataSource: &caom_source - ref: "caom_source" - data: - x: [] - counts: [] -- !HoverTool: &caom_hover - ref: "caom_hover" - tooltips: [('count', '@counts')] -- !FactorRange: &caom_xrange - ref: "caom_xrange" - factors: !self.caom_groups -- !factor_cmap: &caom_cmap - ref: "caom_cmap" - arg: 'x' - palette: !Category20c[!self.caom_bar_colors ] - factors: !self.caom_datacols - start: 1 - end: 2 -- !Figure: &caom_figure - ref: "caom_figure" - x_range: *caom_xrange - plot_height: 250 - tools: [*caom_hover ] - elements: - - kind: 'vbar' - x: 'x' - top: 'counts' - width: 0.9 - source: *caom_source - line_color: 'white' - fill_color: *caom_cmap \ No newline at end of file From 253cf114c627254978e6f880e6d46358d5cf99ec Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Wed, 19 Apr 2023 12:40:09 -0400 Subject: [PATCH 312/449] Remove bokeh templating yaml for filesystem monitor --- .../yaml/monitor_filesystem_interface.yaml | 155 ------------------ 1 file changed, 155 deletions(-) delete mode 100644 jwql/website/apps/jwql/monitor_pages/yaml/monitor_filesystem_interface.yaml diff --git a/jwql/website/apps/jwql/monitor_pages/yaml/monitor_filesystem_interface.yaml b/jwql/website/apps/jwql/monitor_pages/yaml/monitor_filesystem_interface.yaml deleted file mode 100644 index a1da194e9..000000000 --- a/jwql/website/apps/jwql/monitor_pages/yaml/monitor_filesystem_interface.yaml +++ /dev/null @@ -1,155 +0,0 @@ -# Unified x range for all files -- !Range1d: &xr_all - ref: "xr_all" - start: 0 - end: 100 - bounds: !!python/tuple [0, 100] -#File count figure -- !ColumnDataSource: &source_filecount - ref: "source_filecount" - data: - dates: [] - filecount: [] -- !Range1d: &yr_filecount - ref: "yr_filecount" - start: 0 - end: 100 - bounds: !!python/tuple [0, 100] -- !Figure: &fig_filecount - ref: "fig_filecount" - title: "Total File Counts" - y_axis_label: 'Count' - x_range: *xr_all - y_range: *yr_filecount - elements: - - {'kind': 'line', 'x': 'dates', 'y': 'filecount', 'line_color': 'blue', 'source': *source_filecount} -#System stats figure -- !ColumnDataSource: &source_stats - ref: "source_stats" - data: - dates: [] - systemsize: [] - freesize: [] - usedsize: [] -- !Range1d: &yr_stats - ref: "yr_stats" - start: 0 - end: 100 - bounds: !!python/tuple [0, 100] -- !Figure: &fig_stats - ref: "fig_system_stats" - title: "System Stats" - y_axis_label: 'Count' - x_range: *xr_all - y_range: *yr_stats - elements: - - {'kind': 'line', 'x': 'dates', 'y': 'systemsize', 'line_color': 'red', 'source': *source_stats, 'legend': 'Total size'} - - {'kind': 'line', 'x': 'dates', 'y': 'freesize', 'line_color': 'blue', 'source': *source_stats, 'legend': 'Free bytes'} - - {'kind': 'line', 'x': 'dates', 'y': 'usedsize', 'line_color': 'green', 'source': *source_stats, 'legend': 'Used bytes'} - - {'kind': 'circle', 'x': 'dates', 'y': 'systemsize', 'color': 'red', 'source': *source_stats} - - {'kind': 'circle', 'x': 'dates', 'y': 'freesize', 'color': 'blue', 'source': *source_stats} - - {'kind': 'circle', 'x': 'dates', 'y': 'usedsize', 'color': 'green', 'source': *source_stats} -#File types figure -- !ColumnDataSource: &source_files - ref: "source_files" - data: - dates: [] - fits: [] - uncal: [] - cal: [] - rate: [] - rateint: [] - i2d: [] - nrc: [] - nrs: [] - nis: [] - mir: [] - fgs: [] -- !Range1d: &yr_files - ref: "yr_files" - start: 0 - end: 100 - bounds: !!python/tuple [0, 100] -- !Figure: &fig_files - ref: "fig_filecount_type" - title: "Total File Counts by Type" - y_axis_label: 'Count' - x_range: *xr_all - y_range: *yr_files - elements: - - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[0] , 'line_color': !self.types_c[0] , 'source': *source_files, 'legend': !self.types_l[0] } - - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[1] , 'line_color': !self.types_c[1] , 'source': *source_files, 'legend': !self.types_l[1] } - - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[2] , 'line_color': !self.types_c[2] , 'source': *source_files, 'legend': !self.types_l[2] } - - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[3] , 'line_color': !self.types_c[3] , 'source': *source_files, 'legend': !self.types_l[3] } - - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[4] , 'line_color': !self.types_c[4] , 'source': *source_files, 'legend': !self.types_l[4] } - - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[5] , 'line_color': !self.types_c[5] , 'source': *source_files, 'legend': !self.types_l[5] } - - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[6] , 'line_color': !self.types_c[6] , 'source': *source_files, 'legend': !self.types_l[6] } - - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[7] , 'line_color': !self.types_c[7] , 'source': *source_files, 'legend': !self.types_l[7] } - - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[8] , 'line_color': !self.types_c[8] , 'source': *source_files, 'legend': !self.types_l[8] } - - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[9] , 'line_color': !self.types_c[9] , 'source': *source_files, 'legend': !self.types_l[9] } - - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[10] , 'line_color': !self.types_c[10] , 'source': *source_files, 'legend': !self.types_l[10] } - - {'kind': !self.types_k[0] , 'x': 'dates', 'y': !self.types_y[0] , 'color': !self.types_c[0] , 'source': *source_files} - - {'kind': !self.types_k[1] , 'x': 'dates', 'y': !self.types_y[1] , 'color': !self.types_c[1] , 'source': *source_files} - - {'kind': !self.types_k[2] , 'x': 'dates', 'y': !self.types_y[2] , 'color': !self.types_c[2] , 'source': *source_files} - - {'kind': !self.types_k[3] , 'x': 'dates', 'y': !self.types_y[3] , 'color': !self.types_c[3] , 'source': *source_files} - - {'kind': !self.types_k[4] , 'x': 'dates', 'y': !self.types_y[4] , 'color': !self.types_c[4] , 'source': *source_files} - - {'kind': !self.types_k[5] , 'x': 'dates', 'y': !self.types_y[5] , 'color': !self.types_c[5] , 'source': *source_files} - - {'kind': !self.types_k[6] , 'x': 'dates', 'y': !self.types_y[6] , 'color': !self.types_c[6] , 'source': *source_files} - - {'kind': !self.types_k[7] , 'x': 'dates', 'y': !self.types_y[7] , 'color': !self.types_c[7] , 'source': *source_files} - - {'kind': !self.types_k[8] , 'x': 'dates', 'y': !self.types_y[8] , 'color': !self.types_c[8] , 'source': *source_files} - - {'kind': !self.types_k[9] , 'x': 'dates', 'y': !self.types_y[9] , 'color': !self.types_c[9] , 'source': *source_files} - - {'kind': !self.types_k[10] , 'x': 'dates', 'y': !self.types_y[10] , 'color': !self.types_c[10] , 'source': *source_files} -#File sizes figure -- !ColumnDataSource: &source_sizes - ref: "source_sizes" - data: - dates: [] - fits: [] - uncal: [] - cal: [] - rate: [] - rateint: [] - i2d: [] - nrc: [] - nrs: [] - nis: [] - mir: [] - fgs: [] -- !Range1d: &yr_sizes - ref: "yr_sizes" - start: 0 - end: 100 - bounds: !!python/tuple [0, 100] -- !Figure: &fig_sizes - ref: "fig_size_type" - title: "Total File Sizes by Type" - y_axis_label: 'GB' - x_range: *xr_all - y_range: *yr_sizes - elements: - - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[0] , 'line_color': !self.types_c[0] , 'source': *source_sizes, 'legend': !self.types_l[0] } - - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[1] , 'line_color': !self.types_c[1] , 'source': *source_sizes, 'legend': !self.types_l[1] } - - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[2] , 'line_color': !self.types_c[2] , 'source': *source_sizes, 'legend': !self.types_l[2] } - - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[3] , 'line_color': !self.types_c[3] , 'source': *source_sizes, 'legend': !self.types_l[3] } - - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[4] , 'line_color': !self.types_c[4] , 'source': *source_sizes, 'legend': !self.types_l[4] } - - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[5] , 'line_color': !self.types_c[5] , 'source': *source_sizes, 'legend': !self.types_l[5] } - - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[6] , 'line_color': !self.types_c[6] , 'source': *source_sizes, 'legend': !self.types_l[6] } - - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[7] , 'line_color': !self.types_c[7] , 'source': *source_sizes, 'legend': !self.types_l[7] } - - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[8] , 'line_color': !self.types_c[8] , 'source': *source_sizes, 'legend': !self.types_l[8] } - - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[9] , 'line_color': !self.types_c[9] , 'source': *source_sizes, 'legend': !self.types_l[9] } - - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[10] , 'line_color': !self.types_c[10] , 'source': *source_sizes, 'legend': !self.types_l[10] } - - {'kind': !self.types_k[0] , 'x': 'dates', 'y': !self.types_y[0] , 'color': !self.types_c[0] , 'source': *source_sizes} - - {'kind': !self.types_k[1] , 'x': 'dates', 'y': !self.types_y[1] , 'color': !self.types_c[1] , 'source': *source_sizes} - - {'kind': !self.types_k[2] , 'x': 'dates', 'y': !self.types_y[2] , 'color': !self.types_c[2] , 'source': *source_sizes} - - {'kind': !self.types_k[3] , 'x': 'dates', 'y': !self.types_y[3] , 'color': !self.types_c[3] , 'source': *source_sizes} - - {'kind': !self.types_k[4] , 'x': 'dates', 'y': !self.types_y[4] , 'color': !self.types_c[4] , 'source': *source_sizes} - - {'kind': !self.types_k[5] , 'x': 'dates', 'y': !self.types_y[5] , 'color': !self.types_c[5] , 'source': *source_sizes} - - {'kind': !self.types_k[6] , 'x': 'dates', 'y': !self.types_y[6] , 'color': !self.types_c[6] , 'source': *source_sizes} - - {'kind': !self.types_k[7] , 'x': 'dates', 'y': !self.types_y[7] , 'color': !self.types_c[7] , 'source': *source_sizes} - - {'kind': !self.types_k[8] , 'x': 'dates', 'y': !self.types_y[8] , 'color': !self.types_c[8] , 'source': *source_sizes} - - {'kind': !self.types_k[9] , 'x': 'dates', 'y': !self.types_y[9] , 'color': !self.types_c[9] , 'source': *source_sizes} - - {'kind': !self.types_k[10] , 'x': 'dates', 'y': !self.types_y[10] , 'color': !self.types_c[10] , 'source': *source_sizes} -#Widget layout -- !gridplot: - ref: "filesystem_layout" - arg: [[[*fig_filecount, *fig_stats], [*fig_files, *fig_sizes]]] \ No newline at end of file From 120f8126cf10483c96fd5c58c6179c81ba88b21d Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Thu, 20 Apr 2023 13:34:10 -0500 Subject: [PATCH 313/449] Renaming env files and updating ci --- .github/workflows/ci.yml | 9 ++------- ...onment_python_3_10.yml => environment_python_3.10.yml | 0 environment_python_3_9.yml => environment_python_3.9.yml | 0 3 files changed, 2 insertions(+), 7 deletions(-) rename environment_python_3_10.yml => environment_python_3.10.yml (100%) rename environment_python_3_9.yml => environment_python_3.9.yml (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c8792715..3b047377a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,19 +29,14 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest] - python-version: [3.9] + python-version: [3.9, "3.10"] steps: - uses: actions/checkout@v3 - - run: | - echo "major=$(echo ${{ matrix.python-version }} | head -c 1)" >> $GITHUB_OUTPUT - echo "minor=$(echo ${{ matrix.python-version }} | tail -c 2)" >> $GITHUB_OUTPUT - id: python - - uses: mamba-org/provision-with-micromamba@v15 with: - environment-file: ./environment_python_${{ steps.python.outputs.major }}_${{ steps.python.outputs.minor }}.yml + environment-file: ./environment_python_${{ matrix.python-version }}.yml cache-env: true cache-downloads: true diff --git a/environment_python_3_10.yml b/environment_python_3.10.yml similarity index 100% rename from environment_python_3_10.yml rename to environment_python_3.10.yml diff --git a/environment_python_3_9.yml b/environment_python_3.9.yml similarity index 100% rename from environment_python_3_9.yml rename to environment_python_3.9.yml From ea9846e37f6d48761427b1648c82ac7c81e0ad96 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 25 Apr 2023 15:03:51 -0400 Subject: [PATCH 314/449] Fix path to pyproject.toml --- jwql/utils/logging_functions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jwql/utils/logging_functions.py b/jwql/utils/logging_functions.py index ab4d93095..abd82a45e 100644 --- a/jwql/utils/logging_functions.py +++ b/jwql/utils/logging_functions.py @@ -223,7 +223,8 @@ def wrapped(*args, **kwargs): logging.info('Running as PID {}'.format(os.getpid())) # Read in setup.py file to build list of required modules - with open("pyproject.toml", "rb") as f: + toml_file = os.path.join(os.path.dirname(get_config()['setup_file']), 'pyproject.toml') + with open(toml_file, "rb") as f: data = tomllib.load(f) required_modules = data['project']['dependencies'] From f3c8836581a0f50645e6e8490feae20e091434f2 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 8 May 2023 12:39:53 -0400 Subject: [PATCH 315/449] correct logging statement --- jwql/shared_tasks/shared_tasks.py | 34 +++++++++++++++---------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index 516738ed0..c2f7626a1 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -231,7 +231,7 @@ def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_fi process = Popen(command, shell=True, executable="/bin/bash", stderr=PIPE) with process.stderr: log_subprocess_output(process.stderr) - result = process.wait() + result = process.wait() logging.info("Subprocess result was {}".format(result)) if not os.path.isfile(res_file): @@ -241,7 +241,7 @@ def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_fi for line in status: logging.error(line.strip()) return status - + with open(res_file, 'r') as inf: status = inf.readlines() return status @@ -251,17 +251,17 @@ def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_fi def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, step_args={}): """Run the steps of ``calwebb_detector1`` on the input file, saving the result of each step as a separate output file, then return the name-and-path of the file as reduced - in the reduction directory. Once all requested extensions have been produced, the + in the reduction directory. Once all requested extensions have been produced, the pipeline will return. Parameters ---------- input_file_name : str File on which to run the pipeline steps - + short_name : str Name of the file to be calibrated after any extensions have been stripped off. - + ext_or_exts : list List of extensions to be retrieved. @@ -291,7 +291,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, output_dir = os.path.join(config['transfer_dir'], "outgoing") msg = "Input from {}, calibrate in {}, output to {}" logging.info(msg.format(input_dir, cal_dir, output_dir)) - + input_file = os.path.join(input_dir, input_file_name) current_dir = os.path.dirname(__file__) cmd_name = os.path.join(current_dir, "run_pipeline.py") @@ -308,9 +308,9 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, logging.info("Requesting {}".format(calibrated_files)) cores = 'all' - status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, input_file, + status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, input_file, short_name, result_file, cores) - + if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") else: @@ -323,7 +323,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, logging.error("\t{}".format(line.strip())) if core_fail: cores = "half" - status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, + status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, input_file, short_name, result_file, cores) if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") @@ -337,7 +337,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, logging.error("\t{}".format(line.strip())) if core_fail: cores = "none" - status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, + status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, input_file, short_name, result_file, cores) if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") @@ -346,7 +346,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, logging.error("Pipeline subprocess failed.") if not managed: raise ValueError("Pipeline Failed") - + for file in calibrated_files: logging.info("Checking for output {}".format(file)) if not os.path.isfile(os.path.join(cal_dir, file)): @@ -420,14 +420,14 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save short_name = input_file_name.replace("_uncal", "").replace("_0thgroup", "").replace(".fits", "") ensure_dir_exists(cal_dir) output_dir = os.path.join(config["transfer_dir"], "outgoing") - + cmd_name = os.path.join(os.path.dirname(__file__), "run_pipeline.py") result_file = os.path.join(cal_dir, short_name+"_status.txt") cores = 'all' - status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, input_file, + status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, input_file, short_name, result_file, cores) - + if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") else: @@ -440,7 +440,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save logging.error("\t{}".format(line.strip())) if core_fail: cores = "half" - status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, + status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, input_file, short_name, result_file, cores) if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") @@ -454,7 +454,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save logging.error("\t{}".format(line.strip())) if core_fail: cores = "none" - status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, + status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, input_file, short_name, result_file, cores) if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") @@ -549,7 +549,7 @@ def prep_file(input_file, in_ext): logging.critical(msg.format(short_name)) raise ValueError("Redis lock for {} is in an unknown state".format(short_name)) logging.info("\t\tAcquired Lock.") - logging.info("\t\tCopying {} to {}".format(input_file, send_path)) + logging.info("\t\tCopying {} to {}".format(uncal_file, send_path)) copy_files([uncal_file], send_path) return short_name, cal_lock, os.path.join(send_path, uncal_name) From 90d79e38e7882ae40f3eb3e529d849790ad5ea16 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 8 May 2023 15:20:53 -0400 Subject: [PATCH 316/449] update run_save_jump --- jwql/shared_tasks/run_pipeline.py | 57 +++++++++++++++---------------- jwql/shared_tasks/shared_tasks.py | 38 ++++++++++----------- 2 files changed, 47 insertions(+), 48 deletions(-) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 9a196ecdd..124f51a15 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -43,17 +43,17 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co status_file_name = short_name + "_status.txt" status_file = os.path.join(work_directory, status_file_name) uncal_file = os.path.join(work_directory, input_file_basename) - + with open(status_file, 'a+') as status_f: status_f.write("Running run_pipe\n") status_f.write("\t input_file_basename is {} ({})\n".format(input_file_basename, type(input_file_basename))) status_f.write("\t start_dir is {} ({})\n".format(start_dir, type(start_dir))) status_f.write("\t uncal_file is {} ({})\n".format(uncal_file, type(uncal_file))) - + try: copy_files([input_file], work_directory) set_permissions(uncal_file) - + steps = get_pipeline_steps(instrument) first_step_to_be_run = True for step_name in steps: @@ -94,7 +94,7 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co # If the dither_points entry is not populated, then ignore this change pass model[0].save(output_file) - + done = True for output in outputs: output_name = "{}_{}.fits".format(short_name, output) @@ -110,7 +110,7 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co status_f.write("{}\n".format(e)) status_f.write("FAILED") sys.exit(1) - + with open(status_file, "a+") as status_f: status_f.write("SUCCEEDED") # Done. @@ -130,7 +130,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T sys.stderr.write("Starting pipeline\n") with open(status_file, 'a+') as status_f: status_f.write("Starting pipeline\n") - + try: copy_files([input_file], work_directory) set_permissions(uncal_file) @@ -157,51 +157,50 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T # and use the run() method so that we can set parameters # progammatically. model = Detector1Pipeline() + params = {} # Always true if instrument == 'nircam': - model.refpix.odd_even_rows = False + params['refpix'] = dict(odd_even_rows=False) # Default CR rejection threshold is too low - model.jump.rejection_threshold = 15 + #params['jump'] = dict(rejection_threshold=15) # Turn off IPC step until it is put in the right place - model.ipc.skip = True + params['ipc'] = dict(skip=True) - model.jump.save_results = True - model.jump.output_dir = work_directory - model.jump.maximum_cores = max_cores - jump_output = uncal_file.replace('uncal', 'jump') + # Set up to save jump step output + params['jump']['save_results'] = True + params['jump']['output_dir'] = work_directory + params['jump']['maximum_cores'] = max_cores + jump_output = short_name + '_jump.fits' # Check to see if the jump version of the requested file is already # present run_jump = not os.path.isfile(jump_output) if ramp_fit: - model.ramp_fit.save_results = True - model.ramp_fit.maximum_cores = max_cores - # model.save_results = True - model.output_dir = work_directory - # pipe_output = os.path.join(output_dir, input_file_only.replace('uncal', 'rate')) - pipe_output = uncal_file.replace('uncal', '0_ramp_fit') + params['ramp_fit'] = dict(save_results=True, maximum_cores=max_cores) + + pipe_output = os.path.join(work_directory, short_name + '_0_ramp_fit.fits') run_slope = not os.path.isfile(pipe_output) if save_fitopt: - model.ramp_fit.save_opt = True - fitopt_output = uncal_file.replace('uncal', 'fitopt') + params['ramp_fit']['save_opt'] = True + fitopt_output = os.path.join(work_directory, short_name + '_fitopt.fits') run_fitopt = not os.path.isfile(fitopt_output) else: - model.ramp_fit.save_opt = False + params['ramp_fit']['save_opt'] = False fitopt_output = None run_fitopt = False else: - model.ramp_fit.skip = True + params['ramp_fit']['skip'] = True pipe_output = None fitopt_output = None run_slope = False run_fitopt = False if run_jump or (ramp_fit and run_slope) or (save_fitopt and run_fitopt): - model.run(datamodel) + model.call(datamodel, output_dir=work_directory) else: print(("Files with all requested calibration states for {} already present in " "output directory. Skipping pipeline call.".format(uncal_file))) @@ -211,7 +210,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T status_f.write("{}\n".format(e)) status_f.write("FAILED") sys.exit(1) - + with open(status_file, "a+") as status_f: status_f.write("{}\n".format(jump_output)) status_f.write("{}\n".format(pipe_output)) @@ -258,14 +257,14 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T with open(general_status_file, "a+") as status_file: status_file.write("Finished parsing args at {}\n".format(time.ctime())) - + input_file = args.input_file instrument = args.instrument short_name = args.short_name working_path = args.working_path pipe_type = args.pipe outputs = args.outputs - + status_file = os.path.join(working_path, short_name+"_status.txt") with open(status_file, 'w') as out_file: out_file.write("Starting Process\n") @@ -275,10 +274,10 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T out_file.write("\tinstrument is {} ({})\n".format(instrument, type(instrument))) out_file.write("\tinput_file is {} ({})\n".format(input_file, type(input_file))) out_file.write("\tshort_name is {} ({})\n".format(short_name, type(short_name))) - + if not os.path.isfile(args.input_file): raise FileNotFoundError("No input file {}".format(args.input_file)) - + if pipe_type not in ['jump', 'cal']: raise ValueError("Unknown calibration type {}".format(pipe_type)) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index 516738ed0..c4dceee32 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -231,7 +231,7 @@ def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_fi process = Popen(command, shell=True, executable="/bin/bash", stderr=PIPE) with process.stderr: log_subprocess_output(process.stderr) - result = process.wait() + result = process.wait() logging.info("Subprocess result was {}".format(result)) if not os.path.isfile(res_file): @@ -241,7 +241,7 @@ def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_fi for line in status: logging.error(line.strip()) return status - + with open(res_file, 'r') as inf: status = inf.readlines() return status @@ -251,17 +251,17 @@ def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_fi def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, step_args={}): """Run the steps of ``calwebb_detector1`` on the input file, saving the result of each step as a separate output file, then return the name-and-path of the file as reduced - in the reduction directory. Once all requested extensions have been produced, the + in the reduction directory. Once all requested extensions have been produced, the pipeline will return. Parameters ---------- input_file_name : str File on which to run the pipeline steps - + short_name : str Name of the file to be calibrated after any extensions have been stripped off. - + ext_or_exts : list List of extensions to be retrieved. @@ -291,7 +291,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, output_dir = os.path.join(config['transfer_dir'], "outgoing") msg = "Input from {}, calibrate in {}, output to {}" logging.info(msg.format(input_dir, cal_dir, output_dir)) - + input_file = os.path.join(input_dir, input_file_name) current_dir = os.path.dirname(__file__) cmd_name = os.path.join(current_dir, "run_pipeline.py") @@ -308,9 +308,9 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, logging.info("Requesting {}".format(calibrated_files)) cores = 'all' - status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, input_file, + status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, input_file, short_name, result_file, cores) - + if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") else: @@ -323,7 +323,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, logging.error("\t{}".format(line.strip())) if core_fail: cores = "half" - status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, + status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, input_file, short_name, result_file, cores) if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") @@ -337,7 +337,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, logging.error("\t{}".format(line.strip())) if core_fail: cores = "none" - status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, + status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, input_file, short_name, result_file, cores) if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") @@ -346,7 +346,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, logging.error("Pipeline subprocess failed.") if not managed: raise ValueError("Pipeline Failed") - + for file in calibrated_files: logging.info("Checking for output {}".format(file)) if not os.path.isfile(os.path.join(cal_dir, file)): @@ -417,17 +417,17 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save logging.error("File {} not found!".format(input_file)) raise FileNotFoundError("{} not found".format(input_file)) - short_name = input_file_name.replace("_uncal", "").replace("_0thgroup", "").replace(".fits", "") + short_name = input_file_name[0: input_file_name.rfind('_')] ensure_dir_exists(cal_dir) output_dir = os.path.join(config["transfer_dir"], "outgoing") - + cmd_name = os.path.join(os.path.dirname(__file__), "run_pipeline.py") result_file = os.path.join(cal_dir, short_name+"_status.txt") cores = 'all' - status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, input_file, + status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, input_file, short_name, result_file, cores) - + if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") else: @@ -440,7 +440,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save logging.error("\t{}".format(line.strip())) if core_fail: cores = "half" - status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, + status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, input_file, short_name, result_file, cores) if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") @@ -454,7 +454,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save logging.error("\t{}".format(line.strip())) if core_fail: cores = "none" - status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, + status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, input_file, short_name, result_file, cores) if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") @@ -528,7 +528,7 @@ def prep_file(input_file, in_ext): input_path, input_name = os.path.split(input_file) logging.info("\tPath is {}, file is {}".format(input_path, input_name)) - if "uncal" not in in_ext: + if "uncal" not in in_ext and "dark" not in in_ext: logging.info("\tSwitching from {} to uncal".format(in_ext)) uncal_name = os.path.basename(input_file).replace(in_ext, "uncal") uncal_file = filesystem_path(uncal_name, check_existence=True) @@ -762,7 +762,7 @@ def run_parallel_pipeline(input_files, in_ext, ext_or_exts, instrument, jump_pip file_or_files : str or list-of-str Name (or names) of the result file(s), including path(s) """ - logging.info("Pipeline call requestion calibrated extensions {}".format(ext_or_exts)) + logging.info("Pipeline call requested calibrated extensions {}".format(ext_or_exts)) for input_file in input_files: logging.info("\tCalibrating {}".format(input_file)) From 00e9058c6a8189363e43a00b57bafd975b151523 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 8 May 2023 15:45:02 -0400 Subject: [PATCH 317/449] Turn off steps when input is dark.fits --- jwql/shared_tasks/run_pipeline.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 124f51a15..361501ac5 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -166,9 +166,6 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T # Default CR rejection threshold is too low #params['jump'] = dict(rejection_threshold=15) - # Turn off IPC step until it is put in the right place - params['ipc'] = dict(skip=True) - # Set up to save jump step output params['jump']['save_results'] = True params['jump']['output_dir'] = work_directory @@ -199,8 +196,23 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T run_slope = False run_fitopt = False + # If the input file is dark.fits rather than uncal.fits, then skip + # all of the pipeline steps that are run prior to dark subtraction + if 'dark.fits' in input_file: + if instrument.lower() == 'miri': + steps_to_skip = ['group_scale', 'dq_init', 'saturation', 'ipc', 'firstframe', + 'lastframe', 'reset', 'linearity', 'rscd'] + else: + steps_to_skip = ['group_scale', 'dq_init', 'saturation', 'ipc', 'superbias', + 'refpix', 'linearity'] + for step in steps_to_skip: + params[step] = dict(skip: True) + else: + # Turn off IPC step until it is put in the right place + params['ipc'] = dict(skip=True) + if run_jump or (ramp_fit and run_slope) or (save_fitopt and run_fitopt): - model.call(datamodel, output_dir=work_directory) + model.call(datamodel, output_dir=work_directory, steps=params) else: print(("Files with all requested calibration states for {} already present in " "output directory. Skipping pipeline call.".format(uncal_file))) From 858b06eeed934d97995c336b68900d632d5c42b5 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 8 May 2023 17:30:03 -0400 Subject: [PATCH 318/449] Update run_pipe to optionally skip steps --- .../common_monitors/dark_monitor.py | 8 +++---- jwql/shared_tasks/run_pipeline.py | 24 +++++++++++++++---- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index 26d433ed1..8750914d5 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -402,11 +402,11 @@ def exclude_existing_badpix(self, badpix, pixel_type): new_pixels_y : list List of y coordinates of new bad pixels """ - + if len(badpix[0]) == 0: logging.warning("\tNo new {} pixels to check.".format(pixel_type)) return ([], []) - + logging.info("\tChecking {} potential new {} pixels".format(len(badpix[0]), pixel_type)) if pixel_type not in ['hot', 'dead', 'noisy']: @@ -440,7 +440,7 @@ def exclude_existing_badpix(self, badpix, pixel_type): if len(np.intersect1d(ind_x[0], ind_y[0])) == 0: new_pixels_x.append(x) new_pixels_y.append(y) - + logging.info("\t\tKeeping {} {} pixels".format(len(new_pixels_x), pixel_type)) # pixel = (x, y) # if pixel not in already_found: @@ -810,7 +810,7 @@ def process(self, file_list): # Add new noisy pixels to the database logging.info('\tFound {} new noisy pixels'.format(len(new_noisy_pixels[0]))) self.add_bad_pix(new_noisy_pixels, 'noisy', file_list, mean_slope_file, baseline_file, min_time, mid_time, max_time) - + logging.info("Creating Mean Slope Image {}".format(slope_image)) # Create png file of mean slope image. Add bad pixels only for full frame apertures self.create_mean_slope_figure(slope_image, len(slope_files), hotxy=new_hot_pix, deadxy=new_dead_pix, diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 361501ac5..c59126ebe 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -27,7 +27,7 @@ from jwst.saturation import SaturationStep from jwst.superbias import SuperBiasStep -from jwql.instrument_monitors.pipeline_tools import PIPELINE_STEP_MAPPING, get_pipeline_steps +from jwql.instrument_monitors.pipeline_tools import PIPELINE_STEP_MAPPING, completed_pipeline_steps, get_pipeline_steps from jwql.utils.logging_functions import configure_logging from jwql.utils.permissions import set_permissions from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path @@ -55,15 +55,26 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co set_permissions(uncal_file) steps = get_pipeline_steps(instrument) + + # If the input file is a file other than uncal.fits, then we may only need to run a + # subset of steps. Check the completed steps in the input file and set steps such + # that anything that is already complete is not re-run. + if 'uncal.fits' not in input_file: + completed_steps = completed_pipeline_steps(filename) + for step in steps: + if step in completed_steps: + if completed_steps[step]: + steps[step] = False + first_step_to_be_run = True for step_name in steps: - sys.stderr.write("Running step {}\n".format(step_name)) - with open(status_file, 'a+') as status_f: - status_f.write("Running step {}\n".format(step_name)) kwargs = {} if step_name in ['jump', 'rate']: kwargs = {'maximum_cores': max_cores} if steps[step_name]: + sys.stderr.write("Running step {}\n".format(step_name)) + with open(status_file, 'a+') as status_f: + status_f.write("Running step {}\n".format(step_name)) output_file_name = short_name + "_{}.fits".format(step_name) output_file = os.path.join(work_directory, output_file_name) # skip already-done steps @@ -104,6 +115,11 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co if done: sys.stderr.write("Done pipeline.\n") break + else: + sys.stderr.write("Skipping step {}\n".format(step_name)) + with open(status_file, 'a+') as status_f: + status_f.write("Skipping step {}\n".format(step_name)) + except Exception as e: with open(status_file, "a+") as status_f: status_f.write("EXCEPTION\n") From e7527f7b62df8605407741356c67bd5f68e63e35 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 9 May 2023 10:33:36 -0400 Subject: [PATCH 319/449] Fix logic for skipping completed steps --- jwql/shared_tasks/run_pipeline.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index c59126ebe..82e85fd1a 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -57,15 +57,19 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co steps = get_pipeline_steps(instrument) # If the input file is a file other than uncal.fits, then we may only need to run a - # subset of steps. Check the completed steps in the input file and set steps such - # that anything that is already complete is not re-run. + # subset of steps. Check the completed steps in the input file. Find the latest step + # that has been completed, and skip that plus all prior steps if 'uncal.fits' not in input_file: completed_steps = completed_pipeline_steps(filename) for step in steps: - if step in completed_steps: - if completed_steps[step]: - steps[step] = False + steps[step] = not completed_steps[step] + # Special case: if the input file is a dark.fits file, then we want to skip the + # dark_current subtraction step + if 'dark.fits' in input_file: + step['dark_current'] = False + + # Run each specified step first_step_to_be_run = True for step_name in steps: kwargs = {} From 712894b86bdb1d982e38df78c9f9ae605aacaf59 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 9 May 2023 11:02:46 -0400 Subject: [PATCH 320/449] Add missing steps to step list --- jwql/instrument_monitors/pipeline_tools.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/jwql/instrument_monitors/pipeline_tools.py b/jwql/instrument_monitors/pipeline_tools.py index e9754bd17..9390802e8 100644 --- a/jwql/instrument_monitors/pipeline_tools.py +++ b/jwql/instrument_monitors/pipeline_tools.py @@ -165,12 +165,8 @@ def get_pipeline_steps(instrument): # Order is important in 'steps' lists below!! if instrument == 'MIRI': - steps = ['group_scale', 'dq_init', 'saturation', 'ipc', 'firstframe', 'lastframe', - 'linearity', 'rscd', 'dark_current', 'refpix', 'persistence', 'jump', 'rate'] - # No persistence correction for MIRI - steps.remove('persistence') - # MIRI is limited to one frame per group - steps.remove('group_scale') + steps = ['group_scale', 'dq_init', 'saturation', 'ipc', 'firstframe', 'lastframe', 'reset', + 'linearity', 'rscd', 'dark_current', 'refpix', 'jump', 'rate', 'gain_scale'] else: steps = ['group_scale', 'dq_init', 'saturation', 'ipc', 'superbias', 'refpix', 'linearity', 'persistence', 'dark_current', 'jump', 'rate'] From 3cd5cd618526705db3c0ea6eb84221dfbb80498c Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 9 May 2023 12:31:09 -0400 Subject: [PATCH 321/449] rateints support --- jwql/instrument_monitors/common_monitors/dark_monitor.py | 2 +- jwql/shared_tasks/run_pipeline.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index 8750914d5..36b56a5cb 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -712,7 +712,7 @@ def process(self, file_list): logging.info("\t\tAdding {} to calibration set".format(filename)) pipeline_files.append(filename) - outputs = run_parallel_pipeline(pipeline_files, "dark", "rate", self.instrument) + outputs = run_parallel_pipeline(pipeline_files, "dark", ["rate", "rateints"], self.instrument) for filename in file_list: processed_file = filename.replace("_dark", "_rate") if processed_file not in slope_files and os.path.isfile(processed_file): diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 82e85fd1a..aef81fd56 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -60,7 +60,7 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co # subset of steps. Check the completed steps in the input file. Find the latest step # that has been completed, and skip that plus all prior steps if 'uncal.fits' not in input_file: - completed_steps = completed_pipeline_steps(filename) + completed_steps = completed_pipeline_steps(input_file) for step in steps: steps[step] = not completed_steps[step] @@ -109,6 +109,8 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co # If the dither_points entry is not populated, then ignore this change pass model[0].save(output_file) + if 'rateints' in outputs: + model[1].save(output_file.replace('rate', 'rateints')) done = True for output in outputs: From 43facbefd92e8ba1f6a004eab56e3f508397aa5a Mon Sep 17 00:00:00 2001 From: Maria Pena-Guerrero Date: Thu, 11 May 2023 13:47:57 -0400 Subject: [PATCH 322/449] added quick instructions for NRS TA monitors --- jwql/website/apps/jwql/templates/msata_monitor.html | 13 +++++++++++++ jwql/website/apps/jwql/templates/wata_monitor.html | 9 +++++++++ 2 files changed, 22 insertions(+) diff --git a/jwql/website/apps/jwql/templates/msata_monitor.html b/jwql/website/apps/jwql/templates/msata_monitor.html index 3203f069b..0779b7252 100644 --- a/jwql/website/apps/jwql/templates/msata_monitor.html +++ b/jwql/website/apps/jwql/templates/msata_monitor.html @@ -26,6 +26,19 @@

    Updates to this monitor only occur with new data

    This monitor uses all MSATA data available and the plots include all data. The plots will only be updated if new MSATA data is in MAST, otherwise, the monitor will show all data previously obtained. +

    Instructions

    + For general MSATA monitoring you only need to look at the first two plots + below: 1. MSATA Status and 2. MSATA Least Squares Residual V2-V3 Offsets. + Plot 1 shows the successful Visit IDs (blue points, which always correspond + to the second position - 1/2 shutter moved from the first), the unsuccessful + ones (red points), and the in-progress images (gray points, which always + correspond to the first position of the TA). MSATA can do up-to two tries + so you may see 4 entries for the same program ID in the gray points. + Plot 2 shows the V2-V3 least square residual offsets, the closer to the + purple cross (the half facet coordinates) the better. The default limits of + the plot are the acceptable values, if you see any point at the edges or + outside the limits of (-0.5, 0.5) for both axis, please let any of the + NIRSpec TA experts: Tracy Beck, Charles Proffit, and/or Tony Keyes.

    diff --git a/jwql/website/apps/jwql/templates/wata_monitor.html b/jwql/website/apps/jwql/templates/wata_monitor.html index d891e00bd..6403f97c3 100644 --- a/jwql/website/apps/jwql/templates/wata_monitor.html +++ b/jwql/website/apps/jwql/templates/wata_monitor.html @@ -26,6 +26,15 @@

    Updates to this monitor only occur with new data

    This monitor uses all WATA data available and the plots include all data. The plots will only be updated if new WATA data is in MAST, otherwise, the monitor will show all data previously obtained. +

    Instructions

    + For general WATA monitoring you only need to look at the first two plots + below: 1. WATA Status and 2. WATA Residual V2-V3 Offsets. + Plot 1 shows the successful Visit IDs (blue points) and the unsuccessful + ones (red points). + Plot 2 shows the V2-V3 residual offsets, the closer to (0,0) the better. The + default limits of the plot are the acceptable values, if you see any point + at the edges or outside the limits of (-0.5, 0.5) for both axis, please let + any of the NIRSpec TA experts: Tracy Beck, Charles Proffit, and/or Tony Keyes.

    From fb2bb90ca20c206a1b23c09d640cdc589c67703d Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 15 May 2023 10:11:10 -0400 Subject: [PATCH 323/449] trivial commit --- jwql/shared_tasks/shared_tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index c4dceee32..d191311cd 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -702,7 +702,7 @@ def run_pipeline(input_file, in_ext, ext_or_exts, instrument, jump_pipe=False): uncal_name = os.path.basename(uncal_file) result = start_pipeline(uncal_name, short_name, ext_or_exts, instrument, jump_pipe=jump_pipe) logging.info("\t\tStarting with ID {}".format(result.id)) - processed_path = result.get() + #processed_path = result.get() logging.info("\t\tPipeline Complete") output = retrieve_files(short_name, ext_or_exts, retrieve_dir) except Exception as e: @@ -789,7 +789,7 @@ def run_parallel_pipeline(input_files, in_ext, ext_or_exts, instrument, jump_pip for short_name in results: try: logging.info("\tWaiting for {} ({})".format(short_name, results[short_name].id)) - processed_path = results[short_name].get() + #processed_path = results[short_name].get() logging.info("\t{} retrieved".format(short_name)) outputs[input_file_paths[short_name]] = retrieve_files(short_name, ext_or_exts, output_dirs[short_name]) logging.info("\tFiles copied for {}".format(short_name)) From dc97986425be985989ad2385e39c0c59eb7edf62 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 15 May 2023 10:51:33 -0400 Subject: [PATCH 324/449] Move monitor_mast functions into mast_utils. Update tests. --- jwql/jwql_monitors/monitor_mast.py | 1 - ...est_monitor_mast.py => test_mast_utils.py} | 9 +- jwql/utils/mast_utils.py | 251 +++++++++++++++++- 3 files changed, 248 insertions(+), 13 deletions(-) rename jwql/tests/{test_monitor_mast.py => test_mast_utils.py} (89%) diff --git a/jwql/jwql_monitors/monitor_mast.py b/jwql/jwql_monitors/monitor_mast.py index a9fef8362..e9335bf6b 100755 --- a/jwql/jwql_monitors/monitor_mast.py +++ b/jwql/jwql_monitors/monitor_mast.py @@ -35,7 +35,6 @@ from jwql.utils.protect_module import lock_module -# Temporary until JWST operations: switch to test string for MAST request URL ON_GITHUB_ACTIONS = '/home/runner' in os.path.expanduser('~') or '/Users/runner' in os.path.expanduser('~') if not ON_GITHUB_ACTIONS: Mast._portal_api_connection.MAST_REQUEST_URL = get_config()['mast_request_url'] diff --git a/jwql/tests/test_monitor_mast.py b/jwql/tests/test_mast_utils.py similarity index 89% rename from jwql/tests/test_monitor_mast.py rename to jwql/tests/test_mast_utils.py index 23e700128..c4105c96f 100755 --- a/jwql/tests/test_monitor_mast.py +++ b/jwql/tests/test_mast_utils.py @@ -23,6 +23,7 @@ from jwql.jwql_monitors import monitor_mast as mm from jwql.utils.constants import JWST_INSTRUMENT_NAMES +from jwql.utils import mast_utils as mu from jwql.utils.utils import get_config # Temporary until JWST operations: switch to test string for MAST request URL @@ -47,7 +48,7 @@ def test_caom_instrument_keywords(): instruments""" kw = [] for ins in JWST_INSTRUMENT_NAMES: - kw.append(mm.instrument_keywords(ins, caom=True)['keyword'].tolist()) + kw.append(mu.instrument_keywords(ins, caom=True)['keyword'].tolist()) assert kw[0] == kw[1] == kw[2] == kw[3] == kw[4] @@ -57,7 +58,7 @@ def test_filtered_instrument_keywords(): different for all instruments""" kw = [] for ins in JWST_INSTRUMENT_NAMES: - kw.append(mm.instrument_keywords(ins, caom=False)['keyword'].tolist()) + kw.append(mu.instrument_keywords(ins, caom=False)['keyword'].tolist()) assert kw[0] != kw[1] != kw[2] != kw[3] != kw[4] @@ -65,7 +66,7 @@ def test_filtered_instrument_keywords(): def test_instrument_inventory_filtering(): """Test to see that the instrument inventory can be filtered""" filt = 'GR150R' - data = mm.instrument_inventory('niriss', + data = mu.instrument_inventory('niriss', add_filters={'filter': filt}, return_data=True) @@ -78,7 +79,7 @@ def test_instrument_dataproduct_filtering(): """Test to see that the instrument inventory can be filtered by data product""" dp = 'spectrum' - data = mm.instrument_inventory('nirspec', dataproduct=dp, caom=True, + data = mu.instrument_inventory('nirspec', dataproduct=dp, caom=True, return_data=True) dps = [row['dataproduct_type'] for row in data['data']] diff --git a/jwql/utils/mast_utils.py b/jwql/utils/mast_utils.py index d9e2c58bb..6524a8c6a 100644 --- a/jwql/utils/mast_utils.py +++ b/jwql/utils/mast_utils.py @@ -15,8 +15,243 @@ """ -from jwql.jwql_monitors import monitor_mast -from jwql.utils.constants import JWST_DATAPRODUCTS, JWST_INSTRUMENT_NAMES_MIXEDCASE +import logging +import os + +from astroquery.mast import Mast +from bokeh.embed import components +from bokeh.io import save, output_file +import pandas as pd + +from jwql.utils.constants import JWST_DATAPRODUCTS, JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, MAST_QUERY_LIMIT +from jwql.utils.permissions import set_permissions +from jwql.utils.utils import ensure_dir_exists, get_config +from jwql.utils.plotting import bar_chart + + +ON_GITHUB_ACTIONS = '/home/runner' in os.path.expanduser('~') or '/Users/runner' in os.path.expanduser('~') +if not ON_GITHUB_ACTIONS: + Mast._portal_api_connection.MAST_REQUEST_URL = get_config()['mast_request_url'] + +# Increase the limit on the number of entries that can be returned by +# a MAST query. +Mast._portal_api_connection.PAGESIZE = MAST_QUERY_LIMIT + + +def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, + add_filters=None, add_requests=None, + caom=False, return_data=False): + """Get the counts for a given instrument and data product + + Parameters + ---------- + instrument: str + The instrument name, i.e. one of ['niriss','nircam','nirspec', + 'miri','fgs'] + dataproduct: sequence, str + The type of data product to search + add_filters: dict + The ('paramName':'values') pairs to include in the 'filters' + argument of the request e.g. add_filters = {'filter':'GR150R'} + add_requests: dict + The ('request':'value') pairs to include in the request + e.g. add_requests = {'pagesize':1, 'page':1} + caom: bool + Query CAOM service + return_data: bool + Return the actual data instead of counts only + + Returns + ------- + int, dict + The number of database records that satisfy the search criteria + or a dictionary of the data if `return_data=True` + """ + filters = [] + + # Make sure the dataproduct is a list + if isinstance(dataproduct, str): + dataproduct = [dataproduct] + + # Make sure the instrument is supported + if instrument.lower() not in [ins.lower() for ins in JWST_INSTRUMENT_NAMES]: + raise TypeError('Supported instruments include:', JWST_INSTRUMENT_NAMES) + + # CAOM service + if caom: + + # Declare the service + service = 'Mast.Caom.Filtered' + + # Set the filters + filters += [{'paramName': 'obs_collection', 'values': ['JWST']}, + {'paramName': 'instrument_name', 'values': [instrument]}, + {'paramName': 'dataproduct_type', 'values': dataproduct}] + + # Instruent filtered service + else: + + # Declare the service + service = 'Mast.Jwst.Filtered.{}'.format(instrument.title()) + + # Include additonal filters + if isinstance(add_filters, dict): + filters += [{"paramName": name, "values": [val]} + for name, val in add_filters.items()] + + # Assemble the request + params = {'columns': 'COUNT_BIG(*)', + 'filters': filters, + 'removenullcolumns': True} + + # Just get the counts + if return_data: + params['columns'] = '*' + + # Add requests + if isinstance(add_requests, dict): + params.update(add_requests) + + response = Mast.service_request_async(service, params) + result = response[0].json() + + # Return all the data + if return_data: + return result + + # Or just the counts + else: + return result['data'][0]['Column1'] + + +def instrument_keywords(instrument, caom=False): + """Get the keywords for a given instrument service + + Parameters + ---------- + instrument: str + The instrument name, i.e. one of ['niriss','nircam','nirspec', + 'miri','fgs'] + caom: bool + Query CAOM service + + Returns + ------- + pd.DataFrame + A DataFrame of the keywords + """ + # Retrieve one dataset to get header keywords + if not caom: + filter_to_add = {'program': '01440'} + else: + filter_to_add = {'proposal_id': '01440'} + sample = instrument_inventory(instrument, return_data=True, caom=caom, + add_requests={'pagesize': 1, 'page': 1}, + add_filters=filter_to_add) + data = [[i['name'], i['type']] for i in sample['fields']] + keywords = pd.DataFrame(data, columns=('keyword', 'dtype')) + + return keywords + + +def jwst_inventory(instruments=JWST_INSTRUMENT_NAMES, + dataproducts=['image', 'spectrum', 'cube'], + caom=False, plot=False, output_dir=None): + """Gather a full inventory of all JWST data in each instrument + service by instrument/dtype + + Parameters + ---------- + instruments: sequence + The list of instruments to count + dataproducts: sequence + The types of dataproducts to count + caom: bool + Query CAOM service + plot: bool + Return a pie chart of the data + output_dir: str + Directory into which plots are saved + + Returns + ------- + astropy.table.table.Table + The table of record counts for each instrument and mode + """ + if output_dir is None: + output_dir = os.path.join(get_config()['outputs'], 'mast_utils') + ensure_dir_exists(output_dir) + + logging.info('Searching database...') + # Iterate through instruments + inventory, keywords = [], {} + for instrument in instruments: + ins = [instrument] + for dp in dataproducts: + count = instrument_inventory(instrument, dataproduct=dp, caom=caom) + ins.append(count) + + # Get the total + ins.append(sum(ins[-3:])) + + # Add it to the list + inventory.append(ins) + + # Add the keywords to the dict + keywords[instrument] = instrument_keywords(instrument, caom=caom) + + logging.info('Completed database search for {} instruments and {} data products.'. + format(instruments, dataproducts)) + + # Make the table + all_cols = ['instrument'] + dataproducts + ['total'] + table = pd.DataFrame(inventory, columns=all_cols) + + # Plot it + if plot: + if caom: + output_filename = 'database_monitor_caom' + else: + output_filename = 'database_monitor_jwst' + + # Make the plot + plt = bar_chart(table, 'instrument', dataproducts, + title="JWST Inventory") + + # Save the plot as full html + html_filename = output_filename + '.html' + outfile = os.path.join(output_dir, html_filename) + output_file(outfile) + save(plt) + set_permissions(outfile) + + logging.info('Saved Bokeh plots as HTML file: {}'.format(html_filename)) + + # Save the plot as components + plt.sizing_mode = 'stretch_both' + script, div = components(plt) + + div_outfile = os.path.join(output_dir, output_filename + "_component.html") + with open(div_outfile, 'w') as f: + f.write(div) + f.close() + set_permissions(div_outfile) + + script_outfile = os.path.join(output_dir, output_filename + "_component.js") + with open(script_outfile, 'w') as f: + f.write(script) + f.close() + set_permissions(script_outfile) + + logging.info('Saved Bokeh components files: {}_component.html and {}_component.js'.format( + output_filename, output_filename)) + + # Melt the table + table = pd.melt(table, id_vars=['instrument'], + value_vars=dataproducts, + value_name='files', var_name='dataproduct') + + return table, keywords def mast_query(instrument, templates, start_date, end_date, aperture=None, detector=None, filter_name=None, @@ -73,7 +308,7 @@ def mast_query(instrument, templates, start_date, end_date, aperture=None, detec # Make sure instrument is correct case instrument = JWST_INSTRUMENT_NAMES_MIXEDCASE[instrument.lower()] - # monitor_mast.instrument_inventory does not allow list inputs to + # instrument_inventory does not allow list inputs to # the added_filters input (or at least if you do provide a list, then # it becomes a nested list when it sends the query to MAST. The # nested list is subsequently ignored by MAST.) @@ -101,8 +336,8 @@ def mast_query(instrument, templates, start_date, end_date, aperture=None, detec if lamp is not None: parameters["lamp"] = lamp - query = monitor_mast.instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, - add_filters=parameters, return_data=True, caom=False) + query = instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, + add_filters=parameters, return_data=True, caom=False) if len(query['data']) > 0: query_results.extend(query['data']) @@ -147,7 +382,7 @@ def mast_query_miri(detector, aperture, templates, start_date, end_date): instrument = 'MIRI' - # monitor_mast.instrument_inventory does not allow list inputs to + # instrument_inventory does not allow list inputs to # the added_filters input (or at least if you do provide a list, then # it becomes a nested list when it sends the query to MAST. The # nested list is subsequently ignored by MAST.) @@ -164,8 +399,8 @@ def mast_query_miri(detector, aperture, templates, start_date, end_date): parameters = {"date_obs_mjd": {"min": start_date, "max": end_date}, "detector": detector, "exp_type": template_name} - query = monitor_mast.instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, - add_filters=parameters, return_data=True, caom=False) + query = instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, + add_filters=parameters, return_data=True, caom=False) if len(query['data']) > 0: query_results.extend(query['data']) From 9836cc792814b989d10ff9cf2297ccd6ac80f09e Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 15 May 2023 11:02:13 -0400 Subject: [PATCH 325/449] Replace mentions of monitor_mast with mast_utils --- jwql/tests/test_mast_utils.py | 4 ++-- jwql/utils/monitor_utils.py | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/jwql/tests/test_mast_utils.py b/jwql/tests/test_mast_utils.py index c4105c96f..0fc6c877e 100755 --- a/jwql/tests/test_mast_utils.py +++ b/jwql/tests/test_mast_utils.py @@ -1,6 +1,6 @@ #! /usr/bin/env python -"""Tests for the ``monitor_mast`` module. +"""Tests for the ``mast_utils`` module. Authors ------- @@ -14,7 +14,7 @@ suppress verbose output to stdout): :: - pytest -s test_monitor_mast.py + pytest -s test_mast_utils.py """ import os diff --git a/jwql/utils/monitor_utils.py b/jwql/utils/monitor_utils.py index f9258e15f..e4bc13852 100644 --- a/jwql/utils/monitor_utils.py +++ b/jwql/utils/monitor_utils.py @@ -22,9 +22,9 @@ from jwql.database.database_interface import Monitor, engine -from jwql.jwql_monitors import monitor_mast from jwql.utils.constants import ASIC_TEMPLATES, JWST_DATAPRODUCTS, MAST_QUERY_LIMIT from jwql.utils.logging_functions import configure_logging, get_log_status +from jwql.utils import mast_utils from jwql.utils.utils import filename_parser @@ -121,7 +121,7 @@ def mast_query_darks(instrument, aperture, start_date, end_date, readpatt=None): instrument = 'MIRI' dark_template = ['MIR_DARKALL', 'MIR_DARKIMG', 'MIR_DARKMRS'] - # monitor_mast.instrument_inventory does not allow list inputs to + # instrument_inventory does not allow list inputs to # the added_filters input (or at least if you do provide a list, then # it becomes a nested list when it sends the query to MAST. The # nested list is subsequently ignored by MAST.) @@ -137,8 +137,8 @@ def mast_query_darks(instrument, aperture, start_date, end_date, readpatt=None): if readpatt is not None: parameters["readpatt"] = readpatt - query = monitor_mast.instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, - add_filters=parameters, return_data=True, caom=False) + query = mast_utils.instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, + add_filters=parameters, return_data=True, caom=False) if 'data' in query.keys(): if len(query['data']) > 0: query_results.extend(query['data']) @@ -181,7 +181,7 @@ def mast_query_ta(instrument, aperture, start_date, end_date, readpatt=None): else: exp_types = ['NRS_TACQ', 'NRS_MSATA'] - # monitor_mast.instrument_inventory does not allow list inputs to + # instrument_inventory does not allow list inputs to # the added_filters input (or at least if you do provide a list, then # it becomes a nested list when it sends the query to MAST. The # nested list is subsequently ignored by MAST.) @@ -197,8 +197,8 @@ def mast_query_ta(instrument, aperture, start_date, end_date, readpatt=None): if readpatt is not None: parameters["readpatt"] = readpatt - query = monitor_mast.instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, - add_filters=parameters, return_data=True, caom=False) + query = mast_utils.instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, + add_filters=parameters, return_data=True, caom=False) if 'data' in query.keys(): if len(query['data']) > 0: query_results.extend(query['data']) From 4d575dd7e5fc22ab1058494305de2b3afb136c2c Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 15 May 2023 11:06:48 -0400 Subject: [PATCH 326/449] Remove UNUSED monitor_mast imports --- .../common_monitors/dark_monitor.py | 11 +++++------ .../nirspec_monitors/ta_monitors/msata_monitor.py | 1 - .../nirspec_monitors/ta_monitors/wata_monitor.py | 1 - 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index 26d433ed1..e5c6b65c8 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -100,9 +100,8 @@ from jwql.database.database_interface import NIRSpecDarkQueryHistory, NIRSpecDarkPixelStats, NIRSpecDarkDarkCurrent from jwql.database.database_interface import FGSDarkQueryHistory, FGSDarkPixelStats, FGSDarkDarkCurrent from jwql.instrument_monitors import pipeline_tools -from jwql.jwql_monitors import monitor_mast from jwql.shared_tasks.shared_tasks import only_one, run_pipeline, run_parallel_pipeline -from jwql.utils import calculations, instrument_properties, monitor_utils +from jwql.utils import calculations, instrument_properties, mast_utils, monitor_utils from jwql.utils.constants import ASIC_TEMPLATES, DARK_MONITOR_MAX_BADPOINTS_TO_PLOT, JWST_INSTRUMENT_NAMES, FULL_FRAME_APERTURES from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_DATAPRODUCTS, RAPID_READPATTERNS from jwql.utils.logging_functions import log_info, log_fail @@ -402,11 +401,11 @@ def exclude_existing_badpix(self, badpix, pixel_type): new_pixels_y : list List of y coordinates of new bad pixels """ - + if len(badpix[0]) == 0: logging.warning("\tNo new {} pixels to check.".format(pixel_type)) return ([], []) - + logging.info("\tChecking {} potential new {} pixels".format(len(badpix[0]), pixel_type)) if pixel_type not in ['hot', 'dead', 'noisy']: @@ -440,7 +439,7 @@ def exclude_existing_badpix(self, badpix, pixel_type): if len(np.intersect1d(ind_x[0], ind_y[0])) == 0: new_pixels_x.append(x) new_pixels_y.append(y) - + logging.info("\t\tKeeping {} {} pixels".format(len(new_pixels_x), pixel_type)) # pixel = (x, y) # if pixel not in already_found: @@ -810,7 +809,7 @@ def process(self, file_list): # Add new noisy pixels to the database logging.info('\tFound {} new noisy pixels'.format(len(new_noisy_pixels[0]))) self.add_bad_pix(new_noisy_pixels, 'noisy', file_list, mean_slope_file, baseline_file, min_time, mid_time, max_time) - + logging.info("Creating Mean Slope Image {}".format(slope_image)) # Create png file of mean slope image. Add bad pixels only for full frame apertures self.create_mean_slope_figure(slope_image, len(slope_files), hotxy=new_hot_pix, deadxy=new_dead_pix, diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py index d8a34a2a6..d3593ffcb 100644 --- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py +++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py @@ -58,7 +58,6 @@ from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE from jwql.database.database_interface import session, engine from jwql.database.database_interface import NIRSpecTAQueryHistory, NIRSpecTAStats -from jwql.jwql_monitors import monitor_mast from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config, filename_parser diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py index 267f3efc1..337001b4e 100644 --- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py +++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py @@ -54,7 +54,6 @@ from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE from jwql.database.database_interface import session, engine from jwql.database.database_interface import NIRSpecTAQueryHistory, NIRSpecTAStats -from jwql.jwql_monitors import monitor_mast from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config, filename_parser From f9a5a912738f6c0f783c9206007580054bb24d70 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 15 May 2023 11:26:55 -0400 Subject: [PATCH 327/449] Update documentation --- docs/source/jwql_monitors.rst | 6 ------ docs/source/tests.rst | 4 ++-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/docs/source/jwql_monitors.rst b/docs/source/jwql_monitors.rst index 55446ea97..42221e004 100644 --- a/docs/source/jwql_monitors.rst +++ b/docs/source/jwql_monitors.rst @@ -25,9 +25,3 @@ monitor_filesystem.py .. automodule:: jwql.jwql_monitors.monitor_filesystem :members: :undoc-members: - -monitor_mast.py ---------------- -.. automodule:: jwql.jwql_monitors.monitor_mast - :members: - :undoc-members: \ No newline at end of file diff --git a/docs/source/tests.rst b/docs/source/tests.rst index cc7d38d67..576a126e9 100644 --- a/docs/source/tests.rst +++ b/docs/source/tests.rst @@ -74,9 +74,9 @@ test_logging_functions.py :members: :undoc-members: -test_monitor_mast.py +test_mast_utils.py -------------------- -.. automodule:: jwql.tests.test_monitor_mast +.. automodule:: jwql.tests.test_mast_utils :members: :undoc-members: From 1b33f0b43794cc4519dd7c3695cc92a69b0a8c98 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 15 May 2023 11:34:39 -0400 Subject: [PATCH 328/449] Remove db.py from doc list --- docs/source/website.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/source/website.rst b/docs/source/website.rst index 530f367ae..6f0da9b31 100644 --- a/docs/source/website.rst +++ b/docs/source/website.rst @@ -26,12 +26,6 @@ data_containers.py :members: :undoc-members: -db.py ------ -.. automodule:: jwql.website.apps.jwql.db - :members: - :undoc-members: - forms.py -------- .. automodule:: jwql.website.apps.jwql.forms From c03d1eed3d4d44ebd1b65ef994cd706ef1f3b314 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 15 May 2023 11:40:51 -0400 Subject: [PATCH 329/449] Add phinx_rtd_theme to rtd_requirements --- rtd_requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/rtd_requirements.txt b/rtd_requirements.txt index 5ba4b8091..d3f595ab7 100644 --- a/rtd_requirements.txt +++ b/rtd_requirements.txt @@ -10,6 +10,7 @@ pygments==2.14.0 pytest==7.2.1 redis==4.5.1 sphinx>=2 +sphinx_rtd_theme==1.2.0 stsci_rtd_theme==1.0.0 tomli==2.0.1 git+https://github.com/spacetelescope/jwst_reffiles From 653f10d860ea8ae29b3a00f2d9bdc96b63880f70 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 15 May 2023 11:48:57 -0400 Subject: [PATCH 330/449] Remove mast_monitor.py --- jwql/jwql_monitors/monitor_mast.py | 293 ----------------------------- jwql/tests/test_mast_utils.py | 1 - 2 files changed, 294 deletions(-) delete mode 100755 jwql/jwql_monitors/monitor_mast.py diff --git a/jwql/jwql_monitors/monitor_mast.py b/jwql/jwql_monitors/monitor_mast.py deleted file mode 100755 index e9335bf6b..000000000 --- a/jwql/jwql_monitors/monitor_mast.py +++ /dev/null @@ -1,293 +0,0 @@ -#! /usr/bin/env python - -"""This module is home to a suite of MAST queries that gather bulk -properties of available JWST data for JWQL. - -Authors -------- - - Joe Filippazzo - -Use ---- - - To get an inventory of all JWST files do: - :: - - from jwql.jwql_monitors import monitor_mast - inventory, keywords = monitor_mast.jwst_inventory() -""" - -import logging -import os - -from astroquery.mast import Mast -from bokeh.embed import components -from bokeh.io import save, output_file -import pandas as pd - -from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_DATAPRODUCTS, MAST_QUERY_LIMIT -from jwql.utils.logging_functions import log_info, log_fail -from jwql.utils.permissions import set_permissions -from jwql.utils.utils import get_config -from jwql.utils import monitor_utils -from jwql.utils.plotting import bar_chart -from jwql.utils.protect_module import lock_module - - -ON_GITHUB_ACTIONS = '/home/runner' in os.path.expanduser('~') or '/Users/runner' in os.path.expanduser('~') -if not ON_GITHUB_ACTIONS: - Mast._portal_api_connection.MAST_REQUEST_URL = get_config()['mast_request_url'] - -# Increase the limit on the number of entries that can be returned by -# a MAST query. -Mast._portal_api_connection.PAGESIZE = MAST_QUERY_LIMIT - - -def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, - add_filters=None, add_requests=None, - caom=False, return_data=False): - """Get the counts for a given instrument and data product - - Parameters - ---------- - instrument: str - The instrument name, i.e. one of ['niriss','nircam','nirspec', - 'miri','fgs'] - dataproduct: sequence, str - The type of data product to search - add_filters: dict - The ('paramName':'values') pairs to include in the 'filters' - argument of the request e.g. add_filters = {'filter':'GR150R'} - add_requests: dict - The ('request':'value') pairs to include in the request - e.g. add_requests = {'pagesize':1, 'page':1} - caom: bool - Query CAOM service - return_data: bool - Return the actual data instead of counts only - - Returns - ------- - int, dict - The number of database records that satisfy the search criteria - or a dictionary of the data if `return_data=True` - """ - filters = [] - - # Make sure the dataproduct is a list - if isinstance(dataproduct, str): - dataproduct = [dataproduct] - - # Make sure the instrument is supported - if instrument.lower() not in [ins.lower() for ins in JWST_INSTRUMENT_NAMES]: - raise TypeError('Supported instruments include:', JWST_INSTRUMENT_NAMES) - - # CAOM service - if caom: - - # Declare the service - service = 'Mast.Caom.Filtered' - - # Set the filters - filters += [{'paramName': 'obs_collection', 'values': ['JWST']}, - {'paramName': 'instrument_name', 'values': [instrument]}, - {'paramName': 'dataproduct_type', 'values': dataproduct}] - - # Instruent filtered service - else: - - # Declare the service - service = 'Mast.Jwst.Filtered.{}'.format(instrument.title()) - - # Include additonal filters - if isinstance(add_filters, dict): - filters += [{"paramName": name, "values": [val]} - for name, val in add_filters.items()] - - # Assemble the request - params = {'columns': 'COUNT_BIG(*)', - 'filters': filters, - 'removenullcolumns': True} - - # Just get the counts - if return_data: - params['columns'] = '*' - - # Add requests - if isinstance(add_requests, dict): - params.update(add_requests) - - response = Mast.service_request_async(service, params) - result = response[0].json() - - # Return all the data - if return_data: - return result - - # Or just the counts - else: - return result['data'][0]['Column1'] - - -def instrument_keywords(instrument, caom=False): - """Get the keywords for a given instrument service - - Parameters - ---------- - instrument: str - The instrument name, i.e. one of ['niriss','nircam','nirspec', - 'miri','fgs'] - caom: bool - Query CAOM service - - Returns - ------- - pd.DataFrame - A DataFrame of the keywords - """ - # Retrieve one dataset to get header keywords - if not caom: - filter_to_add = {'program': '01440'} - else: - filter_to_add = {'proposal_id': '01440'} - sample = instrument_inventory(instrument, return_data=True, caom=caom, - add_requests={'pagesize': 1, 'page': 1}, - add_filters=filter_to_add) - data = [[i['name'], i['type']] for i in sample['fields']] - keywords = pd.DataFrame(data, columns=('keyword', 'dtype')) - - return keywords - - -def jwst_inventory(instruments=JWST_INSTRUMENT_NAMES, - dataproducts=['image', 'spectrum', 'cube'], - caom=False, plot=False): - """Gather a full inventory of all JWST data in each instrument - service by instrument/dtype - - Parameters - ---------- - instruments: sequence - The list of instruments to count - dataproducts: sequence - The types of dataproducts to count - caom: bool - Query CAOM service - plot: bool - Return a pie chart of the data - - Returns - ------- - astropy.table.table.Table - The table of record counts for each instrument and mode - """ - logging.info('Searching database...') - # Iterate through instruments - inventory, keywords = [], {} - for instrument in instruments: - ins = [instrument] - for dp in dataproducts: - count = instrument_inventory(instrument, dataproduct=dp, caom=caom) - ins.append(count) - - # Get the total - ins.append(sum(ins[-3:])) - - # Add it to the list - inventory.append(ins) - - # Add the keywords to the dict - keywords[instrument] = instrument_keywords(instrument, caom=caom) - - logging.info('Completed database search for {} instruments and {} data products.'. - format(instruments, dataproducts)) - - # Make the table - all_cols = ['instrument'] + dataproducts + ['total'] - table = pd.DataFrame(inventory, columns=all_cols) - - # Plot it - if plot: - # Determine plot location and names - output_dir = get_config()['outputs'] - - if caom: - output_filename = 'database_monitor_caom' - else: - output_filename = 'database_monitor_jwst' - - # Make the plot - plt = bar_chart(table, 'instrument', dataproducts, - title="JWST Inventory") - - # Save the plot as full html - html_filename = output_filename + '.html' - outfile = os.path.join(output_dir, 'monitor_mast', html_filename) - output_file(outfile) - save(plt) - set_permissions(outfile) - - logging.info('Saved Bokeh plots as HTML file: {}'.format(html_filename)) - - # Save the plot as components - plt.sizing_mode = 'stretch_both' - script, div = components(plt) - - div_outfile = os.path.join(output_dir, 'monitor_mast', output_filename + "_component.html") - with open(div_outfile, 'w') as f: - f.write(div) - f.close() - set_permissions(div_outfile) - - script_outfile = os.path.join(output_dir, 'monitor_mast', output_filename + "_component.js") - with open(script_outfile, 'w') as f: - f.write(script) - f.close() - set_permissions(script_outfile) - - logging.info('Saved Bokeh components files: {}_component.html and {}_component.js'.format( - output_filename, output_filename)) - - # Melt the table - table = pd.melt(table, id_vars=['instrument'], - value_vars=dataproducts, - value_name='files', var_name='dataproduct') - - return table, keywords - - -@log_fail -@log_info -def monitor_mast(): - """Tabulates the inventory of all JWST data products in the MAST - archive and generates plots. - """ - logging.info('Beginning database monitoring.') - - # Perform inventory of the JWST service - jwst_inventory(instruments=JWST_INSTRUMENT_NAMES, - dataproducts=['image', 'spectrum', 'cube'], - caom=False, plot=True) - - # Perform inventory of the CAOM service - jwst_inventory(instruments=JWST_INSTRUMENT_NAMES, - dataproducts=['image', 'spectrum', 'cube'], - caom=True, plot=True) - - -@lock_module -def protected_code(): - """Protected code ensures only 1 instance of module will run at any given time""" - - # Configure logging - module = os.path.basename(__file__).strip('.py') - start_time, log_file = monitor_utils.initialize_instrument_monitor(module) - - # Run the monitors - monitor_mast() - monitor_utils.update_monitor_table(module, start_time, log_file) - - -if __name__ == '__main__': - protected_code() diff --git a/jwql/tests/test_mast_utils.py b/jwql/tests/test_mast_utils.py index 0fc6c877e..6267be9bc 100755 --- a/jwql/tests/test_mast_utils.py +++ b/jwql/tests/test_mast_utils.py @@ -21,7 +21,6 @@ from astroquery.mast import Mast -from jwql.jwql_monitors import monitor_mast as mm from jwql.utils.constants import JWST_INSTRUMENT_NAMES from jwql.utils import mast_utils as mu from jwql.utils.utils import get_config From c3fec3aecdde6c8171e03ec4909930a729a051a5 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 15 May 2023 11:59:56 -0400 Subject: [PATCH 331/449] Fix import in CR monitor --- .../common_monitors/cosmic_ray_monitor.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/cosmic_ray_monitor.py b/jwql/instrument_monitors/common_monitors/cosmic_ray_monitor.py index 3db7afee1..19c905806 100755 --- a/jwql/instrument_monitors/common_monitors/cosmic_ray_monitor.py +++ b/jwql/instrument_monitors/common_monitors/cosmic_ray_monitor.py @@ -61,8 +61,8 @@ from jwql.database.database_interface import FGSCosmicRayQueryHistory from jwql.database.database_interface import FGSCosmicRayStats from jwql.database.database_interface import session, engine -from jwql.jwql_monitors import monitor_mast from jwql.shared_tasks.shared_tasks import only_one, run_pipeline, run_parallel_pipeline +from jwql.utils import mast_utils from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_DATAPRODUCTS from jwql.utils.logging_functions import configure_logging from jwql.utils.logging_functions import log_info @@ -557,7 +557,7 @@ def process(self, file_list): files """ for file_chunk in grouper(file_list, 100): - + input_files = [] in_ext = "uncal" out_exts = defaultdict(lambda: ['jump', '0_ramp_fit']) @@ -622,7 +622,7 @@ def process(self, file_list): dir_name = '_'.join(os.path.basename(file_name).split('_')[:2]) # file_name[51:76] self.obs_dir = os.path.join(self.data_dir, dir_name) - + if file_name not in output_files: skip = False head = fits.getheader(file_name) @@ -844,9 +844,9 @@ def query_mast(self): data_product = JWST_DATAPRODUCTS parameters = {"date_obs_mjd": {"min": self.query_start, "max": self.query_end}, "apername": self.aperture} - result = monitor_mast.instrument_inventory(self.instrument, data_product, - add_filters=parameters, - return_data=True) + result = mast_utils.instrument_inventory(self.instrument, data_product, + add_filters=parameters, + return_data=True) return result From 3e3af74672ece149aab129aeaa3cd3c37a7fd363 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 22 May 2023 16:21:47 -0400 Subject: [PATCH 332/449] update step_args in shared_tasks scripts, to be able to skip pipe steps --- .../common_monitors/dark_monitor.py | 6 +- jwql/instrument_monitors/pipeline_tools.py | 23 ++++--- jwql/shared_tasks/run_pipeline.py | 46 ++++++++++--- jwql/shared_tasks/shared_tasks.py | 65 +++++++++++++++---- 4 files changed, 110 insertions(+), 30 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index 36b56a5cb..a04080ed5 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -712,7 +712,11 @@ def process(self, file_list): logging.info("\t\tAdding {} to calibration set".format(filename)) pipeline_files.append(filename) - outputs = run_parallel_pipeline(pipeline_files, "dark", ["rate", "rateints"], self.instrument) + # Specify that we want to skip the dark current correction step + step_args = {'dark_current': {'skip: True'}} + + # Call the pipeline + outputs = run_parallel_pipeline(pipeline_files, "dark", ["rate", "rateints"], self.instrument, step_args=step_args) for filename in file_list: processed_file = filename.replace("_dark", "_rate") if processed_file not in slope_files and os.path.isfile(processed_file): diff --git a/jwql/instrument_monitors/pipeline_tools.py b/jwql/instrument_monitors/pipeline_tools.py index 9390802e8..a5fd4a464 100644 --- a/jwql/instrument_monitors/pipeline_tools.py +++ b/jwql/instrument_monitors/pipeline_tools.py @@ -25,6 +25,7 @@ from jwst.dq_init import DQInitStep from jwst.dark_current import DarkCurrentStep from jwst.firstframe import FirstFrameStep +from jwst.gain_scale import GainScaleStep from jwst.group_scale import GroupScaleStep from jwst.ipc import IPCStep from jwst.jump import JumpStep @@ -34,6 +35,7 @@ from jwst.pipeline.calwebb_detector1 import Detector1Pipeline from jwst.ramp_fitting import RampFitStep from jwst.refpix import RefPixStep +from jwst.reset import ResetStep from jwst.rscd import RscdStep from jwst.saturation import SaturationStep from jwst.superbias import SuperBiasStep @@ -43,16 +45,17 @@ # Define the fits header keyword that accompanies each step PIPE_KEYWORDS = {'S_GRPSCL': 'group_scale', 'S_DQINIT': 'dq_init', 'S_SATURA': 'saturation', - 'S_REFPIX': 'refpix', 'S_SUPERB': 'superbias', + 'S_REFPIX': 'refpix', 'S_SUPERB': 'superbias', 'S_RESET': 'reset', 'S_PERSIS': 'persistence', 'S_DARK': 'dark_current', 'S_LINEAR': 'linearity', 'S_FRSTFR': 'firstframe', 'S_LASTFR': 'lastframe', 'S_RSCD': 'rscd', - 'S_JUMP': 'jump', 'S_RAMP': 'rate'} + 'S_JUMP': 'jump', 'S_RAMP': 'rate', 'S_GANSCL': 'gain_scale', 'S_IPC': 'ipc'} PIPELINE_STEP_MAPPING = {'dq_init': DQInitStep, 'dark_current': DarkCurrentStep, - 'firstframe': FirstFrameStep, 'group_scale': GroupScaleStep, - 'ipc': IPCStep, 'jump': JumpStep, 'lastframe': LastFrameStep, - 'linearity': LinearityStep, 'persistence': PersistenceStep, - 'rate': RampFitStep, 'refpix': RefPixStep, 'rscd': RscdStep, + 'firstframe': FirstFrameStep, 'gain_scale': GainScaleStep, + 'group_scale': GroupScaleStep, 'ipc': IPCStep, 'jump': JumpStep, + 'lastframe': LastFrameStep, 'linearity': LinearityStep, + 'persistence': PersistenceStep, 'rate': RampFitStep, + 'refpix': RefPixStep, 'reset': ResetStep, 'rscd': RscdStep, 'saturation': SaturationStep, 'superbias': SuperBiasStep} # Readout patterns that have nframes != a power of 2. These readout patterns @@ -182,13 +185,13 @@ def get_pipeline_steps(instrument): # IPC correction currently not done for any instrument steps.remove('ipc') - # Initialize using PIPE_KEYWORDS so the steps will be in the right order + # Initialize using OrderedDict so the steps will be in the right order required_steps = OrderedDict({}) for key in steps: required_steps[key] = True - for key in PIPE_KEYWORDS.values(): - if key not in required_steps.keys(): - required_steps[key] = False + #for key in PIPE_KEYWORDS.values(): + # if key not in required_steps.keys(): + # required_steps[key] = False return required_steps diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index aef81fd56..79fa8d260 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -5,6 +5,7 @@ from collections import OrderedDict from copy import deepcopy from glob import glob +import json import os import shutil import sys @@ -33,7 +34,7 @@ from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path -def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_cores='all'): +def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_cores='all', step_args={}): """Run the steps of ``calwebb_detector1`` on the input file, saving the result of each step as a separate output file, then return the name-and-path of the file as reduced in the reduction directory. @@ -61,20 +62,47 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co # that has been completed, and skip that plus all prior steps if 'uncal.fits' not in input_file: completed_steps = completed_pipeline_steps(input_file) + + # Reverse the boolean value, so that now steps answers the question: "Do we need + # to run this step?"" for step in steps: steps[step] = not completed_steps[step] # Special case: if the input file is a dark.fits file, then we want to skip the # dark_current subtraction step - if 'dark.fits' in input_file: - step['dark_current'] = False + #if 'dark.fits' in input_file: + # steps['dark_current'] = False + + # Make sure we don't run steps out of order. Find the latest step that has been + # run, and only run subsequent steps. This protects against cases where some early + # step was not run. In that case, we don't want to go back and run it because running + # pipeline steps out of order doesn't work. + last_run = 'group_scale' # initialize to the first step + for step in steps: + if not steps[step]: + last_run = deepcopy(step) + + for step in steps: + if step == last_run: + break + if step != last_run: + steps[step] = False + + # Set any steps the user specifically asks to skip + for step, step_dict in step_args.items(): + if 'skip' in step_dict: + print(f'SKIP the {step}!!') + if step_dict['skip']: + steps[step] = False # Run each specified step first_step_to_be_run = True for step_name in steps: kwargs = {} + if step_name in step_args: + kwargs = step_args[step_name] if step_name in ['jump', 'rate']: - kwargs = {'maximum_cores': max_cores} + kwargs['maximum_cores'] = max_cores if steps[step_name]: sys.stderr.write("Running step {}\n".format(step_name)) with open(status_file, 'a+') as status_f: @@ -138,7 +166,7 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co # Done. -def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=True, save_fitopt=True, max_cores='all'): +def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=True, save_fitopt=True, max_cores='all', step_args={}): """Call ``calwebb_detector1`` on the provided file, running all steps up to the ``ramp_fit`` step, and save the result. Optionally run the ``ramp_fit`` step and save the resulting slope file as well. @@ -228,7 +256,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T steps_to_skip = ['group_scale', 'dq_init', 'saturation', 'ipc', 'superbias', 'refpix', 'linearity'] for step in steps_to_skip: - params[step] = dict(skip: True) + params[step] = dict(skip=True) else: # Turn off IPC step until it is put in the right place params['ipc'] = dict(skip=True) @@ -269,6 +297,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T out_help = 'Comma-separated list of output extensions (for cal only, otherwise just "all")' name_help = 'Input file name with no path or extensions' cores_help = 'Maximum cores to use (default "all")' + step_args_help = 'Step-specific parameter value nested dictionary' parser = argparse.ArgumentParser(description='Run local calibration') parser.add_argument('pipe', metavar='PIPE', type=str, help=pipe_help) parser.add_argument('outputs', metavar='OUTPUTS', type=str, help=out_help) @@ -277,6 +306,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T parser.add_argument('input_file', metavar='FILE', type=str, help=file_help) parser.add_argument('short_name', metavar='NAME', type=str, help=name_help) parser.add_argument('max_cores', metavar='CORES', type=str, help=cores_help) + parser.add_argument('step_args', metavar='STEP_ARGS', type=json.loads, help=step_args_help) with open(general_status_file, "a+") as status_file: status_file.write("Created argument parser at {}\n".format(time.ctime())) @@ -319,12 +349,12 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T if pipe_type == 'jump': with open(status_file, 'a+') as out_file: out_file.write("Running jump pipeline.\n") - run_save_jump(input_file, short_name, working_path, instrument, ramp_fit=True, save_fitopt=True, max_cores=args.max_cores) + run_save_jump(input_file, short_name, working_path, instrument, ramp_fit=True, save_fitopt=True, max_cores=args.max_cores, step_args=args.step_args) elif pipe_type == 'cal': with open(status_file, 'a+') as out_file: out_file.write("Running cal pipeline.\n") outputs = outputs.split(",") - run_pipe(input_file, short_name, working_path, instrument, outputs, max_cores=args.max_cores) + run_pipe(input_file, short_name, working_path, instrument, outputs, max_cores=args.max_cores, step_args=args.step_args) except Exception as e: with open(status_file, 'a+') as out_file: out_file.write("Exception when starting pipeline.\n") diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index d191311cd..eef421ecd 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -224,9 +224,42 @@ def collect_after_task(**kwargs): gc.collect() -def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_file, cores): - command = "{} {} {} '{}' {} {} {} {}" - command = command.format(name, cmd, outputs, cal_dir, ins, in_file, short_name, cores) +def convert_step_args_to_string(args_dict): + """Convert the nested dictionary containing pipeline step parameter keyword/value pairs + to a string so that it can be passed via command line + + Parameters + ---------- + args_dict : dict + Nested dictionary. Top level keys are pipeline step names. Values are dictionaries containing + keyword value pairs for that step. + + Returns + ------- + args_str : str + String representation of ``args_dict`` + """ + args_str='{' + + for i, step in enumerate(args_dict): + args_str += f'"{step}":' + args_str += '{' + for j, (param, val) in enumerate(args_dict[step].items()): + args_str += f'"{param}":"{val}"' + if j < len(args_dict[step])-1: + args_str += ', ' + args_str += '}' + if i < len(args_dict)-1: + args_str += ',' + return args_str + + +def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_file, cores, step_args): + # Convert step_args dictionary to a string so that it can be passed via command line + step_args_str = convert_step_args_to_string(step_args) + + command = "{} {} {} '{}' {} {} {} {} {}" + command = command.format(name, cmd, outputs, cal_dir, ins, in_file, short_name, cores, step_args_str) logging.info("Running {}".format(command)) process = Popen(command, shell=True, executable="/bin/bash", stderr=PIPE) with process.stderr: @@ -309,7 +342,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, cores = 'all' status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, input_file, - short_name, result_file, cores) + short_name, result_file, cores, step_args) if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") @@ -324,7 +357,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, if core_fail: cores = "half" status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, - input_file, short_name, result_file, cores) + input_file, short_name, result_file, cores, step_args) if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") managed = True @@ -338,7 +371,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, if core_fail: cores = "none" status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, - input_file, short_name, result_file, cores) + input_file, short_name, result_file, cores, step_args) if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") managed = True @@ -554,7 +587,7 @@ def prep_file(input_file, in_ext): return short_name, cal_lock, os.path.join(send_path, uncal_name) -def start_pipeline(input_file, short_name, ext_or_exts, instrument, jump_pipe=False): +def start_pipeline(input_file, short_name, ext_or_exts, instrument, jump_pipe=False, step_args={}): """Starts the standard or save_jump pipeline for the provided file. .. warning:: @@ -591,6 +624,11 @@ def start_pipeline(input_file, short_name, ext_or_exts, instrument, jump_pipe=Fa jump_pipe : bool Whether the detector1 jump pipeline is being used (e.g. the bad pixel monitor) + step_args : dict + Pipeline step arguments to be passed to the pipeline call. Nested dictionary with keys that + are the step names (as seen in pipeline_tools.PIPELINE_STEP_MAPPING). Each value is a + dictionary of keyword value pairs that are relevant for that step. + Returns ------- result : celery.result.AsyncResult @@ -606,9 +644,9 @@ def start_pipeline(input_file, short_name, ext_or_exts, instrument, jump_pipe=Fa ramp_fit = True elif "fitopt" in ext: save_fitopt = True - result = calwebb_detector1_save_jump.delay(input_file, instrument, ramp_fit=ramp_fit, save_fitopt=save_fitopt) + result = calwebb_detector1_save_jump.delay(input_file, instrument, ramp_fit=ramp_fit, save_fitopt=save_fitopt, step_args=step_args) else: - result = run_calwebb_detector1.delay(input_file, short_name, ext_or_exts, instrument) + result = run_calwebb_detector1.delay(input_file, short_name, ext_or_exts, instrument, step_args=step_args) return result @@ -716,7 +754,7 @@ def run_pipeline(input_file, in_ext, ext_or_exts, instrument, jump_pipe=False): return output -def run_parallel_pipeline(input_files, in_ext, ext_or_exts, instrument, jump_pipe=False): +def run_parallel_pipeline(input_files, in_ext, ext_or_exts, instrument, jump_pipe=False, step_args={}): """Convenience function for using the ``run_calwebb_detector1`` function on a list of data files, breaking them into parallel celery calls, collecting the results together, and returning the results as another list. In particular, this function will do the @@ -757,6 +795,11 @@ def run_parallel_pipeline(input_files, in_ext, ext_or_exts, instrument, jump_pip jump_pipe : bool Whether the detector1 jump pipeline is being used (e.g. the bad pixel monitor) + step_args : dict + Pipeline step arguments to be passed to the pipeline call. Nested dictionary with keys that + are the step names (as seen in pipeline_tools.PIPELINE_STEP_MAPPING). Each value is a + dictionary of keyword value pairs that are relevant for that step. + Returns ------- file_or_files : str or list-of-str @@ -782,7 +825,7 @@ def run_parallel_pipeline(input_files, in_ext, ext_or_exts, instrument, jump_pip output_dirs[short_name] = retrieve_dir input_file_paths[short_name] = input_file locks[short_name] = cal_lock - results[short_name] = start_pipeline(uncal_name, short_name, ext_or_exts, instrument, jump_pipe=jump_pipe) + results[short_name] = start_pipeline(uncal_name, short_name, ext_or_exts, instrument, jump_pipe=jump_pipe, step_args=step_args) logging.info("\tStarting {} with ID {}".format(short_name, results[short_name].id)) logging.info("Celery tasks submitted.") logging.info("Waiting for task results") From 458c789fd130249a2accfcac76779bddff540394 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 22 May 2023 16:50:51 -0400 Subject: [PATCH 333/449] Propogate user-input pipeline keywords. --- jwql/shared_tasks/run_pipeline.py | 13 ++++++++++++- jwql/shared_tasks/shared_tasks.py | 9 +++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 79fa8d260..ba321eafa 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -256,11 +256,22 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T steps_to_skip = ['group_scale', 'dq_init', 'saturation', 'ipc', 'superbias', 'refpix', 'linearity'] for step in steps_to_skip: - params[step] = dict(skip=True) + step_dict = dict(skip=True) + if step in params: + params[step] = params[step].update(step_dict) + else: + params[step] = dict(skip=True) else: # Turn off IPC step until it is put in the right place params['ipc'] = dict(skip=True) + # Include any user-specified parameters + for step_name in step_args: + if step_name in params: + params[step_name] = params[step_name].update(step_args[step_name]) + else: + params[step_name] = step_args[step_name] + if run_jump or (ramp_fit and run_slope) or (save_fitopt and run_fitopt): model.call(datamodel, output_dir=work_directory, steps=params) else: diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index eef421ecd..450405058 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -332,8 +332,13 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, result_file = os.path.join(cal_dir, short_name+"_status.txt") if "all" in ext_or_exts: logging.info("All outputs requested") - out_exts = ["dq_init", "saturation", "superbias", "refpix", "linearity", - "persistence", "dark_current", "jump", "rate"] + if instrument.lower() != 'miri': + out_exts = ["dq_init", "saturation", "superbias", "refpix", "linearity", + "persistence", "dark_current", "jump", "rate"] + else: + out_exts = ["group_scale", "dq_init", "saturation", "firstframe", "lastframe", "reset", + "linearity", "rscd", "dark_current", "refpix", "jump", "rate", "gain_scale"] + calibrated_files = ["{}_{}.fits".format(short_name, ext) for ext in out_exts] logging.info("Requesting {}".format(calibrated_files)) else: From 7a1e1858df9ad08aaca299ec6abd71ed883f74c5 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Mon, 22 May 2023 17:10:40 -0400 Subject: [PATCH 334/449] Updating conda requirements --- environment_python_3.10.yml | 28 ++++++++++++++-------------- environment_python_3.9.yml | 28 ++++++++++++++-------------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml index 68a1fa077..6ef4590b0 100644 --- a/environment_python_3.10.yml +++ b/environment_python_3.10.yml @@ -22,36 +22,36 @@ channels: - defaults dependencies: - - astropy=5.2.1 + - astropy=5.2.2 - bokeh=2.4.3 - - beautifulsoup4=4.11.2 + - beautifulsoup4=4.12.2 - celery=5.2.7 - - cryptography=39.0.1 - - django=4.1.7 + - cryptography=40.0.2 + - django=4.2.1 - flake8=6.0.0 - inflection=0.5.1 - - ipython=8.10.0 + - ipython=8.13.2 - jinja2=3.1.2 - jsonschema=4.17.3 - matplotlib=3.7.0 - - nodejs=18.12.1 - - numpy=1.24.2 + - nodejs=18.15.0 + - numpy=1.24.3 - numpydoc=1.5.0 - - pandas=2.0.0 - - pip=23.0 + - pandas=2.0.1 + - pip=23.1.2 - postgresql=15.2 - psycopg2=2.9.3 - - pytest=7.2.1 + - pytest=7.3.1 - pytest-cov=4.0.0 - pytest-mock=3.10.0 - python=3.10.9 - pyyaml=6.0 - redis - - scipy=1.9.3 - - setuptools=67.3.1 - - sphinx=6.1.3 + - scipy=1.10.1 + - setuptools=67.7.2 + - sphinx=6.2.1 - sphinx_rtd_theme=1.2.0 - - sqlalchemy=2.0.8 + - sqlalchemy=2.0.15 - twine=4.0.2 - wtforms=3.0.1 diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml index 2aadf5067..c021f3858 100644 --- a/environment_python_3.9.yml +++ b/environment_python_3.9.yml @@ -22,36 +22,36 @@ channels: - defaults dependencies: - - astropy=5.2.1 + - astropy=5.2.2 - bokeh=2.4.3 - - beautifulsoup4=4.11.2 + - beautifulsoup4=4.12.2 - celery=5.2.7 - - cryptography=39.0.1 - - django=4.1.7 + - cryptography=40.0.2 + - django=4.2.1 - flake8=6.0.0 - inflection=0.5.1 - - ipython=8.10.0 + - ipython=8.13.2 - jinja2=3.1.2 - jsonschema=4.17.3 - matplotlib=3.7.0 - - nodejs=18.12.1 - - numpy=1.24.2 + - nodejs=18.15.0 + - numpy=1.24.3 - numpydoc=1.5.0 - - pandas=2.0.0 - - pip=23.0 + - pandas=2.0.1 + - pip=23.1.2 - postgresql=15.2 - psycopg2=2.9.3 - - pytest=7.2.1 + - pytest=7.3.1 - pytest-cov=4.0.0 - pytest-mock=3.10.0 - python=3.9.16 - pyyaml=6.0 - redis - - scipy=1.9.3 - - setuptools=67.3.1 - - sphinx=6.1.3 + - scipy=1.10.1 + - setuptools=67.7.2 + - sphinx=6.2.1 - sphinx_rtd_theme=1.2.0 - - sqlalchemy=2.0.8 + - sqlalchemy=2.0.15 - twine=4.0.2 - wtforms=3.0.1 From 38cea5b4e9d30ff311ea435d62348054faed11d3 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Tue, 23 May 2023 10:05:34 -0400 Subject: [PATCH 335/449] Updating pip requirements --- environment_python_3.10.yml | 10 +++++----- environment_python_3.9.yml | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml index 6ef4590b0..ea1eff17e 100644 --- a/environment_python_3.10.yml +++ b/environment_python_3.10.yml @@ -57,13 +57,13 @@ dependencies: - pip: - astroquery==0.4.6 - - bandit==1.7.4 - - jwst==1.10.0 + - bandit==1.7.5 + - jwst==1.10.2 - pysiaf==0.19.1 - pysqlite3==0.5.0 - - pyvo==1.4 - - redis==4.5.1 - - selenium==4.8.0 + - pyvo==1.4.1 + - redis==4.5.5 + - selenium==4.9.1 - stdatamodels==1.3.1 - stsci_rtd_theme==1.0.0 - vine==5.0.0 diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml index c021f3858..564ecb0e6 100644 --- a/environment_python_3.9.yml +++ b/environment_python_3.9.yml @@ -57,13 +57,13 @@ dependencies: - pip: - astroquery==0.4.6 - - bandit==1.7.4 - - jwst==1.10.0 + - bandit==1.7.5 + - jwst==1.10.2 - pysiaf==0.19.1 - pysqlite3==0.5.0 - - pyvo==1.4 - - redis==4.5.1 - - selenium==4.8.0 + - pyvo==1.4.1 + - redis==4.5.5 + - selenium==4.9.1 - stdatamodels==1.3.1 - stsci_rtd_theme==1.0.0 - vine==5.0.0 From 1a289b0e3bb803505b987c7514570f8ddc9309b7 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Tue, 23 May 2023 10:51:11 -0400 Subject: [PATCH 336/449] Adding ruff and removing flake8 --- environment_python_3.10.yml | 2 +- environment_python_3.9.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml index ea1eff17e..8e9de9f20 100644 --- a/environment_python_3.10.yml +++ b/environment_python_3.10.yml @@ -28,7 +28,6 @@ dependencies: - celery=5.2.7 - cryptography=40.0.2 - django=4.2.1 - - flake8=6.0.0 - inflection=0.5.1 - ipython=8.13.2 - jinja2=3.1.2 @@ -47,6 +46,7 @@ dependencies: - python=3.10.9 - pyyaml=6.0 - redis + - ruff=0.0.269 - scipy=1.10.1 - setuptools=67.7.2 - sphinx=6.2.1 diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml index 564ecb0e6..27d1b79f9 100644 --- a/environment_python_3.9.yml +++ b/environment_python_3.9.yml @@ -28,7 +28,6 @@ dependencies: - celery=5.2.7 - cryptography=40.0.2 - django=4.2.1 - - flake8=6.0.0 - inflection=0.5.1 - ipython=8.13.2 - jinja2=3.1.2 @@ -47,6 +46,7 @@ dependencies: - python=3.9.16 - pyyaml=6.0 - redis + - ruff=0.0.269 - scipy=1.10.1 - setuptools=67.7.2 - sphinx=6.2.1 From 2acadedace5bcf859a860e3caae9c3c614f868f8 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 23 May 2023 11:35:32 -0400 Subject: [PATCH 337/449] Make step_args optional in run_pipeline.py --- jwql/shared_tasks/run_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index ba321eafa..6f4250349 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -317,7 +317,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T parser.add_argument('input_file', metavar='FILE', type=str, help=file_help) parser.add_argument('short_name', metavar='NAME', type=str, help=name_help) parser.add_argument('max_cores', metavar='CORES', type=str, help=cores_help) - parser.add_argument('step_args', metavar='STEP_ARGS', type=json.loads, help=step_args_help) + parser.add_argument('--step_args', metavar='STEP_ARGS', type=json.loads, default='{}', help=step_args_help) with open(general_status_file, "a+") as status_file: status_file.write("Created argument parser at {}\n".format(time.ctime())) From 8b3ac9ef08e2c12aa17e48c9fc21fb2ecb696c3d Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 23 May 2023 12:00:10 -0400 Subject: [PATCH 338/449] Fix pipeline step test --- jwql/tests/test_pipeline_tools.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/jwql/tests/test_pipeline_tools.py b/jwql/tests/test_pipeline_tools.py index 841cdb6d4..a79d687c8 100644 --- a/jwql/tests/test_pipeline_tools.py +++ b/jwql/tests/test_pipeline_tools.py @@ -107,9 +107,10 @@ def test_get_pipeline_steps(): # MIRI miri_req_steps = pipeline_tools.get_pipeline_steps('miri') - miri_steps = ['dq_init', 'saturation', 'firstframe', 'lastframe', - 'linearity', 'rscd', 'dark_current', 'refpix', 'jump', 'rate'] - not_required = ['group_scale', 'ipc', 'superbias', 'persistence'] + miri_steps = ['group_scale', 'dq_init', 'saturation', 'firstframe', 'lastframe', + 'reset', 'linearity', 'rscd', 'dark_current', 'refpix', 'jump', 'rate', + 'gain_scale'] + not_required = ['ipc', 'superbias', 'persistence'] miri_dict = OrderedDict({}) for step in miri_steps: miri_dict[step] = True From ac4dde8866e5dbe42e8ffaf8ccd792976dcf8227 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 23 May 2023 12:15:03 -0400 Subject: [PATCH 339/449] Clean up --- jwql/instrument_monitors/pipeline_tools.py | 3 --- jwql/shared_tasks/run_pipeline.py | 5 ----- jwql/shared_tasks/shared_tasks.py | 2 +- 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/jwql/instrument_monitors/pipeline_tools.py b/jwql/instrument_monitors/pipeline_tools.py index a5fd4a464..c39e3bda4 100644 --- a/jwql/instrument_monitors/pipeline_tools.py +++ b/jwql/instrument_monitors/pipeline_tools.py @@ -189,9 +189,6 @@ def get_pipeline_steps(instrument): required_steps = OrderedDict({}) for key in steps: required_steps[key] = True - #for key in PIPE_KEYWORDS.values(): - # if key not in required_steps.keys(): - # required_steps[key] = False return required_steps diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 6f4250349..2e34f58e3 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -68,11 +68,6 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co for step in steps: steps[step] = not completed_steps[step] - # Special case: if the input file is a dark.fits file, then we want to skip the - # dark_current subtraction step - #if 'dark.fits' in input_file: - # steps['dark_current'] = False - # Make sure we don't run steps out of order. Find the latest step that has been # run, and only run subsequent steps. This protects against cases where some early # step was not run. In that case, we don't want to go back and run it because running diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index 450405058..047084d62 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -258,7 +258,7 @@ def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_fi # Convert step_args dictionary to a string so that it can be passed via command line step_args_str = convert_step_args_to_string(step_args) - command = "{} {} {} '{}' {} {} {} {} {}" + command = "{} {} {} '{}' {} {} {} {} --step_args {}" command = command.format(name, cmd, outputs, cal_dir, ins, in_file, short_name, cores, step_args_str) logging.info("Running {}".format(command)) process = Popen(command, shell=True, executable="/bin/bash", stderr=PIPE) From 756592f474f38b0e608ebd6a17816b2664c76521 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Tue, 23 May 2023 15:11:54 -0400 Subject: [PATCH 340/449] updating pyproject.toml --- pyproject.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d941dbc56..f17cefdf9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,4 +71,6 @@ namespaces = false [tool.pytest] junit_family = "xunit2" -addopts = "--ignore=jwql/website/apps/jwql/static" + +[tool.pytest.ini_options] +norecursedirs = ["jwql/website/apps/jwql/static"] \ No newline at end of file From b35f6408af574b1bb222e80ee283f9ba132645b5 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Tue, 23 May 2023 17:17:36 -0400 Subject: [PATCH 341/449] Fix code duplication --- .../ta_monitors/wata_monitor.py | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py index 337001b4e..5850701ac 100644 --- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py +++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py @@ -702,27 +702,6 @@ def get_uncal_names(self, file_list): good_files.append(filename) return good_files - def get_uncal_names(self, file_list): - """Replace the last suffix for _uncal and return list. - Parameters - ---------- - file_list : list - List of fits files - Returns - ------- - good_files : list - Filtered list of uncal file names - """ - good_files = [] - for filename in file_list: - # Names look like: jw01133003001_02101_00001_nrs2_cal.fits - if '_uncal' not in filename: - suffix2replace = filename.split('_')[-1] - filename = filename.replace(suffix2replace, 'uncal.fits') - if filename not in good_files: - good_files.append(filename) - return good_files - def update_ta_success_txtfile(self): """Create a text file with all the failed and successful WATA. Parameters From 798691c0b066d2e4a8c0ce48399563798c35417f Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Thu, 25 May 2023 12:56:41 -0400 Subject: [PATCH 342/449] Fix for test with new versioning system --- jwql/tests/test_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/tests/test_setup.py b/jwql/tests/test_setup.py index 4cced8671..5eed720c8 100644 --- a/jwql/tests/test_setup.py +++ b/jwql/tests/test_setup.py @@ -29,4 +29,4 @@ def test_version_number(): assert isinstance(jwql.__version__, str) version_parts = jwql.__version__.split('.') - assert len(version_parts) == 3 + assert len(version_parts) >= 3 From 3caac1adccb09c5e3d9e2636b0bc846831026786 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Thu, 25 May 2023 13:33:50 -0400 Subject: [PATCH 343/449] Decouple TA monitor run from page load --- .../ta_monitors/msata_monitor.py | 28 +++++++++++++++++- .../ta_monitors/wata_monitor.py | 29 ++++++++++++++++++- jwql/website/apps/jwql/monitor_views.py | 24 +++++++-------- jwql/website/apps/jwql/static/js/jwql.js | 6 ++-- .../apps/jwql/templates/msata_monitor.html | 17 ++--------- .../apps/jwql/templates/wata_monitor.html | 17 ++--------- 6 files changed, 73 insertions(+), 48 deletions(-) diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py index d3593ffcb..b8fd7ba70 100644 --- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py +++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py @@ -1062,6 +1062,32 @@ def update_ta_success_txtfile(self): line = "{:<50} {:<50} {:<50}".format(suc, ta_inprogress[idx], ta_failure[idx]) txt.write(line + "\n") + def read_existing_html(self): + """ + This function gets the data from the Bokeh html file created with + the NIRSpec TA monitor script. + """ + self.output_dir = os.path.join(get_config()['outputs'], 'msata_monitor') + ensure_dir_exists(self.output_dir) + + self.output_file_name = os.path.join(self.output_dir, "msata_layout.html") + if not os.path.isfile(self.output_file_name): + return + + # open the html file and get the contents + with open(self.output_file_name, "r") as html_file: + contents = html_file.read() + + soup = BeautifulSoup(contents, 'html.parser').body + + # find the script elements + script1 = str(soup.find('script', type='text/javascript')) + script2 = str(soup.find('script', type='application/json')) + + # find the div element + div = str(soup.find('div', class_='bk-root')) + return div, script1, script2 + @log_fail @log_info def run(self): @@ -1164,7 +1190,7 @@ def run(self): logging.info('\t{} MSATA files were used to make plots.'.format(msata_files_used4plots)) # update the list of successful and failed TAs self.update_ta_success_txtfile() - logging.info('\t{} MSATA status file was updated') + logging.info('\tMSATA status file was updated') else: logging.info('\tMSATA monitor skipped.') diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py index 5850701ac..b330e2dfc 100644 --- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py +++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py @@ -749,6 +749,33 @@ def update_ta_success_txtfile(self): line = "{:<50} {:<50}".format(suc, ta_failure[idx]) txt.write(line + "\n") + def read_existing_html(self): + """ + This function gets the data from the Bokeh html file created with + the NIRSpec TA monitor script. + """ + self.output_dir = os.path.join(get_config()['outputs'], 'wata_monitor') + ensure_dir_exists(self.output_dir) + + self.output_file_name = os.path.join(self.output_dir, "wata_layout.html") + if not os.path.isfile(self.output_file_name): + return + + # open the html file and get the contents + with open(self.output_file_name, "r") as html_file: + contents = html_file.read() + + soup = BeautifulSoup(contents, 'html.parser').body + + # find the script elements + script1 = str(soup.find('script', type='text/javascript')) + script2 = str(soup.find('script', type='application/json')) + + # find the div element + div = str(soup.find('div', class_='bk-root')) + return div, script1, script2 + + @log_fail @log_info def run(self): @@ -849,7 +876,7 @@ def run(self): logging.info('\t{} WATA files were used to make plots.'.format(wata_files_used4plots)) # update the list of successful and failed TAs self.update_ta_success_txtfile() - logging.info('\t{} WATA status file was updated') + logging.info('\tWATA status file was updated') else: logging.info('\tWATA monitor skipped.') diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py index 23b5dafb0..b62f339de 100644 --- a/jwql/website/apps/jwql/monitor_views.py +++ b/jwql/website/apps/jwql/monitor_views.py @@ -270,15 +270,13 @@ def msata_monitoring_ajax(request): JsonResponse object Outgoing response sent to the webpage """ - # run the monitor - module = 'msata_monitor' - start_time, log_file = monitor_utils.initialize_instrument_monitor(module) + # retrieve existing monitor html content monitor = msata_monitor.MSATA() - monitor.run() - monitor_utils.update_monitor_table(module, start_time, log_file) + div, script1, script2 = monitor.read_existing_html() - context = {'script': monitor.script, - 'div': monitor.div} + context = {'script1': script1, + 'script2': script2, + 'div': div} return JsonResponse(context, json_dumps_params={'indent': 2}) @@ -321,14 +319,12 @@ def wata_monitoring_ajax(request): JsonResponse object Outgoing response sent to the webpage """ - # run the monitor - module = 'wata_monitor' - start_time, log_file = monitor_utils.initialize_instrument_monitor(module) + # retrieve existing monitor html content monitor = wata_monitor.WATA() - monitor.run() - monitor_utils.update_monitor_table(module, start_time, log_file) + div, script1, script2 = monitor.read_existing_html() - context = {'script': monitor.script, - 'div': monitor.div} + context = {'script1': script1, + 'script2': script2, + 'div': div} return JsonResponse(context, json_dumps_params={'indent': 2}) diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index f4166d475..e86dfa9c6 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -1001,7 +1001,8 @@ function update_msata_page(base_url) { // Build div content var content = data["div"]; - content += data["script"]; + content += data["script1"]; + content += data["script2"]; /* Add the content to the div * Note: + +
    +
    diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 9d5b5e02c..28b63cb34 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -478,10 +478,24 @@ def archive_thumbnails_query_ajax(request): parameters = request.session.get("query_config", QUERY_CONFIG_TEMPLATE.copy()) filtered_rootnames = get_rootnames_from_query(parameters) - data = thumbnails_query_ajax(filtered_rootnames) - data['thumbnail_sort'] = request.session.get("image_sort", "Ascending") + + paginator = Paginator(filtered_rootnames, 500) + page_number = request.GET.get("page", 1) + page_obj = paginator.get_page(page_number) + + data = thumbnails_query_ajax(page_obj.object_list) + data['thumbnail_sort'] = parameters[QUERY_CONFIG_KEYS.SORT_TYPE] data['thumbnail_group'] = request.session.get("image_group", "Exposure") + # pass pagination info + if page_obj.has_previous(): + data['previous_page'] = page_obj.previous_page_number() + data['current_page'] = page_obj.number + if page_obj.has_next(): + data['next_page'] = page_obj.next_page_number() + data['total_pages'] = paginator.num_pages + data['total_files'] = paginator.count + save_page_navigation_data(request, data) return JsonResponse(data, json_dumps_params={'indent': 2}) @@ -834,11 +848,12 @@ def query_submit(request): template = 'query_submit.html' sort_type = request.session.get('image_sort', 'Ascending') group_type = request.session.get('image_group', 'Exposure') + page_number = request.GET.get("page", 1) context = {'inst': '', 'base_url': get_base_url(), 'sort': sort_type, - 'group': group_type - } + 'group': group_type, + 'page': page_number} return render(request, template, context) From b3956ab12abea3a852c50410313bdb911d559635 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Tue, 6 Jun 2023 10:27:05 -0400 Subject: [PATCH 370/449] Add results per page configuration to query --- jwql/utils/constants.py | 2 ++ jwql/website/apps/jwql/forms.py | 6 ++++++ jwql/website/apps/jwql/static/js/jwql.js | 2 +- jwql/website/apps/jwql/templates/jwql_query.html | 15 ++++++++++++--- jwql/website/apps/jwql/views.py | 3 ++- 5 files changed, 23 insertions(+), 5 deletions(-) diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index 6048dcf8e..ced832ddf 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -443,6 +443,7 @@ class QUERY_CONFIG_KEYS: GRATINGS = "GRATINGS" LOOK_STATUS = "LOOK_STATUS" INSTRUMENTS = "INSTRUMENTS" + NUM_PER_PAGE = "NUM_PER_PAGE" PUPILS = "PUPILS" READ_PATTS = "READ_PATTS" SORT_TYPE = "SORT_TYPE" @@ -460,6 +461,7 @@ class QUERY_CONFIG_KEYS: QUERY_CONFIG_KEYS.FILTERS: {}, QUERY_CONFIG_KEYS.GRATINGS: {}, QUERY_CONFIG_KEYS.INSTRUMENTS: [], + QUERY_CONFIG_KEYS.NUM_PER_PAGE: 100, QUERY_CONFIG_KEYS.PUPILS: {}, QUERY_CONFIG_KEYS.READ_PATTS: {}, QUERY_CONFIG_KEYS.SORT_TYPE: 'Recent', diff --git a/jwql/website/apps/jwql/forms.py b/jwql/website/apps/jwql/forms.py index 6b875b0e1..1163dafc5 100644 --- a/jwql/website/apps/jwql/forms.py +++ b/jwql/website/apps/jwql/forms.py @@ -154,6 +154,12 @@ class JwqlQueryForm(BaseForm): choices=sort_choices, initial=sort_choices[2], widget=forms.RadioSelect) + num_choices = [(50, 50), (100, 100), (200, 200), (500, 500)] + num_per_page = forms.ChoiceField( + required=True, + choices=num_choices, initial=num_choices[1], + widget=forms.RadioSelect) + # instrument specific parameters miri_aper = forms.MultipleChoiceField(required=False, choices=params['miri']['aperture_list'], widget=forms.CheckboxSelectMultiple) nirspec_aper = forms.MultipleChoiceField(required=False, choices=params['nirspec']['aperture_list'], widget=forms.CheckboxSelectMultiple) diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index 731b2ebe0..939aab5b9 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -1244,7 +1244,7 @@ function update_pagination(data) { $("#pagination")[0].innerHTML = content; // Add the total file count to the img_show_count banner - $("#query_total")[0].innerHTML = '

    Query returned ' + data.total_files + ' files total.

    '; + $("#query_total")[0].innerHTML = '

    Query returned ' + data.total_files + ' activities total.

    '; } /** diff --git a/jwql/website/apps/jwql/templates/jwql_query.html b/jwql/website/apps/jwql/templates/jwql_query.html index 9a8d3e724..2cd418947 100644 --- a/jwql/website/apps/jwql/templates/jwql_query.html +++ b/jwql/website/apps/jwql/templates/jwql_query.html @@ -61,10 +61,9 @@

    Dynamic Query Form

    -

    Use this form to query the archive for any instrument or combination of instruments. - Enter the instruments for which you would like to view the archive and other constraints. +

    Use this form to query the archive for data from any instrument or combination of instruments.

    +

    Select at least one instrument, then any other constraints desired. Feel free to leave fields blank if you do not want to further constrain the database search. - Note, however, that it may increase the run time if you search a wider array of images.

    @@ -116,6 +115,16 @@

    Dynamic Query Form

    {% endfor %}
    +
    + Results Per Page +
    + {% for option in form.num_per_page %} +
    + {{ option }} +
    + {% endfor %} +
    +
    diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 28b63cb34..396b0b036 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -479,7 +479,8 @@ def archive_thumbnails_query_ajax(request): parameters = request.session.get("query_config", QUERY_CONFIG_TEMPLATE.copy()) filtered_rootnames = get_rootnames_from_query(parameters) - paginator = Paginator(filtered_rootnames, 500) + paginator = Paginator(filtered_rootnames, + parameters[QUERY_CONFIG_KEYS.NUM_PER_PAGE]) page_number = request.GET.get("page", 1) page_obj = paginator.get_page(page_number) From c65fc63b731e7eba706323d87eed5cc8cf9a73a0 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Tue, 6 Jun 2023 11:06:08 -0400 Subject: [PATCH 371/449] Link query sort preference to session sort preference --- jwql/website/apps/jwql/data_containers.py | 8 ++++---- jwql/website/apps/jwql/static/js/jwql.js | 9 ++++----- jwql/website/apps/jwql/templates/query_submit.html | 2 +- jwql/website/apps/jwql/views.py | 14 ++++++++------ 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index d925285a1..a3a741847 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -1495,8 +1495,8 @@ def get_rootnames_from_query(parameters): Returns ------- - filtered_rootnames : query_set - A query_set of all rootfileinfos filtered from the given parameters + filtered_rootnames : list + A list of all root filenames filtered from the given parameters """ # TODO - This code setup is temporary until the merge to Postgres is complete. # selected_rootfileinfos = RootFileInfo.objects.none() @@ -1572,10 +1572,10 @@ def get_rootnames_from_query(parameters): else: current_ins_rootfileinfos = current_ins_rootfileinfos.order_by('-root_name') - rootnames = [name[0] for name in current_ins_rootfileinfos.values_list("root_name")] + rootnames = [name[0] for name in current_ins_rootfileinfos.values_list('root_name')] filtered_rootnames.extend(rootnames) - return list(set(filtered_rootnames)) + return filtered_rootnames # TODO - BELOW IS THE OUTLINE OF CODE WE WANT TO USE, HOWEVER THIS CAN'T BE IMPLEMENTED WITH DJANGO RUNNING SQLITE # ONCE WE MIGRATE TO POSTGRES WE CAN IMPLEMENT THE BELOW FUNCTIONALITY WHICH SHOULD MAKE THIS CODE MOVE A LITTLE FASTER diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index 939aab5b9..9069b7bba 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -1449,10 +1449,9 @@ function update_thumbnails_per_observation_page(inst, proposal, observation, bas /** * Updates various components on the thumbnails anomaly query page * @param {String} base_url - The base URL for gathering data from the AJAX view. - * @param {String} sort - Sort method string saved in session data image_sort * @param {Int} page - Page number to load */ -function update_thumbnails_query_page(base_url, sort, group, page) { +function update_thumbnails_query_page(base_url, page) { $.ajax({ url: base_url + '/ajax/query_submit/?page=' + page, success: function(data){ @@ -1462,12 +1461,12 @@ function update_thumbnails_query_page(base_url, sort, group, page) { update_thumbnail_array(data); update_filter_options(data, base_url, 'thumbnail'); update_group_options(data, base_url); - update_sort_options(data, base_url); + update_sort_options(data.thumbnail_sort, base_url); update_pagination(data); // Do initial sort and group to match sort button display - group_by_thumbnails(group, base_url); - sort_by_thumbnails(sort, base_url); + group_by_thumbnails(data.thumbnail_group, base_url); + sort_by_thumbnails(data.thumbnail_sort, base_url); // Replace loading screen with the proposal array div document.getElementById("loading").style.display = "none"; diff --git a/jwql/website/apps/jwql/templates/query_submit.html b/jwql/website/apps/jwql/templates/query_submit.html index 1f349843c..0810aee00 100644 --- a/jwql/website/apps/jwql/templates/query_submit.html +++ b/jwql/website/apps/jwql/templates/query_submit.html @@ -55,7 +55,7 @@

    Images of Queried Instruments

    - +
    diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 396b0b036..b874ed89d 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -94,6 +94,7 @@ def jwql_query(request): """The anomaly query form page""" form = JwqlQueryForm(request.POST or None) + form.fields['sort_type'].initial = request.session.get('image_sort', 'Recent') if request.method == 'POST': if form.is_valid(): @@ -399,7 +400,7 @@ def archive_thumbnails_ajax(request, inst, proposal, observation=None): inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()] data = thumbnails_ajax(inst, proposal, obs_num=observation) - data['thumbnail_sort'] = request.session.get("image_sort", "Ascending") + data['thumbnail_sort'] = request.session.get("image_sort", "Recent") data['thumbnail_group'] = request.session.get("image_group", "Exposure") save_page_navigation_data(request, data) @@ -443,7 +444,7 @@ def archive_thumbnails_per_observation(request, inst, proposal, observation): obs_list = sorted(list(set(all_obs))) - sort_type = request.session.get('image_sort', 'Ascending') + sort_type = request.session.get('image_sort', 'Recent') group_type = request.session.get('image_group', 'Exposure') template = 'thumbnails_per_obs.html' context = {'base_url': get_base_url(), @@ -497,12 +498,13 @@ def archive_thumbnails_query_ajax(request): data['total_pages'] = paginator.num_pages data['total_files'] = paginator.count + request.session['image_sort'] = parameters[QUERY_CONFIG_KEYS.SORT_TYPE] save_page_navigation_data(request, data) return JsonResponse(data, json_dumps_params={'indent': 2}) def dashboard(request): - """Generate the dashbaord page + """Generate the dashboard page Parameters ---------- @@ -847,7 +849,7 @@ def query_submit(request): """ template = 'query_submit.html' - sort_type = request.session.get('image_sort', 'Ascending') + sort_type = request.session.get('image_sort', 'Recent') group_type = request.session.get('image_group', 'Exposure') page_number = request.GET.get("page", 1) context = {'inst': '', @@ -1225,7 +1227,7 @@ def view_exposure(request, inst, group_root): # For time based sorting options, sort to "Recent" first to create sorting consistency when times are the same. # This is consistent with how Tinysort is utilized in jwql.js->sort_by_thumbnails - sort_type = request.session.get('image_sort', 'Ascending') + sort_type = request.session.get('image_sort', 'Recent') if sort_type in ['Descending']: matching_rootfiles = sorted(navigation_data, reverse=True) elif sort_type in ['Recent']: @@ -1323,7 +1325,7 @@ def view_image(request, inst, file_root): # sorting consistency when times are the same. # This is consistent with how Tinysort is utilized in # jwql.js->sort_by_thumbnails - sort_type = request.session.get('image_sort', 'Ascending') + sort_type = request.session.get('image_sort', 'Recent') if sort_type in ['Descending']: file_root_list = sorted(navigation_data, reverse=True) elif sort_type in ['Recent']: From 4dae5a7a03c6f5af9a5b9a8888112cfa48710984 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Tue, 6 Jun 2023 11:24:27 -0400 Subject: [PATCH 372/449] Tidy pagination style --- jwql/website/apps/jwql/static/js/jwql.js | 12 ++++++------ jwql/website/apps/jwql/templates/query_submit.html | 1 + 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index 9069b7bba..d5e63a393 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -1229,16 +1229,16 @@ function update_obs_options(data, inst, prop, observation) { * @param {Object} data - The data returned by the query_submit AJAX method */ function update_pagination(data) { - var content = '' + var content = '' if ('previous_page' in data) { - content += '« first ' + - 'previous'; + content += '« first ' + + 'previous'; } - content += 'Page ' + data.current_page + ' of ' + data.total_pages + '' + content += 'Page ' + data.current_page + ' of ' + data.total_pages + '' if ('next_page' in data) { - content += 'next ' + - 'last »'; + content += 'next ' + + 'last »'; } content += ''; $("#pagination")[0].innerHTML = content; diff --git a/jwql/website/apps/jwql/templates/query_submit.html b/jwql/website/apps/jwql/templates/query_submit.html index 0810aee00..585e55111 100644 --- a/jwql/website/apps/jwql/templates/query_submit.html +++ b/jwql/website/apps/jwql/templates/query_submit.html @@ -54,6 +54,7 @@

    Images of Queried Instruments

    +
    From e165c1ca93b1767943ed059bb487c7f56ca3571a Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Tue, 6 Jun 2023 14:04:26 -0400 Subject: [PATCH 373/449] Add summary text for top-level query params --- jwql/utils/constants.py | 4 ++-- jwql/website/apps/jwql/data_containers.py | 6 +++--- jwql/website/apps/jwql/forms.py | 2 +- jwql/website/apps/jwql/static/js/jwql.js | 11 +++++++++-- jwql/website/apps/jwql/templates/jwql_query.html | 4 ++-- jwql/website/apps/jwql/views.py | 12 +++++++++++- 6 files changed, 28 insertions(+), 11 deletions(-) diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index ced832ddf..9e26a155a 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -436,7 +436,7 @@ class QUERY_CONFIG_KEYS: ANOMALIES = "ANOMALIES" APERTURES = "APERTURES" - CAT_TYPE = "CAT_TYPE" + PROPOSAL_CATEGORY = "PROPOSAL_CATEGORY" DETECTORS = "DETECTORS" EXP_TYPES = "EXP_TYPES" FILTERS = "FILTERS" @@ -455,7 +455,7 @@ class QUERY_CONFIG_KEYS: QUERY_CONFIG_TEMPLATE = { QUERY_CONFIG_KEYS.ANOMALIES: {}, QUERY_CONFIG_KEYS.APERTURES: {}, - QUERY_CONFIG_KEYS.CAT_TYPE: [], + QUERY_CONFIG_KEYS.PROPOSAL_CATEGORY: [], QUERY_CONFIG_KEYS.DETECTORS: {}, QUERY_CONFIG_KEYS.EXP_TYPES: {}, QUERY_CONFIG_KEYS.FILTERS: {}, diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index a3a741847..c3828b11a 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -1513,9 +1513,9 @@ def get_rootnames_from_query(parameters): if len(look_status) == 1: viewed = (look_status[0] == 'VIEWED') current_ins_rootfileinfos = current_ins_rootfileinfos.filter(viewed=viewed) - cat_type = parameters[QUERY_CONFIG_KEYS.CAT_TYPE] - if len(cat_type) > 0: - current_ins_rootfileinfos = current_ins_rootfileinfos.filter(obsnum__proposal__category__in=cat_type) + proposal_category = parameters[QUERY_CONFIG_KEYS.PROPOSAL_CATEGORY] + if len(proposal_category) > 0: + current_ins_rootfileinfos = current_ins_rootfileinfos.filter(obsnum__proposal__category__in=proposal_category) # Instrument fields inst_anomalies = parameters[QUERY_CONFIG_KEYS.ANOMALIES][inst] diff --git a/jwql/website/apps/jwql/forms.py b/jwql/website/apps/jwql/forms.py index 1163dafc5..b0554b451 100644 --- a/jwql/website/apps/jwql/forms.py +++ b/jwql/website/apps/jwql/forms.py @@ -145,7 +145,7 @@ class JwqlQueryForm(BaseForm): required=False, choices=look_choices, widget=forms.CheckboxSelectMultiple) cat_choices = [(query_format(choice), query_format(choice)) for choice in PROPOSAL_CATEGORIES] - cat_type = forms.MultipleChoiceField( + proposal_category = forms.MultipleChoiceField( required=False, choices=cat_choices, widget=forms.CheckboxSelectMultiple) sort_choices = [(choice, choice) for choice in SORT_OPTIONS] diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index d5e63a393..173aae822 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -1244,7 +1244,14 @@ function update_pagination(data) { $("#pagination")[0].innerHTML = content; // Add the total file count to the img_show_count banner - $("#query_total")[0].innerHTML = '

    Query returned ' + data.total_files + ' activities total.

    '; + var query_summary = '


    Query returned ' + data.total_files + ' activities total with parameters:

      '; + for (var i = 0; i < Object.keys(data.query_config).length; i++) { + var param = Object.keys(data.query_config)[i]; + query_summary += '
    • ' + param.toUpperCase() + '=' + + data.query_config[param].toString().toUpperCase() + '
      • '; + } + query_summary += '

      '; + $("#query_summary")[0].innerHTML = query_summary; } /** @@ -1256,7 +1263,7 @@ function update_show_count(count, type) { var content = 'Showing ' + count + ' / ' + count + ' ' + type + ''; content += ''; content += 'i'; - content += ''; + content += ''; $("#img_show_count")[0].innerHTML = content; } diff --git a/jwql/website/apps/jwql/templates/jwql_query.html b/jwql/website/apps/jwql/templates/jwql_query.html index 2cd418947..4cd308acd 100644 --- a/jwql/website/apps/jwql/templates/jwql_query.html +++ b/jwql/website/apps/jwql/templates/jwql_query.html @@ -87,8 +87,8 @@

      Dynamic Query Form

    Proposal Category -
    - {% for option in form.cat_type %} +
    + {% for option in form.proposal_category %}
    {{ option }}
    diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index b874ed89d..d073ec766 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -126,7 +126,7 @@ def jwql_query(request): parameters = QUERY_CONFIG_TEMPLATE.copy() parameters[QUERY_CONFIG_KEYS.INSTRUMENTS] = form.cleaned_data['instrument'] parameters[QUERY_CONFIG_KEYS.LOOK_STATUS] = form.cleaned_data['look_status'] - parameters[QUERY_CONFIG_KEYS.CAT_TYPE] = form.cleaned_data['cat_type'] + parameters[QUERY_CONFIG_KEYS.PROPOSAL_CATEGORY] = form.cleaned_data['proposal_category'] parameters[QUERY_CONFIG_KEYS.SORT_TYPE] = form.cleaned_data['sort_type'] parameters[QUERY_CONFIG_KEYS.ANOMALIES] = all_anomalies parameters[QUERY_CONFIG_KEYS.APERTURES] = all_apers @@ -489,6 +489,16 @@ def archive_thumbnails_query_ajax(request): data['thumbnail_sort'] = parameters[QUERY_CONFIG_KEYS.SORT_TYPE] data['thumbnail_group'] = request.session.get("image_group", "Exposure") + # add top level parameters for summarizing + summary_keys = [QUERY_CONFIG_KEYS.INSTRUMENTS, + QUERY_CONFIG_KEYS.SORT_TYPE, QUERY_CONFIG_KEYS.LOOK_STATUS, + QUERY_CONFIG_KEYS.PROPOSAL_CATEGORY] + data['query_config'] = {} + for key in summary_keys: + value = parameters[key] + if value: + data['query_config'][key] = value + # pass pagination info if page_obj.has_previous(): data['previous_page'] = page_obj.previous_page_number() From f181f0a543c31a830b137f3c7a0676c3e13a8f17 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Tue, 6 Jun 2023 15:48:48 -0400 Subject: [PATCH 374/449] Fix for bad test PID --- jwql/tests/test_protect_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/tests/test_protect_module.py b/jwql/tests/test_protect_module.py index d0da6ab79..3bcafa5c1 100644 --- a/jwql/tests/test_protect_module.py +++ b/jwql/tests/test_protect_module.py @@ -97,7 +97,7 @@ def test_lock_module_handles_stale_lock_files(module_lock, do_not_email): delete it, and continue to run successfully """ # create locked file with not running PID in advance of calling protected code with open(module_lock, "w") as lock_file: - lock_file.write(f"{_PID_LOCKFILE_KEY}12345\n") + lock_file.write(f"{_PID_LOCKFILE_KEY}-9999\n") file_created = protected_code_verify_file_exists_true(module_lock) file_exists = os.path.exists(module_lock) # Assert that lock file was created in wrapper, and removed upon exit From cd0c93a9e36923df4f37b7813237584c31701af5 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Tue, 6 Jun 2023 15:55:18 -0400 Subject: [PATCH 375/449] Fix argument to update_sort_options --- jwql/website/apps/jwql/static/js/jwql.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index 173aae822..73f33eae5 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -1468,7 +1468,7 @@ function update_thumbnails_query_page(base_url, page) { update_thumbnail_array(data); update_filter_options(data, base_url, 'thumbnail'); update_group_options(data, base_url); - update_sort_options(data.thumbnail_sort, base_url); + update_sort_options(data, base_url); update_pagination(data); // Do initial sort and group to match sort button display From a9229ca940f9c9836a2c870b0144c3ffe71cda5b Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Wed, 7 Jun 2023 10:23:49 -0400 Subject: [PATCH 376/449] Add all parameters to query summary --- jwql/utils/constants.py | 27 ++++++++++++++------------- jwql/website/apps/jwql/views.py | 12 +++++++----- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index 9e26a155a..dee4123d2 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -433,41 +433,42 @@ # Keep keys defined via class as they are used many places with potential mispellings +# Keys are in sort order from general to instrument specific, then alphabetical +# within instrument specific fields. class QUERY_CONFIG_KEYS: + INSTRUMENTS = "INSTRUMENTS" + PROPOSAL_CATEGORY = "PROPOSAL_CATEGORY" + LOOK_STATUS = "LOOK_STATUS" + NUM_PER_PAGE = "NUM_PER_PAGE" + SORT_TYPE = "SORT_TYPE" ANOMALIES = "ANOMALIES" APERTURES = "APERTURES" - PROPOSAL_CATEGORY = "PROPOSAL_CATEGORY" DETECTORS = "DETECTORS" EXP_TYPES = "EXP_TYPES" FILTERS = "FILTERS" GRATINGS = "GRATINGS" - LOOK_STATUS = "LOOK_STATUS" - INSTRUMENTS = "INSTRUMENTS" - NUM_PER_PAGE = "NUM_PER_PAGE" PUPILS = "PUPILS" READ_PATTS = "READ_PATTS" - SORT_TYPE = "SORT_TYPE" SUBARRAYS = "SUBARRAYS" - THUMBNAILS = "THUMBNAILS" # Template for parameters to be stored in "query_config" session for query_page QUERY_CONFIG_TEMPLATE = { + QUERY_CONFIG_KEYS.INSTRUMENTS: [], + QUERY_CONFIG_KEYS.PROPOSAL_CATEGORY: [], + QUERY_CONFIG_KEYS.LOOK_STATUS: [], + QUERY_CONFIG_KEYS.NUM_PER_PAGE: 100, + QUERY_CONFIG_KEYS.SORT_TYPE: 'Recent', QUERY_CONFIG_KEYS.ANOMALIES: {}, QUERY_CONFIG_KEYS.APERTURES: {}, - QUERY_CONFIG_KEYS.PROPOSAL_CATEGORY: [], QUERY_CONFIG_KEYS.DETECTORS: {}, QUERY_CONFIG_KEYS.EXP_TYPES: {}, QUERY_CONFIG_KEYS.FILTERS: {}, QUERY_CONFIG_KEYS.GRATINGS: {}, - QUERY_CONFIG_KEYS.INSTRUMENTS: [], - QUERY_CONFIG_KEYS.NUM_PER_PAGE: 100, QUERY_CONFIG_KEYS.PUPILS: {}, QUERY_CONFIG_KEYS.READ_PATTS: {}, - QUERY_CONFIG_KEYS.SORT_TYPE: 'Recent', - QUERY_CONFIG_KEYS.SUBARRAYS: {}, - QUERY_CONFIG_KEYS.THUMBNAILS: [] - } + QUERY_CONFIG_KEYS.SUBARRAYS: {} +} # RAPID-style readout patterns for each instrument. Added so we can # differentiate in MAST searches for e.g. the dark current monitor diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index d073ec766..78b09cdba 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -490,13 +490,15 @@ def archive_thumbnails_query_ajax(request): data['thumbnail_group'] = request.session.get("image_group", "Exposure") # add top level parameters for summarizing - summary_keys = [QUERY_CONFIG_KEYS.INSTRUMENTS, - QUERY_CONFIG_KEYS.SORT_TYPE, QUERY_CONFIG_KEYS.LOOK_STATUS, - QUERY_CONFIG_KEYS.PROPOSAL_CATEGORY] data['query_config'] = {} - for key in summary_keys: + for key in parameters: value = parameters[key] - if value: + if isinstance(value, dict): + for subkey in value: + subvalue = value[subkey] + if subvalue: + data['query_config'][f'{key}_{subkey}'] = subvalue + elif value: data['query_config'][key] = value # pass pagination info From b2e0d80805d6dc39948ccd3e1244d0dc37f23580 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Wed, 7 Jun 2023 16:07:13 -0400 Subject: [PATCH 377/449] Fix aspect ratio, default to linear zscale --- jwql/utils/interactive_preview_image.py | 44 ++++++++++++++++--- .../apps/jwql/templates/explore_image.html | 8 ++-- jwql/website/apps/jwql/views.py | 4 +- 3 files changed, 43 insertions(+), 13 deletions(-) diff --git a/jwql/utils/interactive_preview_image.py b/jwql/utils/interactive_preview_image.py index 01c99e808..6ea0c1e9e 100644 --- a/jwql/utils/interactive_preview_image.py +++ b/jwql/utils/interactive_preview_image.py @@ -23,7 +23,9 @@ from astropy.visualization import ZScaleInterval import numpy as np from bokeh.embed import components -from bokeh.models import BasicTicker, ColorBar, HoverTool, LinearColorMapper, LogColorMapper, LogTicker +from bokeh.models import ( + BasicTicker, BoxZoomTool, ColorBar, HoverTool, LinearColorMapper, LogColorMapper, + LogTicker, WheelZoomTool) from bokeh.plotting import figure, output_file, show, save from jwst.datamodels import dqflags @@ -80,9 +82,9 @@ def __init__(self, filename, low_lim=None, high_lim=None, scaling='log', contras self.save_html = save_html # Allow sending in of None without overriding defaults - if(group is None): + if group is None: group = -1 - if(integ is None): + if integ is None: integ = 0 # Determine the min and max values to use for the display @@ -107,13 +109,13 @@ def create_bokeh_image(self): """Method to create the figure """ limits = self.get_scale() - if limits[0] <= 0: - limits = (1e-4, limits[1]) if self.low_lim is not None: limits = (self.low_lim, limits[1]) if self.high_lim is not None: limits = (limits[0], self.high_lim) if self.scaling == 'log': + if limits[0] <= 0: + limits = (1e-4, limits[1]) color_mapper = LogColorMapper( palette="Viridis256", low=limits[0], high=limits[1]) ticker = LogTicker() @@ -128,13 +130,41 @@ def create_bokeh_image(self): if not self.show and self.save_html is not None: output_file(filename=self.save_html, title=os.path.basename(self.filename)) - fig = figure(tools='pan,box_zoom,reset,wheel_zoom,save') + + # fix figure aspect from data aspect + if xd > yd: + plot_width = 800 + plot_height = int(plot_width * yd / xd) + if plot_height < 400: + plot_height = 400 + else: + plot_height = 800 + plot_width = int(plot_height * xd / yd) + if plot_width < 400: + plot_width = 400 + + fig = figure(tools='pan,reset,save', match_aspect=True, + plot_width=plot_width, plot_height=plot_height) + fig.add_tools(BoxZoomTool(match_aspect=True)) + fig.add_tools(WheelZoomTool(zoom_on_axis=False)) + img = fig.image(source=info, image='image', level="image", color_mapper=color_mapper) - color_bar = ColorBar(color_mapper=color_mapper, label_standoff=12, ticker=ticker, title=self.signal_units, bar_line_color='black', + color_bar = ColorBar(color_mapper=color_mapper, label_standoff=12, ticker=ticker, + title=self.signal_units, bar_line_color='black', minor_tick_line_color='black', major_tick_line_color='black') fig.add_layout(color_bar, 'below') + fig.x_range.range_padding = fig.y_range.range_padding = 0 + if xd >= yd: + fig.x_range.start = 0 + fig.x_range.end = xd + fig.x_range.bounds = (0, xd) + if yd >= xd: + fig.y_range.start = 0 + fig.y_range.end = yd + fig.y_range.bounds = (0, yd) + if 'DQ' not in self.extname: hover_tool = HoverTool(tooltips=[("(x,y)", "($x{0.2f}, $y{0.2f})"), ('Value', '@image{0.4f}') diff --git a/jwql/website/apps/jwql/templates/explore_image.html b/jwql/website/apps/jwql/templates/explore_image.html index 205bffc2b..cced80f37 100644 --- a/jwql/website/apps/jwql/templates/explore_image.html +++ b/jwql/website/apps/jwql/templates/explore_image.html @@ -21,10 +21,10 @@

    {{ file_root }}_{{ filetype }}.fits

    Image Settings

    Scaling:   - log      - linear    + log      + linear   
    - Z-Scale Limits:   + Scale Limits:    Low       High     
    @@ -129,4 +129,4 @@
    Submit Anomaly
    -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 8b4712a90..b13599f00 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -892,7 +892,7 @@ def view_header(request, inst, filename, filetype): def explore_image(request, inst, file_root, filetype): - """Generate the header view page + """Generate the explore image page. Parameters ---------- @@ -958,7 +958,7 @@ def explore_image(request, inst, file_root, filetype): return render(request, template, context) -def explore_image_ajax(request, inst, file_root, filetype, scaling="log", low_lim=None, high_lim=None, ext_name="SCI", int1_nr=None, grp1_nr=None, int2_nr=None, grp2_nr=None): +def explore_image_ajax(request, inst, file_root, filetype, scaling="lin", low_lim=None, high_lim=None, ext_name="SCI", int1_nr=None, grp1_nr=None, int2_nr=None, grp2_nr=None): """Generate the page listing all archived images in the database for a certain proposal From 90b04d5e7ef340a66b13da3ef3f079b62ab2bd4e Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Thu, 8 Jun 2023 16:11:39 -0400 Subject: [PATCH 378/449] Move scaling and limit controls to client side --- .../ta_monitors/msata_monitor.py | 322 +++++++++++++----- .../ta_monitors/wata_monitor.py | 232 +++++++++---- jwql/utils/interactive_preview_image.py | 176 ++++++++-- jwql/website/apps/jwql/static/css/jwql.css | 19 +- jwql/website/apps/jwql/static/js/jwql.js | 5 +- .../apps/jwql/templates/explore_image.html | 93 +++-- jwql/website/apps/jwql/urls.py | 2 +- 7 files changed, 607 insertions(+), 242 deletions(-) diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py index bc21a4643..55abcf95e 100755 --- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py +++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py @@ -24,6 +24,7 @@ Author ______ - Maria Pena-Guerrero + - Melanie Clarke Use --- @@ -38,6 +39,7 @@ import os import logging import shutil +from datetime import datetime, timezone, timedelta from random import randint import numpy as np @@ -45,12 +47,13 @@ from astropy.time import Time from astropy.io import fits from bokeh.embed import components -from bokeh.plotting import figure, output_file, save -from bokeh.layouts import gridplot -from bokeh.models import ColumnDataSource, Range1d, Span, Label -from bokeh.models.tools import HoverTool +from bokeh.layouts import gridplot, layout +from bokeh.models import ( + ColumnDataSource, Range1d, CustomJS, CustomJSFilter, CDSView, + Span, Label, DateRangeSlider) +from bokeh.models.tools import HoverTool, BoxSelectTool +from bokeh.plotting import figure, save, output_file from bs4 import BeautifulSoup -from datetime import datetime from sqlalchemy.sql.expression import and_ # jwql imports @@ -59,11 +62,11 @@ from jwql.utils import monitor_utils from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE from jwql.utils.logging_functions import log_info, log_fail -from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config, filename_parser +from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config class MSATA(): - """ Class for executint the NIRSpec MSATA monitor. + """ Class for executing the NIRSpec MSATA monitor. This class will search for new MSATA current files in the file systems for NIRSpec and will run the monitor on these files. The monitor will @@ -136,6 +139,13 @@ def __init__(self): 'planned_v3': {'loc': 'ta_table', 'alt_key': None, 'name': 'planned_v3', 'type': float}, 'FITTARGS': {'loc': 'ta_hdr', 'alt_key': None, 'name': 'stars_in_fit', 'type': int}} + # initialize attributes to be set later + self.source = None + self.share_tools = [] + self.date_range = None + self.date_filter = None + self.date_view = None + def get_tainfo_from_fits(self, fits_file): """ Get the TA information from the fits file Parameters @@ -240,6 +250,19 @@ def get_msata_data(self, new_filenames): msata_df = pd.DataFrame(msata_dict) return msata_df, no_ta_ext_msgs + def add_time_column(self): + """Add time column to data source, to be used by all plots.""" + date_obs = self.source.data['date_obs'] + if 'time_arr' not in self.source.data: + time_arr = [] + for do_str in date_obs: + # convert time string into an array of time (this is in UT) + t = datetime.fromisoformat(do_str) + time_arr.append(t) + + # add to the bokeh data structure + self.source.data["time_arr"] = time_arr + def plt_status(self): """ Plot the MSATA status versus time. Parameters @@ -249,18 +272,12 @@ def plt_status(self): ------- plot: bokeh plot object """ - # ta_status, date_obs = source.data['ta_status'], source.data['date_obs'] - date_obs = self.source.data['date_obs'] ta_status = self.source.data['ta_status'] # check if this column exists in the data already (the other 2 will exist too), else create it - try: - time_arr = self.source.data['time_arr'] - bool_status = self.source.data['bool_status'] - status_colors = self.source.data['status_colors'] - except KeyError: - # bokeh does not like to plot strings, turn into numbers - number_status, time_arr, status_colors = [], [], [] - for tas, do_str in zip(ta_status, date_obs): + if 'bool_status' not in self.source.data: + # bokeh does not like to plot strings, turn into numbers + number_status, status_colors = [], [] + for tas in ta_status: if tas.lower() == 'unsuccessful': number_status.append(0.0) status_colors.append('red') @@ -270,19 +287,19 @@ def plt_status(self): else: number_status.append(1.0) status_colors.append('blue') - # convert time string into an array of time (this is in UT with 0.0 milliseconds) - t = datetime.fromisoformat(do_str) - time_arr.append(t) + # add these to the bokeh data structure - self.source.data["time_arr"] = time_arr self.source.data["number_status"] = number_status self.source.data["status_colors"] = status_colors + # create a new bokeh plot plot = figure(title="MSATA Status [Success=1, In Progress=0.5, Fail=0]", x_axis_label='Time', y_axis_label='MSATA Status', x_axis_type='datetime',) plot.y_range = Range1d(-0.5, 1.5) plot.circle(x='time_arr', y='number_status', source=self.source, - color='status_colors', size=7, fill_alpha=0.3) + color='status_colors', size=7, fill_alpha=0.3, view=self.date_view) + + # add tooltips hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -293,8 +310,11 @@ def plt_status(self): ('Date-Obs', '@date_obs'), ('Subarray', '@subarray'), ('--------', '----------------')] - plot.add_tools(hover) + + # add shared selection tools + for tool in self.share_tools: + plot.add_tools(tool) return plot def plt_residual_offsets(self): @@ -311,7 +331,8 @@ def plt_residual_offsets(self): x_axis_label='Least Squares Residual V2 Offset', y_axis_label='Least Squares Residual V3 Offset') plot.circle(x='lsv2offset', y='lsv3offset', source=self.source, - color="blue", size=7, fill_alpha=0.3) + color="blue", size=7, fill_alpha=0.3, view=self.date_view) + v2halffacet, v3halffacet = self.source.data['v2halffacet'], self.source.data['v3halffacet'] xstart, ystart, ray_length = -1 * v2halffacet[0], -1 * v3halffacet[0], 0.05 plot.ray(x=xstart - ray_length / 2.0, y=ystart, length=ray_length, angle_units="deg", @@ -322,10 +343,13 @@ def plt_residual_offsets(self): plot.add_layout(hflabel) plot.x_range = Range1d(-0.5, 0.5) plot.y_range = Range1d(-0.5, 0.5) + # mark origin lines vline = Span(location=0, dimension='height', line_color='black', line_width=0.7) hline = Span(location=0, dimension='width', line_color='black', line_width=0.7) plot.renderers.extend([vline, hline]) + + # add tooltips hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -338,8 +362,11 @@ def plt_residual_offsets(self): ('LS V2 offset', '@lsv2offset'), ('LS V3 offset', '@lsv3offset'), ('--------', '----------------')] - plot.add_tools(hover) + + # add shared selection tools + for tool in self.share_tools: + plot.add_tools(tool) return plot def plt_v2offset_time(self): @@ -355,8 +382,9 @@ def plt_v2offset_time(self): plot = figure(title="MSATA Least Squares V2 Offset vs Time", x_axis_label='Time', y_axis_label='Least Squares Residual V2 Offset', x_axis_type='datetime') plot.circle(x='time_arr', y='lsv2offset', source=self.source, - color="blue", size=7, fill_alpha=0.3) + color="blue", size=7, fill_alpha=0.3, view=self.date_view) plot.y_range = Range1d(-0.5, 0.5) + # mark origin line hline = Span(location=0, dimension='width', line_color='black', line_width=0.7) time_arr, v2halffacet = self.source.data['time_arr'], self.source.data['v2halffacet'] @@ -364,6 +392,8 @@ def plt_v2offset_time(self): plot.renderers.extend([hline, hfline]) hflabel = Label(x=time_arr[-1], y=-1 * v2halffacet[0], y_units='data', text='-V2 half-facet value') plot.add_layout(hflabel) + + # add tooltips hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -376,8 +406,11 @@ def plt_v2offset_time(self): ('LS V2 offset', '@lsv2offset'), ('LS V3 offset', '@lsv3offset'), ('--------', '----------------')] - plot.add_tools(hover) + + # add shared selection tools + for tool in self.share_tools: + plot.add_tools(tool) return plot def plt_v3offset_time(self): @@ -393,8 +426,9 @@ def plt_v3offset_time(self): plot = figure(title="MSATA Least Squares V3 Offset vs Time", x_axis_label='Time', y_axis_label='Least Squares Residual V3 Offset', x_axis_type='datetime') plot.circle(x='time_arr', y='lsv3offset', source=self.source, - color="blue", size=7, fill_alpha=0.3) + color="blue", size=7, fill_alpha=0.3, view=self.date_view) plot.y_range = Range1d(-0.5, 0.5) + # mark origin line hline = Span(location=0, dimension='width', line_color='black', line_width=0.7) time_arr, v3halffacet = self.source.data['time_arr'], self.source.data['v3halffacet'] @@ -402,6 +436,8 @@ def plt_v3offset_time(self): plot.renderers.extend([hline, hfline]) hflabel = Label(x=time_arr[-1], y=-1 * v3halffacet[0], y_units='data', text='-V3 half-facet value') plot.add_layout(hflabel) + + # add tooltips hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -414,8 +450,11 @@ def plt_v3offset_time(self): ('LS V2 offset', '@lsv2offset'), ('LS V3 offset', '@lsv3offset'), ('--------', '----------------')] - plot.add_tools(hover) + + # add shared selection tools + for tool in self.share_tools: + plot.add_tools(tool) return plot def plt_lsv2v3offsetsigma(self): @@ -432,13 +471,16 @@ def plt_lsv2v3offsetsigma(self): x_axis_label='Least Squares Residual V2 Sigma Offset', y_axis_label='Least Squares Residual V3 Sigma Offset') plot.circle(x='lsv2sigma', y='lsv3sigma', source=self.source, - color="blue", size=7, fill_alpha=0.3) + color="blue", size=7, fill_alpha=0.3, view=self.date_view) plot.x_range = Range1d(-0.1, 0.1) plot.y_range = Range1d(-0.1, 0.1) + # mark origin lines vline = Span(location=0, dimension='height', line_color='black', line_width=0.7) hline = Span(location=0, dimension='width', line_color='black', line_width=0.7) plot.renderers.extend([vline, hline]) + + # add tooltips hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -453,8 +495,11 @@ def plt_lsv2v3offsetsigma(self): ('LS V3 offset', '@lsv3offset'), ('LS V3 sigma', '@lsv3sigma'), ('--------', '----------------')] - plot.add_tools(hover) + + # add shared selection tools + for tool in self.share_tools: + plot.add_tools(tool) return plot def plt_res_offsets_corrected(self): @@ -466,29 +511,30 @@ def plt_res_offsets_corrected(self): ------- plot: bokeh plot object """ - # create a new bokeh plot lsv2offset, lsv3offset = self.source.data['lsv2offset'], self.source.data['lsv3offset'] v2halffacet, v3halffacet = self.source.data['v2halffacet'], self.source.data['v3halffacet'] - # check if this column exists in the data already (the other 2 will exist too), else create it - try: - v2_half_fac_corr = self.source.data['v2_half_fac_corr'] - v3_half_fac_corr = self.source.data['v3_half_fac_corr'] - except KeyError: + + # check if this column exists in the data already, else create it + if 'v2_half_fac_corr' not in self.source.data: v2_half_fac_corr, v3_half_fac_corr = [], [] for idx, v2hf in enumerate(v2halffacet): v3hf = v3halffacet[idx] v2_half_fac_corr.append(lsv2offset[idx] + v2hf) v3_half_fac_corr.append(lsv3offset[idx] + v3hf) + # add these to the bokeh data structure self.source.data["v2_half_fac_corr"] = v2_half_fac_corr self.source.data["v3_half_fac_corr"] = v3_half_fac_corr + + # create a new bokeh plot plot = figure(title="MSATA Least Squares Residual V2-V3 Offsets Half-facet corrected", x_axis_label='Least Squares Residual V2 Offset + half-facet', y_axis_label='Least Squares Residual V3 Offset + half-facet') plot.circle(x='v2_half_fac_corr', y='v3_half_fac_corr', source=self.source, - color="blue", size=7, fill_alpha=0.3) + color="blue", size=7, fill_alpha=0.3, view=self.date_view) plot.x_range = Range1d(-0.5, 0.5) plot.y_range = Range1d(-0.5, 0.5) + # mark origin lines vline = Span(location=0, dimension='height', line_color='black', line_width=0.7) hline = Span(location=0, dimension='width', line_color='black', line_width=0.7) @@ -500,6 +546,8 @@ def plt_res_offsets_corrected(self): angle=90, line_color='purple', line_width=3) hflabel = Label(x=xstart / 3.0, y=ystart, y_units='data', text='-V2, -V3 half-facets values') plot.add_layout(hflabel) + + # add tooltips hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -514,15 +562,16 @@ def plt_res_offsets_corrected(self): ('V2 half-facet', '@v2halffacet'), ('V3 half-facet', '@v3halffacet'), ('--------', '----------------')] - plot.add_tools(hover) + + # add shared selection tools + for tool in self.share_tools: + plot.add_tools(tool) return plot def plt_v2offsigma_time(self): - """ Plot the residual Least Squares V2 sigma Offset versus time - Parameters - ---------- - None + """Plot the residual Least Squares V2 sigma Offset versus time + Returns ------- plot: bokeh plot object @@ -531,11 +580,14 @@ def plt_v2offsigma_time(self): plot = figure(title="MSATA Least Squares V2 Sigma Offset vs Time", x_axis_label='Time', y_axis_label='Least Squares Residual V2 Sigma Offset', x_axis_type='datetime') plot.circle(x='time_arr', y='lsv2sigma', source=self.source, - color="blue", size=7, fill_alpha=0.3) + color="blue", size=7, fill_alpha=0.3, view=self.date_view) plot.y_range = Range1d(-0.1, 0.1) + # mark origin line hline = Span(location=0, dimension='width', line_color='black', line_width=0.7) plot.renderers.extend([hline]) + + # add tooltips hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -547,28 +599,34 @@ def plt_v2offsigma_time(self): ('LS V2 offset', '@lsv2offset'), ('LS V2 sigma', '@lsv2sigma'), ('--------', '----------------')] - plot.add_tools(hover) + + # add shared selection tools + for tool in self.share_tools: + plot.add_tools(tool) return plot def plt_v3offsigma_time(self): - """ Plot the residual Least Squares V3 Offset versus time - Parameters - ---------- - None + """Plot the residual Least Squares V3 Offset versus time + Returns ------- p: bokeh plot object """ # create a new bokeh plot - plot = figure(title="MSATA Least Squares V3 Sigma Offset vs Time", x_axis_label='Time', - y_axis_label='Least Squares Residual V3 Sigma Offset', x_axis_type='datetime') + plot = figure(title="MSATA Least Squares V3 Sigma Offset vs Time", + x_axis_label='Time', + y_axis_label='Least Squares Residual V3 Sigma Offset', + x_axis_type='datetime') plot.circle(x='time_arr', y='lsv3sigma', source=self.source, - color="blue", size=7, fill_alpha=0.3) + color="blue", size=7, fill_alpha=0.3, view=self.date_view) plot.y_range = Range1d(-0.1, 0.1) + # mark origin line hline = Span(location=0, dimension='width', line_color='black', line_width=0.7) plot.renderers.extend([hline]) + + # add tooltips hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -581,8 +639,11 @@ def plt_v3offsigma_time(self): ('LS V3 offset', '@lsv3offset'), ('LS V3 sigma', '@lsv3sigma'), ('--------', '----------------')] - plot.add_tools(hover) + + # add shared selection tools + for tool in self.share_tools: + plot.add_tools(tool) return plot def plt_roll_offset(self): @@ -598,10 +659,12 @@ def plt_roll_offset(self): plot = figure(title="MSATA Least Squares Roll Offset vs Time", x_axis_label='Time', y_axis_label='Least Squares Residual Roll Offset', x_axis_type='datetime') plot.circle(x='time_arr', y='lsrolloffset', source=self.source, - color="blue", size=7, fill_alpha=0.3) + color="blue", size=7, fill_alpha=0.3, view=self.date_view) plot.y_range = Range1d(-600.0, 600.0) + # mark origin line hline = Span(location=0, dimension='width', line_color='black', line_width=0.7) + # Maximum accepted roll line and label time_arr = self.source.data['time_arr'] arlinepos = Span(location=120, dimension='width', line_color='green', line_width=3) @@ -609,6 +672,8 @@ def plt_roll_offset(self): arlabel = Label(x=time_arr[-1], y=125, y_units='data', text='Max accepted roll') plot.add_layout(arlabel) plot.renderers.extend([hline, arlinepos, arlineneg]) + + # add tooltips hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -621,8 +686,11 @@ def plt_roll_offset(self): ('LS V2 offset', '@lsv2offset'), ('LS V3 offset', '@lsv3offset'), ('--------', '----------------')] - plot.add_tools(hover) + + # add shared selection tools + for tool in self.share_tools: + plot.add_tools(tool) return plot def plt_lsoffsetmag(self): @@ -638,11 +706,14 @@ def plt_lsoffsetmag(self): plot = figure(title="MSATA Least Squares Total Magnitude of the Linear V2, V3 Offset Slew vs Time", x_axis_label='Time', y_axis_label='sqrt((V2_off)**2 + (V3_off)**2)', x_axis_type='datetime') plot.circle(x='time_arr', y='lsoffsetmag', source=self.source, - color="blue", size=7, fill_alpha=0.3) + color="blue", size=7, fill_alpha=0.3, view=self.date_view) plot.y_range = Range1d(-0.5, 0.5) + # mark origin line hline = Span(location=0, dimension='width', line_color='black', line_width=0.7) plot.renderers.extend([hline]) + + # add tooltips hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -656,8 +727,11 @@ def plt_lsoffsetmag(self): ('LS V2 offset', '@lsv2offset'), ('LS V3 offset', '@lsv3offset'), ('--------', '----------------')] - plot.add_tools(hover) + + # add shared selection tools + for tool in self.share_tools: + plot.add_tools(tool) return plot def plt_tot_number_of_stars(self): @@ -672,34 +746,34 @@ def plt_tot_number_of_stars(self): # get the number of stars per array visit_id = self.source.data['visit_id'] reference_star_number = self.source.data['reference_star_number'] - stars_in_fit = self.source.data['stars_in_fit'] - date_obs, time_arr = self.source.data['date_obs'], self.source.data['time_arr'] - # check if this column exists in the data already (the other 2 will exist too), else create it - try: - tot_number_of_stars = self.source.data['tot_number_of_stars'] - colors_list = self.source.data['colors_list'] - except KeyError: + + # check if this column exists in the data already, else create it + if 'tot_number_of_stars' not in self.source.data: # create the list of color per visit and tot_number_of_stars colors_list, tot_number_of_stars = [], [] color_dict = {} - for i, _ in enumerate(visit_id): + for i, vid in enumerate(visit_id): tot_stars = len(reference_star_number[i]) tot_number_of_stars.append(tot_stars) ci = '#%06X' % randint(0, 0xFFFFFF) - if visit_id[i] not in color_dict: - color_dict[visit_id[i]] = ci - colors_list.append(color_dict[visit_id[i]]) + if vid not in color_dict: + color_dict[vid] = ci + colors_list.append(color_dict[vid]) + # add these to the bokeh data structure self.source.data["tot_number_of_stars"] = tot_number_of_stars self.source.data["colors_list"] = colors_list + # create a new bokeh plot plot = figure(title="Total Number of Measurements vs Time", x_axis_label='Time', y_axis_label='Total number of measurements', x_axis_type='datetime') plot.circle(x='time_arr', y='tot_number_of_stars', source=self.source, - color='colors_list', size=7, fill_alpha=0.3) + color='colors_list', size=7, fill_alpha=0.3, view=self.date_view) plot.triangle(x='time_arr', y='stars_in_fit', source=self.source, - color='black', size=7, fill_alpha=0.3) + color='black', size=7, fill_alpha=0.3, view=self.date_view) plot.y_range = Range1d(0.0, 40.0) + + # add tooltips hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -714,8 +788,8 @@ def plt_tot_number_of_stars(self): ('LS V2 offset', '@lsv2offset'), ('LS V3 offset', '@lsv3offset'), ('--------', '----------------')] - plot.add_tools(hover) + return plot def plt_mags_time(self): @@ -735,22 +809,24 @@ def plt_mags_time(self): planned_v2 = self.source.data['planned_v2'] planned_v3 = self.source.data['planned_v3'] reference_star_number = self.source.data['reference_star_number'] - visits_stars_mags = self.source.data['reference_star_mag'] box_peak_value = self.source.data['box_peak_value'] date_obs, time_arr = self.source.data['date_obs'], self.source.data['time_arr'] colors_list = self.source.data['colors_list'] detector_list = self.source.data['detector'] + filename = self.source.data['filename'] + # create the structure matching the number of visits and reference stars new_colors_list, vid, dobs, tarr, star_no, status = [], [], [], [], [], [] - peaks, stars_v2, stars_v3, det = [], [], [], [] + peaks, stars_v2, stars_v3, det, fnames = [], [], [], [], [] for i, _ in enumerate(visit_id): - v, d, t, c, s, x, y, dt = [], [], [], [], [], [], [], [] + v, d, t, c, s, x, y, dt, fn = [], [], [], [], [], [], [], [], [] for j in range(len(reference_star_number[i])): v.append(visit_id[i]) d.append(date_obs[i]) t.append(time_arr[i]) c.append(colors_list[i]) dt.append(detector_list[i]) + fn.append(filename[i]) if 'not_removed' in lsf_removed_status[i][j]: s.append('SUCCESS') x.append(planned_v2[i][j]) @@ -769,17 +845,29 @@ def plt_mags_time(self): stars_v3.extend(y) peaks.extend(box_peak_value[i]) det.extend(dt) + fnames.extend(fn) + # now create the mini ColumnDataSource for this particular plot mini_source = {'vid': vid, 'star_no': star_no, 'status': status, - 'dobs': dobs, 'tarr': tarr, 'det': det, + 'dobs': dobs, 'time_arr': tarr, 'det': det, 'fname': fnames, 'peaks': peaks, 'colors_list': new_colors_list, 'stars_v2': stars_v2, 'stars_v3': stars_v3} mini_source = ColumnDataSource(data=mini_source) - # create a the bokeh plot - plot = figure(title="MSATA Counts vs Time", x_axis_label='Time', y_axis_label='box_peak [Counts]', + + # hook up the date range slider to this source as well + callback = CustomJS(args=dict(s=mini_source), code=""" + s.change.emit(); + """) + self.date_range.js_on_change('value', callback) + mini_view = CDSView(source=mini_source, filters=[self.date_filter]) + + # create the bokeh plot + plot = figure(title="MSATA Counts vs Time", x_axis_label='Time', + y_axis_label='box_peak [Counts]', x_axis_type='datetime') - plot.circle(x='tarr', y='peaks', source=mini_source, - color='colors_list', size=7, fill_alpha=0.3) + plot.circle(x='time_arr', y='peaks', source=mini_source, + color='colors_list', size=7, fill_alpha=0.3, view=mini_view) + # add count saturation warning lines loc1, loc2, loc3 = 45000.0, 50000.0, 60000.0 hline1 = Span(location=loc1, dimension='width', line_color='green', line_width=3) @@ -793,9 +881,10 @@ def plt_mags_time(self): plot.add_layout(label2) plot.add_layout(label3) plot.y_range = Range1d(-1000.0, 62000.0) - # add hover + + # add tooltips hover = HoverTool() - hover.tooltips = [('File name', '@filename'), + hover.tooltips = [('File name', '@fname'), ('Visit ID', '@vid'), ('Detector', '@det'), ('Star No.', '@star_no'), @@ -805,18 +894,69 @@ def plt_mags_time(self): ('Measured V2', '@stars_v2'), ('Measured V3', '@stars_v3'), ('--------', '----------------')] - plot.add_tools(hover) + return plot + def setup_date_range(self): + """Set up a date range filter, defaulting to the last week of data.""" + end_date = datetime.now(tz=timezone.utc) + one_week_ago = end_date.date() - timedelta(days=7) + first_data_point = np.min(self.source.data['time_arr']).date() + last_data_point = np.max(self.source.data['time_arr']).date() + if last_data_point < one_week_ago: + # keep at least one point in the plot if there was + # no TA data this week + start_date = last_data_point + else: + start_date = one_week_ago + + # allowed range is from the first ever data point to today + self.date_range = DateRangeSlider( + title="Date range displayed", start=first_data_point, + end=end_date, value=(start_date, end_date), step=1) + + callback = CustomJS(args=dict(s=self.source), code=""" + s.change.emit(); + """) + self.date_range.js_on_change('value', callback) + + self.date_filter = CustomJSFilter(args=dict(slider=self.date_range), code=""" + var indices = []; + var start = slider.value[0]; + var end = slider.value[1]; + + for (var i=0; i < source.get_length(); i++) { + if (source.data['time_arr'][i] >= start + && source.data['time_arr'][i] <= end) { + indices.push(true); + } else { + indices.push(false); + } + } + return indices; + """) + self.date_view = CDSView(source=self.source, filters=[self.date_filter]) + def mk_plt_layout(self): """Create the bokeh plot layout""" self.source = ColumnDataSource(data=self.msata_data) + # make sure all arrays are lists in order to later be able to read the data # from the html file for item in self.source.data: if not isinstance(self.source.data[item], (str, float, int, list)): self.source.data[item] = self.source.data[item].tolist() + + # add a time array to the data source + self.add_time_column() + + # set up selection tools to share + self.share_tools = [BoxSelectTool()] + + # set up a date range filter widget + self.setup_date_range() + # set the output html file name and create the plot grid output_file(self.output_file_name) p1 = self.plt_status() @@ -831,13 +971,15 @@ def mk_plt_layout(self): p10 = self.plt_lsoffsetmag() p12 = self.plt_tot_number_of_stars() p11 = self.plt_mags_time() + # make grid grid = gridplot([p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12], ncols=2, merge_tools=False) - # show(grid) - save(grid) - # return the needed components for embeding the results in the MSATA html template - script, div = components(grid) + box_layout = layout(children=[self.date_range, grid]) + save(box_layout) + + # return the needed components for embedding the results in the MSATA html template + script, div = components(box_layout) return script, div def identify_tables(self): @@ -884,7 +1026,7 @@ def get_data_from_html(self, html_file): File created by the monitor script Returns ------- - prev_data_dict: dictionary + prev_data_dict: dict Dictionary containing all data used in the plots """ @@ -961,7 +1103,7 @@ def prev_data2expected_format(self, prev_data_dict): Date of the latest observation in the previously plotted data """ # remember that the time array created is in milliseconds, removing to get time object - time_in_millis = max(prev_data_dict['tarr']) + time_in_millis = max(prev_data_dict['time_arr']) latest_prev_obs = Time(time_in_millis / 1000., format='unix') latest_prev_obs = latest_prev_obs.mjd prev_data_expected_cols = {} @@ -1051,13 +1193,13 @@ def update_ta_success_txtfile(self): output_success_ta_txtfile = os.path.join(self.output_dir, "msata_success.txt") # check if previous file exsists and read the data from it if os.path.isfile(output_success_ta_txtfile): - # now rename the the previous file, for backup + # now rename the previous file, for backup os.rename(output_success_ta_txtfile, os.path.join(self.output_dir, "prev_msata_success.txt")) # get the new data ta_success, ta_inprogress, ta_failure = [], [], [] filenames, ta_status = self.msata_data.loc[:,'filename'], self.msata_data.loc[:,'ta_status'] for fname, ta_stat in zip(filenames, ta_status): - # select the appriopriate list to append to + # select the appropriate list to append to if ta_stat == 'SUCCESSFUL': ta_success.append(fname) elif ta_stat == 'IN_PROGRESS': diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py index 01e52a44c..f248a1d09 100755 --- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py +++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py @@ -10,7 +10,7 @@ """ -This module contains the code for NIRSpec the Wide Aperture Target +This module contains the code for the NIRSpec Wide Aperture Target Acquisition (WATA) monitor, which monitors the TA offsets. This monitor displays the comparison of desired versus measured TA. @@ -20,6 +20,7 @@ Author ______ - Maria Pena-Guerrero + - Melanie Clarke Use --- @@ -34,21 +35,22 @@ import os import logging import shutil +from datetime import datetime, timezone, timedelta import numpy as np import pandas as pd -from bs4 import BeautifulSoup -from datetime import datetime from astropy.time import Time from astropy.io import fits -from sqlalchemy.sql.expression import and_ -from bokeh.io import output_file -from bokeh.plotting import figure, show, save -from bokeh.models import ColumnDataSource, Range1d -from bokeh.models.tools import HoverTool -from bokeh.layouts import gridplot -from bokeh.models import Span, Label from bokeh.embed import components +from bokeh.io import output_file +from bokeh.layouts import gridplot, layout +from bokeh.models import ( + ColumnDataSource, Range1d, CustomJS, CustomJSFilter, CDSView, + Span, Label, DateRangeSlider) +from bokeh.models.tools import HoverTool, BoxSelectTool +from bokeh.plotting import figure, save +from bs4 import BeautifulSoup +from sqlalchemy.sql.expression import and_ # jwql imports from jwql.utils.logging_functions import log_info, log_fail @@ -60,7 +62,7 @@ class WATA(): - """ Class for executint the NIRSpec WATA monitor. + """ Class for executing the NIRSpec WATA monitor. This class will search for new WATA current files in the file systems for NIRSpec and will run the monitor on these files. The monitor will @@ -130,6 +132,12 @@ def __init__(self): 'SAM_X': {'loc': 'ta_hdr', 'alt_key': None, 'name': 'sam_x'}, 'SAM_Y': {'loc': 'ta_hdr', 'alt_key': None, 'name': 'sam_y'}} + # initialize attributes to be set later + self.source = None + self.share_tools = [] + self.date_range = None + self.date_view = None + def get_tainfo_from_fits(self, fits_file): """ Get the TA information from the fits file Parameters @@ -200,6 +208,19 @@ def get_wata_data(self, new_filenames): wata_df = pd.DataFrame(wata_dict) return wata_df, no_ta_ext_msgs + def add_time_column(self): + """Add time column to data source, to be used by all plots.""" + date_obs = self.source.data['date_obs'] + if 'time_arr' not in self.source.data: + time_arr = [] + for do_str in date_obs: + # convert time string into an array of time (this is in UT) + t = datetime.fromisoformat(do_str) + time_arr.append(t) + + # add to the bokeh data structure + self.source.data["time_arr"] = time_arr + def plt_status(self): """ Plot the WATA status (passed = 0 or failed = 1). Parameters @@ -209,36 +230,32 @@ def plt_status(self): ------- plot: bokeh plot object """ - ta_status, date_obs = self.source.data['ta_status'], self.source.data['date_obs'] - # check if this column exists in the data already (the other 2 will exist too), else create it - try: - time_arr = self.source.data['time_arr'] - bool_status = self.source.data['bool_status'] - status_colors = self.source.data['status_colors'] - except KeyError: - # bokeh does not like to plot strings, turn into binary type - bool_status, time_arr, status_colors = [], [], [] - for tas, do_str in zip(ta_status, date_obs): + ta_status = self.source.data['ta_status'] + + # check if this column exists in the data already, else create it + if 'bool_status' not in self.source.data: + # bokeh does not like to plot strings, turn into binary type + bool_status, status_colors = [], [] + for tas in ta_status: if 'unsuccessful' not in tas.lower(): bool_status.append(1) status_colors.append('blue') else: bool_status.append(0) status_colors.append('red') - # convert time string into an array of time (this is in UT) - t = datetime.fromisoformat(do_str) - time_arr.append(t) + # add these to the bokeh data structure - self.source.data["time_arr"] = time_arr self.source.data["ta_status_bool"] = bool_status self.source.data["status_colors"] = status_colors + # create a new bokeh plot - plot = figure(title="WATA Status [Succes=1, Fail=0]", x_axis_label='Time', + plot = figure(title="WATA Status [Success=1, Fail=0]", x_axis_label='Time', y_axis_label='WATA Status', x_axis_type='datetime',) plot.y_range = Range1d(-0.5, 1.5) plot.circle(x='time_arr', y='ta_status_bool', source=self.source, - color='status_colors', size=7, fill_alpha=0.3) - # output_file("wata_status.html") + color='status_colors', size=7, fill_alpha=0.3, view=self.date_view) + + # make tooltips hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -250,6 +267,10 @@ def plt_status(self): ('--------', '----------------')] plot.add_tools(hover) + + # add shared selection tools + for tool in self.share_tools: + plot.add_tools(tool) return plot def plt_residual_offsets(self): @@ -265,13 +286,16 @@ def plt_residual_offsets(self): plot = figure(title="WATA Residual V2-V3 Offsets", x_axis_label='Residual V2 Offset', y_axis_label='Residual V3 Offset') plot.circle(x='v2_offset', y='v3_offset', source=self.source, - color="blue", size=7, fill_alpha=0.3) + color="blue", size=7, fill_alpha=0.3, view=self.date_view) plot.x_range = Range1d(-0.5, 0.5) plot.y_range = Range1d(-0.5, 0.5) + # mark origin lines vline = Span(location=0, dimension='height', line_color='black', line_width=0.7) hline = Span(location=0, dimension='width', line_color='black', line_width=0.7) plot.renderers.extend([vline, hline]) + + # add tooltips hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -281,8 +305,11 @@ def plt_residual_offsets(self): ('Date-Obs', '@date_obs'), ('Magnitude', '@star_mag'), ('--------', '----------------')] - plot.add_tools(hover) + + # add shared selection tools + for tool in self.share_tools: + plot.add_tools(tool) return plot def plt_v2offset_time(self): @@ -298,11 +325,14 @@ def plt_v2offset_time(self): plot = figure(title="WATA V2 Offset vs Time", x_axis_label='Time', y_axis_label='Residual V2 Offset', x_axis_type='datetime') plot.circle(x='time_arr', y='v2_offset', source=self.source, - color="blue", size=7, fill_alpha=0.3) + color="blue", size=7, fill_alpha=0.3, view=self.date_view) plot.y_range = Range1d(-0.5, 0.5) + # mark origin line hline = Span(location=0, dimension='width', line_color='black', line_width=0.7) plot.renderers.extend([hline]) + + # add tooltips hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -312,8 +342,11 @@ def plt_v2offset_time(self): ('Date-Obs', '@date_obs'), ('Magnitude', '@star_mag'), ('--------', '----------------')] - plot.add_tools(hover) + + # add shared selection tools + for tool in self.share_tools: + plot.add_tools(tool) return plot def plt_v3offset_time(self): @@ -329,11 +362,14 @@ def plt_v3offset_time(self): plot = figure(title="WATA V3 Offset vs Time", x_axis_label='Time', y_axis_label='Residual V3 Offset', x_axis_type='datetime') plot.circle(x='time_arr', y='v3_offset', source=self.source, - color="blue", size=7, fill_alpha=0.3) + color="blue", size=7, fill_alpha=0.3, view=self.date_view) plot.y_range = Range1d(-0.5, 0.5) + # mark origin line hline = Span(location=0, dimension='width', line_color='black', line_width=0.7) plot.renderers.extend([hline]) + + # add tooltips hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -343,8 +379,11 @@ def plt_v3offset_time(self): ('Date-Obs', '@date_obs'), ('Magnitude', '@star_mag'), ('--------', '----------------')] - plot.add_tools(hover) + + # add shared selection tools + for tool in self.share_tools: + plot.add_tools(tool) return plot def plt_mag_time(self): @@ -358,15 +397,9 @@ def plt_mag_time(self): """ # calculate the pseudo magnitudes max_val_box, time_arr = self.source.data['max_val_box'], self.source.data['time_arr'] - # check if this column exists in the data already (the other 2 will exist too), else create it - try: - nrsrapid_f140x = self.source.data["nrsrapid_f140x"] - nrsrapid_f110w = self.source.data["nrsrapid_f110w"] - nrsrapid_clear = self.source.data["nrsrapid_clear"] - nrsrapidd6_f140x = self.source.data["nrsrapidd6_f140x"] - nrsrapidd6_f110w = self.source.data["nrsrapidd6_f110w"] - nrsrapidd6_clear = self.source.data["nrsrapidd6_clear"] - except KeyError: + + # check if this column exists in the data already, else create it + if "nrsrapid_f140x" not in self.source.data: # create the arrays per filter and readout pattern nrsrapid_f140x, nrsrapid_f110w, nrsrapid_clear = [], [], [] nrsrapidd6_f140x, nrsrapidd6_f110w, nrsrapidd6_clear = [], [], [] @@ -417,6 +450,7 @@ def plt_mag_time(self): nrsrapidd6_f140x.append(np.NaN) nrsrapidd6_f110w.append(np.NaN) nrsrapidd6_clear.append(val) + # add to the bokeh data structure self.source.data["nrsrapid_f140x"] = nrsrapid_f140x self.source.data["nrsrapid_f110w"] = nrsrapid_f110w @@ -424,27 +458,30 @@ def plt_mag_time(self): self.source.data["nrsrapidd6_f140x"] = nrsrapidd6_f140x self.source.data["nrsrapidd6_f110w"] = nrsrapidd6_f110w self.source.data["nrsrapidd6_clear"] = nrsrapidd6_clear + # create a new bokeh plot plot = figure(title="WATA Counts vs Time", x_axis_label='Time', y_axis_label='box_peak [Counts]', x_axis_type='datetime') plot.circle(x='time_arr', y='nrsrapid_f140x', source=self.source, - color="purple", size=7, fill_alpha=0.4) + color="purple", size=7, fill_alpha=0.4, view=self.date_view) plot.circle(x='time_arr', y='nrsrapidd6_f140x', source=self.source, - color="purple", size=12, fill_alpha=0.4) + color="purple", size=12, fill_alpha=0.4, view=self.date_view) plot.triangle(x='time_arr', y='nrsrapid_f110w', source=self.source, - color="orange", size=8, fill_alpha=0.4) + color="orange", size=8, fill_alpha=0.4, view=self.date_view) plot.triangle(x='time_arr', y='nrsrapidd6_f110w', source=self.source, - color="orange", size=13, fill_alpha=0.4) + color="orange", size=13, fill_alpha=0.4, view=self.date_view) plot.square(x='time_arr', y='nrsrapid_clear', source=self.source, - color="gray", size=7, fill_alpha=0.4) + color="gray", size=7, fill_alpha=0.4, view=self.date_view) plot.square(x='time_arr', y='nrsrapidd6_clear', source=self.source, - color="gray", size=12, fill_alpha=0.4) + color="gray", size=12, fill_alpha=0.4, view=self.date_view) + # add count saturation warning lines loc1, loc2, loc3 = 45000.0, 50000.0, 60000.0 hline1 = Span(location=loc1, dimension='width', line_color='green', line_width=3) hline2 = Span(location=loc2, dimension='width', line_color='yellow', line_width=3) hline3 = Span(location=loc3, dimension='width', line_color='red', line_width=3) plot.renderers.extend([hline1, hline2, hline3]) + label1 = Label(x=time_arr[-1], y=loc1, y_units='data', text='45000 counts') label2 = Label(x=time_arr[-1], y=loc2, y_units='data', text='50000 counts') label3 = Label(x=time_arr[-1], y=loc3, y_units='data', text='60000 counts') @@ -452,7 +489,8 @@ def plt_mag_time(self): plot.add_layout(label2) plot.add_layout(label3) plot.y_range = Range1d(-1000.0, 62000.0) - # add hover + + # add tooltips hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -462,12 +500,15 @@ def plt_mag_time(self): ('Date-Obs', '@date_obs'), ('Box peak', '@max_val_box'), ('--------', '----------------')] - plot.add_tools(hover) + + # add shared selection tools + for tool in self.share_tools: + plot.add_tools(tool) return plot - def get_unsucessful_ta(self, arr_name): - """ Find unsucessful TAs in this set (to be plotted in red) + def get_unsuccessful_ta(self, arr_name): + """ Find unsuccessful TAs in this set (to be plotted in red) Parameters ---------- arr_name: str, name of the array of interest @@ -497,16 +538,16 @@ def plt_centroid(self): plot: bokeh plot object """ # get the failed TAs to plot in red - try: - corr_col_failed = self.source.data["corr_col_failed"] - except KeyError: - corr_col_failed, corr_col_not_failed = self.get_unsucessful_ta('corr_col') - corr_row_failed, corr_row_not_failed = self.get_unsucessful_ta('corr_row') + if "corr_col_failed" not in self.source.data: + corr_col_failed, corr_col_not_failed = self.get_unsuccessful_ta('corr_col') + corr_row_failed, corr_row_not_failed = self.get_unsuccessful_ta('corr_row') + # add these to the bokeh data structure self.source.data["corr_col_failed"] = corr_col_failed self.source.data["corr_col_not_failed"] = corr_col_not_failed self.source.data["corr_row_failed"] = corr_row_failed self.source.data["corr_row_not_failed"] = corr_row_not_failed + # create a new bokeh plot plot = figure(title="WATA Centroid", x_axis_label='Column', y_axis_label='Row') @@ -514,11 +555,13 @@ def plt_centroid(self): plot.x_range = Range1d(limits[0], limits[1]) plot.y_range = Range1d(limits[0], limits[1]) plot.circle(x='corr_col_not_failed', y='corr_row_not_failed', source=self.source, - color="blue", size=7, fill_alpha=0.5) + color="blue", size=7, fill_alpha=0.5, view=self.date_view) plot.circle(x='corr_col_failed', y='corr_row_failed', source=self.source, - color="red", size=7, fill_alpha=0.5) + color="red", size=7, fill_alpha=0.5, view=self.date_view) plot.x_range = Range1d(0.0, 32.0) plot.y_range = Range1d(0.0, 32.0) + + # add tooltips hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -532,18 +575,72 @@ def plt_centroid(self): ('Det Centr Col', '@detector_final_col'), ('Det Centr Row', '@detector_final_row'), ('--------', '----------------')] - plot.add_tools(hover) + + # add shared selection tools + for tool in self.share_tools: + plot.add_tools(tool) return plot + def setup_date_range(self): + """Set up a date range filter, defaulting to the last week of data.""" + end_date = datetime.now(tz=timezone.utc) + one_week_ago = end_date.date() - timedelta(days=7) + first_data_point = np.min(self.source.data['time_arr']).date() + last_data_point = np.max(self.source.data['time_arr']).date() + if last_data_point < one_week_ago: + # keep at least one point in the plot if there was + # no TA data this week + start_date = last_data_point + else: + start_date = one_week_ago + + # allowed range is from the first ever data point to today + self.date_range = DateRangeSlider( + title="Date range displayed", start=first_data_point, + end=end_date, value=(start_date, end_date), step=1) + + callback = CustomJS(args=dict(s=self.source), code=""" + s.change.emit(); + """) + self.date_range.js_on_change('value', callback) + + filt = CustomJSFilter(args=dict(slider=self.date_range), code=""" + var indices = []; + var start = slider.value[0]; + var end = slider.value[1]; + + for (var i=0; i < source.get_length(); i++) { + if (source.data['time_arr'][i] >= start + && source.data['time_arr'][i] <= end) { + indices.push(true); + } else { + indices.push(false); + } + } + return indices; + """) + self.date_view = CDSView(source=self.source, filters=[filt]) + def mk_plt_layout(self): """Create the bokeh plot layout""" self.source = ColumnDataSource(data=self.wata_data) + # make sure all arrays are lists in order to later be able to read the data # from the html file for item in self.source.data: if not isinstance(self.source.data[item], (str, float, int, list)): self.source.data[item] = self.source.data[item].tolist() + + # add a time array to the data source + self.add_time_column() + + # set up selection tools to share + self.share_tools = [BoxSelectTool()] + + # set up a date range filter widget + self.setup_date_range() + # set the output html file name and create the plot grid output_file(self.output_file_name) p1 = self.plt_status() @@ -552,11 +649,14 @@ def mk_plt_layout(self): p4 = self.plt_v3offset_time() p5 = self.plt_centroid() p6 = self.plt_mag_time() + # make grid grid = gridplot([p1, p2, p3, p4, p5, p6], ncols=2, merge_tools=False) - save(grid) + box_layout = layout(children=[self.date_range, grid]) + save(box_layout) + # return the needed components for embeding the results in the WATA html template - script, div = components(grid) + script, div = components(box_layout) return script, div def identify_tables(self): @@ -729,15 +829,15 @@ def update_ta_success_txtfile(self): Nothing """ output_success_ta_txtfile = os.path.join(self.output_dir, "wata_success.txt") - # check if previous file exsists and read the data from it + # check if previous file exists and read the data from it if os.path.isfile(output_success_ta_txtfile): - # now rename the the previous file, for backup + # now rename the previous file, for backup os.rename(output_success_ta_txtfile, os.path.join(self.output_dir, "prev_wata_success.txt")) # get the new data ta_success, ta_failure = [], [] filenames, ta_status = self.wata_data.loc[:,'filename'], self.wata_data.loc[:,'ta_status'] for fname, ta_stat in zip(filenames, ta_status): - # select the appriopriate list to append to + # select the appropriate list to append to if ta_stat == 'SUCCESSFUL': ta_success.append(fname) else: diff --git a/jwql/utils/interactive_preview_image.py b/jwql/utils/interactive_preview_image.py index 6ea0c1e9e..d0f1c8cea 100644 --- a/jwql/utils/interactive_preview_image.py +++ b/jwql/utils/interactive_preview_image.py @@ -20,18 +20,20 @@ import os from astropy.io import fits -from astropy.visualization import ZScaleInterval +from astropy.visualization import ZScaleInterval, MinMaxInterval, PercentileInterval import numpy as np from bokeh.embed import components +from bokeh.layouts import layout from bokeh.models import ( - BasicTicker, BoxZoomTool, ColorBar, HoverTool, LinearColorMapper, LogColorMapper, - LogTicker, WheelZoomTool) + BasicTicker, BoxZoomTool, Button, ColorBar, CustomJS, Div, HoverTool, + LinearColorMapper, LogColorMapper, LogTicker, RadioGroup, Row, Select, Spacer, + Spinner, WheelZoomTool) from bokeh.plotting import figure, output_file, show, save from jwst.datamodels import dqflags -class InteractivePreviewImg(): +class InteractivePreviewImg: """Class to create the interactive Bokeh figure. """ @@ -113,16 +115,20 @@ def create_bokeh_image(self): limits = (self.low_lim, limits[1]) if self.high_lim is not None: limits = (limits[0], self.high_lim) - if self.scaling == 'log': - if limits[0] <= 0: - limits = (1e-4, limits[1]) - color_mapper = LogColorMapper( - palette="Viridis256", low=limits[0], high=limits[1]) - ticker = LogTicker() - elif self.scaling == 'lin': - color_mapper = LinearColorMapper( - palette="Viridis256", low=limits[0], high=limits[1]) - ticker = BasicTicker() + + # handle log or linear scaling + if limits[0] <= 0: + log_limits = (1e-4, limits[1]) + else: + log_limits = limits + log_color_mapper = LogColorMapper( + palette="Viridis256", low=log_limits[0], high=log_limits[1]) + log_ticker = LogTicker() + lin_color_mapper = LinearColorMapper( + palette="Viridis256", low=limits[0], high=limits[1]) + lin_ticker = BasicTicker() + active = int(self.scaling == 'log') + yd, xd = self.data.shape info = dict(image=[self.data], x=[0], y=[0], dw=[xd], dh=[yd]) if 'DQ' in self.extname: @@ -132,6 +138,8 @@ def create_bokeh_image(self): title=os.path.basename(self.filename)) # fix figure aspect from data aspect + # bokeh throws errors if plot is too small, so make sure + # the smaller dimension has reasonable size if xd > yd: plot_width = 800 plot_height = int(plot_width * yd / xd) @@ -148,13 +156,24 @@ def create_bokeh_image(self): fig.add_tools(BoxZoomTool(match_aspect=True)) fig.add_tools(WheelZoomTool(zoom_on_axis=False)) - img = fig.image(source=info, image='image', - level="image", color_mapper=color_mapper) - color_bar = ColorBar(color_mapper=color_mapper, label_standoff=12, ticker=ticker, - title=self.signal_units, bar_line_color='black', - minor_tick_line_color='black', major_tick_line_color='black') - fig.add_layout(color_bar, 'below') + # make both linear and log scale images to allow toggling between them + images = [] + color_bars = [] + scales = ((lin_color_mapper, lin_ticker), (log_color_mapper, log_ticker)) + for i, config in enumerate(scales): + color_mapper, ticker = config + visible = (i == active) + img = fig.image(source=info, image='image', + level="image", color_mapper=color_mapper, visible=visible) + color_bar = ColorBar(color_mapper=color_mapper, label_standoff=12, ticker=ticker, + title=self.signal_units, bar_line_color='black', + minor_tick_line_color='black', major_tick_line_color='black', + visible=visible) + fig.add_layout(color_bar, 'below') + images.append(img) + color_bars.append(color_bar) + # limit whitespace around image as much as possible fig.x_range.range_padding = fig.y_range.range_padding = 0 if xd >= yd: fig.x_range.start = 0 @@ -168,23 +187,130 @@ def create_bokeh_image(self): if 'DQ' not in self.extname: hover_tool = HoverTool(tooltips=[("(x,y)", "($x{0.2f}, $y{0.2f})"), ('Value', '@image{0.4f}') - ], mode='mouse', renderers=[img]) + ], mode='mouse', renderers=images) else: hover_tool = HoverTool(tooltips=[("(x,y)", "($x{0.2f}, $y{0.2f})"), ('Value', '@dq') - ], mode='mouse', renderers=[img]) + ], mode='mouse', renderers=images) self.create_figure_title() fig.title.text = self.title fig.xaxis.axis_label = 'Pixel' fig.yaxis.axis_label = 'Pixel' fig.tools.append(hover_tool) + + # add interactive widgets + widgets = self.add_interactive_controls(images, color_bars) + box_layout = layout(children=[fig, *widgets]) + # Show figure on screen if requested if self.show: - show(fig) + show(box_layout) elif self.save_html is not None: - save(fig) + save(box_layout) else: - return components(fig) + return components(box_layout) + + def add_interactive_controls(self, images, color_bars): + """ + Add client-side controls for images. + + Currently includes image scaling and limit setting controls. + + Parameters + ---------- + images : list of bokeh.models.Image + 2-element list of images. The first is linear scale, second is log scale. + Only one should be visible at any time. + color_bars : list of bokeh.models.ColorBar + 2-element list of color bars, matching the images. + + Returns + ------- + widgets: list of bokeh.Widget + Widgets to add to the page layout. + """ + # active scaling (0=linear, 1=log) + active = int(self.scaling == 'log') + + tools_label = Div(text="

    Image Settings

    ") + + scale_label = Div(text="Scaling:") + scale_group = RadioGroup(labels=["linear", "log"], + inline=True, active=active) + scale_set = Row(scale_label, scale_group, + css_classes=['mb-4']) + + current_low = images[active].glyph.color_mapper.low + current_high = images[active].glyph.color_mapper.high + preset_limits = {'ZScale': (current_low, current_high), + 'Min/Max': MinMaxInterval().get_limits(self.data), + '99.5%': PercentileInterval(99.5).get_limits(self.data), + '99%': PercentileInterval(99).get_limits(self.data), + '95%': PercentileInterval(95).get_limits(self.data), + '90%': PercentileInterval(90).get_limits(self.data)} + options = [*preset_limits.keys(), 'Custom'] + preset_label = Div(text="Percentile presets:") + preset_select = Select(value='ZScale', options=options, width=120) + preset_set = Row(preset_label, preset_select) + + limit_label = Div(text="Limits:") + limit_low = Spinner(title="Low", value=current_low) + limit_high = Spinner(title="High", value=current_high) + reset = Button(label='Reset', button_type='primary') + limit_set = Row(limit_label, limit_low, limit_high, + css_classes=['mb-4']) + + # JS callbacks for client side controls + + # set alternate image visibility when scale selection changes + scale_group.js_on_click(CustomJS(args={'i1': images[0], 'c1': color_bars[0], + 'i2': images[1], 'c2': color_bars[1]}, + code=""" + if (i1.visible == true) { + i1.visible = false; + c1.visible = false; + i2.visible = true; + c2.visible = true; + } else { + i1.visible = true; + c1.visible = true; + i2.visible = false; + c2.visible = false; + } + """)) + + # set scaling limits from select box on change + limit_reset = CustomJS( + args={'setting': preset_select, 'limits': preset_limits, 'low': limit_low, + 'high': limit_high, 'scale': scale_group}, + code=""" + if (setting.value != "Custom") { + if (scale.active == 1 && limits[setting.value][0] <= 0) { + low.value = 0.0001; + } else { + low.value = limits[setting.value][0]; + } + high.value = limits[setting.value][1]; + } + """) + preset_select.js_on_change('value', limit_reset) + + # set scaling limits from text boxes on change + for i in range(len(images)): + limit_low.js_link('value', images[i].glyph.color_mapper, 'low') + limit_low.js_link('value', color_bars[i].color_mapper, 'low') + limit_high.js_link('value', images[i].glyph.color_mapper, 'high') + limit_high.js_link('value', color_bars[i].color_mapper, 'high') + + # reset boxes to preset range on button click + reset.js_on_click(limit_reset) + + # also reset when swapping limit style + scale_group.js_on_click(limit_reset) + + # return widgets + spacer = Spacer(height=20) + return [tools_label, scale_set, preset_set, limit_set, reset, spacer] def create_figure_title(self): """Create title for the image""" diff --git a/jwql/website/apps/jwql/static/css/jwql.css b/jwql/website/apps/jwql/static/css/jwql.css index 8431c82f3..eb567b25f 100644 --- a/jwql/website/apps/jwql/static/css/jwql.css +++ b/jwql/website/apps/jwql/static/css/jwql.css @@ -23,24 +23,31 @@ width: 100%; margin: auto; } - + + /* Make sure bokeh widgets match default theme */ + .bk-root, .bk-btn { + font-family: 'Overpass', sans-serif !important; + font-size: 1rem !important; + flex-grow: 0 !important; + } + /*Make normal buttons and highlighted outline buttons orange*/ .btn-primary, .btn-primary.disabled, .btn-outline-primary:hover, - .btn-outline-primary.active { + .btn-outline-primary.active, .bk-btn-primary { background-color: #c85108 !important; border-color: #c85108 !important; color: white !important; - border-radius: 0px; + border-radius: 0px !important; text-decoration: none; } /*Make outline buttons and highlighted normal buttons white*/ .btn-primary:hover, .btn-primary.active, .btn-outline-primary, - .show > .btn-primary.dropdown-toggle { + .show > .btn-primary.dropdown-toggle, .bk-btn-primary:hover { background-color: white !important; border-color: #c85108 !important ; color: #c85108 !important; - border-radius: 0px; + border-radius: 0px !important; text-decoration: none; } @@ -544,7 +551,7 @@ } /*Format thumbnail groups when active*/ - .thumbnail-group { + .thumbnail-group { display: inline; font-size: 0.75rem; } diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index e86dfa9c6..705931280 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -1075,9 +1075,6 @@ function update_wata_page(base_url) { var calc_difference = document.getElementById("calcDifference").checked; // Get the arguments to update - var scaling = get_radio_button_value("scaling"); - var low_lim = get_number_or_none("low_lim"); - var high_lim = get_number_or_none("high_lim"); var ext_name = get_radio_button_value("extension"); var int1_nr = get_number_or_none("integration1"); var grp1_nr = get_number_or_none("group1"); @@ -1090,7 +1087,7 @@ function update_wata_page(base_url) { int2_nr="None"; grp2_nr="None"; } - optional_params = optional_params + "/scaling_" + scaling + "/low_" + low_lim + "/high_" + high_lim + "/ext_" + ext_name + "/int1_" + int1_nr + "/grp1_" + grp1_nr + "/int2_" + int2_nr + "/grp2_" + grp2_nr; + optional_params = optional_params + "/ext_" + ext_name + "/int1_" + int1_nr + "/grp1_" + grp1_nr + "/int2_" + int2_nr + "/grp2_" + grp2_nr; } $.ajax({ diff --git a/jwql/website/apps/jwql/templates/explore_image.html b/jwql/website/apps/jwql/templates/explore_image.html index cced80f37..f72a92081 100644 --- a/jwql/website/apps/jwql/templates/explore_image.html +++ b/jwql/website/apps/jwql/templates/explore_image.html @@ -17,59 +17,12 @@

    {{ file_root }}_{{ filetype }}.fits

    View Proposal

    - -

    Image Settings

    -
    - Scaling:   - log      - linear    -
    - Scale Limits:   -  Low      -  High      -
    -
    - Extension:   - {% for extension in extensions %} - {% if extension == 'SCI' %} - {{ extension }}      - {% else %} - {{ extension }}      - {% endif %} - {% endfor %} -
    -
    - - -
    -
    - - -
    -
    -
    - - -
    -
    -
    - - -
    -
    - - -
    -
    -
    -

    - Apply Settings -

    +
    -
    +
    -
    +
    @@ -88,6 +41,46 @@

    Image Settings

    + + +

    Data Settings

    +
    + Extension:   + {% for extension in extensions %} + {% if extension == 'SCI' %} + {{ extension }}      + {% else %} + {{ extension }}      + {% endif %} + {% endfor %} +
    +
    + + +
    +
    + + +
    +
    +
    + + +
    +
    +
    + + +
    +
    + + +
    +
    +
    +

    + Apply Settings +

    diff --git a/jwql/website/apps/jwql/urls.py b/jwql/website/apps/jwql/urls.py index 4a6227047..c0535b378 100644 --- a/jwql/website/apps/jwql/urls.py +++ b/jwql/website/apps/jwql/urls.py @@ -96,7 +96,7 @@ re_path('ajax/query_submit/', views.archive_thumbnails_query_ajax, name='archive_thumb_query_ajax'), re_path(r'^ajax/(?P({}))/archive/$'.format(instruments), views.archived_proposals_ajax, name='archive_ajax'), re_path(r'^ajax/(?P({}))/(?P.+)_(?P.+)/explore_image/$'.format(instruments), views.explore_image_ajax, name='explore_image_ajax'), - re_path(r'^ajax/(?P({}))/(?P.+)_(?P.+)/explore_image/scaling_(?P.+)/low_(?P.+)/high_(?P.+)/ext_(?P.+)/int1_(?P.+)/grp1_(?P.+)/int2_(?P.+)/grp2_(?P.+)/$'.format(instruments), views.explore_image_ajax, name='explore_image_ajax'), + re_path(r'^ajax/(?P({}))/(?P.+)_(?P.+)/explore_image/ext_(?P.+)/int1_(?P.+)/grp1_(?P.+)/int2_(?P.+)/grp2_(?P.+)/$'.format(instruments), views.explore_image_ajax, name='explore_image_ajax'), re_path(r'^ajax/(?P({}))/archive/(?P[\d]{{1,5}})/obs(?P[\d]{{1,3}})/$'.format(instruments), views.archive_thumbnails_ajax, name='archive_thumb_ajax'), re_path(r'^ajax/viewed/(?P.+)/$', views.toggle_viewed_ajax, name='toggle_viewed_ajax'), re_path(r'^ajax/viewed_group/(?P.+)/(?P(viewed|new|Viewed|New))/$', From 6815685f57cc6be99871bcd530ca91a469464502 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Thu, 8 Jun 2023 16:27:33 -0400 Subject: [PATCH 379/449] Fix accidental commit for TA monitors --- .../ta_monitors/msata_monitor.py | 322 +++++------------- .../ta_monitors/wata_monitor.py | 232 ++++--------- 2 files changed, 156 insertions(+), 398 deletions(-) diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py index 55abcf95e..bc21a4643 100755 --- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py +++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py @@ -24,7 +24,6 @@ Author ______ - Maria Pena-Guerrero - - Melanie Clarke Use --- @@ -39,7 +38,6 @@ import os import logging import shutil -from datetime import datetime, timezone, timedelta from random import randint import numpy as np @@ -47,13 +45,12 @@ from astropy.time import Time from astropy.io import fits from bokeh.embed import components -from bokeh.layouts import gridplot, layout -from bokeh.models import ( - ColumnDataSource, Range1d, CustomJS, CustomJSFilter, CDSView, - Span, Label, DateRangeSlider) -from bokeh.models.tools import HoverTool, BoxSelectTool -from bokeh.plotting import figure, save, output_file +from bokeh.plotting import figure, output_file, save +from bokeh.layouts import gridplot +from bokeh.models import ColumnDataSource, Range1d, Span, Label +from bokeh.models.tools import HoverTool from bs4 import BeautifulSoup +from datetime import datetime from sqlalchemy.sql.expression import and_ # jwql imports @@ -62,11 +59,11 @@ from jwql.utils import monitor_utils from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE from jwql.utils.logging_functions import log_info, log_fail -from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config +from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config, filename_parser class MSATA(): - """ Class for executing the NIRSpec MSATA monitor. + """ Class for executint the NIRSpec MSATA monitor. This class will search for new MSATA current files in the file systems for NIRSpec and will run the monitor on these files. The monitor will @@ -139,13 +136,6 @@ def __init__(self): 'planned_v3': {'loc': 'ta_table', 'alt_key': None, 'name': 'planned_v3', 'type': float}, 'FITTARGS': {'loc': 'ta_hdr', 'alt_key': None, 'name': 'stars_in_fit', 'type': int}} - # initialize attributes to be set later - self.source = None - self.share_tools = [] - self.date_range = None - self.date_filter = None - self.date_view = None - def get_tainfo_from_fits(self, fits_file): """ Get the TA information from the fits file Parameters @@ -250,19 +240,6 @@ def get_msata_data(self, new_filenames): msata_df = pd.DataFrame(msata_dict) return msata_df, no_ta_ext_msgs - def add_time_column(self): - """Add time column to data source, to be used by all plots.""" - date_obs = self.source.data['date_obs'] - if 'time_arr' not in self.source.data: - time_arr = [] - for do_str in date_obs: - # convert time string into an array of time (this is in UT) - t = datetime.fromisoformat(do_str) - time_arr.append(t) - - # add to the bokeh data structure - self.source.data["time_arr"] = time_arr - def plt_status(self): """ Plot the MSATA status versus time. Parameters @@ -272,12 +249,18 @@ def plt_status(self): ------- plot: bokeh plot object """ + # ta_status, date_obs = source.data['ta_status'], source.data['date_obs'] + date_obs = self.source.data['date_obs'] ta_status = self.source.data['ta_status'] # check if this column exists in the data already (the other 2 will exist too), else create it - if 'bool_status' not in self.source.data: - # bokeh does not like to plot strings, turn into numbers - number_status, status_colors = [], [] - for tas in ta_status: + try: + time_arr = self.source.data['time_arr'] + bool_status = self.source.data['bool_status'] + status_colors = self.source.data['status_colors'] + except KeyError: + # bokeh does not like to plot strings, turn into numbers + number_status, time_arr, status_colors = [], [], [] + for tas, do_str in zip(ta_status, date_obs): if tas.lower() == 'unsuccessful': number_status.append(0.0) status_colors.append('red') @@ -287,19 +270,19 @@ def plt_status(self): else: number_status.append(1.0) status_colors.append('blue') - + # convert time string into an array of time (this is in UT with 0.0 milliseconds) + t = datetime.fromisoformat(do_str) + time_arr.append(t) # add these to the bokeh data structure + self.source.data["time_arr"] = time_arr self.source.data["number_status"] = number_status self.source.data["status_colors"] = status_colors - # create a new bokeh plot plot = figure(title="MSATA Status [Success=1, In Progress=0.5, Fail=0]", x_axis_label='Time', y_axis_label='MSATA Status', x_axis_type='datetime',) plot.y_range = Range1d(-0.5, 1.5) plot.circle(x='time_arr', y='number_status', source=self.source, - color='status_colors', size=7, fill_alpha=0.3, view=self.date_view) - - # add tooltips + color='status_colors', size=7, fill_alpha=0.3) hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -310,11 +293,8 @@ def plt_status(self): ('Date-Obs', '@date_obs'), ('Subarray', '@subarray'), ('--------', '----------------')] - plot.add_tools(hover) - # add shared selection tools - for tool in self.share_tools: - plot.add_tools(tool) + plot.add_tools(hover) return plot def plt_residual_offsets(self): @@ -331,8 +311,7 @@ def plt_residual_offsets(self): x_axis_label='Least Squares Residual V2 Offset', y_axis_label='Least Squares Residual V3 Offset') plot.circle(x='lsv2offset', y='lsv3offset', source=self.source, - color="blue", size=7, fill_alpha=0.3, view=self.date_view) - + color="blue", size=7, fill_alpha=0.3) v2halffacet, v3halffacet = self.source.data['v2halffacet'], self.source.data['v3halffacet'] xstart, ystart, ray_length = -1 * v2halffacet[0], -1 * v3halffacet[0], 0.05 plot.ray(x=xstart - ray_length / 2.0, y=ystart, length=ray_length, angle_units="deg", @@ -343,13 +322,10 @@ def plt_residual_offsets(self): plot.add_layout(hflabel) plot.x_range = Range1d(-0.5, 0.5) plot.y_range = Range1d(-0.5, 0.5) - # mark origin lines vline = Span(location=0, dimension='height', line_color='black', line_width=0.7) hline = Span(location=0, dimension='width', line_color='black', line_width=0.7) plot.renderers.extend([vline, hline]) - - # add tooltips hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -362,11 +338,8 @@ def plt_residual_offsets(self): ('LS V2 offset', '@lsv2offset'), ('LS V3 offset', '@lsv3offset'), ('--------', '----------------')] - plot.add_tools(hover) - # add shared selection tools - for tool in self.share_tools: - plot.add_tools(tool) + plot.add_tools(hover) return plot def plt_v2offset_time(self): @@ -382,9 +355,8 @@ def plt_v2offset_time(self): plot = figure(title="MSATA Least Squares V2 Offset vs Time", x_axis_label='Time', y_axis_label='Least Squares Residual V2 Offset', x_axis_type='datetime') plot.circle(x='time_arr', y='lsv2offset', source=self.source, - color="blue", size=7, fill_alpha=0.3, view=self.date_view) + color="blue", size=7, fill_alpha=0.3) plot.y_range = Range1d(-0.5, 0.5) - # mark origin line hline = Span(location=0, dimension='width', line_color='black', line_width=0.7) time_arr, v2halffacet = self.source.data['time_arr'], self.source.data['v2halffacet'] @@ -392,8 +364,6 @@ def plt_v2offset_time(self): plot.renderers.extend([hline, hfline]) hflabel = Label(x=time_arr[-1], y=-1 * v2halffacet[0], y_units='data', text='-V2 half-facet value') plot.add_layout(hflabel) - - # add tooltips hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -406,11 +376,8 @@ def plt_v2offset_time(self): ('LS V2 offset', '@lsv2offset'), ('LS V3 offset', '@lsv3offset'), ('--------', '----------------')] - plot.add_tools(hover) - # add shared selection tools - for tool in self.share_tools: - plot.add_tools(tool) + plot.add_tools(hover) return plot def plt_v3offset_time(self): @@ -426,9 +393,8 @@ def plt_v3offset_time(self): plot = figure(title="MSATA Least Squares V3 Offset vs Time", x_axis_label='Time', y_axis_label='Least Squares Residual V3 Offset', x_axis_type='datetime') plot.circle(x='time_arr', y='lsv3offset', source=self.source, - color="blue", size=7, fill_alpha=0.3, view=self.date_view) + color="blue", size=7, fill_alpha=0.3) plot.y_range = Range1d(-0.5, 0.5) - # mark origin line hline = Span(location=0, dimension='width', line_color='black', line_width=0.7) time_arr, v3halffacet = self.source.data['time_arr'], self.source.data['v3halffacet'] @@ -436,8 +402,6 @@ def plt_v3offset_time(self): plot.renderers.extend([hline, hfline]) hflabel = Label(x=time_arr[-1], y=-1 * v3halffacet[0], y_units='data', text='-V3 half-facet value') plot.add_layout(hflabel) - - # add tooltips hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -450,11 +414,8 @@ def plt_v3offset_time(self): ('LS V2 offset', '@lsv2offset'), ('LS V3 offset', '@lsv3offset'), ('--------', '----------------')] - plot.add_tools(hover) - # add shared selection tools - for tool in self.share_tools: - plot.add_tools(tool) + plot.add_tools(hover) return plot def plt_lsv2v3offsetsigma(self): @@ -471,16 +432,13 @@ def plt_lsv2v3offsetsigma(self): x_axis_label='Least Squares Residual V2 Sigma Offset', y_axis_label='Least Squares Residual V3 Sigma Offset') plot.circle(x='lsv2sigma', y='lsv3sigma', source=self.source, - color="blue", size=7, fill_alpha=0.3, view=self.date_view) + color="blue", size=7, fill_alpha=0.3) plot.x_range = Range1d(-0.1, 0.1) plot.y_range = Range1d(-0.1, 0.1) - # mark origin lines vline = Span(location=0, dimension='height', line_color='black', line_width=0.7) hline = Span(location=0, dimension='width', line_color='black', line_width=0.7) plot.renderers.extend([vline, hline]) - - # add tooltips hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -495,11 +453,8 @@ def plt_lsv2v3offsetsigma(self): ('LS V3 offset', '@lsv3offset'), ('LS V3 sigma', '@lsv3sigma'), ('--------', '----------------')] - plot.add_tools(hover) - # add shared selection tools - for tool in self.share_tools: - plot.add_tools(tool) + plot.add_tools(hover) return plot def plt_res_offsets_corrected(self): @@ -511,30 +466,29 @@ def plt_res_offsets_corrected(self): ------- plot: bokeh plot object """ + # create a new bokeh plot lsv2offset, lsv3offset = self.source.data['lsv2offset'], self.source.data['lsv3offset'] v2halffacet, v3halffacet = self.source.data['v2halffacet'], self.source.data['v3halffacet'] - - # check if this column exists in the data already, else create it - if 'v2_half_fac_corr' not in self.source.data: + # check if this column exists in the data already (the other 2 will exist too), else create it + try: + v2_half_fac_corr = self.source.data['v2_half_fac_corr'] + v3_half_fac_corr = self.source.data['v3_half_fac_corr'] + except KeyError: v2_half_fac_corr, v3_half_fac_corr = [], [] for idx, v2hf in enumerate(v2halffacet): v3hf = v3halffacet[idx] v2_half_fac_corr.append(lsv2offset[idx] + v2hf) v3_half_fac_corr.append(lsv3offset[idx] + v3hf) - # add these to the bokeh data structure self.source.data["v2_half_fac_corr"] = v2_half_fac_corr self.source.data["v3_half_fac_corr"] = v3_half_fac_corr - - # create a new bokeh plot plot = figure(title="MSATA Least Squares Residual V2-V3 Offsets Half-facet corrected", x_axis_label='Least Squares Residual V2 Offset + half-facet', y_axis_label='Least Squares Residual V3 Offset + half-facet') plot.circle(x='v2_half_fac_corr', y='v3_half_fac_corr', source=self.source, - color="blue", size=7, fill_alpha=0.3, view=self.date_view) + color="blue", size=7, fill_alpha=0.3) plot.x_range = Range1d(-0.5, 0.5) plot.y_range = Range1d(-0.5, 0.5) - # mark origin lines vline = Span(location=0, dimension='height', line_color='black', line_width=0.7) hline = Span(location=0, dimension='width', line_color='black', line_width=0.7) @@ -546,8 +500,6 @@ def plt_res_offsets_corrected(self): angle=90, line_color='purple', line_width=3) hflabel = Label(x=xstart / 3.0, y=ystart, y_units='data', text='-V2, -V3 half-facets values') plot.add_layout(hflabel) - - # add tooltips hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -562,16 +514,15 @@ def plt_res_offsets_corrected(self): ('V2 half-facet', '@v2halffacet'), ('V3 half-facet', '@v3halffacet'), ('--------', '----------------')] - plot.add_tools(hover) - # add shared selection tools - for tool in self.share_tools: - plot.add_tools(tool) + plot.add_tools(hover) return plot def plt_v2offsigma_time(self): - """Plot the residual Least Squares V2 sigma Offset versus time - + """ Plot the residual Least Squares V2 sigma Offset versus time + Parameters + ---------- + None Returns ------- plot: bokeh plot object @@ -580,14 +531,11 @@ def plt_v2offsigma_time(self): plot = figure(title="MSATA Least Squares V2 Sigma Offset vs Time", x_axis_label='Time', y_axis_label='Least Squares Residual V2 Sigma Offset', x_axis_type='datetime') plot.circle(x='time_arr', y='lsv2sigma', source=self.source, - color="blue", size=7, fill_alpha=0.3, view=self.date_view) + color="blue", size=7, fill_alpha=0.3) plot.y_range = Range1d(-0.1, 0.1) - # mark origin line hline = Span(location=0, dimension='width', line_color='black', line_width=0.7) plot.renderers.extend([hline]) - - # add tooltips hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -599,34 +547,28 @@ def plt_v2offsigma_time(self): ('LS V2 offset', '@lsv2offset'), ('LS V2 sigma', '@lsv2sigma'), ('--------', '----------------')] - plot.add_tools(hover) - # add shared selection tools - for tool in self.share_tools: - plot.add_tools(tool) + plot.add_tools(hover) return plot def plt_v3offsigma_time(self): - """Plot the residual Least Squares V3 Offset versus time - + """ Plot the residual Least Squares V3 Offset versus time + Parameters + ---------- + None Returns ------- p: bokeh plot object """ # create a new bokeh plot - plot = figure(title="MSATA Least Squares V3 Sigma Offset vs Time", - x_axis_label='Time', - y_axis_label='Least Squares Residual V3 Sigma Offset', - x_axis_type='datetime') + plot = figure(title="MSATA Least Squares V3 Sigma Offset vs Time", x_axis_label='Time', + y_axis_label='Least Squares Residual V3 Sigma Offset', x_axis_type='datetime') plot.circle(x='time_arr', y='lsv3sigma', source=self.source, - color="blue", size=7, fill_alpha=0.3, view=self.date_view) + color="blue", size=7, fill_alpha=0.3) plot.y_range = Range1d(-0.1, 0.1) - # mark origin line hline = Span(location=0, dimension='width', line_color='black', line_width=0.7) plot.renderers.extend([hline]) - - # add tooltips hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -639,11 +581,8 @@ def plt_v3offsigma_time(self): ('LS V3 offset', '@lsv3offset'), ('LS V3 sigma', '@lsv3sigma'), ('--------', '----------------')] - plot.add_tools(hover) - # add shared selection tools - for tool in self.share_tools: - plot.add_tools(tool) + plot.add_tools(hover) return plot def plt_roll_offset(self): @@ -659,12 +598,10 @@ def plt_roll_offset(self): plot = figure(title="MSATA Least Squares Roll Offset vs Time", x_axis_label='Time', y_axis_label='Least Squares Residual Roll Offset', x_axis_type='datetime') plot.circle(x='time_arr', y='lsrolloffset', source=self.source, - color="blue", size=7, fill_alpha=0.3, view=self.date_view) + color="blue", size=7, fill_alpha=0.3) plot.y_range = Range1d(-600.0, 600.0) - # mark origin line hline = Span(location=0, dimension='width', line_color='black', line_width=0.7) - # Maximum accepted roll line and label time_arr = self.source.data['time_arr'] arlinepos = Span(location=120, dimension='width', line_color='green', line_width=3) @@ -672,8 +609,6 @@ def plt_roll_offset(self): arlabel = Label(x=time_arr[-1], y=125, y_units='data', text='Max accepted roll') plot.add_layout(arlabel) plot.renderers.extend([hline, arlinepos, arlineneg]) - - # add tooltips hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -686,11 +621,8 @@ def plt_roll_offset(self): ('LS V2 offset', '@lsv2offset'), ('LS V3 offset', '@lsv3offset'), ('--------', '----------------')] - plot.add_tools(hover) - # add shared selection tools - for tool in self.share_tools: - plot.add_tools(tool) + plot.add_tools(hover) return plot def plt_lsoffsetmag(self): @@ -706,14 +638,11 @@ def plt_lsoffsetmag(self): plot = figure(title="MSATA Least Squares Total Magnitude of the Linear V2, V3 Offset Slew vs Time", x_axis_label='Time', y_axis_label='sqrt((V2_off)**2 + (V3_off)**2)', x_axis_type='datetime') plot.circle(x='time_arr', y='lsoffsetmag', source=self.source, - color="blue", size=7, fill_alpha=0.3, view=self.date_view) + color="blue", size=7, fill_alpha=0.3) plot.y_range = Range1d(-0.5, 0.5) - # mark origin line hline = Span(location=0, dimension='width', line_color='black', line_width=0.7) plot.renderers.extend([hline]) - - # add tooltips hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -727,11 +656,8 @@ def plt_lsoffsetmag(self): ('LS V2 offset', '@lsv2offset'), ('LS V3 offset', '@lsv3offset'), ('--------', '----------------')] - plot.add_tools(hover) - # add shared selection tools - for tool in self.share_tools: - plot.add_tools(tool) + plot.add_tools(hover) return plot def plt_tot_number_of_stars(self): @@ -746,34 +672,34 @@ def plt_tot_number_of_stars(self): # get the number of stars per array visit_id = self.source.data['visit_id'] reference_star_number = self.source.data['reference_star_number'] - - # check if this column exists in the data already, else create it - if 'tot_number_of_stars' not in self.source.data: + stars_in_fit = self.source.data['stars_in_fit'] + date_obs, time_arr = self.source.data['date_obs'], self.source.data['time_arr'] + # check if this column exists in the data already (the other 2 will exist too), else create it + try: + tot_number_of_stars = self.source.data['tot_number_of_stars'] + colors_list = self.source.data['colors_list'] + except KeyError: # create the list of color per visit and tot_number_of_stars colors_list, tot_number_of_stars = [], [] color_dict = {} - for i, vid in enumerate(visit_id): + for i, _ in enumerate(visit_id): tot_stars = len(reference_star_number[i]) tot_number_of_stars.append(tot_stars) ci = '#%06X' % randint(0, 0xFFFFFF) - if vid not in color_dict: - color_dict[vid] = ci - colors_list.append(color_dict[vid]) - + if visit_id[i] not in color_dict: + color_dict[visit_id[i]] = ci + colors_list.append(color_dict[visit_id[i]]) # add these to the bokeh data structure self.source.data["tot_number_of_stars"] = tot_number_of_stars self.source.data["colors_list"] = colors_list - # create a new bokeh plot plot = figure(title="Total Number of Measurements vs Time", x_axis_label='Time', y_axis_label='Total number of measurements', x_axis_type='datetime') plot.circle(x='time_arr', y='tot_number_of_stars', source=self.source, - color='colors_list', size=7, fill_alpha=0.3, view=self.date_view) + color='colors_list', size=7, fill_alpha=0.3) plot.triangle(x='time_arr', y='stars_in_fit', source=self.source, - color='black', size=7, fill_alpha=0.3, view=self.date_view) + color='black', size=7, fill_alpha=0.3) plot.y_range = Range1d(0.0, 40.0) - - # add tooltips hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -788,8 +714,8 @@ def plt_tot_number_of_stars(self): ('LS V2 offset', '@lsv2offset'), ('LS V3 offset', '@lsv3offset'), ('--------', '----------------')] - plot.add_tools(hover) + plot.add_tools(hover) return plot def plt_mags_time(self): @@ -809,24 +735,22 @@ def plt_mags_time(self): planned_v2 = self.source.data['planned_v2'] planned_v3 = self.source.data['planned_v3'] reference_star_number = self.source.data['reference_star_number'] + visits_stars_mags = self.source.data['reference_star_mag'] box_peak_value = self.source.data['box_peak_value'] date_obs, time_arr = self.source.data['date_obs'], self.source.data['time_arr'] colors_list = self.source.data['colors_list'] detector_list = self.source.data['detector'] - filename = self.source.data['filename'] - # create the structure matching the number of visits and reference stars new_colors_list, vid, dobs, tarr, star_no, status = [], [], [], [], [], [] - peaks, stars_v2, stars_v3, det, fnames = [], [], [], [], [] + peaks, stars_v2, stars_v3, det = [], [], [], [] for i, _ in enumerate(visit_id): - v, d, t, c, s, x, y, dt, fn = [], [], [], [], [], [], [], [], [] + v, d, t, c, s, x, y, dt = [], [], [], [], [], [], [], [] for j in range(len(reference_star_number[i])): v.append(visit_id[i]) d.append(date_obs[i]) t.append(time_arr[i]) c.append(colors_list[i]) dt.append(detector_list[i]) - fn.append(filename[i]) if 'not_removed' in lsf_removed_status[i][j]: s.append('SUCCESS') x.append(planned_v2[i][j]) @@ -845,29 +769,17 @@ def plt_mags_time(self): stars_v3.extend(y) peaks.extend(box_peak_value[i]) det.extend(dt) - fnames.extend(fn) - # now create the mini ColumnDataSource for this particular plot mini_source = {'vid': vid, 'star_no': star_no, 'status': status, - 'dobs': dobs, 'time_arr': tarr, 'det': det, 'fname': fnames, + 'dobs': dobs, 'tarr': tarr, 'det': det, 'peaks': peaks, 'colors_list': new_colors_list, 'stars_v2': stars_v2, 'stars_v3': stars_v3} mini_source = ColumnDataSource(data=mini_source) - - # hook up the date range slider to this source as well - callback = CustomJS(args=dict(s=mini_source), code=""" - s.change.emit(); - """) - self.date_range.js_on_change('value', callback) - mini_view = CDSView(source=mini_source, filters=[self.date_filter]) - - # create the bokeh plot - plot = figure(title="MSATA Counts vs Time", x_axis_label='Time', - y_axis_label='box_peak [Counts]', + # create a the bokeh plot + plot = figure(title="MSATA Counts vs Time", x_axis_label='Time', y_axis_label='box_peak [Counts]', x_axis_type='datetime') - plot.circle(x='time_arr', y='peaks', source=mini_source, - color='colors_list', size=7, fill_alpha=0.3, view=mini_view) - + plot.circle(x='tarr', y='peaks', source=mini_source, + color='colors_list', size=7, fill_alpha=0.3) # add count saturation warning lines loc1, loc2, loc3 = 45000.0, 50000.0, 60000.0 hline1 = Span(location=loc1, dimension='width', line_color='green', line_width=3) @@ -881,10 +793,9 @@ def plt_mags_time(self): plot.add_layout(label2) plot.add_layout(label3) plot.y_range = Range1d(-1000.0, 62000.0) - - # add tooltips + # add hover hover = HoverTool() - hover.tooltips = [('File name', '@fname'), + hover.tooltips = [('File name', '@filename'), ('Visit ID', '@vid'), ('Detector', '@det'), ('Star No.', '@star_no'), @@ -894,69 +805,18 @@ def plt_mags_time(self): ('Measured V2', '@stars_v2'), ('Measured V3', '@stars_v3'), ('--------', '----------------')] - plot.add_tools(hover) + plot.add_tools(hover) return plot - def setup_date_range(self): - """Set up a date range filter, defaulting to the last week of data.""" - end_date = datetime.now(tz=timezone.utc) - one_week_ago = end_date.date() - timedelta(days=7) - first_data_point = np.min(self.source.data['time_arr']).date() - last_data_point = np.max(self.source.data['time_arr']).date() - if last_data_point < one_week_ago: - # keep at least one point in the plot if there was - # no TA data this week - start_date = last_data_point - else: - start_date = one_week_ago - - # allowed range is from the first ever data point to today - self.date_range = DateRangeSlider( - title="Date range displayed", start=first_data_point, - end=end_date, value=(start_date, end_date), step=1) - - callback = CustomJS(args=dict(s=self.source), code=""" - s.change.emit(); - """) - self.date_range.js_on_change('value', callback) - - self.date_filter = CustomJSFilter(args=dict(slider=self.date_range), code=""" - var indices = []; - var start = slider.value[0]; - var end = slider.value[1]; - - for (var i=0; i < source.get_length(); i++) { - if (source.data['time_arr'][i] >= start - && source.data['time_arr'][i] <= end) { - indices.push(true); - } else { - indices.push(false); - } - } - return indices; - """) - self.date_view = CDSView(source=self.source, filters=[self.date_filter]) - def mk_plt_layout(self): """Create the bokeh plot layout""" self.source = ColumnDataSource(data=self.msata_data) - # make sure all arrays are lists in order to later be able to read the data # from the html file for item in self.source.data: if not isinstance(self.source.data[item], (str, float, int, list)): self.source.data[item] = self.source.data[item].tolist() - - # add a time array to the data source - self.add_time_column() - - # set up selection tools to share - self.share_tools = [BoxSelectTool()] - - # set up a date range filter widget - self.setup_date_range() - # set the output html file name and create the plot grid output_file(self.output_file_name) p1 = self.plt_status() @@ -971,15 +831,13 @@ def mk_plt_layout(self): p10 = self.plt_lsoffsetmag() p12 = self.plt_tot_number_of_stars() p11 = self.plt_mags_time() - # make grid grid = gridplot([p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12], ncols=2, merge_tools=False) - box_layout = layout(children=[self.date_range, grid]) - save(box_layout) - - # return the needed components for embedding the results in the MSATA html template - script, div = components(box_layout) + # show(grid) + save(grid) + # return the needed components for embeding the results in the MSATA html template + script, div = components(grid) return script, div def identify_tables(self): @@ -1026,7 +884,7 @@ def get_data_from_html(self, html_file): File created by the monitor script Returns ------- - prev_data_dict: dict + prev_data_dict: dictionary Dictionary containing all data used in the plots """ @@ -1103,7 +961,7 @@ def prev_data2expected_format(self, prev_data_dict): Date of the latest observation in the previously plotted data """ # remember that the time array created is in milliseconds, removing to get time object - time_in_millis = max(prev_data_dict['time_arr']) + time_in_millis = max(prev_data_dict['tarr']) latest_prev_obs = Time(time_in_millis / 1000., format='unix') latest_prev_obs = latest_prev_obs.mjd prev_data_expected_cols = {} @@ -1193,13 +1051,13 @@ def update_ta_success_txtfile(self): output_success_ta_txtfile = os.path.join(self.output_dir, "msata_success.txt") # check if previous file exsists and read the data from it if os.path.isfile(output_success_ta_txtfile): - # now rename the previous file, for backup + # now rename the the previous file, for backup os.rename(output_success_ta_txtfile, os.path.join(self.output_dir, "prev_msata_success.txt")) # get the new data ta_success, ta_inprogress, ta_failure = [], [], [] filenames, ta_status = self.msata_data.loc[:,'filename'], self.msata_data.loc[:,'ta_status'] for fname, ta_stat in zip(filenames, ta_status): - # select the appropriate list to append to + # select the appriopriate list to append to if ta_stat == 'SUCCESSFUL': ta_success.append(fname) elif ta_stat == 'IN_PROGRESS': diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py index f248a1d09..01e52a44c 100755 --- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py +++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py @@ -10,7 +10,7 @@ """ -This module contains the code for the NIRSpec Wide Aperture Target +This module contains the code for NIRSpec the Wide Aperture Target Acquisition (WATA) monitor, which monitors the TA offsets. This monitor displays the comparison of desired versus measured TA. @@ -20,7 +20,6 @@ Author ______ - Maria Pena-Guerrero - - Melanie Clarke Use --- @@ -35,22 +34,21 @@ import os import logging import shutil -from datetime import datetime, timezone, timedelta import numpy as np import pandas as pd +from bs4 import BeautifulSoup +from datetime import datetime from astropy.time import Time from astropy.io import fits -from bokeh.embed import components -from bokeh.io import output_file -from bokeh.layouts import gridplot, layout -from bokeh.models import ( - ColumnDataSource, Range1d, CustomJS, CustomJSFilter, CDSView, - Span, Label, DateRangeSlider) -from bokeh.models.tools import HoverTool, BoxSelectTool -from bokeh.plotting import figure, save -from bs4 import BeautifulSoup from sqlalchemy.sql.expression import and_ +from bokeh.io import output_file +from bokeh.plotting import figure, show, save +from bokeh.models import ColumnDataSource, Range1d +from bokeh.models.tools import HoverTool +from bokeh.layouts import gridplot +from bokeh.models import Span, Label +from bokeh.embed import components # jwql imports from jwql.utils.logging_functions import log_info, log_fail @@ -62,7 +60,7 @@ class WATA(): - """ Class for executing the NIRSpec WATA monitor. + """ Class for executint the NIRSpec WATA monitor. This class will search for new WATA current files in the file systems for NIRSpec and will run the monitor on these files. The monitor will @@ -132,12 +130,6 @@ def __init__(self): 'SAM_X': {'loc': 'ta_hdr', 'alt_key': None, 'name': 'sam_x'}, 'SAM_Y': {'loc': 'ta_hdr', 'alt_key': None, 'name': 'sam_y'}} - # initialize attributes to be set later - self.source = None - self.share_tools = [] - self.date_range = None - self.date_view = None - def get_tainfo_from_fits(self, fits_file): """ Get the TA information from the fits file Parameters @@ -208,19 +200,6 @@ def get_wata_data(self, new_filenames): wata_df = pd.DataFrame(wata_dict) return wata_df, no_ta_ext_msgs - def add_time_column(self): - """Add time column to data source, to be used by all plots.""" - date_obs = self.source.data['date_obs'] - if 'time_arr' not in self.source.data: - time_arr = [] - for do_str in date_obs: - # convert time string into an array of time (this is in UT) - t = datetime.fromisoformat(do_str) - time_arr.append(t) - - # add to the bokeh data structure - self.source.data["time_arr"] = time_arr - def plt_status(self): """ Plot the WATA status (passed = 0 or failed = 1). Parameters @@ -230,32 +209,36 @@ def plt_status(self): ------- plot: bokeh plot object """ - ta_status = self.source.data['ta_status'] - - # check if this column exists in the data already, else create it - if 'bool_status' not in self.source.data: - # bokeh does not like to plot strings, turn into binary type - bool_status, status_colors = [], [] - for tas in ta_status: + ta_status, date_obs = self.source.data['ta_status'], self.source.data['date_obs'] + # check if this column exists in the data already (the other 2 will exist too), else create it + try: + time_arr = self.source.data['time_arr'] + bool_status = self.source.data['bool_status'] + status_colors = self.source.data['status_colors'] + except KeyError: + # bokeh does not like to plot strings, turn into binary type + bool_status, time_arr, status_colors = [], [], [] + for tas, do_str in zip(ta_status, date_obs): if 'unsuccessful' not in tas.lower(): bool_status.append(1) status_colors.append('blue') else: bool_status.append(0) status_colors.append('red') - + # convert time string into an array of time (this is in UT) + t = datetime.fromisoformat(do_str) + time_arr.append(t) # add these to the bokeh data structure + self.source.data["time_arr"] = time_arr self.source.data["ta_status_bool"] = bool_status self.source.data["status_colors"] = status_colors - # create a new bokeh plot - plot = figure(title="WATA Status [Success=1, Fail=0]", x_axis_label='Time', + plot = figure(title="WATA Status [Succes=1, Fail=0]", x_axis_label='Time', y_axis_label='WATA Status', x_axis_type='datetime',) plot.y_range = Range1d(-0.5, 1.5) plot.circle(x='time_arr', y='ta_status_bool', source=self.source, - color='status_colors', size=7, fill_alpha=0.3, view=self.date_view) - - # make tooltips + color='status_colors', size=7, fill_alpha=0.3) + # output_file("wata_status.html") hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -267,10 +250,6 @@ def plt_status(self): ('--------', '----------------')] plot.add_tools(hover) - - # add shared selection tools - for tool in self.share_tools: - plot.add_tools(tool) return plot def plt_residual_offsets(self): @@ -286,16 +265,13 @@ def plt_residual_offsets(self): plot = figure(title="WATA Residual V2-V3 Offsets", x_axis_label='Residual V2 Offset', y_axis_label='Residual V3 Offset') plot.circle(x='v2_offset', y='v3_offset', source=self.source, - color="blue", size=7, fill_alpha=0.3, view=self.date_view) + color="blue", size=7, fill_alpha=0.3) plot.x_range = Range1d(-0.5, 0.5) plot.y_range = Range1d(-0.5, 0.5) - # mark origin lines vline = Span(location=0, dimension='height', line_color='black', line_width=0.7) hline = Span(location=0, dimension='width', line_color='black', line_width=0.7) plot.renderers.extend([vline, hline]) - - # add tooltips hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -305,11 +281,8 @@ def plt_residual_offsets(self): ('Date-Obs', '@date_obs'), ('Magnitude', '@star_mag'), ('--------', '----------------')] - plot.add_tools(hover) - # add shared selection tools - for tool in self.share_tools: - plot.add_tools(tool) + plot.add_tools(hover) return plot def plt_v2offset_time(self): @@ -325,14 +298,11 @@ def plt_v2offset_time(self): plot = figure(title="WATA V2 Offset vs Time", x_axis_label='Time', y_axis_label='Residual V2 Offset', x_axis_type='datetime') plot.circle(x='time_arr', y='v2_offset', source=self.source, - color="blue", size=7, fill_alpha=0.3, view=self.date_view) + color="blue", size=7, fill_alpha=0.3) plot.y_range = Range1d(-0.5, 0.5) - # mark origin line hline = Span(location=0, dimension='width', line_color='black', line_width=0.7) plot.renderers.extend([hline]) - - # add tooltips hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -342,11 +312,8 @@ def plt_v2offset_time(self): ('Date-Obs', '@date_obs'), ('Magnitude', '@star_mag'), ('--------', '----------------')] - plot.add_tools(hover) - # add shared selection tools - for tool in self.share_tools: - plot.add_tools(tool) + plot.add_tools(hover) return plot def plt_v3offset_time(self): @@ -362,14 +329,11 @@ def plt_v3offset_time(self): plot = figure(title="WATA V3 Offset vs Time", x_axis_label='Time', y_axis_label='Residual V3 Offset', x_axis_type='datetime') plot.circle(x='time_arr', y='v3_offset', source=self.source, - color="blue", size=7, fill_alpha=0.3, view=self.date_view) + color="blue", size=7, fill_alpha=0.3) plot.y_range = Range1d(-0.5, 0.5) - # mark origin line hline = Span(location=0, dimension='width', line_color='black', line_width=0.7) plot.renderers.extend([hline]) - - # add tooltips hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -379,11 +343,8 @@ def plt_v3offset_time(self): ('Date-Obs', '@date_obs'), ('Magnitude', '@star_mag'), ('--------', '----------------')] - plot.add_tools(hover) - # add shared selection tools - for tool in self.share_tools: - plot.add_tools(tool) + plot.add_tools(hover) return plot def plt_mag_time(self): @@ -397,9 +358,15 @@ def plt_mag_time(self): """ # calculate the pseudo magnitudes max_val_box, time_arr = self.source.data['max_val_box'], self.source.data['time_arr'] - - # check if this column exists in the data already, else create it - if "nrsrapid_f140x" not in self.source.data: + # check if this column exists in the data already (the other 2 will exist too), else create it + try: + nrsrapid_f140x = self.source.data["nrsrapid_f140x"] + nrsrapid_f110w = self.source.data["nrsrapid_f110w"] + nrsrapid_clear = self.source.data["nrsrapid_clear"] + nrsrapidd6_f140x = self.source.data["nrsrapidd6_f140x"] + nrsrapidd6_f110w = self.source.data["nrsrapidd6_f110w"] + nrsrapidd6_clear = self.source.data["nrsrapidd6_clear"] + except KeyError: # create the arrays per filter and readout pattern nrsrapid_f140x, nrsrapid_f110w, nrsrapid_clear = [], [], [] nrsrapidd6_f140x, nrsrapidd6_f110w, nrsrapidd6_clear = [], [], [] @@ -450,7 +417,6 @@ def plt_mag_time(self): nrsrapidd6_f140x.append(np.NaN) nrsrapidd6_f110w.append(np.NaN) nrsrapidd6_clear.append(val) - # add to the bokeh data structure self.source.data["nrsrapid_f140x"] = nrsrapid_f140x self.source.data["nrsrapid_f110w"] = nrsrapid_f110w @@ -458,30 +424,27 @@ def plt_mag_time(self): self.source.data["nrsrapidd6_f140x"] = nrsrapidd6_f140x self.source.data["nrsrapidd6_f110w"] = nrsrapidd6_f110w self.source.data["nrsrapidd6_clear"] = nrsrapidd6_clear - # create a new bokeh plot plot = figure(title="WATA Counts vs Time", x_axis_label='Time', y_axis_label='box_peak [Counts]', x_axis_type='datetime') plot.circle(x='time_arr', y='nrsrapid_f140x', source=self.source, - color="purple", size=7, fill_alpha=0.4, view=self.date_view) + color="purple", size=7, fill_alpha=0.4) plot.circle(x='time_arr', y='nrsrapidd6_f140x', source=self.source, - color="purple", size=12, fill_alpha=0.4, view=self.date_view) + color="purple", size=12, fill_alpha=0.4) plot.triangle(x='time_arr', y='nrsrapid_f110w', source=self.source, - color="orange", size=8, fill_alpha=0.4, view=self.date_view) + color="orange", size=8, fill_alpha=0.4) plot.triangle(x='time_arr', y='nrsrapidd6_f110w', source=self.source, - color="orange", size=13, fill_alpha=0.4, view=self.date_view) + color="orange", size=13, fill_alpha=0.4) plot.square(x='time_arr', y='nrsrapid_clear', source=self.source, - color="gray", size=7, fill_alpha=0.4, view=self.date_view) + color="gray", size=7, fill_alpha=0.4) plot.square(x='time_arr', y='nrsrapidd6_clear', source=self.source, - color="gray", size=12, fill_alpha=0.4, view=self.date_view) - + color="gray", size=12, fill_alpha=0.4) # add count saturation warning lines loc1, loc2, loc3 = 45000.0, 50000.0, 60000.0 hline1 = Span(location=loc1, dimension='width', line_color='green', line_width=3) hline2 = Span(location=loc2, dimension='width', line_color='yellow', line_width=3) hline3 = Span(location=loc3, dimension='width', line_color='red', line_width=3) plot.renderers.extend([hline1, hline2, hline3]) - label1 = Label(x=time_arr[-1], y=loc1, y_units='data', text='45000 counts') label2 = Label(x=time_arr[-1], y=loc2, y_units='data', text='50000 counts') label3 = Label(x=time_arr[-1], y=loc3, y_units='data', text='60000 counts') @@ -489,8 +452,7 @@ def plt_mag_time(self): plot.add_layout(label2) plot.add_layout(label3) plot.y_range = Range1d(-1000.0, 62000.0) - - # add tooltips + # add hover hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -500,15 +462,12 @@ def plt_mag_time(self): ('Date-Obs', '@date_obs'), ('Box peak', '@max_val_box'), ('--------', '----------------')] - plot.add_tools(hover) - # add shared selection tools - for tool in self.share_tools: - plot.add_tools(tool) + plot.add_tools(hover) return plot - def get_unsuccessful_ta(self, arr_name): - """ Find unsuccessful TAs in this set (to be plotted in red) + def get_unsucessful_ta(self, arr_name): + """ Find unsucessful TAs in this set (to be plotted in red) Parameters ---------- arr_name: str, name of the array of interest @@ -538,16 +497,16 @@ def plt_centroid(self): plot: bokeh plot object """ # get the failed TAs to plot in red - if "corr_col_failed" not in self.source.data: - corr_col_failed, corr_col_not_failed = self.get_unsuccessful_ta('corr_col') - corr_row_failed, corr_row_not_failed = self.get_unsuccessful_ta('corr_row') - + try: + corr_col_failed = self.source.data["corr_col_failed"] + except KeyError: + corr_col_failed, corr_col_not_failed = self.get_unsucessful_ta('corr_col') + corr_row_failed, corr_row_not_failed = self.get_unsucessful_ta('corr_row') # add these to the bokeh data structure self.source.data["corr_col_failed"] = corr_col_failed self.source.data["corr_col_not_failed"] = corr_col_not_failed self.source.data["corr_row_failed"] = corr_row_failed self.source.data["corr_row_not_failed"] = corr_row_not_failed - # create a new bokeh plot plot = figure(title="WATA Centroid", x_axis_label='Column', y_axis_label='Row') @@ -555,13 +514,11 @@ def plt_centroid(self): plot.x_range = Range1d(limits[0], limits[1]) plot.y_range = Range1d(limits[0], limits[1]) plot.circle(x='corr_col_not_failed', y='corr_row_not_failed', source=self.source, - color="blue", size=7, fill_alpha=0.5, view=self.date_view) + color="blue", size=7, fill_alpha=0.5) plot.circle(x='corr_col_failed', y='corr_row_failed', source=self.source, - color="red", size=7, fill_alpha=0.5, view=self.date_view) + color="red", size=7, fill_alpha=0.5) plot.x_range = Range1d(0.0, 32.0) plot.y_range = Range1d(0.0, 32.0) - - # add tooltips hover = HoverTool() hover.tooltips = [('File name', '@filename'), ('Visit ID', '@visit_id'), @@ -575,72 +532,18 @@ def plt_centroid(self): ('Det Centr Col', '@detector_final_col'), ('Det Centr Row', '@detector_final_row'), ('--------', '----------------')] - plot.add_tools(hover) - # add shared selection tools - for tool in self.share_tools: - plot.add_tools(tool) + plot.add_tools(hover) return plot - def setup_date_range(self): - """Set up a date range filter, defaulting to the last week of data.""" - end_date = datetime.now(tz=timezone.utc) - one_week_ago = end_date.date() - timedelta(days=7) - first_data_point = np.min(self.source.data['time_arr']).date() - last_data_point = np.max(self.source.data['time_arr']).date() - if last_data_point < one_week_ago: - # keep at least one point in the plot if there was - # no TA data this week - start_date = last_data_point - else: - start_date = one_week_ago - - # allowed range is from the first ever data point to today - self.date_range = DateRangeSlider( - title="Date range displayed", start=first_data_point, - end=end_date, value=(start_date, end_date), step=1) - - callback = CustomJS(args=dict(s=self.source), code=""" - s.change.emit(); - """) - self.date_range.js_on_change('value', callback) - - filt = CustomJSFilter(args=dict(slider=self.date_range), code=""" - var indices = []; - var start = slider.value[0]; - var end = slider.value[1]; - - for (var i=0; i < source.get_length(); i++) { - if (source.data['time_arr'][i] >= start - && source.data['time_arr'][i] <= end) { - indices.push(true); - } else { - indices.push(false); - } - } - return indices; - """) - self.date_view = CDSView(source=self.source, filters=[filt]) - def mk_plt_layout(self): """Create the bokeh plot layout""" self.source = ColumnDataSource(data=self.wata_data) - # make sure all arrays are lists in order to later be able to read the data # from the html file for item in self.source.data: if not isinstance(self.source.data[item], (str, float, int, list)): self.source.data[item] = self.source.data[item].tolist() - - # add a time array to the data source - self.add_time_column() - - # set up selection tools to share - self.share_tools = [BoxSelectTool()] - - # set up a date range filter widget - self.setup_date_range() - # set the output html file name and create the plot grid output_file(self.output_file_name) p1 = self.plt_status() @@ -649,14 +552,11 @@ def mk_plt_layout(self): p4 = self.plt_v3offset_time() p5 = self.plt_centroid() p6 = self.plt_mag_time() - # make grid grid = gridplot([p1, p2, p3, p4, p5, p6], ncols=2, merge_tools=False) - box_layout = layout(children=[self.date_range, grid]) - save(box_layout) - + save(grid) # return the needed components for embeding the results in the WATA html template - script, div = components(box_layout) + script, div = components(grid) return script, div def identify_tables(self): @@ -829,15 +729,15 @@ def update_ta_success_txtfile(self): Nothing """ output_success_ta_txtfile = os.path.join(self.output_dir, "wata_success.txt") - # check if previous file exists and read the data from it + # check if previous file exsists and read the data from it if os.path.isfile(output_success_ta_txtfile): - # now rename the previous file, for backup + # now rename the the previous file, for backup os.rename(output_success_ta_txtfile, os.path.join(self.output_dir, "prev_wata_success.txt")) # get the new data ta_success, ta_failure = [], [] filenames, ta_status = self.wata_data.loc[:,'filename'], self.wata_data.loc[:,'ta_status'] for fname, ta_stat in zip(filenames, ta_status): - # select the appropriate list to append to + # select the appriopriate list to append to if ta_stat == 'SUCCESSFUL': ta_success.append(fname) else: From 7efae1e0981760a25f5de7fd9a396076ee8e617a Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 9 Jun 2023 17:38:52 -0400 Subject: [PATCH 380/449] Add row and column plots --- jwql/utils/interactive_preview_image.py | 142 ++++++++++++++++-- jwql/website/apps/jwql/static/js/jwql.js | 7 +- .../apps/jwql/templates/explore_image.html | 19 ++- jwql/website/apps/jwql/urls.py | 2 +- jwql/website/apps/jwql/views.py | 13 +- 5 files changed, 156 insertions(+), 27 deletions(-) diff --git a/jwql/utils/interactive_preview_image.py b/jwql/utils/interactive_preview_image.py index d0f1c8cea..0c4480882 100644 --- a/jwql/utils/interactive_preview_image.py +++ b/jwql/utils/interactive_preview_image.py @@ -16,6 +16,7 @@ Required Arguments: ''filename'' - Name of a fits file containing a JWST observation """ +import datetime from copy import deepcopy import os @@ -23,12 +24,13 @@ from astropy.visualization import ZScaleInterval, MinMaxInterval, PercentileInterval import numpy as np from bokeh.embed import components -from bokeh.layouts import layout +from bokeh.layouts import gridplot, layout from bokeh.models import ( - BasicTicker, BoxZoomTool, Button, ColorBar, CustomJS, Div, HoverTool, - LinearColorMapper, LogColorMapper, LogTicker, RadioGroup, Row, Select, Spacer, - Spinner, WheelZoomTool) + BasicTicker, BoxZoomTool, Button, ColorBar, ColumnDataSource, + CustomJS, CustomJSTransform, DataRange1d, Div, HoverTool, LinearColorMapper, LogColorMapper, + LogTicker, RadioGroup, Range1d, Row, Select, Spacer, Spinner, WheelZoomTool) from bokeh.plotting import figure, output_file, show, save +from bokeh.transform import transform from jwst.datamodels import dqflags @@ -37,8 +39,8 @@ class InteractivePreviewImg: """Class to create the interactive Bokeh figure. """ - def __init__(self, filename, low_lim=None, high_lim=None, scaling='log', contrast=None, extname='SCI', - group=None, integ=None, mask=None, save_html=None, show=False): + def __init__(self, filename, low_lim=None, high_lim=None, scaling='lin', contrast=None, extname='SCI', + group=None, integ=None, mask=None, line_plots=False, save_html=None, show=False): """Populate attributes, read in data, and create the Bokeh figure Parameters ---------- @@ -68,6 +70,9 @@ def __init__(self, filename, low_lim=None, high_lim=None, scaling='log', contras mask : numpy.ndarray Mask to use in order to avoid some pixels when auto-scaling. Pixels with a value other than 0 will be ignored when auto-scaling. + line_plots : bool + If set, column and row plots are added to the layout, to be updated on click in the main figure. + These take some time to create, so are off by default. save_html : str Name of html file to save the figure to. If None, the components are returned instead. show : bool @@ -80,6 +85,7 @@ def __init__(self, filename, low_lim=None, high_lim=None, scaling='log', contras self.contrast = contrast self.extname = extname.upper() self.mask = mask + self.show_line_plots = line_plots self.show = show self.save_html = save_html @@ -105,6 +111,8 @@ def __init__(self, filename, low_lim=None, high_lim=None, scaling='log', contras self.get_data() if 'DQ' in self.extname: self.get_bits() + # col/row plots not available for dq values + self.show_line_plots = False self.script, self.div = self.create_bokeh_image() def create_bokeh_image(self): @@ -140,16 +148,17 @@ def create_bokeh_image(self): # fix figure aspect from data aspect # bokeh throws errors if plot is too small, so make sure # the smaller dimension has reasonable size + max_dim, min_dim = 700, 300 if xd > yd: - plot_width = 800 + plot_width = max_dim plot_height = int(plot_width * yd / xd) - if plot_height < 400: - plot_height = 400 + if plot_height < min_dim: + plot_height = min_dim else: - plot_height = 800 + plot_height = max_dim plot_width = int(plot_height * xd / yd) - if plot_width < 400: - plot_width = 400 + if plot_width < min_dim: + plot_width = min_dim fig = figure(tools='pan,reset,save', match_aspect=True, plot_width=plot_width, plot_height=plot_height) @@ -169,7 +178,10 @@ def create_bokeh_image(self): title=self.signal_units, bar_line_color='black', minor_tick_line_color='black', major_tick_line_color='black', visible=visible) - fig.add_layout(color_bar, 'below') + if self.show_line_plots: + fig.add_layout(color_bar, 'above') + else: + fig.add_layout(color_bar, 'below') images.append(img) color_bars.append(color_bar) @@ -196,11 +208,19 @@ def create_bokeh_image(self): fig.title.text = self.title fig.xaxis.axis_label = 'Pixel' fig.yaxis.axis_label = 'Pixel' - fig.tools.append(hover_tool) + fig.add_tools(hover_tool) # add interactive widgets widgets = self.add_interactive_controls(images, color_bars) - box_layout = layout(children=[fig, *widgets]) + if self.show_line_plots: + # add row and column plots + col_plot, row_plot = self.line_plots(fig) + grid = gridplot([fig, col_plot, row_plot], + ncols=2, merge_tools=False) + else: + grid = fig + + box_layout = layout(children=[grid, *widgets]) # Show figure on screen if requested if self.show: @@ -210,6 +230,98 @@ def create_bokeh_image(self): else: return components(box_layout) + def line_plots(self, main_figure): + new_plots = [] + new_lines = [] + ny, nx = self.data.shape + col_idx, row_idx = np.indices((ny, nx)) + + for direction in ['y', 'x']: + if direction == 'y': + # column plots + fig = figure(plot_width=200, plot_height=main_figure.height, + tools='xwheel_zoom, xpan, reset', + y_axis_location='right') + + fig.x_range = DataRange1d(only_visible=True) + fig.y_range = Range1d() + match_range = fig.y_range + main_range = main_figure.y_range + + x_plot = 'data' + y_plot = 'index' + n_plot = nx + data_source = {'data': self.data, 'index': col_idx} + source = ColumnDataSource(data_source) + else: + # row plots + fig = figure(plot_height=200, plot_width=main_figure.width, + tools='ywheel_zoom, ypan, reset') + + fig.y_range = DataRange1d(only_visible=True) + fig.x_range = Range1d() + match_range = fig.x_range + main_range = main_figure.x_range + + x_plot = 'index' + y_plot = 'data' + n_plot = ny + # indexing is off by 1 for row plots for some reason + data_source = {'data': self.data.T, 'index': row_idx.T + 1} + source = ColumnDataSource(data_source) + + # match one of the axes to the main figure + if main_range.start is not None: + match_range.start = main_range.start + if main_range.end is not None: + match_range.end = main_range.end + main_range.js_link('start', match_range, 'start') + main_range.js_link('start', match_range, 'reset_start') + main_range.js_link('end', match_range, 'end') + main_range.js_link('end', match_range, 'reset_end') + + lines = [] + initial_visible = n_plot // 2 + for i in range(n_plot): + idx_transform = CustomJSTransform( + args={'idx': i}, + v_func=""" + const x_val = new Float64Array(xs.length); + for (let i = 0; i < xs.length; i++) { + x_val[i] = xs[i][idx]; + } + return x_val; + """) + line = fig.step(x=transform(x_plot, idx_transform), + y=transform(y_plot, idx_transform), + mode='before', source=source, + visible=(i == initial_visible)) + lines.append(line) + new_lines.append(lines) + new_plots.append(fig) + + update_plot = CustomJS(args={'lines': new_lines}, code=""" + var x = Math.floor(cb_obj.x); + var y = Math.floor(cb_obj.y); + for (let i=0; i < lines[0].length; i++) { + if (i == x) { + lines[0][i].visible = true; + } else { + lines[0][i].visible = false; + } + } + for (let j=0; j < lines[1].length; j++) { + if (j == y) { + lines[1][j].visible = true; + } else { + lines[1][j].visible = false; + } + } + """) + main_figure.js_on_event('tap', update_plot) + + return new_plots + def add_interactive_controls(self, images, color_bars): """ Add client-side controls for images. diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index 10cbe7f23..273057412 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -423,7 +423,7 @@ function get_radio_button_value(element_name) { } /** - * get_scaling_value + * Get value from a numerical text field * @param {String} element_id - The element id * @returns value - value of element id or "None" if empty or not a number */ @@ -1058,7 +1058,7 @@ function update_wata_page(base_url) { /** * Updates various components on the thumbnails page * @param {String} inst - The instrument of interest (e.g. "FGS") - * @param {String} file_root - The rootname of the file forresponding tot he instrument (e.g. "JW01473015001_04101_00001_MIRIMAGE") + * @param {String} file_root - The rootname of the file corresponding to the instrument (e.g. "JW01473015001_04101_00001_MIRIMAGE") * @param {String} filetype - The type to be viewed (e.g. "cal" or "rate"). * @param {String} base_url - The base URL for gathering data from the AJAX view. * @param {Boolean} do_opt_args - Flag to calculate and send optional arguments in URL @@ -1073,6 +1073,7 @@ function update_wata_page(base_url) { document.getElementById("explore_image").style.display = "none"; document.getElementById("explore_image_fail").style.display = "none"; var calc_difference = document.getElementById("calcDifference").checked; + var show_line_plots = document.getElementById("show_line_plots").checked; // Get the arguments to update var ext_name = get_radio_button_value("extension"); @@ -1087,7 +1088,7 @@ function update_wata_page(base_url) { int2_nr="None"; grp2_nr="None"; } - optional_params = optional_params + "/ext_" + ext_name + "/int1_" + int1_nr + "/grp1_" + grp1_nr + "/int2_" + int2_nr + "/grp2_" + grp2_nr; + optional_params = optional_params + "/plot_" + show_line_plots + "/ext_" + ext_name + "/int1_" + int1_nr + "/grp1_" + grp1_nr + "/int2_" + int2_nr + "/grp2_" + grp2_nr; } $.ajax({ diff --git a/jwql/website/apps/jwql/templates/explore_image.html b/jwql/website/apps/jwql/templates/explore_image.html index f72a92081..af750a8c8 100644 --- a/jwql/website/apps/jwql/templates/explore_image.html +++ b/jwql/website/apps/jwql/templates/explore_image.html @@ -44,7 +44,13 @@

    {{ file_root }}_{{ filetype }}.fits

    Data Settings

    -
    +
    +
    + + +
    + +
    Extension:   {% for extension in extensions %} {% if extension == 'SCI' %} @@ -53,6 +59,8 @@

    Data Settings

    {{ extension }}      {% endif %} {% endfor %} +
    +
    @@ -83,8 +91,11 @@

    Data Settings

    - -
    + +
    + + +
    Submit Anomaly
    @@ -115,8 +126,6 @@
    Submit Anomaly
    - -
    diff --git a/jwql/website/apps/jwql/urls.py b/jwql/website/apps/jwql/urls.py index c0535b378..0b4420080 100644 --- a/jwql/website/apps/jwql/urls.py +++ b/jwql/website/apps/jwql/urls.py @@ -96,7 +96,7 @@ re_path('ajax/query_submit/', views.archive_thumbnails_query_ajax, name='archive_thumb_query_ajax'), re_path(r'^ajax/(?P({}))/archive/$'.format(instruments), views.archived_proposals_ajax, name='archive_ajax'), re_path(r'^ajax/(?P({}))/(?P.+)_(?P.+)/explore_image/$'.format(instruments), views.explore_image_ajax, name='explore_image_ajax'), - re_path(r'^ajax/(?P({}))/(?P.+)_(?P.+)/explore_image/ext_(?P.+)/int1_(?P.+)/grp1_(?P.+)/int2_(?P.+)/grp2_(?P.+)/$'.format(instruments), views.explore_image_ajax, name='explore_image_ajax'), + re_path(r'^ajax/(?P({}))/(?P.+)_(?P.+)/explore_image/plot_(?P(true|false))/ext_(?P.+)/int1_(?P.+)/grp1_(?P.+)/int2_(?P.+)/grp2_(?P.+)/$'.format(instruments), views.explore_image_ajax, name='explore_image_ajax'), re_path(r'^ajax/(?P({}))/archive/(?P[\d]{{1,5}})/obs(?P[\d]{{1,3}})/$'.format(instruments), views.archive_thumbnails_ajax, name='archive_thumb_ajax'), re_path(r'^ajax/viewed/(?P.+)/$', views.toggle_viewed_ajax, name='toggle_viewed_ajax'), re_path(r'^ajax/viewed_group/(?P.+)/(?P(viewed|new|Viewed|New))/$', diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index b0240735f..af4beb1b0 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -994,7 +994,7 @@ def explore_image(request, inst, file_root, filetype): return render(request, template, context) -def explore_image_ajax(request, inst, file_root, filetype, scaling="lin", low_lim=None, high_lim=None, ext_name="SCI", int1_nr=None, grp1_nr=None, int2_nr=None, grp2_nr=None): +def explore_image_ajax(request, inst, file_root, filetype, line_plots='false', low_lim=None, high_lim=None, ext_name="SCI", int1_nr=None, grp1_nr=None, int2_nr=None, grp2_nr=None): """Generate the page listing all archived images in the database for a certain proposal @@ -1008,7 +1008,7 @@ def explore_image_ajax(request, inst, file_root, filetype, scaling="lin", low_li FITS file_root of selected image in filesystem filetype : str Type of file (e.g. ``uncal``) - scaling : str + line_plots : str Scaling to implement in interactive preview image ("log" or "lin") low_lim : str Signal value to use as the lower limit of the displayed image. If "None", it will be calculated using the ZScale function @@ -1067,7 +1067,14 @@ def explore_image_ajax(request, inst, file_root, filetype, scaling="lin", low_li else: integ = int(int1_nr) - int_preview_image = InteractivePreviewImg(full_fits_file, low_lim, high_lim, scaling, None, ext_name, group, integ) + if str(line_plots).strip().lower() == 'true': + line_plots = True + else: + line_plots = False + + int_preview_image = InteractivePreviewImg( + full_fits_file, low_lim=low_lim, high_lim=high_lim, extname=ext_name, + group=group, integ=integ, line_plots=line_plots) context = {'inst': "inst", 'script': int_preview_image.script, From 9938a5460253e3971da4806d6719f9069c5e4290 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Mon, 12 Jun 2023 16:41:59 -0400 Subject: [PATCH 381/449] Readability improvements --- jwql/utils/interactive_preview_image.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/jwql/utils/interactive_preview_image.py b/jwql/utils/interactive_preview_image.py index 0c4480882..262fa2c07 100644 --- a/jwql/utils/interactive_preview_image.py +++ b/jwql/utils/interactive_preview_image.py @@ -148,7 +148,7 @@ def create_bokeh_image(self): # fix figure aspect from data aspect # bokeh throws errors if plot is too small, so make sure # the smaller dimension has reasonable size - max_dim, min_dim = 700, 300 + max_dim, min_dim = 700, 400 if xd > yd: plot_width = max_dim plot_height = int(plot_width * yd / xd) @@ -231,6 +231,19 @@ def create_bokeh_image(self): return components(box_layout) def line_plots(self, main_figure): + """ + Pre-compute column and row plots for each pixel. + + Parameters + ---------- + main_figure : figure + Main figure containing image. + + Returns + ------- + list of figure + New figures to add to the page layout. + """ new_plots = [] new_lines = [] ny, nx = self.data.shape @@ -242,12 +255,15 @@ def line_plots(self, main_figure): fig = figure(plot_width=200, plot_height=main_figure.height, tools='xwheel_zoom, xpan, reset', y_axis_location='right') - fig.x_range = DataRange1d(only_visible=True) fig.y_range = Range1d() match_range = fig.y_range main_range = main_figure.y_range + fig.xaxis.axis_label = self.signal_units + fig.yaxis.axis_label = 'Row pixel (y)' + fig.xaxis.major_label_orientation = np.radians(-45) + x_plot = 'data' y_plot = 'index' n_plot = nx @@ -263,6 +279,9 @@ def line_plots(self, main_figure): match_range = fig.x_range main_range = main_figure.x_range + fig.xaxis.axis_label = 'Column pixel (x)' + fig.yaxis.axis_label = self.signal_units + x_plot = 'index' y_plot = 'data' n_plot = ny From 6e370d49378cbdda45a748080320debc99768468 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Tue, 13 Jun 2023 13:03:53 -0400 Subject: [PATCH 382/449] Move hover tool to separate div --- jwql/utils/interactive_preview_image.py | 116 +++++++++++++++++------- 1 file changed, 82 insertions(+), 34 deletions(-) diff --git a/jwql/utils/interactive_preview_image.py b/jwql/utils/interactive_preview_image.py index 262fa2c07..a414d7e63 100644 --- a/jwql/utils/interactive_preview_image.py +++ b/jwql/utils/interactive_preview_image.py @@ -27,8 +27,9 @@ from bokeh.layouts import gridplot, layout from bokeh.models import ( BasicTicker, BoxZoomTool, Button, ColorBar, ColumnDataSource, - CustomJS, CustomJSTransform, DataRange1d, Div, HoverTool, LinearColorMapper, LogColorMapper, - LogTicker, RadioGroup, Range1d, Row, Select, Spacer, Spinner, WheelZoomTool) + CustomJS, CustomJSTransform, DataRange1d, Div, HoverTool, + LinearColorMapper, LogColorMapper, LogTicker, RadioGroup, Range1d, Row, + Select, Spacer, Spinner, WheelZoomTool) from bokeh.plotting import figure, output_file, show, save from bokeh.transform import transform @@ -138,7 +139,7 @@ def create_bokeh_image(self): active = int(self.scaling == 'log') yd, xd = self.data.shape - info = dict(image=[self.data], x=[0], y=[0], dw=[xd], dh=[yd]) + info = ColumnDataSource(dict(image=[self.data], x=[0], y=[0], dw=[xd], dh=[yd])) if 'DQ' in self.extname: info["dq"] = [self.bit_list] if not self.show and self.save_html is not None: @@ -196,14 +197,8 @@ def create_bokeh_image(self): fig.y_range.end = yd fig.y_range.bounds = (0, yd) - if 'DQ' not in self.extname: - hover_tool = HoverTool(tooltips=[("(x,y)", "($x{0.2f}, $y{0.2f})"), - ('Value', '@image{0.4f}') - ], mode='mouse', renderers=images) - else: - hover_tool = HoverTool(tooltips=[("(x,y)", "($x{0.2f}, $y{0.2f})"), - ('Value', '@dq') - ], mode='mouse', renderers=images) + hover_div, hover_tool = self.add_hover_tool(info, images) + self.create_figure_title() fig.title.text = self.title fig.xaxis.axis_label = 'Pixel' @@ -215,10 +210,10 @@ def create_bokeh_image(self): if self.show_line_plots: # add row and column plots col_plot, row_plot = self.line_plots(fig) - grid = gridplot([fig, col_plot, row_plot], + grid = gridplot([fig, col_plot, row_plot, hover_div], ncols=2, merge_tools=False) else: - grid = fig + grid = gridplot([fig, hover_div], ncols=2, merge_tools=False) box_layout = layout(children=[grid, *widgets]) @@ -249,12 +244,12 @@ def line_plots(self, main_figure): ny, nx = self.data.shape col_idx, row_idx = np.indices((ny, nx)) - for direction in ['y', 'x']: - if direction == 'y': + for index_direction in ['x', 'y']: + if index_direction == 'x': # column plots fig = figure(plot_width=200, plot_height=main_figure.height, tools='xwheel_zoom, xpan, reset', - y_axis_location='right') + y_axis_location='right', margin=(0, 0, 0, 30)) fig.x_range = DataRange1d(only_visible=True) fig.y_range = Range1d() match_range = fig.y_range @@ -314,33 +309,86 @@ def line_plots(self, main_figure): line = fig.step(x=transform(x_plot, idx_transform), y=transform(y_plot, idx_transform), mode='before', source=source, - visible=(i == initial_visible)) + visible=(i == initial_visible), + name=f'Data at {index_direction}={i}') lines.append(line) + fig.title = lines[initial_visible].name new_lines.append(lines) new_plots.append(fig) - update_plot = CustomJS(args={'lines': new_lines}, code=""" - var x = Math.floor(cb_obj.x); - var y = Math.floor(cb_obj.y); - for (let i=0; i < lines[0].length; i++) { - if (i == x) { - lines[0][i].visible = true; - } else { - lines[0][i].visible = false; - } - } - for (let j=0; j < lines[1].length; j++) { - if (j == y) { - lines[1][j].visible = true; - } else { - lines[1][j].visible = false; + update_plot = CustomJS( + args={'lines': new_lines, 'figures': new_plots}, + code=""" + var x = Math.floor(cb_obj.x); + var y = Math.floor(cb_obj.y); + figures[0].title.text = ""; + for (let i=0; i < lines[0].length; i++) { + if (i == x) { + lines[0][i].visible = true; + figures[0].title.text = lines[0][i].name; + } else { + lines[0][i].visible = false; + } } - } - """) + figures[1].title.text = ""; + for (let j=0; j < lines[1].length; j++) { + if (j == y) { + lines[1][j].visible = true; + figures[1].title.text = lines[1][j].name; + } else { + lines[1][j].visible = false; + } + }""") main_figure.js_on_event('tap', update_plot) return new_plots + def add_hover_tool(self, info, images): + """ + Make a hover tool with a div to display text. + + Parameters + ---------- + info : bokeh.models.ColumnDataSource + Data source for the figure. + images : list of bokeh.models.GlyphRenderer + Images to use as renderers for the hover tool. + + Returns + ------- + hover_div : bokeh.models.Div + Div element that will contain text from hover tool. + hover_tool : bokeh.models. + """ + hover_div = Div(height=200, width=200) + hover_callback = CustomJS(args={'s': info, 'd': hover_div}, code=""" + const idx = cb_data.index.image_indices; + if (idx.length > 0) { + var x = idx[0].dim1; + var y = idx[0].dim2; + var val = s.data['image'][0][y][x]; + val = val.toPrecision(5); + d.text = "
    Pixel Value
    " + + "
    " + + "
    " + + "
    (x, y) =
    " + + "
    (" + x + ", " + y + ")
    " + + "
    " + + "
    " + + "
    Value =
    " + + "
    " + val + "
    "; + } else { + d.text = ""; + } + """) + if 'DQ' not in self.extname: + hover_tool = HoverTool(tooltips=None, mode='mouse', renderers=images, + callback=hover_callback) + else: + hover_tool = HoverTool(tooltips=None, mode='mouse', renderers=images, + callback=hover_callback) + return hover_div, hover_tool + def add_interactive_controls(self, images, color_bars): """ Add client-side controls for images. From 4445dde8f786979d4b7eb6e172a81ace8a2aaf3d Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Tue, 13 Jun 2023 14:31:40 -0400 Subject: [PATCH 383/449] Add RA/Dec to hover tool if available --- jwql/utils/interactive_preview_image.py | 63 ++++++++++++++----- .../apps/jwql/templates/explore_image.html | 2 +- 2 files changed, 48 insertions(+), 17 deletions(-) diff --git a/jwql/utils/interactive_preview_image.py b/jwql/utils/interactive_preview_image.py index a414d7e63..615176cd2 100644 --- a/jwql/utils/interactive_preview_image.py +++ b/jwql/utils/interactive_preview_image.py @@ -16,13 +16,13 @@ Required Arguments: ''filename'' - Name of a fits file containing a JWST observation """ -import datetime from copy import deepcopy import os +import numpy as np from astropy.io import fits from astropy.visualization import ZScaleInterval, MinMaxInterval, PercentileInterval -import numpy as np +from astropy.wcs import WCS from bokeh.embed import components from bokeh.layouts import gridplot, layout from bokeh.models import ( @@ -32,7 +32,6 @@ Select, Spacer, Spinner, WheelZoomTool) from bokeh.plotting import figure, output_file, show, save from bokeh.transform import transform - from jwst.datamodels import dqflags @@ -109,6 +108,10 @@ def __init__(self, filename, low_lim=None, high_lim=None, scaling='lin', contras raise ValueError( 'integ must be an integer or 2-element list') self.integ = integ + + self.data = None + self.signal_units = None + self.wcs_coord = None self.get_data() if 'DQ' in self.extname: self.get_bits() @@ -139,9 +142,14 @@ def create_bokeh_image(self): active = int(self.scaling == 'log') yd, xd = self.data.shape - info = ColumnDataSource(dict(image=[self.data], x=[0], y=[0], dw=[xd], dh=[yd])) + info = dict(image=[self.data], x=[0], y=[0], dw=[xd], dh=[yd]) if 'DQ' in self.extname: info["dq"] = [self.bit_list] + if self.wcs_coord is not None and len(self.wcs_coord) == 2: + info["ra"] = [self.wcs_coord[0]] + info["dec"] = [self.wcs_coord[1]] + source = ColumnDataSource(info) + if not self.show and self.save_html is not None: output_file(filename=self.save_html, title=os.path.basename(self.filename)) @@ -173,7 +181,7 @@ def create_bokeh_image(self): for i, config in enumerate(scales): color_mapper, ticker = config visible = (i == active) - img = fig.image(source=info, image='image', + img = fig.image(source=source, image='image', level="image", color_mapper=color_mapper, visible=visible) color_bar = ColorBar(color_mapper=color_mapper, label_standoff=12, ticker=ticker, title=self.signal_units, bar_line_color='black', @@ -197,7 +205,7 @@ def create_bokeh_image(self): fig.y_range.end = yd fig.y_range.bounds = (0, yd) - hover_div, hover_tool = self.add_hover_tool(info, images) + hover_div, hover_tool = self.add_hover_tool(source, images) self.create_figure_title() fig.title.text = self.title @@ -343,13 +351,13 @@ def line_plots(self, main_figure): return new_plots - def add_hover_tool(self, info, images): + def add_hover_tool(self, source, images): """ Make a hover tool with a div to display text. Parameters ---------- - info : bokeh.models.ColumnDataSource + source : bokeh.models.ColumnDataSource Data source for the figure. images : list of bokeh.models.GlyphRenderer Images to use as renderers for the hover tool. @@ -360,23 +368,35 @@ def add_hover_tool(self, info, images): Div element that will contain text from hover tool. hover_tool : bokeh.models. """ - hover_div = Div(height=200, width=200) - hover_callback = CustomJS(args={'s': info, 'd': hover_div}, code=""" + hover_div = Div(height=300, width=300) + hover_callback = CustomJS(args={'s': source, 'd': hover_div, 'u': self.signal_units}, code=""" const idx = cb_data.index.image_indices; if (idx.length > 0) { var x = idx[0].dim1; var y = idx[0].dim2; - var val = s.data['image'][0][y][x]; - val = val.toPrecision(5); + var flat = idx[0].flat_index; + var val = s.data['image'][0][y][x].toPrecision(5); d.text = "
    Pixel Value
    " + "
    " + "
    " + "
    (x, y) =
    " + "
    (" + x + ", " + y + ")
    " + - "
    " + - "
    " + - "
    Value =
    " + - "
    " + val + "
    "; + "
    " + if ('ra' in s.data && 'dec' in s.data) { + var ra = s.data['ra'][0][flat].toPrecision(8); + var dec = s.data['dec'][0][flat].toPrecision(8); + d.text += "
    " + + "
    RA (deg) =
    " + + "
    " + ra + "
    " + + "
    " + + "
    " + + "
    Dec (deg)=
    " + + "
    " + dec + "
    " + + "
    " + } + d.text += "
    " + + "
    Value (" + u + ") =
    " + + "
    " + val + "
    "; } else { d.text = ""; } @@ -548,6 +568,17 @@ def get_data(self): except KeyError: self.signal_units = '' + ny, nx = self.data.shape + col_idx, row_idx = np.indices((ny, nx)) + try: + wcs = WCS(header) + if wcs.has_celestial: + self.wcs_coord = wcs.pixel_to_world_values(col_idx, row_idx) + else: + self.wcs_coord = None + except (ValueError, TypeError): + self.wcs_coord = None + def get_scale(self): """Calculate the limits for the display, following the ZScale function originally created or IRAF. diff --git a/jwql/website/apps/jwql/templates/explore_image.html b/jwql/website/apps/jwql/templates/explore_image.html index af750a8c8..1115c0dc5 100644 --- a/jwql/website/apps/jwql/templates/explore_image.html +++ b/jwql/website/apps/jwql/templates/explore_image.html @@ -10,7 +10,7 @@
    -

    Explore Mode

    +

    Explore Mode

    {{ file_root }}_{{ filetype }}.fits

    View Image From fd8f1251c0e1eaef6aa091d7ef305818f7477cd7 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Tue, 13 Jun 2023 18:22:47 -0400 Subject: [PATCH 384/449] Autoscale plots to windowed data instead of zoom tools --- jwql/utils/interactive_preview_image.py | 127 +++++++++++++++++------- 1 file changed, 92 insertions(+), 35 deletions(-) diff --git a/jwql/utils/interactive_preview_image.py b/jwql/utils/interactive_preview_image.py index 615176cd2..a2e81ea73 100644 --- a/jwql/utils/interactive_preview_image.py +++ b/jwql/utils/interactive_preview_image.py @@ -27,11 +27,9 @@ from bokeh.layouts import gridplot, layout from bokeh.models import ( BasicTicker, BoxZoomTool, Button, ColorBar, ColumnDataSource, - CustomJS, CustomJSTransform, DataRange1d, Div, HoverTool, - LinearColorMapper, LogColorMapper, LogTicker, RadioGroup, Range1d, Row, - Select, Spacer, Spinner, WheelZoomTool) + CustomJS, Div, HoverTool, LinearColorMapper, LogColorMapper, LogTicker, + RadioGroup, Range1d, Row, Select, Spacer, Spinner, WheelZoomTool) from bokeh.plotting import figure, output_file, show, save -from bokeh.transform import transform from jwst.datamodels import dqflags @@ -249,48 +247,59 @@ def line_plots(self, main_figure): """ new_plots = [] new_lines = [] + match_ranges = [] + value_ranges = [] + ny, nx = self.data.shape col_idx, row_idx = np.indices((ny, nx)) - - for index_direction in ['x', 'y']: + directions = ['x', 'y'] + for index_direction in directions: if index_direction == 'x': # column plots - fig = figure(plot_width=200, plot_height=main_figure.height, - tools='xwheel_zoom, xpan, reset', + fig = figure(plot_width=200, plot_height=main_figure.height, tools='', y_axis_location='right', margin=(0, 0, 0, 30)) - fig.x_range = DataRange1d(only_visible=True) + fig.toolbar.logo = None + + fig.x_range = Range1d() fig.y_range = Range1d() match_range = fig.y_range main_range = main_figure.y_range + value_range = fig.x_range fig.xaxis.axis_label = self.signal_units fig.yaxis.axis_label = 'Row pixel (y)' fig.xaxis.major_label_orientation = np.radians(-45) - x_plot = 'data' - y_plot = 'index' n_plot = nx - data_source = {'data': self.data, 'index': col_idx} - source = ColumnDataSource(data_source) + initial_visible = n_plot // 2 + + x = self.data.T + y = col_idx.T + min_val = np.nanmin(x[initial_visible]) + max_val = np.nanmax(x[initial_visible]) + else: # row plots - fig = figure(plot_height=200, plot_width=main_figure.width, - tools='ywheel_zoom, ypan, reset') + fig = figure(plot_height=200, plot_width=main_figure.width, tools='') + fig.toolbar.logo = None - fig.y_range = DataRange1d(only_visible=True) + fig.y_range = Range1d() fig.x_range = Range1d() match_range = fig.x_range main_range = main_figure.x_range + value_range = fig.y_range fig.xaxis.axis_label = 'Column pixel (x)' fig.yaxis.axis_label = self.signal_units - x_plot = 'index' - y_plot = 'data' - n_plot = ny # indexing is off by 1 for row plots for some reason - data_source = {'data': self.data.T, 'index': row_idx.T + 1} - source = ColumnDataSource(data_source) + n_plot = ny + initial_visible = n_plot // 2 + + x = row_idx + 1 + y = self.data + min_val = np.nanmin(y[initial_visible]) + max_val = np.nanmax(y[initial_visible]) # match one of the axes to the main figure if main_range.start is not None: @@ -302,28 +311,29 @@ def line_plots(self, main_figure): main_range.js_link('end', match_range, 'end') main_range.js_link('end', match_range, 'reset_end') + # initialize the other to the data + pad = 0.1 * (max_val - min_val) + value_range.start = min_val - pad + value_range.end = max_val + pad + + # plot a step line for each column and plot + # all but one are hidden to start lines = [] - initial_visible = n_plot // 2 for i in range(n_plot): - idx_transform = CustomJSTransform( - args={'idx': i}, - v_func=""" - const x_val = new Float64Array(xs.length); - for (let i = 0; i < xs.length; i++) { - x_val[i] = xs[i][idx]; - } - return x_val; - """) - line = fig.step(x=transform(x_plot, idx_transform), - y=transform(y_plot, idx_transform), - mode='before', source=source, + line = fig.step(x=x[i], y=y[i], + mode='before', visible=(i == initial_visible), name=f'Data at {index_direction}={i}') lines.append(line) fig.title = lines[initial_visible].name + new_lines.append(lines) new_plots.append(fig) + match_ranges.append(match_range) + value_ranges.append(value_range) + # watch for tap on plot - makes a new line visible, + # matching the selected point update_plot = CustomJS( args={'lines': new_lines, 'figures': new_plots}, code=""" @@ -349,6 +359,48 @@ def line_plots(self, main_figure): }""") main_figure.js_on_event('tap', update_plot) + # watch for changes to matched axis to reset data range on value axis + for i in range(len(directions)): + limit_reset = CustomJS( + args={'line': new_lines[i], + 'direction': directions[i], + 'value_range': value_ranges[i], + 'match_range': match_ranges[i]}, + code=""" + var min_val = Infinity; + var max_val = -Infinity; + for (let i=0; i < line.length; i++) { + if (line[i].visible == true) { + var data, idx; + if (direction == 'x') { + data = line[i].data_source.data['x']; + idx = line[i].data_source.data['y']; + } else { + data = line[i].data_source.data['y']; + idx = line[i].data_source.data['x']; + } + for (let j=0; j < data.length; j++) { + if (idx[j] >= match_range.start + && idx[j] <= match_range.end) { + min_val = Math.min(data[j], min_val); + max_val = Math.max(data[j], max_val); + } + } + break; + } + } + if (min_val < Infinity && min_val != max_val) { + var pad = 0.1 * (max_val - min_val); + value_range.start = min_val - pad; + value_range.end = max_val + pad; + } + """) + match_ranges[i].js_on_change('start', limit_reset) + match_ranges[i].js_on_change('end', limit_reset) + + # also reset the limits when the plot is tapped for a new column/row + main_figure.js_on_event('tap', limit_reset) + return new_plots def add_hover_tool(self, source, images): @@ -375,7 +427,12 @@ def add_hover_tool(self, source, images): var x = idx[0].dim1; var y = idx[0].dim2; var flat = idx[0].flat_index; - var val = s.data['image'][0][y][x].toPrecision(5); + var val = s.data['image'][0][y][x]; + if (!Number.isFinite(val)) { + val = 'NaN'; + } else { + val = val.toPrecision(5); + } d.text = "

    Pixel Value
    " + "
    " + "
    " + From 7b34d6007d1919d3aa0025ca7265e20c063673a2 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Wed, 14 Jun 2023 15:34:08 -0400 Subject: [PATCH 385/449] Fix timing and NaN issues with plot limit setting --- jwql/utils/interactive_preview_image.py | 28 ++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/jwql/utils/interactive_preview_image.py b/jwql/utils/interactive_preview_image.py index a2e81ea73..07545fd6b 100644 --- a/jwql/utils/interactive_preview_image.py +++ b/jwql/utils/interactive_preview_image.py @@ -367,6 +367,13 @@ def line_plots(self, main_figure): 'value_range': value_ranges[i], 'match_range': match_ranges[i]}, code=""" + var timeout; + if (direction == 'x') { + timeout = window._autoscale_timeout_x; + } else { + timeout = window._autoscale_timeout_y; + } + clearTimeout(timeout); var min_val = Infinity; var max_val = -Infinity; for (let i=0; i < line.length; i++) { @@ -382,17 +389,28 @@ def line_plots(self, main_figure): for (let j=0; j < data.length; j++) { if (idx[j] >= match_range.start && idx[j] <= match_range.end) { - min_val = Math.min(data[j], min_val); - max_val = Math.max(data[j], max_val); + if (Number.isFinite(data[j])) { + min_val = Math.min(data[j], min_val); + max_val = Math.max(data[j], max_val); + } } } break; } } - if (min_val < Infinity && min_val != max_val) { + if (Number.isFinite(min_val) && Number.isFinite(max_val) && min_val != max_val) { var pad = 0.1 * (max_val - min_val); - value_range.start = min_val - pad; - value_range.end = max_val + pad; + if (direction == 'x') { + window._autoscale_timeout_x = setTimeout(function() { + value_range.start = min_val - pad; + value_range.end = max_val + pad; + }); + } else { + window._autoscale_timeout_y = setTimeout(function() { + value_range.start = min_val - pad; + value_range.end = max_val + pad; + }); + } } """) match_ranges[i].js_on_change('start', limit_reset) From 82d0360bf449323a24f2b4a831332f64e404bb2d Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 23 Jun 2023 15:29:01 -0400 Subject: [PATCH 386/449] Add download links to image pages --- jwql/website/apps/jwql/static/js/jwql.js | 5 +++++ jwql/website/apps/jwql/templates/explore_image.html | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index 73f33eae5..f13e82a9a 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -125,6 +125,11 @@ document.getElementById("visit_id").innerHTML = parsed_name.visit_id; document.getElementById("detector").innerHTML = file_root.split('_')[3]; + // Add a link to download the file from MAST + document.getElementById("fits_filename").setAttribute('href', + 'https://mast.stsci.edu/api/v0.1/Download/file?uri=mast%3AJWST%2Fproduct%2F' + + fits_filename + '.fits'); + // Show the appropriate image var img = document.getElementById("image_viewer"); var jpg_filepath = '/static/preview_images/' + parsed_name.program + '/' + file_root + '_' + type + '_integ0.jpg'; diff --git a/jwql/website/apps/jwql/templates/explore_image.html b/jwql/website/apps/jwql/templates/explore_image.html index 205bffc2b..d674ebfb7 100644 --- a/jwql/website/apps/jwql/templates/explore_image.html +++ b/jwql/website/apps/jwql/templates/explore_image.html @@ -12,9 +12,10 @@

    Explore Mode

    {{ file_root }}_{{ filetype }}.fits

    -

    +

    View Image View Proposal + Download File

    @@ -129,4 +130,4 @@
    Submit Anomaly
    -{% endblock %} \ No newline at end of file +{% endblock %} From 153727854c50bb1e05753198d38df543d88f982c Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 3 Jul 2023 17:22:38 -0400 Subject: [PATCH 387/449] Tweaks from testing on server --- jwql/instrument_monitors/common_monitors/dark_monitor.py | 2 +- jwql/shared_tasks/run_pipeline.py | 2 ++ jwql/shared_tasks/shared_tasks.py | 7 ++++--- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index b8399b1d9..929d9d099 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -712,7 +712,7 @@ def process(self, file_list): pipeline_files.append(filename) # Specify that we want to skip the dark current correction step - step_args = {'dark_current': {'skip: True'}} + step_args = {'dark_current': {'skip': True}} # Call the pipeline outputs = run_parallel_pipeline(pipeline_files, "dark", ["rate", "rateints"], self.instrument, step_args=step_args) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 2e34f58e3..2ab107318 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -334,6 +334,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T working_path = args.working_path pipe_type = args.pipe outputs = args.outputs + step_args = args.step_args status_file = os.path.join(working_path, short_name+"_status.txt") with open(status_file, 'w') as out_file: @@ -344,6 +345,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T out_file.write("\tinstrument is {} ({})\n".format(instrument, type(instrument))) out_file.write("\tinput_file is {} ({})\n".format(input_file, type(input_file))) out_file.write("\tshort_name is {} ({})\n".format(short_name, type(short_name))) + out_file.write("\tstep_args is {} ({})\n".format(step_args, type(step_args))) if not os.path.isfile(args.input_file): raise FileNotFoundError("No input file {}".format(args.input_file)) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index 047084d62..42ee9021b 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -239,7 +239,7 @@ def convert_step_args_to_string(args_dict): args_str : str String representation of ``args_dict`` """ - args_str='{' + args_str="'{" for i, step in enumerate(args_dict): args_str += f'"{step}":' @@ -248,9 +248,10 @@ def convert_step_args_to_string(args_dict): args_str += f'"{param}":"{val}"' if j < len(args_dict[step])-1: args_str += ', ' - args_str += '}' + args_str += "}" if i < len(args_dict)-1: args_str += ',' + args_str += "}'" return args_str @@ -262,7 +263,7 @@ def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_fi command = command.format(name, cmd, outputs, cal_dir, ins, in_file, short_name, cores, step_args_str) logging.info("Running {}".format(command)) process = Popen(command, shell=True, executable="/bin/bash", stderr=PIPE) - with process.stderr: + with process.stderr: Date: Fri, 7 Jul 2023 12:03:37 -0400 Subject: [PATCH 388/449] Add help message for column/row plots --- jwql/website/apps/jwql/static/js/jwql.js | 7 +++++++ jwql/website/apps/jwql/templates/explore_image.html | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js index dfc3baa48..4bca2470e 100644 --- a/jwql/website/apps/jwql/static/js/jwql.js +++ b/jwql/website/apps/jwql/static/js/jwql.js @@ -1109,6 +1109,13 @@ function update_wata_page(base_url) { */ $('#explore_image').html(content); + // Add a help message for plots + if (show_line_plots === true) { + $('#help').html('iClick on the image to update the column/row plots.'); + } else { + $('#help').html(''); + } + // Replace loading screen document.getElementById("loading").style.display = "none"; document.getElementById("explore_image").style.display = "inline-block"; diff --git a/jwql/website/apps/jwql/templates/explore_image.html b/jwql/website/apps/jwql/templates/explore_image.html index c1629fa9b..62ac88c9b 100644 --- a/jwql/website/apps/jwql/templates/explore_image.html +++ b/jwql/website/apps/jwql/templates/explore_image.html @@ -17,7 +17,7 @@

    {{ file_root }}_{{ filetype }}.fits

    View Proposal Download File

    - +

    From 75b2edb74cc17a8755fdf5cae88781d5707be533 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 7 Jul 2023 16:45:50 -0400 Subject: [PATCH 389/449] Update docstring --- jwql/website/apps/jwql/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index af4beb1b0..f1916dce1 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -1009,7 +1009,7 @@ def explore_image_ajax(request, inst, file_root, filetype, line_plots='false', l filetype : str Type of file (e.g. ``uncal``) line_plots : str - Scaling to implement in interactive preview image ("log" or "lin") + If 'true', column and row plots will be computed and shown with the image. low_lim : str Signal value to use as the lower limit of the displayed image. If "None", it will be calculated using the ZScale function high_lim : str From f8b84a338df7235b76fdf8d2d1df78dd872d0cdc Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 7 Jul 2023 16:46:55 -0400 Subject: [PATCH 390/449] Fix hover tool for DQ image --- jwql/utils/interactive_preview_image.py | 34 +++++++++++++++---------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/jwql/utils/interactive_preview_image.py b/jwql/utils/interactive_preview_image.py index 07545fd6b..d3b93bf73 100644 --- a/jwql/utils/interactive_preview_image.py +++ b/jwql/utils/interactive_preview_image.py @@ -439,17 +439,28 @@ def add_hover_tool(self, source, images): hover_tool : bokeh.models. """ hover_div = Div(height=300, width=300) - hover_callback = CustomJS(args={'s': source, 'd': hover_div, 'u': self.signal_units}, code=""" + + is_dq = ('DQ' in self.extname) + hover_callback = CustomJS(args={'s': source, 'd': hover_div, + 'u': self.signal_units, 'dq': is_dq}, code=""" const idx = cb_data.index.image_indices; if (idx.length > 0) { var x = idx[0].dim1; var y = idx[0].dim2; var flat = idx[0].flat_index; - var val = s.data['image'][0][y][x]; - if (!Number.isFinite(val)) { - val = 'NaN'; + var val; + var label; + if (dq === true) { + val = s.data['dq'][0][y][x].join(', '); + label = "Value"; } else { - val = val.toPrecision(5); + var val = s.data['image'][0][y][x]; + if (!Number.isFinite(val)) { + val = 'NaN'; + } else { + val = val.toPrecision(5); + } + label = "Value (" + u + ")"; } d.text = "
    Pixel Value
    " + "
    " + @@ -461,7 +472,7 @@ def add_hover_tool(self, source, images): var ra = s.data['ra'][0][flat].toPrecision(8); var dec = s.data['dec'][0][flat].toPrecision(8); d.text += "
    " + - "
    RA (deg) =
    " + + "
    RA (deg)=
    " + "
    " + ra + "
    " + "
    " + "
    " + @@ -470,18 +481,15 @@ def add_hover_tool(self, source, images): "
    " } d.text += "
    " + - "
    Value (" + u + ") =
    " + + "
    " + label + "=
    " + "
    " + val + "
    "; } else { d.text = ""; } """) - if 'DQ' not in self.extname: - hover_tool = HoverTool(tooltips=None, mode='mouse', renderers=images, - callback=hover_callback) - else: - hover_tool = HoverTool(tooltips=None, mode='mouse', renderers=images, - callback=hover_callback) + hover_tool = HoverTool(tooltips=None, mode='mouse', renderers=images, + callback=hover_callback) + return hover_div, hover_tool def add_interactive_controls(self, images, color_bars): From 51bcd01fe1e79bfab70680767ebe09547dbed1b8 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 7 Jul 2023 17:32:06 -0400 Subject: [PATCH 391/449] Fix hover tool for uncal images --- jwql/utils/interactive_preview_image.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/jwql/utils/interactive_preview_image.py b/jwql/utils/interactive_preview_image.py index d3b93bf73..211a03340 100644 --- a/jwql/utils/interactive_preview_image.py +++ b/jwql/utils/interactive_preview_image.py @@ -454,7 +454,13 @@ def add_hover_tool(self, source, images): val = s.data['dq'][0][y][x].join(', '); label = "Value"; } else { - var val = s.data['image'][0][y][x]; + // get the data from the array of arrays + val = s.data['image'][0][y][x]; + if (val === undefined) { + // uncal images have to be addressed with the flat index + val = s.data['image'][0][flat]; + } + // report any non-number as NaN if (!Number.isFinite(val)) { val = 'NaN'; } else { From eb45ae1b48502c0af68dc9f6094776a84bc360c4 Mon Sep 17 00:00:00 2001 From: Melanie Clarke Date: Fri, 7 Jul 2023 17:47:58 -0400 Subject: [PATCH 392/449] Handle single-valued DQ pixels --- jwql/utils/interactive_preview_image.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jwql/utils/interactive_preview_image.py b/jwql/utils/interactive_preview_image.py index 211a03340..da9563770 100644 --- a/jwql/utils/interactive_preview_image.py +++ b/jwql/utils/interactive_preview_image.py @@ -451,7 +451,10 @@ def add_hover_tool(self, source, images): var val; var label; if (dq === true) { - val = s.data['dq'][0][y][x].join(', '); + val = s.data['dq'][0][y][x]; + if (Array.isArray(val)) { + val = val.join(', '); + } label = "Value"; } else { // get the data from the array of arrays From 13dcb142b0c71d0136fe03b27f315b0057b9b4dc Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Wed, 12 Jul 2023 10:05:27 -0400 Subject: [PATCH 393/449] prepare models for external postgres db --- jwql/example_config.json | 8 ++++++++ jwql/website/jwql_proj/settings.py | 5 +---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/jwql/example_config.json b/jwql/example_config.json index 2f9c3ad8c..112b13755 100644 --- a/jwql/example_config.json +++ b/jwql/example_config.json @@ -10,6 +10,14 @@ "host" : "", "port" : "" }, + "django_database" : { + "engine" : "", + "name" : "", + "user" : "", + "password" : "", + "host" : "", + "port" : "" + }, "jwql_dir" : "", "jwql_version": "", "server_type": "", diff --git a/jwql/website/jwql_proj/settings.py b/jwql/website/jwql_proj/settings.py index 676d164d9..6dc070ca7 100644 --- a/jwql/website/jwql_proj/settings.py +++ b/jwql/website/jwql_proj/settings.py @@ -108,10 +108,7 @@ # Database # https://docs.djangoproject.com/en/2.0/ref/settings/#databases DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), - }, + 'default': get_config()['django_database'] } # Password validation From 06458e6968c30a8be05b098724f00d6b95f87f89 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Wed, 12 Jul 2023 11:56:52 -0400 Subject: [PATCH 394/449] db keys must be all caps --- jwql/example_config.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/jwql/example_config.json b/jwql/example_config.json index 112b13755..937a1d63b 100644 --- a/jwql/example_config.json +++ b/jwql/example_config.json @@ -11,12 +11,12 @@ "port" : "" }, "django_database" : { - "engine" : "", - "name" : "", - "user" : "", - "password" : "", - "host" : "", - "port" : "" + "ENGINE" : "", + "NAME" : "", + "USER" : "", + "PASSWORD" : "", + "HOST" : "", + "PORT" : "" }, "jwql_dir" : "", "jwql_version": "", From 9f97ca9a799b6db2fe4f8532b0d56c176596d87b Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Thu, 13 Jul 2023 17:39:46 -0400 Subject: [PATCH 395/449] update tab_service search for all intents --- jwql/website/apps/jwql/data_containers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index c3828b11a..1b6cdcae7 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -1248,9 +1248,9 @@ def get_instrument_proposals(instrument): List of proposals for the given instrument """ tap_service = vo.dal.TAPService("https://vao.stsci.edu/caomtap/tapservice.aspx") - tap_results = tap_service.search(f"select distinct proposal_id from dbo.ObsPointing where obs_collection='JWST' and calib_level>0 and instrument_name like '{instrument.lower()}%'") + tap_results = tap_service.search(f"select distinct prpID from CaomObservation where collection='JWST' and maxLevel>0 and insName like '{instrument.lower()}%'") prop_table = tap_results.to_table() - proposals = prop_table['proposal_id'].data + proposals = prop_table['prpID'].data inst_proposals = sorted(proposals.compressed(), reverse=True) return inst_proposals From 02a6b63ef686357b7920ad8603ddd15b828080a4 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 25 Jul 2023 12:47:08 -0400 Subject: [PATCH 396/449] rateints case working --- jwql/shared_tasks/run_pipeline.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 2ab107318..7e743bbf8 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -50,6 +50,7 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co status_f.write("\t input_file_basename is {} ({})\n".format(input_file_basename, type(input_file_basename))) status_f.write("\t start_dir is {} ({})\n".format(start_dir, type(start_dir))) status_f.write("\t uncal_file is {} ({})\n".format(uncal_file, type(uncal_file))) + status_f.write(f"\t outputs is {outputs}\n") try: copy_files([input_file], work_directory) @@ -133,8 +134,12 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co pass model[0].save(output_file) if 'rateints' in outputs: - model[1].save(output_file.replace('rate', 'rateints')) - + outbase = os.path.basename(output_file) + outbase = outbase.replace('rate', 'rateints') + output_file = os.path.join(work_directory, outbase) + model[1].save(output_file) + with open(status_file, 'a+') as status_f: + status_f.write(f"Saved rateints model to {output_file}\n") done = True for output in outputs: output_name = "{}_{}.fits".format(short_name, output) From 9f5f05d1fc8271faaf36554d21aff59dbbcacabd Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 25 Jul 2023 12:54:01 -0400 Subject: [PATCH 397/449] Uncomment the get() lines --- jwql/shared_tasks/shared_tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index 42ee9021b..244d1a3e8 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -746,7 +746,7 @@ def run_pipeline(input_file, in_ext, ext_or_exts, instrument, jump_pipe=False): uncal_name = os.path.basename(uncal_file) result = start_pipeline(uncal_name, short_name, ext_or_exts, instrument, jump_pipe=jump_pipe) logging.info("\t\tStarting with ID {}".format(result.id)) - #processed_path = result.get() + processed_path = result.get() logging.info("\t\tPipeline Complete") output = retrieve_files(short_name, ext_or_exts, retrieve_dir) except Exception as e: @@ -838,7 +838,7 @@ def run_parallel_pipeline(input_files, in_ext, ext_or_exts, instrument, jump_pip for short_name in results: try: logging.info("\tWaiting for {} ({})".format(short_name, results[short_name].id)) - #processed_path = results[short_name].get() + processed_path = results[short_name].get() logging.info("\t{} retrieved".format(short_name)) outputs[input_file_paths[short_name]] = retrieve_files(short_name, ext_or_exts, output_dirs[short_name]) logging.info("\tFiles copied for {}".format(short_name)) From 21e0d3eb5ad810fccf8a930a4ee5c67f239dedf3 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 25 Jul 2023 15:07:18 -0400 Subject: [PATCH 398/449] Remove bad cut and paste text --- jwql/shared_tasks/shared_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index e83dd66ce..b35410291 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -263,7 +263,7 @@ def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_fi command = command.format(name, cmd, outputs, cal_dir, ins, in_file, short_name, cores, step_args_str) logging.info("Running {}".format(command)) process = Popen(command, shell=True, executable="/bin/bash", stderr=PIPE) - with process.stderr: Date: Tue, 25 Jul 2023 15:20:48 -0400 Subject: [PATCH 399/449] Return dark monitor to specifying rate outputs. rateints will be another PR --- jwql/instrument_monitors/common_monitors/dark_monitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index 929d9d099..8ffab9146 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -715,7 +715,7 @@ def process(self, file_list): step_args = {'dark_current': {'skip': True}} # Call the pipeline - outputs = run_parallel_pipeline(pipeline_files, "dark", ["rate", "rateints"], self.instrument, step_args=step_args) + outputs = run_parallel_pipeline(pipeline_files, "dark", ["rate"], self.instrument, step_args=step_args) for filename in file_list: processed_file = filename.replace("_dark", "_rate") if processed_file not in slope_files and os.path.isfile(processed_file): From 95de7d98f945b681d951897e52d74d0298275e1a Mon Sep 17 00:00:00 2001 From: Bradley Sappington <101193271+BradleySappington@users.noreply.github.com> Date: Tue, 25 Jul 2023 16:39:30 -0400 Subject: [PATCH 400/449] Update README.md --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5909b8889..70b4e173b 100644 --- a/README.md +++ b/README.md @@ -154,10 +154,11 @@ Any questions about the `jwql` project or its software can be directed to `jwql@ - Mees Fix (Technical Lead, INS) [@mfixstsci](https://github.com/mfixstsci) - Misty Cracraft (INS) [@cracraft](https://github.com/cracraft) - Mike Engesser (INS) [@mengesser](https://github.com/mengesser) -- Shannon Osborne (INS) [@shanosborne](https://github.com/shanosborne) - Maria Pena-Guerrero [@penaguerrero](https://github.com/penaguerrero) - Ben Sunnquist (INS) [@bsunnquist](https://github.com/bsunnquist) - Brian York (INS) [@york-stsci](https://github.com/york-stsci) +- Bradley Sappington (INS) [@bradleysappington](https://github.com/bradleysappington) +- Melanie Clarke (INS) [@melanieclarke](https://github.com/melanieclarke) ## Past Development Team Members - Matthew Bourque (INS) [@bourque](https://github.com/bourque) @@ -168,7 +169,7 @@ Any questions about the `jwql` project or its software can be directed to `jwql@ - Sara Ogaz (DMD) [@SaOgaz](https://github.com/SaOgaz) - Catherine Martlin (INS) [@catherine-martlin](https://github.com/catherine-martlin) - Johannes Sahlmann (INS) [@Johannes-Sahlmann](https://github.com/johannes-sahlmann) - +- Shannon Osborne (INS) [@shanosborne](https://github.com/shanosborne) ## Acknowledgments: - Faith Abney (DMD) From 4adc5853acfc0398b3035f8fbbdcafd048afd480 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 27 Jul 2023 15:28:36 -0400 Subject: [PATCH 401/449] use json loads rather than custom function. resp to review comments --- jwql/shared_tasks/run_pipeline.py | 3 -- jwql/shared_tasks/shared_tasks.py | 54 +++++-------------------------- 2 files changed, 8 insertions(+), 49 deletions(-) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 7e743bbf8..d9af19086 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -213,9 +213,6 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T if instrument == 'nircam': params['refpix'] = dict(odd_even_rows=False) - # Default CR rejection threshold is too low - #params['jump'] = dict(rejection_threshold=15) - # Set up to save jump step output params['jump']['save_results'] = True params['jump']['output_dir'] = work_directory diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index b35410291..b7164b042 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -75,6 +75,7 @@ def some_function(some_arguments): from copy import deepcopy import gc from glob import glob +import json import logging from logging import FileHandler, StreamHandler import os @@ -224,40 +225,9 @@ def collect_after_task(**kwargs): gc.collect() -def convert_step_args_to_string(args_dict): - """Convert the nested dictionary containing pipeline step parameter keyword/value pairs - to a string so that it can be passed via command line - - Parameters - ---------- - args_dict : dict - Nested dictionary. Top level keys are pipeline step names. Values are dictionaries containing - keyword value pairs for that step. - - Returns - ------- - args_str : str - String representation of ``args_dict`` - """ - args_str="'{" - - for i, step in enumerate(args_dict): - args_str += f'"{step}":' - args_str += '{' - for j, (param, val) in enumerate(args_dict[step].items()): - args_str += f'"{param}":"{val}"' - if j < len(args_dict[step])-1: - args_str += ', ' - args_str += "}" - if i < len(args_dict)-1: - args_str += ',' - args_str += "}'" - return args_str - - def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_file, cores, step_args): # Convert step_args dictionary to a string so that it can be passed via command line - step_args_str = convert_step_args_to_string(step_args) + step_args_str = json.loads(step_args) command = "{} {} {} '{}' {} {} {} {} --step_args {}" command = command.format(name, cmd, outputs, cal_dir, ins, in_file, short_name, cores, step_args_str) @@ -557,7 +527,7 @@ def prep_file(input_file, in_ext): short_name : str The exposure ID with the calibration tag and the fits extension chopped off. - uncal_file : str + input_name : str The raw file to be calibrated """ config = get_config() @@ -569,16 +539,8 @@ def prep_file(input_file, in_ext): input_path, input_name = os.path.split(input_file) logging.info("\tPath is {}, file is {}".format(input_path, input_name)) - if "uncal" not in in_ext and "dark" not in in_ext: - logging.info("\tSwitching from {} to uncal".format(in_ext)) - uncal_name = os.path.basename(input_file).replace(in_ext, "uncal") - uncal_file = filesystem_path(uncal_name, check_existence=True) - else: - uncal_file = input_file - uncal_name = input_name - - if not os.path.isfile(uncal_file): - raise FileNotFoundError("Input File {} does not exist.".format(uncal_file)) + if not os.path.isfile(input_file): + raise FileNotFoundError("Input File {} does not exist.".format(input_file)) output_file_or_files = [] short_name = input_name.replace("_" + in_ext, "").replace(".fits", "") @@ -590,9 +552,9 @@ def prep_file(input_file, in_ext): logging.critical(msg.format(short_name)) raise ValueError("Redis lock for {} is in an unknown state".format(short_name)) logging.info("\t\tAcquired Lock.") - logging.info("\t\tCopying {} to {}".format(uncal_file, send_path)) - copy_files([uncal_file], send_path) - return short_name, cal_lock, os.path.join(send_path, uncal_name) + logging.info("\t\tCopying {} to {}".format(input_file, send_path)) + copy_files([input_file], send_path) + return short_name, cal_lock, os.path.join(send_path, input_name) def start_pipeline(input_file, short_name, ext_or_exts, instrument, jump_pipe=False, step_args={}): From 5fdcec1fee1a96732786f3e1aef1fe41b1b7d8ab Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 28 Jul 2023 12:33:20 -0400 Subject: [PATCH 402/449] Returning to custom function for dictioanry to string conversion --- jwql/shared_tasks/run_pipeline.py | 1 - jwql/shared_tasks/shared_tasks.py | 35 +++++++++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index d9af19086..74754afd0 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -87,7 +87,6 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co # Set any steps the user specifically asks to skip for step, step_dict in step_args.items(): if 'skip' in step_dict: - print(f'SKIP the {step}!!') if step_dict['skip']: steps[step] = False diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index b7164b042..7a05f6f92 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -224,10 +224,41 @@ def after_setup_celery_logger(logger, **kwargs): def collect_after_task(**kwargs): gc.collect() +def convert_step_args_to_string(args_dict): + """Convert the nested dictionary containing pipeline step parameter keyword/value pairs + to a string so that it can be passed via command line + + Parameters + ---------- + args_dict : dict + Nested dictionary. Top level keys are pipeline step names. Values are dictionaries containing + keyword value pairs for that step. + + Returns + ------- + args_str : str + String representation of ``args_dict`` + """ + args_str="'{" + + for i, step in enumerate(args_dict): + args_str += f'"{step}":' + args_str += '{' + for j, (param, val) in enumerate(args_dict[step].items()): + args_str += f'"{param}":"{val}"' + if j < len(args_dict[step])-1: + args_str += ', ' + args_str += "}" + if i < len(args_dict)-1: + args_str += ',' + args_str += "}'" + return args_str + def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_file, cores, step_args): - # Convert step_args dictionary to a string so that it can be passed via command line - step_args_str = json.loads(step_args) + # Convert step_args dictionary to a string so that it can be passed via command line. + # For some reason, json.dumps() doesn't seem to work correctly, so we use a custom function. + step_args_str = convert_step_args_to_string(step_args) command = "{} {} {} '{}' {} {} {} {} --step_args {}" command = command.format(name, cmd, outputs, cal_dir, ins, in_file, short_name, cores, step_args_str) From 7b1fe988bd52cbfbd8ba4052e58b3efd02831da1 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 28 Jul 2023 13:54:06 -0400 Subject: [PATCH 403/449] Return jump threshold statement. --- jwql/shared_tasks/run_pipeline.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 74754afd0..a79167e0b 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -212,6 +212,9 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T if instrument == 'nircam': params['refpix'] = dict(odd_even_rows=False) + # Default CR rejection threshold is too low + params['jump']['rejection_threshold'] = 15 + # Set up to save jump step output params['jump']['save_results'] = True params['jump']['output_dir'] = work_directory From dd136258b7436e32685e2c09874ed87453b014af Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Fri, 28 Jul 2023 14:48:26 -0400 Subject: [PATCH 404/449] update anomaly search backend --- jwql/website/apps/jwql/data_containers.py | 42 +++-------------------- 1 file changed, 5 insertions(+), 37 deletions(-) diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py index 1b6cdcae7..f4ecb6aa7 100644 --- a/jwql/website/apps/jwql/data_containers.py +++ b/jwql/website/apps/jwql/data_containers.py @@ -1498,8 +1498,7 @@ def get_rootnames_from_query(parameters): filtered_rootnames : list A list of all root filenames filtered from the given parameters """ - # TODO - This code setup is temporary until the merge to Postgres is complete. - # selected_rootfileinfos = RootFileInfo.objects.none() + filtered_rootnames = [] # Each Query Selection is Instrument specific for inst in parameters[QUERY_CONFIG_KEYS.INSTRUMENTS]: @@ -1544,23 +1543,13 @@ def get_rootnames_from_query(parameters): current_ins_rootfileinfos = current_ins_rootfileinfos.filter(read_patt__in=inst_read_patt) if (inst_subarray != []): current_ins_rootfileinfos = current_ins_rootfileinfos.filter(subarray__in=inst_subarray) - - # TODO - This uncommented CODE is what we will use while DJANGO is still using the SQLITE3 DB. - # ONCE DB IS MIGRATED TO POSTGRES WE CAN REPLACE THIS CODE WITH THE UNTESTED CODE COMMENTED OUT BELOW - # >>> START CODE PRE DB MIGRATION HERE <<< if (inst_anomalies != []): - # If the rootfile info has any of the marked anomalies we want it - # Make union of marked anomalies from our current query set, then save intersection. - anomaly_filters = None + anomaly_rootfileinfos = RootFileInfo.objects.none() for anomaly in inst_anomalies: + # If the rootfile info has any of the marked anomalies we want it anomaly_filter = "anomalies__" + str(anomaly).lower() - this_anomaly_rootnames = current_ins_rootfileinfos.filter(**{anomaly_filter: True}) - if anomaly_filters is None: - anomaly_filters = this_anomaly_rootnames - else: - anomaly_filters |= this_anomaly_rootnames - current_ins_rootfileinfos = anomaly_filters - # >>> END CODE PRE DB MIGRATION HERE <<< + anomaly_rootfileinfos = anomaly_rootfileinfos.union(current_ins_rootfileinfos.filter(**{anomaly_filter: True})) + current_ins_rootfileinfos = current_ins_rootfileinfos.intersection(anomaly_rootfileinfos) # sort as desired if sort_type.upper() == 'ASCENDING': @@ -1577,27 +1566,6 @@ def get_rootnames_from_query(parameters): return filtered_rootnames - # TODO - BELOW IS THE OUTLINE OF CODE WE WANT TO USE, HOWEVER THIS CAN'T BE IMPLEMENTED WITH DJANGO RUNNING SQLITE - # ONCE WE MIGRATE TO POSTGRES WE CAN IMPLEMENT THE BELOW FUNCTIONALITY WHICH SHOULD MAKE THIS CODE MOVE A LITTLE FASTER - # NOTE: THIS CODE OUTLINE IS UNTESTED and is only a rough outline! - # >>> START CODE AFTER DB MIGRATION HERE <<< - # if (inst_anomalies != []): - # # If the rootfile info has any of the marked anomalies we want it - # # Make union of marked anomalies from our current query set, then save intersection. - # anomaly_rootfileinfos = RootFileInfo.objects.none() - - # for anomaly in inst_anomalies: - # anomaly_filter = "anomalies__" + str(anomaly).lower() - # anomaly_rootfileinfos.union(current_ins_rootfileinfos.filter(**{anomaly_filter: True})) - - # current_ins_rootfileinfos = current_ins_rootfileinfos.intersection(anomaly_rootfileinfos) - - # # Add this instrument's query set to our return queryset - # selected_rootfileinfos = selected_rootfileinfos.union(current_ins_rootfileinfos) - - # return selected_rootfileinfos # OR RETURN LIST OF ROOTNAMES - # >>> END CODE AFTER DB MIGRATION HERE <<< - def get_thumbnails_by_instrument(inst): """Return a list of thumbnails available in the filesystem for the From 7f4334566f787a1f75dc94b8ab85dd76d9437ab4 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 1 Aug 2023 16:37:32 -0400 Subject: [PATCH 405/449] exploring --- environment_python_3.10.yml | 64 ++++++++++++++++++------------------- environment_python_3.9.yml | 64 ++++++++++++++++++------------------- 2 files changed, 64 insertions(+), 64 deletions(-) diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml index 207f9d658..32bef050d 100644 --- a/environment_python_3.10.yml +++ b/environment_python_3.10.yml @@ -15,58 +15,58 @@ # To remove the environment entirely, run the following command: # $ conda env remove -n jwql-3.10 -name: jwql-3.10 +name: jwql-3.10-deptest channels: - conda-forge - defaults dependencies: - - astropy=5.2.2 + - astropy=5.3.1 - bokeh=2.4.3 - beautifulsoup4=4.12.2 - - celery=5.2.7 - - cryptography=40.0.2 - - django=4.2.1 + - celery=5.3.1 + - cryptography=41.0.2 + - django=4.2.3 - inflection=0.5.1 - - ipython=8.13.2 + - ipython=8.14.0 - jinja2=3.1.2 - jsonschema=4.17.3 - - matplotlib=3.7.0 - - nodejs=18.15.0 - - numpy=1.24.3 + - matplotlib=3.7.2 + - nodejs=18.16.1 + - numpy=1.25.1 - numpydoc=1.5.0 - - pandas=2.0.1 - - pip=23.1.2 - - postgresql=15.2 - - psycopg2=2.9.3 - - pytest=7.3.1 - - pytest-cov=4.0.0 - - pytest-mock=3.10.0 - - python=3.10.9 + - pandas=2.0.3 + - pip=23.2.1 + - postgresql=15.3 + - psycopg2=2.9.6 + - pytest=7.4.0 + - pytest-cov=4.1.0 + - pytest-mock=3.11.1 + - python=3.10.12 - pyyaml=6.0 - redis - - ruff=0.0.269 + - ruff=0.0.280 - scipy=1.9.3 - - setuptools=67.7.2 + - setuptools=68.0.0 - sphinx=6.2.1 - - sphinx_rtd_theme=1.2.0 - - sqlalchemy=2.0.15 + - sphinx_rtd_theme=1.2.2 + - sqlalchemy=2.0.19 - twine=4.0.2 - wtforms=3.0.1 - pip: - - astroquery==0.4.6 - - bandit==1.7.5 - - jwst==1.10.2 - - pysiaf==0.19.1 - - pysqlite3==0.5.0 - - pyvo==1.4.1 - - redis==4.5.5 - - selenium==4.9.1 - - stdatamodels==1.3.1 - - stsci_rtd_theme==1.0.0 - - vine==5.0.0 + - astroquery=0.4.6 + - bandit=1.7.5 + - jwst=1.11.3 + - pysiaf=0.20.0 + - pysqlite3=0.5.1 + - pyvo=1.4.1 + - redis=4.6.0 + - selenium=4.10.0 + - stdatamodels=1.7.1 + - stsci_rtd_theme=1.0.0 + - vine=5.0.0 - git+https://github.com/spacetelescope/jwst_reffiles # Current package diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml index cf20b4108..dd677fa50 100644 --- a/environment_python_3.9.yml +++ b/environment_python_3.9.yml @@ -15,58 +15,58 @@ # To remove the environment entirely, run the following command: # $ conda env remove -n jwql-3.9 -name: jwql-3.9 +name: jwql-3.9-deptest channels: - conda-forge - defaults dependencies: - - astropy=5.2.2 + - astropy=5.3.1 - bokeh=2.4.3 - beautifulsoup4=4.12.2 - - celery=5.2.7 - - cryptography=40.0.2 - - django=4.2.1 + - celery=5.3.1 + - cryptography=41.0.2 + - django=4.2.3 - inflection=0.5.1 - - ipython=8.13.2 + - ipython=8.14.0 - jinja2=3.1.2 - jsonschema=4.17.3 - - matplotlib=3.7.0 - - nodejs=18.15.0 - - numpy=1.24.3 + - matplotlib=3.7.2 + - nodejs=18.16.1 + - numpy=1.25.1 - numpydoc=1.5.0 - - pandas=2.0.1 - - pip=23.1.2 - - postgresql=15.2 - - psycopg2=2.9.3 - - pytest=7.3.1 - - pytest-cov=4.0.0 - - pytest-mock=3.10.0 - - python=3.9.16 + - pandas=2.0.3 + - pip=23.2.1 + - postgresql=15.3 + - psycopg2=2.9.6 + - pytest=7.4.0 + - pytest-cov=4.1.0 + - pytest-mock=3.11.1 + - python=3.9.17 - pyyaml=6.0 - redis - - ruff=0.0.269 + - ruff=0.0.280 - scipy=1.9.3 - - setuptools=67.7.2 + - setuptools=68.0.0 - sphinx=6.2.1 - - sphinx_rtd_theme=1.2.0 - - sqlalchemy=2.0.15 + - sphinx_rtd_theme=1.2.2 + - sqlalchemy=2.0.19 - twine=4.0.2 - wtforms=3.0.1 - pip: - - astroquery==0.4.6 - - bandit==1.7.5 - - jwst==1.10.2 - - pysiaf==0.19.1 - - pysqlite3==0.5.0 - - pyvo==1.4.1 - - redis==4.5.5 - - selenium==4.9.1 - - stdatamodels==1.3.1 - - stsci_rtd_theme==1.0.0 - - vine==5.0.0 + - astroquery=0.4.6 + - bandit=1.7.5 + - jwst=1.11.3 + - pysiaf=0.20.0 + - pysqlite3=0.5.1 + - pyvo=1.4.1 + - redis=4.6.0 + - selenium=4.10.0 + - stdatamodels=1.7.1 + - stsci_rtd_theme=1.0.0 + - vine=5.0.0 - git+https://github.com/spacetelescope/jwst_reffiles # Current package From 2673b3cee46078019dfac099826482358aba2a8b Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 1 Aug 2023 16:46:30 -0400 Subject: [PATCH 406/449] Smarter about base filename --- jwql/shared_tasks/shared_tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index 7a05f6f92..be232935d 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -459,7 +459,8 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save logging.error("File {} not found!".format(input_file)) raise FileNotFoundError("{} not found".format(input_file)) - short_name = input_file_name[0: input_file_name.rfind('_')] + parts = input_file_name.split('_') + short_name = f'{parts[0]}_{parts[1]}_{parts[2]}_{parts[3]}' ensure_dir_exists(cal_dir) output_dir = os.path.join(config["transfer_dir"], "outgoing") From 99bcb7a83bcab8a7c7205ada5b51ab7b1275745c Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Wed, 2 Aug 2023 12:55:54 -0400 Subject: [PATCH 407/449] Remove unused badpix values from columndatasource --- .../common_monitors/dark_monitor.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index 8ffab9146..75cdf1062 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -649,8 +649,7 @@ def overplot_bad_pix(self, pix_type, coords, values): values = [] sources[pix_type] = ColumnDataSource(data=dict(pixels_x=coords[0], - pixels_y=coords[1], - values=values + pixels_y=coords[1] ) ) @@ -658,14 +657,6 @@ def overplot_bad_pix(self, pix_type, coords, values): badpixplots[pix_type] = self.plot.circle(x=f'pixels_x', y=f'pixels_y', source=sources[pix_type], color=colors[pix_type]) - # Create hover tools for the bad pixel types - #hover_tools[pix_type] = HoverTool(tooltips=[(f'{pix_type} (x, y):', '(@pixels_x, @pixels_y)'), - # ('value:', f'@values'), - # ], - # renderers=[badpixplots[pix_type]]) - # Add tool to plot - #self.plot.tools.append(hover_tools[pix_type]) - # Add to the legend if numpix > 0: if numpix <= DARK_MONITOR_MAX_BADPOINTS_TO_PLOT: @@ -814,7 +805,7 @@ def process(self, file_list): logging.info('\tFound {} new noisy pixels'.format(len(new_noisy_pixels[0]))) self.add_bad_pix(new_noisy_pixels, 'noisy', file_list, mean_slope_file, baseline_file, min_time, mid_time, max_time) - logging.info("Creating Mean Slope Image {}".format(slope_image)) + logging.info("Creating Mean Slope Image") # Create png file of mean slope image. Add bad pixels only for full frame apertures self.create_mean_slope_figure(slope_image, len(slope_files), hotxy=new_hot_pix, deadxy=new_dead_pix, noisyxy=new_noisy_pixels, baseline_file=baseline_file) From 9f45a404b5ea7f0f3e6fb5021e1800848c5fa7b0 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Wed, 2 Aug 2023 13:22:53 -0400 Subject: [PATCH 408/449] Set first pipeline step name per instrument --- jwql/shared_tasks/run_pipeline.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index a79167e0b..0531f0337 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -61,7 +61,7 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co # If the input file is a file other than uncal.fits, then we may only need to run a # subset of steps. Check the completed steps in the input file. Find the latest step # that has been completed, and skip that plus all prior steps - if 'uncal.fits' not in input_file: + if 'uncal' not in input_file: completed_steps = completed_pipeline_steps(input_file) # Reverse the boolean value, so that now steps answers the question: "Do we need @@ -73,7 +73,11 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co # run, and only run subsequent steps. This protects against cases where some early # step was not run. In that case, we don't want to go back and run it because running # pipeline steps out of order doesn't work. - last_run = 'group_scale' # initialize to the first step + if instrument in ['miri', 'nirspec']: + last_run = 'group_scale' # initialize to the first step + else: + last_run = 'dq_init' + for step in steps: if not steps[step]: last_run = deepcopy(step) From 0b4791ba9024c6a9828c82edadda3f42d453dae8 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Wed, 2 Aug 2023 15:02:05 -0400 Subject: [PATCH 409/449] avoid logging large numpy arrays --- jwql/instrument_monitors/common_monitors/bias_monitor.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/jwql/instrument_monitors/common_monitors/bias_monitor.py b/jwql/instrument_monitors/common_monitors/bias_monitor.py index 120cfab0e..9e1f9ad32 100755 --- a/jwql/instrument_monitors/common_monitors/bias_monitor.py +++ b/jwql/instrument_monitors/common_monitors/bias_monitor.py @@ -432,7 +432,13 @@ def process(self, file_list): # Add this new entry to the bias database table with engine.begin() as connection: connection.execute(self.stats_table.__table__.insert(), bias_db_entry) - logging.info('\tNew entry added to bias database table: {}'.format(bias_db_entry)) + + # Don't print long arrays of numbers to the log file + log_dict = {} + for key in bias_db_entry: + if key not in ['collapsed_rows', 'collapsed_columns', 'counts', 'bin_centers']: + log_dict[key] = bias_db_entry[key] + logging.info('\tNew entry added to bias database table: {}'.format(log_dict)) # Remove the raw and calibrated files to save memory space os.remove(filename) From 8737521c9ab8c4ea45389faf47ef4b27ef0d94e6 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Wed, 2 Aug 2023 15:36:02 -0400 Subject: [PATCH 410/449] Make plotting functions robust against missing db entries --- .../jwql/monitor_pages/monitor_bias_bokeh.py | 54 +++++++++++-------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py index 7f14c9fb2..35cd6dbe0 100644 --- a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py @@ -466,31 +466,39 @@ def create_plot(self, colname): title_text = 'Column' axis_text = 'Row Number' - #datestr = self.data['expstart'][0].strftime("%m/%d/%Y") - datestr = self.data['expstart_str'].iloc[0] - title_str = f'Calibrated data: Collapsed {title_text}, {datestr}' - - if len(self.data[colname].iloc[0]) > 0: - plot = figure(title=title_str, tools='pan,box_zoom,reset,wheel_zoom,save', - background_fill_color="#fafafa") - - # Add a column containing pixel numbers to plot against - pix_num = np.arange(len(self.data[colname].iloc[0])) - self.data['pixel'] = [pix_num] - - series = self.data.iloc[0] - series = series[['pixel', colname]] - source = ColumnDataSource(dict(series)) - plot.scatter(x='pixel', y=colname, fill_color="#C85108", line_color="#C85108", - alpha=0.75, source=source) - - hover_text = axis_text.split(' ')[0] - hover_tool = HoverTool(tooltips=f'{hover_text} @pixel: @{colname}') - plot.tools.append(hover_tool) - plot.xaxis.axis_label = axis_text - plot.yaxis.axis_label = 'Median Signal (DN)' + # Make sure there is data present + if len(self.data) > 0: + # Make sure that the colname column is not empty + if len(self.data[colname].iloc[0]) > 0: + datestr = self.data['expstart_str'].iloc[0] + title_str = f'Calibrated data: Collapsed {title_text}, {datestr}' + + plot = figure(title=title_str, tools='pan,box_zoom,reset,wheel_zoom,save', + background_fill_color="#fafafa") + + # Add a column containing pixel numbers to plot against + pix_num = np.arange(len(self.data[colname].iloc[0])) + self.data['pixel'] = [pix_num] + + series = self.data.iloc[0] + series = series[['pixel', colname]] + source = ColumnDataSource(dict(series)) + plot.scatter(x='pixel', y=colname, fill_color="#C85108", line_color="#C85108", + alpha=0.75, source=source) + + hover_text = axis_text.split(' ')[0] + hover_tool = HoverTool(tooltips=f'{hover_text} @pixel: @{colname}') + plot.tools.append(hover_tool) + plot.xaxis.axis_label = axis_text + plot.yaxis.axis_label = 'Median Signal (DN)' + else: + # If there is a latest_data entry, but the collapsed_row or collapsed_col + # columns are empty, then make a placeholder plot. + title_str = f'Calibrated data: Collapsed {title_text}' + plot = PlaceholderPlot(title_str, axis_text, 'Median Signal (DN)').plot else: # If there are no data, then create an empty placeholder plot + title_str = f'Calibrated data: Collapsed {title_text}' plot = PlaceholderPlot(title_str, axis_text, 'Median Signal (DN)').plot return plot From 8dd5bbd1a57f226f5e12bfcd45431c2b234bfb94 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Wed, 2 Aug 2023 15:43:49 -0400 Subject: [PATCH 411/449] Make other plotting functions robust against missing db entries --- .../jwql/monitor_pages/monitor_bias_bokeh.py | 53 ++++++++++--------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py index 35cd6dbe0..47a85b8b8 100644 --- a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py @@ -382,31 +382,36 @@ def create_plot(self): """ x_label = 'Signal (DN)' y_label = '# Pixels' - if len(self.data['counts'].iloc[0]) > 0: - - # In order to use Bokeh's quad, we need left and right bin edges, rather than bin centers - bin_centers = np.array(self.data['bin_centers'][0]) - half_widths = (bin_centers[1:] - bin_centers[0:-1]) / 2 - half_widths = np.insert(half_widths, 0, half_widths[0]) - self.data['bin_left'] = [bin_centers - half_widths] - self.data['bin_right'] = [bin_centers + half_widths] - - datestr = self.data['expstart_str'].iloc[0] - self.plot = figure(title=f'Calibrated data: Histogram, {datestr}', tools='pan,box_zoom,reset,wheel_zoom,save', - background_fill_color="#fafafa") - - # Keep only the columns where the data are a list - series = self.data.iloc[0] - series = series[['counts', 'bin_left', 'bin_right', 'bin_centers']] - source = ColumnDataSource(dict(series)) - self.plot.quad(top='counts', bottom=0, left='bin_left', right='bin_right', - fill_color="#C85108", line_color="#C85108", alpha=0.75, source=source) - - hover_tool = HoverTool(tooltips=f'@bin_centers DN: @counts') - self.plot.tools.append(hover_tool) - self.plot.xaxis.axis_label = x_label - self.plot.yaxis.axis_label = y_label + # Be sure data is not empty + if len(data) > 0: + # Be sure the array of histogram information is not empty + if len(self.data['counts'].iloc[0]) > 0: + + # In order to use Bokeh's quad, we need left and right bin edges, rather than bin centers + bin_centers = np.array(self.data['bin_centers'][0]) + half_widths = (bin_centers[1:] - bin_centers[0:-1]) / 2 + half_widths = np.insert(half_widths, 0, half_widths[0]) + self.data['bin_left'] = [bin_centers - half_widths] + self.data['bin_right'] = [bin_centers + half_widths] + datestr = self.data['expstart_str'].iloc[0] + self.plot = figure(title=f'Calibrated data: Histogram, {datestr}', tools='pan,box_zoom,reset,wheel_zoom,save', + background_fill_color="#fafafa") + + # Keep only the columns where the data are a list + series = self.data.iloc[0] + series = series[['counts', 'bin_left', 'bin_right', 'bin_centers']] + source = ColumnDataSource(dict(series)) + self.plot.quad(top='counts', bottom=0, left='bin_left', right='bin_right', + fill_color="#C85108", line_color="#C85108", alpha=0.75, source=source) + + hover_tool = HoverTool(tooltips=f'@bin_centers DN: @counts') + self.plot.tools.append(hover_tool) + self.plot.xaxis.axis_label = x_label + self.plot.yaxis.axis_label = y_label + + else: + self.plot = PlaceholderPlot('Calibrated data: Histogram', x_label, y_label).plot else: self.plot = PlaceholderPlot('Calibrated data: Histogram', x_label, y_label).plot From 40bd0dae87e1c704c87914b56ae55c81d7ded34c Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Wed, 2 Aug 2023 15:46:38 -0400 Subject: [PATCH 412/449] typo --- jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py index 47a85b8b8..4b5a6a6de 100644 --- a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py @@ -383,7 +383,7 @@ def create_plot(self): x_label = 'Signal (DN)' y_label = '# Pixels' # Be sure data is not empty - if len(data) > 0: + if len(self.data) > 0: # Be sure the array of histogram information is not empty if len(self.data['counts'].iloc[0]) > 0: From 45c1f8f64ade3edb4c1818a2334dd8f1c6ee593d Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 4 Aug 2023 14:02:46 -0400 Subject: [PATCH 413/449] Check fidelity of file copy from mast --- .../common_monitors/dark_monitor.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index 75cdf1062..c1c2ecc4c 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -1022,6 +1022,19 @@ def run(self): # Copy files from filesystem dark_files, not_copied = copy_files(new_filenames, self.data_dir) + # Check that there were no problems with the file copying. If any of the copied + # files have different sizes between the MAST filesystem and the JWQL filesystem, + # then throw them out. + for dark_file in dark_files: + copied_size = os.stat(dark_file).st_size + orig_size = os.stat(filesystem_path(os.path.basename(dark_file))).st_size + if orig_size != copied_size: + logging.info(f"\tProblem copying {os.path.basename(dark_file)} from the filesystem.") + logging.info(f"Size in filesystem: {orig_size}, size of copy: {copied_size}. Skipping file.") + not_copied.append(dark_file) + dark_files.remove(dark_file) + os.remove(dark_file) + logging.info('\tNew_filenames: {}'.format(new_filenames)) logging.info('\tData dir: {}'.format(self.data_dir)) logging.info('\tCopied to working dir: {}'.format(dark_files)) From dc7e2720f1ea7675936e76bb2bdaa6ae97702608 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 17 Aug 2023 12:01:15 -0400 Subject: [PATCH 414/449] env updates with correct nodejs --- environment_python_3.10.yml | 40 +++++++++++++++---------------- environment_python_3.9.yml | 40 +++++++++++++++---------------- pyproject.toml | 3 +-- requirements.txt | 48 ++++++++++++++++++------------------- rtd_requirements.txt | 20 ++++++++-------- 5 files changed, 75 insertions(+), 76 deletions(-) diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml index 32bef050d..291e94710 100644 --- a/environment_python_3.10.yml +++ b/environment_python_3.10.yml @@ -22,51 +22,51 @@ channels: - defaults dependencies: - - astropy=5.3.1 + - astropy=5.3.2 - bokeh=2.4.3 - beautifulsoup4=4.12.2 - celery=5.3.1 - - cryptography=41.0.2 + - cryptography=41.0.3 - django=4.2.3 - inflection=0.5.1 - ipython=8.14.0 - jinja2=3.1.2 - - jsonschema=4.17.3 + - jsonschema=4.19.0 - matplotlib=3.7.2 - - nodejs=18.16.1 - - numpy=1.25.1 + - nodejs=18.16.0 + - numpy=1.25.2 - numpydoc=1.5.0 - pandas=2.0.3 - pip=23.2.1 - - postgresql=15.3 - - psycopg2=2.9.6 + - postgresql=15.2 + - psycopg2=2.9.3 - pytest=7.4.0 - pytest-cov=4.1.0 - pytest-mock=3.11.1 - python=3.10.12 - pyyaml=6.0 - redis - - ruff=0.0.280 + - ruff=0.0.284 - scipy=1.9.3 - setuptools=68.0.0 - sphinx=6.2.1 - sphinx_rtd_theme=1.2.2 - - sqlalchemy=2.0.19 + - sqlalchemy=2.0.20 - twine=4.0.2 - wtforms=3.0.1 - pip: - - astroquery=0.4.6 - - bandit=1.7.5 - - jwst=1.11.3 - - pysiaf=0.20.0 - - pysqlite3=0.5.1 - - pyvo=1.4.1 - - redis=4.6.0 - - selenium=4.10.0 - - stdatamodels=1.7.1 - - stsci_rtd_theme=1.0.0 - - vine=5.0.0 + - astroquery==0.4.6 + - bandit==1.7.5 + - jwst==1.11.4 + - pysiaf==0.20.0 + - pysqlite3==0.5.1 + - pyvo==1.4.2 + - redis==5.0.0 + - selenium==4.11.2 + - stdatamodels==1.7.2 + - stsci_rtd_theme==1.0.0 + - vine==5.0.0 - git+https://github.com/spacetelescope/jwst_reffiles # Current package diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml index dd677fa50..a42b9e770 100644 --- a/environment_python_3.9.yml +++ b/environment_python_3.9.yml @@ -22,51 +22,51 @@ channels: - defaults dependencies: - - astropy=5.3.1 + - astropy=5.3.2 - bokeh=2.4.3 - beautifulsoup4=4.12.2 - celery=5.3.1 - - cryptography=41.0.2 + - cryptography=41.0.3 - django=4.2.3 - inflection=0.5.1 - ipython=8.14.0 - jinja2=3.1.2 - - jsonschema=4.17.3 + - jsonschema=4.19.0 - matplotlib=3.7.2 - - nodejs=18.16.1 - - numpy=1.25.1 + - nodejs=18.16.0 + - numpy=1.25.2 - numpydoc=1.5.0 - pandas=2.0.3 - pip=23.2.1 - - postgresql=15.3 - - psycopg2=2.9.6 + - postgresql=15.2 + - psycopg2=2.9.3 - pytest=7.4.0 - pytest-cov=4.1.0 - pytest-mock=3.11.1 - python=3.9.17 - pyyaml=6.0 - redis - - ruff=0.0.280 + - ruff=0.0.284 - scipy=1.9.3 - setuptools=68.0.0 - sphinx=6.2.1 - sphinx_rtd_theme=1.2.2 - - sqlalchemy=2.0.19 + - sqlalchemy=2.0.20 - twine=4.0.2 - wtforms=3.0.1 - pip: - - astroquery=0.4.6 - - bandit=1.7.5 - - jwst=1.11.3 - - pysiaf=0.20.0 - - pysqlite3=0.5.1 - - pyvo=1.4.1 - - redis=4.6.0 - - selenium=4.10.0 - - stdatamodels=1.7.1 - - stsci_rtd_theme=1.0.0 - - vine=5.0.0 + - astroquery==0.4.6 + - bandit==1.7.5 + - jwst==1.11.4 + - pysiaf==0.20.0 + - pysqlite3==0.5.1 + - pyvo==1.4.2 + - redis==5.0.0 + - selenium==4.11.2 + - stdatamodels==1.7.2 + - stsci_rtd_theme==1.0.0 + - vine==5.0.0 - git+https://github.com/spacetelescope/jwst_reffiles # Current package diff --git a/pyproject.toml b/pyproject.toml index f17cefdf9..c17d3dc28 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,6 @@ dependencies = [ "jwst", "jwst_reffiles", "matplotlib", - "nodejs", "numpy", "numpydoc", "pandas", @@ -46,7 +45,7 @@ test = [ "pytest", "pytest-cov", "pytest-mock", -] +] docs = [ "sphinx", "sphinx_rtd_theme", diff --git a/requirements.txt b/requirements.txt index 501641339..9571c33c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,38 +1,38 @@ -astropy==5.2.2 +astropy==5.3.2 astroquery==0.4.6 bandit==1.7.5 beautifulsoup4==4.12.2 bokeh==2.4.3 -celery==5.2.7 -cryptography==40.0.2 -django==4.2.1 +celery==5.3.1 +cryptography==41.0.3 +django==4.2.3 inflection==0.5.1 -ipython==8.13.2 +ipython==8.14.0 jinja2==3.1.2 -jsonschema==4.17.3 -jwst==1.10.2 -matplotlib==3.7.0 -nodejs==0.1.1 -numpy==1.24.3 +jsonschema==4.19.0 +jwst==1.11.4 +matplotlib==3.7.2 +nodejs==18.16.0 +numpy==1.25.2 numpydoc==1.5.0 -pandas==2.0.1 +pandas==2.0.3 psycopg2-binary==2.9.3 -pysiaf==0.19.1 -pysqlite3==0.5.0 -pytest==7.3.1 -pytest-cov==4.0.0 -pytest-mock==3.10.0 -pyvo==1.4.1 +pysiaf==0.20.0 +pysqlite3==0.5.1 +pytest==7.4.0 +pytest-cov==4.1.0 +pytest-mock==3.11.1 +pyvo==1.4.2 pyyaml==6.0 -redis==4.5.5 -ruff==0.0.269 +redis==5.0.0 +ruff==0.0.284 scipy==1.9.3 -selenium==4.9.1 -setuptools==67.7.2 +selenium==4.11.2 +setuptools==68.0.0 sphinx==6.2.1 -sphinx_rtd_theme==1.2.0 -sqlalchemy==2.0.15 -stdatamodels==1.3.1 +sphinx_rtd_theme==1.2.2 +sqlalchemy==2.0.20 +stdatamodels==1.7.2 stsci_rtd_theme==1.0.0 twine==4.0.2 vine==5.0.0 diff --git a/rtd_requirements.txt b/rtd_requirements.txt index ba1ecf587..aa1047b5e 100644 --- a/rtd_requirements.txt +++ b/rtd_requirements.txt @@ -1,15 +1,15 @@ -sphinx_automodapi==0.14.1 +sphinx_automodapi==0.15.0 bokeh==2.4.3 -celery==5.2.7 -cython==0.29.33 -django==4.2.1 +celery==5.3.1 +cython==3.0.0 +django==4.2.3 docutils==0.18.1 -jwst==1.10.2 -pygments==2.14.0 -pytest==7.3.1 -redis==4.5.5 -sphinx>=2 -sphinx_rtd_theme==1.2.0 +jwst==1.11.4 +pygments==2.16.1 +pytest==7.4.0 +redis==5.0.0 +sphinx==6.2.1 +sphinx_rtd_theme==1.2.2 stsci_rtd_theme==1.0.0 tomli==2.0.1 git+https://github.com/spacetelescope/jwst_reffiles From bc3b546ad514d5b2b73e4279ee3214a9774f2e10 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 17 Aug 2023 12:02:00 -0400 Subject: [PATCH 415/449] Return env name back to nominal --- environment_python_3.10.yml | 2 +- environment_python_3.9.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml index 291e94710..f554506b0 100644 --- a/environment_python_3.10.yml +++ b/environment_python_3.10.yml @@ -15,7 +15,7 @@ # To remove the environment entirely, run the following command: # $ conda env remove -n jwql-3.10 -name: jwql-3.10-deptest +name: jwql-3.10 channels: - conda-forge diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml index a42b9e770..3986b0829 100644 --- a/environment_python_3.9.yml +++ b/environment_python_3.9.yml @@ -15,7 +15,7 @@ # To remove the environment entirely, run the following command: # $ conda env remove -n jwql-3.9 -name: jwql-3.9-deptest +name: jwql-3.9 channels: - conda-forge From a6beb9415c8e2bf6335f24a6f47f42becc4b3078 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 18 Aug 2023 12:29:09 -0400 Subject: [PATCH 416/449] bring nodejs up to most recent version --- environment_python_3.10.yml | 14 +++++++------- environment_python_3.9.yml | 14 +++++++------- requirements.txt | 6 +++--- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml index f554506b0..f61973680 100644 --- a/environment_python_3.10.yml +++ b/environment_python_3.10.yml @@ -1,10 +1,10 @@ # This file describes a conda environment that can be to install jwql # # Run the following command to set up this environment: -# $ conda env create -f environment_python_3_10.yml +# $ conda env create -f environment_python_3.10.yml # # The environment name can be overridden with the following command: -# $ conda env create -n -f environment_python_3_10.yml +# $ conda env create -n -f environment_python_3.10.yml # # Run the following command to activate the environment: # $ source activate jwql-3.10 @@ -23,8 +23,8 @@ channels: dependencies: - astropy=5.3.2 - - bokeh=2.4.3 - beautifulsoup4=4.12.2 + - bokeh=2.4.3 - celery=5.3.1 - cryptography=41.0.3 - django=4.2.3 @@ -33,20 +33,20 @@ dependencies: - jinja2=3.1.2 - jsonschema=4.19.0 - matplotlib=3.7.2 - - nodejs=18.16.0 + - nodejs=20.5.1 - numpy=1.25.2 - numpydoc=1.5.0 - pandas=2.0.3 - pip=23.2.1 - - postgresql=15.2 - - psycopg2=2.9.3 + - postgresql=15.4 + - psycopg2=2.9.6 - pytest=7.4.0 - pytest-cov=4.1.0 - pytest-mock=3.11.1 - python=3.10.12 - pyyaml=6.0 - redis - - ruff=0.0.284 + - ruff=0.0.285 - scipy=1.9.3 - setuptools=68.0.0 - sphinx=6.2.1 diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml index 3986b0829..d3724b1d8 100644 --- a/environment_python_3.9.yml +++ b/environment_python_3.9.yml @@ -1,10 +1,10 @@ # This file describes a conda environment that can be to install jwql # # Run the following command to set up this environment: -# $ conda env create -f environment_python_3_9.yml +# $ conda env create -f environment_python_3.9.yml # # The environment name can be overridden with the following command: -# $ conda env create -n -f environment_python_3_9.yml +# $ conda env create -n -f environment_python_3.9.yml # # Run the following command to activate the environment: # $ source activate jwql-3.9 @@ -23,8 +23,8 @@ channels: dependencies: - astropy=5.3.2 - - bokeh=2.4.3 - beautifulsoup4=4.12.2 + - bokeh=2.4.3 - celery=5.3.1 - cryptography=41.0.3 - django=4.2.3 @@ -33,20 +33,20 @@ dependencies: - jinja2=3.1.2 - jsonschema=4.19.0 - matplotlib=3.7.2 - - nodejs=18.16.0 + - nodejs=20.5.1 - numpy=1.25.2 - numpydoc=1.5.0 - pandas=2.0.3 - pip=23.2.1 - - postgresql=15.2 - - psycopg2=2.9.3 + - postgresql=15.4 + - psycopg2=2.9.6 - pytest=7.4.0 - pytest-cov=4.1.0 - pytest-mock=3.11.1 - python=3.9.17 - pyyaml=6.0 - redis - - ruff=0.0.284 + - ruff=0.0.285 - scipy=1.9.3 - setuptools=68.0.0 - sphinx=6.2.1 diff --git a/requirements.txt b/requirements.txt index 9571c33c9..4391be8b3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,11 +12,11 @@ jinja2==3.1.2 jsonschema==4.19.0 jwst==1.11.4 matplotlib==3.7.2 -nodejs==18.16.0 +nodejs==20.5.1 numpy==1.25.2 numpydoc==1.5.0 pandas==2.0.3 -psycopg2-binary==2.9.3 +psycopg2-binary==2.9.7 pysiaf==0.20.0 pysqlite3==0.5.1 pytest==7.4.0 @@ -25,7 +25,7 @@ pytest-mock==3.11.1 pyvo==1.4.2 pyyaml==6.0 redis==5.0.0 -ruff==0.0.284 +ruff==0.0.285 scipy==1.9.3 selenium==4.11.2 setuptools==68.0.0 From fd7c838616b7518db13ea035cfdd79dcd71638f1 Mon Sep 17 00:00:00 2001 From: "york@stsci.edu" Date: Mon, 11 Sep 2023 15:23:09 -0400 Subject: [PATCH 417/449] Adding more logging --- jwql/shared_tasks/run_pipeline.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 0531f0337..e73c5bf1e 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -51,18 +51,27 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co status_f.write("\t start_dir is {} ({})\n".format(start_dir, type(start_dir))) status_f.write("\t uncal_file is {} ({})\n".format(uncal_file, type(uncal_file))) status_f.write(f"\t outputs is {outputs}\n") + sys.stderr.write("Running run_pipe\n") + sys.stderr.write("\t input_file_basename is {} ({})\n".format(input_file_basename, type(input_file_basename))) + sys.stderr.write("\t start_dir is {} ({})\n".format(start_dir, type(start_dir))) + sys.stderr.write("\t uncal_file is {} ({})\n".format(uncal_file, type(uncal_file))) + sys.stderr.write(f"\t outputs is {outputs}\n") try: + sys.stderr.write("Copying file {} to working directory.\n".format(input_file)) copy_files([input_file], work_directory) + sys.stderr.write("Setting permissions on {}\n".format(uncal_file)) set_permissions(uncal_file) steps = get_pipeline_steps(instrument) + sys.stderr.write("Pipeline steps initialized to {}\n".format(steps)) # If the input file is a file other than uncal.fits, then we may only need to run a # subset of steps. Check the completed steps in the input file. Find the latest step # that has been completed, and skip that plus all prior steps if 'uncal' not in input_file: completed_steps = completed_pipeline_steps(input_file) + sys.stderr.write("Steps {} already completed.\n".format(completed_steps)) # Reverse the boolean value, so that now steps answers the question: "Do we need # to run this step?"" @@ -80,18 +89,21 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co for step in steps: if not steps[step]: + sys.stderr.write("Setting last_run to {}.\n".format(step)) last_run = deepcopy(step) for step in steps: if step == last_run: break if step != last_run: + sys.stderr.write("Setting {} to skip while looking for last_run.\n".format(step)) steps[step] = False # Set any steps the user specifically asks to skip for step, step_dict in step_args.items(): if 'skip' in step_dict: if step_dict['skip']: + sys.stderr.write("Setting step {} to skip by user request.\n".format(step)) steps[step] = False # Run each specified step From 1834232cfa409f4339b333b368fd9ea2ddb308ce Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Wed, 13 Sep 2023 15:12:15 -0400 Subject: [PATCH 418/449] Add step_args to calwebb_detector1_save_jump --- jwql/shared_tasks/shared_tasks.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index be232935d..1e8e95549 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -408,7 +408,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, @celery_app.task(name='jwql.shared_tasks.shared_tasks.calwebb_detector1_save_jump') -def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save_fitopt=True): +def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save_fitopt=True, step_args={}): """Call ``calwebb_detector1`` on the provided file, running all steps up to the ``ramp_fit`` step, and save the result. Optionally run the ``ramp_fit`` step and save the resulting slope file as well. @@ -430,6 +430,13 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save If ``True``, the file of optional outputs from the ramp fitting step of the pipeline is saved. + step_args : dict + A dictionary containing custom arguments to supply to individual pipeline steps. + When a step is run, the dictionary will be checked for a key matching the step + name (as defined in jwql.utils.utils.get_pipeline_steps() for the provided + instrument). The value matching the step key should, itself, be a dictionary that + can be spliced in to step.call() via dereferencing (**dict) + Returns ------- jump_output : str @@ -469,7 +476,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save cores = 'all' status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, input_file, - short_name, result_file, cores) + short_name, result_file, cores, step_args) if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") @@ -484,7 +491,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save if core_fail: cores = "half" status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, - input_file, short_name, result_file, cores) + input_file, short_name, result_file, cores, step_args) if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") managed = True @@ -498,7 +505,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save if core_fail: cores = "none" status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, - input_file, short_name, result_file, cores) + input_file, short_name, result_file, cores, step_args) if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") managed = True From 477072f37d93b36ed907d77d43204d74f1f600f8 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 21 Sep 2023 15:22:52 -0400 Subject: [PATCH 419/449] create wrapper func for saving png files --- .../common_monitors/dark_monitor.py | 5 ++-- jwql/utils/utils.py | 23 ++++++++++++++++++- .../monitor_pages/monitor_bad_pixel_bokeh.py | 6 ++--- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index c1c2ecc4c..467a8bd1c 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -84,7 +84,6 @@ from astropy.modeling import models from astropy.stats import sigma_clipped_stats from astropy.time import Time -from bokeh.io import export_png from bokeh.models import ColorBar, ColumnDataSource, HoverTool, Legend from bokeh.models import LinearColorMapper from bokeh.plotting import figure @@ -106,7 +105,7 @@ from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_DATAPRODUCTS, RAPID_READPATTERNS from jwql.utils.logging_functions import log_info, log_fail from jwql.utils.permissions import set_permissions -from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path +from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path, save_png THRESHOLDS_FILE = os.path.join(os.path.split(__file__)[0], 'dark_monitor_file_thresholds.txt') @@ -350,7 +349,7 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no self.plot.add_layout(legend, 'below') # Save the plot in a png - export_png(self.plot, filename=output_filename) + save_png(self.plot, filename=output_filename) set_permissions(output_filename) diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index 33d0c272c..75cf49ca6 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -45,6 +45,7 @@ from bokeh.plotting import figure import numpy as np from PIL import Image +from selenium import webdriver from jwql.utils import permissions from jwql.utils.constants import FILE_AC_CAR_ID_LEN, FILE_AC_O_ID_LEN, FILE_ACT_LEN, \ @@ -170,7 +171,7 @@ def create_png_from_fits(filename, outdir): # Save the plot in a png output_filename = os.path.join(outdir, os.path.basename(filename).replace('fits','png')) - export_png(plot, filename=output_filename) + save_png(plot, filename=output_filename) permissions.set_permissions(output_filename) return output_filename else: @@ -756,6 +757,26 @@ def read_png(filename): return img +def save_png(fig, filename=''): + """Starting with selenium version 4.10.0, our testing has shown that on the JWQL + servers, we need to specify an instance of a web driver when exporting a Bokeh + figure as a png. This is a wrapper function that creates the web driver instance + and calls Bokeh's export_png function. + + Parameters + ---------- + fig : bokeh.plotting.figure + Bokeh figure to be saved as a png + + filename : str + Filename to use for the png file + """ + options = webdriver.FirefoxOptions() + options.add_argument('-headless') + driver = webdriver.Firefox(options=options) + export_png(fig, filename=filename, webdriver=driver) + + def grouper(iterable, chunksize): """ Take a list of items (iterable), and group it into chunks of chunksize, with the diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py index 48c47e216..b172df2d4 100755 --- a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py @@ -22,7 +22,7 @@ from astropy.stats import sigma_clipped_stats from astropy.time import Time from bokeh.embed import components, file_html -from bokeh.io import export_png, show +from bokeh.io import show from bokeh.layouts import layout from bokeh.models import ColumnDataSource, DatetimeTickFormatter, HoverTool, Legend, LinearColorMapper, Panel, Tabs, Text, Title from bokeh.plotting import figure @@ -40,7 +40,7 @@ from jwql.utils.constants import BAD_PIXEL_MONITOR_MAX_POINTS_TO_PLOT, BAD_PIXEL_TYPES, DARKS_BAD_PIXEL_TYPES from jwql.utils.constants import DETECTOR_PER_INSTRUMENT, FLATS_BAD_PIXEL_TYPES, JWST_INSTRUMENT_NAMES_MIXEDCASE from jwql.utils.permissions import set_permissions -from jwql.utils.utils import filesystem_path, get_config, read_png +from jwql.utils.utils import filesystem_path, get_config, read_png, save_png SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) OUTPUT_DIR = get_config()['outputs'] @@ -593,7 +593,7 @@ def switch_to_png(self, filename, title): Title to add to the Figure """ # Save the figure as a png - export_png(self.plot, filename=filename) + save_png(self.plot, filename=filename) set_permissions(filename) # Read in the png and insert into a replacement figure From 03da9a8916d3dc438a7caaae96c0710a2708092b Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 21 Sep 2023 15:38:38 -0400 Subject: [PATCH 420/449] Quit driver when done. Add test --- jwql/tests/test_utils.py | 19 ++++++++++++++++++- jwql/utils/utils.py | 1 + 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/jwql/tests/test_utils.py b/jwql/tests/test_utils.py index c2f6795c1..27bee1b5e 100644 --- a/jwql/tests/test_utils.py +++ b/jwql/tests/test_utils.py @@ -22,7 +22,11 @@ from pathlib import Path import pytest -from jwql.utils.utils import copy_files, get_config, filename_parser, filesystem_path, _validate_config +from bokeh.models import LinearColorMapper +from bokeh.plotting import figure +import numpy as np + +from jwql.utils.utils import copy_files, get_config, filename_parser, filesystem_path, save_png, _validate_config # Determine if tests are being run on Github Actions @@ -479,6 +483,19 @@ def test_filesystem_path(): assert check == location +def test_save_png(): + """Test that we can create a png file""" + plot = figure(title='test',tools='') + image = np.zeros((200,200)) + image[100:105, 100:105] = 1 + ny, nx = image.shape + mapper = LinearColorMapper(palette='Viridis256', low=0 ,high=1.1) + imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, color_mapper=mapper, level="image") + #save_png(plot, filename='test.png') + from bokeh.io import export_png + export_png(plot, filename='test.png') + + @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') def test_validate_config(): """Test that the config validator works.""" diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index 75cf49ca6..182c18191 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -775,6 +775,7 @@ def save_png(fig, filename=''): options.add_argument('-headless') driver = webdriver.Firefox(options=options) export_png(fig, filename=filename, webdriver=driver) + driver.quit() def grouper(iterable, chunksize): From f239ef2ccc290c3f05c657611e0d7903044fcf7a Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 21 Sep 2023 15:54:07 -0400 Subject: [PATCH 421/449] RTD needs selenium now --- rtd_requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/rtd_requirements.txt b/rtd_requirements.txt index aa1047b5e..d4b88e1fd 100644 --- a/rtd_requirements.txt +++ b/rtd_requirements.txt @@ -8,6 +8,7 @@ jwst==1.11.4 pygments==2.16.1 pytest==7.4.0 redis==5.0.0 +selenium==4.11.3 sphinx==6.2.1 sphinx_rtd_theme==1.2.2 stsci_rtd_theme==1.0.0 From 23d02e79a7a3dd9752c4505fdccc2a36f791059c Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 21 Sep 2023 16:05:08 -0400 Subject: [PATCH 422/449] Fix test. Adjust rtd version --- jwql/tests/test_utils.py | 4 +--- rtd_requirements.txt | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/jwql/tests/test_utils.py b/jwql/tests/test_utils.py index 27bee1b5e..813bb0432 100644 --- a/jwql/tests/test_utils.py +++ b/jwql/tests/test_utils.py @@ -491,9 +491,7 @@ def test_save_png(): ny, nx = image.shape mapper = LinearColorMapper(palette='Viridis256', low=0 ,high=1.1) imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, color_mapper=mapper, level="image") - #save_png(plot, filename='test.png') - from bokeh.io import export_png - export_png(plot, filename='test.png') + save_png(plot, filename='test.png') @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') diff --git a/rtd_requirements.txt b/rtd_requirements.txt index d4b88e1fd..39500c2b6 100644 --- a/rtd_requirements.txt +++ b/rtd_requirements.txt @@ -8,7 +8,7 @@ jwst==1.11.4 pygments==2.16.1 pytest==7.4.0 redis==5.0.0 -selenium==4.11.3 +selenium==4.11.2 sphinx==6.2.1 sphinx_rtd_theme==1.2.2 stsci_rtd_theme==1.0.0 From b85a1b0f7080afc1a58a1e8070b22377cf878b58 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Sep 2023 21:21:42 +0000 Subject: [PATCH 423/449] Bump cryptography from 41.0.3 to 41.0.4 Bumps [cryptography](https://github.com/pyca/cryptography) from 41.0.3 to 41.0.4. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/41.0.3...41.0.4) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4391be8b3..d4f3a5616 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ bandit==1.7.5 beautifulsoup4==4.12.2 bokeh==2.4.3 celery==5.3.1 -cryptography==41.0.3 +cryptography==41.0.4 django==4.2.3 inflection==0.5.1 ipython==8.14.0 From 2f0e8258c8edb0952c1b95c6a80bcf5470b4d3b6 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 22 Sep 2023 12:37:11 -0400 Subject: [PATCH 424/449] Better logging. Fix param def. Allow empty dark file list --- .../common_monitors/bad_pixel_monitor.py | 7 ++++--- jwql/shared_tasks/run_pipeline.py | 8 ++++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py index 546fc6d80..cc4b7fb68 100755 --- a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py +++ b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py @@ -984,9 +984,10 @@ def process(self, illuminated_raw_files, illuminated_slope_files, flat_file_coun else: index += 1 - min_dark_time = min(dark_obstimes) - max_dark_time = max(dark_obstimes) - mid_dark_time = instrument_properties.mean_time(dark_obstimes) + if len(dark_slope_files) > 0: + min_dark_time = min(dark_obstimes) + max_dark_time = max(dark_obstimes) + mid_dark_time = instrument_properties.mean_time(dark_obstimes) # Check whether there are still enough files left to meet the threshold if illuminated_slope_files is None: diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 0531f0337..82ea0689e 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -10,6 +10,7 @@ import shutil import sys import time +import traceback from jwst import datamodels from jwst.dq_init import DQInitStep @@ -161,7 +162,8 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co with open(status_file, "a+") as status_f: status_f.write("EXCEPTION\n") status_f.write("{}\n".format(e)) - status_f.write("FAILED") + status_f.write("FAILED\n") + status_f.write(traceback.format_exc()) sys.exit(1) with open(status_file, "a+") as status_f: @@ -217,6 +219,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T params['refpix'] = dict(odd_even_rows=False) # Default CR rejection threshold is too low + params['jump'] = {} params['jump']['rejection_threshold'] = 15 # Set up to save jump step output @@ -284,7 +287,8 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T with open(status_file, "a+") as status_f: status_f.write("EXCEPTION\n") status_f.write("{}\n".format(e)) - status_f.write("FAILED") + status_f.write("FAILED\n") + status_f.write(traceback.format_exc()) sys.exit(1) with open(status_file, "a+") as status_f: From 7ce0d9c89e9ace520c98f2487762e5cb208156e4 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 25 Sep 2023 13:19:07 -0400 Subject: [PATCH 425/449] Fix typo in env file names in readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 70b4e173b..377005a11 100644 --- a/README.md +++ b/README.md @@ -83,16 +83,16 @@ source activate base/root **Note:** If you have added a step activating conda to your default terminal/shell (e.g. the `.bashrc`, `.zshrc`, or `.profile` file) then you don't need to do the above step. -Lastly, create the `jwql` environment via one of the `environment.yml` files (currently `environment_python_3_9.yml`, for python 3.9, and `environment_python_3.10.yml`, for python 3.10, are supported by `jwql`): +Lastly, create the `jwql` environment via one of the `environment.yml` files (currently `environment_python_3.9.yml`, for python 3.9, and `environment_python_3.10.yml`, for python 3.10, are supported by `jwql`): ``` -conda env create -f environment_python_3_9.yml +conda env create -f environment_python_3.9.yml ``` or ``` -conda env create -f environment_python_3_10.yml +conda env create -f environment_python_3.10.yml ``` ### Configuration File From 55b12f399cc7504e05879157e2c3bfae7b85866c Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 25 Sep 2023 13:49:40 -0400 Subject: [PATCH 426/449] pep8 fixes --- jwql/shared_tasks/run_pipeline.py | 2 +- jwql/shared_tasks/shared_tasks.py | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 82ea0689e..991e08d47 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -348,7 +348,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T outputs = args.outputs step_args = args.step_args - status_file = os.path.join(working_path, short_name+"_status.txt") + status_file = os.path.join(working_path, short_name + "_status.txt") with open(status_file, 'w') as out_file: out_file.write("Starting Process\n") out_file.write("\tpipeline is {} ({})\n".format(pipe_type, type(pipe_type))) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index 1e8e95549..f076e2035 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -1,4 +1,4 @@ - #! /usr/bin/env python +#! /usr/bin/env python """This module contains code for the celery application, which is used for any demanding work which should be restricted in terms of how many iterations are run simultaneously, or @@ -204,7 +204,7 @@ def log_subprocess_output(pipe): If a subprocess STDOUT has been set to subprocess.PIPE, this function will log each line to the logging output. """ - for line in iter(pipe.readline, b''): # b'\n'-separated lines + for line in iter(pipe.readline, b''): # b'\n'-separated lines logging.info("\t{}".format(line.decode('UTF-8').strip())) @@ -224,6 +224,7 @@ def after_setup_celery_logger(logger, **kwargs): def collect_after_task(**kwargs): gc.collect() + def convert_step_args_to_string(args_dict): """Convert the nested dictionary containing pipeline step parameter keyword/value pairs to a string so that it can be passed via command line @@ -239,17 +240,17 @@ def convert_step_args_to_string(args_dict): args_str : str String representation of ``args_dict`` """ - args_str="'{" + args_str = "'{" for i, step in enumerate(args_dict): args_str += f'"{step}":' args_str += '{' for j, (param, val) in enumerate(args_dict[step].items()): args_str += f'"{param}":"{val}"' - if j < len(args_dict[step])-1: + if j < len(args_dict[step]) - 1: args_str += ', ' args_str += "}" - if i < len(args_dict)-1: + if i < len(args_dict) - 1: args_str += ',' args_str += "}'" return args_str @@ -331,7 +332,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, current_dir = os.path.dirname(__file__) cmd_name = os.path.join(current_dir, "run_pipeline.py") outputs = ",".join(ext_or_exts) - result_file = os.path.join(cal_dir, short_name+"_status.txt") + result_file = os.path.join(cal_dir, short_name + "_status.txt") if "all" in ext_or_exts: logging.info("All outputs requested") if instrument.lower() != 'miri': @@ -399,7 +400,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, set_permissions(os.path.join(output_dir, file)) logging.info("Removing local files.") - files_to_remove = glob(os.path.join(cal_dir, short_name+"*")) + files_to_remove = glob(os.path.join(cal_dir, short_name + "*")) for file_name in files_to_remove: logging.info("\tRemoving {}".format(file_name)) os.remove(file_name) @@ -472,7 +473,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save output_dir = os.path.join(config["transfer_dir"], "outgoing") cmd_name = os.path.join(os.path.dirname(__file__), "run_pipeline.py") - result_file = os.path.join(cal_dir, short_name+"_status.txt") + result_file = os.path.join(cal_dir, short_name + "_status.txt") cores = 'all' status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, input_file, @@ -531,7 +532,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save files["fitopt_output"] = os.path.join(output_dir, file) logging.info("Removing local files.") - files_to_remove = glob(os.path.join(cal_dir, short_name+"*")) + files_to_remove = glob(os.path.join(cal_dir, short_name + "*")) for file_name in files_to_remove: logging.info("\tRemoving {}".format(file_name)) os.remove(file_name) From ebf05a9e22ca555ee3f7d9b48fcd48dd0068de5c Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Thu, 28 Sep 2023 10:59:37 -0400 Subject: [PATCH 427/449] Include migrations in source control --- .../apps/jwql/migrations/0001_initial.py | 45 +++++++++ .../migrations/0002_auto_20220913_1525.py | 27 ++++++ .../migrations/0003_auto_20220921_0955.py | 58 +++++++++++ .../migrations/0004_auto_20220922_0911.py | 33 +++++++ .../migrations/0005_auto_20220922_1422.py | 23 +++++ .../migrations/0006_auto_20230214_1624.py | 95 +++++++++++++++++++ .../migrations/0007_auto_20230222_1157.py | 22 +++++ .../migrations/0008_rootfileinfo_exp_type.py | 18 ++++ .../migrations/0009_auto_20230303_0930.py | 63 ++++++++++++ .../migrations/0010_auto_20230313_1053.py | 23 +++++ jwql/website/apps/jwql/migrations/__init__.py | 0 11 files changed, 407 insertions(+) create mode 100644 jwql/website/apps/jwql/migrations/0001_initial.py create mode 100644 jwql/website/apps/jwql/migrations/0002_auto_20220913_1525.py create mode 100644 jwql/website/apps/jwql/migrations/0003_auto_20220921_0955.py create mode 100644 jwql/website/apps/jwql/migrations/0004_auto_20220922_0911.py create mode 100644 jwql/website/apps/jwql/migrations/0005_auto_20220922_1422.py create mode 100644 jwql/website/apps/jwql/migrations/0006_auto_20230214_1624.py create mode 100644 jwql/website/apps/jwql/migrations/0007_auto_20230222_1157.py create mode 100644 jwql/website/apps/jwql/migrations/0008_rootfileinfo_exp_type.py create mode 100644 jwql/website/apps/jwql/migrations/0009_auto_20230303_0930.py create mode 100644 jwql/website/apps/jwql/migrations/0010_auto_20230313_1053.py create mode 100644 jwql/website/apps/jwql/migrations/__init__.py diff --git a/jwql/website/apps/jwql/migrations/0001_initial.py b/jwql/website/apps/jwql/migrations/0001_initial.py new file mode 100644 index 000000000..9de5ca6a9 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0001_initial.py @@ -0,0 +1,45 @@ +# Generated by Django 3.1.7 on 2022-09-09 17:47 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='ImageData', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('inst', models.CharField(choices=[('FGS', 'FGS'), ('MIRI', 'MIRI'), ('NIRCam', 'NIRCam'), ('NIRISS', 'NIRISS'), ('NIRSpec', 'NIRSpec')], default=None, max_length=7, verbose_name='instrument')), + ('pub_date', models.DateTimeField(verbose_name='date published')), + ('filepath', models.FilePathField(path='/user/lchambers/jwql/')), + ], + options={ + 'verbose_name_plural': 'image data', + 'db_table': 'imagedata', + }, + ), + migrations.CreateModel( + name='InstrumentFilterHandler', + fields=[ + ('instrument', models.CharField(max_length=10, primary_key=True, serialize=False)), + ], + ), + migrations.CreateModel( + name='ThumbnailFilterInfo', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('observation', models.PositiveIntegerField()), + ('proposal', models.PositiveIntegerField()), + ('root_name', models.CharField(max_length=300)), + ('marked_viewed', models.BooleanField(default=False)), + ('inst_handler', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jwql.instrumentfilterhandler')), + ], + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0002_auto_20220913_1525.py b/jwql/website/apps/jwql/migrations/0002_auto_20220913_1525.py new file mode 100644 index 000000000..049901811 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0002_auto_20220913_1525.py @@ -0,0 +1,27 @@ +# Generated by Django 3.1.7 on 2022-09-13 20:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0001_initial'), + ] + + operations = [ + migrations.RemoveField( + model_name='thumbnailfilterinfo', + name='id', + ), + migrations.AlterField( + model_name='instrumentfilterhandler', + name='instrument', + field=models.TextField(max_length=10, primary_key=True, serialize=False), + ), + migrations.AlterField( + model_name='thumbnailfilterinfo', + name='root_name', + field=models.TextField(max_length=300, primary_key=True, serialize=False), + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0003_auto_20220921_0955.py b/jwql/website/apps/jwql/migrations/0003_auto_20220921_0955.py new file mode 100644 index 000000000..df7bc6965 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0003_auto_20220921_0955.py @@ -0,0 +1,58 @@ +# Generated by Django 3.1.7 on 2022-09-21 14:55 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0002_auto_20220913_1525'), + ] + + operations = [ + migrations.CreateModel( + name='Archive', + fields=[ + ('instrument', models.CharField(help_text='Instrument name', max_length=7, primary_key=True, serialize=False)), + ], + options={ + 'ordering': ['instrument'], + }, + ), + migrations.CreateModel( + name='Observation', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('obsnum', models.CharField(help_text='Observation number, as a 3 digit string', max_length=3)), + ('number_of_files', models.IntegerField(default=0, help_text='Number of files in the proposal')), + ('obsstart', models.FloatField(default=0.0, help_text='Time of the beginning of the observation in MJD')), + ('obsend', models.FloatField(default=0.0, help_text='Time of the end of the observation in MJD')), + ('exptypes', models.CharField(default='', help_text='Comma-separated list of exposure types', max_length=100)), + ], + options={ + 'ordering': ['-obsnum'], + }, + ), + migrations.CreateModel( + name='Proposal', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('prop_id', models.CharField(help_text='5-digit proposal ID string', max_length=5)), + ('thumbnail_path', models.CharField(default='', help_text='Path to the proposal thumbnail', max_length=100)), + ('archive', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jwql.archive')), + ], + options={ + 'ordering': ['-prop_id'], + 'unique_together': {('prop_id', 'archive')}, + }, + ), + migrations.DeleteModel( + name='ImageData', + ), + migrations.AddField( + model_name='observation', + name='proposal', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jwql.proposal'), + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0004_auto_20220922_0911.py b/jwql/website/apps/jwql/migrations/0004_auto_20220922_0911.py new file mode 100644 index 000000000..998dd2ce4 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0004_auto_20220922_0911.py @@ -0,0 +1,33 @@ +# Generated by Django 3.1.7 on 2022-09-22 14:11 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0003_auto_20220921_0955'), + ] + + operations = [ + migrations.AddField( + model_name='thumbnailfilterinfo', + name='obsnum', + field=models.CharField(default=11, help_text='Observation number, as a 3 digit string', max_length=3), + preserve_default=False, + ), + migrations.CreateModel( + name='RootFileInfo', + fields=[ + ('instrument', models.CharField(help_text='Instrument name', max_length=7)), + ('proposal', models.CharField(help_text='5-digit proposal ID string', max_length=5)), + ('root_name', models.TextField(max_length=300, primary_key=True, serialize=False)), + ('viewed', models.BooleanField(default=False)), + ('obsnum', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jwql.observation')), + ], + options={ + 'ordering': ['-root_name'], + }, + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0005_auto_20220922_1422.py b/jwql/website/apps/jwql/migrations/0005_auto_20220922_1422.py new file mode 100644 index 000000000..3c51cbe23 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0005_auto_20220922_1422.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1.7 on 2022-09-22 19:22 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0004_auto_20220922_0911'), + ] + + operations = [ + migrations.RemoveField( + model_name='thumbnailfilterinfo', + name='inst_handler', + ), + migrations.DeleteModel( + name='InstrumentFilterHandler', + ), + migrations.DeleteModel( + name='ThumbnailFilterInfo', + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0006_auto_20230214_1624.py b/jwql/website/apps/jwql/migrations/0006_auto_20230214_1624.py new file mode 100644 index 000000000..493608df3 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0006_auto_20230214_1624.py @@ -0,0 +1,95 @@ +# Generated by Django 3.1.7 on 2023-02-14 21:24 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0005_auto_20220922_1422'), + ] + + operations = [ + migrations.CreateModel( + name='Anomalies', + fields=[ + ('root_file_info', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='jwql.rootfileinfo')), + ('cosmic_ray_shower', models.BooleanField(default=False)), + ('diffraction_spike', models.BooleanField(default=False)), + ('excessive_saturation', models.BooleanField(default=False)), + ('guidestar_failure', models.BooleanField(default=False)), + ('persistence', models.BooleanField(default=False)), + ('crosstalk', models.BooleanField(default=False)), + ('data_transfer_error', models.BooleanField(default=False)), + ('ghost', models.BooleanField(default=False)), + ('snowball', models.BooleanField(default=False)), + ('column_pull_up', models.BooleanField(default=False)), + ('column_pull_down', models.BooleanField(default=False)), + ('dominant_msa_leakage', models.BooleanField(default=False)), + ('dragons_breath', models.BooleanField(default=False)), + ('mrs_glow', models.BooleanField(default=False)), + ('mrs_zipper', models.BooleanField(default=False)), + ('internal_reflection', models.BooleanField(default=False)), + ('optical_short', models.BooleanField(default=False)), + ('row_pull_up', models.BooleanField(default=False)), + ('row_pull_down', models.BooleanField(default=False)), + ('lrs_contamination', models.BooleanField(default=False)), + ('tree_rings', models.BooleanField(default=False)), + ('scattered_light', models.BooleanField(default=False)), + ('claws', models.BooleanField(default=False)), + ('wisps', models.BooleanField(default=False)), + ('tilt_event', models.BooleanField(default=False)), + ('light_saber', models.BooleanField(default=False)), + ('other', models.BooleanField(default=False)), + ], + options={ + 'ordering': ['-root_file_info'], + }, + ), + migrations.AddField( + model_name='proposal', + name='cat_type', + field=models.CharField(default='', help_text='Category Type', max_length=10), + ), + migrations.AddField( + model_name='rootfileinfo', + name='aperature', + field=models.CharField(default='', help_text='Aperature', max_length=40), + ), + migrations.AddField( + model_name='rootfileinfo', + name='detector', + field=models.CharField(default='', help_text='Detector', max_length=40), + ), + migrations.AddField( + model_name='rootfileinfo', + name='filter', + field=models.CharField(default='', help_text='Instrument name', max_length=7), + ), + migrations.AddField( + model_name='rootfileinfo', + name='grating', + field=models.CharField(default='', help_text='Grating', max_length=40), + ), + migrations.AddField( + model_name='rootfileinfo', + name='pupil', + field=models.CharField(default='', help_text='Pupil', max_length=40), + ), + migrations.AddField( + model_name='rootfileinfo', + name='read_patt', + field=models.CharField(default='', help_text='Read Pattern', max_length=40), + ), + migrations.AddField( + model_name='rootfileinfo', + name='read_patt_num', + field=models.IntegerField(default=0, help_text='Read Pattern Number'), + ), + migrations.AddField( + model_name='rootfileinfo', + name='subarray', + field=models.CharField(default='', help_text='Subarray', max_length=40), + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0007_auto_20230222_1157.py b/jwql/website/apps/jwql/migrations/0007_auto_20230222_1157.py new file mode 100644 index 000000000..2d4ffdba3 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0007_auto_20230222_1157.py @@ -0,0 +1,22 @@ +# Generated by Django 3.1.7 on 2023-02-22 16:57 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0006_auto_20230214_1624'), + ] + + operations = [ + migrations.RemoveField( + model_name='rootfileinfo', + name='aperature', + ), + migrations.AddField( + model_name='rootfileinfo', + name='aperture', + field=models.CharField(default='', help_text='Aperture', max_length=40), + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0008_rootfileinfo_exp_type.py b/jwql/website/apps/jwql/migrations/0008_rootfileinfo_exp_type.py new file mode 100644 index 000000000..2c67cd955 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0008_rootfileinfo_exp_type.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.7 on 2023-02-22 17:01 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0007_auto_20230222_1157'), + ] + + operations = [ + migrations.AddField( + model_name='rootfileinfo', + name='exp_type', + field=models.CharField(default='', help_text='Exposure Type', max_length=40), + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0009_auto_20230303_0930.py b/jwql/website/apps/jwql/migrations/0009_auto_20230303_0930.py new file mode 100644 index 000000000..d0dbafc76 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0009_auto_20230303_0930.py @@ -0,0 +1,63 @@ +# Generated by Django 3.1.7 on 2023-03-03 14:30 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0008_rootfileinfo_exp_type'), + ] + + operations = [ + migrations.RenameField( + model_name='proposal', + old_name='cat_type', + new_name='category', + ), + migrations.AddField( + model_name='rootfileinfo', + name='expstart', + field=models.FloatField(default=0.0, help_text='Exposure Start Time'), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='aperture', + field=models.CharField(blank=True, default='', help_text='Aperture', max_length=40, null=True), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='detector', + field=models.CharField(blank=True, default='', help_text='Detector', max_length=40, null=True), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='exp_type', + field=models.CharField(blank=True, default='', help_text='Exposure Type', max_length=40, null=True), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='filter', + field=models.CharField(blank=True, default='', help_text='Instrument name', max_length=7, null=True), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='grating', + field=models.CharField(blank=True, default='', help_text='Grating', max_length=40, null=True), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='pupil', + field=models.CharField(blank=True, default='', help_text='Pupil', max_length=40, null=True), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='read_patt', + field=models.CharField(blank=True, default='', help_text='Read Pattern', max_length=40, null=True), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='subarray', + field=models.CharField(blank=True, default='', help_text='Subarray', max_length=40, null=True), + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0010_auto_20230313_1053.py b/jwql/website/apps/jwql/migrations/0010_auto_20230313_1053.py new file mode 100644 index 000000000..3d80e95fe --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0010_auto_20230313_1053.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1.7 on 2023-03-13 15:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0009_auto_20230303_0930'), + ] + + operations = [ + migrations.AddField( + model_name='anomalies', + name='flag_date', + field=models.DateTimeField(blank=True, help_text='flag date', null=True), + ), + migrations.AddField( + model_name='anomalies', + name='user', + field=models.CharField(blank=True, default='', help_text='user', max_length=50, null=True), + ), + ] diff --git a/jwql/website/apps/jwql/migrations/__init__.py b/jwql/website/apps/jwql/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb From a22b9ccc95958ff5a29f6ddbdc35978cbdd64682 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 28 Sep 2023 14:51:41 -0400 Subject: [PATCH 428/449] pep8 fixes --- .../common_monitors/dark_monitor.py | 20 ++++--------- jwql/utils/utils.py | 10 +++++-- .../monitor_pages/monitor_bad_pixel_bokeh.py | 28 +++++-------------- 3 files changed, 20 insertions(+), 38 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index 467a8bd1c..c8d30c92a 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -275,14 +275,14 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no # Create figure start_time = Time(float(self.query_start), format='mjd').tt.datetime.strftime("%m/%d/%Y") - end_time = Time(float(self.query_end), format='mjd').tt.datetime.strftime("%m/%d/%Y") + end_time = Time(float(self.query_end), format='mjd').tt.datetime.strftime("%m/%d/%Y") self.plot = figure(title=f'{self.aperture}: {num_files} files. {start_time} to {end_time}', tools='') # tools='pan,box_zoom,reset,wheel_zoom,save') self.plot.x_range.range_padding = self.plot.y_range.range_padding = 0 # Create the color mapper that will be used to scale the image - mapper = LinearColorMapper(palette='Viridis256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev)) + mapper = LinearColorMapper(palette='Viridis256', low=(img_med - (5 * img_dev)), high=(img_med + (5 * img_dev))) # Plot image and add color bar imgplot = self.plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, @@ -291,13 +291,6 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no color_bar = ColorBar(color_mapper=mapper, width=8, title='DN/sec') self.plot.add_layout(color_bar, 'right') - # Add hover tool for all pixel values - #hover_tool = HoverTool(tooltips=[('(x, y):', '($x{int}, $y{int})'), - # ('value:', '@image') - # ], - # renderers=[imgplot]) - #self.plot.tools.append(hover_tool) - if (('FULL' in self.aperture) or ('_CEN' in self.aperture)): if hotxy is not None: @@ -337,14 +330,14 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no base_start = Time(float(base_parts[3]), format='mjd').tt.datetime base_end = Time(float(base_parts[5]), format='mjd').tt.datetime base_start_time = base_start.strftime("%m/%d/%Y") - base_end_time = base_end.strftime("%m/%d/%Y") + base_end_time = base_end.strftime("%m/%d/%Y") legend_title = f'Compared to dark from {base_start_time} to {base_end_time}' else: legend_title = 'Compared to previous mean dark' legend = Legend(items=[hot_legend, dead_legend, noisy_legend], location="center", orientation='vertical', - title = legend_title) + title=legend_title) self.plot.add_layout(legend, 'below') @@ -352,7 +345,6 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no save_png(self.plot, filename=output_filename) set_permissions(output_filename) - def get_metadata(self, filename): """Collect basic metadata from a fits file @@ -650,14 +642,14 @@ def overplot_bad_pix(self, pix_type, coords, values): sources[pix_type] = ColumnDataSource(data=dict(pixels_x=coords[0], pixels_y=coords[1] ) - ) + ) # Overplot the bad pixel locations badpixplots[pix_type] = self.plot.circle(x=f'pixels_x', y=f'pixels_y', source=sources[pix_type], color=colors[pix_type]) # Add to the legend - if numpix > 0: + if numpix > 0: if numpix <= DARK_MONITOR_MAX_BADPOINTS_TO_PLOT: text = f"{numpix} pix {adjective[pix_type]} than baseline" else: diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index 182c18191..4b8951717 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -158,8 +158,7 @@ def create_png_from_fits(filename, outdir): plot.ygrid.visible = False # Create the color mapper that will be used to scale the image - #mapper = LogColorMapper(palette='Viridis256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev)) - mapper = LogColorMapper(palette='Greys256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev)) + mapper = LogColorMapper(palette='Greys256', low=(img_med - (5 * img_dev)),high=(img_med + (5 * img_dev))) # Plot image imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, @@ -170,8 +169,13 @@ def create_png_from_fits(filename, outdir): plot.yaxis.visible = False # Save the plot in a png +<<<<<<< Updated upstream output_filename = os.path.join(outdir, os.path.basename(filename).replace('fits','png')) save_png(plot, filename=output_filename) +======= + output_filename = os.path.join(outdir, os.path.basename(filename).replace('fits', 'png')) + export_png(plot, filename=output_filename) +>>>>>>> Stashed changes permissions.set_permissions(output_filename) return output_filename else: @@ -750,7 +754,7 @@ def read_png(filename): # Copy the RGBA image into view, flipping it so it comes right-side up # with a lower-left origin - view[:,:,:] = np.flipud(np.asarray(rgba_img)) + view[:, :, :] = np.flipud(np.asarray(rgba_img)) else: view = None # Return the 2D version diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py index b172df2d4..385efaa1f 100755 --- a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py @@ -74,8 +74,6 @@ def __init__(self, instrument): # Get the relevant database tables self.identify_tables() - #self.detectors = get_unique_values_per_column(self.pixel_table, 'detector') - self.detectors = sorted(DETECTOR_PER_INSTRUMENT[self.instrument]) if self.instrument == 'miri': self.detectors = ['MIRIMAGE'] @@ -128,7 +126,6 @@ def modify_bokeh_saved_html(self): self._html = "".join(newlines) - def run(self): # Right now, the aperture name in the query history table is used as the title of the @@ -163,7 +160,7 @@ def run(self): # Insert into our html template and save template_dir = os.path.join(os.path.dirname(__file__), '../templates') template_file = os.path.join(template_dir, 'bad_pixel_monitor_savefile_basic.html') - temp_vars = {'inst': self.instrument, 'plot_script': script, 'plot_div':div} + temp_vars = {'inst': self.instrument, 'plot_script': script, 'plot_div': div} self._html = file_html(tabs, CDN, f'{self.instrument} bad pix monitor', template_file, temp_vars) # Modify the html such that our Django-related lines are kept in place, @@ -264,7 +261,6 @@ def __init__(self, pixel_table, instrument, detector): for badtype in self.badtypes: self.get_trending_data(badtype) - def get_most_recent_entry(self): """Get all nedded data from the database tables. """ @@ -312,7 +308,7 @@ def get_trending_data(self, badpix_type): # Query database for all data in the table with a matching detector and bad pixel type all_entries_by_type = session.query(self.pixel_table.type, self.pixel_table.detector, func.array_length(self.pixel_table.x_coord, 1), self.pixel_table.obs_mid_time) \ - .filter(and_(self.pixel_table.detector == self.detector, self.pixel_table.type == badpix_type)) \ + .filter(and_(self.pixel_table.detector == self.detector, self.pixel_table.type == badpix_type)) \ .all() # Organize the results @@ -463,7 +459,7 @@ def create_plot(self): self.plot = figure(title=title_text, tools=tools, x_axis_label="Pixel Number", y_axis_label="Pixel Number") else: - self.plot = figure(tools='') #, x_axis_label="Pixel Number", y_axis_label="Pixel Number") + self.plot = figure(tools='') self.plot.toolbar.logo = None self.plot.toolbar_location = None self.plot.min_border = 0 @@ -475,18 +471,9 @@ def create_plot(self): # Plot image if image is not None: ny, nx = image.shape - #imgplot = self.plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, color_mapper=mapper, level="image") - - - # Shift the figure title slightly right in this case to get it # to align with the axes - #self.plot = figure(title=title, x_range=(0, self._detlen), y_range=(0, self._detlen), width=xdim, height=ydim*, - # tools='pan,box_zoom,reset,wheel_zoom,save', x_axis_label="Pixel Number", y_axis_label="Pixel Number") self.plot.image_rgba(image=[image], x=0, y=0, dw=self._detlen, dh=self._detlen, alpha=0.5) - - - else: # If the background image is not present, manually set the x and y range self.plot.x_range.start = 0 @@ -511,11 +498,10 @@ def create_plot(self): legend = Legend(items=[plot_legend], location="center", orientation='vertical', - title = legend_title) + title=legend_title) self.plot.add_layout(legend, 'below') - def overplot_bad_pix(self): """Add a scatter plot of potential new bad pixels to the plot @@ -537,7 +523,7 @@ def overplot_bad_pix(self): # If there are no new bad pixels, write text within the figure mentioning that txt_source = ColumnDataSource(data=dict(x=[self._detlen / 10], y=[self._detlen / 2], text=[f'No new {self.badpix_type} pixels found'])) - glyph = Text(x="x", y="y", text="text", angle=0., text_color="navy", text_font_size={'value':'20px'}) + glyph = Text(x="x", y="y", text="text", angle=0., text_color="navy", text_font_size={'value': '20px'}) self.plot.add_glyph(txt_source, glyph) # Insert a fake one, in order to get the plot to be made @@ -668,7 +654,7 @@ def create_plot(self): string_time=string_times, value=[self.badpix_type] * len(self.num_pix) ) - ) + ) self.plot = figure(title=f'{self.detector}: New {self.badpix_type} Pixels', tools='pan,box_zoom,reset,wheel_zoom,save', background_fill_color="#fafafa") @@ -696,7 +682,7 @@ def create_plot(self): time_pad = datetime.timedelta(days=1) self.plot.x_range.start = min(self.time) - time_pad self.plot.x_range.end = max(self.time) + time_pad - self.plot.grid.grid_line_color="white" + self.plot.grid.grid_line_color = "white" self.plot.xaxis.axis_label = 'Date' self.plot.yaxis.axis_label = f'Number of {self.badpix_type} pixels' From 7c2cd6618ee81e39cb7b9b09723f96f4beca4c51 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 28 Sep 2023 15:03:00 -0400 Subject: [PATCH 429/449] more pep8 --- jwql/instrument_monitors/common_monitors/dark_monitor.py | 4 ++-- jwql/tests/test_utils.py | 6 +++--- jwql/utils/utils.py | 7 +------ .../apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py | 6 +++--- 4 files changed, 9 insertions(+), 14 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index c8d30c92a..c73dd3543 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -641,8 +641,8 @@ def overplot_bad_pix(self, pix_type, coords, values): sources[pix_type] = ColumnDataSource(data=dict(pixels_x=coords[0], pixels_y=coords[1] - ) - ) + ) + ) # Overplot the bad pixel locations badpixplots[pix_type] = self.plot.circle(x=f'pixels_x', y=f'pixels_y', diff --git a/jwql/tests/test_utils.py b/jwql/tests/test_utils.py index 813bb0432..cd50c6a01 100644 --- a/jwql/tests/test_utils.py +++ b/jwql/tests/test_utils.py @@ -485,11 +485,11 @@ def test_filesystem_path(): def test_save_png(): """Test that we can create a png file""" - plot = figure(title='test',tools='') - image = np.zeros((200,200)) + plot = figure(title='test', tools='') + image = np.zeros((200, 200)) image[100:105, 100:105] = 1 ny, nx = image.shape - mapper = LinearColorMapper(palette='Viridis256', low=0 ,high=1.1) + mapper = LinearColorMapper(palette='Viridis256', low=0, high=1.1) imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, color_mapper=mapper, level="image") save_png(plot, filename='test.png') diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index 4b8951717..ca4df70a3 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -169,13 +169,8 @@ def create_png_from_fits(filename, outdir): plot.yaxis.visible = False # Save the plot in a png -<<<<<<< Updated upstream - output_filename = os.path.join(outdir, os.path.basename(filename).replace('fits','png')) - save_png(plot, filename=output_filename) -======= output_filename = os.path.join(outdir, os.path.basename(filename).replace('fits', 'png')) - export_png(plot, filename=output_filename) ->>>>>>> Stashed changes + save_png(plot, filename=output_filename) permissions.set_permissions(output_filename) return output_filename else: diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py index 385efaa1f..5e522f7cd 100755 --- a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py @@ -308,8 +308,8 @@ def get_trending_data(self, badpix_type): # Query database for all data in the table with a matching detector and bad pixel type all_entries_by_type = session.query(self.pixel_table.type, self.pixel_table.detector, func.array_length(self.pixel_table.x_coord, 1), self.pixel_table.obs_mid_time) \ - .filter(and_(self.pixel_table.detector == self.detector, self.pixel_table.type == badpix_type)) \ - .all() + .filter(and_(self.pixel_table.detector == self.detector, self.pixel_table.type == badpix_type)) \ + .all() # Organize the results num_pix = [] @@ -654,7 +654,7 @@ def create_plot(self): string_time=string_times, value=[self.badpix_type] * len(self.num_pix) ) - ) + ) self.plot = figure(title=f'{self.detector}: New {self.badpix_type} Pixels', tools='pan,box_zoom,reset,wheel_zoom,save', background_fill_color="#fafafa") From 6fc7ad384071ed1bdd884c63a6c7ed22aa33174f Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 28 Sep 2023 15:04:49 -0400 Subject: [PATCH 430/449] pep8, make up your mind --- jwql/utils/utils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index ca4df70a3..113dd66c9 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -49,11 +49,11 @@ from jwql.utils import permissions from jwql.utils.constants import FILE_AC_CAR_ID_LEN, FILE_AC_O_ID_LEN, FILE_ACT_LEN, \ - FILE_DATETIME_LEN, FILE_EPOCH_LEN, FILE_GUIDESTAR_ATTMPT_LEN_MIN, \ - FILE_GUIDESTAR_ATTMPT_LEN_MAX, FILE_OBS_LEN, FILE_PARALLEL_SEQ_ID_LEN, \ - FILE_PROG_ID_LEN, FILE_SEG_LEN, FILE_SOURCE_ID_LEN, FILE_SUFFIX_TYPES, \ - FILE_TARG_ID_LEN, FILE_VISIT_GRP_LEN, FILE_VISIT_LEN, FILETYPE_WO_STANDARD_SUFFIX, \ - JWST_INSTRUMENT_NAMES_SHORTHAND + FILE_DATETIME_LEN, FILE_EPOCH_LEN, FILE_GUIDESTAR_ATTMPT_LEN_MIN, \ + FILE_GUIDESTAR_ATTMPT_LEN_MAX, FILE_OBS_LEN, FILE_PARALLEL_SEQ_ID_LEN, \ + FILE_PROG_ID_LEN, FILE_SEG_LEN, FILE_SOURCE_ID_LEN, FILE_SUFFIX_TYPES, \ + FILE_TARG_ID_LEN, FILE_VISIT_GRP_LEN, FILE_VISIT_LEN, FILETYPE_WO_STANDARD_SUFFIX, \ + JWST_INSTRUMENT_NAMES_SHORTHAND __location__ = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) @@ -158,7 +158,7 @@ def create_png_from_fits(filename, outdir): plot.ygrid.visible = False # Create the color mapper that will be used to scale the image - mapper = LogColorMapper(palette='Greys256', low=(img_med - (5 * img_dev)),high=(img_med + (5 * img_dev))) + mapper = LogColorMapper(palette='Greys256', low=(img_med - (5 * img_dev)), high=(img_med + (5 * img_dev))) # Plot image imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, From f81de385fa3105aa2470b774014bddcb243d00d4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Oct 2023 12:50:14 +0000 Subject: [PATCH 431/449] Bump scipy from 1.9.3 to 1.10.0 Bumps [scipy](https://github.com/scipy/scipy) from 1.9.3 to 1.10.0. - [Release notes](https://github.com/scipy/scipy/releases) - [Commits](https://github.com/scipy/scipy/compare/v1.9.3...v1.10.0) --- updated-dependencies: - dependency-name: scipy dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d4f3a5616..4270e9d45 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,7 +26,7 @@ pyvo==1.4.2 pyyaml==6.0 redis==5.0.0 ruff==0.0.285 -scipy==1.9.3 +scipy==1.10.0 selenium==4.11.2 setuptools==68.0.0 sphinx==6.2.1 From 1e6073ec1b0083ebd9366d7241bc4dc63b6e8516 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Mon, 2 Oct 2023 15:03:05 -0400 Subject: [PATCH 432/449] Removed all templating, adding new calls to monitor views to display proper html --- .../monitor_pages/monitor_readnoise_bokeh.py | 118 ++++++++++++++++-- jwql/website/apps/jwql/monitor_views.py | 4 +- 2 files changed, 109 insertions(+), 13 deletions(-) diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py index 3a5d22a6a..5e2b0c80f 100644 --- a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py @@ -12,7 +12,7 @@ :: - from jwql.website.apps.jwql import monitor_pages + . monitor_template = monitor_pages.ReadnoiseMonitor() monitor_template.input_parameters = ('NIRCam', 'NRCA1_FULL') """ @@ -20,12 +20,17 @@ from datetime import datetime, timedelta import os +from bokeh.embed import components +from bokeh.layouts import column, row +from bokeh.models import Panel, Tabs # bokeh <= 3.0 +from bokeh.models import ColumnDataSource, HoverTool +# from bokeh.models import TabPanel, Tabs # bokeh >= 3.0 +from bokeh.plotting import figure import numpy as np -from jwql.bokeh_templating import BokehTemplate from jwql.database.database_interface import session from jwql.database.database_interface import FGSReadnoiseStats, MIRIReadnoiseStats, NIRCamReadnoiseStats, NIRISSReadnoiseStats, NIRSpecReadnoiseStats -from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE +from jwql.utils.constants import FULL_FRAME_APERTURES, JWST_INSTRUMENT_NAMES_MIXEDCASE class ReadnoiseMonitorData(): @@ -74,8 +79,24 @@ def load_data(self): session.close() -class ReadNoisePlots(): - """Class to make readnoise plots. + +class ReadNoiseFigure(): + """Generate tabbed plot displayed in JWQL web application + """ + def __init__(self, instrument): + instrument_apertures = FULL_FRAME_APERTURES[instrument.upper()] + + self.tabs = [] + for aperture in instrument_apertures: + readnoise_tab = ReadNoisePlotTab(instrument, aperture) + self.tabs.append(readnoise_tab.tab) + + self.plot = Tabs(tabs=self.tabs) + self.tab_components = components(self.plot) + + +class ReadNoisePlotTab(): + """Class to make instrument/aperture panels """ def __init__(self, instrument, aperture): self.instrument = instrument @@ -83,16 +104,89 @@ def __init__(self, instrument, aperture): self.db = ReadnoiseMonitorData(self.instrument, self.aperture) + self.plot_readnoise_amplifers() + self.plot_readnoise_difference_image() + self.plot_readnoise_histogram() - def read_noise_amp_plots(self): - """Make read noise monitor trending plots per amplifier - """ - print('make plot here') - # for amp in ['1', '2', '3', '4']: - # expstarts_iso = np.array([result.expstart for result in self.db]) - # readnoise_vals = np.array([getattr(result, 'amp{}_mean'.format(amp)) for result in self.query_results]) + self.tab = Panel(child=column(row(*self.amp_plots), self.diff_image_plot, self.readnoise_histogram), title= self.aperture) + + def plot_readnoise_amplifers(self): + + self.amp_plots = [] + for amp in ['1', '2', '3', '4']: + + amp_plot = figure(width=350, height=350, x_axis_type='datetime') + + readnoise_vals = np.array([getattr(result, 'amp{}_mean'.format(amp)) for result in self.db.query_results]) + filenames = [os.path.basename(result.uncal_filename).replace('_uncal.fits', '') for result in self.db.query_results] + expstarts_iso = np.array([result.expstart for result in self.db.query_results]) + expstarts = np.array([datetime.strptime(date, '%Y-%m-%dT%H:%M:%S.%f') for date in expstarts_iso]) + nints = [result.nints for result in self.db.query_results] + ngroups = [result.ngroups for result in self.db.query_results] + + source = ColumnDataSource(data=dict( + file=filenames, + expstarts=expstarts, + nints=nints, + ngroups=ngroups, + readnoise=readnoise_vals, + )) + + amp_plot.add_tools(HoverTool(tooltips=[("file", "@filenames"), + ("time", "@expstarts"), + ("nints", "@nints"), + ("ngroups", "@ngroups"), + ("readnoise", "@readnoise")])) + + amp_plot.circle(x='expstarts', y='readnoise', source=source) + + amp_plot.xaxis.axis_label = 'Date' + amp_plot.yaxis.axis_label = 'Mean Readnoise [DN]' + + self.amp_plots.append(amp_plot) + + def plot_readnoise_difference_image(self): + """Updates the readnoise difference image""" + + # Update the readnoise difference image and histogram, if data exists + + self.diff_image_plot = figure(height=700, width=700) + + if len(self.db.query_results) != 0: + diff_image_png = self.db.query_results[-1].readnoise_diff_image + self.diff_image_plot.image_url(url=[diff_image_png], x=1, y=1, w=2048, h=2048, anchor="bottom_left") + self.diff_image_plot.xaxis.visible = False + self.diff_image_plot.yaxis.visible = False + self.diff_image_plot.xgrid.grid_line_color = None + self.diff_image_plot.ygrid.grid_line_color = None + self.diff_image_plot.title.text_font_size = '22px' + self.diff_image_plot.title.align = 'center' + + def plot_readnoise_histogram(self): + """Updates the readnoise histogram""" + + diff_image_n = np.array(self.db.query_results[-1].diff_image_n) + diff_image_bin_centers = np.array(self.db.query_results[-1].diff_image_bin_centers) + + hist_xr_start = diff_image_bin_centers.min() + hist_xr_end = diff_image_bin_centers.max() + hist_yr_start = diff_image_n.min() + hist_yr_end = diff_image_n.max() + diff_image_n.max() * 0.05 + + self.readnoise_histogram = figure(height=800, width=800, + x_range=(hist_xr_start, hist_xr_end), + y_range=(hist_yr_start, hist_yr_end)) + + source = ColumnDataSource(data=dict( + x=diff_image_bin_centers, + y=diff_image_n, + )) + self.readnoise_histogram.add_tools(HoverTool(tooltips=[("Data (x, y)", "(@x, @y)"),])) + self.readnoise_histogram.circle(x='x', y='y', source=source) + self.readnoise_histogram.xaxis.axis_label = 'Readnoise Difference [DN]' + self.readnoise_histogram.yaxis.axis_label = 'Number of Pixels' diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py index b62f339de..09cf423f6 100644 --- a/jwql/website/apps/jwql/monitor_views.py +++ b/jwql/website/apps/jwql/monitor_views.py @@ -37,12 +37,14 @@ from . import bokeh_containers from jwql.website.apps.jwql import bokeh_containers +from jwql.website.apps.jwql.monitor_pages.monitor_readnoise_bokeh import ReadNoiseFigure from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE from jwql.utils.utils import get_config, get_base_url from jwql.instrument_monitors.nirspec_monitors.ta_monitors import msata_monitor from jwql.instrument_monitors.nirspec_monitors.ta_monitors import wata_monitor from jwql.utils import monitor_utils + CONFIG = get_config() FILESYSTEM_DIR = os.path.join(CONFIG['jwql_dir'], 'filesystem') @@ -219,7 +221,7 @@ def readnoise_monitor(request, inst): inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()] # Get the html and JS needed to render the readnoise tab plots - tabs_components = bokeh_containers.readnoise_monitor_tabs(inst) + tabs_components = ReadNoiseFigure(inst).tab_components template = "readnoise_monitor.html" From d4db60902588ceb844ac06218274c0cb6e466b45 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Mon, 2 Oct 2023 15:50:28 -0400 Subject: [PATCH 433/449] Remove import from __init__ and updating typos for plotting, updating source for readnoise scatter --- jwql/website/apps/jwql/monitor_pages/__init__.py | 3 +-- .../apps/jwql/monitor_pages/monitor_readnoise_bokeh.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/jwql/website/apps/jwql/monitor_pages/__init__.py b/jwql/website/apps/jwql/monitor_pages/__init__.py index 1405523c0..8a4054c76 100644 --- a/jwql/website/apps/jwql/monitor_pages/__init__.py +++ b/jwql/website/apps/jwql/monitor_pages/__init__.py @@ -1,2 +1 @@ -from .monitor_cosmic_rays_bokeh import CosmicRayMonitor -from .monitor_readnoise_bokeh import ReadnoiseMonitor +from .monitor_cosmic_rays_bokeh import CosmicRayMonitor \ No newline at end of file diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py index 5e2b0c80f..c4e4a350f 100644 --- a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py @@ -133,7 +133,7 @@ def plot_readnoise_amplifers(self): readnoise=readnoise_vals, )) - amp_plot.add_tools(HoverTool(tooltips=[("file", "@filenames"), + amp_plot.add_tools(HoverTool(tooltips=[("file", "@file"), ("time", "@expstarts"), ("nints", "@nints"), ("ngroups", "@ngroups"), @@ -155,7 +155,7 @@ def plot_readnoise_difference_image(self): if len(self.db.query_results) != 0: diff_image_png = self.db.query_results[-1].readnoise_diff_image - self.diff_image_plot.image_url(url=[diff_image_png], x=1, y=1, w=2048, h=2048, anchor="bottom_left") + self.diff_image_plot.image_url(url=[diff_image_png], x=0, y=0, w=2048, h=2048, anchor="bottom_left") self.diff_image_plot.xaxis.visible = False self.diff_image_plot.yaxis.visible = False self.diff_image_plot.xgrid.grid_line_color = None From 8c2fd616cdaf51ca280ff0dd77cf0a2dbf922e82 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Tue, 3 Oct 2023 16:07:04 -0400 Subject: [PATCH 434/449] update deps in environment files --- environment_python_3.10.yml | 50 ++++++++++++------------ environment_python_3.9.yml | 46 +++++++++++----------- requirements.txt | 78 ++++++++++++++++++------------------- 3 files changed, 87 insertions(+), 87 deletions(-) diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml index f61973680..7856cf201 100644 --- a/environment_python_3.10.yml +++ b/environment_python_3.10.yml @@ -22,49 +22,49 @@ channels: - defaults dependencies: - - astropy=5.3.2 + - astropy=5.3.3 - beautifulsoup4=4.12.2 - bokeh=2.4.3 - - celery=5.3.1 - - cryptography=41.0.3 - - django=4.2.3 + - celery=5.3.4 + - cryptography=41.0.4 + - django=4.2.5 - inflection=0.5.1 - - ipython=8.14.0 + - ipython=8.16.1 - jinja2=3.1.2 - - jsonschema=4.19.0 - - matplotlib=3.7.2 - - nodejs=20.5.1 + - jsonschema=4.19.1 + - matplotlib=3.8.0 + - nodejs>=18.15.0 - numpy=1.25.2 - numpydoc=1.5.0 - - pandas=2.0.3 + - pandas=2.1.1 - pip=23.2.1 - - postgresql=15.4 - - psycopg2=2.9.6 - - pytest=7.4.0 + - postgresql>=15.2 + - psycopg2>=2.9.3 + - pytest=7.4.2 - pytest-cov=4.1.0 - pytest-mock=3.11.1 - - python=3.10.12 - - pyyaml=6.0 - - redis - - ruff=0.0.285 + - python=3.10.13 + - pyyaml=6.0.1 + - redis=5.0.1 + - ruff=0.0.292 - scipy=1.9.3 - - setuptools=68.0.0 - - sphinx=6.2.1 - - sphinx_rtd_theme=1.2.2 - - sqlalchemy=2.0.20 + - setuptools=68.2.2 + - sphinx=7.2.6 + - sphinx_rtd_theme=1.3.0 + - sqlalchemy=2.0.21 - twine=4.0.2 - wtforms=3.0.1 - pip: - astroquery==0.4.6 - bandit==1.7.5 - - jwst==1.11.4 + - jwst==1.12.2 - pysiaf==0.20.0 - - pysqlite3==0.5.1 + - pysqlite3==0.5.2 - pyvo==1.4.2 - - redis==5.0.0 - - selenium==4.11.2 - - stdatamodels==1.7.2 + - redis==5.0.1 + - selenium==4.13.0 + - stdatamodels==1.8.3 - stsci_rtd_theme==1.0.0 - vine==5.0.0 - git+https://github.com/spacetelescope/jwst_reffiles diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml index d3724b1d8..554355df3 100644 --- a/environment_python_3.9.yml +++ b/environment_python_3.9.yml @@ -22,49 +22,49 @@ channels: - defaults dependencies: - - astropy=5.3.2 + - astropy=5.3.3 - beautifulsoup4=4.12.2 - bokeh=2.4.3 - - celery=5.3.1 - - cryptography=41.0.3 - - django=4.2.3 + - celery=5.3.4 + - cryptography=41.0.4 + - django=4.2.5 - inflection=0.5.1 - - ipython=8.14.0 + - ipython=8.16.1 - jinja2=3.1.2 - - jsonschema=4.19.0 - - matplotlib=3.7.2 - - nodejs=20.5.1 + - jsonschema=4.19.1 + - matplotlib=3.8.0 + - nodejs=20.8.0 - numpy=1.25.2 - numpydoc=1.5.0 - - pandas=2.0.3 + - pandas=2.1.1 - pip=23.2.1 - postgresql=15.4 - - psycopg2=2.9.6 - - pytest=7.4.0 + - psycopg2=2.9.7 + - pytest=7.4.2 - pytest-cov=4.1.0 - pytest-mock=3.11.1 - python=3.9.17 - - pyyaml=6.0 - - redis - - ruff=0.0.285 + - pyyaml=6.0.1 + - redis=5.0.1 + - ruff=0.0.292 - scipy=1.9.3 - - setuptools=68.0.0 - - sphinx=6.2.1 - - sphinx_rtd_theme=1.2.2 - - sqlalchemy=2.0.20 + - setuptools=68.2.2 + - sphinx=7.2.6 + - sphinx_rtd_theme=1.3.0 + - sqlalchemy=2.0.21 - twine=4.0.2 - wtforms=3.0.1 - pip: - astroquery==0.4.6 - bandit==1.7.5 - - jwst==1.11.4 + - jwst==1.12.2 - pysiaf==0.20.0 - - pysqlite3==0.5.1 + - pysqlite3==0.5.2 - pyvo==1.4.2 - - redis==5.0.0 - - selenium==4.11.2 - - stdatamodels==1.7.2 + - redis==5.0.1 + - selenium==4.13.0 + - stdatamodels==1.8.3 - stsci_rtd_theme==1.0.0 - vine==5.0.0 - git+https://github.com/spacetelescope/jwst_reffiles diff --git a/requirements.txt b/requirements.txt index 4270e9d45..6b8d88d41 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,40 +1,40 @@ -astropy==5.3.2 -astroquery==0.4.6 -bandit==1.7.5 -beautifulsoup4==4.12.2 -bokeh==2.4.3 -celery==5.3.1 -cryptography==41.0.4 -django==4.2.3 -inflection==0.5.1 -ipython==8.14.0 -jinja2==3.1.2 -jsonschema==4.19.0 -jwst==1.11.4 -matplotlib==3.7.2 -nodejs==20.5.1 -numpy==1.25.2 -numpydoc==1.5.0 -pandas==2.0.3 -psycopg2-binary==2.9.7 -pysiaf==0.20.0 -pysqlite3==0.5.1 -pytest==7.4.0 -pytest-cov==4.1.0 -pytest-mock==3.11.1 -pyvo==1.4.2 -pyyaml==6.0 -redis==5.0.0 -ruff==0.0.285 -scipy==1.10.0 -selenium==4.11.2 -setuptools==68.0.0 -sphinx==6.2.1 -sphinx_rtd_theme==1.2.2 -sqlalchemy==2.0.20 -stdatamodels==1.7.2 -stsci_rtd_theme==1.0.0 -twine==4.0.2 -vine==5.0.0 -wtforms==3.0.1 +astropy +astroquery +bandit +beautifulsoup4 +bokeh +celery +cryptography +django +inflection +ipython +jinja2 +jsonschema +jwst +matplotlib +nodejs +numpy +numpydoc +pandas +psycopg2-binary +pysiaf +pysqlite3 +pytest +pytest-cov +pytest-mock +pyvo +pyyaml +redis +ruff +scipy +selenium +setuptools +sphinx +sphinx_rtd_theme +sqlalchemy +stdatamodels +stsci_rtd_theme +twine +vine +wtforms git+https://github.com/spacetelescope/jwst_reffiles#egg=jwst_reffiles From c0de0112325fa00644caa9cbe619d25bb63aa670 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Tue, 3 Oct 2023 16:18:34 -0400 Subject: [PATCH 435/449] update versions in requirements.txt --- requirements.txt | 78 ++++++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6b8d88d41..bb289de5a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,40 +1,40 @@ -astropy -astroquery -bandit -beautifulsoup4 -bokeh -celery -cryptography -django -inflection -ipython -jinja2 -jsonschema -jwst -matplotlib -nodejs -numpy -numpydoc -pandas -psycopg2-binary -pysiaf -pysqlite3 -pytest -pytest-cov -pytest-mock -pyvo -pyyaml -redis -ruff -scipy -selenium -setuptools -sphinx -sphinx_rtd_theme -sqlalchemy -stdatamodels -stsci_rtd_theme -twine -vine -wtforms +astropy==5.3.3 +astroquery==0.4.6 +bandit==1.7.5 +beautifulsoup4==4.12.2 +bokeh==2.4.3 +celery==5.3.4 +cryptography==41.0.4 +django==4.2.5 +inflection==0.5.1 +ipython==8.16.1 +jinja2==3.1.2 +jsonschema==4.19.1 +jwst==1.12.2 +matplotlib==3.8.0 +nodejs==20.8.0 +numpy==1.25.2 +numpydoc==1.5.0 +pandas==2.1.1 +psycopg2-binary==2.9.7 +pysiaf==0.20.0 +pysqlite3==0.5.2 +pytest==7.4.2 +pytest-cov==4.1.0 +pytest-mock==3.11.1 +pyvo==1.4.2 +pyyaml==6.0.1 +redis==5.0.1 +ruff==0.0.292 +scipy==1.9.3 +selenium==4.13.0 +setuptools==68.2.2 +sphinx==7.2.6 +sphinx_rtd_theme==1.3.0 +sqlalchemy==2.0.21 +stdatamodels==1.8.3 +stsci_rtd_theme==1.0.0 +twine==4.0.2 +vine==5.0.0 +wtforms==3.0.1 git+https://github.com/spacetelescope/jwst_reffiles#egg=jwst_reffiles From 8ae4da424b1cf99c8a6e20cabe504c938285c228 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Tue, 3 Oct 2023 16:21:14 -0400 Subject: [PATCH 436/449] put redis back to 5.0.0 --- environment_python_3.10.yml | 2 +- jwql/utils/logging_functions.py | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml index 7856cf201..505bdb7c0 100644 --- a/environment_python_3.10.yml +++ b/environment_python_3.10.yml @@ -45,7 +45,7 @@ dependencies: - pytest-mock=3.11.1 - python=3.10.13 - pyyaml=6.0.1 - - redis=5.0.1 + - redis=5.0.0 - ruff=0.0.292 - scipy=1.9.3 - setuptools=68.2.2 diff --git a/jwql/utils/logging_functions.py b/jwql/utils/logging_functions.py index abd82a45e..3ef464a40 100644 --- a/jwql/utils/logging_functions.py +++ b/jwql/utils/logging_functions.py @@ -106,7 +106,7 @@ def configure_logging(module): logging.basicConfig(filename=log_file, format='%(asctime)s %(levelname)s: %(message)s', datefmt='%m/%d/%Y %H:%M:%S %p', - level=logging.INFO) + level=logging.DEBUG) print('Log file initialized to {}'.format(log_file)) set_permissions(log_file) diff --git a/requirements.txt b/requirements.txt index bb289de5a..d82a3409f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,7 +24,7 @@ pytest-cov==4.1.0 pytest-mock==3.11.1 pyvo==1.4.2 pyyaml==6.0.1 -redis==5.0.1 +redis==5.0.0. ruff==0.0.292 scipy==1.9.3 selenium==4.13.0 From 9ef1cd9356335913eab0562282cad68e16f1f579 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Tue, 3 Oct 2023 16:23:10 -0400 Subject: [PATCH 437/449] whoops, accidental logging change --- jwql/utils/logging_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/utils/logging_functions.py b/jwql/utils/logging_functions.py index 3ef464a40..abd82a45e 100644 --- a/jwql/utils/logging_functions.py +++ b/jwql/utils/logging_functions.py @@ -106,7 +106,7 @@ def configure_logging(module): logging.basicConfig(filename=log_file, format='%(asctime)s %(levelname)s: %(message)s', datefmt='%m/%d/%Y %H:%M:%S %p', - level=logging.DEBUG) + level=logging.INFO) print('Log file initialized to {}'.format(log_file)) set_permissions(log_file) From b4e4c38e856adfa88e82483dcfc4f81b6e5bbb52 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Fri, 6 Oct 2023 11:04:47 -0400 Subject: [PATCH 438/449] revert all redis references to 5.0.0 --- environment_python_3.10.yml | 2 +- environment_python_3.9.yml | 4 ++-- requirements.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml index 505bdb7c0..518fbc902 100644 --- a/environment_python_3.10.yml +++ b/environment_python_3.10.yml @@ -62,7 +62,7 @@ dependencies: - pysiaf==0.20.0 - pysqlite3==0.5.2 - pyvo==1.4.2 - - redis==5.0.1 + - redis==5.0.0 - selenium==4.13.0 - stdatamodels==1.8.3 - stsci_rtd_theme==1.0.0 diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml index 554355df3..5b70010a8 100644 --- a/environment_python_3.9.yml +++ b/environment_python_3.9.yml @@ -45,7 +45,7 @@ dependencies: - pytest-mock=3.11.1 - python=3.9.17 - pyyaml=6.0.1 - - redis=5.0.1 + - redis=5.0.0 - ruff=0.0.292 - scipy=1.9.3 - setuptools=68.2.2 @@ -62,7 +62,7 @@ dependencies: - pysiaf==0.20.0 - pysqlite3==0.5.2 - pyvo==1.4.2 - - redis==5.0.1 + - redis==5.0.0 - selenium==4.13.0 - stdatamodels==1.8.3 - stsci_rtd_theme==1.0.0 diff --git a/requirements.txt b/requirements.txt index d82a3409f..01aeb4f7f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,7 +24,7 @@ pytest-cov==4.1.0 pytest-mock==3.11.1 pyvo==1.4.2 pyyaml==6.0.1 -redis==5.0.0. +redis==5.0.0 ruff==0.0.292 scipy==1.9.3 selenium==4.13.0 From b874639b0e45e0a67b8813a9b36a4d12f02af7a1 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Fri, 6 Oct 2023 12:51:46 -0400 Subject: [PATCH 439/449] mamba didn't like 3.10.13 --- environment_python_3.10.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml index 518fbc902..f60c31d27 100644 --- a/environment_python_3.10.yml +++ b/environment_python_3.10.yml @@ -43,7 +43,7 @@ dependencies: - pytest=7.4.2 - pytest-cov=4.1.0 - pytest-mock=3.11.1 - - python=3.10.13 + - python=3.10.12 - pyyaml=6.0.1 - redis=5.0.0 - ruff=0.0.292 From 6edf1262793467580a131f5c745682e980d8cecb Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Fri, 6 Oct 2023 15:32:45 -0400 Subject: [PATCH 440/449] update for 3.10.12 --- environment_python_3.10.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml index f60c31d27..fd927ce30 100644 --- a/environment_python_3.10.yml +++ b/environment_python_3.10.yml @@ -22,24 +22,24 @@ channels: - defaults dependencies: - - astropy=5.3.3 + - astropy=5.3.4 - beautifulsoup4=4.12.2 - bokeh=2.4.3 - celery=5.3.4 - cryptography=41.0.4 - - django=4.2.5 + - django=4.2.6 - inflection=0.5.1 - ipython=8.16.1 - jinja2=3.1.2 - jsonschema=4.19.1 - matplotlib=3.8.0 - - nodejs>=18.15.0 + - nodejs=20.8.0 - numpy=1.25.2 - numpydoc=1.5.0 - pandas=2.1.1 - pip=23.2.1 - - postgresql>=15.2 - - psycopg2>=2.9.3 + - postgresql=15.4 + - psycopg2=2.9.7 - pytest=7.4.2 - pytest-cov=4.1.0 - pytest-mock=3.11.1 From 91feef45fdbf47a9e994c28ef8b3921ecae7d68f Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Fri, 6 Oct 2023 16:29:43 -0400 Subject: [PATCH 441/449] update rtd_requirements.txt --- rtd_requirements.txt | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/rtd_requirements.txt b/rtd_requirements.txt index 39500c2b6..f4f42830e 100644 --- a/rtd_requirements.txt +++ b/rtd_requirements.txt @@ -1,16 +1,16 @@ -sphinx_automodapi==0.15.0 +sphinx_automodapi>=0.15.0 bokeh==2.4.3 -celery==5.3.1 -cython==3.0.0 -django==4.2.3 -docutils==0.18.1 -jwst==1.11.4 +celery==5.3.4 +cython>=3.0.0 +django==4.2.5 +docutils>=0.18.1 +jwst==1.12.3 pygments==2.16.1 -pytest==7.4.0 +pytest==7.4.2 redis==5.0.0 -selenium==4.11.2 -sphinx==6.2.1 -sphinx_rtd_theme==1.2.2 +selenium==4.13.0 +sphinx==7.2.6 +sphinx_rtd_theme==1.30 stsci_rtd_theme==1.0.0 tomli==2.0.1 git+https://github.com/spacetelescope/jwst_reffiles From 059310ac90171af382c1c91b130861e20cdb0c6d Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Fri, 6 Oct 2023 16:35:59 -0400 Subject: [PATCH 442/449] jwst changed out from under me --- environment_python_3.10.yml | 2 +- environment_python_3.9.yml | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml index fd927ce30..9f2ed409b 100644 --- a/environment_python_3.10.yml +++ b/environment_python_3.10.yml @@ -58,7 +58,7 @@ dependencies: - pip: - astroquery==0.4.6 - bandit==1.7.5 - - jwst==1.12.2 + - jwst==1.12.3 - pysiaf==0.20.0 - pysqlite3==0.5.2 - pyvo==1.4.2 diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml index 5b70010a8..4f4159244 100644 --- a/environment_python_3.9.yml +++ b/environment_python_3.9.yml @@ -58,7 +58,7 @@ dependencies: - pip: - astroquery==0.4.6 - bandit==1.7.5 - - jwst==1.12.2 + - jwst==1.12.3 - pysiaf==0.20.0 - pysqlite3==0.5.2 - pyvo==1.4.2 diff --git a/requirements.txt b/requirements.txt index 01aeb4f7f..6a7f670cd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ inflection==0.5.1 ipython==8.16.1 jinja2==3.1.2 jsonschema==4.19.1 -jwst==1.12.2 +jwst==1.12.3 matplotlib==3.8.0 nodejs==20.8.0 numpy==1.25.2 From fdfd4b674a9a6db8468fe16e72a235e537894079 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Fri, 6 Oct 2023 16:40:32 -0400 Subject: [PATCH 443/449] update setuptools version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c17d3dc28..133af8c79 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,7 @@ file = "LICENSE" content-type = "text/plain" [build-system] -requires = ["setuptools>=61.2,<=65.5.0", "numpy", "wheel", "setuptools_scm"] +requires = ["setuptools=68.2.2", "numpy", "wheel", "setuptools_scm"] build-backend = "setuptools.build_meta" [tool.setuptools] From f365329f7023f0d18596e59ee6f1979d76d51f31 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Fri, 6 Oct 2023 16:51:14 -0400 Subject: [PATCH 444/449] setuptools greater than --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 133af8c79..24f701717 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,7 @@ file = "LICENSE" content-type = "text/plain" [build-system] -requires = ["setuptools=68.2.2", "numpy", "wheel", "setuptools_scm"] +requires = ["setuptools>=68.0.0", "numpy", "wheel", "setuptools_scm"] build-backend = "setuptools.build_meta" [tool.setuptools] From f7ecd691e9034e3828a21406ebe00372721ec09a Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Fri, 6 Oct 2023 16:54:54 -0400 Subject: [PATCH 445/449] no thank you sphinx_theme. stsci_theme 4ever --- rtd_requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/rtd_requirements.txt b/rtd_requirements.txt index f4f42830e..abac6b128 100644 --- a/rtd_requirements.txt +++ b/rtd_requirements.txt @@ -10,7 +10,6 @@ pytest==7.4.2 redis==5.0.0 selenium==4.13.0 sphinx==7.2.6 -sphinx_rtd_theme==1.30 stsci_rtd_theme==1.0.0 tomli==2.0.1 git+https://github.com/spacetelescope/jwst_reffiles From 385ec3d6243ffba2399d0f66951169a91c99c249 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Fri, 13 Oct 2023 11:22:39 -0400 Subject: [PATCH 446/449] Adding string slicing for path to readnoise diff image --- .../monitor_pages/monitor_readnoise_bokeh.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py index c4e4a350f..551623606 100644 --- a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py @@ -108,14 +108,18 @@ def __init__(self, instrument, aperture): self.plot_readnoise_difference_image() self.plot_readnoise_histogram() - self.tab = Panel(child=column(row(*self.amp_plots), self.diff_image_plot, self.readnoise_histogram), title= self.aperture) + self.tab = Panel(child=column(row(*self.amp_plots), + self.diff_image_plot, + self.readnoise_histogram), + title= self.aperture) def plot_readnoise_amplifers(self): self.amp_plots = [] for amp in ['1', '2', '3', '4']: - amp_plot = figure(width=350, height=350, x_axis_type='datetime') + amp_plot = figure(title='Amp {}'.format(amp), width=280, height=280, x_axis_type='datetime') + amp_plot.xaxis[0].ticker.desired_num_ticks = 4 readnoise_vals = np.array([getattr(result, 'amp{}_mean'.format(amp)) for result in self.db.query_results]) @@ -151,10 +155,12 @@ def plot_readnoise_difference_image(self): # Update the readnoise difference image and histogram, if data exists - self.diff_image_plot = figure(height=700, width=700) + self.diff_image_plot = figure(title='Readnoise Difference (most recent dark - pipeline reffile)', + height=500, width=500, sizing_mode='scale_width') if len(self.db.query_results) != 0: diff_image_png = self.db.query_results[-1].readnoise_diff_image + diff_image_png = os.path.join('/static', '/'.join(diff_image_png.split('/')[-6:])) self.diff_image_plot.image_url(url=[diff_image_png], x=0, y=0, w=2048, h=2048, anchor="bottom_left") self.diff_image_plot.xaxis.visible = False self.diff_image_plot.yaxis.visible = False @@ -174,9 +180,10 @@ def plot_readnoise_histogram(self): hist_yr_start = diff_image_n.min() hist_yr_end = diff_image_n.max() + diff_image_n.max() * 0.05 - self.readnoise_histogram = figure(height=800, width=800, + self.readnoise_histogram = figure(height=500, width=500, x_range=(hist_xr_start, hist_xr_end), - y_range=(hist_yr_start, hist_yr_end)) + y_range=(hist_yr_start, hist_yr_end), + sizing_mode='scale_width') source = ColumnDataSource(data=dict( @@ -190,3 +197,5 @@ def plot_readnoise_histogram(self): self.readnoise_histogram.xaxis.axis_label = 'Readnoise Difference [DN]' self.readnoise_histogram.yaxis.axis_label = 'Number of Pixels' + self.readnoise_histogram.xaxis.axis_label_text_font_size = "15pt" + self.readnoise_histogram.yaxis.axis_label_text_font_size = "15pt" From 6cdb3f7527cb680514ae04f2862992ee8eab6d07 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Fri, 13 Oct 2023 13:11:23 -0400 Subject: [PATCH 447/449] pep8speaks --- .../apps/jwql/monitor_pages/__init__.py | 2 +- .../monitor_pages/monitor_readnoise_bokeh.py | 34 +++++++++---------- jwql/website/apps/jwql/monitor_views.py | 2 +- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/jwql/website/apps/jwql/monitor_pages/__init__.py b/jwql/website/apps/jwql/monitor_pages/__init__.py index 8a4054c76..ed184d7ff 100644 --- a/jwql/website/apps/jwql/monitor_pages/__init__.py +++ b/jwql/website/apps/jwql/monitor_pages/__init__.py @@ -1 +1 @@ -from .monitor_cosmic_rays_bokeh import CosmicRayMonitor \ No newline at end of file +from .monitor_cosmic_rays_bokeh import CosmicRayMonitor diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py index 551623606..7657aeeed 100644 --- a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py @@ -35,21 +35,21 @@ class ReadnoiseMonitorData(): """Class to hold bias data to be plotted - + Parameters ---------- - + instrument : str Instrument name (e.g. nircam) - + Attributes ---------- - + instrument : str Instrument name (e.g. nircam) aperture : str Aperture name (e.g. apername) - + """ def __init__(self, instrument, aperture): @@ -57,7 +57,6 @@ def __init__(self, instrument, aperture): self.aperture = aperture self.load_data() - def identify_tables(self): """Determine which database tables to use for the given instrument""" @@ -96,7 +95,7 @@ def __init__(self, instrument): class ReadNoisePlotTab(): - """Class to make instrument/aperture panels + """Class to make instrument/aperture panels """ def __init__(self, instrument, aperture): self.instrument = instrument @@ -108,10 +107,10 @@ def __init__(self, instrument, aperture): self.plot_readnoise_difference_image() self.plot_readnoise_histogram() - self.tab = Panel(child=column(row(*self.amp_plots), - self.diff_image_plot, - self.readnoise_histogram), - title= self.aperture) + self.tab = Panel(child=column(row(*self.amp_plots), + self.diff_image_plot, + self.readnoise_histogram), + title=self.aperture) def plot_readnoise_amplifers(self): @@ -122,7 +121,7 @@ def plot_readnoise_amplifers(self): amp_plot.xaxis[0].ticker.desired_num_ticks = 4 readnoise_vals = np.array([getattr(result, 'amp{}_mean'.format(amp)) for result in self.db.query_results]) - + filenames = [os.path.basename(result.uncal_filename).replace('_uncal.fits', '') for result in self.db.query_results] expstarts_iso = np.array([result.expstart for result in self.db.query_results]) expstarts = np.array([datetime.strptime(date, '%Y-%m-%dT%H:%M:%S.%f') for date in expstarts_iso]) @@ -138,10 +137,10 @@ def plot_readnoise_amplifers(self): )) amp_plot.add_tools(HoverTool(tooltips=[("file", "@file"), - ("time", "@expstarts"), - ("nints", "@nints"), - ("ngroups", "@ngroups"), - ("readnoise", "@readnoise")])) + ("time", "@expstarts"), + ("nints", "@nints"), + ("ngroups", "@ngroups"), + ("readnoise", "@readnoise")])) amp_plot.circle(x='expstarts', y='readnoise', source=source) @@ -155,7 +154,7 @@ def plot_readnoise_difference_image(self): # Update the readnoise difference image and histogram, if data exists - self.diff_image_plot = figure(title='Readnoise Difference (most recent dark - pipeline reffile)', + self.diff_image_plot = figure(title='Readnoise Difference (most recent dark - pipeline reffile)', height=500, width=500, sizing_mode='scale_width') if len(self.db.query_results) != 0: @@ -185,7 +184,6 @@ def plot_readnoise_histogram(self): y_range=(hist_yr_start, hist_yr_end), sizing_mode='scale_width') - source = ColumnDataSource(data=dict( x=diff_image_bin_centers, y=diff_image_n, diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py index 09cf423f6..3835f744b 100644 --- a/jwql/website/apps/jwql/monitor_views.py +++ b/jwql/website/apps/jwql/monitor_views.py @@ -69,7 +69,7 @@ def bad_pixel_monitor(request, inst): context = { 'inst': inst, - } + } return render(request, template, context) From e6e43bef746b0fabb2e2fdcf4b78439fdd7332d8 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Thu, 19 Oct 2023 15:21:28 -0400 Subject: [PATCH 448/449] Pep8 fixes --- .../jwql/monitor_pages/monitor_readnoise_bokeh.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py index 7657aeeed..82f0fe914 100644 --- a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py @@ -129,12 +129,11 @@ def plot_readnoise_amplifers(self): ngroups = [result.ngroups for result in self.db.query_results] source = ColumnDataSource(data=dict( - file=filenames, - expstarts=expstarts, - nints=nints, - ngroups=ngroups, - readnoise=readnoise_vals, - )) + file=filenames, + expstarts=expstarts, + nints=nints, + ngroups=ngroups, + readnoise=readnoise_vals)) amp_plot.add_tools(HoverTool(tooltips=[("file", "@file"), ("time", "@expstarts"), From 33c8ddb5e5af9b3afce9d9e0fa694840495abdda Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Fri, 20 Oct 2023 11:03:29 -0400 Subject: [PATCH 449/449] Addressing Bryans comments --- .../monitor_pages/monitor_readnoise_bokeh.py | 39 +++++++++++++------ 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py index 82f0fe914..552cac71f 100644 --- a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py @@ -41,6 +41,8 @@ class ReadnoiseMonitorData(): instrument : str Instrument name (e.g. nircam) + aperture : str + Aperture name (e.g. apername) Attributes ---------- @@ -49,7 +51,12 @@ class ReadnoiseMonitorData(): Instrument name (e.g. nircam) aperture : str Aperture name (e.g. apername) - + query_results : list + Results from read noise statistics table based on + instrument, aperture and exposure start time + stats_table : sqlalchemy.orm.decl_api.DeclarativeMeta + Statistics table object to query based on instrument + and aperture """ def __init__(self, instrument, aperture): @@ -113,14 +120,18 @@ def __init__(self, instrument, aperture): title=self.aperture) def plot_readnoise_amplifers(self): - + """Class to create readnoise scatter plots per amplifier. + """ self.amp_plots = [] for amp in ['1', '2', '3', '4']: amp_plot = figure(title='Amp {}'.format(amp), width=280, height=280, x_axis_type='datetime') amp_plot.xaxis[0].ticker.desired_num_ticks = 4 - readnoise_vals = np.array([getattr(result, 'amp{}_mean'.format(amp)) for result in self.db.query_results]) + if self.db.query_results: + readnoise_vals = np.array([getattr(result, 'amp{}_mean'.format(amp)) for result in self.db.query_results]) + else: + readnoise_vals = np.array(list()) filenames = [os.path.basename(result.uncal_filename).replace('_uncal.fits', '') for result in self.db.query_results] expstarts_iso = np.array([result.expstart for result in self.db.query_results]) @@ -160,18 +171,24 @@ def plot_readnoise_difference_image(self): diff_image_png = self.db.query_results[-1].readnoise_diff_image diff_image_png = os.path.join('/static', '/'.join(diff_image_png.split('/')[-6:])) self.diff_image_plot.image_url(url=[diff_image_png], x=0, y=0, w=2048, h=2048, anchor="bottom_left") - self.diff_image_plot.xaxis.visible = False - self.diff_image_plot.yaxis.visible = False - self.diff_image_plot.xgrid.grid_line_color = None - self.diff_image_plot.ygrid.grid_line_color = None - self.diff_image_plot.title.text_font_size = '22px' - self.diff_image_plot.title.align = 'center' + + self.diff_image_plot.xaxis.visible = False + self.diff_image_plot.yaxis.visible = False + self.diff_image_plot.xgrid.grid_line_color = None + self.diff_image_plot.ygrid.grid_line_color = None + self.diff_image_plot.title.text_font_size = '22px' + self.diff_image_plot.title.align = 'center' + def plot_readnoise_histogram(self): """Updates the readnoise histogram""" - diff_image_n = np.array(self.db.query_results[-1].diff_image_n) - diff_image_bin_centers = np.array(self.db.query_results[-1].diff_image_bin_centers) + if len(self.db.query_results) != 0: + diff_image_n = np.array(self.db.query_results[-1].diff_image_n) + diff_image_bin_centers = np.array(self.db.query_results[-1].diff_image_bin_centers) + else: + diff_image_n = np.array(list()) + diff_image_bin_centers = np.array(list()) hist_xr_start = diff_image_bin_centers.min() hist_xr_end = diff_image_bin_centers.max()