From b462352dc0a5ec7e43ad4f5e069e91bc24174b7f Mon Sep 17 00:00:00 2001 From: Michael Engesser Date: Wed, 23 Sep 2020 15:50:52 -0400 Subject: [PATCH 0001/1618] Cosmic Ray Monitor Adding cosmic ray monitor, as well as unit test code and relevant database files. Note that testing will require test files and an edited config.json. Reach out for a copy of mine if needed. --- jwql/database/database_interface.py | 7 + .../miri/miri_cosmic_ray_query_history.txt | 6 + .../miri/miri_cosmic_ray_stats.txt | 6 + .../miri_monitors/cosmic_ray_monitor.py | 628 ++++++++++++++++++ jwql/tests/test_cosmic_ray_monitor.py | 121 ++++ 5 files changed, 768 insertions(+) create mode 100644 jwql/database/monitor_table_definitions/miri/miri_cosmic_ray_query_history.txt create mode 100644 jwql/database/monitor_table_definitions/miri/miri_cosmic_ray_stats.txt create mode 100644 jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py create mode 100644 jwql/tests/test_cosmic_ray_monitor.py diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py index 855a5c67e..44b6fd8ff 100755 --- a/jwql/database/database_interface.py +++ b/jwql/database/database_interface.py @@ -423,6 +423,13 @@ class : obj NIRCamReadnoiseStats = monitor_orm_factory('nircam_readnoise_stats') NIRISSReadnoiseQueryHistory = monitor_orm_factory('niriss_readnoise_query_history') NIRISSReadnoiseStats = monitor_orm_factory('niriss_readnoise_stats') +<<<<<<< Updated upstream +MIRICosmicRayStats = monitor_orm_factory('miri_cosmic_ray_stats') +MIRICosmicRayQueryHistory = monitor_orm_factory('miri_cosmic_ray_query_history') +======= +MIRICosmicRayQueryHistory = monitor_orm_factory('miri_cosmic_ray_query_history') +MIRICosmicRayStats = monitor_orm_factory('miri_cosmic_ray_stats') +>>>>>>> Stashed changes if __name__ == '__main__': diff --git a/jwql/database/monitor_table_definitions/miri/miri_cosmic_ray_query_history.txt b/jwql/database/monitor_table_definitions/miri/miri_cosmic_ray_query_history.txt new file mode 100644 index 000000000..bbea160bb --- /dev/null +++ b/jwql/database/monitor_table_definitions/miri/miri_cosmic_ray_query_history.txt @@ -0,0 +1,6 @@ +INSTRUMENT, string +APERTURE, string +START_TIME_MJD, float +END_TIME_MJD, float +FILES_FOUND, integer +RUN_MONITOR, bool \ No newline at end of file diff --git a/jwql/database/monitor_table_definitions/miri/miri_cosmic_ray_stats.txt b/jwql/database/monitor_table_definitions/miri/miri_cosmic_ray_stats.txt new file mode 100644 index 000000000..97259bfb4 --- /dev/null +++ b/jwql/database/monitor_table_definitions/miri/miri_cosmic_ray_stats.txt @@ -0,0 +1,6 @@ +APERTURE, string +SOURCE_FILE, string +OBS_START_TIME, datetime +OBS_END_TIME, datetime +JUMP_COUNT, integer +MAGNITUDE, float_array_1d \ No newline at end of file diff --git a/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py b/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py new file mode 100644 index 000000000..c998cd619 --- /dev/null +++ b/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py @@ -0,0 +1,628 @@ +#! /usr/bin/env python + +<<<<<<< Updated upstream +""" + +""" + +import os +import logging + +from astroquery.mast import Mast +from jwst import datamodels +#from bokeh.charts import Donut +#from bokeh.embed import components + +# Functions for logging +======= +"""This module contains code for the cosmic ray monitor, which currently checks +the number and magnitude of jumps in all observations performed with MIRI. + +The code first checks MAST for any new MIRI observations that have not yet been +run through the monitor. It then copies those files to a working directory, +where they are run through the pipeline, and for which the output is stored +in a new directory for each observation. + +Each observation is then analyzed for jumps due to cosmic rays, of which the number +and magnitude are recorded. This information is then inserted into the +"MiriCosmicRayStats" database table. + + +Author +------ + + -Mike Engesser + +Use +--- + + This module can be used from the command line as such: + + :: + + python cosmic_ray_monitor.py +""" + + +# Functions for logging +import logging +>>>>>>> Stashed changes +from jwql.utils.logging_functions import configure_logging +from jwql.utils.logging_functions import log_info +from jwql.utils.logging_functions import log_fail + +<<<<<<< Updated upstream +# Function for setting permissions of files/directories +#from jwql.permissions.permissions import set_permissions + +# Function for parsing filenames +from jwql.utils.utils import filename_parser + +# Objects for hard-coded information +from jwql.utils.utils import get_config +from jwql.utils.constants import JWST_DATAPRODUCTS, JWST_INSTRUMENT_NAMES + +#My imports +from jwql.jwql_monitors import monitor_mast +from astropy.time import Time +from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_DATAPRODUCTS +======= +# Funtions for database interaction +from jwql.jwql_monitors import monitor_mast +from jwql.utils.constants import JWST_DATAPRODUCTS +>>>>>>> Stashed changes +from jwql.database.database_interface import session +from jwql.database.database_interface import MIRICosmicRayQueryHistory +from jwql.database.database_interface import MIRICosmicRayStats +from sqlalchemy import func +from sqlalchemy.sql.expression import and_ +<<<<<<< Updated upstream +from pysiaf import Siaf +from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path +import os +from astropy.io import fits +import numpy as np +from jwql.instrument_monitors import pipeline_tools +import shutil +======= + +# Functions for file manipulation +from jwql.utils.utils import copy_files, ensure_dir_exists, get_config + +# Astropy functions +from astropy.io import fits +from astropy.time import Time + +# Utility functions +from jwql.instrument_monitors import pipeline_tools +import os +import numpy as np +import shutil +import julian +import datetime +>>>>>>> Stashed changes + + +@log_fail +@log_info +def cosmic_ray_monitor(): +<<<<<<< Updated upstream + """ + The main function of the ``monitor_template`` module. +======= + """ +>>>>>>> Stashed changes + Queries MAST for new MIRI data and copies it to a working + directory where it is run through the JWST pipeline. The output of the 'jump' + and 'rate' steps is used to determine the number and magnitudes of cosmic rays + which is then saved to the database. +<<<<<<< Updated upstream + + + + +======= +>>>>>>> Stashed changes + """ + + logging.info('Begin logging for cosmic_ray_monitor') + + aperture = 'MIRIM_FULL' +<<<<<<< Updated upstream + query_table = eval('MIRICosmicRayQueryHistory') + stats_table = eval('MIRICosmicRayStats') +======= + query_table = MIRICosmicRayQueryHistory + stats_table = MIRICosmicRayStats +>>>>>>> Stashed changes + + #We start by querying MAST for new data + start_date = most_recent_search(aperture, query_table) + end_date = Time.now().mjd + logging.info('\tMost recent query: {}'.format(start_date)) + + new_entries = query_mast(start_date, end_date) +<<<<<<< Updated upstream + + new_filenames = ['jw04192001001_01101_00001_MIRIMAGE_uncal.fits'] +======= + + #for testing purposes only + new_filenames = get_config()['local_test_data'] + +>>>>>>> Stashed changes + #for file_entry in new_entries['data']: + # try: + # new_filenames.append(filesystem_path(file_entry['filename'])) + # except FileNotFoundError: + # logging.info('\t{} not found in target directory'.format(file_entry['filename'])) + # except ValueError: + # logging.info( + # '\tProvided file {} does not follow JWST naming conventions.'.format(file_entry['filename'])) + + #Next we copy new files to the working directory + output_dir = os.path.join(get_config()['outputs'], 'cosmic_ray_monitor') +<<<<<<< Updated upstream + data_dir = '/Users/mengesser/Documents/MIRI-JWQL-Monitors/'#os.path.join(output_dir,'data') +======= + + data_dir = get_config()['local_test_dir'] #for testing purposes only + + #data_dir = os.path.join(output_dir,'data') +>>>>>>> Stashed changes + ensure_dir_exists(data_dir) + + cosmic_ray_files, not_copied = copy_files(new_filenames, data_dir) + + for file_name in cosmic_ray_files: + if 'uncal' in file_name: + dir_name = file_name[51:76] #Gets the observation identifier. Should be constant? + obs_dir = os.path.join(data_dir, dir_name) + ensure_dir_exists(obs_dir) +<<<<<<< Updated upstream + +======= + + head = fits.getheader(file_name) + nints = head['NINTS'] + +>>>>>>> Stashed changes + try: + shutil.copy2(file_name, obs_dir) + except: + logging.info('Failed to copy {} to observation dir.'.format(file_name)) + + #Next we run the pipeline on the files to get the proper outputs + MIRI_steps = pipeline_tools.get_pipeline_steps('MIRI') + + file = os.path.join(obs_dir, os.path.basename(file_name)) + + completed_steps = pipeline_tools.completed_pipeline_steps(file) + steps = pipeline_tools.steps_to_run(MIRI_steps,completed_steps) + + try: +<<<<<<< Updated upstream + pipeline_tools.run_calwebb_detector1_steps(file,steps,steps) +======= + pipeline_tools.calwebb_detector1_save_jump(file, obs_dir, ramp_fit=True, save_fitopt=False) +>>>>>>> Stashed changes + except: + logging.info('Failed to complete pipeline steps on {}.'.format(file)) + + + #Temporarily, we need to move the rateints file to the new folder manually +<<<<<<< Updated upstream + f = 'rateint_fitopt.fits' + shutil.move(os.path.join(data_dir, f), os.path.join(obs_dir, f)) +======= + #f = 'rateint_fitopt.fits' + #shutil.move(os.path.join(data_dir, f), os.path.join(obs_dir, f)) +>>>>>>> Stashed changes + + #Next we analyze the cosmic rays in the new data + for output_file in os.listdir(obs_dir): + + if 'jump' in output_file: + jump_file = os.path.join(obs_dir,output_file) +<<<<<<< Updated upstream + + if 'rateint' in output_file: + rate_file = os.path.join(obs_dir,output_file) +======= + + if nints == 1: + if '0_ramp_fit' in output_file: + rate_file = os.path.join(obs_dir, output_file) + + elif nints > 1: + if '1_ramp_fit' in output_file: + rate_file = os.path.join(obs_dir,output_file) +>>>>>>> Stashed changes + + try: + jump_head, jump_data, jump_dq = get_jump_data(jump_file) + except: + logging.info('Could not open jump file: {}'.format(jump_file)) + + try: + rate_data = get_rate_data(rate_file) + except: + logging.info('Could not open rate file: {}'.format(rate_file)) + +<<<<<<< Updated upstream + jump_locs = get_jump_locs(jump_dq) + jump_locs_pre = group_before(jump_locs) + + eff_time = jump_head['EFFEXPTM'] + + cosmic_ray_num = len(jump_locs) + #cosmic_ray_rate = get_cr_rate(jump_locs, eff_time) + cosmic_ray_mags = get_cr_mags(jump_locs, jump_locs_pre, rate_data, jump_data, jump_head) + + #Insert new data into database + cosmic_ray_db_entry = {'aperture': aperture, + 'source_files': file_name, + 'obs_start_time': start_time, + 'obs_end_time': end_time, + 'jump_count': cosmic_ray_num, + 'magnitude': cosmic_ray_mags + } + stats_table.__table__.insert().execute(cosmic_ray_db_entry) + +def get_cr_mags(jump_locs, jump_locs_pre, rateints, jump_data, jump_head): +======= + jump_locs = get_jump_locs(jump_dq,nints) + jump_locs_pre = group_before(jump_locs,nints) + + eff_time = jump_head['EFFEXPTM'] + + # Get observation time info + obs_start_time = jump_head['EXPSTART'] + obs_end_time = jump_head['EXPEND'] + start_time = julian.from_jd(obs_start_time, fmt='mjd') + end_time = julian.from_jd(obs_end_time, fmt='mjd') + + cosmic_ray_num = len(jump_locs) + #cosmic_ray_rate = get_cr_rate(jump_locs, eff_time) + cosmic_ray_mags = get_cr_mags(jump_locs, jump_locs_pre, rate_data, jump_data, jump_head, nints) + + # Insert new data into database + try: + cosmic_ray_db_entry = {'entry_date': datetime.datetime.now(), + 'aperture': aperture, + 'source_file': file_name, + 'obs_start_time': start_time, + 'obs_end_time': end_time, + 'jump_count': cosmic_ray_num, + 'magnitude': cosmic_ray_mags + } + stats_table.__table__.insert().execute(cosmic_ray_db_entry) + + logging.info("Successfully inserted into database. \n") + except: + logging.info("Could not insert entry into database. \n") + +def get_cr_mags(jump_locs, jump_locs_pre, rateints, jump_data, jump_head, nints): +>>>>>>> Stashed changes + """ + Computes a list of magnitudes using the coordinate of the detected jump compared to + the magnitude of the same pixel in the group prior to the jump. + + Parameters: + ---------- + + jump_locs: list + List of coordinates to a pixel marked with a jump. + + jump_locs_pre: list + List of matching coordinates one group before jump_locs. + + head: FITS header + Header containing file information. + + rateints: ndarray + Array in DN/s. + + Returns: + ------- + + mags: list + A list of cosmic ray magnitudes corresponding to each jump. + + """ + + mags = [] + for coord, coord_gb in zip(jump_locs, jump_locs_pre): +<<<<<<< Updated upstream + mags.append(magnitude(coord, coord_gb, rateints, jump_data, jump_head)) + +======= + mags.append(magnitude(coord, coord_gb, rateints, jump_data, jump_head, nints)) + +>>>>>>> Stashed changes + return mags + +def get_cr_rate(jump_locs, t): + """ + Computes the rate of cosmic ray impacts in a given time. + + Parameters: + ---------- + jump_locs: list + List of coordinates to a pixel marked with a jump. + + t: int or float + Time over which to compute the rate. + Nominally the effective exposure time. + + """ + + return len(jump_locs)/t + +def get_jump_data(jump_filename): + """ + Opens and reads a given .FITS file containing cosmic rays. + + Parameters: + ---------- + jump_filename: str + Path to file. + + Returns: + ------- + head: FITS header + Header containing file information + + data: NoneType + FITS data + + dq: ndarray + Data Quality array containing jump flags. + + """ + + hdu = fits.open(jump_filename) + + head = hdu[0].header + data = hdu[1].data + + dq = hdu[3].data + + hdu.close() + + return head, data, dq + +<<<<<<< Updated upstream +def get_jump_locs(dq): +======= +def get_jump_locs(dq,nints): +>>>>>>> Stashed changes + """ + Uses the data quality array to find the location of all jumps in the data. + + Parameters: + ---------- + dq: ndarray + Data Quality array containing jump flags. + + Returns: + ------- + jump_locs: list + List of coordinates to a pixel marked with a jump. + """ + + temp = np.where(dq==4) + + jump_locs = [] +<<<<<<< Updated upstream + for i in range(len(temp[0])): + jump_locs.append((temp[0][i],temp[1][i],temp[2][i],temp[3][i])) + + return jump_locs + +def group_before(jump_locs): + """ + Creates a list of coordinates one group before given jump coordinates. + + Parameters: + ---------- + jump_locs: list + List of coordinates to a pixel marked with a jump. + + Returns: + ------- + jump_locs_pre: list + List of matching coordinates one group before jump_locs. + + """ + jump_locs_pre = [] + for coord in jump_locs: + jump_locs_pre.append((coord[0],coord[1]-1,coord[2],coord[3])) + + return jump_locs_pre +======= + + if nints > 1: + for i in range(len(temp[0])): + jump_locs.append((temp[0][i],temp[1][i],temp[2][i],temp[3][i])) + else: + for i in range(len(temp[0])): + jump_locs.append((temp[0][i],temp[1][i],temp[2][i])) + + return jump_locs +>>>>>>> Stashed changes + +def get_rate_data(rate_filename): + """ + Opens and reads a given .FITS file. + + Parameters: + ---------- + rate_filename: str + Path to file. + + Returns: + ------- + data: NoneType + FITS data + """ + + data = fits.getdata(rate_filename) + + return data + +<<<<<<< Updated upstream +def magnitude(coord, coord_gb, rateints, data, head): +======= + +def group_before(jump_locs, nints): + """ + Creates a list of coordinates one group before given jump coordinates. + + Parameters: + ---------- + jump_locs: list + List of coordinates to a pixel marked with a jump. + + Returns: + ------- + jump_locs_pre: list + List of matching coordinates one group before jump_locs. + + """ + jump_locs_pre = [] + + if nints > 1: + for coord in jump_locs: + jump_locs_pre.append((coord[0], coord[1] - 1, coord[2], coord[3])) + else: + for coord in jump_locs: + jump_locs_pre.append((coord[0] - 1, coord[1], coord[2])) + + return jump_locs_pre + +def magnitude(coord, coord_gb, rateints, data, head, nints): +>>>>>>> Stashed changes + """ + Calculates the magnitude of a list of jumps given their coordinates + in an array of pixels. + + Parameters: + ---------- + coord: tuple + Coordinate of jump. + + coord_gb: tuple + Coordinate of jump pixel one group before. + + head: FITS header + Header containing file information. + + rateints: ndarray + Array in DN/s. + + Returns: + ------- + cr_mag: + + """ +<<<<<<< Updated upstream + + rate = rateints[coord[0]][coord[2]][coord[3]] + grouptime = head['TGROUP'] + cr_mag = data[coord] - data[coord_gb] - rate*grouptime + +======= + + grouptime = head['TGROUP'] + + if nints == 1: + rate = rateints[coord[-2]][coord[-1]] + cr_mag = data[0][coord[0]][coord[1]][coord[2]] \ + - data[0][coord_gb[0]][coord_gb[1]][coord_gb[2]] \ + - rate * grouptime + else: + rate = rateints[coord[0]][coord[-2]][coord[-1]] + cr_mag = data[coord] - data[coord_gb] - rate * grouptime + +>>>>>>> Stashed changes + return cr_mag + +def most_recent_search(aperture, query_table): + """Adapted from Dark Monitor (Bryan Hilbert) + + Query the query history database and return the information + on the most recent query for the given ``aperture_name`` where + the cosmic ray monitor was executed. + + Returns: + ------- + query_result : float + Date (in MJD) of the ending range of the previous MAST query + where the cosmic ray monitor was run. + """ + + sub_query = session.query(query_table.aperture, + func.max(query_table.end_time_mjd).label('maxdate') + ).group_by(query_table.aperture).subquery('t2') + + # Note that "self.query_table.run_monitor == True" below is + # intentional. Switching = to "is" results in an error in the query. + query = session.query(query_table).join( + sub_query, + and_( + query_table.aperture == aperture, + query_table.end_time_mjd == sub_query.c.maxdate, + query_table.run_monitor == True + ) + ).all() + + query_count = len(query) + if query_count == 0: + query_result = 57357.0 # a.k.a. Dec 1, 2015 == CV3 + logging.info(('\tNo query history for {}. Beginning search date will be set to {}.' + .format(aperture, query_result))) + elif query_count > 1: + raise ValueError('More than one "most recent" query?') + else: + query_result = query[0].end_time_mjd + + return query_result + +def query_mast(start_date, end_date): + """ + Use astroquery to search MAST for cosmic ray data + + Parameters: + ---------- + start_date : float + Starting date for the search in MJD + end_date : float + Ending date for the search in MJD + + Returns + ------- + result : list + List of dictionaries containing the query results + + """ + + instrument = 'MIRI' + dataproduct = JWST_DATAPRODUCTS + parameters = {"date_obs_mjd": {"min": start_date, "max": end_date}} + + result = monitor_mast.instrument_inventory(instrument, dataproduct, + add_filters=parameters, + return_data=True) + + return result + + +if __name__ == '__main__': + + # Configure logging + module = os.path.basename(__file__).strip('.py') + configure_logging(module) + + # Call the main function + cosmic_ray_monitor() diff --git a/jwql/tests/test_cosmic_ray_monitor.py b/jwql/tests/test_cosmic_ray_monitor.py new file mode 100644 index 000000000..f55a138d7 --- /dev/null +++ b/jwql/tests/test_cosmic_ray_monitor.py @@ -0,0 +1,121 @@ +import pytest +import cosmic_ray_monitor as crm +import numpy as np +from astropy.io import fits + +from jwql.database.database_interface import MIRICosmicRayQueryHistory + +def define_test_data(nints): + if nints == 1: + data = np.ones((2,5,10,10)) + rate_data = np.ones((10,10)) + else: + data = np.ones((2,5,10,10)) + rate_data = np.ones((2,10,10)) + + + file = "jw00000000000_00000_00000_MIRIMAGE_uncal.fits" + aperture = "MIRIM_FULL" + + return data, rate_data, file, aperture + + +def test_get_jump_data(): + _ = define_test_data(2) + file = _[2] + + head, data, dq = crm.get_jump_data(file) + + assert type(head) == fits.header.Header + assert type(data) == np.ndarray + assert type(dq) == np.ndarray + + +def test_get_rate_data(): + _ = define_test_data(1) + file = _[2] + + data = crm.get_rate_data(file) + + assert type(data) == np.ndarray + + +def test_get_cr_rate(): + jump_locs = np.arange(100).tolist() + t = 5 + + rate = crm.get_cr_rate(jump_locs, t) + + assert rate == 20.0 + + +def test_group_before(): + jump_locs = [(2,1,1)] + nints = 1 + + assert crm.group_before(jump_locs, nints) == [(1,1,1)] + + jump_locs = [(1,2,1,1)] + nints = 2 + + assert crm.group_before(jump_locs, nints) == [(1,1,1,1)] + + +def test_magnitude(): + + nints = 5 + data, rate_data, file, aperture = define_test_data(nints) + head = fits.getheader(file) + coord = (1,2,1,1) + coord_gb = (1,1,1,1) + mag = crm.magnitude(coord, coord_gb, rate_data, data, head, nints) + assert mag == -2.77504 + + nints = 1 + data, rate_data, file, aperture = define_test_data(nints) + coord = (1,1,1) + coord_gb = (0,1,1) + mag = crm.magnitude(coord, coord_gb, rate_data, data, head, nints) + assert mag == -2.77504 + + +def test_get_cr_mags(): + + jump_locs = [(2,1,1)] + jump_locs_pre = [(1,1,1)] + nints = 1 + data, rate_data, file, aperture = define_test_data(nints) + head = fits.getheader(file) + + mags = crm.get_cr_mags(jump_locs, jump_locs_pre, rate_data, data, head, nints) + assert mags == [-2.77504] + + jump_locs = [(1,2,1,1)] + jump_locs_pre = [(1,1,1,1)] + nints = 5 + data, rate_data, file, aperture = define_test_data(nints) + + mags = crm.get_cr_mags(jump_locs, jump_locs_pre, rate_data, data, head, nints) + assert mags == [-2.77504] + + +def test_most_recent_search(): + + _ = define_test_data(1) + aperture = _[3] + + query_table = MIRICosmicRayQueryHistory + + result = crm.most_recent_search(aperture,query_table) + + assert result == 57357.0 + + +def test_query_mast(): + + start_date = 57357.0 + end_date = 57405.0 + + result = crm.query_mast(start_date, end_date) + + assert len(result) == 5 From d4dbae77b46a37461645b1ce61e21d5464dde8f9 Mon Sep 17 00:00:00 2001 From: Michael Engesser Date: Thu, 1 Oct 2020 01:18:51 -0400 Subject: [PATCH 0002/1618] [WIP] Building cosmic ray monitor bokeh plots Work in progress --- jwql/bokeh_templating/template.py | 2 +- jwql/utils/constants.py | 10 +- .../apps/jwql/monitor_pages/__init__.py | 2 + .../monitor_cosmic_rays_bokeh.py | 168 ++++++++++++++++++ .../yaml/cosmic_ray_monitor_interface.yaml | 74 ++++++++ jwql/website/apps/jwql/monitor_views.py | 30 ++++ .../jwql/templates/cosmic_ray_monitor.html | 26 +++ jwql/website/apps/jwql/urls.py | 4 +- jwql/website/apps/jwql/views.py | 24 +-- 9 files changed, 324 insertions(+), 16 deletions(-) create mode 100644 jwql/website/apps/jwql/monitor_pages/monitor_cosmic_rays_bokeh.py create mode 100644 jwql/website/apps/jwql/monitor_pages/yaml/cosmic_ray_monitor_interface.yaml create mode 100644 jwql/website/apps/jwql/templates/cosmic_ray_monitor.html diff --git a/jwql/bokeh_templating/template.py b/jwql/bokeh_templating/template.py index 073d67782..f62328073 100644 --- a/jwql/bokeh_templating/template.py +++ b/jwql/bokeh_templating/template.py @@ -138,7 +138,7 @@ def _parse_interface(self): # variable, since the constructors store everything in self.refs # (and self.document, for the document). try: - yaml.load_all(interface) + self.full_stream = list(yaml.load_all(interface, Loader=yaml.FullLoader)) except yaml.YAMLError as exc: raise BokehTemplateParserError(exc) diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index e19e4dc5f..1fa002799 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -78,6 +78,11 @@ # Defines the possible anomalies (with rendered name) to flag through the web app ANOMALY_CHOICES = [(anomaly, inflection.titleize(anomaly)) for anomaly in ANOMALIES_PER_INSTRUMENT] +# Bad pixel types by the type of data used to find them +BAD_PIXEL_TYPES = ['DEAD', 'HOT', 'LOW_QE', 'RC', 'OPEN', 'ADJ_OPEN', 'TELEGRAPH', 'OTHER_BAD_PIXEL'] +DARKS_BAD_PIXEL_TYPES = ['HOT', 'RC', 'OTHER_BAD_PIXEL', 'TELEGRAPH'] +FLATS_BAD_PIXEL_TYPES = ['DEAD', 'OPEN', 'ADJ_OPEN', 'LOW_QE'] + # Possible exposure types for dark current data DARK_EXP_TYPES = {'nircam': ['NRC_DARK'], 'niriss': ['NIS_DARK'], @@ -116,7 +121,8 @@ 'NRCB4_FULL', 'NRCB5_FULL'], 'NIRISS': ['NIS_CEN'], 'NIRSPEC': ['NRS1_FULL', 'NRS2_FULL'], - 'MIRI': ['MIRIM_FULL'] + 'MIRI': ['MIRIM_FULL'], + 'FGS': ['FGS1_FULL', 'FGS2_FULL'] } # Possible suffix types for nominal files @@ -167,7 +173,7 @@ 'miri': [('Dark Current Monitor', '#'), ('Data Trending', '/miri/miri_data_trending'), ('Bad Pixel Monitor', '#'), - ('Cosmic Ray Monitor', '#'), + ('Cosmic Ray Monitor', '/miri/cosmic_ray_monitor'), ('Photometry Monitor', '#'), ('TA Failure Monitor', '#'), ('Blind Pointing Accuracy Monitor', '#'), diff --git a/jwql/website/apps/jwql/monitor_pages/__init__.py b/jwql/website/apps/jwql/monitor_pages/__init__.py index 29f975f8a..bba755ce5 100644 --- a/jwql/website/apps/jwql/monitor_pages/__init__.py +++ b/jwql/website/apps/jwql/monitor_pages/__init__.py @@ -1,3 +1,5 @@ +from .monitor_badpixel_bokeh import BadPixelMonitor +from .monitor_cosmic_rays_bokeh import CosmicRayMonitor from .monitor_dark_bokeh import DarkMonitor from .monitor_filesystem_bokeh import MonitorFilesystem from .monitor_mast_bokeh import MastMonitor diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_cosmic_rays_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_cosmic_rays_bokeh.py new file mode 100644 index 000000000..07085081a --- /dev/null +++ b/jwql/website/apps/jwql/monitor_pages/monitor_cosmic_rays_bokeh.py @@ -0,0 +1,168 @@ +"""This module contains code for the cosmic ray monitor Bokeh plots. +Author +------ + - Mike Engesser +Use +--- + This module can be used from the command line as such: + :: + +""" + +import os + +from astropy.io import fits +from astropy.time import Time +import datetime +import numpy as np +import matplotlib.pyplot as plt + +from jwql.database.database_interface import session +from jwql.database.database_interface import MIRICosmicRayQueryHistory, MIRICosmicRayStats +from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE +from jwql.utils.utils import get_config, filesystem_path +from jwql.bokeh_templating import BokehTemplate + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + +class CosmicRayMonitor(BokehTemplate): + + # Combine instrument and aperture into a single property because we + # do not want to invoke the setter unless both are updated + @property + def aperture_info(self): + return (self._instrument, self._aperture) + + @aperture_info.setter + def aperture_info(self, info): + self._instrument, self._aperture = info + self.pre_init() + self.post_init() + + + def pre_init(self): + # Start with default values for instrument and aperture because + # BokehTemplate's __init__ method does not allow input arguments + try: + dummy_instrument = self._instrument + dummy_aperture = self._aperture + except AttributeError: + self._instrument = 'MIRI' + self._aperture = 'MIRIM_FULL' + + self._embed = True + + # App design + self.format_string = None + self.interface_file = os.path.join(SCRIPT_DIR, "yaml", "cosmic_ray_monitor_interface.yaml") + + # Load data tables + self.load_data() + self.get_history_data() + + # Get dates and coordinates of the most recent entries + #self.most_recent_data() + + + def post_init(self): + self._update_cosmic_ray_v_time() + self._update_cosmic_ray_histogram() + + + def get_histogram_data(self): + """Get data required to create cosmic ray histogram from the + database query. + """ + + last_hist_index = -1 + hist = plt.hist(self.mags[last_hist_index]) + + self.bin_left = np.array( [bar.get_x() for bar in hist[2]] ) + self.amplitude = [bar.get_height() for bar in hist[2]] + self.bottom = [bar.get_y() for bar in hist[2]] + deltas = self.bin_left[1:] - self.bin_left[0: -1] + self.bin_width = np.append(deltas[0], deltas) + + + def get_history_data(self): + """Extract data on the history of cosmic ray numbers from the + database query result + """ + self.cosmic_ray_history = {} + + self.times = [row.obs_end_time for row in self.cosmic_ray_table] + self.num = [row.jump_count for row in self.cosmic_ray_table] + + self.mags = [row.magnitude for row in self.cosmic_ray_table] + + hover_values = np.array([datetime.datetime.strftime(t, "%d-%b-%Y") for t in self.times]) + self.cosmic_ray_history['Cosmic Rays'] = (self.times, self.num, hover_values) + + + def identify_tables(self): + """Determine which database tables as associated with + a given instrument""" + mixed_case_name = JWST_INSTRUMENT_NAMES_MIXEDCASE[self._instrument.lower()] + self.query_table = eval('{}CosmicRayQueryHistory'.format(mixed_case_name)) + self.stats_table = eval('{}CosmicRayStats'.format(mixed_case_name)) + + + def load_data(self): + """Query the database tables to get data""" + + # Determine which database tables are needed based on instrument + self.identify_tables() + + # Query database for all data with a matching aperture + self.cosmic_ray_table = session.query(self.stats_table) \ + .filter(self.stats_table.aperture == self._aperture) \ + .all() + + + def most_recent_data(self): + """Get the cosmic ray magnitudes associated with the most + recent run of the monitor. + """ + + cosmic_ray_times = [row.obs_end_time for row in self.cosmic_ray_table] + + if len(cosmic_ray_times) > 0: + self.most_recent_obs = max(cosmic_ray_times) + else: + self.most_recent_obs = datetime.datetime(1999, 10, 31) + + + def _update_cosmic_ray_v_time(self): + """Update the plot properties for the plots of the number of cosmic rays + versus time. + """ + + self.refs['cosmic_ray_x_range'].start = min(self.times) + self.refs['cosmic_ray_x_range'].end = max(self.times) + self.refs['cosmic_ray_y_range'].start = min(self.num) + self.refs['cosmic_ray_y_range'].end = max(self.num) + + self.refs['cosmic_ray_history_figure'].title.text = 'MIRIM_FULL Cosmic Ray History' + self.refs['cosmic_ray_history_figure'].title.align = "center" + self.refs['cosmic_ray_history_figure'].title.text_font_size = "20px" + + + def _update_cosmic_ray_histogram(): + + mags = [row.magnitude for row in self.cosmic_ray_table] + + self.refs['hist_x_range'].start = 0 + self.refs['hist_x_range'].end = max(mags) + + self.refs['hist_y_range'].start = 0 + self.refs['hist_y_range'].end = max(mags) + + self.refs['cosmic_ray_histogram'].title.text = 'MIRIM_FULL Cosmic Ray Intensities' + self.refs['cosmic_ray_histogram'].title.align = "center" + self.refs['cosmic_ray_histogram'].title.text_font_size = "20px" + + + +# Uncomment the line below when testing via the command line: +# bokeh serve --show monitor_cosmic_rays_bokeh.py +CosmicRayMonitor() \ No newline at end of file diff --git a/jwql/website/apps/jwql/monitor_pages/yaml/cosmic_ray_monitor_interface.yaml b/jwql/website/apps/jwql/monitor_pages/yaml/cosmic_ray_monitor_interface.yaml new file mode 100644 index 000000000..c776a45d7 --- /dev/null +++ b/jwql/website/apps/jwql/monitor_pages/yaml/cosmic_ray_monitor_interface.yaml @@ -0,0 +1,74 @@ +# YAML file defining bokeh figures for the cosmic ray monitor +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Cosmic Rays v. Time Figure +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- !ColumnDataSource: &cosmic_ray_history_source + ref: "cosmic_ray_history_source" + data: + time: !self.cosmic_ray_history['Cosmic Rays'][0] + cosmic_rays: !self.cosmic_ray_history['Cosmic Rays'][1] + time_labels: !self.cosmic_ray_history['Cosmic Rays'][2] + +- !Range1d: &cosmic_ray_history_xrange + ref: "cosmic_ray_history_xrange" + start: 0 + end: 1 + bounds: 'auto' #!!python/tuple [0, 1] + +- !Range1d: &cosmic_ray_history_yrange + ref: "cosmic_ray_history_yrange" + start: 0 + end: 1 + bounds: 'auto' # !!python/tuple [-1, 1] + +- !Figure: &cosmic_ray_history_figure + ref: "cosmic_ray_history_figure" + x_axis_label: "Date" + x_axis_type: "datetime" + y_axis_label: "Number of cosmic rays" + x_range: *cosmic_ray_history_xrange + y_range: *cosmic_ray_history_yrange + tools: 'hover, zoom_in, zoom_out, box_zoom, pan, reset, save' + tooltips: "@time_labels Cosmic Rays: @cosmic_rays" + elements: + - {'kind': 'circle', 'x': 'time', 'y': 'cosmic_rays', line_width: 5, 'source': *cosmic_ray_history_source} + + +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Cosmic Ray Histogram Figure +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- !ColumnDataSource: &cosmic_ray_history_source + ref: "cosmic_ray_history_source" + data: + bin_center: !self.bin_center + amplitude: !self.amplitude + bottom: !self.bottom + bin_width: !self.bin_width + +- !Range1d: &hist_x_range + ref: "hist_x_range" + #start: 0 + #end: 1 + #bounds: 'auto' #!!python/tuple [0, 1] + +- !Range1d: &hist_y_range + ref: "hist_y_range" + #start: 0 + #end: 1 + #bounds: !!python/tuple [0, 1] + +- !Figure: &cosmic_ray_histogram + ref: "cosmic_ray_histogram" + x_axis_label: "Cosmic Ray Magnitude (DN)" + y_axis_label: "Number of Cosmic Rays" + x_range: *hist_x_range + y_range: *hist_y_range + elements: + - {'kind': 'vbar', 'x': 'bin_center', 'y': 'bin_width', 'top': 'amplitude', 'bottom': 'bottom', 'source': *cosmic_ray_history_source} +# - {'kind': 'text', 'x': 0, 'y': 20000, 'id': 1001} + +#- !Document: +# - !row: +# - *cosmic_ray_history_figure +# - !row: +# - *cosmic_ray_histogram \ No newline at end of file diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py index 1668072d0..b6761a28f 100644 --- a/jwql/website/apps/jwql/monitor_views.py +++ b/jwql/website/apps/jwql/monitor_views.py @@ -37,6 +37,36 @@ FILESYSTEM_DIR = os.path.join(get_config()['jwql_dir'], 'filesystem') +def cosmic_ray_monitor(request, inst='MIRI'): + """Generate the cosmic ray monitor page for a given instrument + Parameters + ---------- + request : HttpRequest object + Incoming request from the webpage + inst : str + Name of JWST instrument + Returns + ------- + HttpResponse object + Outgoing response sent to the webpage + """ + + # Ensure the instrument is correctly capitalized + inst = 'MIRI' #other instruments not currently supported + #JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()] + + tabs_components = bokeh_containers.cosmic_ray_monitor_tabs(inst) + + template = "cosmic_ray_monitor.html" + + context = { + 'inst': inst, + 'tabs_components': tabs_components, + } + + # Return a HTTP response with the template and dictionary of variables + return render(request, template, context) + def dark_monitor(request, inst): """Generate the dark monitor page for a given instrument diff --git a/jwql/website/apps/jwql/templates/cosmic_ray_monitor.html b/jwql/website/apps/jwql/templates/cosmic_ray_monitor.html new file mode 100644 index 000000000..e7ae04f8d --- /dev/null +++ b/jwql/website/apps/jwql/templates/cosmic_ray_monitor.html @@ -0,0 +1,26 @@ +{% extends "base.html" %} + +{% block preamble %} + + {{ inst }} Cosmic Ray Monitor- JWQL + +{% endblock %} + +{% block content %} + +
+ +

{{ inst }} Cosmic Ray Monitor

+
+ +
+ + {{ tabs_components[0] | safe }} +
+ + + {{ tabs_components[1] | safe }} + +
+ +{% endblock %} diff --git a/jwql/website/apps/jwql/urls.py b/jwql/website/apps/jwql/urls.py index ef19ac736..9ceccd4e1 100644 --- a/jwql/website/apps/jwql/urls.py +++ b/jwql/website/apps/jwql/urls.py @@ -64,12 +64,14 @@ # MIRI-specific views path('miri/miri_data_trending/', views.miri_data_trending, name='miri_data_trending'), + path('miri/cosmic_ray_monitor/', monitor_views.cosmic_ray_monitor, name='cosmic_ray_monitor'), # NIRSpec-specific views path('nirspec/nirspec_data_trending/', views.nirspec_data_trending, name='nirspec_data_trending'), # Common monitor views - re_path(r'^(?P({}))/.+_monitor/$'.format(instruments), monitor_views.dark_monitor, name='dark_monitor'), + re_path(r'^(?P({}))/dark_monitor/$'.format(instruments), monitor_views.dark_monitor, name='dark_monitor'), + re_path(r'^(?P({}))/badpixel_monitor/$'.format(instruments), monitor_views.badpixel_monitor, name='badpixel_monitor'), # Main site views path('about/', views.about, name='about'), diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 29af0156b..593401f41 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -196,8 +196,8 @@ def api_landing(request): return render(request, template, context) -@auth_required -def archived_proposals(request, user, inst): +# @auth_required +def archived_proposals(request, inst): #request, user, inst): """Generate the page listing all archived proposals in the database Parameters @@ -223,8 +223,8 @@ def archived_proposals(request, user, inst): return render(request, template, context) -@auth_required -def archived_proposals_ajax(request, user, inst): +# @auth_required +def archived_proposals_ajax(request, inst): # request, user, inst): """Generate the page listing all archived proposals in the database Parameters @@ -259,8 +259,8 @@ def archived_proposals_ajax(request, user, inst): return JsonResponse(context, json_dumps_params={'indent': 2}) -@auth_required -def archive_thumbnails(request, user, inst, proposal): +# @auth_required +def archive_thumbnails(request, inst, proposal): # request, user, inst, proposal): """Generate the page listing all archived images in the database for a certain proposal @@ -290,8 +290,8 @@ def archive_thumbnails(request, user, inst, proposal): return render(request, template, context) -@auth_required -def archive_thumbnails_ajax(request, user, inst, proposal): +# @auth_required +def archive_thumbnails_ajax(request, inst, proposal): # request, user, inst, proposal): """Generate the page listing all archived images in the database for a certain proposal @@ -346,8 +346,8 @@ def dashboard(request): return render(request, template, context) -@auth_info -def engineering_database(request, user): +# @auth_info +def engineering_database(request): # request, user): """Generate the EDB page. Parameters @@ -745,8 +745,8 @@ def view_header(request, inst, filename): return render(request, template, context) -@auth_required -def view_image(request, user, inst, file_root, rewrite=False): +# @auth_required +def view_image(request, inst, file_root, rewrite=False): # request, user, inst, file_root, rewrite=False): """Generate the image view page Parameters From 8851f0415d7526b28280e5cedef232f5cbf92a4c Mon Sep 17 00:00:00 2001 From: Michael Engesser Date: Thu, 1 Oct 2020 01:26:21 -0400 Subject: [PATCH 0003/1618] [WIP] Building Cosmic Ray Bokeh Plots work in progress --- jwql/bokeh_templating/template.py | 2 +- jwql/utils/constants.py | 10 +- .../apps/jwql/monitor_pages/__init__.py | 2 - .../monitor_cosmic_rays_bokeh.py | 168 ------------------ .../yaml/cosmic_ray_monitor_interface.yaml | 74 -------- jwql/website/apps/jwql/monitor_views.py | 30 ---- .../jwql/templates/cosmic_ray_monitor.html | 26 --- jwql/website/apps/jwql/urls.py | 4 +- jwql/website/apps/jwql/views.py | 24 +-- 9 files changed, 16 insertions(+), 324 deletions(-) delete mode 100644 jwql/website/apps/jwql/monitor_pages/monitor_cosmic_rays_bokeh.py delete mode 100644 jwql/website/apps/jwql/monitor_pages/yaml/cosmic_ray_monitor_interface.yaml delete mode 100644 jwql/website/apps/jwql/templates/cosmic_ray_monitor.html diff --git a/jwql/bokeh_templating/template.py b/jwql/bokeh_templating/template.py index f62328073..073d67782 100644 --- a/jwql/bokeh_templating/template.py +++ b/jwql/bokeh_templating/template.py @@ -138,7 +138,7 @@ def _parse_interface(self): # variable, since the constructors store everything in self.refs # (and self.document, for the document). try: - self.full_stream = list(yaml.load_all(interface, Loader=yaml.FullLoader)) + yaml.load_all(interface) except yaml.YAMLError as exc: raise BokehTemplateParserError(exc) diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index 1fa002799..e19e4dc5f 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -78,11 +78,6 @@ # Defines the possible anomalies (with rendered name) to flag through the web app ANOMALY_CHOICES = [(anomaly, inflection.titleize(anomaly)) for anomaly in ANOMALIES_PER_INSTRUMENT] -# Bad pixel types by the type of data used to find them -BAD_PIXEL_TYPES = ['DEAD', 'HOT', 'LOW_QE', 'RC', 'OPEN', 'ADJ_OPEN', 'TELEGRAPH', 'OTHER_BAD_PIXEL'] -DARKS_BAD_PIXEL_TYPES = ['HOT', 'RC', 'OTHER_BAD_PIXEL', 'TELEGRAPH'] -FLATS_BAD_PIXEL_TYPES = ['DEAD', 'OPEN', 'ADJ_OPEN', 'LOW_QE'] - # Possible exposure types for dark current data DARK_EXP_TYPES = {'nircam': ['NRC_DARK'], 'niriss': ['NIS_DARK'], @@ -121,8 +116,7 @@ 'NRCB4_FULL', 'NRCB5_FULL'], 'NIRISS': ['NIS_CEN'], 'NIRSPEC': ['NRS1_FULL', 'NRS2_FULL'], - 'MIRI': ['MIRIM_FULL'], - 'FGS': ['FGS1_FULL', 'FGS2_FULL'] + 'MIRI': ['MIRIM_FULL'] } # Possible suffix types for nominal files @@ -173,7 +167,7 @@ 'miri': [('Dark Current Monitor', '#'), ('Data Trending', '/miri/miri_data_trending'), ('Bad Pixel Monitor', '#'), - ('Cosmic Ray Monitor', '/miri/cosmic_ray_monitor'), + ('Cosmic Ray Monitor', '#'), ('Photometry Monitor', '#'), ('TA Failure Monitor', '#'), ('Blind Pointing Accuracy Monitor', '#'), diff --git a/jwql/website/apps/jwql/monitor_pages/__init__.py b/jwql/website/apps/jwql/monitor_pages/__init__.py index bba755ce5..29f975f8a 100644 --- a/jwql/website/apps/jwql/monitor_pages/__init__.py +++ b/jwql/website/apps/jwql/monitor_pages/__init__.py @@ -1,5 +1,3 @@ -from .monitor_badpixel_bokeh import BadPixelMonitor -from .monitor_cosmic_rays_bokeh import CosmicRayMonitor from .monitor_dark_bokeh import DarkMonitor from .monitor_filesystem_bokeh import MonitorFilesystem from .monitor_mast_bokeh import MastMonitor diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_cosmic_rays_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_cosmic_rays_bokeh.py deleted file mode 100644 index 07085081a..000000000 --- a/jwql/website/apps/jwql/monitor_pages/monitor_cosmic_rays_bokeh.py +++ /dev/null @@ -1,168 +0,0 @@ -"""This module contains code for the cosmic ray monitor Bokeh plots. -Author ------- - - Mike Engesser -Use ---- - This module can be used from the command line as such: - :: - -""" - -import os - -from astropy.io import fits -from astropy.time import Time -import datetime -import numpy as np -import matplotlib.pyplot as plt - -from jwql.database.database_interface import session -from jwql.database.database_interface import MIRICosmicRayQueryHistory, MIRICosmicRayStats -from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE -from jwql.utils.utils import get_config, filesystem_path -from jwql.bokeh_templating import BokehTemplate - -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) - -class CosmicRayMonitor(BokehTemplate): - - # Combine instrument and aperture into a single property because we - # do not want to invoke the setter unless both are updated - @property - def aperture_info(self): - return (self._instrument, self._aperture) - - @aperture_info.setter - def aperture_info(self, info): - self._instrument, self._aperture = info - self.pre_init() - self.post_init() - - - def pre_init(self): - # Start with default values for instrument and aperture because - # BokehTemplate's __init__ method does not allow input arguments - try: - dummy_instrument = self._instrument - dummy_aperture = self._aperture - except AttributeError: - self._instrument = 'MIRI' - self._aperture = 'MIRIM_FULL' - - self._embed = True - - # App design - self.format_string = None - self.interface_file = os.path.join(SCRIPT_DIR, "yaml", "cosmic_ray_monitor_interface.yaml") - - # Load data tables - self.load_data() - self.get_history_data() - - # Get dates and coordinates of the most recent entries - #self.most_recent_data() - - - def post_init(self): - self._update_cosmic_ray_v_time() - self._update_cosmic_ray_histogram() - - - def get_histogram_data(self): - """Get data required to create cosmic ray histogram from the - database query. - """ - - last_hist_index = -1 - hist = plt.hist(self.mags[last_hist_index]) - - self.bin_left = np.array( [bar.get_x() for bar in hist[2]] ) - self.amplitude = [bar.get_height() for bar in hist[2]] - self.bottom = [bar.get_y() for bar in hist[2]] - deltas = self.bin_left[1:] - self.bin_left[0: -1] - self.bin_width = np.append(deltas[0], deltas) - - - def get_history_data(self): - """Extract data on the history of cosmic ray numbers from the - database query result - """ - self.cosmic_ray_history = {} - - self.times = [row.obs_end_time for row in self.cosmic_ray_table] - self.num = [row.jump_count for row in self.cosmic_ray_table] - - self.mags = [row.magnitude for row in self.cosmic_ray_table] - - hover_values = np.array([datetime.datetime.strftime(t, "%d-%b-%Y") for t in self.times]) - self.cosmic_ray_history['Cosmic Rays'] = (self.times, self.num, hover_values) - - - def identify_tables(self): - """Determine which database tables as associated with - a given instrument""" - mixed_case_name = JWST_INSTRUMENT_NAMES_MIXEDCASE[self._instrument.lower()] - self.query_table = eval('{}CosmicRayQueryHistory'.format(mixed_case_name)) - self.stats_table = eval('{}CosmicRayStats'.format(mixed_case_name)) - - - def load_data(self): - """Query the database tables to get data""" - - # Determine which database tables are needed based on instrument - self.identify_tables() - - # Query database for all data with a matching aperture - self.cosmic_ray_table = session.query(self.stats_table) \ - .filter(self.stats_table.aperture == self._aperture) \ - .all() - - - def most_recent_data(self): - """Get the cosmic ray magnitudes associated with the most - recent run of the monitor. - """ - - cosmic_ray_times = [row.obs_end_time for row in self.cosmic_ray_table] - - if len(cosmic_ray_times) > 0: - self.most_recent_obs = max(cosmic_ray_times) - else: - self.most_recent_obs = datetime.datetime(1999, 10, 31) - - - def _update_cosmic_ray_v_time(self): - """Update the plot properties for the plots of the number of cosmic rays - versus time. - """ - - self.refs['cosmic_ray_x_range'].start = min(self.times) - self.refs['cosmic_ray_x_range'].end = max(self.times) - self.refs['cosmic_ray_y_range'].start = min(self.num) - self.refs['cosmic_ray_y_range'].end = max(self.num) - - self.refs['cosmic_ray_history_figure'].title.text = 'MIRIM_FULL Cosmic Ray History' - self.refs['cosmic_ray_history_figure'].title.align = "center" - self.refs['cosmic_ray_history_figure'].title.text_font_size = "20px" - - - def _update_cosmic_ray_histogram(): - - mags = [row.magnitude for row in self.cosmic_ray_table] - - self.refs['hist_x_range'].start = 0 - self.refs['hist_x_range'].end = max(mags) - - self.refs['hist_y_range'].start = 0 - self.refs['hist_y_range'].end = max(mags) - - self.refs['cosmic_ray_histogram'].title.text = 'MIRIM_FULL Cosmic Ray Intensities' - self.refs['cosmic_ray_histogram'].title.align = "center" - self.refs['cosmic_ray_histogram'].title.text_font_size = "20px" - - - -# Uncomment the line below when testing via the command line: -# bokeh serve --show monitor_cosmic_rays_bokeh.py -CosmicRayMonitor() \ No newline at end of file diff --git a/jwql/website/apps/jwql/monitor_pages/yaml/cosmic_ray_monitor_interface.yaml b/jwql/website/apps/jwql/monitor_pages/yaml/cosmic_ray_monitor_interface.yaml deleted file mode 100644 index c776a45d7..000000000 --- a/jwql/website/apps/jwql/monitor_pages/yaml/cosmic_ray_monitor_interface.yaml +++ /dev/null @@ -1,74 +0,0 @@ -# YAML file defining bokeh figures for the cosmic ray monitor -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Cosmic Rays v. Time Figure -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -- !ColumnDataSource: &cosmic_ray_history_source - ref: "cosmic_ray_history_source" - data: - time: !self.cosmic_ray_history['Cosmic Rays'][0] - cosmic_rays: !self.cosmic_ray_history['Cosmic Rays'][1] - time_labels: !self.cosmic_ray_history['Cosmic Rays'][2] - -- !Range1d: &cosmic_ray_history_xrange - ref: "cosmic_ray_history_xrange" - start: 0 - end: 1 - bounds: 'auto' #!!python/tuple [0, 1] - -- !Range1d: &cosmic_ray_history_yrange - ref: "cosmic_ray_history_yrange" - start: 0 - end: 1 - bounds: 'auto' # !!python/tuple [-1, 1] - -- !Figure: &cosmic_ray_history_figure - ref: "cosmic_ray_history_figure" - x_axis_label: "Date" - x_axis_type: "datetime" - y_axis_label: "Number of cosmic rays" - x_range: *cosmic_ray_history_xrange - y_range: *cosmic_ray_history_yrange - tools: 'hover, zoom_in, zoom_out, box_zoom, pan, reset, save' - tooltips: "@time_labels Cosmic Rays: @cosmic_rays" - elements: - - {'kind': 'circle', 'x': 'time', 'y': 'cosmic_rays', line_width: 5, 'source': *cosmic_ray_history_source} - - -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Cosmic Ray Histogram Figure -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -- !ColumnDataSource: &cosmic_ray_history_source - ref: "cosmic_ray_history_source" - data: - bin_center: !self.bin_center - amplitude: !self.amplitude - bottom: !self.bottom - bin_width: !self.bin_width - -- !Range1d: &hist_x_range - ref: "hist_x_range" - #start: 0 - #end: 1 - #bounds: 'auto' #!!python/tuple [0, 1] - -- !Range1d: &hist_y_range - ref: "hist_y_range" - #start: 0 - #end: 1 - #bounds: !!python/tuple [0, 1] - -- !Figure: &cosmic_ray_histogram - ref: "cosmic_ray_histogram" - x_axis_label: "Cosmic Ray Magnitude (DN)" - y_axis_label: "Number of Cosmic Rays" - x_range: *hist_x_range - y_range: *hist_y_range - elements: - - {'kind': 'vbar', 'x': 'bin_center', 'y': 'bin_width', 'top': 'amplitude', 'bottom': 'bottom', 'source': *cosmic_ray_history_source} -# - {'kind': 'text', 'x': 0, 'y': 20000, 'id': 1001} - -#- !Document: -# - !row: -# - *cosmic_ray_history_figure -# - !row: -# - *cosmic_ray_histogram \ No newline at end of file diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py index b6761a28f..1668072d0 100644 --- a/jwql/website/apps/jwql/monitor_views.py +++ b/jwql/website/apps/jwql/monitor_views.py @@ -37,36 +37,6 @@ FILESYSTEM_DIR = os.path.join(get_config()['jwql_dir'], 'filesystem') -def cosmic_ray_monitor(request, inst='MIRI'): - """Generate the cosmic ray monitor page for a given instrument - Parameters - ---------- - request : HttpRequest object - Incoming request from the webpage - inst : str - Name of JWST instrument - Returns - ------- - HttpResponse object - Outgoing response sent to the webpage - """ - - # Ensure the instrument is correctly capitalized - inst = 'MIRI' #other instruments not currently supported - #JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()] - - tabs_components = bokeh_containers.cosmic_ray_monitor_tabs(inst) - - template = "cosmic_ray_monitor.html" - - context = { - 'inst': inst, - 'tabs_components': tabs_components, - } - - # Return a HTTP response with the template and dictionary of variables - return render(request, template, context) - def dark_monitor(request, inst): """Generate the dark monitor page for a given instrument diff --git a/jwql/website/apps/jwql/templates/cosmic_ray_monitor.html b/jwql/website/apps/jwql/templates/cosmic_ray_monitor.html deleted file mode 100644 index e7ae04f8d..000000000 --- a/jwql/website/apps/jwql/templates/cosmic_ray_monitor.html +++ /dev/null @@ -1,26 +0,0 @@ -{% extends "base.html" %} - -{% block preamble %} - - {{ inst }} Cosmic Ray Monitor- JWQL - -{% endblock %} - -{% block content %} - -
- -

{{ inst }} Cosmic Ray Monitor

-
- -
- - {{ tabs_components[0] | safe }} -
- - - {{ tabs_components[1] | safe }} - -
- -{% endblock %} diff --git a/jwql/website/apps/jwql/urls.py b/jwql/website/apps/jwql/urls.py index 9ceccd4e1..ef19ac736 100644 --- a/jwql/website/apps/jwql/urls.py +++ b/jwql/website/apps/jwql/urls.py @@ -64,14 +64,12 @@ # MIRI-specific views path('miri/miri_data_trending/', views.miri_data_trending, name='miri_data_trending'), - path('miri/cosmic_ray_monitor/', monitor_views.cosmic_ray_monitor, name='cosmic_ray_monitor'), # NIRSpec-specific views path('nirspec/nirspec_data_trending/', views.nirspec_data_trending, name='nirspec_data_trending'), # Common monitor views - re_path(r'^(?P({}))/dark_monitor/$'.format(instruments), monitor_views.dark_monitor, name='dark_monitor'), - re_path(r'^(?P({}))/badpixel_monitor/$'.format(instruments), monitor_views.badpixel_monitor, name='badpixel_monitor'), + re_path(r'^(?P({}))/.+_monitor/$'.format(instruments), monitor_views.dark_monitor, name='dark_monitor'), # Main site views path('about/', views.about, name='about'), diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 593401f41..29af0156b 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -196,8 +196,8 @@ def api_landing(request): return render(request, template, context) -# @auth_required -def archived_proposals(request, inst): #request, user, inst): +@auth_required +def archived_proposals(request, user, inst): """Generate the page listing all archived proposals in the database Parameters @@ -223,8 +223,8 @@ def archived_proposals(request, inst): #request, user, inst): return render(request, template, context) -# @auth_required -def archived_proposals_ajax(request, inst): # request, user, inst): +@auth_required +def archived_proposals_ajax(request, user, inst): """Generate the page listing all archived proposals in the database Parameters @@ -259,8 +259,8 @@ def archived_proposals_ajax(request, inst): # request, user, inst): return JsonResponse(context, json_dumps_params={'indent': 2}) -# @auth_required -def archive_thumbnails(request, inst, proposal): # request, user, inst, proposal): +@auth_required +def archive_thumbnails(request, user, inst, proposal): """Generate the page listing all archived images in the database for a certain proposal @@ -290,8 +290,8 @@ def archive_thumbnails(request, inst, proposal): # request, user, inst, proposa return render(request, template, context) -# @auth_required -def archive_thumbnails_ajax(request, inst, proposal): # request, user, inst, proposal): +@auth_required +def archive_thumbnails_ajax(request, user, inst, proposal): """Generate the page listing all archived images in the database for a certain proposal @@ -346,8 +346,8 @@ def dashboard(request): return render(request, template, context) -# @auth_info -def engineering_database(request): # request, user): +@auth_info +def engineering_database(request, user): """Generate the EDB page. Parameters @@ -745,8 +745,8 @@ def view_header(request, inst, filename): return render(request, template, context) -# @auth_required -def view_image(request, inst, file_root, rewrite=False): # request, user, inst, file_root, rewrite=False): +@auth_required +def view_image(request, user, inst, file_root, rewrite=False): """Generate the image view page Parameters From 3b73b64fb6c11d41e45c09b08010a7f41518561f Mon Sep 17 00:00:00 2001 From: Michael Engesser Date: Thu, 1 Oct 2020 01:29:58 -0400 Subject: [PATCH 0004/1618] Revert "[WIP] Building Cosmic Ray Bokeh Plots" This reverts commit 8851f0415d7526b28280e5cedef232f5cbf92a4c. --- jwql/bokeh_templating/template.py | 2 +- jwql/utils/constants.py | 10 +- .../apps/jwql/monitor_pages/__init__.py | 2 + .../monitor_cosmic_rays_bokeh.py | 168 ++++++++++++++++++ .../yaml/cosmic_ray_monitor_interface.yaml | 74 ++++++++ jwql/website/apps/jwql/monitor_views.py | 30 ++++ .../jwql/templates/cosmic_ray_monitor.html | 26 +++ jwql/website/apps/jwql/urls.py | 4 +- jwql/website/apps/jwql/views.py | 24 +-- 9 files changed, 324 insertions(+), 16 deletions(-) create mode 100644 jwql/website/apps/jwql/monitor_pages/monitor_cosmic_rays_bokeh.py create mode 100644 jwql/website/apps/jwql/monitor_pages/yaml/cosmic_ray_monitor_interface.yaml create mode 100644 jwql/website/apps/jwql/templates/cosmic_ray_monitor.html diff --git a/jwql/bokeh_templating/template.py b/jwql/bokeh_templating/template.py index 073d67782..f62328073 100644 --- a/jwql/bokeh_templating/template.py +++ b/jwql/bokeh_templating/template.py @@ -138,7 +138,7 @@ def _parse_interface(self): # variable, since the constructors store everything in self.refs # (and self.document, for the document). try: - yaml.load_all(interface) + self.full_stream = list(yaml.load_all(interface, Loader=yaml.FullLoader)) except yaml.YAMLError as exc: raise BokehTemplateParserError(exc) diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index e19e4dc5f..1fa002799 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -78,6 +78,11 @@ # Defines the possible anomalies (with rendered name) to flag through the web app ANOMALY_CHOICES = [(anomaly, inflection.titleize(anomaly)) for anomaly in ANOMALIES_PER_INSTRUMENT] +# Bad pixel types by the type of data used to find them +BAD_PIXEL_TYPES = ['DEAD', 'HOT', 'LOW_QE', 'RC', 'OPEN', 'ADJ_OPEN', 'TELEGRAPH', 'OTHER_BAD_PIXEL'] +DARKS_BAD_PIXEL_TYPES = ['HOT', 'RC', 'OTHER_BAD_PIXEL', 'TELEGRAPH'] +FLATS_BAD_PIXEL_TYPES = ['DEAD', 'OPEN', 'ADJ_OPEN', 'LOW_QE'] + # Possible exposure types for dark current data DARK_EXP_TYPES = {'nircam': ['NRC_DARK'], 'niriss': ['NIS_DARK'], @@ -116,7 +121,8 @@ 'NRCB4_FULL', 'NRCB5_FULL'], 'NIRISS': ['NIS_CEN'], 'NIRSPEC': ['NRS1_FULL', 'NRS2_FULL'], - 'MIRI': ['MIRIM_FULL'] + 'MIRI': ['MIRIM_FULL'], + 'FGS': ['FGS1_FULL', 'FGS2_FULL'] } # Possible suffix types for nominal files @@ -167,7 +173,7 @@ 'miri': [('Dark Current Monitor', '#'), ('Data Trending', '/miri/miri_data_trending'), ('Bad Pixel Monitor', '#'), - ('Cosmic Ray Monitor', '#'), + ('Cosmic Ray Monitor', '/miri/cosmic_ray_monitor'), ('Photometry Monitor', '#'), ('TA Failure Monitor', '#'), ('Blind Pointing Accuracy Monitor', '#'), diff --git a/jwql/website/apps/jwql/monitor_pages/__init__.py b/jwql/website/apps/jwql/monitor_pages/__init__.py index 29f975f8a..bba755ce5 100644 --- a/jwql/website/apps/jwql/monitor_pages/__init__.py +++ b/jwql/website/apps/jwql/monitor_pages/__init__.py @@ -1,3 +1,5 @@ +from .monitor_badpixel_bokeh import BadPixelMonitor +from .monitor_cosmic_rays_bokeh import CosmicRayMonitor from .monitor_dark_bokeh import DarkMonitor from .monitor_filesystem_bokeh import MonitorFilesystem from .monitor_mast_bokeh import MastMonitor diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_cosmic_rays_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_cosmic_rays_bokeh.py new file mode 100644 index 000000000..07085081a --- /dev/null +++ b/jwql/website/apps/jwql/monitor_pages/monitor_cosmic_rays_bokeh.py @@ -0,0 +1,168 @@ +"""This module contains code for the cosmic ray monitor Bokeh plots. +Author +------ + - Mike Engesser +Use +--- + This module can be used from the command line as such: + :: + +""" + +import os + +from astropy.io import fits +from astropy.time import Time +import datetime +import numpy as np +import matplotlib.pyplot as plt + +from jwql.database.database_interface import session +from jwql.database.database_interface import MIRICosmicRayQueryHistory, MIRICosmicRayStats +from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE +from jwql.utils.utils import get_config, filesystem_path +from jwql.bokeh_templating import BokehTemplate + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + +class CosmicRayMonitor(BokehTemplate): + + # Combine instrument and aperture into a single property because we + # do not want to invoke the setter unless both are updated + @property + def aperture_info(self): + return (self._instrument, self._aperture) + + @aperture_info.setter + def aperture_info(self, info): + self._instrument, self._aperture = info + self.pre_init() + self.post_init() + + + def pre_init(self): + # Start with default values for instrument and aperture because + # BokehTemplate's __init__ method does not allow input arguments + try: + dummy_instrument = self._instrument + dummy_aperture = self._aperture + except AttributeError: + self._instrument = 'MIRI' + self._aperture = 'MIRIM_FULL' + + self._embed = True + + # App design + self.format_string = None + self.interface_file = os.path.join(SCRIPT_DIR, "yaml", "cosmic_ray_monitor_interface.yaml") + + # Load data tables + self.load_data() + self.get_history_data() + + # Get dates and coordinates of the most recent entries + #self.most_recent_data() + + + def post_init(self): + self._update_cosmic_ray_v_time() + self._update_cosmic_ray_histogram() + + + def get_histogram_data(self): + """Get data required to create cosmic ray histogram from the + database query. + """ + + last_hist_index = -1 + hist = plt.hist(self.mags[last_hist_index]) + + self.bin_left = np.array( [bar.get_x() for bar in hist[2]] ) + self.amplitude = [bar.get_height() for bar in hist[2]] + self.bottom = [bar.get_y() for bar in hist[2]] + deltas = self.bin_left[1:] - self.bin_left[0: -1] + self.bin_width = np.append(deltas[0], deltas) + + + def get_history_data(self): + """Extract data on the history of cosmic ray numbers from the + database query result + """ + self.cosmic_ray_history = {} + + self.times = [row.obs_end_time for row in self.cosmic_ray_table] + self.num = [row.jump_count for row in self.cosmic_ray_table] + + self.mags = [row.magnitude for row in self.cosmic_ray_table] + + hover_values = np.array([datetime.datetime.strftime(t, "%d-%b-%Y") for t in self.times]) + self.cosmic_ray_history['Cosmic Rays'] = (self.times, self.num, hover_values) + + + def identify_tables(self): + """Determine which database tables as associated with + a given instrument""" + mixed_case_name = JWST_INSTRUMENT_NAMES_MIXEDCASE[self._instrument.lower()] + self.query_table = eval('{}CosmicRayQueryHistory'.format(mixed_case_name)) + self.stats_table = eval('{}CosmicRayStats'.format(mixed_case_name)) + + + def load_data(self): + """Query the database tables to get data""" + + # Determine which database tables are needed based on instrument + self.identify_tables() + + # Query database for all data with a matching aperture + self.cosmic_ray_table = session.query(self.stats_table) \ + .filter(self.stats_table.aperture == self._aperture) \ + .all() + + + def most_recent_data(self): + """Get the cosmic ray magnitudes associated with the most + recent run of the monitor. + """ + + cosmic_ray_times = [row.obs_end_time for row in self.cosmic_ray_table] + + if len(cosmic_ray_times) > 0: + self.most_recent_obs = max(cosmic_ray_times) + else: + self.most_recent_obs = datetime.datetime(1999, 10, 31) + + + def _update_cosmic_ray_v_time(self): + """Update the plot properties for the plots of the number of cosmic rays + versus time. + """ + + self.refs['cosmic_ray_x_range'].start = min(self.times) + self.refs['cosmic_ray_x_range'].end = max(self.times) + self.refs['cosmic_ray_y_range'].start = min(self.num) + self.refs['cosmic_ray_y_range'].end = max(self.num) + + self.refs['cosmic_ray_history_figure'].title.text = 'MIRIM_FULL Cosmic Ray History' + self.refs['cosmic_ray_history_figure'].title.align = "center" + self.refs['cosmic_ray_history_figure'].title.text_font_size = "20px" + + + def _update_cosmic_ray_histogram(): + + mags = [row.magnitude for row in self.cosmic_ray_table] + + self.refs['hist_x_range'].start = 0 + self.refs['hist_x_range'].end = max(mags) + + self.refs['hist_y_range'].start = 0 + self.refs['hist_y_range'].end = max(mags) + + self.refs['cosmic_ray_histogram'].title.text = 'MIRIM_FULL Cosmic Ray Intensities' + self.refs['cosmic_ray_histogram'].title.align = "center" + self.refs['cosmic_ray_histogram'].title.text_font_size = "20px" + + + +# Uncomment the line below when testing via the command line: +# bokeh serve --show monitor_cosmic_rays_bokeh.py +CosmicRayMonitor() \ No newline at end of file diff --git a/jwql/website/apps/jwql/monitor_pages/yaml/cosmic_ray_monitor_interface.yaml b/jwql/website/apps/jwql/monitor_pages/yaml/cosmic_ray_monitor_interface.yaml new file mode 100644 index 000000000..c776a45d7 --- /dev/null +++ b/jwql/website/apps/jwql/monitor_pages/yaml/cosmic_ray_monitor_interface.yaml @@ -0,0 +1,74 @@ +# YAML file defining bokeh figures for the cosmic ray monitor +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Cosmic Rays v. Time Figure +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- !ColumnDataSource: &cosmic_ray_history_source + ref: "cosmic_ray_history_source" + data: + time: !self.cosmic_ray_history['Cosmic Rays'][0] + cosmic_rays: !self.cosmic_ray_history['Cosmic Rays'][1] + time_labels: !self.cosmic_ray_history['Cosmic Rays'][2] + +- !Range1d: &cosmic_ray_history_xrange + ref: "cosmic_ray_history_xrange" + start: 0 + end: 1 + bounds: 'auto' #!!python/tuple [0, 1] + +- !Range1d: &cosmic_ray_history_yrange + ref: "cosmic_ray_history_yrange" + start: 0 + end: 1 + bounds: 'auto' # !!python/tuple [-1, 1] + +- !Figure: &cosmic_ray_history_figure + ref: "cosmic_ray_history_figure" + x_axis_label: "Date" + x_axis_type: "datetime" + y_axis_label: "Number of cosmic rays" + x_range: *cosmic_ray_history_xrange + y_range: *cosmic_ray_history_yrange + tools: 'hover, zoom_in, zoom_out, box_zoom, pan, reset, save' + tooltips: "@time_labels Cosmic Rays: @cosmic_rays" + elements: + - {'kind': 'circle', 'x': 'time', 'y': 'cosmic_rays', line_width: 5, 'source': *cosmic_ray_history_source} + + +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Cosmic Ray Histogram Figure +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- !ColumnDataSource: &cosmic_ray_history_source + ref: "cosmic_ray_history_source" + data: + bin_center: !self.bin_center + amplitude: !self.amplitude + bottom: !self.bottom + bin_width: !self.bin_width + +- !Range1d: &hist_x_range + ref: "hist_x_range" + #start: 0 + #end: 1 + #bounds: 'auto' #!!python/tuple [0, 1] + +- !Range1d: &hist_y_range + ref: "hist_y_range" + #start: 0 + #end: 1 + #bounds: !!python/tuple [0, 1] + +- !Figure: &cosmic_ray_histogram + ref: "cosmic_ray_histogram" + x_axis_label: "Cosmic Ray Magnitude (DN)" + y_axis_label: "Number of Cosmic Rays" + x_range: *hist_x_range + y_range: *hist_y_range + elements: + - {'kind': 'vbar', 'x': 'bin_center', 'y': 'bin_width', 'top': 'amplitude', 'bottom': 'bottom', 'source': *cosmic_ray_history_source} +# - {'kind': 'text', 'x': 0, 'y': 20000, 'id': 1001} + +#- !Document: +# - !row: +# - *cosmic_ray_history_figure +# - !row: +# - *cosmic_ray_histogram \ No newline at end of file diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py index 1668072d0..b6761a28f 100644 --- a/jwql/website/apps/jwql/monitor_views.py +++ b/jwql/website/apps/jwql/monitor_views.py @@ -37,6 +37,36 @@ FILESYSTEM_DIR = os.path.join(get_config()['jwql_dir'], 'filesystem') +def cosmic_ray_monitor(request, inst='MIRI'): + """Generate the cosmic ray monitor page for a given instrument + Parameters + ---------- + request : HttpRequest object + Incoming request from the webpage + inst : str + Name of JWST instrument + Returns + ------- + HttpResponse object + Outgoing response sent to the webpage + """ + + # Ensure the instrument is correctly capitalized + inst = 'MIRI' #other instruments not currently supported + #JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()] + + tabs_components = bokeh_containers.cosmic_ray_monitor_tabs(inst) + + template = "cosmic_ray_monitor.html" + + context = { + 'inst': inst, + 'tabs_components': tabs_components, + } + + # Return a HTTP response with the template and dictionary of variables + return render(request, template, context) + def dark_monitor(request, inst): """Generate the dark monitor page for a given instrument diff --git a/jwql/website/apps/jwql/templates/cosmic_ray_monitor.html b/jwql/website/apps/jwql/templates/cosmic_ray_monitor.html new file mode 100644 index 000000000..e7ae04f8d --- /dev/null +++ b/jwql/website/apps/jwql/templates/cosmic_ray_monitor.html @@ -0,0 +1,26 @@ +{% extends "base.html" %} + +{% block preamble %} + + {{ inst }} Cosmic Ray Monitor- JWQL + +{% endblock %} + +{% block content %} + +
+ +

{{ inst }} Cosmic Ray Monitor

+
+ +
+ + {{ tabs_components[0] | safe }} +
+ + + {{ tabs_components[1] | safe }} + +
+ +{% endblock %} diff --git a/jwql/website/apps/jwql/urls.py b/jwql/website/apps/jwql/urls.py index ef19ac736..9ceccd4e1 100644 --- a/jwql/website/apps/jwql/urls.py +++ b/jwql/website/apps/jwql/urls.py @@ -64,12 +64,14 @@ # MIRI-specific views path('miri/miri_data_trending/', views.miri_data_trending, name='miri_data_trending'), + path('miri/cosmic_ray_monitor/', monitor_views.cosmic_ray_monitor, name='cosmic_ray_monitor'), # NIRSpec-specific views path('nirspec/nirspec_data_trending/', views.nirspec_data_trending, name='nirspec_data_trending'), # Common monitor views - re_path(r'^(?P({}))/.+_monitor/$'.format(instruments), monitor_views.dark_monitor, name='dark_monitor'), + re_path(r'^(?P({}))/dark_monitor/$'.format(instruments), monitor_views.dark_monitor, name='dark_monitor'), + re_path(r'^(?P({}))/badpixel_monitor/$'.format(instruments), monitor_views.badpixel_monitor, name='badpixel_monitor'), # Main site views path('about/', views.about, name='about'), diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 29af0156b..593401f41 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -196,8 +196,8 @@ def api_landing(request): return render(request, template, context) -@auth_required -def archived_proposals(request, user, inst): +# @auth_required +def archived_proposals(request, inst): #request, user, inst): """Generate the page listing all archived proposals in the database Parameters @@ -223,8 +223,8 @@ def archived_proposals(request, user, inst): return render(request, template, context) -@auth_required -def archived_proposals_ajax(request, user, inst): +# @auth_required +def archived_proposals_ajax(request, inst): # request, user, inst): """Generate the page listing all archived proposals in the database Parameters @@ -259,8 +259,8 @@ def archived_proposals_ajax(request, user, inst): return JsonResponse(context, json_dumps_params={'indent': 2}) -@auth_required -def archive_thumbnails(request, user, inst, proposal): +# @auth_required +def archive_thumbnails(request, inst, proposal): # request, user, inst, proposal): """Generate the page listing all archived images in the database for a certain proposal @@ -290,8 +290,8 @@ def archive_thumbnails(request, user, inst, proposal): return render(request, template, context) -@auth_required -def archive_thumbnails_ajax(request, user, inst, proposal): +# @auth_required +def archive_thumbnails_ajax(request, inst, proposal): # request, user, inst, proposal): """Generate the page listing all archived images in the database for a certain proposal @@ -346,8 +346,8 @@ def dashboard(request): return render(request, template, context) -@auth_info -def engineering_database(request, user): +# @auth_info +def engineering_database(request): # request, user): """Generate the EDB page. Parameters @@ -745,8 +745,8 @@ def view_header(request, inst, filename): return render(request, template, context) -@auth_required -def view_image(request, user, inst, file_root, rewrite=False): +# @auth_required +def view_image(request, inst, file_root, rewrite=False): # request, user, inst, file_root, rewrite=False): """Generate the image view page Parameters From c30069e8537dd4790eebf78d1c7ced6a4ecf339e Mon Sep 17 00:00:00 2001 From: Michael Engesser Date: Mon, 19 Oct 2020 18:35:02 -0400 Subject: [PATCH 0005/1618] Update cosmic_ray_monitor.py Fixing merge conflicts --- .../miri_monitors/cosmic_ray_monitor.py | 641 +++++++----------- 1 file changed, 240 insertions(+), 401 deletions(-) diff --git a/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py b/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py index c998cd619..b06c9e099 100644 --- a/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py +++ b/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py @@ -1,90 +1,49 @@ #! /usr/bin/env python -<<<<<<< Updated upstream -""" - -""" - -import os -import logging - -from astroquery.mast import Mast -from jwst import datamodels -#from bokeh.charts import Donut -#from bokeh.embed import components - -# Functions for logging -======= """This module contains code for the cosmic ray monitor, which currently checks -the number and magnitude of jumps in all observations performed with MIRI. - -The code first checks MAST for any new MIRI observations that have not yet been -run through the monitor. It then copies those files to a working directory, -where they are run through the pipeline, and for which the output is stored -in a new directory for each observation. - -Each observation is then analyzed for jumps due to cosmic rays, of which the number -and magnitude are recorded. This information is then inserted into the -"MiriCosmicRayStats" database table. - - -Author ------- - + the number and magnitude of jumps in all observations performed with MIRI. + + The code first checks MAST for any new MIRI observations that have not yet been + run through the monitor. It then copies those files to a working directory, + where they are run through the pipeline, and for which the output is stored + in a new directory for each observation. + + Each observation is then analyzed for jumps due to cosmic rays, of which the number + and magnitude are recorded. This information is then inserted into the + "MiriCosmicRayStats" database table. + + + Author + ------ + -Mike Engesser - -Use ---- - + + Use + --- + This module can be used from the command line as such: - + :: - - python cosmic_ray_monitor.py -""" + + python cosmic_ray_monitor.py + """ # Functions for logging import logging ->>>>>>> Stashed changes from jwql.utils.logging_functions import configure_logging from jwql.utils.logging_functions import log_info from jwql.utils.logging_functions import log_fail -<<<<<<< Updated upstream -# Function for setting permissions of files/directories -#from jwql.permissions.permissions import set_permissions - -# Function for parsing filenames -from jwql.utils.utils import filename_parser - -# Objects for hard-coded information -from jwql.utils.utils import get_config -from jwql.utils.constants import JWST_DATAPRODUCTS, JWST_INSTRUMENT_NAMES - -#My imports -from jwql.jwql_monitors import monitor_mast -from astropy.time import Time -from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_DATAPRODUCTS -======= # Funtions for database interaction from jwql.jwql_monitors import monitor_mast from jwql.utils.constants import JWST_DATAPRODUCTS ->>>>>>> Stashed changes + from jwql.database.database_interface import session from jwql.database.database_interface import MIRICosmicRayQueryHistory from jwql.database.database_interface import MIRICosmicRayStats from sqlalchemy import func from sqlalchemy.sql.expression import and_ -<<<<<<< Updated upstream -from pysiaf import Siaf -from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path -import os -from astropy.io import fits -import numpy as np -from jwql.instrument_monitors import pipeline_tools -import shutil -======= # Functions for file manipulation from jwql.utils.utils import copy_files, ensure_dir_exists, get_config @@ -100,41 +59,23 @@ import shutil import julian import datetime ->>>>>>> Stashed changes - @log_fail @log_info def cosmic_ray_monitor(): -<<<<<<< Updated upstream - """ - The main function of the ``monitor_template`` module. -======= - """ ->>>>>>> Stashed changes - Queries MAST for new MIRI data and copies it to a working - directory where it is run through the JWST pipeline. The output of the 'jump' - and 'rate' steps is used to determine the number and magnitudes of cosmic rays - which is then saved to the database. -<<<<<<< Updated upstream - - - - -======= ->>>>>>> Stashed changes """ + The main function of the ``monitor_template`` module. + Queries MAST for new MIRI data and copies it to a working + directory where it is run through the JWST pipeline. The output of the 'jump' + and 'rate' steps is used to determine the number and magnitudes of cosmic rays + which is then saved to the database. + """ logging.info('Begin logging for cosmic_ray_monitor') - + aperture = 'MIRIM_FULL' -<<<<<<< Updated upstream - query_table = eval('MIRICosmicRayQueryHistory') - stats_table = eval('MIRICosmicRayStats') -======= query_table = MIRICosmicRayQueryHistory stats_table = MIRICosmicRayStats ->>>>>>> Stashed changes #We start by querying MAST for new data start_date = most_recent_search(aperture, query_table) @@ -142,15 +83,10 @@ def cosmic_ray_monitor(): logging.info('\tMost recent query: {}'.format(start_date)) new_entries = query_mast(start_date, end_date) -<<<<<<< Updated upstream - new_filenames = ['jw04192001001_01101_00001_MIRIMAGE_uncal.fits'] -======= - #for testing purposes only new_filenames = get_config()['local_test_data'] - ->>>>>>> Stashed changes + #for file_entry in new_entries['data']: # try: # new_filenames.append(filesystem_path(file_entry['filename'])) @@ -159,19 +95,14 @@ def cosmic_ray_monitor(): # except ValueError: # logging.info( # '\tProvided file {} does not follow JWST naming conventions.'.format(file_entry['filename'])) - + #Next we copy new files to the working directory output_dir = os.path.join(get_config()['outputs'], 'cosmic_ray_monitor') -<<<<<<< Updated upstream - data_dir = '/Users/mengesser/Documents/MIRI-JWQL-Monitors/'#os.path.join(output_dir,'data') -======= - + data_dir = get_config()['local_test_dir'] #for testing purposes only - #data_dir = os.path.join(output_dir,'data') ->>>>>>> Stashed changes - ensure_dir_exists(data_dir) + ensure_dir_exists(data_dir) cosmic_ray_files, not_copied = copy_files(new_filenames, data_dir) for file_name in cosmic_ray_files: @@ -179,207 +110,160 @@ def cosmic_ray_monitor(): dir_name = file_name[51:76] #Gets the observation identifier. Should be constant? obs_dir = os.path.join(data_dir, dir_name) ensure_dir_exists(obs_dir) -<<<<<<< Updated upstream -======= - head = fits.getheader(file_name) nints = head['NINTS'] - ->>>>>>> Stashed changes + try: shutil.copy2(file_name, obs_dir) except: logging.info('Failed to copy {} to observation dir.'.format(file_name)) - + #Next we run the pipeline on the files to get the proper outputs MIRI_steps = pipeline_tools.get_pipeline_steps('MIRI') - + file = os.path.join(obs_dir, os.path.basename(file_name)) - + completed_steps = pipeline_tools.completed_pipeline_steps(file) steps = pipeline_tools.steps_to_run(MIRI_steps,completed_steps) try: -<<<<<<< Updated upstream - pipeline_tools.run_calwebb_detector1_steps(file,steps,steps) -======= pipeline_tools.calwebb_detector1_save_jump(file, obs_dir, ramp_fit=True, save_fitopt=False) ->>>>>>> Stashed changes except: logging.info('Failed to complete pipeline steps on {}.'.format(file)) - - - #Temporarily, we need to move the rateints file to the new folder manually -<<<<<<< Updated upstream - f = 'rateint_fitopt.fits' - shutil.move(os.path.join(data_dir, f), os.path.join(obs_dir, f)) -======= - #f = 'rateint_fitopt.fits' - #shutil.move(os.path.join(data_dir, f), os.path.join(obs_dir, f)) ->>>>>>> Stashed changes - - #Next we analyze the cosmic rays in the new data - for output_file in os.listdir(obs_dir): - - if 'jump' in output_file: - jump_file = os.path.join(obs_dir,output_file) -<<<<<<< Updated upstream + + + #Next we analyze the cosmic rays in the new data + for output_file in os.listdir(obs_dir): + + if 'jump' in output_file: + jump_file = os.path.join(obs_dir,output_file) if 'rateint' in output_file: rate_file = os.path.join(obs_dir,output_file) -======= - - if nints == 1: - if '0_ramp_fit' in output_file: - rate_file = os.path.join(obs_dir, output_file) - + + if nints == 1: + if '0_ramp_fit' in output_file: + rate_file = os.path.join(obs_dir, output_file) + elif nints > 1: if '1_ramp_fit' in output_file: rate_file = os.path.join(obs_dir,output_file) ->>>>>>> Stashed changes - - try: - jump_head, jump_data, jump_dq = get_jump_data(jump_file) - except: - logging.info('Could not open jump file: {}'.format(jump_file)) - - try: - rate_data = get_rate_data(rate_file) - except: - logging.info('Could not open rate file: {}'.format(rate_file)) - -<<<<<<< Updated upstream - jump_locs = get_jump_locs(jump_dq) - jump_locs_pre = group_before(jump_locs) - - eff_time = jump_head['EFFEXPTM'] - - cosmic_ray_num = len(jump_locs) - #cosmic_ray_rate = get_cr_rate(jump_locs, eff_time) - cosmic_ray_mags = get_cr_mags(jump_locs, jump_locs_pre, rate_data, jump_data, jump_head) - - #Insert new data into database - cosmic_ray_db_entry = {'aperture': aperture, - 'source_files': file_name, - 'obs_start_time': start_time, - 'obs_end_time': end_time, - 'jump_count': cosmic_ray_num, - 'magnitude': cosmic_ray_mags - } - stats_table.__table__.insert().execute(cosmic_ray_db_entry) - -def get_cr_mags(jump_locs, jump_locs_pre, rateints, jump_data, jump_head): -======= - jump_locs = get_jump_locs(jump_dq,nints) - jump_locs_pre = group_before(jump_locs,nints) - - eff_time = jump_head['EFFEXPTM'] - - # Get observation time info - obs_start_time = jump_head['EXPSTART'] - obs_end_time = jump_head['EXPEND'] - start_time = julian.from_jd(obs_start_time, fmt='mjd') - end_time = julian.from_jd(obs_end_time, fmt='mjd') - - cosmic_ray_num = len(jump_locs) - #cosmic_ray_rate = get_cr_rate(jump_locs, eff_time) - cosmic_ray_mags = get_cr_mags(jump_locs, jump_locs_pre, rate_data, jump_data, jump_head, nints) - - # Insert new data into database - try: - cosmic_ray_db_entry = {'entry_date': datetime.datetime.now(), - 'aperture': aperture, - 'source_file': file_name, - 'obs_start_time': start_time, - 'obs_end_time': end_time, - 'jump_count': cosmic_ray_num, - 'magnitude': cosmic_ray_mags - } - stats_table.__table__.insert().execute(cosmic_ray_db_entry) + + try: + jump_head, jump_data, jump_dq = get_jump_data(jump_file) + except: + logging.info('Could not open jump file: {}'.format(jump_file)) + + try: + rate_data = get_rate_data(rate_file) + except: + logging.info('Could not open rate file: {}'.format(rate_file)) + + + jump_locs = get_jump_locs(jump_dq,nints) + jump_locs_pre = group_before(jump_locs,nints) + + eff_time = jump_head['EFFEXPTM'] + + # Get observation time info + obs_start_time = jump_head['EXPSTART'] + obs_end_time = jump_head['EXPEND'] + start_time = julian.from_jd(obs_start_time, fmt='mjd') + end_time = julian.from_jd(obs_end_time, fmt='mjd') + + cosmic_ray_num = len(jump_locs) + #cosmic_ray_rate = get_cr_rate(jump_locs, eff_time) + cosmic_ray_mags = get_cr_mags(jump_locs, jump_locs_pre, rate_data, jump_data, jump_head, nints) + + # Insert new data into database + try: + cosmic_ray_db_entry = {'entry_date': datetime.datetime.now(), + 'aperture': aperture, + 'source_file': file_name, + 'obs_start_time': start_time, + 'obs_end_time': end_time, + 'jump_count': cosmic_ray_num, + 'magnitude': cosmic_ray_mags + } + stats_table.__table__.insert().execute(cosmic_ray_db_entry) + + logging.info("Successfully inserted into database. \n") + except: + logging.info("Could not insert entry into database. \n") - logging.info("Successfully inserted into database. \n") - except: - logging.info("Could not insert entry into database. \n") - def get_cr_mags(jump_locs, jump_locs_pre, rateints, jump_data, jump_head, nints): ->>>>>>> Stashed changes """ - Computes a list of magnitudes using the coordinate of the detected jump compared to - the magnitude of the same pixel in the group prior to the jump. - - Parameters: - ---------- - - jump_locs: list + Computes a list of magnitudes using the coordinate of the detected jump compared to + the magnitude of the same pixel in the group prior to the jump. + + Parameters: + ---------- + + jump_locs: list List of coordinates to a pixel marked with a jump. - jump_locs_pre: list + jump_locs_pre: list List of matching coordinates one group before jump_locs. - head: FITS header + head: FITS header Header containing file information. - rateints: ndarray - Array in DN/s. + rateints: ndarray + Array in DN/s. - Returns: - ------- - - mags: list - A list of cosmic ray magnitudes corresponding to each jump. - - """ + Returns: + ------- + + mags: list + A list of cosmic ray magnitudes corresponding to each jump. + + """ mags = [] for coord, coord_gb in zip(jump_locs, jump_locs_pre): -<<<<<<< Updated upstream - mags.append(magnitude(coord, coord_gb, rateints, jump_data, jump_head)) - -======= mags.append(magnitude(coord, coord_gb, rateints, jump_data, jump_head, nints)) - ->>>>>>> Stashed changes - return mags + + return mags def get_cr_rate(jump_locs, t): """ - Computes the rate of cosmic ray impacts in a given time. - - Parameters: - ---------- - jump_locs: list + Computes the rate of cosmic ray impacts in a given time. + + Parameters: + ---------- + jump_locs: list List of coordinates to a pixel marked with a jump. - t: int or float - Time over which to compute the rate. + t: int or float + Time over which to compute the rate. Nominally the effective exposure time. - - """ + + """ return len(jump_locs)/t - + def get_jump_data(jump_filename): """ - Opens and reads a given .FITS file containing cosmic rays. - - Parameters: - ---------- - jump_filename: str + Opens and reads a given .FITS file containing cosmic rays. + + Parameters: + ---------- + jump_filename: str Path to file. - Returns: - ------- - head: FITS header + Returns: + ------- + head: FITS header Header containing file information - - data: NoneType + + data: NoneType FITS data - - dq: ndarray + + dq: ndarray Data Quality array containing jump flags. - """ + """ hdu = fits.open(jump_filename) @@ -390,108 +274,72 @@ def get_jump_data(jump_filename): hdu.close() - return head, data, dq + return head, data, dq -<<<<<<< Updated upstream -def get_jump_locs(dq): -======= def get_jump_locs(dq,nints): ->>>>>>> Stashed changes """ - Uses the data quality array to find the location of all jumps in the data. - - Parameters: - ---------- - dq: ndarray + Uses the data quality array to find the location of all jumps in the data. + + Parameters: + ---------- + dq: ndarray Data Quality array containing jump flags. - Returns: - ------- - jump_locs: list + Returns: + ------- + jump_locs: list List of coordinates to a pixel marked with a jump. - """ + """ temp = np.where(dq==4) jump_locs = [] -<<<<<<< Updated upstream - for i in range(len(temp[0])): - jump_locs.append((temp[0][i],temp[1][i],temp[2][i],temp[3][i])) - - return jump_locs - -def group_before(jump_locs): - """ - Creates a list of coordinates one group before given jump coordinates. - - Parameters: - ---------- - jump_locs: list - List of coordinates to a pixel marked with a jump. - - Returns: - ------- - jump_locs_pre: list - List of matching coordinates one group before jump_locs. - """ - jump_locs_pre = [] - for coord in jump_locs: - jump_locs_pre.append((coord[0],coord[1]-1,coord[2],coord[3])) - - return jump_locs_pre -======= - if nints > 1: for i in range(len(temp[0])): jump_locs.append((temp[0][i],temp[1][i],temp[2][i],temp[3][i])) else: for i in range(len(temp[0])): jump_locs.append((temp[0][i],temp[1][i],temp[2][i])) - - return jump_locs ->>>>>>> Stashed changes + +return jump_locs def get_rate_data(rate_filename): """ - Opens and reads a given .FITS file. - - Parameters: - ---------- - rate_filename: str + Opens and reads a given .FITS file. + + Parameters: + ---------- + rate_filename: str Path to file. - Returns: - ------- - data: NoneType + Returns: + ------- + data: NoneType FITS data - """ + """ data = fits.getdata(rate_filename) return data -<<<<<<< Updated upstream -def magnitude(coord, coord_gb, rateints, data, head): -======= - def group_before(jump_locs, nints): """ - Creates a list of coordinates one group before given jump coordinates. - - Parameters: - ---------- - jump_locs: list + Creates a list of coordinates one group before given jump coordinates. + + Parameters: + ---------- + jump_locs: list List of coordinates to a pixel marked with a jump. - - Returns: - ------- - jump_locs_pre: list + + Returns: + ------- + jump_locs_pre: list List of matching coordinates one group before jump_locs. - - """ + + """ jump_locs_pre = [] - + if nints > 1: for coord in jump_locs: jump_locs_pre.append((coord[0], coord[1] - 1, coord[2], coord[3])) @@ -499,130 +347,121 @@ def group_before(jump_locs, nints): for coord in jump_locs: jump_locs_pre.append((coord[0] - 1, coord[1], coord[2])) - return jump_locs_pre +return jump_locs_pre def magnitude(coord, coord_gb, rateints, data, head, nints): ->>>>>>> Stashed changes """ - Calculates the magnitude of a list of jumps given their coordinates - in an array of pixels. - - Parameters: - ---------- - coord: tuple + Calculates the magnitude of a list of jumps given their coordinates + in an array of pixels. + + Parameters: + ---------- + coord: tuple Coordinate of jump. - - coord_gb: tuple + + coord_gb: tuple Coordinate of jump pixel one group before. - - head: FITS header + + head: FITS header Header containing file information. - rateints: ndarray - Array in DN/s. + rateints: ndarray + Array in DN/s. - Returns: - ------- - cr_mag: - - """ -<<<<<<< Updated upstream + Returns: + ------- + cr_mag: + + """ - rate = rateints[coord[0]][coord[2]][coord[3]] grouptime = head['TGROUP'] - cr_mag = data[coord] - data[coord_gb] - rate*grouptime -======= - - grouptime = head['TGROUP'] - if nints == 1: rate = rateints[coord[-2]][coord[-1]] cr_mag = data[0][coord[0]][coord[1]][coord[2]] \ - - data[0][coord_gb[0]][coord_gb[1]][coord_gb[2]] \ - - rate * grouptime + - data[0][coord_gb[0]][coord_gb[1]][coord_gb[2]] \ + - rate * grouptime else: rate = rateints[coord[0]][coord[-2]][coord[-1]] cr_mag = data[coord] - data[coord_gb] - rate * grouptime - ->>>>>>> Stashed changes + return cr_mag def most_recent_search(aperture, query_table): """Adapted from Dark Monitor (Bryan Hilbert) - - Query the query history database and return the information - on the most recent query for the given ``aperture_name`` where - the cosmic ray monitor was executed. - - Returns: - ------- - query_result : float + + Query the query history database and return the information + on the most recent query for the given ``aperture_name`` where + the cosmic ray monitor was executed. + + Returns: + ------- + query_result : float Date (in MJD) of the ending range of the previous MAST query where the cosmic ray monitor was run. - """ + """ sub_query = session.query(query_table.aperture, func.max(query_table.end_time_mjd).label('maxdate') ).group_by(query_table.aperture).subquery('t2') - - # Note that "self.query_table.run_monitor == True" below is - # intentional. Switching = to "is" results in an error in the query. - query = session.query(query_table).join( - sub_query, - and_( - query_table.aperture == aperture, - query_table.end_time_mjd == sub_query.c.maxdate, - query_table.run_monitor == True - ) - ).all() - - query_count = len(query) - if query_count == 0: - query_result = 57357.0 # a.k.a. Dec 1, 2015 == CV3 - logging.info(('\tNo query history for {}. Beginning search date will be set to {}.' - .format(aperture, query_result))) - elif query_count > 1: - raise ValueError('More than one "most recent" query?') - else: - query_result = query[0].end_time_mjd - - return query_result + + # Note that "self.query_table.run_monitor == True" below is + # intentional. Switching = to "is" results in an error in the query. + query = session.query(query_table).join( + sub_query, + and_( + query_table.aperture == aperture, + query_table.end_time_mjd == sub_query.c.maxdate, + query_table.run_monitor == True + ) + ).all() + + query_count = len(query) + if query_count == 0: + query_result = 57357.0 # a.k.a. Dec 1, 2015 == CV3 + logging.info(('\tNo query history for {}. Beginning search date will be set to {}.' + .format(aperture, query_result))) + elif query_count > 1: + raise ValueError('More than one "most recent" query?') + else: + query_result = query[0].end_time_mjd + + return query_result def query_mast(start_date, end_date): """ - Use astroquery to search MAST for cosmic ray data - - Parameters: - ---------- - start_date : float + Use astroquery to search MAST for cosmic ray data + + Parameters: + ---------- + start_date : float Starting date for the search in MJD - end_date : float + end_date : float Ending date for the search in MJD - - Returns - ------- - result : list + + Returns + ------- + result : list List of dictionaries containing the query results - - """ + + """ instrument = 'MIRI' dataproduct = JWST_DATAPRODUCTS parameters = {"date_obs_mjd": {"min": start_date, "max": end_date}} - - result = monitor_mast.instrument_inventory(instrument, dataproduct, - add_filters=parameters, - return_data=True) + result = monitor_mast.instrument_inventory(instrument, dataproduct, + add_filters=parameters, + return_data=True) + return result if __name__ == '__main__': - + # Configure logging module = os.path.basename(__file__).strip('.py') configure_logging(module) - + # Call the main function cosmic_ray_monitor() From b17ae992b82ba7c1aae18de01216bd3257201fbb Mon Sep 17 00:00:00 2001 From: Michael Engesser Date: Mon, 19 Oct 2020 18:36:45 -0400 Subject: [PATCH 0006/1618] Update test_cosmic_ray_monitor.py Fixing PEP8 conflicts --- jwql/tests/test_cosmic_ray_monitor.py | 79 +++++++++++++-------------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/jwql/tests/test_cosmic_ray_monitor.py b/jwql/tests/test_cosmic_ray_monitor.py index f55a138d7..d218abae4 100644 --- a/jwql/tests/test_cosmic_ray_monitor.py +++ b/jwql/tests/test_cosmic_ray_monitor.py @@ -7,25 +7,24 @@ def define_test_data(nints): if nints == 1: - data = np.ones((2,5,10,10)) - rate_data = np.ones((10,10)) + data = np.ones((2, 5, 10, 10)) + rate_data = np.ones((10, 10)) else: - data = np.ones((2,5,10,10)) - rate_data = np.ones((2,10,10)) - + data = np.ones((2, 5, 10, 10)) + rate_data = np.ones((2, 10, 10)) file = "jw00000000000_00000_00000_MIRIMAGE_uncal.fits" aperture = "MIRIM_FULL" - return data, rate_data, file, aperture +return data, rate_data, file, aperture def test_get_jump_data(): _ = define_test_data(2) file = _[2] - + head, data, dq = crm.get_jump_data(file) - + assert type(head) == fits.header.Header assert type(data) == np.ndarray assert type(dq) == np.ndarray @@ -34,88 +33,88 @@ def test_get_jump_data(): def test_get_rate_data(): _ = define_test_data(1) file = _[2] - + data = crm.get_rate_data(file) - + assert type(data) == np.ndarray def test_get_cr_rate(): jump_locs = np.arange(100).tolist() t = 5 - + rate = crm.get_cr_rate(jump_locs, t) - + assert rate == 20.0 def test_group_before(): - jump_locs = [(2,1,1)] + jump_locs = [(2, 1, 1)] nints = 1 - - assert crm.group_before(jump_locs, nints) == [(1,1,1)] - - jump_locs = [(1,2,1,1)] + + assert crm.group_before(jump_locs, nints) == [(1, 1, 1)] + + jump_locs = [(1, 2, 1, 1)] nints = 2 - - assert crm.group_before(jump_locs, nints) == [(1,1,1,1)] + + assert crm.group_before(jump_locs, nints) == [(1, 1, 1, 1)] def test_magnitude(): - + nints = 5 data, rate_data, file, aperture = define_test_data(nints) head = fits.getheader(file) - coord = (1,2,1,1) - coord_gb = (1,1,1,1) + coord = (1, 2, 1, 1) + coord_gb = (1, 1, 1, 1) mag = crm.magnitude(coord, coord_gb, rate_data, data, head, nints) assert mag == -2.77504 - + nints = 1 data, rate_data, file, aperture = define_test_data(nints) - coord = (1,1,1) - coord_gb = (0,1,1) + coord = (1, 1, 1) + coord_gb = (0, 1, 1) mag = crm.magnitude(coord, coord_gb, rate_data, data, head, nints) assert mag == -2.77504 def test_get_cr_mags(): - - jump_locs = [(2,1,1)] - jump_locs_pre = [(1,1,1)] + + jump_locs = [(2, 1, 1)] + jump_locs_pre = [(1, 1, 1)] nints = 1 data, rate_data, file, aperture = define_test_data(nints) head = fits.getheader(file) - + mags = crm.get_cr_mags(jump_locs, jump_locs_pre, rate_data, data, head, nints) assert mags == [-2.77504] - - jump_locs = [(1,2,1,1)] - jump_locs_pre = [(1,1,1,1)] + + jump_locs = [(1, 2, 1, 1)] + jump_locs_pre = [(1, 1, 1, 1)] nints = 5 data, rate_data, file, aperture = define_test_data(nints) - + mags = crm.get_cr_mags(jump_locs, jump_locs_pre, rate_data, data, head, nints) assert mags == [-2.77504] def test_most_recent_search(): - + _ = define_test_data(1) aperture = _[3] - + query_table = MIRICosmicRayQueryHistory - + result = crm.most_recent_search(aperture,query_table) - + assert result == 57357.0 def test_query_mast(): - + start_date = 57357.0 end_date = 57405.0 - + result = crm.query_mast(start_date, end_date) - + assert len(result) == 5 From a92462d5d479803b4b7b9d70c04414d5cae77496 Mon Sep 17 00:00:00 2001 From: Michael Engesser Date: Mon, 19 Oct 2020 18:37:22 -0400 Subject: [PATCH 0007/1618] Update database_interface.py Fixing merge conflicts --- jwql/database/database_interface.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py index 44b6fd8ff..65ce8d6e3 100755 --- a/jwql/database/database_interface.py +++ b/jwql/database/database_interface.py @@ -423,13 +423,8 @@ class : obj NIRCamReadnoiseStats = monitor_orm_factory('nircam_readnoise_stats') NIRISSReadnoiseQueryHistory = monitor_orm_factory('niriss_readnoise_query_history') NIRISSReadnoiseStats = monitor_orm_factory('niriss_readnoise_stats') -<<<<<<< Updated upstream -MIRICosmicRayStats = monitor_orm_factory('miri_cosmic_ray_stats') -MIRICosmicRayQueryHistory = monitor_orm_factory('miri_cosmic_ray_query_history') -======= MIRICosmicRayQueryHistory = monitor_orm_factory('miri_cosmic_ray_query_history') MIRICosmicRayStats = monitor_orm_factory('miri_cosmic_ray_stats') ->>>>>>> Stashed changes if __name__ == '__main__': From b78f82c426cd52899c62115b233b7f976c476696 Mon Sep 17 00:00:00 2001 From: Michael Engesser Date: Mon, 19 Oct 2020 18:39:57 -0400 Subject: [PATCH 0008/1618] Update test_cosmic_ray_monitor.py Adding module docstring --- jwql/tests/test_cosmic_ray_monitor.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/jwql/tests/test_cosmic_ray_monitor.py b/jwql/tests/test_cosmic_ray_monitor.py index d218abae4..cd9cc0e5b 100644 --- a/jwql/tests/test_cosmic_ray_monitor.py +++ b/jwql/tests/test_cosmic_ray_monitor.py @@ -1,3 +1,22 @@ +#! /usr/bin/env python + +"""Tests for the cosmic ray monitor module. + + Authors + ------- + + - Mike Engesser + + Use + --- + + These tests can be run via the command line (omit the ``-s`` to + suppress verbose output to stdout): + :: + + pytest -s test_cosmic_ray_monitor.py + """ + import pytest import cosmic_ray_monitor as crm import numpy as np From 8ea3e195e9499c7f9a3ee4109b38f5e09d1acb75 Mon Sep 17 00:00:00 2001 From: Michael Engesser Date: Thu, 21 Jan 2021 12:37:21 -0500 Subject: [PATCH 0009/1618] Addressing PR comments Making fixes requested by @bourque --- .../miri_monitors/cosmic_ray_monitor.py | 840 ++++++++++-------- jwql/tests/test_cosmic_ray_monitor.py | 33 +- jwql/website/apps/jwql/views.py | 12 +- 3 files changed, 492 insertions(+), 393 deletions(-) diff --git a/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py b/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py index b06c9e099..d676050ff 100644 --- a/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py +++ b/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py @@ -1,467 +1,563 @@ #! /usr/bin/env python -"""This module contains code for the cosmic ray monitor, which currently checks - the number and magnitude of jumps in all observations performed with MIRI. - - The code first checks MAST for any new MIRI observations that have not yet been - run through the monitor. It then copies those files to a working directory, - where they are run through the pipeline, and for which the output is stored - in a new directory for each observation. - - Each observation is then analyzed for jumps due to cosmic rays, of which the number - and magnitude are recorded. This information is then inserted into the - "MiriCosmicRayStats" database table. - - - Author - ------ - - -Mike Engesser - - Use - --- - +"""This module runs the Cosmic Ray Monitor. + +This module contains code for the cosmic ray monitor, which currently +checks the number and magnitude of jumps in all observations performed +with MIRI and NIRCam. The code first checks MAST for any new MIRI or +NIRCam observations that have not yet been run through the monitor. It +then copies those files to a working directory, where they are run +through the pipeline, and for which the output is stored in a new +directory for each observation. Each observation is then analyzed for +jumps due to cosmic rays, of which the number and magnitude are +recorded. This information is then inserted into the stats database +table. + +Authors +------- + + - Mike Engesser + +Use +--- + This module can be used from the command line as such: - - :: - - python cosmic_ray_monitor.py - """ + :: + python cosmic_ray_monitor.py +""" -# Functions for logging +# Native Imports +import datetime import logging +import os +import shutil + +# Third-Party Imports +from astropy.io import fits +from astropy.time import Time +import julian +import numpy as np +from pysiaf import Siaf +from sqlalchemy import func +from sqlalchemy.sql.expression import and_ + +# Local imports +from jwql.database.database_interface import MIRICosmicRayQueryHistory +from jwql.database.database_interface import MIRICosmicRayStats +from jwql.database.database_interface import NIRCamCosmicRayQueryHistory +from jwql.database.database_interface import NIRCamCosmicRayStats +from jwql.database.database_interface import session +from jwql.instrument_monitors import pipeline_tools +from jwql.jwql_monitors import monitor_mast +from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_DATAPRODUCTS from jwql.utils.logging_functions import configure_logging from jwql.utils.logging_functions import log_info from jwql.utils.logging_functions import log_fail +from jwql.utils.utils import copy_files, ensure_dir_exists, get_config -# Funtions for database interaction -from jwql.jwql_monitors import monitor_mast -from jwql.utils.constants import JWST_DATAPRODUCTS -from jwql.database.database_interface import session -from jwql.database.database_interface import MIRICosmicRayQueryHistory -from jwql.database.database_interface import MIRICosmicRayStats -from sqlalchemy import func -from sqlalchemy.sql.expression import and_ +class CosmicRay: + """Class for executing the cosmic ray monitor. -# Functions for file manipulation -from jwql.utils.utils import copy_files, ensure_dir_exists, get_config + This class will search for new (since the previous instance of the + class) MIRI and NIRCam data in the file system. It will loop over + instrument/aperture combinations and find the number of new files + available. It will copy the files over to a working directory and + run the monitor. This will count the number and magnitude of all + cosmic rays in each new exposure. Results are all saved to + database tables. -# Astropy functions -from astropy.io import fits -from astropy.time import Time + Attributes + ---------- + output_dir : str + Path into which outputs will be placed -# Utility functions -from jwql.instrument_monitors import pipeline_tools -import os -import numpy as np -import shutil -import julian -import datetime + data_dir : str + Path into which new files will be copied to be worked on + + query_start : float + MJD start date to use for querying MAST + + query_end : float + MJD end date to use for querying MAST + + instrument : str + Name of instrument used to collect the dark current data + + aperture : str + Name of the aperture used for the dark current (e.g. + ``NRCA1_FULL``) -@log_fail -@log_info -def cosmic_ray_monitor(): + query_table : sqlalchemy table + Table containing the history of cosmic ray monitor queries to MAST + for each instrument/aperture combination + + stats_table : sqlalchemy table + Table containing cosmic ray analysis results. Number and + magnitude of cosmic rays, etc. + + Raises + ------ + ValueError + If encountering a file not following the JWST file naming + convention + + ValueError + If the most recent query search returns more than one entry """ - The main function of the ``monitor_template`` module. - Queries MAST for new MIRI data and copies it to a working - directory where it is run through the JWST pipeline. The output of the 'jump' - and 'rate' steps is used to determine the number and magnitudes of cosmic rays - which is then saved to the database. + + def __init__(self): + """Initialize an instance of the ``Cosmic_Ray`` class.""" + + def identify_tables(self): + """Determine which database tables to use for a run of the + cosmic ray monitor. + + Uses the instrument variable to get the mixed-case instrument + name, and uses that name to find the query and stats tables + for that instrument. """ - - logging.info('Begin logging for cosmic_ray_monitor') - - aperture = 'MIRIM_FULL' - query_table = MIRICosmicRayQueryHistory - stats_table = MIRICosmicRayStats - - #We start by querying MAST for new data - start_date = most_recent_search(aperture, query_table) - end_date = Time.now().mjd - logging.info('\tMost recent query: {}'.format(start_date)) - - new_entries = query_mast(start_date, end_date) - - #for testing purposes only - new_filenames = get_config()['local_test_data'] - - #for file_entry in new_entries['data']: - # try: - # new_filenames.append(filesystem_path(file_entry['filename'])) - # except FileNotFoundError: - # logging.info('\t{} not found in target directory'.format(file_entry['filename'])) - # except ValueError: - # logging.info( - # '\tProvided file {} does not follow JWST naming conventions.'.format(file_entry['filename'])) - - #Next we copy new files to the working directory - output_dir = os.path.join(get_config()['outputs'], 'cosmic_ray_monitor') - - data_dir = get_config()['local_test_dir'] #for testing purposes only - #data_dir = os.path.join(output_dir,'data') - - ensure_dir_exists(data_dir) - cosmic_ray_files, not_copied = copy_files(new_filenames, data_dir) - - for file_name in cosmic_ray_files: - if 'uncal' in file_name: - dir_name = file_name[51:76] #Gets the observation identifier. Should be constant? - obs_dir = os.path.join(data_dir, dir_name) - ensure_dir_exists(obs_dir) - - head = fits.getheader(file_name) - nints = head['NINTS'] - - try: - shutil.copy2(file_name, obs_dir) - except: - logging.info('Failed to copy {} to observation dir.'.format(file_name)) - - #Next we run the pipeline on the files to get the proper outputs - MIRI_steps = pipeline_tools.get_pipeline_steps('MIRI') - - file = os.path.join(obs_dir, os.path.basename(file_name)) - - completed_steps = pipeline_tools.completed_pipeline_steps(file) - steps = pipeline_tools.steps_to_run(MIRI_steps,completed_steps) - - try: - pipeline_tools.calwebb_detector1_save_jump(file, obs_dir, ramp_fit=True, save_fitopt=False) - except: - logging.info('Failed to complete pipeline steps on {}.'.format(file)) - - - #Next we analyze the cosmic rays in the new data - for output_file in os.listdir(obs_dir): - - if 'jump' in output_file: - jump_file = os.path.join(obs_dir,output_file) - - if 'rateint' in output_file: - rate_file = os.path.join(obs_dir,output_file) - - if nints == 1: - if '0_ramp_fit' in output_file: - rate_file = os.path.join(obs_dir, output_file) - - elif nints > 1: - if '1_ramp_fit' in output_file: - rate_file = os.path.join(obs_dir,output_file) - - try: - jump_head, jump_data, jump_dq = get_jump_data(jump_file) - except: - logging.info('Could not open jump file: {}'.format(jump_file)) - - try: - rate_data = get_rate_data(rate_file) - except: - logging.info('Could not open rate file: {}'.format(rate_file)) - - - jump_locs = get_jump_locs(jump_dq,nints) - jump_locs_pre = group_before(jump_locs,nints) - - eff_time = jump_head['EFFEXPTM'] - - # Get observation time info - obs_start_time = jump_head['EXPSTART'] - obs_end_time = jump_head['EXPEND'] - start_time = julian.from_jd(obs_start_time, fmt='mjd') - end_time = julian.from_jd(obs_end_time, fmt='mjd') - - cosmic_ray_num = len(jump_locs) - #cosmic_ray_rate = get_cr_rate(jump_locs, eff_time) - cosmic_ray_mags = get_cr_mags(jump_locs, jump_locs_pre, rate_data, jump_data, jump_head, nints) - - # Insert new data into database - try: - cosmic_ray_db_entry = {'entry_date': datetime.datetime.now(), - 'aperture': aperture, - 'source_file': file_name, - 'obs_start_time': start_time, - 'obs_end_time': end_time, - 'jump_count': cosmic_ray_num, - 'magnitude': cosmic_ray_mags - } - stats_table.__table__.insert().execute(cosmic_ray_db_entry) - - logging.info("Successfully inserted into database. \n") - except: - logging.info("Could not insert entry into database. \n") - -def get_cr_mags(jump_locs, jump_locs_pre, rateints, jump_data, jump_head, nints): - """ - Computes a list of magnitudes using the coordinate of the detected jump compared to - the magnitude of the same pixel in the group prior to the jump. - + + mixed_case_name = JWST_INSTRUMENT_NAMES_MIXEDCASE[self.instrument] + self.query_table = eval('{}CosmicRayQueryHistory'.format(mixed_case_name)) + self.stats_table = eval('{}CosmicRayStats'.format(mixed_case_name)) + + def get_cr_mags(self, jump_locs, jump_locs_pre, rateints, jump_data, jump_head): + """Gets the magnitude of each cosmic ray. + + Computes a list of magnitudes using the coordinate of the + detected jump compared to the magnitude of the same pixel in + the group prior to the jump. + Parameters: ---------- - + jump_locs: list - List of coordinates to a pixel marked with a jump. - + List of coordinates to a pixel marked with a jump. + jump_locs_pre: list - List of matching coordinates one group before jump_locs. - + List of matching coordinates one group before jump_locs. + head: FITS header - Header containing file information. - + Header containing file information. + rateints: ndarray - Array in DN/s. - + Array in DN/s. + Returns: ------- - + mags: list - A list of cosmic ray magnitudes corresponding to each jump. - + A list of cosmic ray magnitudes corresponding to each jump. + """ - - mags = [] - for coord, coord_gb in zip(jump_locs, jump_locs_pre): - mags.append(magnitude(coord, coord_gb, rateints, jump_data, jump_head, nints)) - - return mags - -def get_cr_rate(jump_locs, t): - """ - Computes the rate of cosmic ray impacts in a given time. - + + mags = [] + for coord, coord_gb in zip(jump_locs, jump_locs_pre): + mags.append(self.magnitude(coord, coord_gb, rateints, jump_data, jump_head)) + + return mags + + def get_cr_rate(self, jump_locs, time): + """Computes the rate of cosmic ray impacts in a given time. + Parameters: ---------- jump_locs: list - List of coordinates to a pixel marked with a jump. - + List of coordinates to a pixel marked with a jump. + t: int or float - Time over which to compute the rate. - Nominally the effective exposure time. - + Time over which to compute the rate. + Nominally the effective exposure time. + """ - - return len(jump_locs)/t -def get_jump_data(jump_filename): - """ - Opens and reads a given .FITS file containing cosmic rays. - + return len(jump_locs) / time + + def get_jump_data(self, jump_filename): + """Opens and reads a given .FITS file containing cosmic rays. + Parameters: ---------- jump_filename: str - Path to file. - + Path to file. + Returns: ------- head: FITS header - Header containing file information - + Header containing file information + data: NoneType - FITS data - + FITS data + dq: ndarray - Data Quality array containing jump flags. - + Data Quality array containing jump flags. + """ - - hdu = fits.open(jump_filename) - - head = hdu[0].header - data = hdu[1].data - - dq = hdu[3].data - - hdu.close() - - return head, data, dq - -def get_jump_locs(dq,nints): - """ - Uses the data quality array to find the location of all jumps in the data. - + + hdu = fits.open(jump_filename) + + head = hdu[0].header + data = hdu[1].data + + dq = hdu[3].data + + hdu.close() + + return head, data, dq + + def get_jump_locs(self, dq): + """Uses the data quality array to find the location of all + jumps in the data. + Parameters: ---------- dq: ndarray - Data Quality array containing jump flags. - + Data Quality array containing jump flags. + Returns: ------- jump_locs: list - List of coordinates to a pixel marked with a jump. + List of coordinates to a pixel marked with a jump. """ - - temp = np.where(dq==4) - - jump_locs = [] - - if nints > 1: - for i in range(len(temp[0])): - jump_locs.append((temp[0][i],temp[1][i],temp[2][i],temp[3][i])) - else: - for i in range(len(temp[0])): - jump_locs.append((temp[0][i],temp[1][i],temp[2][i])) - -return jump_locs - -def get_rate_data(rate_filename): - """ - Opens and reads a given .FITS file. - + + temp = np.where(dq == 4) + + jump_locs = [] + + if self.nints > 1: + for i in range(len(temp[0])): + jump_locs.append((temp[0][i], temp[1][i], temp[2][i], temp[3][i])) + else: + for i in range(len(temp[0])): + jump_locs.append((temp[0][i], temp[1][i], temp[2][i])) + + return jump_locs + + def get_rate_data(self, rate_filename): + """Opens and reads a given .FITS file. + Parameters: ---------- rate_filename: str - Path to file. - + Path to file. + Returns: ------- data: NoneType - FITS data + FITS data """ - - data = fits.getdata(rate_filename) - - return data -def group_before(jump_locs, nints): - """ - Creates a list of coordinates one group before given jump coordinates. - + data = fits.getdata(rate_filename) + + return data + + def group_before(self, jump_locs): + """Creates a list of coordinates one group before given jump + coordinates. + Parameters: ---------- jump_locs: list - List of coordinates to a pixel marked with a jump. - + List of coordinates to a pixel marked with a jump. + Returns: ------- jump_locs_pre: list - List of matching coordinates one group before jump_locs. - + List of matching coordinates one group before jump_locs. """ - jump_locs_pre = [] - - if nints > 1: - for coord in jump_locs: - jump_locs_pre.append((coord[0], coord[1] - 1, coord[2], coord[3])) - else: - for coord in jump_locs: - jump_locs_pre.append((coord[0] - 1, coord[1], coord[2])) - -return jump_locs_pre - -def magnitude(coord, coord_gb, rateints, data, head, nints): - """ - Calculates the magnitude of a list of jumps given their coordinates - in an array of pixels. - + + jump_locs_pre = [] + + if self.nints > 1: + for coord in jump_locs: + jump_locs_pre.append((coord[0], coord[1] - 1, coord[2], coord[3])) + else: + for coord in jump_locs: + jump_locs_pre.append((coord[0] - 1, coord[1], coord[2])) + + return jump_locs_pre + + def magnitude(self, coord, coord_gb, rateints, data, head): + """Calculates the magnitude of a list of jumps given their + coordinates in an array of pixels. + Parameters: ---------- coord: tuple - Coordinate of jump. - + Coordinate of jump. + coord_gb: tuple - Coordinate of jump pixel one group before. - + Coordinate of jump pixel one group before. + head: FITS header - Header containing file information. - + Header containing file information. + rateints: ndarray - Array in DN/s. - + Array in DN/s. + Returns: ------- - cr_mag: - + cr_mag: float + the magnitude of the cosmic ray """ - - grouptime = head['TGROUP'] - - if nints == 1: - rate = rateints[coord[-2]][coord[-1]] - cr_mag = data[0][coord[0]][coord[1]][coord[2]] \ - - data[0][coord_gb[0]][coord_gb[1]][coord_gb[2]] \ - - rate * grouptime - else: - rate = rateints[coord[0]][coord[-2]][coord[-1]] - cr_mag = data[coord] - data[coord_gb] - rate * grouptime - - return cr_mag - -def most_recent_search(aperture, query_table): - """Adapted from Dark Monitor (Bryan Hilbert) - + + grouptime = head['TGROUP'] + + if self.nints == 1: + rate = rateints[coord[-2]][coord[-1]] + cr_mag = data[0][coord[0]][coord[1]][coord[2]] \ + - data[0][coord_gb[0]][coord_gb[1]][coord_gb[2]] \ + - rate * grouptime + + else: + rate = rateints[coord[0]][coord[-2]][coord[-1]] + cr_mag = data[coord] - data[coord_gb] - rate * grouptime + + return cr_mag + + + def most_recent_search(self): + """Adapted from Dark Monitor (Bryan Hilbert) + Query the query history database and return the information on the most recent query for the given ``aperture_name`` where the cosmic ray monitor was executed. - + Returns: ------- query_result : float - Date (in MJD) of the ending range of the previous MAST query - where the cosmic ray monitor was run. + Date (in MJD) of the ending range of the previous MAST + query where the cosmic ray monitor was run. """ - - sub_query = session.query(query_table.aperture, - func.max(query_table.end_time_mjd).label('maxdate') - ).group_by(query_table.aperture).subquery('t2') - - # Note that "self.query_table.run_monitor == True" below is - # intentional. Switching = to "is" results in an error in the query. - query = session.query(query_table).join( - sub_query, - and_( - query_table.aperture == aperture, - query_table.end_time_mjd == sub_query.c.maxdate, - query_table.run_monitor == True - ) - ).all() - - query_count = len(query) - if query_count == 0: - query_result = 57357.0 # a.k.a. Dec 1, 2015 == CV3 - logging.info(('\tNo query history for {}. Beginning search date will be set to {}.' - .format(aperture, query_result))) - elif query_count > 1: - raise ValueError('More than one "most recent" query?') - else: - query_result = query[0].end_time_mjd - - return query_result - -def query_mast(start_date, end_date): - """ - Use astroquery to search MAST for cosmic ray data - + + sub_query = session.query(self.query_table.aperture, + func.max(self.query_table.end_time_mjd).label('maxdate') + ).group_by(self.query_table.aperture).subquery('t2') + + # Note that "self.query_table.run_monitor == True" below is + # intentional. Switching = to "is" results in an error in the query. + query = session.query(self.query_table).join( + sub_query, + and_( + self.query_table.aperture == self.aperture, + self.query_table.end_time_mjd == sub_query.c.maxdate, + self.query_table.run_monitor == True + ) + ).all() + + query_count = len(query) + if query_count == 0: + query_result = 57357.0 # a.k.a. Dec 1, 2015 == CV3 + logging.info(('\tNo query history for {}. Beginning search date will be set to {}.' + .format(self.aperture, query_result))) + #elif query_count > 1: + # raise ValueError('More than one "most recent" query?') + else: + query_result = query[0].end_time_mjd + + return query_result + + def process(self, file_list): + """The main method for processing files. See module docstrings + for further details. + + Parameters + ---------- + file_list : list + List of filenames (including full paths) to the cosmic ray + files + """ + + for file_name in file_list: + if 'uncal' in file_name: + dir_name = '_'.join(file_name.split('_')[:4]) # file_name[51:76] + + self.obs_dir = os.path.join(self.data_dir, dir_name) + ensure_dir_exists(self.obs_dir) + + head = fits.getheader(file_name) + self.nints = head['NINTS'] + + try: + copy_files([file_name], self.obs_dir) + except: + logging.info('Failed to copy {} to observation dir.'.format(file_name)) + + # Next we run the pipeline on the files to get the proper outputs + uncal_file = os.path.join(self.obs_dir, os.path.basename(file_name)) + + try: + pipeline_tools.calwebb_detector1_save_jump(uncal_file, self.obs_dir, ramp_fit=True, save_fitopt=False) + except: + logging.info('Failed to complete pipeline steps on {}.'.format(uncal_file)) + + # Next we analyze the cosmic rays in the new data + for output_file in os.listdir(self.obs_dir): + + if 'jump' in output_file: + jump_file = os.path.join(self.obs_dir, output_file) + + if self.nints == 1: + if '0_ramp_fit' in output_file: + rate_file = os.path.join(self.obs_dir, output_file) + + elif self.nints > 1: + if '1_ramp_fit' in output_file: + rate_file = os.path.join(self.obs_dir, output_file) + + try: + jump_head, jump_data, jump_dq = self.get_jump_data(jump_file) + except: + logging.info('Could not open jump file: {}'.format(jump_file)) + + try: + rate_data = self.get_rate_data(rate_file) + except: + logging.info('Could not open rate file: {}'.format(rate_file)) + + jump_locs = self.get_jump_locs(jump_dq) + jump_locs_pre = self.group_before(jump_locs) + + eff_time = jump_head['EFFEXPTM'] + + # Get observation time info + obs_start_time = jump_head['EXPSTART'] + obs_end_time = jump_head['EXPEND'] + start_time = julian.from_jd(obs_start_time, fmt='mjd') + end_time = julian.from_jd(obs_end_time, fmt='mjd') + + cosmic_ray_num = len(jump_locs) + # cosmic_ray_rate = get_cr_rate(jump_locs, eff_time) + cosmic_ray_mags = self.get_cr_mags(jump_locs, jump_locs_pre, rate_data, jump_data, jump_head) + + # Insert new data into database + try: + cosmic_ray_db_entry = {'entry_date': datetime.datetime.now(), + 'aperture': self.aperture, + 'source_file': file_name, + 'obs_start_time': start_time, + 'obs_end_time': end_time, + 'jump_count': cosmic_ray_num, + 'magnitude': cosmic_ray_mags + } + self.stats_table.__table__.insert().execute(cosmic_ray_db_entry) + + logging.info("Successfully inserted into database. \n") + except: + logging.info("Could not insert entry into database. \n") + + @log_fail + @log_info + def run(self): + """The main method. See module docstrings for additional info + + Queries MAST for new MIRI data and copies it to a working + directory where it is run through the JWST pipeline. The output + of the 'jump' and 'rate' steps is used to determine the number + and magnitudes of cosmic rays which is then saved to the + database. + """ + + logging.info('Begin logging for cosmic_ray_monitor') + + self.query_end = Time.now().mjd + + for instrument in JWST_INSTRUMENT_NAMES: + if instrument == 'miri' or instrument == 'nircam': + self.instrument = instrument + + # Identify which tables to use + self.identify_tables() + + # Get a list of possible apertures + possible_apertures = list(Siaf(instrument).apernames) + + # Use this line instead to save time while testing + #possible_apertures = ['MIRIM_FULL', 'NRCB4_FULL'] + + for aperture in possible_apertures: + + logging.info('') + logging.info('Working on aperture {} in {}'.format(aperture, instrument)) + + self.aperture = aperture + + # We start by querying MAST for new data + self.query_start = self.most_recent_search() + + logging.info('\tMost recent query: {}'.format(self.query_start)) + + new_entries = self.query_mast() + + # for testing purposes only + new_filenames = get_config()['local_test_data'] + + # for file_entry in new_entries['data']: + # try: + # new_filenames.append(filesystem_path(file_entry['filename'])) + # except FileNotFoundError: + # logging.info('\t{} not found in target directory'.format(file_entry['filename'])) + # except ValueError: + # logging.info( + # '\tProvided file {} does not follow JWST naming conventions.'.format(file_entry['filename'])) + + # Next we copy new files to the working directory + output_dir = os.path.join(get_config()['outputs'], 'cosmic_ray_monitor') + + self.data_dir = get_config()['local_test_dir'] # for testing purposes only + + # self.data_dir = os.path.join(output_dir,'data') + ensure_dir_exists(self.data_dir) + + cosmic_ray_files, not_copied = copy_files(new_filenames, self.data_dir) + + self.process(cosmic_ray_files) + + monitor_run = True + + new_entry = {'instrument': self.instrument, + 'aperture': self.aperture, + 'start_time_mjd': self.query_start, + 'end_time_mjd': self.query_end, + 'files_found': len(new_entries), + 'run_monitor': monitor_run, + 'entry_date': datetime.datetime.now()} + self.query_table.__table__.insert().execute(new_entry) + logging.info('\tUpdated the query history table') + + def query_mast(self): + """Use astroquery to search MAST for cosmic ray data + Parameters: ---------- start_date : float - Starting date for the search in MJD + Starting date for the search in MJD end_date : float - Ending date for the search in MJD - + Ending date for the search in MJD + Returns ------- result : list - List of dictionaries containing the query results - + List of dictionaries containing the query results """ - - instrument = 'MIRI' - dataproduct = JWST_DATAPRODUCTS - parameters = {"date_obs_mjd": {"min": start_date, "max": end_date}} - - result = monitor_mast.instrument_inventory(instrument, dataproduct, - add_filters=parameters, - return_data=True) - - return result + + data_product = JWST_DATAPRODUCTS + parameters = {"date_obs_mjd": {"min": self.query_start, "max": self.query_end}, "apername": self.aperture} + + result = monitor_mast.instrument_inventory(self.instrument, data_product, + add_filters=parameters, + return_data=True) + + return result if __name__ == '__main__': - # Configure logging module = os.path.basename(__file__).strip('.py') configure_logging(module) - + # Call the main function - cosmic_ray_monitor() + monitor = CosmicRay() + monitor.run() diff --git a/jwql/tests/test_cosmic_ray_monitor.py b/jwql/tests/test_cosmic_ray_monitor.py index cd9cc0e5b..c2f80e360 100644 --- a/jwql/tests/test_cosmic_ray_monitor.py +++ b/jwql/tests/test_cosmic_ray_monitor.py @@ -17,13 +17,16 @@ pytest -s test_cosmic_ray_monitor.py """ -import pytest -import cosmic_ray_monitor as crm -import numpy as np +# Third Party Imports from astropy.io import fits +import numpy as np +import pytest +# Local Imports +from cosmic_ray_monitor import Cosmic_Ray from jwql.database.database_interface import MIRICosmicRayQueryHistory + def define_test_data(nints): if nints == 1: data = np.ones((2, 5, 10, 10)) @@ -35,14 +38,14 @@ def define_test_data(nints): file = "jw00000000000_00000_00000_MIRIMAGE_uncal.fits" aperture = "MIRIM_FULL" -return data, rate_data, file, aperture + return data, rate_data, file, aperture def test_get_jump_data(): _ = define_test_data(2) file = _[2] - head, data, dq = crm.get_jump_data(file) + head, data, dq = Cosmic_Ray.get_jump_data(file) assert type(head) == fits.header.Header assert type(data) == np.ndarray @@ -53,7 +56,7 @@ def test_get_rate_data(): _ = define_test_data(1) file = _[2] - data = crm.get_rate_data(file) + data = Cosmic_Ray.get_rate_data(file) assert type(data) == np.ndarray @@ -62,7 +65,7 @@ def test_get_cr_rate(): jump_locs = np.arange(100).tolist() t = 5 - rate = crm.get_cr_rate(jump_locs, t) + rate = Cosmic_Ray.get_cr_rate(jump_locs, t) assert rate == 20.0 @@ -71,12 +74,12 @@ def test_group_before(): jump_locs = [(2, 1, 1)] nints = 1 - assert crm.group_before(jump_locs, nints) == [(1, 1, 1)] + assert Cosmic_Ray.group_before(jump_locs, nints) == [(1, 1, 1)] jump_locs = [(1, 2, 1, 1)] nints = 2 - assert crm.group_before(jump_locs, nints) == [(1, 1, 1, 1)] + assert Cosmic_Ray.group_before(jump_locs, nints) == [(1, 1, 1, 1)] def test_magnitude(): @@ -86,14 +89,14 @@ def test_magnitude(): head = fits.getheader(file) coord = (1, 2, 1, 1) coord_gb = (1, 1, 1, 1) - mag = crm.magnitude(coord, coord_gb, rate_data, data, head, nints) + mag = Cosmic_Ray.magnitude(coord, coord_gb, rate_data, data, head, nints) assert mag == -2.77504 nints = 1 data, rate_data, file, aperture = define_test_data(nints) coord = (1, 1, 1) coord_gb = (0, 1, 1) - mag = crm.magnitude(coord, coord_gb, rate_data, data, head, nints) + mag = Cosmic_Ray.magnitude(coord, coord_gb, rate_data, data, head, nints) assert mag == -2.77504 @@ -105,7 +108,7 @@ def test_get_cr_mags(): data, rate_data, file, aperture = define_test_data(nints) head = fits.getheader(file) - mags = crm.get_cr_mags(jump_locs, jump_locs_pre, rate_data, data, head, nints) + mags = Cosmic_Ray.get_cr_mags(jump_locs, jump_locs_pre, rate_data, data, head, nints) assert mags == [-2.77504] jump_locs = [(1, 2, 1, 1)] @@ -113,7 +116,7 @@ def test_get_cr_mags(): nints = 5 data, rate_data, file, aperture = define_test_data(nints) - mags = crm.get_cr_mags(jump_locs, jump_locs_pre, rate_data, data, head, nints) + mags = Cosmic_Ray.get_cr_mags(jump_locs, jump_locs_pre, rate_data, data, head, nints) assert mags == [-2.77504] @@ -124,7 +127,7 @@ def test_most_recent_search(): query_table = MIRICosmicRayQueryHistory - result = crm.most_recent_search(aperture,query_table) + result = Cosmic_Ray.most_recent_search(aperture,query_table) assert result == 57357.0 @@ -134,6 +137,6 @@ def test_query_mast(): start_date = 57357.0 end_date = 57405.0 - result = crm.query_mast(start_date, end_date) + result = Cosmic_Ray.query_mast(start_date, end_date) assert len(result) == 5 diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 67442db41..ebfdb3e83 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -201,7 +201,7 @@ def api_landing(request): # @auth_required -def archived_proposals(request, inst): #request, user, inst): +def archived_proposals(request, user, inst): """Generate the page listing all archived proposals in the database Parameters @@ -228,7 +228,7 @@ def archived_proposals(request, inst): #request, user, inst): # @auth_required -def archived_proposals_ajax(request, inst): # request, user, inst): +def archived_proposals_ajax(request, user, inst): """Generate the page listing all archived proposals in the database Parameters @@ -264,7 +264,7 @@ def archived_proposals_ajax(request, inst): # request, user, inst): # @auth_required -def archive_thumbnails(request, inst, proposal): # request, user, inst, proposal): +def archive_thumbnails(request, user, inst, proposal): """Generate the page listing all archived images in the database for a certain proposal @@ -295,7 +295,7 @@ def archive_thumbnails(request, inst, proposal): # request, user, inst, proposa # @auth_required -def archive_thumbnails_ajax(request, inst, proposal): # request, user, inst, proposal): +def archive_thumbnails_ajax(request, user, inst, proposal): """Generate the page listing all archived images in the database for a certain proposal @@ -351,7 +351,7 @@ def dashboard(request): # @auth_info -def engineering_database(request): # request, user): +def engineering_database(request, user): """Generate the EDB page. Parameters @@ -797,7 +797,7 @@ def view_header(request, inst, filename): # @auth_required -def view_image(request, inst, file_root, rewrite=False): # request, user, inst, file_root, rewrite=False): +def view_image(request, user, inst, file_root, rewrite=False): """Generate the image view page Parameters From 4320405068505e801404972ac47b3176c2eca2d5 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Wed, 24 Feb 2021 15:59:48 -0500 Subject: [PATCH 0010/1618] Fixed various issues raised by @pep8speaks --- .../miri_monitors/cosmic_ray_monitor.py | 6 --- jwql/tests/test_cosmic_ray_monitor.py | 54 +++++++++---------- jwql/utils/constants.py | 2 +- .../monitor_cosmic_rays_bokeh.py | 44 ++++++--------- jwql/website/apps/jwql/monitor_views.py | 2 +- 5 files changed, 46 insertions(+), 62 deletions(-) diff --git a/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py b/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py index d676050ff..1adbee70e 100644 --- a/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py +++ b/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py @@ -317,7 +317,6 @@ def magnitude(self, coord, coord_gb, rateints, data, head): return cr_mag - def most_recent_search(self): """Adapted from Dark Monitor (Bryan Hilbert) @@ -352,8 +351,6 @@ def most_recent_search(self): query_result = 57357.0 # a.k.a. Dec 1, 2015 == CV3 logging.info(('\tNo query history for {}. Beginning search date will be set to {}.' .format(self.aperture, query_result))) - #elif query_count > 1: - # raise ValueError('More than one "most recent" query?') else: query_result = query[0].end_time_mjd @@ -474,9 +471,6 @@ def run(self): # Get a list of possible apertures possible_apertures = list(Siaf(instrument).apernames) - # Use this line instead to save time while testing - #possible_apertures = ['MIRIM_FULL', 'NRCB4_FULL'] - for aperture in possible_apertures: logging.info('') diff --git a/jwql/tests/test_cosmic_ray_monitor.py b/jwql/tests/test_cosmic_ray_monitor.py index c2f80e360..8227d62ce 100644 --- a/jwql/tests/test_cosmic_ray_monitor.py +++ b/jwql/tests/test_cosmic_ray_monitor.py @@ -1,19 +1,19 @@ #! /usr/bin/env python """Tests for the cosmic ray monitor module. - + Authors ------- - + - Mike Engesser - + Use --- - + These tests can be run via the command line (omit the ``-s`` to suppress verbose output to stdout): :: - + pytest -s test_cosmic_ray_monitor.py """ @@ -44,9 +44,9 @@ def define_test_data(nints): def test_get_jump_data(): _ = define_test_data(2) file = _[2] - + head, data, dq = Cosmic_Ray.get_jump_data(file) - + assert type(head) == fits.header.Header assert type(data) == np.ndarray assert type(dq) == np.ndarray @@ -55,35 +55,35 @@ def test_get_jump_data(): def test_get_rate_data(): _ = define_test_data(1) file = _[2] - + data = Cosmic_Ray.get_rate_data(file) - + assert type(data) == np.ndarray def test_get_cr_rate(): jump_locs = np.arange(100).tolist() t = 5 - + rate = Cosmic_Ray.get_cr_rate(jump_locs, t) - + assert rate == 20.0 def test_group_before(): jump_locs = [(2, 1, 1)] nints = 1 - + assert Cosmic_Ray.group_before(jump_locs, nints) == [(1, 1, 1)] - + jump_locs = [(1, 2, 1, 1)] nints = 2 - + assert Cosmic_Ray.group_before(jump_locs, nints) == [(1, 1, 1, 1)] def test_magnitude(): - + nints = 5 data, rate_data, file, aperture = define_test_data(nints) head = fits.getheader(file) @@ -91,7 +91,7 @@ def test_magnitude(): coord_gb = (1, 1, 1, 1) mag = Cosmic_Ray.magnitude(coord, coord_gb, rate_data, data, head, nints) assert mag == -2.77504 - + nints = 1 data, rate_data, file, aperture = define_test_data(nints) coord = (1, 1, 1) @@ -101,42 +101,42 @@ def test_magnitude(): def test_get_cr_mags(): - + jump_locs = [(2, 1, 1)] jump_locs_pre = [(1, 1, 1)] nints = 1 data, rate_data, file, aperture = define_test_data(nints) head = fits.getheader(file) - + mags = Cosmic_Ray.get_cr_mags(jump_locs, jump_locs_pre, rate_data, data, head, nints) assert mags == [-2.77504] - + jump_locs = [(1, 2, 1, 1)] jump_locs_pre = [(1, 1, 1, 1)] nints = 5 data, rate_data, file, aperture = define_test_data(nints) - + mags = Cosmic_Ray.get_cr_mags(jump_locs, jump_locs_pre, rate_data, data, head, nints) assert mags == [-2.77504] def test_most_recent_search(): - + _ = define_test_data(1) aperture = _[3] - + query_table = MIRICosmicRayQueryHistory - + result = Cosmic_Ray.most_recent_search(aperture,query_table) - + assert result == 57357.0 def test_query_mast(): - + start_date = 57357.0 end_date = 57405.0 - + result = Cosmic_Ray.query_mast(start_date, end_date) - + assert len(result) == 5 diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index 11f77a29d..4b07ee5f8 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -121,7 +121,7 @@ # Bad pixel types by the type of data used to find them BAD_PIXEL_TYPES = ['DEAD', 'HOT', 'LOW_QE', 'RC', 'OPEN', 'ADJ_OPEN', 'TELEGRAPH', 'OTHER_BAD_PIXEL'] -DARKS_BAD_PIXEL_TYPES = ['HOT', 'RC', 'OTHER_BAD_PIXEL', 'TELEGRAPH'] +DARKS_BAD_PIXEL_TYPES = ['HOT', 'RC', 'OTHER_BAD_PIXEL', 'TELEGRAPH'] FLATS_BAD_PIXEL_TYPES = ['DEAD', 'OPEN', 'ADJ_OPEN', 'LOW_QE'] # Possible exposure types for dark current data diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_cosmic_rays_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_cosmic_rays_bokeh.py index 07085081a..46b91478d 100644 --- a/jwql/website/apps/jwql/monitor_pages/monitor_cosmic_rays_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_cosmic_rays_bokeh.py @@ -6,7 +6,7 @@ --- This module can be used from the command line as such: :: - + """ import os @@ -25,6 +25,7 @@ SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + class CosmicRayMonitor(BokehTemplate): # Combine instrument and aperture into a single property because we @@ -39,7 +40,6 @@ def aperture_info(self, info): self.pre_init() self.post_init() - def pre_init(self): # Start with default values for instrument and aperture because # BokehTemplate's __init__ method does not allow input arguments @@ -63,42 +63,38 @@ def pre_init(self): # Get dates and coordinates of the most recent entries #self.most_recent_data() - def post_init(self): self._update_cosmic_ray_v_time() self._update_cosmic_ray_histogram() - - + def get_histogram_data(self): """Get data required to create cosmic ray histogram from the database query. """ - + last_hist_index = -1 hist = plt.hist(self.mags[last_hist_index]) - - self.bin_left = np.array( [bar.get_x() for bar in hist[2]] ) + + self.bin_left = np.array([bar.get_x() for bar in hist[2]]) self.amplitude = [bar.get_height() for bar in hist[2]] self.bottom = [bar.get_y() for bar in hist[2]] deltas = self.bin_left[1:] - self.bin_left[0: -1] self.bin_width = np.append(deltas[0], deltas) - def get_history_data(self): """Extract data on the history of cosmic ray numbers from the database query result """ self.cosmic_ray_history = {} - + self.times = [row.obs_end_time for row in self.cosmic_ray_table] self.num = [row.jump_count for row in self.cosmic_ray_table] - + self.mags = [row.magnitude for row in self.cosmic_ray_table] - + hover_values = np.array([datetime.datetime.strftime(t, "%d-%b-%Y") for t in self.times]) self.cosmic_ray_history['Cosmic Rays'] = (self.times, self.num, hover_values) - def identify_tables(self): """Determine which database tables as associated with a given instrument""" @@ -106,7 +102,6 @@ def identify_tables(self): self.query_table = eval('{}CosmicRayQueryHistory'.format(mixed_case_name)) self.stats_table = eval('{}CosmicRayStats'.format(mixed_case_name)) - def load_data(self): """Query the database tables to get data""" @@ -118,12 +113,11 @@ def load_data(self): .filter(self.stats_table.aperture == self._aperture) \ .all() - def most_recent_data(self): """Get the cosmic ray magnitudes associated with the most recent run of the monitor. """ - + cosmic_ray_times = [row.obs_end_time for row in self.cosmic_ray_table] if len(cosmic_ray_times) > 0: @@ -131,12 +125,11 @@ def most_recent_data(self): else: self.most_recent_obs = datetime.datetime(1999, 10, 31) - def _update_cosmic_ray_v_time(self): - """Update the plot properties for the plots of the number of cosmic rays + """Update the plot properties for the plots of the number of cosmic rays versus time. """ - + self.refs['cosmic_ray_x_range'].start = min(self.times) self.refs['cosmic_ray_x_range'].end = max(self.times) self.refs['cosmic_ray_y_range'].start = min(self.num) @@ -145,24 +138,21 @@ def _update_cosmic_ray_v_time(self): self.refs['cosmic_ray_history_figure'].title.text = 'MIRIM_FULL Cosmic Ray History' self.refs['cosmic_ray_history_figure'].title.align = "center" self.refs['cosmic_ray_history_figure'].title.text_font_size = "20px" - - + def _update_cosmic_ray_histogram(): - + mags = [row.magnitude for row in self.cosmic_ray_table] - + self.refs['hist_x_range'].start = 0 self.refs['hist_x_range'].end = max(mags) - + self.refs['hist_y_range'].start = 0 self.refs['hist_y_range'].end = max(mags) - + self.refs['cosmic_ray_histogram'].title.text = 'MIRIM_FULL Cosmic Ray Intensities' self.refs['cosmic_ray_histogram'].title.align = "center" self.refs['cosmic_ray_histogram'].title.text_font_size = "20px" - - # Uncomment the line below when testing via the command line: # bokeh serve --show monitor_cosmic_rays_bokeh.py CosmicRayMonitor() \ No newline at end of file diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py index 3a7287031..caec86d0a 100644 --- a/jwql/website/apps/jwql/monitor_views.py +++ b/jwql/website/apps/jwql/monitor_views.py @@ -37,6 +37,7 @@ FILESYSTEM_DIR = os.path.join(get_config()['jwql_dir'], 'filesystem') + def cosmic_ray_monitor(request, inst='MIRI'): """Generate the cosmic ray monitor page for a given instrument Parameters @@ -53,7 +54,6 @@ def cosmic_ray_monitor(request, inst='MIRI'): # Ensure the instrument is correctly capitalized inst = 'MIRI' #other instruments not currently supported - #JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()] tabs_components = bokeh_containers.cosmic_ray_monitor_tabs(inst) From 723f009713b41f591a1cb4fde670f1070f549ca8 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Fri, 26 Feb 2021 15:58:05 -0500 Subject: [PATCH 0011/1618] Added julian dependency (used in cosmic ray monitor) --- environment_python_3_7.yml | 1 + environment_python_3_8.yml | 1 + requirements.txt | 1 + setup.py | 1 + 4 files changed, 4 insertions(+) diff --git a/environment_python_3_7.yml b/environment_python_3_7.yml index 18b0440a4..36a7884f4 100644 --- a/environment_python_3_7.yml +++ b/environment_python_3_7.yml @@ -35,6 +35,7 @@ dependencies: - pip: - asdf==2.7.1 - crds==10.1.0 + - julian==0.14 - jwedb==0.0.6 - jwst==0.17.1 - pysqlite3==0.4.3 diff --git a/environment_python_3_8.yml b/environment_python_3_8.yml index 54dbfafbc..262547187 100644 --- a/environment_python_3_8.yml +++ b/environment_python_3_8.yml @@ -33,6 +33,7 @@ dependencies: - asdf==2.7.1 - astroquery==0.4.1 - crds==10.1.0 + - julian==0.14 - jwedb==0.0.6 - jwst==0.17.1 - pysiaf==0.9.0 diff --git a/requirements.txt b/requirements.txt index b1b24b5d2..cb3ee5119 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,7 @@ inflection==0.5.1 ipython==7.19.0 jinja2==2.11.3 jsonschema==3.2.0 +julian==0.14 jwedb>=0.0.6 jwst==0.18.3 matplotlib==3.3.4 diff --git a/setup.py b/setup.py index fea5cebeb..d9ce35b3e 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,7 @@ 'ipython', 'jinja2', 'jsonschema', + 'julian', 'jwedb>=0.0.3', 'jwst', 'matplotlib', From 366cfa03a8841101060c8c53ffee635f42f6f74e Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Fri, 26 Feb 2021 15:59:01 -0500 Subject: [PATCH 0012/1618] Added NIRCam cosmic ray tables and table definitions --- jwql/database/database_interface.py | 2 ++ .../nircam/nircam_cosmic_ray_query_history.txt | 6 ++++++ .../nircam/nircam_cosmic_ray_stats.txt | 6 ++++++ 3 files changed, 14 insertions(+) create mode 100644 jwql/database/monitor_table_definitions/nircam/nircam_cosmic_ray_query_history.txt create mode 100644 jwql/database/monitor_table_definitions/nircam/nircam_cosmic_ray_stats.txt diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py index 72767f21a..93e8c9fe9 100755 --- a/jwql/database/database_interface.py +++ b/jwql/database/database_interface.py @@ -428,6 +428,8 @@ class : obj MIRIReadnoiseStats = monitor_orm_factory('miri_readnoise_stats') FGSReadnoiseQueryHistory = monitor_orm_factory('fgs_readnoise_query_history') FGSReadnoiseStats = monitor_orm_factory('fgs_readnoise_stats') +NIRCamCosmicRayQueryHistory = monitor_orm_factory('nircam_cosmic_ray_query_history') +NIRCamCosmicRayStats = monitor_orm_factory('nircam_cosmic_ray_stats') MIRICosmicRayQueryHistory = monitor_orm_factory('miri_cosmic_ray_query_history') MIRICosmicRayStats = monitor_orm_factory('miri_cosmic_ray_stats') diff --git a/jwql/database/monitor_table_definitions/nircam/nircam_cosmic_ray_query_history.txt b/jwql/database/monitor_table_definitions/nircam/nircam_cosmic_ray_query_history.txt new file mode 100644 index 000000000..bbea160bb --- /dev/null +++ b/jwql/database/monitor_table_definitions/nircam/nircam_cosmic_ray_query_history.txt @@ -0,0 +1,6 @@ +INSTRUMENT, string +APERTURE, string +START_TIME_MJD, float +END_TIME_MJD, float +FILES_FOUND, integer +RUN_MONITOR, bool \ No newline at end of file diff --git a/jwql/database/monitor_table_definitions/nircam/nircam_cosmic_ray_stats.txt b/jwql/database/monitor_table_definitions/nircam/nircam_cosmic_ray_stats.txt new file mode 100644 index 000000000..97259bfb4 --- /dev/null +++ b/jwql/database/monitor_table_definitions/nircam/nircam_cosmic_ray_stats.txt @@ -0,0 +1,6 @@ +APERTURE, string +SOURCE_FILE, string +OBS_START_TIME, datetime +OBS_END_TIME, datetime +JUMP_COUNT, integer +MAGNITUDE, float_array_1d \ No newline at end of file From 45017d92730e0c47f3e13ec36a8e67f0d2d6c11c Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Fri, 26 Feb 2021 15:59:17 -0500 Subject: [PATCH 0013/1618] Removed redundant code --- jwql/utils/constants.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index 4b07ee5f8..1083c757e 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -119,11 +119,6 @@ DARKS_BAD_PIXEL_TYPES = ['HOT', 'RC', 'OTHER_BAD_PIXEL', 'TELEGRAPH'] FLATS_BAD_PIXEL_TYPES = ['DEAD', 'OPEN', 'ADJ_OPEN', 'LOW_QE'] -# Bad pixel types by the type of data used to find them -BAD_PIXEL_TYPES = ['DEAD', 'HOT', 'LOW_QE', 'RC', 'OPEN', 'ADJ_OPEN', 'TELEGRAPH', 'OTHER_BAD_PIXEL'] -DARKS_BAD_PIXEL_TYPES = ['HOT', 'RC', 'OTHER_BAD_PIXEL', 'TELEGRAPH'] -FLATS_BAD_PIXEL_TYPES = ['DEAD', 'OPEN', 'ADJ_OPEN', 'LOW_QE'] - # Possible exposure types for dark current data DARK_EXP_TYPES = {'nircam': ['NRC_DARK'], 'niriss': ['NIS_DARK'], From 174e278959c01252b875da920df1ddd11607affc Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Fri, 26 Feb 2021 15:59:42 -0500 Subject: [PATCH 0014/1618] Various fixed to get (most) tests to pass --- jwql/tests/test_cosmic_ray_monitor.py | 133 ++++++++++++++++---------- 1 file changed, 83 insertions(+), 50 deletions(-) diff --git a/jwql/tests/test_cosmic_ray_monitor.py b/jwql/tests/test_cosmic_ray_monitor.py index 8227d62ce..d1083d6ab 100644 --- a/jwql/tests/test_cosmic_ray_monitor.py +++ b/jwql/tests/test_cosmic_ray_monitor.py @@ -14,20 +14,27 @@ suppress verbose output to stdout): :: - pytest -s test_cosmic_ray_monitor.py + pytest -s test_cosmic_ray_monitor.py """ -# Third Party Imports +import os + from astropy.io import fits import numpy as np import pytest -# Local Imports -from cosmic_ray_monitor import Cosmic_Ray +from jwql.instrument_monitors.miri_monitors.cosmic_ray_monitor import CosmicRay from jwql.database.database_interface import MIRICosmicRayQueryHistory - +from jwql.utils.utils import get_config def define_test_data(nints): + """Define the data to test with. + + Parameters + ---------- + nints : int + The number of integrations + """ if nints == 1: data = np.ones((2, 5, 10, 10)) rate_data = np.ones((10, 10)) @@ -35,108 +42,134 @@ def define_test_data(nints): data = np.ones((2, 5, 10, 10)) rate_data = np.ones((2, 10, 10)) - file = "jw00000000000_00000_00000_MIRIMAGE_uncal.fits" - aperture = "MIRIM_FULL" + filesystem = get_config()['filesystem'] + filename = os.path.join(filesystem, 'jw00608', 'jw00608002001_02101_00001_mirimage_rate.fits') + aperture = 'MIRIM_FULL' - return data, rate_data, file, aperture + return data, rate_data, filename, aperture def test_get_jump_data(): - _ = define_test_data(2) - file = _[2] + """Test the ``get_jumpy_data`` function""" + + cr = CosmicRay() + _, _, filename, _ = define_test_data(2) - head, data, dq = Cosmic_Ray.get_jump_data(file) + header, data, dq = cr.get_jump_data(filename) - assert type(head) == fits.header.Header + assert type(header) == fits.header.Header assert type(data) == np.ndarray assert type(dq) == np.ndarray def test_get_rate_data(): - _ = define_test_data(1) - file = _[2] + """Test the ``get_rate_data`` function""" + + cr = CosmicRay() + _, _, filename, _ = define_test_data(2) - data = Cosmic_Ray.get_rate_data(file) + data = cr.get_rate_data(filename) assert type(data) == np.ndarray def test_get_cr_rate(): - jump_locs = np.arange(100).tolist() - t = 5 + """Test the ``get_cr_rate`` function""" - rate = Cosmic_Ray.get_cr_rate(jump_locs, t) + cr = CosmicRay() + jump_locations = np.arange(100).tolist() + time = 5 + + rate = cr.get_cr_rate(jump_locations, time) assert rate == 20.0 def test_group_before(): - jump_locs = [(2, 1, 1)] - nints = 1 + """Test the ``group_before`` function""" + + cr = CosmicRay() + + jump_locations = [(2, 1, 1)] + cr.nints = 1 - assert Cosmic_Ray.group_before(jump_locs, nints) == [(1, 1, 1)] + assert cr.group_before(jump_locations) == [(1, 1, 1)] - jump_locs = [(1, 2, 1, 1)] - nints = 2 + jump_locations = [(1, 2, 1, 1)] + cr.nints = 2 - assert Cosmic_Ray.group_before(jump_locs, nints) == [(1, 1, 1, 1)] + assert cr.group_before(jump_locations) == [(1, 1, 1, 1)] def test_magnitude(): + """Test the ``magnitude`` method""" - nints = 5 - data, rate_data, file, aperture = define_test_data(nints) - head = fits.getheader(file) + cr = CosmicRay() + + cr.nints = 5 + data, rate_data, filename, aperture = define_test_data(cr.nints) + header = fits.getheader(filename) coord = (1, 2, 1, 1) coord_gb = (1, 1, 1, 1) - mag = Cosmic_Ray.magnitude(coord, coord_gb, rate_data, data, head, nints) + mag = cr.magnitude(coord, coord_gb, rate_data, data, header) assert mag == -2.77504 - nints = 1 - data, rate_data, file, aperture = define_test_data(nints) + cr.nints = 1 + data, rate_data, filename, aperture = define_test_data(cr.nints) coord = (1, 1, 1) coord_gb = (0, 1, 1) - mag = Cosmic_Ray.magnitude(coord, coord_gb, rate_data, data, head, nints) + mag = cr.magnitude(coord, coord_gb, rate_data, data, header) assert mag == -2.77504 def test_get_cr_mags(): + """Test the ``get_cr_mags`` function""" + + cr = CosmicRay() - jump_locs = [(2, 1, 1)] - jump_locs_pre = [(1, 1, 1)] - nints = 1 - data, rate_data, file, aperture = define_test_data(nints) - head = fits.getheader(file) + jump_locations = [(2, 1, 1)] + jump_locations_pre = [(1, 1, 1)] + cr.nints = 1 + data, rate_data, filename, aperture = define_test_data(cr.nints) + header = fits.getheader(filename) - mags = Cosmic_Ray.get_cr_mags(jump_locs, jump_locs_pre, rate_data, data, head, nints) + mags = cr.get_cr_mags(jump_locations, jump_locations_pre, rate_data, data, header) assert mags == [-2.77504] - jump_locs = [(1, 2, 1, 1)] - jump_locs_pre = [(1, 1, 1, 1)] - nints = 5 - data, rate_data, file, aperture = define_test_data(nints) + jump_locations = [(1, 2, 1, 1)] + jump_locations_pre = [(1, 1, 1, 1)] + cr.nints = 5 + data, rate_data, filename, aperture = define_test_data(cr.nints) - mags = Cosmic_Ray.get_cr_mags(jump_locs, jump_locs_pre, rate_data, data, head, nints) + mags = cr.get_cr_mags(jump_locations, jump_locations_pre, rate_data, data, header) assert mags == [-2.77504] def test_most_recent_search(): + """Test the ``most_recent_search`` function""" - _ = define_test_data(1) - aperture = _[3] + cr = CosmicRay() + _, _, _, aperture = define_test_data(1) - query_table = MIRICosmicRayQueryHistory + cr.aperture = aperture + cr.query_table = MIRICosmicRayQueryHistory - result = Cosmic_Ray.most_recent_search(aperture,query_table) + result = cr.most_recent_search() - assert result == 57357.0 + assert isinstance(result, float) def test_query_mast(): + """Test the ``query_mast`` function""" + + cr = CosmicRay() + _, _, _, aperture = define_test_data(1) - start_date = 57357.0 - end_date = 57405.0 + cr.aperture = aperture + cr.instrument = 'miri' + cr.query_start = 57357.0 + cr.query_end = 57405.0 - result = Cosmic_Ray.query_mast(start_date, end_date) + result = cr.query_mast() assert len(result) == 5 From f67663855eb19d5f58e827f8b5f2b277f9dfb274 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Fri, 26 Feb 2021 16:00:39 -0500 Subject: [PATCH 0015/1618] Turned back on authentication requirements for certain page views --- jwql/website/apps/jwql/views.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index 8bfc7aa23..f4d6e196d 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -236,7 +236,7 @@ def api_landing(request): return render(request, template, context) -# @auth_required +@auth_required def archived_proposals(request, user, inst): """Generate the page listing all archived proposals in the database @@ -263,7 +263,7 @@ def archived_proposals(request, user, inst): return render(request, template, context) -# @auth_required +@auth_required def archived_proposals_ajax(request, user, inst): """Generate the page listing all archived proposals in the database @@ -298,7 +298,7 @@ def archived_proposals_ajax(request, user, inst): return JsonResponse(context, json_dumps_params={'indent': 2}) -# @auth_required +@auth_required def archive_thumbnails(request, user, inst, proposal): """Generate the page listing all archived images in the database for a certain proposal @@ -328,7 +328,7 @@ def archive_thumbnails(request, user, inst, proposal): return render(request, template, context) -# @auth_required +@auth_required def archive_thumbnails_ajax(request, user, inst, proposal): """Generate the page listing all archived images in the database for a certain proposal @@ -416,7 +416,7 @@ def dashboard(request): return render(request, template, context) -# @auth_info +@auth_info def engineering_database(request, user): """Generate the EDB page. @@ -725,7 +725,7 @@ def view_header(request, inst, filename): return render(request, template, context) -# @auth_required +@auth_required def view_image(request, user, inst, file_root, rewrite=False): """Generate the image view page From 5d386e89b5c9b950a77daa4a1e7e448d2d79a176 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Fri, 26 Feb 2021 16:01:50 -0500 Subject: [PATCH 0016/1618] Various changes to standardize docstrings according to PEP257 and increase readibility with more explicit variable names, amongst other minor tweaks --- .../miri_monitors/cosmic_ray_monitor.py | 282 +++++++++--------- .../monitor_cosmic_rays_bokeh.py | 28 +- jwql/website/apps/jwql/monitor_views.py | 33 +- 3 files changed, 175 insertions(+), 168 deletions(-) diff --git a/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py b/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py index 1adbee70e..82a33ad07 100644 --- a/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py +++ b/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py @@ -90,8 +90,8 @@ class CosmicRay: ``NRCA1_FULL``) query_table : sqlalchemy table - Table containing the history of cosmic ray monitor queries to MAST - for each instrument/aperture combination + Table containing the history of cosmic ray monitor queries to + MAST for each instrument/aperture combination stats_table : sqlalchemy table Table containing cosmic ray analysis results. Number and @@ -108,144 +108,129 @@ class CosmicRay: """ def __init__(self): - """Initialize an instance of the ``Cosmic_Ray`` class.""" + """Initialize an instance of the ``CosmicRay`` class.""" - def identify_tables(self): - """Determine which database tables to use for a run of the - cosmic ray monitor. - - Uses the instrument variable to get the mixed-case instrument - name, and uses that name to find the query and stats tables - for that instrument. - """ - - mixed_case_name = JWST_INSTRUMENT_NAMES_MIXEDCASE[self.instrument] - self.query_table = eval('{}CosmicRayQueryHistory'.format(mixed_case_name)) - self.stats_table = eval('{}CosmicRayStats'.format(mixed_case_name)) - - def get_cr_mags(self, jump_locs, jump_locs_pre, rateints, jump_data, jump_head): + def get_cr_mags(self, jump_locations, jump_locations_pre, rateints, jump_data, jump_head): """Gets the magnitude of each cosmic ray. Computes a list of magnitudes using the coordinate of the detected jump compared to the magnitude of the same pixel in the group prior to the jump. - Parameters: + Parameters ---------- - - jump_locs: list + jump_locations : list List of coordinates to a pixel marked with a jump. - jump_locs_pre: list - List of matching coordinates one group before jump_locs. + jump_locations_pre : list + List of matching coordinates one group before + ``jump_locations``. - head: FITS header - Header containing file information. + rateints : ndarray obj + Array in ``DN/s``. - rateints: ndarray - Array in DN/s. + jump_data : - Returns: - ------- + jump_head : - mags: list + Returns + ------- + mags : list A list of cosmic ray magnitudes corresponding to each jump. - """ mags = [] - for coord, coord_gb in zip(jump_locs, jump_locs_pre): + for coord, coord_gb in zip(jump_locations, jump_locations_pre): mags.append(self.magnitude(coord, coord_gb, rateints, jump_data, jump_head)) return mags - def get_cr_rate(self, jump_locs, time): + def get_cr_rate(self, jump_locations, time): """Computes the rate of cosmic ray impacts in a given time. - Parameters: + Parameters ---------- - jump_locs: list + jump_locations : list List of coordinates to a pixel marked with a jump. - t: int or float - Time over which to compute the rate. - Nominally the effective exposure time. - + time : int or float + Time over which to compute the rate. Nominally the + effective exposure time. """ - return len(jump_locs) / time + return len(jump_locations) / time def get_jump_data(self, jump_filename): - """Opens and reads a given .FITS file containing cosmic rays. + """Opens and reads a given ``.FITS`` file containing cosmic + rays. - Parameters: + Parameters ---------- - jump_filename: str + jump_filename : str Path to file. - Returns: + Returns ------- - head: FITS header + header : FITS header Header containing file information - data: NoneType + data : obj FITS data - dq: ndarray + dq : ndarray obj Data Quality array containing jump flags. - """ hdu = fits.open(jump_filename) - head = hdu[0].header + header = hdu[0].header data = hdu[1].data dq = hdu[3].data hdu.close() - return head, data, dq + return header, data, dq - def get_jump_locs(self, dq): + def get_jump_locations(self, dq): """Uses the data quality array to find the location of all jumps in the data. - Parameters: + Parameters ---------- - dq: ndarray + dq : ndarray obj Data Quality array containing jump flags. - Returns: + Returns ------- - jump_locs: list + jump_locations : list List of coordinates to a pixel marked with a jump. """ - temp = np.where(dq == 4) + flag_locations = np.where(dq == 4) - jump_locs = [] + jump_locations = [] if self.nints > 1: - for i in range(len(temp[0])): - jump_locs.append((temp[0][i], temp[1][i], temp[2][i], temp[3][i])) + for i in range(len(flag_locations[0])): + jump_locations.append((flag_locations[0][i], flag_locations[1][i], flag_locations[2][i], flag_locations[3][i])) else: - for i in range(len(temp[0])): - jump_locs.append((temp[0][i], temp[1][i], temp[2][i])) + for i in range(len(flag_locations[0])): + jump_locations.append((flag_locations[0][i], flag_locations[1][i], flag_locations[2][i])) - return jump_locs + return jump_locations def get_rate_data(self, rate_filename): - """Opens and reads a given .FITS file. + """Opens and reads a given rate ``.FITS`` file. - Parameters: + Parameters ---------- - rate_filename: str + rate_filename : str Path to file. Returns: ------- - data: NoneType + data : obj FITS data """ @@ -253,78 +238,92 @@ def get_rate_data(self, rate_filename): return data - def group_before(self, jump_locs): + def group_before(self, jump_locations): """Creates a list of coordinates one group before given jump coordinates. - Parameters: + Parameters ---------- - jump_locs: list + jump_locations : list List of coordinates to a pixel marked with a jump. - Returns: + Returns ------- - jump_locs_pre: list - List of matching coordinates one group before jump_locs. + jump_locations_pre : list + List of matching coordinates one group before + ``jump_locations``. """ - jump_locs_pre = [] + jump_locations_pre = [] if self.nints > 1: - for coord in jump_locs: - jump_locs_pre.append((coord[0], coord[1] - 1, coord[2], coord[3])) + for coord in jump_locations: + jump_locations_pre.append((coord[0], coord[1] - 1, coord[2], coord[3])) else: - for coord in jump_locs: - jump_locs_pre.append((coord[0] - 1, coord[1], coord[2])) + for coord in jump_locations: + jump_locations_pre.append((coord[0] - 1, coord[1], coord[2])) - return jump_locs_pre + return jump_locations_pre - def magnitude(self, coord, coord_gb, rateints, data, head): + def identify_tables(self): + """Determine which database tables to use for a run of the + cosmic ray monitor. + + Uses the instrument variable to get the mixed-case instrument + name, and uses that name to find the query and stats tables + for that instrument. + """ + + mixed_case_name = JWST_INSTRUMENT_NAMES_MIXEDCASE[self.instrument] + self.query_table = eval('{}CosmicRayQueryHistory'.format(mixed_case_name)) + self.stats_table = eval('{}CosmicRayStats'.format(mixed_case_name)) + + def magnitude(self, coord, coord_gb, rateints, data, header): """Calculates the magnitude of a list of jumps given their coordinates in an array of pixels. - Parameters: + Parameters ---------- - coord: tuple + coord : tuple Coordinate of jump. - coord_gb: tuple + coord_gb : tuple Coordinate of jump pixel one group before. - head: FITS header + header : FITS header Header containing file information. - rateints: ndarray - Array in DN/s. + rateints : ndarray + Array in ``DN/s``. - Returns: + Returns ------- - cr_mag: float + cr_mag : float the magnitude of the cosmic ray """ - grouptime = head['TGROUP'] + group_time = header['TGROUP'] if self.nints == 1: rate = rateints[coord[-2]][coord[-1]] cr_mag = data[0][coord[0]][coord[1]][coord[2]] \ - data[0][coord_gb[0]][coord_gb[1]][coord_gb[2]] \ - - rate * grouptime + - rate * group_time else: rate = rateints[coord[0]][coord[-2]][coord[-1]] - cr_mag = data[coord] - data[coord_gb] - rate * grouptime + cr_mag = data[coord] - data[coord_gb] - rate * group_time return cr_mag def most_recent_search(self): - """Adapted from Dark Monitor (Bryan Hilbert) + """Adapted from the Dark Monitor (Bryan Hilbert) - Query the query history database and return the information - on the most recent query for the given ``aperture_name`` where - the cosmic ray monitor was executed. + Query the query history table in the ``jwqldb`` database and + return the information on the most recent query for the given + ``aperture_name`` where the cosmic ray monitor was executed. - Returns: + Returns ------- query_result : float Date (in MJD) of the ending range of the previous MAST @@ -363,46 +362,47 @@ def process(self, file_list): Parameters ---------- file_list : list - List of filenames (including full paths) to the cosmic ray - files + List of filenames (including full paths) to the cosmic ray + files """ for file_name in file_list: - if 'uncal' in file_name: - dir_name = '_'.join(file_name.split('_')[:4]) # file_name[51:76] - self.obs_dir = os.path.join(self.data_dir, dir_name) - ensure_dir_exists(self.obs_dir) + # Only process uncal files + if 'uncal' in file_name: - head = fits.getheader(file_name) - self.nints = head['NINTS'] + # Define some useful parameters + dir_name = '_'.join(file_name.split('_')[:4]) # aka file_name[51:76] + self.nints = fits.getheader(file_name)['NINTS'] + # Ensure the directory to the data exists. If so, copy over files + observation_dir = os.path.join(self.data_dir, dir_name) + ensure_dir_exists(observation_dir) try: - copy_files([file_name], self.obs_dir) + copy_files([file_name], observation_dir) except: logging.info('Failed to copy {} to observation dir.'.format(file_name)) # Next we run the pipeline on the files to get the proper outputs - uncal_file = os.path.join(self.obs_dir, os.path.basename(file_name)) - + uncal_file = os.path.join(observation_dir, os.path.basename(file_name)) try: - pipeline_tools.calwebb_detector1_save_jump(uncal_file, self.obs_dir, ramp_fit=True, save_fitopt=False) + pipeline_tools.calwebb_detector1_save_jump(uncal_file, observation_dir, ramp_fit=True, save_fitopt=False) except: logging.info('Failed to complete pipeline steps on {}.'.format(uncal_file)) # Next we analyze the cosmic rays in the new data - for output_file in os.listdir(self.obs_dir): + for output_file in os.listdir(observation_dir): if 'jump' in output_file: - jump_file = os.path.join(self.obs_dir, output_file) + jump_file = os.path.join(observation_dir, output_file) if self.nints == 1: if '0_ramp_fit' in output_file: - rate_file = os.path.join(self.obs_dir, output_file) + rate_file = os.path.join(observation_dir, output_file) elif self.nints > 1: if '1_ramp_fit' in output_file: - rate_file = os.path.join(self.obs_dir, output_file) + rate_file = os.path.join(observation_dir, output_file) try: jump_head, jump_data, jump_dq = self.get_jump_data(jump_file) @@ -414,8 +414,8 @@ def process(self, file_list): except: logging.info('Could not open rate file: {}'.format(rate_file)) - jump_locs = self.get_jump_locs(jump_dq) - jump_locs_pre = self.group_before(jump_locs) + jump_locations = self.get_jump_locations(jump_dq) + jump_locations_pre = self.group_before(jump_locations) eff_time = jump_head['EFFEXPTM'] @@ -425,9 +425,9 @@ def process(self, file_list): start_time = julian.from_jd(obs_start_time, fmt='mjd') end_time = julian.from_jd(obs_end_time, fmt='mjd') - cosmic_ray_num = len(jump_locs) - # cosmic_ray_rate = get_cr_rate(jump_locs, eff_time) - cosmic_ray_mags = self.get_cr_mags(jump_locs, jump_locs_pre, rate_data, jump_data, jump_head) + cosmic_ray_num = len(jump_locations) + # cosmic_ray_rate = get_cr_rate(jump_locations, eff_time) + cosmic_ray_mags = self.get_cr_mags(jump_locations, jump_locations_pre, rate_data, jump_data, jump_head) # Insert new data into database try: @@ -440,11 +440,32 @@ def process(self, file_list): 'magnitude': cosmic_ray_mags } self.stats_table.__table__.insert().execute(cosmic_ray_db_entry) - logging.info("Successfully inserted into database. \n") except: logging.info("Could not insert entry into database. \n") + def query_mast(self): + """Use ``astroquery`` to search MAST for cosmic ray data + + Parameters: + ---------- + start_date : float + Starting date for the search in MJD + end_date : float + Ending date for the search in MJD + + Returns + ------- + result : list + List of dictionaries containing the query results + """ + + data_product = JWST_DATAPRODUCTS + parameters = {"date_obs_mjd": {"min": self.query_start, "max": self.query_end}, "apername": self.aperture} + result = monitor_mast.instrument_inventory(self.instrument, data_product, add_filters=parameters, return_data=True) + + return result + @log_fail @log_info def run(self): @@ -462,7 +483,10 @@ def run(self): self.query_end = Time.now().mjd for instrument in JWST_INSTRUMENT_NAMES: + + # Currently only supports MIRI and NIRCam if instrument == 'miri' or instrument == 'nircam': + self.instrument = instrument # Identify which tables to use @@ -521,33 +545,9 @@ def run(self): self.query_table.__table__.insert().execute(new_entry) logging.info('\tUpdated the query history table') - def query_mast(self): - """Use astroquery to search MAST for cosmic ray data - - Parameters: - ---------- - start_date : float - Starting date for the search in MJD - end_date : float - Ending date for the search in MJD - - Returns - ------- - result : list - List of dictionaries containing the query results - """ - - data_product = JWST_DATAPRODUCTS - parameters = {"date_obs_mjd": {"min": self.query_start, "max": self.query_end}, "apername": self.aperture} - - result = monitor_mast.instrument_inventory(self.instrument, data_product, - add_filters=parameters, - return_data=True) - - return result - if __name__ == '__main__': + # Configure logging module = os.path.basename(__file__).strip('.py') configure_logging(module) diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_cosmic_rays_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_cosmic_rays_bokeh.py index 46b91478d..e218809e7 100644 --- a/jwql/website/apps/jwql/monitor_pages/monitor_cosmic_rays_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_cosmic_rays_bokeh.py @@ -1,27 +1,32 @@ """This module contains code for the cosmic ray monitor Bokeh plots. -Author ------- + +Authors +------- + - Mike Engesser Use --- - This module can be used from the command line as such: + + This module is intended to be imported and use as such: :: + from jwql.website.apps.jwql import monitor_pages + monitor_template = monitor_pages.CosmicRayMonitor() """ +import datetime import os from astropy.io import fits from astropy.time import Time -import datetime -import numpy as np import matplotlib.pyplot as plt +import numpy as np +from jwql.bokeh_templating import BokehTemplate from jwql.database.database_interface import session from jwql.database.database_interface import MIRICosmicRayQueryHistory, MIRICosmicRayStats from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE from jwql.utils.utils import get_config, filesystem_path -from jwql.bokeh_templating import BokehTemplate SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -88,12 +93,12 @@ def get_history_data(self): self.cosmic_ray_history = {} self.times = [row.obs_end_time for row in self.cosmic_ray_table] - self.num = [row.jump_count for row in self.cosmic_ray_table] + self.count = [row.jump_count for row in self.cosmic_ray_table] self.mags = [row.magnitude for row in self.cosmic_ray_table] hover_values = np.array([datetime.datetime.strftime(t, "%d-%b-%Y") for t in self.times]) - self.cosmic_ray_history['Cosmic Rays'] = (self.times, self.num, hover_values) + self.cosmic_ray_history['Cosmic Rays'] = (self.times, self.count, hover_values) def identify_tables(self): """Determine which database tables as associated with @@ -132,8 +137,8 @@ def _update_cosmic_ray_v_time(self): self.refs['cosmic_ray_x_range'].start = min(self.times) self.refs['cosmic_ray_x_range'].end = max(self.times) - self.refs['cosmic_ray_y_range'].start = min(self.num) - self.refs['cosmic_ray_y_range'].end = max(self.num) + self.refs['cosmic_ray_y_range'].start = min(self.count) + self.refs['cosmic_ray_y_range'].end = max(self.count) self.refs['cosmic_ray_history_figure'].title.text = 'MIRIM_FULL Cosmic Ray History' self.refs['cosmic_ray_history_figure'].title.align = "center" @@ -153,6 +158,7 @@ def _update_cosmic_ray_histogram(): self.refs['cosmic_ray_histogram'].title.align = "center" self.refs['cosmic_ray_histogram'].title.text_font_size = "20px" + # Uncomment the line below when testing via the command line: # bokeh serve --show monitor_cosmic_rays_bokeh.py -CosmicRayMonitor() \ No newline at end of file +CosmicRayMonitor() diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py index caec86d0a..137b34363 100644 --- a/jwql/website/apps/jwql/monitor_views.py +++ b/jwql/website/apps/jwql/monitor_views.py @@ -38,14 +38,16 @@ FILESYSTEM_DIR = os.path.join(get_config()['jwql_dir'], 'filesystem') -def cosmic_ray_monitor(request, inst='MIRI'): - """Generate the cosmic ray monitor page for a given instrument +def bad_pixel_monitor(request, inst): + """Generate the dark monitor page for a given instrument + Parameters ---------- request : HttpRequest object Incoming request from the webpage inst : str Name of JWST instrument + Returns ------- HttpResponse object @@ -53,11 +55,11 @@ def cosmic_ray_monitor(request, inst='MIRI'): """ # Ensure the instrument is correctly capitalized - inst = 'MIRI' #other instruments not currently supported + inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()] - tabs_components = bokeh_containers.cosmic_ray_monitor_tabs(inst) + tabs_components = bokeh_containers.bad_pixel_monitor_tabs(inst) - template = "cosmic_ray_monitor.html" + template = "bad_pixel_monitor.html" context = { 'inst': inst, @@ -68,8 +70,8 @@ def cosmic_ray_monitor(request, inst='MIRI'): return render(request, template, context) -def bad_pixel_monitor(request, inst): - """Generate the dark monitor page for a given instrument +def bias_monitor(request, inst): + """Generate the bias monitor page for a given instrument Parameters ---------- @@ -87,9 +89,10 @@ def bad_pixel_monitor(request, inst): # Ensure the instrument is correctly capitalized inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()] - tabs_components = bokeh_containers.bad_pixel_monitor_tabs(inst) + # Get the html and JS needed to render the bias tab plots + tabs_components = bokeh_containers.bias_monitor_tabs(inst) - template = "bad_pixel_monitor.html" + template = "bias_monitor.html" context = { 'inst': inst, @@ -100,8 +103,8 @@ def bad_pixel_monitor(request, inst): return render(request, template, context) -def bias_monitor(request, inst): - """Generate the bias monitor page for a given instrument +def cosmic_ray_monitor(request, inst='MIRI'): + """Generate the cosmic ray monitor page for a given instrument Parameters ---------- @@ -109,7 +112,6 @@ def bias_monitor(request, inst): Incoming request from the webpage inst : str Name of JWST instrument - Returns ------- HttpResponse object @@ -117,12 +119,11 @@ def bias_monitor(request, inst): """ # Ensure the instrument is correctly capitalized - inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()] + inst = 'MIRI' #other instruments not currently supported - # Get the html and JS needed to render the bias tab plots - tabs_components = bokeh_containers.bias_monitor_tabs(inst) + tabs_components = bokeh_containers.cosmic_ray_monitor_tabs(inst) - template = "bias_monitor.html" + template = "cosmic_ray_monitor.html" context = { 'inst': inst, From 6979ecc7e803b6c1089d3b81f8be44c1a527d381 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Mon, 8 Mar 2021 11:16:30 -0500 Subject: [PATCH 0017/1618] Trying to use MAST archive instead of local test data --- .../miri_monitors/cosmic_ray_monitor.py | 62 +++++++++---------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py b/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py index 82a33ad07..4a0654bfd 100644 --- a/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py +++ b/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py @@ -54,7 +54,7 @@ from jwql.utils.logging_functions import configure_logging from jwql.utils.logging_functions import log_info from jwql.utils.logging_functions import log_fail -from jwql.utils.utils import copy_files, ensure_dir_exists, get_config +from jwql.utils.utils import copy_files, ensure_dir_exists, filesystem_path, get_config class CosmicRay: @@ -502,48 +502,48 @@ def run(self): self.aperture = aperture - # We start by querying MAST for new data + # Query MAST for new data self.query_start = self.most_recent_search() - logging.info('\tMost recent query: {}'.format(self.query_start)) - new_entries = self.query_mast() + new_filenames = [] - # for testing purposes only - new_filenames = get_config()['local_test_data'] + if not new_entries['data']: + logging.info('\tNo new data to process') - # for file_entry in new_entries['data']: - # try: - # new_filenames.append(filesystem_path(file_entry['filename'])) - # except FileNotFoundError: - # logging.info('\t{} not found in target directory'.format(file_entry['filename'])) - # except ValueError: - # logging.info( - # '\tProvided file {} does not follow JWST naming conventions.'.format(file_entry['filename'])) + else: + for file_entry in new_entries['data']: + try: + new_filenames.append(filesystem_path(file_entry['filename'])) + except FileNotFoundError: + logging.info('\t{} not found in target directory'.format(file_entry['filename'])) + except ValueError: + logging.info( + '\tProvided file {} does not follow JWST naming conventions.'.format(file_entry['filename'])) - # Next we copy new files to the working directory - output_dir = os.path.join(get_config()['outputs'], 'cosmic_ray_monitor') + # Next we copy new files to the working directory + output_dir = os.path.join(get_config()['outputs'], 'cosmic_ray_monitor') - self.data_dir = get_config()['local_test_dir'] # for testing purposes only + # self.data_dir = get_config()['local_test_dir'] # for testing purposes only - # self.data_dir = os.path.join(output_dir,'data') - ensure_dir_exists(self.data_dir) + self.data_dir = os.path.join(output_dir,'data') + ensure_dir_exists(self.data_dir) - cosmic_ray_files, not_copied = copy_files(new_filenames, self.data_dir) + cosmic_ray_files, not_copied = copy_files(new_filenames, self.data_dir) - self.process(cosmic_ray_files) + self.process(cosmic_ray_files) - monitor_run = True + monitor_run = True - new_entry = {'instrument': self.instrument, - 'aperture': self.aperture, - 'start_time_mjd': self.query_start, - 'end_time_mjd': self.query_end, - 'files_found': len(new_entries), - 'run_monitor': monitor_run, - 'entry_date': datetime.datetime.now()} - self.query_table.__table__.insert().execute(new_entry) - logging.info('\tUpdated the query history table') + new_entry = {'instrument': self.instrument, + 'aperture': self.aperture, + 'start_time_mjd': self.query_start, + 'end_time_mjd': self.query_end, + 'files_found': len(new_entries), + 'run_monitor': monitor_run, + 'entry_date': datetime.datetime.now()} + self.query_table.__table__.insert().execute(new_entry) + logging.info('\tUpdated the query history table') if __name__ == '__main__': From 9a0de82ac492f24053f6d89c6dc3ca11b9a2fc8e Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Mon, 8 Mar 2021 12:18:11 -0500 Subject: [PATCH 0018/1618] Trying to get around bug in defining jump_file --- .../miri_monitors/cosmic_ray_monitor.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py b/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py index 4a0654bfd..095f523cd 100644 --- a/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py +++ b/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py @@ -394,7 +394,9 @@ def process(self, file_list): for output_file in os.listdir(observation_dir): if 'jump' in output_file: - jump_file = os.path.join(observation_dir, output_file) + jump_filename = os.path.join(observation_dir, output_file) + else: + jump_filename = None if self.nints == 1: if '0_ramp_fit' in output_file: @@ -405,9 +407,9 @@ def process(self, file_list): rate_file = os.path.join(observation_dir, output_file) try: - jump_head, jump_data, jump_dq = self.get_jump_data(jump_file) + jump_head, jump_data, jump_dq = self.get_jump_data(jump_filename) except: - logging.info('Could not open jump file: {}'.format(jump_file)) + logging.info('Could not open jump file: {}'.format(jump_filename)) try: rate_data = self.get_rate_data(rate_file) From 4aa87ec765c2c88be2d2a6e08852d31051fc8234 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Mon, 8 Mar 2021 12:23:45 -0500 Subject: [PATCH 0019/1618] Trying to get around bug in defining rate_file --- jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py b/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py index 095f523cd..a605b7af3 100644 --- a/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py +++ b/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py @@ -401,10 +401,14 @@ def process(self, file_list): if self.nints == 1: if '0_ramp_fit' in output_file: rate_file = os.path.join(observation_dir, output_file) + else: + rate_file = None elif self.nints > 1: if '1_ramp_fit' in output_file: rate_file = os.path.join(observation_dir, output_file) + else: + rate_file = None try: jump_head, jump_data, jump_dq = self.get_jump_data(jump_filename) From 246b11bd814babbed41995d03f0212dd233525b0 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Mon, 8 Mar 2021 12:35:45 -0500 Subject: [PATCH 0020/1618] Trying to debug what is going on with these missing files/variables --- jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py b/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py index a605b7af3..b024fcea1 100644 --- a/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py +++ b/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py @@ -393,6 +393,8 @@ def process(self, file_list): # Next we analyze the cosmic rays in the new data for output_file in os.listdir(observation_dir): + logging.info(output_file) + if 'jump' in output_file: jump_filename = os.path.join(observation_dir, output_file) else: From 440ec7589e47eb50e8792161f1eaba2659ee31a6 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Mon, 8 Mar 2021 12:38:46 -0500 Subject: [PATCH 0021/1618] Trying to debug what is going on with these missing files/variables --- jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py b/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py index b024fcea1..f11aed8dc 100644 --- a/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py +++ b/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py @@ -393,6 +393,7 @@ def process(self, file_list): # Next we analyze the cosmic rays in the new data for output_file in os.listdir(observation_dir): + logging.info('output file') logging.info(output_file) if 'jump' in output_file: From 12ada1dbc23fc7faa261ddfc5d85f0db44c9df52 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Tue, 27 Jul 2021 10:27:56 -0400 Subject: [PATCH 0022/1618] Applied updates from @mengesser from his "develop" branch to this feature branch --- .../miri_monitors/cosmic_ray_monitor.py | 62 +++++++++++++++++-- 1 file changed, 58 insertions(+), 4 deletions(-) diff --git a/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py b/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py index f11aed8dc..20328aba0 100644 --- a/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py +++ b/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py @@ -355,6 +355,60 @@ def most_recent_search(self): return query_result + def possible_apers(self, inst): + """Return possible apertures to check for cosmic rays + + Parameters: + ---------- + inst: str + The name of the instrument of interest + + Returns: + ------- + apers: list + A list of possible apertures to check for the given + instrument + """ + if inst.lower() == 'nircam': + apers = ['NRCA1_FULL', + 'NRCA2_FULL', + 'NRCA3_FULL', + 'NRCA4_FULL', + 'NRCA5_FULL', + + 'NRCB1_FULL', + 'NRCB2_FULL', + 'NRCB3_FULL', + 'NRCB4_FULL', + 'NRCB5_FULL'] + + if inst.lower() == 'miri': + apers = ['MIRIM_FULL', + 'MIRIM_ILLUM', + 'MIRIM_BRIGHTSKY', + 'MIRIM_SUB256', + 'MIRIM_SUB128', + 'MIRIM_SUB64', + 'MIRIM_CORON1065', + 'MIRIM_CORON1140', + 'MIRIM_CORON1550', + 'MIRIM_CORONLYOT', + 'MIRIM_SLITLESSPRISM', + 'MIRIFU_CHANNEL1A', + 'MIRIFU_CHANNEL1B' + 'MIRIFU_CHANNEL1C', + 'MIRIFU_CHANNEL2A', + 'MIRIFU_CHANNEL2B' + 'MIRIFU_CHANNEL2C', + 'MIRIFU_CHANNEL3A', + 'MIRIFU_CHANNEL3B' + 'MIRIFU_CHANNEL3C', + 'MIRIFU_CHANNEL4A', + 'MIRIFU_CHANNEL4B', + 'MIRIFU_CHANNEL4C'] + + return apers + def process(self, file_list): """The main method for processing files. See module docstrings for further details. @@ -382,6 +436,7 @@ def process(self, file_list): copy_files([file_name], observation_dir) except: logging.info('Failed to copy {} to observation dir.'.format(file_name)) + pass # Next we run the pipeline on the files to get the proper outputs uncal_file = os.path.join(observation_dir, os.path.basename(file_name)) @@ -389,6 +444,7 @@ def process(self, file_list): pipeline_tools.calwebb_detector1_save_jump(uncal_file, observation_dir, ramp_fit=True, save_fitopt=False) except: logging.info('Failed to complete pipeline steps on {}.'.format(uncal_file)) + pass # Next we analyze the cosmic rays in the new data for output_file in os.listdir(observation_dir): @@ -502,7 +558,7 @@ def run(self): self.identify_tables() # Get a list of possible apertures - possible_apertures = list(Siaf(instrument).apernames) + possible_apertures = self.possible_apers(instrument) for aperture in possible_apertures: @@ -533,9 +589,7 @@ def run(self): # Next we copy new files to the working directory output_dir = os.path.join(get_config()['outputs'], 'cosmic_ray_monitor') - # self.data_dir = get_config()['local_test_dir'] # for testing purposes only - - self.data_dir = os.path.join(output_dir,'data') + self.data_dir = os.path.join(output_dir,'data') ensure_dir_exists(self.data_dir) cosmic_ray_files, not_copied = copy_files(new_filenames, self.data_dir) From d0a060667f4e2117cb23e7ff8e11a11e0aeac022 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Tue, 27 Jul 2021 17:41:29 -0400 Subject: [PATCH 0023/1618] Updated tests to reflect new filesystem structure --- jwql/tests/test_cosmic_ray_monitor.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/jwql/tests/test_cosmic_ray_monitor.py b/jwql/tests/test_cosmic_ray_monitor.py index d1083d6ab..95cf9f008 100644 --- a/jwql/tests/test_cosmic_ray_monitor.py +++ b/jwql/tests/test_cosmic_ray_monitor.py @@ -43,7 +43,7 @@ def define_test_data(nints): rate_data = np.ones((2, 10, 10)) filesystem = get_config()['filesystem'] - filename = os.path.join(filesystem, 'jw00608', 'jw00608002001_02101_00001_mirimage_rate.fits') + filename = os.path.join(filesystem, 'public', 'jw00608', 'jw00608002001', 'jw00608002001_02101_00001_mirimage_rate.fits') aperture = 'MIRIM_FULL' return data, rate_data, filename, aperture @@ -112,14 +112,14 @@ def test_magnitude(): coord = (1, 2, 1, 1) coord_gb = (1, 1, 1, 1) mag = cr.magnitude(coord, coord_gb, rate_data, data, header) - assert mag == -2.77504 + assert mag == -0.15904 cr.nints = 1 data, rate_data, filename, aperture = define_test_data(cr.nints) coord = (1, 1, 1) coord_gb = (0, 1, 1) mag = cr.magnitude(coord, coord_gb, rate_data, data, header) - assert mag == -2.77504 + assert mag == -0.15904 def test_get_cr_mags(): @@ -134,7 +134,7 @@ def test_get_cr_mags(): header = fits.getheader(filename) mags = cr.get_cr_mags(jump_locations, jump_locations_pre, rate_data, data, header) - assert mags == [-2.77504] + assert mags == [-0.15904] jump_locations = [(1, 2, 1, 1)] jump_locations_pre = [(1, 1, 1, 1)] @@ -142,7 +142,7 @@ def test_get_cr_mags(): data, rate_data, filename, aperture = define_test_data(cr.nints) mags = cr.get_cr_mags(jump_locations, jump_locations_pre, rate_data, data, header) - assert mags == [-2.77504] + assert mags == [-0.15904] def test_most_recent_search(): From 527cfe7a2bdbd0b1fb0f1ec4af09f5679ad37b87 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Mon, 2 Aug 2021 10:21:50 -0400 Subject: [PATCH 0024/1618] Rearranged some code to reduce number of if/else and try/except blocks --- .../miri_monitors/cosmic_ray_monitor.py | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py b/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py index 20328aba0..bbc1e7fc2 100644 --- a/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py +++ b/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py @@ -469,30 +469,32 @@ def process(self, file_list): else: rate_file = None - try: + if jump_filename: + jump_head, jump_data, jump_dq = self.get_jump_data(jump_filename) - except: - logging.info('Could not open jump file: {}'.format(jump_filename)) + jump_locations = self.get_jump_locations(jump_dq) + jump_locations_pre = self.group_before(jump_locations) - try: - rate_data = self.get_rate_data(rate_file) - except: - logging.info('Could not open rate file: {}'.format(rate_file)) + # Get observation time info + eff_time = jump_head['EFFEXPTM'] + obs_start_time = jump_head['EXPSTART'] + obs_end_time = jump_head['EXPEND'] + start_time = julian.from_jd(obs_start_time, fmt='mjd') + end_time = julian.from_jd(obs_end_time, fmt='mjd') - jump_locations = self.get_jump_locations(jump_dq) - jump_locations_pre = self.group_before(jump_locations) + cosmic_ray_num = len(jump_locations) + # cosmic_ray_rate = get_cr_rate(jump_locations, eff_time) - eff_time = jump_head['EFFEXPTM'] + else: + break - # Get observation time info - obs_start_time = jump_head['EXPSTART'] - obs_end_time = jump_head['EXPEND'] - start_time = julian.from_jd(obs_start_time, fmt='mjd') - end_time = julian.from_jd(obs_end_time, fmt='mjd') + if rate_file: + rate_data = self.get_rate_data(rate_file) + else: + rate_data = None - cosmic_ray_num = len(jump_locations) - # cosmic_ray_rate = get_cr_rate(jump_locations, eff_time) - cosmic_ray_mags = self.get_cr_mags(jump_locations, jump_locations_pre, rate_data, jump_data, jump_head) + if jump_data and rate_data: + cosmic_ray_mags = self.get_cr_mags(jump_locations, jump_locations_pre, rate_data, jump_data, jump_head) # Insert new data into database try: From 82c76fbbcebff4c02cbf52ee54c8a3d8803a2115 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Mon, 2 Aug 2021 10:29:46 -0400 Subject: [PATCH 0025/1618] Updated tests to use a file that exists in the new DMS build filesystem --- jwql/tests/test_cosmic_ray_monitor.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/jwql/tests/test_cosmic_ray_monitor.py b/jwql/tests/test_cosmic_ray_monitor.py index 95cf9f008..cee26a3f7 100644 --- a/jwql/tests/test_cosmic_ray_monitor.py +++ b/jwql/tests/test_cosmic_ray_monitor.py @@ -43,7 +43,7 @@ def define_test_data(nints): rate_data = np.ones((2, 10, 10)) filesystem = get_config()['filesystem'] - filename = os.path.join(filesystem, 'public', 'jw00608', 'jw00608002001', 'jw00608002001_02101_00001_mirimage_rate.fits') + filename = os.path.join(filesystem, 'public', 'jw00623', 'jw00623087001', 'jw00623087001_07101_00001_mirimage_rate.fits') aperture = 'MIRIM_FULL' return data, rate_data, filename, aperture @@ -112,14 +112,14 @@ def test_magnitude(): coord = (1, 2, 1, 1) coord_gb = (1, 1, 1, 1) mag = cr.magnitude(coord, coord_gb, rate_data, data, header) - assert mag == -0.15904 + assert mag == -2.77504 cr.nints = 1 data, rate_data, filename, aperture = define_test_data(cr.nints) coord = (1, 1, 1) coord_gb = (0, 1, 1) mag = cr.magnitude(coord, coord_gb, rate_data, data, header) - assert mag == -0.15904 + assert mag == -2.77504 def test_get_cr_mags(): @@ -134,7 +134,7 @@ def test_get_cr_mags(): header = fits.getheader(filename) mags = cr.get_cr_mags(jump_locations, jump_locations_pre, rate_data, data, header) - assert mags == [-0.15904] + assert mags == [-2.77504] jump_locations = [(1, 2, 1, 1)] jump_locations_pre = [(1, 1, 1, 1)] @@ -142,7 +142,7 @@ def test_get_cr_mags(): data, rate_data, filename, aperture = define_test_data(cr.nints) mags = cr.get_cr_mags(jump_locations, jump_locations_pre, rate_data, data, header) - assert mags == [-0.15904] + assert mags == [-2.77504] def test_most_recent_search(): From 0dd76137321d1fa3b355a693d98c848c1f5b2d08 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Mon, 2 Aug 2021 11:09:58 -0400 Subject: [PATCH 0026/1618] Added check for tests running on GitHub Actions to be skipped if central storage is needed --- jwql/tests/test_cosmic_ray_monitor.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/jwql/tests/test_cosmic_ray_monitor.py b/jwql/tests/test_cosmic_ray_monitor.py index cee26a3f7..06aaa17aa 100644 --- a/jwql/tests/test_cosmic_ray_monitor.py +++ b/jwql/tests/test_cosmic_ray_monitor.py @@ -27,6 +27,9 @@ from jwql.database.database_interface import MIRICosmicRayQueryHistory from jwql.utils.utils import get_config +ON_GITHUB_ACTIONS = '/home/runner' in os.path.expanduser('~') or '/Users/runner' in os.path.expanduser('~') + + def define_test_data(nints): """Define the data to test with. @@ -49,6 +52,7 @@ def define_test_data(nints): return data, rate_data, filename, aperture +@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') def test_get_jump_data(): """Test the ``get_jumpy_data`` function""" @@ -62,6 +66,7 @@ def test_get_jump_data(): assert type(dq) == np.ndarray +@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') def test_get_rate_data(): """Test the ``get_rate_data`` function""" @@ -101,6 +106,7 @@ def test_group_before(): assert cr.group_before(jump_locations) == [(1, 1, 1, 1)] +@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') def test_magnitude(): """Test the ``magnitude`` method""" @@ -122,6 +128,7 @@ def test_magnitude(): assert mag == -2.77504 +@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') def test_get_cr_mags(): """Test the ``get_cr_mags`` function""" @@ -145,6 +152,7 @@ def test_get_cr_mags(): assert mags == [-2.77504] +@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') def test_most_recent_search(): """Test the ``most_recent_search`` function""" @@ -159,6 +167,7 @@ def test_most_recent_search(): assert isinstance(result, float) +@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') def test_query_mast(): """Test the ``query_mast`` function""" From 2e13ddea4f8d13a1180792676acf5558d9e0d5f5 Mon Sep 17 00:00:00 2001 From: Michael Engesser Date: Mon, 16 Aug 2021 17:04:14 -0400 Subject: [PATCH 0027/1618] Addressing PR comments -Removed unused get_cr_rate function and lines invoking it. -Moved to common_monitors directory -Added API info to get_cr_mags function --- .../cosmic_ray_monitor.py | 386 +++++++++--------- 1 file changed, 186 insertions(+), 200 deletions(-) rename jwql/instrument_monitors/{miri_monitors => common_monitors}/cosmic_ray_monitor.py (64%) diff --git a/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py b/jwql/instrument_monitors/common_monitors/cosmic_ray_monitor.py similarity index 64% rename from jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py rename to jwql/instrument_monitors/common_monitors/cosmic_ray_monitor.py index bbc1e7fc2..662b892d5 100644 --- a/jwql/instrument_monitors/miri_monitors/cosmic_ray_monitor.py +++ b/jwql/instrument_monitors/common_monitors/cosmic_ray_monitor.py @@ -54,7 +54,7 @@ from jwql.utils.logging_functions import configure_logging from jwql.utils.logging_functions import log_info from jwql.utils.logging_functions import log_fail -from jwql.utils.utils import copy_files, ensure_dir_exists, filesystem_path, get_config +from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path class CosmicRay: @@ -90,8 +90,8 @@ class CosmicRay: ``NRCA1_FULL``) query_table : sqlalchemy table - Table containing the history of cosmic ray monitor queries to - MAST for each instrument/aperture combination + Table containing the history of cosmic ray monitor queries to MAST + for each instrument/aperture combination stats_table : sqlalchemy table Table containing cosmic ray analysis results. Number and @@ -108,129 +108,132 @@ class CosmicRay: """ def __init__(self): - """Initialize an instance of the ``CosmicRay`` class.""" + """Initialize an instance of the ``Cosmic_Ray`` class.""" - def get_cr_mags(self, jump_locations, jump_locations_pre, rateints, jump_data, jump_head): + def identify_tables(self): + """Determine which database tables to use for a run of the + cosmic ray monitor. + + Uses the instrument variable to get the mixed-case instrument + name, and uses that name to find the query and stats tables + for that instrument. + """ + + mixed_case_name = JWST_INSTRUMENT_NAMES_MIXEDCASE[self.instrument] + self.query_table = eval('{}CosmicRayQueryHistory'.format(mixed_case_name)) + self.stats_table = eval('{}CosmicRayStats'.format(mixed_case_name)) + + def get_cr_mags(self, jump_locs, jump_locs_pre, rateints, jump_data, jump_head): """Gets the magnitude of each cosmic ray. Computes a list of magnitudes using the coordinate of the detected jump compared to the magnitude of the same pixel in the group prior to the jump. - Parameters + Parameters: ---------- - jump_locations : list + + jump_locs: list List of coordinates to a pixel marked with a jump. - jump_locations_pre : list - List of matching coordinates one group before - ``jump_locations``. + jump_locs_pre: list + List of matching coordinates one group before jump_locs. - rateints : ndarray obj - Array in ``DN/s``. + rateints: ndarray + Array in DN/s. - jump_data : + jump_data: ndarray + Ndarray containing image data cube - jump_head : + jump_head: FITS Header + FITS header unit containing information about the jump data - Returns + Returns: ------- - mags : list + + mags: list A list of cosmic ray magnitudes corresponding to each jump. + """ mags = [] - for coord, coord_gb in zip(jump_locations, jump_locations_pre): + for coord, coord_gb in zip(jump_locs, jump_locs_pre): mags.append(self.magnitude(coord, coord_gb, rateints, jump_data, jump_head)) return mags - - def get_cr_rate(self, jump_locations, time): - """Computes the rate of cosmic ray impacts in a given time. - - Parameters - ---------- - jump_locations : list - List of coordinates to a pixel marked with a jump. - - time : int or float - Time over which to compute the rate. Nominally the - effective exposure time. - """ - - return len(jump_locations) / time + def get_jump_data(self, jump_filename): - """Opens and reads a given ``.FITS`` file containing cosmic - rays. + """Opens and reads a given .FITS file containing cosmic rays. - Parameters + Parameters: ---------- - jump_filename : str + jump_filename: str Path to file. - Returns + Returns: ------- - header : FITS header + head: FITS header Header containing file information - data : obj + data: NoneType FITS data - dq : ndarray obj + dq: ndarray Data Quality array containing jump flags. + """ hdu = fits.open(jump_filename) - header = hdu[0].header + head = hdu[0].header data = hdu[1].data dq = hdu[3].data hdu.close() - return header, data, dq + return head, data, dq - def get_jump_locations(self, dq): + def get_jump_locs(self, dq): """Uses the data quality array to find the location of all jumps in the data. - Parameters + Parameters: ---------- - dq : ndarray obj + dq: ndarray Data Quality array containing jump flags. - Returns + Returns: ------- - jump_locations : list + jump_locs: list List of coordinates to a pixel marked with a jump. """ - flag_locations = np.where(dq == 4) + temp = np.where(dq == 4) - jump_locations = [] + jump_locs = [] if self.nints > 1: - for i in range(len(flag_locations[0])): - jump_locations.append((flag_locations[0][i], flag_locations[1][i], flag_locations[2][i], flag_locations[3][i])) + for i in range(len(temp[0])): + jump_locs.append((temp[0][i], temp[1][i], temp[2][i], temp[3][i])) else: - for i in range(len(flag_locations[0])): - jump_locations.append((flag_locations[0][i], flag_locations[1][i], flag_locations[2][i])) + for i in range(len(temp[0])): + jump_locs.append((temp[0][i], temp[1][i], temp[2][i])) - return jump_locations + return jump_locs def get_rate_data(self, rate_filename): - """Opens and reads a given rate ``.FITS`` file. + """Opens and reads a given .FITS file. - Parameters + Parameters: ---------- - rate_filename : str + rate_filename: str Path to file. Returns: ------- - data : obj + data: NoneType FITS data """ @@ -238,92 +241,79 @@ def get_rate_data(self, rate_filename): return data - def group_before(self, jump_locations): + def group_before(self, jump_locs): """Creates a list of coordinates one group before given jump coordinates. - Parameters + Parameters: ---------- - jump_locations : list + jump_locs: list List of coordinates to a pixel marked with a jump. - Returns + Returns: ------- - jump_locations_pre : list - List of matching coordinates one group before - ``jump_locations``. + jump_locs_pre: list + List of matching coordinates one group before jump_locs. """ - jump_locations_pre = [] + jump_locs_pre = [] if self.nints > 1: - for coord in jump_locations: - jump_locations_pre.append((coord[0], coord[1] - 1, coord[2], coord[3])) + for coord in jump_locs: + jump_locs_pre.append((coord[0], coord[1] - 1, coord[2], coord[3])) else: - for coord in jump_locations: - jump_locations_pre.append((coord[0] - 1, coord[1], coord[2])) + for coord in jump_locs: + jump_locs_pre.append((coord[0] - 1, coord[1], coord[2])) - return jump_locations_pre - - def identify_tables(self): - """Determine which database tables to use for a run of the - cosmic ray monitor. + return jump_locs_pre - Uses the instrument variable to get the mixed-case instrument - name, and uses that name to find the query and stats tables - for that instrument. - """ - - mixed_case_name = JWST_INSTRUMENT_NAMES_MIXEDCASE[self.instrument] - self.query_table = eval('{}CosmicRayQueryHistory'.format(mixed_case_name)) - self.stats_table = eval('{}CosmicRayStats'.format(mixed_case_name)) - - def magnitude(self, coord, coord_gb, rateints, data, header): + def magnitude(self, coord, coord_gb, rateints, data, head): """Calculates the magnitude of a list of jumps given their coordinates in an array of pixels. - Parameters + Parameters: ---------- - coord : tuple + coord: tuple Coordinate of jump. - coord_gb : tuple + coord_gb: tuple Coordinate of jump pixel one group before. - header : FITS header + head: FITS header Header containing file information. - rateints : ndarray - Array in ``DN/s``. + rateints: ndarray + Array in DN/s. - Returns + Returns: ------- - cr_mag : float + cr_mag: float the magnitude of the cosmic ray """ - group_time = header['TGROUP'] + grouptime = head['TGROUP'] if self.nints == 1: rate = rateints[coord[-2]][coord[-1]] cr_mag = data[0][coord[0]][coord[1]][coord[2]] \ - data[0][coord_gb[0]][coord_gb[1]][coord_gb[2]] \ - - rate * group_time + - rate * grouptime else: rate = rateints[coord[0]][coord[-2]][coord[-1]] - cr_mag = data[coord] - data[coord_gb] - rate * group_time + cr_mag = data[coord] - data[coord_gb] - rate * grouptime return cr_mag + def most_recent_search(self): - """Adapted from the Dark Monitor (Bryan Hilbert) + """Adapted from Dark Monitor (Bryan Hilbert) - Query the query history table in the ``jwqldb`` database and - return the information on the most recent query for the given - ``aperture_name`` where the cosmic ray monitor was executed. + Query the query history database and return the information + on the most recent query for the given ``aperture_name`` where + the cosmic ray monitor was executed. - Returns + Returns: ------- query_result : float Date (in MJD) of the ending range of the previous MAST @@ -350,6 +340,8 @@ def most_recent_search(self): query_result = 57357.0 # a.k.a. Dec 1, 2015 == CV3 logging.info(('\tNo query history for {}. Beginning search date will be set to {}.' .format(self.aperture, query_result))) + #elif query_count > 1: + # raise ValueError('More than one "most recent" query?') else: query_result = query[0].end_time_mjd @@ -421,80 +413,67 @@ def process(self, file_list): """ for file_name in file_list: - - # Only process uncal files if 'uncal' in file_name: + dir_name = '_'.join(file_name.split('_')[:4]) # file_name[51:76] - # Define some useful parameters - dir_name = '_'.join(file_name.split('_')[:4]) # aka file_name[51:76] - self.nints = fits.getheader(file_name)['NINTS'] + self.obs_dir = os.path.join(self.data_dir, dir_name) + ensure_dir_exists(self.obs_dir) + + head = fits.getheader(file_name) + self.nints = head['NINTS'] - # Ensure the directory to the data exists. If so, copy over files - observation_dir = os.path.join(self.data_dir, dir_name) - ensure_dir_exists(observation_dir) try: - copy_files([file_name], observation_dir) + copy_files([file_name], self.obs_dir) except: logging.info('Failed to copy {} to observation dir.'.format(file_name)) pass # Next we run the pipeline on the files to get the proper outputs - uncal_file = os.path.join(observation_dir, os.path.basename(file_name)) + uncal_file = os.path.join(self.obs_dir, os.path.basename(file_name)) + try: - pipeline_tools.calwebb_detector1_save_jump(uncal_file, observation_dir, ramp_fit=True, save_fitopt=False) + pipeline_tools.calwebb_detector1_save_jump(uncal_file, self.obs_dir, ramp_fit=True, save_fitopt=False) except: logging.info('Failed to complete pipeline steps on {}.'.format(uncal_file)) pass # Next we analyze the cosmic rays in the new data - for output_file in os.listdir(observation_dir): - - logging.info('output file') - logging.info(output_file) + for output_file in os.listdir(self.obs_dir): if 'jump' in output_file: - jump_filename = os.path.join(observation_dir, output_file) - else: - jump_filename = None + jump_file = os.path.join(self.obs_dir, output_file) if self.nints == 1: if '0_ramp_fit' in output_file: - rate_file = os.path.join(observation_dir, output_file) - else: - rate_file = None + rate_file = os.path.join(self.obs_dir, output_file) elif self.nints > 1: if '1_ramp_fit' in output_file: - rate_file = os.path.join(observation_dir, output_file) - else: - rate_file = None - - if jump_filename: + rate_file = os.path.join(self.obs_dir, output_file) - jump_head, jump_data, jump_dq = self.get_jump_data(jump_filename) - jump_locations = self.get_jump_locations(jump_dq) - jump_locations_pre = self.group_before(jump_locations) + try: + jump_head, jump_data, jump_dq = self.get_jump_data(jump_file) + except: + logging.info('Could not open jump file: {}'.format(jump_file)) - # Get observation time info - eff_time = jump_head['EFFEXPTM'] - obs_start_time = jump_head['EXPSTART'] - obs_end_time = jump_head['EXPEND'] - start_time = julian.from_jd(obs_start_time, fmt='mjd') - end_time = julian.from_jd(obs_end_time, fmt='mjd') + try: + rate_data = self.get_rate_data(rate_file) + except: + logging.info('Could not open rate file: {}'.format(rate_file)) - cosmic_ray_num = len(jump_locations) - # cosmic_ray_rate = get_cr_rate(jump_locations, eff_time) + jump_locs = self.get_jump_locs(jump_dq) + jump_locs_pre = self.group_before(jump_locs) - else: - break + eff_time = jump_head['EFFEXPTM'] - if rate_file: - rate_data = self.get_rate_data(rate_file) - else: - rate_data = None + # Get observation time info + obs_start_time = jump_head['EXPSTART'] + obs_end_time = jump_head['EXPEND'] + start_time = julian.from_jd(obs_start_time, fmt='mjd') + end_time = julian.from_jd(obs_end_time, fmt='mjd') - if jump_data and rate_data: - cosmic_ray_mags = self.get_cr_mags(jump_locations, jump_locations_pre, rate_data, jump_data, jump_head) + cosmic_ray_num = len(jump_locs) + cosmic_ray_mags = self.get_cr_mags(jump_locs, jump_locs_pre, rate_data, jump_data, jump_head) # Insert new data into database try: @@ -507,32 +486,11 @@ def process(self, file_list): 'magnitude': cosmic_ray_mags } self.stats_table.__table__.insert().execute(cosmic_ray_db_entry) + logging.info("Successfully inserted into database. \n") except: logging.info("Could not insert entry into database. \n") - def query_mast(self): - """Use ``astroquery`` to search MAST for cosmic ray data - - Parameters: - ---------- - start_date : float - Starting date for the search in MJD - end_date : float - Ending date for the search in MJD - - Returns - ------- - result : list - List of dictionaries containing the query results - """ - - data_product = JWST_DATAPRODUCTS - parameters = {"date_obs_mjd": {"min": self.query_start, "max": self.query_end}, "apername": self.aperture} - result = monitor_mast.instrument_inventory(self.instrument, data_product, add_filters=parameters, return_data=True) - - return result - @log_fail @log_info def run(self): @@ -550,18 +508,19 @@ def run(self): self.query_end = Time.now().mjd for instrument in JWST_INSTRUMENT_NAMES: - - # Currently only supports MIRI and NIRCam if instrument == 'miri' or instrument == 'nircam': - self.instrument = instrument # Identify which tables to use self.identify_tables() # Get a list of possible apertures + #possible_apertures = list(Siaf(instrument).apernames) possible_apertures = self.possible_apers(instrument) + # Use this line instead to save time while testing + #possible_apertures = ['MIRIM_FULL', 'NRCB4_FULL'] + for aperture in possible_apertures: logging.info('') @@ -569,50 +528,77 @@ def run(self): self.aperture = aperture - # Query MAST for new data + # We start by querying MAST for new data self.query_start = self.most_recent_search() + logging.info('\tMost recent query: {}'.format(self.query_start)) + new_entries = self.query_mast() + + # for testing purposes only + #new_filenames = get_config()['local_test_data'] new_filenames = [] - if not new_entries['data']: - logging.info('\tNo new data to process') + for file_entry in new_entries['data']: + try: + new_filenames.append(filesystem_path(file_entry['filename'])) + except FileNotFoundError: + logging.info('\t{} not found in target directory'.format(file_entry['filename'])) + except ValueError: + logging.info( + '\tProvided file {} does not follow JWST naming conventions.'.format(file_entry['filename'])) - else: - for file_entry in new_entries['data']: - try: - new_filenames.append(filesystem_path(file_entry['filename'])) - except FileNotFoundError: - logging.info('\t{} not found in target directory'.format(file_entry['filename'])) - except ValueError: - logging.info( - '\tProvided file {} does not follow JWST naming conventions.'.format(file_entry['filename'])) + # Next we copy new files to the working directory + output_dir = os.path.join(get_config()['outputs'], 'cosmic_ray_monitor') - # Next we copy new files to the working directory - output_dir = os.path.join(get_config()['outputs'], 'cosmic_ray_monitor') + #self.data_dir = get_config()['local_test_dir'] # for testing purposes only - self.data_dir = os.path.join(output_dir,'data') - ensure_dir_exists(self.data_dir) + self.data_dir = os.path.join(output_dir,'data') + ensure_dir_exists(self.data_dir) - cosmic_ray_files, not_copied = copy_files(new_filenames, self.data_dir) + cosmic_ray_files, not_copied = copy_files(new_filenames, self.data_dir) - self.process(cosmic_ray_files) + self.process(cosmic_ray_files) - monitor_run = True + monitor_run = True - new_entry = {'instrument': self.instrument, - 'aperture': self.aperture, - 'start_time_mjd': self.query_start, - 'end_time_mjd': self.query_end, - 'files_found': len(new_entries), - 'run_monitor': monitor_run, - 'entry_date': datetime.datetime.now()} - self.query_table.__table__.insert().execute(new_entry) - logging.info('\tUpdated the query history table') + new_entry = {'instrument': self.instrument, + 'aperture': self.aperture, + 'start_time_mjd': self.query_start, + 'end_time_mjd': self.query_end, + 'files_found': len(new_entries), + 'run_monitor': monitor_run, + 'entry_date': datetime.datetime.now()} + self.query_table.__table__.insert().execute(new_entry) + logging.info('\tUpdated the query history table') + def query_mast(self): + """Use astroquery to search MAST for cosmic ray data -if __name__ == '__main__': + Parameters: + ---------- + start_date : float + Starting date for the search in MJD + end_date : float + Ending date for the search in MJD + + Returns + ------- + result : list + List of dictionaries containing the query results + """ + data_product = JWST_DATAPRODUCTS + parameters = {"date_obs_mjd": {"min": self.query_start, "max": self.query_end}, "apername": self.aperture} + + result = monitor_mast.instrument_inventory(self.instrument, data_product, + add_filters=parameters, + return_data=True) + + return result + + +if __name__ == '__main__': # Configure logging module = os.path.basename(__file__).strip('.py') configure_logging(module) From a14a144d4db9d93af6bdcb3f4a30c4b0847ce223 Mon Sep 17 00:00:00 2001 From: Michael Engesser Date: Tue, 21 Sep 2021 17:02:48 -0400 Subject: [PATCH 0028/1618] Updating Cosmic Ray Monitor Plots Adding support for NIRCam plots. Still needs testing - waiting on monitor to be working. --- jwql/utils/constants.py | 2 + jwql/website/apps/jwql/bokeh_containers.py | 88 +++++++++++++++++++ .../monitor_cosmic_rays_bokeh.py | 5 +- jwql/website/apps/jwql/monitor_views.py | 4 +- jwql/website/apps/jwql/urls.py | 2 +- 5 files changed, 96 insertions(+), 5 deletions(-) diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py index eb2cc2a60..8885091d3 100644 --- a/jwql/utils/constants.py +++ b/jwql/utils/constants.py @@ -210,6 +210,7 @@ INSTRUMENT_MONITOR_DATABASE_TABLES = { 'dark_monitor': ['_dark_dark_current', '_dark_pixel_stats', '_dark_query_history'], 'bad_pixel_monitor': ['_bad_pixel_stats', '_bad_pixel_query_history']} + 'cosmic_ray_monitor': ['_cosmic_ray_stats', '_cosmic_ray_query_history']} INSTRUMENT_SERVICE_MATCH = { 'FGS': 'Mast.Jwst.Filtered.Fgs', @@ -268,6 +269,7 @@ ('Mean Dark Current Rate Monitor', '/nircam/dark_monitor'), ('Bad Pixel Monitor', '/nircam/bad_pixel_monitor'), ('Photometric Stability Monitor', '#')], + ('Cosmic Ray Monitor', '/nircam/cosmic_ray_monitor'), 'niriss': [('Bad Pixel Monitor', '/niriss/bad_pixel_monitor'), ('Readnoise Monitor', '/niriss/readnoise_monitor'), ('AMI Calibrator Monitor', '#'), diff --git a/jwql/website/apps/jwql/bokeh_containers.py b/jwql/website/apps/jwql/bokeh_containers.py index b375657d9..3d0b2c49d 100644 --- a/jwql/website/apps/jwql/bokeh_containers.py +++ b/jwql/website/apps/jwql/bokeh_containers.py @@ -181,7 +181,95 @@ def bias_monitor_tabs(instrument): return div, script +def cosmic_ray_monitor_tabs(instrument): + """Creates the various tabs of the cosmic monitor results page. + Parameters + ---------- + instrument : str + The JWST instrument of interest (e.g. ``nircam``). + + Returns + ------- + div : str + The HTML div to render cosmic ray monitor plots + script : str + The JS script to render cosmic ray monitor plots + """ + + full_apertures = FULL_FRAME_APERTURES[instrument.upper()] + + templates_all_apertures = {} + for aperture in full_apertures: + + # Start with default values for instrument and aperture because + # BokehTemplate's __init__ method does not allow input arguments + monitor_template = monitor_pages.CosmicRayMonitor() + + # Set instrument and monitor using CosmicRayMonitor's setters + monitor_template.aperture_info = (instrument, aperture) + templates_all_apertures[aperture] = monitor_template + + # Histogram tab + histograms_all_apertures = [] + for aperture_name, template in templates_all_apertures.items(): + histogram = template.refs["cosmic_ray_histogram"] + histogram.sizing_mode = "scale_width" # Make sure the sizing is adjustable + histograms_all_apertures.append(histogram) + + if instrument == 'NIRCam': + a1, a2, a3, a4, a5, b1, b2, b3, b4, b5 = histograms_all_apertures + histogram_layout = layout( + [a2, a4, b3, b1], + [a1, a3, b4, b2], + [a5, b5] + ) + + elif instrument in ['MIRI']: + single_aperture = histograms_all_apertures[0] + histogram_layout = layout( + [single_aperture] + ) + + histogram_layout.sizing_mode = "scale_width" # Make sure the sizing is adjustable + histogram_tab = Panel(child=histogram_layout, title="Histogram") + + # Cosmic Ray v. time tab + lines_all_apertures = [] + for aperture_name, template in templates_all_apertures.items(): + line = template.refs["cosmic_ray_history_figure"] + line.title.align = "center" + line.title.text_font_size = "20px" + line.sizing_mode = "scale_width" # Make sure the sizing is adjustable + lines_all_apertures.append(line) + + if instrument == 'NIRCam': + a1, a2, a3, a4, a5, b1, b2, b3, b4, b5 = lines_all_apertures + line_layout = layout( + [a2, a4, b3, b1], + [a1, a3, b4, b2], + [a5, b5] + ) + + elif instrument in ['MIRI']: + single_aperture = lines_all_apertures[0] + line_layout = layout( + [single_aperture] + ) + + + line_layout.sizing_mode = "scale_width" # Make sure the sizing is adjustable + line_tab = Panel(child=line_layout, title="Trending") + + # Build tabs + tabs = Tabs(tabs=[histogram_tab, line_tab]) + + # Return tab HTML and JavaScript to web app + script, div = components(tabs) + + return div, script + + def dark_monitor_tabs(instrument): """Creates the various tabs of the dark monitor results page. diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_cosmic_rays_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_cosmic_rays_bokeh.py index e218809e7..db1a04a31 100644 --- a/jwql/website/apps/jwql/monitor_pages/monitor_cosmic_rays_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_cosmic_rays_bokeh.py @@ -25,6 +25,7 @@ from jwql.bokeh_templating import BokehTemplate from jwql.database.database_interface import session from jwql.database.database_interface import MIRICosmicRayQueryHistory, MIRICosmicRayStats +from jwql.database.database_interface import NIRCamCosmicRayQueryHistory, NIRCamCosmicRayStats from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE from jwql.utils.utils import get_config, filesystem_path @@ -140,7 +141,7 @@ def _update_cosmic_ray_v_time(self): self.refs['cosmic_ray_y_range'].start = min(self.count) self.refs['cosmic_ray_y_range'].end = max(self.count) - self.refs['cosmic_ray_history_figure'].title.text = 'MIRIM_FULL Cosmic Ray History' + self.refs['cosmic_ray_history_figure'].title.text = '{} Cosmic Ray History'.format(self.aperture) self.refs['cosmic_ray_history_figure'].title.align = "center" self.refs['cosmic_ray_history_figure'].title.text_font_size = "20px" @@ -154,7 +155,7 @@ def _update_cosmic_ray_histogram(): self.refs['hist_y_range'].start = 0 self.refs['hist_y_range'].end = max(mags) - self.refs['cosmic_ray_histogram'].title.text = 'MIRIM_FULL Cosmic Ray Intensities' + self.refs['cosmic_ray_histogram'].title.text = '{}} Cosmic Ray Intensities'.format(self.aperture) self.refs['cosmic_ray_histogram'].title.align = "center" self.refs['cosmic_ray_histogram'].title.text_font_size = "20px" diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py index c091d1733..176e9cb61 100644 --- a/jwql/website/apps/jwql/monitor_views.py +++ b/jwql/website/apps/jwql/monitor_views.py @@ -103,7 +103,7 @@ def bias_monitor(request, inst): return render(request, template, context) -def cosmic_ray_monitor(request, inst='MIRI'): +def cosmic_ray_monitor(request, inst): """Generate the cosmic ray monitor page for a given instrument Parameters @@ -119,7 +119,7 @@ def cosmic_ray_monitor(request, inst='MIRI'): """ # Ensure the instrument is correctly capitalized - inst = 'MIRI' #other instruments not currently supported + inst = inst.upper() tabs_components = bokeh_containers.cosmic_ray_monitor_tabs(inst) diff --git a/jwql/website/apps/jwql/urls.py b/jwql/website/apps/jwql/urls.py index 6b5e36795..5fb5832fa 100644 --- a/jwql/website/apps/jwql/urls.py +++ b/jwql/website/apps/jwql/urls.py @@ -64,7 +64,6 @@ # MIRI-specific views path('miri/miri_data_trending/', views.miri_data_trending, name='miri_data_trending'), - path('miri/cosmic_ray_monitor/', monitor_views.cosmic_ray_monitor, name='cosmic_ray_monitor'), # NIRSpec-specific views path('nirspec/nirspec_data_trending/', views.nirspec_data_trending, name='nirspec_data_trending'), @@ -74,6 +73,7 @@ re_path(r'^(?P({}))/bad_pixel_monitor/$'.format(instruments), monitor_views.bad_pixel_monitor, name='bad_pixel_monitor'), re_path(r'^(?P({}))/bias_monitor/$'.format(instruments), monitor_views.bias_monitor, name='bias_monitor'), re_path(r'^(?P({}))/readnoise_monitor/$'.format(instruments), monitor_views.readnoise_monitor, name='readnoise_monitor'), + re_path(r'^(?P({}))/cosmic_ray_monitor/$'.format(instruments), monitor_views.cosmic_ray_monitor, name='cosmic_ray_monitor'), # Main site views path('about/', views.about, name='about'), From 05493866971a076e28d2db4b1e118d02e8b92f9d Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 19 Oct 2021 13:54:50 -0400 Subject: [PATCH 0029/1618] Add empty monitor file --- .../common_monitors/edb_mnemonics_monitor.py | 139 ++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 jwql/instrument_monitors/common_monitors/edb_mnemonics_monitor.py diff --git a/jwql/instrument_monitors/common_monitors/edb_mnemonics_monitor.py b/jwql/instrument_monitors/common_monitors/edb_mnemonics_monitor.py new file mode 100644 index 000000000..00cfa7383 --- /dev/null +++ b/jwql/instrument_monitors/common_monitors/edb_mnemonics_monitor.py @@ -0,0 +1,139 @@ +#! /usr/bin/env python + +"""Engineering Database Mnemonics Trending Monitor (EDB Trending Monitor) + +more description here +""" + + + + +from jwql.edb import engineering_database as ed + + + +# To query the EDB for a single mnemonic +starttime = Time('2019-01-16T00:00:00.000') +endtime = Time('2019-01-16T00:01:00.000') +mnemonic = 'IMIR_HK_ICE_SEC_VOLT4' +mnemonic_data = ed.get_mnemonic(mnemonic, starttime, endtime) + +""" +m.mnemonic_identifier +Out[12]: 'IMIR_HK_ICE_SEC_VOLT4' + +In [14]: m.requested_start_time +Out[14]: