diff --git a/CHANGES.rst b/CHANGES.rst index 2452df232..6280f83ca 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,79 @@ +## What's Changed + +1.2.0 (2023-11-21) +================== + +Web Application +~~~~~~~~~~~~~~~ +- Hover over proposal thumbnails by @shanosborne in https://github.com/spacetelescope/jwql/pull/927 +- Not found images replaced with default image by @BradleySappington in https://github.com/spacetelescope/jwql/pull/949 +- Remove second set of colorbar tick labels from preview images by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/963 +- Split program page by obsnum by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/948 +- Interactive preview images by @BradleySappington in https://github.com/spacetelescope/jwql/pull/965 +- Alert user to non-existant proposal number by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/994 +- Fix bad thumbnails by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/997 +- Explore by group type by @BradleySappington in https://github.com/spacetelescope/jwql/pull/1000 +- Default scaling params for preview images where all pix are NaN by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1006 +- Explore int grp difference by @BradleySappington in https://github.com/spacetelescope/jwql/pull/1020 +- Add inst select home search by @mengesser in https://github.com/spacetelescope/jwql/pull/1010 +- Allow proposal thumbnails to come from dark thumbnails by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1001 +- Adding Next and Previous Buttons by @mfixstsci in https://github.com/spacetelescope/jwql/pull/978 +- Build archive page from database by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1030 +- Select anomaly name formatting by @rcooper295 in https://github.com/spacetelescope/jwql/pull/1053 +- Implement "Viewed" Button and Filtering by @BradleySappington in https://github.com/spacetelescope/jwql/pull/1049 +- fixes for no data found on server by @penaguerrero in https://github.com/spacetelescope/jwql/pull/1042 +- Filter archive page by @BradleySappington in https://github.com/spacetelescope/jwql/pull/1065 +- Exposure page: Default to show rate img, keep order of radio buttons constant by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1111 +- Filter type by @BradleySappington in https://github.com/spacetelescope/jwql/pull/1092 +- Sort obs date by @BradleySappington in https://github.com/spacetelescope/jwql/pull/1113 +- added scattered light, light saber anomalies for niriss by @rcooper295 in https://github.com/spacetelescope/jwql/pull/1133 +- Add new NIRCam anomalies by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1136 +- Add Logging View by @mfixstsci in https://github.com/spacetelescope/jwql/pull/1080 +- Remove bokeh templating from Dark monitor by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1153 +- Fix API view for filenames per proposal and preview images by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1046 +- Create archive page asynchronously by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1203 +- Adding sort by category by @mfixstsci in https://github.com/spacetelescope/jwql/pull/1155 +- Improve bad pixel monitor plots by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1202 +- List viewed/new status by instrument by @melanieclarke in https://github.com/spacetelescope/jwql/pull/1197 +- Display all detectors by @melanieclarke in https://github.com/spacetelescope/jwql/pull/1213 +- Remove bokeh templating from Bias Monitor by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1230 +- Reformat query page functionality by @BradleySappington in https://github.com/spacetelescope/jwql/pull/1240 +- Add download links to image pages by @melanieclarke in https://github.com/spacetelescope/jwql/pull/1283 +- Improve explore image by @melanieclarke in https://github.com/spacetelescope/jwql/pull/1282 +- Remove Bokeh Templating from Readnoise Monitor by @mfixstsci in https://github.com/spacetelescope/jwql/pull/1238 +- Move Date Range selection to Query Page by @BradleySappington in https://github.com/spacetelescope/jwql/pull/1396 + +Project & API Documentation +~~~~~~~~~~~~~~~~~~~~~~~~~~~ +- Make Readnoise Monitor less memory-intensive by @bsunnquist in https://github.com/spacetelescope/jwql/pull/922 +- Speed up MAST queries by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/942 +- Lock python by @BradleySappington in https://github.com/spacetelescope/jwql/pull/955 +- Reduce memory usage by the dark monitor by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/908 +- Fix clipping logic in preview image scaling by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1011 +- Cosmic ray monitor (via mengesser:develop) by @bourque in https://github.com/spacetelescope/jwql/pull/700 +- nirspec TA monitors: wata and msata by @penaguerrero in https://github.com/spacetelescope/jwql/pull/888 +- allow per-instrument database table reset by @york-stsci in https://github.com/spacetelescope/jwql/pull/1031 +- Celery monitors by @york-stsci in https://github.com/spacetelescope/jwql/pull/921 +- Raise the upper limit on number of returned Mast entries by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1048 +- Added chunked iteration to CR monitor by @york-stsci in https://github.com/spacetelescope/jwql/pull/1066 +- Created a clone_tables script by @york-stsci in https://github.com/spacetelescope/jwql/pull/1059 +- Add EDB telemetry monitor by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/832 +- Tweaks to mnemonics queried by the EDB monitor by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1115 +- Fix failing MIRI dark montior by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1129 +- Unit test cleanup by @melanieclarke in https://github.com/spacetelescope/jwql/pull/1167 +- Catch missing obs in navigation data by @melanieclarke in https://github.com/spacetelescope/jwql/pull/1206 +- Update preview image generator to handle new NaN pixels by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1212 +- drop support for Python 3.8 by @zacharyburnett in https://github.com/spacetelescope/jwql/pull/1249 +- Adding Python 3.10 Support to Testing Matrix by @mfixstsci in https://github.com/spacetelescope/jwql/pull/1257 +- Remove mast monitor by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1256 +- Improve general JWQL queries by @melanieclarke in https://github.com/spacetelescope/jwql/pull/1281 +- Prepare models for external postgres db by @BradleySappington in https://github.com/spacetelescope/jwql/pull/1287 +- Pipeline skip already run steps by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1262 +- Include migrations in source control by @BradleySappington in https://github.com/spacetelescope/jwql/pull/1364 +- Add step_args to calwebb_detector1_save_jump by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1358 + +--Full Changelog--: https://github.com/spacetelescope/jwql/compare/1.1.0...1.2.0 + Unreleased ========== diff --git a/jwql/instrument_monitors/common_monitors/readnoise_monitor.py b/jwql/instrument_monitors/common_monitors/readnoise_monitor.py index 5e4f7bb7f..833db0fb7 100755 --- a/jwql/instrument_monitors/common_monitors/readnoise_monitor.py +++ b/jwql/instrument_monitors/common_monitors/readnoise_monitor.py @@ -413,7 +413,7 @@ def process(self, file_list): processed_file = file.replace("uncal", "refpix") if not os.path.isfile(processed_file): files_to_calibrate.append(file) - + # Run the files through the necessary pipeline steps outputs = run_parallel_pipeline(files_to_calibrate, "uncal", "refpix", self.instrument) @@ -422,7 +422,7 @@ def process(self, file_list): # Get relevant header information for this file self.get_metadata(filename) - + if filename in outputs: processed_file = outputs[filename] else: @@ -498,7 +498,10 @@ def process(self, file_list): # Construct new entry for this file for the readnoise database table. # Can't insert values with numpy.float32 datatypes into database # so need to change the datatypes of these values. - readnoise_db_entry = {'uncal_filename': filename, + # + # Store files as file name only (file path will be built at retrieval based + # on runtime configuration) + readnoise_db_entry = {'uncal_filename': os.path.basename(filename), 'aperture': self.aperture, 'detector': self.detector, 'subarray': self.subarray, @@ -506,12 +509,12 @@ def process(self, file_list): 'nints': self.nints, 'ngroups': self.ngroups, 'expstart': self.expstart, - 'readnoise_filename': readnoise_outfile, + 'readnoise_filename': os.path.basename(readnoise_outfile), 'full_image_mean': float(full_image_mean), 'full_image_stddev': float(full_image_stddev), 'full_image_n': full_image_n.astype(float), 'full_image_bin_centers': full_image_bin_centers.astype(float), - 'readnoise_diff_image': readnoise_diff_png, + 'readnoise_diff_image': os.path.basename(readnoise_diff_png), 'diff_image_mean': float(diff_image_mean), 'diff_image_stddev': float(diff_image_stddev), 'diff_image_n': diff_image_n.astype(float), diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py index 552cac71f..41ca929b9 100644 --- a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py @@ -26,11 +26,15 @@ from bokeh.models import ColumnDataSource, HoverTool # from bokeh.models import TabPanel, Tabs # bokeh >= 3.0 from bokeh.plotting import figure +from django.templatetags.static import static import numpy as np from jwql.database.database_interface import session from jwql.database.database_interface import FGSReadnoiseStats, MIRIReadnoiseStats, NIRCamReadnoiseStats, NIRISSReadnoiseStats, NIRSpecReadnoiseStats from jwql.utils.constants import FULL_FRAME_APERTURES, JWST_INSTRUMENT_NAMES_MIXEDCASE +from jwql.utils.utils import get_config + +OUTPUTS_DIR = get_config()['outputs'] class ReadnoiseMonitorData(): @@ -107,9 +111,14 @@ class ReadNoisePlotTab(): def __init__(self, instrument, aperture): self.instrument = instrument self.aperture = aperture + self.ins_ap = "{}_{}".format(self.instrument.lower(), self.aperture.lower()) self.db = ReadnoiseMonitorData(self.instrument, self.aperture) + # Use outputs directory to obtain server name in path. + server_name = OUTPUTS_DIR.split("outputs/", 1)[1] + self.file_path = static(os.path.join("outputs", server_name, "readnoise_monitor", "data", self.ins_ap)) + self.plot_readnoise_amplifers() self.plot_readnoise_difference_image() self.plot_readnoise_histogram() @@ -133,7 +142,7 @@ def plot_readnoise_amplifers(self): else: readnoise_vals = np.array(list()) - filenames = [os.path.basename(result.uncal_filename).replace('_uncal.fits', '') for result in self.db.query_results] + filenames = [result.uncal_filename.replace('_uncal.fits', '') for result in self.db.query_results] expstarts_iso = np.array([result.expstart for result in self.db.query_results]) expstarts = np.array([datetime.strptime(date, '%Y-%m-%dT%H:%M:%S.%f') for date in expstarts_iso]) nints = [result.nints for result in self.db.query_results] @@ -168,10 +177,9 @@ def plot_readnoise_difference_image(self): height=500, width=500, sizing_mode='scale_width') if len(self.db.query_results) != 0: - diff_image_png = self.db.query_results[-1].readnoise_diff_image - diff_image_png = os.path.join('/static', '/'.join(diff_image_png.split('/')[-6:])) + diff_image_png = os.path.join(self.file_path, self.db.query_results[-1].readnoise_diff_image) self.diff_image_plot.image_url(url=[diff_image_png], x=0, y=0, w=2048, h=2048, anchor="bottom_left") - + self.diff_image_plot.xaxis.visible = False self.diff_image_plot.yaxis.visible = False self.diff_image_plot.xgrid.grid_line_color = None @@ -179,7 +187,6 @@ def plot_readnoise_difference_image(self): self.diff_image_plot.title.text_font_size = '22px' self.diff_image_plot.title.align = 'center' - def plot_readnoise_histogram(self): """Updates the readnoise histogram""" @@ -200,12 +207,9 @@ def plot_readnoise_histogram(self): y_range=(hist_yr_start, hist_yr_end), sizing_mode='scale_width') - source = ColumnDataSource(data=dict( - x=diff_image_bin_centers, - y=diff_image_n, - )) + source = ColumnDataSource(data=dict(x=diff_image_bin_centers, y=diff_image_n, )) - self.readnoise_histogram.add_tools(HoverTool(tooltips=[("Data (x, y)", "(@x, @y)"),])) + self.readnoise_histogram.add_tools(HoverTool(tooltips=[("Data (x, y)", "(@x, @y)"), ])) self.readnoise_histogram.circle(x='x', y='y', source=source)