Merge branch 'develop' into scsb145

CHANGES.rst

@@ -1,5 +1,57 @@
 ## What's Changed
 
+1.2.8 (2024-04-18)
+==================
+
+Web Application
+~~~~~~~~~~~~~~~
+
+Project & API Documentation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+- final model define for faking by @BradleySappington
+
+
+1.2.7 (2024-04-18)
+==================
+
+Web Application
+~~~~~~~~~~~~~~~
+
+Project & API Documentation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+- Import monitor models by @bhilbert4
+
+
+1.2.6 (2024-04-15)
+==================
+
+Web Application
+~~~~~~~~~~~~~~~
+- Update NIRCam Background Monitor plots to handle new Claw Monitor columns by @bsunnquist
+
+Project & API Documentation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+- add default to read_patt_num by @BradleySappington
+- migration work for next RC by @BradleySappington
+- Fix database empty fields by @BradleySappington
+
+
+1.2.5 (2024-03-19)
+==================
+
+Web Application
+~~~~~~~~~~~~~~~
+- Fix Bokeh `file_html` Call by @mfixstsci
+- Update Bad Pix Exclude Line by @mfixstsci
+- Interactive preview image - updates for Bokeh 3 by @bhilbert4
+
+Project & API Documentation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+- Allow creation of pngs from 3D and 4D arrays by @bhilbert4
+- Add max length to charfield by @BradleySappington
+- Header fix by @BradleySappington
+
+
 1.2.4 (2024-03-11)
 ==================
 

jwql/instrument_monitors/nircam_monitors/claw_monitor.py

@@ -192,6 +192,7 @@ def make_background_plots(self, plot_type='bkg'):
                     df = df[df['stddev'] != 0]  # older data has no accurate stddev measures
                     plot_data = df['stddev'].values
                 if plot_type == 'model':
+                    df = df[np.isfinite(df['total_bkg'])]  # the claw monitor did not track model measurements at first
                     plot_data = df['median'].values / df['total_bkg'].values
                 plot_expstarts = df['expstart_mjd'].values
 
@@ -300,7 +301,11 @@ def process(self):
 
                 # Get predicted background level using JWST background tool
                 ra, dec = hdu[1].header['RA_V1'], hdu[1].header['DEC_V1']
-                wv = self.filter_wave[self.fltr.upper()]
+                if ('N' in self.pupil.upper()) | ('M' in self.pupil.upper()):
+                    fltr_wv = self.pupil.upper()
+                else:
+                    fltr_wv = self.fltr.upper()
+                wv = self.filter_wave[fltr_wv]
                 date = hdu[0].header['DATE-BEG']
                 doy = int(Time(date).yday.split(':')[1])
                 try:
@@ -332,7 +337,7 @@ def process(self):
                                  'skyflat_filename': os.path.basename(self.outfile),
                                  'doy': float(doy),
                                  'total_bkg': float(total_bkg),
-                                 'entry_date': datetime.datetime.now()
+                                 'entry_date': datetime.datetime.now(datetime.timezone.utc)
                                  }
                 entry = self.stats_table(**claw_db_entry)
                 entry.save()
@@ -423,11 +428,13 @@ def run(self):
         mast_table = self.query_mast()
         logging.info('{} files found between {} and {}.'.format(len(mast_table), self.query_start_mjd, self.query_end_mjd))
 
-        # Define pivot wavelengths
-        self.filter_wave = {'F070W': 0.704, 'F090W': 0.902, 'F115W': 1.154, 'F150W': 1.501, 'F150W2': 1.659,
-                            'F200W': 1.989, 'F212N': 2.121, 'F250M': 2.503, 'F277W': 2.762, 'F300M': 2.989,
-                            'F322W2': 3.232, 'F356W': 3.568, 'F410M': 4.082, 'F430M': 4.281, 'F444W': 4.408,
-                            'F480M': 4.874}
+        # Define pivot wavelengths - last downloaded March 8 2024 from:
+        # https://jwst-docs.stsci.edu/jwst-near-infrared-camera/nircam-instrumentation/nircam-filters
+        self.filter_wave = {'F070W': 0.704, 'F090W': 0.901, 'F115W': 1.154, 'F140M': 1.404, 'F150W': 1.501, 'F162M': 1.626, 'F164N': 1.644,
+                            'F150W2': 1.671, 'F182M': 1.845, 'F187N': 1.874, 'F200W': 1.99, 'F210M': 2.093, 'F212N': 2.12, 'F250M': 2.503,
+                            'F277W': 2.786, 'F300M': 2.996, 'F322W2': 3.247, 'F323N': 3.237, 'F335M': 3.365, 'F356W': 3.563, 'F360M': 3.621,
+                            'F405N': 4.055, 'F410M': 4.092, 'F430M': 4.28, 'F444W': 4.421, 'F460M': 4.624, 'F466N': 4.654, 'F470N': 4.707,
+                            'F480M': 4.834}
 
         # Create observation-level median stacks for each filter/pupil combo, in pixel-space
         combos = np.array(['{}_{}_{}_{}'.format(str(row['program']), row['observtn'], row['filter'], row['pupil']).lower() for row in mast_table])
@@ -469,7 +476,7 @@ def run(self):
                      'start_time_mjd': self.query_start_mjd,
                      'end_time_mjd': self.query_end_mjd,
                      'run_monitor': monitor_run,
-                     'entry_date': datetime.datetime.now()}
+                     'entry_date': datetime.datetime.now(datetime.timezone.utc)}
         entry = self.query_table(**new_entry)
         entry.save()
 

jwql/pull_jwql_branch.sh

@@ -62,6 +62,7 @@ echo "Reset: $reset";
 echo "Notify: $notify $recipient";
 
 # 1. Pull updated code from GitHub deployment branch (keep second checkout in case its already defined for some weird reason)
+git fetch origin
 git checkout -b $branch_name --track origin/$branch_name
 git checkout $branch_name
 git fetch origin $branch_name

jwql/tests/test_data_containers.py

@@ -31,7 +31,7 @@
 import pandas as pd
 import pytest
 
-from jwql.utils.constants import ON_GITHUB_ACTIONS
+from jwql.utils.constants import ON_GITHUB_ACTIONS, DEFAULT_MODEL_CHARFIELD
 
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jwql.website.jwql_proj.settings")
 
@@ -45,7 +45,7 @@
     from jwql.utils.utils import get_config  # noqa: E402 (module level import not at top of file)
     from jwql.website.apps.jwql.models import RootFileInfo
 
-    
+
 @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to django models.')
 def test_build_table():
     tab = data_containers.build_table('filesystem_general')
@@ -199,7 +199,6 @@ def test_get_all_proposals():
                            (['uncal', 'rate', 'o001_crf', 'o006_crfints', 'bad'], {'bad'})),
                           (False, ['rate', 'uncal', 'bad', 'o006_crfints', 'o001_crf'],
                            ['uncal', 'rate', 'o001_crf', 'o006_crfints', 'bad'])])
-
 def test_get_available_suffixes(untracked, input_suffixes, expected):
     result = data_containers.get_available_suffixes(
         input_suffixes, return_untracked=untracked)
@@ -339,6 +338,7 @@ def test_get_anomaly_form_post_group(mocker):
     assert update_mock.call_count == 2
 """
 
+
 @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to django models.')
 def test_get_dashboard_components():
     request = MockPostRequest()
@@ -607,42 +607,42 @@ def test_mast_query_by_rootname():
     instrument = 'NIRCam'
     rootname1 = 'jw02767002001_02103_00005_nrcb4'
     dict_stuff = data_containers.mast_query_by_rootname(instrument, rootname1)
-    defaults = dict(filter=dict_stuff.get('filter', ''),
-                    detector=dict_stuff.get('detector', ''),
-                    exp_type=dict_stuff.get('exp_type', ''),
-                    read_pat=dict_stuff.get('readpatt', ''),
-                    grating=dict_stuff.get('grating', ''),
+    defaults = dict(filter=dict_stuff.get('filter', DEFAULT_MODEL_CHARFIELD),
+                    detector=dict_stuff.get('detector', DEFAULT_MODEL_CHARFIELD),
+                    exp_type=dict_stuff.get('exp_type', DEFAULT_MODEL_CHARFIELD),
+                    read_pat=dict_stuff.get('readpatt', DEFAULT_MODEL_CHARFIELD),
+                    grating=dict_stuff.get('grating', DEFAULT_MODEL_CHARFIELD),
                     patt_num=dict_stuff.get('patt_num', 0),
-                    aperture=dict_stuff.get('apername', ''),
-                    subarray=dict_stuff.get('subarray', ''),
-                    pupil=dict_stuff.get('pupil', ''))
+                    aperture=dict_stuff.get('apername', DEFAULT_MODEL_CHARFIELD),
+                    subarray=dict_stuff.get('subarray', DEFAULT_MODEL_CHARFIELD),
+                    pupil=dict_stuff.get('pupil', DEFAULT_MODEL_CHARFIELD))
     assert isinstance(defaults, dict)
 
     rootname2 = 'jw02084001001_04103_00001-seg003_nrca3'
     dict_stuff = data_containers.mast_query_by_rootname(instrument, rootname2)
-    defaults = dict(filter=dict_stuff.get('filter', ''),
-                    detector=dict_stuff.get('detector', ''),
-                    exp_type=dict_stuff.get('exp_type', ''),
-                    read_pat=dict_stuff.get('readpatt', ''),
-                    grating=dict_stuff.get('grating', ''),
+    defaults = dict(filter=dict_stuff.get('filter', DEFAULT_MODEL_CHARFIELD),
+                    detector=dict_stuff.get('detector', DEFAULT_MODEL_CHARFIELD),
+                    exp_type=dict_stuff.get('exp_type', DEFAULT_MODEL_CHARFIELD),
+                    read_pat=dict_stuff.get('readpatt', DEFAULT_MODEL_CHARFIELD),
+                    grating=dict_stuff.get('grating', DEFAULT_MODEL_CHARFIELD),
                     patt_num=dict_stuff.get('patt_num', 0),
-                    aperture=dict_stuff.get('apername', ''),
-                    subarray=dict_stuff.get('subarray', ''),
-                    pupil=dict_stuff.get('pupil', ''))
+                    aperture=dict_stuff.get('apername', DEFAULT_MODEL_CHARFIELD),
+                    subarray=dict_stuff.get('subarray', DEFAULT_MODEL_CHARFIELD),
+                    pupil=dict_stuff.get('pupil', DEFAULT_MODEL_CHARFIELD))
     assert isinstance(defaults, dict)
 
     instrument2 = 'FGS'
     rootname3 = 'jw01029003001_06201_00001_guider2'
     dict_stuff = data_containers.mast_query_by_rootname(instrument2, rootname3)
-    defaults = dict(filter=dict_stuff.get('filter', ''),
-                    detector=dict_stuff.get('detector', ''),
-                    exp_type=dict_stuff.get('exp_type', ''),
-                    read_pat=dict_stuff.get('readpatt', ''),
-                    grating=dict_stuff.get('grating', ''),
+    defaults = dict(filter=dict_stuff.get('filter', DEFAULT_MODEL_CHARFIELD),
+                    detector=dict_stuff.get('detector', DEFAULT_MODEL_CHARFIELD),
+                    exp_type=dict_stuff.get('exp_type', DEFAULT_MODEL_CHARFIELD),
+                    read_pat=dict_stuff.get('readpatt', DEFAULT_MODEL_CHARFIELD),
+                    grating=dict_stuff.get('grating', DEFAULT_MODEL_CHARFIELD),
                     patt_num=dict_stuff.get('patt_num', 0),
-                    aperture=dict_stuff.get('apername', ''),
-                    subarray=dict_stuff.get('subarray', ''),
-                    pupil=dict_stuff.get('pupil', ''))
+                    aperture=dict_stuff.get('apername', DEFAULT_MODEL_CHARFIELD),
+                    subarray=dict_stuff.get('subarray', DEFAULT_MODEL_CHARFIELD),
+                    pupil=dict_stuff.get('pupil', DEFAULT_MODEL_CHARFIELD))
     assert isinstance(defaults, dict)
 
 

jwql/utils/constants.py

@@ -364,6 +364,9 @@
     "wfscmb",
 ]
 
+# Default Model Values
+DEFAULT_MODEL_CHARFIELD = "empty"
+
 # Filename Component Lengths
 FILE_AC_CAR_ID_LEN = 4
 FILE_AC_O_ID_LEN = 3
@@ -700,6 +703,7 @@
     "nircam_dark_dark_current", "nircam_dark_pixel_stats", "nircam_dark_query_history",
     "niriss_dark_dark_current", "niriss_dark_pixel_stats", "niriss_dark_query_history",
     "nirspec_dark_dark_current", "nirspec_dark_pixel_stats", "nirspec_dark_query_history",
+    "nirspec_grating_query_history",
     "fgs_edb_blocks_stats", "fgs_edb_daily_stats", "fgs_edb_every_change_stats", "fgs_edb_time_interval_stats", "fgs_edb_time_stats",
     "miri_edb_blocks_stats", "miri_edb_daily_stats", "miri_edb_every_change_stats", "miri_edb_time_interval_stats", "miri_edb_time_stats",
     "nircam_edb_blocks_stats", "nircam_edb_daily_stats", "nircam_edb_every_change_stats", "nircam_edb_time_interval_stats", "nircam_edb_time_stats",

jwql/utils/interactive_preview_image.py

@@ -554,7 +554,7 @@ def add_interactive_controls(self, images, color_bars):
         # JS callbacks for client side controls
 
         # set alternate image visibility when scale selection changes
-        scale_group.js_on_click(CustomJS(args={'i1': images[0], 'c1': color_bars[0],
+        scale_group.js_on_change('active', CustomJS(args={'i1': images[0], 'c1': color_bars[0],
                                                'i2': images[1], 'c2': color_bars[1]},
                                          code="""
             if (i1.visible == true) {
@@ -594,10 +594,10 @@ def add_interactive_controls(self, images, color_bars):
             limit_high.js_link('value', color_bars[i].color_mapper, 'high')
 
         # reset boxes to preset range on button click
-        reset.js_on_click(limit_reset)
+        reset.js_on_event('button_click', limit_reset)
 
         # also reset when swapping limit style
-        scale_group.js_on_click(limit_reset)
+        scale_group.js_on_change('active', limit_reset)
 
         # return widgets
         spacer = Spacer(height=20)