Merge branch 'main' into shower_flagging_enhancement

CHANGES.rst

@@ -1,4 +1,17 @@
-1.6.2 (unreleased)
+1.7.0 (unreleased)
+==================
+
+Changes to API
+--------------
+
+- 
+
+Bug Fixes
+---------
+
+- 
+
+1.6.2 (2024-03-21)
 ==================
 
 Changes to API
@@ -24,9 +37,24 @@ jump
   number of groups to trigger the detection, and use all integrations to determine
   the median value. [#248]
 
+ramp_fitting
+~~~~~~~~~~~~
+
+- Changed the data type of three variables that are used in measuring
+  the jump free segments of integrations. The variables were uint8 and
+  they would yield wrong results for integrations with more than 256
+  groups. [#251]
+
 Other
 -----
 
+jump
+~~~~
+
+- Enable the use of multiple integrations to find outliers. Also,
+  when the number of groups is above a threshold, use single pass
+  outlier flagging rather than the iterative flagging. [#242]
+
 1.6.1 (2024-02-29)
 ==================
 

src/stcal/jump/jump.py

@@ -2,6 +2,7 @@
 import multiprocessing
 import time
 import warnings
+
 import numpy as np
 import cv2 as cv
 import astropy.stats as stats
@@ -18,10 +19,10 @@
 
 def detect_jumps(
     frames_per_group,
-    data,
+    indata,
     gdq,
     pdq,
-    err,
+    inerr,
     gain_2d,
     readnoise_2d,
     rejection_thresh,
@@ -56,6 +57,7 @@ def detect_jumps(
     minimum_groups=3,
     minimum_sigclip_groups=100,
     only_use_ints=True,
+    min_diffs_single_pass=10,
     mask_persist_grps_next_int=True,
     persist_grps_flagged=25,
 ):
@@ -81,7 +83,7 @@ def detect_jumps(
     frames_per_group : int
         number of frames per group
 
-    data : float, 4D array
+    indata : float, 4D array
         science array
 
     gdq : int, 4D array
@@ -90,7 +92,7 @@ def detect_jumps(
     pdq : int, 2D array
         pixelg dq array
 
-    err : float, 4D array
+    inerr : float, 4D array
         error array
 
     gain_2d : float, 2D array
@@ -109,7 +111,7 @@ def detect_jumps(
     four_grp_thresh : float
         cosmic ray sigma rejection threshold for ramps having 4 groups
 
-    max_cores: str
+    max_cores : str
         Maximum number of cores to use for multiprocessing. Available choices
         are 'none' (which will create one process), 'quarter', 'half', 'all'
         (of available cpu cores).
@@ -124,11 +126,11 @@ def detect_jumps(
         neighbors (marginal detections). Any primary jump below this value will
         not have its neighbors flagged.
 
-    flag_4_neighbors: bool
+    flag_4_neighbors : bool
         if set to True (default is True), it will cause the four perpendicular
         neighbors of all detected jumps to also be flagged as a jump.
 
-    dqflags: dict
+    dqflags : dict
         A dictionary with at least the following keywords:
         DO_NOT_USE, SATURATED, JUMP_DET, NO_GAIN_VALUE, GOOD
 
@@ -209,6 +211,15 @@ def detect_jumps(
     min_sat_radius_extend : float
         The minimum radius of the saturated core of a snowball for the core to
         be extended
+    minimum_groups : int
+       The minimum number of groups for jump detection
+    minimum_sigclip_groups : int
+       The minimum number of groups required to use sigma clipping to find outliers.
+    only_use_ints : boolean
+       In sigma clipping, if True only differences between integrations are compared. If False,
+       then all differences are processed at once.
+    min_diffs_single_pass : int
+       The minimum number of groups to switch to flagging all outliers in a single pass.
 
     Returns
     -------
@@ -235,13 +246,12 @@ def detect_jumps(
 
     # Apply gain to the SCI, ERR, and readnoise arrays so they're in units
     # of electrons
-    data *= gain_2d
-    err *= gain_2d
+    data = indata * gain_2d
+    err = inerr * gain_2d
     readnoise_2d *= gain_2d
     # also apply to the after_jump thresholds
     after_jump_flag_e1 = after_jump_flag_dn1 * np.nanmedian(gain_2d)
     after_jump_flag_e2 = after_jump_flag_dn2 * np.nanmedian(gain_2d)
-
     # Apply the 2-point difference method as a first pass
     log.info("Executing two-point difference method")
     start = time.time()
@@ -280,6 +290,7 @@ def detect_jumps(
             minimum_groups=3,
             minimum_sigclip_groups=minimum_sigclip_groups,
             only_use_ints=only_use_ints,
+            min_diffs_single_pass=min_diffs_single_pass,
         )
         # remove redundant bits in pixels that have jump flagged but were
         # already flagged as do_not_use or saturated.
@@ -364,6 +375,7 @@ def detect_jumps(
                     minimum_groups,
                     minimum_sigclip_groups,
                     only_use_ints,
+                    min_diffs_single_pass,
                 ),
             )
 
@@ -390,6 +402,7 @@ def detect_jumps(
                 minimum_groups,
                 minimum_sigclip_groups,
                 only_use_ints,
+                min_diffs_single_pass,
             ),
         )
         log.info("Creating %d processes for jump detection ", n_slices)
@@ -903,21 +916,24 @@ def find_faint_extended(
           emission.
       min_shower_area : int
           The minimum area for a group of pixels to be flagged as a shower.
-      inner: int
+      inner : int
           The inner radius of the ring_2D_kernal used for the convolution.
       outer : int
           The outer radius of the ring_2D_kernal used for the convolution.
       sat_flag : int
           The integer value of the saturation flag.
       jump_flag : int
           The integer value of the jump flag
-      ellipse_expand: float
+      ellipse_expand : float
           The relative increase in the size of the fitted ellipse to be
           applied to the shower.
-    num_grps_masked: int
-        The number of groups after the detected shower to be flagged as jump.
-    max_extended_radius: int
-        The upper limit for the extension of saturation and jump
+      num_grps_masked : int
+          The number of groups after the detected shower to be flagged as jump.
+      max_extended_radius : int
+          The upper limit for the extension of saturation and jump
+      minimum_sigclip_groups : int
+          The minimum number of groups to use sigma clipping.
+
 
     Returns
     -------