Merge branch '418-readwrite-of-flags-in-sdfits-scan-and-spectrum' of …

…github.com:GreenBankObservatory/dysh into 418-readwrite-of-flags-in-sdfits-scan-and-spectrum

docs/source/how-tos/index.rst

@@ -86,6 +86,21 @@ Practical step-by-step guides to help you achieve a specific goal. Most useful w
 
             Align Spectra
 
+    .. grid-item-card::
+        :shadow: md
+        :margin: 2 2 0 0
+
+        :material-outlined:`flag;3em;green` **Flagging**
+
+        How to read and save data
+
+        .. button-link:: examples/flagging.html
+            :color: primary
+            :outline:
+            :click-parent:
+
+            Flagging
+
 .. toctree::
    :maxdepth: 4
    :hidden:
@@ -95,3 +110,4 @@ Practical step-by-step guides to help you achieve a specific goal. Most useful w
    examples/smoothing
    examples/dataIO
    examples/align_spectra
+   examples/flagging

src/dysh/fits/tests/test_gbtfitsload.py

@@ -714,3 +714,17 @@ def test_additive_flags(self, tmp_path):
         sdf.flag_channel(channel=[[80000, 100000]])
         sdf.apply_flags()
         assert np.all(sdf._sdf[0]._flagmask[0] == flag1 | flag2)
+
+    def test_read_gbtidl_flags(self):
+        """
+        Test reading a flag file generated by GBTIDL.
+        """
+        fits_path = util.get_project_testdata() / "AGBT17A_404_01/AGBT17A_404_01.raw.vegas"
+        sdf = gbtfitsload.GBTFITSLoad(fits_path)
+        # The data should not be flagged, as the flags are only for intnum=arange(43,52)
+        ps = sdf.getps(scan=19, plnum=0, apply_flags=True).timeaverage()
+        assert np.all(ps.mask == False)
+        # The data should be flagged for these integrations.
+        ps = sdf.getps(scan=19, plnum=0, apply_flags=True, intnum=[i for i in range(43, 52)]).timeaverage()
+        assert np.all(ps.mask[2299:] == True)
+        assert np.all(ps.mask[:2299] == False)

src/dysh/util/selection.py

@@ -1218,6 +1218,11 @@ def read(self, fileobj, **kwargs):
         # [flags]
         # #RECNUM,SCAN,INTNUM,PLNUM,IFNUM,FDNUM,BCHAN,ECHAN,IDSTRING
         # *|6|*|*|2|0|3072|3072|VEGAS_SPUR
+        #
+        # It is possible there is a space after the *GBTIDL flag files can also indicate ranges with a : and can indicate upper or lower limits
+        # by not including a number. For instancer here is scan range 42 to 51 and channel range with
+        # lower limit of 2299
+        # *|20|42:51|*|*|*|2299|*|unspecified
 
         # Because the table header and table row delimeters are different,
         # Table.read() can't work.  So construct it row by row.
@@ -1235,14 +1240,21 @@ def read(self, fileobj, **kwargs):
             else:
                 values = l.split("|")
                 for i, v in enumerate(values):
-                    if v == "*":
+                    if v.strip() == "*":
                         continue
                     else:
                         if header[i] == "IDSTRING":
                             vdict[header[i]] = v
                         else:
+                            # handle comma-separated lists
                             if "," in v:
                                 vdict[header[i]] = [int(float(x)) for x in v.split(",")]
+                            # handle colon-separated ranges by expanding into a comma-separated list.
+                            elif ":" in v:
+                                vdict[header[i]] = [int(float(x)) for x in range(*map(int, v.split(":")))] + [
+                                    int(v.split(":")[-1])
+                                ]
+                            # handle single values
                             else:
                                 vdict[header[i]] = int(float(v))
 
@@ -1259,6 +1271,21 @@ def read(self, fileobj, **kwargs):
                     echan = [int(float(x)) for x in echan]
                     # pair up echan and bchan
                     vdict["channel"] = list(zip(bchan, echan))
+                elif bchan is not None and echan is None:
+                    if not isinstance(bchan, list):
+                        bchan = [bchan]
+                    bchan = [int(float(x)) for x in bchan]
+                    echan = [2**25] * len(
+                        bchan
+                    )  # Set to a large number so it effectively spans the whole range from `bchan`.
+                    vdict["channel"] = tuple(zip(bchan, echan))
+                elif bchan is None and echan is not None:
+                    if not isinstance(echan, list):
+                        echan = [echan]
+                    echan = [int(float(x)) for x in echan]
+                    bchan = [0] * len(echan)
+                    vdict["channel"] = tuple(zip(bchan, echan))
+
                 if kwargs is not None:
                     vdict.update(kwargs)
                 logger.debug(f"flag({tag=},{vdict})")

testdata/AGBT17A_404_01/AGBT17A_404_01.raw.vegas/AGBT17A_404_01.raw.vegas.A.flag

@@ -0,0 +1,7 @@
+[header]
+created = Fri Nov 22 19:04:55 2024                                                                                                                                                                                                                              
+version = 1.0                                                                                                                                                                                                                                                   
+created_by = gbtidl                                                                                                                                                                                                                                             
+[flags]
+#RECNUM,SCAN,INTNUM,PLNUM,IFNUM,FDNUM,BCHAN,ECHAN,IDSTRING
+ *|20|42:51|*|*|*|2299|*|unspecified