Merge pull request #35 from ImperialCollegeLondon/vNext

Update software to v3.6.0

.github/workflows/matlab.yml

@@ -38,7 +38,7 @@ jobs:
     if: github.ref == 'refs/heads/main'
     needs: test
     env:
-      VERSION: "3.5.0"
+      VERSION: "3.6.0"
     steps:
       - name: Check out repository
         uses: actions/checkout@v3

resources/ReleaseNotes.md

@@ -1,8 +1,16 @@
+# App
+
+- Add drop down and text field to browse and view all processing steps separately
+- Add check box to select whether to plot PSD
+- Add tooltips for meta data text areas and toolstrip buttons
+
 # Software
 
-- Make dependence on Parallel Computing Toolbox optional
-- Convert addition of missing row at end of file data into a processing step (`mag.process.Separate`)
-- Add `mag.process.Separate` to HK processing
-- Add export of whole science data
-- Fix issue with processing data ready time in `mag.process.Units` when not enough digits are present
-- Fix issue with plotting last event mode and range in `mag.graphics.view.HK` when data is cropped
+- Add fine-grained definitions of quality flags (see `mag.meta.Quality`)
+- Update previous uses of quality flag
+- Allow cropping of `mag.Science` with negative duration (filters from the end)
+- Allow cropping of I-ALiRT with separate filter for primary and secondary
+- Add conversion method `eventtable` in `mag.event.Event` to simplify logic in science analysis
+- Derivative of `mag.TimeSeries` variables now has missing value as first element instead of last
+- Delete unused processing steps
+- Add tests for `mag.event.Event`, `mag.event.ModeChange`, `mag.event.RangeChange` and `mag.event.RampMode`

src/analyze/+mag/+process/Filter.m

@@ -111,7 +111,7 @@
                 if (nnz(locConfig) == 2) && any(contains([events.Label(idxConfig(1):idxConfig(end))], "Ramp"))
 
                     configRange = timerange(events.Time(idxConfig(1)), events.Time(idxConfig(end)), "closed");
-                    data{configRange, "quality"} = false;
+                    data{configRange, "quality"} = mag.meta.Quality.Bad;
                 end
             end
         end
@@ -135,7 +135,7 @@
             for t = times
 
                 if isa(range, "duration")
-                    data{timerange(t + range(1), t + range(2), "closed"), "quality"} = false;
+                    data{timerange(t + range(1), t + range(2), "closed"), "quality"} = mag.meta.Quality.Bad;
                 else
 
                     tEvent = data(withtol(t, dt), :).Properties.RowTimes;
@@ -155,7 +155,7 @@
                     idxRemove = idxTime + r;
                     idxRemove(idxRemove < 1) = [];
 
-                    data{idxRemove, "quality"} = false;
+                    data{idxRemove, "quality"} = mag.meta.Quality.Bad;
                 end
             end
         end

src/analyze/+mag/+process/Separate.m

@@ -11,6 +11,8 @@
     properties
         % DISCRIMINATIONVARIABLE Name of variable to increase in row.
         DiscriminationVariable (1, 1) string
+        % QUALITYVARIABLE Name of quality variable.
+        QualityVariable string {mustBeScalarOrEmpty}
         % VARIABLES Variables to be set to missing.
         Variables (1, :) string
     end
@@ -57,9 +59,14 @@
             end
 
             finalRow = data(end, :);
+
             finalRow.(this.DiscriminationVariable) = finalRow.(this.DiscriminationVariable) + eps();
             finalRow{:, variables} = missing();
 
+            if ~isempty(this.QualityVariable)
+                finalRow.(this.QualityVariable) = mag.meta.Quality.Artificial;
+            end
+
             data = [data; finalRow];
         end
     end

src/analyze/+mag/@IMAPTestingAnalysis/IMAPTestingAnalysis.m

@@ -28,7 +28,7 @@
             mag.process.Timestamp(), ...
             mag.process.DateTime(), ...
             mag.process.SignedInteger(CompressionVariable = "compression", Variables = ["x", "y", "z"]), ...
-            mag.process.Separate(DiscriminationVariable = "t", Variables = ["x", "y", "z"])]
+            mag.process.Separate(DiscriminationVariable = "t", QualityVariable = "quality", Variables = ["x", "y", "z"])]
         % WHOLEDATAPROCESSING Steps needed to process all of imported data.
         WholeDataProcessing (1, :) mag.process.Step = [ ...
             mag.process.Sort(), ...
@@ -54,7 +54,7 @@
         HKProcessing (1, :) mag.process.Step = [ ...
             mag.process.DateTime(), ...
             mag.process.Units(), ...
-            mag.process.Separate(DiscriminationVariable = "t", Variables = "*")]
+            mag.process.Separate(DiscriminationVariable = "t", QualityVariable = string.empty(), Variables = "*")]
     end
 
     properties (Dependent)

src/analyze/+mag/@IMAPTestingAnalysis/generateEventTable.m

@@ -17,8 +17,6 @@
     sensor = string(this.Results.getSensor(primaryOrSecondary));
 
     % Adapt existing event properties.
-    sensorEvents.Reason = repmat("Command", height(sensorEvents), 1);
-
     if contains("Sensor", sensorEvents.Properties.VariableNames)
 
         sensorEvents = sensorEvents(ismissing([sensorEvents.Sensor]) | ([sensorEvents.Sensor] == sensor), :);
@@ -30,29 +28,6 @@
     % Improve ramp mode timestamp estimates.
     sensorEvents = updateRampModeTimestamps(sensorEvents, data);
 
-    % Add automatic transitions.
-    locTimedCommand = ~ismissing(sensorEvents.Duration) & (sensorEvents.Duration ~= 0);
-
-    idxTimedCommand = find(locTimedCommand);
-    idxBaselineCommand = find(~locTimedCommand);
-
-    for i = idxTimedCommand(:)'
-
-        idx = idxBaselineCommand(idxBaselineCommand < i);
-
-        if isempty(idx)
-            error("Cannot determine initial event.");
-        end
-
-        autoEvent = sensorEvents(idx(end), :);
-        autoEvent.Time = sensorEvents.Time(i) + seconds(sensorEvents.Duration(i));
-        autoEvent.Reason = "Auto";
-
-        sensorEvents = [sensorEvents; autoEvent]; %#ok<AGROW>
-    end
-
-    sensorEvents = sortrows(sensorEvents);
-
     % Improve estimates of mode changes.
     sensorEvents = findModeChanges(data, sensorEvents);
 

src/analyze/+mag/@IMAPTestingAnalysis/loadIALiRTData.m

@@ -35,8 +35,8 @@ function loadIALiRTData(this, primaryMetaData, secondaryMetaData)
         primary.compression = false(height(primary), 1);
         secondary.compression = false(height(secondary), 1);
 
-        primary.quality = true(height(primary), 1);
-        secondary.quality = true(height(secondary), 1);
+        primary.quality = repmat(mag.meta.Quality.Regular, height(primary), 1);
+        secondary.quality = repmat(mag.meta.Quality.Regular, height(secondary), 1);
 
         % Current file meta data.
         pmd = primaryMetaData.copy();
@@ -90,7 +90,8 @@ function loadIALiRTData(this, primaryMetaData, secondaryMetaData)
         Duration = double.empty(0, 1), ...
         Range = double.empty(0, 1), ...
         Sensor = string.empty(0, 1), ...
-        Label = string.empty(0, 1)));
+        Label = string.empty(0, 1), ...
+        Reason = string.empty(0, 1)));
     sensorEvents = table2timetable(sensorEvents, RowTimes = "Time");
 
     primaryData.Properties.Events = this.generateEventTable("Primary", sensorEvents, primaryData);

src/analyze/+mag/@IMAPTestingAnalysis/loadScienceData.m

@@ -29,8 +29,8 @@ function loadScienceData(this, primaryMetaData, secondaryMetaData)
         primary.compression = false(height(primary), 1);
         secondary.compression = false(height(secondary), 1);
 
-        primary.quality = true(height(primary), 1);
-        secondary.quality = true(height(secondary), 1);
+        primary.quality = repmat(mag.meta.Quality.Regular, height(primary), 1);
+        secondary.quality = repmat(mag.meta.Quality.Regular, height(secondary), 1);
 
         % Current file meta data.
         [mode, primaryFrequency, secondaryFrequency, packetFrequency] = extractFileMetaData(this.ScienceFileNames(i));
@@ -75,7 +75,7 @@ function loadScienceData(this, primaryMetaData, secondaryMetaData)
 
     %% Add Mode and Range Change Events
 
-    sensorEvents = timetable(this.Results.Events);
+    sensorEvents = eventtable(this.Results.Events);
     sensorEvents = sensorEvents(timerange(startTime - seconds(1), endTime, "closed"), :);
 
     primaryData.Properties.Events = this.generateEventTable("Primary", sensorEvents, primaryData);

src/data/+mag/+event/Event.m

@@ -70,6 +70,41 @@
             timetableThis{contains(timetableThis.Label, "Config"), ["PrimaryNormalRate", "SecondaryNormalRate", "PacketNormalFrequency", "PrimaryBurstRate", "SecondaryBurstRate", "PacketBurstFrequency", "Duration"]} = missing();
             timetableThis{contains(timetableThis.Label, "Ramp"), "Range"} = missing();
         end
+
+        function eventtableThis = eventtable(this)
+        % EVENTTABLE Convert evnets to eventtable.
+
+            eventtableThis = this.timetable();
+            eventtableThis.Reason = repmat("Command", height(eventtableThis), 1);
+
+            locTimedCommand = ~ismissing(eventtableThis.Duration) & (eventtableThis.Duration ~= 0);
+
+            idxTimedCommand = find(locTimedCommand);
+            idxBaselineCommand = find(~locTimedCommand);
+
+            for i = idxTimedCommand(:)'
+
+                idx = idxBaselineCommand(idxBaselineCommand < i);
+                assert(~isempty(idx), "Cannot determine initial event.");
+
+                autoEvent = eventtableThis(i, :);
+                autoEvent.Time = eventtableThis.Time(i) + seconds(eventtableThis.Duration(i));
+                autoEvent.Mode = eventtableThis.Mode(idx(end));
+                autoEvent.Duration = 0;
+                autoEvent.Reason = "Auto";
+
+                if isequal(autoEvent.Mode, "Normal")
+                    autoEvent.Label = compose("Normal (%d, %d)", autoEvent.PrimaryNormalRate, autoEvent.SecondaryNormalRate);
+                else
+                    autoEvent.Label = compose("Burst (%d, %d)", autoEvent.PrimaryBurstRate, autoEvent.SecondaryBurstRate);
+                end
+
+                eventtableThis = [eventtableThis; autoEvent]; %#ok<AGROW>
+            end
+
+            eventtableThis = sortrows(eventtableThis);
+            eventtableThis = eventtable(eventtableThis, EventLabelsVariable = "Label");
+        end
     end
 
     methods (Abstract, Access = protected)