Merge pull request #29 from ImperialCollegeLondon/vNext

Update software to v3.3.1

.github/workflows/matlab.yml

@@ -38,7 +38,7 @@ jobs:
     if: github.ref == 'refs/heads/main'
     needs: test
     env:
-      VERSION: "3.3.0"
+      VERSION: "3.3.1"
     steps:
       - name: Check out repository
         uses: actions/checkout@v3

buildfile.m

@@ -19,10 +19,10 @@
         IncludeSubfolders = true);
 
     % Add the "test" task to run tests.
-    testFolder = "tests";
+    testFolders = ["tests/system", "tests/unit"];
 
-    plan("test") = matlab.buildtool.tasks.TestTask(testFolder, ...
-        SourceFiles = sourceFolders, ...
+    plan("test") = matlab.buildtool.tasks.TestTask(testFolders, ...
+        SourceFiles = [sourceFolders, "tests/tool"], ...
         IncludeSubfolders = true, ...
         TestResults = fullfile("artifacts/results.xml"), ...
         CodeCoverageResults = fullfile("artifacts/coverage.xml"));

resources/ReleaseNotes.md

@@ -1,29 +1,9 @@
 # Software
 
-## Compression
-
-- Add processing step `mag.process.Compression` to correct for compression factor
-- Allow filtering around compression changes in `mag.process.Filter`
-- Fix issue in `mag.process.SignedInteger` with compression still using 16th bit for signedness, instead of 18th
-- Fix typos in compression event plot
-
-## Export
-
-- Export formats are defined in `mag.io.Type`
-- Export formats accept data structures `mag.Instrument` and `mag.HK`, instead of custom structure
-- Add `Compression` flag to science export
-- Add `PROCSTAT` to HK export
-
-## Other
-
-- Add variable continuity definition for science variables in `timetable`
-- Add shutdown event as final event in `mag.Science` event table
-- Add property `Harness` in `mag.meta.Science` to describe sensor harness
-- Add ability to specify final event end time in `mag.graphics.view.Field`
-- Simplify how columns are filtered in loading science and I-ALiRT
-- Make `mag.process.Calibration` more flexible to custom variable names
-- Allow plotting more than one event in `mag.graphics.view.Field`
-- Change event type to `categorical` when data is not numeric
-- Add option to create folder in `mag.graphics.savePlots`
-- Fix issue with loading Excel files containing sensor meta data
-- Fix issue in custom events chart when event value is not `double` or `single`
+- Add I-ALiRT plot to compare full science to I-ALiRT
+- Add processing step to cast variable data type (`mag.process.Cast`)
+- Add option to filter out data after a long pause in measurements in `mag.process.Filter`
+- Allow adding input arguments to conversion function in `mag.graphics.operation.Convert`
+- Fix issue with setting I-ALiRT mode when loading
+- Fix issues with exporting I-ALiRT data
+- Add tests for `mag.graphics.style.Colormap` and `mag.graphics.style.Default`

resources/project/EEtUlUb-dLAdf0KpMVivaUlztwA/mUpHVeZpZ6sxD_WsHI2gc2cdlnwd.xml

@@ -0,0 +1,2 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<Info Ref="tests/tool" Type="Relative"/>

resources/project/EEtUlUb-dLAdf0KpMVivaUlztwA/mUpHVeZpZ6sxD_WsHI2gc2cdlnwp.xml

@@ -0,0 +1,2 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<Info location="2ff31828-3d31-433d-9e34-f623cbcc58e3" type="Reference"/>

src/analyze/+mag/+process/Cast.m

@@ -0,0 +1,47 @@
+classdef Cast < mag.process.Step
+% CAST Apply data type cast.
+
+    properties (Dependent)
+        Name
+        Description
+        DetailedDescription
+    end
+
+    properties
+        % DATATYPE Data type to convert to.
+        DataType (1, 1) string
+        % VARIABLES Variables to be cast.
+        Variables (1, :) string
+    end
+
+    methods
+
+        function this = Cast(options)
+
+            arguments
+                options.?mag.process.Cast
+            end
+
+            this.assignProperties(options);
+        end
+
+        function value = get.Name(~)
+            value = "Cast to Data Type";
+        end
+
+        function value = get.Description(this)
+            value = "Cast " + join(compose("""%s""", this.Variables), ", ") + " to """ + this.DataType + """.";
+        end
+
+        function value = get.DetailedDescription(this)
+            value = this.Description;
+        end
+
+        function data = apply(this, data, ~)
+
+            for v = this.Variables
+                data.(v) = cast(data.(v), this.DataType);
+            end
+        end
+    end
+end

src/analyze/+mag/+process/Filter.m

@@ -21,6 +21,9 @@
         % ONCOMPRESSIONCHANGE How many vectors to remove when compression
         % changes.
         OnCompressionChange (1, 2) {mustBeA(OnCompressionChange, ["double", "duration"])} = zeros(1, 2)
+        % ONLONGPAUSE How many vectors to remove when long pause in the
+        % data is detected.
+        OnLongPause (1, 2) {mustBeA(OnLongPause, ["double", "duration"])} = zeros(1, 2)
     end
 
     methods
@@ -71,17 +74,30 @@
 
             % Filter data points at mode changes.
             if ~isequal(this.OnModeChange, zeros(1, 2))
-                data = this.cropDataWithRange(events, data, this.ModeVariable, this.OnModeChange);
+                data = this.cropDataWithEvents(events, data, this.ModeVariable, this.OnModeChange);
             end
 
             % Filter duration at range changes.
             if ~isequal(this.OnRangeChange, zeros(1, 2))
-                data = this.cropDataWithRange(events, data, this.RangeVariable, this.OnRangeChange);
+                data = this.cropDataWithEvents(events, data, this.RangeVariable, this.OnRangeChange);
             end
 
             % Filter duration at compression changes.
             if ~isequal(this.OnCompressionChange, zeros(1, 2))
-                data = this.cropDataWithRange(data, data, this.CompressionVariable, this.OnCompressionChange);
+                data = this.cropDataWithEvents(data, data, this.CompressionVariable, this.OnCompressionChange);
+            end
+
+            % Filter out after long pauses.
+            if ~isequal(this.OnLongPause, zeros(1, 2))
+
+                times = data.Properties.RowTimes;
+
+                locTimes = diff(times) > seconds(1);
+                locTimes = [false; locTimes];
+
+                times = times(locTimes);
+
+                data = this.cropDataWithRange(data, times', this.OnLongPause);
             end
 
             % Filter out between config and ramp mode.
@@ -101,14 +117,22 @@
         end
     end
 
+    methods (Access = private)
+
+        function data = cropDataWithEvents(this, events, data, name, range)
+
+            locEvent = [false; diff(events.(name)) ~= 0];
+            this.cropDataWithRange(data, events.Properties.RowTimes(locEvent)', range);
+        end
+    end
+
     methods (Static, Access = private)
 
-        function data = cropDataWithRange(events, data, name, range)
+        function data = cropDataWithRange(data, times, range)
 
             dt = mode(diff(data.Properties.RowTimes));
-            locEvent = [false; diff(events.(name)) ~= 0];
 
-            for t = events.Properties.RowTimes(locEvent)'
+            for t = times
 
                 if isa(range, "duration")
                     data{timerange(t + range(1), t + range(2), "closed"), "quality"} = false;

src/analyze/+mag/+process/Range.m

@@ -51,18 +51,18 @@
 
     methods (Hidden)
 
-        function unscaledData = applyRange(this, unscaledData, ranges)
+        function data = applyRange(this, data, ranges)
 
             arguments (Input)
                 this
-                unscaledData (:, :) double
+                data (:, :) double
                 ranges (:, 1) double
             end
 
             for sf = 0:3
 
                 locScaleFactor = ranges == sf;
-                unscaledData(locScaleFactor, :) = this.ScaleFactors(sf + 1) * unscaledData(locScaleFactor, :);
+                data(locScaleFactor, :) = this.ScaleFactors(sf + 1) * data(locScaleFactor, :);
             end
         end
     end

src/analyze/+mag/@IMAPTestingAnalysis/export.m

@@ -42,7 +42,7 @@ function export(this, exportStrategy, options)
         iALiRT.crop(period);
         iALiRTData = scienceExportFormat.formatForExport(iALiRT);
 
-        exportStrategy.ExportFileName = fullfile(options.Location, compose("%s %s (%.2f, %.2f)", datestr(m.Primary.MetaData.Timestamp, "ddmmyy-hhMM"), m.Primary.MetaData.Mode, m.Primary.MetaData.DataFrequency, m.Secondary.MetaData.DataFrequency) + extension); %#ok<DATST>
+        exportStrategy.ExportFileName = fullfile(options.Location, compose("%s %s (%.2f, %.2f)", datestr(iALiRT.Primary.MetaData.Timestamp, "ddmmyy-hhMM"), iALiRT.Primary.MetaData.Mode, iALiRT.Primary.MetaData.DataFrequency, iALiRT.Secondary.MetaData.DataFrequency) + extension); %#ok<DATST>
         exportStrategy.export(iALiRTData);
     end
 

src/analyze/+mag/@IMAPTestingAnalysis/loadIALiRTData.m

@@ -6,10 +6,10 @@ function loadIALiRTData(this, primaryMetaData, secondaryMetaData)
     end
 
     primaryMetaData = primaryMetaData.copy();
-    primaryMetaData.set(Mode = "I-ALiRT", DataFrequency = 1/4, PacketFrequency = 4);
+    primaryMetaData.set(Mode = "IALiRT", DataFrequency = 1/4, PacketFrequency = 4);
 
     secondaryMetaData = secondaryMetaData.copy();
-    secondaryMetaData.set(Mode = "I-ALiRT", DataFrequency = 1/4, PacketFrequency = 4);
+    secondaryMetaData.set(Mode = "IALiRT", DataFrequency = 1/4, PacketFrequency = 4);
 
     [~, ~, extension] = fileparts(this.IALiRTPattern);
     rawIALiRT = this.dispatchExtension(extension, ImportFileNames = this.IALiRTFileNames).import();

src/io/+mag/+io/+format/ScienceMAT.m

@@ -7,7 +7,7 @@
 
             arguments
                 ~
-                data (1, 1) mag.Instrument
+                data (1, 1) {mustBeA(data, ["mag.Instrument", "mag.IALiRT"])}
             end
 
             exportedData.B.P.Time = data.Primary.Time;

src/visualize/+mag/+graphics/+operation/Convert.m

@@ -4,6 +4,8 @@
     properties
         % CONVERSION Conversion variable to use.
         Conversion (1, 1) function_handle = @sqrt
+        % ARGUMENTS Other arguments for conversion function.
+        Arguments (1, :) cell = cell.empty()
     end
 
     methods
@@ -24,7 +26,7 @@
                 originalData
             end
 
-            plottableData = this.Conversion(originalData);
+            plottableData = this.Conversion(originalData, this.Arguments{:});
         end
     end
 end

src/visualize/+mag/+graphics/+view/IALiRT.m

@@ -32,10 +32,23 @@ function visualize(this)
                 LinkXAxes = true, ...
                 WindowState = "maximized");
 
-            % Plot I-ALiRT and science.
+            % Plot I-ALiRT and science (full).
             primaryScience = this.Results.Primary;
             secondaryScience = this.Results.Secondary;
 
+            primaryOverlay = this.generateOverlayGraph(primaryScience, primaryIALiRT);
+            secondaryOverlay = this.generateOverlayGraph(secondaryScience, secondaryIALiRT);
+
+            this.Figures(2) = mag.graphics.visualize( ...
+                primaryOverlay{:}, secondaryOverlay{:}, ...
+                Name = "Science vs. I-ALiRT (Full)", ...
+                Arrangement = [3, 2], ...
+                GlobalLegend = ["Science", "I-ALiRT"], ...
+                LinkXAxes = true, ...
+                TileIndexing = "columnmajor", ...
+                WindowState = "maximized");
+
+            % Plot I-ALiRT and science (closest vector).
             primaryComparison = synchronize(timetable(primaryIALiRT.Time, primaryIALiRT.X, primaryIALiRT.Y, primaryIALiRT.Z, primaryIALiRT.Quality, VariableNames = ["xi", "yi", "zi", "qi"]), ...
                 timetable(primaryScience.Time, primaryScience.X, primaryScience.Y, primaryScience.Z, primaryScience.Quality, VariableNames = ["xs", "ys", "zs", "qs"]), "first", "nearest");
             secondaryComparison = synchronize(timetable(secondaryIALiRT.Time, secondaryIALiRT.X, secondaryIALiRT.Y, secondaryIALiRT.Z, secondaryIALiRT.Quality, VariableNames = ["xi", "yi", "zi", "qi"]), ...
@@ -44,9 +57,9 @@ function visualize(this)
             primaryGraphs = this.generateComparisonGraph(primaryIALiRT, primaryComparison);
             secondaryGraphs = this.generateComparisonGraph(secondaryIALiRT, secondaryComparison);
 
-            this.Figures(2) = mag.graphics.visualize( ...
+            this.Figures(3) = mag.graphics.visualize( ...
                 primaryComparison, primaryGraphs, secondaryComparison, secondaryGraphs, ...
-                Name = "Science vs. I-ALiRT", ...
+                Name = "Science vs. I-ALiRT (Closest Vector)", ...
                 Arrangement = [9, 2], ...
                 GlobalLegend = ["Science", "I-ALiRT"], ...
                 LinkXAxes = true, ...
@@ -60,7 +73,7 @@ function visualize(this)
             timestampComparison = table(primaryScience.Time(idxPriMin), primaryIALiRT.Time, secondaryScience.Time(idxSecMin), secondaryIALiRT.Time, ...
                 VariableNames = ["ps", "pi", "ss", "si"]);
 
-            this.Figures(3) = mag.graphics.visualize( ...
+            this.Figures(4) = mag.graphics.visualize( ...
                 timestampComparison, ...
                 [mag.graphics.style.Default(Title = "I-ALiRT FOB vs. FIB", YLabel = "\Deltat [ms]", Charts = mag.graphics.chart.Plot(XVariable = "pi", YVariables = this.getTimingOperation("pi", "si"))), ...
                 mag.graphics.style.Default(Title = "Primary Science vs. I-ALiRT", YLabel = "\Deltat [ms]", Charts = mag.graphics.chart.Plot(XVariable = "ps", YVariables = this.getTimingOperation("ps", "pi"))), ...
@@ -74,6 +87,16 @@ function visualize(this)
 
     methods (Access = private)
 
+        function overlayGraphs = generateOverlayGraph(this, scienceData, iALiRTData)
+
+            combinedData = outerjoin(scienceData.Data(scienceData.Quality, :), iALiRTData.Data(iALiRTData.Quality, :));
+
+            overlayGraphs = {combinedData, ...
+                [mag.graphics.style.Default(Title = this.getFieldTitle(iALiRTData), YLabel = "x [nT]", Charts = [mag.graphics.chart.Plot(YVariables = "x_left", Filter = ~ismissing(combinedData.x_left)), mag.graphics.chart.Scatter(YVariables = "x_right", Filter = ~ismissing(combinedData.x_right), Marker = "x")]), ...
+                mag.graphics.style.Default(YLabel = "y [nT]", Charts = [mag.graphics.chart.Plot(YVariables = "y_left", Filter = ~ismissing(combinedData.y_left)), mag.graphics.chart.Scatter(YVariables = "y_right", Filter = ~ismissing(combinedData.y_right), Marker = "x")]), ...
+                mag.graphics.style.Default(YLabel = "z [nT]", Charts = [mag.graphics.chart.Plot(YVariables = "z_left", Filter = ~ismissing(combinedData.z_left)), mag.graphics.chart.Scatter(YVariables = "z_right", Filter = ~ismissing(combinedData.z_right), Marker = "x")])]};
+        end
+
         function comparisonGraphs = generateComparisonGraph(this, iALiRTData, comparisonData)
 
             defaultColors = colororder();
@@ -91,7 +114,8 @@ function visualize(this)
 
         function action = getTimingOperation(y1, y2)
 
-            action = mag.graphics.operation.Composition(Operations = [mag.graphics.operation.Subtract(Minuend = y1, Subtrahend = y2), ...
+            action = mag.graphics.operation.Composition(Operations = [ ...
+                mag.graphics.operation.Subtract(Minuend = y1, Subtrahend = y2), ...
                 mag.graphics.operation.Convert(Conversion = @milliseconds)]);
         end
     end

tests/unit/visualize/chart/ColorSupportTestCase.m

@@ -1,4 +1,4 @@
-classdef (Abstract) ColorSupportTestCase < MAGVisualizationTestCase
+classdef (Abstract) ColorSupportTestCase < MAGChartTestCase
 % COLORSUPPORTTESTCASE Base class for all charts that support colors.
 
     properties (TestParameter)
@@ -12,7 +12,7 @@
         function setColorProperty(testCase, ColorProperties)
 
             % Set up.
-            [tl, ax] = GraphicsTestUtilities.createFigure(testCase);
+            [tl, ax] = mag.test.GraphicsTestUtilities.createFigure(testCase);
 
             args = testCase.getExtraArguments();
 
@@ -24,11 +24,11 @@ function setColorProperty(testCase, ColorProperties)
             assembledGraph = chart.plot(testCase.Data, ax, tl);
 
             % Verify.
-            graph = GraphicsTestUtilities.getChildrenGraph(testCase, tl, ax, testCase.GraphClassName);
+            graph = mag.test.GraphicsTestUtilities.getChildrenGraph(testCase, tl, ax, testCase.GraphClassName);
 
             testCase.verifySameHandle(assembledGraph, graph, "Chart should return assembled graph.");
 
-            [~, verifiableValue] = GraphicsTestUtilities.getVerifiables(ColorProperties);
+            [~, verifiableValue] = mag.test.GraphicsTestUtilities.getVerifiables(ColorProperties);
             testCase.verifyEqual(graph.(testCase.getColorPropertyName()), verifiableValue, compose("""%s"" property value should match.", ColorProperties.Name));
         end
     end

tests/unit/visualize/chart/MAGVisualizationTestCase.m → tests/unit/visualize/chart/MAGChartTestCase.m

@@ -1,5 +1,5 @@
-classdef (Abstract) MAGVisualizationTestCase < matlab.unittest.TestCase
-% MAGVISUALIZATIONTESTCASE Base class for all MAG visualization tests.
+classdef (Abstract) MAGChartTestCase < matlab.unittest.TestCase
+% MAGCHARTTESTCASE Base class for all MAG chart tests.
 
     properties (Constant)
         % DATA Test data.

tests/unit/visualize/chart/MarkerSupportTestCase.m

@@ -1,4 +1,4 @@
-classdef (Abstract) MarkerSupportTestCase < MAGVisualizationTestCase
+classdef (Abstract) MarkerSupportTestCase < MAGChartTestCase
 % MARKERSUPPORTTESTCASE Base class for all charts that support markers.
 
     properties (TestParameter)
@@ -15,7 +15,7 @@
         function setMarkerProperty(testCase, MarkerProperties)
 
             % Set up.
-            [tl, ax] = GraphicsTestUtilities.createFigure(testCase);
+            [tl, ax] = mag.test.GraphicsTestUtilities.createFigure(testCase);
 
             args = testCase.getExtraArguments();
 
@@ -27,11 +27,11 @@ function setMarkerProperty(testCase, MarkerProperties)
             assembledGraph = chart.plot(testCase.Data, ax, tl);
 
             % Verify.
-            graph = GraphicsTestUtilities.getChildrenGraph(testCase, tl, ax, testCase.GraphClassName);
+            graph = mag.test.GraphicsTestUtilities.getChildrenGraph(testCase, tl, ax, testCase.GraphClassName);
 
             testCase.verifySameHandle(assembledGraph, graph, "Chart should return assembled graph.");
 
-            [verifiableName, verifiableValue] = GraphicsTestUtilities.getVerifiables(MarkerProperties);
+            [verifiableName, verifiableValue] = mag.test.GraphicsTestUtilities.getVerifiables(MarkerProperties);
             testCase.verifyEqual(graph.(verifiableName), verifiableValue, compose("""%s"" property value should match.", MarkerProperties.Name));
         end
     end