Merge pull request #28 from ImperialCollegeLondon/v321

Update software to v3.3.0

.github/workflows/matlab.yml

@@ -38,7 +38,7 @@ jobs:
     if: github.ref == 'refs/heads/main'
     needs: test
     env:
-      VERSION: "3.2.0"
+      VERSION: "3.3.0"
     steps:
       - name: Check out repository
         uses: actions/checkout@v3

resources/ReleaseNotes.md

@@ -1,15 +1,29 @@
 # Software
 
-- Add compression flag to `mag.Science`
-- Add settings file to specify name of `timetable` properties for `mag.Science`
-- Add `Compression` property to supported events for plotting
-- Add `YAxisLocation` for `mag.graphics.style.Default` and `mag.graphics.style.Stackedplot`
-- Shift `YAxisLocation` to "right" for plots on right-hand side of some views: `mag.graphics.view.Field`, `mag.graphics.view.Frequency`, `mag.graphics.view.HK`, `mag.graphics.view.IALiRT`, and `mag.graphics.view.RampMode`
-- Do not show figure until it is fully populated in `mag.graphics.visualize`
-- Create `mag.meta.Mode` enumeration to capture sensor science mode
-- Move definition of time constants to shared utility file `mag.time.Constant`
-- Make `mag.process.Range` and `mag.process.SignedInteger` more flexible to custom variable names
-- Make sure converted values in `mag.process.SignedInteger` are returned as `double`
-- Fix [#10](https://github.com/ImperialCollegeLondon/MAG-Data-Visualization-Toolbox/issues/10)
-- Fix [#19](https://github.com/ImperialCollegeLondon/MAG-Data-Visualization-Toolbox/issues/19)
-- Add tests for processing step documentation properties
+## Compression
+
+- Add processing step `mag.process.Compression` to correct for compression factor
+- Allow filtering around compression changes in `mag.process.Filter`
+- Fix issue in `mag.process.SignedInteger` with compression still using 16th bit for signedness, instead of 18th
+- Fix typos in compression event plot
+
+## Export
+
+- Export formats are defined in `mag.io.Type`
+- Export formats accept data structures `mag.Instrument` and `mag.HK`, instead of custom structure
+- Add `Compression` flag to science export
+- Add `PROCSTAT` to HK export
+
+## Other
+
+- Add variable continuity definition for science variables in `timetable`
+- Add shutdown event as final event in `mag.Science` event table
+- Add property `Harness` in `mag.meta.Science` to describe sensor harness
+- Add ability to specify final event end time in `mag.graphics.view.Field`
+- Simplify how columns are filtered in loading science and I-ALiRT
+- Make `mag.process.Calibration` more flexible to custom variable names
+- Allow plotting more than one event in `mag.graphics.view.Field`
+- Change event type to `categorical` when data is not numeric
+- Add option to create folder in `mag.graphics.savePlots`
+- Fix issue with loading Excel files containing sensor meta data
+- Fix issue in custom events chart when event value is not `double` or `single`

src/analyze/+mag/+process/Calibration.m

@@ -19,6 +19,9 @@
         % DEFAULTCALIBRATIONFILE Default file containing scale factor,
         % misalignment and offset information.
         DefaultCalibrationFile (1, 1) string {mustBeFile} = fullfile(mag.process.Calibration.FileLocation, "default.txt")
+        % VARIABLES Variables to be converted using calibration
+        % information.
+        Variables (1, :) string
     end
 
     methods
@@ -36,8 +39,8 @@
             value = "Calibration";
         end
 
-        function value = get.Description(~)
-            value = "Calibrate science data by applying scale factor, misalignment and offset.";
+        function value = get.Description(this)
+            value = "Calibrate " + join(compose("""%s""", this.Variables), ", ") + " data by applying scale factor, misalignment and offset.";
         end
 
         function value = get.DetailedDescription(this)
@@ -62,7 +65,7 @@
                 locRange = data.range == r;
 
                 calibrationFile = this.getFileName(r, metaData.Model);
-                data{locRange, ["x", "y", "z"]} = this.applyCalibration(data{locRange, ["x", "y", "z"]}, calibrationFile);
+                data{locRange, this.Variables} = this.applyCalibration(data{locRange, this.Variables}, calibrationFile);
             end
         end
     end

src/analyze/+mag/+process/Compression.m

@@ -0,0 +1,47 @@
+classdef Compression < mag.process.Step
+% COMPRESSION Apply correction for compressed data.
+
+    properties (Dependent)
+        Name
+        Description
+        DetailedDescription
+    end
+
+    properties
+        % COMPRESSIONVARIABLE Name of compression variable.
+        CompressionVariable (1, 1) string
+        % VARIABLES Variables to be corrected using compression
+        % information.
+        Variables (1, :) string
+    end
+
+    methods
+
+        function this = Compression(options)
+
+            arguments
+                options.?mag.process.Compression
+            end
+
+            this.assignProperties(options);
+        end
+
+        function value = get.Name(~)
+            value = "Apply Compression Correction";
+        end
+
+        function value = get.Description(this)
+            value = "Apply correction to " + join(compose("""%s""", this.Variables), ", ") + " based on compression """ + this.CompressionVariable + """.";
+        end
+
+        function value = get.DetailedDescription(this)
+            value = this.Description;
+        end
+
+        function data = apply(this, data, ~)
+
+            locCompressed = logical(data.(this.CompressionVariable));
+            data{locCompressed, this.Variables} = data{locCompressed, this.Variables} / 4;
+        end
+    end
+end

src/analyze/+mag/+process/Filter.m

@@ -8,10 +8,19 @@
     end
 
     properties
-        % ONRANGECHANGE How many vectors to remove when range changes.
-        OnRangeChange (1, 2) {mustBeA(OnRangeChange, ["double", "duration"])} = zeros(1, 2)
+        % MODEVARIABLE Name of mode change variable.
+        ModeVariable (1, 1) string = "DataFrequency"
         % ONMODECHANGE How many vectors to remove when mode changes.
         OnModeChange (1, 2) {mustBeA(OnModeChange, ["double", "duration"])} = zeros(1, 2)
+        % RANGEVARIABLE Name of range variable.
+        RangeVariable (1, 1) string = "Range"
+        % ONRANGECHANGE How many vectors to remove when range changes.
+        OnRangeChange (1, 2) {mustBeA(OnRangeChange, ["double", "duration"])} = zeros(1, 2)
+        % COMPRESSIONVARIABLE Name of compression variable.
+        CompressionVariable (1, 1) string = "Compression"
+        % ONCOMPRESSIONCHANGE How many vectors to remove when compression
+        % changes.
+        OnCompressionChange (1, 2) {mustBeA(OnCompressionChange, ["double", "duration"])} = zeros(1, 2)
     end
 
     methods
@@ -38,8 +47,9 @@
             value = this.Description + " After said events, onboard filtering " + ...
                 "needs time to adjust, thus some data points are dropped for display purposes. " + ...
                 "For range changes, " + string(this.OnRangeChange(1)) + " before and " + string(this.OnRangeChange(2)) + ...
-                " after are dropped, and for mode changes, " + string(this.OnModeChange(1)) + " before and " + ...
-                string(this.OnModeChange(2)) + " after are dropped.";
+                " after are dropped, for mode changes, " + string(this.OnModeChange(1)) + " before and " + ...
+                string(this.OnModeChange(2)) + " after are dropped, and for compression changes, " + ...
+                string(this.OnCompressionChange(1)) + " before and " + string(this.OnCompressionChange(2)) + " after are dropped.";
         end
 
         function data = apply(this, data, ~)
@@ -50,29 +60,43 @@
                 ~
             end
 
-            [startTime, endTime] = bounds(data.t);
-
             events = data.Properties.Events;
-            events = events(timerange(startTime, endTime, "closed"), :);
+            [startTime, endTime] = bounds(data.Properties.RowTimes);
+
+            if isempty(events)
+                events = data;
+            else
+                events = events(timerange(startTime, endTime, "closed"), :);
+            end
 
             % Filter data points at mode changes.
             if ~isequal(this.OnModeChange, zeros(1, 2))
-                data = this.cropDataWithRange(events, data, "DataFrequency", this.OnModeChange);
+                data = this.cropDataWithRange(events, data, this.ModeVariable, this.OnModeChange);
             end
 
             % Filter duration at range changes.
             if ~isequal(this.OnRangeChange, zeros(1, 2))
-                data = this.cropDataWithRange(events, data, "Range", this.OnRangeChange);
+                data = this.cropDataWithRange(events, data, this.RangeVariable, this.OnRangeChange);
+            end
+
+            % Filter duration at compression changes.
+            if ~isequal(this.OnCompressionChange, zeros(1, 2))
+                data = this.cropDataWithRange(data, data, this.CompressionVariable, this.OnCompressionChange);
             end
 
             % Filter out between config and ramp mode.
             % Ramp mode is surrounded by two config modes. Remove data from
             % the first to the last config.
-            locConfig = contains(events.Label, "Config");
-            idxConfig = find(locConfig);
+            if isa(events, "eventtable")
 
-            if (nnz(locConfig) == 2) && any(contains([events.Label(idxConfig(1):idxConfig(end))], "Ramp"))
-                data{timerange(events.Time(idxConfig(1)), events.Time(idxConfig(end)), "closed"), "quality"} = false;
+                locConfig = contains(events.Label, "Config");
+                idxConfig = find(locConfig);
+
+                if (nnz(locConfig) == 2) && any(contains([events.Label(idxConfig(1):idxConfig(end))], "Ramp"))
+
+                    configRange = timerange(events.Time(idxConfig(1)), events.Time(idxConfig(end)), "closed");
+                    data{configRange, "quality"} = false;
+                end
             end
         end
     end
@@ -81,23 +105,23 @@
 
         function data = cropDataWithRange(events, data, name, range)
 
-            dt = mode(diff(data.t));
-            locEvent = [true; diff(events.(name)) ~= 0];
+            dt = mode(diff(data.Properties.RowTimes));
+            locEvent = [false; diff(events.(name)) ~= 0];
 
-            for t = events.Time(locEvent)'
+            for t = events.Properties.RowTimes(locEvent)'
 
                 if isa(range, "duration")
                     data{timerange(t + range(1), t + range(2), "closed"), "quality"} = false;
                 else
 
-                    tEvent = data(withtol(t, dt), :).t;
+                    tEvent = data(withtol(t, dt), :).Properties.RowTimes;
 
                     if isempty(tEvent)
                         continue;
                     elseif isscalar(tEvent)
-                        idxTime = find(data.t == tEvent);
+                        idxTime = find(data.Properties.RowTimes == tEvent);
                     else
-                        [~, idxTime] = min(abs(data.t - t));
+                        [~, idxTime] = min(abs(data.Properties.RowTimes - t));
                     end
 
                     r = range(1):range(2);

src/analyze/+mag/+process/SignedInteger.m

@@ -9,6 +9,8 @@
     end
 
     properties
+        % COMPRESSIONVARIABLE Name of compression variable.
+        CompressionVariable (1, 1) string
         % VARIABLES Variables to be converted to signed integer.
         Variables (1, :) string
     end
@@ -38,7 +40,13 @@
 
         function data = apply(this, data, ~)
 
-            data{:, this.Variables} = this.convertToSignedInteger(data{:, this.Variables});
+            rf = rowfilter(data);
+
+            uncompressed = rf.(this.CompressionVariable) == false;
+            compressed = rf.(this.CompressionVariable) == true;
+
+            data{uncompressed, this.Variables} = this.convertToSignedInteger(data{uncompressed, this.Variables}, 16);
+            data{compressed, this.Variables} = this.convertToSignedInteger(data{compressed, this.Variables}, 18);
 
             for v = this.Variables
                 data.(v) = cast(data.(v), "double");
@@ -48,36 +56,27 @@
 
     methods (Hidden)
 
-        function signedData = convertToSignedInteger(this, unsignedData)
+        function signedData = convertToSignedInteger(~, unsignedData, signedBit)
 
             arguments (Input)
-                this
-                unsignedData (:, :) double
+                ~
+                unsignedData {mustBeNumeric}
+                signedBit (1, 1) double = 16
             end
 
-            arguments (Output)
-                signedData (:, :) double
+            if isa(unsignedData, "double")
+                assumedType = {"int16"};
+            else
+                assumedType = {};
             end
 
-            try
-                signedData = this.doConvert(unsignedData);
-            catch exception
+            if isempty(unsignedData)
+                signedData = unsignedData;
+            else
 
-                if isequal(exception.identifier, "MATLAB:bitget:outOfRange")
-                    signedData = this.doConvert(unsignedData, "int16");
-                else
-                    exception.rethrow();
-                end
+                isNegative = bitget(unsignedData, signedBit, assumedType{:});
+                signedData = bitset(unsignedData, signedBit, 0, assumedType{:}) + ((-2 ^ (signedBit - 1)) * isNegative);
             end
         end
     end
-
-    methods (Static, Access = private)
-
-        function signedData = doConvert(unsignedData, varargin)
-
-            isNegative = bitget(unsignedData, 16, varargin{:});
-            signedData = bitset(unsignedData, 16, 0, varargin{:}) + ((-2 ^ 15) * isNegative);
-        end
-    end
 end

src/analyze/+mag/@IMAPTestingAnalysis/IMAPTestingAnalysis.m

@@ -26,7 +26,7 @@
             mag.process.Missing(Variables = ["x", "y", "z"]), ...
             mag.process.Timestamp(), ...
             mag.process.DateTime(), ...
-            mag.process.SignedInteger(Variables = ["x", "y", "z"])]
+            mag.process.SignedInteger(CompressionVariable = "compression", Variables = ["x", "y", "z"])]
         % WHOLEDATAPROCESSING Steps needed to process all of imported data.
         WholeDataProcessing (1, :) mag.process.Step = [ ...
             mag.process.Sort(), ...
@@ -36,13 +36,15 @@
         ScienceProcessing (1, :) mag.process.Step = [
             mag.process.Filter(OnModeChange = [0, 1], OnRangeChange = [-1, 5]), ...
             mag.process.Range(RangeVariable = "range", Variables = ["x", "y", "z"]), ...
-            mag.process.Calibration()]
+            mag.process.Calibration(Variables = ["x", "y", "z"]), ...
+            mag.process.Compression(CompressionVariable = "compression", Variables = ["x", "y", "z"])]
         % IALIRTPROCESSING Steps needed to process only I-ALiRT data.
         IALiRTProcessing (1, :) mag.process.Step = [
             mag.process.Filter(OnRangeChange = [0, 1]), ...
             mag.process.AllZero(Variables = ["x", "y", "z"]), ...
             mag.process.Range(RangeVariable = "range", Variables = ["x", "y", "z"]), ...
-            mag.process.Calibration()]
+            mag.process.Calibration(Variables = ["x", "y", "z"]), ...
+            mag.process.Compression(CompressionVariable = "compression", Variables = ["x", "y", "z"])]
         % RAMPPROCESSING Steps needed to process only ramp mode data.
         RampProcessing (1, :) mag.process.Step = [ ...
             mag.process.Unwrap(Variables = ["x", "y", "z"]), ...
@@ -344,7 +346,7 @@ function load(this)
 
                 if any(events{end-2:end, "Mode"} == "Normal")
 
-                    events = events((events.Mode == "Normal") & (events.DataFrequency == 2), :);
+                    events = events((events.Mode == "Normal") & (events.DataFrequency == 2) & ~contains(events.Label, "Shutdown"), :);
                     period = timerange(events.Time(end), endTime, "closed");
                 else
                     period = timerange(NaT(TimeZone = "UTC"), NaT(TimeZone = "UTC"));