Merge pull request #41 from ImperialCollegeLondon/io

Update software to v4.0.0

.github/workflows/matlab.yml

@@ -38,7 +38,7 @@ jobs:
     if: github.ref == 'refs/heads/main'
     needs: test
     env:
-      VERSION: "3.7.1"
+      VERSION: "4.0.0"
     steps:
       - name: Check out repository
         uses: actions/checkout@v3

resources/ReleaseNotes.md

@@ -1,6 +1,22 @@
+# App
+
+- Adapt to refactoring of import/export
+- Add support for CDF import/export
+- Remove support for DAT export
+
 # Software
 
-- Rename `mag.IMAPTestingAnalysis` to `mag.IMAPAnalysis`
-- Fix issue with mode being mistakenly confused for "Ramp" when first range is `NaN`
-- Fix issue in `mag.graphics.sftPlots` with cropped analysis not defined when `Filter` is empty
-- Add system and unit tests for `mag.IMAPAnalysis`
+## IO
+
+- Refactor import/export of science and HK data
+- Add support for CDF science import/export
+- Remove support for DAT export
+
+## Other
+
+- Add `Science` property to `mag.Instrument` and `mag.IALiRT` as back-end for `Primary` and `Secondary` properties
+- Add `Outboard` and `Inboard` property to `mag.Instrument`
+- Add support in `mag.meta.log.Word` for EM documents
+- Add `mag.mixin.Struct` to enable support to convert to `struct` for meta data classes
+- Move description of sensor setup to `mag.meta.Setup`
+- Add tests for IO classes and utilities

src/analyze/+mag/+process/Calibration.m

@@ -2,7 +2,7 @@
 % CALIBRATION Correct data by applying scale factor, misalignment and
 % offset.
 
-    properties (Constant, Hidden)
+    properties (Constant)
         % FILELOCATION Location of calibration files.
         FileLocation (1, 1) string = fullfile(fileparts(mfilename("fullpath")), "../../calibration")
     end
@@ -60,11 +60,17 @@
 
             ranges = unique(data.range);
 
+            if isempty(metaData.Setup)
+                modelName = string.empty();
+            else
+                modelName = metaData.Setup.Model;
+            end
+
             for r = ranges'
 
                 locRange = data.range == r;
 
-                calibrationFile = this.getFileName(r, metaData.Model);
+                calibrationFile = this.getFileName(r, modelName);
                 data{locRange, this.Variables} = this.applyCalibration(data{locRange, this.Variables}, calibrationFile);
             end
         end

src/analyze/+mag/+process/Separate.m

@@ -50,7 +50,7 @@
 
             if isequal(this.Variables, "*")
 
-                locMissingCompatible = varfun(@this.isMissingCompatible, data, OutputFormat = "uniform");
+                locMissingCompatible = varfun(@mag.internal.isMissingCompatible, data, OutputFormat = "uniform");
                 variables = data.Properties.VariableNames(locMissingCompatible);
 
                 variables(variables == this.DiscriminationVariable) = [];
@@ -70,11 +70,4 @@
             data = [data; finalRow];
         end
     end
-
-    methods (Static, Access = private)
-
-        function tf = isMissingCompatible(x)
-            tf = isa(x, "single") | isa(x, "double") | isa(x, "duration") | isa(x, "calendarDuration") | isa(x, "datetime") | isa(x, "categorical") | isa(x, "string");
-        end
-    end
 end

src/analyze/+mag/+process/Units.m

@@ -64,7 +64,7 @@
 
             arguments
                 this
-                data table
+                data tabular
                 metaData (1, 1) mag.meta.HK
             end
 

src/analyze/+mag/@IMAPAnalysis/IMAPAnalysis.m

@@ -11,7 +11,8 @@
         % EVENTPATTERN Pattern of event files.
         EventPattern (1, :) string = fullfile("*", "Event", "*.html")
         % METADATAPATTERN Pattern of meta data files.
-        MetaDataPattern (1, :) string = [fullfile("*.msg"), fullfile("IMAP-MAG-TE-ICL-061*.xlsx"), fullfile("IMAP-MAG-TE-ICL-071*.docx")]
+        MetaDataPattern (1, :) string = [fullfile("*.msg"), fullfile("IMAP-MAG-TE-ICL-058*.xlsx"), fullfile("IMAP-MAG-TE-ICL-061*.xlsx"), ...
+            fullfile("IMAP-MAG-TE-ICL-071*.docx"), fullfile("IMAP-OPS-TE-ICL-001*.docx")]
         % SCIENCEPATTERN Pattern of science data files.
         SciencePattern (1, :) string = fullfile("MAGScience-*-(*)-*.csv")
         % IALIRTPATTERN Pattern of I-ALiRT data files.
@@ -23,10 +24,7 @@
             fullfile("*", "Export", "idle_export_proc.*.csv")]
         % PERFILEPROCESSING Steps needed to process single files of data.
         PerFileProcessing (1, :) mag.process.Step = [ ...
-            mag.process.Missing(Variables = ["x", "y", "z"]), ...
             mag.process.AllZero(Variables = ["coarse", "fine", "x", "y", "z"]), ...
-            mag.process.Timestamp(), ...
-            mag.process.DateTime(), ...
             mag.process.SignedInteger(CompressionVariable = "compression", Variables = ["x", "y", "z"]), ...
             mag.process.Separate(DiscriminationVariable = "t", QualityVariable = "quality", Variables = ["x", "y", "z"])]
         % WHOLEDATAPROCESSING Steps needed to process all of imported data.
@@ -52,7 +50,6 @@
             mag.process.Ramp()]
         % HKPROCESSING Steps needed to process imported HK data.
         HKProcessing (1, :) mag.process.Step = [ ...
-            mag.process.DateTime(), ...
             mag.process.Units(), ...
             mag.process.Separate(DiscriminationVariable = "t", QualityVariable = string.empty(), Variables = "*")]
     end
@@ -170,13 +167,13 @@ function load(this)
 
             this.loadEventsData();
 
-            [primaryMetaData, secondaryMetaData, hkMetaData] = this.loadMetaData();
+            [primarySetup, secondarySetup] = this.loadMetaData();
 
-            this.loadScienceData(primaryMetaData, secondaryMetaData);
+            this.loadScienceData(primarySetup, secondarySetup);
 
-            this.loadIALiRTData(primaryMetaData, secondaryMetaData);
+            this.loadIALiRTData(primarySetup, secondarySetup);
 
-            this.loadHKData(hkMetaData);
+            this.loadHKData();
         end
 
         function modes = getAllModes(this)
@@ -327,9 +324,7 @@ function load(this)
             end
 
             rampMode = this.Results.copy();
-
-            rampMode.Primary = this.PrimaryRamp;
-            rampMode.Secondary = this.SecondaryRamp;
+            rampMode.Science = [this.PrimaryRamp, this.SecondaryRamp];
 
             if rampMode.HasScience
                 rampMode.cropToMatch();
@@ -416,13 +411,16 @@ function load(this)
         loadEventsData(this)
 
         % LOADMETADATA Load meta data.
-        [primaryMetaData, secondaryMetaData, hkMetaData] = loadMetaData(this)
+        [primarySetup, secondarySetup] = loadMetaData(this)
 
         % LOADSCIENCEDATA Load science data.
-        loadScienceData(this, primaryMetaData, secondaryMetaData)
+        loadScienceData(this, primarySetup, secondarySetup)
+
+        % LOADIALIRTDATA Load I-ALiRT data.
+        loadIALiRTData(this, primarySetup, secondarySetup)
 
         % LOADHKDATA Load HK data.
-        loadHKData(this, hkMetaData)
+        loadHKData(this)
 
         % GENERATEEVENTTABLE Create an event table for a sensor, based on
         % detected events and science data.
@@ -494,26 +492,28 @@ function load(this)
 
     methods (Static, Access = private)
 
-        function importExportStrategy = dispatchExtension(extension, options)
+        function importStrategy = dispatchExtension(extension, type)
         % DISPATCHEXTENSION Dispatch extension to correct I/O strategy.
 
             arguments (Input)
                 extension
-                options.?mag.io.Type
+                type (1, 1) string {mustBeMember(type, ["Science", "HK"])}
             end
 
             arguments (Output)
-                importExportStrategy (1, 1) mag.io.Type
+                importStrategy (1, 1) mag.io.in.Format
             end
 
-            args = namedargs2cell(options);
-
             switch extension
-                case cellstr(mag.io.CSV.Extension)
-                    importExportStrategy = mag.io.CSV(args{:});
+                case mag.io.in.CSV.Extension
+                    format = "CSV";
+                case mag.io.in.CDF.Extension
+                    format = "CDF";
                 otherwise
                     error("Unsupported extension ""%s"" for science data import.", extension);
             end
+
+            importStrategy = feval("mag.io.in." + type + format);
         end
     end
 end

src/analyze/+mag/@IMAPAnalysis/export.m

@@ -1,13 +1,18 @@
-function export(this, exportStrategy, options)
+function export(this, exportType, options)
 
     arguments
         this (1, 1) mag.IMAPAnalysis
-        exportStrategy (1, 1) mag.io.Type
+        exportType (1, 1) string {mustBeMember(exportType, ["MAT", "CDF"])}
         options.Location (1, 1) string {mustBeFolder} = "results"
         options.StartTime (1, 1) datetime = NaT(TimeZone = "UTC")
         options.EndTime (1, 1) datetime = NaT(TimeZone = "UTC")
     end
 
+    % Determine export classes.
+    scienceFormat = feval("mag.io.out.Science" + exportType);
+    hkFormat = feval("mag.io.out.HK" + exportType);
+
+    % Determine export window.
     if ismissing(options.StartTime)
         options.StartTime = datetime("-Inf", TimeZone = "UTC");
     end
@@ -18,20 +23,13 @@ function export(this, exportStrategy, options)
 
     period = timerange(options.StartTime, options.EndTime, "closed");
 
-    extension = exportStrategy.Extension;
-    scienceExportFormat = exportStrategy.ScienceExportFormat;
-    hkExportFormat = exportStrategy.HKExportFormat;
-
     % Export full science.
     if this.Results.HasScience
 
-        r = this.Results.copy();
-        r.crop(period);
+        results = this.Results.copy();
+        results.crop(period);
 
-        scienceData = scienceExportFormat.formatForExport(r);
-
-        exportStrategy.ExportFileName = fullfile(options.Location, compose("%s %s (%d, %d)", datestr(r.Primary.MetaData.Timestamp, "ddmmyy-hhMM"), r.Primary.MetaData.Mode, r.Primary.MetaData.DataFrequency, r.Secondary.MetaData.DataFrequency) + extension); %#ok<DATST>
-        exportStrategy.export(scienceData);
+        mag.io.export(results, Location = options.Location, Format = scienceFormat);
     end
 
     % Export each mode.
@@ -40,10 +38,7 @@ function export(this, exportStrategy, options)
     for m = modes
 
         m.crop(period);
-        modeData = scienceExportFormat.formatForExport(m);
-
-        exportStrategy.ExportFileName = fullfile(options.Location, compose("%s %s (%d, %d)", datestr(m.Primary.MetaData.Timestamp, "ddmmyy-hhMM"), m.Primary.MetaData.Mode, m.Primary.MetaData.DataFrequency, m.Secondary.MetaData.DataFrequency) + extension); %#ok<DATST>
-        exportStrategy.export(modeData);
+        mag.io.export(m, Location = options.Location, Format = scienceFormat);
     end
 
     % Export I-ALiRT.
@@ -52,44 +47,33 @@ function export(this, exportStrategy, options)
     if ~isempty(iALiRT)
 
         iALiRT.crop(period);
-        iALiRTData = scienceExportFormat.formatForExport(iALiRT);
-
-        exportStrategy.ExportFileName = fullfile(options.Location, compose("%s %s (%.2f, %.2f)", datestr(iALiRT.Primary.MetaData.Timestamp, "ddmmyy-hhMM"), iALiRT.Primary.MetaData.Mode, iALiRT.Primary.MetaData.DataFrequency, iALiRT.Secondary.MetaData.DataFrequency) + extension); %#ok<DATST>
-        exportStrategy.export(iALiRTData);
+        mag.io.export(iALiRT, Location = options.Location, Format = scienceFormat);
     end
 
     % Export range cycling.
     rangeCycling = this.getRangeCycling();
 
     if ~isempty(rangeCycling)
 
-        rangeData = scienceExportFormat.formatForExport(rangeCycling);
-
-        exportStrategy.ExportFileName = fullfile(options.Location, compose("%s Range Cycling", datestr(rangeCycling.MetaData.Timestamp, "ddmmyy-hhMM")) + extension); %#ok<DATST>
-        exportStrategy.export(rangeData);
+        mag.io.export(rangeCycling, Location = options.Location, Format = scienceFormat, ...
+            FileName = compose("%s Range Cycling", datestr(rangeCycling.MetaData.Timestamp, "ddmmyy-hhMM")) + scienceFormat.Extension); %#ok<DATST>
     end
 
     % Export ramp mode.
     rampMode = this.getRampMode();
 
     if ~isempty(rampMode)
 
-        rampData = scienceExportFormat.formatForExport(rampMode);
-
-        exportStrategy.ExportFileName = fullfile(options.Location, compose("%s Ramp Mode", datestr(rampMode.MetaData.Timestamp, "ddmmyy-hhMM")) + extension); %#ok<DATST>
-        exportStrategy.export(rampData);
+        mag.io.export(rampMode, Location = options.Location, Format = scienceFormat, ...
+            FileName = compose("%s Ramp Mode", datestr(rampMode.MetaData.Timestamp, "ddmmyy-hhMM")) + scienceFormat.Extension); %#ok<DATST>
     end
 
     % Export HK data.
-    if ~isempty(hkExportFormat) && ~isempty(this.Results.HK)
+    if ~isempty(this.Results.HK)
 
         hk = this.Results.HK.copy();
         hk.crop(period);
 
-        hkData = hkExportFormat.formatForExport(hk);
-        hkMetaData = [hk.MetaData];
-
-        exportStrategy.ExportFileName = fullfile(options.Location, compose("%s HK", datestr(min([hkMetaData.Timestamp]), "ddmmyy-hhMM")) + extension); %#ok<DATST>
-        exportStrategy.export(hkData);
+        mag.io.export(hk, Location = options.Location, Format = hkFormat);
     end
 end

src/analyze/+mag/@IMAPAnalysis/generateEventTable.m

@@ -1,20 +1,23 @@
-function eventTable = generateEventTable(this, primaryOrSecondary, sensorEvents, data)
+function eventTable = generateEventTable(~, data, sensorEvents)
 
     arguments (Input)
-        this (1, 1) mag.IMAPAnalysis
-        primaryOrSecondary (1, 1) string {mustBeMember(primaryOrSecondary, ["Primary", "Secondary"])}
+        ~
+        data (1, 1) mag.Science
         sensorEvents timetable
-        data timetable
     end
 
     arguments (Output)
         eventTable eventtable
     end
 
-    ranges = sortrows(data(:, "range"));
-
     % Select sensor.
-    sensor = string(this.Results.getSensor(primaryOrSecondary));
+    sensor = string(data.MetaData.Sensor);
+
+    if data.MetaData.Primary
+        primaryOrSecondary = "Primary";
+    else
+        primaryOrSecondary = "Secondary";
+    end
 
     % Adapt existing event properties.
     if contains("Sensor", sensorEvents.Properties.VariableNames)
@@ -26,10 +29,10 @@
     sensorEvents = removeUninterestingVariables(sensorEvents, primaryOrSecondary);
 
     % Improve ramp mode timestamp estimates.
-    sensorEvents = updateRampModeTimestamps(sensorEvents, data);
+    sensorEvents = updateRampModeTimestamps(sensorEvents, data.Data);
 
     % Improve estimates of mode changes.
-    sensorEvents = findModeChanges(data, sensorEvents);
+    sensorEvents = findModeChanges(data.Data, sensorEvents);
 
     % Basic structure for events table.
     emptyTime = datetime.empty();
@@ -48,6 +51,8 @@
     % Process range changes.
     % Range changes can be automatic, so add automatic transitions. If they
     % are commanded, correct the times at which they occur.
+    ranges = sortrows(data.Data(:, data.Settings.Range));
+
     if ~isempty(ranges)
 
         [rangeTable, sensorEvents] = findRangeChanges(ranges, sensorEvents, primaryOrSecondary);
@@ -67,7 +72,7 @@
     end
 
     % Add sensor shutdown.
-    shutDownTable = array2timetable(repmat(missing(), [1, numel(eventTable.Properties.VariableNames)]), RowTimes = max(data.t) + eps(), VariableNames = eventTable.Properties.VariableNames);
+    shutDownTable = array2timetable(NaN(1, numel(eventTable.Properties.VariableNames)), RowTimes = max(data.Time) + eps(), VariableNames = eventTable.Properties.VariableNames);
     shutDownTable.Label = primaryOrSecondary + " Shutdown";
     shutDownTable.Reason = "Command";
 
@@ -81,7 +86,7 @@
     eventTable.Reason = categorical(eventTable.Reason);
 
     eventTable{contains(eventTable.Label, "Config"), ["DataFrequency", "PacketFrequency", "Duration"]} = missing();
-    eventTable{contains(eventTable.Label, "Ramp"), "Range"} = missing();
+    eventTable{contains(eventTable.Label, "Ramp"), "Range"} = NaN();
 
     % Convert to event table.
     eventTable = eventtable(eventTable, EventLabelsVariable = "Label");