Merge pull request #135 from IndicoDataSolutions/nate/resolve_unit_tests

[CAT-892] resolve unit tests
IndicoDataSolutions · Jan 31, 2024 · b411c12 · b411c12
2 parents c972c6c + 3244f76
commit b411c12
Show file tree

Hide file tree

Showing 10 changed files with 41 additions and 30 deletions.
diff --git a/IndicoV2.Abstractions/Submissions/Models/SubmissionStatus.cs b/IndicoV2.Abstractions/Submissions/Models/SubmissionStatus.cs
@@ -3,10 +3,10 @@
     public enum SubmissionStatus
     {
         PROCESSING,
-        FAILED,
-        COMPLETE,
-        PENDING_ADMIN_REVIEW,
+        PENDING_AUTO_REVIEW,
         PENDING_REVIEW,
-        PENDING_AUTO_REVIEW
+        PENDING_ADMIN_REVIEW,
+        COMPLETE,
+        FAILED
     }
 }
diff --git a/IndicoV2.IntegrationTests/DataSets/DataSetClientTests.cs b/IndicoV2.IntegrationTests/DataSets/DataSetClientTests.cs
@@ -17,6 +17,8 @@ public class DataSetClientTests
         private DataHelper _dataHelper;
         private IndicoConfigs _indicoConfigs;
         private int _dataSetId;
+        private int _documentDataSetId;
+        private int _csvDataSetId;
 
         [SetUp]
         public async Task SetUp()
@@ -29,12 +31,16 @@ public async Task SetUp()
             var _rawDataSetId = _indicoConfigs.DatasetId;
             if (_rawDataSetId == 0)
             {
-                var dataset = (await _dataHelper.DataSets().GetAny());
+                var dataset = await _dataHelper.DataSets().GetAny();
                 _dataSetId = dataset.Id;
+                _documentDataSetId = _dataSetId;
+                _csvDataSetId = _dataSetId;
             }
             else
             {
                 _dataSetId = _rawDataSetId;
+                _documentDataSetId = _indicoConfigs.DocumentDatasetId;
+                _csvDataSetId = _indicoConfigs.CsvDatasetId;
             }
         }
 
@@ -88,15 +94,14 @@ public async Task ProcessFiles_ShouldStartProcessing()
         {
             // Arrange
             var files = new[] {_dataHelper.Files().GetSampleFilePath()};
-            await _dataSetClient.AddFilesAsync(_dataSetId, files, default);
+            await _dataSetClient.AddFilesAsync(_documentDataSetId, files, default);
             var downloadedFiles =
-                (await _dataSetClient.FileUploadStatusAsync(_dataSetId, default))
+                (await _dataSetClient.FileUploadStatusAsync(_documentDataSetId, default))
                 .Dataset.Files
                 .Where(f => f.Status == FileStatus.Downloaded)
                 .Select(f => f.Id.Value);
-
             // Act
-            var result = await _dataSetClient.ProcessFileAsync(_dataSetId, downloadedFiles, default);
+            var result = await _dataSetClient.ProcessFileAsync(_documentDataSetId, downloadedFiles, default);
 
             // Assert
             result.Should().NotBeNull();
@@ -107,15 +112,14 @@ public async Task ProcessCsv_ShouldStartProcessing()
         {
             // Arrange
             var files = new[] { _dataHelper.Files().GetSampleCsvPath() };
-            await _dataSetClient.AddFilesAsync(_dataSetId, files, default);
+            await _dataSetClient.AddFilesAsync(_csvDataSetId, files, default);
             var downloadedFiles =
-                (await _dataSetClient.FileUploadStatusAsync(_dataSetId, default))
+                (await _dataSetClient.FileUploadStatusAsync(_csvDataSetId, default))
                 .Dataset.Files
                 .Where(f => f.Status == FileStatus.Downloaded)
                 .Select(f => f.Id.Value);
-
             // Act
-            var result = await _dataSetClient.ProcessCsvAsync(_dataSetId, downloadedFiles, default);
+            var result = await _dataSetClient.ProcessCsvAsync(_csvDataSetId, downloadedFiles, default);
 
             // Assert
             result.Should().NotBeNull();

diff --git a/IndicoV2.IntegrationTests/Submissions/SubmissionClientTests.cs b/IndicoV2.IntegrationTests/Submissions/SubmissionClientTests.cs
@@ -77,7 +77,7 @@ public async Task CreateAsync_ShouldCreateSubmission_FromFilePath()
             var filePath = _dataHelper.Files().GetSampleFilePath();
 
             // Act
-            var submissionIds = await _submissionsClient.CreateAsync(_workflowId, new[] { filePath });
+            var submissionIds = await _submissionsClient.CreateAsync(_workflowId, paths: new[] { filePath });
 
             // Assert
             var submissionId = submissionIds.Single();
@@ -91,13 +91,13 @@ public async Task CreateAsync_ShouldCreateSubmission_FromUri()
             var uri = _dataHelper.Uris().GetSampleUri();
 
             // Act
-            var submissionIds = await _submissionsClient.CreateAsync(_workflowId, new[] { uri });
+            var submissionIds = await _submissionsClient.CreateAsync(_workflowId, uris: new[] { uri });
 
             // Assert
             var submissionId = submissionIds.Single();
             submissionId.Should().BeGreaterThan(0);
         }
-        
+
         [Test]
         public async Task GetAsync_ShouldFetchSubmission()
         {

diff --git a/IndicoV2.IntegrationTests/Utils/Configs/IndicoConfigs.cs b/IndicoV2.IntegrationTests/Utils/Configs/IndicoConfigs.cs
@@ -8,6 +8,9 @@ internal class IndicoConfigs
 
         public int DatasetId => ParseEnvVar("INDICO_TEST_DATASET_ID");
 
+        public int DocumentDatasetId => ParseEnvVar("INDICO_TEST_DOCUMENT_DATASET_ID");
+        public int CsvDatasetId => ParseEnvVar("INDICO_TEST_CSV_DATASET_ID");
+
         public int ModelGroupId => ParseEnvVar("INDICO_TEST_MODELGROUP_ID");
 
         public static int ParseEnvVar(string varName)

diff --git a/IndicoV2.IntegrationTests/Utils/DataHelpers/Files/workflow-sample.pdf b/IndicoV2.IntegrationTests/Utils/DataHelpers/Files/workflow-sample.pdf
diff --git a/IndicoV2.StrawberryShake/DataSets/DataSetSsClient.cs b/IndicoV2.StrawberryShake/DataSets/DataSetSsClient.cs
@@ -40,7 +40,7 @@ public Task<IDataSetGetDatasetsResult> ListAsync(CancellationToken cancellationT
 
         public Task<IDataSetGetDatasetsFullResult> ListFullAsync(int? limit, CancellationToken cancellationToken) =>
             ExecuteAsync(() => _services.GetRequiredService<DataSetGetDatasetsFullQuery>().ExecuteAsync(
-                limit,
+                limit == null ? 100 : limit,
                 cancellationToken));
     }
 }
diff --git a/IndicoV2.StrawberryShake/Exceptions/ErrorHandlingWrapper.cs b/IndicoV2.StrawberryShake/Exceptions/ErrorHandlingWrapper.cs
@@ -11,7 +11,6 @@ protected async Task<TResult> ExecuteAsync<TResult>(Func<Task<IOperationResult<T
             where TResult : class
         {
             var result = await executeAsync();
-
             if (result.Errors != null && result.Errors.Any())
             {
                 throw new GraphQlException(result);

diff --git a/IndicoV2.StrawberryShake/Submissions/SubmissionSsClient.cs b/IndicoV2.StrawberryShake/Submissions/SubmissionSsClient.cs
@@ -20,7 +20,7 @@ public async Task<IEnumerable<int>> CreateUri(int workflowId, IEnumerable<Uri> u
             CancellationToken cancellationToken = default, SubmissionResultVersion? resultsFileVersion = null) =>
             (await ExecuteAsync(async () => await _services.GetRequiredService<WorkflowUrlSubmissionMutation>().ExecuteAsync(
                 workflowId,
-                (IReadOnlyList<string>)uris,
+                uris.Select(s => s.ToString()).ToList(),
                 resultsFileVersion,
                 cancellationToken
             )))

diff --git a/IndicoV2/Ocr/OcrClient.cs b/IndicoV2/Ocr/OcrClient.cs
@@ -33,7 +33,7 @@ internal async Task<JArray> Upload(List<string> filePaths)
         public async Task<string> ExtractDocumentAsync(string filePath, DocumentExtractionPreset preset, CancellationToken cancellationToken)
         {
             JArray fileMetadata;
-            var files = new List<object>();
+            var files = new List<FileInput>();
             fileMetadata = await Upload(new List<string> { filePath });
             foreach (JObject uploadMeta in fileMetadata)
             {
@@ -44,10 +44,10 @@ public async Task<string> ExtractDocumentAsync(string filePath, DocumentExtracti
                     { "upload_type", (string)uploadMeta.GetValue("upload_type") }
                 };
 
-                var file = new
+                var file = new FileInput
                 {
-                    filename = (string)uploadMeta.GetValue("name"),
-                    filemeta = meta.ToString()
+                    Filename = (string)uploadMeta.GetValue("name"),
+                    Filemeta = meta.ToString()
                 };
 
                 files.Add(file);
@@ -62,7 +62,7 @@ public async Task<string> ExtractDocumentAsync(string filePath, DocumentExtracti
             return job.DocumentExtraction.JobIds.First();
         }
         public async Task<string> GetExtractionResultAsync(Uri documentUri, CancellationToken cancellationToken) =>
-            (await GetExtractionResultAsync<JToken>(documentUri, cancellationToken)).Value<string>("text");
+            (await GetExtractionResultAsync<JToken>(documentUri, cancellationToken)).ToString();
 
         public async Task<TResult> GetExtractionResultAsync<TResult>(Uri documentUri, CancellationToken cancellationToken)
         {

diff --git a/IndicoV2/Submissions/SubmissionsClient.cs b/IndicoV2/Submissions/SubmissionsClient.cs
@@ -68,7 +68,8 @@ public async Task<IEnumerable<int>> CreateAsync(int workflowId, IEnumerable<Uri>
             await _strawberryShakeClient.Submissions().CreateUri(workflowId, uris, cancellationToken);
 
         public async Task<IEnumerable<int>> CreateAsync(int workflowId, IEnumerable<string> paths,
-          CancellationToken cancellationToken, SubmissionResultsFileVersion? resultsFileVersion = null) {
+          CancellationToken cancellationToken, SubmissionResultsFileVersion? resultsFileVersion = null)
+        {
             var filesToUpload = new List<(string Name, Stream content)>();
             foreach (var path in paths)
             {
@@ -83,12 +84,16 @@ public async Task<IEnumerable<int>> CreateAsync(int workflowId, IEnumerable<stri
                 }
 
             }
-            return await _strawberryShakeClient.Submissions().Create(workflowId, (IEnumerable<(string Name, string Meta)>)filesToUpload, cancellationToken, (SubmissionResultVersion?)resultsFileVersion);
+            return await CreateAsync(workflowId, filesToUpload: filesToUpload, cancellationToken, resultsFileVersion);
         }
 
         [Obsolete("This is the Legacy version and will be deprecated. Please use ListAsync instead.")]
         public async Task<IEnumerable<ISubmission>> ListAsync(IEnumerable<int> submissionIds, IEnumerable<int> workflowIds, IFilter filters, int limit = 1000,
-            CancellationToken cancellationToken = default) => (IEnumerable<ISubmission>)await ListAsync(submissionIds, workflowIds, filters, null, limit, cancellationToken);
+            CancellationToken cancellationToken = default)
+        {
+            var result = await ListAsync(submissionIds, workflowIds, filters, null, limit, cancellationToken);
+            return result.Data;
+        }
 
 
         public async Task<IHasCursor<IEnumerable<ISubmission>>> ListAsync(IEnumerable<int> submissionIds, IEnumerable<int> workflowIds, IFilter filters, int? after, int limit = 1000, CancellationToken cancellationToken = default)
@@ -116,14 +121,14 @@ public async Task<IHasCursor<IEnumerable<ISubmission>>> ListAsync(IEnumerable<in
         public async Task<ISubmission> GetAsync(int submissionId, CancellationToken cancellationToken = default)
         {
             var result = await _strawberryShakeClient.Submissions().Get(submissionId, cancellationToken);
-            if (!Enum.TryParse(result.Status.ToString().ToUpper(),out Models.SubmissionStatus parsed))
+            if (!Enum.IsDefined(typeof(StrawberryShake.SubmissionStatus), result.Status))
             {
                 throw new NotSupportedException($"Cannot read submission status: {result.Status}");
             }
             return new Submission
             {
                 Id = result.Id ?? 0,
-                Status = parsed,
+                Status = (Models.SubmissionStatus) result.Status,
                 DatasetId = result.DatasetId ?? 0,
                 WorkflowId = result.WorkflowId ?? 0,
                 InputFile = result.InputFile,
@@ -141,7 +146,7 @@ public async Task<string> GenerateSubmissionResultAsync(int submissionId, Cancel
         public async Task<ISubmission> MarkSubmissionAsRetrieved(int submissionId, bool retrieved = true, CancellationToken cancellationToken = default)
         {
             var resultId = await _strawberryShakeClient.Submissions().MarkRetrieved(submissionId, retrieved, cancellationToken);
-            var result = await _strawberryShakeClient.Submissions().List((IReadOnlyList<int?>)new List<int>(submissionId).AsReadOnly(), default, default, default, default, cancellationToken);
+            var result = await _strawberryShakeClient.Submissions().List(new List<int?>(submissionId).AsReadOnly(), default, default, default, default, cancellationToken);
             return new SubmissionSs(result?.Submissions?[0]);
         }