From e5509784c5b6ec31f059234ee59b41bf8e5a4b07 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Fri, 10 Jan 2025 15:27:31 -0500 Subject: [PATCH 1/2] Add tests for data correctness --- python/tests/test_stream.py | 41 ++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/python/tests/test_stream.py b/python/tests/test_stream.py index a540b7e..0b25ff1 100644 --- a/python/tests/test_stream.py +++ b/python/tests/test_stream.py @@ -245,15 +245,13 @@ def test_stream_data_to_filesystem( del stream # close the stream, flush the files group = zarr.open(settings.store_path, mode="r") - data = group["0"] + array = group["0"] - assert data.shape == ( - settings.dimensions[0].chunk_size_px, - settings.dimensions[1].array_size_px, - settings.dimensions[2].array_size_px, - ) + assert array.shape == data.shape + for i in range(array.shape[0]): + assert np.array_equal(array[i, :, :], data[i, :, :]) - metadata = data.metadata + metadata = array.metadata if compression_codec is not None: if version == ZarrVersion.V2: cname = ( @@ -360,25 +358,22 @@ def test_stream_data_to_s3( secret=settings.s3.secret_access_key, client_kwargs={"endpoint_url": settings.s3.endpoint}, ) - store = s3fs.S3Map( - root=f"{s3_settings.bucket_name}/{settings.store_path}", s3=s3 - ) - cache = ( - zarr.LRUStoreCache(store, max_size=2**28) - if version == ZarrVersion.V2 - else zarr.LRUStoreCacheV3(store, max_size=2**28) + store = zarr.storage.FsspecStore.from_url( + f"s3://{settings.s3.bucket_name}/{settings.store_path}", + storage_options={ + "key": s3_settings.access_key_id, + "secret": s3_settings.secret_access_key, + "client_kwargs": {"endpoint_url": s3_settings.endpoint}, + }, ) - group = zarr.group(store=cache) + group = zarr.group(store=store) + array = group["0"] - data = group["0"] - - assert data.shape == ( - settings.dimensions[0].chunk_size_px, - settings.dimensions[1].array_size_px, - settings.dimensions[2].array_size_px, - ) + assert array.shape == data.shape + for i in range(array.shape[0]): + assert np.array_equal(array[i, :, :], data[i, :, :]) - metadata = data.metadata + metadata = array.metadata if compression_codec is not None: if version == ZarrVersion.V2: cname = ( From 421cef428fd6517682babda433fdb09343001b0b Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Mon, 13 Jan 2025 15:07:08 -0500 Subject: [PATCH 2/2] Use a visually interesting data pattern in zarrv3-raw-s3.c. --- examples/zarrv3-raw-s3.c | 39 +++++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/examples/zarrv3-raw-s3.c b/examples/zarrv3-raw-s3.c index 6535d26..77dfca1 100644 --- a/examples/zarrv3-raw-s3.c +++ b/examples/zarrv3-raw-s3.c @@ -1,6 +1,7 @@ /// @file zarrv3-raw-s3.c /// @brief Zarr V3 with uncompressed data to S3 #include "acquire.zarr.h" +#include #include #include @@ -59,6 +60,8 @@ int main() { // Create stream ZarrStream* stream = ZarrStream_create(&settings); + ZarrStreamSettings_destroy_dimension_array(&settings); + if (!stream) { fprintf(stderr, "Failed to create stream\n"); return 1; @@ -69,12 +72,40 @@ int main() { const size_t height = 48; uint16_t* frame = (uint16_t*)malloc(width * height * sizeof(uint16_t)); + int centerX = width / 2; + int centerY = height / 2; + // Write frames size_t bytes_written; - for (int i = 0; i < 10; i++) { - // Fill frame with sample data - for (size_t j = 0; j < width * height; j++) { - frame[j] = i * 1000 + j; + for (int t = 0; t < 50; t++) { + // Fill frame with a moving diagonal pattern + for (size_t y = 0; y < height; y++) { + int dy = y - centerY; + for (size_t x = 0; x < width; x++) { + // Create a diagonal pattern that moves with time + // and varies intensity based on position + int diagonal = (x + y + t * 8) % 32; + + // Create intensity variation + uint16_t intensity; + if (diagonal < 16) { + intensity = (uint16_t)((diagonal * 4096)); // Ramp up + } else { + intensity = (uint16_t)((31 - diagonal) * 4096); // Ramp down + } + + int dx = x - centerX; + + // Add some circular features + int radius = (int)sqrt(dx * dx + dy * dy); + + // Modulate the pattern with concentric circles + if (radius % 16 < 8) { + intensity = (uint16_t)(intensity * 0.7); + } + + frame[y * width + x] = intensity; + } } ZarrStatusCode status = ZarrStream_append(