Update Zarr V3 writer to latest spec (#33)

* update zarr-python dependency to 3.0.0b3 * Explicitly depend on ZLIB * Update group metadata. * wip: array up to V3 spec * Tests passing. * Add another example (V3 raw filesystem). * Fix memory leak in examples. * Use uint32_t for CRC32 checksum, as unsigned long is 8 bytes on some systems. * Metadata file 0/.zattrs is no longer necessary. * Update Python tests. * Update Python project to v0.0.4. * Update test Python to 3.11 and zarr-python dependency to 3.0.0rc1. * dataset/0 will not exist because dataset/0/.zattrs is no longer created. * revert examples changes * Respond to PR comments.
acquire-project · Jan 3, 2025 · 873d314 · 873d314
1 parent 9f101d4
commit 873d314
Show file tree

Hide file tree

Showing 23 changed files with 573 additions and 503 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -155,10 +155,10 @@ jobs:
           submodules: true
           ref: ${{ github.event.pull_request.head.sha }}
 
-      - name: Set up Python 3.10
+      - name: Set up Python 3.11
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.11"
 
       - name: Install vcpkg
         run: |

diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt
@@ -14,4 +14,5 @@ target_link_libraries(${tgt} PRIVATE
         acquire-zarr
         nlohmann_json::nlohmann_json
         miniocpp::miniocpp
+        ZLIB::ZLIB
 )
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
@@ -18,10 +18,10 @@ foreach (name ${examples})
             ${PROJECT_SOURCE_DIR}/include
     )
     target_link_libraries(${tgt} PRIVATE
-            acquire-logger
             acquire-zarr
             nlohmann_json::nlohmann_json
             miniocpp::miniocpp
+            ZLIB::ZLIB
     )
 
     add_test(NAME example-${tgt} COMMAND ${tgt})

diff --git a/examples/zarrv3-compressed-filesystem.c b/examples/zarrv3-compressed-filesystem.c
@@ -1,6 +1,8 @@
 /// @file zarr-v3-compressed-filesystem.c
 /// @brief Zarr V3 with LZ4 compression to filesystem
 #include "acquire.zarr.h"
+
+#include <math.h>
 #include <stdio.h>
 #include <stdlib.h>
 
@@ -79,21 +81,44 @@ int main() {
 
     // Write frames
     size_t bytes_written;
-    for (int i = 0; i < 10; i++) {
-        // Fill frame with sample data
-        for (size_t j = 0; j < width * height; j++) {
-            frame[j] = i * 1000 + j;
+    int centerX = width / 2;
+    int centerY = height / 2;
+    for (int t = 0; t < 10; t++) {
+        // Fill frame with a moving diagonal pattern
+        for (size_t y = 0; y < height; y++) {
+            int dy = y - centerY;
+            for (size_t x = 0; x < width; x++) {
+                // Create a diagonal pattern that moves with time
+                // and varies intensity based on position
+                int diagonal = (x + y + t * 8) % 32;
+
+                // Create intensity variation
+                uint16_t intensity;
+                if (diagonal < 16) {
+                    intensity = (uint16_t)((diagonal * 4096)); // Ramp up
+                } else {
+                    intensity = (uint16_t)((31 - diagonal) * 4096); // Ramp down
+                }
+
+                // Add some circular features
+                int dx = x - centerX;
+                int radius = (int)sqrt(dx*dx + dy*dy);
+
+                // Modulate the pattern with concentric circles
+                if (radius % 16 < 8) {
+                    intensity = (uint16_t)(intensity * 0.7);
+                }
+
+                frame[y * width + x] = intensity;
+            }
         }
 
         ZarrStatusCode status = ZarrStream_append(
-          stream,
-          frame,
-          width * height * sizeof(uint16_t),
-          &bytes_written
-        );
+          stream, frame, width * height * sizeof(uint16_t), &bytes_written);
 
         if (status != ZarrStatusCode_Success) {
-            fprintf(stderr, "Failed to append frame: %s\n",
+            fprintf(stderr,
+                    "Failed to append frame: %s\n",
                     Zarr_get_status_message(status));
             break;
         }

diff --git a/pyproject.toml b/pyproject.toml
@@ -10,7 +10,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "acquire-zarr"
-version = "0.0.2"
+version = "0.0.4"
 description = "Python bindings for acquire-zarr"
 authors = [
     {name = "Alan Liddell", email = "[email protected]"}
@@ -30,7 +30,7 @@ testing = [
     "ruff",
     "s3fs",
     "tifffile",
-    "zarr",
+    "zarr==3.0.0rc1",
     "python-dotenv",
 ]
 

diff --git a/python/tests/test_settings.py b/python/tests/test_settings.py
@@ -62,49 +62,57 @@ def test_set_compression_settings(settings):
     settings.compression = compression_settings
     assert settings.compression is not None
     assert settings.compression.compressor == acquire_zarr.Compressor.BLOSC1
-    assert settings.compression.codec == acquire_zarr.CompressionCodec.BLOSC_ZSTD
+    assert (
+        settings.compression.codec == acquire_zarr.CompressionCodec.BLOSC_ZSTD
+    )
     assert settings.compression.level == 5
     assert settings.compression.shuffle == 2
 
 
 def test_set_dimensions(settings):
     assert len(settings.dimensions) == 0
 
-    settings.dimensions.append(acquire_zarr.Dimension(
-        name="foo",
-        kind=acquire_zarr.DimensionType.TIME,
-        array_size_px=1,
-        chunk_size_px=2,
-        shard_size_chunks=3,
-    ))
+    settings.dimensions.append(
+        acquire_zarr.Dimension(
+            name="foo",
+            kind=acquire_zarr.DimensionType.TIME,
+            array_size_px=1,
+            chunk_size_px=2,
+            shard_size_chunks=3,
+        )
+    )
     assert len(settings.dimensions) == 1
     assert settings.dimensions[0].name == "foo"
     assert settings.dimensions[0].kind == acquire_zarr.DimensionType.TIME
     assert settings.dimensions[0].array_size_px == 1
     assert settings.dimensions[0].chunk_size_px == 2
     assert settings.dimensions[0].shard_size_chunks == 3
 
-    settings.dimensions.append(acquire_zarr.Dimension(
-        name="bar",
-        kind=acquire_zarr.DimensionType.SPACE,
-        array_size_px=4,
-        chunk_size_px=5,
-        shard_size_chunks=6,
-    ))
+    settings.dimensions.append(
+        acquire_zarr.Dimension(
+            name="bar",
+            kind=acquire_zarr.DimensionType.SPACE,
+            array_size_px=4,
+            chunk_size_px=5,
+            shard_size_chunks=6,
+        )
+    )
     assert len(settings.dimensions) == 2
     assert settings.dimensions[1].name == "bar"
     assert settings.dimensions[1].kind == acquire_zarr.DimensionType.SPACE
     assert settings.dimensions[1].array_size_px == 4
     assert settings.dimensions[1].chunk_size_px == 5
     assert settings.dimensions[1].shard_size_chunks == 6
 
-    settings.dimensions.append(acquire_zarr.Dimension(
-        name="baz",
-        kind=acquire_zarr.DimensionType.OTHER,
-        array_size_px=7,
-        chunk_size_px=8,
-        shard_size_chunks=9,
-    ))
+    settings.dimensions.append(
+        acquire_zarr.Dimension(
+            name="baz",
+            kind=acquire_zarr.DimensionType.OTHER,
+            array_size_px=7,
+            chunk_size_px=8,
+            shard_size_chunks=9,
+        )
+    )
     assert len(settings.dimensions) == 3
     assert settings.dimensions[2].name == "baz"
     assert settings.dimensions[2].kind == acquire_zarr.DimensionType.OTHER