Merge pull request #199 from constantinpape/cpp-fixes

Cpp fixes

include/z5/compression/blosc_compressor.hxx

@@ -75,16 +75,16 @@ namespace compression {
         void init(const DatasetMetadata & metadata) {
             const auto & cOpts = metadata.compressionOptions;
 
-            clevel_     = boost::get<int>(cOpts.at("level"));
-            shuffle_    = boost::get<int>(cOpts.at("shuffle"));
-            compressor_ = boost::get<std::string>(cOpts.at("codec"));
-            blocksize_ = boost::get<int>(cOpts.at("blocksize"));
+            clevel_     = std::get<int>(cOpts.at("level"));
+            shuffle_    = std::get<int>(cOpts.at("shuffle"));
+            compressor_ = std::get<std::string>(cOpts.at("codec"));
+            blocksize_ = std::get<int>(cOpts.at("blocksize"));
 
             // set nthreads with a default value of 1
             nthreads_ = 1;
             auto threadsIt = cOpts.find("nthreads");
             if(threadsIt != cOpts.end()) {
-                nthreads_ = boost::get<int>(threadsIt->second);
+                nthreads_ = std::get<int>(threadsIt->second);
             }
         }
 

include/z5/compression/bzip2_compressor.hxx

@@ -105,7 +105,7 @@ namespace compression {
 
     private:
         void init(const DatasetMetadata & metadata) {
-            clevel_ = boost::get<int>(metadata.compressionOptions.at("level"));
+            clevel_ = std::get<int>(metadata.compressionOptions.at("level"));
         }
 
         // compression level

include/z5/compression/lz4_compressor.hxx

@@ -71,10 +71,10 @@ namespace compression {
     private:
         void init(const DatasetMetadata & metadata) {
             // appropriate for hc compression
-            // level_ = boost::get<int>(metadata.compressionOptions.at("level"));
+            // level_ = std::get<int>(metadata.compressionOptions.at("level"));
 
             // appropriate for fast compression
-            level_ = 10 - boost::get<int>(metadata.compressionOptions.at("level"));
+            level_ = 10 - std::get<int>(metadata.compressionOptions.at("level"));
         }
 
         // compression level

include/z5/compression/xz_compressor.hxx

@@ -132,7 +132,7 @@ namespace compression {
 
     private:
         void init(const DatasetMetadata & metadata) {
-            level_ = boost::get<int>(metadata.compressionOptions.at("level"));
+            level_ = std::get<int>(metadata.compressionOptions.at("level"));
         }
 
         // compression level

include/z5/compression/zlib_compressor.hxx

@@ -148,8 +148,8 @@ namespace compression {
 
     private:
         void init(const DatasetMetadata & metadata) {
-            clevel_ = boost::get<int>(metadata.compressionOptions.at("level"));
-            useZlibEncoding_ = boost::get<bool>(metadata.compressionOptions.at("useZlib"));
+            clevel_ = std::get<int>(metadata.compressionOptions.at("level"));
+            useZlibEncoding_ = std::get<bool>(metadata.compressionOptions.at("useZlib"));
         }
 
         // compression level

include/z5/metadata.hxx

@@ -121,7 +121,11 @@ namespace z5 {
 
         void fromJsonZarr(const nlohmann::json & j) {
             checkJson(j);
-            dtype = types::Datatypes::zarrToDtype().at(j["dtype"]);
+            try {
+                dtype = types::Datatypes::zarrToDtype().at(j["dtype"]);
+            } catch(std::out_of_range) {
+                throw std::runtime_error("Unsupported zarr dtype: " + static_cast<std::string>(j["dtype"]));
+            }
             shape = types::ShapeType(j["shape"].begin(), j["shape"].end());
             chunkShape = types::ShapeType(j["chunks"].begin(), j["chunks"].end());
 

include/z5/types/types.hxx

@@ -3,7 +3,7 @@
 #include <vector>
 #include <string>
 #include <map>
-#include "boost/variant.hpp"
+#include <variant>
 
 #include "nlohmann/json.hpp"
 
@@ -207,7 +207,7 @@ namespace types {
     //
     // Compression Options and Fill Value
     //
-    typedef std::map<std::string, boost::variant<int, bool, std::string>> CompressionOptions;
+    typedef std::map<std::string, std::variant<int, bool, std::string>> CompressionOptions;
 
     inline void readZarrCompressionOptionsFromJson(Compressor compressor,
                                                    const nlohmann::json & jOpts,
@@ -254,22 +254,22 @@ namespace types {
 
         switch(compressor) {
             #ifdef WITH_BLOSC
-            case blosc: jOpts["cname"]   = boost::get<std::string>(options.at("codec"));
-                        jOpts["clevel"]  = boost::get<int>(options.at("level"));
-                        jOpts["shuffle"] = boost::get<int>(options.at("shuffle"));
-                        jOpts["blocksize"] = boost::get<int>(options.at("blocksize"));
+            case blosc: jOpts["cname"]   = std::get<std::string>(options.at("codec"));
+                        jOpts["clevel"]  = std::get<int>(options.at("level"));
+                        jOpts["shuffle"] = std::get<int>(options.at("shuffle"));
+                        jOpts["blocksize"] = std::get<int>(options.at("blocksize"));
                         break;
             #endif
             #ifdef WITH_ZLIB
-            case zlib: jOpts["id"] = boost::get<bool>(options.at("useZlib")) ? "zlib" : "gzip";
-                       jOpts["level"] = boost::get<int>(options.at("level"));
+            case zlib: jOpts["id"] = std::get<bool>(options.at("useZlib")) ? "zlib" : "gzip";
+                       jOpts["level"] = std::get<int>(options.at("level"));
                        break;
             #endif
             #ifdef WITH_BZIP2
-            case bzip2: jOpts["level"] = boost::get<int>(options.at("level")); break;
+            case bzip2: jOpts["level"] = std::get<int>(options.at("level")); break;
             #endif
             #ifdef WITH_LZ4
-            case lz4: jOpts["acceleration"] = boost::get<int>(options.at("level")); break;
+            case lz4: jOpts["acceleration"] = std::get<int>(options.at("level")); break;
             #endif
             // raw compression has no parameters
             default: break;
@@ -323,25 +323,25 @@ namespace types {
 
         switch(compressor) {
             #ifdef WITH_ZLIB
-            case zlib: jOpts["level"] = boost::get<int>(options.at("level"));
+            case zlib: jOpts["level"] = std::get<int>(options.at("level"));
                        break;
             #endif
             #ifdef WITH_BZIP2
-            case bzip2: jOpts["blockSize"] = boost::get<int>(options.at("level"));
+            case bzip2: jOpts["blockSize"] = std::get<int>(options.at("level"));
                         break;
             #endif
             #ifdef WITH_XZ
-            case xz: jOpts["preset"] = boost::get<int>(options.at("level")); break;
+            case xz: jOpts["preset"] = std::get<int>(options.at("level")); break;
             #endif
             #ifdef WITH_LZ4
-            case lz4: jOpts["blockSize"] = boost::get<int>(options.at("level")); break;
+            case lz4: jOpts["blockSize"] = std::get<int>(options.at("level")); break;
             #endif
             #ifdef WITH_BLOSC
-            case blosc: jOpts["cname"] = boost::get<std::string>(options.at("codec"));
-                        jOpts["clevel"] = boost::get<int>(options.at("level"));
-                        jOpts["shuffle"] = boost::get<int>(options.at("shuffle"));
-                        jOpts["blocksize"] = boost::get<int>(options.at("blocksize"));
-                        jOpts["nthreads"] = boost::get<int>(options.at("nthreads"));
+            case blosc: jOpts["cname"] = std::get<std::string>(options.at("codec"));
+                        jOpts["clevel"] = std::get<int>(options.at("level"));
+                        jOpts["shuffle"] = std::get<int>(options.at("shuffle"));
+                        jOpts["blocksize"] = std::get<int>(options.at("blocksize"));
+                        jOpts["nthreads"] = std::get<int>(options.at("nthreads"));
                         break;
             #endif
             // raw compression has no parameters
@@ -356,9 +356,10 @@ namespace types {
 
         switch(compressor) {
             #ifdef WITH_BLOSC
-            case blosc: if(options.find("name") == options.end()){options["name"] = "lz4";}
+            case blosc: if(options.find("codec") == options.end()){options["codec"] = std::string("lz4");}
                         if(options.find("level") == options.end()){options["level"] = 5;}
                         if(options.find("shuffle") == options.end()){options["shuffle"] = 1;}
+                        if(options.find("blocksize") == options.end()){options["blocksize"] = 0;}
                         break;
             #endif
             #ifdef WITH_ZLIB
@@ -407,13 +408,16 @@ namespace types {
     inline void compressionTypeToJson(const CompressionOptions & opts, nlohmann::json & j) {
         for(auto & elem : opts) {
             const auto & val = elem.second;
-            const int type_id = val.which();
-            switch(type_id) {
-                case 0: j[elem.first] = boost::get<int>(val); break;
-                case 1: j[elem.first] = boost::get<bool>(val); break;
-                case 2: j[elem.first] = boost::get<std::string>(val); break;
-                default: throw std::runtime_error("Invalid type conversion for compression type");
+            if(std::holds_alternative<int>(val)){
+                j[elem.first] = std::get<int>(val);
+            } else if(std::holds_alternative<bool>(val)){
+                j[elem.first] = std::get<bool>(val);
+            } else if(std::holds_alternative<std::string>(val)){
+                j[elem.first] = std::get<std::string>(val);
+            } else {
+                throw std::runtime_error("Invalid type conversion for compression type");
             }
+
         }
     }
 

src/test/test_dataset.cxx

@@ -222,4 +222,44 @@ namespace z5 {
         }
     }
 
+
+    TEST_F(DatasetTest, CreateBloscDataset) {
+
+        types::ShapeType shape({100, 100, 100});
+        types::ShapeType chunks({10, 10, 10});
+
+        Metadata fMeta(true);
+        filesystem::writeMetadata(fileHandle_, fMeta);
+        auto ds = createDataset(fileHandle_, "blosc-ds", "float32", shape, chunks, "blosc");
+        const auto & chunksPerDim = ds->chunksPerDimension();
+
+        std::default_random_engine generator;
+
+        // test uninitialized chunk -> this is expected to throw a runtime error
+        float dataTmp[size_];
+        ASSERT_THROW(ds->readChunk(types::ShapeType({0, 0, 0}), dataTmp), std::runtime_error);
+
+        // test for 10 random chunks
+        for(unsigned t = 0; t < 10; ++t) {
+
+            // get a random chunk
+            types::ShapeType chunkId(ds->dimension());
+            for(unsigned d = 0; d < ds->dimension(); ++d) {
+                std::uniform_int_distribution<std::size_t> distr(0, chunksPerDim[d] - 1);
+                chunkId[d] = distr(generator);
+            }
+
+            ds->writeChunk(chunkId, dataFloat_);
+
+            // read a chunk
+            float dataTmp[size_];
+            ds->readChunk(chunkId, dataTmp);
+
+            // check
+            for(std::size_t i = 0; i < size_; ++i) {
+                ASSERT_EQ(dataTmp[i], dataFloat_[i]);
+            }
+        }
+    }
+
 }

src/test/test_metadata.cxx

@@ -82,9 +82,9 @@ namespace z5 {
         // check compressr
         ASSERT_EQ(metadata.compressor, types::Compressors::zarrToCompressor()[compressor["id"]]);
         // check compression options
-        ASSERT_EQ(boost::get<int>(metadata.compressionOptions["level"]), compressor["clevel"]);
-        ASSERT_EQ(boost::get<std::string>(metadata.compressionOptions["codec"]), compressor["cname"]);
-        ASSERT_EQ(boost::get<int>(metadata.compressionOptions["shuffle"]), compressor["shuffle"]);
+        ASSERT_EQ(std::get<int>(metadata.compressionOptions["level"]), compressor["clevel"]);
+        ASSERT_EQ(std::get<std::string>(metadata.compressionOptions["codec"]), compressor["cname"]);
+        ASSERT_EQ(std::get<int>(metadata.compressionOptions["shuffle"]), compressor["shuffle"]);
         // check dtype, fillvalue and order
         ASSERT_EQ(metadata.dtype, types::Datatypes::zarrToDtype()[jZarr["dtype"]]);
         ASSERT_EQ(metadata.fillValue, jZarr["fill_value"]);
@@ -105,7 +105,7 @@ namespace z5 {
         #ifdef WITH_ZLIB
         ASSERT_EQ(metadata.compressor, types::zlib);
         #endif
-        ASSERT_EQ(boost::get<bool>(metadata.compressionOptions.at("useZlib")), false);
+        ASSERT_EQ(std::get<bool>(metadata.compressionOptions.at("useZlib")), false);
         ASSERT_EQ(metadata.dtype, types::Datatypes::n5ToDtype()[jN5["dataType"]]);
     }
 
@@ -158,12 +158,12 @@ namespace z5 {
         // check compression
         ASSERT_EQ(metaRead.compressor, metaWrite.compressor);
         // check compression options
-        ASSERT_EQ(boost::get<int>(metaRead.compressionOptions.at("level")),
-                  boost::get<int>(metaWrite.compressionOptions.at("level")));
-        ASSERT_EQ(boost::get<int>(metaRead.compressionOptions.at("shuffle")),
-                  boost::get<int>(metaWrite.compressionOptions.at("shuffle")));
-        ASSERT_EQ(boost::get<std::string>(metaRead.compressionOptions.at("codec")),
-                  boost::get<std::string>(metaWrite.compressionOptions.at("codec")));
+        ASSERT_EQ(std::get<int>(metaRead.compressionOptions.at("level")),
+                  std::get<int>(metaWrite.compressionOptions.at("level")));
+        ASSERT_EQ(std::get<int>(metaRead.compressionOptions.at("shuffle")),
+                  std::get<int>(metaWrite.compressionOptions.at("shuffle")));
+        ASSERT_EQ(std::get<std::string>(metaRead.compressionOptions.at("codec")),
+                  std::get<std::string>(metaWrite.compressionOptions.at("codec")));
         // check dtype, fill value, order
         ASSERT_EQ(metaRead.dtype,             metaWrite.dtype);
         ASSERT_EQ(metaRead.fillValue, metaWrite.fillValue);
@@ -194,7 +194,7 @@ namespace z5 {
         #ifdef WITH_ZLIB
         ASSERT_EQ(metaRead.compressor, types::zlib);
         #endif
-        ASSERT_EQ(boost::get<bool>(metaRead.compressionOptions["useZlib"]), false);
+        ASSERT_EQ(std::get<bool>(metaRead.compressionOptions["useZlib"]), false);
         ASSERT_EQ(metaRead.dtype, types::Datatypes::n5ToDtype()[jN5["dataType"]]);
     }