[on hold] RNTuple example

CMakeLists.txt

@@ -86,6 +86,12 @@ if (LLAMA_BUILD_EXAMPLES)
 		message(WARNING "Could not find alpaka. Alpaka examples are disabled.")
 	endif()
 
+	# ROOT examples
+	find_package(ROOT QUIET)
+	if (ROOT_FOUND)
+		add_subdirectory("examples/hep_rntuple")
+	endif()
+
 	# CUDA examples
 	include(CheckLanguage)
 	check_language(CUDA)

examples/bufferguard/bufferguard.cpp

@@ -68,9 +68,11 @@ struct GuardMapping2D : llama::ArrayExtentsDynamic<2>
         std::abort();
     }
 
-    template<std::size_t... RecordCoords>
-    constexpr auto blobNrAndOffset(ArrayIndex ai, llama::RecordCoord<RecordCoords...> rc = {}) const
-        -> llama::NrAndOffset
+    template<std::size_t... RecordCoords, std::size_t N = 0>
+    constexpr auto blobNrAndOffset(
+        ArrayIndex ai,
+        llama::Array<std::size_t, N> = {},
+        llama::RecordCoord<RecordCoords...> rc = {}) const -> llama::NrAndOffset
     {
         // [0][0] is at left top
         const auto [row, col] = ai;

examples/common/ttjet_13tev_june2019.hpp

@@ -7,7 +7,7 @@
 
 using bit = bool;
 using byte = unsigned char;
-using Index = std::uint64_t;
+using Index = std::uint32_t;
 
 // clang-format off
 struct run {};
@@ -1538,7 +1538,7 @@ using Electron = llama::Record<
     llama::Field<Electron_pdgId, std::int32_t>,
     llama::Field<Electron_photonIdx, std::int32_t>,
     llama::Field<Electron_tightCharge, std::int32_t>,
-    llama::Field<Electron_vidNestedWPbitmap, std::int32_t>,
+    //llama::Field<Electron_vidNestedWPbitmap, std::int32_t>,
     llama::Field<Electron_convVeto, bit>,
     llama::Field<Electron_cutBased_HEEP, bit>,
     llama::Field<Electron_isPFcand, bit>,
@@ -1947,6 +1947,7 @@ using Event = llama::Record<
     llama::Field<ChsMET_sumEt, float>,
     //llama::Field<nCorrT1METJet, Index>,
     //llama::Field<nElectron, Index>,
+    llama::Field<nElectron, Electron[]>,
     llama::Field<Flag_ecalBadCalibFilterV2, bit>,
     //llama::Field<nFatJet, Index>,
     //llama::Field<nGenJetAK8, Index>,

examples/hep_rntuple/CMakeLists.txt

@@ -0,0 +1,11 @@
+cmake_minimum_required (VERSION 3.15)
+project(llama-hep_rntuple)
+
+set(CMAKE_CXX_STANDARD 17)
+
+find_package(ROOT REQUIRED)
+if (NOT TARGET llama::llama)
+	find_package(llama REQUIRED)
+endif()
+add_executable(${PROJECT_NAME} hep_rntuple.cpp)
+target_link_libraries(${PROJECT_NAME} PRIVATE ROOT::Hist ROOT::Graf ROOT::Gpad ROOT::ROOTNTuple llama::llama)

examples/hep_rntuple/hep_rntuple.cpp

@@ -0,0 +1,117 @@
+// This example uses a non-public CMS NanoAOD file called: ttjet_13tev_june2019_lzma.
+// Please contact us if you need it.
+
+#include "../common/ttjet_13tev_june2019.hpp"
+
+#include <RConfigure.h>
+#define R__HAS_STD_STRING_VIEW
+#include <ROOT/RNTuple.hxx>
+#include <ROOT/RNTupleDS.hxx>
+#include <ROOT/RNTupleModel.hxx>
+#include <ROOT/RNTupleOptions.hxx>
+#include <ROOT/RNTupleView.hxx>
+#include <chrono>
+#include <llama/DumpMapping.hpp>
+#include <llama/llama.hpp>
+
+using SmallEvent = boost::mp11::mp_take_c<Event, 100>;
+
+int main(int argc, const char* argv[])
+{
+    if (argc != 2)
+    {
+        fmt::print("Please specify input file!\n");
+        return 1;
+    }
+
+    using namespace std::chrono;
+    using namespace ROOT::Experimental;
+
+    // auto ntuple
+    //    = RNTupleReader::Open(RNTupleModel::Create(), "NTuple", "/mnt/c/dev/llama/ttjet_13tev_june2019_lzma.root");
+    auto ntuple = RNTupleReader::Open(RNTupleModel::Create(), "NTuple", argv[1]);
+    // try
+    //{
+    //    ntuple->PrintInfo(ROOT::Experimental::ENTupleInfo::kStorageDetails);
+    //}
+    // catch (const std::exception& e)
+    //{
+    //    fmt::print("PrintInfo error: {}", e.what());
+    //}
+    const auto eventCount = ntuple->GetNEntries();
+    const auto& d = ntuple->GetDescriptor();
+    const auto electronCount
+        = d.GetNElements(d.FindColumnId(d.FindFieldId("nElectron.nElectron.Electron_deltaEtaSC"), 0));
+    fmt::print("File contains {} events with {} electrons\n", eventCount, electronCount);
+
+    auto start = steady_clock::now();
+    auto mapping = llama::mapping::OffsetTable<llama::ArrayDims<1>, SmallEvent>{
+        llama::ArrayDims{eventCount},
+        llama::ArrayDims{electronCount}};
+    auto view = llama::allocView(mapping);
+    fmt::print("Alloc LLAMA view: {}ms\n", duration_cast<milliseconds>(steady_clock::now() - start).count());
+
+    std::size_t totalSize = 0;
+    for (auto i = 0u; i < view.mapping.blobCount; i++)
+        totalSize += view.mapping.blobSize(i);
+    fmt::print("Total LLAMA view memory: {}MiB in {} blobs\n", totalSize / 1024 / 1024, view.mapping.blobCount);
+
+    // fill offset table
+    start = steady_clock::now();
+    std::size_t offset = 0;
+    auto electronViewCollection = ntuple->GetViewCollection("nElectron");
+    for (std::size_t i = 0; i < eventCount; i++)
+    {
+        offset += electronViewCollection(i);
+        view(i)(llama::EndOffset<nElectron>{}) = offset;
+        assert(offset <= electronCount);
+    }
+    fmt::print("Fill offset table: {}ms\n", duration_cast<milliseconds>(steady_clock::now() - start).count());
+
+    using AugmentedSmallEvent = typename decltype(mapping)::RecordDim;
+    start = steady_clock::now();
+    llama::forEachLeaf<AugmentedSmallEvent>(
+        [&](auto coord)
+        {
+            using Coord = decltype(coord);
+            using LeafTag = llama::GetTag<AugmentedSmallEvent, Coord>;
+            using Type = llama::GetType<AugmentedSmallEvent, Coord>;
+
+            fmt::print("Copying {}\n", llama::structName<LeafTag>());
+            if constexpr (
+                !llama::mapping::internal::isEndOffsetField<LeafTag> && !llama::mapping::internal::isSizeField<LeafTag>)
+            {
+                if constexpr (boost::mp11::mp_contains<typename Coord::List, boost::mp11::mp_size_t<llama::dynamic>>::
+                                  value)
+                {
+                    using Before = llama::mapping::internal::BeforeDynamic<Coord>;
+                    using BeforeBefore = llama::RecordCoordFromList<boost::mp11::mp_pop_front<typename Before::List>>;
+                    using After = llama::mapping::internal::AfterDynamic<Coord>;
+                    using SubCollectionTag = llama::GetTag<AugmentedSmallEvent, Before>;
+
+                    auto collectionColumn = ntuple->GetViewCollection(llama::structName<SubCollectionTag>());
+                    auto column = collectionColumn.template GetView<Type>(
+                        llama::structName<SubCollectionTag>() + "." + llama::structName<LeafTag>());
+                    for (std::size_t i = 0; i < eventCount; i++)
+                    {
+                        const auto subCollectionCount = view(i)(BeforeBefore{})(llama::Size<SubCollectionTag>{});
+                        for (std::size_t j = 0; j < subCollectionCount; j++)
+                        {
+                            const auto value = column(j);
+                            auto& dst = view(i)(Before{})(j) (After{});
+                            dst = value;
+                        }
+                    }
+                }
+                else
+                {
+                    auto column = ntuple->GetView<Type>(llama::structName<LeafTag>());
+                    for (std::size_t i = 0; i < eventCount; i++)
+                        view(i)(coord) = column(i);
+                }
+            }
+        });
+    fmt::print("Copy RNTuple -> LLAMA view: {}ms\n", duration_cast<milliseconds>(steady_clock::now() - start).count());
+
+    start = steady_clock::now();
+}

include/llama/Concepts.hpp

@@ -23,7 +23,7 @@ namespace llama
         { m.blobSize(std::size_t{}) } -> std::same_as<std::size_t>;
         { m.blobNrAndOffset(typename M::ArrayIndex{}) } -> std::same_as<NrAndOffset>;
         { m.template blobNrAndOffset<0>(typename M::ArrayIndex{}) } -> std::same_as<NrAndOffset>;
-        { m.blobNrAndOffset(typename M::ArrayIndex{}, llama::RecordCoord<0>{}) } -> std::same_as<NrAndOffset>;
+        { m.blobNrAndOffset(typename M::ArrayIndex{}, {}, llama::RecordCoord<0>{}) } -> std::same_as<NrAndOffset>;
     };
     // clang-format on
 

include/llama/Core.hpp

@@ -30,6 +30,15 @@ namespace llama
     template<typename T>
     inline constexpr bool isAllowedFieldType = std::is_trivially_destructible_v<T>;
 
+    template<typename... Fields>
+    inline constexpr bool isAllowedFieldType<Record<Fields...>> = true;
+
+    template<typename T, std::size_t N>
+    inline constexpr bool isAllowedFieldType<T[N]> = isAllowedFieldType<T>;
+
+    template<typename T>
+    inline constexpr bool isAllowedFieldType<T[]> = isAllowedFieldType<T>;
+
     /// Record dimension tree node which may either be a leaf or refer to a child tree presented as another \ref
     /// Record.
     /// \tparam Tag Name of the node. May be any type (struct, class).
@@ -101,6 +110,14 @@ namespace llama
                 = boost::mp11::mp_push_front<typename GetTagsImpl<ChildType, RecordCoord<Coords...>>::type, ChildTag>;
         };
 
+        template<typename ChildType, std::size_t... Coords>
+        struct GetTagsImpl<ChildType[], RecordCoord<dynamic, Coords...>>
+        {
+            using ChildTag = RecordCoord<dynamic>;
+            using type
+                = boost::mp11::mp_push_front<typename GetTagsImpl<ChildType, RecordCoord<Coords...>>::type, ChildTag>;
+        };
+
         template<typename T>
         struct GetTagsImpl<T, RecordCoord<>>
         {
@@ -198,6 +215,16 @@ namespace llama
                 typename GetCoordFromTagsImpl<ChildType, RecordCoord<ResultCoords..., FirstTag::front>, Tags...>::type;
         };
 
+        template<typename ChildType, std::size_t... ResultCoords, typename FirstTag, typename... Tags>
+        struct GetCoordFromTagsImpl<ChildType[], RecordCoord<ResultCoords...>, FirstTag, Tags...>
+        {
+            static_assert(
+                std::is_same_v<FirstTag, RecordCoord<dynamic>>,
+                "Please use a RecordCoord<dynamic> to index into dynamic arrays");
+            using type =
+                typename GetCoordFromTagsImpl<ChildType, RecordCoord<ResultCoords..., FirstTag::front>, Tags...>::type;
+        };
+
         template<typename RecordDim, typename RecordCoord>
         struct GetCoordFromTagsImpl<RecordDim, RecordCoord>
         {
@@ -242,6 +269,13 @@ namespace llama
             using type = typename GetTypeImpl<ChildType, RecordCoord<TailCoords...>>::type;
         };
 
+        template<typename ChildType, std::size_t HeadCoord, std::size_t... TailCoords>
+        struct GetTypeImpl<ChildType[], RecordCoord<HeadCoord, TailCoords...>>
+        {
+            static_assert(HeadCoord == dynamic, "Record coord at a dynamic array must be llama::dynamic");
+            using type = typename GetTypeImpl<ChildType, RecordCoord<TailCoords...>>::type;
+        };
+
         template<typename T>
         struct GetTypeImpl<T, RecordCoord<>>
         {
@@ -289,6 +323,12 @@ namespace llama
             }
             using type = decltype(help(std::make_index_sequence<N>{}));
         };
+
+        template<typename Child, std::size_t... RCs>
+        struct LeafRecordCoordsImpl<Child[], RecordCoord<RCs...>>
+        {
+            using type = typename LeafRecordCoordsImpl<Child, RecordCoord<RCs..., dynamic>>::type;
+        };
     } // namespace internal
 
     /// Returns a flat type list containing all record coordinates to all leaves of the given record dimension.
@@ -557,6 +597,19 @@ namespace llama
         struct IsBoundedArray<T[N]> : std::true_type
         {
         };
+
+        template<class T>
+        struct is_unbounded_array : std::false_type
+        {
+        };
+
+        template<class T>
+        struct is_unbounded_array<T[]> : std::true_type
+        {
+        };
+
+        template<typename T>
+        inline constexpr bool is_unbounded_array_v = is_unbounded_array<T>::value;
     } // namespace internal
 
     namespace internal

include/llama/DumpMapping.hpp

@@ -8,6 +8,7 @@
 
 #include "ArrayIndexRange.hpp"
 #include "Core.hpp"
+#include "mapping/OffsetTable.hpp"
 
 #include <boost/functional/hash.hpp>
 #include <fmt/format.h>
@@ -75,14 +76,33 @@ namespace llama
                             {ai,
                              internal::toVec(rc),
                              recordCoordTags<RecordDim>(rc),
-                             mapping.blobNrAndOffset(ai, rc),
+                             mapping.blobNrAndOffset(ai, {}, rc),
                              sizeof(GetType<RecordDim, decltype(rc)>)});
                     });
             }
 
             return infos;
         }
 
+        template<typename ArrayExtents, typename RecordDim, typename SubMappings>
+        auto boxesFromMapping(const mapping::OffsetTable<ArrayExtents, RecordDim, SubMappings>& mapping)
+            -> std::vector<FieldBox<ArrayExtents::rank>>
+        {
+            std::size_t previousBlobs = 0;
+            std::vector<FieldBox<ArrayExtents::rank>> infos;
+            boost::mp11::mp_for_each<boost::mp11::mp_iota<boost::mp11::mp_size<decltype(mapping.subMappings)>>>(
+                [&](auto ic)
+                {
+                    const auto& subMapping = get<decltype(ic)::value>(mapping.subMappings);
+                    auto subBoxes = boxesFromMapping(subMapping);
+                    for(auto& box : subBoxes)
+                        box.nrAndOffset.nr += previousBlobs;
+                    infos.insert(infos.end(), subBoxes.begin(), subBoxes.end());
+                    previousBlobs += std::decay_t<decltype(subMapping)>::blobCount;
+                });
+            return infos;
+        }
+
         template<std::size_t Dim>
         auto breakBoxes(std::vector<FieldBox<Dim>> boxes, std::size_t wrapByteCount) -> std::vector<FieldBox<Dim>>
         {

include/llama/Proofs.hpp

@@ -73,7 +73,8 @@ namespace llama
                                                           {
                                                               using Type
                                                                   = GetType<typename Mapping::RecordDim, decltype(rc)>;
-                                                              const auto [blob, offset] = m.blobNrAndOffset(ai, rc);
+                                                              const auto [blob, offset]
+                                                                  = m.blobNrAndOffset(ai, {}, rc);
                                                               for(std::size_t b = 0; b < sizeof(Type); b++)
                                                                   if(testAndSet(blob, offset + b))
                                                                   {
@@ -105,7 +106,8 @@ namespace llama
                                                           {
                                                               using Type
                                                                   = GetType<typename Mapping::RecordDim, decltype(rc)>;
-                                                              const auto [blob, offset] = m.blobNrAndOffset(ai, rc);
+                                                              const auto [blob, offset]
+                                                                  = m.blobNrAndOffset(ai, {}, rc);
                                                               if(flatIndex % PieceLength != 0
                                                                  && (lastBlob != blob
                                                                      || lastOffset + sizeof(Type) != offset))

include/llama/RecordCoord.hpp

@@ -6,11 +6,14 @@
 #include "Meta.hpp"
 
 #include <array>
+#include <limits>
 #include <ostream>
 #include <type_traits>
 
 namespace llama
 {
+    inline constexpr auto dynamic = std::numeric_limits<std::size_t>::max();
+
     /// Represents a coordinate for a record inside the record dimension tree.
     /// \tparam Coords... the compile time coordinate.
     template<std::size_t... Coords>