From dd45eccffcb457690b1042b8b3616993d95feddc Mon Sep 17 00:00:00 2001 From: Bernhard Manfred Gruber Date: Sun, 7 Mar 2021 17:44:36 +0100 Subject: [PATCH] implement support for sub arrays of dynamic size --- examples/bufferguard/bufferguard.cpp | 4 +- include/llama/Core.hpp | 47 ++++ include/llama/RecordCoord.hpp | 3 + include/llama/Tuple.hpp | 37 +-- include/llama/View.hpp | 29 ++- include/llama/VirtualRecord.hpp | 95 +++++++- include/llama/llama.hpp | 1 + include/llama/mapping/AoS.hpp | 6 +- include/llama/mapping/AoSoA.hpp | 6 +- include/llama/mapping/Heatmap.hpp | 4 +- include/llama/mapping/OffsetTable.hpp | 315 +++++++++++++++++++++++++ include/llama/mapping/One.hpp | 5 +- include/llama/mapping/SoA.hpp | 6 +- include/llama/mapping/Split.hpp | 19 +- include/llama/mapping/Trace.hpp | 5 +- include/llama/mapping/tree/Mapping.hpp | 5 +- tests/computedprop.cpp | 15 +- tests/core.cpp | 30 ++- tests/recorddimension.cpp | 158 +++++++++++++ 19 files changed, 720 insertions(+), 70 deletions(-) create mode 100644 include/llama/mapping/OffsetTable.hpp diff --git a/examples/bufferguard/bufferguard.cpp b/examples/bufferguard/bufferguard.cpp index ebc10971ef..d80ff7172e 100644 --- a/examples/bufferguard/bufferguard.cpp +++ b/examples/bufferguard/bufferguard.cpp @@ -69,8 +69,8 @@ struct GuardMapping2D std::abort(); } - template - constexpr auto blobNrAndOffset(ArrayDims coord) const -> llama::NrAndOffset + template + constexpr auto blobNrAndOffset(ArrayDims coord, llama::Array = {}) const -> llama::NrAndOffset { // [0][0] is at left top const auto [row, col] = coord; diff --git a/include/llama/Core.hpp b/include/llama/Core.hpp index 8870e7b179..77c9aae5ea 100644 --- a/include/llama/Core.hpp +++ b/include/llama/Core.hpp @@ -58,6 +58,15 @@ namespace llama inline constexpr bool isAllowedFieldType = std::is_trivially_constructible_v&& std::is_trivially_destructible_v; + template + inline constexpr bool isAllowedFieldType> = true; + + template + inline constexpr bool isAllowedFieldType = isAllowedFieldType; + + template + inline constexpr bool isAllowedFieldType = isAllowedFieldType; + /// Record dimension tree node which may either be a leaf or refer to a child tree presented as another \ref /// Record. /// \tparam Tag Name of the node. May be any type (struct, class). @@ -134,6 +143,14 @@ namespace llama mp_push_front>::type, CurrTag>; }; + template + struct GetTagsImpl> + { + using ChildTag = RecordCoord; + using type = boost::mp11:: + mp_push_front>::type, CurrTag>; + }; + template struct GetTagsImpl> { @@ -212,6 +229,16 @@ namespace llama typename GetCoordFromTagsImpl, Tags...>::type; }; + template + struct GetCoordFromTagsImpl, FirstTag, Tags...> + { + static_assert( + std::is_same_v>, + "Please use a RecordCoord to index into dynamic arrays"); + using type = + typename GetCoordFromTagsImpl, Tags...>::type; + }; + template struct GetCoordFromTagsImpl { @@ -244,6 +271,13 @@ namespace llama using type = typename GetTypeImpl>::type; }; + template + struct GetTypeImpl> + { + static_assert(HeadCoord == dynamic, "Record coord at a dynamic array must be llama::dynamic"); + using type = typename GetTypeImpl>::type; + }; + template struct GetTypeImpl> { @@ -565,5 +599,18 @@ namespace llama struct is_bounded_array : std::true_type { }; + + template + struct is_unbounded_array : std::false_type + { + }; + + template + struct is_unbounded_array : std::true_type + { + }; + + template + inline constexpr bool is_unbounded_array_v = is_unbounded_array::value; } // namespace internal } // namespace llama diff --git a/include/llama/RecordCoord.hpp b/include/llama/RecordCoord.hpp index 635b7a7aee..b4a7cd0511 100644 --- a/include/llama/RecordCoord.hpp +++ b/include/llama/RecordCoord.hpp @@ -5,10 +5,13 @@ #include #include +#include #include namespace llama { + inline constexpr auto dynamic = std::numeric_limits::max(); + /// Represents a coordinate for a record inside the record dimension tree. /// \tparam Coords... the compile time coordinate. template diff --git a/include/llama/Tuple.hpp b/include/llama/Tuple.hpp index 776cfd2861..64657e6b8e 100644 --- a/include/llama/Tuple.hpp +++ b/include/llama/Tuple.hpp @@ -27,33 +27,25 @@ namespace llama constexpr Tuple() = default; LLAMA_FN_HOST_ACC_INLINE - constexpr Tuple(FirstElement first, Elements... rest) : first(first), rest(rest...) + constexpr Tuple(FirstElement first, Elements... rest) : first(std::move(first)), rest(std::move(rest)...) { } - LLAMA_FN_HOST_ACC_INLINE - constexpr Tuple(FirstElement first, Tuple rest) : first(first), rest(rest) + /// Converting constructor + template + LLAMA_FN_HOST_ACC_INLINE constexpr Tuple(T firstArg, Ts... restArgs) + : first(std::forward(firstArg)) + , rest(std::forward(restArgs)...) { } - FirstElement first; ///< the first element (if existing) - RestTuple rest; ///< the remaining elements - }; - - template - struct Tuple - { - using FirstElement = T_FirstElement; - using RestTuple = Tuple<>; - - constexpr Tuple() = default; - LLAMA_FN_HOST_ACC_INLINE - constexpr Tuple(FirstElement const first, Tuple<> const rest = Tuple<>()) : first(first) + constexpr Tuple(FirstElement first, Tuple rest) : first(std::move(first)), rest(std::move(rest)) { } - FirstElement first; + FirstElement first; ///< the first element (if existing) + [[no_unique_address]] RestTuple rest; ///< the remaining elements }; template @@ -63,7 +55,16 @@ namespace llama using TupleElement = boost::mp11::mp_at_c; template - LLAMA_FN_HOST_ACC_INLINE auto get(const Tuple& tuple) + LLAMA_FN_HOST_ACC_INLINE auto get(Tuple& tuple) -> auto& + { + if constexpr (Pos == 0) + return tuple.first; + else + return get(tuple.rest); + } + + template + LLAMA_FN_HOST_ACC_INLINE auto get(const Tuple& tuple) -> const auto& { if constexpr (Pos == 0) return tuple.first; diff --git a/include/llama/View.hpp b/include/llama/View.hpp index c79479fd7a..151aa48fa4 100644 --- a/include/llama/View.hpp +++ b/include/llama/View.hpp @@ -257,12 +257,12 @@ namespace llama if constexpr (isRecord || internal::is_bounded_array::value) { LLAMA_FORCE_INLINE_RECURSIVE - return VirtualRecordTypeConst{arrayDims, *this}; + return VirtualRecordTypeConst{*this, arrayDims}; } else { LLAMA_FORCE_INLINE_RECURSIVE - return accessor(arrayDims, RecordCoord<>{}); + return accessor(arrayDims, Array{}, RecordCoord<>{}); } } @@ -271,12 +271,12 @@ namespace llama if constexpr (isRecord || internal::is_bounded_array::value) { LLAMA_FORCE_INLINE_RECURSIVE - return VirtualRecordType{arrayDims, *this}; + return VirtualRecordType{*this, arrayDims}; } else { LLAMA_FORCE_INLINE_RECURSIVE - return accessor(arrayDims, RecordCoord<>{}); + return accessor(arrayDims, Array{}, RecordCoord<>{}); } } @@ -367,29 +367,34 @@ namespace llama friend struct VirtualRecord; LLAMA_SUPPRESS_HOST_DEVICE_WARNING - template - LLAMA_FN_HOST_ACC_INLINE auto accessor(ArrayDims arrayDims, RecordCoord dc = {}) const - -> decltype(auto) + template + LLAMA_FN_HOST_ACC_INLINE auto accessor( + ArrayDims arrayDims, + Array dynamicArrayExtents, + RecordCoord dc = {}) const -> decltype(auto) { if constexpr (internal::isComputed>::value) return mapping.compute(arrayDims, dc, storageBlobs); else { - const auto [nr, offset] = mapping.template blobNrAndOffset(arrayDims); + const auto [nr, offset] = mapping.template blobNrAndOffset(arrayDims, dynamicArrayExtents); using Type = GetType>; return reinterpret_cast(storageBlobs[nr][offset]); } } LLAMA_SUPPRESS_HOST_DEVICE_WARNING - template - LLAMA_FN_HOST_ACC_INLINE auto accessor(ArrayDims arrayDims, RecordCoord dc = {}) -> decltype(auto) + template + LLAMA_FN_HOST_ACC_INLINE auto accessor( + ArrayDims arrayDims, + Array dynamicArrayExtents, + RecordCoord dc = {}) -> decltype(auto) { if constexpr (internal::isComputed>::value) - return mapping.compute(arrayDims, dc, storageBlobs); + return mapping.compute(arrayDims, dynamicArrayExtents, dc, storageBlobs); else { - const auto [nr, offset] = mapping.template blobNrAndOffset(arrayDims); + const auto [nr, offset] = mapping.template blobNrAndOffset(arrayDims, dynamicArrayExtents); using Type = GetType>; return reinterpret_cast(storageBlobs[nr][offset]); } diff --git a/include/llama/VirtualRecord.hpp b/include/llama/VirtualRecord.hpp index 2cb030cea7..97698944af 100644 --- a/include/llama/VirtualRecord.hpp +++ b/include/llama/VirtualRecord.hpp @@ -309,6 +309,21 @@ namespace llama template typename Tuple, typename... Args> constexpr inline auto isDirectListInitializableFromTuple> = isDirectListInitializable; + + template + constexpr inline auto unboundArraysUntil = []() constexpr + { + std::size_t count = 0; + boost::mp11::mp_for_each>( + [&](auto i) constexpr + { + using RC = RecordCoordFromList>; + using TypeAtRC = GetType; + count += static_cast(internal::is_unbounded_array_v); + }); + return count; + } + (); } // namespace internal /// Virtual record type returned by \ref View after resolving an array dimensions coordinate or partially resolving @@ -324,9 +339,11 @@ namespace llama private: using ArrayDims = typename View::Mapping::ArrayDims; using RecordDim = typename View::Mapping::RecordDim; + using DynamicArrayExtentsArray = Array>; - [[no_unique_address]] const ArrayDims arrayDimsCoord; std::conditional_t view; + [[no_unique_address]] const ArrayDims arrayDimsCoord; + [[no_unique_address]] const DynamicArrayExtentsArray dynamicArrayExtents; public: /// Subtree of the record dimension of View starting at BoundRecordCoord. If BoundRecordCoord is `RecordCoord<>` @@ -337,15 +354,20 @@ namespace llama LLAMA_FN_HOST_ACC_INLINE VirtualRecord() /* requires(OwnView) */ : arrayDimsCoord({}) + , dynamicArrayExtents({}) , view{allocViewStack<0, RecordDim>()} { static_assert(OwnView, "The default constructor of VirtualRecord is only available if it owns the view."); } LLAMA_FN_HOST_ACC_INLINE - VirtualRecord(ArrayDims arrayDimsCoord, std::conditional_t view) - : arrayDimsCoord(arrayDimsCoord) - , view{static_cast(view)} + VirtualRecord( + std::conditional_t view, + ArrayDims arrayDimsCoord, + DynamicArrayExtentsArray dynamicArrayExtents = {}) + : view{static_cast(view)} + , arrayDimsCoord(arrayDimsCoord) + , dynamicArrayExtents{dynamicArrayExtents} { } @@ -388,15 +410,21 @@ namespace llama { using AbsolutCoord = Cat>; using AccessedType = GetType; - if constexpr (isRecord || internal::is_bounded_array::value) + if constexpr ( + isRecord || internal::is_bounded_array::value + || internal::is_unbounded_array_v) { LLAMA_FORCE_INLINE_RECURSIVE - return VirtualRecord{arrayDimsCoord, this->view}; + return VirtualRecord{ + this->view, + arrayDimsCoord, + dynamicArrayExtents, + }; } else { LLAMA_FORCE_INLINE_RECURSIVE - return this->view.accessor(arrayDimsCoord, AbsolutCoord{}); + return this->view.accessor(arrayDimsCoord, dynamicArrayExtents, AbsolutCoord{}); } } @@ -406,22 +434,24 @@ namespace llama { using AbsolutCoord = Cat>; using AccessedType = GetType; - if constexpr (isRecord || internal::is_bounded_array::value) + if constexpr ( + isRecord || internal::is_bounded_array::value + || internal::is_unbounded_array_v) { LLAMA_FORCE_INLINE_RECURSIVE - return VirtualRecord{arrayDimsCoord, this->view}; + return VirtualRecord{this->view, arrayDimsCoord, dynamicArrayExtents}; } else { LLAMA_FORCE_INLINE_RECURSIVE - return this->view.accessor(arrayDimsCoord, AbsolutCoord{}); + return this->view.accessor(arrayDimsCoord, dynamicArrayExtents, AbsolutCoord{}); } } /// Access a record in the record dimension underneath the current virtual record using a series of tags. If the /// access resolves to a leaf, a reference to a variable inside the \ref View storage is returned, otherwise /// another virtual record. - template + template ...>, bool> = true> LLAMA_FN_HOST_ACC_INLINE auto operator()(Tags...) const -> decltype(auto) { using RecordCoord = GetCoordFromTagsRelative; @@ -431,7 +461,7 @@ namespace llama } // FIXME(bgruber): remove redundancy - template + template ...>, bool> = true> LLAMA_FN_HOST_ACC_INLINE auto operator()(Tags...) -> decltype(auto) { using RecordCoord = GetCoordFromTagsRelative; @@ -440,6 +470,47 @@ namespace llama return operator()(RecordCoord{}); } + template < + typename ADD = AccessibleRecordDim, + std::enable_if_t, bool> = true> + LLAMA_FN_HOST_ACC_INLINE auto operator()(std::size_t i) const -> decltype(auto) + { + using AbsolutCoord = Cat>; + using ResolvedType = GetType; + auto newDynamicArrayExtents = push_back(dynamicArrayExtents, i); + if constexpr (isRecord || internal::is_unbounded_array_v) + { + LLAMA_FORCE_INLINE_RECURSIVE + return VirtualRecord{this->view, arrayDimsCoord, newDynamicArrayExtents}; + } + else + { + LLAMA_FORCE_INLINE_RECURSIVE + return this->view.accessor(arrayDimsCoord, newDynamicArrayExtents, AbsolutCoord{}); + } + } + + // FIXME(bgruber): remove redundancy + template < + typename ADD = AccessibleRecordDim, + std::enable_if_t, bool> = true> + LLAMA_FN_HOST_ACC_INLINE auto operator()(std::size_t i) -> decltype(auto) + { + using AbsolutCoord = Cat>; + using ResolvedType = GetType; + auto newDynamicArrayExtents = push_back(dynamicArrayExtents, i); + if constexpr (isRecord || internal::is_unbounded_array_v) + { + LLAMA_FORCE_INLINE_RECURSIVE + return VirtualRecord{this->view, arrayDimsCoord, newDynamicArrayExtents}; + } + else + { + LLAMA_FORCE_INLINE_RECURSIVE + return this->view.accessor(arrayDimsCoord, newDynamicArrayExtents, AbsolutCoord{}); + } + } + // we need this one to disable the compiler generated copy assignment LLAMA_FN_HOST_ACC_INLINE auto operator=(const VirtualRecord& other) -> VirtualRecord& { diff --git a/include/llama/llama.hpp b/include/llama/llama.hpp index c552ac7170..e2c5740785 100644 --- a/include/llama/llama.hpp +++ b/include/llama/llama.hpp @@ -41,6 +41,7 @@ #include "mapping/AoS.hpp" #include "mapping/AoSoA.hpp" #include "mapping/Heatmap.hpp" +#include "mapping/OffsetTable.hpp" #include "mapping/One.hpp" #include "mapping/SoA.hpp" #include "mapping/Split.hpp" diff --git a/include/llama/mapping/AoS.hpp b/include/llama/mapping/AoS.hpp index 4944c29cd0..cfbfd702ea 100644 --- a/include/llama/mapping/AoS.hpp +++ b/include/llama/mapping/AoS.hpp @@ -40,8 +40,10 @@ namespace llama::mapping return LinearizeArrayDimsFunctor{}.size(arrayDimsSize) * sizeOf; } - template - LLAMA_FN_HOST_ACC_INLINE constexpr auto blobNrAndOffset(ArrayDims coord) const -> NrAndOffset + template + LLAMA_FN_HOST_ACC_INLINE constexpr auto blobNrAndOffset( + ArrayDims coord, + Array dynamicArrayExtents = {}) const -> NrAndOffset { const auto offset = LinearizeArrayDimsFunctor{}(coord, arrayDimsSize) * sizeOf + offsetOf, AlignAndPad>; diff --git a/include/llama/mapping/AoSoA.hpp b/include/llama/mapping/AoSoA.hpp index 1f29f0f7e8..0b63d564d3 100644 --- a/include/llama/mapping/AoSoA.hpp +++ b/include/llama/mapping/AoSoA.hpp @@ -55,8 +55,10 @@ namespace llama::mapping return LinearizeArrayDimsFunctor{}.size(arrayDimsSize) * sizeOf; } - template - LLAMA_FN_HOST_ACC_INLINE constexpr auto blobNrAndOffset(ArrayDims coord) const -> NrAndOffset + template + LLAMA_FN_HOST_ACC_INLINE constexpr auto blobNrAndOffset( + ArrayDims coord, + Array dynamicArrayExtents = {}) const -> NrAndOffset { const auto flatArrayIndex = LinearizeArrayDimsFunctor{}(coord, arrayDimsSize); const auto blockIndex = flatArrayIndex / Lanes; diff --git a/include/llama/mapping/Heatmap.hpp b/include/llama/mapping/Heatmap.hpp index 34564dca48..92f016a0cc 100644 --- a/include/llama/mapping/Heatmap.hpp +++ b/include/llama/mapping/Heatmap.hpp @@ -44,8 +44,8 @@ namespace llama::mapping return mapping.blobSize(i); } - template - LLAMA_FN_HOST_ACC_INLINE auto blobNrAndOffset(ArrayDims coord) const -> NrAndOffset + template + LLAMA_FN_HOST_ACC_INLINE auto blobNrAndOffset(ArrayDims coord, Array = {}) const -> NrAndOffset { const auto nao = mapping.template blobNrAndOffset(coord); for (auto i = 0; i < sizeof(GetType>); i++) diff --git a/include/llama/mapping/OffsetTable.hpp b/include/llama/mapping/OffsetTable.hpp new file mode 100644 index 0000000000..abe7d975b6 --- /dev/null +++ b/include/llama/mapping/OffsetTable.hpp @@ -0,0 +1,315 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include "../Tuple.hpp" +#include "AoS.hpp" // TODO +#include "Common.hpp" + +namespace llama +{ + using EndOffsetType = std::size_t; + using SizeType = std::size_t; + + template + struct EndOffset + { + }; + template + struct Size + { + }; +} // namespace llama + +namespace llama::mapping +{ + namespace internal + { + using namespace boost::mp11; + + template + inline constexpr bool isEndOffsetField = false; + + template + inline constexpr bool isEndOffsetField> = true; + + template + inline constexpr bool isSizeField = false; + + template + inline constexpr bool isSizeField> = true; + + template + struct AddOffsetAndSizeFieldsImpl + { + using type = Record; + }; + + template + struct AddOffsetAndSizeFieldsImpl> + { + using type = Record, Field, EndOffsetType>, Field, SizeType>>; + }; + + template + using AddOffsetAndSizeFields = typename AddOffsetAndSizeFieldsImpl::type; + + template + struct ReplaceDynamicSubarrays + { + using Replaced = T; + using SubRecordDims = mp_list<>; + using SplitCoords = mp_list<>; + using Augmented = T; + }; + + template + struct ReplaceDynamicSubarrays> + { + using Replaced = EndOffsetType; // offset table entry + using SubRecordDims = mp_list>::Replaced>; + using SplitCoords = mp_push_front< + typename ReplaceDynamicSubarrays>::SplitCoords, + RecordCoord>; + using Augmented = T[]; + }; + + template + struct ReplaceDynamicSubarraysHelp; + + template + struct ReplaceDynamicSubarraysHelp, std::index_sequence> + { + using Replaced = Record>, + typename ReplaceDynamicSubarrays>, RecordCoord>::Replaced>...>; + using SubRecordDims + = mp_append>, RecordCoord>:: + SubRecordDims...>; + using SplitCoords + = mp_append>, RecordCoord>:: + SplitCoords...>; + + using Augmented = mp_flatten>; + }; + + template + struct ReplaceDynamicSubarrays, RecordCoord> + : ReplaceDynamicSubarraysHelp< + Record, + RecordCoord, + std::make_index_sequence> + { + }; + + template + using BeforeDynamic + = RecordCoordFromList>>>; + + template + using AfterDynamic = RecordCoordFromList>::value + 1, RC::size)>>>; + + template + using OffsetLastCoord = RecordCoordFromList< + mp_push_back, mp_size_t>>; + + template + struct ShiftRecordCoord; + + template + struct ShiftRecordCoord> + { + using Coord = RecordCoord<>; + }; + + template + struct ShiftRecordCoord> + { + template + using IsUnboundArrayField = llama::internal::is_unbounded_array>; + + using ShiftedFirst + = RecordCoord, IsUnboundArrayField>::value>; + using ShiftedRest = typename ShiftRecordCoord, RecordCoord>::Coord; + + using Coord = Cat; + }; + } // namespace internal + + template + struct OffsetTable + { + using RDS = internal::ReplaceDynamicSubarrays>; + using SubRecordDims = boost::mp11::mp_push_front; + using SplitCoords = typename RDS::SplitCoords; + + template + using MapRecordDim = AlignedAoS, SubRecordDim>; // TODO: make this configureable + using MappedSubRecordDims = boost::mp11::mp_transform; + + boost::mp11::mp_rename subMappings; + + using ArrayDims = T_ArrayDims; + using RecordDim = typename RDS::Augmented; + static constexpr std::size_t blobCount = []() constexpr + { + std::size_t count = 0; + boost::mp11::mp_for_each>( + [&](auto subMapping) { count += decltype(subMapping)::type::blobCount; }); + return count; + } + (); + + constexpr OffsetTable() = default; + + template + LLAMA_FN_HOST_ACC_INLINE constexpr OffsetTable(ArrayDims... sizes) : subMappings(sizes...) + { + } + + LLAMA_FN_HOST_ACC_INLINE constexpr auto arrayDims() const -> ArrayDims + { + return get<0>(subMappings).arrayDims(); + } + + LLAMA_FN_HOST_ACC_INLINE constexpr auto blobSize(std::size_t i) const -> std::size_t + { + std::size_t result = 0; + boost::mp11::mp_for_each>>( + [&](auto jc) + { + constexpr auto j = decltype(jc)::value; + constexpr auto subBlobs = TupleElement::blobCount; + if (i < subBlobs) + result = get(subMappings).blobSize(i); + i -= subBlobs; + }); + return result; + } + + template + LLAMA_FN_HOST_ACC_INLINE static constexpr auto isComputed(RecordCoord) + { + return true; + } + + template + LLAMA_FN_HOST_ACC_INLINE constexpr auto compute( + ArrayDims coord, + Array dynamicArrayExtents, + RecordCoord rc, + Array& blobs) const -> decltype(auto) + { + return computeRecursive<0>(llama::RecordCoord{}, rc, coord, dynamicArrayExtents, blobs); + } + + private: + template < + std::size_t MappingIndex, + typename ResolvedRecordCoord, + typename UnresolvedRecordCoord, + std::size_t N, + typename Blob> + LLAMA_FN_HOST_ACC_INLINE constexpr auto computeRecursive( + ResolvedRecordCoord, + UnresolvedRecordCoord, + ArrayDims coord, + Array dynamicArrayExtents, + Array& blobs) const -> decltype(auto) + { + static_assert( + ArrayDims::rank == 1, + "Not implemented"); // this would need a way to get the prev of coord, also ArrayDims can be a different + // type during recursive instantiation + + using UnresolvedBeforeDynamic = internal::BeforeDynamic; + using UnresolvedAfterDynamic = internal::AfterDynamic; + using ResolvedSoFar = Cat; + + auto loadBeginOffset = [&](auto unresolvedBeforeDynamic) -> EndOffsetType + { + if (coord == ArrayDims{}) [[unlikely]] + return 0; + auto prevCoord = coord; + prevCoord[0]--; + return reinterpret_cast( + *mapToAddress(ResolvedRecordCoord{}, unresolvedBeforeDynamic, prevCoord, blobs)); + }; + + using Tag = GetTag; + if constexpr (internal::isEndOffsetField) + // load offset from dynamic array member field at prev record coord + return reinterpret_cast(*mapToAddress( + ResolvedRecordCoord{}, + internal::OffsetLastCoord{}, + coord, + blobs)); + else if constexpr (internal::isSizeField) + { + // compute size from end offset and prev end offset (or 0 for the first sub array) + const auto begin = loadBeginOffset(internal::OffsetLastCoord{}); + const auto end = reinterpret_cast(*mapToAddress( + ResolvedRecordCoord{}, + internal::OffsetLastCoord{}, + coord, + blobs)); + return static_cast(end - begin); + } + else if constexpr (std::is_same_v) + { + // no dynamic sub arrays anymore, proceed with access + static_assert(N == 0); + using Type = GetType; + return reinterpret_cast( + *mapToAddress(ResolvedRecordCoord{}, UnresolvedBeforeDynamic{}, coord, blobs)); + } + else + { + // continue resolving with next submapping + using ShiftedCoord = typename internal::ShiftRecordCoord::Coord; + constexpr auto nextSubMappingIndex = boost::mp11::mp_find::value + 1; + static_assert(nextSubMappingIndex < boost::mp11::mp_size::value); + const auto dynamicSubIndex = loadBeginOffset(UnresolvedBeforeDynamic{}) + dynamicArrayExtents[0]; + return computeRecursive( + Cat>{}, + UnresolvedAfterDynamic{}, + llama::ArrayDims{dynamicSubIndex}, + pop_front(dynamicArrayExtents), + blobs); + } + } + + template < + std::size_t MappingIndex, + typename RecordCoordBeforeThisMapping, + typename RecordCoordForThisMapping, + typename Blob> + LLAMA_FN_HOST_ACC_INLINE constexpr auto mapToAddress( + RecordCoordBeforeThisMapping, + RecordCoordForThisMapping, + ArrayDims coord, + Array& blobs) const -> std::byte* + { + // we need to shift the record coord before mapping, because the user exposed RecordDim contains the + // artificial EndOffset and Size fields, which the RecordDim of the submappings don't have. + using ExposedSubRecordDim = GetType; + using ShiftedCoord = + typename internal::ShiftRecordCoord::Coord; + auto [nr, offset] = blobNrAndOffset(get(subMappings), ShiftedCoord{}, coord); + boost::mp11::mp_for_each>( + [nr = std::ref(nr)](auto i) { nr += boost::mp11::mp_at::blobCount; }); + return &blobs[nr][offset]; + } + + template + LLAMA_FN_HOST_ACC_INLINE constexpr auto blobNrAndOffset( + const Mapping& mapping, + RecordCoord, + ArrayDims coord) const -> NrAndOffset + { + return mapping.template blobNrAndOffset(coord); + } + }; +} // namespace llama::mapping diff --git a/include/llama/mapping/One.hpp b/include/llama/mapping/One.hpp index 936078bdd8..0acfdea84e 100644 --- a/include/llama/mapping/One.hpp +++ b/include/llama/mapping/One.hpp @@ -38,8 +38,9 @@ namespace llama::mapping return sizeOf; } - template - LLAMA_FN_HOST_ACC_INLINE constexpr auto blobNrAndOffset(ArrayDims) const -> NrAndOffset + template + LLAMA_FN_HOST_ACC_INLINE constexpr auto blobNrAndOffset(ArrayDims, Array = {}) const + -> NrAndOffset { constexpr auto offset = offsetOf>; return {0, offset}; diff --git a/include/llama/mapping/SoA.hpp b/include/llama/mapping/SoA.hpp index 3b2096a81d..af69305eb8 100644 --- a/include/llama/mapping/SoA.hpp +++ b/include/llama/mapping/SoA.hpp @@ -56,8 +56,10 @@ namespace llama::mapping return LinearizeArrayDimsFunctor{}.size(arrayDimsSize) * sizeOf; } - template - LLAMA_FN_HOST_ACC_INLINE constexpr auto blobNrAndOffset(ArrayDims coord) const -> NrAndOffset + template + LLAMA_FN_HOST_ACC_INLINE constexpr auto blobNrAndOffset( + ArrayDims coord, + Array dynamicArrayExtents = {}) const -> NrAndOffset { if constexpr (SeparateBuffers) { diff --git a/include/llama/mapping/Split.hpp b/include/llama/mapping/Split.hpp index d6b187b893..49c77c4125 100644 --- a/include/llama/mapping/Split.hpp +++ b/include/llama/mapping/Split.hpp @@ -103,8 +103,10 @@ namespace llama::mapping return mapping1.blobSize(0) + mapping2.blobSize(0); } - template - LLAMA_FN_HOST_ACC_INLINE constexpr auto blobNrAndOffset(ArrayDims coord) const -> NrAndOffset + template + LLAMA_FN_HOST_ACC_INLINE constexpr auto blobNrAndOffset( + ArrayDims coord, + Array dynamicArrayExtents = {}) const -> NrAndOffset { if constexpr (RecordCoordCommonPrefixIsSame>) { @@ -112,13 +114,17 @@ namespace llama::mapping // zero all coordinate values that are part of RecordCoordForMapping1 using Prefix = mp_repeat_c, RecordCoordForMapping1::size>; using Suffix = mp_drop_c, RecordCoordForMapping1::size>; - return blobNrAndOffset(RecordCoordFromList>{}, coord, mapping1); + return blobNrAndOffset( + RecordCoordFromList>{}, + coord, + dynamicArrayExtents, + mapping1); } else { constexpr auto dstCoord = internal::offsetCoord(RecordCoord{}, RecordCoordForMapping1{}); - auto nrAndOffset = blobNrAndOffset(dstCoord, coord, mapping2); + auto nrAndOffset = blobNrAndOffset(dstCoord, coord, dynamicArrayExtents, mapping2); if constexpr (SeparateBlobs) nrAndOffset.nr += Mapping1::blobCount; else @@ -131,13 +137,14 @@ namespace llama::mapping } private: - template + template LLAMA_FN_HOST_ACC_INLINE constexpr auto blobNrAndOffset( RecordCoord, ArrayDims coord, + Array dynamicArrayExtents, const Mapping& mapping) const -> NrAndOffset { - return mapping.template blobNrAndOffset(coord); + return mapping.template blobNrAndOffset(coord, dynamicArrayExtents); } public: diff --git a/include/llama/mapping/Trace.hpp b/include/llama/mapping/Trace.hpp index a4492cf358..6f31daf4a9 100644 --- a/include/llama/mapping/Trace.hpp +++ b/include/llama/mapping/Trace.hpp @@ -75,8 +75,9 @@ namespace llama::mapping return mapping.blobSize(i); } - template - LLAMA_FN_HOST_ACC_INLINE auto blobNrAndOffset(ArrayDims coord) const -> NrAndOffset + template + LLAMA_FN_HOST_ACC_INLINE auto blobNrAndOffset(ArrayDims coord, Array dynamicArrayExtents = {}) + const -> NrAndOffset { const static auto name = internal::coordName(RecordCoord{}); fieldHits.at(name)++; diff --git a/include/llama/mapping/tree/Mapping.hpp b/include/llama/mapping/tree/Mapping.hpp index e826bc8180..0fded6707f 100644 --- a/include/llama/mapping/tree/Mapping.hpp +++ b/include/llama/mapping/tree/Mapping.hpp @@ -202,8 +202,9 @@ namespace llama::mapping::tree return internal::getTreeBlobSize(resultTree); } - template - LLAMA_FN_HOST_ACC_INLINE auto blobNrAndOffset(ArrayDims coord) const -> NrAndOffset + template + LLAMA_FN_HOST_ACC_INLINE auto blobNrAndOffset(ArrayDims coord, Array dynamicArrayExtents = {}) + const -> NrAndOffset { auto const basicTreeCoord = createTreeCoord>(coord); auto const resultTreeCoord = mergedFunctors.basicCoordToResultCoord(basicTreeCoord, basicTree); diff --git a/tests/computedprop.cpp b/tests/computedprop.cpp index a7aa0b97a4..edb374f4aa 100644 --- a/tests/computedprop.cpp +++ b/tests/computedprop.cpp @@ -23,9 +23,10 @@ namespace return llama::RecordCoordCommonPrefixIsSame, llama::RecordCoord<3>>; } - template + template constexpr auto compute( ArrayDims coord, + llama::Array dynamicArrayExtents, llama::RecordCoord, llama::Array& storageBlobs) const { @@ -128,9 +129,12 @@ namespace return true; } - template - constexpr auto compute(ArrayDims coord, llama::RecordCoord, llama::Array&) - const -> std::size_t + template + constexpr auto compute( + ArrayDims coord, + llama::Array dynamicArrayExtents, + llama::RecordCoord, + llama::Array&) const -> std::size_t { return std::reduce(std::begin(coord), std::end(coord), std::size_t{1}, std::multiplies<>{}); } @@ -202,9 +206,10 @@ namespace return true; } - template + template constexpr auto compute( ArrayDims coord, + llama::Array dynamicArrayExtents, llama::RecordCoord, llama::Array& blobs) const -> BoolRef { diff --git a/tests/core.cpp b/tests/core.cpp index c17bc660f2..e8507c793e 100644 --- a/tests/core.cpp +++ b/tests/core.cpp @@ -277,4 +277,32 @@ TEST_CASE("flatRecordCoord") STATIC_REQUIRE(llama::flatRecordCoord> == 8); STATIC_REQUIRE(llama::flatRecordCoord> == 9); STATIC_REQUIRE(llama::flatRecordCoord> == 10); -} \ No newline at end of file +} + +TEST_CASE("unboundArrays") +{ + struct Tag + { + }; + + using Int0 = int; + using Int1 = int[]; + using Int2 = llama::Record>[]; + using Int3 = llama::Record>[]>>[]; + + using llama::internal::unboundArraysUntil; + STATIC_REQUIRE(unboundArraysUntil> == 0); + STATIC_REQUIRE(unboundArraysUntil> == 0); + STATIC_REQUIRE(unboundArraysUntil> == 1); + STATIC_REQUIRE(unboundArraysUntil> == 0); + STATIC_REQUIRE(unboundArraysUntil> == 1); + STATIC_REQUIRE(unboundArraysUntil> == 1); + STATIC_REQUIRE(unboundArraysUntil> == 2); + STATIC_REQUIRE(unboundArraysUntil> == 0); + STATIC_REQUIRE(unboundArraysUntil> == 1); + STATIC_REQUIRE(unboundArraysUntil> == 1); + STATIC_REQUIRE(unboundArraysUntil> == 2); + STATIC_REQUIRE(unboundArraysUntil> == 2); + STATIC_REQUIRE( + unboundArraysUntil> == 3); +} diff --git a/tests/recorddimension.cpp b/tests/recorddimension.cpp index 374090591f..420952fc0c 100644 --- a/tests/recorddimension.cpp +++ b/tests/recorddimension.cpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include #include @@ -267,3 +269,159 @@ TEST_CASE("recorddim.record_with_arrays") view(0u)(A3{}, 1_RC, 0_RC); view(0u)(A3{}, 1_RC, 1_RC); } + +TEST_CASE("dynamic array") +{ + struct Tag + { + }; + using RecordDim = llama::Record>; + auto mapping + = llama::mapping::OffsetTable, RecordDim>{llama::ArrayDims{2}, llama::ArrayDims{5}}; + auto view = allocView(mapping); + + view(0)(llama::EndOffset{}) = 3; + view(1)(llama::EndOffset{}) = 5; + + CHECK(view(0)(llama::Size{}) == 3); + int& e0 = view(0)(Tag{})(0); + int& e1 = view(0)(Tag{})(1); + int& e2 = view(0)(Tag{})(2); + CHECK(view(1)(llama::Size{}) == 2); + int& e3 = view(1)(Tag{})(0); + int& e4 = view(1)(Tag{})(1); + + e0 = 1; + e1 = 2; + e2 = 3; + e3 = 4; + e4 = 5; + CHECK(e0 == 1); + CHECK(e1 == 2); + CHECK(e2 == 3); + CHECK(e3 == 4); + CHECK(e4 == 5); +} + +namespace +{ + // clang-format off + struct run {}; + struct luminosityBlock {}; + struct Electrons {}; + struct Muons {}; + struct Eta{}; + struct Mass{}; + struct Phi{}; + + using Electron = llama::Record< + llama::Field, + llama::Field, + llama::Field + >; + using Muon = llama::Record< + llama::Field, + llama::Field, + llama::Field + >; + using Event = llama::Record< + llama::Field, + llama::Field, + llama::Field, + llama::Field + >; + // clang-format on +} // namespace + +TEST_CASE("edm") +{ + // 3 events with 5 electrons and 4 muons + auto mapping = llama::mapping::OffsetTable, Event>{ + llama::ArrayDims{3}, + llama::ArrayDims{5}, + llama::ArrayDims{4}}; + // std::ofstream{"edm.svg"} << llama::toSvg(mapping); + auto view = llama::allocView(mapping); + + // setup offset table + view(0)(llama::EndOffset{}) = 3; + view(1)(llama::EndOffset{}) = 3; + view(2)(llama::EndOffset{}) = 5; + + view(0)(llama::EndOffset{}) = 0; + view(1)(llama::EndOffset{}) = 3; + view(2)(llama::EndOffset{}) = 4; + + // fill with values + int value = 1; + for (auto i = 0; i < 3; i++) + { + auto event = view(i); + event(run{}) = value++; + event(luminosityBlock{}) = value++; + for (auto j = 0; j < event(llama::Size{}); j++) + { + auto electron = event(Electrons{})(j); + electron(Eta{}) = value++; + electron(Mass{}) = value++; + electron(Phi{}) = value++; + } + for (auto j = 0; j < event(llama::Size{}); j++) + { + auto muon = event(Muons{})(j); + muon(Eta{}) = value++; + muon(Mass{}) = value++; + muon(Phi{}) = value++; + } + } + + // check all values + value = 1; + CHECK(view(0)(run{}) == value++); + CHECK(view(0)(luminosityBlock{}) == value++); + CHECK(view(0)(llama::EndOffset{}) == 3); + CHECK(view(0)(llama::Size{}) == 3); + CHECK(view(0)(Electrons{})(0) (Eta{}) == value++); + CHECK(view(0)(Electrons{})(0) (Mass{}) == value++); + CHECK(view(0)(Electrons{})(0) (Phi{}) == value++); + CHECK(view(0)(Electrons{})(1) (Eta{}) == value++); + CHECK(view(0)(Electrons{})(1) (Mass{}) == value++); + CHECK(view(0)(Electrons{})(1) (Phi{}) == value++); + CHECK(view(0)(Electrons{})(2) (Eta{}) == value++); + CHECK(view(0)(Electrons{})(2) (Mass{}) == value++); + CHECK(view(0)(Electrons{})(2) (Phi{}) == value++); + CHECK(view(0)(llama::EndOffset{}) == 0); + CHECK(view(0)(llama::Size{}) == 0); + + CHECK(view(1)(run{}) == value++); + CHECK(view(1)(luminosityBlock{}) == value++); + CHECK(view(1)(llama::EndOffset{}) == 3); + CHECK(view(1)(llama::Size{}) == 0); + CHECK(view(1)(llama::EndOffset{}) == 3); + CHECK(view(1)(llama::Size{}) == 3); + CHECK(view(1)(Muons{})(0) (Eta{}) == value++); + CHECK(view(1)(Muons{})(0) (Mass{}) == value++); + CHECK(view(1)(Muons{})(0) (Phi{}) == value++); + CHECK(view(1)(Muons{})(1) (Eta{}) == value++); + CHECK(view(1)(Muons{})(1) (Mass{}) == value++); + CHECK(view(1)(Muons{})(1) (Phi{}) == value++); + CHECK(view(1)(Muons{})(2) (Eta{}) == value++); + CHECK(view(1)(Muons{})(2) (Mass{}) == value++); + CHECK(view(1)(Muons{})(2) (Phi{}) == value++); + + CHECK(view(2)(run{}) == value++); + CHECK(view(2)(luminosityBlock{}) == value++); + CHECK(view(2)(llama::EndOffset{}) == 5); + CHECK(view(2)(llama::Size{}) == 2); + CHECK(view(2)(Electrons{})(0) (Eta{}) == value++); + CHECK(view(2)(Electrons{})(0) (Mass{}) == value++); + CHECK(view(2)(Electrons{})(0) (Phi{}) == value++); + CHECK(view(2)(Electrons{})(1) (Eta{}) == value++); + CHECK(view(2)(Electrons{})(1) (Mass{}) == value++); + CHECK(view(2)(Electrons{})(1) (Phi{}) == value++); + CHECK(view(2)(llama::EndOffset{}) == 4); + CHECK(view(2)(llama::Size{}) == 1); + CHECK(view(2)(Muons{})(0) (Eta{}) == value++); + CHECK(view(2)(Muons{})(0) (Mass{}) == value++); + CHECK(view(2)(Muons{})(0) (Phi{}) == value++); +}