diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ac55a13..65d9d17 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 with: - toolchain: stable + toolchain: beta # FIXME: revert to stable when 1.65 lands components: clippy - name: "clippy --all" run: cargo clippy --all --tests -- -D warnings @@ -27,7 +27,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 with: - toolchain: stable + toolchain: beta # FIXME: revert to stable when 1.65 lands components: rustfmt - name: Run run: cargo fmt --all -- --check @@ -39,7 +39,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 with: - toolchain: stable + toolchain: beta # FIXME: revert to stable when 1.65 lands - uses: Swatinem/rust-cache@v1 - uses: taiki-e/install-action@nextest - name: Run @@ -53,7 +53,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions-rs/toolchain@v1 with: - toolchain: stable + toolchain: beta # FIXME: revert to stable when 1.65 lands components: llvm-tools-preview - name: Install cargo-llvm-cov uses: taiki-e/install-action@cargo-llvm-cov diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..7dc1d05 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,34 @@ +## Changelog + +All notable changes to `arrow2-convert` project will be documented in this file. + +## [Unreleased] +## [0.4.0] +### Changed +- MSRV is bumped to 1.65.0 due to dependency on GATs (#73, @aldanor). +- `ArrowDeserialize` can now be used as a standalone trait bound (#73, @aldanor). + +## [0.3.1] - 2022-09-29 +### Changed +- Update arrow2 version to 0.14 (#66, @ncpenke). + +## [0.3.0] - 2022-08-25 +### Added +- Add support for converting to `Chunk` (#44, @ncpenke). +- Add support for `i128` (#48, @ncpenke). +- Add support for enums (#37, @ncpenke). +- Add support for flattening chunks (#56, @nielsmeima). + +### Changed +- Serialize escaped Rust identifiers unescaped (#59, @teymour-aldridge). +- Update arrow2 version to 0.13 (#61, @teymour-aldridge). + +## [0.2.0] - 2022-06-13 +### Added +- Add support for `FixedSizeBinary` and `FixedSizeList` (#30, @ncpenke). + +### Changed +- Update arrow2 version to 0.12 (#38, @joshuataylor). + +## [0.1.0] - 2022-03-03 +Initial crate release. diff --git a/arrow2_convert/src/deserialize.rs b/arrow2_convert/src/deserialize.rs index c3a83f8..78de9fc 100644 --- a/arrow2_convert/src/deserialize.rs +++ b/arrow2_convert/src/deserialize.rs @@ -5,18 +5,38 @@ use chrono::{NaiveDate, NaiveDateTime}; use crate::field::*; -/// Implemented by [`ArrowField`] that can be deserialized from arrow -pub trait ArrowDeserialize: ArrowField + Sized +#[doc(hidden)] +/// Type whose reference can be used to create an iterator. +pub trait IterRef { + /// Iterator type. + type Iter<'a>: Iterator + where + Self: 'a; + + /// Converts `&self` into an iterator. + fn iter_ref(&self) -> Self::Iter<'_>; +} + +impl IterRef for T where - Self::ArrayType: ArrowArray, - for<'a> &'a Self::ArrayType: IntoIterator, + for<'a> &'a T: IntoIterator, { + type Iter<'a> = <&'a T as IntoIterator>::IntoIter where Self: 'a; + + #[inline] + fn iter_ref(&self) -> Self::Iter<'_> { + self.into_iter() + } +} + +/// Implemented by [`ArrowField`] that can be deserialized from arrow +pub trait ArrowDeserialize: ArrowField + Sized { /// The `arrow2::Array` type corresponding to this field - type ArrayType; + type ArrayType: ArrowArray; /// Deserialize this field from arrow fn arrow_deserialize( - v: <&Self::ArrayType as IntoIterator>::Item, + v: <::Iter<'_> as Iterator>::Item, ) -> Option<::Type>; #[inline] @@ -28,7 +48,7 @@ where /// something like for<'a> &'a T::ArrayType: IntoIterator>, /// However, the E parameter seems to confuse the borrow checker if it's a reference. fn arrow_deserialize_internal( - v: <&Self::ArrayType as IntoIterator>::Item, + v: <::Iter<'_> as Iterator>::Item, ) -> ::Type { Self::arrow_deserialize(v).unwrap() } @@ -36,18 +56,15 @@ where /// Internal trait used to support deserialization and iteration of structs, and nested struct lists /// -/// Trivial pass-thru implementations are provided for arrow2 arrays that implement IntoIterator. +/// Trivial pass-thru implementations are provided for arrow2 arrays that auto-implement IterRef. /// /// The derive macro generates implementations for typed struct arrays. #[doc(hidden)] -pub trait ArrowArray -where - for<'a> &'a Self: IntoIterator, -{ +pub trait ArrowArray: IterRef { type BaseArrayType: Array; // Returns a typed iterator to the underlying elements of the array from an untyped Array reference. - fn iter_from_array_ref(b: &dyn Array) -> <&Self as IntoIterator>::IntoIter; + fn iter_from_array_ref(b: &dyn Array) -> ::Iter<'_>; } // Macro to facilitate implementation for numeric types and numeric arrays. @@ -71,11 +88,11 @@ macro_rules! impl_arrow_array { impl ArrowArray for $array { type BaseArrayType = Self; - fn iter_from_array_ref(b: &dyn Array) -> <&Self as IntoIterator>::IntoIter { + fn iter_from_array_ref(b: &dyn Array) -> ::Iter<'_> { b.as_any() .downcast_ref::() .unwrap() - .into_iter() + .iter_ref() } } }; @@ -85,21 +102,19 @@ macro_rules! impl_arrow_array { impl ArrowDeserialize for Option where T: ArrowDeserialize, - T::ArrayType: 'static + ArrowArray, - for<'a> &'a T::ArrayType: IntoIterator, { type ArrayType = ::ArrayType; #[inline] fn arrow_deserialize( - v: <&Self::ArrayType as IntoIterator>::Item, + v: <::Iter<'_> as Iterator>::Item, ) -> Option<::Type> { Self::arrow_deserialize_internal(v).map(Some) } #[inline] fn arrow_deserialize_internal( - v: <&Self::ArrayType as IntoIterator>::Item, + v: <::Iter<'_> as Iterator>::Item, ) -> ::Type { ::arrow_deserialize(v) } @@ -204,7 +219,6 @@ fn arrow_deserialize_vec_helper( ) -> Option< as ArrowField>::Type> where T: ArrowDeserialize + ArrowEnableVecForType + 'static, - for<'a> &'a T::ArrayType: IntoIterator, { use std::ops::Deref; v.map(|t| { @@ -217,8 +231,6 @@ where impl ArrowDeserialize for Vec where T: ArrowDeserialize + ArrowEnableVecForType + 'static, - ::ArrayType: 'static, - for<'b> &'b ::ArrayType: IntoIterator, { type ArrayType = ListArray; @@ -230,8 +242,6 @@ where impl ArrowDeserialize for LargeVec where T: ArrowDeserialize + ArrowEnableVecForType + 'static, - ::ArrayType: 'static, - for<'b> &'b ::ArrayType: IntoIterator, { type ArrayType = ListArray; @@ -243,8 +253,6 @@ where impl ArrowDeserialize for FixedSizeVec where T: ArrowDeserialize + ArrowEnableVecForType + 'static, - ::ArrayType: 'static, - for<'b> &'b ::ArrayType: IntoIterator, { type ArrayType = FixedSizeListArray; @@ -276,17 +284,15 @@ where /// useful when the same rust type maps to one or more Arrow types for example `LargeString`. fn try_into_collection_as_type(self) -> arrow2::error::Result where - ArrowType: ArrowDeserialize + ArrowField + 'static, - for<'b> &'b ::ArrayType: IntoIterator; + ArrowType: ArrowDeserialize + ArrowField + 'static; } /// Helper to return an iterator for elements from a [`arrow2::array::Array`]. fn arrow_array_deserialize_iterator_internal<'a, Element, Field>( - b: &'a dyn arrow2::array::Array, + b: &'a dyn Array, ) -> impl Iterator + 'a where Field: ArrowDeserialize + ArrowField + 'static, - for<'b> &'b ::ArrayType: IntoIterator, { <::ArrayType as ArrowArray>::iter_from_array_ref(b) .map(::arrow_deserialize_internal) @@ -294,12 +300,11 @@ where /// Returns a typed iterator to a target type from an `arrow2::Array` pub fn arrow_array_deserialize_iterator_as_type<'a, Element, ArrowType>( - arr: &'a dyn arrow2::array::Array, + arr: &'a dyn Array, ) -> arrow2::error::Result + 'a> where Element: 'static, ArrowType: ArrowDeserialize + ArrowField + 'static, - for<'b> &'b ::ArrayType: IntoIterator, { if &::data_type() != arr.data_type() { Err(arrow2::error::Error::InvalidArgumentError( @@ -315,11 +320,10 @@ where /// Return an iterator that deserializes an [`Array`] to an element of type T pub fn arrow_array_deserialize_iterator<'a, T>( - arr: &'a dyn arrow2::array::Array, + arr: &'a dyn Array, ) -> arrow2::error::Result + 'a> where T: ArrowDeserialize + ArrowField + 'static, - for<'b> &'b ::ArrayType: IntoIterator, { arrow_array_deserialize_iterator_as_type::(arr) } @@ -327,7 +331,6 @@ where impl TryIntoCollection for ArrowArray where Element: ArrowDeserialize + ArrowField + 'static, - for<'b> &'b ::ArrayType: IntoIterator, ArrowArray: std::borrow::Borrow, Collection: FromIterator, { @@ -338,7 +341,6 @@ where fn try_into_collection_as_type(self) -> arrow2::error::Result where ArrowType: ArrowDeserialize + ArrowField + 'static, - for<'b> &'b ::ArrayType: IntoIterator, { Ok( arrow_array_deserialize_iterator_as_type::(self.borrow())?