refactor: Bump arrow 52 and datafusion 39 (#105)

* refactor: Bump arrow 52 and datafusion 39 Signed-off-by: Xuanwo <[email protected]> * Fix union Signed-off-by: Xuanwo <[email protected]> * Format code Signed-off-by: Xuanwo <[email protected]> --------- Signed-off-by: Xuanwo <[email protected]>
datafusion-contrib · Jul 1, 2024 · 0b937e0 · 0b937e0
1 parent 1e61096
commit 0b937e0
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 15 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -13,23 +13,23 @@ rust-version = "1.70"
 
 [dependencies]
 anyhow = { version = "1.0", optional = true }
-arrow = { version = "51", features = ["prettyprint", "chrono-tz"] }
+arrow = { version = "52", features = ["prettyprint", "chrono-tz"] }
 async-trait = { version = "0.1.77", optional = true }
 bytes = "1.4"
 chrono = { version = "0.4.37", default-features = false, features = ["std"] }
 chrono-tz = "0.8.6"
 clap = { version = "4.5.4", features = ["derive"], optional = true }
-datafusion = { version = "37.0.0", optional = true }
-datafusion-expr = { version = "37.0.0", optional = true }
-datafusion-physical-expr = { version = "37.0.0", optional = true }
+datafusion = { version = "39.0.0", optional = true }
+datafusion-expr = { version = "39.0.0", optional = true }
+datafusion-physical-expr = { version = "39.0.0", optional = true }
 fallible-streaming-iterator = { version = "0.1" }
 flate2 = "1"
 futures = { version = "0.3", optional = true, default-features = false, features = ["std"] }
 futures-util = { version = "0.3", optional = true }
 lz4_flex = "0.11"
 lzokay-native = "0.1"
 num = "0.4.1"
-object_store = { version = "0.9.0", optional = true }
+object_store = { version = "0.10.1", optional = true }
 prost = { version = "0.11" }
 snafu = "0.7"
 snap = "1.1"

diff --git a/src/array_decoder/union.rs b/src/array_decoder/union.rs
@@ -119,15 +119,8 @@ impl ArrayBatchDecoder for UnionArrayDecoder {
             .collect::<Result<Vec<_>>>()?;
 
         // Currently default to decoding as Sparse UnionArray so no value offsets
-        let field_type_ids = (0..self.variants.len() as i8).collect::<Vec<_>>();
-        let type_ids = Buffer::from_vec(tags);
-        let child_arrays = self
-            .fields
-            .iter()
-            .map(|(_id, field)| field.as_ref().clone())
-            .zip(child_arrays)
-            .collect::<Vec<_>>();
-        let array = UnionArray::try_new(&field_type_ids, type_ids, None, child_arrays)
+        let type_ids = Buffer::from_vec(tags.clone()).into();
+        let array = UnionArray::try_new(self.fields.clone(), type_ids, None, child_arrays)
             .context(ArrowSnafu)?;
         let array = Arc::new(array);
         Ok(array)

diff --git a/src/bin/orc-metadata.rs b/src/bin/orc-metadata.rs
@@ -41,7 +41,6 @@ fn main() -> Result<(), Box<dyn Error>> {
                 &mut f,
                 &metadata,
                 metadata.root_data_type(),
-                i,
                 stripe_metadata,
             )?;
             println!("stripe index: {i}");