Merge branch 'main' into avro-reader-schema-store

Author: jecsand838

.github/workflows/arrow.yml

@@ -68,7 +68,10 @@ jobs:
       - name: Test arrow-schema
         run: cargo test -p arrow-schema --all-features
       - name: Test arrow-array
-        run: cargo test -p arrow-array --all-features
+        run: |
+          cargo test -p arrow-array --all-features
+          # Disable feature `force_validate`
+          cargo test -p arrow-array --features=ffi
       - name: Test arrow-select
         run: cargo test -p arrow-select --all-features
       - name: Test arrow-cast

CHANGELOG-old.md

@@ -89,8 +89,7 @@ You can also use rust's official docker image:
 docker run --rm -v $(pwd):/arrow-rs -it rust /bin/bash -c "cd /arrow-rs && rustup component add rustfmt && cargo build"
 ```
 
-The command above assumes that are in the root directory of the project, not in the same
-directory as this README.md.
+The command above assumes that are in the root directory of the project.
 
 You can also compile specific workspaces:
 

CHANGELOG.md

@@ -67,7 +67,7 @@ exclude = [
 ]
 
 [workspace.package]
-version = "55.2.0"
+version = "56.0.0"
 homepage = "https://github.com/apache/arrow-rs"
 repository = "https://github.com/apache/arrow-rs"
 authors = ["Apache Arrow <dev@arrow.apache.org>"]
@@ -84,22 +84,22 @@ edition = "2021"
 rust-version = "1.84"
 
 [workspace.dependencies]
-arrow = { version = "55.2.0", path = "./arrow", default-features = false }
-arrow-arith = { version = "55.2.0", path = "./arrow-arith" }
-arrow-array = { version = "55.2.0", path = "./arrow-array" }
-arrow-buffer = { version = "55.2.0", path = "./arrow-buffer" }
-arrow-cast = { version = "55.2.0", path = "./arrow-cast" }
-arrow-csv = { version = "55.2.0", path = "./arrow-csv" }
-arrow-data = { version = "55.2.0", path = "./arrow-data" }
-arrow-ipc = { version = "55.2.0", path = "./arrow-ipc" }
-arrow-json = { version = "55.2.0", path = "./arrow-json" }
-arrow-ord = { version = "55.2.0", path = "./arrow-ord" }
-arrow-pyarrow = { version = "55.2.0", path = "./arrow-pyarrow" }
-arrow-row = { version = "55.2.0", path = "./arrow-row" }
-arrow-schema = { version = "55.2.0", path = "./arrow-schema" }
-arrow-select = { version = "55.2.0", path = "./arrow-select" }
-arrow-string = { version = "55.2.0", path = "./arrow-string" }
-parquet = { version = "55.2.0", path = "./parquet", default-features = false }
+arrow = { version = "56.0.0", path = "./arrow", default-features = false }
+arrow-arith = { version = "56.0.0", path = "./arrow-arith" }
+arrow-array = { version = "56.0.0", path = "./arrow-array" }
+arrow-buffer = { version = "56.0.0", path = "./arrow-buffer" }
+arrow-cast = { version = "56.0.0", path = "./arrow-cast" }
+arrow-csv = { version = "56.0.0", path = "./arrow-csv" }
+arrow-data = { version = "56.0.0", path = "./arrow-data" }
+arrow-ipc = { version = "56.0.0", path = "./arrow-ipc" }
+arrow-json = { version = "56.0.0", path = "./arrow-json" }
+arrow-ord = { version = "56.0.0", path = "./arrow-ord" }
+arrow-pyarrow = { version = "56.0.0", path = "./arrow-pyarrow" }
+arrow-row = { version = "56.0.0", path = "./arrow-row" }
+arrow-schema = { version = "56.0.0", path = "./arrow-schema" }
+arrow-select = { version = "56.0.0", path = "./arrow-select" }
+arrow-string = { version = "56.0.0", path = "./arrow-string" }
+parquet = { version = "56.0.0", path = "./parquet", default-features = false }
 
 # These crates have not yet been released and thus do not use the workspace version
 parquet-variant = { version = "0.1.0", path = "./parquet-variant" }

CONTRIBUTING.md

@@ -108,7 +108,7 @@ The deprecated version is the next version which will be released (please
 consult the list above). To mark the API as deprecated, use the
 `#[deprecated(since = "...", note = "...")]` attribute.
 
-Foe example
+For example
 
 ```rust
 #[deprecated(since = "51.0.0", note = "Use `date_part` instead")]

Cargo.toml

@@ -408,7 +408,17 @@ impl ImportedArrowArray<'_> {
             .map(|index| {
                 let len = self.buffer_len(index, variadic_buffer_lens, &self.data_type)?;
                 match unsafe { create_buffer(self.owner.clone(), self.array, index, len) } {
-                    Some(buf) => Ok(buf),
+                    Some(buf) => {
+                        // External libraries may use a dangling pointer for a buffer with length 0.
+                        // We respect the array length specified in the C Data Interface. Actually,
+                        // if the length is incorrect, we cannot create a correct buffer even if
+                        // the pointer is valid.
+                        if buf.is_empty() {
+                            Ok(MutableBuffer::new(0).into())
+                        } else {
+                            Ok(buf)
+                        }
+                    }
                     None if len == 0 => {
                         // Null data buffer, which Rust doesn't allow. So create
                         // an empty buffer.
@@ -1296,9 +1306,15 @@ mod tests_to_then_from_ffi {
 
 #[cfg(test)]
 mod tests_from_ffi {
+    #[cfg(not(feature = "force_validate"))]
+    use std::ptr::NonNull;
     use std::sync::Arc;
 
+    #[cfg(feature = "force_validate")]
     use arrow_buffer::{bit_util, buffer::Buffer};
+    #[cfg(not(feature = "force_validate"))]
+    use arrow_buffer::{bit_util, buffer::Buffer, ScalarBuffer};
+
     use arrow_data::transform::MutableArrayData;
     use arrow_data::ArrayData;
     use arrow_schema::{DataType, Field};
@@ -1660,6 +1676,25 @@ mod tests_from_ffi {
         }
     }
 
+    #[test]
+    #[cfg(not(feature = "force_validate"))]
+    fn test_utf8_view_ffi_from_dangling_pointer() {
+        let empty = GenericByteViewBuilder::<StringViewType>::new().finish();
+        let buffers = empty.data_buffers().to_vec();
+        let nulls = empty.nulls().cloned();
+
+        // Create a dangling pointer to a view buffer with zero length.
+        let alloc = Arc::new(1);
+        let buffer = unsafe { Buffer::from_custom_allocation(NonNull::<u8>::dangling(), 0, alloc) };
+        let views = unsafe { ScalarBuffer::new_unchecked(buffer) };
+
+        let str_view: GenericByteViewArray<StringViewType> =
+            unsafe { GenericByteViewArray::new_unchecked(views, buffers, nulls) };
+        let imported = roundtrip_byte_view_array(str_view);
+        assert_eq!(imported.len(), 0);
+        assert_eq!(&imported, &empty);
+    }
+
     #[test]
     fn test_round_trip_byte_view() {
         fn test_case<T>()

README.md

@@ -59,6 +59,7 @@ strum_macros = "0.27"
 uuid = "1.17"
 indexmap = "2.10"
 
+
 [dev-dependencies]
 arrow-data = { workspace = true }
 rand = { version = "0.9.1", default-features = false, features = [

arrow-array/src/ffi.rs

@@ -34,8 +34,12 @@
 //! # use std::fs::File;
 //! # use std::io::BufReader;
 //! # use arrow_avro::reader::ReaderBuilder;
-//!
-//! let file = File::open("../testing/data/avro/alltypes_plain.avro").unwrap();
+//! # let path = "avro/alltypes_plain.avro";
+//! # let path = match std::env::var("ARROW_TEST_DATA") {
+//! #   Ok(dir) => format!("{dir}/{path}"),
+//! #   Err(_) => format!("../testing/data/{path}")
+//! # };
+//! let file = File::open(path).unwrap();
 //! let mut avro = ReaderBuilder::new().build(BufReader::new(file)).unwrap();
 //! let batch = avro.next().unwrap();
 //! ```

arrow-avro/Cargo.toml

@@ -821,6 +821,20 @@ impl ToPrimitive for i256 {
         }
     }
 
+    fn to_f64(&self) -> Option<f64> {
+        let mag = if let Some(u) = self.checked_abs() {
+            let (low, high) = u.to_parts();
+            (high as f64) * 2_f64.powi(128) + (low as f64)
+        } else {
+            // self == MIN
+            2_f64.powi(255)
+        };
+        if *self < i256::ZERO {
+            Some(-mag)
+        } else {
+            Some(mag)
+        }
+    }
     fn to_u64(&self) -> Option<u64> {
         let as_i128 = self.low as i128;
 
@@ -1264,4 +1278,29 @@ mod tests {
             }
         }
     }
+
+    #[test]
+    fn test_decimal256_to_f64_typical_values() {
+        let v = i256::from_i128(42_i128);
+        assert_eq!(v.to_f64().unwrap(), 42.0);
+
+        let v = i256::from_i128(-123456789012345678i128);
+        assert_eq!(v.to_f64().unwrap(), -123456789012345678.0);
+    }
+
+    #[test]
+    fn test_decimal256_to_f64_large_positive_value() {
+        let max_f = f64::MAX;
+        let big = i256::from_f64(max_f * 2.0).unwrap_or(i256::MAX);
+        let out = big.to_f64().unwrap();
+        assert!(out.is_finite() && out.is_sign_positive());
+    }
+
+    #[test]
+    fn test_decimal256_to_f64_large_negative_value() {
+        let max_f = f64::MAX;
+        let big_neg = i256::from_f64(-(max_f * 2.0)).unwrap_or(i256::MIN);
+        let out = big_neg.to_f64().unwrap();
+        assert!(out.is_finite() && out.is_sign_negative());
+    }
 }