diff --git a/Cargo.lock b/Cargo.lock index def7f88448aeb..ed90dd25bda7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -164,9 +164,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "58.3.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471" +checksum = "ffaaa3e009861fd829d0a24dd6f115aa8e4634324bb092147d43baafe69ca4a7" dependencies = [ "arrow-arith", "arrow-array", @@ -187,9 +187,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.3.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738" +checksum = "3ac95125e1d71c4a252b5a9c729aef111e80418f08aaa6dbabd1ba66918247fc" dependencies = [ "arrow-array", "arrow-buffer", @@ -201,9 +201,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.3.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" +checksum = "0c60c79628e9a97cb90d7a0dc3e944f216a902f837d4ecabc14d524bddbbc137" dependencies = [ "ahash", "arrow-buffer", @@ -220,9 +220,9 @@ dependencies = [ [[package]] name = "arrow-avro" -version = "58.3.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "049230728cd6e093088c8d231b4beede184e35cad7777c1505c0d5a8571f4376" +checksum = "2835d67df2b69bf5de251ee6d289f85650d41b8169dcee0f950ea88747812c32" dependencies = [ "arrow-array", "arrow-buffer", @@ -244,9 +244,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.3.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0" +checksum = "6026f638c400e9878c1b1cc05c3cfd46fbf381285916ab408678701c1df46c1a" dependencies = [ "bytes", "half", @@ -256,9 +256,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "58.3.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f" +checksum = "c82c236c3caf8df5664284f3f1fbe89938852163998c3fdbf37e84ac220445e9" dependencies = [ "arrow-array", "arrow-buffer", @@ -278,9 +278,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.3.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de" +checksum = "12714e5fb7954159af1e26d4e0d37108bcf1a2ad5ee5c5bf02a944d564d588b7" dependencies = [ "arrow-array", "arrow-cast", @@ -293,9 +293,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.3.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" +checksum = "7bd568aa70c4ec5947027b0d5caee94877433b661a0bb9e8ddceeeb5f0c9b1ab" dependencies = [ "arrow-buffer", "arrow-schema", @@ -306,9 +306,9 @@ dependencies = [ [[package]] name = "arrow-flight" -version = "58.3.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28abfe8bf9f124e5fc83b334af4fa58f8d0323ad25312ccb2d1da50178415704" +checksum = "68365401e834743d708094927e2ca727a32d639fe900df04b936e07a36701b74" dependencies = [ "arrow-arith", "arrow-array", @@ -334,9 +334,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.3.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f" +checksum = "e57ee4d470eab1a021bc4b63fa2b2c15d572892bf227b0a982d3b755a6c662b5" dependencies = [ "arrow-array", "arrow-buffer", @@ -350,9 +350,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "58.3.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b" +checksum = "38f47e0e7a284e1f3707a780dc8cd5451b1614e9e398ea2d9ca03c7a2fe9a9ed" dependencies = [ "arrow-array", "arrow-buffer", @@ -375,9 +375,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.3.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0" +checksum = "a79cf73ad2eba8686ec2aa9bbf8671208e509025f166afc040cedbd94ffe4983" dependencies = [ "arrow-array", "arrow-buffer", @@ -388,9 +388,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.3.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c" +checksum = "cea0f7d8ed6182f14952761e2c0f989852d5aa334fcbc49f73a9f2247c25b879" dependencies = [ "arrow-array", "arrow-buffer", @@ -401,9 +401,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.3.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" +checksum = "80b3e786a0dd9103acd583a6fb486dbf2f3268466cc0bd571dcf34cef231c1f1" dependencies = [ "bitflags", "serde", @@ -413,9 +413,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "58.3.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" +checksum = "067a67e0361f6c31f4a7248759f36ca4ca71b187a941ed4d49da1c7d3d4db624" dependencies = [ "ahash", "arrow-array", @@ -427,9 +427,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.3.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42" +checksum = "99bc95847f3ff62a2b03d6f8ce2e3e78f01362060549a2a311898dd442f6256d" dependencies = [ "arrow-array", "arrow-buffer", @@ -3730,12 +3730,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "integer-encoding" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" - [[package]] name = "ipnet" version = "2.12.0" @@ -4389,15 +4383,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" -[[package]] -name = "ordered-float" -version = "2.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" -dependencies = [ - "num-traits", -] - [[package]] name = "outref" version = "0.5.2" @@ -4445,9 +4430,9 @@ dependencies = [ [[package]] name = "parquet" -version = "58.3.0" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908" +checksum = "970dff83e97d953c827ae8176f6bf4e9f77bf62daacc01ec5df348ec5eacd913" dependencies = [ "ahash", "arrow-array", @@ -4474,7 +4459,6 @@ dependencies = [ "seq-macro", "simdutf8", "snap", - "thrift", "tokio", "twox-hash", "zstd", @@ -4834,7 +4818,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ "heck", - "itertools 0.14.0", + "itertools 0.13.0", "log", "multimap", "petgraph", @@ -4853,7 +4837,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.117", @@ -6147,17 +6131,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "thrift" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" -dependencies = [ - "byteorder", - "integer-encoding", - "ordered-float", -] - [[package]] name = "time" version = "0.3.47" diff --git a/Cargo.toml b/Cargo.toml index 61b999d8184be..7e4c8169ceb00 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -89,30 +89,30 @@ version = "54.0.0" # # See for more details: https://github.com/rust-lang/cargo/issues/11329 apache-avro = { version = "0.21", default-features = false } -arrow = { version = "58.3.0", features = [ +arrow = { version = "59.0.0", features = [ "prettyprint", "chrono-tz", ] } -arrow-avro = { version = "58.3.0", default-features = false, features = [ +arrow-avro = { version = "59.0.0", default-features = false, features = [ "deflate", "snappy", "zstd", "bzip2", "xz", ] } -arrow-buffer = { version = "58.3.0", default-features = false } -arrow-data = { version = "58.3.0", default-features = false } -arrow-flight = { version = "58.3.0", features = [ +arrow-buffer = { version = "59.0.0", default-features = false } +arrow-data = { version = "59.0.0", default-features = false } +arrow-flight = { version = "59.0.0", features = [ "flight-sql-experimental", ] } # Both codecs are required here to make sure that code paths like # file-spilling have access to all compression codecs. -arrow-ipc = { version = "58.3.0", default-features = false, features = [ +arrow-ipc = { version = "59.0.0", default-features = false, features = [ "lz4", "zstd", ] } -arrow-ord = { version = "58.3.0", default-features = false } -arrow-schema = { version = "58.3.0", default-features = false } +arrow-ord = { version = "59.0.0", default-features = false } +arrow-schema = { version = "59.0.0", default-features = false } async-trait = "0.1.89" bigdecimal = "0.4.8" bytes = "1.11" @@ -178,7 +178,7 @@ memchr = "2.8.1" num-traits = { version = "0.2" } object_store = { version = "0.13.2", default-features = false } parking_lot = "0.12" -parquet = { version = "58.3.0", default-features = false, features = [ +parquet = { version = "59.0.0", default-features = false, features = [ "arrow", "async", "object_store", @@ -290,4 +290,4 @@ incremental = false debug = false debug-assertions = false strip = "debuginfo" -incremental = false +incremental = false \ No newline at end of file diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rs index 935bf0a9744dd..0d8ada1367826 100644 --- a/datafusion-cli/src/main.rs +++ b/datafusion-cli/src/main.rs @@ -613,9 +613,9 @@ mod tests { +-----------------------------------+-----------------+---------------------+------+------------------+ | filename | file_size_bytes | metadata_size_bytes | hits | extra | +-----------------------------------+-----------------+---------------------+------+------------------+ - | alltypes_plain.parquet | 1851 | 8882 | 2 | page_index=false | - | alltypes_tiny_pages.parquet | 454233 | 269074 | 2 | page_index=true | - | lz4_raw_compressed_larger.parquet | 380836 | 1339 | 2 | page_index=false | + | alltypes_plain.parquet | 1851 | 8794 | 2 | page_index=false | + | alltypes_tiny_pages.parquet | 454233 | 268970 | 2 | page_index=true | + | lz4_raw_compressed_larger.parquet | 380836 | 1331 | 2 | page_index=false | +-----------------------------------+-----------------+---------------------+------+------------------+ "); @@ -644,9 +644,9 @@ mod tests { +-----------------------------------+-----------------+---------------------+------+------------------+ | filename | file_size_bytes | metadata_size_bytes | hits | extra | +-----------------------------------+-----------------+---------------------+------+------------------+ - | alltypes_plain.parquet | 1851 | 8882 | 5 | page_index=false | - | alltypes_tiny_pages.parquet | 454233 | 269074 | 2 | page_index=true | - | lz4_raw_compressed_larger.parquet | 380836 | 1339 | 3 | page_index=false | + | alltypes_plain.parquet | 1851 | 8794 | 5 | page_index=false | + | alltypes_tiny_pages.parquet | 454233 | 268970 | 2 | page_index=true | + | lz4_raw_compressed_larger.parquet | 380836 | 1331 | 3 | page_index=false | +-----------------------------------+-----------------+---------------------+------+------------------+ "); diff --git a/datafusion/common/src/file_options/mod.rs b/datafusion/common/src/file_options/mod.rs index 5d2abd23172ed..97b4a44f03223 100644 --- a/datafusion/common/src/file_options/mod.rs +++ b/datafusion/common/src/file_options/mod.rs @@ -114,14 +114,14 @@ mod tests { properties .bloom_filter_properties(&ColumnPath::from("")) .expect("expected bloom properties!") - .fpp, + .fpp(), 0.123 ); assert_eq!( properties .bloom_filter_properties(&ColumnPath::from("")) .expect("expected bloom properties!") - .ndv, + .ndv(), 123 ); @@ -242,7 +242,7 @@ mod tests { properties .bloom_filter_properties(&col1) .expect("expected bloom properties!") - .fpp, + .fpp(), 0.123 ); @@ -250,7 +250,7 @@ mod tests { properties .bloom_filter_properties(&col2_nested) .expect("expected bloom properties!") - .fpp, + .fpp(), 0.456 ); @@ -258,7 +258,7 @@ mod tests { properties .bloom_filter_properties(&col1) .expect("expected bloom properties!") - .ndv, + .ndv(), 123 ); @@ -266,7 +266,7 @@ mod tests { properties .bloom_filter_properties(&col2_nested) .expect("expected bloom properties!") - .ndv, + .ndv(), 456 ); diff --git a/datafusion/common/src/file_options/parquet_writer.rs b/datafusion/common/src/file_options/parquet_writer.rs index a5b270a8f57b6..320bfcf33e488 100644 --- a/datafusion/common/src/file_options/parquet_writer.rs +++ b/datafusion/common/src/file_options/parquet_writer.rs @@ -157,8 +157,8 @@ impl TryFrom<&TableParquetOptions> for WriterPropertiesBuilder { } if let Some(bloom_filter_ndv) = options.bloom_filter_ndv { - builder = - builder.set_column_bloom_filter_ndv(path.clone(), bloom_filter_ndv); + builder = builder + .set_column_bloom_filter_max_ndv(path.clone(), bloom_filter_ndv); } } @@ -273,7 +273,7 @@ impl ParquetOptions { builder = builder.set_bloom_filter_fpp(*bloom_filter_fpp); }; if let Some(bloom_filter_ndv) = bloom_filter_ndv { - builder = builder.set_bloom_filter_ndv(*bloom_filter_ndv); + builder = builder.set_bloom_filter_max_ndv(*bloom_filter_ndv); }; if let Some(dictionary_enabled) = dictionary_enabled { builder = builder.set_dictionary_enabled(*dictionary_enabled); @@ -534,8 +534,8 @@ mod tests { } .into(), ), - bloom_filter_fpp: bloom_filter_default_props.map(|p| p.fpp), - bloom_filter_ndv: bloom_filter_default_props.map(|p| p.ndv), + bloom_filter_fpp: bloom_filter_default_props.map(|p| p.fpp()), + bloom_filter_ndv: bloom_filter_default_props.map(|p| p.ndv()), } } @@ -830,10 +830,12 @@ mod tests { ); assert_eq!( default_writer_props.bloom_filter_properties(&"default".into()), - Some(&BloomFilterProperties { - fpp: 0.42, - ndv: DEFAULT_BLOOM_FILTER_NDV - }), + Some( + &BloomFilterProperties::builder() + .with_fpp(0.42) + .with_max_ndv(DEFAULT_BLOOM_FILTER_NDV) + .build() + ), "should have only the fpp set, and the ndv at default", ); } @@ -937,7 +939,7 @@ mod tests { // the WriterProperties::default, with only ndv set let default_writer_props = WriterProperties::builder() .set_bloom_filter_enabled(true) - .set_bloom_filter_ndv(42) + .set_bloom_filter_max_ndv(42) .build(); assert_eq!( @@ -947,10 +949,12 @@ mod tests { ); assert_eq!( default_writer_props.bloom_filter_properties(&"default".into()), - Some(&BloomFilterProperties { - fpp: DEFAULT_BLOOM_FILTER_FPP, - ndv: 42 - }), + Some( + &BloomFilterProperties::builder() + .with_fpp(DEFAULT_BLOOM_FILTER_FPP) + .with_max_ndv(42) + .build() + ), "should have only the ndv set, and the fpp at default", ); } diff --git a/datafusion/core/tests/datasource/object_store_access.rs b/datafusion/core/tests/datasource/object_store_access.rs index 83b84f6f9284e..25150ae284cc0 100644 --- a/datafusion/core/tests/datasource/object_store_access.rs +++ b/datafusion/core/tests/datasource/object_store_access.rs @@ -904,7 +904,7 @@ async fn query_single_parquet_file_with_single_predicate() { RequestCountingObjectStore() Total Requests: 2 - GET (opts) path=parquet_table.parquet head=true - - GET (ranges) path=parquet_table.parquet ranges=1064-1481,1481-1594,1594-2011,2011-2124 + - GET (ranges) path=parquet_table.parquet ranges=1064-1594,1594-2124 " ); } @@ -928,8 +928,8 @@ async fn query_single_parquet_file_multi_row_groups_multiple_predicates() { RequestCountingObjectStore() Total Requests: 3 - GET (opts) path=parquet_table.parquet head=true - - GET (ranges) path=parquet_table.parquet ranges=4-421,421-534,534-951,951-1064 - - GET (ranges) path=parquet_table.parquet ranges=1064-1481,1481-1594,1594-2011,2011-2124 + - GET (ranges) path=parquet_table.parquet ranges=4-534,534-1064 + - GET (ranges) path=parquet_table.parquet ranges=1064-1594,1594-2124 " ); } diff --git a/datafusion/core/tests/extension_types/pretty_printing.rs b/datafusion/core/tests/extension_types/pretty_printing.rs index c0796887b8b6e..f097b5bec97fc 100644 --- a/datafusion/core/tests/extension_types/pretty_printing.rs +++ b/datafusion/core/tests/extension_types/pretty_printing.rs @@ -40,10 +40,16 @@ async fn create_test_table() -> Result { // define data. let batch = RecordBatch::try_new( schema, - vec![Arc::new(FixedSizeBinaryArray::from(vec![ - &[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 5, 6], - ]))], + vec![Arc::new( + FixedSizeBinaryArray::try_from_iter( + vec![ + &[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 5, 6], + ] + .into_iter(), + ) + .unwrap(), + )], )?; let state = SessionStateBuilder::default() diff --git a/datafusion/core/tests/parquet/mod.rs b/datafusion/core/tests/parquet/mod.rs index 0e936a79ebe9f..12296f8498d9f 100644 --- a/datafusion/core/tests/parquet/mod.rs +++ b/datafusion/core/tests/parquet/mod.rs @@ -726,11 +726,11 @@ fn make_bytearray_batch( let name: StringArray = std::iter::repeat_n(Some(name), num_rows).collect(); let service_string: StringArray = string_values.iter().map(Some).collect(); let service_binary: BinaryArray = binary_values.iter().map(Some).collect(); - let service_fixedsize: FixedSizeBinaryArray = fixedsize_values - .iter() - .map(|value| Some(value.as_slice())) - .collect::>() - .into(); + let service_fixedsize = FixedSizeBinaryArray::try_from_sparse_iter_with_size( + fixedsize_values.iter().map(|value| Some(value.as_slice())), + 3, + ) + .unwrap(); let service_large_binary: LargeBinaryArray = large_binary_values.iter().map(Some).collect(); diff --git a/datafusion/datasource-parquet/src/row_group_filter.rs b/datafusion/datasource-parquet/src/row_group_filter.rs index 07f4fe92cf308..1e9b0636e59e9 100644 --- a/datafusion/datasource-parquet/src/row_group_filter.rs +++ b/datafusion/datasource-parquet/src/row_group_filter.rs @@ -955,10 +955,7 @@ mod tests { let schema = Arc::new(Schema::new(vec![Field::new("c1", Decimal128(9, 2), false)])); let field = PrimitiveTypeField::new("c1", PhysicalType::INT32) - .with_logical_type(LogicalType::Decimal { - scale: 2, - precision: 9, - }) + .with_logical_type(LogicalType::decimal(2, 9)) .with_scale(2) .with_precision(9); let schema_descr = get_test_schema_descr(vec![field]); @@ -1023,10 +1020,7 @@ mod tests { Arc::new(Schema::new(vec![Field::new("c1", Decimal128(9, 0), false)])); let field = PrimitiveTypeField::new("c1", PhysicalType::INT32) - .with_logical_type(LogicalType::Decimal { - scale: 0, - precision: 9, - }) + .with_logical_type(LogicalType::decimal(0, 9)) .with_scale(0) .with_precision(9); let schema_descr = get_test_schema_descr(vec![field]); @@ -1118,10 +1112,7 @@ mod tests { false, )])); let field = PrimitiveTypeField::new("c1", PhysicalType::INT64) - .with_logical_type(LogicalType::Decimal { - scale: 2, - precision: 18, - }) + .with_logical_type(LogicalType::decimal(2, 18)) .with_scale(2) .with_precision(18); let schema_descr = get_test_schema_descr(vec![field]); @@ -1176,10 +1167,7 @@ mod tests { false, )])); let field = PrimitiveTypeField::new("c1", PhysicalType::FIXED_LEN_BYTE_ARRAY) - .with_logical_type(LogicalType::Decimal { - scale: 2, - precision: 18, - }) + .with_logical_type(LogicalType::decimal(2, 18)) .with_scale(2) .with_precision(18) .with_byte_len(16); @@ -1254,10 +1242,7 @@ mod tests { false, )])); let field = PrimitiveTypeField::new("c1", PhysicalType::BYTE_ARRAY) - .with_logical_type(LogicalType::Decimal { - scale: 2, - precision: 18, - }) + .with_logical_type(LogicalType::decimal(2, 18)) .with_scale(2) .with_precision(18) .with_byte_len(16); diff --git a/datafusion/physical-expr-common/src/utils.rs b/datafusion/physical-expr-common/src/utils.rs index e469885f83316..117da23df2f3e 100644 --- a/datafusion/physical-expr-common/src/utils.rs +++ b/datafusion/physical-expr-common/src/utils.rs @@ -614,11 +614,9 @@ mod tests { #[test] fn scatter_fixed_size_binary_test() -> Result<()> { - let truthy = Arc::new(FixedSizeBinaryArray::from(vec![ - &[1u8, 2][..], - &[3, 4][..], - &[5, 6][..], - ])); + let truthy = Arc::new(FixedSizeBinaryArray::try_from_iter( + vec![&[1u8, 2][..], &[3, 4][..], &[5, 6][..]].into_iter(), + )?); let mask = BooleanArray::from(vec![true, false, true, false, true]); let result = scatter(&mask, truthy.as_ref())?; diff --git a/datafusion/physical-expr/benches/in_list_strategy.rs b/datafusion/physical-expr/benches/in_list_strategy.rs index 5c4922fdcf8a9..3eff1f5cf3dff 100644 --- a/datafusion/physical-expr/benches/in_list_strategy.rs +++ b/datafusion/physical-expr/benches/in_list_strategy.rs @@ -993,7 +993,7 @@ fn bench_fixed_size_binary_inner( .collect(); let refs: Vec<&[u8]> = values.iter().map(|v| v.as_slice()).collect(); - let array = FixedSizeBinaryArray::from(refs); + let array = FixedSizeBinaryArray::try_from_iter(refs.into_iter()).unwrap(); let schema = Schema::new(vec![Field::new("a", array.data_type().clone(), true)]); let exprs: Vec<_> = haystack diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs index c4377b3189ff7..b1fdf3ddabb5a 100644 --- a/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs +++ b/datafusion/physical-plan/src/joins/sort_merge_join/tests.rs @@ -175,7 +175,7 @@ fn build_fixed_size_binary_table( let batch = RecordBatch::try_new( Arc::new(schema), vec![ - Arc::new(FixedSizeBinaryArray::from(a.1.clone())), + Arc::new(FixedSizeBinaryArray::try_from_iter(a.1.iter().copied()).unwrap()), Arc::new(Int32Array::from(b.1.clone())), Arc::new(Int32Array::from(c.1.clone())), ], diff --git a/datafusion/spark/src/function/hash/xxhash64.rs b/datafusion/spark/src/function/hash/xxhash64.rs index 5dca47bcb8984..9d02a51b2217e 100644 --- a/datafusion/spark/src/function/hash/xxhash64.rs +++ b/datafusion/spark/src/function/hash/xxhash64.rs @@ -363,12 +363,17 @@ mod tests { #[test] fn test_xxhash64_fixed_size_binary() { - let array = FixedSizeBinaryArray::from(vec![ - Some(&[0x01, 0x02, 0x03, 0x04][..]), - Some(&[0x05, 0x06, 0x07, 0x08][..]), - None, - Some(&[0x00, 0x00, 0x00, 0x00][..]), - ]); + let array = FixedSizeBinaryArray::try_from_sparse_iter_with_size( + vec![ + Some(&[0x01, 0x02, 0x03, 0x04][..]), + Some(&[0x05, 0x06, 0x07, 0x08][..]), + None, + Some(&[0x00, 0x00, 0x00, 0x00][..]), + ] + .into_iter(), + 4, + ) + .unwrap(); let array_ref: ArrayRef = Arc::new(array); let mut hashes = vec![DEFAULT_SEED; 4];