andygrove commented on code in PR #68:
URL: https://github.com/apache/datafusion-spark/pull/68#discussion_r4604029021
##########
crates/spark/src/function/hash/murmur3.rs:
##########
@@ -0,0 +1,359 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::array::{ArrayRef, Int32Array, types::ArrowDictionaryKeyType};
+use arrow::buffer::{Buffer, ScalarBuffer};
+use arrow::datatypes::{DataType, Field, FieldRef};
+use datafusion_common::{Result, ScalarValue, internal_err};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs,
ScalarUDFImpl,
+ Signature, Volatility,
+};
+use datafusion_macros::user_doc;
+
+use super::utils::hash_dictionary_values;
+use crate::create_hashes_internal;
+
+const DEFAULT_SEED: u32 = 42;
+
+/// Spark-compatible `hash` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#hash>
+#[user_doc(
+ doc_section(label = "Hashing Functions"),
+ description = "Returns the 32-bit Murmur3 hash (seed 42) of the arguments
as an INT, matching Spark's `hash`. \
+ DataFusion has no built-in equivalent. Arguments are hashed in order,
and each argument's hash seeds the next, \
+ so `hash(a, b)` equals `hash(named_struct('a', a, 'b', b))` and
argument order matters. \
+ NULL arguments, NULL list elements and NULL struct fields are skipped,
so the result is never NULL: \
+ `hash(NULL)` returns 42. The result depends on the input type, as in
Spark: INT hashes 4 bytes and BIGINT 8, \
+ and integer literals are BIGINT in DataFusion but INT in Spark, so
cast them to INT to match Spark's results. \
+ `-0.0` hashes like `0.0`, and every NaN hashes like the canonical NaN.
Dates hash as INT days and \
+ timestamps as BIGINT microseconds, whatever the Arrow unit
(DataFusion's TIMESTAMP literals are \
+ nanoseconds, floored to microseconds). Decimals with precision up to
18 hash as their unscaled long, and wider decimals hash \
+ the minimal big-endian two's complement bytes of the unscaled value.
Lists hash each element in order \
+ (nested lists flatten into the same chain, and an empty list leaves
the seed unchanged), structs hash each field, \
+ and maps hash each key then its value. This crate hashes maps as if
Spark's `spark.sql.legacy.allowHashOnMapType` \
+ were set, where Spark 4 rejects them by default (see
https://github.com/apache/datafusion-spark/issues/69). \
+ Dictionary-encoded values hash like their decoded values. At least one
argument is required.",
+ syntax_example = "hash(expr1[, ..., exprN])",
+ sql_example = r#"```sql
+> SELECT hash('Spark', [CAST(123 AS INT)], CAST(2 AS INT));
+-1321691492
+> SELECT hash(CAST(1 AS INT));
+-559580957
+> SELECT hash(1);
+-1712319331
+> SELECT hash(NULL);
+42
+```"#,
+ argument(
+ name = "expr1",
+ description = "Expression of any type to hash. NULL is skipped."
+ ),
+ argument(
+ name = "exprN",
+ description = "Further expressions, hashed in order with the previous
hash as the seed."
+ )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkHash {
+ signature: Signature,
+}
+
+impl Default for SparkHash {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkHash {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::variadic_any(Volatility::Immutable),
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkHash {
+ fn name(&self) -> &str {
+ "hash"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+ internal_err!("return_field_from_args should be used instead")
+ }
+
+ fn return_field_from_args(&self, _args: ReturnFieldArgs) ->
Result<FieldRef> {
+ // Spark's HashExpression overrides nullable to false: NULL inputs are
+ // skipped and the seed is used, so the result is never null.
+ Ok(Arc::new(Field::new(self.name(), DataType::Int32, false)))
+ }
+
+ fn documentation(&self) -> Option<&Documentation> {
+ self.doc()
+ }
+
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) ->
Result<ColumnarValue> {
+ // When every argument is a scalar the result is the same for every
row,
+ // so hash a single row and return a scalar. Otherwise expand scalars
to
+ // the batch length so every array has `number_rows` entries.
+ let all_scalars = args
+ .args
+ .iter()
+ .all(|arg| matches!(arg, ColumnarValue::Scalar(_)));
+ let num_rows = if all_scalars { 1 } else { args.number_rows };
+ let arrays = args
+ .args
+ .iter()
+ .map(|arg| arg.to_array(num_rows))
+ .collect::<Result<Vec<_>>>()?;
+
+ let mut hashes: Vec<u32> = vec![DEFAULT_SEED; num_rows];
+ create_murmur3_hashes(&arrays, &mut hashes)?;
+
+ if all_scalars {
+ Ok(ColumnarValue::Scalar(ScalarValue::Int32(Some(
+ hashes[0] as i32,
+ ))))
+ } else {
+ // Reinterpret Vec<u32> as ScalarBuffer<i32> without copying.
+ let buffer = ScalarBuffer::<i32>::from(Buffer::from_vec(hashes));
+ Ok(ColumnarValue::Array(Arc::new(Int32Array::new(
+ buffer, None,
+ ))))
+ }
+ }
+}
+
+#[inline]
+fn mix_k1(k1: u32) -> u32 {
+ k1.wrapping_mul(0xcc9e2d51)
+ .rotate_left(15)
+ .wrapping_mul(0x1b873593)
+}
+
+#[inline]
+fn mix_h1(h1: u32, k1: u32) -> u32 {
+ (h1 ^ k1)
+ .rotate_left(13)
+ .wrapping_mul(5)
+ .wrapping_add(0xe6546b64)
+}
+
+#[inline]
+fn fmix(mut h1: u32, len: u32) -> u32 {
+ h1 ^= len;
+ h1 ^= h1 >> 16;
+ h1 = h1.wrapping_mul(0x85ebca6b);
+ h1 ^= h1 >> 13;
+ h1 = h1.wrapping_mul(0xc2b2ae35);
+ h1 ^= h1 >> 16;
+ h1
+}
+
+/// Spark's `Murmur3_x86_32.hashUnsafeBytes`: whole 4-byte little-endian words
+/// first, then each remaining tail byte is mixed separately as a sign-extended
+/// word (this differs from the canonical MurmurHash3 tail handling).
+#[inline]
+fn spark_compatible_murmur3_hash<T: AsRef<[u8]>>(data: T, seed: u32) -> u32 {
+ let data = data.as_ref();
+ let (words, tail) = data.as_chunks::<4>();
+ let mut h1 = seed;
+ for word in words {
+ h1 = mix_h1(h1, mix_k1(u32::from_le_bytes(*word)));
+ }
+ for &byte in tail {
+ h1 = mix_h1(h1, mix_k1(byte as i8 as i32 as u32));
+ }
+ fmix(h1, data.len() as u32)
+}
+
+/// Hash the values in a dictionary array using murmur3.
+fn create_murmur3_hashes_dictionary<K: ArrowDictionaryKeyType>(
+ array: &ArrayRef,
+ hashes_buffer: &mut [u32],
+ seeds_are_pristine: bool,
+) -> Result<()> {
+ hash_dictionary_values::<K, u32>(
+ array,
+ hashes_buffer,
+ seeds_are_pristine,
+ create_murmur3_hashes,
+ )
+}
+
+/// Create murmur3 hash values for every row, based on the values in the
columns.
+///
+/// The number of rows to hash is determined by `hashes_buffer.len()`.
+/// `hashes_buffer` should be pre-sized appropriately and seeded with the
+/// initial hash value (Spark uses `42`).
+fn create_murmur3_hashes(arrays: &[ArrayRef], hashes_buffer: &mut [u32]) ->
Result<()> {
+ create_hashes_internal!(
+ arrays,
+ hashes_buffer,
+ spark_compatible_murmur3_hash,
+ create_murmur3_hashes_dictionary,
+ create_murmur3_hashes
+ );
+ Ok(())
+}
Review Comment:
Done in e395b27. Ported Comet's arms for `Time64(Nanosecond)`,
`Interval(YearMonth)`, `Duration(Microsecond)` and `Interval(MonthDayNano)`,
with Comet's note on leaving out the days. `hash(TIME '12:00:00')` is
-1984942870, and a `Duration(Microsecond)` of one day is 892267096. DataFusion
reads `INTERVAL '1' DAY` as a calendar interval, so the slt builds the Duration
with `arrow_cast`. The arms are shared, so `xxhash64` now hashes these types
too, and its values match Spark 4.2.0. One thing I found: Spark 4.2.0's
interpreted hash includes the days of a calendar interval, while the generated
code doesn't. Over `range()`, `hash(make_interval(0, 0, 0, d))` is 1081828689
for both 0 and 1, which is what this PR returns. A literal or a `VALUES` row is
constant-folded through the interpreted path, so Spark gives 1954791903 and
-1328321721 there. The slt case notes this and is marked `skipif spark`.
##########
crates/spark/src/function/hash/murmur3.rs:
##########
@@ -0,0 +1,359 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::array::{ArrayRef, Int32Array, types::ArrowDictionaryKeyType};
+use arrow::buffer::{Buffer, ScalarBuffer};
+use arrow::datatypes::{DataType, Field, FieldRef};
+use datafusion_common::{Result, ScalarValue, internal_err};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs,
ScalarUDFImpl,
+ Signature, Volatility,
+};
+use datafusion_macros::user_doc;
+
+use super::utils::hash_dictionary_values;
+use crate::create_hashes_internal;
+
+const DEFAULT_SEED: u32 = 42;
+
+/// Spark-compatible `hash` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#hash>
+#[user_doc(
+ doc_section(label = "Hashing Functions"),
+ description = "Returns the 32-bit Murmur3 hash (seed 42) of the arguments
as an INT, matching Spark's `hash`. \
+ DataFusion has no built-in equivalent. Arguments are hashed in order,
and each argument's hash seeds the next, \
+ so `hash(a, b)` equals `hash(named_struct('a', a, 'b', b))` and
argument order matters. \
+ NULL arguments, NULL list elements and NULL struct fields are skipped,
so the result is never NULL: \
+ `hash(NULL)` returns 42. The result depends on the input type, as in
Spark: INT hashes 4 bytes and BIGINT 8, \
+ and integer literals are BIGINT in DataFusion but INT in Spark, so
cast them to INT to match Spark's results. \
+ `-0.0` hashes like `0.0`, and every NaN hashes like the canonical NaN.
Dates hash as INT days and \
+ timestamps as BIGINT microseconds, whatever the Arrow unit
(DataFusion's TIMESTAMP literals are \
+ nanoseconds, floored to microseconds). Decimals with precision up to
18 hash as their unscaled long, and wider decimals hash \
+ the minimal big-endian two's complement bytes of the unscaled value.
Lists hash each element in order \
+ (nested lists flatten into the same chain, and an empty list leaves
the seed unchanged), structs hash each field, \
+ and maps hash each key then its value. This crate hashes maps as if
Spark's `spark.sql.legacy.allowHashOnMapType` \
+ were set, where Spark 4 rejects them by default (see
https://github.com/apache/datafusion-spark/issues/69). \
+ Dictionary-encoded values hash like their decoded values. At least one
argument is required.",
+ syntax_example = "hash(expr1[, ..., exprN])",
+ sql_example = r#"```sql
+> SELECT hash('Spark', [CAST(123 AS INT)], CAST(2 AS INT));
+-1321691492
+> SELECT hash(CAST(1 AS INT));
+-559580957
+> SELECT hash(1);
+-1712319331
+> SELECT hash(NULL);
+42
+```"#,
+ argument(
+ name = "expr1",
+ description = "Expression of any type to hash. NULL is skipped."
+ ),
+ argument(
+ name = "exprN",
+ description = "Further expressions, hashed in order with the previous
hash as the seed."
+ )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkHash {
+ signature: Signature,
+}
+
+impl Default for SparkHash {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkHash {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::variadic_any(Volatility::Immutable),
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkHash {
+ fn name(&self) -> &str {
+ "hash"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+ internal_err!("return_field_from_args should be used instead")
+ }
+
+ fn return_field_from_args(&self, _args: ReturnFieldArgs) ->
Result<FieldRef> {
+ // Spark's HashExpression overrides nullable to false: NULL inputs are
+ // skipped and the seed is used, so the result is never null.
+ Ok(Arc::new(Field::new(self.name(), DataType::Int32, false)))
+ }
+
+ fn documentation(&self) -> Option<&Documentation> {
+ self.doc()
+ }
+
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) ->
Result<ColumnarValue> {
+ // When every argument is a scalar the result is the same for every
row,
+ // so hash a single row and return a scalar. Otherwise expand scalars
to
+ // the batch length so every array has `number_rows` entries.
+ let all_scalars = args
+ .args
+ .iter()
+ .all(|arg| matches!(arg, ColumnarValue::Scalar(_)));
+ let num_rows = if all_scalars { 1 } else { args.number_rows };
+ let arrays = args
+ .args
+ .iter()
+ .map(|arg| arg.to_array(num_rows))
+ .collect::<Result<Vec<_>>>()?;
+
+ let mut hashes: Vec<u32> = vec![DEFAULT_SEED; num_rows];
+ create_murmur3_hashes(&arrays, &mut hashes)?;
+
+ if all_scalars {
+ Ok(ColumnarValue::Scalar(ScalarValue::Int32(Some(
+ hashes[0] as i32,
+ ))))
+ } else {
+ // Reinterpret Vec<u32> as ScalarBuffer<i32> without copying.
+ let buffer = ScalarBuffer::<i32>::from(Buffer::from_vec(hashes));
+ Ok(ColumnarValue::Array(Arc::new(Int32Array::new(
+ buffer, None,
+ ))))
+ }
+ }
+}
+
+#[inline]
+fn mix_k1(k1: u32) -> u32 {
+ k1.wrapping_mul(0xcc9e2d51)
+ .rotate_left(15)
+ .wrapping_mul(0x1b873593)
+}
+
+#[inline]
+fn mix_h1(h1: u32, k1: u32) -> u32 {
+ (h1 ^ k1)
+ .rotate_left(13)
+ .wrapping_mul(5)
+ .wrapping_add(0xe6546b64)
+}
+
+#[inline]
+fn fmix(mut h1: u32, len: u32) -> u32 {
+ h1 ^= len;
+ h1 ^= h1 >> 16;
+ h1 = h1.wrapping_mul(0x85ebca6b);
+ h1 ^= h1 >> 13;
+ h1 = h1.wrapping_mul(0xc2b2ae35);
+ h1 ^= h1 >> 16;
+ h1
+}
+
+/// Spark's `Murmur3_x86_32.hashUnsafeBytes`: whole 4-byte little-endian words
+/// first, then each remaining tail byte is mixed separately as a sign-extended
+/// word (this differs from the canonical MurmurHash3 tail handling).
+#[inline]
+fn spark_compatible_murmur3_hash<T: AsRef<[u8]>>(data: T, seed: u32) -> u32 {
+ let data = data.as_ref();
+ let (words, tail) = data.as_chunks::<4>();
+ let mut h1 = seed;
+ for word in words {
+ h1 = mix_h1(h1, mix_k1(u32::from_le_bytes(*word)));
+ }
+ for &byte in tail {
+ h1 = mix_h1(h1, mix_k1(byte as i8 as i32 as u32));
+ }
+ fmix(h1, data.len() as u32)
+}
+
+/// Hash the values in a dictionary array using murmur3.
+fn create_murmur3_hashes_dictionary<K: ArrowDictionaryKeyType>(
+ array: &ArrayRef,
+ hashes_buffer: &mut [u32],
+ seeds_are_pristine: bool,
+) -> Result<()> {
+ hash_dictionary_values::<K, u32>(
+ array,
+ hashes_buffer,
+ seeds_are_pristine,
+ create_murmur3_hashes,
+ )
+}
+
+/// Create murmur3 hash values for every row, based on the values in the
columns.
+///
+/// The number of rows to hash is determined by `hashes_buffer.len()`.
+/// `hashes_buffer` should be pre-sized appropriately and seeded with the
+/// initial hash value (Spark uses `42`).
+fn create_murmur3_hashes(arrays: &[ArrayRef], hashes_buffer: &mut [u32]) ->
Result<()> {
+ create_hashes_internal!(
+ arrays,
+ hashes_buffer,
+ spark_compatible_murmur3_hash,
+ create_murmur3_hashes_dictionary,
+ create_murmur3_hashes
+ );
+ Ok(())
+}
+
+#[cfg(test)]
Review Comment:
Ported 27 of the 29 in e395b27, in a `tests::comet` module, with their
expected values unchanged. The two left out test Comet's gather-eligibility
selection, which this crate doesn't have. All 27 pass, including
`test_ansi_intervals_hash_like_int_and_long` and
`test_lists_of_null_type_and_intervals`.
##########
crates/sqllogictest/test_files/hash/hash.slt:
##########
@@ -0,0 +1,586 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under
one or more contributor license agreements
+
+# Tests for Spark-compatible hash (32-bit Murmur3, seed 42)
+# Reference: https://spark.apache.org/docs/latest/api/sql/index.html#hash
+# Expected values were checked against Spark 4.2.0 (PySpark).
+
+# Int32
+
+query I
+SELECT hash(arrow_cast(1, 'Int32'));
+----
+-559580957
+
+query I
+SELECT hash(arrow_cast(0, 'Int32'));
+----
+933211791
+
+query I
+SELECT hash(arrow_cast(-1, 'Int32'));
+----
+-1604776387
+
+query I
+SELECT hash(arrow_cast(2147483647, 'Int32'));
+----
+133916647
+
+query I
+SELECT hash(arrow_cast(-2147483648, 'Int32'));
+----
+723455942
+
+# Int64
+
+query I
+SELECT hash(arrow_cast(1, 'Int64'));
+----
+-1712319331
+
+query I
+SELECT hash(arrow_cast(0, 'Int64'));
+----
+-1670924195
+
+query I
+SELECT hash(arrow_cast(-1, 'Int64'));
+----
+-939490007
+
+query I
+SELECT hash(arrow_cast(9223372036854775807, 'Int64'));
+----
+-1604625029
+
+query I
+SELECT hash(arrow_cast(-9223372036854775808, 'Int64'));
+----
+-853646085
+
+# Float64: -0.0 hashes like 0.0
+
+query I
+SELECT hash(arrow_cast(1.0, 'Float64'));
+----
+-460888942
+
+query I
+SELECT hash(arrow_cast(0.0, 'Float64'));
+----
+-1670924195
+
+query I
+SELECT hash(arrow_cast(-0.0, 'Float64'));
+----
+-1670924195
+
+query I
+SELECT hash(arrow_cast(-1.0, 'Float64'));
+----
+332927831
+
+# Float32: -0.0 hashes like 0.0
+
+query I
+SELECT hash(arrow_cast(1.0, 'Float32'));
+----
+-466301895
+
+query I
+SELECT hash(arrow_cast(0.0, 'Float32'));
+----
+933211791
+
+query I
+SELECT hash(arrow_cast(-0.0, 'Float32'));
+----
+933211791
+
+query I
+SELECT hash(arrow_cast(-1.0, 'Float32'));
+----
+-603084565
+
+# Strings: every tail length, multi-byte UTF-8
+
+query I
+SELECT hash('hello');
+----
+-1008564952
+
+query I
+SELECT hash('bar');
+----
+-1808790533
+
+query I
+SELECT hash('');
+----
+142593372
+
+query I
+SELECT hash('a');
+----
+1485273170
+
+query I
+SELECT hash('ab');
+----
+-97053317
+
+query I
+SELECT hash('abc');
+----
+1322437556
+
+query I
+SELECT hash('abcd');
+----
+-396302900
+
+query I
+SELECT hash('abcde');
+----
+814637928
+
+query I
+SELECT hash('😁');
+----
+885025535
+
+query I
+SELECT hash('天地');
+----
+-1899966402
+
+# NULL leaves the seed (42) unchanged; the result is never NULL
+query I
+SELECT hash(NULL);
+----
+42
+
+query I
+SELECT hash(arrow_cast(NULL, 'Int32'));
+----
+42
+
+# NULL arguments are skipped when hashing several columns
+query B
+SELECT hash(1, NULL) = hash(1);
+----
+true
+
+query B
+SELECT hash(NULL, 1) = hash(1);
+----
+true
+
+# Arguments chain: the hash of the previous argument seeds the next
+query B
+SELECT hash(1, 2) = hash(2, 1);
+----
+false
+
+# Spark's documented example; DataFusion types 123 and 2 as BIGINT, Spark as
INT
+query I
+SELECT hash('Spark', [CAST(123 AS INT)], CAST(2 AS INT));
+----
+-1321691492
+
+query I
+SELECT hash('Spark', [CAST(123 AS INT)]);
+----
+1262264271
+
+# Hashing several arguments equals hashing a struct of them
+query I
+SELECT hash(CAST(1 AS INT), 'x');
+----
+-1305545736
+
+# Decimals with precision <= 18 hash as the unscaled long
+
+query I
+SELECT hash(CAST(123.45 AS DECIMAL(10, 2)));
+----
+1416086240
+
+query I
+SELECT hash(CAST(-123.45 AS DECIMAL(10, 2)));
+----
+-1959512858
+
+# Decimals with precision > 18 hash the minimal big-endian two's complement
+# bytes of the unscaled value (Java's BigInteger.toByteArray)
+
+query I
+SELECT hash(CAST(1 AS DECIMAL(20, 0)));
+----
+-386724586
+
+query I
+SELECT hash(CAST(0 AS DECIMAL(20, 0)));
+----
+-783713497
+
+query I
+SELECT hash(CAST(-1 AS DECIMAL(20, 0)));
+----
+1398487324
+
+# 127 and -128 fit in one byte; 128 and -129 need a leading sign byte
+query I
+SELECT hash(CAST(127 AS DECIMAL(20, 0)));
+----
+1185089389
+
+query I
+SELECT hash(CAST(128 AS DECIMAL(20, 0)));
+----
+-544401882
+
+query I
+SELECT hash(CAST(-128 AS DECIMAL(20, 0)));
+----
+775851899
+
+query I
+SELECT hash(CAST(-129 AS DECIMAL(20, 0)));
+----
+-771458971
+
+query I
+SELECT hash(CAST(1.5 AS DECIMAL(20, 1)));
+----
+-357711666
+
+query I
+SELECT hash(CAST('12345678901234567890.1234567890' AS DECIMAL(38, 10)));
+----
+269448356
+
+query I
+SELECT hash(CAST('-12345678901234567890.1234567890' AS DECIMAL(38, 10)));
+----
+694017569
+
+query I
+SELECT hash(CAST('99999999999999999999999999999999999999' AS DECIMAL(38, 0)));
+----
+-817514053
+
+query I
+SELECT hash(CAST('-99999999999999999999999999999999999999' AS DECIMAL(38, 0)));
+----
+1400911110
+
+query I
+SELECT hash(CAST(NULL AS DECIMAL(20, 0)));
+----
+42
+
+# Lists hash each element in order, chaining the running hash; NULL elements
+# are skipped and an empty or NULL list leaves the seed unchanged
+
+query I
+SELECT hash([CAST(1 AS INT), CAST(2 AS INT), CAST(3 AS INT)]);
+----
+-912918097
+
+query I
+SELECT hash([1, 2, 3]);
+----
+1045631400
+
+query I
+SELECT hash(['a', 'b']);
+----
+190734147
+
+query I
+SELECT hash([CAST(1 AS INT), NULL, CAST(3 AS INT)]);
+----
+1066699649
+
+query I
+SELECT hash(arrow_cast([], 'List(Int32)'));
+----
+42
+
+query I
+SELECT hash(arrow_cast(NULL, 'List(Int32)'));
+----
+42
+
+query I
+SELECT hash([CAST(1 AS DECIMAL(20, 0)), CAST(-129 AS DECIMAL(20, 0))]);
+----
+-265473650
+
+query I
+SELECT hash([CAST(123.45 AS DECIMAL(10, 2))]);
+----
+1416086240
+
+# Nested lists flatten into the same chain as a flat list
+query I
+SELECT hash([[CAST(1 AS INT)], [CAST(2 AS INT), CAST(3 AS INT)]]);
+----
+-912918097
+
+# Structs hash each field in order; NULL fields are skipped
+
+query I
+SELECT hash(named_struct('a', CAST(1 AS INT), 'b', 'x'));
+----
+-1305545736
+
+query I
+SELECT hash(named_struct('a', CAST(1 AS INT), 'b', CAST(NULL AS VARCHAR)));
+----
+-559580957
+
+query I
+SELECT hash(named_struct('a', CAST(1 AS DECIMAL(20, 0))));
+----
+-386724586
+
+# Maps hash each key then its value, entry by entry. Spark 4.2.0 rejects hash
on
+# maps by default (DATATYPE_MISMATCH.HASH_MAP_TYPE); these values are from
Spark
+# with spark.sql.legacy.allowHashOnMapType=true.
+
+query I
+SELECT hash(map([CAST(1 AS INT), CAST(2 AS INT)], ['a', 'b']));
+----
+-1706464220
+
+query I
+SELECT hash(map(['a'], [CAST(1 AS INT)]));
+----
+532252686
+
+# Dictionary-encoded values hash like their decoded values
+
+query I
+SELECT hash(arrow_cast('hello', 'Dictionary(Int32, Utf8)'));
+----
+-1008564952
+
+query I
+SELECT hash(arrow_cast(CAST(1 AS INT), 'Dictionary(Int8, Int32)'));
+----
+-559580957
+
+query I
+SELECT hash(arrow_cast(column1, 'Dictionary(Int32, Utf8)'))
+FROM VALUES ('hello'), ('bar'), (NULL), ('hello');
+----
+-1008564952
+-1808790533
+42
+-1008564952
+
+# A dictionary after another argument is seeded per row by the running hash
+query I
+SELECT hash(CAST(1 AS INT), arrow_cast(column1, 'Dictionary(Int32, Utf8)'))
+FROM VALUES ('x'), ('x');
+----
+-1305545736
+-1305545736
+
+# Booleans hash as INT 1 or 0
+
+query I
+SELECT hash(true);
+----
+-559580957
+
+query I
+SELECT hash(false);
+----
+933211791
+
+# TINYINT and SMALLINT widen to INT before hashing
+
+query I
+SELECT hash(CAST(1 AS TINYINT));
+----
+-559580957
+
+query I
+SELECT hash(CAST(0 AS TINYINT));
+----
+933211791
+
+query I
+SELECT hash(CAST(-1 AS TINYINT));
+----
+-1604776387
+
+query I
+SELECT hash(CAST(127 AS TINYINT));
+----
+1135925485
+
+query I
+SELECT hash(CAST(-128 AS TINYINT));
+----
+1110053733
+
+query I
+SELECT hash(CAST(1 AS SMALLINT));
+----
+-559580957
+
+query I
+SELECT hash(CAST(-1 AS SMALLINT));
+----
+-1604776387
+
+query I
+SELECT hash(CAST(32767 AS SMALLINT));
+----
+1249274084
+
+query I
+SELECT hash(CAST(-32768 AS SMALLINT));
+----
+-1871935946
+
+# LargeUtf8 and Utf8View hash like Utf8
+
+query I
+SELECT hash(arrow_cast('hello', 'LargeUtf8'));
+----
+-1008564952
+
+query I
+SELECT hash(arrow_cast('hello', 'Utf8View'));
+----
+-1008564952
+
+# Binary hashes its bytes
+
+query I
+SELECT hash(X'0102');
+----
+1964783052
+
+# Dates hash as INT days since the epoch; Date64 (milliseconds) is floored to
days
+
+query I
+SELECT hash(DATE '2020-01-01');
+----
+-725858435
+
+query I
+SELECT hash(arrow_cast(DATE '2020-01-01', 'Date64'));
+----
+-725858435
+
+query I
+SELECT hash(arrow_cast(-1, 'Date64'));
+----
+-1604776387
+
+# Timestamps hash as BIGINT microseconds since the epoch, whatever the Arrow
unit.
+# DataFusion's TIMESTAMP literals are nanoseconds. Spark values are with
+# spark.sql.session.timeZone=UTC.
+
+query I
+SELECT hash(TIMESTAMP '2020-01-01 00:00:00');
+----
+-1486022740
+
+query I
+SELECT hash(arrow_cast(TIMESTAMP '2020-01-01 00:00:00', 'Timestamp(Second,
None)'));
+----
+-1486022740
+
+query I
+SELECT hash(arrow_cast(TIMESTAMP '2020-01-01 00:00:00',
'Timestamp(Millisecond, None)'));
+----
+-1486022740
+
+query I
+SELECT hash(arrow_cast(TIMESTAMP '2020-01-01 00:00:00',
'Timestamp(Microsecond, None)'));
+----
+-1486022740
+
+query I
+SELECT hash(arrow_cast(TIMESTAMP '2020-01-01 00:00:00', 'Timestamp(Nanosecond,
Some("UTC"))'));
+----
+-1486022740
+
+query I
+SELECT hash([TIMESTAMP '2020-01-01 00:00:00']);
+----
+-1486022740
+
+# -1 ns floors to -1 us (Spark: timestamp_micros(-1)); -1 s is -1000000 us
+query I
+SELECT hash(arrow_cast(-1, 'Timestamp(Nanosecond, None)'));
+----
+-939490007
+
+query I
+SELECT hash(arrow_cast(-1, 'Timestamp(Second, None)'));
+----
+346627249
+
+# Every NaN hashes like the canonical NaN; negating NaN flips its sign bit
+
+query I
+SELECT hash(CAST('NaN' AS DOUBLE));
+----
+-1281358385
+
+query I
+SELECT hash(-CAST('NaN' AS DOUBLE));
+----
+-1281358385
+
+query I
+SELECT hash(CAST('NaN' AS FLOAT));
+----
+-349261430
+
+query I
+SELECT hash(-CAST('NaN' AS FLOAT));
+----
+-349261430
+
+query I
+SELECT hash([CAST('NaN' AS DOUBLE), -CAST('NaN' AS DOUBLE)]);
+----
+1050078426
+
+query I
+SELECT hash([-CAST('NaN' AS FLOAT)]);
+----
+-349261430
+
+# Table data (VALUES integers are Int64, which hash as 8 bytes)
+statement ok
+CREATE TABLE hash_t AS VALUES (1, 'hello'), (NULL, 'bar'), (0, NULL);
+
+query I
+SELECT hash(column1) FROM hash_t;
+----
+-1712319331
+42
+-1670924195
+
+query I
+SELECT hash(column2) FROM hash_t;
+----
+-1008564952
+-1808790533
+42
Review Comment:
Added all four in e395b27. They pass.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]