geoffreyclaude commented on code in PR #24088: URL: https://github.com/apache/datafusion/pull/24088#discussion_r3982186342
########## datafusion/physical-expr/src/expressions/in_list/byte_view_filter.rs: ########## @@ -0,0 +1,223 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Filters for `Utf8View` and `BinaryView` `IN` lists whose non-null values are +//! at most 12 bytes. +//! +//! Arrow stores such values directly in a `u128` view: the low 32 bits contain +//! the length and the remaining bits contain zero-padded bytes. Comparing two +//! inline views compares their complete values. Long views contain a prefix and +//! buffer location instead, so a list containing one uses the generic filter. +//! +//! Lists with up to four non-null values use the existing 128-bit branchless +//! filter. Larger lists use the primitive hash-set filter over the same 128 +//! bits. `Decimal128Type` is only a carrier; no decimal operations are used. +//! A long input value cannot match an inline list value because its length is +//! part of the view. + +use std::marker::PhantomData; +use std::sync::Arc; + +use arrow::array::{ + Array, ArrayRef, AsArray, BooleanArray, GenericByteViewArray, MAX_INLINE_VIEW_LEN, + PrimitiveArray, +}; +use arrow::buffer::ScalarBuffer; +use arrow::datatypes::{ + BinaryViewType, ByteViewType, DataType, Decimal128Type, StringViewType, +}; +use arrow::util::bit_iterator::BitIndexIterator; +use datafusion_common::{Result, exec_datafusion_err, internal_datafusion_err}; + +use super::primitive_filter::instantiate_primitive_filter; +use super::static_filter::{StaticFilter, StaticFilterRef}; + +fn view_len(view: u128) -> u32 { + view as u32 +} + +fn downcast_byte_view<T: ByteViewType>( + array: &dyn Array, +) -> Result<&GenericByteViewArray<T>> { + array.as_byte_view_opt::<T>().ok_or_else(|| { + exec_datafusion_err!( + "Expected concrete {} array, got {}", + T::DATA_TYPE, + array.data_type() + ) + }) +} + +fn all_inline<T: ByteViewType>(array: &GenericByteViewArray<T>) -> bool { + let is_inline = |idx: usize| view_len(array.views()[idx]) <= MAX_INLINE_VIEW_LEN; + match array.nulls() { + Some(nulls) => { + BitIndexIterator::new(nulls.validity(), nulls.offset(), nulls.len()) + .all(is_inline) + } + None => (0..array.len()).all(is_inline), + } +} + +fn as_decimal128<T: ByteViewType>( + array: &GenericByteViewArray<T>, +) -> PrimitiveArray<Decimal128Type> { + let views = array.views(); + // `views.inner()` is already sliced to the array's offset. + let values = ScalarBuffer::<i128>::new(views.inner().clone(), 0, views.len()); + PrimitiveArray::<Decimal128Type>::new(values, array.nulls().cloned()) +} + +/// Adapts the selected primitive filter to the original byte-view type. +/// +/// Arrow requires unused inline bytes to be zero, so equal values have equal +/// `u128` views. +struct ByteViewFilter<T: ByteViewType> { + inner: StaticFilterRef, + _marker: PhantomData<T>, Review Comment: We could, but there are only two possible `ByteViewType`s, so we won't win much here! -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
