jayzhan211 commented on code in PR #12116: URL: https://github.com/apache/datafusion/pull/12116#discussion_r1731979504
########## datafusion/functions/src/core/union_extract.rs: ########## @@ -0,0 +1,722 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::cmp::Ordering; +use std::sync::Arc; + +use arrow::array::{ + layout, make_array, new_empty_array, new_null_array, Array, ArrayRef, BooleanArray, + Int32Array, Scalar, UnionArray, +}; +use arrow::compute::take; +use arrow::datatypes::{DataType, FieldRef, UnionFields, UnionMode}; + +use arrow::buffer::{BooleanBuffer, MutableBuffer, NullBuffer, ScalarBuffer}; +use arrow::util::bit_util; +use datafusion_common::cast::as_union_array; +use datafusion_common::{ + exec_datafusion_err, exec_err, internal_err, ExprSchema, Result, ScalarValue, +}; +use datafusion_expr::{ColumnarValue, Expr}; +use datafusion_expr::{ScalarUDFImpl, Signature, Volatility}; + +#[derive(Debug)] +pub struct UnionExtractFun { + signature: Signature, +} + +impl Default for UnionExtractFun { + fn default() -> Self { + Self::new() + } +} + +impl UnionExtractFun { + pub fn new() -> Self { + Self { + signature: Signature::any(2, Volatility::Immutable), + } + } +} + +impl ScalarUDFImpl for UnionExtractFun { + fn as_any(&self) -> &dyn std::any::Any { + self + } + + fn name(&self) -> &str { + "union_extract" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _: &[DataType]) -> Result<DataType> { + // should be using return_type_from_exprs and not calling the default implementation + internal_err!("union_extract should return type from exprs") + } + + fn return_type_from_exprs( + &self, + args: &[Expr], + _: &dyn ExprSchema, + arg_types: &[DataType], + ) -> Result<DataType> { + if args.len() != 2 { + return exec_err!( + "union_extract expects 2 arguments, got {} instead", + args.len() + ); + } + + let fields = if let DataType::Union(fields, _) = &arg_types[0] { + fields + } else { + return exec_err!( + "union_extract first argument must be a union, got {} instead", + arg_types[0] + ); + }; + + let field_name = if let Expr::Literal(ScalarValue::Utf8(Some(field_name))) = + &args[1] + { + field_name + } else { + return exec_err!( + "union_extract second argument must be a non-null string literal, got {} instead", + arg_types[1] + ); + }; + + let field = find_field(fields, field_name)?.1; + + Ok(field.data_type().clone()) + } + + fn invoke(&self, args: &[ColumnarValue]) -> Result<ColumnarValue> { + if args.len() != 2 { + return exec_err!( + "union_extract expects 2 arguments, got {} instead", + args.len() + ); + } + + let union = &args[0]; + + let target_name = match &args[1] { + ColumnarValue::Scalar(ScalarValue::Utf8(Some(target_name))) => Ok(target_name), + ColumnarValue::Scalar(ScalarValue::Utf8(None)) => exec_err!("union_extract second argument must be a non-null string literal, got a null instead"), + _ => exec_err!("union_extract second argument must be a non-null string literal, got {} instead", &args[1].data_type()), + }; + + match union { + ColumnarValue::Array(array) => { + let union_array = as_union_array(&array).map_err(|_| { + exec_datafusion_err!( + "union_extract first argument must be a union, got {} instead", + array.data_type() + ) + })?; + + let (fields, mode) = match union_array.data_type() { + DataType::Union(fields, mode) => (fields, mode), + _ => unreachable!(), + }; + + let target_type_id = find_field(fields, target_name?)?.0; + + match mode { + UnionMode::Sparse => { + Ok(extract_sparse(union_array, fields, target_type_id)?) + } + UnionMode::Dense => { + Ok(extract_dense(union_array, fields, target_type_id)?) + } + } + } + ColumnarValue::Scalar(ScalarValue::Union(value, fields, _)) => { + let target_name = target_name?; + let (target_type_id, target) = find_field(fields, target_name)?; + + let result = match value { + Some((type_id, value)) if target_type_id == *type_id => { + *value.clone() + } + _ => ScalarValue::try_from(target.data_type())?, + }; + + Ok(ColumnarValue::Scalar(result)) + } + other => exec_err!( + "union_extract first argument must be a union, got {} instead", + other.data_type() + ), + } + } +} + +fn find_field<'a>(fields: &'a UnionFields, name: &str) -> Result<(i8, &'a FieldRef)> { + fields + .iter() + .find(|field| field.1.name() == name) + .ok_or_else(|| exec_datafusion_err!("field {name} not found on union")) +} + +fn extract_sparse( + union_array: &UnionArray, + fields: &UnionFields, + target_type_id: i8, +) -> Result<ColumnarValue> { + let target = union_array.child(target_type_id); + + if fields.len() == 1 // case 1.1: if there is a single field, all type ids are the same, and since union doesn't have a null mask, the result array is exactly the same as it only child + || union_array.is_empty() // case 1.2: sparse union length and childrens length must match, if the union is empty, so is any children + || target.null_count() == target.len() || target.data_type().is_null() + // case 1.3: if all values of the target children are null, regardless of selected type ids, the result will also be completely null + { + Ok(ColumnarValue::Array(Arc::clone(target))) + } else { + match eq_scalar(union_array.type_ids(), target_type_id) { + // case 2: all type ids equals our target, and since unions doesn't have a null mask, the result array is exactly the same as our target + BoolValue::Scalar(true) => Ok(ColumnarValue::Array(Arc::clone(target))), + // case 3: none type_id matches our target, the result is a null array + BoolValue::Scalar(false) => { + if layout(target.data_type()).can_contain_null_mask { Review Comment: ```suggestion new_null_array(target.data_type(), target.len()); ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org For additional commands, e-mail: github-h...@datafusion.apache.org
