andygrove commented on code in PR #73:
URL: https://github.com/apache/datafusion-spark/pull/73#discussion_r4603852086


##########
crates/spark/src/function/string/regexp_extract.rs:
##########
@@ -0,0 +1,325 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::array::{
+    Array, ArrayRef, AsArray, Int32Array, StringArray, StringBuilder, 
new_null_array,
+};
+use arrow::compute::can_cast_types;
+use arrow::datatypes::DataType;
+use datafusion_common::cast::as_int32_array;
+use datafusion_common::{
+    DataFusionError, Result, ScalarValue, exec_datafusion_err, 
internal_datafusion_err,
+    internal_err, plan_err,
+};
+use datafusion_expr::{
+    ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+    Volatility,
+};
+use datafusion_macros::user_doc;
+use regex::{CaptureLocations, Regex};
+
+use crate::function::error_utils::invalid_arg_count_exec_err;
+
+const NAME: &str = "regexp_extract";
+
+/// Spark-compatible `regexp_extract` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#regexp_extract>
+#[user_doc(
+    doc_section(label = "Regular Expression Functions"),
+    description = "Returns the text matched by group `idx` of the first match 
of `regexp` in `str`. \
+        Unlike DataFusion's `regexp_match`, which returns a list of every 
capture group, this returns a single string. \
+        Returns an empty string if there is no match, or if the group did not 
take part in the match. \
+        `idx` is only checked when the pattern matches: an `idx` outside 
`0..=groupCount` is an error \
+        (`INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX`, as in Spark 3.5 and 
later; Spark 3.4 raised \
+        `_LEGACY_ERROR_TEMP_2006`/`2007`) on a row that matches, and an empty 
string on a row that does not. \
+        An invalid pattern is an error (`INVALID_PARAMETER_VALUE.PATTERN`). 
Returns NULL if any argument is NULL, \
+        and a NULL row is not checked, so an invalid pattern is only reported 
on a row it is applied to. \
+        As in Spark, `str` and `regexp` are cast to strings and `idx` to INT. \
+        Known differences from Spark 
(https://github.com/apache/datafusion-spark/issues/255): patterns use the Rust \
+        `regex` engine, not Java's `java.util.regex`. Look-around, 
backreferences, and patterns that exceed the Rust \
+        engine's compiled-size limit raise an \"unsupported regex pattern\" 
error, where Spark evaluates them. \
+        Possessive quantifiers are read as nested greedy repetition (`a++` as 
`(a+)+`), so they can match \
+        where Java's would not. \
+        `\\d`, `\\w` and `\\s` match Unicode characters, where Java's match 
only ASCII. \
+        `.` matches `\\r`, where Java's does not. `$` matches only at the very 
end of the input, where Java's also \
+        matches before a final line terminator. `(?i)` folds case using 
Unicode rules (so `k` matches the Kelvin \
+        sign), where Java folds only ASCII by default. Rust's `(?P<name>...)` 
named groups are accepted, where Java \
+        rejects them as an invalid pattern.",
+    syntax_example = "regexp_extract(str, regexp[, idx])",
+    sql_example = r#"```sql
+> SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)');
+1
+> SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 0);
+1a
+> SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 2);
+a
+> SELECT regexp_extract('abc', '(\d+)', 1);
+-- empty string
+> SELECT regexp_extract('a1', '(\d+)', NULL);
+NULL
+```"#,
+    argument(
+        name = "str",
+        description = "String expression to search. Values of other types, 
such as numbers and binary, are cast to a string. NULL returns NULL."
+    ),
+    argument(
+        name = "regexp",
+        description = "String regular expression in Rust `regex` syntax. 
Values of other types are cast to a string. NULL returns NULL."
+    ),
+    argument(
+        name = "idx",
+        description = "Optional capture group index, cast to INT (numeric and 
string values are accepted). Defaults to 1; 0 is the whole match. NULL returns 
NULL."
+    )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkRegexpExtract {
+    signature: Signature,
+}
+
+impl Default for SparkRegexpExtract {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl SparkRegexpExtract {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+impl ScalarUDFImpl for SparkRegexpExtract {
+    fn name(&self) -> &str {
+        NAME
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+        Ok(DataType::Utf8)
+    }
+
+    fn documentation(&self) -> Option<&Documentation> {
+        self.doc()
+    }
+
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+        if !(2..=3).contains(&arg_types.len()) {
+            return Err(invalid_arg_count_exec_err(NAME, (2, 3), 
arg_types.len()));
+        }
+        // Like Spark's implicit casts: str and regexp are cast to strings, 
idx to INT.
+        let castable_to_string = |t: &DataType| {
+            *t == DataType::Null || (!t.is_nested() && can_cast_types(t, 
&DataType::Utf8))
+        };
+        for (name, arg_type) in ["str", "regexp"].iter().zip(arg_types) {
+            if !castable_to_string(arg_type) {
+                return plan_err!("{NAME} expects a string {name}, got 
{arg_type}");
+            }
+        }
+        let mut coerced = vec![DataType::Utf8, DataType::Utf8];
+        if let Some(idx) = arg_types.get(2) {
+            let castable_to_int = idx.is_numeric()
+                || matches!(
+                    idx,
+                    DataType::Null
+                        | DataType::Utf8
+                        | DataType::LargeUtf8
+                        | DataType::Utf8View
+                );
+            if !castable_to_int {
+                return plan_err!("{NAME} expects an INT idx, got {idx}");
+            }
+            coerced.push(DataType::Int32);
+        }
+        Ok(coerced)
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> 
Result<ColumnarValue> {
+        let all_scalar = args
+            .args
+            .iter()
+            .all(|arg| matches!(arg, ColumnarValue::Scalar(_)));
+        let len = if all_scalar { 1 } else { args.number_rows };
+
+        // The usual call has a literal pattern and idx: keep those as scalars 
so
+        // the pattern is compiled once and no per-row copies are allocated.
+        let pattern = match &args.args[1] {
+            ColumnarValue::Scalar(scalar) => match scalar_str(scalar)? {
+                Some(pattern) => Pattern::Scalar(pattern),
+                None => return null_result(len, all_scalar),
+            },
+            ColumnarValue::Array(array) => 
Pattern::Array(array.as_string::<i32>()),
+        };
+        let index = match args.args.get(2) {
+            None => Index::Scalar(1),
+            Some(ColumnarValue::Scalar(ScalarValue::Int32(Some(idx)))) => {
+                Index::Scalar(*idx)
+            }
+            Some(ColumnarValue::Scalar(ScalarValue::Int32(None) | 
ScalarValue::Null)) => {
+                return null_result(len, all_scalar);
+            }
+            Some(ColumnarValue::Scalar(other)) => {
+                return internal_err!(
+                    "{NAME} expects an INT idx, got {}",
+                    other.data_type()
+                );
+            }
+            Some(ColumnarValue::Array(array)) => 
Index::Array(as_int32_array(array)?),
+        };
+
+        let strings = args.args[0].to_array(len)?;
+        let result = extract(strings.as_string::<i32>(), pattern, index)?;
+        if all_scalar {
+            Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(
+                &result, 0,
+            )?))
+        } else {
+            Ok(ColumnarValue::Array(result))
+        }
+    }
+}
+
+#[derive(Clone, Copy)]
+enum Pattern<'a> {
+    Scalar(&'a str),
+    Array(&'a StringArray),
+}
+
+#[derive(Clone, Copy)]
+enum Index<'a> {
+    Scalar(i32),
+    Array(&'a Int32Array),
+}
+
+fn scalar_str(scalar: &ScalarValue) -> Result<Option<&str>> {
+    match scalar {
+        ScalarValue::Null => Ok(None),
+        other => other.try_as_str().ok_or_else(|| {
+            internal_datafusion_err!(
+                "{NAME} expects a string regexp, got {}",
+                other.data_type()
+            )
+        }),
+    }
+}
+
+fn null_result(len: usize, all_scalar: bool) -> Result<ColumnarValue> {
+    Ok(if all_scalar {
+        ColumnarValue::Scalar(ScalarValue::Utf8(None))
+    } else {
+        ColumnarValue::Array(new_null_array(&DataType::Utf8, len))
+    })
+}
+
+fn extract(strings: &StringArray, pattern: Pattern, index: Index) -> 
Result<ArrayRef> {
+    // Each result is a substring of its input, so the input bytes bound the 
output.
+    let mut builder =
+        StringBuilder::with_capacity(strings.len(), 
strings.value_data().len());

Review Comment:
   Thanks for the probe. Fixed in 8741567: the capacity is now 
`offset_span_len(strings.offsets())`, or the sum of view lengths for 
`Utf8View`. `sliced_subject_allocates_for_the_slice` checks an empty and a 
one-row slice with 64 KiB of padding on each side, for both the values and an 
allocation under 1 KiB. It fails on the old bound with 131074 bytes.



##########
crates/sqllogictest/test_files/string/regexp_extract.slt:
##########
@@ -0,0 +1,406 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under 
one or more contributor license agreements
+
+# Tests for Spark-compatible regexp_extract
+# Reference: 
https://spark.apache.org/docs/latest/api/sql/index.html#regexp_extract
+# Expected values come from Spark's sql-tests/results/regexp-functions.sql.out
+# or were verified against Spark 4.2.0.
+# Audited against Spark 4.2.0 on 2026-11-15.
+# Expected values verified with pyspark==4.2.0.
+# Known divergences are captured as commented-out queries below.
+# Patterns use the Rust regex engine, not java.util.regex; the differences are
+# tracked by https://github.com/apache/datafusion-spark/issues/255
+# The REGEX_GROUP_INDEX error matches Spark 3.5 and later (Spark 3.4 raised
+# _LEGACY_ERROR_TEMP_2006 / _LEGACY_ERROR_TEMP_2007 instead).
+
+query T
+SELECT regexp_extract('1a 2b 14m', '\d+', 0);
+----
+1
+
+# Default group index is 1
+query T
+SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)');
+----
+1
+
+query T
+SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 0);
+----
+1a
+
+query T
+SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 1);
+----
+1
+
+query T
+SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 2);
+----
+a
+
+# A group that did not take part in the match, or no match at all, gives ''
+query T
+SELECT regexp_extract('1a 2b 14m', '(\d+)?', 1);
+----
+1
+
+query T
+SELECT regexp_extract('a b m', '(\d+)?', 1);
+----
+(empty)
+
+query T
+SELECT regexp_extract('1a 2b 14m', '(\d+)?([a-z]+)', 1);
+----
+1
+
+query T
+SELECT regexp_extract('a b m', '(\d+)?([a-z]+)', 1);
+----
+(empty)
+
+query T
+SELECT regexp_extract('abc', '(\d+)', 1);
+----
+(empty)
+
+# idx is only checked when the pattern matches, so no match gives '' even
+# when idx is out of range (including the default idx 1 with no groups)
+query T
+SELECT regexp_extract('abc', '\d+');
+----
+(empty)
+
+query T
+SELECT regexp_extract('abc', '\d+', 5);
+----
+(empty)
+
+query T
+SELECT regexp_extract('abc', '\d+', -1);
+----
+(empty)
+
+query T
+SELECT regexp_extract('abc', '(\d+)', 2);
+----
+(empty)
+
+query T
+SELECT regexp_extract('', '\d+', 3);
+----
+(empty)
+
+# NULL in any argument gives NULL
+query T
+SELECT regexp_extract(NULL, '(\d+)', 1);
+----
+NULL
+
+query T
+SELECT regexp_extract('a1', NULL, 1);
+----
+NULL
+
+query T
+SELECT regexp_extract('a1', '(\d+)', NULL);
+----
+NULL
+
+# A NULL row returns NULL before idx or the pattern is checked
+query T
+SELECT regexp_extract(NULL, '(unclosed', 0);
+----
+NULL
+
+query T
+SELECT regexp_extract('a', '(unclosed', NULL);
+----
+NULL
+
+# A NULL idx returns NULL before idx is checked
+query T
+SELECT regexp_extract('a', NULL, 99);
+----
+NULL
+
+# Group index out of range
+statement error INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX.*Expects group index 
between 0 and 0, but got 1
+SELECT regexp_extract('1a 2b 14m', '\d+');
+
+statement error Expects group index between 0 and 0, but got 1
+SELECT regexp_extract('1a 2b 14m', '\d+', 1);
+
+statement error Expects group index between 0 and 0, but got 2
+SELECT regexp_extract('1a 2b 14m', '\d+', 2);
+
+statement error Expects group index between 0 and 0, but got -1
+SELECT regexp_extract('1a 2b 14m', '\d+', -1);
+
+statement error Expects group index between 0 and 2, but got 3
+SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 3);
+
+statement error Expects group index between 0 and 2, but got -1
+SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', -1);
+
+# Invalid pattern
+statement error INVALID_PARAMETER_VALUE.PATTERN
+SELECT regexp_extract('1a 2b 14m', '(?l)');
+
+statement error INVALID_PARAMETER_VALUE.PATTERN.*'\(unclosed'
+SELECT regexp_extract('x', '(unclosed', 0);
+
+# The pattern is quoted like Spark's toSQLValue, with \ and ' escaped
+statement error is invalid: '\(\\\\d'\.
+SELECT regexp_extract('x', '(\d', 0);
+
+statement error is invalid: 'it\\'s\('\.
+SELECT regexp_extract('x', 'it''s(', 0);
+
+# An empty pattern matches the empty string
+query T
+SELECT regexp_extract('abc', '', 0);

Review Comment:
   Added both in 8741567.



##########
crates/sqllogictest/test_files/string/regexp_extract.slt:
##########
@@ -0,0 +1,406 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under 
one or more contributor license agreements
+
+# Tests for Spark-compatible regexp_extract
+# Reference: 
https://spark.apache.org/docs/latest/api/sql/index.html#regexp_extract
+# Expected values come from Spark's sql-tests/results/regexp-functions.sql.out
+# or were verified against Spark 4.2.0.
+# Audited against Spark 4.2.0 on 2026-11-15.
+# Expected values verified with pyspark==4.2.0.
+# Known divergences are captured as commented-out queries below.
+# Patterns use the Rust regex engine, not java.util.regex; the differences are
+# tracked by https://github.com/apache/datafusion-spark/issues/255
+# The REGEX_GROUP_INDEX error matches Spark 3.5 and later (Spark 3.4 raised
+# _LEGACY_ERROR_TEMP_2006 / _LEGACY_ERROR_TEMP_2007 instead).
+
+query T
+SELECT regexp_extract('1a 2b 14m', '\d+', 0);
+----
+1
+
+# Default group index is 1
+query T
+SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)');
+----
+1
+
+query T
+SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 0);
+----
+1a
+
+query T
+SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 1);
+----
+1
+
+query T
+SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 2);
+----
+a
+
+# A group that did not take part in the match, or no match at all, gives ''
+query T
+SELECT regexp_extract('1a 2b 14m', '(\d+)?', 1);
+----
+1
+
+query T
+SELECT regexp_extract('a b m', '(\d+)?', 1);
+----
+(empty)
+
+query T
+SELECT regexp_extract('1a 2b 14m', '(\d+)?([a-z]+)', 1);
+----
+1
+
+query T
+SELECT regexp_extract('a b m', '(\d+)?([a-z]+)', 1);
+----
+(empty)
+
+query T
+SELECT regexp_extract('abc', '(\d+)', 1);
+----
+(empty)
+
+# idx is only checked when the pattern matches, so no match gives '' even
+# when idx is out of range (including the default idx 1 with no groups)
+query T
+SELECT regexp_extract('abc', '\d+');
+----
+(empty)
+
+query T
+SELECT regexp_extract('abc', '\d+', 5);
+----
+(empty)
+
+query T
+SELECT regexp_extract('abc', '\d+', -1);
+----
+(empty)
+
+query T
+SELECT regexp_extract('abc', '(\d+)', 2);
+----
+(empty)
+
+query T
+SELECT regexp_extract('', '\d+', 3);
+----
+(empty)
+
+# NULL in any argument gives NULL
+query T
+SELECT regexp_extract(NULL, '(\d+)', 1);
+----
+NULL
+
+query T
+SELECT regexp_extract('a1', NULL, 1);
+----
+NULL
+
+query T
+SELECT regexp_extract('a1', '(\d+)', NULL);
+----
+NULL
+
+# A NULL row returns NULL before idx or the pattern is checked
+query T
+SELECT regexp_extract(NULL, '(unclosed', 0);
+----
+NULL
+
+query T
+SELECT regexp_extract('a', '(unclosed', NULL);
+----
+NULL
+
+# A NULL idx returns NULL before idx is checked
+query T
+SELECT regexp_extract('a', NULL, 99);
+----
+NULL
+
+# Group index out of range
+statement error INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX.*Expects group index 
between 0 and 0, but got 1
+SELECT regexp_extract('1a 2b 14m', '\d+');
+
+statement error Expects group index between 0 and 0, but got 1
+SELECT regexp_extract('1a 2b 14m', '\d+', 1);
+
+statement error Expects group index between 0 and 0, but got 2
+SELECT regexp_extract('1a 2b 14m', '\d+', 2);
+
+statement error Expects group index between 0 and 0, but got -1
+SELECT regexp_extract('1a 2b 14m', '\d+', -1);
+
+statement error Expects group index between 0 and 2, but got 3
+SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 3);
+
+statement error Expects group index between 0 and 2, but got -1
+SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', -1);
+
+# Invalid pattern
+statement error INVALID_PARAMETER_VALUE.PATTERN
+SELECT regexp_extract('1a 2b 14m', '(?l)');
+
+statement error INVALID_PARAMETER_VALUE.PATTERN.*'\(unclosed'
+SELECT regexp_extract('x', '(unclosed', 0);
+
+# The pattern is quoted like Spark's toSQLValue, with \ and ' escaped
+statement error is invalid: '\(\\\\d'\.
+SELECT regexp_extract('x', '(\d', 0);
+
+statement error is invalid: 'it\\'s\('\.
+SELECT regexp_extract('x', 'it''s(', 0);
+
+# An empty pattern matches the empty string
+query T
+SELECT regexp_extract('abc', '', 0);
+----
+(empty)
+
+# Known differences between the Rust regex engine and java.util.regex
+# https://github.com/apache/datafusion-spark/issues/255
+
+# Look-around and backreferences are not supported by the Rust regex engine
+statement error Unsupported regex pattern 'a\(\?=b\)' in `regexp_extract`: 
look-around
+SELECT regexp_extract('ab', 'a(?=b)', 0);
+
+statement error Unsupported regex pattern .* in `regexp_extract`: 
backreferences
+SELECT regexp_extract('aa', '(a)\1', 0);
+
+# Spark 4.2.0 supports look-around
+# https://github.com/apache/datafusion-spark/issues/255
+# query T
+# SELECT regexp_extract('ab', 'a(?=b)', 0);
+# ----
+# a
+
+# Spark 4.2.0 supports backreferences
+# https://github.com/apache/datafusion-spark/issues/255
+# query T
+# SELECT regexp_extract('aa', '(a)\1', 0);
+# ----
+# aa
+
+# Spark 4.2.0 treats a++ as a possessive quantifier that does not give back,
+# so a++a cannot match; the Rust regex engine reads a++ as (a+)+ and returns 
'aaa'
+# https://github.com/apache/datafusion-spark/issues/255
+# query T
+# SELECT regexp_extract('aaa', '(a++a)', 1);
+# ----
+# (empty)
+
+# A pattern over the Rust regex engine's compiled-size limit is not supported
+statement error Unsupported regex pattern .* in `regexp_extract`: the compiled 
pattern exceeds the Rust regex size limit
+SELECT regexp_extract('a', '\w{1000}\w{1000}\w{1000}', 0);
+
+# Spark 4.2.0 matches \d against ASCII digits only; DataFusion returns '٣'
+# https://github.com/apache/datafusion-spark/issues/255
+# query T
+# SELECT regexp_extract('x٣y', '\d', 0);
+# ----
+# (empty)
+
+# Spark 4.2.0's . does not match a carriage return; DataFusion returns 'a\rb'
+# https://github.com/apache/datafusion-spark/issues/255
+# query T
+# SELECT regexp_extract('a' || chr(13) || 'b', '(a.b)', 1);
+# ----
+# (empty)
+
+# Spark 4.2.0's $ also matches before a final line terminator; DataFusion 
returns ''
+# https://github.com/apache/datafusion-spark/issues/255
+# query T
+# SELECT regexp_extract('abc' || chr(10), '(abc$)', 1);
+# ----
+# abc
+
+# Spark 4.2.0's (?i) folds ASCII case only, so k does not match the Kelvin sign
+# (U+212A); DataFusion returns it
+# https://github.com/apache/datafusion-spark/issues/255
+# query T
+# SELECT regexp_extract('K', '((?i)k)', 1);
+# ----
+# (empty)
+
+# Spark 4.2.0 rejects Python-style named groups; DataFusion returns 'a'
+# https://github.com/apache/datafusion-spark/issues/255
+# statement error INVALID_PARAMETER_VALUE.PATTERN
+# SELECT regexp_extract('a', '(?P<n>a)', 1);
+
+# Wrong number of arguments
+statement error
+SELECT regexp_extract('abc');
+
+# Column input, including NULL rows and a per-row pattern
+statement ok
+CREATE TABLE regexp_extract_t AS VALUES ('100-200', '(\d+)-(\d+)'), (NULL, 
'(\d+)'), ('abc', '([a-z])');
+
+query T
+SELECT regexp_extract(column1, column2, 1) FROM regexp_extract_t;
+----
+100
+NULL
+a
+
+# Per-row patterns alternating between rows, with NULL string and pattern rows
+statement ok
+CREATE TABLE regexp_extract_patterns AS VALUES ('a1', '(\d)'), (NULL, '(\d)'), 
('b2', NULL), ('c3', '([a-z])'), ('d4', '(\d)'), ('e5', '([a-z])(\d)');
+
+query T
+SELECT regexp_extract(column1, column2, 1) FROM regexp_extract_patterns;
+----
+1
+NULL
+NULL
+c
+4
+e
+
+# Per-row idx, including NULL and an out-of-range idx on a row that does not 
match
+statement ok
+CREATE TABLE regexp_extract_idx AS VALUES ('a1', '(\d)', 1), ('d4', '(\d)', 
NULL), ('xy', '(\d)', 7), ('c3', '([a-z])(\d)', 2);
+
+query T
+SELECT regexp_extract(column1, column2, column3) FROM regexp_extract_idx;
+----
+1
+NULL
+(empty)
+3
+
+query T
+SELECT regexp_extract(column1, '(\d+)', column2) FROM (VALUES ('abc', 3), 
('def', -1), (NULL, 9));
+----
+(empty)
+(empty)
+NULL
+
+# An out-of-range idx fails once a row matches
+statement error Expects group index between 0 and 1, but got 3
+SELECT regexp_extract(column1, '(\d+)', column2) FROM (VALUES ('abc', 3), 
('12', 3));
+
+# Literal str with a column pattern
+query T
+SELECT regexp_extract('a1', column1, 1) FROM (VALUES ('(\d)'), ('([a-z])'));
+----
+1
+a
+
+# Literal str and pattern with a column idx
+query T
+SELECT regexp_extract('a1b2', '([a-z])(\d)', column1) FROM (VALUES (0), (1), 
(2));
+----
+a1
+a
+1
+
+# An invalid pattern in a later row of a pattern column is an error
+statement error INVALID_PARAMETER_VALUE.PATTERN.*is invalid: '\('\.
+SELECT regexp_extract(column1, column2, 0) FROM (VALUES ('a', 'a'), ('b', 
'('));
+
+# A NULL str row is not checked against its invalid pattern
+query T
+SELECT regexp_extract(column1, column2, 0) FROM (VALUES (CAST(NULL AS STRING), 
'('), ('b', 'b'));
+----
+NULL
+b
+
+# Literal pattern with no match on any row and the default idx
+query T
+SELECT regexp_extract(column1, '\d+') FROM (VALUES ('abc'), ('xyz'));
+----
+(empty)
+(empty)
+
+# An invalid literal pattern is not compiled when every string is NULL, as in
+# Spark, so a literal pattern is not validated at planning time
+query T
+SELECT regexp_extract(column1, '(unclosed', 0) FROM (VALUES (CAST(NULL AS 
STRING)), (CAST(NULL AS STRING)));
+----
+NULL
+NULL
+
+# Other string types are accepted
+query T
+SELECT regexp_extract(arrow_cast('1a 2b', 'LargeUtf8'), 
arrow_cast('(\d+)([a-z])', 'Utf8View'), 2);
+----
+a
+
+# Like Spark, str and regexp of other types are cast to strings
+query T
+SELECT regexp_extract(12345, '(\d)', 1);
+----
+1
+
+query T
+SELECT regexp_extract(column1, '(\d)', 1) FROM (VALUES (12345), (67));
+----
+1
+6
+
+query T
+SELECT regexp_extract(X'6131', '(\d)', 1);
+----
+1
+
+query T
+SELECT regexp_extract('a1', 1, 0);
+----
+1
+
+# Like Spark, idx of other numeric types and strings is cast to INT
+query T
+SELECT regexp_extract('1a 2b', '(\d+)([a-z])', arrow_cast(2, 'Int8'));
+----
+a
+
+query T

Review Comment:
   Added in 8741567. It returns `a1`, `a`, `1` and NULL, as in Spark 4.2.0.



##########
crates/spark/src/function/string/regexp_extract.rs:
##########
@@ -0,0 +1,325 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::array::{
+    Array, ArrayRef, AsArray, Int32Array, StringArray, StringBuilder, 
new_null_array,
+};
+use arrow::compute::can_cast_types;
+use arrow::datatypes::DataType;
+use datafusion_common::cast::as_int32_array;
+use datafusion_common::{
+    DataFusionError, Result, ScalarValue, exec_datafusion_err, 
internal_datafusion_err,
+    internal_err, plan_err,
+};
+use datafusion_expr::{
+    ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+    Volatility,
+};
+use datafusion_macros::user_doc;
+use regex::{CaptureLocations, Regex};
+
+use crate::function::error_utils::invalid_arg_count_exec_err;
+
+const NAME: &str = "regexp_extract";
+
+/// Spark-compatible `regexp_extract` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#regexp_extract>
+#[user_doc(
+    doc_section(label = "Regular Expression Functions"),
+    description = "Returns the text matched by group `idx` of the first match 
of `regexp` in `str`. \
+        Unlike DataFusion's `regexp_match`, which returns a list of every 
capture group, this returns a single string. \
+        Returns an empty string if there is no match, or if the group did not 
take part in the match. \
+        `idx` is only checked when the pattern matches: an `idx` outside 
`0..=groupCount` is an error \
+        (`INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX`, as in Spark 3.5 and 
later; Spark 3.4 raised \
+        `_LEGACY_ERROR_TEMP_2006`/`2007`) on a row that matches, and an empty 
string on a row that does not. \
+        An invalid pattern is an error (`INVALID_PARAMETER_VALUE.PATTERN`). 
Returns NULL if any argument is NULL, \
+        and a NULL row is not checked, so an invalid pattern is only reported 
on a row it is applied to. \
+        As in Spark, `str` and `regexp` are cast to strings and `idx` to INT. \
+        Known differences from Spark 
(https://github.com/apache/datafusion-spark/issues/255): patterns use the Rust \
+        `regex` engine, not Java's `java.util.regex`. Look-around, 
backreferences, and patterns that exceed the Rust \
+        engine's compiled-size limit raise an \"unsupported regex pattern\" 
error, where Spark evaluates them. \
+        Possessive quantifiers are read as nested greedy repetition (`a++` as 
`(a+)+`), so they can match \
+        where Java's would not. \
+        `\\d`, `\\w` and `\\s` match Unicode characters, where Java's match 
only ASCII. \
+        `.` matches `\\r`, where Java's does not. `$` matches only at the very 
end of the input, where Java's also \
+        matches before a final line terminator. `(?i)` folds case using 
Unicode rules (so `k` matches the Kelvin \
+        sign), where Java folds only ASCII by default. Rust's `(?P<name>...)` 
named groups are accepted, where Java \
+        rejects them as an invalid pattern.",
+    syntax_example = "regexp_extract(str, regexp[, idx])",
+    sql_example = r#"```sql
+> SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)');
+1
+> SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 0);
+1a
+> SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 2);
+a
+> SELECT regexp_extract('abc', '(\d+)', 1);
+-- empty string
+> SELECT regexp_extract('a1', '(\d+)', NULL);
+NULL
+```"#,
+    argument(
+        name = "str",
+        description = "String expression to search. Values of other types, 
such as numbers and binary, are cast to a string. NULL returns NULL."
+    ),
+    argument(
+        name = "regexp",
+        description = "String regular expression in Rust `regex` syntax. 
Values of other types are cast to a string. NULL returns NULL."
+    ),
+    argument(
+        name = "idx",
+        description = "Optional capture group index, cast to INT (numeric and 
string values are accepted). Defaults to 1; 0 is the whole match. NULL returns 
NULL."
+    )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkRegexpExtract {
+    signature: Signature,
+}
+
+impl Default for SparkRegexpExtract {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl SparkRegexpExtract {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+impl ScalarUDFImpl for SparkRegexpExtract {
+    fn name(&self) -> &str {
+        NAME
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+        Ok(DataType::Utf8)
+    }
+
+    fn documentation(&self) -> Option<&Documentation> {
+        self.doc()
+    }
+
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+        if !(2..=3).contains(&arg_types.len()) {
+            return Err(invalid_arg_count_exec_err(NAME, (2, 3), 
arg_types.len()));
+        }
+        // Like Spark's implicit casts: str and regexp are cast to strings, 
idx to INT.
+        let castable_to_string = |t: &DataType| {
+            *t == DataType::Null || (!t.is_nested() && can_cast_types(t, 
&DataType::Utf8))
+        };
+        for (name, arg_type) in ["str", "regexp"].iter().zip(arg_types) {
+            if !castable_to_string(arg_type) {
+                return plan_err!("{NAME} expects a string {name}, got 
{arg_type}");
+            }
+        }
+        let mut coerced = vec![DataType::Utf8, DataType::Utf8];
+        if let Some(idx) = arg_types.get(2) {
+            let castable_to_int = idx.is_numeric()
+                || matches!(
+                    idx,
+                    DataType::Null
+                        | DataType::Utf8
+                        | DataType::LargeUtf8
+                        | DataType::Utf8View
+                );
+            if !castable_to_int {
+                return plan_err!("{NAME} expects an INT idx, got {idx}");
+            }
+            coerced.push(DataType::Int32);
+        }
+        Ok(coerced)
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> 
Result<ColumnarValue> {
+        let all_scalar = args
+            .args
+            .iter()
+            .all(|arg| matches!(arg, ColumnarValue::Scalar(_)));
+        let len = if all_scalar { 1 } else { args.number_rows };
+
+        // The usual call has a literal pattern and idx: keep those as scalars 
so
+        // the pattern is compiled once and no per-row copies are allocated.
+        let pattern = match &args.args[1] {
+            ColumnarValue::Scalar(scalar) => match scalar_str(scalar)? {
+                Some(pattern) => Pattern::Scalar(pattern),
+                None => return null_result(len, all_scalar),
+            },
+            ColumnarValue::Array(array) => 
Pattern::Array(array.as_string::<i32>()),
+        };
+        let index = match args.args.get(2) {
+            None => Index::Scalar(1),
+            Some(ColumnarValue::Scalar(ScalarValue::Int32(Some(idx)))) => {
+                Index::Scalar(*idx)
+            }
+            Some(ColumnarValue::Scalar(ScalarValue::Int32(None) | 
ScalarValue::Null)) => {
+                return null_result(len, all_scalar);
+            }
+            Some(ColumnarValue::Scalar(other)) => {
+                return internal_err!(
+                    "{NAME} expects an INT idx, got {}",
+                    other.data_type()
+                );
+            }
+            Some(ColumnarValue::Array(array)) => 
Index::Array(as_int32_array(array)?),
+        };
+
+        let strings = args.args[0].to_array(len)?;
+        let result = extract(strings.as_string::<i32>(), pattern, index)?;

Review Comment:
   Done in 8741567. `extract` is generic over `StringArrayType`, so `Utf8`, 
`LargeUtf8` and `Utf8View` subjects work and the result is always `Utf8`. A 
pattern column of another string type is cast to `Utf8`, and any other type is 
an error rather than a panic. I ported Comet's test and extended it to 
`Utf8View`, and added tests for a `LargeUtf8` pattern column and a non-string 
subject.



##########
crates/spark/src/function/string/regexp_extract.rs:
##########
@@ -0,0 +1,325 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::array::{
+    Array, ArrayRef, AsArray, Int32Array, StringArray, StringBuilder, 
new_null_array,
+};
+use arrow::compute::can_cast_types;
+use arrow::datatypes::DataType;
+use datafusion_common::cast::as_int32_array;
+use datafusion_common::{
+    DataFusionError, Result, ScalarValue, exec_datafusion_err, 
internal_datafusion_err,
+    internal_err, plan_err,
+};
+use datafusion_expr::{
+    ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+    Volatility,
+};
+use datafusion_macros::user_doc;
+use regex::{CaptureLocations, Regex};
+
+use crate::function::error_utils::invalid_arg_count_exec_err;
+
+const NAME: &str = "regexp_extract";
+
+/// Spark-compatible `regexp_extract` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#regexp_extract>
+#[user_doc(
+    doc_section(label = "Regular Expression Functions"),
+    description = "Returns the text matched by group `idx` of the first match 
of `regexp` in `str`. \
+        Unlike DataFusion's `regexp_match`, which returns a list of every 
capture group, this returns a single string. \
+        Returns an empty string if there is no match, or if the group did not 
take part in the match. \
+        `idx` is only checked when the pattern matches: an `idx` outside 
`0..=groupCount` is an error \
+        (`INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX`, as in Spark 3.5 and 
later; Spark 3.4 raised \
+        `_LEGACY_ERROR_TEMP_2006`/`2007`) on a row that matches, and an empty 
string on a row that does not. \
+        An invalid pattern is an error (`INVALID_PARAMETER_VALUE.PATTERN`). 
Returns NULL if any argument is NULL, \
+        and a NULL row is not checked, so an invalid pattern is only reported 
on a row it is applied to. \
+        As in Spark, `str` and `regexp` are cast to strings and `idx` to INT. \
+        Known differences from Spark 
(https://github.com/apache/datafusion-spark/issues/255): patterns use the Rust \
+        `regex` engine, not Java's `java.util.regex`. Look-around, 
backreferences, and patterns that exceed the Rust \
+        engine's compiled-size limit raise an \"unsupported regex pattern\" 
error, where Spark evaluates them. \
+        Possessive quantifiers are read as nested greedy repetition (`a++` as 
`(a+)+`), so they can match \
+        where Java's would not. \
+        `\\d`, `\\w` and `\\s` match Unicode characters, where Java's match 
only ASCII. \
+        `.` matches `\\r`, where Java's does not. `$` matches only at the very 
end of the input, where Java's also \
+        matches before a final line terminator. `(?i)` folds case using 
Unicode rules (so `k` matches the Kelvin \
+        sign), where Java folds only ASCII by default. Rust's `(?P<name>...)` 
named groups are accepted, where Java \
+        rejects them as an invalid pattern.",
+    syntax_example = "regexp_extract(str, regexp[, idx])",
+    sql_example = r#"```sql
+> SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)');
+1
+> SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 0);
+1a
+> SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 2);
+a
+> SELECT regexp_extract('abc', '(\d+)', 1);
+-- empty string
+> SELECT regexp_extract('a1', '(\d+)', NULL);
+NULL
+```"#,
+    argument(
+        name = "str",
+        description = "String expression to search. Values of other types, 
such as numbers and binary, are cast to a string. NULL returns NULL."
+    ),
+    argument(
+        name = "regexp",
+        description = "String regular expression in Rust `regex` syntax. 
Values of other types are cast to a string. NULL returns NULL."
+    ),
+    argument(
+        name = "idx",
+        description = "Optional capture group index, cast to INT (numeric and 
string values are accepted). Defaults to 1; 0 is the whole match. NULL returns 
NULL."
+    )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkRegexpExtract {
+    signature: Signature,
+}
+
+impl Default for SparkRegexpExtract {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl SparkRegexpExtract {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+impl ScalarUDFImpl for SparkRegexpExtract {
+    fn name(&self) -> &str {
+        NAME
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+        Ok(DataType::Utf8)
+    }
+
+    fn documentation(&self) -> Option<&Documentation> {
+        self.doc()
+    }
+
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+        if !(2..=3).contains(&arg_types.len()) {
+            return Err(invalid_arg_count_exec_err(NAME, (2, 3), 
arg_types.len()));
+        }
+        // Like Spark's implicit casts: str and regexp are cast to strings, 
idx to INT.
+        let castable_to_string = |t: &DataType| {
+            *t == DataType::Null || (!t.is_nested() && can_cast_types(t, 
&DataType::Utf8))
+        };
+        for (name, arg_type) in ["str", "regexp"].iter().zip(arg_types) {
+            if !castable_to_string(arg_type) {
+                return plan_err!("{NAME} expects a string {name}, got 
{arg_type}");
+            }
+        }
+        let mut coerced = vec![DataType::Utf8, DataType::Utf8];
+        if let Some(idx) = arg_types.get(2) {
+            let castable_to_int = idx.is_numeric()
+                || matches!(
+                    idx,
+                    DataType::Null
+                        | DataType::Utf8
+                        | DataType::LargeUtf8
+                        | DataType::Utf8View
+                );
+            if !castable_to_int {
+                return plan_err!("{NAME} expects an INT idx, got {idx}");
+            }
+            coerced.push(DataType::Int32);
+        }
+        Ok(coerced)
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> 
Result<ColumnarValue> {
+        let all_scalar = args
+            .args
+            .iter()
+            .all(|arg| matches!(arg, ColumnarValue::Scalar(_)));
+        let len = if all_scalar { 1 } else { args.number_rows };
+
+        // The usual call has a literal pattern and idx: keep those as scalars 
so
+        // the pattern is compiled once and no per-row copies are allocated.
+        let pattern = match &args.args[1] {
+            ColumnarValue::Scalar(scalar) => match scalar_str(scalar)? {
+                Some(pattern) => Pattern::Scalar(pattern),
+                None => return null_result(len, all_scalar),
+            },
+            ColumnarValue::Array(array) => 
Pattern::Array(array.as_string::<i32>()),
+        };
+        let index = match args.args.get(2) {
+            None => Index::Scalar(1),
+            Some(ColumnarValue::Scalar(ScalarValue::Int32(Some(idx)))) => {
+                Index::Scalar(*idx)
+            }
+            Some(ColumnarValue::Scalar(ScalarValue::Int32(None) | 
ScalarValue::Null)) => {
+                return null_result(len, all_scalar);
+            }
+            Some(ColumnarValue::Scalar(other)) => {
+                return internal_err!(
+                    "{NAME} expects an INT idx, got {}",
+                    other.data_type()
+                );
+            }
+            Some(ColumnarValue::Array(array)) => 
Index::Array(as_int32_array(array)?),
+        };
+
+        let strings = args.args[0].to_array(len)?;
+        let result = extract(strings.as_string::<i32>(), pattern, index)?;
+        if all_scalar {
+            Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(
+                &result, 0,
+            )?))
+        } else {
+            Ok(ColumnarValue::Array(result))
+        }
+    }
+}
+
+#[derive(Clone, Copy)]
+enum Pattern<'a> {
+    Scalar(&'a str),
+    Array(&'a StringArray),
+}
+
+#[derive(Clone, Copy)]
+enum Index<'a> {
+    Scalar(i32),
+    Array(&'a Int32Array),
+}
+
+fn scalar_str(scalar: &ScalarValue) -> Result<Option<&str>> {
+    match scalar {
+        ScalarValue::Null => Ok(None),
+        other => other.try_as_str().ok_or_else(|| {
+            internal_datafusion_err!(
+                "{NAME} expects a string regexp, got {}",
+                other.data_type()
+            )
+        }),
+    }
+}
+
+fn null_result(len: usize, all_scalar: bool) -> Result<ColumnarValue> {
+    Ok(if all_scalar {
+        ColumnarValue::Scalar(ScalarValue::Utf8(None))
+    } else {
+        ColumnarValue::Array(new_null_array(&DataType::Utf8, len))
+    })
+}
+
+fn extract(strings: &StringArray, pattern: Pattern, index: Index) -> 
Result<ArrayRef> {
+    // Each result is a substring of its input, so the input bytes bound the 
output.
+    let mut builder =
+        StringBuilder::with_capacity(strings.len(), 
strings.value_data().len());
+    // A scalar pattern is compiled on the first non-NULL row and reused. A 
column
+    // pattern keeps only the last one compiled, since it rarely changes per 
row.
+    // The capture buffer is reused across rows instead of allocated per match.
+    let mut cached: Option<(&str, Regex, CaptureLocations)> = None;

Review Comment:
   Done in 8741567. `SparkRegexpExtract` has a `PatternCache` keyed by pattern, 
cleared when it reaches 64 entries, and `PartialEq` and `Hash` ignore it. 
Literal patterns come from the cache on the first non-NULL row, so NULL input 
still isn't checked. Column patterns keep the per-row cache. 
`literal_patterns_are_cached_across_batches` checks the reuse.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to