mbutrovich commented on code in PR #73:
URL: https://github.com/apache/datafusion-spark/pull/73#discussion_r4602396174


##########
crates/spark/src/function/string/regexp_extract.rs:
##########
@@ -0,0 +1,325 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::array::{
+    Array, ArrayRef, AsArray, Int32Array, StringArray, StringBuilder, 
new_null_array,
+};
+use arrow::compute::can_cast_types;
+use arrow::datatypes::DataType;
+use datafusion_common::cast::as_int32_array;
+use datafusion_common::{
+    DataFusionError, Result, ScalarValue, exec_datafusion_err, 
internal_datafusion_err,
+    internal_err, plan_err,
+};
+use datafusion_expr::{
+    ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+    Volatility,
+};
+use datafusion_macros::user_doc;
+use regex::{CaptureLocations, Regex};
+
+use crate::function::error_utils::invalid_arg_count_exec_err;
+
+const NAME: &str = "regexp_extract";
+
+/// Spark-compatible `regexp_extract` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#regexp_extract>
+#[user_doc(
+    doc_section(label = "Regular Expression Functions"),
+    description = "Returns the text matched by group `idx` of the first match 
of `regexp` in `str`. \
+        Unlike DataFusion's `regexp_match`, which returns a list of every 
capture group, this returns a single string. \
+        Returns an empty string if there is no match, or if the group did not 
take part in the match. \
+        `idx` is only checked when the pattern matches: an `idx` outside 
`0..=groupCount` is an error \
+        (`INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX`, as in Spark 3.5 and 
later; Spark 3.4 raised \
+        `_LEGACY_ERROR_TEMP_2006`/`2007`) on a row that matches, and an empty 
string on a row that does not. \
+        An invalid pattern is an error (`INVALID_PARAMETER_VALUE.PATTERN`). 
Returns NULL if any argument is NULL, \
+        and a NULL row is not checked, so an invalid pattern is only reported 
on a row it is applied to. \
+        As in Spark, `str` and `regexp` are cast to strings and `idx` to INT. \
+        Known differences from Spark 
(https://github.com/apache/datafusion-spark/issues/255): patterns use the Rust \
+        `regex` engine, not Java's `java.util.regex`. Look-around, 
backreferences, and patterns that exceed the Rust \
+        engine's compiled-size limit raise an \"unsupported regex pattern\" 
error, where Spark evaluates them. \
+        Possessive quantifiers are read as nested greedy repetition (`a++` as 
`(a+)+`), so they can match \
+        where Java's would not. \
+        `\\d`, `\\w` and `\\s` match Unicode characters, where Java's match 
only ASCII. \
+        `.` matches `\\r`, where Java's does not. `$` matches only at the very 
end of the input, where Java's also \
+        matches before a final line terminator. `(?i)` folds case using 
Unicode rules (so `k` matches the Kelvin \
+        sign), where Java folds only ASCII by default. Rust's `(?P<name>...)` 
named groups are accepted, where Java \
+        rejects them as an invalid pattern.",
+    syntax_example = "regexp_extract(str, regexp[, idx])",
+    sql_example = r#"```sql
+> SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)');
+1
+> SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 0);
+1a
+> SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 2);
+a
+> SELECT regexp_extract('abc', '(\d+)', 1);
+-- empty string
+> SELECT regexp_extract('a1', '(\d+)', NULL);
+NULL
+```"#,
+    argument(
+        name = "str",
+        description = "String expression to search. Values of other types, 
such as numbers and binary, are cast to a string. NULL returns NULL."
+    ),
+    argument(
+        name = "regexp",
+        description = "String regular expression in Rust `regex` syntax. 
Values of other types are cast to a string. NULL returns NULL."
+    ),
+    argument(
+        name = "idx",
+        description = "Optional capture group index, cast to INT (numeric and 
string values are accepted). Defaults to 1; 0 is the whole match. NULL returns 
NULL."
+    )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkRegexpExtract {
+    signature: Signature,
+}
+
+impl Default for SparkRegexpExtract {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl SparkRegexpExtract {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+impl ScalarUDFImpl for SparkRegexpExtract {
+    fn name(&self) -> &str {
+        NAME
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+        Ok(DataType::Utf8)
+    }
+
+    fn documentation(&self) -> Option<&Documentation> {
+        self.doc()
+    }
+
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+        if !(2..=3).contains(&arg_types.len()) {
+            return Err(invalid_arg_count_exec_err(NAME, (2, 3), 
arg_types.len()));
+        }
+        // Like Spark's implicit casts: str and regexp are cast to strings, 
idx to INT.
+        let castable_to_string = |t: &DataType| {
+            *t == DataType::Null || (!t.is_nested() && can_cast_types(t, 
&DataType::Utf8))
+        };
+        for (name, arg_type) in ["str", "regexp"].iter().zip(arg_types) {
+            if !castable_to_string(arg_type) {
+                return plan_err!("{NAME} expects a string {name}, got 
{arg_type}");
+            }
+        }
+        let mut coerced = vec![DataType::Utf8, DataType::Utf8];
+        if let Some(idx) = arg_types.get(2) {
+            let castable_to_int = idx.is_numeric()
+                || matches!(
+                    idx,
+                    DataType::Null
+                        | DataType::Utf8
+                        | DataType::LargeUtf8
+                        | DataType::Utf8View
+                );
+            if !castable_to_int {
+                return plan_err!("{NAME} expects an INT idx, got {idx}");
+            }
+            coerced.push(DataType::Int32);
+        }
+        Ok(coerced)
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> 
Result<ColumnarValue> {
+        let all_scalar = args
+            .args
+            .iter()
+            .all(|arg| matches!(arg, ColumnarValue::Scalar(_)));
+        let len = if all_scalar { 1 } else { args.number_rows };
+
+        // The usual call has a literal pattern and idx: keep those as scalars 
so
+        // the pattern is compiled once and no per-row copies are allocated.
+        let pattern = match &args.args[1] {
+            ColumnarValue::Scalar(scalar) => match scalar_str(scalar)? {
+                Some(pattern) => Pattern::Scalar(pattern),
+                None => return null_result(len, all_scalar),
+            },
+            ColumnarValue::Array(array) => 
Pattern::Array(array.as_string::<i32>()),
+        };
+        let index = match args.args.get(2) {
+            None => Index::Scalar(1),
+            Some(ColumnarValue::Scalar(ScalarValue::Int32(Some(idx)))) => {
+                Index::Scalar(*idx)
+            }
+            Some(ColumnarValue::Scalar(ScalarValue::Int32(None) | 
ScalarValue::Null)) => {
+                return null_result(len, all_scalar);
+            }
+            Some(ColumnarValue::Scalar(other)) => {
+                return internal_err!(
+                    "{NAME} expects an INT idx, got {}",
+                    other.data_type()
+                );
+            }
+            Some(ColumnarValue::Array(array)) => 
Index::Array(as_int32_array(array)?),
+        };
+
+        let strings = args.args[0].to_array(len)?;
+        let result = extract(strings.as_string::<i32>(), pattern, index)?;

Review Comment:
   `as_string::<i32>()` panics if the subject isn't `Utf8`. `coerce_types` 
guarantees `Utf8` for SQL, but Comet calls `invoke_with_args()` directly with 
whatever its planner hands over. Comet's `regexp_extract` accepts `LargeUtf8` 
([`regexp_extract.rs#L54-L72`](https://github.com/apache/datafusion-comet/blob/ba16e1a53127c90ba00fdc7058703945a8bb9f34/native/spark-expr/src/string_funcs/regexp_extract.rs#L54-L72))
 and has a test for it 
([`large_utf8_subject_returns_utf8_array`](https://github.com/apache/datafusion-comet/blob/ba16e1a53127c90ba00fdc7058703945a8bb9f34/native/spark-expr/src/string_funcs/regexp_extract.rs#L301-L327)).
 Here, calling `invoke_with_args` with a `LargeStringArray` or 
`StringViewArray` subject panics with `string array`. The column-pattern path 
on line 159 has the same problem.
   
   Could `extract` take `Utf8`, `LargeUtf8`, and `Utf8View` for the subject and 
the pattern column (for example, generic over `StringArrayType`), still return 
`Utf8`, and return an error for any other type? Could we also port Comet's 
test? Unit tests run in CI now (#318), so it can stay a Rust test.



##########
crates/sqllogictest/test_files/string/regexp_extract.slt:
##########
@@ -0,0 +1,406 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under 
one or more contributor license agreements
+
+# Tests for Spark-compatible regexp_extract
+# Reference: 
https://spark.apache.org/docs/latest/api/sql/index.html#regexp_extract
+# Expected values come from Spark's sql-tests/results/regexp-functions.sql.out
+# or were verified against Spark 4.2.0.
+# Audited against Spark 4.2.0 on 2026-11-15.
+# Expected values verified with pyspark==4.2.0.
+# Known divergences are captured as commented-out queries below.
+# Patterns use the Rust regex engine, not java.util.regex; the differences are
+# tracked by https://github.com/apache/datafusion-spark/issues/255
+# The REGEX_GROUP_INDEX error matches Spark 3.5 and later (Spark 3.4 raised
+# _LEGACY_ERROR_TEMP_2006 / _LEGACY_ERROR_TEMP_2007 instead).
+
+query T
+SELECT regexp_extract('1a 2b 14m', '\d+', 0);
+----
+1
+
+# Default group index is 1
+query T
+SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)');
+----
+1
+
+query T
+SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 0);
+----
+1a
+
+query T
+SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 1);
+----
+1
+
+query T
+SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 2);
+----
+a
+
+# A group that did not take part in the match, or no match at all, gives ''
+query T
+SELECT regexp_extract('1a 2b 14m', '(\d+)?', 1);
+----
+1
+
+query T
+SELECT regexp_extract('a b m', '(\d+)?', 1);
+----
+(empty)
+
+query T
+SELECT regexp_extract('1a 2b 14m', '(\d+)?([a-z]+)', 1);
+----
+1
+
+query T
+SELECT regexp_extract('a b m', '(\d+)?([a-z]+)', 1);
+----
+(empty)
+
+query T
+SELECT regexp_extract('abc', '(\d+)', 1);
+----
+(empty)
+
+# idx is only checked when the pattern matches, so no match gives '' even
+# when idx is out of range (including the default idx 1 with no groups)
+query T
+SELECT regexp_extract('abc', '\d+');
+----
+(empty)
+
+query T
+SELECT regexp_extract('abc', '\d+', 5);
+----
+(empty)
+
+query T
+SELECT regexp_extract('abc', '\d+', -1);
+----
+(empty)
+
+query T
+SELECT regexp_extract('abc', '(\d+)', 2);
+----
+(empty)
+
+query T
+SELECT regexp_extract('', '\d+', 3);
+----
+(empty)
+
+# NULL in any argument gives NULL
+query T
+SELECT regexp_extract(NULL, '(\d+)', 1);
+----
+NULL
+
+query T
+SELECT regexp_extract('a1', NULL, 1);
+----
+NULL
+
+query T
+SELECT regexp_extract('a1', '(\d+)', NULL);
+----
+NULL
+
+# A NULL row returns NULL before idx or the pattern is checked
+query T
+SELECT regexp_extract(NULL, '(unclosed', 0);
+----
+NULL
+
+query T
+SELECT regexp_extract('a', '(unclosed', NULL);
+----
+NULL
+
+# A NULL idx returns NULL before idx is checked
+query T
+SELECT regexp_extract('a', NULL, 99);
+----
+NULL
+
+# Group index out of range
+statement error INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX.*Expects group index 
between 0 and 0, but got 1
+SELECT regexp_extract('1a 2b 14m', '\d+');
+
+statement error Expects group index between 0 and 0, but got 1
+SELECT regexp_extract('1a 2b 14m', '\d+', 1);
+
+statement error Expects group index between 0 and 0, but got 2
+SELECT regexp_extract('1a 2b 14m', '\d+', 2);
+
+statement error Expects group index between 0 and 0, but got -1
+SELECT regexp_extract('1a 2b 14m', '\d+', -1);
+
+statement error Expects group index between 0 and 2, but got 3
+SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 3);
+
+statement error Expects group index between 0 and 2, but got -1
+SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', -1);
+
+# Invalid pattern
+statement error INVALID_PARAMETER_VALUE.PATTERN
+SELECT regexp_extract('1a 2b 14m', '(?l)');
+
+statement error INVALID_PARAMETER_VALUE.PATTERN.*'\(unclosed'
+SELECT regexp_extract('x', '(unclosed', 0);
+
+# The pattern is quoted like Spark's toSQLValue, with \ and ' escaped
+statement error is invalid: '\(\\\\d'\.
+SELECT regexp_extract('x', '(\d', 0);
+
+statement error is invalid: 'it\\'s\('\.
+SELECT regexp_extract('x', 'it''s(', 0);
+
+# An empty pattern matches the empty string
+query T
+SELECT regexp_extract('abc', '', 0);
+----
+(empty)
+
+# Known differences between the Rust regex engine and java.util.regex
+# https://github.com/apache/datafusion-spark/issues/255
+
+# Look-around and backreferences are not supported by the Rust regex engine
+statement error Unsupported regex pattern 'a\(\?=b\)' in `regexp_extract`: 
look-around
+SELECT regexp_extract('ab', 'a(?=b)', 0);
+
+statement error Unsupported regex pattern .* in `regexp_extract`: 
backreferences
+SELECT regexp_extract('aa', '(a)\1', 0);
+
+# Spark 4.2.0 supports look-around
+# https://github.com/apache/datafusion-spark/issues/255
+# query T
+# SELECT regexp_extract('ab', 'a(?=b)', 0);
+# ----
+# a
+
+# Spark 4.2.0 supports backreferences
+# https://github.com/apache/datafusion-spark/issues/255
+# query T
+# SELECT regexp_extract('aa', '(a)\1', 0);
+# ----
+# aa
+
+# Spark 4.2.0 treats a++ as a possessive quantifier that does not give back,
+# so a++a cannot match; the Rust regex engine reads a++ as (a+)+ and returns 
'aaa'
+# https://github.com/apache/datafusion-spark/issues/255
+# query T
+# SELECT regexp_extract('aaa', '(a++a)', 1);
+# ----
+# (empty)
+
+# A pattern over the Rust regex engine's compiled-size limit is not supported
+statement error Unsupported regex pattern .* in `regexp_extract`: the compiled 
pattern exceeds the Rust regex size limit
+SELECT regexp_extract('a', '\w{1000}\w{1000}\w{1000}', 0);
+
+# Spark 4.2.0 matches \d against ASCII digits only; DataFusion returns '٣'
+# https://github.com/apache/datafusion-spark/issues/255
+# query T
+# SELECT regexp_extract('x٣y', '\d', 0);
+# ----
+# (empty)
+
+# Spark 4.2.0's . does not match a carriage return; DataFusion returns 'a\rb'
+# https://github.com/apache/datafusion-spark/issues/255
+# query T
+# SELECT regexp_extract('a' || chr(13) || 'b', '(a.b)', 1);
+# ----
+# (empty)
+
+# Spark 4.2.0's $ also matches before a final line terminator; DataFusion 
returns ''
+# https://github.com/apache/datafusion-spark/issues/255
+# query T
+# SELECT regexp_extract('abc' || chr(10), '(abc$)', 1);
+# ----
+# abc
+
+# Spark 4.2.0's (?i) folds ASCII case only, so k does not match the Kelvin sign
+# (U+212A); DataFusion returns it
+# https://github.com/apache/datafusion-spark/issues/255
+# query T
+# SELECT regexp_extract('K', '((?i)k)', 1);
+# ----
+# (empty)
+
+# Spark 4.2.0 rejects Python-style named groups; DataFusion returns 'a'
+# https://github.com/apache/datafusion-spark/issues/255
+# statement error INVALID_PARAMETER_VALUE.PATTERN
+# SELECT regexp_extract('a', '(?P<n>a)', 1);
+
+# Wrong number of arguments
+statement error
+SELECT regexp_extract('abc');
+
+# Column input, including NULL rows and a per-row pattern
+statement ok
+CREATE TABLE regexp_extract_t AS VALUES ('100-200', '(\d+)-(\d+)'), (NULL, 
'(\d+)'), ('abc', '([a-z])');
+
+query T
+SELECT regexp_extract(column1, column2, 1) FROM regexp_extract_t;
+----
+100
+NULL
+a
+
+# Per-row patterns alternating between rows, with NULL string and pattern rows
+statement ok
+CREATE TABLE regexp_extract_patterns AS VALUES ('a1', '(\d)'), (NULL, '(\d)'), 
('b2', NULL), ('c3', '([a-z])'), ('d4', '(\d)'), ('e5', '([a-z])(\d)');
+
+query T
+SELECT regexp_extract(column1, column2, 1) FROM regexp_extract_patterns;
+----
+1
+NULL
+NULL
+c
+4
+e
+
+# Per-row idx, including NULL and an out-of-range idx on a row that does not 
match
+statement ok
+CREATE TABLE regexp_extract_idx AS VALUES ('a1', '(\d)', 1), ('d4', '(\d)', 
NULL), ('xy', '(\d)', 7), ('c3', '([a-z])(\d)', 2);
+
+query T
+SELECT regexp_extract(column1, column2, column3) FROM regexp_extract_idx;
+----
+1
+NULL
+(empty)
+3
+
+query T
+SELECT regexp_extract(column1, '(\d+)', column2) FROM (VALUES ('abc', 3), 
('def', -1), (NULL, 9));
+----
+(empty)
+(empty)
+NULL
+
+# An out-of-range idx fails once a row matches
+statement error Expects group index between 0 and 1, but got 3
+SELECT regexp_extract(column1, '(\d+)', column2) FROM (VALUES ('abc', 3), 
('12', 3));
+
+# Literal str with a column pattern
+query T
+SELECT regexp_extract('a1', column1, 1) FROM (VALUES ('(\d)'), ('([a-z])'));
+----
+1
+a
+
+# Literal str and pattern with a column idx
+query T
+SELECT regexp_extract('a1b2', '([a-z])(\d)', column1) FROM (VALUES (0), (1), 
(2));
+----
+a1
+a
+1
+
+# An invalid pattern in a later row of a pattern column is an error
+statement error INVALID_PARAMETER_VALUE.PATTERN.*is invalid: '\('\.
+SELECT regexp_extract(column1, column2, 0) FROM (VALUES ('a', 'a'), ('b', 
'('));
+
+# A NULL str row is not checked against its invalid pattern
+query T
+SELECT regexp_extract(column1, column2, 0) FROM (VALUES (CAST(NULL AS STRING), 
'('), ('b', 'b'));
+----
+NULL
+b
+
+# Literal pattern with no match on any row and the default idx
+query T
+SELECT regexp_extract(column1, '\d+') FROM (VALUES ('abc'), ('xyz'));
+----
+(empty)
+(empty)
+
+# An invalid literal pattern is not compiled when every string is NULL, as in
+# Spark, so a literal pattern is not validated at planning time
+query T
+SELECT regexp_extract(column1, '(unclosed', 0) FROM (VALUES (CAST(NULL AS 
STRING)), (CAST(NULL AS STRING)));
+----
+NULL
+NULL
+
+# Other string types are accepted
+query T
+SELECT regexp_extract(arrow_cast('1a 2b', 'LargeUtf8'), 
arrow_cast('(\d+)([a-z])', 'Utf8View'), 2);
+----
+a
+
+# Like Spark, str and regexp of other types are cast to strings
+query T
+SELECT regexp_extract(12345, '(\d)', 1);
+----
+1
+
+query T
+SELECT regexp_extract(column1, '(\d)', 1) FROM (VALUES (12345), (67));
+----
+1
+6
+
+query T
+SELECT regexp_extract(X'6131', '(\d)', 1);
+----
+1
+
+query T
+SELECT regexp_extract('a1', 1, 0);
+----
+1
+
+# Like Spark, idx of other numeric types and strings is cast to INT
+query T
+SELECT regexp_extract('1a 2b', '(\d+)([a-z])', arrow_cast(2, 'Int8'));
+----
+a
+
+query T
+SELECT regexp_extract('a1', '(\d)', '1');
+----
+1
+
+query T
+SELECT regexp_extract('a1', '(\d)', 1.0::DOUBLE);
+----
+1
+
+query T
+SELECT regexp_extract('a1', '(\d)', 1.5::DOUBLE);
+----
+1
+
+# An idx that cannot be cast to INT is an error, as in Spark
+# (Spark raises CAST_OVERFLOW and CAST_INVALID_INPUT)
+statement error Can't cast value 4294967297 to type Int32
+SELECT regexp_extract('a1', '(\d)', 4294967297::BIGINT);
+
+statement error Cannot cast string 'x' to value of Int32 type
+SELECT regexp_extract('a1', '(\d)', 'x');

Review Comment:
   These queries assert errors "as in Spark", but this file runs with ANSI mode 
off. With ANSI off, PySpark 4.2.0 returns `'1'` for `regexp_extract('a1', 
'(\\d)', 4294967297L)` because the BIGINT wraps to 1. It returns NULL for 
`regexp_extract('a1', '(\\d)', 'x')`. Spark raises `CAST_OVERFLOW` and 
`CAST_INVALID_INPUT` only with ANSI on. A DOUBLE `NaN` idx is another case. 
With ANSI off Spark casts it to 0 and returns `'1'`, while this PR fails with 
`Failed to cast field 'lit' from Float64 to Int32`.
   
   Could these follow the layout in 
[`date_from_unix_date.slt`](https://github.com/apache/datafusion-spark/blob/125600164aa825e2fe805887dfdb384d4182f2a8/crates/sqllogictest/test_files/datetime/date_from_unix_date.slt?plain=1#L54-L68)?
 It asserts the error under `set datafusion.execution.enable_ansi_mode = true` 
and records the ANSI-off Spark result as a commented-out query that cites #88.



##########
crates/spark/src/function/string/regexp_extract.rs:
##########
@@ -0,0 +1,325 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::array::{
+    Array, ArrayRef, AsArray, Int32Array, StringArray, StringBuilder, 
new_null_array,
+};
+use arrow::compute::can_cast_types;
+use arrow::datatypes::DataType;
+use datafusion_common::cast::as_int32_array;
+use datafusion_common::{
+    DataFusionError, Result, ScalarValue, exec_datafusion_err, 
internal_datafusion_err,
+    internal_err, plan_err,
+};
+use datafusion_expr::{
+    ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+    Volatility,
+};
+use datafusion_macros::user_doc;
+use regex::{CaptureLocations, Regex};
+
+use crate::function::error_utils::invalid_arg_count_exec_err;
+
+const NAME: &str = "regexp_extract";
+
+/// Spark-compatible `regexp_extract` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#regexp_extract>
+#[user_doc(
+    doc_section(label = "Regular Expression Functions"),
+    description = "Returns the text matched by group `idx` of the first match 
of `regexp` in `str`. \
+        Unlike DataFusion's `regexp_match`, which returns a list of every 
capture group, this returns a single string. \
+        Returns an empty string if there is no match, or if the group did not 
take part in the match. \
+        `idx` is only checked when the pattern matches: an `idx` outside 
`0..=groupCount` is an error \
+        (`INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX`, as in Spark 3.5 and 
later; Spark 3.4 raised \
+        `_LEGACY_ERROR_TEMP_2006`/`2007`) on a row that matches, and an empty 
string on a row that does not. \
+        An invalid pattern is an error (`INVALID_PARAMETER_VALUE.PATTERN`). 
Returns NULL if any argument is NULL, \
+        and a NULL row is not checked, so an invalid pattern is only reported 
on a row it is applied to. \
+        As in Spark, `str` and `regexp` are cast to strings and `idx` to INT. \
+        Known differences from Spark 
(https://github.com/apache/datafusion-spark/issues/255): patterns use the Rust \
+        `regex` engine, not Java's `java.util.regex`. Look-around, 
backreferences, and patterns that exceed the Rust \
+        engine's compiled-size limit raise an \"unsupported regex pattern\" 
error, where Spark evaluates them. \
+        Possessive quantifiers are read as nested greedy repetition (`a++` as 
`(a+)+`), so they can match \
+        where Java's would not. \
+        `\\d`, `\\w` and `\\s` match Unicode characters, where Java's match 
only ASCII. \
+        `.` matches `\\r`, where Java's does not. `$` matches only at the very 
end of the input, where Java's also \
+        matches before a final line terminator. `(?i)` folds case using 
Unicode rules (so `k` matches the Kelvin \
+        sign), where Java folds only ASCII by default. Rust's `(?P<name>...)` 
named groups are accepted, where Java \
+        rejects them as an invalid pattern.",
+    syntax_example = "regexp_extract(str, regexp[, idx])",
+    sql_example = r#"```sql
+> SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)');
+1
+> SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 0);
+1a
+> SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 2);
+a
+> SELECT regexp_extract('abc', '(\d+)', 1);
+-- empty string
+> SELECT regexp_extract('a1', '(\d+)', NULL);
+NULL
+```"#,
+    argument(
+        name = "str",
+        description = "String expression to search. Values of other types, 
such as numbers and binary, are cast to a string. NULL returns NULL."
+    ),
+    argument(
+        name = "regexp",
+        description = "String regular expression in Rust `regex` syntax. 
Values of other types are cast to a string. NULL returns NULL."
+    ),
+    argument(
+        name = "idx",
+        description = "Optional capture group index, cast to INT (numeric and 
string values are accepted). Defaults to 1; 0 is the whole match. NULL returns 
NULL."
+    )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkRegexpExtract {
+    signature: Signature,
+}
+
+impl Default for SparkRegexpExtract {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl SparkRegexpExtract {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+impl ScalarUDFImpl for SparkRegexpExtract {
+    fn name(&self) -> &str {
+        NAME
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+        Ok(DataType::Utf8)
+    }
+
+    fn documentation(&self) -> Option<&Documentation> {
+        self.doc()
+    }
+
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+        if !(2..=3).contains(&arg_types.len()) {
+            return Err(invalid_arg_count_exec_err(NAME, (2, 3), 
arg_types.len()));
+        }
+        // Like Spark's implicit casts: str and regexp are cast to strings, 
idx to INT.
+        let castable_to_string = |t: &DataType| {
+            *t == DataType::Null || (!t.is_nested() && can_cast_types(t, 
&DataType::Utf8))
+        };
+        for (name, arg_type) in ["str", "regexp"].iter().zip(arg_types) {
+            if !castable_to_string(arg_type) {
+                return plan_err!("{NAME} expects a string {name}, got 
{arg_type}");
+            }
+        }
+        let mut coerced = vec![DataType::Utf8, DataType::Utf8];
+        if let Some(idx) = arg_types.get(2) {
+            let castable_to_int = idx.is_numeric()
+                || matches!(
+                    idx,
+                    DataType::Null
+                        | DataType::Utf8
+                        | DataType::LargeUtf8
+                        | DataType::Utf8View
+                );
+            if !castable_to_int {
+                return plan_err!("{NAME} expects an INT idx, got {idx}");
+            }
+            coerced.push(DataType::Int32);
+        }
+        Ok(coerced)
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> 
Result<ColumnarValue> {
+        let all_scalar = args
+            .args
+            .iter()
+            .all(|arg| matches!(arg, ColumnarValue::Scalar(_)));
+        let len = if all_scalar { 1 } else { args.number_rows };
+
+        // The usual call has a literal pattern and idx: keep those as scalars 
so
+        // the pattern is compiled once and no per-row copies are allocated.
+        let pattern = match &args.args[1] {
+            ColumnarValue::Scalar(scalar) => match scalar_str(scalar)? {
+                Some(pattern) => Pattern::Scalar(pattern),
+                None => return null_result(len, all_scalar),
+            },
+            ColumnarValue::Array(array) => 
Pattern::Array(array.as_string::<i32>()),
+        };
+        let index = match args.args.get(2) {
+            None => Index::Scalar(1),
+            Some(ColumnarValue::Scalar(ScalarValue::Int32(Some(idx)))) => {
+                Index::Scalar(*idx)
+            }
+            Some(ColumnarValue::Scalar(ScalarValue::Int32(None) | 
ScalarValue::Null)) => {
+                return null_result(len, all_scalar);
+            }
+            Some(ColumnarValue::Scalar(other)) => {
+                return internal_err!(
+                    "{NAME} expects an INT idx, got {}",
+                    other.data_type()
+                );
+            }
+            Some(ColumnarValue::Array(array)) => 
Index::Array(as_int32_array(array)?),
+        };
+
+        let strings = args.args[0].to_array(len)?;
+        let result = extract(strings.as_string::<i32>(), pattern, index)?;
+        if all_scalar {
+            Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(
+                &result, 0,
+            )?))
+        } else {
+            Ok(ColumnarValue::Array(result))
+        }
+    }
+}
+
+#[derive(Clone, Copy)]
+enum Pattern<'a> {
+    Scalar(&'a str),
+    Array(&'a StringArray),
+}
+
+#[derive(Clone, Copy)]
+enum Index<'a> {
+    Scalar(i32),
+    Array(&'a Int32Array),
+}
+
+fn scalar_str(scalar: &ScalarValue) -> Result<Option<&str>> {
+    match scalar {
+        ScalarValue::Null => Ok(None),
+        other => other.try_as_str().ok_or_else(|| {
+            internal_datafusion_err!(
+                "{NAME} expects a string regexp, got {}",
+                other.data_type()
+            )
+        }),
+    }
+}
+
+fn null_result(len: usize, all_scalar: bool) -> Result<ColumnarValue> {
+    Ok(if all_scalar {
+        ColumnarValue::Scalar(ScalarValue::Utf8(None))
+    } else {
+        ColumnarValue::Array(new_null_array(&DataType::Utf8, len))
+    })
+}
+
+fn extract(strings: &StringArray, pattern: Pattern, index: Index) -> 
Result<ArrayRef> {
+    // Each result is a substring of its input, so the input bytes bound the 
output.
+    let mut builder =
+        StringBuilder::with_capacity(strings.len(), 
strings.value_data().len());
+    // A scalar pattern is compiled on the first non-NULL row and reused. A 
column
+    // pattern keeps only the last one compiled, since it rarely changes per 
row.
+    // The capture buffer is reused across rows instead of allocated per match.
+    let mut cached: Option<(&str, Regex, CaptureLocations)> = None;
+
+    let scalar_pattern = matches!(pattern, Pattern::Scalar(_));
+
+    for row in 0..strings.len() {
+        let row_pattern = match pattern {
+            Pattern::Scalar(pattern) => Some(pattern),
+            Pattern::Array(patterns) => {
+                (!patterns.is_null(row)).then(|| patterns.value(row))
+            }
+        };
+        let idx = match index {
+            Index::Scalar(idx) => Some(idx),
+            Index::Array(indexes) => (!indexes.is_null(row)).then(|| 
indexes.value(row)),
+        };
+        let (false, Some(pattern), Some(idx)) = (strings.is_null(row), 
row_pattern, idx)
+        else {
+            builder.append_null();
+            continue;
+        };
+        let (_, regex, locations) = match &mut cached {
+            Some(entry) if scalar_pattern || entry.0 == pattern => entry,
+            _ => {
+                let regex = compile(pattern)?;
+                let locations = regex.capture_locations();
+                cached.insert((pattern, regex, locations))
+            }
+        };
+        let subject = strings.value(row);
+        // Like Spark, idx is only checked when the pattern matches.
+        let text = match regex.captures_read(locations, subject) {
+            None => "",
+            Some(_) => locations
+                .get(check_group_index(regex, idx)?)
+                .map_or("", |(start, end)| &subject[start..end]),
+        };
+        builder.append_value(text);
+    }
+    Ok(Arc::new(builder.finish()))
+}
+
+fn check_group_index(regex: &Regex, idx: i32) -> Result<usize> {
+    let group_count = regex.captures_len() - 1;
+    usize::try_from(idx)
+        .ok()
+        .filter(|group| *group <= group_count)
+        .ok_or_else(|| {
+            exec_datafusion_err!(
+                "[INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX] The value of 
parameter(s) \
+                 `idx` in `{NAME}` is invalid: Expects group index between 0 
and \
+                 {group_count}, but got {idx}."

Review Comment:
   Spark 4.2.0 ends both messages with ` SQLSTATE: 22023` (PySpark: ``...but 
got 2. SQLSTATE: 22023`` and ``...is invalid: '('. SQLSTATE: 22023``). `encode` 
includes it in its `INVALID_PARAMETER_VALUE.CHARSET` message 
([`encode.rs#L118`](https://github.com/apache/datafusion-spark/blob/125600164aa825e2fe805887dfdb384d4182f2a8/crates/spark/src/function/string/encode.rs#L118)).
 Could both messages here add it too, the `PATTERN` one on line 294 included?
   
   ```suggestion
                   "[INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX] The value of 
parameter(s) \
                    `idx` in `{NAME}` is invalid: Expects group index between 0 
and \
                    {group_count}, but got {idx}. SQLSTATE: 22023"
   ```



##########
crates/spark/src/function/string/regexp_extract.rs:
##########
@@ -0,0 +1,325 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::array::{
+    Array, ArrayRef, AsArray, Int32Array, StringArray, StringBuilder, 
new_null_array,
+};
+use arrow::compute::can_cast_types;
+use arrow::datatypes::DataType;
+use datafusion_common::cast::as_int32_array;
+use datafusion_common::{
+    DataFusionError, Result, ScalarValue, exec_datafusion_err, 
internal_datafusion_err,
+    internal_err, plan_err,
+};
+use datafusion_expr::{
+    ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+    Volatility,
+};
+use datafusion_macros::user_doc;
+use regex::{CaptureLocations, Regex};
+
+use crate::function::error_utils::invalid_arg_count_exec_err;
+
+const NAME: &str = "regexp_extract";
+
+/// Spark-compatible `regexp_extract` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#regexp_extract>
+#[user_doc(
+    doc_section(label = "Regular Expression Functions"),
+    description = "Returns the text matched by group `idx` of the first match 
of `regexp` in `str`. \
+        Unlike DataFusion's `regexp_match`, which returns a list of every 
capture group, this returns a single string. \
+        Returns an empty string if there is no match, or if the group did not 
take part in the match. \
+        `idx` is only checked when the pattern matches: an `idx` outside 
`0..=groupCount` is an error \
+        (`INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX`, as in Spark 3.5 and 
later; Spark 3.4 raised \
+        `_LEGACY_ERROR_TEMP_2006`/`2007`) on a row that matches, and an empty 
string on a row that does not. \
+        An invalid pattern is an error (`INVALID_PARAMETER_VALUE.PATTERN`). 
Returns NULL if any argument is NULL, \
+        and a NULL row is not checked, so an invalid pattern is only reported 
on a row it is applied to. \
+        As in Spark, `str` and `regexp` are cast to strings and `idx` to INT. \
+        Known differences from Spark 
(https://github.com/apache/datafusion-spark/issues/255): patterns use the Rust \
+        `regex` engine, not Java's `java.util.regex`. Look-around, 
backreferences, and patterns that exceed the Rust \
+        engine's compiled-size limit raise an \"unsupported regex pattern\" 
error, where Spark evaluates them. \
+        Possessive quantifiers are read as nested greedy repetition (`a++` as 
`(a+)+`), so they can match \
+        where Java's would not. \
+        `\\d`, `\\w` and `\\s` match Unicode characters, where Java's match 
only ASCII. \
+        `.` matches `\\r`, where Java's does not. `$` matches only at the very 
end of the input, where Java's also \
+        matches before a final line terminator. `(?i)` folds case using 
Unicode rules (so `k` matches the Kelvin \
+        sign), where Java folds only ASCII by default. Rust's `(?P<name>...)` 
named groups are accepted, where Java \
+        rejects them as an invalid pattern.",
+    syntax_example = "regexp_extract(str, regexp[, idx])",
+    sql_example = r#"```sql
+> SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)');
+1
+> SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 0);
+1a
+> SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 2);
+a
+> SELECT regexp_extract('abc', '(\d+)', 1);
+-- empty string
+> SELECT regexp_extract('a1', '(\d+)', NULL);
+NULL
+```"#,
+    argument(
+        name = "str",
+        description = "String expression to search. Values of other types, 
such as numbers and binary, are cast to a string. NULL returns NULL."
+    ),
+    argument(
+        name = "regexp",
+        description = "String regular expression in Rust `regex` syntax. 
Values of other types are cast to a string. NULL returns NULL."
+    ),
+    argument(
+        name = "idx",
+        description = "Optional capture group index, cast to INT (numeric and 
string values are accepted). Defaults to 1; 0 is the whole match. NULL returns 
NULL."
+    )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkRegexpExtract {
+    signature: Signature,
+}
+
+impl Default for SparkRegexpExtract {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl SparkRegexpExtract {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+impl ScalarUDFImpl for SparkRegexpExtract {
+    fn name(&self) -> &str {
+        NAME
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+        Ok(DataType::Utf8)
+    }
+
+    fn documentation(&self) -> Option<&Documentation> {
+        self.doc()
+    }
+
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+        if !(2..=3).contains(&arg_types.len()) {
+            return Err(invalid_arg_count_exec_err(NAME, (2, 3), 
arg_types.len()));
+        }
+        // Like Spark's implicit casts: str and regexp are cast to strings, 
idx to INT.
+        let castable_to_string = |t: &DataType| {
+            *t == DataType::Null || (!t.is_nested() && can_cast_types(t, 
&DataType::Utf8))
+        };
+        for (name, arg_type) in ["str", "regexp"].iter().zip(arg_types) {
+            if !castable_to_string(arg_type) {
+                return plan_err!("{NAME} expects a string {name}, got 
{arg_type}");
+            }
+        }
+        let mut coerced = vec![DataType::Utf8, DataType::Utf8];
+        if let Some(idx) = arg_types.get(2) {
+            let castable_to_int = idx.is_numeric()
+                || matches!(
+                    idx,
+                    DataType::Null
+                        | DataType::Utf8
+                        | DataType::LargeUtf8
+                        | DataType::Utf8View
+                );
+            if !castable_to_int {
+                return plan_err!("{NAME} expects an INT idx, got {idx}");
+            }
+            coerced.push(DataType::Int32);
+        }
+        Ok(coerced)
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> 
Result<ColumnarValue> {
+        let all_scalar = args
+            .args
+            .iter()
+            .all(|arg| matches!(arg, ColumnarValue::Scalar(_)));
+        let len = if all_scalar { 1 } else { args.number_rows };
+
+        // The usual call has a literal pattern and idx: keep those as scalars 
so
+        // the pattern is compiled once and no per-row copies are allocated.
+        let pattern = match &args.args[1] {
+            ColumnarValue::Scalar(scalar) => match scalar_str(scalar)? {
+                Some(pattern) => Pattern::Scalar(pattern),
+                None => return null_result(len, all_scalar),
+            },
+            ColumnarValue::Array(array) => 
Pattern::Array(array.as_string::<i32>()),
+        };
+        let index = match args.args.get(2) {
+            None => Index::Scalar(1),
+            Some(ColumnarValue::Scalar(ScalarValue::Int32(Some(idx)))) => {
+                Index::Scalar(*idx)
+            }
+            Some(ColumnarValue::Scalar(ScalarValue::Int32(None) | 
ScalarValue::Null)) => {
+                return null_result(len, all_scalar);
+            }
+            Some(ColumnarValue::Scalar(other)) => {
+                return internal_err!(
+                    "{NAME} expects an INT idx, got {}",
+                    other.data_type()
+                );
+            }
+            Some(ColumnarValue::Array(array)) => 
Index::Array(as_int32_array(array)?),
+        };
+
+        let strings = args.args[0].to_array(len)?;
+        let result = extract(strings.as_string::<i32>(), pattern, index)?;
+        if all_scalar {
+            Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(
+                &result, 0,
+            )?))
+        } else {
+            Ok(ColumnarValue::Array(result))
+        }
+    }
+}
+
+#[derive(Clone, Copy)]
+enum Pattern<'a> {
+    Scalar(&'a str),
+    Array(&'a StringArray),
+}
+
+#[derive(Clone, Copy)]
+enum Index<'a> {
+    Scalar(i32),
+    Array(&'a Int32Array),
+}
+
+fn scalar_str(scalar: &ScalarValue) -> Result<Option<&str>> {
+    match scalar {
+        ScalarValue::Null => Ok(None),
+        other => other.try_as_str().ok_or_else(|| {
+            internal_datafusion_err!(
+                "{NAME} expects a string regexp, got {}",
+                other.data_type()
+            )
+        }),
+    }
+}
+
+fn null_result(len: usize, all_scalar: bool) -> Result<ColumnarValue> {
+    Ok(if all_scalar {
+        ColumnarValue::Scalar(ScalarValue::Utf8(None))
+    } else {
+        ColumnarValue::Array(new_null_array(&DataType::Utf8, len))
+    })
+}
+
+fn extract(strings: &StringArray, pattern: Pattern, index: Index) -> 
Result<ArrayRef> {
+    // Each result is a substring of its input, so the input bytes bound the 
output.
+    let mut builder =
+        StringBuilder::with_capacity(strings.len(), 
strings.value_data().len());
+    // A scalar pattern is compiled on the first non-NULL row and reused. A 
column
+    // pattern keeps only the last one compiled, since it rarely changes per 
row.
+    // The capture buffer is reused across rows instead of allocated per match.
+    let mut cached: Option<(&str, Regex, CaptureLocations)> = None;

Review Comment:
   This compiles a literal pattern once per `invoke_with_args` call, so once 
per batch. Comet compiles it once per planned expression, after 
apache/datafusion-comet#5612 measured the per-batch compile. Its 
`regexp_extract` benchmark went from 862 us to 705 us per 8192-row batch, and a 
512-row run was 2.1x faster. The "Port ... from Comet" issues plan for Comet to 
use the datafusion-spark version and remove its own copy (for example #125), so 
Comet would lose that gain when it switches.
   
   Could we carry over Comet's cache 
([`pattern_cache.rs`](https://github.com/apache/datafusion-comet/blob/ba16e1a53127c90ba00fdc7058703945a8bb9f34/native/spark-expr/src/string_funcs/pattern_cache.rs),
 a `Mutex<Option<(String, Regex)>>`) as a field on `SparkRegexpExtract`, in 
this PR? Comet's one-slot design assumes one cache per planned expression. Here 
one UDF instance serves every query in the session, so two expressions with 
different patterns would evict each other on every batch. Keying the cache by 
pattern avoids that. `PartialEq` and `Hash` would then need manual impls that 
ignore the cache.



##########
crates/spark/src/function/string/regexp_extract.rs:
##########
@@ -0,0 +1,325 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::array::{
+    Array, ArrayRef, AsArray, Int32Array, StringArray, StringBuilder, 
new_null_array,
+};
+use arrow::compute::can_cast_types;
+use arrow::datatypes::DataType;
+use datafusion_common::cast::as_int32_array;
+use datafusion_common::{
+    DataFusionError, Result, ScalarValue, exec_datafusion_err, 
internal_datafusion_err,
+    internal_err, plan_err,
+};
+use datafusion_expr::{
+    ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+    Volatility,
+};
+use datafusion_macros::user_doc;
+use regex::{CaptureLocations, Regex};
+
+use crate::function::error_utils::invalid_arg_count_exec_err;
+
+const NAME: &str = "regexp_extract";
+
+/// Spark-compatible `regexp_extract` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#regexp_extract>
+#[user_doc(
+    doc_section(label = "Regular Expression Functions"),
+    description = "Returns the text matched by group `idx` of the first match 
of `regexp` in `str`. \
+        Unlike DataFusion's `regexp_match`, which returns a list of every 
capture group, this returns a single string. \
+        Returns an empty string if there is no match, or if the group did not 
take part in the match. \
+        `idx` is only checked when the pattern matches: an `idx` outside 
`0..=groupCount` is an error \
+        (`INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX`, as in Spark 3.5 and 
later; Spark 3.4 raised \
+        `_LEGACY_ERROR_TEMP_2006`/`2007`) on a row that matches, and an empty 
string on a row that does not. \
+        An invalid pattern is an error (`INVALID_PARAMETER_VALUE.PATTERN`). 
Returns NULL if any argument is NULL, \
+        and a NULL row is not checked, so an invalid pattern is only reported 
on a row it is applied to. \
+        As in Spark, `str` and `regexp` are cast to strings and `idx` to INT. \
+        Known differences from Spark 
(https://github.com/apache/datafusion-spark/issues/255): patterns use the Rust \
+        `regex` engine, not Java's `java.util.regex`. Look-around, 
backreferences, and patterns that exceed the Rust \
+        engine's compiled-size limit raise an \"unsupported regex pattern\" 
error, where Spark evaluates them. \
+        Possessive quantifiers are read as nested greedy repetition (`a++` as 
`(a+)+`), so they can match \
+        where Java's would not. \
+        `\\d`, `\\w` and `\\s` match Unicode characters, where Java's match 
only ASCII. \
+        `.` matches `\\r`, where Java's does not. `$` matches only at the very 
end of the input, where Java's also \
+        matches before a final line terminator. `(?i)` folds case using 
Unicode rules (so `k` matches the Kelvin \
+        sign), where Java folds only ASCII by default. Rust's `(?P<name>...)` 
named groups are accepted, where Java \
+        rejects them as an invalid pattern.",
+    syntax_example = "regexp_extract(str, regexp[, idx])",
+    sql_example = r#"```sql
+> SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)');
+1
+> SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 0);
+1a
+> SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 2);
+a
+> SELECT regexp_extract('abc', '(\d+)', 1);
+-- empty string
+> SELECT regexp_extract('a1', '(\d+)', NULL);
+NULL
+```"#,
+    argument(
+        name = "str",
+        description = "String expression to search. Values of other types, 
such as numbers and binary, are cast to a string. NULL returns NULL."
+    ),
+    argument(
+        name = "regexp",
+        description = "String regular expression in Rust `regex` syntax. 
Values of other types are cast to a string. NULL returns NULL."
+    ),
+    argument(
+        name = "idx",
+        description = "Optional capture group index, cast to INT (numeric and 
string values are accepted). Defaults to 1; 0 is the whole match. NULL returns 
NULL."
+    )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkRegexpExtract {
+    signature: Signature,
+}
+
+impl Default for SparkRegexpExtract {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl SparkRegexpExtract {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+impl ScalarUDFImpl for SparkRegexpExtract {
+    fn name(&self) -> &str {
+        NAME
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+        Ok(DataType::Utf8)
+    }
+
+    fn documentation(&self) -> Option<&Documentation> {
+        self.doc()
+    }
+
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+        if !(2..=3).contains(&arg_types.len()) {
+            return Err(invalid_arg_count_exec_err(NAME, (2, 3), 
arg_types.len()));
+        }
+        // Like Spark's implicit casts: str and regexp are cast to strings, 
idx to INT.
+        let castable_to_string = |t: &DataType| {
+            *t == DataType::Null || (!t.is_nested() && can_cast_types(t, 
&DataType::Utf8))
+        };
+        for (name, arg_type) in ["str", "regexp"].iter().zip(arg_types) {
+            if !castable_to_string(arg_type) {
+                return plan_err!("{NAME} expects a string {name}, got 
{arg_type}");
+            }
+        }
+        let mut coerced = vec![DataType::Utf8, DataType::Utf8];
+        if let Some(idx) = arg_types.get(2) {
+            let castable_to_int = idx.is_numeric()
+                || matches!(
+                    idx,
+                    DataType::Null
+                        | DataType::Utf8
+                        | DataType::LargeUtf8
+                        | DataType::Utf8View
+                );
+            if !castable_to_int {
+                return plan_err!("{NAME} expects an INT idx, got {idx}");
+            }
+            coerced.push(DataType::Int32);
+        }
+        Ok(coerced)
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> 
Result<ColumnarValue> {
+        let all_scalar = args
+            .args
+            .iter()
+            .all(|arg| matches!(arg, ColumnarValue::Scalar(_)));
+        let len = if all_scalar { 1 } else { args.number_rows };
+
+        // The usual call has a literal pattern and idx: keep those as scalars 
so
+        // the pattern is compiled once and no per-row copies are allocated.
+        let pattern = match &args.args[1] {
+            ColumnarValue::Scalar(scalar) => match scalar_str(scalar)? {
+                Some(pattern) => Pattern::Scalar(pattern),
+                None => return null_result(len, all_scalar),
+            },
+            ColumnarValue::Array(array) => 
Pattern::Array(array.as_string::<i32>()),
+        };
+        let index = match args.args.get(2) {
+            None => Index::Scalar(1),
+            Some(ColumnarValue::Scalar(ScalarValue::Int32(Some(idx)))) => {
+                Index::Scalar(*idx)
+            }
+            Some(ColumnarValue::Scalar(ScalarValue::Int32(None) | 
ScalarValue::Null)) => {
+                return null_result(len, all_scalar);
+            }
+            Some(ColumnarValue::Scalar(other)) => {
+                return internal_err!(
+                    "{NAME} expects an INT idx, got {}",
+                    other.data_type()
+                );
+            }
+            Some(ColumnarValue::Array(array)) => 
Index::Array(as_int32_array(array)?),
+        };
+
+        let strings = args.args[0].to_array(len)?;
+        let result = extract(strings.as_string::<i32>(), pattern, index)?;
+        if all_scalar {
+            Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(
+                &result, 0,
+            )?))
+        } else {
+            Ok(ColumnarValue::Array(result))
+        }
+    }
+}
+
+#[derive(Clone, Copy)]
+enum Pattern<'a> {
+    Scalar(&'a str),
+    Array(&'a StringArray),
+}
+
+#[derive(Clone, Copy)]
+enum Index<'a> {
+    Scalar(i32),
+    Array(&'a Int32Array),
+}
+
+fn scalar_str(scalar: &ScalarValue) -> Result<Option<&str>> {
+    match scalar {
+        ScalarValue::Null => Ok(None),
+        other => other.try_as_str().ok_or_else(|| {
+            internal_datafusion_err!(
+                "{NAME} expects a string regexp, got {}",
+                other.data_type()
+            )
+        }),
+    }
+}
+
+fn null_result(len: usize, all_scalar: bool) -> Result<ColumnarValue> {
+    Ok(if all_scalar {
+        ColumnarValue::Scalar(ScalarValue::Utf8(None))
+    } else {
+        ColumnarValue::Array(new_null_array(&DataType::Utf8, len))
+    })
+}
+
+fn extract(strings: &StringArray, pattern: Pattern, index: Index) -> 
Result<ArrayRef> {
+    // Each result is a substring of its input, so the input bytes bound the 
output.
+    let mut builder =
+        StringBuilder::with_capacity(strings.len(), 
strings.value_data().len());
+    // A scalar pattern is compiled on the first non-NULL row and reused. A 
column
+    // pattern keeps only the last one compiled, since it rarely changes per 
row.
+    // The capture buffer is reused across rows instead of allocated per match.
+    let mut cached: Option<(&str, Regex, CaptureLocations)> = None;
+
+    let scalar_pattern = matches!(pattern, Pattern::Scalar(_));
+
+    for row in 0..strings.len() {
+        let row_pattern = match pattern {
+            Pattern::Scalar(pattern) => Some(pattern),
+            Pattern::Array(patterns) => {
+                (!patterns.is_null(row)).then(|| patterns.value(row))
+            }
+        };
+        let idx = match index {
+            Index::Scalar(idx) => Some(idx),
+            Index::Array(indexes) => (!indexes.is_null(row)).then(|| 
indexes.value(row)),
+        };
+        let (false, Some(pattern), Some(idx)) = (strings.is_null(row), 
row_pattern, idx)
+        else {
+            builder.append_null();
+            continue;
+        };
+        let (_, regex, locations) = match &mut cached {
+            Some(entry) if scalar_pattern || entry.0 == pattern => entry,
+            _ => {
+                let regex = compile(pattern)?;
+                let locations = regex.capture_locations();
+                cached.insert((pattern, regex, locations))
+            }
+        };
+        let subject = strings.value(row);
+        // Like Spark, idx is only checked when the pattern matches.
+        let text = match regex.captures_read(locations, subject) {
+            None => "",
+            Some(_) => locations
+                .get(check_group_index(regex, idx)?)
+                .map_or("", |(start, end)| &subject[start..end]),
+        };
+        builder.append_value(text);
+    }
+    Ok(Arc::new(builder.finish()))
+}
+
+fn check_group_index(regex: &Regex, idx: i32) -> Result<usize> {
+    let group_count = regex.captures_len() - 1;
+    usize::try_from(idx)
+        .ok()
+        .filter(|group| *group <= group_count)
+        .ok_or_else(|| {
+            exec_datafusion_err!(
+                "[INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX] The value of 
parameter(s) \
+                 `idx` in `{NAME}` is invalid: Expects group index between 0 
and \
+                 {group_count}, but got {idx}."
+            )
+        })
+}
+
+fn compile(pattern: &str) -> Result<Regex> {
+    Regex::new(pattern).map_err(|err| {
+        let pattern = to_sql_value(pattern);
+        match unsupported_reason(&err) {
+            Some(reason) => DataFusionError::NotImplemented(format!(
+                "Unsupported regex pattern {pattern} in `{NAME}`: {reason}. \
+                 See https://github.com/apache/datafusion-spark/issues/255";
+            )),
+            None => exec_datafusion_err!(
+                "[INVALID_PARAMETER_VALUE.PATTERN] The value of parameter(s) 
`regexp` in \
+                 `{NAME}` is invalid: {pattern}."
+            ),
+        }
+    })
+}
+
+/// Returns why a pattern the Rust engine rejects may still be valid in Java,
+/// for the errors the `regex` crate reports for features it does not support.
+fn unsupported_reason(err: &regex::Error) -> Option<&'static str> {
+    match err {
+        regex::Error::CompiledTooBig(_) => {
+            Some("the compiled pattern exceeds the Rust regex size limit")
+        }
+        regex::Error::Syntax(msg)
+            if msg.contains(
+                "look-around, including look-ahead and look-behind, is not 
supported",
+            ) =>
+        {
+            Some("look-around is not supported by the Rust regex engine")
+        }
+        regex::Error::Syntax(msg) if msg.contains("backreferences are not 
supported") => {
+            Some("backreferences are not supported by the Rust regex engine")
+        }
+        _ => None,
+    }
+}

Review Comment:
   Java accepts several constructs that the Rust parser rejects. Patterns that 
use them get `INVALID_PARAMETER_VALUE.PATTERN`, which tells the user that a 
valid Spark pattern is invalid. PySpark 4.2.0 gives the same results with ANSI 
mode on and off:
   
   | Pattern | Subject | Spark 4.2.0 | This PR |
   |---|---|---|---|
   | `\Q.\E` | `a.b` | `.` | `INVALID_PARAMETER_VALUE.PATTERN` |
   | `(?>a+)b` | `aab` | `aab` | `INVALID_PARAMETER_VALUE.PATTERN` |
   | `b\Z` | `ab` | `b` | `INVALID_PARAMETER_VALUE.PATTERN` |
   | `\h` | `a b` | ` ` (one space) | `INVALID_PARAMETER_VALUE.PATTERN` |
   | `(?d)b` | `ab` | `b` | `INVALID_PARAMETER_VALUE.PATTERN` |
   
   #255 recommends a distinct "unsupported pattern" error, not 
`INVALID_PARAMETER_VALUE.PATTERN`, for anything outside the verified subset. 
Matching on `regex::Error` message text also depends on wording that the 
`regex` crate doesn't treat as API.
   
   Could we classify the failure with `regex_syntax::ast::parse::Parser` 
instead? `regex-syntax` is already in the lock file through `regex`. It reports 
`EscapeUnrecognized` for `\Q`, `\Z`, and `\h`, and `FlagUnrecognized` for `(?>` 
and `(?d)`. Every syntax error I tried that Java also rejects has one of six 
kinds: `GroupUnclosed` (`(`), `GroupUnopened` (`)`), `ClassUnclosed` (`[a`), 
`ClassRangeInvalid` (`[z-a]`), `RepetitionMissing` (`**`), and 
`RepetitionCountInvalid` (`a{2,1}`). Reporting those six kinds as 
`INVALID_PARAMETER_VALUE.PATTERN` and everything else as unsupported would keep 
the golden-file errors and stop rejecting valid patterns as invalid. The cost 
is that a few patterns Java also rejects, such as `\y`, would get the 
unsupported error. The doc's list of differences could then name these 
constructs too.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to