kosiew commented on code in PR #73:
URL: https://github.com/apache/datafusion-spark/pull/73#discussion_r4598513114


##########
crates/sqllogictest/test_files/string/regexp_extract.slt:
##########
@@ -0,0 +1,406 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under 
one or more contributor license agreements
+
+# Tests for Spark-compatible regexp_extract
+# Reference: 
https://spark.apache.org/docs/latest/api/sql/index.html#regexp_extract
+# Expected values come from Spark's sql-tests/results/regexp-functions.sql.out
+# or were verified against Spark 4.2.0.
+# Audited against Spark 4.2.0 on 2026-11-15.
+# Expected values verified with pyspark==4.2.0.
+# Known divergences are captured as commented-out queries below.
+# Patterns use the Rust regex engine, not java.util.regex; the differences are
+# tracked by https://github.com/apache/datafusion-spark/issues/255
+# The REGEX_GROUP_INDEX error matches Spark 3.5 and later (Spark 3.4 raised
+# _LEGACY_ERROR_TEMP_2006 / _LEGACY_ERROR_TEMP_2007 instead).
+
+query T
+SELECT regexp_extract('1a 2b 14m', '\d+', 0);
+----
+1
+
+# Default group index is 1
+query T
+SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)');
+----
+1
+
+query T
+SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 0);
+----
+1a
+
+query T
+SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 1);
+----
+1
+
+query T
+SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 2);
+----
+a
+
+# A group that did not take part in the match, or no match at all, gives ''
+query T
+SELECT regexp_extract('1a 2b 14m', '(\d+)?', 1);
+----
+1
+
+query T
+SELECT regexp_extract('a b m', '(\d+)?', 1);
+----
+(empty)
+
+query T
+SELECT regexp_extract('1a 2b 14m', '(\d+)?([a-z]+)', 1);
+----
+1
+
+query T
+SELECT regexp_extract('a b m', '(\d+)?([a-z]+)', 1);
+----
+(empty)
+
+query T
+SELECT regexp_extract('abc', '(\d+)', 1);
+----
+(empty)
+
+# idx is only checked when the pattern matches, so no match gives '' even
+# when idx is out of range (including the default idx 1 with no groups)
+query T
+SELECT regexp_extract('abc', '\d+');
+----
+(empty)
+
+query T
+SELECT regexp_extract('abc', '\d+', 5);
+----
+(empty)
+
+query T
+SELECT regexp_extract('abc', '\d+', -1);
+----
+(empty)
+
+query T
+SELECT regexp_extract('abc', '(\d+)', 2);
+----
+(empty)
+
+query T
+SELECT regexp_extract('', '\d+', 3);
+----
+(empty)
+
+# NULL in any argument gives NULL
+query T
+SELECT regexp_extract(NULL, '(\d+)', 1);
+----
+NULL
+
+query T
+SELECT regexp_extract('a1', NULL, 1);
+----
+NULL
+
+query T
+SELECT regexp_extract('a1', '(\d+)', NULL);
+----
+NULL
+
+# A NULL row returns NULL before idx or the pattern is checked
+query T
+SELECT regexp_extract(NULL, '(unclosed', 0);
+----
+NULL
+
+query T
+SELECT regexp_extract('a', '(unclosed', NULL);
+----
+NULL
+
+# A NULL idx returns NULL before idx is checked
+query T
+SELECT regexp_extract('a', NULL, 99);
+----
+NULL
+
+# Group index out of range
+statement error INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX.*Expects group index 
between 0 and 0, but got 1
+SELECT regexp_extract('1a 2b 14m', '\d+');
+
+statement error Expects group index between 0 and 0, but got 1
+SELECT regexp_extract('1a 2b 14m', '\d+', 1);
+
+statement error Expects group index between 0 and 0, but got 2
+SELECT regexp_extract('1a 2b 14m', '\d+', 2);
+
+statement error Expects group index between 0 and 0, but got -1
+SELECT regexp_extract('1a 2b 14m', '\d+', -1);
+
+statement error Expects group index between 0 and 2, but got 3
+SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 3);
+
+statement error Expects group index between 0 and 2, but got -1
+SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', -1);
+
+# Invalid pattern
+statement error INVALID_PARAMETER_VALUE.PATTERN
+SELECT regexp_extract('1a 2b 14m', '(?l)');
+
+statement error INVALID_PARAMETER_VALUE.PATTERN.*'\(unclosed'
+SELECT regexp_extract('x', '(unclosed', 0);
+
+# The pattern is quoted like Spark's toSQLValue, with \ and ' escaped
+statement error is invalid: '\(\\\\d'\.
+SELECT regexp_extract('x', '(\d', 0);
+
+statement error is invalid: 'it\\'s\('\.
+SELECT regexp_extract('x', 'it''s(', 0);
+
+# An empty pattern matches the empty string
+query T
+SELECT regexp_extract('abc', '', 0);

Review Comment:
   Could we extend the empty-pattern tests with `regexp_extract('éx', '()', 1)` 
returning an empty string and `regexp_extract('éx', '(é)', 1)` returning `'é'`? 
These would cover zero-width captures and UTF-8 byte-boundary slicing.



##########
crates/sqllogictest/test_files/string/regexp_extract.slt:
##########
@@ -0,0 +1,406 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under 
one or more contributor license agreements
+
+# Tests for Spark-compatible regexp_extract
+# Reference: 
https://spark.apache.org/docs/latest/api/sql/index.html#regexp_extract
+# Expected values come from Spark's sql-tests/results/regexp-functions.sql.out
+# or were verified against Spark 4.2.0.
+# Audited against Spark 4.2.0 on 2026-11-15.
+# Expected values verified with pyspark==4.2.0.
+# Known divergences are captured as commented-out queries below.
+# Patterns use the Rust regex engine, not java.util.regex; the differences are
+# tracked by https://github.com/apache/datafusion-spark/issues/255
+# The REGEX_GROUP_INDEX error matches Spark 3.5 and later (Spark 3.4 raised
+# _LEGACY_ERROR_TEMP_2006 / _LEGACY_ERROR_TEMP_2007 instead).
+
+query T
+SELECT regexp_extract('1a 2b 14m', '\d+', 0);
+----
+1
+
+# Default group index is 1
+query T
+SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)');
+----
+1
+
+query T
+SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 0);
+----
+1a
+
+query T
+SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 1);
+----
+1
+
+query T
+SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 2);
+----
+a
+
+# A group that did not take part in the match, or no match at all, gives ''
+query T
+SELECT regexp_extract('1a 2b 14m', '(\d+)?', 1);
+----
+1
+
+query T
+SELECT regexp_extract('a b m', '(\d+)?', 1);
+----
+(empty)
+
+query T
+SELECT regexp_extract('1a 2b 14m', '(\d+)?([a-z]+)', 1);
+----
+1
+
+query T
+SELECT regexp_extract('a b m', '(\d+)?([a-z]+)', 1);
+----
+(empty)
+
+query T
+SELECT regexp_extract('abc', '(\d+)', 1);
+----
+(empty)
+
+# idx is only checked when the pattern matches, so no match gives '' even
+# when idx is out of range (including the default idx 1 with no groups)
+query T
+SELECT regexp_extract('abc', '\d+');
+----
+(empty)
+
+query T
+SELECT regexp_extract('abc', '\d+', 5);
+----
+(empty)
+
+query T
+SELECT regexp_extract('abc', '\d+', -1);
+----
+(empty)
+
+query T
+SELECT regexp_extract('abc', '(\d+)', 2);
+----
+(empty)
+
+query T
+SELECT regexp_extract('', '\d+', 3);
+----
+(empty)
+
+# NULL in any argument gives NULL
+query T
+SELECT regexp_extract(NULL, '(\d+)', 1);
+----
+NULL
+
+query T
+SELECT regexp_extract('a1', NULL, 1);
+----
+NULL
+
+query T
+SELECT regexp_extract('a1', '(\d+)', NULL);
+----
+NULL
+
+# A NULL row returns NULL before idx or the pattern is checked
+query T
+SELECT regexp_extract(NULL, '(unclosed', 0);
+----
+NULL
+
+query T
+SELECT regexp_extract('a', '(unclosed', NULL);
+----
+NULL
+
+# A NULL idx returns NULL before idx is checked
+query T
+SELECT regexp_extract('a', NULL, 99);
+----
+NULL
+
+# Group index out of range
+statement error INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX.*Expects group index 
between 0 and 0, but got 1
+SELECT regexp_extract('1a 2b 14m', '\d+');
+
+statement error Expects group index between 0 and 0, but got 1
+SELECT regexp_extract('1a 2b 14m', '\d+', 1);
+
+statement error Expects group index between 0 and 0, but got 2
+SELECT regexp_extract('1a 2b 14m', '\d+', 2);
+
+statement error Expects group index between 0 and 0, but got -1
+SELECT regexp_extract('1a 2b 14m', '\d+', -1);
+
+statement error Expects group index between 0 and 2, but got 3
+SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 3);
+
+statement error Expects group index between 0 and 2, but got -1
+SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', -1);
+
+# Invalid pattern
+statement error INVALID_PARAMETER_VALUE.PATTERN
+SELECT regexp_extract('1a 2b 14m', '(?l)');
+
+statement error INVALID_PARAMETER_VALUE.PATTERN.*'\(unclosed'
+SELECT regexp_extract('x', '(unclosed', 0);
+
+# The pattern is quoted like Spark's toSQLValue, with \ and ' escaped
+statement error is invalid: '\(\\\\d'\.
+SELECT regexp_extract('x', '(\d', 0);
+
+statement error is invalid: 'it\\'s\('\.
+SELECT regexp_extract('x', 'it''s(', 0);
+
+# An empty pattern matches the empty string
+query T
+SELECT regexp_extract('abc', '', 0);
+----
+(empty)
+
+# Known differences between the Rust regex engine and java.util.regex
+# https://github.com/apache/datafusion-spark/issues/255
+
+# Look-around and backreferences are not supported by the Rust regex engine
+statement error Unsupported regex pattern 'a\(\?=b\)' in `regexp_extract`: 
look-around
+SELECT regexp_extract('ab', 'a(?=b)', 0);
+
+statement error Unsupported regex pattern .* in `regexp_extract`: 
backreferences
+SELECT regexp_extract('aa', '(a)\1', 0);
+
+# Spark 4.2.0 supports look-around
+# https://github.com/apache/datafusion-spark/issues/255
+# query T
+# SELECT regexp_extract('ab', 'a(?=b)', 0);
+# ----
+# a
+
+# Spark 4.2.0 supports backreferences
+# https://github.com/apache/datafusion-spark/issues/255
+# query T
+# SELECT regexp_extract('aa', '(a)\1', 0);
+# ----
+# aa
+
+# Spark 4.2.0 treats a++ as a possessive quantifier that does not give back,
+# so a++a cannot match; the Rust regex engine reads a++ as (a+)+ and returns 
'aaa'
+# https://github.com/apache/datafusion-spark/issues/255
+# query T
+# SELECT regexp_extract('aaa', '(a++a)', 1);
+# ----
+# (empty)
+
+# A pattern over the Rust regex engine's compiled-size limit is not supported
+statement error Unsupported regex pattern .* in `regexp_extract`: the compiled 
pattern exceeds the Rust regex size limit
+SELECT regexp_extract('a', '\w{1000}\w{1000}\w{1000}', 0);
+
+# Spark 4.2.0 matches \d against ASCII digits only; DataFusion returns '٣'
+# https://github.com/apache/datafusion-spark/issues/255
+# query T
+# SELECT regexp_extract('x٣y', '\d', 0);
+# ----
+# (empty)
+
+# Spark 4.2.0's . does not match a carriage return; DataFusion returns 'a\rb'
+# https://github.com/apache/datafusion-spark/issues/255
+# query T
+# SELECT regexp_extract('a' || chr(13) || 'b', '(a.b)', 1);
+# ----
+# (empty)
+
+# Spark 4.2.0's $ also matches before a final line terminator; DataFusion 
returns ''
+# https://github.com/apache/datafusion-spark/issues/255
+# query T
+# SELECT regexp_extract('abc' || chr(10), '(abc$)', 1);
+# ----
+# abc
+
+# Spark 4.2.0's (?i) folds ASCII case only, so k does not match the Kelvin sign
+# (U+212A); DataFusion returns it
+# https://github.com/apache/datafusion-spark/issues/255
+# query T
+# SELECT regexp_extract('K', '((?i)k)', 1);
+# ----
+# (empty)
+
+# Spark 4.2.0 rejects Python-style named groups; DataFusion returns 'a'
+# https://github.com/apache/datafusion-spark/issues/255
+# statement error INVALID_PARAMETER_VALUE.PATTERN
+# SELECT regexp_extract('a', '(?P<n>a)', 1);
+
+# Wrong number of arguments
+statement error
+SELECT regexp_extract('abc');
+
+# Column input, including NULL rows and a per-row pattern
+statement ok
+CREATE TABLE regexp_extract_t AS VALUES ('100-200', '(\d+)-(\d+)'), (NULL, 
'(\d+)'), ('abc', '([a-z])');
+
+query T
+SELECT regexp_extract(column1, column2, 1) FROM regexp_extract_t;
+----
+100
+NULL
+a
+
+# Per-row patterns alternating between rows, with NULL string and pattern rows
+statement ok
+CREATE TABLE regexp_extract_patterns AS VALUES ('a1', '(\d)'), (NULL, '(\d)'), 
('b2', NULL), ('c3', '([a-z])'), ('d4', '(\d)'), ('e5', '([a-z])(\d)');
+
+query T
+SELECT regexp_extract(column1, column2, 1) FROM regexp_extract_patterns;
+----
+1
+NULL
+NULL
+c
+4
+e
+
+# Per-row idx, including NULL and an out-of-range idx on a row that does not 
match
+statement ok
+CREATE TABLE regexp_extract_idx AS VALUES ('a1', '(\d)', 1), ('d4', '(\d)', 
NULL), ('xy', '(\d)', 7), ('c3', '([a-z])(\d)', 2);
+
+query T
+SELECT regexp_extract(column1, column2, column3) FROM regexp_extract_idx;
+----
+1
+NULL
+(empty)
+3
+
+query T
+SELECT regexp_extract(column1, '(\d+)', column2) FROM (VALUES ('abc', 3), 
('def', -1), (NULL, 9));
+----
+(empty)
+(empty)
+NULL
+
+# An out-of-range idx fails once a row matches
+statement error Expects group index between 0 and 1, but got 3
+SELECT regexp_extract(column1, '(\d+)', column2) FROM (VALUES ('abc', 3), 
('12', 3));
+
+# Literal str with a column pattern
+query T
+SELECT regexp_extract('a1', column1, 1) FROM (VALUES ('(\d)'), ('([a-z])'));
+----
+1
+a
+
+# Literal str and pattern with a column idx
+query T
+SELECT regexp_extract('a1b2', '([a-z])(\d)', column1) FROM (VALUES (0), (1), 
(2));
+----
+a1
+a
+1
+
+# An invalid pattern in a later row of a pattern column is an error
+statement error INVALID_PARAMETER_VALUE.PATTERN.*is invalid: '\('\.
+SELECT regexp_extract(column1, column2, 0) FROM (VALUES ('a', 'a'), ('b', 
'('));
+
+# A NULL str row is not checked against its invalid pattern
+query T
+SELECT regexp_extract(column1, column2, 0) FROM (VALUES (CAST(NULL AS STRING), 
'('), ('b', 'b'));
+----
+NULL
+b
+
+# Literal pattern with no match on any row and the default idx
+query T
+SELECT regexp_extract(column1, '\d+') FROM (VALUES ('abc'), ('xyz'));
+----
+(empty)
+(empty)
+
+# An invalid literal pattern is not compiled when every string is NULL, as in
+# Spark, so a literal pattern is not validated at planning time
+query T
+SELECT regexp_extract(column1, '(unclosed', 0) FROM (VALUES (CAST(NULL AS 
STRING)), (CAST(NULL AS STRING)));
+----
+NULL
+NULL
+
+# Other string types are accepted
+query T
+SELECT regexp_extract(arrow_cast('1a 2b', 'LargeUtf8'), 
arrow_cast('(\d+)([a-z])', 'Utf8View'), 2);
+----
+a
+
+# Like Spark, str and regexp of other types are cast to strings
+query T
+SELECT regexp_extract(12345, '(\d)', 1);
+----
+1
+
+query T
+SELECT regexp_extract(column1, '(\d)', 1) FROM (VALUES (12345), (67));
+----
+1
+6
+
+query T
+SELECT regexp_extract(X'6131', '(\d)', 1);
+----
+1
+
+query T
+SELECT regexp_extract('a1', 1, 0);
+----
+1
+
+# Like Spark, idx of other numeric types and strings is cast to INT
+query T
+SELECT regexp_extract('1a 2b', '(\d+)([a-z])', arrow_cast(2, 'Int8'));
+----
+a
+
+query T

Review Comment:
   Could we add `SELECT regexp_extract('a1', '([a-z])(\d)', column1) FROM 
(VALUES ('0'), ('1'), ('2'), (CAST(NULL AS STRING)))`, expecting `'a1'`, `'a'`, 
`'1'`, and NULL? Existing tests cover scalar string indices and integer index 
columns, but not string-column-to-Int32 coercion before the `Index::Array` path.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to