kosiew commented on code in PR #26895: URL: https://github.com/apache/datafusion/pull/26895#discussion_r4599355634
########## datafusion/sqllogictest/test_files/metadata.slt: ########## @@ -687,6 +691,224 @@ NULL l_bar baz +# Tests for coalesce / nvl / ifnull / nvl2 propagate field metadata +# https://github.com/apache/datafusion/pull/24865 + +# The tests below use the id, name and l_name columns. Each column carries +# its own 'metadata_key' entry, so the output of arrow_metadata() shows which +# argument the result's metadata came from. +query ITT +select id, name, l_name from table_with_metadata; +---- +1 NULL NULL +NULL bar l_bar +3 baz l_baz + +query ??? +select arrow_metadata(id), arrow_metadata(name), arrow_metadata(l_name) +from table_with_metadata limit 1; +---- +{metadata_key: the id field} {metadata_key: the name field} {metadata_key: the l_name field} + +# Metadata is merged from the value-contributing arguments; earlier arguments win. Review Comment: Could we clarify that executable assertions capture current behavior while commented outputs describe intended behavior? The duplicate expected-output block at lines 736-739 can also be removed. ########## datafusion/sqllogictest/test_files/metadata.slt: ########## @@ -687,6 +691,224 @@ NULL l_bar baz +# Tests for coalesce / nvl / ifnull / nvl2 propagate field metadata +# https://github.com/apache/datafusion/pull/24865 + +# The tests below use the id, name and l_name columns. Each column carries +# its own 'metadata_key' entry, so the output of arrow_metadata() shows which +# argument the result's metadata came from. +query ITT +select id, name, l_name from table_with_metadata; +---- +1 NULL NULL +NULL bar l_bar +3 baz l_baz + +query ??? +select arrow_metadata(id), arrow_metadata(name), arrow_metadata(l_name) +from table_with_metadata limit 1; +---- +{metadata_key: the id field} {metadata_key: the name field} {metadata_key: the l_name field} + +# Metadata is merged from the value-contributing arguments; earlier arguments win. +query TT +select coalesce(name, l_name), arrow_metadata(coalesce(name, l_name), 'metadata_key') +from table_with_metadata; +---- +NULL NULL +bar NULL +baz NULL + +# Expected +#NULL the name field +#bar the name field +#baz the name field + +# An untyped NULL literal argument does not block metadata. +query TT +select coalesce(NULL, name), arrow_metadata(coalesce(NULL, name), 'metadata_key') +from table_with_metadata; +---- +NULL the name field +bar the name field +baz the name field + +# Expected +#NULL the name field +#bar the name field +#baz the name field + +# An argument without metadata does not block metadata from later arguments. +query TT +select coalesce(upper(name), l_name), arrow_metadata(coalesce(upper(name), l_name), 'metadata_key') +from table_with_metadata; +---- +NULL NULL +BAR NULL +BAZ NULL + +# Expected +#NULL the l_name field +#BAR the l_name field +#BAZ the l_name field + +# Common extension types are propagated. +query T +select arrow_metadata( + coalesce( + with_metadata(name, 'ARROW:extension:name', 'ext.type'), + with_metadata(l_name, 'ARROW:extension:name', 'ext.type') + ), + 'ARROW:extension:name') +from table_with_metadata limit 1; +---- +NULL + +# Conflicting extension types are a planning error; see the tests above. + +# nvl +query TT +select nvl(name, l_name), arrow_metadata(nvl(name, l_name), 'metadata_key') +from table_with_metadata; +---- +NULL NULL +bar NULL +baz NULL + +# Expected +#NULL the name field +#bar the name field +#baz the name field + +query T +select arrow_metadata(ifnull(name, l_name), 'metadata_key') from table_with_metadata limit 1; +---- +NULL + +# nvl2: the result comes from the second or third argument. +query TT +select nvl2(id, name, l_name), arrow_metadata(nvl2(id, name, l_name), 'metadata_key') +from table_with_metadata; +---- +NULL NULL +l_bar NULL +baz NULL + +# Expected +#NULL the name field +#l_bar the name field +#baz the name field + +# The first argument of nvl2 is only tested for NULL; its metadata does not leak. +query T +select arrow_metadata( + nvl2(with_metadata(id, 'ARROW:extension:name', 'test.type'), name, l_name), + 'ARROW:extension:name') +from table_with_metadata limit 1; +---- +NULL + +# CASE: metadata is merged from the THEN and ELSE branches; earlier branches win. +# coalesce, nvl, ifnull and nvl2 are all rewritten to CASE during +# simplification, so CASE must propagate metadata for them to do so. +query TT +select + case when name is not null then name else l_name end, + arrow_metadata(case when name is not null then name else l_name end, 'metadata_key') +from table_with_metadata; +---- +NULL NULL +bar NULL +baz NULL + +# Expected +#NULL the name field +#bar the name field +#baz the name field + +# CASE without ELSE +query TT +select + case when name is not null then name end, + arrow_metadata(case when name is not null then name end, 'metadata_key') +from table_with_metadata; +---- +NULL NULL +bar NULL +baz NULL + +# Expected +#NULL the name field +#bar the name field +#baz the name field + +# Simple CASE form +query TT +select + case id when 1 then name else l_name end, + arrow_metadata(case id when 1 then name else l_name end, 'metadata_key') +from table_with_metadata; +---- +NULL NULL +l_bar NULL +l_baz NULL + +# Expected +#NULL the name field +#l_bar the name field +#l_baz the name field + +# A branch without metadata does not block metadata from later branches. +query TT +select + case when id is null then upper(name) else l_name end, + arrow_metadata(case when id is null then upper(name) else l_name end, 'metadata_key') +from table_with_metadata; +---- +NULL NULL +BAR NULL +l_baz NULL + +# Expected +#NULL the l_name field +#BAR the l_name field +#l_baz the l_name field + +# Agreeing extension types are propagated. +query T +select arrow_metadata( + case when name is not null + then with_metadata(name, 'ARROW:extension:name', 'ext.type') + else with_metadata(l_name, 'ARROW:extension:name', 'ext.type') + end, + 'ARROW:extension:name') +from table_with_metadata limit 1; +---- +NULL + +# Conflicting extension types should be a planning error, as for coalesce: +# Error during planning: Cannot merge field metadata with differing values of 'ARROW:extension:name': 'a.type' and 'b.type' +query T +select + case when name is not null + then with_metadata(name, 'ARROW:extension:name', 'a.type') + else with_metadata(l_name, 'ARROW:extension:name', 'b.type') + end +from table_with_metadata; +---- +NULL +bar +baz + +# The CASE operand and WHEN conditions are only tested; their metadata does not leak. Review Comment: This heading mentions WHEN-condition metadata isolation, but the test only covers the CASE operand. Could we narrow the heading or add a separate condition-isolation test? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
