yihua commented on code in PR #11678:
URL: https://github.com/apache/hudi/pull/11678#discussion_r1691800246
##########
hudi-spark-datasource/hudi-spark-common/src/test/scala/org/apache/hudi/TestRecordLevelIndexSupport.scala:
##########
@@ -43,46 +48,62 @@ class TestRecordLevelIndexSupport {
assertTrue(result.isEmpty)
// Case 3: EqualTo filters on simple AttributeReference and non-Literal
should return empty result
- testFilter = EqualTo(AttributeReference("_row_key", StringType, nullable =
true)(), fromUnixTime)
+ testFilter = EqualTo(AttributeReference(filterColumnName, StringType,
nullable = true)(), fromUnixTime)
result = RecordLevelIndexSupport.filterQueryWithRecordKey(testFilter,
Option.empty)
assertTrue(result.isEmpty)
// Case 4: EqualTo filters on simple AttributeReference and Literal which
should return non-empty result
- testFilter = EqualTo(AttributeReference("_row_key", StringType, nullable =
true)(), Literal("row1"))
- result = RecordLevelIndexSupport.filterQueryWithRecordKey(testFilter,
Option.apply(HoodieMetadataField.RECORD_KEY_METADATA_FIELD.getFieldName))
+ testFilter = EqualTo(AttributeReference(filterColumnName, StringType,
nullable = true)(), Literal("row1"))
+ result = RecordLevelIndexSupport.filterQueryWithRecordKey(testFilter,
Option.apply(recordKeyField))
assertTrue(result.isDefined)
assertEquals(result, Option.apply(testFilter, List.apply("row1")))
- // case 5: EqualTo on fields other than record key should return empty
result
+ // case 5: EqualTo on fields other than record key should return empty
result unless it's _hoodie_record_key
result = RecordLevelIndexSupport.filterQueryWithRecordKey(testFilter,
Option.apply("blah"))
- assertTrue(result.isEmpty)
+ if
(filterColumnName.equals(HoodieMetadataField.RECORD_KEY_METADATA_FIELD.getFieldName))
{
+ assertTrue(result.isDefined)
+ assertEquals(result, Option.apply(testFilter, List.apply("row1")))
+ } else {
+ assertTrue(result.isEmpty)
+ }
- // Case 6: In filter on fields other than record key should return empty
result
- testFilter = In(AttributeReference("_row_key", StringType, nullable =
true)(), List.apply(Literal("xyz"), Literal("abc")))
+ // Case 6: In filter on fields other than record key should return empty
result unless it's _hoodie_record_key
+ testFilter = In(AttributeReference(filterColumnName, StringType, nullable
= true)(), List.apply(Literal("xyz"), Literal("abc")))
result = RecordLevelIndexSupport.filterQueryWithRecordKey(testFilter,
Option.apply("blah"))
- assertTrue(result.isEmpty)
+ if
(filterColumnName.equals(HoodieMetadataField.RECORD_KEY_METADATA_FIELD.getFieldName))
{
+ assertTrue(result.isDefined)
+ assertEquals(result, Option.apply(testFilter, List.apply("xyz", "abc")))
+ } else {
+ assertTrue(result.isEmpty)
+ }
// Case 7: In filter on record key should return non-empty result
- testFilter = In(AttributeReference("_row_key", StringType, nullable =
true)(), List.apply(Literal("xyz"), Literal("abc")))
- result = RecordLevelIndexSupport.filterQueryWithRecordKey(testFilter,
Option.apply(HoodieMetadataField.RECORD_KEY_METADATA_FIELD.getFieldName))
+ testFilter = In(AttributeReference(filterColumnName, StringType, nullable
= true)(), List.apply(Literal("xyz"), Literal("abc")))
+ result = RecordLevelIndexSupport.filterQueryWithRecordKey(testFilter,
Option.apply(recordKeyField))
assertTrue(result.isDefined)
+ assertEquals(result, Option.apply(testFilter, List.apply("xyz", "abc")))
// Case 8: In filter on simple AttributeReference(on record-key) and
non-Literal should return empty result
- testFilter = In(AttributeReference("_row_key", StringType, nullable =
true)(), List.apply(fromUnixTime))
+ testFilter = In(AttributeReference(filterColumnName, StringType, nullable
= true)(), List.apply(fromUnixTime))
result = RecordLevelIndexSupport.filterQueryWithRecordKey(testFilter,
Option.apply(HoodieMetadataField.RECORD_KEY_METADATA_FIELD.getFieldName))
assertTrue(result.isEmpty)
// Case 9: Anything other than EqualTo and In predicate is not supported.
Hence it returns empty result
- testFilter = Not(In(AttributeReference("_row_key", StringType, nullable =
true)(), List.apply(Literal("xyz"), Literal("abc"))))
+ testFilter = Not(In(AttributeReference(filterColumnName, StringType,
nullable = true)(), List.apply(Literal("xyz"), Literal("abc"))))
result = RecordLevelIndexSupport.filterQueryWithRecordKey(testFilter,
Option.apply(HoodieMetadataField.RECORD_KEY_METADATA_FIELD.getFieldName))
assertTrue(result.isEmpty)
- testFilter = Not(In(AttributeReference("_row_key", StringType, nullable =
true)(), List.apply(fromUnixTime)))
+ testFilter = Not(In(AttributeReference(filterColumnName, StringType,
nullable = true)(), List.apply(fromUnixTime)))
result = RecordLevelIndexSupport.filterQueryWithRecordKey(testFilter,
Option.apply(HoodieMetadataField.RECORD_KEY_METADATA_FIELD.getFieldName))
assertTrue(result.isEmpty)
- testFilter = GreaterThan(AttributeReference("_row_key", StringType,
nullable = true)(), Literal("row1"))
+ testFilter = GreaterThan(AttributeReference(filterColumnName, StringType,
nullable = true)(), Literal("row1"))
result = RecordLevelIndexSupport.filterQueryWithRecordKey(testFilter,
Option.apply(HoodieMetadataField.RECORD_KEY_METADATA_FIELD.getFieldName))
assertTrue(result.isEmpty)
+
+ testFilter =
EqualTo(AttributeReference(HoodieMetadataField.RECORD_KEY_METADATA_FIELD.getFieldName,
StringType, nullable = true)(), Literal("row1"))
+ result = RecordLevelIndexSupport.filterQueryWithRecordKey(testFilter,
Option.apply(recordKeyField))
+ assertTrue(result.isDefined)
+ assertEquals(result, Option.apply(testFilter, List.apply("row1")))
Review Comment:
This is already covered so we can remove this case.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
