markap14 commented on code in PR #11857: URL: https://github.com/apache/nifi/pull/11857#discussion_r4485652852
########## nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaRecordTombstoneIT.java: ########## @@ -0,0 +1,260 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.kafka.processors; + +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.avro.Schema; +import org.apache.avro.file.DataFileWriter; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.nifi.avro.AvroReader; +import org.apache.nifi.kafka.processors.consumer.ProcessingStrategy; +import org.apache.nifi.kafka.processors.producer.wrapper.WrapperRecord; +import org.apache.nifi.kafka.service.api.consumer.AutoOffsetReset; +import org.apache.nifi.kafka.shared.property.OutputStrategy; +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.schema.access.SchemaAccessUtils; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; + +/** + * RECORD output for a batch that contains a non-null empty value, a null tombstone, and a real value. + * A null tombstone is omitted from success and parse.failure. A JSON reader emits only the real value. + * An Avro reader routes the non-null empty value to parse.failure and writes the real value to success. + */ +class ConsumeKafkaRecordTombstoneIT extends AbstractConsumeKafkaIT { + + private static final String NON_EMPTY_VALUE = "{\"name\":\"A\"}"; + + private static final byte[] NON_EMPTY_BYTES = NON_EMPTY_VALUE.getBytes(StandardCharsets.UTF_8); + + private static final byte[] EMPTY_BYTES = new byte[0]; + + private static final String AVRO_SCHEMA = """ + { + "type": "record", + "name": "message", + "fields": [ + {"name": "name", "type": "string"} + ] + } + """; + + private static final String MESSAGE_KEY = "key"; + + private static final int FIRST_PARTITION = 0; + + private static final Duration POLL_TIMEOUT = Duration.ofSeconds(30); + + private TestRunner runner; + + private String topic; + + @BeforeEach + void setRunner() throws InitializationException { + runner = TestRunners.newTestRunner(ConsumeKafka.class); + addKafkaConnectionService(runner); + runner.setProperty(ConsumeKafka.CONNECTION_SERVICE, CONNECTION_SERVICE_ID); + addRecordReaderService(runner); + addRecordWriterService(runner); + + topic = UUID.randomUUID().toString(); + runner.setProperty(ConsumeKafka.GROUP_ID, topic); + runner.setProperty(ConsumeKafka.TOPICS, topic); + runner.setProperty(ConsumeKafka.PROCESSING_STRATEGY, ProcessingStrategy.RECORD); + runner.setProperty(ConsumeKafka.AUTO_OFFSET_RESET, AutoOffsetReset.EARLIEST.getValue()); + } + + @Test + void testOutputStrategyUseValue() throws ExecutionException, InterruptedException, IOException { + setOutputStrategy(OutputStrategy.USE_VALUE); + produceJsonRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + assertEquals("A", record.get("name").asText()); + } + + @Test + void testOutputStrategyUseWrapper() throws ExecutionException, InterruptedException, IOException { + setOutputStrategy(OutputStrategy.USE_WRAPPER); + produceJsonRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + final ObjectNode value = assertInstanceOf(ObjectNode.class, record.get(WrapperRecord.VALUE)); + assertEquals("A", value.get("name").asText()); + } + + @Test + void testOutputStrategyInjectMetadata() throws ExecutionException, InterruptedException, IOException { + setOutputStrategy(OutputStrategy.INJECT_METADATA); + produceJsonRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + assertEquals("A", record.get("name").asText()); + } + + @Test + void testOutputStrategyInjectOffset() throws ExecutionException, InterruptedException, IOException { + setOutputStrategy(OutputStrategy.INJECT_OFFSET); + produceJsonRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + assertEquals("A", record.get("name").asText()); + } Review Comment: [Claude Opus-5.5] These four JSON reader tests pass on `main` without this change. As your table in the earlier comment shows, with the JSON reader a null value already produced no FlowFile on either `success` or `parse.failure`. So they don't protect against the bug this PR fixes, and each one costs a full round trip to Kafka. I'd suggest removing them. ########## nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaRecordTombstoneIT.java: ########## @@ -0,0 +1,260 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.kafka.processors; + +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.avro.Schema; +import org.apache.avro.file.DataFileWriter; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.nifi.avro.AvroReader; +import org.apache.nifi.kafka.processors.consumer.ProcessingStrategy; +import org.apache.nifi.kafka.processors.producer.wrapper.WrapperRecord; +import org.apache.nifi.kafka.service.api.consumer.AutoOffsetReset; +import org.apache.nifi.kafka.shared.property.OutputStrategy; +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.schema.access.SchemaAccessUtils; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; + +/** + * RECORD output for a batch that contains a non-null empty value, a null tombstone, and a real value. + * A null tombstone is omitted from success and parse.failure. A JSON reader emits only the real value. + * An Avro reader routes the non-null empty value to parse.failure and writes the real value to success. + */ +class ConsumeKafkaRecordTombstoneIT extends AbstractConsumeKafkaIT { + + private static final String NON_EMPTY_VALUE = "{\"name\":\"A\"}"; + + private static final byte[] NON_EMPTY_BYTES = NON_EMPTY_VALUE.getBytes(StandardCharsets.UTF_8); + + private static final byte[] EMPTY_BYTES = new byte[0]; + + private static final String AVRO_SCHEMA = """ + { + "type": "record", + "name": "message", + "fields": [ + {"name": "name", "type": "string"} + ] + } + """; + + private static final String MESSAGE_KEY = "key"; + + private static final int FIRST_PARTITION = 0; + + private static final Duration POLL_TIMEOUT = Duration.ofSeconds(30); + + private TestRunner runner; + + private String topic; + + @BeforeEach + void setRunner() throws InitializationException { + runner = TestRunners.newTestRunner(ConsumeKafka.class); + addKafkaConnectionService(runner); + runner.setProperty(ConsumeKafka.CONNECTION_SERVICE, CONNECTION_SERVICE_ID); + addRecordReaderService(runner); + addRecordWriterService(runner); + + topic = UUID.randomUUID().toString(); + runner.setProperty(ConsumeKafka.GROUP_ID, topic); + runner.setProperty(ConsumeKafka.TOPICS, topic); + runner.setProperty(ConsumeKafka.PROCESSING_STRATEGY, ProcessingStrategy.RECORD); + runner.setProperty(ConsumeKafka.AUTO_OFFSET_RESET, AutoOffsetReset.EARLIEST.getValue()); + } + + @Test + void testOutputStrategyUseValue() throws ExecutionException, InterruptedException, IOException { + setOutputStrategy(OutputStrategy.USE_VALUE); + produceJsonRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + assertEquals("A", record.get("name").asText()); + } + + @Test + void testOutputStrategyUseWrapper() throws ExecutionException, InterruptedException, IOException { + setOutputStrategy(OutputStrategy.USE_WRAPPER); + produceJsonRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + final ObjectNode value = assertInstanceOf(ObjectNode.class, record.get(WrapperRecord.VALUE)); + assertEquals("A", value.get("name").asText()); + } + + @Test + void testOutputStrategyInjectMetadata() throws ExecutionException, InterruptedException, IOException { + setOutputStrategy(OutputStrategy.INJECT_METADATA); + produceJsonRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + assertEquals("A", record.get("name").asText()); + } + + @Test + void testOutputStrategyInjectOffset() throws ExecutionException, InterruptedException, IOException { + setOutputStrategy(OutputStrategy.INJECT_OFFSET); + produceJsonRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + assertEquals("A", record.get("name").asText()); + } + + @Test + void testOutputStrategyUseValueWithAvroReader() throws ExecutionException, InterruptedException, IOException, InitializationException { + setOutputStrategy(OutputStrategy.USE_VALUE); + useAvroReader(); + produceAvroRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + assertEquals("A", record.get("name").asText()); + } + + @Test + void testOutputStrategyUseWrapperWithAvroReader() throws ExecutionException, InterruptedException, IOException, InitializationException { + setOutputStrategy(OutputStrategy.USE_WRAPPER); + useAvroReader(); + produceAvroRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + final ObjectNode value = assertInstanceOf(ObjectNode.class, record.get(WrapperRecord.VALUE)); + assertEquals("A", value.get("name").asText()); + } + + @Test + void testOutputStrategyInjectMetadataWithAvroReader() throws ExecutionException, InterruptedException, IOException, InitializationException { + setOutputStrategy(OutputStrategy.INJECT_METADATA); + useAvroReader(); + produceAvroRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + assertEquals("A", record.get("name").asText()); + } + + @Test + void testOutputStrategyInjectOffsetWithAvroReader() throws ExecutionException, InterruptedException, IOException, InitializationException { + setOutputStrategy(OutputStrategy.INJECT_OFFSET); + useAvroReader(); + produceAvroRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + assertEquals("A", record.get("name").asText()); + } Review Comment: [Claude Opus-5.5] The Avro tests are the ones that actually reproduce the bug, but the four of them differ only by Output Strategy. Tombstones are skipped in `AbstractRecordStreamKafkaMessageConverter.toFlowFiles` before any code specific to an Output Strategy runs, so the strategy doesn't change the result. A single Avro test (or at most `USE_VALUE` and `USE_WRAPPER`) would give the same protection with far fewer Kafka round trips. ########## nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaTest.java: ########## @@ -177,6 +184,55 @@ public void testVerifyFailed() throws InitializationException { assertNotNull(firstResult.getExplanation()); } + @Test + public void testVerifySkipsTombstoneAndParsesFollowingRecord() throws Exception { + final RecordReaderFactory readerFactory = recordReaderFactory(); + final RecordReader recordReader = mock(RecordReader.class); + when(readerFactory.createRecordReader(any(), any(), anyLong(), any())).thenAnswer(invocation -> { + final long length = invocation.getArgument(2); + if (length == 0L) { + throw new IOException("empty value"); + } + + return recordReader; + }); + + final ByteRecord tombstone = new ByteRecord(TEST_TOPIC_NAME, FIRST_PARTITION, 0L, 0L, List.of(), null, new byte[0], 1, true); + final ByteRecord dataRecord = new ByteRecord(TEST_TOPIC_NAME, FIRST_PARTITION, 1L, 0L, List.of(), null, FIRST_RECORD_VALUE, 1); + when(kafkaConsumerService.poll(any(Duration.class))).thenReturn(List.of(tombstone, dataRecord)); + + final ConfigVerificationResult parseResult = parseResult(verifyRecords()); + assertEquals(Outcome.SUCCESSFUL, parseResult.getOutcome()); + assertEquals("Successfully parsed 1 records (skipped 1 tombstones)", parseResult.getExplanation()); + } + + @Test + public void testVerifySkipsWhenSampleContainsOnlyTombstones() throws Exception { + final RecordReaderFactory readerFactory = recordReaderFactory(); + final ByteRecord tombstone = new ByteRecord(TEST_TOPIC_NAME, FIRST_PARTITION, 0L, 0L, List.of(), null, new byte[0], 1, true); + when(kafkaConsumerService.poll(any(Duration.class))).thenReturn(List.of(tombstone)); + + final ConfigVerificationResult parseResult = parseResult(verifyRecords()); + assertEquals(Outcome.SKIPPED, parseResult.getOutcome()); + assertEquals("No records were parsed (skipped 1 tombstones)", parseResult.getExplanation()); + verify(readerFactory, never()).createRecordReader(any(), any(), anyLong(), any()); + } + + @Test + public void testVerifyReportsDataRecordWhenTombstonePrecedesParseFailure() throws Exception { + final RecordReaderFactory readerFactory = recordReaderFactory(); + when(readerFactory.createRecordReader(any(), any(), anyLong(), any())).thenThrow(new IOException("empty value")); + + final ByteRecord tombstone = new ByteRecord(TEST_TOPIC_NAME, FIRST_PARTITION, 0L, 0L, List.of(), null, new byte[0], 1, true); + final ByteRecord dataRecord = new ByteRecord(TEST_TOPIC_NAME, FIRST_PARTITION, 1L, 0L, List.of(), null, FIRST_RECORD_VALUE, 1); + when(kafkaConsumerService.poll(any(Duration.class))).thenReturn(List.of(tombstone, dataRecord)); + + final ConfigVerificationResult parseResult = parseResult(verifyRecords()); + assertEquals(Outcome.FAILED, parseResult.getOutcome()); + assertEquals("Failed to parse Record number 2: java.io.IOException: empty value", parseResult.getExplanation()); + verify(readerFactory).createRecordReader(any(), any(), anyLong(), any()); + } Review Comment: [Claude Opus-5.5] `testVerifySkipsTombstoneAndParsesFollowingRecord` and `testVerifyReportsDataRecordWhenTombstonePrecedesParseFailure` use the same two records and differ only in how the mock reader behaves. They could be combined into one test: run verify and check the successful result, then change the reader to throw and run verify again to check the failure message and record number. ########## nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/kafka/processors/consumer/convert/RecordStreamKafkaMessageConverterTest.java: ########## @@ -176,4 +179,40 @@ void testParseFailureIncrementsParseErrorCounterImmediately() throws Exception { KafkaFlowFileAttribute.KAFKA_PARTITION, Integer.toString(FIRST_PARTITION))), eq(CommitTiming.NOW)); } + + @Test + void testTombstoneSkipsReaderAndEmptyValueStillFailsParsing() throws Exception { + final OffsetTracker tombstoneOffsetTracker = new OffsetTracker(); + final RecordReaderFactory readerFactory = mock(RecordReaderFactory.class); + when(readerFactory.createRecordReader(any(), any(), anyLong(), any())) + .thenThrow(new MalformedRecordException("empty value")); + + final FlowFile flowFile = new MockFlowFile(1); + when(session.create()).thenReturn(flowFile); + when(session.putAllAttributes(any(FlowFile.class), any())).thenReturn(flowFile); + when(session.write(any(FlowFile.class), any(OutputStreamCallback.class))).thenReturn(flowFile); + + final RecordStreamKafkaMessageConverter converter = new RecordStreamKafkaMessageConverter( + readerFactory, + writerFactory, + value -> new String(value, StandardCharsets.UTF_8), + Pattern.compile(".*"), + KeyEncoding.UTF8, + true, + tombstoneOffsetTracker, + logger, + BROKER_URI, + new CreateNewFlowFileGrouping(writerFactory, logger, BROKER_URI, true) + ); + + final ByteRecord tombstone = new ByteRecord(FIRST_TOPIC, FIRST_PARTITION, 4L, 1000L, List.of(), null, new byte[0], 1L, true); + final ByteRecord emptyValue = new ByteRecord(FIRST_TOPIC, FIRST_PARTITION, 5L, 1000L, List.of(), null, new byte[0], 1L); + converter.toFlowFiles(session, List.of(tombstone, emptyValue).iterator()); + + verify(readerFactory, times(1)).createRecordReader(any(), any(), anyLong(), any()); + verify(session).transfer(flowFile, ConsumeKafka.PARSE_FAILURE); + verify(session, times(2)).adjustCounter(eq("Records Received from " + FIRST_TOPIC), eq(1L), eq(false)); + verify(session).adjustCounter(eq("Tombstones Skipped for " + FIRST_TOPIC), eq(1L), eq(false)); + assertEquals(2L, tombstoneOffsetTracker.getRecordCounts().get(FIRST_TOPIC)); + } Review Comment: [Claude Opus-5.5] This repeats nearly all of the setup in `testParseFailureIncrementsParseErrorCounterImmediately` just above. Could we add the tombstone record to that existing test instead, along with the counter and offset assertions from here? That keeps the number of tests the same. Also, the separate `tombstoneOffsetTracker` isn't needed: JUnit creates a new instance of the test class for every test, so the `offsetTracker` field is already fresh. And `times(1)` is Mockito's default, so `verify(readerFactory)` is enough. ########## nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaTest.java: ########## @@ -365,6 +421,32 @@ public void testOnTriggerDemarcatorRecordsOriginalKafkaBytes() throws Initializa runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); } + private RecordReaderFactory recordReaderFactory() throws InitializationException { Review Comment: [Claude Opus-5.5] Small naming suggestion: method names read best when they say what the method does. Besides creating the mock, this adds and enables the service and switches the processor to RECORD mode. Something like `configureRecordReader()` would make that clearer where it's called. ########## nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaTest.java: ########## @@ -365,6 +421,32 @@ public void testOnTriggerDemarcatorRecordsOriginalKafkaBytes() throws Initializa runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); } + private RecordReaderFactory recordReaderFactory() throws InitializationException { + final RecordReaderFactory readerFactory = mock(RecordReaderFactory.class); + when(readerFactory.getIdentifier()).thenReturn(READER_ID); + runner.addControllerService(READER_ID, readerFactory); + runner.enableControllerService(readerFactory); + runner.setProperty(ConsumeKafka.PROCESSING_STRATEGY, ProcessingStrategy.RECORD); + runner.setProperty(ConsumeKafka.RECORD_READER, READER_ID); + return readerFactory; + } + + private List<ConfigVerificationResult> verifyRecords() throws InitializationException { + when(kafkaConsumerService.getPartitionStates()).thenReturn(List.of(new PartitionState(TEST_TOPIC_NAME, FIRST_PARTITION))); + setConnectionService(); + when(kafkaConnectionService.getConsumerService(any())).thenReturn(kafkaConsumerService); + runner.setProperty(TOPICS, TEST_TOPIC_NAME); + runner.setProperty(GROUP_ID, CONSUMER_GROUP_ID); + return processor.verify(runner.getProcessContext(), runner.getLogger(), Collections.emptyMap()); + } + + private static ConfigVerificationResult parseResult(final List<ConfigVerificationResult> results) { Review Comment: [Claude Opus-5.5] Same idea here: `findParseResult()` would say what this method does. ########## nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaRecordTombstoneIT.java: ########## @@ -0,0 +1,260 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.kafka.processors; + +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.avro.Schema; +import org.apache.avro.file.DataFileWriter; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.nifi.avro.AvroReader; +import org.apache.nifi.kafka.processors.consumer.ProcessingStrategy; +import org.apache.nifi.kafka.processors.producer.wrapper.WrapperRecord; +import org.apache.nifi.kafka.service.api.consumer.AutoOffsetReset; +import org.apache.nifi.kafka.shared.property.OutputStrategy; +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.schema.access.SchemaAccessUtils; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; + +/** + * RECORD output for a batch that contains a non-null empty value, a null tombstone, and a real value. + * A null tombstone is omitted from success and parse.failure. A JSON reader emits only the real value. + * An Avro reader routes the non-null empty value to parse.failure and writes the real value to success. + */ +class ConsumeKafkaRecordTombstoneIT extends AbstractConsumeKafkaIT { + + private static final String NON_EMPTY_VALUE = "{\"name\":\"A\"}"; + + private static final byte[] NON_EMPTY_BYTES = NON_EMPTY_VALUE.getBytes(StandardCharsets.UTF_8); + + private static final byte[] EMPTY_BYTES = new byte[0]; + + private static final String AVRO_SCHEMA = """ + { + "type": "record", + "name": "message", + "fields": [ + {"name": "name", "type": "string"} + ] + } + """; + + private static final String MESSAGE_KEY = "key"; + + private static final int FIRST_PARTITION = 0; + + private static final Duration POLL_TIMEOUT = Duration.ofSeconds(30); + + private TestRunner runner; + + private String topic; + + @BeforeEach + void setRunner() throws InitializationException { + runner = TestRunners.newTestRunner(ConsumeKafka.class); + addKafkaConnectionService(runner); + runner.setProperty(ConsumeKafka.CONNECTION_SERVICE, CONNECTION_SERVICE_ID); + addRecordReaderService(runner); + addRecordWriterService(runner); + + topic = UUID.randomUUID().toString(); + runner.setProperty(ConsumeKafka.GROUP_ID, topic); + runner.setProperty(ConsumeKafka.TOPICS, topic); + runner.setProperty(ConsumeKafka.PROCESSING_STRATEGY, ProcessingStrategy.RECORD); + runner.setProperty(ConsumeKafka.AUTO_OFFSET_RESET, AutoOffsetReset.EARLIEST.getValue()); + } + + @Test + void testOutputStrategyUseValue() throws ExecutionException, InterruptedException, IOException { + setOutputStrategy(OutputStrategy.USE_VALUE); + produceJsonRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + assertEquals("A", record.get("name").asText()); + } + + @Test + void testOutputStrategyUseWrapper() throws ExecutionException, InterruptedException, IOException { + setOutputStrategy(OutputStrategy.USE_WRAPPER); + produceJsonRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + final ObjectNode value = assertInstanceOf(ObjectNode.class, record.get(WrapperRecord.VALUE)); + assertEquals("A", value.get("name").asText()); + } + + @Test + void testOutputStrategyInjectMetadata() throws ExecutionException, InterruptedException, IOException { + setOutputStrategy(OutputStrategy.INJECT_METADATA); + produceJsonRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + assertEquals("A", record.get("name").asText()); + } + + @Test + void testOutputStrategyInjectOffset() throws ExecutionException, InterruptedException, IOException { + setOutputStrategy(OutputStrategy.INJECT_OFFSET); + produceJsonRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + assertEquals("A", record.get("name").asText()); + } + + @Test + void testOutputStrategyUseValueWithAvroReader() throws ExecutionException, InterruptedException, IOException, InitializationException { + setOutputStrategy(OutputStrategy.USE_VALUE); + useAvroReader(); + produceAvroRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + assertEquals("A", record.get("name").asText()); + } + + @Test + void testOutputStrategyUseWrapperWithAvroReader() throws ExecutionException, InterruptedException, IOException, InitializationException { + setOutputStrategy(OutputStrategy.USE_WRAPPER); + useAvroReader(); + produceAvroRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + final ObjectNode value = assertInstanceOf(ObjectNode.class, record.get(WrapperRecord.VALUE)); + assertEquals("A", value.get("name").asText()); + } + + @Test + void testOutputStrategyInjectMetadataWithAvroReader() throws ExecutionException, InterruptedException, IOException, InitializationException { + setOutputStrategy(OutputStrategy.INJECT_METADATA); + useAvroReader(); + produceAvroRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + assertEquals("A", record.get("name").asText()); + } + + @Test + void testOutputStrategyInjectOffsetWithAvroReader() throws ExecutionException, InterruptedException, IOException, InitializationException { + setOutputStrategy(OutputStrategy.INJECT_OFFSET); + useAvroReader(); + produceAvroRecords(); + + runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1); + runner.assertTransferCount(ConsumeKafka.SUCCESS, 1); + final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst(); + final ArrayNode records = assertInstanceOf(ArrayNode.class, objectMapper.readTree(flowFile.getContent())); + assertEquals(1, records.size()); + final ObjectNode record = assertInstanceOf(ObjectNode.class, records.get(0)); + assertEquals("A", record.get("name").asText()); + } + + private void setOutputStrategy(final OutputStrategy outputStrategy) { + runner.setProperty(ConsumeKafka.OUTPUT_STRATEGY, outputStrategy); + } + + private void produceJsonRecords() throws ExecutionException, InterruptedException { Review Comment: [Claude Opus-5.5] Naming suggestion: `produceJsonRecords()` and `produceAvroRecords()` also start the processor and run it until output appears, which the names don't suggest. Something like `produceAndConsumeJsonRecords()` / `produceAndConsumeAvroRecords()` would be clearer. Similarly, `avroRecord()` below would read better as `createAvroRecord()`. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
