rzepinskip commented on code in PR #11857:
URL: https://github.com/apache/nifi/pull/11857#discussion_r4492807803


##########
nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaRecordTombstoneIT.java:
##########
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors;
+
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.avro.Schema;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericDatumWriter;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.nifi.avro.AvroReader;
+import org.apache.nifi.kafka.processors.consumer.ProcessingStrategy;
+import org.apache.nifi.kafka.processors.producer.wrapper.WrapperRecord;
+import org.apache.nifi.kafka.service.api.consumer.AutoOffsetReset;
+import org.apache.nifi.kafka.shared.property.OutputStrategy;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.schema.access.SchemaAccessUtils;
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+
+/**
+ * RECORD output for a batch that contains a non-null empty value, a null 
tombstone, and a real value.
+ * A null tombstone is omitted from success and parse.failure. A JSON reader 
emits only the real value.
+ * An Avro reader routes the non-null empty value to parse.failure and writes 
the real value to success.
+ */
+class ConsumeKafkaRecordTombstoneIT extends AbstractConsumeKafkaIT {
+
+    private static final String NON_EMPTY_VALUE = "{\"name\":\"A\"}";
+
+    private static final byte[] NON_EMPTY_BYTES = 
NON_EMPTY_VALUE.getBytes(StandardCharsets.UTF_8);
+
+    private static final byte[] EMPTY_BYTES = new byte[0];
+
+    private static final String AVRO_SCHEMA = """
+            {
+              "type": "record",
+              "name": "message",
+              "fields": [
+                {"name": "name", "type": "string"}
+              ]
+            }
+            """;
+
+    private static final String MESSAGE_KEY = "key";
+
+    private static final int FIRST_PARTITION = 0;
+
+    private static final Duration POLL_TIMEOUT = Duration.ofSeconds(30);
+
+    private TestRunner runner;
+
+    private String topic;
+
+    @BeforeEach
+    void setRunner() throws InitializationException {
+        runner = TestRunners.newTestRunner(ConsumeKafka.class);
+        addKafkaConnectionService(runner);
+        runner.setProperty(ConsumeKafka.CONNECTION_SERVICE, 
CONNECTION_SERVICE_ID);
+        addRecordReaderService(runner);
+        addRecordWriterService(runner);
+
+        topic = UUID.randomUUID().toString();
+        runner.setProperty(ConsumeKafka.GROUP_ID, topic);
+        runner.setProperty(ConsumeKafka.TOPICS, topic);
+        runner.setProperty(ConsumeKafka.PROCESSING_STRATEGY, 
ProcessingStrategy.RECORD);
+        runner.setProperty(ConsumeKafka.AUTO_OFFSET_RESET, 
AutoOffsetReset.EARLIEST.getValue());
+    }
+
+    @Test
+    void testOutputStrategyUseValue() throws ExecutionException, 
InterruptedException, IOException {
+        setOutputStrategy(OutputStrategy.USE_VALUE);
+        produceJsonRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        assertEquals("A", record.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyUseWrapper() throws ExecutionException, 
InterruptedException, IOException {
+        setOutputStrategy(OutputStrategy.USE_WRAPPER);
+        produceJsonRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        final ObjectNode value = assertInstanceOf(ObjectNode.class, 
record.get(WrapperRecord.VALUE));
+        assertEquals("A", value.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyInjectMetadata() throws ExecutionException, 
InterruptedException, IOException {
+        setOutputStrategy(OutputStrategy.INJECT_METADATA);
+        produceJsonRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        assertEquals("A", record.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyInjectOffset() throws ExecutionException, 
InterruptedException, IOException {
+        setOutputStrategy(OutputStrategy.INJECT_OFFSET);
+        produceJsonRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        assertEquals("A", record.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyUseValueWithAvroReader() throws ExecutionException, 
InterruptedException, IOException, InitializationException {
+        setOutputStrategy(OutputStrategy.USE_VALUE);
+        useAvroReader();
+        produceAvroRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        assertEquals("A", record.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyUseWrapperWithAvroReader() throws 
ExecutionException, InterruptedException, IOException, InitializationException {
+        setOutputStrategy(OutputStrategy.USE_WRAPPER);
+        useAvroReader();
+        produceAvroRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        final ObjectNode value = assertInstanceOf(ObjectNode.class, 
record.get(WrapperRecord.VALUE));
+        assertEquals("A", value.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyInjectMetadataWithAvroReader() throws 
ExecutionException, InterruptedException, IOException, InitializationException {
+        setOutputStrategy(OutputStrategy.INJECT_METADATA);
+        useAvroReader();
+        produceAvroRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        assertEquals("A", record.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyInjectOffsetWithAvroReader() throws 
ExecutionException, InterruptedException, IOException, InitializationException {
+        setOutputStrategy(OutputStrategy.INJECT_OFFSET);
+        useAvroReader();
+        produceAvroRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        assertEquals("A", record.get("name").asText());
+    }

Review Comment:
   Right, we can bring them back if we decide to differentiate behavior by 
Output Strategy.



##########
nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaRecordTombstoneIT.java:
##########
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors;
+
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.avro.Schema;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericDatumWriter;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.nifi.avro.AvroReader;
+import org.apache.nifi.kafka.processors.consumer.ProcessingStrategy;
+import org.apache.nifi.kafka.processors.producer.wrapper.WrapperRecord;
+import org.apache.nifi.kafka.service.api.consumer.AutoOffsetReset;
+import org.apache.nifi.kafka.shared.property.OutputStrategy;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.schema.access.SchemaAccessUtils;
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+
+/**
+ * RECORD output for a batch that contains a non-null empty value, a null 
tombstone, and a real value.
+ * A null tombstone is omitted from success and parse.failure. A JSON reader 
emits only the real value.
+ * An Avro reader routes the non-null empty value to parse.failure and writes 
the real value to success.
+ */
+class ConsumeKafkaRecordTombstoneIT extends AbstractConsumeKafkaIT {
+
+    private static final String NON_EMPTY_VALUE = "{\"name\":\"A\"}";
+
+    private static final byte[] NON_EMPTY_BYTES = 
NON_EMPTY_VALUE.getBytes(StandardCharsets.UTF_8);
+
+    private static final byte[] EMPTY_BYTES = new byte[0];
+
+    private static final String AVRO_SCHEMA = """
+            {
+              "type": "record",
+              "name": "message",
+              "fields": [
+                {"name": "name", "type": "string"}
+              ]
+            }
+            """;
+
+    private static final String MESSAGE_KEY = "key";
+
+    private static final int FIRST_PARTITION = 0;
+
+    private static final Duration POLL_TIMEOUT = Duration.ofSeconds(30);
+
+    private TestRunner runner;
+
+    private String topic;
+
+    @BeforeEach
+    void setRunner() throws InitializationException {
+        runner = TestRunners.newTestRunner(ConsumeKafka.class);
+        addKafkaConnectionService(runner);
+        runner.setProperty(ConsumeKafka.CONNECTION_SERVICE, 
CONNECTION_SERVICE_ID);
+        addRecordReaderService(runner);
+        addRecordWriterService(runner);
+
+        topic = UUID.randomUUID().toString();
+        runner.setProperty(ConsumeKafka.GROUP_ID, topic);
+        runner.setProperty(ConsumeKafka.TOPICS, topic);
+        runner.setProperty(ConsumeKafka.PROCESSING_STRATEGY, 
ProcessingStrategy.RECORD);
+        runner.setProperty(ConsumeKafka.AUTO_OFFSET_RESET, 
AutoOffsetReset.EARLIEST.getValue());
+    }
+
+    @Test
+    void testOutputStrategyUseValue() throws ExecutionException, 
InterruptedException, IOException {
+        setOutputStrategy(OutputStrategy.USE_VALUE);
+        produceJsonRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        assertEquals("A", record.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyUseWrapper() throws ExecutionException, 
InterruptedException, IOException {
+        setOutputStrategy(OutputStrategy.USE_WRAPPER);
+        produceJsonRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        final ObjectNode value = assertInstanceOf(ObjectNode.class, 
record.get(WrapperRecord.VALUE));
+        assertEquals("A", value.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyInjectMetadata() throws ExecutionException, 
InterruptedException, IOException {
+        setOutputStrategy(OutputStrategy.INJECT_METADATA);
+        produceJsonRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        assertEquals("A", record.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyInjectOffset() throws ExecutionException, 
InterruptedException, IOException {
+        setOutputStrategy(OutputStrategy.INJECT_OFFSET);
+        produceJsonRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        assertEquals("A", record.get("name").asText());
+    }

Review Comment:
   Right, we can bring them back if we decide to differentiate behavior by 
Output Strategy.



##########
nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaTest.java:
##########
@@ -177,6 +184,55 @@ public void testVerifyFailed() throws 
InitializationException {
         assertNotNull(firstResult.getExplanation());
     }
 
+    @Test
+    public void testVerifySkipsTombstoneAndParsesFollowingRecord() throws 
Exception {
+        final RecordReaderFactory readerFactory = recordReaderFactory();
+        final RecordReader recordReader = mock(RecordReader.class);
+        when(readerFactory.createRecordReader(any(), any(), anyLong(), 
any())).thenAnswer(invocation -> {
+            final long length = invocation.getArgument(2);
+            if (length == 0L) {
+                throw new IOException("empty value");
+            }
+
+            return recordReader;
+        });
+
+        final ByteRecord tombstone = new ByteRecord(TEST_TOPIC_NAME, 
FIRST_PARTITION, 0L, 0L, List.of(), null, new byte[0], 1, true);
+        final ByteRecord dataRecord = new ByteRecord(TEST_TOPIC_NAME, 
FIRST_PARTITION, 1L, 0L, List.of(), null, FIRST_RECORD_VALUE, 1);
+        
when(kafkaConsumerService.poll(any(Duration.class))).thenReturn(List.of(tombstone,
 dataRecord));
+
+        final ConfigVerificationResult parseResult = 
parseResult(verifyRecords());
+        assertEquals(Outcome.SUCCESSFUL, parseResult.getOutcome());
+        assertEquals("Successfully parsed 1 records (skipped 1 tombstones)", 
parseResult.getExplanation());
+    }
+
+    @Test
+    public void testVerifySkipsWhenSampleContainsOnlyTombstones() throws 
Exception {
+        final RecordReaderFactory readerFactory = recordReaderFactory();
+        final ByteRecord tombstone = new ByteRecord(TEST_TOPIC_NAME, 
FIRST_PARTITION, 0L, 0L, List.of(), null, new byte[0], 1, true);
+        
when(kafkaConsumerService.poll(any(Duration.class))).thenReturn(List.of(tombstone));
+
+        final ConfigVerificationResult parseResult = 
parseResult(verifyRecords());
+        assertEquals(Outcome.SKIPPED, parseResult.getOutcome());
+        assertEquals("No records were parsed (skipped 1 tombstones)", 
parseResult.getExplanation());
+        verify(readerFactory, never()).createRecordReader(any(), any(), 
anyLong(), any());
+    }
+
+    @Test
+    public void testVerifyReportsDataRecordWhenTombstonePrecedesParseFailure() 
throws Exception {
+        final RecordReaderFactory readerFactory = recordReaderFactory();
+        when(readerFactory.createRecordReader(any(), any(), anyLong(), 
any())).thenThrow(new IOException("empty value"));
+
+        final ByteRecord tombstone = new ByteRecord(TEST_TOPIC_NAME, 
FIRST_PARTITION, 0L, 0L, List.of(), null, new byte[0], 1, true);
+        final ByteRecord dataRecord = new ByteRecord(TEST_TOPIC_NAME, 
FIRST_PARTITION, 1L, 0L, List.of(), null, FIRST_RECORD_VALUE, 1);
+        
when(kafkaConsumerService.poll(any(Duration.class))).thenReturn(List.of(tombstone,
 dataRecord));
+
+        final ConfigVerificationResult parseResult = 
parseResult(verifyRecords());
+        assertEquals(Outcome.FAILED, parseResult.getOutcome());
+        assertEquals("Failed to parse Record number 2: java.io.IOException: 
empty value", parseResult.getExplanation());
+        verify(readerFactory).createRecordReader(any(), any(), anyLong(), 
any());
+    }

Review Comment:
   I'd rather keep these as two tests. They share the same two records, but 
they check different verify() outcomes: a successful parse after skipping the 
tombstone, vs a failure whose record number still counts the skipped tombstone. 
Keeping them separate allows to make this outcomes explicit in test names.



##########
nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaTest.java:
##########
@@ -365,6 +421,32 @@ public void 
testOnTriggerDemarcatorRecordsOriginalKafkaBytes() throws Initializa
         runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
     }
 
+    private RecordReaderFactory recordReaderFactory() throws 
InitializationException {
+        final RecordReaderFactory readerFactory = 
mock(RecordReaderFactory.class);
+        when(readerFactory.getIdentifier()).thenReturn(READER_ID);
+        runner.addControllerService(READER_ID, readerFactory);
+        runner.enableControllerService(readerFactory);
+        runner.setProperty(ConsumeKafka.PROCESSING_STRATEGY, 
ProcessingStrategy.RECORD);
+        runner.setProperty(ConsumeKafka.RECORD_READER, READER_ID);
+        return readerFactory;
+    }
+
+    private List<ConfigVerificationResult> verifyRecords() throws 
InitializationException {
+        when(kafkaConsumerService.getPartitionStates()).thenReturn(List.of(new 
PartitionState(TEST_TOPIC_NAME, FIRST_PARTITION)));
+        setConnectionService();
+        
when(kafkaConnectionService.getConsumerService(any())).thenReturn(kafkaConsumerService);
+        runner.setProperty(TOPICS, TEST_TOPIC_NAME);
+        runner.setProperty(GROUP_ID, CONSUMER_GROUP_ID);
+        return processor.verify(runner.getProcessContext(), 
runner.getLogger(), Collections.emptyMap());
+    }
+
+    private static ConfigVerificationResult parseResult(final 
List<ConfigVerificationResult> results) {

Review Comment:
   Ok, fixed.



##########
nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-service-api/src/main/java/org/apache/nifi/kafka/service/api/record/ByteRecord.java:
##########
@@ -45,6 +46,20 @@ public ByteRecord(
             final byte[] key,
             final byte[] value,
             final long bundledCount
+    ) {
+        this(topic, partition, offset, timestamp, headers, key, value, 
bundledCount, false);
+    }
+
+    public ByteRecord(
+            final String topic,
+            final int partition,
+            final long offset,
+            final long timestamp,
+            final List<RecordHeader> headers,
+            final byte[] key,
+            final byte[] value,
+            final long bundledCount,
+            final boolean tombstone

Review Comment:
   The 8-arg constructor is the previous public API on nifi-kafka-service-api. 
Production still uses it in ByteRecordBundler (DEMARCATOR strategy), and 
defaulting tombstone to false is correct there (bundled records are not Kafka 
tombstones). 
   
   Kafka consumer code uses the new 9-arg constructor with the real flag (true 
or false depending on 
`org.apache.kafka.clients.consumer.ConsumerRecord.value()`). 
   
   I could remove existing 8-arg constructor and use new 9-arg one for 
ByteRecordBundler as well but wouldn't that be an API break (that would mean?



##########
nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-3-integration/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaRecordTombstoneIT.java:
##########
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.kafka.processors;
+
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.avro.Schema;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericDatumWriter;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.nifi.avro.AvroReader;
+import org.apache.nifi.kafka.processors.consumer.ProcessingStrategy;
+import org.apache.nifi.kafka.processors.producer.wrapper.WrapperRecord;
+import org.apache.nifi.kafka.service.api.consumer.AutoOffsetReset;
+import org.apache.nifi.kafka.shared.property.OutputStrategy;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.schema.access.SchemaAccessUtils;
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+
+/**
+ * RECORD output for a batch that contains a non-null empty value, a null 
tombstone, and a real value.
+ * A null tombstone is omitted from success and parse.failure. A JSON reader 
emits only the real value.
+ * An Avro reader routes the non-null empty value to parse.failure and writes 
the real value to success.
+ */
+class ConsumeKafkaRecordTombstoneIT extends AbstractConsumeKafkaIT {
+
+    private static final String NON_EMPTY_VALUE = "{\"name\":\"A\"}";
+
+    private static final byte[] NON_EMPTY_BYTES = 
NON_EMPTY_VALUE.getBytes(StandardCharsets.UTF_8);
+
+    private static final byte[] EMPTY_BYTES = new byte[0];
+
+    private static final String AVRO_SCHEMA = """
+            {
+              "type": "record",
+              "name": "message",
+              "fields": [
+                {"name": "name", "type": "string"}
+              ]
+            }
+            """;
+
+    private static final String MESSAGE_KEY = "key";
+
+    private static final int FIRST_PARTITION = 0;
+
+    private static final Duration POLL_TIMEOUT = Duration.ofSeconds(30);
+
+    private TestRunner runner;
+
+    private String topic;
+
+    @BeforeEach
+    void setRunner() throws InitializationException {
+        runner = TestRunners.newTestRunner(ConsumeKafka.class);
+        addKafkaConnectionService(runner);
+        runner.setProperty(ConsumeKafka.CONNECTION_SERVICE, 
CONNECTION_SERVICE_ID);
+        addRecordReaderService(runner);
+        addRecordWriterService(runner);
+
+        topic = UUID.randomUUID().toString();
+        runner.setProperty(ConsumeKafka.GROUP_ID, topic);
+        runner.setProperty(ConsumeKafka.TOPICS, topic);
+        runner.setProperty(ConsumeKafka.PROCESSING_STRATEGY, 
ProcessingStrategy.RECORD);
+        runner.setProperty(ConsumeKafka.AUTO_OFFSET_RESET, 
AutoOffsetReset.EARLIEST.getValue());
+    }
+
+    @Test
+    void testOutputStrategyUseValue() throws ExecutionException, 
InterruptedException, IOException {
+        setOutputStrategy(OutputStrategy.USE_VALUE);
+        produceJsonRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        assertEquals("A", record.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyUseWrapper() throws ExecutionException, 
InterruptedException, IOException {
+        setOutputStrategy(OutputStrategy.USE_WRAPPER);
+        produceJsonRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        final ObjectNode value = assertInstanceOf(ObjectNode.class, 
record.get(WrapperRecord.VALUE));
+        assertEquals("A", value.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyInjectMetadata() throws ExecutionException, 
InterruptedException, IOException {
+        setOutputStrategy(OutputStrategy.INJECT_METADATA);
+        produceJsonRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        assertEquals("A", record.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyInjectOffset() throws ExecutionException, 
InterruptedException, IOException {
+        setOutputStrategy(OutputStrategy.INJECT_OFFSET);
+        produceJsonRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 0);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        assertEquals("A", record.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyUseValueWithAvroReader() throws ExecutionException, 
InterruptedException, IOException, InitializationException {
+        setOutputStrategy(OutputStrategy.USE_VALUE);
+        useAvroReader();
+        produceAvroRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        assertEquals("A", record.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyUseWrapperWithAvroReader() throws 
ExecutionException, InterruptedException, IOException, InitializationException {
+        setOutputStrategy(OutputStrategy.USE_WRAPPER);
+        useAvroReader();
+        produceAvroRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        final ObjectNode value = assertInstanceOf(ObjectNode.class, 
record.get(WrapperRecord.VALUE));
+        assertEquals("A", value.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyInjectMetadataWithAvroReader() throws 
ExecutionException, InterruptedException, IOException, InitializationException {
+        setOutputStrategy(OutputStrategy.INJECT_METADATA);
+        useAvroReader();
+        produceAvroRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        assertEquals("A", record.get("name").asText());
+    }
+
+    @Test
+    void testOutputStrategyInjectOffsetWithAvroReader() throws 
ExecutionException, InterruptedException, IOException, InitializationException {
+        setOutputStrategy(OutputStrategy.INJECT_OFFSET);
+        useAvroReader();
+        produceAvroRecords();
+
+        runner.assertTransferCount(ConsumeKafka.PARSE_FAILURE, 1);
+        runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
+        final MockFlowFile flowFile = 
runner.getFlowFilesForRelationship(ConsumeKafka.SUCCESS).getFirst();
+        final ArrayNode records = assertInstanceOf(ArrayNode.class, 
objectMapper.readTree(flowFile.getContent()));
+        assertEquals(1, records.size());
+        final ObjectNode record = assertInstanceOf(ObjectNode.class, 
records.get(0));
+        assertEquals("A", record.get("name").asText());
+    }
+
+    private void setOutputStrategy(final OutputStrategy outputStrategy) {
+        runner.setProperty(ConsumeKafka.OUTPUT_STRATEGY, outputStrategy);
+    }
+
+    private void produceJsonRecords() throws ExecutionException, 
InterruptedException {

Review Comment:
   Ok, fixed.



##########
nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/kafka/processors/consumer/convert/RecordStreamKafkaMessageConverterTest.java:
##########
@@ -176,4 +179,40 @@ void 
testParseFailureIncrementsParseErrorCounterImmediately() throws Exception {
                         KafkaFlowFileAttribute.KAFKA_PARTITION, 
Integer.toString(FIRST_PARTITION))),
                 eq(CommitTiming.NOW));
     }
+
+    @Test
+    void testTombstoneSkipsReaderAndEmptyValueStillFailsParsing() throws 
Exception {
+        final OffsetTracker tombstoneOffsetTracker = new OffsetTracker();
+        final RecordReaderFactory readerFactory = 
mock(RecordReaderFactory.class);
+        when(readerFactory.createRecordReader(any(), any(), anyLong(), any()))
+                .thenThrow(new MalformedRecordException("empty value"));
+
+        final FlowFile flowFile = new MockFlowFile(1);
+        when(session.create()).thenReturn(flowFile);
+        when(session.putAllAttributes(any(FlowFile.class), 
any())).thenReturn(flowFile);
+        when(session.write(any(FlowFile.class), 
any(OutputStreamCallback.class))).thenReturn(flowFile);
+
+        final RecordStreamKafkaMessageConverter converter = new 
RecordStreamKafkaMessageConverter(
+                readerFactory,
+                writerFactory,
+                value -> new String(value, StandardCharsets.UTF_8),
+                Pattern.compile(".*"),
+                KeyEncoding.UTF8,
+                true,
+                tombstoneOffsetTracker,
+                logger,
+                BROKER_URI,
+                new CreateNewFlowFileGrouping(writerFactory, logger, 
BROKER_URI, true)
+        );
+
+        final ByteRecord tombstone = new ByteRecord(FIRST_TOPIC, 
FIRST_PARTITION, 4L, 1000L, List.of(), null, new byte[0], 1L, true);
+        final ByteRecord emptyValue = new ByteRecord(FIRST_TOPIC, 
FIRST_PARTITION, 5L, 1000L, List.of(), null, new byte[0], 1L);
+        converter.toFlowFiles(session, List.of(tombstone, 
emptyValue).iterator());
+
+        verify(readerFactory, times(1)).createRecordReader(any(), any(), 
anyLong(), any());
+        verify(session).transfer(flowFile, ConsumeKafka.PARSE_FAILURE);
+        verify(session, times(2)).adjustCounter(eq("Records Received from " + 
FIRST_TOPIC), eq(1L), eq(false));
+        verify(session).adjustCounter(eq("Tombstones Skipped for " + 
FIRST_TOPIC), eq(1L), eq(false));
+        assertEquals(2L, 
tombstoneOffsetTracker.getRecordCounts().get(FIRST_TOPIC));
+    }

Review Comment:
   Ok, fixed.



##########
nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/test/java/org/apache/nifi/kafka/processors/ConsumeKafkaTest.java:
##########
@@ -365,6 +421,32 @@ public void 
testOnTriggerDemarcatorRecordsOriginalKafkaBytes() throws Initializa
         runner.assertTransferCount(ConsumeKafka.SUCCESS, 1);
     }
 
+    private RecordReaderFactory recordReaderFactory() throws 
InitializationException {

Review Comment:
   Ok, fixed.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to