[
https://issues.apache.org/jira/browse/NIFI-4142?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16124100#comment-16124100
]
ASF GitHub Bot commented on NIFI-4142:
--------------------------------------
Github user joewitt commented on a diff in the pull request:
https://github.com/apache/nifi/pull/2015#discussion_r132786485
--- Diff:
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ValidateRecord.java
---
@@ -0,0 +1,457 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.standard;
+
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+import org.apache.avro.Schema;
+import org.apache.avro.Schema.Parser;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.avro.AvroSchemaValidator;
+import org.apache.nifi.avro.AvroTypeUtil;
+import org.apache.nifi.components.AllowableValue;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.schema.access.SchemaField;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.schema.validation.SchemaValidationContext;
+import org.apache.nifi.schema.validation.StandardSchemaValidator;
+import org.apache.nifi.schemaregistry.services.SchemaRegistry;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.WriteResult;
+import org.apache.nifi.serialization.record.RawRecordWriter;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordSchema;
+import
org.apache.nifi.serialization.record.validation.RecordSchemaValidator;
+import
org.apache.nifi.serialization.record.validation.SchemaValidationResult;
+import org.apache.nifi.serialization.record.validation.ValidationError;
+
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@InputRequirement(Requirement.INPUT_REQUIRED)
+@Tags({"record", "schema", "validate"})
+@CapabilityDescription("Validates the Records of an incoming FlowFile
against a given schema. All records that adhere to the schema are routed to the
\"valid\" relationship while "
+ + "records that do not adhere to hte schema are routed to the
\"invalid\" relationship. It is therefore possible for a single incoming
FlowFile to be split into two individual "
+ + "FlowFiles if some records are valid according to the schema and
others are not. Any FlowFile that is routed to the \"invalid\" relationship
will emit a ROUTE Provenance Event "
+ + "with the Details field populated to explain why records were
invalid. In addition, to gain further explanation of why records were invalid,
DEBUG-level logging can be enabled "
+ + "for the \"org.apache.nifi.processors.standard.ValidateRecord\"
logger.")
+public class ValidateRecord extends AbstractProcessor {
+
+ static final AllowableValue SCHEMA_NAME_PROPERTY = new
AllowableValue("schema-name-property", "Use Schema Name Property",
+ "The schema to validate the data against is determined by looking
at the 'Schema Name' Property and looking up the schema in the configured
Schema Registry");
+ static final AllowableValue SCHEMA_TEXT_PROPERTY = new
AllowableValue("schema-text-property", "Use Schema Text Property",
+ "The schema to validate the data against is determined by looking
at the 'Schema Text' Property and parsing the schema as an Avro schema");
+ static final AllowableValue READER_SCHEMA = new
AllowableValue("reader-schema", "Used Reader's Schema",
--- End diff --
Ah here it is. "Used Reader's Schema" should be "Use Reader Schema".
> Implement a ValidateRecord Processor
> ------------------------------------
>
> Key: NIFI-4142
> URL: https://issues.apache.org/jira/browse/NIFI-4142
> Project: Apache NiFi
> Issue Type: New Feature
> Components: Extensions
> Reporter: Mark Payne
> Assignee: Mark Payne
> Fix For: 1.4.0
>
>
> We need a processor that is capable of validating that all Records in a
> FlowFile adhere to the proper schema.
> The Processor should be configured with a Record Reader and should route each
> record to either 'valid' or 'invalid' based on whether or not the record
> adheres to the reader's schema. A record would be invalid in any of the
> following cases:
> - Missing field that is required according to the schema
> - Extra field that is not present in schema (it should be configurable
> whether or not this is a failure)
> - Field requires coercion and strict type checking enabled (this should also
> be configurable)
> - Field is invalid, such as the value "hello" when it should be an integer
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
