Copilot commented on code in PR #2291:
URL: https://github.com/apache/nifi-minifi-cpp/pull/2291#discussion_r4630261734


##########
extensions/atlas/ReportLineageToAtlas.cpp:
##########
@@ -0,0 +1,388 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "ReportLineageToAtlas.h"
+
+#include <algorithm>
+#include <regex>
+#include <string>
+#include <string_view>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "AtlasClient.h"
+#include "FlowPathBuilder.h"
+#include "core/Resource.h"
+#include "minifi-cpp/Exception.h"
+#include "minifi-cpp/core/ProcessContext.h"
+#include "utils/ParsingUtils.h"
+#include "utils/StringUtils.h"
+
+namespace org::apache::nifi::minifi::extensions::atlas {
+
+namespace {
+
+// Maps the extractor's neutral system name to the Atlas entity type this
+// reporting task chose to represent it with. Configurable systems (S3, 
filesystem)
+// consult the reporting task's properties before mapping.
+std::string atlasTypeFor(const Dataset& dataset, 
ReportLineageToAtlas::S3ModelVersion s3_version) {
+  if (dataset.system == "kafka") {
+    return "kafka_topic";
+  }
+  if (dataset.system == "s3") {
+    return s3_version == ReportLineageToAtlas::S3ModelVersion::V1 ? 
"aws_s3_pseudo_dir" : "aws_s3_v2_directory";
+  }
+  if (dataset.system == "file") {
+    return "fs_path";
+  }
+  if (dataset.system == "http") {
+    return "http_endpoint";
+  }
+  if (dataset.system == "jdbc") {
+    return "rdbms_instance";
+  }
+  // The remote port's real kind, agreed on by both instances so their 
entities merge in Atlas:
+  // a SEND targets a remote input port, a RECEIVE pulls from a remote output 
port.
+  if (dataset.system == "site-to-site-input-port") {
+    return "nifi_input_port";
+  }
+  if (dataset.system == "site-to-site-output-port") {
+    return "nifi_output_port";
+  }
+  return dataset.system;
+}
+
+// Reduces a filesystem path to its containing directory, i.e. strips the last
+// component. Applied only when Filesystem Path Level = DIRECTORY.
+std::string dirOf(std::string_view path) {
+  const auto last_slash = path.rfind('/');
+  if (last_slash == std::string::npos) {
+    return std::string{path};
+  }
+  return std::string{path.substr(0, last_slash + 1)};
+}
+
+}  // namespace
+
+void ReportLineageToAtlas::initialize() {
+  setSupportedProperties(Properties);
+}
+
+AtlasEntity ReportLineageToAtlas::datasetToEntity(const Dataset& dataset, 
FsPathLevel fs_path_level, S3ModelVersion s3_model_version,
+    const NamespaceResolver& namespace_resolver) {
+  AtlasEntity entry;
+  entry.type_name = atlasTypeFor(dataset, s3_model_version);
+  entry.string_attributes = dataset.attributes;
+  std::string identifier = dataset.identifier;
+  if (dataset.system == "file" && fs_path_level == FsPathLevel::DIRECTORY) {
+    identifier = dirOf(identifier);
+    // The entity now describes the directory, not the file, so `path` has to 
follow
+    // `name`. Leaving it as the file path would make the directory entity's 
`path`
+    // whichever file happened to be seen last, since Atlas upserts by 
qualifiedName.
+    entry.string_attributes["name"] = identifier;
+    entry.string_attributes["path"] = identifier;
+  }
+  const auto namespace_name = 
namespace_resolver.resolve(dataset.host.value_or(""));
+  entry.qualified_name = identifier + "@" + namespace_name;
+  // Atlas' base "Asset" trait declares `name` as mandatory. Every dataset 
type we emit
+  // (fs_path, kafka_topic, nifi_output_port, nifi_input_port, aws_s3_v2_*, 
hive_table)
+  // inherits from Asset, so a missing name gets the whole bulk POST rejected 
with
+  // "mandatory attribute value missing in type Asset". Use the identifier as a
+  // sensible default when the extractor didn't already supply a `name`.
+  if (!entry.string_attributes.contains("name")) {
+    entry.string_attributes["name"] = identifier;
+  }
+  return entry;
+}
+
+bool ReportLineageToAtlas::isRemoteNiFiPort(std::string_view system) {
+  return system == "site-to-site-input-port" || system == 
"site-to-site-output-port";
+}
+
+std::string ReportLineageToAtlas::resolveRemotePortName(AtlasClient& atlas, 
const AtlasEntity& entity, const std::string& fallback) {
+  if (const auto cached = remote_port_names_.find(entity.qualified_name); 
cached != remote_port_names_.end()) {
+    return cached->second;
+  }
+  auto existing = atlas.getEntityName(entity.type_name, entity.qualified_name);
+  if (!existing) {
+    // Couldn't tell — send the fallback rather than skip the entity, and 
don't cache, so
+    // the next trigger tries again.
+    logger_->log_debug("Could not read the current name of '{}': {}", 
entity.qualified_name, existing.error());
+    return fallback;
+  }
+  // No name yet, or the only name there is the one we put there: the owner 
hasn't named
+  // this port, so keep the fallback and keep checking. Caching here would 
freeze our
+  // placeholder in place and overwrite NiFi's name once it finally sets one.
+  if (!*existing || (*existing)->empty() || **existing == fallback) {
+    return fallback;
+  }
+  logger_->log_debug("Preserving existing name '{}' for remote port '{}'", 
**existing, entity.qualified_name);
+  return remote_port_names_.emplace(entity.qualified_name, 
**existing).first->second;
+}
+
+void ReportLineageToAtlas::onSchedule(core::reporting::ReportingTaskContext& 
context) {
+  atlas_base_urls_.clear();
+  if (auto atlas_urls = context.getProperty(AtlasUrls); atlas_urls && 
!atlas_urls->empty()) {
+    atlas_base_urls_ = utils::string::splitAndTrimRemovingEmpty(*atlas_urls, 
",");
+  }
+  if (atlas_base_urls_.empty()) {
+    throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Atlas URLs property is 
required");
+  }
+
+  atlas_username_ = std::nullopt;
+  if (auto username = context.getProperty(AtlasUsername); username && 
!username->empty()) {
+    atlas_username_ = *username;
+  }
+  atlas_password_ = std::nullopt;
+  if (auto password = context.getProperty(AtlasPassword); password && 
!password->empty()) {
+    atlas_password_ = *password;
+  }
+
+  ssl_context_service_ = nullptr;
+  if (auto service_name = context.getProperty(SSLContext); service_name && 
!service_name->empty()) {
+    auto service = context.getControllerService(*service_name, uuid_);
+    if (!service) {
+      throw Exception(PROCESS_SCHEDULE_EXCEPTION, "SSL Context Service '" + 
*service_name + "' not found");
+    }
+    ssl_context_service_ = 
std::dynamic_pointer_cast<controllers::SSLContextServiceInterface>(service);
+    if (!ssl_context_service_) {
+      throw Exception(PROCESS_SCHEDULE_EXCEPTION, "SSL Context Service '" + 
*service_name + "' is not an SSLContextServiceInterface");
+    }
+  }
+
+  if (auto nifi_url = context.getProperty(NiFiUrl); nifi_url) {
+    nifi_url_ = *nifi_url;
+  }
+
+  namespace_resolver_ = NamespaceResolver{};
+  if (auto default_namespace = context.getProperty(DefaultNamespace); 
default_namespace && !default_namespace->empty()) {
+    namespace_resolver_.setDefaultNamespace(*default_namespace);
+  } else {
+    throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Default Metadata Namespace 
property is required");
+  }
+  // Dynamic properties: hostnamePattern.<ns> = <regex1> <regex2> ...
+  for (const auto& key : context.getDynamicPropertyKeys()) {
+    static constexpr std::string_view prefix = "hostnamePattern.";
+    if (!key.starts_with(prefix)) {
+      continue;
+    }
+    const auto namespace_name = key.substr(prefix.size());
+    if (namespace_name.empty()) {
+      continue;
+    }
+    if (auto value = context.getRawDynamicProperty(key); value) {
+      try {
+        namespace_resolver_.addRuleFromWhitespaceList(namespace_name, *value);
+      } catch (const std::regex_error& err) {
+        // std::regex_error is a std::logic_error, not a minifi Exception, so 
without this
+        // it escapes onSchedule as a bare "Unexpected end of regex" with no 
hint as to
+        // which property is malformed.
+        throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Invalid regex in dynamic 
property '" + key + "': " + err.what());
+      }
+    }
+  }
+
+  if (auto s3_version = context.getProperty(AwsS3ModelVersion); s3_version) {
+    s3_model_version_ = (*s3_version == "v1") ? S3ModelVersion::V1 : 
S3ModelVersion::V2;
+  }
+  if (auto path_level = context.getProperty(FilesystemPathLevel); path_level) {
+    fs_path_level_ = (*path_level == "FILE") ? FsPathLevel::FILE : 
FsPathLevel::DIRECTORY;
+  }
+  if (auto batch_size = context.getProperty(ProvenanceBatchSize); batch_size) {
+    if (const auto parsed = parsing::parseIntegral<std::size_t>(*batch_size)) {
+      provenance_batch_size_ = *parsed;
+    }
+  }

Review Comment:
   A configured `Provenance Batch Size` of `0` is accepted, causing every 
trigger to read an empty batch and report no lineage. Malformed values are also 
silently ignored, leaving the previous or default size in use. Reject 
non-numeric and zero values during scheduling instead of silently accepting 
them.



##########
extensions/atlas/ReportLineageToAtlas.cpp:
##########
@@ -0,0 +1,388 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "ReportLineageToAtlas.h"
+
+#include <algorithm>
+#include <regex>
+#include <string>
+#include <string_view>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "AtlasClient.h"
+#include "FlowPathBuilder.h"
+#include "core/Resource.h"
+#include "minifi-cpp/Exception.h"
+#include "minifi-cpp/core/ProcessContext.h"
+#include "utils/ParsingUtils.h"
+#include "utils/StringUtils.h"
+
+namespace org::apache::nifi::minifi::extensions::atlas {
+
+namespace {
+
+// Maps the extractor's neutral system name to the Atlas entity type this
+// reporting task chose to represent it with. Configurable systems (S3, 
filesystem)
+// consult the reporting task's properties before mapping.
+std::string atlasTypeFor(const Dataset& dataset, 
ReportLineageToAtlas::S3ModelVersion s3_version) {
+  if (dataset.system == "kafka") {
+    return "kafka_topic";
+  }
+  if (dataset.system == "s3") {
+    return s3_version == ReportLineageToAtlas::S3ModelVersion::V1 ? 
"aws_s3_pseudo_dir" : "aws_s3_v2_directory";
+  }
+  if (dataset.system == "file") {
+    return "fs_path";
+  }
+  if (dataset.system == "http") {
+    return "http_endpoint";
+  }
+  if (dataset.system == "jdbc") {
+    return "rdbms_instance";
+  }
+  // The remote port's real kind, agreed on by both instances so their 
entities merge in Atlas:
+  // a SEND targets a remote input port, a RECEIVE pulls from a remote output 
port.
+  if (dataset.system == "site-to-site-input-port") {
+    return "nifi_input_port";
+  }
+  if (dataset.system == "site-to-site-output-port") {
+    return "nifi_output_port";
+  }
+  return dataset.system;
+}
+
+// Reduces a filesystem path to its containing directory, i.e. strips the last
+// component. Applied only when Filesystem Path Level = DIRECTORY.
+std::string dirOf(std::string_view path) {
+  const auto last_slash = path.rfind('/');
+  if (last_slash == std::string::npos) {
+    return std::string{path};
+  }
+  return std::string{path.substr(0, last_slash + 1)};
+}
+
+}  // namespace
+
+void ReportLineageToAtlas::initialize() {
+  setSupportedProperties(Properties);
+}
+
+AtlasEntity ReportLineageToAtlas::datasetToEntity(const Dataset& dataset, 
FsPathLevel fs_path_level, S3ModelVersion s3_model_version,
+    const NamespaceResolver& namespace_resolver) {
+  AtlasEntity entry;
+  entry.type_name = atlasTypeFor(dataset, s3_model_version);
+  entry.string_attributes = dataset.attributes;
+  std::string identifier = dataset.identifier;
+  if (dataset.system == "file" && fs_path_level == FsPathLevel::DIRECTORY) {
+    identifier = dirOf(identifier);
+    // The entity now describes the directory, not the file, so `path` has to 
follow
+    // `name`. Leaving it as the file path would make the directory entity's 
`path`
+    // whichever file happened to be seen last, since Atlas upserts by 
qualifiedName.
+    entry.string_attributes["name"] = identifier;
+    entry.string_attributes["path"] = identifier;
+  }
+  const auto namespace_name = 
namespace_resolver.resolve(dataset.host.value_or(""));
+  entry.qualified_name = identifier + "@" + namespace_name;
+  // Atlas' base "Asset" trait declares `name` as mandatory. Every dataset 
type we emit
+  // (fs_path, kafka_topic, nifi_output_port, nifi_input_port, aws_s3_v2_*, 
hive_table)
+  // inherits from Asset, so a missing name gets the whole bulk POST rejected 
with
+  // "mandatory attribute value missing in type Asset". Use the identifier as a
+  // sensible default when the extractor didn't already supply a `name`.
+  if (!entry.string_attributes.contains("name")) {
+    entry.string_attributes["name"] = identifier;
+  }
+  return entry;
+}
+
+bool ReportLineageToAtlas::isRemoteNiFiPort(std::string_view system) {
+  return system == "site-to-site-input-port" || system == 
"site-to-site-output-port";
+}
+
+std::string ReportLineageToAtlas::resolveRemotePortName(AtlasClient& atlas, 
const AtlasEntity& entity, const std::string& fallback) {
+  if (const auto cached = remote_port_names_.find(entity.qualified_name); 
cached != remote_port_names_.end()) {
+    return cached->second;
+  }
+  auto existing = atlas.getEntityName(entity.type_name, entity.qualified_name);
+  if (!existing) {
+    // Couldn't tell — send the fallback rather than skip the entity, and 
don't cache, so
+    // the next trigger tries again.
+    logger_->log_debug("Could not read the current name of '{}': {}", 
entity.qualified_name, existing.error());
+    return fallback;
+  }
+  // No name yet, or the only name there is the one we put there: the owner 
hasn't named
+  // this port, so keep the fallback and keep checking. Caching here would 
freeze our
+  // placeholder in place and overwrite NiFi's name once it finally sets one.
+  if (!*existing || (*existing)->empty() || **existing == fallback) {
+    return fallback;
+  }
+  logger_->log_debug("Preserving existing name '{}' for remote port '{}'", 
**existing, entity.qualified_name);
+  return remote_port_names_.emplace(entity.qualified_name, 
**existing).first->second;
+}
+
+void ReportLineageToAtlas::onSchedule(core::reporting::ReportingTaskContext& 
context) {
+  atlas_base_urls_.clear();
+  if (auto atlas_urls = context.getProperty(AtlasUrls); atlas_urls && 
!atlas_urls->empty()) {
+    atlas_base_urls_ = utils::string::splitAndTrimRemovingEmpty(*atlas_urls, 
",");
+  }
+  if (atlas_base_urls_.empty()) {
+    throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Atlas URLs property is 
required");
+  }
+
+  atlas_username_ = std::nullopt;
+  if (auto username = context.getProperty(AtlasUsername); username && 
!username->empty()) {
+    atlas_username_ = *username;
+  }
+  atlas_password_ = std::nullopt;
+  if (auto password = context.getProperty(AtlasPassword); password && 
!password->empty()) {
+    atlas_password_ = *password;
+  }
+
+  ssl_context_service_ = nullptr;
+  if (auto service_name = context.getProperty(SSLContext); service_name && 
!service_name->empty()) {
+    auto service = context.getControllerService(*service_name, uuid_);
+    if (!service) {
+      throw Exception(PROCESS_SCHEDULE_EXCEPTION, "SSL Context Service '" + 
*service_name + "' not found");
+    }
+    ssl_context_service_ = 
std::dynamic_pointer_cast<controllers::SSLContextServiceInterface>(service);
+    if (!ssl_context_service_) {
+      throw Exception(PROCESS_SCHEDULE_EXCEPTION, "SSL Context Service '" + 
*service_name + "' is not an SSLContextServiceInterface");
+    }
+  }
+
+  if (auto nifi_url = context.getProperty(NiFiUrl); nifi_url) {
+    nifi_url_ = *nifi_url;
+  }
+
+  namespace_resolver_ = NamespaceResolver{};
+  if (auto default_namespace = context.getProperty(DefaultNamespace); 
default_namespace && !default_namespace->empty()) {
+    namespace_resolver_.setDefaultNamespace(*default_namespace);
+  } else {
+    throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Default Metadata Namespace 
property is required");
+  }
+  // Dynamic properties: hostnamePattern.<ns> = <regex1> <regex2> ...
+  for (const auto& key : context.getDynamicPropertyKeys()) {
+    static constexpr std::string_view prefix = "hostnamePattern.";
+    if (!key.starts_with(prefix)) {
+      continue;
+    }
+    const auto namespace_name = key.substr(prefix.size());
+    if (namespace_name.empty()) {
+      continue;
+    }
+    if (auto value = context.getRawDynamicProperty(key); value) {
+      try {
+        namespace_resolver_.addRuleFromWhitespaceList(namespace_name, *value);
+      } catch (const std::regex_error& err) {
+        // std::regex_error is a std::logic_error, not a minifi Exception, so 
without this
+        // it escapes onSchedule as a bare "Unexpected end of regex" with no 
hint as to
+        // which property is malformed.
+        throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Invalid regex in dynamic 
property '" + key + "': " + err.what());
+      }
+    }
+  }
+
+  if (auto s3_version = context.getProperty(AwsS3ModelVersion); s3_version) {
+    s3_model_version_ = (*s3_version == "v1") ? S3ModelVersion::V1 : 
S3ModelVersion::V2;
+  }
+  if (auto path_level = context.getProperty(FilesystemPathLevel); path_level) {
+    fs_path_level_ = (*path_level == "FILE") ? FsPathLevel::FILE : 
FsPathLevel::DIRECTORY;
+  }
+  if (auto batch_size = context.getProperty(ProvenanceBatchSize); batch_size) {
+    if (const auto parsed = parsing::parseIntegral<std::size_t>(*batch_size)) {
+      provenance_batch_size_ = *parsed;
+    }
+  }
+
+  type_defs_registered_ = false;
+  logger_->log_info("ReportLineageToAtlas scheduled: atlas={} namespace={} 
batch={} paths={}",
+      atlas_base_urls_.front(), namespace_resolver_.defaultNamespace(), 
provenance_batch_size_,
+      fs_path_level_ == FsPathLevel::DIRECTORY ? "DIRECTORY" : "FILE");
+}
+
+void ReportLineageToAtlas::onTrigger(core::reporting::ReportingTaskContext& 
context) {
+  // 1. Fresh AtlasClient per trigger — matches NiFi's design, keeps auth state
+  //    scoped, and makes each trigger independently recoverable.
+  AtlasClient atlas{{
+      .base_urls = atlas_base_urls_,
+      .username = atlas_username_,
+      .password = atlas_password_,
+      .ssl_context_service = ssl_context_service_,
+      .logger = logger_,
+  }};
+
+  // 2. Ensure both typedef bundles exist, on the first successful trigger. 
The flag is
+  //    set only on success: latching it after a partial registration would 
leave the
+  //    schema incomplete with no further attempt to repair it, and every 
later publish
+  //    would fail with "Type ENTITY ... does not exist".
+  if (!type_defs_registered_) {
+    if (auto res = atlas.registerTypeDefs(); !res) {
+      logger_->log_error("Failed to register Atlas typedefs: {}", res.error());
+      context.yield();
+      return;
+    }
+    type_defs_registered_ = true;
+  }
+
+  // 3. Snapshot the flow topology and translate it into Atlas entities. The 
flow rarely
+  //    changes, so skip both the translation and the (blocking) publish while 
it matches
+  //    what we last published successfully. The caches are updated only on 
success, so a
+  //    failed publish is retried on the next trigger.
+  auto topology = context.getFlowTopology();
+  if (!published_topology_ || *published_topology_ != topology) {
+    auto flow_result = FlowPathBuilder::build(topology, 
namespace_resolver_.defaultNamespace(), nifi_url_);
+    if (auto res = atlas.createOrUpdateEntities(flow_result.entities); !res) {
+      logger_->log_error("Failed to publish flow topology to Atlas: {}", 
res.error());
+      context.yield();
+      return;
+    }
+    logger_->log_debug("Published flow topology to Atlas: {} entities", 
flow_result.entities.size());
+    published_flow_result_ = std::move(flow_result);
+    published_topology_ = std::move(topology);
+  } else {
+    logger_->log_debug("Flow topology unchanged since last publish; skipping 
republish");
+  }
+  const auto& flow_result = *published_flow_result_;
+
+  // 4. Consume provenance events, dispatch each through the extractors, and
+  //    accumulate inputs/outputs per owning flow_path. Persist the cursor so
+  //    a restart resumes where we left off.
+  auto repo = context.getProvenanceRepository();
+  if (!repo) {
+    logger_->log_error("No provenance repository available; cannot report 
lineage edges");
+    context.yield();
+    return;
+  }
+  auto* state_manager = context.getStateManager();
+  if (!state_manager) {
+    logger_->log_error("No state manager available; cannot checkpoint 
provenance cursor");
+    context.yield();
+    return;
+  }
+
+  std::string cursor_str;
+  {
+    std::unordered_map<std::string, std::string> state;
+    if (state_manager->get(state) && state.contains("cursor")) {
+      cursor_str = state.at("cursor");
+    }
+  }
+  auto cursor = repo->cursorFromString(cursor_str);
+  if (!cursor_str.empty() && !cursor) {
+    logger_->log_warn("Failed to parse persisted cursor; starting from 
beginning");
+    cursor = repo->cursorFromString("");
+  }
+
+  auto batch_result = repo->getEvents(provenance_batch_size_, cursor.get());
+  if (!batch_result) {
+    logger_->log_error("Failed to read provenance events: {}", 
batch_result.error());
+    context.yield();
+    return;
+  }
+  const auto& events = batch_result.value();
+  if (events.empty()) {
+    logger_->log_debug("No new provenance events");
+    return;
+  }
+
+  std::unordered_map<std::string, AtlasEntity> external_entities;
+  // For each event, look up the owning flow path (via componentId) and append
+  // the extracted Datasets to that path's accumulated inputs/outputs.
+  std::unordered_map<std::string, std::unordered_set<AtlasEntity::Reference>> 
path_inputs;
+  std::unordered_map<std::string, std::unordered_set<AtlasEntity::Reference>> 
path_outputs;
+  for (const auto& event : events) {
+    if (!event) {
+      continue;
+    }
+    const auto refs = extractor_dispatcher_.dispatch(*event);
+    if (refs.empty()) {
+      continue;
+    }
+    const auto proc_it = 
flow_result.processor_to_path.find(event->getComponentId());
+    if (proc_it == flow_result.processor_to_path.end()) {
+      // Event from a processor no longer in the flow (deleted since last 
trigger?). Skip.
+      continue;
+    }
+    const auto& path_qn = proc_it->second.flow_path_qualified_name;
+    const auto convert = [&](const Dataset& dataset) {
+      auto entry = datasetToEntity(dataset, fs_path_level_, s3_model_version_, 
namespace_resolver_);
+      // A remote port belongs to the NiFi on the other end, which describes 
it too. Atlas
+      // demands Asset.name on update as well as create, so we cannot simply 
leave it out;
+      // instead hand back the name that NiFi has already set, when it has set 
one.
+      if (isRemoteNiFiPort(dataset.system)) {
+        entry.string_attributes["name"] = resolveRemotePortName(atlas, entry, 
entry.string_attributes["name"]);
+      }
+      external_entities[entry.qualified_name] = entry;
+      return AtlasEntity::Reference{entry.type_name, entry.qualified_name};
+    };
+    for (const auto& input : refs.inputs) {
+      path_inputs[path_qn].insert(convert(input));
+    }
+    for (const auto& output : refs.outputs) {
+      path_outputs[path_qn].insert(convert(output));
+    }
+  }
+
+  // 5. Build partial-update entities: one minifi_flow_path per path that saw 
new
+  //    events, with the accumulated (deduped) inputs/outputs. Atlas upserts by
+  //    qualifiedName, so this merges into whatever's already there.
+  std::unordered_map<std::string, AtlasEntity> updates;
+  const auto touch_path = [&](const std::string& path_qn) -> AtlasEntity& {
+    auto entity_it = std::ranges::find_if(flow_result.entities, [&](const 
AtlasEntity& entity) {
+      return entity.qualified_name == path_qn;
+    });
+    return updates.insert({path_qn, *entity_it}).first->second;
+  };
+
+  for (auto& [path_qn, refs] : path_inputs) {
+    touch_path(path_qn).ref_list_attributes["inputs"] = refs;
+  }
+  for (auto& [path_qn, refs] : path_outputs) {
+    touch_path(path_qn).ref_list_attributes["outputs"] = refs;

Review Comment:
   Atlas bulk upserts replace supplied `inputs`/`outputs` arrays, so publishing 
file B in a later batch removes file A's lineage. These assignments also 
discard structural `minifi_queue` references copied from the topology. Merge 
each batch with topology edges and previously published lineage, restoring that 
state across restarts or using an additive Atlas update.



##########
extensions/atlas/NamespaceResolver.cpp:
##########
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "NamespaceResolver.h"
+
+#include <cctype>
+#include <utility>
+#include "utils/StringUtils.h"
+
+namespace org::apache::nifi::minifi::extensions::atlas {
+
+void NamespaceResolver::addRule(std::string namespace_name, 
std::vector<std::regex> patterns) {
+  if (patterns.empty() || namespace_name.empty()) {
+    return;
+  }
+  rules_.push_back({std::move(namespace_name), std::move(patterns)});
+}
+
+void NamespaceResolver::addRuleFromWhitespaceList(std::string namespace_name, 
std::string_view whitespace_separated_regexes) {
+  std::vector<std::regex> patterns;
+  for (const auto& raw : 
utils::string::splitAndTrimRemovingEmpty(whitespace_separated_regexes, " ")) {

Review Comment:
   The advertised whitespace-separated list is split only on literal spaces. A 
YAML block-scalar value with one hostname regex per line (or a tab-separated 
value) is compiled as a single regex containing that whitespace, so neither 
hostname matches and both silently use the default namespace. Please tokenize 
all whitespace, not just spaces.



##########
extensions/atlas/ReportLineageToAtlas.cpp:
##########
@@ -0,0 +1,388 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "ReportLineageToAtlas.h"
+
+#include <algorithm>
+#include <regex>
+#include <string>
+#include <string_view>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "AtlasClient.h"
+#include "FlowPathBuilder.h"
+#include "core/Resource.h"
+#include "minifi-cpp/Exception.h"
+#include "minifi-cpp/core/ProcessContext.h"
+#include "utils/ParsingUtils.h"
+#include "utils/StringUtils.h"
+
+namespace org::apache::nifi::minifi::extensions::atlas {
+
+namespace {
+
+// Maps the extractor's neutral system name to the Atlas entity type this
+// reporting task chose to represent it with. Configurable systems (S3, 
filesystem)
+// consult the reporting task's properties before mapping.
+std::string atlasTypeFor(const Dataset& dataset, 
ReportLineageToAtlas::S3ModelVersion s3_version) {
+  if (dataset.system == "kafka") {
+    return "kafka_topic";
+  }
+  if (dataset.system == "s3") {
+    return s3_version == ReportLineageToAtlas::S3ModelVersion::V1 ? 
"aws_s3_pseudo_dir" : "aws_s3_v2_directory";
+  }
+  if (dataset.system == "file") {
+    return "fs_path";
+  }
+  if (dataset.system == "http") {
+    return "http_endpoint";
+  }
+  if (dataset.system == "jdbc") {
+    return "rdbms_instance";
+  }
+  // The remote port's real kind, agreed on by both instances so their 
entities merge in Atlas:
+  // a SEND targets a remote input port, a RECEIVE pulls from a remote output 
port.
+  if (dataset.system == "site-to-site-input-port") {
+    return "nifi_input_port";
+  }
+  if (dataset.system == "site-to-site-output-port") {
+    return "nifi_output_port";
+  }
+  return dataset.system;
+}
+
+// Reduces a filesystem path to its containing directory, i.e. strips the last
+// component. Applied only when Filesystem Path Level = DIRECTORY.
+std::string dirOf(std::string_view path) {
+  const auto last_slash = path.rfind('/');
+  if (last_slash == std::string::npos) {
+    return std::string{path};
+  }
+  return std::string{path.substr(0, last_slash + 1)};
+}
+
+}  // namespace
+
+void ReportLineageToAtlas::initialize() {
+  setSupportedProperties(Properties);
+}
+
+AtlasEntity ReportLineageToAtlas::datasetToEntity(const Dataset& dataset, 
FsPathLevel fs_path_level, S3ModelVersion s3_model_version,
+    const NamespaceResolver& namespace_resolver) {
+  AtlasEntity entry;
+  entry.type_name = atlasTypeFor(dataset, s3_model_version);
+  entry.string_attributes = dataset.attributes;
+  std::string identifier = dataset.identifier;
+  if (dataset.system == "file" && fs_path_level == FsPathLevel::DIRECTORY) {
+    identifier = dirOf(identifier);
+    // The entity now describes the directory, not the file, so `path` has to 
follow
+    // `name`. Leaving it as the file path would make the directory entity's 
`path`
+    // whichever file happened to be seen last, since Atlas upserts by 
qualifiedName.
+    entry.string_attributes["name"] = identifier;
+    entry.string_attributes["path"] = identifier;
+  }
+  const auto namespace_name = 
namespace_resolver.resolve(dataset.host.value_or(""));
+  entry.qualified_name = identifier + "@" + namespace_name;
+  // Atlas' base "Asset" trait declares `name` as mandatory. Every dataset 
type we emit
+  // (fs_path, kafka_topic, nifi_output_port, nifi_input_port, aws_s3_v2_*, 
hive_table)
+  // inherits from Asset, so a missing name gets the whole bulk POST rejected 
with
+  // "mandatory attribute value missing in type Asset". Use the identifier as a
+  // sensible default when the extractor didn't already supply a `name`.
+  if (!entry.string_attributes.contains("name")) {
+    entry.string_attributes["name"] = identifier;
+  }
+  return entry;
+}
+
+bool ReportLineageToAtlas::isRemoteNiFiPort(std::string_view system) {
+  return system == "site-to-site-input-port" || system == 
"site-to-site-output-port";
+}
+
+std::string ReportLineageToAtlas::resolveRemotePortName(AtlasClient& atlas, 
const AtlasEntity& entity, const std::string& fallback) {
+  if (const auto cached = remote_port_names_.find(entity.qualified_name); 
cached != remote_port_names_.end()) {
+    return cached->second;
+  }
+  auto existing = atlas.getEntityName(entity.type_name, entity.qualified_name);
+  if (!existing) {
+    // Couldn't tell — send the fallback rather than skip the entity, and 
don't cache, so
+    // the next trigger tries again.
+    logger_->log_debug("Could not read the current name of '{}': {}", 
entity.qualified_name, existing.error());
+    return fallback;
+  }
+  // No name yet, or the only name there is the one we put there: the owner 
hasn't named
+  // this port, so keep the fallback and keep checking. Caching here would 
freeze our
+  // placeholder in place and overwrite NiFi's name once it finally sets one.
+  if (!*existing || (*existing)->empty() || **existing == fallback) {
+    return fallback;
+  }
+  logger_->log_debug("Preserving existing name '{}' for remote port '{}'", 
**existing, entity.qualified_name);
+  return remote_port_names_.emplace(entity.qualified_name, 
**existing).first->second;
+}
+
+void ReportLineageToAtlas::onSchedule(core::reporting::ReportingTaskContext& 
context) {
+  atlas_base_urls_.clear();
+  if (auto atlas_urls = context.getProperty(AtlasUrls); atlas_urls && 
!atlas_urls->empty()) {
+    atlas_base_urls_ = utils::string::splitAndTrimRemovingEmpty(*atlas_urls, 
",");
+  }
+  if (atlas_base_urls_.empty()) {
+    throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Atlas URLs property is 
required");
+  }
+
+  atlas_username_ = std::nullopt;
+  if (auto username = context.getProperty(AtlasUsername); username && 
!username->empty()) {
+    atlas_username_ = *username;
+  }
+  atlas_password_ = std::nullopt;
+  if (auto password = context.getProperty(AtlasPassword); password && 
!password->empty()) {
+    atlas_password_ = *password;
+  }
+
+  ssl_context_service_ = nullptr;
+  if (auto service_name = context.getProperty(SSLContext); service_name && 
!service_name->empty()) {
+    auto service = context.getControllerService(*service_name, uuid_);
+    if (!service) {
+      throw Exception(PROCESS_SCHEDULE_EXCEPTION, "SSL Context Service '" + 
*service_name + "' not found");
+    }
+    ssl_context_service_ = 
std::dynamic_pointer_cast<controllers::SSLContextServiceInterface>(service);
+    if (!ssl_context_service_) {
+      throw Exception(PROCESS_SCHEDULE_EXCEPTION, "SSL Context Service '" + 
*service_name + "' is not an SSLContextServiceInterface");
+    }
+  }
+
+  if (auto nifi_url = context.getProperty(NiFiUrl); nifi_url) {
+    nifi_url_ = *nifi_url;
+  }
+
+  namespace_resolver_ = NamespaceResolver{};
+  if (auto default_namespace = context.getProperty(DefaultNamespace); 
default_namespace && !default_namespace->empty()) {
+    namespace_resolver_.setDefaultNamespace(*default_namespace);
+  } else {
+    throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Default Metadata Namespace 
property is required");
+  }
+  // Dynamic properties: hostnamePattern.<ns> = <regex1> <regex2> ...
+  for (const auto& key : context.getDynamicPropertyKeys()) {
+    static constexpr std::string_view prefix = "hostnamePattern.";
+    if (!key.starts_with(prefix)) {
+      continue;
+    }
+    const auto namespace_name = key.substr(prefix.size());
+    if (namespace_name.empty()) {
+      continue;
+    }
+    if (auto value = context.getRawDynamicProperty(key); value) {
+      try {
+        namespace_resolver_.addRuleFromWhitespaceList(namespace_name, *value);
+      } catch (const std::regex_error& err) {
+        // std::regex_error is a std::logic_error, not a minifi Exception, so 
without this
+        // it escapes onSchedule as a bare "Unexpected end of regex" with no 
hint as to
+        // which property is malformed.
+        throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Invalid regex in dynamic 
property '" + key + "': " + err.what());
+      }
+    }
+  }
+
+  if (auto s3_version = context.getProperty(AwsS3ModelVersion); s3_version) {
+    s3_model_version_ = (*s3_version == "v1") ? S3ModelVersion::V1 : 
S3ModelVersion::V2;
+  }
+  if (auto path_level = context.getProperty(FilesystemPathLevel); path_level) {
+    fs_path_level_ = (*path_level == "FILE") ? FsPathLevel::FILE : 
FsPathLevel::DIRECTORY;
+  }
+  if (auto batch_size = context.getProperty(ProvenanceBatchSize); batch_size) {
+    if (const auto parsed = parsing::parseIntegral<std::size_t>(*batch_size)) {
+      provenance_batch_size_ = *parsed;
+    }
+  }
+
+  type_defs_registered_ = false;
+  logger_->log_info("ReportLineageToAtlas scheduled: atlas={} namespace={} 
batch={} paths={}",
+      atlas_base_urls_.front(), namespace_resolver_.defaultNamespace(), 
provenance_batch_size_,
+      fs_path_level_ == FsPathLevel::DIRECTORY ? "DIRECTORY" : "FILE");
+}
+
+void ReportLineageToAtlas::onTrigger(core::reporting::ReportingTaskContext& 
context) {
+  // 1. Fresh AtlasClient per trigger — matches NiFi's design, keeps auth state
+  //    scoped, and makes each trigger independently recoverable.
+  AtlasClient atlas{{
+      .base_urls = atlas_base_urls_,
+      .username = atlas_username_,
+      .password = atlas_password_,
+      .ssl_context_service = ssl_context_service_,
+      .logger = logger_,
+  }};
+
+  // 2. Ensure both typedef bundles exist, on the first successful trigger. 
The flag is
+  //    set only on success: latching it after a partial registration would 
leave the
+  //    schema incomplete with no further attempt to repair it, and every 
later publish
+  //    would fail with "Type ENTITY ... does not exist".
+  if (!type_defs_registered_) {
+    if (auto res = atlas.registerTypeDefs(); !res) {
+      logger_->log_error("Failed to register Atlas typedefs: {}", res.error());
+      context.yield();
+      return;
+    }
+    type_defs_registered_ = true;
+  }
+
+  // 3. Snapshot the flow topology and translate it into Atlas entities. The 
flow rarely
+  //    changes, so skip both the translation and the (blocking) publish while 
it matches
+  //    what we last published successfully. The caches are updated only on 
success, so a
+  //    failed publish is retried on the next trigger.
+  auto topology = context.getFlowTopology();
+  if (!published_topology_ || *published_topology_ != topology) {
+    auto flow_result = FlowPathBuilder::build(topology, 
namespace_resolver_.defaultNamespace(), nifi_url_);
+    if (auto res = atlas.createOrUpdateEntities(flow_result.entities); !res) {
+      logger_->log_error("Failed to publish flow topology to Atlas: {}", 
res.error());
+      context.yield();
+      return;
+    }
+    logger_->log_debug("Published flow topology to Atlas: {} entities", 
flow_result.entities.size());
+    published_flow_result_ = std::move(flow_result);
+    published_topology_ = std::move(topology);
+  } else {
+    logger_->log_debug("Flow topology unchanged since last publish; skipping 
republish");
+  }
+  const auto& flow_result = *published_flow_result_;
+
+  // 4. Consume provenance events, dispatch each through the extractors, and
+  //    accumulate inputs/outputs per owning flow_path. Persist the cursor so
+  //    a restart resumes where we left off.
+  auto repo = context.getProvenanceRepository();
+  if (!repo) {
+    logger_->log_error("No provenance repository available; cannot report 
lineage edges");
+    context.yield();
+    return;
+  }
+  auto* state_manager = context.getStateManager();
+  if (!state_manager) {
+    logger_->log_error("No state manager available; cannot checkpoint 
provenance cursor");
+    context.yield();
+    return;
+  }
+
+  std::string cursor_str;
+  {
+    std::unordered_map<std::string, std::string> state;
+    if (state_manager->get(state) && state.contains("cursor")) {
+      cursor_str = state.at("cursor");
+    }
+  }
+  auto cursor = repo->cursorFromString(cursor_str);
+  if (!cursor_str.empty() && !cursor) {
+    logger_->log_warn("Failed to parse persisted cursor; starting from 
beginning");
+    cursor = repo->cursorFromString("");
+  }
+
+  auto batch_result = repo->getEvents(provenance_batch_size_, cursor.get());
+  if (!batch_result) {
+    logger_->log_error("Failed to read provenance events: {}", 
batch_result.error());
+    context.yield();
+    return;
+  }
+  const auto& events = batch_result.value();
+  if (events.empty()) {
+    logger_->log_debug("No new provenance events");
+    return;
+  }
+
+  std::unordered_map<std::string, AtlasEntity> external_entities;
+  // For each event, look up the owning flow path (via componentId) and append
+  // the extracted Datasets to that path's accumulated inputs/outputs.
+  std::unordered_map<std::string, std::unordered_set<AtlasEntity::Reference>> 
path_inputs;
+  std::unordered_map<std::string, std::unordered_set<AtlasEntity::Reference>> 
path_outputs;
+  for (const auto& event : events) {
+    if (!event) {
+      continue;
+    }
+    const auto refs = extractor_dispatcher_.dispatch(*event);
+    if (refs.empty()) {
+      continue;
+    }
+    const auto proc_it = 
flow_result.processor_to_path.find(event->getComponentId());
+    if (proc_it == flow_result.processor_to_path.end()) {
+      // Event from a processor no longer in the flow (deleted since last 
trigger?). Skip.
+      continue;
+    }
+    const auto& path_qn = proc_it->second.flow_path_qualified_name;
+    const auto convert = [&](const Dataset& dataset) {
+      auto entry = datasetToEntity(dataset, fs_path_level_, s3_model_version_, 
namespace_resolver_);
+      // A remote port belongs to the NiFi on the other end, which describes 
it too. Atlas
+      // demands Asset.name on update as well as create, so we cannot simply 
leave it out;
+      // instead hand back the name that NiFi has already set, when it has set 
one.
+      if (isRemoteNiFiPort(dataset.system)) {
+        entry.string_attributes["name"] = resolveRemotePortName(atlas, entry, 
entry.string_attributes["name"]);
+      }

Review Comment:
   This performs an Atlas lookup for every event, even when hundreds of events 
in the batch target the same remote port. `resolveRemotePortName()` 
deliberately does not cache missing/fallback names, so when the receiving NiFi 
is not publishing its own names, the default 1,000-event batch can make 1,000 
serial GETs before its bulk POST. Please reuse the name already resolved for 
that port in `external_entities` for the remainder of the batch, while allowing 
another lookup next trigger.



##########
extensions/atlas/tests/features/containers/atlas_server_container.py:
##########
@@ -0,0 +1,257 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Wrapper around a single, shared Apache Atlas container.
+
+Atlas is heavy (HBase + Solr + Kafka + its own server, ~90s cold start), so 
the whole Atlas
+behave suite - merged into one feature file - shares ONE container from
+`adamdebreceni/atlas:latest` (see extensions/atlas/tests/features/README.md 
for how the
+image is built). `environment.py` deploys it once in `before_all` on a 
persistent network,
+runs every scenario's MiNiFi/NiFi/Kafka containers on that same network, and 
clears its
+entities between scenarios (`clear_entities()`) instead of restarting it. 
`deploy()` starts
+the container via the shared `LinuxContainer` base, resolves the 
Docker-assigned (ephemeral)
+REST host port, then polls `/api/atlas/admin/status` until ACTIVE (up to
+`COLD_START_TIMEOUT_SECONDS`).
+
+Atlas stays single-homed on its network for the whole run: MiNiFi reaches it 
over that
+network via `in_network_url` (`atlas-shared:21000`), and the host-side REST
+probes/assertions/cleanup in this class reach it via the published host port. 
We do NOT
+connect the running Atlas to per-scenario networks - multi-homing it breaks 
host access to
+its published port - so the scenario containers join Atlas's network instead.
+"""
+
+from __future__ import annotations
+
+import logging
+import urllib.parse
+
+import requests
+from docker.models.networks import Network
+from minifi_behave.containers.container_linux import LinuxContainer
+from minifi_behave.core.helpers import wait_for_condition
+from minifi_behave.core.minifi_test_context import MinifiTestContext
+from requests.auth import HTTPBasicAuth
+
+logger = logging.getLogger(__name__)
+
+
+class AtlasServerContainer(LinuxContainer):
+    """Single shared Apache Atlas container. Publishes the REST API on a 
Docker-assigned host port."""
+
+    IMAGE = "adamdebreceni/atlas:latest"

Review Comment:
   The suite unconditionally pulls the mutable `adamdebreceni/atlas:latest` 
image, so CI can run different Atlas environments for the same commit. Pin a 
reviewed image digest and document its build source or provide a reproducible 
Dockerfile; the referenced `extensions/atlas/tests/features/README.md` is 
absent.



##########
extensions/atlas/ReportLineageToAtlas.cpp:
##########
@@ -0,0 +1,388 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "ReportLineageToAtlas.h"
+
+#include <algorithm>
+#include <regex>
+#include <string>
+#include <string_view>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "AtlasClient.h"
+#include "FlowPathBuilder.h"
+#include "core/Resource.h"
+#include "minifi-cpp/Exception.h"
+#include "minifi-cpp/core/ProcessContext.h"
+#include "utils/ParsingUtils.h"
+#include "utils/StringUtils.h"
+
+namespace org::apache::nifi::minifi::extensions::atlas {
+
+namespace {
+
+// Maps the extractor's neutral system name to the Atlas entity type this
+// reporting task chose to represent it with. Configurable systems (S3, 
filesystem)
+// consult the reporting task's properties before mapping.
+std::string atlasTypeFor(const Dataset& dataset, 
ReportLineageToAtlas::S3ModelVersion s3_version) {
+  if (dataset.system == "kafka") {
+    return "kafka_topic";
+  }
+  if (dataset.system == "s3") {
+    return s3_version == ReportLineageToAtlas::S3ModelVersion::V1 ? 
"aws_s3_pseudo_dir" : "aws_s3_v2_directory";
+  }
+  if (dataset.system == "file") {
+    return "fs_path";
+  }
+  if (dataset.system == "http") {
+    return "http_endpoint";
+  }
+  if (dataset.system == "jdbc") {
+    return "rdbms_instance";
+  }
+  // The remote port's real kind, agreed on by both instances so their 
entities merge in Atlas:
+  // a SEND targets a remote input port, a RECEIVE pulls from a remote output 
port.
+  if (dataset.system == "site-to-site-input-port") {
+    return "nifi_input_port";
+  }
+  if (dataset.system == "site-to-site-output-port") {
+    return "nifi_output_port";
+  }
+  return dataset.system;
+}
+
+// Reduces a filesystem path to its containing directory, i.e. strips the last
+// component. Applied only when Filesystem Path Level = DIRECTORY.
+std::string dirOf(std::string_view path) {
+  const auto last_slash = path.rfind('/');
+  if (last_slash == std::string::npos) {
+    return std::string{path};
+  }
+  return std::string{path.substr(0, last_slash + 1)};
+}
+
+}  // namespace
+
+void ReportLineageToAtlas::initialize() {
+  setSupportedProperties(Properties);
+}
+
+AtlasEntity ReportLineageToAtlas::datasetToEntity(const Dataset& dataset, 
FsPathLevel fs_path_level, S3ModelVersion s3_model_version,
+    const NamespaceResolver& namespace_resolver) {
+  AtlasEntity entry;
+  entry.type_name = atlasTypeFor(dataset, s3_model_version);
+  entry.string_attributes = dataset.attributes;
+  std::string identifier = dataset.identifier;
+  if (dataset.system == "file" && fs_path_level == FsPathLevel::DIRECTORY) {
+    identifier = dirOf(identifier);
+    // The entity now describes the directory, not the file, so `path` has to 
follow
+    // `name`. Leaving it as the file path would make the directory entity's 
`path`
+    // whichever file happened to be seen last, since Atlas upserts by 
qualifiedName.
+    entry.string_attributes["name"] = identifier;
+    entry.string_attributes["path"] = identifier;
+  }
+  const auto namespace_name = 
namespace_resolver.resolve(dataset.host.value_or(""));
+  entry.qualified_name = identifier + "@" + namespace_name;
+  // Atlas' base "Asset" trait declares `name` as mandatory. Every dataset 
type we emit
+  // (fs_path, kafka_topic, nifi_output_port, nifi_input_port, aws_s3_v2_*, 
hive_table)
+  // inherits from Asset, so a missing name gets the whole bulk POST rejected 
with
+  // "mandatory attribute value missing in type Asset". Use the identifier as a
+  // sensible default when the extractor didn't already supply a `name`.
+  if (!entry.string_attributes.contains("name")) {
+    entry.string_attributes["name"] = identifier;
+  }
+  return entry;
+}
+
+bool ReportLineageToAtlas::isRemoteNiFiPort(std::string_view system) {
+  return system == "site-to-site-input-port" || system == 
"site-to-site-output-port";
+}
+
+std::string ReportLineageToAtlas::resolveRemotePortName(AtlasClient& atlas, 
const AtlasEntity& entity, const std::string& fallback) {
+  if (const auto cached = remote_port_names_.find(entity.qualified_name); 
cached != remote_port_names_.end()) {
+    return cached->second;
+  }
+  auto existing = atlas.getEntityName(entity.type_name, entity.qualified_name);
+  if (!existing) {
+    // Couldn't tell — send the fallback rather than skip the entity, and 
don't cache, so
+    // the next trigger tries again.
+    logger_->log_debug("Could not read the current name of '{}': {}", 
entity.qualified_name, existing.error());
+    return fallback;
+  }
+  // No name yet, or the only name there is the one we put there: the owner 
hasn't named
+  // this port, so keep the fallback and keep checking. Caching here would 
freeze our
+  // placeholder in place and overwrite NiFi's name once it finally sets one.
+  if (!*existing || (*existing)->empty() || **existing == fallback) {
+    return fallback;
+  }
+  logger_->log_debug("Preserving existing name '{}' for remote port '{}'", 
**existing, entity.qualified_name);
+  return remote_port_names_.emplace(entity.qualified_name, 
**existing).first->second;
+}
+
+void ReportLineageToAtlas::onSchedule(core::reporting::ReportingTaskContext& 
context) {
+  atlas_base_urls_.clear();
+  if (auto atlas_urls = context.getProperty(AtlasUrls); atlas_urls && 
!atlas_urls->empty()) {
+    atlas_base_urls_ = utils::string::splitAndTrimRemovingEmpty(*atlas_urls, 
",");
+  }
+  if (atlas_base_urls_.empty()) {
+    throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Atlas URLs property is 
required");
+  }
+
+  atlas_username_ = std::nullopt;
+  if (auto username = context.getProperty(AtlasUsername); username && 
!username->empty()) {
+    atlas_username_ = *username;
+  }
+  atlas_password_ = std::nullopt;
+  if (auto password = context.getProperty(AtlasPassword); password && 
!password->empty()) {
+    atlas_password_ = *password;
+  }
+
+  ssl_context_service_ = nullptr;
+  if (auto service_name = context.getProperty(SSLContext); service_name && 
!service_name->empty()) {
+    auto service = context.getControllerService(*service_name, uuid_);
+    if (!service) {
+      throw Exception(PROCESS_SCHEDULE_EXCEPTION, "SSL Context Service '" + 
*service_name + "' not found");
+    }
+    ssl_context_service_ = 
std::dynamic_pointer_cast<controllers::SSLContextServiceInterface>(service);
+    if (!ssl_context_service_) {
+      throw Exception(PROCESS_SCHEDULE_EXCEPTION, "SSL Context Service '" + 
*service_name + "' is not an SSLContextServiceInterface");
+    }
+  }
+
+  if (auto nifi_url = context.getProperty(NiFiUrl); nifi_url) {
+    nifi_url_ = *nifi_url;
+  }
+
+  namespace_resolver_ = NamespaceResolver{};
+  if (auto default_namespace = context.getProperty(DefaultNamespace); 
default_namespace && !default_namespace->empty()) {
+    namespace_resolver_.setDefaultNamespace(*default_namespace);
+  } else {
+    throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Default Metadata Namespace 
property is required");
+  }
+  // Dynamic properties: hostnamePattern.<ns> = <regex1> <regex2> ...
+  for (const auto& key : context.getDynamicPropertyKeys()) {
+    static constexpr std::string_view prefix = "hostnamePattern.";
+    if (!key.starts_with(prefix)) {
+      continue;
+    }
+    const auto namespace_name = key.substr(prefix.size());
+    if (namespace_name.empty()) {
+      continue;
+    }
+    if (auto value = context.getRawDynamicProperty(key); value) {
+      try {
+        namespace_resolver_.addRuleFromWhitespaceList(namespace_name, *value);
+      } catch (const std::regex_error& err) {
+        // std::regex_error is a std::logic_error, not a minifi Exception, so 
without this
+        // it escapes onSchedule as a bare "Unexpected end of regex" with no 
hint as to
+        // which property is malformed.
+        throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Invalid regex in dynamic 
property '" + key + "': " + err.what());
+      }
+    }
+  }
+
+  if (auto s3_version = context.getProperty(AwsS3ModelVersion); s3_version) {
+    s3_model_version_ = (*s3_version == "v1") ? S3ModelVersion::V1 : 
S3ModelVersion::V2;
+  }
+  if (auto path_level = context.getProperty(FilesystemPathLevel); path_level) {
+    fs_path_level_ = (*path_level == "FILE") ? FsPathLevel::FILE : 
FsPathLevel::DIRECTORY;
+  }
+  if (auto batch_size = context.getProperty(ProvenanceBatchSize); batch_size) {
+    if (const auto parsed = parsing::parseIntegral<std::size_t>(*batch_size)) {
+      provenance_batch_size_ = *parsed;
+    }
+  }
+
+  type_defs_registered_ = false;

Review Comment:
   `onSchedule()` can run again on the same task after stop/start, but only the 
typedef flag is reset. If the Atlas endpoint, default namespace, or flow URL is 
changed while stopped, an unchanged flow still matches `published_topology_`, 
so topology publication is skipped and lineage updates use the old cached 
qualified names/URL (or reference topology absent from the new Atlas server). 
Please invalidate the configuration-dependent caches when scheduling again.



##########
extensions/atlas/ReportLineageToAtlas.cpp:
##########
@@ -0,0 +1,388 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "ReportLineageToAtlas.h"
+
+#include <algorithm>
+#include <regex>
+#include <string>
+#include <string_view>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "AtlasClient.h"
+#include "FlowPathBuilder.h"
+#include "core/Resource.h"
+#include "minifi-cpp/Exception.h"
+#include "minifi-cpp/core/ProcessContext.h"
+#include "utils/ParsingUtils.h"
+#include "utils/StringUtils.h"
+
+namespace org::apache::nifi::minifi::extensions::atlas {
+
+namespace {
+
+// Maps the extractor's neutral system name to the Atlas entity type this
+// reporting task chose to represent it with. Configurable systems (S3, 
filesystem)
+// consult the reporting task's properties before mapping.
+std::string atlasTypeFor(const Dataset& dataset, 
ReportLineageToAtlas::S3ModelVersion s3_version) {
+  if (dataset.system == "kafka") {
+    return "kafka_topic";
+  }
+  if (dataset.system == "s3") {
+    return s3_version == ReportLineageToAtlas::S3ModelVersion::V1 ? 
"aws_s3_pseudo_dir" : "aws_s3_v2_directory";
+  }
+  if (dataset.system == "file") {
+    return "fs_path";
+  }
+  if (dataset.system == "http") {
+    return "http_endpoint";
+  }
+  if (dataset.system == "jdbc") {
+    return "rdbms_instance";
+  }
+  // The remote port's real kind, agreed on by both instances so their 
entities merge in Atlas:
+  // a SEND targets a remote input port, a RECEIVE pulls from a remote output 
port.
+  if (dataset.system == "site-to-site-input-port") {
+    return "nifi_input_port";
+  }
+  if (dataset.system == "site-to-site-output-port") {
+    return "nifi_output_port";
+  }
+  return dataset.system;
+}
+
+// Reduces a filesystem path to its containing directory, i.e. strips the last
+// component. Applied only when Filesystem Path Level = DIRECTORY.
+std::string dirOf(std::string_view path) {
+  const auto last_slash = path.rfind('/');
+  if (last_slash == std::string::npos) {
+    return std::string{path};
+  }
+  return std::string{path.substr(0, last_slash + 1)};
+}
+
+}  // namespace
+
+void ReportLineageToAtlas::initialize() {
+  setSupportedProperties(Properties);
+}
+
+AtlasEntity ReportLineageToAtlas::datasetToEntity(const Dataset& dataset, 
FsPathLevel fs_path_level, S3ModelVersion s3_model_version,
+    const NamespaceResolver& namespace_resolver) {
+  AtlasEntity entry;
+  entry.type_name = atlasTypeFor(dataset, s3_model_version);
+  entry.string_attributes = dataset.attributes;
+  std::string identifier = dataset.identifier;
+  if (dataset.system == "file" && fs_path_level == FsPathLevel::DIRECTORY) {
+    identifier = dirOf(identifier);
+    // The entity now describes the directory, not the file, so `path` has to 
follow
+    // `name`. Leaving it as the file path would make the directory entity's 
`path`
+    // whichever file happened to be seen last, since Atlas upserts by 
qualifiedName.
+    entry.string_attributes["name"] = identifier;
+    entry.string_attributes["path"] = identifier;
+  }
+  const auto namespace_name = 
namespace_resolver.resolve(dataset.host.value_or(""));
+  entry.qualified_name = identifier + "@" + namespace_name;
+  // Atlas' base "Asset" trait declares `name` as mandatory. Every dataset 
type we emit
+  // (fs_path, kafka_topic, nifi_output_port, nifi_input_port, aws_s3_v2_*, 
hive_table)
+  // inherits from Asset, so a missing name gets the whole bulk POST rejected 
with
+  // "mandatory attribute value missing in type Asset". Use the identifier as a
+  // sensible default when the extractor didn't already supply a `name`.
+  if (!entry.string_attributes.contains("name")) {
+    entry.string_attributes["name"] = identifier;
+  }
+  return entry;
+}
+
+bool ReportLineageToAtlas::isRemoteNiFiPort(std::string_view system) {
+  return system == "site-to-site-input-port" || system == 
"site-to-site-output-port";
+}
+
+std::string ReportLineageToAtlas::resolveRemotePortName(AtlasClient& atlas, 
const AtlasEntity& entity, const std::string& fallback) {
+  if (const auto cached = remote_port_names_.find(entity.qualified_name); 
cached != remote_port_names_.end()) {
+    return cached->second;
+  }
+  auto existing = atlas.getEntityName(entity.type_name, entity.qualified_name);
+  if (!existing) {
+    // Couldn't tell — send the fallback rather than skip the entity, and 
don't cache, so
+    // the next trigger tries again.
+    logger_->log_debug("Could not read the current name of '{}': {}", 
entity.qualified_name, existing.error());
+    return fallback;
+  }
+  // No name yet, or the only name there is the one we put there: the owner 
hasn't named
+  // this port, so keep the fallback and keep checking. Caching here would 
freeze our
+  // placeholder in place and overwrite NiFi's name once it finally sets one.
+  if (!*existing || (*existing)->empty() || **existing == fallback) {
+    return fallback;
+  }
+  logger_->log_debug("Preserving existing name '{}' for remote port '{}'", 
**existing, entity.qualified_name);
+  return remote_port_names_.emplace(entity.qualified_name, 
**existing).first->second;
+}
+
+void ReportLineageToAtlas::onSchedule(core::reporting::ReportingTaskContext& 
context) {
+  atlas_base_urls_.clear();
+  if (auto atlas_urls = context.getProperty(AtlasUrls); atlas_urls && 
!atlas_urls->empty()) {
+    atlas_base_urls_ = utils::string::splitAndTrimRemovingEmpty(*atlas_urls, 
",");
+  }
+  if (atlas_base_urls_.empty()) {
+    throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Atlas URLs property is 
required");
+  }
+
+  atlas_username_ = std::nullopt;
+  if (auto username = context.getProperty(AtlasUsername); username && 
!username->empty()) {
+    atlas_username_ = *username;
+  }
+  atlas_password_ = std::nullopt;
+  if (auto password = context.getProperty(AtlasPassword); password && 
!password->empty()) {
+    atlas_password_ = *password;
+  }
+
+  ssl_context_service_ = nullptr;
+  if (auto service_name = context.getProperty(SSLContext); service_name && 
!service_name->empty()) {
+    auto service = context.getControllerService(*service_name, uuid_);
+    if (!service) {
+      throw Exception(PROCESS_SCHEDULE_EXCEPTION, "SSL Context Service '" + 
*service_name + "' not found");
+    }
+    ssl_context_service_ = 
std::dynamic_pointer_cast<controllers::SSLContextServiceInterface>(service);
+    if (!ssl_context_service_) {
+      throw Exception(PROCESS_SCHEDULE_EXCEPTION, "SSL Context Service '" + 
*service_name + "' is not an SSLContextServiceInterface");
+    }
+  }
+
+  if (auto nifi_url = context.getProperty(NiFiUrl); nifi_url) {
+    nifi_url_ = *nifi_url;
+  }
+
+  namespace_resolver_ = NamespaceResolver{};
+  if (auto default_namespace = context.getProperty(DefaultNamespace); 
default_namespace && !default_namespace->empty()) {
+    namespace_resolver_.setDefaultNamespace(*default_namespace);
+  } else {
+    throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Default Metadata Namespace 
property is required");
+  }
+  // Dynamic properties: hostnamePattern.<ns> = <regex1> <regex2> ...
+  for (const auto& key : context.getDynamicPropertyKeys()) {
+    static constexpr std::string_view prefix = "hostnamePattern.";
+    if (!key.starts_with(prefix)) {
+      continue;
+    }
+    const auto namespace_name = key.substr(prefix.size());
+    if (namespace_name.empty()) {
+      continue;
+    }
+    if (auto value = context.getRawDynamicProperty(key); value) {
+      try {
+        namespace_resolver_.addRuleFromWhitespaceList(namespace_name, *value);
+      } catch (const std::regex_error& err) {
+        // std::regex_error is a std::logic_error, not a minifi Exception, so 
without this
+        // it escapes onSchedule as a bare "Unexpected end of regex" with no 
hint as to
+        // which property is malformed.
+        throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Invalid regex in dynamic 
property '" + key + "': " + err.what());
+      }
+    }
+  }
+
+  if (auto s3_version = context.getProperty(AwsS3ModelVersion); s3_version) {
+    s3_model_version_ = (*s3_version == "v1") ? S3ModelVersion::V1 : 
S3ModelVersion::V2;
+  }
+  if (auto path_level = context.getProperty(FilesystemPathLevel); path_level) {
+    fs_path_level_ = (*path_level == "FILE") ? FsPathLevel::FILE : 
FsPathLevel::DIRECTORY;
+  }
+  if (auto batch_size = context.getProperty(ProvenanceBatchSize); batch_size) {
+    if (const auto parsed = parsing::parseIntegral<std::size_t>(*batch_size)) {
+      provenance_batch_size_ = *parsed;
+    }
+  }
+
+  type_defs_registered_ = false;
+  logger_->log_info("ReportLineageToAtlas scheduled: atlas={} namespace={} 
batch={} paths={}",
+      atlas_base_urls_.front(), namespace_resolver_.defaultNamespace(), 
provenance_batch_size_,
+      fs_path_level_ == FsPathLevel::DIRECTORY ? "DIRECTORY" : "FILE");
+}
+
+void ReportLineageToAtlas::onTrigger(core::reporting::ReportingTaskContext& 
context) {

Review Comment:
   `remote_port_names_` survives triggers, so after NiFi renames a port, later 
batches write the cached old name back to Atlas. Clear this cache at the start 
of each trigger, retaining name reuse only within that batch.



##########
extensions/atlas/extractors/FilePathExtractor.h:
##########
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <optional>
+#include <regex>
+#include <string>
+#include <string_view>
+#include <utility>
+
+#include "DatasetExtractor.h"
+
+namespace org::apache::nifi::minifi::extensions::atlas::extractors {
+
+// Local filesystem: matched on transit URI (^file:/.*). Whether the 
identifier is
+// the file or its containing directory is controlled by the reporting task's
+// "Filesystem Path Level" property (dispatcher passes it via 
ExtractorContext).
+// This extractor emits the file path; the reporting task collapses it to a
+// directory at serialize time if configured to.
+class FilePathExtractor : public DatasetExtractor {
+ public:
+  std::optional<std::reference_wrapper<const std::regex>> transitUriPattern() 
const override {
+    static const std::regex pattern{"^file:/.*"};
+    return std::cref(pattern);
+  }
+
+  DatasetReferences extract(const provenance::ProvenanceEventRecord& event) 
const override {
+    DatasetReferences refs;
+    const auto uri = event.getTransitUri();
+    // Accept both "file:///abs/path" and "file:/abs/path"; extract everything 
after
+    // the scheme's leading "file:".
+    constexpr std::string_view scheme = "file:";
+    if (!uri.starts_with(scheme)) return refs;
+    auto path = std::string_view{uri}.substr(scheme.size());
+    // Strip leading //authority if present (RFC 3986). We accept 
authority-less
+    // file URIs since MiNiFi's PutFile emits "file:///path".
+    while (!path.empty() && path.front() == '/') {
+      if (path.size() >= 2 && path[1] == '/') {
+        path = path.substr(1);
+        continue;
+      }
+      break;
+    }
+    if (path.empty()) return refs;
+
+    Dataset ds{.system = "file", .identifier = std::string{path}, .host = 
std::string{"localhost"}, .attributes = {}};
+    ds.attributes["name"] = std::filesystem::path{path}.string();
+    ds.attributes["path"] = std::filesystem::path{path}.string();

Review Comment:
   The new `pathToUri()` producer encodes `/tmp/a b.txt` as 
`file:///tmp/a%20b.txt`, but this extractor publishes the escaped text as the 
filesystem identifier and attributes. It also turns `file://server/share/f` 
into `/server/share/f` on `localhost`, preventing namespace rules for the 
server from matching. Parse the optional authority separately, preserve the 
path's own leading slashes, and percent-decode the path once before building 
the dataset.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to