lordgamez commented on code in PR #1875: URL: https://github.com/apache/nifi-minifi-cpp/pull/1875#discussion_r1853752585
########## extensions/couchbase/controllerservices/CouchbaseClusterService.cpp: ########## @@ -0,0 +1,167 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "CouchbaseClusterService.h" +#include "couchbase/codec/raw_binary_transcoder.hxx" +#include "couchbase/codec/raw_string_transcoder.hxx" +#include "couchbase/codec/raw_json_transcoder.hxx" + +#include "core/Resource.h" + +namespace org::apache::nifi::minifi::couchbase { + +namespace { + +constexpr std::array<::couchbase::errc::common, 9> temporary_connection_errors = { + ::couchbase::errc::common::temporary_failure, + ::couchbase::errc::common::request_canceled, + ::couchbase::errc::common::internal_server_failure, + ::couchbase::errc::common::cas_mismatch, + ::couchbase::errc::common::ambiguous_timeout, + ::couchbase::errc::common::unambiguous_timeout, + ::couchbase::errc::common::rate_limited, + ::couchbase::errc::common::quota_limited +}; Review Comment: Good idea, updated in https://github.com/apache/nifi-minifi-cpp/pull/1875/commits/e99afe0d8127c14de5379fbd0f5ee8da7f2ace9c ########## extensions/couchbase/controllerservices/CouchbaseClusterService.h: ########## @@ -0,0 +1,162 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <memory> +#include <string> +#include <utility> +#include <mutex> + +#include "core/controller/ControllerService.h" +#include "core/PropertyDefinition.h" +#include "core/PropertyDefinitionBuilder.h" +#include "core/PropertyType.h" +#include "couchbase/cluster.hxx" +#include "core/ProcessContext.h" +#include "core/logging/LoggerConfiguration.h" + +namespace org::apache::nifi::minifi::couchbase { + +struct CouchbaseCollection { + std::string bucket_name; + std::string scope_name; + std::string collection_name; +}; + +struct CouchbaseUpsertResult { + std::string bucket_name; + std::uint64_t cas{0}; + std::uint64_t sequence_number{0}; + std::uint64_t partition_uuid{0}; + std::uint16_t partition_id{0}; +}; + +enum class CouchbaseValueType { + Json, + Binary, + String +}; + +enum class CouchbaseErrorType { + FATAL, + TEMPORARY, +}; + +class CouchbaseClient { + public: + CouchbaseClient(std::string connection_string, std::string username, std::string password, const std::shared_ptr<core::logging::Logger>& logger) + : connection_string_(std::move(connection_string)), username_(std::move(username)), password_(std::move(password)), logger_(logger) { + } + + ~CouchbaseClient() { + close(); + } + + CouchbaseClient(const CouchbaseClient&) = delete; + CouchbaseClient(CouchbaseClient&&) = delete; + CouchbaseClient& operator=(CouchbaseClient&&) = delete; + CouchbaseClient& operator=(const CouchbaseClient&) = delete; + + nonstd::expected<CouchbaseUpsertResult, CouchbaseErrorType> upsert(const CouchbaseCollection& collection, CouchbaseValueType document_type, const std::string& document_id, + const std::vector<std::byte>& buffer, const ::couchbase::upsert_options& options); + nonstd::expected<void, CouchbaseErrorType> establishConnection(); + void close(); + + private: + nonstd::expected<::couchbase::collection, CouchbaseErrorType> getCollection(const CouchbaseCollection& collection); + + std::string connection_string_; + std::string username_; + std::string password_; + std::mutex cluster_mutex_; + std::optional<::couchbase::cluster> cluster_; + std::shared_ptr<core::logging::Logger> logger_; +}; + +namespace controllers { + +class CouchbaseClusterService : public core::controller::ControllerService { + public: + explicit CouchbaseClusterService(std::string_view name, const minifi::utils::Identifier &uuid = {}) + : ControllerService(name, uuid) { + } + + explicit CouchbaseClusterService(std::string_view name, const std::shared_ptr<Configure>& /*configuration*/) + : ControllerService(name) { + } + + EXTENSIONAPI static constexpr const char* Description = "Provides a centralized Couchbase connection and bucket passwords management. Bucket passwords can be specified via dynamic properties."; + + EXTENSIONAPI static constexpr auto ConnectionString = core::PropertyDefinitionBuilder<>::createProperty("Connection String") + .withDescription("The hostnames or ip addresses of the bootstraping nodes and optional parameters. Syntax) couchbase://node1,node2,nodeN?param1=value1¶m2=value2¶mN=valueN") Review Comment: I actually just copied the NiFi description where it is used like that, but I think it's better the latter way, updated in https://github.com/apache/nifi-minifi-cpp/pull/1875/commits/e99afe0d8127c14de5379fbd0f5ee8da7f2ace9c ########## LICENSE: ########## @@ -214,6 +214,7 @@ This product bundles 'Kubernetes Client Library for C' (kubernetes-client/c), wh This project bundles a configuration file from 'Kubernetes Metrics Server' (kubernetes-sigs/metrics-server), which is available under an ALv2 license This project bundles 'OpenSSL' which is available under an ALv2 license This project bundles 'gRPC' which is available under an ALv2 license +This project bundles 'couchbase-cxx-clien' which is available under an ALv2 license Review Comment: Fixed in https://github.com/apache/nifi-minifi-cpp/pull/1875/commits/e99afe0d8127c14de5379fbd0f5ee8da7f2ace9c ########## thirdparty/bustache/add-append.patch: ########## @@ -0,0 +1,42 @@ +fmt library 11.0.2 requires the implementation of the append function aside +from push_back method when formatting the bustache specific container + +diff --git a/include/bustache/model.hpp b/include/bustache/model.hpp +index 575969a..6df2cff 100644 +--- a/include/bustache/model.hpp ++++ b/include/bustache/model.hpp +@@ -8,7 +8,7 @@ + #define BUSTACHE_MODEL_HPP_INCLUDED + + #include <bustache/format.hpp> +-#include <version> ++#include <version> + #include <vector> + #include <cstring> + #include <concepts> +@@ -330,6 +330,16 @@ namespace bustache::detail + buf[count++] = c; + } + ++ template <typename U> ++ void append(const U* begin, const U* end) ++ { ++ size_t content_count = end - begin; ++ for (size_t i = 0; i < content_count; ++i) { ++ push_back(begin[i]); ++ } ++ begin += content_count; ++ } Review Comment: Good point, updated in https://github.com/apache/nifi-minifi-cpp/pull/1875/commits/e99afe0d8127c14de5379fbd0f5ee8da7f2ace9c ########## libminifi/include/Exception.h: ########## @@ -43,11 +43,12 @@ enum ExceptionType { REGEX_EXCEPTION, REPOSITORY_EXCEPTION, PARAMETER_EXCEPTION, + CONTROLLER_ENABLE_EXCEPTION, Review Comment: Removed in https://github.com/apache/nifi-minifi-cpp/pull/1875/commits/e99afe0d8127c14de5379fbd0f5ee8da7f2ace9c ########## thirdparty/couchbase/remove-thirdparty.patch: ########## Review Comment: Agreed, updated in https://github.com/apache/nifi-minifi-cpp/pull/1875/commits/e99afe0d8127c14de5379fbd0f5ee8da7f2ace9c ########## extensions/couchbase/controllerservices/CouchbaseClusterService.cpp: ########## @@ -0,0 +1,167 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "CouchbaseClusterService.h" +#include "couchbase/codec/raw_binary_transcoder.hxx" +#include "couchbase/codec/raw_string_transcoder.hxx" +#include "couchbase/codec/raw_json_transcoder.hxx" + +#include "core/Resource.h" + +namespace org::apache::nifi::minifi::couchbase { + +namespace { + +constexpr std::array<::couchbase::errc::common, 9> temporary_connection_errors = { + ::couchbase::errc::common::temporary_failure, + ::couchbase::errc::common::request_canceled, + ::couchbase::errc::common::internal_server_failure, + ::couchbase::errc::common::cas_mismatch, + ::couchbase::errc::common::ambiguous_timeout, + ::couchbase::errc::common::unambiguous_timeout, + ::couchbase::errc::common::rate_limited, + ::couchbase::errc::common::quota_limited +}; + +CouchbaseErrorType getErrorType(const std::error_code& error_code) { + for (const auto& temporary_error : temporary_connection_errors) { + if (static_cast<int>(temporary_error) == error_code.value()) { + return CouchbaseErrorType::TEMPORARY; + } + } + return CouchbaseErrorType::FATAL; +} + +} // namespace + +nonstd::expected<::couchbase::collection, CouchbaseErrorType> CouchbaseClient::getCollection(const CouchbaseCollection& collection) { + auto connection_result = establishConnection(); + if (!connection_result) { + return nonstd::make_unexpected(connection_result.error()); + } + std::lock_guard<std::mutex> lock(cluster_mutex_); + return cluster_->bucket(collection.bucket_name).scope(collection.scope_name).collection(collection.collection_name); +} + +nonstd::expected<CouchbaseUpsertResult, CouchbaseErrorType> CouchbaseClient::upsert(const CouchbaseCollection& collection, + CouchbaseValueType document_type, const std::string& document_id, const std::vector<std::byte>& buffer, const ::couchbase::upsert_options& options) { + auto collection_result = getCollection(collection); + if (!collection_result.has_value()) { + return nonstd::make_unexpected(collection_result.error()); + } + + std::pair<::couchbase::error, ::couchbase::mutation_result> result; + if (document_type == CouchbaseValueType::Json) { + result = collection_result->upsert<::couchbase::codec::raw_json_transcoder>(document_id, buffer, options).get(); + } else if (document_type == CouchbaseValueType::String) { + std::string data_str(reinterpret_cast<const char*>(buffer.data()), buffer.size()); + result = collection_result->upsert<::couchbase::codec::raw_string_transcoder>(document_id, data_str, options).get(); + } else { + result = collection_result->upsert<::couchbase::codec::raw_binary_transcoder>(document_id, buffer, options).get(); + } + auto& [upsert_err, upsert_resp] = result; + if (upsert_err.ec()) { + // ambiguous_timeout should not be retried as we do not know if the insert was successful or not + if (getErrorType(upsert_err.ec()) == CouchbaseErrorType::TEMPORARY && upsert_err.ec().value() != static_cast<int>(::couchbase::errc::common::ambiguous_timeout)) { + logger_->log_error("Failed to upsert document '{}' to collection '{}.{}.{}' due to temporary issue, error code: '{}', message: '{}'", + document_id, collection.bucket_name, collection.scope_name, collection.collection_name, upsert_err.ec(), upsert_err.message()); + return nonstd::make_unexpected(CouchbaseErrorType::TEMPORARY); + } + logger_->log_error("Failed to upsert document '{}' to collection '{}.{}.{}' with error code: '{}', message: '{}'", + document_id, collection.bucket_name, collection.scope_name, collection.collection_name, upsert_err.ec(), upsert_err.message()); + return nonstd::make_unexpected(CouchbaseErrorType::FATAL); + } else { + const uint64_t partition_uuid = (upsert_resp.mutation_token().has_value() ? upsert_resp.mutation_token()->partition_uuid() : 0); + const uint64_t sequence_number = (upsert_resp.mutation_token().has_value() ? upsert_resp.mutation_token()->sequence_number() : 0); + const uint16_t partition_id = (upsert_resp.mutation_token().has_value() ? upsert_resp.mutation_token()->partition_id() : 0); + return CouchbaseUpsertResult { + collection.bucket_name, + upsert_resp.cas().value(), + partition_uuid, + sequence_number, + partition_id + }; + } +} + +void CouchbaseClient::close() { + std::lock_guard<std::mutex> lock(cluster_mutex_); + if (cluster_) { + cluster_->close().wait(); Review Comment: Makes sense, updated in https://github.com/apache/nifi-minifi-cpp/pull/1875/commits/e99afe0d8127c14de5379fbd0f5ee8da7f2ace9c -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
