lordgamez commented on code in PR #1875: URL: https://github.com/apache/nifi-minifi-cpp/pull/1875#discussion_r1842239941
########## extensions/couchbase/controllerservices/CouchbaseClusterService.h: ########## @@ -0,0 +1,163 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <memory> +#include <string> +#include <utility> + +#include "core/controller/ControllerService.h" +#include "core/PropertyDefinition.h" +#include "core/PropertyDefinitionBuilder.h" +#include "core/PropertyType.h" +#include "couchbase/cluster.hxx" +#include "core/ProcessContext.h" +#include "core/logging/LoggerConfiguration.h" + +namespace org::apache::nifi::minifi::couchbase { + +struct CouchbaseCollection { + std::string bucket_name; + std::string scope_name; + std::string collection_name; +}; + +struct CouchbaseUpsertResult { + std::string bucket_name; + std::uint64_t cas{0}; + std::uint64_t sequence_number{0}; + std::uint64_t partition_uuid{0}; + std::uint16_t partition_id{0}; +}; + +enum class CouchbaseValueType { + Json, + Binary, + String +}; + +enum class CouchbaseErrorType { + FATAL, + TEMPORARY, +}; + +class CouchbaseClient { + public: + CouchbaseClient(std::string connection_string, std::string username, std::string password, const std::shared_ptr<core::logging::Logger>& logger) + : connection_string_(std::move(connection_string)), username_(std::move(username)), password_(std::move(password)), logger_(logger) { + } + + nonstd::expected<CouchbaseUpsertResult, CouchbaseErrorType> upsert(const CouchbaseCollection& collection, CouchbaseValueType document_type, const std::string& document_id, + const std::vector<std::byte>& buffer, const ::couchbase::upsert_options& options); + nonstd::expected<void, CouchbaseErrorType> establishConnection(); + void close(); + + private: + static constexpr std::array<::couchbase::errc::common, 9> temporary_connection_errors = { + ::couchbase::errc::common::temporary_failure, + ::couchbase::errc::common::request_canceled, + ::couchbase::errc::common::internal_server_failure, + ::couchbase::errc::common::cas_mismatch, + ::couchbase::errc::common::ambiguous_timeout, + ::couchbase::errc::common::unambiguous_timeout, + ::couchbase::errc::common::rate_limited, + ::couchbase::errc::common::quota_limited + }; Review Comment: Moved to .cpp file in https://github.com/apache/nifi-minifi-cpp/pull/1875/commits/1b2393bc8eeff24748c1d39cce0e3cea24aa5a4b I would keep the `getErrorType` function because we return the error type and also set it in testing and it it more explicit and readable that way than a bool would be. ########## extensions/couchbase/controllerservices/CouchbaseClusterService.cpp: ########## @@ -0,0 +1,148 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "CouchbaseClusterService.h" +#include "couchbase/codec/raw_binary_transcoder.hxx" +#include "couchbase/codec/raw_string_transcoder.hxx" +#include "couchbase/codec/raw_json_transcoder.hxx" + +#include "core/Resource.h" + +namespace org::apache::nifi::minifi::couchbase { + +CouchbaseErrorType CouchbaseClient::getErrorType(const std::error_code& error_code) { + for (const auto& temporary_error : temporary_connection_errors) { + if (static_cast<int>(temporary_error) == error_code.value()) { + return CouchbaseErrorType::TEMPORARY; + } + } + return CouchbaseErrorType::FATAL; +} + +nonstd::expected<::couchbase::collection, CouchbaseErrorType> CouchbaseClient::getCollection(const CouchbaseCollection& collection) { + auto connection_result = establishConnection(); + if (!connection_result) { + return nonstd::make_unexpected(connection_result.error()); + } + return cluster_->bucket(collection.bucket_name).scope(collection.scope_name).collection(collection.collection_name); +} + +nonstd::expected<CouchbaseUpsertResult, CouchbaseErrorType> CouchbaseClient::upsert(const CouchbaseCollection& collection, + CouchbaseValueType document_type, const std::string& document_id, const std::vector<std::byte>& buffer, const ::couchbase::upsert_options& options) { + auto collection_result = getCollection(collection); + if (!collection_result.has_value()) { + return nonstd::make_unexpected(collection_result.error()); + } + + std::pair<::couchbase::error, ::couchbase::mutation_result> result; + if (document_type == CouchbaseValueType::Json) { + result = collection_result->upsert<::couchbase::codec::raw_json_transcoder>(document_id, buffer, options).get(); + } else if (document_type == CouchbaseValueType::String) { + std::string data_str(reinterpret_cast<const char*>(buffer.data()), buffer.size()); + result = collection_result->upsert<::couchbase::codec::raw_string_transcoder>(document_id, data_str, options).get(); + } else { + result = collection_result->upsert<::couchbase::codec::raw_binary_transcoder>(document_id, buffer, options).get(); + } + auto& [upsert_err, upsert_resp] = result; + if (upsert_err.ec()) { + // ambiguous_timeout should not be retried as we do not know if the insert was successful or not + if (getErrorType(upsert_err.ec()) == CouchbaseErrorType::TEMPORARY && upsert_err.ec().value() != static_cast<int>(::couchbase::errc::common::ambiguous_timeout)) { + logger_->log_error("Failed to upsert document '{}' to collection '{}.{}.{}' due to temporary issue, error code: '{}', message: '{}'", + document_id, collection.bucket_name, collection.scope_name, collection.collection_name, upsert_err.ec(), upsert_err.message()); + return nonstd::make_unexpected(CouchbaseErrorType::TEMPORARY); + } + logger_->log_error("Failed to upsert document '{}' to collection '{}.{}.{}' with error code: '{}', message: '{}'", + document_id, collection.bucket_name, collection.scope_name, collection.collection_name, upsert_err.ec(), upsert_err.message()); + return nonstd::make_unexpected(CouchbaseErrorType::FATAL); + } else { + const uint64_t partition_uuid = (upsert_resp.mutation_token().has_value() ? upsert_resp.mutation_token()->partition_uuid() : 0); + const uint64_t sequence_number = (upsert_resp.mutation_token().has_value() ? upsert_resp.mutation_token()->sequence_number() : 0); + const uint16_t partition_id = (upsert_resp.mutation_token().has_value() ? upsert_resp.mutation_token()->partition_id() : 0); + return CouchbaseUpsertResult { + collection.bucket_name, + upsert_resp.cas().value(), + partition_uuid, + sequence_number, + partition_id + }; + } +} + +void CouchbaseClient::close() { + if (cluster_) { + cluster_->close().wait(); + } +} + Review Comment: Good point, updated in https://github.com/apache/nifi-minifi-cpp/pull/1875/commits/1b2393bc8eeff24748c1d39cce0e3cea24aa5a4b ########## extensions/couchbase/controllerservices/CouchbaseClusterService.cpp: ########## @@ -0,0 +1,148 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "CouchbaseClusterService.h" +#include "couchbase/codec/raw_binary_transcoder.hxx" +#include "couchbase/codec/raw_string_transcoder.hxx" +#include "couchbase/codec/raw_json_transcoder.hxx" + +#include "core/Resource.h" + +namespace org::apache::nifi::minifi::couchbase { + +CouchbaseErrorType CouchbaseClient::getErrorType(const std::error_code& error_code) { + for (const auto& temporary_error : temporary_connection_errors) { + if (static_cast<int>(temporary_error) == error_code.value()) { + return CouchbaseErrorType::TEMPORARY; + } + } + return CouchbaseErrorType::FATAL; +} + +nonstd::expected<::couchbase::collection, CouchbaseErrorType> CouchbaseClient::getCollection(const CouchbaseCollection& collection) { + auto connection_result = establishConnection(); + if (!connection_result) { + return nonstd::make_unexpected(connection_result.error()); + } + return cluster_->bucket(collection.bucket_name).scope(collection.scope_name).collection(collection.collection_name); +} + +nonstd::expected<CouchbaseUpsertResult, CouchbaseErrorType> CouchbaseClient::upsert(const CouchbaseCollection& collection, + CouchbaseValueType document_type, const std::string& document_id, const std::vector<std::byte>& buffer, const ::couchbase::upsert_options& options) { + auto collection_result = getCollection(collection); + if (!collection_result.has_value()) { + return nonstd::make_unexpected(collection_result.error()); + } + + std::pair<::couchbase::error, ::couchbase::mutation_result> result; + if (document_type == CouchbaseValueType::Json) { + result = collection_result->upsert<::couchbase::codec::raw_json_transcoder>(document_id, buffer, options).get(); + } else if (document_type == CouchbaseValueType::String) { + std::string data_str(reinterpret_cast<const char*>(buffer.data()), buffer.size()); + result = collection_result->upsert<::couchbase::codec::raw_string_transcoder>(document_id, data_str, options).get(); + } else { + result = collection_result->upsert<::couchbase::codec::raw_binary_transcoder>(document_id, buffer, options).get(); + } + auto& [upsert_err, upsert_resp] = result; + if (upsert_err.ec()) { + // ambiguous_timeout should not be retried as we do not know if the insert was successful or not + if (getErrorType(upsert_err.ec()) == CouchbaseErrorType::TEMPORARY && upsert_err.ec().value() != static_cast<int>(::couchbase::errc::common::ambiguous_timeout)) { + logger_->log_error("Failed to upsert document '{}' to collection '{}.{}.{}' due to temporary issue, error code: '{}', message: '{}'", + document_id, collection.bucket_name, collection.scope_name, collection.collection_name, upsert_err.ec(), upsert_err.message()); + return nonstd::make_unexpected(CouchbaseErrorType::TEMPORARY); + } + logger_->log_error("Failed to upsert document '{}' to collection '{}.{}.{}' with error code: '{}', message: '{}'", + document_id, collection.bucket_name, collection.scope_name, collection.collection_name, upsert_err.ec(), upsert_err.message()); + return nonstd::make_unexpected(CouchbaseErrorType::FATAL); + } else { + const uint64_t partition_uuid = (upsert_resp.mutation_token().has_value() ? upsert_resp.mutation_token()->partition_uuid() : 0); + const uint64_t sequence_number = (upsert_resp.mutation_token().has_value() ? upsert_resp.mutation_token()->sequence_number() : 0); + const uint16_t partition_id = (upsert_resp.mutation_token().has_value() ? upsert_resp.mutation_token()->partition_id() : 0); + return CouchbaseUpsertResult { + collection.bucket_name, + upsert_resp.cas().value(), + partition_uuid, + sequence_number, + partition_id + }; + } +} + +void CouchbaseClient::close() { + if (cluster_) { + cluster_->close().wait(); + } +} + +nonstd::expected<void, CouchbaseErrorType> CouchbaseClient::establishConnection() { + if (cluster_) { + return {}; + } + + auto options = ::couchbase::cluster_options(username_, password_); + auto [connect_err, cluster] = ::couchbase::cluster::connect(connection_string_, options).get(); + if (connect_err.ec()) { + logger_->log_error("Failed to connect to Couchbase cluster with error code: '{}' and message: '{}'", connect_err.ec(), connect_err.message()); + return nonstd::make_unexpected(getErrorType(connect_err.ec())); + } + cluster_ = std::move(cluster); + return {}; +} Review Comment: I checked and the connection handling should be okay, but you are right that the `cluster_` object should be synchronized between threads. Added a synchronization with lock guards for that object in https://github.com/apache/nifi-minifi-cpp/pull/1875/commits/1b2393bc8eeff24748c1d39cce0e3cea24aa5a4b -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
