bobpaulin commented on code in PR #11872: URL: https://github.com/apache/nifi/pull/11872#discussion_r4622361567
########## nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/ParameterSecretReferences.java: ########## @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.components.connector; + +import org.apache.nifi.asset.Asset; +import org.apache.nifi.components.connector.components.ParameterValue; +import org.apache.nifi.components.connector.secrets.SecretsManager; +import org.apache.nifi.flow.VersionedConnector; +import org.apache.nifi.flow.VersionedConnectorValueReference; +import org.apache.nifi.flow.VersionedParameter; +import org.apache.nifi.flow.VersionedParameterContext; +import org.apache.nifi.parameter.Parameter; +import org.apache.nifi.parameter.ParameterContext; +import org.apache.nifi.security.encryption.PropertyEncryptionEncoder; +import org.apache.nifi.security.encryption.PropertyEncryptionException; +import org.apache.nifi.security.encryption.PropertyEncryptionProvider; +import org.apache.nifi.security.encryption.SensitivePropertyCodec; +import org.apache.nifi.security.encryption.SensitivePropertyContext; +import org.apache.nifi.security.encryption.SensitivePropertyContextFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Matches and resolves relationships between Connector-managed Parameters and Secret References. Sensitive parameter + * values are persisted encrypted with the sensitive properties key, as in any other Parameter Context. A sensitive + * parameter whose value is exactly one configured secret is also bound to that Secret Reference, and on restore takes the + * current value of the secret from the {@link SecretsManager} instead of its persisted value. + */ +final class ParameterSecretReferences { + private static final Logger logger = LoggerFactory.getLogger(ParameterSecretReferences.class); + + private ParameterSecretReferences() { + } + + /** + * Converts a persisted Connector's managed parameter secret references into runtime Secret References. + * Entries that are null or not a secret reference are omitted. + */ + static Map<String, SecretReference> convertManagedParameterSecretReferences(final VersionedConnector versionedConnector) { + final Map<String, VersionedConnectorValueReference> versionedReferences = versionedConnector.getManagedParameterSecretReferences(); + if (versionedReferences == null || versionedReferences.isEmpty()) { + return Map.of(); + } + + final Map<String, SecretReference> references = new HashMap<>(); + for (final Map.Entry<String, VersionedConnectorValueReference> entry : versionedReferences.entrySet()) { + final VersionedConnectorValueReference versionedReference = entry.getValue(); + if (versionedReference == null || !ConnectorValueType.SECRET_REFERENCE.name().equals(versionedReference.getValueType())) { + continue; + } + + references.put(entry.getKey(), new SecretReference(versionedReference.getProviderId(), versionedReference.getProviderName(), + versionedReference.getSecretName(), versionedReference.getFullyQualifiedSecretName())); + } + return references; + } + + /** + * Returns the Secret Reference for each sensitive parameter whose value equals exactly one of the supplied secrets. + * A parameter whose value matches more than one reference is omitted. + */ + static Map<String, SecretReference> matchSensitiveParametersToSecretReferences(final ParameterContext parameterContext, final Map<SecretReference, Secret> secrets) { + final Map<String, SecretReference> references = new HashMap<>(); + if (parameterContext == null || secrets == null || secrets.isEmpty()) { + return references; + } + + for (final Parameter parameter : parameterContext.getParameters().values()) { + final String value = parameter.getValue(); + final List<Asset> assets = parameter.getReferencedAssets(); + if (!parameter.getDescriptor().isSensitive() || value == null || value.isEmpty() || assets != null && !assets.isEmpty()) { + continue; + } + + SecretReference matchingReference = null; + boolean ambiguous = false; + for (final Map.Entry<SecretReference, Secret> entry : secrets.entrySet()) { + final Secret secret = entry.getValue(); + if (secret == null || !value.equals(secret.getValue())) { + continue; + } + + if (matchingReference == null) { + matchingReference = entry.getKey(); + } else if (!matchingReference.equals(entry.getKey())) { + ambiguous = true; + break; + } + } + + if (matchingReference != null && !ambiguous) { + references.put(parameter.getDescriptor().getName(), matchingReference); + } + } + + return references; + } + + /** + * Resolves the values of a persisted managed Parameter Context without modifying it. A sensitive parameter bound to a + * Secret Reference takes the current value of that secret, or is left null and reported when the secret cannot be + * resolved. Any other sensitive parameter takes its persisted value decrypted with the supplied provider, or is left null + * and reported when the value cannot be decrypted. + */ + static Resolution resolve(final VersionedParameterContext parameterContext, final Map<String, SecretReference> parameterSecretReferences, + final SecretsManager secretsManager, final PropertyEncryptionProvider propertyEncryptionProvider) { + if (parameterContext == null) { + return new Resolution(null, List.of(), Map.of(), Set.of()); + } + + final VersionedParameterContext resolved = copyContext(parameterContext); + final Map<String, SecretReference> references = parameterSecretReferences == null ? Map.of() : parameterSecretReferences; + final Set<SecretReference> distinctReferences = new HashSet<>(references.values()); + final Map<SecretReference, Secret> secrets = getSecrets(distinctReferences, secretsManager); + final Map<String, SecretReference> unresolved = new HashMap<>(); + final Set<String> undecryptable = new HashSet<>(); + final List<ParameterValue> resolvedValues = new ArrayList<>(); + + if (resolved.getParameters() != null) { + for (final VersionedParameter parameter : resolved.getParameters()) { + if (!parameter.isSensitive()) { + continue; + } + + final SecretReference reference = references.get(parameter.getName()); + if (reference == null) { + final String value = parameter.getValue(); + if (PropertyEncryptionEncoder.isEncrypted(value)) { + final SensitivePropertyContext context = SensitivePropertyContextFactory.forParameter(resolved.getName(), parameter.getName()); + try { + parameter.setValue(SensitivePropertyCodec.decrypt(propertyEncryptionProvider, PropertyEncryptionEncoder.getDecoded(value), context)); + } catch (final PropertyEncryptionException e) { + parameter.setValue(null); + undecryptable.add(parameter.getName()); + } + } + + continue; + } + + final Secret secret = secrets.get(reference); + if (secret == null) { + parameter.setValue(null); + unresolved.put(parameter.getName(), reference); + continue; + } + + parameter.setValue(secret.getValue()); + resolvedValues.add(new ParameterValue.Builder() + .name(parameter.getName()) + .value(secret.getValue()) + .sensitive(true) + .build()); + } + } + + return new Resolution(resolved, resolvedValues, unresolved, undecryptable); + } + + /** + * A Secrets Provider that cannot be reached leaves every reference unresolved, so a restore installs the flow with its + * components stopped instead of failing. + */ + private static Map<SecretReference, Secret> getSecrets(final Set<SecretReference> references, final SecretsManager secretsManager) { + if (references.isEmpty()) { + return Map.of(); + } + + try { + return secretsManager.getSecrets(references, false); + } catch (final RuntimeException e) { + logger.warn("Failed to retrieve {} Secret References from the Secrets Manager; treating them as unresolved", references.size(), e); Review Comment: [Claude Opus 5.5] Good catch. ParameterSecretReferences.Resolution now has a secretsProviderFailure field that holds the reason the provider call failed. When it's set, flow evolution fails with "the secrets provider could not be reached: . Restore access to the provider, then start the Connector to resume the upgrade." It no longer names a secret as missing. A restore still treats the references as unresolved, so the flow comes up with its components stopped rather than failing. I extended testFlowEvolutionWithMissingSecretThenRestoredSecret to cover it in three stages: the provider is unreachable, then it's reachable but the secret is missing, then the secret is restored. ########## nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/AbstractNarSwapUpgradeDrainIT.java: ########## @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.tests.system.connectors; + +import java.io.IOException; +import java.util.Map; + +/** + * Upgrade-drain support whose trigger is a NAR swap and restart. + */ +public abstract class AbstractNarSwapUpgradeDrainIT extends AbstractConnectorUpgradeDrainIT { Review Comment: [Claude Opus 5.5] Removed. ClusteredConnectorUpgradeDrainIT now extends AbstractConnectorUpgradeDrainIT directly and provides the connector and bundle overrides itself. triggerUpgrade is no longer abstract. It's a plain method on AbstractChangeVersionDrainIT, the only place that uses it. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
