markap14 commented on code in PR #11872:
URL: https://github.com/apache/nifi/pull/11872#discussion_r4619093326


##########
nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/ParameterSecretReferences.java:
##########
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.components.connector;
+
+import org.apache.nifi.asset.Asset;
+import org.apache.nifi.components.connector.components.ParameterValue;
+import org.apache.nifi.components.connector.secrets.SecretsManager;
+import org.apache.nifi.flow.VersionedConnector;
+import org.apache.nifi.flow.VersionedConnectorValueReference;
+import org.apache.nifi.flow.VersionedParameter;
+import org.apache.nifi.flow.VersionedParameterContext;
+import org.apache.nifi.parameter.Parameter;
+import org.apache.nifi.parameter.ParameterContext;
+import org.apache.nifi.security.encryption.PropertyEncryptionEncoder;
+import org.apache.nifi.security.encryption.PropertyEncryptionException;
+import org.apache.nifi.security.encryption.PropertyEncryptionProvider;
+import org.apache.nifi.security.encryption.SensitivePropertyCodec;
+import org.apache.nifi.security.encryption.SensitivePropertyContext;
+import org.apache.nifi.security.encryption.SensitivePropertyContextFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Matches and resolves relationships between Connector-managed Parameters and 
Secret References. Sensitive parameter
+ * values are persisted encrypted with the sensitive properties key, as in any 
other Parameter Context. A sensitive
+ * parameter whose value is exactly one configured secret is also bound to 
that Secret Reference, and on restore takes the
+ * current value of the secret from the {@link SecretsManager} instead of its 
persisted value.
+ */
+final class ParameterSecretReferences {
+    private static final Logger logger = 
LoggerFactory.getLogger(ParameterSecretReferences.class);
+
+    private ParameterSecretReferences() {
+    }
+
+    /**
+     * Converts a persisted Connector's managed parameter secret references 
into runtime Secret References.
+     * Entries that are null or not a secret reference are omitted.
+     */
+    static Map<String, SecretReference> 
convertManagedParameterSecretReferences(final VersionedConnector 
versionedConnector) {
+        final Map<String, VersionedConnectorValueReference> 
versionedReferences = versionedConnector.getManagedParameterSecretReferences();
+        if (versionedReferences == null || versionedReferences.isEmpty()) {
+            return Map.of();
+        }
+
+        final Map<String, SecretReference> references = new HashMap<>();
+        for (final Map.Entry<String, VersionedConnectorValueReference> entry : 
versionedReferences.entrySet()) {
+            final VersionedConnectorValueReference versionedReference = 
entry.getValue();
+            if (versionedReference == null || 
!ConnectorValueType.SECRET_REFERENCE.name().equals(versionedReference.getValueType()))
 {
+                continue;
+            }
+
+            references.put(entry.getKey(), new 
SecretReference(versionedReference.getProviderId(), 
versionedReference.getProviderName(),
+                versionedReference.getSecretName(), 
versionedReference.getFullyQualifiedSecretName()));
+        }
+        return references;
+    }
+
+    /**
+     * Returns the Secret Reference for each sensitive parameter whose value 
equals exactly one of the supplied secrets.
+     * A parameter whose value matches more than one reference is omitted.
+     */
+    static Map<String, SecretReference> 
matchSensitiveParametersToSecretReferences(final ParameterContext 
parameterContext, final Map<SecretReference, Secret> secrets) {
+        final Map<String, SecretReference> references = new HashMap<>();
+        if (parameterContext == null || secrets == null || secrets.isEmpty()) {
+            return references;
+        }
+
+        for (final Parameter parameter : 
parameterContext.getParameters().values()) {
+            final String value = parameter.getValue();
+            final List<Asset> assets = parameter.getReferencedAssets();
+            if (!parameter.getDescriptor().isSensitive() || value == null || 
value.isEmpty() || assets != null && !assets.isEmpty()) {
+                continue;
+            }
+
+            SecretReference matchingReference = null;
+            boolean ambiguous = false;
+            for (final Map.Entry<SecretReference, Secret> entry : 
secrets.entrySet()) {
+                final Secret secret = entry.getValue();
+                if (secret == null || !value.equals(secret.getValue())) {
+                    continue;
+                }
+
+                if (matchingReference == null) {
+                    matchingReference = entry.getKey();
+                } else if (!matchingReference.equals(entry.getKey())) {
+                    ambiguous = true;
+                    break;
+                }
+            }
+
+            if (matchingReference != null && !ambiguous) {
+                references.put(parameter.getDescriptor().getName(), 
matchingReference);
+            }
+        }
+
+        return references;
+    }
+
+    /**
+     * Resolves the values of a persisted managed Parameter Context without 
modifying it. A sensitive parameter bound to a
+     * Secret Reference takes the current value of that secret, or is left 
null and reported when the secret cannot be
+     * resolved. Any other sensitive parameter takes its persisted value 
decrypted with the supplied provider, or is left null
+     * and reported when the value cannot be decrypted.
+     */
+    static Resolution resolve(final VersionedParameterContext 
parameterContext, final Map<String, SecretReference> parameterSecretReferences,
+            final SecretsManager secretsManager, final 
PropertyEncryptionProvider propertyEncryptionProvider) {
+        if (parameterContext == null) {
+            return new Resolution(null, List.of(), Map.of(), Set.of());
+        }
+
+        final VersionedParameterContext resolved = 
copyContext(parameterContext);
+        final Map<String, SecretReference> references = 
parameterSecretReferences == null ? Map.of() : parameterSecretReferences;
+        final Set<SecretReference> distinctReferences = new 
HashSet<>(references.values());
+        final Map<SecretReference, Secret> secrets = 
getSecrets(distinctReferences, secretsManager);
+        final Map<String, SecretReference> unresolved = new HashMap<>();
+        final Set<String> undecryptable = new HashSet<>();
+        final List<ParameterValue> resolvedValues = new ArrayList<>();
+
+        if (resolved.getParameters() != null) {
+            for (final VersionedParameter parameter : 
resolved.getParameters()) {
+                if (!parameter.isSensitive()) {
+                    continue;
+                }
+
+                final SecretReference reference = 
references.get(parameter.getName());
+                if (reference == null) {
+                    final String value = parameter.getValue();
+                    if (PropertyEncryptionEncoder.isEncrypted(value)) {
+                        final SensitivePropertyContext context = 
SensitivePropertyContextFactory.forParameter(resolved.getName(), 
parameter.getName());
+                        try {
+                            
parameter.setValue(SensitivePropertyCodec.decrypt(propertyEncryptionProvider, 
PropertyEncryptionEncoder.getDecoded(value), context));
+                        } catch (final PropertyEncryptionException e) {
+                            parameter.setValue(null);
+                            undecryptable.add(parameter.getName());
+                        }
+                    }
+
+                    continue;
+                }
+
+                final Secret secret = secrets.get(reference);
+                if (secret == null) {
+                    parameter.setValue(null);
+                    unresolved.put(parameter.getName(), reference);
+                    continue;
+                }
+
+                parameter.setValue(secret.getValue());
+                resolvedValues.add(new ParameterValue.Builder()
+                    .name(parameter.getName())
+                    .value(secret.getValue())
+                    .sensitive(true)
+                    .build());
+            }
+        }
+
+        return new Resolution(resolved, resolvedValues, unresolved, 
undecryptable);
+    }
+
+    /**
+     * A Secrets Provider that cannot be reached leaves every reference 
unresolved, so a restore installs the flow with its
+     * components stopped instead of failing.
+     */
+    private static Map<SecretReference, Secret> getSecrets(final 
Set<SecretReference> references, final SecretsManager secretsManager) {
+        if (references.isEmpty()) {
+            return Map.of();
+        }
+
+        try {
+            return secretsManager.getSecrets(references, false);
+        } catch (final RuntimeException e) {
+            logger.warn("Failed to retrieve {} Secret References from the 
Secrets Manager; treating them as unresolved", references.size(), e);

Review Comment:
   [Claude Opus 5.5] Restoring the flow with its components stopped when the 
secrets provider can't be reached is the right behavior. But once this catch 
returns an empty map, a provider that is down looks exactly like secrets that 
don't exist, and that difference never reaches the user.
   
   For example, starting a pending Connector while the provider is down fails 
with the message from 
[`refreshPreUpgradeParameterSecrets`](https://github.com/apache/nifi/blob/a8faae0748f5d5afbb5345ed0b119f527bc5c0c5/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/ConnectorFlowEvolution.java#L306):
 "secret [x] used by parameter [y] could not be resolved. Restore the secret in 
its provider, then start the Connector to resume the upgrade." That sends the 
user looking for a missing secret when the real problem is that the provider 
can't be reached. Only this warning in the server log says what actually 
happened.
   
   Could `Resolution` carry the provider failure, so callers can tell the two 
cases apart? Then the explanation can say something like "the secrets provider 
could not be reached: <reason>", and the restore path can still leave the 
components stopped.
   
   
   ---
   <sub>🤖 Posted by an AI agent (Cursor · skill `apache-github-pr`) on behalf 
of @markap14.</sub>
   <!-- agent-attribution v1: agent=cursor; skill=apache-github-pr; 
operator=markap14 -->



##########
nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/connectors/AbstractNarSwapUpgradeDrainIT.java:
##########
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.tests.system.connectors;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Upgrade-drain support whose trigger is a NAR swap and restart.
+ */
+public abstract class AbstractNarSwapUpgradeDrainIT extends 
AbstractConnectorUpgradeDrainIT {

Review Comment:
   [Claude Opus 5.5] Now that `ConnectorUpgradeDrainProviderIT` is gone, 
`ClusteredConnectorUpgradeDrainIT` is the only subclass, and it doesn't use 
`triggerUpgrade` or `swapAndRestart`. It repeats the stop, swap and start steps 
itself, because it edits the provider files in the middle. Let's move what it 
still needs from this class (the bundle names, `createOutgoingConnector`, 
`outgoingBundle` and `incomingBundle`) into `ClusteredConnectorUpgradeDrainIT` 
and remove this class.
   
   
   ---
   <sub>🤖 Posted by an AI agent (Cursor · skill `apache-github-pr`) on behalf 
of @markap14.</sub>
   <!-- agent-attribution v1: agent=cursor; skill=apache-github-pr; 
operator=markap14 -->



##########
nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/VersionedComponentFlowMapper.java:
##########
@@ -1117,18 +1124,32 @@ public VersionedConnector mapConnector(final 
ConnectorNode connectorNode, final
         versionedConnector.setName(connectorNode.getName());
         
versionedConnector.setScheduledState(mapConnectorState(connectorNode.getDesiredState()));

Review Comment:
   [Claude Opus 5.5] Following up on our discussion about removing the drain 
timeout: once it's gone, whether a Connector comes back in the right state 
after a restart depends entirely on the state saved here. So 
`mapConnectorState` needs a deliberate answer for every `ConnectorState` that 
the desired state can hold. Right now everything except RUNNING, STARTING and 
TROUBLESHOOTING falls into `default -> ENABLED`.
   
   The case that concerns me most is an ordinary configuration update:
   - `transitionStateForUpdating` changes the desired state from RUNNING to 
UPDATING, and `applyUpdate` then changes it to UPDATED.
   - `completeUpdate` saves the flow right after `applyUpdate` 
([StandardConnectorRepository line 
887](https://github.com/apache/nifi/blob/a8faae0748f5d5afbb5345ed0b119f527bc5c0c5/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorRepository.java#L887)),
 so that save records ENABLED for a Connector that was running.
   - The Connector is only saved as RUNNING again by the next save, after 
`updateConnector` calls `start` ([line 
868](https://github.com/apache/nifi/blob/a8faae0748f5d5afbb5345ed0b119f527bc5c0c5/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/connector/StandardConnectorRepository.java#L868)).
   
   If NiFi restarts between those two saves, a Connector that was running comes 
back stopped. UPDATE_FAILED has the same question.
   
   Could we:
   - replace the `default` branch with an explicit case for every 
`ConnectorState`, so a new state can't silently map to ENABLED;
   - decide what UPDATING, UPDATED and UPDATE_FAILED should save? They describe 
an update in progress rather than what the Connector should do after a restart. 
One option is to save the state the Connector will return to (the 
`initialDesiredState` the repository already tracks) instead of ENABLED;
   - add a unit test that maps a Connector in each desired state and checks the 
saved value? Nothing tests `mapConnector` today.
   
   
   ---
   <sub>🤖 Posted by an AI agent (Cursor · skill `apache-github-pr`) on behalf 
of @markap14.</sub>
   <!-- agent-attribution v1: agent=cursor; skill=apache-github-pr; 
operator=markap14 -->



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to