bobpaulin commented on code in PR #11872:
URL: https://github.com/apache/nifi/pull/11872#discussion_r4552816308
##########
nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/connector/TestStandardConnectorNode.java:
##########
@@ -2039,6 +2123,329 @@ public void
testInheritingConfigurationKeepsTransitivelyGatedRequiredPropertyIrr
connectorNode.verifyCanStart();
}
+
+ @Test
+ public void testEvolveFlowReturnsSameTaskWhileHookIsPending() throws
Exception {
+ final FlowEvolutionHookConnector connector = new
FlowEvolutionHookConnector();
+ connector.leaveFutureIncomplete = true;
+ final StandardConnectorNode connectorNode =
createConnectorNode(connector);
+ connectorNode.adoptAuthoritativeSnapshot(flowGroup("pre-upgrade"),
null, Map.of(), outgoingSnapshotBundle());
+
+ connectorNode.prepareFlowEvolution();
+ final Future<Void> evolution =
connectorNode.evolveFlow(flowEvolutionExecutor);
+ assertTrue(connector.evolutionFuturePublished.await(5,
TimeUnit.SECONDS));
+
+ assertSame(evolution, connectorNode.evolveFlow(flowEvolutionExecutor));
+ assertEquals(FlowEvolutionStatus.PHASE_ONE_PENDING,
connectorNode.getFlowEvolutionStatus());
+ // The hook stays blocked, so the task has to be interrupted before
the executor is closed.
+ evolution.cancel(true);
+ }
+
+ enum HookCancel {
+ STOP,
+ CANCEL_DRAIN,
+ ABORT
+ }
+
+ @ParameterizedTest
+ @EnumSource(HookCancel.class)
+ public void testCancelsBlockedFlowEvolutionHook(final HookCancel action)
throws Exception {
+ assertHookCancelSetsDesiredStopped(switch (action) {
+ case STOP -> node -> node.stop(scheduler);
+ case CANCEL_DRAIN -> StandardConnectorNode::cancelDrainFlowFiles;
+ case ABORT -> node -> node.abortUpdate(new
FlowUpdateException("cancel"));
+ });
+ }
+
+ @Test
+ public void testEvolveFlowAfterAbortDoesNotInvokeHook() throws Exception {
+ final FlowEvolutionHookConnector connector = new
FlowEvolutionHookConnector();
+ final StandardConnectorNode connectorNode =
createConnectorNode(connector);
+ connectorNode.adoptAuthoritativeSnapshot(flowGroup("pre-upgrade"),
null, Map.of(), outgoingSnapshotBundle());
+
+ connectorNode.prepareFlowEvolution();
+ connectorNode.abortUpdate(new FlowUpdateException("cancel"));
+
+ assertThrows(FlowUpdateException.class, () ->
connectorNode.evolveFlow(flowEvolutionExecutor));
+ assertEquals(0, connector.hookCalls);
+ }
+
+ @Test
+ public void testResumeSkipsHook() throws Exception {
+ final FlowEvolutionHookConnector connector = new
FlowEvolutionHookConnector();
+ final StandardConnectorNode connectorNode =
createConnectorNode(connector);
+ connectorNode.adoptAuthoritativeSnapshot(flowGroup("pre-upgrade"),
null, Map.of(), outgoingSnapshotBundle());
+
+ connectorNode.prepareFlowEvolution();
+ connectorNode.evolveFlow(flowEvolutionExecutor).get(5,
TimeUnit.SECONDS);
+ connectorNode.abortUpdate(new FlowUpdateException("persist failed"));
+ connectorNode.prepareFlowEvolution();
+
+ final Future<Void> resumed =
connectorNode.evolveFlow(flowEvolutionExecutor);
+ resumed.get(5, TimeUnit.SECONDS);
+ assertEquals(1, connector.hookCalls);
+ assertEquals(FlowEvolutionStatus.PHASE_ONE_COMPLETE,
connectorNode.getFlowEvolutionStatus());
+ }
+
+ @Test
+ public void testStaleEvolutionTaskCannotWriteIntoLaterEvolution() throws
Exception {
+ final GatedActiveFlowConnector connector = new
GatedActiveFlowConnector();
+ final StandardConnectorNode connectorNode =
createConnectorNode(connector);
+ connectorNode.adoptAuthoritativeSnapshot(flowGroup("pre-upgrade"),
null, Map.of(), outgoingSnapshotBundle());
+ connectorNode.prepareFlowEvolution();
+ final Future<Void> evolution =
connectorNode.evolveFlow(flowEvolutionExecutor);
+ assertTrue(connector.reachedActiveFlow.await(5, TimeUnit.SECONDS));
+
+ connectorNode.abortUpdate(new FlowUpdateException("cancel"));
+ connectorNode.prepareFlowEvolution();
+ connector.releaseActiveFlow.countDown();
+
+ assertThrows(CancellationException.class, () -> evolution.get(5,
TimeUnit.SECONDS));
+ assertEquals(FlowEvolutionStatus.PHASE_ONE_PENDING,
connectorNode.getFlowEvolutionStatus());
+ assertEquals(ConnectorState.DRAINING, connectorNode.getCurrentState());
+
assertTrue(connectorNode.getValidationErrors().stream().noneMatch(result ->
StandardConnectorNode.FLOW_EVOLUTION_VALIDATION_SUBJECT.equals(result.getSubject())));
+ }
+
+ private void assertHookCancelSetsDesiredStopped(final
Consumer<StandardConnectorNode> cancel) throws Exception {
+ final FlowEvolutionHookConnector connector = new
FlowEvolutionHookConnector();
+ connector.leaveFutureIncomplete = true;
+ final StandardConnectorNode connectorNode =
createConnectorNode(connector);
+ connectorNode.adoptAuthoritativeSnapshot(flowGroup("pre-upgrade"),
null, Map.of(), outgoingSnapshotBundle());
+
+ connectorNode.prepareFlowEvolution();
+ final Future<Void> evolution =
connectorNode.evolveFlow(flowEvolutionExecutor);
+ assertTrue(connector.hookEntered.await(5, TimeUnit.SECONDS));
+ assertTrue(connector.evolutionFuturePublished.await(5,
TimeUnit.SECONDS));
+
+ cancel.accept(connectorNode);
+ assertEquals(ConnectorState.STOPPED, connectorNode.getDesiredState());
+ assertThrows(CancellationException.class, () -> evolution.get(2,
TimeUnit.SECONDS));
+ assertTrue(connector.evolutionFuture.isCancelled());
+ assertEquals(FlowEvolutionStatus.FAILED,
connectorNode.getFlowEvolutionStatus());
+ assertTrue(connectorNode.isPendingFlowEvolution());
+ assertEvolutionContextClosed(connector.context);
+ }
+
+ @Test
+ public void
testFlowEvolutionFailureIsIndependentOfValidationStateUntilNextAttempt() throws
Exception {
+ final ConnectorValidationTrigger validationTrigger =
mock(ConnectorValidationTrigger.class);
+ final StandardConnectorNode connectorNode = createConnectorNode(new
FlowEvolutionHookConnector(), validationTrigger);
+ connectorNode.markInvalid("Existing Failure", "existing failure");
+
+ connectorNode.failFlowEvolution("first evolution failure", null);
+ assertEquals(FlowEvolutionStatus.FAILED,
connectorNode.getFlowEvolutionStatus());
+ assertEquals(2, connectorNode.getValidationErrors().size());
+ assertEquals(1, connectorNode.getValidationErrors().stream()
+ .filter(result ->
StandardConnectorNode.FLOW_EVOLUTION_VALIDATION_SUBJECT.equals(result.getSubject()))
+ .count());
+
+ connectorNode.performValidation();
+ assertEquals(1, connectorNode.getValidationErrors().stream()
+ .filter(result ->
StandardConnectorNode.FLOW_EVOLUTION_VALIDATION_SUBJECT.equals(result.getSubject()))
+ .count());
+ connectorNode.resetValidationState();
+ assertEquals(1, connectorNode.getValidationErrors().stream()
+ .filter(result ->
StandardConnectorNode.FLOW_EVOLUTION_VALIDATION_SUBJECT.equals(result.getSubject()))
+ .count());
+
+ connectorNode.markInvalid("Replacement Failure", "replacement
failure");
+ connectorNode.failFlowEvolution("duplicate evolution failure", null);
+ assertEquals(2, connectorNode.getValidationErrors().size());
+ final ValidationResult evolutionFailure =
connectorNode.getValidationErrors().stream()
+ .filter(result ->
StandardConnectorNode.FLOW_EVOLUTION_VALIDATION_SUBJECT.equals(result.getSubject()))
+ .findFirst()
+ .orElseThrow();
+ assertEquals("first evolution failure",
evolutionFailure.getExplanation());
+ assertEquals(1, ((MockComponentLog)
connectorNode.getComponentLog()).getErrorMessages().stream()
+ .filter(message -> message.getMsg().contains("Flow evolution
failed"))
+ .count());
+
+ connectorNode.adoptAuthoritativeSnapshot(flowGroup("pre-upgrade"),
null, Map.of(), outgoingSnapshotBundle());
+ connectorNode.prepareFlowEvolution();
+
+ assertEquals(FlowEvolutionStatus.PHASE_ONE_PENDING,
connectorNode.getFlowEvolutionStatus());
+ assertTrue(connectorNode.getValidationErrors().stream()
+ .noneMatch(result ->
StandardConnectorNode.FLOW_EVOLUTION_VALIDATION_SUBJECT.equals(result.getSubject())));
+ assertTrue(connectorNode.getValidationErrors().stream()
+ .anyMatch(result -> "Replacement
Failure".equals(result.getSubject())));
+ }
+
+ @Test
+ public void testEvolveFlowRejectsMissingManagedComponents() throws
Exception {
+ final FlowEvolutionHookConnector connector = new
FlowEvolutionHookConnector();
+ final ProcessorNode missingProcessor = mock(ProcessorNode.class);
+ when(missingProcessor.isExtensionMissing()).thenReturn(true);
+
when(missingProcessor.getCanonicalClassName()).thenReturn("org.example.MissingProcessor");
+ when(missingProcessor.getBundleCoordinate()).thenReturn(new
BundleCoordinate("org.example", "missing-processor", "1.0.0"));
+
when(managedProcessGroup.findAllProcessors()).thenReturn(List.of(missingProcessor));
+ final StandardConnectorNode processorNode =
createConnectorNode(connector);
+ processorNode.adoptAuthoritativeSnapshot(flowGroup("pre-upgrade"),
null, Map.of(), outgoingSnapshotBundle());
+
+ processorNode.prepareFlowEvolution();
+ final ExecutionException processorFailure =
assertThrows(ExecutionException.class,
+ () -> processorNode.evolveFlow(flowEvolutionExecutor).get(5,
TimeUnit.SECONDS));
+
assertTrue(processorFailure.getCause().getMessage().contains("MissingProcessor"),
processorFailure.getCause().getMessage());
+ assertEquals(FlowEvolutionStatus.FAILED,
processorNode.getFlowEvolutionStatus());
+ assertEquals(ConnectorState.STOPPED, processorNode.getCurrentState());
+ assertEquals(0, connector.hookCalls);
+ }
+
+ @Test
+ public void
testPendingEvolutionBlocksEndTroubleshootingWithQueuedFlowFiles() throws
Exception {
+ final StandardConnectorNode connectorNode = createConnectorNode(new
FlowEvolutionHookConnector());
+ connectorNode.adoptAuthoritativeSnapshot(flowGroup("pre-upgrade"),
null, Map.of(), outgoingSnapshotBundle());
+ connectorNode.restoreTroubleshootingState();
+ when(managedProcessGroup.getQueueSize()).thenReturn(new QueueSize(3,
30L));
+
+ final IllegalStateException failure =
assertThrows(IllegalStateException.class,
connectorNode::verifyCanEndTroubleshooting);
+
+ assertTrue(failure.getMessage().contains("3 FlowFiles"),
failure.getMessage());
+ assertTrue(failure.getMessage().contains("upgrade is pending"),
failure.getMessage());
+ }
+
+ @Test
+ public void testFlowEvolutionRetriesAfterMissingSecretRestored() throws
Exception {
+ final SecretReference secretReference = new
SecretReference("provider-id", "Provider", "password",
"Provider.group.password");
+ final AtomicReference<Secret> currentSecret = new AtomicReference<>();
+ when(secretsManager.getSecrets(anySet(),
eq(false))).thenAnswer(invocation -> {
+ final Secret secret = currentSecret.get();
+ return secret == null ? Map.of() : Map.of(secretReference, secret);
+ });
+
+ final StandardConnectorNode connectorNode = createConnectorNode(new
FlowEvolutionHookConnector(), secretsManager);
+ connectorNode.adoptAuthoritativeSnapshot(flowGroup("pre-upgrade"),
+ strippedParameterContext("Source Database Password"),
Map.of("Source Database Password", secretReference),
+ outgoingSnapshotBundle());
+
+ connectorNode.prepareFlowEvolution();
+ final ExecutionException failure =
assertThrows(ExecutionException.class,
+ () -> connectorNode.evolveFlow(flowEvolutionExecutor).get(5,
TimeUnit.SECONDS));
+
assertTrue(failure.getCause().getMessage().contains("Provider.group.password"),
failure.getCause().getMessage());
+ assertTrue(failure.getCause().getMessage().contains("Source Database
Password"), failure.getCause().getMessage());
+ assertEquals(FlowEvolutionStatus.FAILED,
connectorNode.getFlowEvolutionStatus());
+ assertEquals(ConnectorState.STOPPED, connectorNode.getCurrentState());
+
+ final Secret restoredSecret = mock(Secret.class);
+ when(restoredSecret.getValue()).thenReturn("current-password");
+ currentSecret.set(restoredSecret);
+ connectorNode.prepareFlowEvolution();
+ assertEquals(FlowEvolutionStatus.PHASE_ONE_PENDING,
connectorNode.getFlowEvolutionStatus());
+ assertTrue(connectorNode.getValidationErrors().stream()
+ .noneMatch(result ->
StandardConnectorNode.FLOW_EVOLUTION_VALIDATION_SUBJECT.equals(result.getSubject())));
+ connectorNode.evolveFlow(flowEvolutionExecutor).get(5,
TimeUnit.SECONDS);
+
+ final ArgumentCaptor<Collection<ParameterValue>> valuesCaptor =
ArgumentCaptor.forClass(Collection.class);
+
verify(parameterContextFacade).updateParameters(valuesCaptor.capture());
+ assertEquals("current-password",
valuesCaptor.getValue().iterator().next().getValue());
+ assertEquals(FlowEvolutionStatus.PHASE_ONE_COMPLETE,
connectorNode.getFlowEvolutionStatus());
+ }
+
+ @Test
+ public void
testCaptureAuthoritativeSnapshotPersistsStructuredParameterSecretReference()
throws Exception {
+ final String parameterName = "Source Database Password";
+ final String secretValue = "current-password";
+ final SecretReference secretReference = new
SecretReference("provider-id", "Provider", "password",
"Provider.group.password");
+ final Secret secret = mock(Secret.class);
+ when(secret.getValue()).thenReturn(secretValue);
+
when(secretsManager.getSecrets(anySet())).thenReturn(Map.of(secretReference,
secret));
+
+ final Parameter parameter = new Parameter.Builder()
+ .name(parameterName)
+ .value(secretValue)
+ .sensitive(true)
+ .build();
+ final ParameterContext parameterContext = mock(ParameterContext.class);
+
when(parameterContext.getIdentifier()).thenReturn("managed-parameters");
+ when(parameterContext.getName()).thenReturn("Managed Parameters");
+
when(parameterContext.getInheritedParameterContextNames()).thenReturn(List.of());
+
when(parameterContext.getParameters()).thenReturn(Map.of(parameter.getDescriptor(),
parameter));
+
+ final StandardConnectorNode connectorNode = createConnectorNode(new
FlowEvolutionHookConnector(), secretsManager);
+
when(managedProcessGroup.getParameterContext()).thenReturn(parameterContext);
+ seedActiveConfiguration(connectorNode, REQUIRED_STEP,
Map.of(REQUIRED_SECRET, secretReference));
+
+ connectorNode.captureAuthoritativeSnapshot();
+
+ final ConnectorFlowSnapshot snapshot =
connectorNode.getAuthoritativeFlowSnapshot();
+ final VersionedParameter persistedParameter =
snapshot.parameterContext().getParameters().iterator().next();
+ assertFalse(persistedParameter.isProvided());
+ assertNull(persistedParameter.getValue());
+ assertEquals(Map.of(parameterName, secretReference),
snapshot.parameterSecretReferences());
+ assertEquals(connectorNode.getBundleCoordinate().getVersion(),
snapshot.bundle().getVersion());
+ assertEquals(secretReference,
snapshot.activeConfiguration().getNamedStepConfiguration(REQUIRED_STEP)
+ .configuration().getPropertyValues().get(REQUIRED_SECRET));
+ }
+
+ @Test
+ public void
testRestorePersistedFlowKeepsComponentsStoppedWhenSecretMissing() throws
Exception {
+ final SecretReference secretReference = new
SecretReference("provider-id", "Provider", "password",
"Provider.group.password");
+ when(secretsManager.getSecrets(anySet(),
eq(false))).thenReturn(Map.of());
+ final StandardConnectorNode connectorNode = createConnectorNode(new
FlowEvolutionHookConnector(), secretsManager);
+ clearInvocations(managedProcessGroup);
+
+ connectorNode.restorePersistedFlow(flowGroup("pre-upgrade"),
+ strippedParameterContext("Source Database Password"),
Map.of("Source Database Password", secretReference), true);
+
+
verify(managedProcessGroup).restoreFlowPreservingIdentifiers(any(VersionedExternalFlow.class),
eq(false));
+ }
+
+ @Test
+ public void
testRestorePersistedFlowUsesConfiguredAutoResumePolicyWhenSecretsResolved()
throws Exception {
+ final StandardConnectorNode connectorNode = createConnectorNode(new
FlowEvolutionHookConnector(), secretsManager);
+ clearInvocations(managedProcessGroup);
+
+ connectorNode.restorePersistedFlow(flowGroup("pre-upgrade"), null,
Map.of(), true);
+
+
verify(managedProcessGroup).restoreFlowPreservingIdentifiers(any(VersionedExternalFlow.class));
+ verify(managedProcessGroup,
never()).restoreFlowPreservingIdentifiers(any(VersionedExternalFlow.class),
anyBoolean());
+ }
+
+ @Test
+ public void
testPreUpgradeFlowContextUsesSnapshotBundleAndRejectsFlowUpdate() throws
Exception {
Review Comment:
Removed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]