Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.apache.nifi.controller.ScheduledState;
import org.apache.nifi.controller.flow.FlowManager;
import org.apache.nifi.events.BulletinFactory;
import org.apache.nifi.flow.ExecutionEngine;
import org.apache.nifi.groups.ComponentScheduler;
import org.apache.nifi.groups.DefaultComponentScheduler;
import org.apache.nifi.groups.ProcessGroup;
Expand Down Expand Up @@ -148,17 +149,27 @@ public Set<ComponentNode> scheduleReferencingComponents(final ControllerServiceN
}
}

// start all of the components that are not disabled
// A processor in a stateless group is started through its group (the group is a single scheduling unit), and
// each stateless group is started at most once.
final Set<ComponentNode> updated = new HashSet<>();
final Set<ProcessGroup> startedStatelessGroups = new HashSet<>();
for (final ProcessorNode node : processors) {
if (candidates != null && !candidates.contains(node)) {
continue;
}

if (node.getScheduledState() != ScheduledState.DISABLED) {
if (node.getScheduledState() == ScheduledState.DISABLED) {
continue;
}

final ProcessGroup statelessGroup = getTopMostStatelessGroup(node.getProcessGroup());
if (statelessGroup == null) {
componentScheduler.startComponent(node);
updated.add(node);
} else if (startedStatelessGroups.add(statelessGroup)) {
componentScheduler.startStatelessGroup(statelessGroup);
}

updated.add(node);
}
for (final ReportingTaskNode node : reportingTasks) {
if (candidates != null && !candidates.contains(node)) {
Expand Down Expand Up @@ -194,15 +205,30 @@ public Map<ComponentNode, Future<Void>> unscheduleReferencingComponents(final Co

final Map<ComponentNode, Future<Void>> updated = new HashMap<>();

// A processor in a stateless group is stopped through the group (a single scheduling unit); standard processors
// are stopped individually.
final Map<ProcessGroup, List<ProcessorNode>> statelessMembersByGroup = new HashMap<>();
final List<ProcessorNode> standardProcessors = new ArrayList<>();
for (final ProcessorNode node : processors) {
if (!isRunningOrStarting(node)) {
continue;
}

final ProcessGroup statelessGroup = getTopMostStatelessGroup(node.getProcessGroup());
if (statelessGroup == null) {
standardProcessors.add(node);
} else {
statelessMembersByGroup.computeIfAbsent(statelessGroup, group -> new ArrayList<>()).add(node);
}
}

// verify that we can stop all components (that are running or starting) before doing anything
// Note: We check both RUNNING and STARTING states because a processor might be stuck in STARTING
// state if it references an invalid controller service (e.g., after a restart when the controller
// service configuration became invalid). Such processors need to be stopped before the controller
// service can be disabled.
for (final ProcessorNode node : processors) {
if (isRunningOrStarting(node)) {
node.verifyCanStop();
}
// service can be disabled. Stateless-group members are verified and stopped through their group.
for (final ProcessorNode node : standardProcessors) {
node.verifyCanStop();
}
for (final ReportingTaskNode node : reportingTasks) {
if (isRunningOrStarting(node)) {
Expand All @@ -215,13 +241,19 @@ public Map<ComponentNode, Future<Void>> unscheduleReferencingComponents(final Co
}
}

// stop all of the components that are running or starting
for (final ProcessorNode node : processors) {
if (isRunningOrStarting(node)) {
final Future<Void> future = node.getProcessGroup().stopProcessor(node);
updated.put(node, future);
// stop each stateless group once as a single unit, mapping the group's single future to every affected member
for (final Map.Entry<ProcessGroup, List<ProcessorNode>> entry : statelessMembersByGroup.entrySet()) {
final Future<Void> future = entry.getKey().stopProcessing();
for (final ProcessorNode member : entry.getValue()) {
updated.put(member, future);
}
}

// stop the standard processors
for (final ProcessorNode node : standardProcessors) {
final Future<Void> future = node.getProcessGroup().stopProcessor(node);
updated.put(node, future);
}
for (final ReportingTaskNode node : reportingTasks) {
if (isRunningOrStarting(node)) {
final Future<Void> future = processScheduler.unschedule(node);
Expand Down Expand Up @@ -264,6 +296,27 @@ private boolean isRunningOrStarting(final ReportingTaskNode node) {
return scheduledState == ScheduledState.RUNNING || scheduledState == ScheduledState.STARTING;
}

/**
* Returns the top-most Process Group that runs using the Stateless Engine and contains the given Process Group, or
* {@code null} if the given Process Group is {@code null} or does not run using the Stateless Engine. Only the
* top-most stateless group may be started or stopped directly; {@link ProcessGroup#startProcessing()} and
* {@link ProcessGroup#stopProcessing()} are no-ops on a nested stateless group.
*/
private ProcessGroup getTopMostStatelessGroup(final ProcessGroup start) {
ProcessGroup topMost = null;

// a Standard-Engine group cannot be a child of a stateless group, so the stateless chain is contiguous
for (ProcessGroup group = start; group != null; group = group.getParent()) {
if (group.resolveExecutionEngine() != ExecutionEngine.STATELESS) {
break;
}

topMost = group;
}

return topMost;
}

@Override
public CompletableFuture<Void> enableControllerService(final ControllerServiceNode serviceNode) {
if (serviceNode.isActive()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.apache.nifi.controller.flow.FlowManager;
import org.apache.nifi.controller.service.ControllerServiceProvider;
import org.apache.nifi.encrypt.PropertyEncryptor;
import org.apache.nifi.flow.ExecutionEngine;
import org.apache.nifi.nar.ExtensionManager;
import org.apache.nifi.registry.flow.VersionControlInformation;
import org.apache.nifi.registry.flow.VersionedFlowStatus;
Expand All @@ -41,13 +42,18 @@
import java.io.IOException;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

@ExtendWith(MockitoExtension.class)
Expand Down Expand Up @@ -144,6 +150,26 @@ void setProcessGroup() throws IOException {
);
}

@Test
void testStartAndStopProcessingDoNothingWhenParentIsStateless() throws Exception {
// Callers that resolve a component's owning stateless group rely on this no-op: only the top-most stateless
// group may be started or stopped directly. It is also what terminates the recursive stopComponents() walk,
// so making it throw or delegate upward would break stateless shutdown.
when(parentProcessGroup.getExecutionEngine()).thenReturn(ExecutionEngine.STATELESS);
when(parentProcessGroup.resolveExecutionEngine()).thenReturn(ExecutionEngine.STATELESS);
processGroup.setParent(parentProcessGroup);
processGroup.setExecutionEngine(ExecutionEngine.STATELESS);

processGroup.startProcessing();

final CompletableFuture<Void> stopFuture = processGroup.stopProcessing();

assertTrue(stopFuture.isDone());
assertNull(stopFuture.get());
verify(processScheduler, never()).startStatelessGroup(any());
verify(processScheduler, never()).stopStatelessGroup(any());
}

@Test
void testGetLoggingAttributesWithoutParentProcessGroup() {
processGroup.setName(NAME);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,15 @@ public interface ControllerServiceProvider extends ControllerServiceLookup {
* Controller services that reference this one, its schedulable referencing
* components will also be unscheduled.
*
* A referencing processor that belongs to a stateless process group is not
* stopped individually; instead its owning stateless group is stopped as a
* single unit, and the group's single stop {@link Future} is mapped to every
* affected processor in that group in the returned map.
*
* @param serviceNode the node
*
* @return a map of each affected component to the {@link Future} that
* completes when the component (or its stateless group) has stopped
*/
Map<ComponentNode, Future<Void>> unscheduleReferencingComponents(ControllerServiceNode serviceNode);

Expand Down Expand Up @@ -199,12 +207,19 @@ public interface ControllerServiceProvider extends ControllerServiceLookup {
* recursively, so if a Processor is referencing Service A, which is
* referencing serviceNode, then the Processor will also be started.
*
* A referencing processor that belongs to a stateless process group is not
* started individually; instead its owning stateless group is started as a
* single unit. Each affected stateless group is started at most once even
* when several of its processors reference the service.
*
* @param serviceNode the node
*/
Set<ComponentNode> scheduleReferencingComponents(ControllerServiceNode serviceNode);

/**
* Schedules any of the candidate components that are currently referencing the given Controller Service to run.
* A candidate processor that belongs to a stateless process group causes its owning stateless group to be started
* as a single unit; the group is started once even if only one of its processors is among the candidates.
* @return the components that were scheduled
*/
Set<ComponentNode> scheduleReferencingComponents(ControllerServiceNode serviceNode, Set<ComponentNode> candidates, ComponentScheduler componentScheduler);
Expand Down
Loading
Loading