Skip to content

Commit

Permalink
Merge pull request #408 from Vlatombe/resuming-node-fails-if-not-assi…
Browse files Browse the repository at this point in the history
…gned

Build fails to resume if controller crashes before queue is saved
  • Loading branch information
jglick authored Nov 22, 2024
2 parents 6ae8123 + 680ce20 commit f6c9e89
Show file tree
Hide file tree
Showing 3 changed files with 180 additions and 7 deletions.
16 changes: 11 additions & 5 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
<parent>
<groupId>org.jenkins-ci.plugins</groupId>
<artifactId>plugin</artifactId>
<version>4.88</version>
<version>5.3</version>
<relativePath/>
</parent>
<groupId>org.jenkins-ci.plugins.workflow</groupId>
Expand Down Expand Up @@ -67,8 +67,9 @@
<properties>
<changelist>999999-SNAPSHOT</changelist>
<!-- TODO Until in plugin-pom -->
<jenkins-test-harness.version>2182.v0138ccb_c0b_cb_</jenkins-test-harness.version>
<jenkins.version>2.440.1</jenkins.version>
<jenkins-test-harness.version>2357.vf2a_982b_b_910f</jenkins-test-harness.version>
<jenkins.baseline>2.479</jenkins.baseline>
<jenkins.version>${jenkins.baseline}.1</jenkins.version>
<useBeta>true</useBeta>
<gitHubRepo>jenkinsci/${project.artifactId}-plugin</gitHubRepo>
<hpi.compatibleSinceVersion>2.40</hpi.compatibleSinceVersion>
Expand All @@ -77,8 +78,8 @@
<dependencies>
<dependency>
<groupId>io.jenkins.tools.bom</groupId>
<artifactId>bom-2.440.x</artifactId>
<version>2907.vcb_35d6f2f7de</version>
<artifactId>bom-${jenkins.baseline}.x</artifactId>
<version>3482.vc10d4f6da_28a_</version>
<scope>import</scope>
<type>pom</type>
</dependency>
Expand Down Expand Up @@ -116,6 +117,11 @@
<artifactId>workflow-basic-steps</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jenkins-ci.plugins</groupId>
<artifactId>pipeline-input-step</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jenkins-ci.plugins</groupId>
<artifactId>pipeline-stage-step</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -215,8 +215,26 @@ public void stop(@NonNull Throwable cause) throws Exception {
@Override public void onResume() {
try {
if (state == null) {
Run<?, ?> run = getContext().get(Run.class);
LOGGER.fine(() -> "No ExecutorStepDynamicContext found for node block in " + run + "; perhaps loading from a historical build record, hoping for the best");
var flowNode = getContext().get(FlowNode.class);
LOGGER.fine(() -> "node block " + getContext() + " not yet scheduled, checking for an existing queue item");
if (flowNode == null) {

Check warning on line 220 in src/main/java/org/jenkinsci/plugins/workflow/support/steps/ExecutorStepExecution.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Partially covered line

Line 220 is only partially covered, one branch is missing
LOGGER.fine(() -> "No FlowNode found for node block " + getContext() + "; can't recover" );

Check warning on line 221 in src/main/java/org/jenkinsci/plugins/workflow/support/steps/ExecutorStepExecution.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Not covered line

Line 221 is not covered by tests
} else {
var action = flowNode.getAction(QueueItemActionImpl.class);
if (action == null) {

Check warning on line 224 in src/main/java/org/jenkinsci/plugins/workflow/support/steps/ExecutorStepExecution.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Partially covered line

Line 224 is only partially covered, one branch is missing
LOGGER.fine(() -> "No QueueItemAction found for node block " + getContext() + "; can't recover");

Check warning on line 225 in src/main/java/org/jenkinsci/plugins/workflow/support/steps/ExecutorStepExecution.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Not covered line

Line 225 is not covered by tests
} else {
LOGGER.fine(() -> "QueueItemAction with id=" + action.id + " found for node block " + getContext());
var queueItem = action.itemInQueue();
if (queueItem == null) {
LOGGER.fine(() -> "Could not find queue item " + action.id + ", rescheduling it");
flowNode.removeActions(QueueItemActionImpl.class);
start();
} else {
LOGGER.fine(() -> "Found Queue.Item " + queueItem + " for node block " + getContext() + "; should be fine");
}
}
}
return;
}
state.resume(getContext());
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
/*
* The MIT License
*
* Copyright (c) 2024, CloudBees, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/

package org.jenkinsci.plugins.workflow.support.steps;

import static org.awaitility.Awaitility.await;
import static org.hamcrest.Matchers.allOf;
import static org.hamcrest.Matchers.arrayWithSize;
import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.iterableWithSize;
import static org.junit.Assert.assertTrue;

import java.io.IOException;
import java.time.Duration;
import java.util.List;
import java.util.concurrent.TimeoutException;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.hamcrest.Description;
import org.hamcrest.TypeSafeMatcher;
import org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition;
import org.jenkinsci.plugins.workflow.flow.FlowExecutionList;
import org.jenkinsci.plugins.workflow.graphanalysis.DepthFirstScanner;
import org.jenkinsci.plugins.workflow.graphanalysis.NodeStepTypePredicate;
import org.jenkinsci.plugins.workflow.job.WorkflowJob;
import org.jenkinsci.plugins.workflow.support.steps.input.InputAction;
import org.junit.Rule;
import org.junit.Test;
import org.jvnet.hudson.test.InboundAgentRule;
import org.jvnet.hudson.test.JenkinsRule;
import org.jvnet.hudson.test.PrefixedOutputStream;
import org.jvnet.hudson.test.RealJenkinsRule;
import org.jvnet.hudson.test.TailLog;

public class ExecutorStepExecutionRJRTest {
private static final Logger LOGGER = Logger.getLogger(ExecutorStepExecutionRJRTest.class.getName());

@Rule public RealJenkinsRule rjr = new RealJenkinsRule().withColor(PrefixedOutputStream.Color.GREEN).withPackageLogger(ExecutorStepExecution.class, Level.FINE).withPackageLogger(FlowExecutionList.class, Level.FINE);

@Rule
public InboundAgentRule iar = new InboundAgentRule();

@Test public void restartWhileWaitingForANode() throws Throwable {
rjr.startJenkins();
try (var tailLog = new TailLog(rjr, "p", 1)) {
iar.createAgent(rjr, InboundAgentRule.Options.newBuilder().name("J").label("mib").color(PrefixedOutputStream.Color.YELLOW).webSocket().build());
rjr.runRemotely(ExecutorStepExecutionRJRTest::setupJobAndStart);
rjr.stopJenkinsForcibly();
rjr.startJenkins();
rjr.runRemotely(ExecutorStepExecutionRJRTest::resumeCompleteBranch1ThenBranch2);
}
}

private static void resumeCompleteBranch1ThenBranch2(JenkinsRule r) throws Throwable {
var p = r.jenkins.getItemByFullName("p", WorkflowJob.class);
var b = p.getBuildByNumber(1);
await("Waiting for agent J to reconnect").atMost(Duration.ofSeconds(30)).until(() -> r.jenkins.getComputer("J").isOnline());
var actions = await().until(() -> b.getActions(InputAction.class), allOf(iterableWithSize(1), hasItem(new InputActionWithId("Branch1"))));
proceed(actions, "Branch1", p.getName() + "#" + b.number);
// This is quicker than waitForMessage that can wait for up to 10 minutes
actions = await().until(() -> b.getActions(InputAction.class), allOf(iterableWithSize(1), hasItem(new InputActionWithId("Branch2"))));
r.waitForMessage("Complete branch 2 ?", b);
proceed(actions, "Branch2", p.getName() + "#" + b.number);
r.waitForCompletion(b);
}

private static class InputActionWithId extends TypeSafeMatcher<InputAction> {
private final String inputId;

private InputActionWithId(String inputId) {
this.inputId = inputId;
}


@Override
protected boolean matchesSafely(InputAction inputAction) {
try {
return inputAction.getExecutions().stream().anyMatch(execution -> inputId.equals(execution.getId()));
} catch (InterruptedException | TimeoutException e) {
return false;
}
}

@Override
public void describeTo(Description description) {
description.appendText("has input with id ").appendValue(inputId);
}
}

private static void proceed(List<InputAction> actions, String inputId, String name) throws InterruptedException, TimeoutException, IOException {
for (var action : actions) {
if (action.getExecutions().isEmpty()) {
continue;
}
var inputStepExecution = action.getExecutions().get(0);
if (inputId.equals(inputStepExecution.getId())) {
LOGGER.info(() -> "proceeding " + name);
inputStepExecution.proceed(null);
break;
}
}
}

private static void setupJobAndStart(JenkinsRule r) throws Exception {
var p = r.createProject(WorkflowJob.class, "p");
p.setDefinition(new CpsFlowDefinition("""
parallel 'Branch 1': {
node('mib') {
input id: 'Branch1', message: 'Complete branch 1 ?'
}
}, 'Branch 2': {
sleep 1
node('mib') {
input id:'Branch2', message: 'Complete branch 2 ?'
}
}
""", true));
var b = p.scheduleBuild2(0).waitForStart();
r.waitForMessage("Complete branch 1 ?", b);
assertTrue(b.isBuilding());
await().until(() -> r.jenkins.getQueue().getItems(), arrayWithSize(1));
LOGGER.info("Node steps: " + new DepthFirstScanner().filteredNodes(b.getExecution(), new NodeStepTypePredicate("node")));
// "Branch 1" step start + "Branch 1" body start + "Branch 2" step start
await().until(() -> new DepthFirstScanner().filteredNodes(b.getExecution(), new NodeStepTypePredicate("node")), hasSize(3));
}
}

0 comments on commit f6c9e89

Please sign in to comment.