fix(worker): stop invalid Spot requests from retrying On-Demand

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## 0.41.3 - Unreleased
 
+### Fixed
+
+- Stopped market-independent AWS Spot launch request errors from being retried as On-Demand while preserving fallback for Spot-recoverable capacity, quota, and unsupported-market failures. Thanks @vincentkoc.
+
 ## 0.41.2 - 2026-08-10
 
 ### Fixed

docs/features/capacity-fallback.md

@@ -103,11 +103,14 @@ is `on-demand-after-120s`) enables the On-Demand retry pass; set it to `none`
 (or leave it empty) to never fall back.
 
 The On-Demand pass runs after every Spot candidate in the class chain has been
-tried and rejected — it reruns the same chain on On-Demand. AWS fallback fires
-on provider rejection. Azure also treats a slow Spot VM provisioning operation
-as a capacity miss after the configured `on-demand-after-*` duration, or after
-the default 120 seconds when on-demand fallback is disabled with `spot-only` or
-`none`.
+tried and rejected. AWS retries only the candidates whose individual failures
+are Spot-recoverable, including Spot capacity or quota errors and request errors
+that explicitly identify Spot as unsupported. Market-independent request or
+image errors, such as an invalid block-device mapping, are not retried on
+On-Demand. Azure
+also treats a slow Spot VM provisioning operation as a capacity miss after the
+configured `on-demand-after-*` duration, or after the default 120 seconds when
+on-demand fallback is disabled with `spot-only` or `none`.
 Slow Azure On-Demand creates are bounded too, so the coordinator can keep trying
 the class chain before the CLI lease wait expires.
 

internal/cli/aws.go

@@ -560,6 +560,7 @@ func (c *AWSClient) createServerWithFallbackInRegion(ctx context.Context, cfg Co
 	}
 	candidates := awsLaunchCandidates(cfg)
 	useSpot := cfg.Capacity.Market != "on-demand"
+	var marketFallbackCandidates []string
 	var errs []error
 	for i, instanceType := range candidates {
 		next := cfg
@@ -573,6 +574,9 @@ func (c *AWSClient) createServerWithFallbackInRegion(ctx context.Context, cfg Co
 			if !isRetryableAWSProvisioningError(err) {
 				return Server{}, next, joinErrors(errs)
 			}
+			if useSpot {
+				marketFallbackCandidates = appendAWSMarketFallbackCandidate(marketFallbackCandidates, instanceType, err)
+			}
 			continue
 		}
 		server, err := c.createServer(ctx, next, publicKey, leaseID, slug, keep, imageID, securityGroupID, useSpot, control)
@@ -583,9 +587,12 @@ func (c *AWSClient) createServerWithFallbackInRegion(ctx context.Context, cfg Co
 		if !isRetryableAWSProvisioningError(err) {
 			return Server{}, next, joinErrors(errs)
 		}
+		if useSpot {
+			marketFallbackCandidates = appendAWSMarketFallbackCandidate(marketFallbackCandidates, instanceType, err)
+		}
 	}
-	if useSpot && strings.HasPrefix(cfg.Capacity.Fallback, "on-demand") {
-		for _, instanceType := range candidates {
+	if len(marketFallbackCandidates) > 0 && strings.HasPrefix(cfg.Capacity.Fallback, "on-demand") {
+		for _, instanceType := range marketFallbackCandidates {
 			next := cfg
 			next.ServerType = instanceType
 			if logf != nil {
@@ -1429,6 +1436,7 @@ func awsTagsWithName(labels map[string]string, name string) []types.Tag {
 func isRetryableAWSProvisioningError(err error) bool {
 	s := err.Error()
 	return strings.Contains(s, "InsufficientInstanceCapacity") ||
+		strings.Contains(s, "UnfulfillableCapacity") ||
 		strings.Contains(s, "MaxSpotInstanceCountExceeded") ||
 		strings.Contains(s, "VcpuLimitExceeded") ||
 		strings.Contains(s, "InvalidHostID.NotFound") ||
@@ -1443,6 +1451,31 @@ func isRetryableAWSProvisioningError(err error) bool {
 				strings.Contains(s, "instance type")))
 }
 
+func isAWSMarketFallbackError(err error) bool {
+	s := err.Error()
+	if strings.Contains(s, "InsufficientInstanceCapacity") ||
+		strings.Contains(s, "UnfulfillableCapacity") ||
+		strings.Contains(s, "MaxSpotInstanceCountExceeded") ||
+		strings.Contains(s, "VcpuLimitExceeded") {
+		return true
+	}
+	spotSpecific := strings.Contains(strings.ToLower(s), "spot")
+	return spotSpecific && (strings.Contains(s, "Unsupported") ||
+		strings.Contains(s, "InvalidParameterValue") ||
+		(strings.Contains(s, "InvalidParameterCombination") &&
+			(strings.Contains(s, "Free Tier") ||
+				strings.Contains(s, "eligible") ||
+				strings.Contains(s, "InstanceType") ||
+				strings.Contains(s, "instance type"))))
+}
+
+func appendAWSMarketFallbackCandidate(candidates []string, instanceType string, err error) []string {
+	if !isAWSMarketFallbackError(err) {
+		return candidates
+	}
+	return append(candidates, instanceType)
+}
+
 func isRetryableAWSRegionProvisioningError(err error) bool {
 	s := err.Error()
 	return isRetryableAWSProvisioningError(err) ||

internal/cli/aws_test.go

@@ -104,6 +104,37 @@ func TestAWSFixedAttemptAttestationIsNonCircularAndSecretFree(t *testing.T) {
 	}
 }
 
+func TestAWSMarketFallbackError(t *testing.T) {
+	tests := []struct {
+		name string
+		err  error
+		want bool
+	}{
+		{name: "spot capacity", err: errors.New("UnfulfillableCapacity: no Spot capacity"), want: true},
+		{name: "spot quota", err: errors.New("MaxSpotInstanceCountExceeded: quota"), want: true},
+		{name: "spot unsupported", err: errors.New("UnsupportedOperation: Spot is not supported"), want: true},
+		{name: "parameter independent", err: errors.New("InvalidParameterValue: invalid subnet"), want: false},
+		{name: "unsupported independent", err: errors.New("UnsupportedOperation: architecture is not supported"), want: false},
+		{name: "image independent", err: errors.New("no AWS AMI found in eu-west-1"), want: false},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if got := isAWSMarketFallbackError(test.err); got != test.want {
+				t.Fatalf("isAWSMarketFallbackError(%q) = %v, want %v", test.err, got, test.want)
+			}
+		})
+	}
+	if !isRetryableAWSProvisioningError(errors.New("UnfulfillableCapacity: no Spot capacity")) {
+		t.Fatal("UnfulfillableCapacity must remain eligible for type and market fallback")
+	}
+	var candidates []string
+	candidates = appendAWSMarketFallbackCandidate(candidates, "t3.small", errors.New("InvalidParameterValue: invalid subnet"))
+	candidates = appendAWSMarketFallbackCandidate(candidates, "t3.medium", errors.New("UnfulfillableCapacity: no Spot capacity"))
+	if len(candidates) != 1 || candidates[0] != "t3.medium" {
+		t.Fatalf("mixed market fallback candidates = %v, want [t3.medium]", candidates)
+	}
+}
+
 func TestAWSFixedPinnedAttemptBlocksBroadRegionFallback(t *testing.T) {
 	control := &AWSFixedCreateControl{PinnedAttempt: &AWSLaunchAttempt{ClientToken: "pinned"}}
 	err := errors.New("transport closed while waiting for capacity response")

worker/src/aws.ts

@@ -947,6 +947,7 @@ export class EC2SpotClient {
       const attempts: ProvisioningAttempt[] = [];
       const quotaCache = new Map<string, number | undefined>();
       const imageCache = new Map<string, string>();
+      const marketFallbackCandidates: string[] = [];
       const pinnedMacOSImageID =
         config.target === "macos" ? config.awsAMI || this.env.CRABBOX_AWS_AMI || "" : "";
       const resolveCandidateImageID = async (candidateConfig: LeaseConfig): Promise<string> => {
@@ -993,6 +994,9 @@ export class EC2SpotClient {
         if (preflight) {
           attempts.push(preflight);
           failures.push(`${serverType}: ${preflight.message}`);
+          if (config.capacityMarket === "spot") {
+            marketFallbackCandidates.push(serverType);
+          }
           continue;
         }
         try {
@@ -1029,12 +1033,17 @@ export class EC2SpotClient {
           });
           failures.push(`${serverType}: ${message}`);
           if (!isRetryableAWSProvisioningError(message)) {
+            marketFallbackCandidates.length = 0;
             break;
           }
+          if (config.capacityMarket === "spot" && isAWSMarketFallbackError(message)) {
+            marketFallbackCandidates.push(serverType);
+          }
         }
       }
-      if (config.capacityMarket === "spot" && config.capacityFallback.startsWith("on-demand")) {
-        for (const serverType of candidates) {
+      // Retry only candidates whose Spot failure can be recovered by On-Demand.
+      if (marketFallbackCandidates.length > 0 && config.capacityFallback.startsWith("on-demand")) {
+        for (const serverType of marketFallbackCandidates) {
           // oxlint-disable-next-line eslint/no-await-in-loop -- on-demand fallback must stay sequential.
           const preflight = await this.quotaPreflightAttempt(serverType, "on-demand", quotaCache);
           if (preflight) {
@@ -3455,6 +3464,7 @@ export function awsProvisioningErrorCategory(message: string): string {
   }
   if (
     message.includes("InsufficientInstanceCapacity") ||
+    message.includes("UnfulfillableCapacity") ||
     isAWSInsufficientCapacityOnHostError(message)
   ) {
     return "capacity";
@@ -3480,6 +3490,19 @@ export function awsProvisioningErrorCategory(message: string): string {
   return "";
 }
 
+export function isAWSMarketFallbackError(message: string): boolean {
+  if (
+    message.includes("InsufficientInstanceCapacity") ||
+    message.includes("UnfulfillableCapacity") ||
+    message.includes("MaxSpotInstanceCountExceeded") ||
+    message.includes("VcpuLimitExceeded")
+  ) {
+    return true;
+  }
+  const category = awsProvisioningErrorCategory(message);
+  return /\bspot\b/i.test(message) && (category === "unsupported" || category === "policy");
+}
+
 function isOpaqueAWSHTTP400XMLOnlyError(message: string): boolean {
   const entries = message
     .split(";")