CNTRLPLANE-2834: add e2e test for etcd snapshot backup method on AWS

test/e2e/nodepool_test.go

@@ -470,7 +470,7 @@ func validateCAPIConditionBubblingDuringProvisioning(t *testing.T, ctx context.C
 	machinesUnreadyObserved := pollForConditionFalseWithAggregatedMessage(t, ctx, client, nodePool,
 		hyperv1.NodePoolAllMachinesReadyConditionType, "ready")
 	if !machinesUnreadyObserved {
-		t.Logf("AllMachinesReady was not observed as False with aggregated message during provisioning "+
+		t.Logf("AllMachinesReady was not observed as False with aggregated message during provisioning " +
 			"(CAPI provider may have set Ready condition before we could observe nil state)")
 	}
 }

test/e2e/v2/backuprestore/cli.go

@@ -59,6 +59,7 @@ type OADPBackupOptions struct {
 	Render                   bool     // Render the backup object to STDOUT instead of creating it
 	IncludedResources        []string // Comma-separated list of resources to include
 	IncludeNamespaces        []string // Additional namespaces to include
+	UseEtcdSnapshot          bool     // Use etcd snapshot mode (--use-etcd-snapshot)
 }
 
 // OADPRestoreOptions contains options for creating an OADP restore
@@ -79,6 +80,7 @@ type OADPRestoreOptions struct {
 	Render                 bool     // Render the restore object to STDOUT instead of creating it
 	RestorePVs             *bool    // Restore persistent volumes (default: true)
 	PreserveNodePorts      *bool    // Preserve NodePort assignments during restore (default: true)
+	UseEtcdSnapshot        bool     // Use etcd snapshot mode (--use-etcd-snapshot)
 }
 
 // OADPScheduleOptions contains options for creating an OADP backup schedule
@@ -101,6 +103,7 @@ type OADPScheduleOptions struct {
 	Paused                   bool     // Create schedule in paused state
 	UseOwnerReferences       bool     // Use owner references in backup objects
 	SkipImmediately          bool     // Skip immediate backup after schedule creation
+	UseEtcdSnapshot          bool     // Use etcd snapshot mode (--use-etcd-snapshot)
 }
 
 // FixDrOidcIamOptions contains options for fixing OIDC identity provider during disaster recovery
@@ -271,6 +274,9 @@ func buildBackupArgs(opts *OADPBackupOptions) []string {
 	if opts.Render {
 		args = append(args, "--render")
 	}
+	if opts.UseEtcdSnapshot {
+		args = append(args, "--use-etcd-snapshot")
+	}
 
 	// Slice flags
 	if len(opts.IncludedResources) > 0 {
@@ -328,6 +334,9 @@ func buildRestoreArgs(opts *OADPRestoreOptions) []string {
 	if opts.Render {
 		args = append(args, "--render")
 	}
+	if opts.UseEtcdSnapshot {
+		args = append(args, "--use-etcd-snapshot")
+	}
 
 	// Slice flags
 	if len(opts.IncludeNamespaces) > 0 {
@@ -383,6 +392,9 @@ func buildScheduleArgs(opts *OADPScheduleOptions) []string {
 	if opts.SkipImmediately {
 		args = append(args, "--skip-immediately")
 	}
+	if opts.UseEtcdSnapshot {
+		args = append(args, "--use-etcd-snapshot")
+	}
 
 	// Slice flags
 	if len(opts.IncludedResources) > 0 {

test/e2e/v2/backuprestore/etcd_snapshot.go

@@ -0,0 +1,184 @@
+//go:build e2ev2 && backuprestore
+
+package backuprestore
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/go-logr/logr"
+	hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1"
+	"github.com/openshift/hypershift/test/e2e/v2/internal"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	crclient "sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	// EtcdPodName is the name of the etcd pod whose init container logs are verified.
+	EtcdPodName = "etcd-0"
+	// EtcdInitContainerName is the name of the init container in the etcd pod.
+	EtcdInitContainerName = "etcd-init"
+
+	// HCPEtcdBackupNamePrefix is the prefix used by the OADP plugin when creating
+	// HCPEtcdBackup resources. The full name follows the pattern: oadp-<BackupName>-<random>.
+	HCPEtcdBackupNamePrefix = "oadp-"
+
+	// logRestoringSnapshot is emitted by etcdutl/etcdctl when starting a snapshot restore.
+	logRestoringSnapshot = "restoring snapshot"
+	// logRestoredSnapshot is emitted by etcdutl/etcdctl when snapshot restore completes.
+	logRestoredSnapshot = "restored snapshot"
+	// logNotRestoringSnapshot indicates the restore was skipped because data already existed.
+	logNotRestoringSnapshot = "not empty, not restoring snapshot"
+)
+
+// MatchesHCPEtcdBackupName checks whether an HCPEtcdBackup resource name matches the
+// expected naming pattern for a given OADP backup name. The OADP plugin creates
+// HCPEtcdBackup resources with the naming pattern: oadp-<BackupName>-<random>.
+func MatchesHCPEtcdBackupName(hcpEtcdBackupName, oadpBackupName string) bool {
+	return strings.HasPrefix(hcpEtcdBackupName, HCPEtcdBackupNamePrefix+oadpBackupName+"-")
+}
+
+// WaitForHCPEtcdBackupCondition waits for an HCPEtcdBackup resource matching the given
+// OADP backup name to have a BackupCompleted condition with the specified status.
+// HCPEtcdBackup names follow the pattern: oadp-<BackupName>-<random>.
+func WaitForHCPEtcdBackupCondition(testCtx *internal.TestContext, backupName string, expectedStatus metav1.ConditionStatus) error {
+	return wait.PollUntilContextTimeout(testCtx.Context, PollInterval, BackupTimeout, true, func(ctx context.Context) (bool, error) {
+		hcpEtcdBackupList := &hyperv1.HCPEtcdBackupList{}
+		if err := testCtx.MgmtClient.List(ctx, hcpEtcdBackupList, crclient.InNamespace(testCtx.ControlPlaneNamespace)); err != nil {
+			return false, fmt.Errorf("failed to list HCPEtcdBackup resources: %w", err)
+		}
+
+		for _, backup := range hcpEtcdBackupList.Items {
+			if !MatchesHCPEtcdBackupName(backup.Name, backupName) {
+				continue
+			}
+			condition := meta.FindStatusCondition(backup.Status.Conditions, string(hyperv1.BackupCompleted))
+			if condition == nil {
+				return false, nil
+			}
+			if condition.Status == expectedStatus {
+				return true, nil
+			}
+			// If the condition is explicitly False, the backup failed - stop polling.
+			if expectedStatus == metav1.ConditionTrue && condition.Status == metav1.ConditionFalse {
+				return false, fmt.Errorf("HCPEtcdBackup %s has BackupCompleted=False: reason=%s, message=%s",
+					backup.Name, condition.Reason, condition.Message)
+			}
+			return false, nil
+		}
+		return false, nil
+	})
+}
+
+// VerifyEtcdInitLogs retrieves the etcd-init container logs from the etcd-0 pod in the
+// control plane namespace and verifies that they contain expected snapshot restore traces.
+// The expected log lines from etcdutl/etcdctl indicate a successful snapshot restore:
+//   - "restoring snapshot" (restore started)
+//   - "restored snapshot" (restore completed)
+//
+// It also checks that the restore was not skipped due to existing data:
+//   - "not empty, not restoring snapshot" must NOT be present
+func VerifyEtcdInitLogs(ctx context.Context, logger logr.Logger, kubeClient kubernetes.Interface, controlPlaneNamespace string) error {
+	podLogOpts := &corev1.PodLogOptions{
+		Container: EtcdInitContainerName,
+	}
+
+	req := kubeClient.CoreV1().Pods(controlPlaneNamespace).GetLogs(EtcdPodName, podLogOpts)
+	logStream, err := req.Stream(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to stream %s container logs from %s: %w", EtcdInitContainerName, EtcdPodName, err)
+	}
+	defer logStream.Close()
+
+	result, err := parseEtcdInitLogs(logStream)
+	if err != nil {
+		return err
+	}
+
+	logger.Info("etcd-init container logs scanned", "lines", result.lineCount)
+
+	if result.restoreSkipped {
+		for _, line := range result.tailLines {
+			logger.V(1).Info("etcd-init tail", "log", line)
+		}
+		return fmt.Errorf("etcd-init logs contain '%s'; restore was skipped because data directory was not empty", logNotRestoringSnapshot)
+	}
+	if !result.restoreStarted {
+		for _, line := range result.tailLines {
+			logger.V(1).Info("etcd-init tail", "log", line)
+		}
+		return fmt.Errorf("etcd-init logs do not contain '%s'; snapshot restore may not have started", logRestoringSnapshot)
+	}
+	if !result.restoreCompleted {
+		for _, line := range result.tailLines {
+			logger.V(1).Info("etcd-init tail", "log", line)
+		}
+		return fmt.Errorf("etcd-init logs do not contain '%s'; snapshot restore may have failed", logRestoredSnapshot)
+	}
+
+	return nil
+}
+
+// etcdInitLogResult holds the results of parsing etcd-init container logs.
+type etcdInitLogResult struct {
+	restoreStarted   bool
+	restoreCompleted bool
+	restoreSkipped   bool
+	lineCount        int
+	tailLines        []string
+}
+
+// parseEtcdInitLogs scans etcd-init container log output and checks for expected
+// snapshot restore trace messages from etcdutl/etcdctl.
+func parseEtcdInitLogs(reader io.Reader) (*etcdInitLogResult, error) {
+	const tailSize = 50
+
+	result := &etcdInitLogResult{}
+
+	// Use a ring buffer so old strings become eligible for GC immediately
+	// instead of being retained by the underlying slice array.
+	ring := make([]string, tailSize)
+	ringIdx := 0
+	ringLen := 0
+
+	scanner := bufio.NewScanner(reader)
+	buf := make([]byte, 256*1024)
+	scanner.Buffer(buf, 512*1024)
+	for scanner.Scan() {
+		line := scanner.Text()
+		result.lineCount++
+		ring[ringIdx] = line
+		ringIdx = (ringIdx + 1) % tailSize
+		if ringLen < tailSize {
+			ringLen++
+		}
+		lower := strings.ToLower(line)
+		if strings.Contains(lower, logNotRestoringSnapshot) {
+			result.restoreSkipped = true
+		} else if strings.Contains(lower, logRestoredSnapshot) {
+			result.restoreCompleted = true
+		} else if strings.Contains(lower, logRestoringSnapshot) {
+			result.restoreStarted = true
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("error reading etcd-init logs: %w", err)
+	}
+
+	// Flatten the ring buffer into chronological order.
+	result.tailLines = make([]string, ringLen)
+	start := (ringIdx - ringLen + tailSize) % tailSize
+	for i := range ringLen {
+		result.tailLines[i] = ring[(start+i)%tailSize]
+	}
+
+	return result, nil
+}

test/e2e/v2/backuprestore/etcd_snapshot_test.go

@@ -0,0 +1,125 @@
+//go:build e2ev2 && backuprestore
+
+package backuprestore
+
+import (
+	"strings"
+	"testing"
+
+	. "github.com/onsi/gomega"
+)
+
+func TestParseEtcdInitLogs(t *testing.T) {
+	tests := []struct {
+		name             string
+		logs             string
+		restoreStarted   bool
+		restoreCompleted bool
+		restoreSkipped   bool
+		lineCount        int
+	}{
+		{
+			name: "When etcd-init logs show successful snapshot restore it should detect both restoring and restored",
+			logs: `INFO: using etcdutl (etcd 3.6+)
++----------+----------+------------+------------+---------+
+|   HASH   | REVISION | TOTAL KEYS | TOTAL SIZE | VERSION |
++----------+----------+------------+------------+---------+
+| 5643d825 |   578454 |       3209 |      49 MB |   3.6.0 |
++----------+----------+------------+------------+---------+
+2026-04-13T07:33:20Z    info    snapshot/v3_snapshot.go:305     restoring snapshot      {"path": "/tmp/snapshot"}
+2026-04-13T07:33:20Z    info    membership/cluster.go:424       added member
+2026-04-13T07:33:20Z    info    snapshot/v3_snapshot.go:333     restored snapshot       {"path": "/tmp/snapshot"}`,
+			restoreStarted:   true,
+			restoreCompleted: true,
+			restoreSkipped:   false,
+			lineCount:        9,
+		},
+		{
+			name: "When data directory is not empty it should detect restore was skipped",
+			logs: `/var/lib/data not empty, not restoring snapshot`,
+			restoreStarted:   false,
+			restoreCompleted: false,
+			restoreSkipped:   true,
+			lineCount:        1,
+		},
+		{
+			name:             "When logs contain only curl progress it should detect neither restore started nor completed",
+			logs:             `  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current`,
+			restoreStarted:   false,
+			restoreCompleted: false,
+			restoreSkipped:   false,
+			lineCount:        1,
+		},
+		{
+			name: "When restore starts but does not complete it should detect only restoring",
+			logs: `INFO: using etcdutl (etcd 3.6+)
+2026-04-13T07:33:20Z    info    snapshot/v3_snapshot.go:305     restoring snapshot      {"path": "/tmp/snapshot"}`,
+			restoreStarted:   true,
+			restoreCompleted: false,
+			restoreSkipped:   false,
+			lineCount:        2,
+		},
+		{
+			name:             "When logs are empty it should detect nothing",
+			logs:             ``,
+			restoreStarted:   false,
+			restoreCompleted: false,
+			restoreSkipped:   false,
+			lineCount:        0,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			g := NewWithT(t)
+			reader := strings.NewReader(tt.logs)
+			result, err := parseEtcdInitLogs(reader)
+			g.Expect(err).NotTo(HaveOccurred())
+			g.Expect(result.restoreStarted).To(Equal(tt.restoreStarted), "restoreStarted mismatch")
+			g.Expect(result.restoreCompleted).To(Equal(tt.restoreCompleted), "restoreCompleted mismatch")
+			g.Expect(result.restoreSkipped).To(Equal(tt.restoreSkipped), "restoreSkipped mismatch")
+			g.Expect(result.lineCount).To(Equal(tt.lineCount), "lineCount mismatch")
+		})
+	}
+}
+
+func TestMatchesHCPEtcdBackupName(t *testing.T) {
+	tests := []struct {
+		name               string
+		hcpEtcdBackupName  string
+		oadpBackupName     string
+		expectedMatch      bool
+	}{
+		{
+			name:              "When HCPEtcdBackup name matches the oadp pattern it should return true",
+			hcpEtcdBackupName: "oadp-mycluster-mynamespace-abc123-xyz78",
+			oadpBackupName:    "mycluster-mynamespace-abc123",
+			expectedMatch:     true,
+		},
+		{
+			name:              "When HCPEtcdBackup name does not match it should return false",
+			hcpEtcdBackupName: "some-other-backup",
+			oadpBackupName:    "mycluster-mynamespace-abc123",
+			expectedMatch:     false,
+		},
+		{
+			name:              "When HCPEtcdBackup name is the exact backup name without prefix it should return false",
+			hcpEtcdBackupName: "mycluster-mynamespace-abc123",
+			oadpBackupName:    "mycluster-mynamespace-abc123",
+			expectedMatch:     false,
+		},
+		{
+			name:              "When HCPEtcdBackup name only shares a backup name prefix it should return false",
+			hcpEtcdBackupName: "oadp-mycluster-mynamespace-abc1234-xyz78",
+			oadpBackupName:    "mycluster-mynamespace-abc123",
+			expectedMatch:     false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			g := NewWithT(t)
+			g.Expect(MatchesHCPEtcdBackupName(tt.hcpEtcdBackupName, tt.oadpBackupName)).To(Equal(tt.expectedMatch))
+		})
+	}
+}