diff --git a/cmd/openshift-tests/disaster-recovery.go b/cmd/openshift-tests/disaster-recovery.go new file mode 100644 index 000000000000..cc87044eb3a5 --- /dev/null +++ b/cmd/openshift-tests/disaster-recovery.go @@ -0,0 +1,81 @@ +package main + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "k8s.io/kubernetes/pkg/kubectl/util/templates" + + "github.com/openshift/origin/pkg/test/ginkgo" + _ "github.com/openshift/origin/test/e2e/dr" +) + +// disasterRecoverySuites are all known disaster recovery test suites this binary should run +var disasterRecoverySuites = []*ginkgo.TestSuite{ + { + Name: "all", + Description: templates.LongDesc(` + Run all tests. + `), + Matches: func(name string) bool { + return strings.Contains(name, "[Feature:DisasterRecovery]") + }, + TestTimeout: 120 * time.Minute, + }, +} + +// DisasterRecoveryOptions lists all options for disaster recovery tests +type DisasterRecoveryOptions struct { + Suite string + TestOptions []string +} + +func (o *DisasterRecoveryOptions) OptionsMap() (map[string]string, error) { + options := make(map[string]string) + for _, option := range o.TestOptions { + parts := strings.SplitN(option, "=", 2) + switch { + case len(parts) != 2, len(parts[0]) == 0: + return nil, fmt.Errorf("test option %q is not valid, must be KEY=VALUE", option) + } + _, exists := options[parts[0]] + if exists { + return nil, fmt.Errorf("option %q declared twice", parts[0]) + } + options[parts[0]] = parts[1] + } + return options, nil +} + +func (o *DisasterRecoveryOptions) ToEnv() string { + out, err := json.Marshal(o) + if err != nil { + panic(err) + } + return string(out) +} + +func initDRSnapshotRestore(value string) error { + if len(value) == 0 { + return nil + } + var opt DisasterRecoveryOptions + if err := json.Unmarshal([]byte(value), &opt); err != nil { + return err + } + for _, suite := range disasterRecoverySuites { + if suite.Name == opt.Suite { + o, err := opt.OptionsMap() + if err != nil { + return err + } + if suite.Init != nil { + return suite.Init(o) + } + return nil + } + } + return fmt.Errorf("unrecognized test info") +} diff --git a/cmd/openshift-tests/openshift-tests.go b/cmd/openshift-tests/openshift-tests.go index 1af46da7a5a2..bca56bf92d47 100644 --- a/cmd/openshift-tests/openshift-tests.go +++ b/cmd/openshift-tests/openshift-tests.go @@ -49,6 +49,7 @@ func main() { newRunUpgradeCommand(), newRunTestCommand(), newRunMonitorCommand(), + newRunDisasterRecoveryRestoreSnapshotCommand(), ) pflag.CommandLine = pflag.NewFlagSet("empty", pflag.ExitOnError) @@ -192,6 +193,45 @@ func newRunUpgradeCommand() *cobra.Command { return cmd } +func newRunDisasterRecoveryRestoreSnapshotCommand() *cobra.Command { + opt := &testginkgo.Options{Suites: disasterRecoverySuites} + disasterRecoveryOpt := &DisasterRecoveryOptions{} + + cmd := &cobra.Command{ + Use: "run-dr-restore-snapshot SUITE", + Short: "Run a restore snapshot disaster recovery suite", + Long: templates.LongDesc(` + Help, I'm being held at long description factory + + `) + testginkgo.SuitesString(opt.Suites, "\n\nAvailable test suites:\n\n"), + + SilenceUsage: true, + SilenceErrors: true, + RunE: func(cmd *cobra.Command, args []string) error { + return mirrorToFile(opt, func() error { + if err := initProvider(opt.Provider); err != nil { + return err + } + + disasterRecoveryOpt.Suite = "all" + value := disasterRecoveryOpt.ToEnv() + if err := initDRSnapshotRestore(value); err != nil { + return err + } + opt.SuiteOptions = value + + e2e.AfterReadingAllFlags(exutil.TestContext) + e2e.TestContext.DumpLogsOnFailure = true + exutil.TestContext.DumpLogsOnFailure = true + return opt.Run(args) + }) + }, + } + + bindOptions(opt, cmd.Flags()) + return cmd +} + func newRunTestCommand() *cobra.Command { testOpt := &testginkgo.TestOptions{ Out: os.Stdout, diff --git a/test/e2e/dr/restore-from-snapshot.go b/test/e2e/dr/restore-from-snapshot.go new file mode 100644 index 000000000000..06d27dc7f2eb --- /dev/null +++ b/test/e2e/dr/restore-from-snapshot.go @@ -0,0 +1,342 @@ +package dr + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/dynamic" + e2e "k8s.io/kubernetes/test/e2e/framework" + + "github.com/openshift/origin/test/e2e/upgrade" + exutil "github.com/openshift/origin/test/extended/util" + + g "github.com/onsi/ginkgo" + o "github.com/onsi/gomega" +) + +const ( + sshOpts = "-o StrictHostKeyChecking=no -o LogLevel=error -o ServerAliveInterval=30 -o ConnectionAttempts=100 -o ConnectTimeout=30" + proxyTemplate = "ssh -A %s -W %%h:%%p core@%s 2>/dev/null" + scpTemplate = "scp %s -o ProxyCommand=\"%s\" %s core@%s:%s" + sshTemplate = "ssh %s -o ProxyCommand=\"%s\" core@%s \"%s\"" + rollBackMachineConfig = "99-rollback-test" +) + +var _ = g.Describe("[Feature:DisasterRecovery][Disruptive]", func() { + f := e2e.NewDefaultFramework("disaster-recovery") + f.SkipNamespaceCreation = true + f.SkipPrivilegedPSPBinding = true + + oc := exutil.NewCLIWithoutNamespace("disaster-recovery") + + g.It("Cluster should restore itself from etcd snapshot", func() { + config, err := e2e.LoadConfig() + o.Expect(err).NotTo(o.HaveOccurred()) + dynamicClient := dynamic.NewForConfigOrDie(config) + mcps := dynamicClient.Resource(schema.GroupVersionResource{ + Group: "machineconfiguration.openshift.io", + Version: "v1", + Resource: "machineconfigpools", + }) + mc := dynamicClient.Resource(schema.GroupVersionResource{ + Group: "machineconfiguration.openshift.io", + Version: "v1", + Resource: "machineconfigs", + }) + + bastionHost := setupSSHBastion(oc) + proxy := fmt.Sprintf(proxyTemplate, sshOpts, bastionHost) + defer removeSSHBastion(oc) + + setMachineConfig("rollback-A.yaml", oc, mcps) + + masters := getAllMasters(oc) + e2e.Logf("masters: %v", masters) + o.Expect(masters).NotTo(o.BeEmpty()) + firstMaster := masters[0] + e2e.Logf("first master: %v", firstMaster) + + e2e.Logf("Make etcd backup on first master") + runViaBastionSSH(firstMaster, proxy, + "sudo -i /bin/bash -x /usr/local/bin/etcd-snapshot-backup.sh /root/assets/backup/snapshot.db") + runViaBastionSSH(firstMaster, proxy, + "sudo -i install -o core -g core /root/assets/backup/snapshot.db /tmp/snapshot.db") + setMachineConfig("rollback-B.yaml", oc, mcps) + + scpFileToHost(os.Getenv("KUBE_SSH_KEY_PATH"), proxy, "/home/core/.ssh/id_rsa", firstMaster) + runViaBastionSSH(firstMaster, proxy, "chmod 0600 /home/core/.ssh/id_rsa") + for _, master := range masters { + if master == firstMaster { + continue + } + runViaBastionSSH(firstMaster, proxy, + fmt.Sprintf("scp -o StrictHostKeyChecking=no /tmp/snapshot.db core@%s:/tmp/snapshot.db", master)) + } + + etcdConnectionString := constructEtcdConnectionString(masters, proxy) + e2e.Logf("etcd connstring: '%s'", etcdConnectionString) + for _, master := range masters { + runViaBastionSSH(master, proxy, + fmt.Sprintf("sudo -i /bin/bash -x /usr/local/bin/etcd-snapshot-restore.sh /tmp/snapshot.db %s", etcdConnectionString)) + } + + waitForAPIServer(oc) + waitForMastersToUpdate(oc, mcps) + + rollBackInMC := getRollbackContentsInMachineConfig(oc, mc, rollBackMachineConfig) + o.Expect(rollBackInMC).To(o.BeEquivalentTo("data:,A")) + + for _, master := range masters { + rollBackFile := fetchRollbackFileContents(master, proxy) + o.Expect(rollBackFile).To(o.BeEquivalentTo("A")) + } + }) +}) + +func setupSSHBastion(oc *exutil.CLI) string { + e2e.Logf("Setting up ssh bastion host") + const ( + ns = "ssh-bastion" + ) + + var ( + bastionHost = "" + sshBastionBaseDir = exutil.FixturePath("testdata", "disaster-recovery", "ssh-bastion") + files = []string{ + "service.yaml", + "serviceaccount.yaml", + "role.yaml", + "rolebinding.yaml", + "clusterrole.yaml", + "clusterrolebinding.yaml", + "deployment.yaml", + } + keyTypes = []string{"rsa", "ecdsa", "ed25519"} + tmpFiles = make([]string, len(keyTypes)) + ) + + _, err := oc.AdminProjectClient().Project().Projects().Get(ns, metav1.GetOptions{}) + if err != nil { + err = oc.Run("new-project").Args(ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + } + + e2e.Logf("Creating ssh keys") + _, err = oc.AdminKubeClient().CoreV1().Secrets(ns).Get("ssh-host-keys", metav1.GetOptions{}) + if kerrors.IsNotFound(err) { + tmpDir, err := ioutil.TempDir("/tmp", "ssh-keys") + o.Expect(err).NotTo(o.HaveOccurred()) + defer os.RemoveAll(tmpDir) + + for index, keyType := range keyTypes { + keyPath := filepath.Join(tmpDir, keyType) + e2e.Logf("Generating %s key in %s", keyType, keyPath) + out, err := exec.Command( + "ssh-keygen", + "-q", // silence + "-t", keyType, // type + "-f", keyPath, // output file + "-C", "", // no comment + "-N", "", // no passphrase + ).Output() + if err != nil { + e2e.Logf("ssh-keygen output:\n%s", out) + } + o.Expect(err).NotTo(o.HaveOccurred()) + tmpFiles[index] = keyPath + } + + secretKeyArgs := fmt.Sprintf( + "ssh_host_rsa_key=%s,ssh_host_ecdsa_key=%s,ssh_host_ed25519_key=%s,sshd_config=%s", + tmpFiles[0], tmpFiles[1], tmpFiles[2], filepath.Join(sshBastionBaseDir, "sshd_config"), + ) + _, err = oc.Run("create").Args("-n", ns, "secret", "generic", "ssh-host-keys", "--from-file", secretKeyArgs).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + } else { + o.Expect(err).NotTo(o.HaveOccurred()) + } + + e2e.Logf("Deploying ssh bastion") + for _, file := range files { + testDataPath := filepath.Join(sshBastionBaseDir, file) + err := oc.Run("apply").Args("-f", testDataPath).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + } + + e2e.Logf("Waiting for load balancer to be created") + err = wait.Poll(10*time.Second, 5*time.Minute, func() (done bool, err error) { + svc, err := oc.AdminKubeClient().CoreV1().Services(ns).Get("ssh-bastion", metav1.GetOptions{}) + if err != nil { + return false, nil + } + if svc.Spec.Type != corev1.ServiceTypeLoadBalancer { + return true, fmt.Errorf("Incorrect service type: %v", svc.Spec.Type) + } + if len(svc.Status.LoadBalancer.Ingress) == 0 { + return false, nil + } + bastionHost = svc.Status.LoadBalancer.Ingress[0].Hostname + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("Bastion host: %s", bastionHost) + + e2e.Logf("Waiting for host to be resolvable") + err = wait.Poll(10*time.Second, 5*time.Minute, func() (done bool, err error) { + _, err = exec.Command("nslookup", bastionHost).Output() + if err != nil { + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred()) + + return bastionHost +} + +func runCommandAndRetry(command string) string { + const ( + maxRetries = 10 + pause = 10 + ) + var ( + retryCount = 0 + out []byte + err error + ) + e2e.Logf("command '%s'", command) + for retryCount = 0; retryCount <= maxRetries; retryCount++ { + out, err = exec.Command("bash", "-c", command).CombinedOutput() + e2e.Logf("output:\n%s", out) + if err == nil { + break + } + e2e.Logf("%v", err) + time.Sleep(time.Second * pause) + } + o.Expect(retryCount).NotTo(o.Equal(maxRetries + 1)) + return string(out) +} + +func scpFileToHost(src string, proxy string, dest string, destHost string) { + e2e.Logf("Copying %s to %s at host '%s' via %s", src, dest, destHost, proxy) + + command := fmt.Sprintf(scpTemplate, sshOpts, proxy, src, destHost, dest) + runCommandAndRetry(command) +} + +func runViaBastionSSH(host string, proxy string, remoteCommand string) string { + e2e.Logf("Running '%s' on host %s via %s", remoteCommand, host, proxy) + + command := fmt.Sprintf(sshTemplate, sshOpts, proxy, host, remoteCommand) + return runCommandAndRetry(command) +} + +func setMachineConfig(rollbackFileName string, oc *exutil.CLI, mcps dynamic.NamespaceableResourceInterface) { + e2e.Logf("Update MachineConfig using %s file on masters", rollbackFileName) + machineConfigTemplate := exutil.FixturePath("testdata", "disaster-recovery", rollbackFileName) + err := oc.Run("apply").Args("-f", machineConfigTemplate).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + waitForMastersToUpdate(oc, mcps) +} + +func getRollbackContentsInMachineConfig(oc *exutil.CLI, mcs dynamic.NamespaceableResourceInterface, mcName string) string { + e2e.Logf("Reading contents of rollback MachineConfig") + pool, err := mcs.Get(mcName, metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + files, found, err := unstructured.NestedSlice(pool.Object, "spec", "config", "storage", "files") + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(found).To(o.BeTrue()) + o.Expect(files).NotTo(o.BeEmpty()) + + file := files[0].(map[string]interface{}) + actual, found, err := unstructured.NestedString(file, "contents", "source") + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(found).To(o.BeTrue()) + + return actual +} + +func waitForMastersToUpdate(oc *exutil.CLI, mcps dynamic.NamespaceableResourceInterface) { + e2e.Logf("Waiting for MachineConfig master to finish rolling out") + err := wait.Poll(30*time.Second, 30*time.Minute, func() (done bool, err error) { + return upgrade.IsPoolUpdated(mcps, "master") + }) + o.Expect(err).NotTo(o.HaveOccurred()) +} + +func getAllMasters(oc *exutil.CLI) []string { + nodeNames := sets.NewString() + + e2e.Logf("Fetching a list of masters") + + masterNodes, err := oc.AdminKubeClient().CoreV1().Nodes().List(metav1.ListOptions{ + LabelSelector: "node-role.kubernetes.io/master", + }) + for i := range masterNodes.Items { + node := &masterNodes.Items[i] + nodeNames.Insert(node.ObjectMeta.Name) + } + + o.Expect(err).NotTo(o.HaveOccurred()) + + return nodeNames.List() +} + +func constructEtcdConnectionString(masters []string, proxy string) string { + //TODO vrutkovs: replace this nonsense with `etcdctl member list -w json ...` + etcdConnectionString := "" + e2e.Logf("Construct etcd connection string") + for _, master := range masters { + hostname := runViaBastionSSH(master, proxy, "hostname -f") + o.Expect(hostname).NotTo(o.BeEmpty()) + hostname = strings.TrimSpace(hostname) + + etcdEnv := runViaBastionSSH(master, proxy, "cat /run/etcd/environment") + var entry string + for _, entry = range strings.Split(etcdEnv, "\n") { + if strings.HasPrefix(entry, "ETCD_DNS_NAME=") { + break + } + } + etcdDNSName := strings.Split(entry, "=")[1] + o.Expect(etcdDNSName).NotTo(o.BeEmpty()) + etcdConnectionString = fmt.Sprintf("%setcd-member-%s=https://%s:2380,", etcdConnectionString, hostname, etcdDNSName) + } + return etcdConnectionString[:len(etcdConnectionString)-1] +} + +func waitForAPIServer(oc *exutil.CLI) { + e2e.Logf("Waiting for API server to restore") + err := wait.Poll(10*time.Second, 5*time.Minute, func() (done bool, err error) { + _, err = oc.AdminKubeClient().CoreV1().Nodes().List(metav1.ListOptions{}) + if err != nil { + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred()) +} + +func fetchRollbackFileContents(master string, proxy string) string { + e2e.Logf("Fetching /etc/rollback-test file contents from %s", master) + return runViaBastionSSH(master, proxy, "cat /etc/rollback-test") +} + +func removeSSHBastion(oc *exutil.CLI) { + e2e.Logf("Removing ssh bastion") +} diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index f2f4913e0bd8..2baaac98de13 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -495,7 +495,7 @@ func clusterUpgrade(c configv1client.Interface, dc dynamic.Interface, config *re } allUpdated := true for _, p := range pools.Items { - updated, err := isPoolUpdated(mcps, p.GetName()) + updated, err := IsPoolUpdated(mcps, p.GetName()) if err != nil { framework.Logf("error checking pool %s: %v", p.GetName(), err) return false, nil @@ -512,7 +512,7 @@ func clusterUpgrade(c configv1client.Interface, dc dynamic.Interface, config *re } // TODO(runcom): drop this when MCO types are in openshift/api and we can use the typed client directly -func isPoolUpdated(dc dynamic.NamespaceableResourceInterface, name string) (bool, error) { +func IsPoolUpdated(dc dynamic.NamespaceableResourceInterface, name string) (bool, error) { pool, err := dc.Get(name, metav1.GetOptions{}) if err != nil { framework.Logf("error getting pool %s: %v", name, err) diff --git a/test/extended/testdata/bindata.go b/test/extended/testdata/bindata.go index f24bb3d51c48..6778e57edb12 100644 --- a/test/extended/testdata/bindata.go +++ b/test/extended/testdata/bindata.go @@ -117,6 +117,17 @@ // test/extended/testdata/deployments/tag-images-deployment.yaml // test/extended/testdata/deployments/test-deployment-broken.yaml // test/extended/testdata/deployments/test-deployment-test.yaml +// test/extended/testdata/disaster-recovery/rollback-A.yaml +// test/extended/testdata/disaster-recovery/rollback-B.yaml +// test/extended/testdata/disaster-recovery/ssh-bastion/clusterrole.yaml +// test/extended/testdata/disaster-recovery/ssh-bastion/clusterrolebinding.yaml +// test/extended/testdata/disaster-recovery/ssh-bastion/deployment.yaml +// test/extended/testdata/disaster-recovery/ssh-bastion/namespace.yaml +// test/extended/testdata/disaster-recovery/ssh-bastion/role.yaml +// test/extended/testdata/disaster-recovery/ssh-bastion/rolebinding.yaml +// test/extended/testdata/disaster-recovery/ssh-bastion/service.yaml +// test/extended/testdata/disaster-recovery/ssh-bastion/serviceaccount.yaml +// test/extended/testdata/disaster-recovery/ssh-bastion/sshd_config // test/extended/testdata/forcepull-test.json // test/extended/testdata/gssapi/config/kubeconfig // test/extended/testdata/gssapi/config/oauth_config.json @@ -6362,6 +6373,384 @@ func testExtendedTestdataDeploymentsTestDeploymentTestYaml() (*asset, error) { return a, nil } +var _testExtendedTestdataDisasterRecoveryRollbackAYaml = []byte(`apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + machineconfiguration.openshift.io/role: master + name: 99-rollback-test +spec: + config: + ignition: + version: 2.2.0 + storage: + files: + - contents: + source: data:,A + filesystem: root + mode: 420 + path: /etc/rollback-test +`) + +func testExtendedTestdataDisasterRecoveryRollbackAYamlBytes() ([]byte, error) { + return _testExtendedTestdataDisasterRecoveryRollbackAYaml, nil +} + +func testExtendedTestdataDisasterRecoveryRollbackAYaml() (*asset, error) { + bytes, err := testExtendedTestdataDisasterRecoveryRollbackAYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "test/extended/testdata/disaster-recovery/rollback-A.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _testExtendedTestdataDisasterRecoveryRollbackBYaml = []byte(`apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + machineconfiguration.openshift.io/role: master + name: 99-rollback-test +spec: + config: + ignition: + version: 2.2.0 + storage: + files: + - contents: + source: data:,B + filesystem: root + mode: 420 + path: /etc/rollback-test +`) + +func testExtendedTestdataDisasterRecoveryRollbackBYamlBytes() ([]byte, error) { + return _testExtendedTestdataDisasterRecoveryRollbackBYaml, nil +} + +func testExtendedTestdataDisasterRecoveryRollbackBYaml() (*asset, error) { + bytes, err := testExtendedTestdataDisasterRecoveryRollbackBYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "test/extended/testdata/disaster-recovery/rollback-B.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _testExtendedTestdataDisasterRecoverySshBastionClusterroleYaml = []byte(`apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ssh-bastion +rules: +- apiGroups: + - "machineconfiguration.openshift.io" + resources: + - "machineconfigs" + verbs: + - get +- apiGroups: + - "" + resources: + - "nodes" + verbs: + - list + - get +`) + +func testExtendedTestdataDisasterRecoverySshBastionClusterroleYamlBytes() ([]byte, error) { + return _testExtendedTestdataDisasterRecoverySshBastionClusterroleYaml, nil +} + +func testExtendedTestdataDisasterRecoverySshBastionClusterroleYaml() (*asset, error) { + bytes, err := testExtendedTestdataDisasterRecoverySshBastionClusterroleYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "test/extended/testdata/disaster-recovery/ssh-bastion/clusterrole.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _testExtendedTestdataDisasterRecoverySshBastionClusterrolebindingYaml = []byte(`apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + annotations: + openshift.io/description: Allows ssh-pod to read nodes and machineconfigs + name: ssh-bastion +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ssh-bastion +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:serviceaccount:ssh-bastion:ssh-bastion +`) + +func testExtendedTestdataDisasterRecoverySshBastionClusterrolebindingYamlBytes() ([]byte, error) { + return _testExtendedTestdataDisasterRecoverySshBastionClusterrolebindingYaml, nil +} + +func testExtendedTestdataDisasterRecoverySshBastionClusterrolebindingYaml() (*asset, error) { + bytes, err := testExtendedTestdataDisasterRecoverySshBastionClusterrolebindingYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "test/extended/testdata/disaster-recovery/ssh-bastion/clusterrolebinding.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _testExtendedTestdataDisasterRecoverySshBastionDeploymentYaml = []byte(`apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + run: ssh-bastion + name: ssh-bastion + namespace: ssh-bastion +spec: + replicas: 1 + selector: + matchLabels: + run: ssh-bastion + template: + metadata: + labels: + run: ssh-bastion + spec: + serviceAccountName: "ssh-bastion" + containers: + - image: quay.io/eparis/ssh:latest + imagePullPolicy: Always + name: ssh-bastion + ports: + - containerPort: 22 + name: ssh + protocol: TCP + volumeMounts: + - name: ssh-host-keys + mountPath: "/etc/ssh/" + readOnly: true + securityContext: + privileged: true + volumes: + - name: ssh-host-keys + secret: + secretName: ssh-host-keys + items: + - key: ssh_host_rsa_key + path: ssh_host_rsa_key + mode: 256 + - key: ssh_host_ecdsa_key + path: ssh_host_ecdsa_key + mode: 256 + - key: ssh_host_ed25519_key + path: ssh_host_ed25519_key + mode: 256 + - key: sshd_config + path: sshd_config + restartPolicy: Always +`) + +func testExtendedTestdataDisasterRecoverySshBastionDeploymentYamlBytes() ([]byte, error) { + return _testExtendedTestdataDisasterRecoverySshBastionDeploymentYaml, nil +} + +func testExtendedTestdataDisasterRecoverySshBastionDeploymentYaml() (*asset, error) { + bytes, err := testExtendedTestdataDisasterRecoverySshBastionDeploymentYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "test/extended/testdata/disaster-recovery/ssh-bastion/deployment.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _testExtendedTestdataDisasterRecoverySshBastionNamespaceYaml = []byte(`apiVersion: v1 +kind: Namespace +metadata: + name: ssh-bastion + labels: + openshift.io/run-level: "0" + +`) + +func testExtendedTestdataDisasterRecoverySshBastionNamespaceYamlBytes() ([]byte, error) { + return _testExtendedTestdataDisasterRecoverySshBastionNamespaceYaml, nil +} + +func testExtendedTestdataDisasterRecoverySshBastionNamespaceYaml() (*asset, error) { + bytes, err := testExtendedTestdataDisasterRecoverySshBastionNamespaceYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "test/extended/testdata/disaster-recovery/ssh-bastion/namespace.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _testExtendedTestdataDisasterRecoverySshBastionRoleYaml = []byte(`apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ssh-bastion + namespace: ssh-bastion +rules: +- apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + verbs: + - use + resourceNames: + - privileged +`) + +func testExtendedTestdataDisasterRecoverySshBastionRoleYamlBytes() ([]byte, error) { + return _testExtendedTestdataDisasterRecoverySshBastionRoleYaml, nil +} + +func testExtendedTestdataDisasterRecoverySshBastionRoleYaml() (*asset, error) { + bytes, err := testExtendedTestdataDisasterRecoverySshBastionRoleYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "test/extended/testdata/disaster-recovery/ssh-bastion/role.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _testExtendedTestdataDisasterRecoverySshBastionRolebindingYaml = []byte(`apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: + openshift.io/description: Allows ssh-pod to run as root + name: ssh-bastion + namespace: ssh-bastion +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ssh-bastion +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:serviceaccount:ssh-bastion:ssh-bastion +`) + +func testExtendedTestdataDisasterRecoverySshBastionRolebindingYamlBytes() ([]byte, error) { + return _testExtendedTestdataDisasterRecoverySshBastionRolebindingYaml, nil +} + +func testExtendedTestdataDisasterRecoverySshBastionRolebindingYaml() (*asset, error) { + bytes, err := testExtendedTestdataDisasterRecoverySshBastionRolebindingYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "test/extended/testdata/disaster-recovery/ssh-bastion/rolebinding.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _testExtendedTestdataDisasterRecoverySshBastionServiceYaml = []byte(`apiVersion: v1 +kind: Service +metadata: + labels: + run: ssh-bastion + name: ssh-bastion + namespace: ssh-bastion +spec: + externalTrafficPolicy: Local + ports: + - name: ssh + port: 22 + protocol: TCP + targetPort: ssh + selector: + run: ssh-bastion + type: LoadBalancer +`) + +func testExtendedTestdataDisasterRecoverySshBastionServiceYamlBytes() ([]byte, error) { + return _testExtendedTestdataDisasterRecoverySshBastionServiceYaml, nil +} + +func testExtendedTestdataDisasterRecoverySshBastionServiceYaml() (*asset, error) { + bytes, err := testExtendedTestdataDisasterRecoverySshBastionServiceYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "test/extended/testdata/disaster-recovery/ssh-bastion/service.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _testExtendedTestdataDisasterRecoverySshBastionServiceaccountYaml = []byte(`apiVersion: v1 +kind: ServiceAccount +metadata: + name: ssh-bastion + namespace: ssh-bastion +`) + +func testExtendedTestdataDisasterRecoverySshBastionServiceaccountYamlBytes() ([]byte, error) { + return _testExtendedTestdataDisasterRecoverySshBastionServiceaccountYaml, nil +} + +func testExtendedTestdataDisasterRecoverySshBastionServiceaccountYaml() (*asset, error) { + bytes, err := testExtendedTestdataDisasterRecoverySshBastionServiceaccountYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "test/extended/testdata/disaster-recovery/ssh-bastion/serviceaccount.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _testExtendedTestdataDisasterRecoverySshBastionSshd_config = []byte(`HostKey /etc/ssh/ssh_host_rsa_key +HostKey /etc/ssh/ssh_host_ecdsa_key +HostKey /etc/ssh/ssh_host_ed25519_key +SyslogFacility AUTHPRIV +PermitRootLogin no +AuthorizedKeysFile /home/core/.ssh/authorized_keys +PasswordAuthentication no +ChallengeResponseAuthentication no +GSSAPIAuthentication yes +GSSAPICleanupCredentials no +UsePAM yes +X11Forwarding yes +PrintMotd no +AcceptEnv LANG LC_CTYPE LC_NUMERIC LC_TIME LC_COLLATE LC_MONETARY LC_MESSAGES +AcceptEnv LC_PAPER LC_NAME LC_ADDRESS LC_TELEPHONE LC_MEASUREMENT +AcceptEnv LC_IDENTIFICATION LC_ALL LANGUAGE +AcceptEnv XMODIFIERS +Subsystem sftp /usr/libexec/openssh/sftp-server +`) + +func testExtendedTestdataDisasterRecoverySshBastionSshd_configBytes() ([]byte, error) { + return _testExtendedTestdataDisasterRecoverySshBastionSshd_config, nil +} + +func testExtendedTestdataDisasterRecoverySshBastionSshd_config() (*asset, error) { + bytes, err := testExtendedTestdataDisasterRecoverySshBastionSshd_configBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "test/extended/testdata/disaster-recovery/ssh-bastion/sshd_config", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + var _testExtendedTestdataForcepullTestJson = []byte(`{ "kind": "List", "apiVersion": "v1", @@ -32666,6 +33055,17 @@ var _bindata = map[string]func() (*asset, error){ "test/extended/testdata/deployments/tag-images-deployment.yaml": testExtendedTestdataDeploymentsTagImagesDeploymentYaml, "test/extended/testdata/deployments/test-deployment-broken.yaml": testExtendedTestdataDeploymentsTestDeploymentBrokenYaml, "test/extended/testdata/deployments/test-deployment-test.yaml": testExtendedTestdataDeploymentsTestDeploymentTestYaml, + "test/extended/testdata/disaster-recovery/rollback-A.yaml": testExtendedTestdataDisasterRecoveryRollbackAYaml, + "test/extended/testdata/disaster-recovery/rollback-B.yaml": testExtendedTestdataDisasterRecoveryRollbackBYaml, + "test/extended/testdata/disaster-recovery/ssh-bastion/clusterrole.yaml": testExtendedTestdataDisasterRecoverySshBastionClusterroleYaml, + "test/extended/testdata/disaster-recovery/ssh-bastion/clusterrolebinding.yaml": testExtendedTestdataDisasterRecoverySshBastionClusterrolebindingYaml, + "test/extended/testdata/disaster-recovery/ssh-bastion/deployment.yaml": testExtendedTestdataDisasterRecoverySshBastionDeploymentYaml, + "test/extended/testdata/disaster-recovery/ssh-bastion/namespace.yaml": testExtendedTestdataDisasterRecoverySshBastionNamespaceYaml, + "test/extended/testdata/disaster-recovery/ssh-bastion/role.yaml": testExtendedTestdataDisasterRecoverySshBastionRoleYaml, + "test/extended/testdata/disaster-recovery/ssh-bastion/rolebinding.yaml": testExtendedTestdataDisasterRecoverySshBastionRolebindingYaml, + "test/extended/testdata/disaster-recovery/ssh-bastion/service.yaml": testExtendedTestdataDisasterRecoverySshBastionServiceYaml, + "test/extended/testdata/disaster-recovery/ssh-bastion/serviceaccount.yaml": testExtendedTestdataDisasterRecoverySshBastionServiceaccountYaml, + "test/extended/testdata/disaster-recovery/ssh-bastion/sshd_config": testExtendedTestdataDisasterRecoverySshBastionSshd_config, "test/extended/testdata/forcepull-test.json": testExtendedTestdataForcepullTestJson, "test/extended/testdata/gssapi/config/kubeconfig": testExtendedTestdataGssapiConfigKubeconfig, "test/extended/testdata/gssapi/config/oauth_config.json": testExtendedTestdataGssapiConfigOauth_configJson, @@ -33109,6 +33509,21 @@ var _bintree = &bintree{nil, map[string]*bintree{ "test-deployment-broken.yaml": &bintree{testExtendedTestdataDeploymentsTestDeploymentBrokenYaml, map[string]*bintree{}}, "test-deployment-test.yaml": &bintree{testExtendedTestdataDeploymentsTestDeploymentTestYaml, map[string]*bintree{}}, }}, + "disaster-recovery": &bintree{nil, map[string]*bintree{ + "rollback-A.yaml": &bintree{testExtendedTestdataDisasterRecoveryRollbackAYaml, map[string]*bintree{}}, + "rollback-B.yaml": &bintree{testExtendedTestdataDisasterRecoveryRollbackBYaml, map[string]*bintree{}}, + "ssh-bastion": &bintree{nil, map[string]*bintree{ + "clusterrole.yaml": &bintree{testExtendedTestdataDisasterRecoverySshBastionClusterroleYaml, map[string]*bintree{}}, + "clusterrolebinding.yaml": &bintree{testExtendedTestdataDisasterRecoverySshBastionClusterrolebindingYaml, map[string]*bintree{}}, + "deployment.yaml": &bintree{testExtendedTestdataDisasterRecoverySshBastionDeploymentYaml, map[string]*bintree{}}, + "namespace.yaml": &bintree{testExtendedTestdataDisasterRecoverySshBastionNamespaceYaml, map[string]*bintree{}}, + "role.yaml": &bintree{testExtendedTestdataDisasterRecoverySshBastionRoleYaml, map[string]*bintree{}}, + "rolebinding.yaml": &bintree{testExtendedTestdataDisasterRecoverySshBastionRolebindingYaml, map[string]*bintree{}}, + "service.yaml": &bintree{testExtendedTestdataDisasterRecoverySshBastionServiceYaml, map[string]*bintree{}}, + "serviceaccount.yaml": &bintree{testExtendedTestdataDisasterRecoverySshBastionServiceaccountYaml, map[string]*bintree{}}, + "sshd_config": &bintree{testExtendedTestdataDisasterRecoverySshBastionSshd_config, map[string]*bintree{}}, + }}, + }}, "forcepull-test.json": &bintree{testExtendedTestdataForcepullTestJson, map[string]*bintree{}}, "gssapi": &bintree{nil, map[string]*bintree{ "config": &bintree{nil, map[string]*bintree{ diff --git a/test/extended/testdata/disaster-recovery/rollback-A.yaml b/test/extended/testdata/disaster-recovery/rollback-A.yaml new file mode 100644 index 000000000000..9634a704cf9e --- /dev/null +++ b/test/extended/testdata/disaster-recovery/rollback-A.yaml @@ -0,0 +1,17 @@ +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + machineconfiguration.openshift.io/role: master + name: 99-rollback-test +spec: + config: + ignition: + version: 2.2.0 + storage: + files: + - contents: + source: data:,A + filesystem: root + mode: 420 + path: /etc/rollback-test diff --git a/test/extended/testdata/disaster-recovery/rollback-B.yaml b/test/extended/testdata/disaster-recovery/rollback-B.yaml new file mode 100644 index 000000000000..f731af036ab7 --- /dev/null +++ b/test/extended/testdata/disaster-recovery/rollback-B.yaml @@ -0,0 +1,17 @@ +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + machineconfiguration.openshift.io/role: master + name: 99-rollback-test +spec: + config: + ignition: + version: 2.2.0 + storage: + files: + - contents: + source: data:,B + filesystem: root + mode: 420 + path: /etc/rollback-test diff --git a/test/extended/testdata/disaster-recovery/ssh-bastion/clusterrole.yaml b/test/extended/testdata/disaster-recovery/ssh-bastion/clusterrole.yaml new file mode 100644 index 000000000000..f7ce7f35641c --- /dev/null +++ b/test/extended/testdata/disaster-recovery/ssh-bastion/clusterrole.yaml @@ -0,0 +1,18 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ssh-bastion +rules: +- apiGroups: + - "machineconfiguration.openshift.io" + resources: + - "machineconfigs" + verbs: + - get +- apiGroups: + - "" + resources: + - "nodes" + verbs: + - list + - get diff --git a/test/extended/testdata/disaster-recovery/ssh-bastion/clusterrolebinding.yaml b/test/extended/testdata/disaster-recovery/ssh-bastion/clusterrolebinding.yaml new file mode 100644 index 000000000000..cdad0df9e50f --- /dev/null +++ b/test/extended/testdata/disaster-recovery/ssh-bastion/clusterrolebinding.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + annotations: + openshift.io/description: Allows ssh-pod to read nodes and machineconfigs + name: ssh-bastion +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ssh-bastion +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:serviceaccount:ssh-bastion:ssh-bastion diff --git a/test/extended/testdata/disaster-recovery/ssh-bastion/deployment.yaml b/test/extended/testdata/disaster-recovery/ssh-bastion/deployment.yaml new file mode 100644 index 000000000000..284978607e3f --- /dev/null +++ b/test/extended/testdata/disaster-recovery/ssh-bastion/deployment.yaml @@ -0,0 +1,49 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + run: ssh-bastion + name: ssh-bastion + namespace: ssh-bastion +spec: + replicas: 1 + selector: + matchLabels: + run: ssh-bastion + template: + metadata: + labels: + run: ssh-bastion + spec: + serviceAccountName: "ssh-bastion" + containers: + - image: quay.io/eparis/ssh:latest + imagePullPolicy: Always + name: ssh-bastion + ports: + - containerPort: 22 + name: ssh + protocol: TCP + volumeMounts: + - name: ssh-host-keys + mountPath: "/etc/ssh/" + readOnly: true + securityContext: + privileged: true + volumes: + - name: ssh-host-keys + secret: + secretName: ssh-host-keys + items: + - key: ssh_host_rsa_key + path: ssh_host_rsa_key + mode: 256 + - key: ssh_host_ecdsa_key + path: ssh_host_ecdsa_key + mode: 256 + - key: ssh_host_ed25519_key + path: ssh_host_ed25519_key + mode: 256 + - key: sshd_config + path: sshd_config + restartPolicy: Always diff --git a/test/extended/testdata/disaster-recovery/ssh-bastion/namespace.yaml b/test/extended/testdata/disaster-recovery/ssh-bastion/namespace.yaml new file mode 100644 index 000000000000..41fe6775c02c --- /dev/null +++ b/test/extended/testdata/disaster-recovery/ssh-bastion/namespace.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: ssh-bastion + labels: + openshift.io/run-level: "0" + diff --git a/test/extended/testdata/disaster-recovery/ssh-bastion/role.yaml b/test/extended/testdata/disaster-recovery/ssh-bastion/role.yaml new file mode 100644 index 000000000000..825d93b554ac --- /dev/null +++ b/test/extended/testdata/disaster-recovery/ssh-bastion/role.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ssh-bastion + namespace: ssh-bastion +rules: +- apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + verbs: + - use + resourceNames: + - privileged diff --git a/test/extended/testdata/disaster-recovery/ssh-bastion/rolebinding.yaml b/test/extended/testdata/disaster-recovery/ssh-bastion/rolebinding.yaml new file mode 100644 index 000000000000..ba2e2f2b4bdb --- /dev/null +++ b/test/extended/testdata/disaster-recovery/ssh-bastion/rolebinding.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + annotations: + openshift.io/description: Allows ssh-pod to run as root + name: ssh-bastion + namespace: ssh-bastion +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ssh-bastion +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:serviceaccount:ssh-bastion:ssh-bastion diff --git a/test/extended/testdata/disaster-recovery/ssh-bastion/service.yaml b/test/extended/testdata/disaster-recovery/ssh-bastion/service.yaml new file mode 100644 index 000000000000..63fb71775799 --- /dev/null +++ b/test/extended/testdata/disaster-recovery/ssh-bastion/service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + run: ssh-bastion + name: ssh-bastion + namespace: ssh-bastion +spec: + externalTrafficPolicy: Local + ports: + - name: ssh + port: 22 + protocol: TCP + targetPort: ssh + selector: + run: ssh-bastion + type: LoadBalancer diff --git a/test/extended/testdata/disaster-recovery/ssh-bastion/serviceaccount.yaml b/test/extended/testdata/disaster-recovery/ssh-bastion/serviceaccount.yaml new file mode 100644 index 000000000000..729a2330c7e3 --- /dev/null +++ b/test/extended/testdata/disaster-recovery/ssh-bastion/serviceaccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ssh-bastion + namespace: ssh-bastion diff --git a/test/extended/testdata/disaster-recovery/ssh-bastion/sshd_config b/test/extended/testdata/disaster-recovery/ssh-bastion/sshd_config new file mode 100644 index 000000000000..1f1b17167049 --- /dev/null +++ b/test/extended/testdata/disaster-recovery/ssh-bastion/sshd_config @@ -0,0 +1,18 @@ +HostKey /etc/ssh/ssh_host_rsa_key +HostKey /etc/ssh/ssh_host_ecdsa_key +HostKey /etc/ssh/ssh_host_ed25519_key +SyslogFacility AUTHPRIV +PermitRootLogin no +AuthorizedKeysFile /home/core/.ssh/authorized_keys +PasswordAuthentication no +ChallengeResponseAuthentication no +GSSAPIAuthentication yes +GSSAPICleanupCredentials no +UsePAM yes +X11Forwarding yes +PrintMotd no +AcceptEnv LANG LC_CTYPE LC_NUMERIC LC_TIME LC_COLLATE LC_MONETARY LC_MESSAGES +AcceptEnv LC_PAPER LC_NAME LC_ADDRESS LC_TELEPHONE LC_MEASUREMENT +AcceptEnv LC_IDENTIFICATION LC_ALL LANGUAGE +AcceptEnv XMODIFIERS +Subsystem sftp /usr/libexec/openssh/sftp-server