Skip to content

test/e2e: More refactoring #765

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
openshift-merge-robot merged 1 commit into from
May 17, 2019
Merged

test/e2e: More refactoring #765

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
172 changes: 82 additions & 90 deletions test/e2e/mcd_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"time"

igntypes "github.com/coreos/ignition/config/v2_2/types"
"github.com/stretchr/testify/assert"
mcv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
"github.com/openshift/machine-config-operator/pkg/daemon/constants"
Expand Down Expand Up @@ -93,23 +94,49 @@ func createMCToAddFile(name, filename, data, fs string) *mcv1.MachineConfig {
// included the given mcName in its config, and returns the new
// rendered config name.
func waitForRenderedConfig(t *testing.T, cs *framework.ClientSet, pool, mcName string) (string, error) {
var newMCName string
var renderedConfig string
startTime := time.Now()
if err := wait.PollImmediate(2*time.Second, 5*time.Minute, func() (bool, error) {
mcp, err := cs.MachineConfigPools().Get(pool, metav1.GetOptions{})
if err != nil {
return false, err
}
for _, mc := range mcp.Status.Configuration.Source {
if mc.Name == mcName {
newMCName = mcp.Status.Configuration.Name
renderedConfig = mcp.Status.Configuration.Name
return true, nil
}
}
return false, nil
}); err != nil {
return "", errors.Wrapf(err, "machine config %s hasn't been picked by pool %s", mcName, pool)
}
return newMCName, nil
t.Logf("Pool %s has rendered config %s with %s (waited %v)", pool, mcName, renderedConfig, time.Since(startTime))
return renderedConfig, nil
}

// waitForPoolComplete polls a pool until it has completed an update to target
func waitForPoolComplete(t *testing.T, cs *framework.ClientSet, pool, target string) error {
// We need a spec and status separation https://github.com/openshift/machine-config-operator/pull/765#issuecomment-493136113
time.Sleep(time.Second * 13)
startTime := time.Now()
if err := wait.Poll(2*time.Second, 10*time.Minute, func() (bool, error) {
mcp, err := cs.MachineConfigPools().Get(pool, metav1.GetOptions{})
if err != nil {
return false, err
}
if mcp.Status.Configuration.Name != target {
return false, nil
}
if mcv1.IsMachineConfigPoolConditionTrue(mcp.Status.Conditions, mcv1.MachineConfigPoolUpdated) {
return true, nil
}
return false, nil
}); err != nil {
return errors.Wrapf(err, "pool %s didn't report updated to %s", pool, target)
}
t.Logf("Pool %s has completed %s (waited %v)", pool, target, time.Since(startTime))
return nil
}

func TestMCDeployed(t *testing.T) {
Expand All @@ -118,6 +145,7 @@ func TestMCDeployed(t *testing.T) {

// TODO: bring this back to 10
for i := 0; i < 5; i++ {
startTime := time.Now()
mcadd := createMCToAddFile("add-a-file", fmt.Sprintf("/etc/mytestconf%d", i), "test", "root")

// create the dummy MC now
Expand All @@ -126,34 +154,23 @@ func TestMCDeployed(t *testing.T) {
t.Errorf("failed to create machine config %v", err)
}

newMCName, err := waitForRenderedConfig(t, cs, "worker", mcadd.Name)
t.Logf("Created %s", mcadd.Name)
renderedConfig, err := waitForRenderedConfig(t, cs, "worker", mcadd.Name)
if err != nil {
t.Errorf("%v", err)
}

visited := make(map[string]bool)
if err := wait.Poll(2*time.Second, 30*time.Minute, func() (bool, error) {
nodes, err := getNodesByRole(cs, "worker")
if err != nil {
return false, nil
}
for _, node := range nodes {
if visited[node.Name] {
continue
}
if node.Annotations[constants.CurrentMachineConfigAnnotationKey] == newMCName &&
node.Annotations[constants.MachineConfigDaemonStateAnnotationKey] == constants.MachineConfigDaemonStateDone {
visited[node.Name] = true
if len(visited) == len(nodes) {
return true, nil
}
continue
}
}
return false, nil
}); err != nil {
t.Errorf("machine config didn't result in file being on any worker: %v", err)
if err := waitForPoolComplete(t, cs, "worker", renderedConfig); err != nil {
t.Fatal(err)
}
nodes, err := getNodesByRole(cs, "worker")
if err != nil {
t.Fatal(err)
}
for _, node := range nodes {
assert.Equal(t, renderedConfig, node.Annotations[constants.CurrentMachineConfigAnnotationKey])
assert.Equal(t, constants.MachineConfigDaemonStateDone, node.Annotations[constants.MachineConfigDaemonStateAnnotationKey])
}
t.Logf("All nodes updated with %s (%s elapsed)", mcadd.Name, time.Since(startTime))
}
}

Expand Down Expand Up @@ -199,65 +216,49 @@ func TestUpdateSSH(t *testing.T) {
if err != nil {
t.Errorf("failed to create machine config %v", err)
}
t.Logf("Created %s", mcadd.Name)

// grab the latest worker- MC
newMCName, err := waitForRenderedConfig(t, cs, "worker", mcadd.Name)
renderedConfig, err := waitForRenderedConfig(t, cs, "worker", mcadd.Name)
if err != nil {
t.Errorf("%v", err)
t.Fatal(err)
}
if err := waitForPoolComplete(t, cs, "worker", renderedConfig); err != nil {
t.Fatal(err)
}
nodes, err := getNodesByRole(cs, "worker")
if err != nil {
t.Fatal(err)
}
for _, node := range nodes {
assert.Equal(t, node.Annotations[constants.CurrentMachineConfigAnnotationKey], renderedConfig)
assert.Equal(t, node.Annotations[constants.MachineConfigDaemonStateAnnotationKey], constants.MachineConfigDaemonStateDone)
// find the MCD pod that has spec.nodeNAME = node.Name and get its name:
listOptions := metav1.ListOptions{
FieldSelector: fields.SelectorFromSet(fields.Set{"spec.nodeName": node.Name}).String(),
}
listOptions.LabelSelector = labels.SelectorFromSet(labels.Set{"k8s-app": "machine-config-daemon"}).String()

visited := make(map[string]bool)
if err := wait.Poll(2*time.Second, 10*time.Minute, func() (bool, error) {
nodes, err := getNodesByRole(cs, "worker")
mcdList, err := cs.Pods("openshift-machine-config-operator").List(listOptions)
if err != nil {
return false, err
t.Fatal(err)
}
for _, node := range nodes {
if visited[node.Name] {
continue
}
// check that the new MC is in the annotations and that the MCD state is Done.
if node.Annotations[constants.CurrentMachineConfigAnnotationKey] == newMCName &&
node.Annotations[constants.MachineConfigDaemonStateAnnotationKey] == constants.MachineConfigDaemonStateDone {
// find the MCD pod that has spec.nodeNAME = node.Name and get its name:
listOptions := metav1.ListOptions{
FieldSelector: fields.SelectorFromSet(fields.Set{"spec.nodeName": node.Name}).String(),
}
listOptions.LabelSelector = labels.SelectorFromSet(labels.Set{"k8s-app": "machine-config-daemon"}).String()

mcdList, err := cs.Pods("openshift-machine-config-operator").List(listOptions)
if err != nil {
return false, nil
}
if len(mcdList.Items) != 1 {
t.Logf("did not find any mcd pods")
return false, nil
}
mcdName := mcdList.Items[0].ObjectMeta.Name

// now rsh into that daemon and grep the authorized key file to check if 1234_test was written
// must do both commands in same shell, combine commands into one exec.Command()
found, err := exec.Command("oc", "rsh", "-n", "openshift-machine-config-operator", mcdName,
"grep", "1234_test", "/rootfs/home/core/.ssh/authorized_keys").CombinedOutput()
if err != nil {
t.Logf("unable to read authorized_keys on daemon: %s got: %s got err: %v", mcdName, found, err)
return false, nil
}
if !strings.Contains(string(found), "1234_test") {
t.Logf("updated ssh keys not found in authorized_keys, got %s", found)
return false, nil
}
if len(mcdList.Items) != 1 {
t.Fatalf("Failed to find MCD for node %s", node.Name)
}
mcdName := mcdList.Items[0].ObjectMeta.Name

visited[node.Name] = true
if len(visited) == len(nodes) {
return true, nil
}
continue
}
// now rsh into that daemon and grep the authorized key file to check if 1234_test was written
// must do both commands in same shell, combine commands into one exec.Command()
found, err := exec.Command("oc", "rsh", "-n", "openshift-machine-config-operator", mcdName,
"grep", "1234_test", "/rootfs/home/core/.ssh/authorized_keys").CombinedOutput()
if err != nil {
t.Fatalf("unable to read authorized_keys on daemon: %s got: %s got err: %v", mcdName, found, err)
}
return false, nil
}); err != nil {
t.Errorf("machine config didn't result in ssh keys being on any worker: %v", err)
if !strings.Contains(string(found), "1234_test") {
t.Fatalf("updated ssh keys not found in authorized_keys, got %s", found)
}
t.Logf("Node %s has SSH key", node.Name)
}
}

Expand Down Expand Up @@ -340,7 +341,7 @@ func TestReconcileAfterBadMC(t *testing.T) {
t.Errorf("failed to create machine config %v", err)
}

newMCName, err := waitForRenderedConfig(t, cs, "worker", mcadd.Name)
renderedConfig, err := waitForRenderedConfig(t, cs, "worker", mcadd.Name)
if err != nil {
t.Errorf("%v", err)
}
Expand All @@ -352,7 +353,7 @@ func TestReconcileAfterBadMC(t *testing.T) {
return false, err
}
for _, node := range nodes {
if node.Annotations[constants.DesiredMachineConfigAnnotationKey] == newMCName &&
if node.Annotations[constants.DesiredMachineConfigAnnotationKey] == renderedConfig &&
node.Annotations[constants.MachineConfigDaemonStateAnnotationKey] != constants.MachineConfigDaemonStateDone {
// just check that we have the annotation here, w/o strings checking anything that can flip fast causing flakes
if node.Annotations[constants.MachineConfigDaemonReasonAnnotationKey] != "" {
Expand Down Expand Up @@ -385,17 +386,8 @@ func TestReconcileAfterBadMC(t *testing.T) {
}

// wait for the mcp to go back to previous config
if err := wait.PollImmediate(2*time.Second, 5*time.Minute, func() (bool, error) {
mcp, err := cs.MachineConfigPools().Get("worker", metav1.GetOptions{})
if err != nil {
return false, err
}
if mcp.Status.Configuration.Name == workerOldMc {
return true, nil
}
return false, nil
}); err != nil {
t.Errorf("old machine config hasn't been picked by the pool: %v", err)
if err := waitForPoolComplete(t, cs, "worker", workerOldMc); err != nil {
t.Fatal(err)
}

visited := make(map[string]bool)
Expand Down