diff --git a/Makefile b/Makefile index c68ab3b8eb..e2830519a7 100644 --- a/Makefile +++ b/Makefile @@ -105,7 +105,7 @@ test-unit: test-integration: # TODO(dmage): remove DOCKER_API_VERSION when our CI will upgrade Docker - DOCKER_API_VERSION=1.24 go test ./test/integration/... + GOTEST_FLAGS="$(TESTFLAGS)" DOCKER_API_VERSION=1.24 hack/test-go.sh test/integration/* .PHONY: test-integration # Remove all build artifacts. diff --git a/pkg/testframework/master.go b/pkg/testframework/master.go index 1589239278..539fa59e33 100644 --- a/pkg/testframework/master.go +++ b/pkg/testframework/master.go @@ -11,6 +11,7 @@ import ( "net/http" "os" "path" + "path/filepath" "strconv" "testing" "time" @@ -22,6 +23,7 @@ import ( "github.com/docker/docker/pkg/archive" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" kubeclient "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -29,15 +31,71 @@ import ( projectapiv1 "github.com/openshift/api/project/v1" ) +type MasterInterface interface { + Stop() error + WaitHealthz(configDir string) error + AdminKubeConfigPath() string +} + +type MasterProcess struct { + kubeconfig string +} + +func StartMasterProcess(kubeconfig string) (MasterInterface, error) { + if err := os.Setenv("KUBECONFIG", kubeconfig); err != nil { + return nil, err + } + return &MasterProcess{ + kubeconfig: kubeconfig, + }, nil +} + +func (p *MasterProcess) AdminKubeConfigPath() string { + return p.kubeconfig +} + +func (p *MasterProcess) Stop() error { return nil } + +func (p *MasterProcess) WaitHealthz(configDir string) error { + config, err := ConfigFromFile(p.kubeconfig) + if err != nil { + return err + } + u, _, err := rest.DefaultServerURL(config.Host, config.APIPath, schema.GroupVersion{}, true) + if err != nil { + return err + } + tlsConfig := &tls.Config{ + InsecureSkipVerify: true, + } + rt := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + TLSClientConfig: tlsConfig, + } + + return WaitHTTP(rt, fmt.Sprintf("https://%s/healthz", u.Host)) +} + type MasterContainer struct { ID string Port int NetworkSettings struct { IPAddress string } + + KubeConfigPath string } -func StartMasterContainer(configDir string) (*MasterContainer, error) { +func StartMasterContainer(configDir string) (MasterInterface, error) { cli, err := client.NewEnvClient() if err != nil { return nil, err @@ -75,8 +133,9 @@ func StartMasterContainer(configDir string) (*MasterContainer, error) { } c := &MasterContainer{ - ID: resp.ID, - Port: 8443, + ID: resp.ID, + Port: 8443, + KubeConfigPath: filepath.Join(configDir, "master", "admin.kubeconfig"), } inspectResult, err := cli.ContainerInspect(ctx, resp.ID) @@ -106,6 +165,10 @@ func StartMasterContainer(configDir string) (*MasterContainer, error) { return c, nil } +func (m *MasterContainer) AdminKubeConfigPath() string { + return m.KubeConfigPath +} + func (c *MasterContainer) WriteConfigs(configDir string) error { cli, err := client.NewEnvClient() if err != nil { @@ -216,8 +279,9 @@ func (r *Repository) Transport() http.RoundTripper { type Master struct { t *testing.T tmpDir string - container *MasterContainer + container MasterInterface adminKubeConfig *rest.Config + namespaces []string } func NewMaster(t *testing.T) *Master { @@ -226,7 +290,12 @@ func NewMaster(t *testing.T) *Master { t.Fatalf("failed to create a temporary directory for the master container: %v", err) } - container, err := StartMasterContainer(tmpDir) + var container MasterInterface + if path, ok := os.LookupEnv("TEST_KUBECONFIG"); ok { + container, err = StartMasterProcess(path) + } else { + container, err = StartMasterContainer(tmpDir) + } if err != nil { if removeErr := os.RemoveAll(tmpDir); removeErr != nil { t.Logf("failed to remove the temporary directory: %v", removeErr) @@ -286,6 +355,14 @@ func (m *Master) WaitForRoles() error { } func (m *Master) Close() { + if kubeClient, err := kubeclient.NewForConfig(m.AdminKubeConfig()); err == nil { + for _, ns := range m.namespaces { + if err := kubeClient.Core().Namespaces().Delete(ns, nil); err != nil { + m.t.Logf("failed to cleanup namespace %s: %v", ns, err) + } + } + } + if err := m.container.Stop(); err != nil { m.t.Logf("failed to stop the master container: %v", err) } @@ -295,16 +372,12 @@ func (m *Master) Close() { } } -func (m *Master) AdminKubeConfigPath() string { - return path.Join(m.tmpDir, "master", "admin.kubeconfig") -} - func (m *Master) AdminKubeConfig() *rest.Config { if m.adminKubeConfig != nil { return m.adminKubeConfig } - config, err := ConfigFromFile(m.AdminKubeConfigPath()) + config, err := ConfigFromFile(m.container.AdminKubeConfigPath()) if err != nil { m.t.Fatalf("failed to read the admin kubeconfig file: %v", err) } @@ -315,7 +388,7 @@ func (m *Master) AdminKubeConfig() *rest.Config { } func (m *Master) StartRegistry(t *testing.T, options ...RegistryOption) *Registry { - ln, closeFn := StartTestRegistry(t, m.AdminKubeConfigPath(), options...) + ln, closeFn := StartTestRegistry(t, m.container.AdminKubeConfigPath(), options...) return &Registry{ t: t, listener: ln, @@ -336,5 +409,6 @@ func (m *Master) CreateUser(username string, password string) *User { } func (m *Master) CreateProject(namespace, user string) *projectapiv1.Project { + m.namespaces = append(m.namespaces, namespace) return CreateProject(m.t, m.AdminKubeConfig(), namespace, user) } diff --git a/test/integration/crossmount/crossmount_test.go b/test/integration/crossmount/crossmount_test.go index b85723d789..710710f8a3 100644 --- a/test/integration/crossmount/crossmount_test.go +++ b/test/integration/crossmount/crossmount_test.go @@ -98,10 +98,7 @@ func TestCrossMount(t *testing.T) { // Upload a random image to a new project. uploadedImage := func(user *testframework.User, repoName string) sourceGenerator { return func(t *testing.T, registry *testframework.Registry) (*projectapiv1.Project, reference.Named, distribution.Manifest, closeFn) { - project := testframework.CreateProject(t, adminKubeConfig, uniqueName(user.Name), user.Name) - close := func() { - testframework.DeleteProject(t, adminKubeConfig, project.Name) - } + project := master.CreateProject(uniqueName(user.Name), user.Name) repo := registry.Repository(project.Name, repoName, user) manifest, err := testutil.UploadSchema2Image(context.Background(), repo, "latest") @@ -115,7 +112,7 @@ func TestCrossMount(t *testing.T) { t.Fatal(err) } - return project, named, manifest, close + return project, named, manifest, noopClose } } diff --git a/test/integration/offline/offline_test.go b/test/integration/offline/offline_test.go index 010279b0d6..0a30a21396 100644 --- a/test/integration/offline/offline_test.go +++ b/test/integration/offline/offline_test.go @@ -160,7 +160,7 @@ func TestPullthroughBlob(t *testing.T) { defer master.Close() testuser := master.CreateUser("testuser", "testp@ssw0rd") - testproject := master.CreateProject("testproject", testuser.Name) + testproject := master.CreateProject("test-offline-image-pullthrough", testuser.Name) teststreamName := "pullthrough" t.Log("=== import image") diff --git a/test/integration/pullthroughblob/pullthroughblob_test.go b/test/integration/pullthroughblob/pullthroughblob_test.go index cf277f5d08..af083f0291 100644 --- a/test/integration/pullthroughblob/pullthroughblob_test.go +++ b/test/integration/pullthroughblob/pullthroughblob_test.go @@ -30,7 +30,7 @@ func TestPullthroughBlob(t *testing.T) { defer master.Close() testuser := master.CreateUser("testuser", "testp@ssw0rd") - testproject := master.CreateProject("testproject", testuser.Name) + testproject := master.CreateProject("test-image-pullthrough-blob", testuser.Name) teststreamName := "pullthrough" requestCounter := counter.New()