Add an initial importer for Image Repositories

Image Repositories that have "dockerImageRepository" set will have
images imported if "tags" is empty or if there are "tags" that have
an empty string value. Once complete, or if sufficient failures
occur, the annotation "openshift.io/image.dockerRepositoryCheck"
will be set to a timestamp (time completed) or a string message.

examples/image-repositories/image-repositories.json

@@ -9,7 +9,6 @@
         "name": "ruby-20-centos7"
       },
       "tags": {
-        "latest": "latest"
       }
     },
     {
@@ -20,7 +19,6 @@
         "name": "nodejs-010-centos7"
       },
       "tags": {
-        "latest": "latest"
       }
     },
     {
@@ -31,7 +29,6 @@
         "name": "wildfly-8-centos"
       },
       "tags": {
-        "latest": "latest"
       }
     }
   ],

examples/sample-app/application-template-dockerbuild.json

@@ -39,7 +39,8 @@
       "kind": "ImageRepository",
       "metadata": {
         "name": "ruby-20-centos7"
-      }
+      },
+      "dockerImageRepository": "openshift/ruby-20-centos7"
     },
     {
       "apiVersion": "v1beta1",

examples/sample-app/application-template-stibuild.json

@@ -39,7 +39,8 @@
       "kind": "ImageRepository",
       "metadata": {
         "name": "ruby-20-centos7"
-      }
+      },
+      "dockerImageRepository": "openshift/ruby-20-centos7"
     },
     {
       "apiVersion": "v1beta1",

hack/test-cmd.sh

@@ -185,10 +185,9 @@ osc create -f examples/image-repositories/image-repositories.json
 [ -n "$(osc get imageRepositories wildfly-8-centos -t "{{.status.dockerImageRepository}}")" ]
 osc delete imageRepositories ruby-20-centos7
 osc delete imageRepositories nodejs-010-centos7
-osc delete imageRepositories wildfly-8-centos
 [ -z "$(osc get imageRepositories ruby-20-centos7 -t "{{.status.dockerImageRepository}}")" ]
 [ -z "$(osc get imageRepositories nodejs-010-centos7 -t "{{.status.dockerImageRepository}}")" ]
-[ -z "$(osc get imageRepositories wildfly-8-centos -t "{{.status.dockerImageRepository}}")" ]
+# don't delete wildfly-8-centos
 echo "imageRepositories: ok"
 
 osc create -f test/integration/fixtures/test-image-repository.json
@@ -284,4 +283,8 @@ openshift ex registry --create --credentials="${KUBECONFIG}"
 [ "$(openshift ex registry | grep 'service exists')" ]
 echo "ex registry: ok"
 
+# verify the image repository had its tags populated
+[ -n "$(osc get imageRepositories wildfly-8-centos -t "{{.tags.latest}}")" ]
+[ -n "$(osc get imageRepositories wildfly-8-centos -t "{{ index .metadata.annotations \"openshift.io/image.dockerRepositoryCheck\"}}")" ]
+
 osc get minions,pods

hack/test-end-to-end.sh

@@ -250,7 +250,7 @@ osc create -n test -f "${STI_CONFIG_FILE}"
 
 # Trigger build
 echo "[INFO] Starting build from ${STI_CONFIG_FILE} and streaming its logs..."
-osc start-build -n test ruby-sample-build --follow
+#osc start-build -n test ruby-sample-build --follow
 wait_for_build "test"
 wait_for_app "test"
 

pkg/cmd/server/origin/master.go

@@ -52,6 +52,7 @@ import (
 	deployetcd "github.com/openshift/origin/pkg/deploy/registry/etcd"
 	deployrollback "github.com/openshift/origin/pkg/deploy/rollback"
 	"github.com/openshift/origin/pkg/dns"
+	imagecontroller "github.com/openshift/origin/pkg/image/controller"
 	"github.com/openshift/origin/pkg/image/registry/image"
 	imageetcd "github.com/openshift/origin/pkg/image/registry/image/etcd"
 	"github.com/openshift/origin/pkg/image/registry/imagerepository"
@@ -154,6 +155,11 @@ func (c *MasterConfig) ImageChangeControllerClient() *osclient.Client {
 	return c.OSClient
 }
 
+// DeploymentClient returns the deployment client object
+func (c *MasterConfig) ImageImportControllerClient() *osclient.Client {
+	return c.OSClient
+}
+
 // DeploymentControllerClients returns the deployment controller client object
 func (c *MasterConfig) DeploymentControllerClients() (*osclient.Client, *kclient.Client) {
 	return c.OSClient, c.MasterConfigParameters.KubeClient
@@ -739,6 +745,15 @@ func (c *MasterConfig) RunDeploymentImageChangeTriggerController() {
 	controller.Run()
 }
 
+func (c *MasterConfig) RunImageImportController() {
+	osclient := c.ImageImportControllerClient()
+	factory := imagecontroller.ImportControllerFactory{
+		Client: osclient,
+	}
+	controller := factory.Create()
+	controller.Run()
+}
+
 // ensureCORSAllowedOrigins takes a string list of origins and attempts to covert them to CORS origin
 // regexes, or exits if it cannot.
 func (c *MasterConfig) ensureCORSAllowedOrigins() []*regexp.Regexp {

pkg/cmd/server/start.go

@@ -92,6 +92,7 @@ func (cfg Config) startMaster() error {
 	openshiftConfig.RunDeploymentConfigController()
 	openshiftConfig.RunDeploymentConfigChangeController()
 	openshiftConfig.RunDeploymentImageChangeTriggerController()
+	openshiftConfig.RunImageImportController()
 	openshiftConfig.RunProjectAuthorizationCache()
 
 	return nil

pkg/dockerregistry/client.go

@@ -18,6 +18,12 @@ type Client interface {
 
 // Connection allows you to retrieve data from a Docker V1 registry.
 type Connection interface {
+	// ImageTags will return a map of the tags for the image by namespace image by
+	// namespace (if not specified, will be "library") and name.
+	ImageTags(namespace, name string) (map[string]string, error)
+	// ImageByID will return the requested image by namespace (if not specified,
+	// will be "library"), name, and ID.
+	ImageByID(namespace, name, id string) (*docker.Image, error)
 	// ImageByTag will return the requested image by namespace (if not specified,
 	// will be "library"), name, and tag (if not specified, "latest").
 	ImageByTag(namespace, name, tag string) (*docker.Image, error)
@@ -60,12 +66,14 @@ func convertConnectionError(registry string, err error) error {
 type connection struct {
 	client *http.Client
 	host   string
+	cached map[string]*repository
 }
 
 func newConnection(name string) connection {
 	return connection{
 		host:   name,
 		client: http.DefaultClient,
+		cached: make(map[string]*repository),
 	}
 }
 
@@ -75,6 +83,41 @@ type repository struct {
 	token    string
 }
 
+// ImageTags returns the tags for the named Docker image repository.
+func (c connection) ImageTags(namespace, name string) (map[string]string, error) {
+	if len(namespace) == 0 {
+		namespace = "library"
+	}
+	if len(name) == 0 {
+		return nil, fmt.Errorf("image name must be specified")
+	}
+
+	repo, err := c.getCachedRepository(fmt.Sprintf("%s/%s", namespace, name))
+	if err != nil {
+		return nil, err
+	}
+
+	return c.getTags(repo)
+}
+
+// ImageByID returns the specified image within the named Docker image repository
+func (c connection) ImageByID(namespace, name, imageID string) (*docker.Image, error) {
+	if len(namespace) == 0 {
+		namespace = "library"
+	}
+	if len(name) == 0 {
+		return nil, fmt.Errorf("image name must be specified")
+	}
+
+	repo, err := c.getCachedRepository(fmt.Sprintf("%s/%s", namespace, name))
+	if err != nil {
+		return nil, err
+	}
+
+	return c.getImage(repo, imageID, "")
+}
+
+// ImageByTag returns the specified image within the named Docker image repository
 func (c connection) ImageByTag(namespace, name, tag string) (*docker.Image, error) {
 	if len(namespace) == 0 {
 		namespace = "library"
@@ -87,7 +130,7 @@ func (c connection) ImageByTag(namespace, name, tag string) (*docker.Image, erro
 		searchTag = "latest"
 	}
 
-	repo, err := c.getRepository(fmt.Sprintf("%s/%s", namespace, name))
+	repo, err := c.getCachedRepository(fmt.Sprintf("%s/%s", namespace, name))
 	if err != nil {
 		return nil, err
 	}
@@ -100,6 +143,18 @@ func (c connection) ImageByTag(namespace, name, tag string) (*docker.Image, erro
 	return c.getImage(repo, imageID, tag)
 }
 
+func (c connection) getCachedRepository(name string) (*repository, error) {
+	if cached, ok := c.cached[name]; ok {
+		return cached, nil
+	}
+	repo, err := c.getRepository(name)
+	if err != nil {
+		return nil, err
+	}
+	c.cached[name] = repo
+	return repo, nil
+}
+
 func (c connection) getRepository(name string) (*repository, error) {
 	req, err := http.NewRequest("GET", fmt.Sprintf("https://%s/v1/repositories/%s/images", c.host, name), nil)
 	if err != nil {
@@ -123,6 +178,29 @@ func (c connection) getRepository(name string) (*repository, error) {
 	}, nil
 }
 
+func (c connection) getTags(repo *repository) (map[string]string, error) {
+	req, err := http.NewRequest("GET", fmt.Sprintf("https://%s/v1/repositories/%s/tags", repo.endpoint, repo.name), nil)
+	if err != nil {
+		return nil, fmt.Errorf("error creating request: %v", err)
+	}
+	req.Header.Add("Authorization", "Token "+repo.token)
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, convertConnectionError(c.host, fmt.Errorf("error getting image tags for %s: %v", repo.name, err))
+	}
+	switch code := resp.StatusCode; {
+	case code == http.StatusNotFound:
+		return nil, errRepositoryNotFound{repo.name}
+	case code >= 300 || resp.StatusCode < 200:
+		return nil, fmt.Errorf("error retrieving tags: server returned %d", resp.StatusCode)
+	}
+	tags := make(map[string]string)
+	if err := json.NewDecoder(resp.Body).Decode(&tags); err != nil {
+		return nil, fmt.Errorf("error decoding image %s tags: %v", repo.name, err)
+	}
+	return tags, nil
+}
+
 func (c connection) getTag(repo *repository, tag, userTag string) (string, error) {
 	req, err := http.NewRequest("GET", fmt.Sprintf("https://%s/v1/repositories/%s/tags/%s", repo.endpoint, repo.name, tag), nil)
 	if err != nil {
@@ -158,7 +236,7 @@ func (c connection) getImage(repo *repository, image, userTag string) (*docker.I
 	}
 	switch code := resp.StatusCode; {
 	case code == http.StatusNotFound:
-		return nil, errImageNotFound{userTag, image, repo.name}
+		return nil, NewImageNotFoundError(repo.name, image, userTag)
 	case code >= 300 || resp.StatusCode < 200:
 		return nil, fmt.Errorf("error retrieving image %s: server returned %d", req.URL, resp.StatusCode)
 	}
@@ -197,8 +275,15 @@ type errImageNotFound struct {
 	repository string
 }
 
+func NewImageNotFoundError(repository, image, tag string) error {
+	return errImageNotFound{tag, image, repository}
+}
+
 func (e errImageNotFound) Error() string {
-	return fmt.Sprintf("the image %q in repository %q with tag %q was not found and may have been deleted", e.tag, e.image, e.repository)
+	if len(e.tag) == 0 {
+		return fmt.Sprintf("the image %q in repository %q was not found and may have been deleted", e.image, e.repository)
+	}
+	return fmt.Sprintf("the image %q in repository %q with tag %q was not found and may have been deleted", e.image, e.repository, e.tag)
 }
 
 type errRegistryNotFound struct {