diff --git a/go.mod b/go.mod index 289114f389..96e94689c7 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,9 @@ require ( github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46 github.com/aws/aws-sdk-go v1.34.5 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible + github.com/docker/distribution v2.7.1+incompatible + github.com/docker/go-metrics v0.0.1 // indirect + github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect github.com/envoyproxy/protoc-gen-validate v0.1.0 github.com/fsouza/fake-gcs-server v1.21.0 github.com/golang/mock v1.4.4 @@ -19,6 +22,8 @@ require ( github.com/hashicorp/golang-lru v0.5.1 github.com/klauspost/compress v1.10.11 // indirect github.com/minio/minio-go/v7 v7.0.5 + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.0.1 // indirect github.com/prometheus/client_golang v1.6.0 github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.9.1 diff --git a/go.sum b/go.sum index f53eb46d6e..afa018c240 100644 --- a/go.sum +++ b/go.sum @@ -96,6 +96,12 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= +github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= +github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= +github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4= +github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= @@ -268,6 +274,7 @@ github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2 github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= @@ -293,6 +300,7 @@ github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -334,6 +342,10 @@ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= +github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= @@ -347,6 +359,7 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_golang v1.6.0 h1:YVPodQOcK15POxhgARIvnDRVpLcuK8mglnMrWfyrw6A= github.com/prometheus/client_golang v1.6.0/go.mod h1:ZLOG9ck3JLRdB5MgO8f+lLTe83AXG6ro35rLTxvnIl4= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -356,10 +369,12 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.11 h1:DhHlBtkHWPYi8O2y31JkK0TF+DGM+51OopZjH/Ia5qI= github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -372,6 +387,7 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -545,6 +561,7 @@ golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/pkg/app/piped/imageprovider/BUILD.bazel b/pkg/app/piped/imageprovider/BUILD.bazel new file mode 100644 index 0000000000..090250a5c1 --- /dev/null +++ b/pkg/app/piped/imageprovider/BUILD.bazel @@ -0,0 +1,15 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["provider.go"], + importpath = "github.com/pipe-cd/pipe/pkg/app/piped/imageprovider", + visibility = ["//visibility:public"], + deps = [ + "//pkg/app/piped/imageprovider/gcr:go_default_library", + "//pkg/config:go_default_library", + "//pkg/model:go_default_library", + "@com_github_docker_distribution//registry/client/auth/challenge:go_default_library", + "@org_uber_go_zap//:go_default_library", + ], +) diff --git a/pkg/app/piped/imageprovider/gcr/BUILD.bazel b/pkg/app/piped/imageprovider/gcr/BUILD.bazel index 385cef7a25..530e074bd3 100644 --- a/pkg/app/piped/imageprovider/gcr/BUILD.bazel +++ b/pkg/app/piped/imageprovider/gcr/BUILD.bazel @@ -5,4 +5,13 @@ go_library( srcs = ["gcr.go"], importpath = "github.com/pipe-cd/pipe/pkg/app/piped/imageprovider/gcr", visibility = ["//visibility:public"], + deps = [ + "//pkg/config:go_default_library", + "//pkg/model:go_default_library", + "@com_github_docker_distribution//registry/client:go_default_library", + "@com_github_docker_distribution//registry/client/auth:go_default_library", + "@com_github_docker_distribution//registry/client/auth/challenge:go_default_library", + "@com_github_docker_distribution//registry/client/transport:go_default_library", + "@org_uber_go_zap//:go_default_library", + ], ) diff --git a/pkg/app/piped/imageprovider/gcr/gcr.go b/pkg/app/piped/imageprovider/gcr/gcr.go index 6392f3566d..e1f8ee24a4 100644 --- a/pkg/app/piped/imageprovider/gcr/gcr.go +++ b/pkg/app/piped/imageprovider/gcr/gcr.go @@ -13,3 +13,94 @@ // limitations under the License. package gcr + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "github.com/docker/distribution/registry/client" + "github.com/docker/distribution/registry/client/auth" + "github.com/docker/distribution/registry/client/auth/challenge" + "github.com/docker/distribution/registry/client/transport" + "go.uber.org/zap" + + "github.com/pipe-cd/pipe/pkg/config" + "github.com/pipe-cd/pipe/pkg/model" +) + +type Provider struct { + name string + baseURL url.URL + transport http.RoundTripper + + logger *zap.Logger +} + +type determineURL func(manager challenge.Manager, tx http.RoundTripper, domain string) (*url.URL, error) + +func NewProvider(name string, cfg *config.ImageProviderGCRConfig, fn determineURL, logger *zap.Logger) (*Provider, error) { + var tx http.RoundTripper = &http.Transport{ + MaxIdleConns: 10, + IdleConnTimeout: 10 * time.Second, + Proxy: http.ProxyFromEnvironment, + } + manager := challenge.NewSimpleManager() + + u, err := fn(manager, tx, cfg.Address) + if err != nil { + return nil, fmt.Errorf("failed to determine registry URL: %w", err) + } + a := newAuthorizer(tx, manager) + return &Provider{ + name: name, + baseURL: *u, + transport: transport.NewTransport(tx, a), + logger: logger.Named("gcr-provider"), + }, nil +} + +func (p *Provider) Name() string { + return p.name +} + +func (p *Provider) Type() model.ImageProviderType { + return model.ImageProviderTypeGCR +} + +func (p *Provider) ParseImage(image string) (*model.ImageName, error) { + ss := strings.SplitN(image, "/", 2) + if len(ss) < 2 { + return nil, fmt.Errorf("invalid image format (e.g. gcr.io/pipecd/helloworld)") + } + return &model.ImageName{ + Domain: ss[0], + Repo: ss[1], + }, nil +} + +func (p *Provider) GetLatestImage(ctx context.Context, image *model.ImageName) (*model.ImageRef, error) { + repository, err := client.NewRepository(image, p.baseURL.String(), p.transport) + if err != nil { + return nil, err + } + // TODO: Stop listing all tags + _, err = repository.Tags(ctx).All(ctx) + if err != nil { + return nil, err + } + // TODO: Give back latest image from GCR + return nil, nil +} + +func newAuthorizer(tx http.RoundTripper, manager challenge.Manager) transport.RequestModifier { + // TODO: Use credentials for GCR configured by user + authHandlers := []auth.AuthenticationHandler{ + auth.NewTokenHandler(tx, nil, "", "pull"), + auth.NewBasicHandler(nil), + } + return auth.NewAuthorizer(manager, authHandlers...) +} diff --git a/pkg/app/piped/imageprovider/provider.go b/pkg/app/piped/imageprovider/provider.go new file mode 100644 index 0000000000..b9fb02ceb9 --- /dev/null +++ b/pkg/app/piped/imageprovider/provider.go @@ -0,0 +1,73 @@ +// Copyright 2020 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package imageprovider + +import ( + "context" + "fmt" + "net/http" + "net/url" + + "github.com/docker/distribution/registry/client/auth/challenge" + "go.uber.org/zap" + + "github.com/pipe-cd/pipe/pkg/app/piped/imageprovider/gcr" + "github.com/pipe-cd/pipe/pkg/config" + "github.com/pipe-cd/pipe/pkg/model" +) + +// Provider acts as a container registry client. +type Provider interface { + // Name gives back the provider name that is unique in the Piped. + Name() string + // Type indicates which container registry client to act as. + Type() model.ImageProviderType + // ParseImage converts the given string into structured image. + ParseImage(image string) (*model.ImageName, error) + // GetLatestImages gives back an image with the latest tag. + GetLatestImage(ctx context.Context, image *model.ImageName) (*model.ImageRef, error) +} + +// NewProvider yields an appropriate provider according to the given config. +func NewProvider(cfg *config.PipedImageProvider, logger *zap.Logger) (Provider, error) { + switch cfg.Type { + case model.ImageProviderTypeGCR: + return gcr.NewProvider(cfg.Name, cfg.GCRConfig, doChallenge, logger) + case model.ImageProviderTypeDockerHub: + return nil, fmt.Errorf("not implemented yet") + case model.ImageProviderTypeECR: + return nil, fmt.Errorf("not implemented yet") + default: + return nil, fmt.Errorf("unknown image provider type: %s", cfg.Type) + } +} + +func doChallenge(manager challenge.Manager, tx http.RoundTripper, domain string) (*url.URL, error) { + registryURL := url.URL{ + Scheme: "https", + Host: domain, + Path: "/v2/", + } + cs, err := manager.GetChallenges(registryURL) + if err != nil { + return nil, err + } + if len(cs) == 0 { + // TODO: Handle the no challenge case + // referring to https://github.com/fluxcd/flux/blob/72743f209207453a4326757ba89fb03cb514b34d/pkg/registry/client_factory.go#L64-L91 + } + + return ®istryURL, nil +} diff --git a/pkg/app/piped/imagewatcher/BUILD.bazel b/pkg/app/piped/imagewatcher/BUILD.bazel index deb8ad750d..11fa942c48 100644 --- a/pkg/app/piped/imagewatcher/BUILD.bazel +++ b/pkg/app/piped/imagewatcher/BUILD.bazel @@ -5,5 +5,10 @@ go_library( srcs = ["watcher.go"], importpath = "github.com/pipe-cd/pipe/pkg/app/piped/imagewatcher", visibility = ["//visibility:public"], - deps = ["@org_uber_go_zap//:go_default_library"], + deps = [ + "//pkg/app/piped/imageprovider:go_default_library", + "//pkg/config:go_default_library", + "//pkg/git:go_default_library", + "@org_uber_go_zap//:go_default_library", + ], ) diff --git a/pkg/app/piped/imagewatcher/watcher.go b/pkg/app/piped/imagewatcher/watcher.go index c7a86fb350..d860271e86 100644 --- a/pkg/app/piped/imagewatcher/watcher.go +++ b/pkg/app/piped/imagewatcher/watcher.go @@ -13,68 +13,176 @@ // limitations under the License. // Package imagewatcher provides a piped component -// that periodically checks the image registry and updates +// that periodically checks the container registry and updates // the image if there are differences with Git. package imagewatcher import ( "context" + "sync" "time" "go.uber.org/zap" + + "github.com/pipe-cd/pipe/pkg/app/piped/imageprovider" + "github.com/pipe-cd/pipe/pkg/config" + "github.com/pipe-cd/pipe/pkg/git" ) type Watcher interface { Run(context.Context) error } -type watcher struct { - timer *time.Timer - logger *zap.Logger +type gitClient interface { + Clone(ctx context.Context, repoID, remote, branch, destination string) (git.Repo, error) } -type imageRepos map[string]imageRepo -type imageRepo struct { +type watcher struct { + config *config.PipedSpec + gitClient gitClient + logger *zap.Logger + wg sync.WaitGroup + mu sync.Mutex + + // Indexed by repo id. + gitRepos map[string]git.Repo } -func NewWatcher(interval time.Duration, logger *zap.Logger) Watcher { +func NewWatcher(cfg *config.PipedSpec, gitClient gitClient, logger *zap.Logger) Watcher { return &watcher{ - timer: time.NewTimer(interval), - logger: logger, + config: cfg, + gitClient: gitClient, + logger: logger.Named("image-watcher"), } } +// Run spawns goroutines for each image provider. func (w *watcher) Run(ctx context.Context) error { + // Pre-clone to cache the registered git repositories. + for _, r := range w.config.Repositories { + repo, err := w.gitClient.Clone(ctx, r.RepoID, r.Remote, r.Branch, "") + if err != nil { + w.logger.Error("failed to clone repository", + zap.String("repo-id", r.RepoID), + zap.Error(err), + ) + return err + } + w.gitRepos[r.RepoID] = repo + } + + for _, cfg := range w.config.ImageProviders { + p, err := imageprovider.NewProvider(&cfg, w.logger) + if err != nil { + return err + } + + w.wg.Add(1) + go w.run(ctx, p, cfg.PullInterval.Duration()) + } + w.wg.Wait() + return nil +} + +// run periodically compares the image stored in the given provider and one stored in git. +// And then pushes those with differences. +func (w *watcher) run(ctx context.Context, provider imageprovider.Provider, interval time.Duration) { + defer w.wg.Done() + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { select { case <-ctx.Done(): - return nil - case <-w.timer.C: - reposInReg := w.fetchFromRegistry() - reposInGit := w.fetchFromGit() - outdated := calculateChanges(reposInReg, reposInGit) - if err := w.update(outdated); err != nil { + return + case <-ticker.C: + targets := w.collectTargets(ctx, provider) + outdated, err := determineUpdates(ctx, targets, provider) + if err != nil { + w.logger.Error("failed to determine which one should be updated", zap.Error(err)) + continue + } + if len(outdated) == 0 { + w.logger.Info("no image to be updated") + continue + } + if err := update(outdated); err != nil { w.logger.Error("failed to update image", zap.Error(err)) + continue } } } - return nil } -func (w *watcher) fetchFromRegistry() imageRepos { - return nil +// collectTarget collects target images for each git repository. +func (w *watcher) collectTargets(ctx context.Context, provider imageprovider.Provider) (targets []config.ImageWatcherTarget) { + for id, repo := range w.gitRepos { + branch := repo.GetClonedBranch() + w.mu.Lock() + err := repo.Pull(ctx, branch) + w.mu.Unlock() + if err != nil { + w.logger.Error("failed to update repository branch", + zap.String("repo-id", id), + zap.Error(err), + ) + continue + } + + includes := make([]string, 0) + excludes := make([]string, 0) + for _, target := range w.config.ImageWatcher.Repos { + if target.RepoID != id { + continue + } + includes = append(includes, target.Includes...) + excludes = append(excludes, target.Excludes...) + } + cfg, ok, err := config.LoadImageWatcher(repo.GetPath(), includes, excludes) + if err != nil { + w.logger.Error("failed to load configuration file for Image Watcher", zap.Error(err)) + continue + } + if !ok { + w.logger.Error("configuration file for Image Watcher not found", zap.Error(err)) + continue + } + t := filterTargets(provider.Name(), cfg.Targets) + targets = append(targets, t...) + } + return } -func (w *watcher) fetchFromGit() imageRepos { - return nil +// filterTargets gives back the targets corresponding to the given provider. +func filterTargets(provider string, targets []config.ImageWatcherTarget) (filtered []config.ImageWatcherTarget) { + for _, t := range targets { + if t.Provider == provider { + filtered = append(filtered, t) + } + } + return } -func (w *watcher) update(targets imageRepos) error { - return nil +// determineUpdates gives back target images to be updated. +func determineUpdates(ctx context.Context, targets []config.ImageWatcherTarget, provider imageprovider.Provider) (outdated []config.ImageWatcherTarget, err error) { + for _, target := range targets { + i, err := provider.ParseImage(target.Image) + if err != nil { + return nil, err + } + // TODO: Control not to reach the rate limit + _, err = provider.GetLatestImage(ctx, i) + if err != nil { + return nil, err + } + // TODO: Compares between image repos in the image registry and image repos in git + // And then gives back image repos to be updated. + } + + return } -// calculateChanges compares between image repos in the image registry and -// image repos in git. And then gives back image repos to be updated. -func calculateChanges(x, y imageRepos) imageRepos { +func update(targets []config.ImageWatcherTarget) error { + // TODO: Make it possible to push outdated images to Git return nil } diff --git a/pkg/config/BUILD.bazel b/pkg/config/BUILD.bazel index 78d1857624..49e90326f3 100644 --- a/pkg/config/BUILD.bazel +++ b/pkg/config/BUILD.bazel @@ -13,6 +13,7 @@ go_library( "deployment_lambda.go", "deployment_terraform.go", "duration.go", + "image_watcher.go", "piped.go", "replicas.go", "sealed_secret.go", diff --git a/pkg/config/config.go b/pkg/config/config.go index 3396f72a93..409d4a31db 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -61,6 +61,8 @@ const ( // This configuration file should be placed in .pipe directory // at the root of the repository. KindAnalysisTemplate Kind = "AnalysisTemplate" + // KindImageWatcher represents configuration for Repo Watcher. + KindImageWatcher Kind = "ImageWatcher" ) // Config represents configuration data load from file. @@ -79,6 +81,7 @@ type Config struct { PipedSpec *PipedSpec ControlPlaneSpec *ControlPlaneSpec AnalysisTemplateSpec *AnalysisTemplateSpec + ImageWatcherSpec *ImageWatcherSpec SealedSecretSpec *SealedSecretSpec } @@ -138,6 +141,10 @@ func (c *Config) init(kind Kind, apiVersion string) error { c.SealedSecretSpec = &SealedSecretSpec{} c.spec = c.SealedSecretSpec + case KindImageWatcher: + c.ImageWatcherSpec = &ImageWatcherSpec{} + c.spec = c.ImageWatcherSpec + default: return fmt.Errorf("unsupported kind: %s", c.Kind) } diff --git a/pkg/config/deployment.go b/pkg/config/deployment.go index 8ed507498f..cfe79f9d24 100644 --- a/pkg/config/deployment.go +++ b/pkg/config/deployment.go @@ -263,21 +263,6 @@ type TemplatableAnalysisHTTP struct { Template AnalysisTemplateRef `json:"template"` } -type DeploymentImageWatcher struct { - Targets []ImageWatcherTarget `json:"targets"` -} - -type ImageWatcherTarget struct { - Provider string `json:"provider"` - Image string `json:"image"` - Path ImageWatcherTargetPath `json:"path"` -} - -type ImageWatcherTargetPath struct { - Filename string `json:"filename"` - Field string `json:"field"` -} - type SealedSecretMapping struct { // Relative path from the application directory to sealed secret file. Path string `json:"path"` diff --git a/pkg/config/deployment_kubernetes.go b/pkg/config/deployment_kubernetes.go index bd34f8cf9e..2da86ddfac 100644 --- a/pkg/config/deployment_kubernetes.go +++ b/pkg/config/deployment_kubernetes.go @@ -34,8 +34,6 @@ type KubernetesDeploymentSpec struct { Workloads []K8sResourceReference `json:"workloads"` // Which method should be used for traffic routing. TrafficRouting *KubernetesTrafficRouting `json:"trafficRouting"` - // Configuration for automatic image updates. - ImageWatcher DeploymentImageWatcher `json:"imageWatcher"` } // Validate returns an error if any wrong configuration value was found. diff --git a/pkg/config/image_watcher.go b/pkg/config/image_watcher.go new file mode 100644 index 0000000000..1feecbf002 --- /dev/null +++ b/pkg/config/image_watcher.go @@ -0,0 +1,38 @@ +// Copyright 2020 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +type ImageWatcherSpec struct { + Targets []ImageWatcherTarget `json:"targets"` +} + +type ImageWatcherTarget struct { + Provider string `json:"provider"` + Image string `json:"image"` + FilePath string `json:"filePath"` + Field string `json:"field"` +} + +// LoadImageWatcher finds the config files for the image watcher in the .pipe directory first up. +// And returns parsed config, False is returned as the second returned value if not found. +func LoadImageWatcher(repoRoot string, includes, excludes []string) (*ImageWatcherSpec, bool, error) { + // TODO: Load image watcher config + // referring to AnalysisTemplateSpec + return nil, false, nil +} + +func (s *ImageWatcherSpec) Validate() error { + return nil +} diff --git a/pkg/config/piped.go b/pkg/config/piped.go index 3164590259..37580fbb53 100644 --- a/pkg/config/piped.go +++ b/pkg/config/piped.go @@ -59,6 +59,8 @@ type PipedSpec struct { Notifications Notifications `json:"notifications"` // How the sealed secret should be managed. SealedSecretManagement *SealedSecretManagement `json:"sealedSecretManagement"` + // Configuration for image watcher. + ImageWatcher PipedImageWatcher `json:"imageWatcher"` } // Validate validates configured data of all fields. @@ -374,18 +376,22 @@ type AnalysisProviderStackdriverConfig struct { } type PipedImageProvider struct { - Name string - Type model.ImageProviderType + Name string `json:"name"` + Type model.ImageProviderType `json:"type"` + // Default is 5m. + PullInterval Duration `json:"pullInterval"` - DockerhubConfig *ImageProviderDockerhubConfig + DockerHubConfig *ImageProviderDockerHubConfig GCRConfig *ImageProviderGCRConfig ECRConfig *ImageProviderECRConfig } type genericPipedImageProvider struct { - Name string `json:"name"` - Type model.ImageProviderType `json:"type"` - Config json.RawMessage `json:"config"` + Name string `json:"name"` + Type model.ImageProviderType `json:"type"` + PullInterval Duration `json:"pullInterval"` + + Config json.RawMessage `json:"config"` } func (p *PipedImageProvider) UnmarshalJSON(data []byte) error { @@ -396,12 +402,16 @@ func (p *PipedImageProvider) UnmarshalJSON(data []byte) error { } p.Name = gp.Name p.Type = gp.Type + p.PullInterval = gp.PullInterval + if p.PullInterval == 0 { + p.PullInterval = Duration(5 * time.Minute) + } switch p.Type { - case model.ImageProviderTypeDockerhub: - p.DockerhubConfig = &ImageProviderDockerhubConfig{} + case model.ImageProviderTypeDockerHub: + p.DockerHubConfig = &ImageProviderDockerHubConfig{} if len(gp.Config) > 0 { - err = json.Unmarshal(gp.Config, p.DockerhubConfig) + err = json.Unmarshal(gp.Config, p.DockerHubConfig) } case model.ImageProviderTypeGCR: p.GCRConfig = &ImageProviderGCRConfig{} @@ -420,14 +430,18 @@ func (p *PipedImageProvider) UnmarshalJSON(data []byte) error { } type ImageProviderGCRConfig struct { + Address string `json:"address"` + CredentialsFile string `json:"credentialsFile"` } -type ImageProviderDockerhubConfig struct { +type ImageProviderDockerHubConfig struct { Username string `json:"username"` PasswordFile string `json:"passwordFile"` } type ImageProviderECRConfig struct { + Address string `json:"address"` + TokenFile string `json:"tokenFile"` } type Notifications struct { @@ -554,3 +568,15 @@ func (p *SealedSecretManagement) UnmarshalJSON(data []byte) error { } return err } + +type PipedImageWatcher struct { + Repos []PipedImageWatcherRepoTarget `json:"repos"` +} + +type PipedImageWatcherRepoTarget struct { + RepoID string `json:"repoId"` + // The paths to ImageWatcher files to be included. + Includes []string `json:"includes"` + // The paths to ImageWatcher files to be excluded. + Excludes []string `json:"excludes"` +} diff --git a/pkg/config/piped_test.go b/pkg/config/piped_test.go index d3a280e946..fc5e846252 100644 --- a/pkg/config/piped_test.go +++ b/pkg/config/piped_test.go @@ -150,13 +150,32 @@ func TestPipedConfig(t *testing.T) { }, ImageProviders: []PipedImageProvider{ { - Name: "my-dockerhub", - Type: "DOCKERHUB", - DockerhubConfig: &ImageProviderDockerhubConfig{ + Name: "my-dockerhub", + Type: "DOCKER_HUB", + PullInterval: Duration(time.Minute * 5), + DockerHubConfig: &ImageProviderDockerHubConfig{ Username: "foo", PasswordFile: "/etc/piped-secret/dockerhub-pass", }, }, + { + Name: "my-gcr", + Type: "GCR", + PullInterval: Duration(time.Minute * 5), + GCRConfig: &ImageProviderGCRConfig{ + Address: "asia.gcr.io", + CredentialsFile: "/etc/piped-secret/gcr-service-account", + }, + }, + { + Name: "my-ecr", + Type: "ECR", + PullInterval: Duration(time.Minute * 5), + ECRConfig: &ImageProviderECRConfig{ + Address: "012345678910.dkr.ecr.us-east-1.amazonaws.com", + TokenFile: "/etc/piped-secret/ecr-authorization-token", + }, + }, }, Notifications: Notifications{ Routes: []NotificationRoute{ @@ -204,6 +223,14 @@ func TestPipedConfig(t *testing.T) { PublicKeyFile: "/etc/piped-secret/sealing-public-key", }, }, + ImageWatcher: PipedImageWatcher{ + Repos: []PipedImageWatcherRepoTarget{ + { + RepoID: "foo", + Includes: []string{".pipe/imagewatcher-dev.yaml"}, + }, + }, + }, }, expectedError: nil, }, diff --git a/pkg/config/testdata/.pipe/image-watcher.yaml b/pkg/config/testdata/.pipe/image-watcher.yaml new file mode 100644 index 0000000000..2f83a5c7c5 --- /dev/null +++ b/pkg/config/testdata/.pipe/image-watcher.yaml @@ -0,0 +1,12 @@ +apiVersion: pipecd.dev/v1beta1 +kind: ImageWatcher +spec: + targets: + - image: gcr.io/pipecd/foo + provider: my-gcr + filePath: foo/deployment.yaml + field: spec.containers[0].image + - image: pipecd/bar + provider: my-dockerhub + filePath: bar/deployment.yaml + field: spec.containers[0].image diff --git a/pkg/config/testdata/piped/piped-config.yaml b/pkg/config/testdata/piped/piped-config.yaml index 1d47843730..74ef3a124f 100644 --- a/pkg/config/testdata/piped/piped-config.yaml +++ b/pkg/config/testdata/piped/piped-config.yaml @@ -79,10 +79,23 @@ spec: imageProviders: - name: my-dockerhub - type: DOCKERHUB + type: DOCKER_HUB + pullInterval: 5m config: username: foo passwordFile: /etc/piped-secret/dockerhub-pass + - name: my-gcr + type: GCR + pullInterval: 5m + config: + address: asia.gcr.io + credentialsFile: /etc/piped-secret/gcr-service-account + - name: my-ecr + type: ECR + pullInterval: 5m + config: + address: 012345678910.dkr.ecr.us-east-1.amazonaws.com + tokenFile: /etc/piped-secret/ecr-authorization-token notifications: routes: @@ -120,3 +133,9 @@ spec: # keyName: key-name # decryptServiceAccountFile: /etc/piped-secret/decrypt-service-account.json # encryptServiceAccountFile: /etc/piped-secret/encrypt-service-account.json + + imageWatcher: + repos: + - repoId: foo + includes: + - .pipe/imagewatcher-dev.yaml \ No newline at end of file diff --git a/pkg/model/BUILD.bazel b/pkg/model/BUILD.bazel index f2c0765703..18a7cb4019 100644 --- a/pkg/model/BUILD.bazel +++ b/pkg/model/BUILD.bazel @@ -53,6 +53,7 @@ go_library( "environment.go", "event.go", "filestore.go", + "image_name.go", "imageprovider.go", "model.go", "piped.go", @@ -75,6 +76,7 @@ go_test( size = "small", srcs = [ "apikey_test.go", + "image_name_test.go", "model_test.go", "piped_test.go", "project_test.go", diff --git a/pkg/model/image_name.go b/pkg/model/image_name.go new file mode 100644 index 0000000000..02da26aeac --- /dev/null +++ b/pkg/model/image_name.go @@ -0,0 +1,62 @@ +// Copyright 2020 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "fmt" + "path" +) + +// ImageName represents an untagged image. Note that images may have +// the domain omitted (e.g. Docker Hub). If they only have single path element, +// the prefix `library` is implied. +// +// Examples: +// - alpine +// - library/alpine +// - gcr.io/pipecd/helloworld +type ImageName struct { + Domain string + Repo string +} + +func (i ImageName) String() string { + return path.Join(i.Domain, i.Repo) +} + +// Name gives back just repository name without domain. +func (i ImageName) Name() string { + return i.Repo +} + +// ImageRef represents a tagged image. The tag is allowed to be +// empty, though it is in general undefined what that means +// +// Examples: +// - alpine:3.0 +// - library/alpine:3.0 +// - gcr.io/pipecd/helloworld:0.1.0 +type ImageRef struct { + ImageName + Tag string +} + +func (i ImageRef) String() string { + if i.Tag == "" { + return i.ImageName.String() + } + + return fmt.Sprintf("%s:%s", i.ImageName.String(), i.Tag) +} diff --git a/pkg/model/image_name_test.go b/pkg/model/image_name_test.go new file mode 100644 index 0000000000..3f0bf66151 --- /dev/null +++ b/pkg/model/image_name_test.go @@ -0,0 +1,83 @@ +package model + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestImageNameString(t *testing.T) { + testcases := []struct { + name string + domain string + repo string + want string + }{ + { + name: "empty repo", + want: "", + }, + { + name: "domain omitted", + repo: "repo", + want: "repo", + }, + { + name: "with domain", + domain: "domain", + repo: "repo", + want: "domain/repo", + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + i := ImageName{ + Repo: tc.repo, + Domain: tc.domain, + } + got := i.String() + assert.Equal(t, tc.want, got) + }) + } +} + +func TestImageRefString(t *testing.T) { + testcases := []struct { + name string + imageName ImageName + tag string + want string + }{ + { + name: "empty repo", + want: "", + }, + { + name: "tag omitted", + imageName: ImageName{ + Domain: "domain", + Repo: "repo", + }, + want: "domain/repo", + }, + { + name: "with tag", + imageName: ImageName{ + Domain: "domain", + Repo: "repo", + }, + tag: "tag", + want: "domain/repo:tag", + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + i := ImageRef{ + ImageName: tc.imageName, + Tag: tc.tag, + } + got := i.String() + assert.Equal(t, tc.want, got) + }) + } +} diff --git a/pkg/model/imageprovider.go b/pkg/model/imageprovider.go index 2e4d5a603b..48a8223c5d 100644 --- a/pkg/model/imageprovider.go +++ b/pkg/model/imageprovider.go @@ -17,7 +17,7 @@ package model type ImageProviderType string const ( - ImageProviderTypeDockerhub ImageProviderType = "DOCKERHUB" + ImageProviderTypeDockerHub ImageProviderType = "DOCKER_HUB" ImageProviderTypeGCR ImageProviderType = "GCR" ImageProviderTypeECR ImageProviderType = "ECR" ) diff --git a/repositories.bzl b/repositories.bzl index 877e656c3a..1d1fedfa7a 100644 --- a/repositories.bzl +++ b/repositories.bzl @@ -231,6 +231,25 @@ def go_repositories(): sum = "h1:RMLoZVzv4GliuWafOuPuQDKSm1SJph7uCRnnS61JAn4=", version = "v0.0.0-20181026042036-e10d5fee7954", ) + go_repository( + name = "com_github_docker_distribution", + importpath = "github.com/docker/distribution", + sum = "h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=", + version = "v2.7.1+incompatible", + ) + go_repository( + name = "com_github_docker_go_metrics", + importpath = "github.com/docker/go-metrics", + sum = "h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8=", + version = "v0.0.1", + ) + go_repository( + name = "com_github_docker_libtrust", + importpath = "github.com/docker/libtrust", + sum = "h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4=", + version = "v0.0.0-20160708172513-aabc10ec26b7", + ) + go_repository( name = "com_github_docker_spdystream", importpath = "github.com/docker/spdystream", @@ -1070,6 +1089,18 @@ def go_repositories(): sum = "h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=", version = "v1.7.0", ) + go_repository( + name = "com_github_opencontainers_go_digest", + importpath = "github.com/opencontainers/go-digest", + sum = "h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=", + version = "v1.0.0", + ) + go_repository( + name = "com_github_opencontainers_image_spec", + importpath = "github.com/opencontainers/image-spec", + sum = "h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI=", + version = "v1.0.1", + ) go_repository( name = "com_github_openpeedeep_depguard",