Skip to content

Make GCR image provider available #1344

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pipecd-bot merged 2 commits into from
Jan 6, 2021
Merged

Make GCR image provider available #1344

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fa1eac8
Make GCR image provider available
nakabonne Jan 4, 2021
87a5339
Move go_repository as a workaround
nakabonne Jan 6, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 25 additions & 13 deletions WORKSPACE
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,19 +28,6 @@ load(
"go_rules_dependencies",
)

go_rules_dependencies()

go_register_toolchains(
go_version = "1.14.10",
)

load(
"@io_bazel_rules_go//extras:embed_data_deps.bzl",
"go_embed_data_dependencies",
)

go_embed_data_dependencies()

http_archive(
name = "bazel_gazelle",
sha256 = "b85f48fa105c4403326e9525ad2b2cc437babaa6e15a3fc0b1dbab0ab064bc7c",
Expand All @@ -56,6 +43,31 @@ load(
"go_repository",
)

# TODO: Investigate why "go-containerregistry" package is overridden
# The downloaded source inside bazel-pipe/external/com_github_google_go_containerregistry is not containing some functions.
# Maybe gazelle downloaded the wrong version.
# As a workaround, we moved the go_repository part from repositories.bzl to here. We have to stop doing that ASAP.
# See: https://github.com/pipe-cd/pipe/pull/1344#issuecomment-755075915
go_repository(
name = "com_github_google_go_containerregistry",
importpath = "github.com/google/go-containerregistry",
sum = "h1:+vqpHdgIbD7xSeufHJq0iuAx7ILcEeh3fR5Og2nW1R0=",
version = "v0.3.0",
)

go_rules_dependencies()

go_register_toolchains(
go_version = "1.14.10",
)

load(
"@io_bazel_rules_go//extras:embed_data_deps.bzl",
"go_embed_data_dependencies",
)

go_embed_data_dependencies()

gazelle_dependencies()

### Google Protobuf
Expand Down
16 changes: 6 additions & 10 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,37 +9,33 @@ require (
github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46
github.com/aws/aws-sdk-go v1.34.5
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/docker/distribution v2.7.1+incompatible
github.com/docker/go-metrics v0.0.1 // indirect
github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
github.com/envoyproxy/protoc-gen-validate v0.1.0
github.com/fsouza/fake-gcs-server v1.21.0
github.com/goccy/go-yaml v1.8.4
github.com/golang/mock v1.4.4
github.com/golang/protobuf v1.4.2
github.com/gomodule/redigo v2.0.0+incompatible
github.com/google/go-containerregistry v0.3.0
github.com/google/go-github/v29 v29.0.3
github.com/google/uuid v1.1.1
github.com/hashicorp/golang-lru v0.5.1
github.com/hashicorp/golang-lru v0.5.3
github.com/klauspost/compress v1.10.11 // indirect
github.com/minio/minio-go/v7 v7.0.5
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.1 // indirect
github.com/prometheus/client_golang v1.6.0
github.com/prometheus/client_model v0.2.0
github.com/prometheus/common v0.9.1
github.com/robfig/cron/v3 v3.0.1 // indirect
github.com/spf13/cobra v0.0.5
github.com/robfig/cron/v3 v3.0.1
github.com/spf13/cobra v1.0.0
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.5.1
go.mongodb.org/mongo-driver v1.4.0
go.uber.org/atomic v1.7.0
go.uber.org/multierr v1.2.0 // indirect
go.uber.org/zap v1.10.1-0.20190709142728-9a9fa7d4b5f0
golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de
golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0
golang.org/x/net v0.0.0-20200822124328-c89045814202
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
google.golang.org/api v0.31.0
google.golang.org/grpc v1.31.1
google.golang.org/protobuf v1.25.0
Expand Down
251 changes: 231 additions & 20 deletions go.sum
View file
Open in desktop

Large diffs are not rendered by default.

1 change: 0 additions & 1 deletion pkg/app/piped/imageprovider/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ go_library(
"//pkg/app/piped/imageprovider/gcr:go_default_library",
"//pkg/config:go_default_library",
"//pkg/model:go_default_library",
"@com_github_docker_distribution//registry/client/auth/challenge:go_default_library",
"@org_uber_go_zap//:go_default_library",
],
)
8 changes: 3 additions & 5 deletions pkg/app/piped/imageprovider/gcr/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,10 @@ go_library(
importpath = "github.com/pipe-cd/pipe/pkg/app/piped/imageprovider/gcr",
visibility = ["//visibility:public"],
deps = [
"//pkg/config:go_default_library",
"//pkg/model:go_default_library",
"@com_github_docker_distribution//registry/client:go_default_library",
"@com_github_docker_distribution//registry/client/auth:go_default_library",
"@com_github_docker_distribution//registry/client/auth/challenge:go_default_library",
"@com_github_docker_distribution//registry/client/transport:go_default_library",
"@com_github_google_go_containerregistry//pkg/authn:go_default_library",
"@com_github_google_go_containerregistry//pkg/name:go_default_library",
"@com_github_google_go_containerregistry//pkg/v1/google:go_default_library",
"@org_uber_go_zap//:go_default_library",
],
)
127 changes: 78 additions & 49 deletions pkg/app/piped/imageprovider/gcr/gcr.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,61 +17,70 @@ package gcr
import (
"context"
"fmt"
"net/http"
"net/url"
"io/ioutil"
"sort"
"strings"
"time"

"github.com/docker/distribution/registry/client"
"github.com/docker/distribution/registry/client/auth"
"github.com/docker/distribution/registry/client/auth/challenge"
"github.com/docker/distribution/registry/client/transport"
"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/name"
"github.com/google/go-containerregistry/pkg/v1/google"
"go.uber.org/zap"

"github.com/pipe-cd/pipe/pkg/config"
"github.com/pipe-cd/pipe/pkg/model"
)

type Provider struct {
name string
baseURL url.URL
transport http.RoundTripper
type GCR struct {
name string
serviceAccountFile string
// If nil, treated as an anonymous user.
authenticator authn.Authenticator
logger *zap.Logger
}

type Option func(*GCR)

logger *zap.Logger
func WithServiceAccountFile(path string) Option {
return func(e *GCR) {
e.serviceAccountFile = path
}
}

type determineURL func(manager challenge.Manager, tx http.RoundTripper, domain string) (*url.URL, error)
func WithLogger(logger *zap.Logger) Option {
return func(e *GCR) {
e.logger = logger
}
}

func NewProvider(name string, cfg *config.ImageProviderGCRConfig, fn determineURL, logger *zap.Logger) (*Provider, error) {
var tx http.RoundTripper = &http.Transport{
MaxIdleConns: 10,
IdleConnTimeout: 10 * time.Second,
Proxy: http.ProxyFromEnvironment,
// NewGCR generates a GCR client with an anonymous user if no authenticate method set.
func NewGCR(name string, opts ...Option) (*GCR, error) {
g := &GCR{
name: name,
logger: zap.NewNop(),
}
for _, opt := range opts {
opt(g)
}
manager := challenge.NewSimpleManager()
g.logger = g.logger.Named("gcr-provider")

u, err := fn(manager, tx, cfg.Address)
if err != nil {
return nil, fmt.Errorf("failed to determine registry URL: %w", err)
if g.serviceAccountFile != "" {
b, err := ioutil.ReadFile(g.serviceAccountFile)
if err != nil {
return nil, fmt.Errorf("failed to open the service account file: %w", err)
}
g.authenticator = google.NewJSONKeyAuthenticator(string(b))
}
a := newAuthorizer(tx, manager)
return &Provider{
name: name,
baseURL: *u,
transport: transport.NewTransport(tx, a),
logger: logger.Named("gcr-provider"),
}, nil
return g, nil
}

func (p *Provider) Name() string {
return p.name
func (g *GCR) Name() string {
return g.name
}

func (p *Provider) Type() model.ImageProviderType {
func (g *GCR) Type() model.ImageProviderType {
return model.ImageProviderTypeGCR
}

func (p *Provider) ParseImage(image string) (*model.ImageName, error) {
func (g *GCR) ParseImage(image string) (*model.ImageName, error) {
ss := strings.SplitN(image, "/", 2)
if len(ss) < 2 {
return nil, fmt.Errorf("invalid image format (e.g. gcr.io/pipecd/helloworld)")
Expand All @@ -82,25 +91,45 @@ func (p *Provider) ParseImage(image string) (*model.ImageName, error) {
}, nil
}

func (p *Provider) GetLatestImage(ctx context.Context, image *model.ImageName) (*model.ImageRef, error) {
repository, err := client.NewRepository(image, p.baseURL.String(), p.transport)
func (g *GCR) GetLatestImage(ctx context.Context, image *model.ImageName) (*model.ImageRef, error) {
repo, err := name.NewRepository(image.String())
if err != nil {
return nil, err
return nil, fmt.Errorf("%s is invalid repository: %w", image, err)
}
options := []google.ListerOption{
google.WithContext(ctx),
}
// TODO: Stop listing all tags
_, err = repository.Tags(ctx).All(ctx)
if g.authenticator != nil {
options = append(options, google.WithAuth(g.authenticator))
}
// TODO: Use pagination to retrieve image tags from GCR
// Currently, the result could be quite large size if there are a lot of tags.
// "google/go-containerregistry" doesn't provide any option to paginate.
// We can propose it to them, or just borrow and modify for us.
// See more: https://docs.docker.com/registry/spec/api/#listing-image-tags
res, err := google.List(repo, options...)
if err != nil {
return nil, err
return nil, fmt.Errorf("failed to list tags: %w", err)
}
if len(res.Manifests) == 0 {
return nil, fmt.Errorf("no manifests found in %s", repo.Name())
}
// TODO: Give back latest image from GCR
return nil, nil
}

func newAuthorizer(tx http.RoundTripper, manager challenge.Manager) transport.RequestModifier {
// TODO: Use credentials for GCR configured by user
authHandlers := []auth.AuthenticationHandler{
auth.NewTokenHandler(tx, nil, "", "pull"),
auth.NewBasicHandler(nil),
// Determine the latest by sorting by the uploaded time.
manifests := make([]google.ManifestInfo, 0, len(res.Manifests))
for _, m := range res.Manifests {
manifests = append(manifests, m)
}
sort.Slice(manifests, func(i, j int) bool {
return manifests[i].Uploaded.After(manifests[j].Uploaded)
})
latest := manifests[0]
if len(latest.Tags) == 0 {
return nil, fmt.Errorf("no tag is associated to the latest image")
}
return auth.NewAuthorizer(manager, authHandlers...)
return &model.ImageRef{
ImageName: *image,
// TODO: Enable to specify the tag if multiple tags are associated to an image
Copy link
Member Author

@nakabonne nakabonne Jan 4, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm looking into using regexp to specify which tag will be used. It would be nice to add the tagRegex field to ImageWatcher config under the .pipe/ directory.

apiVersion: pipecd.dev/v1beta1
kind: ImageWatcher
spec:
  targets:
    - image: gcr.io/pipecd/foo
      provider: my-gcr
      filePath: foo/deployment.yaml
      field: $.spec.template.spec.containers[0].image
      tagRegex: ^([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+)?$

Alternatively, simply providing useSemver or something like that might be nice.

Copy link
Member Author

@nakabonne nakabonne Jan 4, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This option is needed right now for us whose images' have multiple tags: https://gcr.io/pipecd/server

Copy link

@jonjohnsonjr jonjohnsonjr Jan 6, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It would be wonderful if you could leverage pubsub instead of polling every 5 minutes for ImageWatcher, but I understand that you would need to do something specific for each registry, since everything implements this slightly differently.

Copy link
Member Author

@nakabonne nakabonne Jan 6, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@jonjohnsonjr Thank you for taking a look at our project! You're quite right. We don't have to be worried about the pull rate limit if we could support the event hooks for cloud-provider-specific. However, we decided not to do that so as to reduce the burden on users and avoid complexity.

FYI: Alternatively, we're looking into a completely different solution to deal with the rate limit: #1341

Tag: latest.Tags[0],
}, nil
}
31 changes: 7 additions & 24 deletions pkg/app/piped/imageprovider/provider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,7 @@ package imageprovider
import (
"context"
"fmt"
"net/http"
"net/url"

"github.com/docker/distribution/registry/client/auth/challenge"
"go.uber.org/zap"

"github.com/pipe-cd/pipe/pkg/app/piped/imageprovider/ecr"
Expand All @@ -45,9 +42,11 @@ type Provider interface {
func NewProvider(cfg *config.PipedImageProvider, logger *zap.Logger) (Provider, error) {
switch cfg.Type {
case model.ImageProviderTypeGCR:
return gcr.NewProvider(cfg.Name, cfg.GCRConfig, doChallenge, logger)
case model.ImageProviderTypeDockerHub:
return nil, fmt.Errorf("not implemented yet")
options := []gcr.Option{
gcr.WithServiceAccountFile(cfg.GCRConfig.ServiceAccountFile),
gcr.WithLogger(logger),
}
return gcr.NewGCR(cfg.Name, options...)
case model.ImageProviderTypeECR:
options := []ecr.Option{
ecr.WithRegistryID(cfg.ECRConfig.RegistryID),
Expand All @@ -56,25 +55,9 @@ func NewProvider(cfg *config.PipedImageProvider, logger *zap.Logger) (Provider,
ecr.WithLogger(logger),
}
return ecr.NewECR(cfg.Name, cfg.ECRConfig.Region, options...)
case model.ImageProviderTypeDockerHub:
return nil, fmt.Errorf("not implemented yet")
default:
return nil, fmt.Errorf("unknown image provider type: %s", cfg.Type)
}
}

func doChallenge(manager challenge.Manager, tx http.RoundTripper, domain string) (*url.URL, error) {
registryURL := url.URL{
Scheme: "https",
Host: domain,
Path: "/v2/",
}
cs, err := manager.GetChallenges(registryURL)
if err != nil {
return nil, err
}
if len(cs) == 0 {
// TODO: Handle the no challenge case
// referring to https://github.com/fluxcd/flux/blob/72743f209207453a4326757ba89fb03cb514b34d/pkg/registry/client_factory.go#L64-L91
}

return &registryURL, nil
}
4 changes: 2 additions & 2 deletions pkg/config/piped.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -428,8 +428,8 @@ func (p *PipedImageProvider) UnmarshalJSON(data []byte) error {
}

type ImageProviderGCRConfig struct {
Address string `json:"address"`
CredentialsFile string `json:"credentialsFile"`
// Path to the json file of service account with the required "roles/storage.objectViewer" role.
ServiceAccountFile string `json:"serviceAccountFile"`
}

type ImageProviderDockerHubConfig struct {
Expand Down
3 changes: 1 addition & 2 deletions pkg/config/piped_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,8 +161,7 @@ func TestPipedConfig(t *testing.T) {
Name: "my-gcr",
Type: "GCR",
GCRConfig: &ImageProviderGCRConfig{
Address: "asia.gcr.io",
CredentialsFile: "/etc/piped-secret/gcr-service-account",
ServiceAccountFile: "/etc/piped-secret/gcr-service-account.json",
},
},
{
Expand Down
3 changes: 1 addition & 2 deletions pkg/config/testdata/piped/piped-config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,8 +86,7 @@ spec:
- name: my-gcr
type: GCR
config:
address: asia.gcr.io
credentialsFile: /etc/piped-secret/gcr-service-account
serviceAccountFile: /etc/piped-secret/gcr-service-account.json
- name: my-ecr
type: ECR
config:
Expand Down
Loading