From 5c83f5d4cbc6801b892039d37bdd668d0eaa3582 Mon Sep 17 00:00:00 2001 From: jonpan <1191243580@qq.com> Date: Fri, 20 Oct 2023 22:25:09 +0800 Subject: [PATCH 01/12] feat: stash Signed-off-by: jonpan <1191243580@qq.com> --- go.mod | 1 + go.sum | 4 +++ manager/job/preheat.go | 72 ++++++++++++++++++++++++++++++++++++++++-- manager/job/types.go | 13 ++++++++ 4 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 manager/job/types.go diff --git a/go.mod b/go.mod index 6577d3da88b..d81f926e38a 100644 --- a/go.mod +++ b/go.mod @@ -112,6 +112,7 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 // indirect github.com/envoyproxy/protoc-gen-validate v1.0.2 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect diff --git a/go.sum b/go.sum index e276c31646e..47876ce437e 100644 --- a/go.sum +++ b/go.sum @@ -233,6 +233,8 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 h1:ZClxb8laGDf5arXfYcAtECDFgAgHklGI8CxgjHnXKJ4= +github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= @@ -524,6 +526,8 @@ github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI= diff --git a/manager/job/preheat.go b/manager/job/preheat.go index fc58a89e961..d6b751a93bf 100644 --- a/manager/job/preheat.go +++ b/manager/job/preheat.go @@ -32,7 +32,10 @@ import ( "time" machineryv1tasks "github.com/RichardKnop/machinery/v1/tasks" - "github.com/distribution/distribution/v3" + distribution "github.com/distribution/distribution/v3" + "github.com/distribution/distribution/v3/manifest/manifestlist" + "github.com/distribution/distribution/v3/manifest/ocischema" + "github.com/distribution/distribution/v3/manifest/schema1" "github.com/distribution/distribution/v3/manifest/schema2" "github.com/go-http-utils/headers" "github.com/google/uuid" @@ -218,7 +221,12 @@ func (p *preheat) getManifests(ctx context.Context, url string, header http.Head } req.Header = header - req.Header.Add(headers.Accept, schema2.MediaTypeManifest) + // 参考 docker pull 源码 moby https://github.com/moby/moby/blob/6040283f23efbd085eed90fa72d12ae73119837e/vendor/github.com/docker/distribution/registry/client/repository.go#L313 + for _, d := range distribution.ManifestMediaTypes() { + req.Header.Add(headers.Accept, d) + } + + // TOOD: 根据不同类型解析 manifest client := &http.Client{ Timeout: timeout, @@ -243,11 +251,30 @@ func (p *preheat) parseLayers(resp *http.Response, tag, filter string, header ht return nil, err } - manifest, _, err := distribution.UnmarshalManifest(schema2.MediaTypeManifest, body) + mt := resp.Header.Get("Content-Type") + manifest, _, err := distribution.UnmarshalManifest(mt, body) if err != nil { return nil, err } + // TODO: manifest type + + var layers []internaljob.PreheatRequest + for _, v := range manifest.References() { + layer := internaljob.PreheatRequest{ + URL: layerURL(image.protocol, image.domain, image.name, v.Digest.String()), + Tag: tag, + Filter: filter, + Headers: nethttp.HeaderToMap(header), + } + + layers = append(layers, layer) + } + + return layers, nil +} + +func (p *preheat) parseManifestReference(manifest manifestReference, tag, filter string, header http.Header, image *preheatImage) ([]internaljob.PreheatRequest, error) { var layers []internaljob.PreheatRequest for _, v := range manifest.References() { layer := internaljob.PreheatRequest{ @@ -263,6 +290,45 @@ func (p *preheat) parseLayers(resp *http.Response, tag, filter string, header ht return layers, nil } +func (p *preheat) parseManifest(resp *http.Response, tag, filter string, header http.Header, image *preheatImage) ([]internaljob.PreheatRequest, error) { + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + mt := resp.Header.Get("Content-Type") + manifest, _, err := distribution.UnmarshalManifest(mt, body) + if err != nil { + return nil, err + } + + if manifest == nil { + return nil, fmt.Errorf("image manifest does not exist for tag or digest %q", image.tag) + } + + // + switch v := manifest.(type) { + case *schema1.SignedManifest: + return p.parseManifestReference(v, tag, filter, header, image) + case *schema2.DeserializedManifest: + return p.parseManifestReference(v, tag, filter, header, image) + case *ocischema.DeserializedManifest: + return p.parseManifestReference(v, tag, filter, header, image) + case *manifestlist.DeserializedManifestList: + fmt.Println(v) + default: + return nil, invalidManifestFormatError{} + } + // raise no match error + return nil, nil +} + +func (p *preheat) parseManifestList(mfstList *manifestlist.DeserializedManifestList, image *preheatImage) error { + // 解析 manifest list + + return nil +} + // getAuthToken gets auth token from registry. func getAuthToken(ctx context.Context, header http.Header, timeout time.Duration, rootCAs *x509.CertPool) (string, error) { ctx, span := tracer.Start(ctx, config.SpanAuthWithRegistry, trace.WithSpanKind(trace.SpanKindProducer)) diff --git a/manager/job/types.go b/manager/job/types.go new file mode 100644 index 00000000000..570d4be9ddd --- /dev/null +++ b/manager/job/types.go @@ -0,0 +1,13 @@ +package job + +import distribution "github.com/distribution/distribution/v3" + +type invalidManifestFormatError struct{} + +func (invalidManifestFormatError) Error() string { + return "unsupported manifest format" +} + +type manifestReference interface { + References() []distribution.Descriptor +} From ee2867bf154d6e1b3b1b16dc308d8ccff6bbe21d Mon Sep 17 00:00:00 2001 From: jonpan <1191243580@qq.com> Date: Sat, 21 Oct 2023 18:14:12 +0800 Subject: [PATCH 02/12] feat: preheat image supports authentication and parse multi manifest mediatype Signed-off-by: jonpan <1191243580@qq.com> --- go.mod | 11 +- go.sum | 19 ++ manager/job/docker_auth_client.go | 164 +++++++++++++++++ manager/job/preheat.go | 292 +++++++++++------------------- manager/job/preheat_test.go | 70 +++++++ manager/job/types.go | 40 +++- manager/types/job.go | 7 + 7 files changed, 411 insertions(+), 192 deletions(-) create mode 100644 manager/job/docker_auth_client.go create mode 100644 manager/job/preheat_test.go diff --git a/go.mod b/go.mod index d81f926e38a..fb31f222862 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,10 @@ require ( github.com/casbin/casbin/v2 v2.77.2 github.com/casbin/gorm-adapter/v3 v3.5.0 github.com/colinmarc/hdfs/v2 v2.3.0 + github.com/containerd/containerd v1.3.4 github.com/distribution/distribution/v3 v3.0.0-20220620080156-3e4f8a0ab147 + github.com/docker/distribution v2.7.1+incompatible + github.com/docker/docker v20.10.7+incompatible github.com/docker/go-connections v0.4.0 github.com/docker/go-units v0.4.0 github.com/gaius-qi/ping v1.0.0 @@ -51,6 +54,7 @@ require ( github.com/montanaflynn/stats v0.7.1 github.com/onsi/ginkgo/v2 v2.12.0 github.com/onsi/gomega v1.28.0 + github.com/opencontainers/image-spec v1.0.2 github.com/orcaman/concurrent-map/v2 v2.0.1 github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 github.com/prometheus/client_golang v1.16.0 @@ -99,6 +103,7 @@ require ( cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v1.1.1 // indirect cloud.google.com/go/pubsub v1.33.0 // indirect + github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible // indirect github.com/KyleBanks/depth v1.2.1 // indirect github.com/RichardKnop/logging v0.0.0-20190827224416-1a693bdd4fae // indirect @@ -112,6 +117,7 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 // indirect github.com/envoyproxy/protoc-gen-validate v1.0.2 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect @@ -131,6 +137,7 @@ require ( github.com/go-stack/stack v1.8.1 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/goccy/go-json v0.10.2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.4.3 // indirect github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect github.com/golang-sql/sqlexp v0.1.0 // indirect @@ -142,6 +149,7 @@ require ( github.com/google/s2a-go v0.1.5 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect + github.com/gorilla/mux v1.8.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/hashicorp/hcl v1.0.0 // indirect @@ -174,10 +182,11 @@ require ( github.com/mdlayher/socket v0.4.1 // indirect github.com/microsoft/go-mssqldb v0.17.0 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect + github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/morikuni/aec v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.0.2 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/pkg/errors v0.9.1 // indirect diff --git a/go.sum b/go.sum index 47876ce437e..4c1857625e6 100644 --- a/go.sum +++ b/go.sum @@ -59,6 +59,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.0.0/go.mod h1:uGG2W01BaETf0Ozp+Q github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.0.0/go.mod h1:+6sju8gk8FRmSajX3Oz4G5Gm7P+mbqE9FVaXXFYTkCM= github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -193,6 +195,7 @@ github.com/colinmarc/hdfs/v2 v2.3.0/go.mod h1:nsyY1uyQOomU34KVQk9Qb/lDJobN1MQ/9W github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.4 h1:3o0smo5SKY7H6AJCmJhsnCjR2/V2T8VmiHt7seN2/kI= github.com/containerd/containerd v1.3.4/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= @@ -213,6 +216,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -226,11 +231,15 @@ github.com/distribution/distribution/v3 v3.0.0-20220620080156-3e4f8a0ab147/go.mo github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/cli v20.10.7+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v1.4.2-0.20200319182547-c7ad2b866182/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v20.10.7+incompatible h1:Z6O9Nhsjv+ayUEeI1IojKbYcsGdgYSNqxe1s2MYzUhQ= github.com/docker/docker v20.10.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= +github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 h1:ZClxb8laGDf5arXfYcAtECDFgAgHklGI8CxgjHnXKJ4= @@ -817,6 +826,8 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= +github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA= +github.com/moby/term v0.0.0-20221205130635-1aeaba878587/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -829,6 +840,7 @@ github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJ github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -932,6 +944,7 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= @@ -949,6 +962,7 @@ github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJ github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= @@ -960,6 +974,7 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= @@ -1405,6 +1420,7 @@ golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1456,6 +1472,7 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1780,7 +1797,9 @@ gorm.io/plugin/dbresolver v1.3.0 h1:uFDX3bIuH9Lhj5LY2oyqR/bU6pqWuDgas35NAPF4X3M= gorm.io/plugin/dbresolver v1.3.0/go.mod h1:Pr7p5+JFlgDaiM6sOrli5olekJD16YRunMyA2S7ZfKk= gorm.io/plugin/soft_delete v1.2.1 h1:qx9D/c4Xu6w5KT8LviX8DgLcB9hkKl6JC9f44Tj7cGU= gorm.io/plugin/soft_delete v1.2.1/go.mod h1:Zv7vQctOJTGOsJ/bWgrN1n3od0GBAZgnLjEx+cApLGk= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gotest.tools/v3 v3.0.2 h1:kG1BFyqVHuQoVQiR1bWGnfz/fmHvvuiSPIV7rvl360E= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/manager/job/docker_auth_client.go b/manager/job/docker_auth_client.go new file mode 100644 index 00000000000..cdce84a8eec --- /dev/null +++ b/manager/job/docker_auth_client.go @@ -0,0 +1,164 @@ +package job + +import ( + "crypto/tls" + "encoding/base64" + "fmt" + "net" + "net/http" + "net/url" + "time" + + "github.com/docker/distribution" + "github.com/docker/distribution/registry/client/auth" + "github.com/docker/distribution/registry/client/transport" + "github.com/docker/docker/api/types" + "github.com/docker/docker/registry" +) + +type BasicAuth struct { + Username string + Password string + Auth string +} + +func (b *BasicAuth) token() string { + auth := fmt.Sprintf("%s:%s", b.Username, b.Password) + return base64.StdEncoding.EncodeToString([]byte(auth)) +} + +type imageAuthClient struct { + client *http.Client + baseTr *http.Transport + basic *BasicAuth + headerModifier http.Header + tokenInterceptor *InterceptorTokenHandler +} + +type dockerAuthClientOption func(*imageAuthClient) + +func WithBasicAuth(b *BasicAuth) dockerAuthClientOption { + return func(o *imageAuthClient) { + o.basic = b + } +} + +func WithImageRepo(repo string) dockerAuthClientOption { + return func(o *imageAuthClient) { + o.basic.Auth = repo + } +} + +func WithHeaderModifier(h http.Header) dockerAuthClientOption { + return func(o *imageAuthClient) { + o.headerModifier = h + } +} + +func WithTransport(tr *http.Transport) dockerAuthClientOption { + return func(o *imageAuthClient) { + o.baseTr = tr + } +} + +func WithClient(c *http.Client) dockerAuthClientOption { + return func(o *imageAuthClient) { + o.client = c + } +} + +func NewImageAuthClient(image *preheatImage, opts ...dockerAuthClientOption) (*imageAuthClient, error) { + d := &imageAuthClient{} + for _, opt := range opts { + opt(d) + } + + if d.client == nil { + d.client = &http.Client{} + } + + if d.baseTr == nil { + direct := &net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + } + + d.baseTr = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: direct.DialContext, + TLSHandshakeTimeout: 10 * time.Second, + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + DisableKeepAlives: true, + } + } + + authTransport := transport.NewTransport(d.baseTr, transport.NewHeaderRequestModifier(d.headerModifier)) + challengeManager, _, err := registry.PingV2Registry(&url.URL{Scheme: image.protocol, Host: image.domain}, authTransport) + if err != nil { + return nil, err + } + + scope := auth.RepositoryScope{ + Repository: image.name, + Actions: []string{"pull"}, + } + creds := registry.NewStaticCredentialStore(&types.AuthConfig{Username: d.basic.Username, Password: d.basic.Password, Auth: d.basic.token()}) + tokenHandlerOptions := auth.TokenHandlerOptions{ + Transport: authTransport, + Credentials: creds, + Scopes: []auth.Scope{scope}, + ClientID: registry.AuthClientID, + } + tokenHandler := auth.NewTokenHandlerWithOptions(tokenHandlerOptions) + basicHandler := auth.NewBasicHandler(creds) + interceptor := NewInterceptorTokenHandler() + d.tokenInterceptor = interceptor + d.client.Transport = transport.NewTransport(d.baseTr, auth.NewAuthorizer(challengeManager, tokenHandler, interceptor, basicHandler)) + return d, nil +} + +func (d *imageAuthClient) Do(req *http.Request) (*http.Response, error) { + return d.client.Do(req) +} + +func (d *imageAuthClient) GetBearerToken() string { + return fmt.Sprintf("Bearer %s", d.tokenInterceptor.GetAuthToken()) +} + +// GetManifestMediaTypeAcceptHeader +// get manifest/tag need accept header +func GetManifestMediaTypeAcceptHeader(h http.Header) http.Header { + if h == nil { + h = http.Header{} + } + header := h.Clone() + for _, v := range distribution.ManifestMediaTypes() { + header.Add("Accept", v) + } + + return header +} + +// InterceptorTokenHandler is a token interceptor +// intercept bearer token from auth handler +type InterceptorTokenHandler struct { + auth.AuthenticationHandler + token string +} + +func NewInterceptorTokenHandler() *InterceptorTokenHandler { + return &InterceptorTokenHandler{} +} + +func (h *InterceptorTokenHandler) Scheme() string { + return "bearer" +} + +func (h *InterceptorTokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error { + h.token = req.Header.Get("Authorization") + return nil +} + +func (h *InterceptorTokenHandler) GetAuthToken() string { + return h.token +} diff --git a/manager/job/preheat.go b/manager/job/preheat.go index d6b751a93bf..dc4b2891b33 100644 --- a/manager/job/preheat.go +++ b/manager/job/preheat.go @@ -22,23 +22,23 @@ import ( "context" "crypto/tls" "crypto/x509" - "encoding/json" "errors" "fmt" "io" "net/http" "regexp" - "strings" "time" machineryv1tasks "github.com/RichardKnop/machinery/v1/tasks" - distribution "github.com/distribution/distribution/v3" - "github.com/distribution/distribution/v3/manifest/manifestlist" - "github.com/distribution/distribution/v3/manifest/ocischema" - "github.com/distribution/distribution/v3/manifest/schema1" - "github.com/distribution/distribution/v3/manifest/schema2" - "github.com/go-http-utils/headers" + "github.com/containerd/containerd/platforms" + "github.com/docker/distribution" + "github.com/docker/distribution/manifest/manifestlist" + "github.com/docker/distribution/manifest/ocischema" + "github.com/docker/distribution/manifest/schema1" + "github.com/docker/distribution/manifest/schema2" + registryClient "github.com/docker/distribution/registry/client" "github.com/google/uuid" + specs "github.com/opencontainers/image-spec/specs-go/v1" "go.opentelemetry.io/otel/trace" logger "d7y.io/dragonfly/v2/internal/dflog" @@ -75,14 +75,6 @@ type preheat struct { rootCAs *x509.CertPool } -// preheatImage is image information for preheat. -type preheatImage struct { - protocol string - domain string - name string - tag string -} - // newPreheat creates a new Preheat. func newPreheat(job *internaljob.Job, httpRequestTimeout time.Duration, rootCAs *x509.CertPool) (Preheat, error) { return &preheat{job, httpRequestTimeout, rootCAs}, nil @@ -96,35 +88,26 @@ func (p *preheat) CreatePreheat(ctx context.Context, schedulers []models.Schedul span.SetAttributes(config.AttributePreheatURL.String(json.URL)) defer span.End() - url := json.URL - tag := json.Tag - filter := json.Filter - rawheader := json.Headers - // Initialize queues. queues := getSchedulerQueues(schedulers) // Generate download files. var files []internaljob.PreheatRequest + var err error switch PreheatType(json.Type) { case PreheatImageType: - // Parse image manifest url. - image, err := parseAccessURL(url) - if err != nil { - return nil, err - } - files, err = p.getLayers(ctx, url, tag, filter, nethttp.MapToHeader(rawheader), image) + files, err = p.getImageLayers(ctx, json) if err != nil { return nil, err } case PreheatFileType: files = []internaljob.PreheatRequest{ { - URL: url, - Tag: tag, - Filter: filter, - Headers: rawheader, + URL: json.URL, + Tag: json.Tag, + Filter: json.Filter, + Headers: json.Headers, }, } default: @@ -177,223 +160,156 @@ func (p *preheat) createGroupJob(ctx context.Context, files []internaljob.Prehea }, nil } -// getLayers gets layers of image. -func (p *preheat) getLayers(ctx context.Context, url, tag, filter string, header http.Header, image *preheatImage) ([]internaljob.PreheatRequest, error) { +// getImageLayers gets layers of image. +func (p *preheat) getImageLayers(ctx context.Context, args types.PreheatArgs) ([]internaljob.PreheatRequest, error) { ctx, span := tracer.Start(ctx, config.SpanGetLayers, trace.WithSpanKind(trace.SpanKindProducer)) defer span.End() - resp, err := p.getManifests(ctx, url, header, p.httpRequestTimeout) + // Parse image manifest url. + image, err := parseAccessURL(args.URL) if err != nil { return nil, err } - defer resp.Body.Close() - - if resp.StatusCode/100 != 2 { - if resp.StatusCode == http.StatusUnauthorized { - token, err := getAuthToken(ctx, resp.Header, p.httpRequestTimeout, p.rootCAs) - if err != nil { - return nil, err - } - - header.Add(headers.Authorization, fmt.Sprintf("Bearer %s", token)) - resp, err = p.getManifests(ctx, url, header, p.httpRequestTimeout) - if err != nil { - return nil, err - } - } else { - return nil, fmt.Errorf("request registry %d", resp.StatusCode) - } - } - layers, err := p.parseLayers(resp, tag, filter, header, image) + // init docker auth client + client, err := NewImageAuthClient( + image, + WithClient(&http.Client{Timeout: p.httpRequestTimeout}), + WithTransport(&http.Transport{TLSClientConfig: &tls.Config{RootCAs: p.rootCAs}}), + WithBasicAuth(&BasicAuth{Username: args.Username, Password: args.Password}), + ) if err != nil { return nil, err } - return layers, nil -} + // Parse platform + platform := platforms.DefaultSpec() + if args.Platform != "" { + platform, err = platforms.Parse(args.Platform) + if err != nil { + return nil, err + } + } -// getManifests gets manifests of image. -func (p *preheat) getManifests(ctx context.Context, url string, header http.Header, timeout time.Duration) (*http.Response, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + // Get manifests + header := nethttp.MapToHeader(args.Headers).Clone() + manifests, err := p.getManifests(ctx, client, image, header, platform) if err != nil { return nil, err } - req.Header = header - // 参考 docker pull 源码 moby https://github.com/moby/moby/blob/6040283f23efbd085eed90fa72d12ae73119837e/vendor/github.com/docker/distribution/registry/client/repository.go#L313 - for _, d := range distribution.ManifestMediaTypes() { - req.Header.Add(headers.Accept, d) + // no matching manifest for platform in the manifest list entries + if len(manifests) == 0 { + return nil, noMatchesErr{} } - // TOOD: 根据不同类型解析 manifest + // set authorization header + header.Set("Authorization", client.GetBearerToken()) - client := &http.Client{ - Timeout: timeout, - Transport: &http.Transport{ - DialContext: nethttp.NewSafeDialer().DialContext, - TLSClientConfig: &tls.Config{RootCAs: p.rootCAs}, - }, - } - - resp, err := client.Do(req) + // prase image layers to preheat + layers, err := p.parseLayers(manifests, args, header, image) if err != nil { return nil, err } - return resp, nil + return layers, nil } -// parseLayers parses layers of image. -func (p *preheat) parseLayers(resp *http.Response, tag, filter string, header http.Header, image *preheatImage) ([]internaljob.PreheatRequest, error) { - body, err := io.ReadAll(resp.Body) +// getManifests gets manifests of image. +func (p *preheat) getManifests(ctx context.Context, client *imageAuthClient, image *preheatImage, header http.Header, pp specs.Platform) ([]distribution.Manifest, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, image.buildManifestUrl(image.tag), nil) if err != nil { return nil, err } - mt := resp.Header.Get("Content-Type") - manifest, _, err := distribution.UnmarshalManifest(mt, body) + req.Header = GetManifestMediaTypeAcceptHeader(header) + resp, err := client.Do(req) if err != nil { return nil, err } - // TODO: manifest type - - var layers []internaljob.PreheatRequest - for _, v := range manifest.References() { - layer := internaljob.PreheatRequest{ - URL: layerURL(image.protocol, image.domain, image.name, v.Digest.String()), - Tag: tag, - Filter: filter, - Headers: nethttp.HeaderToMap(header), - } - - layers = append(layers, layer) - } - - return layers, nil -} - -func (p *preheat) parseManifestReference(manifest manifestReference, tag, filter string, header http.Header, image *preheatImage) ([]internaljob.PreheatRequest, error) { - var layers []internaljob.PreheatRequest - for _, v := range manifest.References() { - layer := internaljob.PreheatRequest{ - URL: layerURL(image.protocol, image.domain, image.name, v.Digest.String()), - Tag: tag, - Filter: filter, - Headers: nethttp.HeaderToMap(header), - } + defer resp.Body.Close() - layers = append(layers, layer) + if resp.StatusCode == http.StatusNotModified { + return nil, distribution.ErrManifestNotModified + } else if !registryClient.SuccessStatus(resp.StatusCode) { + return nil, registryClient.HandleErrorResponse(resp) } - return layers, nil -} - -func (p *preheat) parseManifest(resp *http.Response, tag, filter string, header http.Header, image *preheatImage) ([]internaljob.PreheatRequest, error) { + ctHeader := resp.Header.Get("Content-Type") body, err := io.ReadAll(resp.Body) if err != nil { return nil, err } - - mt := resp.Header.Get("Content-Type") - manifest, _, err := distribution.UnmarshalManifest(mt, body) + manifest, _, err := distribution.UnmarshalManifest(ctHeader, body) if err != nil { return nil, err } - if manifest == nil { - return nil, fmt.Errorf("image manifest does not exist for tag or digest %q", image.tag) - } - - // switch v := manifest.(type) { - case *schema1.SignedManifest: - return p.parseManifestReference(v, tag, filter, header, image) - case *schema2.DeserializedManifest: - return p.parseManifestReference(v, tag, filter, header, image) - case *ocischema.DeserializedManifest: - return p.parseManifestReference(v, tag, filter, header, image) + case *schema1.SignedManifest, *schema2.DeserializedManifest, *ocischema.DeserializedManifest: + return []distribution.Manifest{v}, nil case *manifestlist.DeserializedManifestList: - fmt.Println(v) - default: - return nil, invalidManifestFormatError{} - } - // raise no match error - return nil, nil -} - -func (p *preheat) parseManifestList(mfstList *manifestlist.DeserializedManifestList, image *preheatImage) error { - // 解析 manifest list - - return nil -} - -// getAuthToken gets auth token from registry. -func getAuthToken(ctx context.Context, header http.Header, timeout time.Duration, rootCAs *x509.CertPool) (string, error) { - ctx, span := tracer.Start(ctx, config.SpanAuthWithRegistry, trace.WithSpanKind(trace.SpanKindProducer)) - defer span.End() + callback := func(m manifestlist.ManifestDescriptor) ([]distribution.Manifest, error) { + image.tag = m.Digest.String() + return p.getManifests(ctx, client, image, header, pp) + } - authURL := authURL(header.Values(headers.WWWAuthenticate)) - if len(authURL) == 0 { - return "", errors.New("authURL is empty") + return p.getManifestsFromManifestList(ctx, v, pp, callback) } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, authURL, nil) - if err != nil { - return "", err - } + return nil, invalidManifestFormatError{} +} - client := &http.Client{ - Timeout: timeout, - Transport: &http.Transport{ - DialContext: nethttp.NewSafeDialer().DialContext, - TLSClientConfig: &tls.Config{RootCAs: rootCAs}, - }, - } +func (p *preheat) getManifestsFromManifestList( + ctx context.Context, + manifestList *manifestlist.DeserializedManifestList, + pp specs.Platform, + callback func(manifestlist.ManifestDescriptor) ([]distribution.Manifest, error), +) ([]distribution.Manifest, error) { + var ms []distribution.Manifest + for _, v := range p.filterManifests(manifestList.Manifests, pp) { + manifestList, err := callback(v) + if err != nil { + return nil, err + } - resp, err := client.Do(req) - if err != nil { - return "", err + ms = append(ms, manifestList...) } - defer resp.Body.Close() - - body, _ := io.ReadAll(resp.Body) - var result map[string]any - if err := json.Unmarshal(body, &result); err != nil { - return "", err - } + return ms, nil +} - if result["token"] == nil { - return "", errors.New("token is empty") +// filterManifests +func (p *preheat) filterManifests(manifests []manifestlist.ManifestDescriptor, pp specs.Platform) []manifestlist.ManifestDescriptor { + var matches []manifestlist.ManifestDescriptor + for _, desc := range manifests { + if desc.Platform.Architecture == pp.Architecture && desc.Platform.OS == pp.OS { + matches = append(matches, desc) + } } - token := fmt.Sprintf("%v", result["token"]) - return token, nil + return matches } -// authURL gets auth url from www-authenticate header. -func authURL(wwwAuth []string) string { - // Bearer realm="",service="",scope="repository::pull" - if len(wwwAuth) == 0 { - return "" - } +// parseLayers parses layers of image. +func (p *preheat) parseLayers(manifests []distribution.Manifest, args types.PreheatArgs, header http.Header, image *preheatImage) ([]internaljob.PreheatRequest, error) { + var layers []internaljob.PreheatRequest + for _, m := range manifests { + for _, v := range m.References() { + h := header.Clone() + h.Set("Accept", v.MediaType) + layer := internaljob.PreheatRequest{ + URL: image.buildBlobsUrl(v.Digest.String()), + Tag: args.Tag, + Filter: args.Filter, + Headers: nethttp.HeaderToMap(h), + } - polished := make([]string, 0) - for _, it := range wwwAuth { - polished = append(polished, strings.ReplaceAll(it, "\"", "")) + layers = append(layers, layer) + } } - - fields := strings.Split(polished[0], ",") - host := strings.Split(fields[0], "=")[1] - query := strings.Join(fields[1:], "&") - return fmt.Sprintf("%s?%s", host, query) -} - -// layerURL gets layer url. -func layerURL(protocol string, domain string, name string, digest string) string { - return fmt.Sprintf("%s://%s/v2/%s/blobs/%s", protocol, domain, name, digest) + return layers, nil } // parseAccessURL parses access url. diff --git a/manager/job/preheat_test.go b/manager/job/preheat_test.go new file mode 100644 index 00000000000..c291bf03c10 --- /dev/null +++ b/manager/job/preheat_test.go @@ -0,0 +1,70 @@ +package job + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "d7y.io/dragonfly/v2/manager/types" +) + +type testCase struct { + args types.PreheatArgs + desc string +} + +func TestGetManifest(t *testing.T) { + p := &preheat{} + + testCases := []testCase{ + { + args: types.PreheatArgs{ + URL: "https://registry-1.docker.io/v2/dragonflyoss/busybox/manifests/1.35.0", + Type: "image", + }, + desc: "get opensource image layers", + }, + { + args: types.PreheatArgs{ + URL: "https://registry-1.docker.io/v2/library/nginx/manifests/latest", + Platform: "linux/amd64", + }, + desc: "get docker official multi arch image layers", + }, + { + args: types.PreheatArgs{ + URL: "xxx", + Username: "xxx", + Password: "xxx", + }, + desc: "get harbor image layers", + }, + { + args: types.PreheatArgs{ + URL: "xxx", + Tag: "xxx", + Username: "xxx", + Password: "xxx", + }, + desc: "get ali image layers", + }, + { + args: types.PreheatArgs{ + URL: "xxx", + Username: "xxx", + Password: "xxx", + }, + desc: "get huawei image layers", + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + t.Log(tc.desc) + layers, err := p.getImageLayers(context.Background(), tc.args) + require.NoError(t, err) + require.NotEmpty(t, layers) + }) + } +} diff --git a/manager/job/types.go b/manager/job/types.go index 570d4be9ddd..14df4bf2b64 100644 --- a/manager/job/types.go +++ b/manager/job/types.go @@ -1,13 +1,47 @@ package job -import distribution "github.com/distribution/distribution/v3" +import ( + "fmt" + "github.com/containerd/containerd/platforms" + specs "github.com/opencontainers/image-spec/specs-go/v1" +) + +// preheatImage is image information for preheat. +type preheatImage struct { + protocol string + domain string + name string + tag string +} + +func (p *preheatImage) buildManifestUrl(digest string) string { + return fmt.Sprintf("%s://%s/v2/%s/manifests/%s", p.protocol, p.domain, p.name, digest) +} + +func (p *preheatImage) buildBlobsUrl(digest string) string { + return fmt.Sprintf("%s://%s/v2/%s/blobs/%s", p.protocol, p.domain, p.name, digest) +} + +// invalidManifestFormatError is error for invalid manifest format. type invalidManifestFormatError struct{} func (invalidManifestFormatError) Error() string { return "unsupported manifest format" } -type manifestReference interface { - References() []distribution.Descriptor +// noMatchesErr is error for no matching manifest. +type noMatchesErr struct { + platform specs.Platform +} + +func (e noMatchesErr) Error() string { + return fmt.Sprintf("no matching manifest for %s in the manifest list entries", formatPlatform(e.platform)) +} + +func formatPlatform(platform specs.Platform) string { + if platform.OS == "" { + platform = platforms.DefaultSpec() + } + return platforms.Format(platform) } diff --git a/manager/types/job.go b/manager/types/job.go index b0c2fbbe120..e17f639e5c1 100644 --- a/manager/types/job.go +++ b/manager/types/job.go @@ -58,4 +58,11 @@ type PreheatArgs struct { Tag string `json:"tag" binding:"omitempty"` Filter string `json:"filter" binding:"omitempty"` Headers map[string]string `json:"headers" binding:"omitempty"` + + // private image registry need username and password to get auth token + Username string `json:"username" binding:"omitempty"` + Password string `json:"password" binding:"omitempty"` + + // The image type preheating task can specify the image architecture type. eg: linux/amd64 + Platform string `json:"platform" binding:"omitempty"` } From 8cd9ff5f9a2187866b0fd66d0e295ccb9515cad6 Mon Sep 17 00:00:00 2001 From: jonpan <1191243580@qq.com> Date: Sat, 21 Oct 2023 18:32:27 +0800 Subject: [PATCH 03/12] feat: unified naming Signed-off-by: jonpan <1191243580@qq.com> --- manager/job/docker_auth_client.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/manager/job/docker_auth_client.go b/manager/job/docker_auth_client.go index cdce84a8eec..1e1261f010c 100644 --- a/manager/job/docker_auth_client.go +++ b/manager/job/docker_auth_client.go @@ -35,39 +35,39 @@ type imageAuthClient struct { tokenInterceptor *InterceptorTokenHandler } -type dockerAuthClientOption func(*imageAuthClient) +type imageAuthClientOption func(*imageAuthClient) -func WithBasicAuth(b *BasicAuth) dockerAuthClientOption { +func WithBasicAuth(b *BasicAuth) imageAuthClientOption { return func(o *imageAuthClient) { o.basic = b } } -func WithImageRepo(repo string) dockerAuthClientOption { +func WithImageRepo(repo string) imageAuthClientOption { return func(o *imageAuthClient) { o.basic.Auth = repo } } -func WithHeaderModifier(h http.Header) dockerAuthClientOption { +func WithHeaderModifier(h http.Header) imageAuthClientOption { return func(o *imageAuthClient) { o.headerModifier = h } } -func WithTransport(tr *http.Transport) dockerAuthClientOption { +func WithTransport(tr *http.Transport) imageAuthClientOption { return func(o *imageAuthClient) { o.baseTr = tr } } -func WithClient(c *http.Client) dockerAuthClientOption { +func WithClient(c *http.Client) imageAuthClientOption { return func(o *imageAuthClient) { o.client = c } } -func NewImageAuthClient(image *preheatImage, opts ...dockerAuthClientOption) (*imageAuthClient, error) { +func NewImageAuthClient(image *preheatImage, opts ...imageAuthClientOption) (*imageAuthClient, error) { d := &imageAuthClient{} for _, opt := range opts { opt(d) From 04d6ba871f4bfb9308ffd4ac172dc86bc4435775 Mon Sep 17 00:00:00 2001 From: jonpan <1191243580@qq.com> Date: Sat, 21 Oct 2023 18:45:25 +0800 Subject: [PATCH 04/12] fix: change file name Signed-off-by: jonpan <1191243580@qq.com> --- manager/job/{docker_auth_client.go => image_auth_client.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename manager/job/{docker_auth_client.go => image_auth_client.go} (100%) diff --git a/manager/job/docker_auth_client.go b/manager/job/image_auth_client.go similarity index 100% rename from manager/job/docker_auth_client.go rename to manager/job/image_auth_client.go From b27ef28347c4d5062c515ff763671d6ca1e072a4 Mon Sep 17 00:00:00 2001 From: jonpan <1191243580@qq.com> Date: Sun, 22 Oct 2023 09:02:01 +0800 Subject: [PATCH 05/12] fix: duplicate auth schema Signed-off-by: jonpan <1191243580@qq.com> --- manager/job/image_auth_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manager/job/image_auth_client.go b/manager/job/image_auth_client.go index 1e1261f010c..199a9af578f 100644 --- a/manager/job/image_auth_client.go +++ b/manager/job/image_auth_client.go @@ -122,7 +122,7 @@ func (d *imageAuthClient) Do(req *http.Request) (*http.Response, error) { } func (d *imageAuthClient) GetBearerToken() string { - return fmt.Sprintf("Bearer %s", d.tokenInterceptor.GetAuthToken()) + return d.tokenInterceptor.GetAuthToken() } // GetManifestMediaTypeAcceptHeader From 3486b0cf83952ffb122cbec9ed6ef4a478203b32 Mon Sep 17 00:00:00 2001 From: jonpan <1191243580@qq.com> Date: Sun, 22 Oct 2023 11:35:55 +0800 Subject: [PATCH 06/12] fix: golint failure Signed-off-by: jonpan <1191243580@qq.com> --- manager/job/image_auth_client.go | 32 ++++++++++++++++---------------- manager/job/preheat.go | 6 +++--- manager/job/types.go | 4 ++-- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/manager/job/image_auth_client.go b/manager/job/image_auth_client.go index 199a9af578f..51c03120e0a 100644 --- a/manager/job/image_auth_client.go +++ b/manager/job/image_auth_client.go @@ -27,7 +27,7 @@ func (b *BasicAuth) token() string { return base64.StdEncoding.EncodeToString([]byte(auth)) } -type imageAuthClient struct { +type ImageAuthClient struct { client *http.Client baseTr *http.Transport basic *BasicAuth @@ -35,40 +35,40 @@ type imageAuthClient struct { tokenInterceptor *InterceptorTokenHandler } -type imageAuthClientOption func(*imageAuthClient) +type ImageAuthClientOption func(*ImageAuthClient) -func WithBasicAuth(b *BasicAuth) imageAuthClientOption { - return func(o *imageAuthClient) { +func WithBasicAuth(b *BasicAuth) ImageAuthClientOption { + return func(o *ImageAuthClient) { o.basic = b } } -func WithImageRepo(repo string) imageAuthClientOption { - return func(o *imageAuthClient) { +func WithImageRepo(repo string) ImageAuthClientOption { + return func(o *ImageAuthClient) { o.basic.Auth = repo } } -func WithHeaderModifier(h http.Header) imageAuthClientOption { - return func(o *imageAuthClient) { +func WithHeaderModifier(h http.Header) ImageAuthClientOption { + return func(o *ImageAuthClient) { o.headerModifier = h } } -func WithTransport(tr *http.Transport) imageAuthClientOption { - return func(o *imageAuthClient) { +func WithTransport(tr *http.Transport) ImageAuthClientOption { + return func(o *ImageAuthClient) { o.baseTr = tr } } -func WithClient(c *http.Client) imageAuthClientOption { - return func(o *imageAuthClient) { +func WithClient(c *http.Client) ImageAuthClientOption { + return func(o *ImageAuthClient) { o.client = c } } -func NewImageAuthClient(image *preheatImage, opts ...imageAuthClientOption) (*imageAuthClient, error) { - d := &imageAuthClient{} +func NewImageAuthClient(image *preheatImage, opts ...ImageAuthClientOption) (*ImageAuthClient, error) { + d := &ImageAuthClient{} for _, opt := range opts { opt(d) } @@ -117,11 +117,11 @@ func NewImageAuthClient(image *preheatImage, opts ...imageAuthClientOption) (*im return d, nil } -func (d *imageAuthClient) Do(req *http.Request) (*http.Response, error) { +func (d *ImageAuthClient) Do(req *http.Request) (*http.Response, error) { return d.client.Do(req) } -func (d *imageAuthClient) GetBearerToken() string { +func (d *ImageAuthClient) GetBearerToken() string { return d.tokenInterceptor.GetAuthToken() } diff --git a/manager/job/preheat.go b/manager/job/preheat.go index dc4b2891b33..5c2714c74e4 100644 --- a/manager/job/preheat.go +++ b/manager/job/preheat.go @@ -216,8 +216,8 @@ func (p *preheat) getImageLayers(ctx context.Context, args types.PreheatArgs) ([ } // getManifests gets manifests of image. -func (p *preheat) getManifests(ctx context.Context, client *imageAuthClient, image *preheatImage, header http.Header, pp specs.Platform) ([]distribution.Manifest, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, image.buildManifestUrl(image.tag), nil) +func (p *preheat) getManifests(ctx context.Context, client *ImageAuthClient, image *preheatImage, header http.Header, pp specs.Platform) ([]distribution.Manifest, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, image.buildManifestURL(image.tag), nil) if err != nil { return nil, err } @@ -300,7 +300,7 @@ func (p *preheat) parseLayers(manifests []distribution.Manifest, args types.Preh h := header.Clone() h.Set("Accept", v.MediaType) layer := internaljob.PreheatRequest{ - URL: image.buildBlobsUrl(v.Digest.String()), + URL: image.buildBlobsURL(v.Digest.String()), Tag: args.Tag, Filter: args.Filter, Headers: nethttp.HeaderToMap(h), diff --git a/manager/job/types.go b/manager/job/types.go index 14df4bf2b64..a1312040d2c 100644 --- a/manager/job/types.go +++ b/manager/job/types.go @@ -15,11 +15,11 @@ type preheatImage struct { tag string } -func (p *preheatImage) buildManifestUrl(digest string) string { +func (p *preheatImage) buildManifestURL(digest string) string { return fmt.Sprintf("%s://%s/v2/%s/manifests/%s", p.protocol, p.domain, p.name, digest) } -func (p *preheatImage) buildBlobsUrl(digest string) string { +func (p *preheatImage) buildBlobsURL(digest string) string { return fmt.Sprintf("%s://%s/v2/%s/blobs/%s", p.protocol, p.domain, p.name, digest) } From 72b610fd8d08963b451d99ee3da2eb240507f8ca Mon Sep 17 00:00:00 2001 From: jonpan <1191243580@qq.com> Date: Sun, 22 Oct 2023 11:46:13 +0800 Subject: [PATCH 07/12] refactor: optimize the initialization code of default http client Signed-off-by: jonpan <1191243580@qq.com> --- manager/job/image_auth_client.go | 16 +++++----------- manager/job/preheat.go | 9 +++++++-- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/manager/job/image_auth_client.go b/manager/job/image_auth_client.go index 51c03120e0a..bc45476ef81 100644 --- a/manager/job/image_auth_client.go +++ b/manager/job/image_auth_client.go @@ -29,7 +29,6 @@ func (b *BasicAuth) token() string { type ImageAuthClient struct { client *http.Client - baseTr *http.Transport basic *BasicAuth headerModifier http.Header tokenInterceptor *InterceptorTokenHandler @@ -55,12 +54,6 @@ func WithHeaderModifier(h http.Header) ImageAuthClientOption { } } -func WithTransport(tr *http.Transport) ImageAuthClientOption { - return func(o *ImageAuthClient) { - o.baseTr = tr - } -} - func WithClient(c *http.Client) ImageAuthClientOption { return func(o *ImageAuthClient) { o.client = c @@ -77,13 +70,13 @@ func NewImageAuthClient(image *preheatImage, opts ...ImageAuthClientOption) (*Im d.client = &http.Client{} } - if d.baseTr == nil { + if d.client.Transport == nil { direct := &net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, } - d.baseTr = &http.Transport{ + d.client.Transport = &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: direct.DialContext, TLSHandshakeTimeout: 10 * time.Second, @@ -92,7 +85,8 @@ func NewImageAuthClient(image *preheatImage, opts ...ImageAuthClientOption) (*Im } } - authTransport := transport.NewTransport(d.baseTr, transport.NewHeaderRequestModifier(d.headerModifier)) + baseTransport := d.client.Transport + authTransport := transport.NewTransport(baseTransport, transport.NewHeaderRequestModifier(d.headerModifier)) challengeManager, _, err := registry.PingV2Registry(&url.URL{Scheme: image.protocol, Host: image.domain}, authTransport) if err != nil { return nil, err @@ -113,7 +107,7 @@ func NewImageAuthClient(image *preheatImage, opts ...ImageAuthClientOption) (*Im basicHandler := auth.NewBasicHandler(creds) interceptor := NewInterceptorTokenHandler() d.tokenInterceptor = interceptor - d.client.Transport = transport.NewTransport(d.baseTr, auth.NewAuthorizer(challengeManager, tokenHandler, interceptor, basicHandler)) + d.client.Transport = transport.NewTransport(baseTransport, auth.NewAuthorizer(challengeManager, tokenHandler, interceptor, basicHandler)) return d, nil } diff --git a/manager/job/preheat.go b/manager/job/preheat.go index 5c2714c74e4..d33dba6a18d 100644 --- a/manager/job/preheat.go +++ b/manager/job/preheat.go @@ -174,8 +174,13 @@ func (p *preheat) getImageLayers(ctx context.Context, args types.PreheatArgs) ([ // init docker auth client client, err := NewImageAuthClient( image, - WithClient(&http.Client{Timeout: p.httpRequestTimeout}), - WithTransport(&http.Transport{TLSClientConfig: &tls.Config{RootCAs: p.rootCAs}}), + WithClient(&http.Client{ + Timeout: p.httpRequestTimeout, + Transport: &http.Transport{ + DialContext: nethttp.NewSafeDialer().DialContext, + TLSClientConfig: &tls.Config{RootCAs: p.rootCAs}, + }, + }), WithBasicAuth(&BasicAuth{Username: args.Username, Password: args.Password}), ) if err != nil { From 25e8935603a4b5bc177ed002925e3aaa099de121 Mon Sep 17 00:00:00 2001 From: jonpan <1191243580@qq.com> Date: Sun, 22 Oct 2023 11:59:43 +0800 Subject: [PATCH 08/12] fix: security check (InsecureSkipVerify should not be used in production code.) Signed-off-by: jonpan <1191243580@qq.com> --- manager/job/image_auth_client.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/manager/job/image_auth_client.go b/manager/job/image_auth_client.go index bc45476ef81..e37e8fb8974 100644 --- a/manager/job/image_auth_client.go +++ b/manager/job/image_auth_client.go @@ -2,6 +2,7 @@ package job import ( "crypto/tls" + "crypto/x509" "encoding/base64" "fmt" "net" @@ -80,7 +81,7 @@ func NewImageAuthClient(image *preheatImage, opts ...ImageAuthClientOption) (*Im Proxy: http.ProxyFromEnvironment, DialContext: direct.DialContext, TLSHandshakeTimeout: 10 * time.Second, - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + TLSClientConfig: &tls.Config{RootCAs: x509.NewCertPool()}, DisableKeepAlives: true, } } From 0ac91805ef3621bea9272e104f03d478e252380a Mon Sep 17 00:00:00 2001 From: jonpan <1191243580@qq.com> Date: Sun, 22 Oct 2023 12:08:24 +0800 Subject: [PATCH 09/12] test: remove invalid test cases Signed-off-by: jonpan <1191243580@qq.com> --- manager/job/preheat_test.go | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/manager/job/preheat_test.go b/manager/job/preheat_test.go index c291bf03c10..2fc24e63870 100644 --- a/manager/job/preheat_test.go +++ b/manager/job/preheat_test.go @@ -32,31 +32,6 @@ func TestGetManifest(t *testing.T) { }, desc: "get docker official multi arch image layers", }, - { - args: types.PreheatArgs{ - URL: "xxx", - Username: "xxx", - Password: "xxx", - }, - desc: "get harbor image layers", - }, - { - args: types.PreheatArgs{ - URL: "xxx", - Tag: "xxx", - Username: "xxx", - Password: "xxx", - }, - desc: "get ali image layers", - }, - { - args: types.PreheatArgs{ - URL: "xxx", - Username: "xxx", - Password: "xxx", - }, - desc: "get huawei image layers", - }, } for _, tc := range testCases { From c7c603147e78f0223cf5b1ae4fcefe52474fb920 Mon Sep 17 00:00:00 2001 From: jonpan <1191243580@qq.com> Date: Fri, 3 Nov 2023 16:31:21 +0800 Subject: [PATCH 10/12] test: add preheat image multi-platform e2e test cases Signed-off-by: jonpan <1191243580@qq.com> --- test/e2e/manager/preheat.go | 117 +++++++++++++++++++++++++++++++++++- 1 file changed, 115 insertions(+), 2 deletions(-) diff --git a/test/e2e/manager/preheat.go b/test/e2e/manager/preheat.go index a033453e5d9..d8bda72b6d3 100644 --- a/test/e2e/manager/preheat.go +++ b/test/e2e/manager/preheat.go @@ -24,12 +24,11 @@ import ( "strings" "time" + commonv1 "d7y.io/api/v2/pkg/apis/common/v1" machineryv1tasks "github.com/RichardKnop/machinery/v1/tasks" . "github.com/onsi/ginkgo/v2" //nolint . "github.com/onsi/gomega" //nolint - commonv1 "d7y.io/api/v2/pkg/apis/common/v1" - internaljob "d7y.io/dragonfly/v2/internal/job" "d7y.io/dragonfly/v2/manager/models" "d7y.io/dragonfly/v2/manager/types" @@ -138,6 +137,120 @@ var _ = Describe("Preheat with manager", func() { Expect(sha256sum1[i]).To(Equal(sha256sum)) } }) + + It("preheat image for linux/amd64 platform should be ok", Label("preheat", "image", "platform", "linux/amd64"), func() { + url := "https://index.docker.io/v2/dragonflyoss/scheduler/manifests/v2.1.0" + fmt.Println("download image: " + url) + + var ( + seedPeerTaskIDs = []string{ + "c8ca6a17354d3a79397eef26803e5af84d00a3fd64b0f823922086a31ebdee18", + "b8de5865e2ebf537279683adfbdb5f858b0c7212e5744a1df233086496c245d7", + "e4bf0d4b551afda56f9627c81ee02ab4360865d37c7dd43586e37f26f4386806", + "7da0721fd078dd46a63298747ffde8fcbe12b53378f282c9def693615ac7993e", + "3639c8c5712e77acd3751142c83150c0a12284a54fa41224a1c7acc0e343020d", + } + sha256sum1 = []string{ + "f1f1039835051ecc04909f939530e86a20f02d2ce5ad7a81c0fa3616f7303944", + "c1d6d1b2d5a367259e6e51a7f4d1ccd66a28cc9940d6599d8a8ea9544dd4b4a8", + "871ab018db94b4ae7b137764837bc4504393a60656ba187189e985cd809064f7", + "f1a1d290795d904815786e41d39a41dc1af5de68a9e9020baba8bd83b32d8f95", + "f1ffc4b5459e82dc8e7ddd1d1a2ec469e85a1f076090c22851a1f2ce6f71e1a6", + } + ) + + var seedPeerPods [3]*e2eutil.PodExec + for i := 0; i < 3; i++ { + seedPeerPods[i] = getSeedPeerExec(i) + } + fsPod := getFileServerExec() + + // preheat file + req, err := structure.StructToMap(types.CreatePreheatJobRequest{ + Type: internaljob.PreheatJob, + Args: types.PreheatArgs{ + Type: "image", + URL: url, + Platform: "linux/amd64", + }, + }) + Expect(err).NotTo(HaveOccurred()) + + out, err := fsPod.CurlCommand("POST", map[string]string{"Content-Type": "application/json"}, req, + fmt.Sprintf("http://%s:%s/%s", managerService, managerPort, preheatPath)).CombinedOutput() + fmt.Println(string(out)) + Expect(err).NotTo(HaveOccurred()) + + // wait for success + job := &models.Job{} + err = json.Unmarshal(out, job) + Expect(err).NotTo(HaveOccurred()) + done := waitForDone(job, fsPod) + Expect(done).Should(BeTrue()) + + for i, seedPeerTaskID := range seedPeerTaskIDs { + sha256sum, err := checkPreheatResult(seedPeerPods, seedPeerTaskID) + Expect(err).NotTo(HaveOccurred()) + Expect(sha256sum1[i]).To(Equal(sha256sum)) + } + }) + + It("preheat image for linux/arm64 platform should be ok", Label("preheat", "image", "platform", "linux/arm64"), func() { + url := "https://index.docker.io/v2/dragonflyoss/scheduler/manifests/v2.1.0" + fmt.Println("download image: " + url) + + var ( + seedPeerTaskIDs = []string{ + "9869dbb01ac214e90e4ae667e42d50210c2ff1e63292d73b14f0a7a2226c0320", + "ab049caee13f77d91568d954a5d32f5d2354497cab098887a8a663656daa9840", + "e4bf0d4b551afda56f9627c81ee02ab4360865d37c7dd43586e37f26f4386806", + "a26e1ac8b70926f45766fcf886f23a833793c39c62237bcda9ffeb158131c0d6", + "7376f665077e91cd0dc410c00242ab88775e3eae19eca4b7b3a29ded14fc3754", + } + sha256sum1 = []string{ + "a0d7a8f11f7e25ca59f0bf470187dd9aa27e7ca951cf67a53c750deea5d3b076", + "a880266d3b77f75696023df2da1ef66c3c565e0f70596242395c9e68de955c7c", + "871ab018db94b4ae7b137764837bc4504393a60656ba187189e985cd809064f7", + "9b5952218d7711195c6c6fbddbef2780507d20851ca68845d180397d1348f0d8", + "889f4c960ac4ff70774e9c4cfa64efc4823ade0702d0f96c20ff0054ffbbe504", + } + ) + + var seedPeerPods [3]*e2eutil.PodExec + for i := 0; i < 3; i++ { + seedPeerPods[i] = getSeedPeerExec(i) + } + fsPod := getFileServerExec() + + // preheat file + req, err := structure.StructToMap(types.CreatePreheatJobRequest{ + Type: internaljob.PreheatJob, + Args: types.PreheatArgs{ + Type: "image", + URL: url, + Platform: "linux/arm64", + }, + }) + Expect(err).NotTo(HaveOccurred()) + + out, err := fsPod.CurlCommand("POST", map[string]string{"Content-Type": "application/json"}, req, + fmt.Sprintf("http://%s:%s/%s", managerService, managerPort, preheatPath)).CombinedOutput() + fmt.Println(string(out)) + Expect(err).NotTo(HaveOccurred()) + + // wait for success + job := &models.Job{} + err = json.Unmarshal(out, job) + Expect(err).NotTo(HaveOccurred()) + done := waitForDone(job, fsPod) + Expect(done).Should(BeTrue()) + + for i, seedPeerTaskID := range seedPeerTaskIDs { + sha256sum, err := checkPreheatResult(seedPeerPods, seedPeerTaskID) + Expect(err).NotTo(HaveOccurred()) + Expect(sha256sum1[i]).To(Equal(sha256sum)) + } + }) }) }) From ca091d522134edefc84fca035307e5075a42f234 Mon Sep 17 00:00:00 2001 From: jonpan <1191243580@qq.com> Date: Mon, 6 Nov 2023 11:55:12 +0800 Subject: [PATCH 11/12] fix: lint error Signed-off-by: jonpan <1191243580@qq.com> --- test/e2e/manager/preheat.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/e2e/manager/preheat.go b/test/e2e/manager/preheat.go index d8bda72b6d3..228a8b355d3 100644 --- a/test/e2e/manager/preheat.go +++ b/test/e2e/manager/preheat.go @@ -24,11 +24,12 @@ import ( "strings" "time" - commonv1 "d7y.io/api/v2/pkg/apis/common/v1" machineryv1tasks "github.com/RichardKnop/machinery/v1/tasks" . "github.com/onsi/ginkgo/v2" //nolint . "github.com/onsi/gomega" //nolint + commonv1 "d7y.io/api/v2/pkg/apis/common/v1" + internaljob "d7y.io/dragonfly/v2/internal/job" "d7y.io/dragonfly/v2/manager/models" "d7y.io/dragonfly/v2/manager/types" From eaf60e9371ec363b72ccd777b878f2bc57e981b3 Mon Sep 17 00:00:00 2001 From: jonpan <1191243580@qq.com> Date: Mon, 6 Nov 2023 12:08:03 +0800 Subject: [PATCH 12/12] feat: add preheat multi arch image feature gate to skip e2e test Signed-off-by: jonpan <1191243580@qq.com> --- test/e2e/dfget_test.go | 18 +++++----- test/e2e/e2e_test.go | 57 ++++---------------------------- test/e2e/e2eutil/feature_gate.go | 57 ++++++++++++++++++++++++++++++++ test/e2e/manager/preheat.go | 14 ++++++-- 4 files changed, 84 insertions(+), 62 deletions(-) create mode 100644 test/e2e/e2eutil/feature_gate.go diff --git a/test/e2e/dfget_test.go b/test/e2e/dfget_test.go index a7c8fc5fddc..b7d14dc2417 100644 --- a/test/e2e/dfget_test.go +++ b/test/e2e/dfget_test.go @@ -51,7 +51,7 @@ func getFileSizes() map[string]int { files = e2eutil.GetFileList() ) - if featureGates.Enabled(featureGateEmptyFile) { + if e2eutil.FeatureGates.Enabled(e2eutil.FeatureGateEmptyFile) { fmt.Printf("dfget-empty-file feature gate enabled\n") files = append(files, "/tmp/empty-file") } @@ -104,7 +104,7 @@ func singleDfgetTest(name, ns, label, podNamePrefix, container string) { Expect(strings.HasPrefix(podName, podNamePrefix)).Should(BeTrue()) // copy test tools into container - if featureGates.Enabled(featureGateRange) { + if e2eutil.FeatureGates.Enabled(e2eutil.FeatureGateRange) { out, err = e2eutil.KubeCtlCommand("-n", ns, "cp", "-c", container, "/tmp/sha256sum-offset", fmt.Sprintf("%s:/bin/", podName)).CombinedOutput() if err != nil { @@ -129,21 +129,21 @@ func singleDfgetTest(name, ns, label, podNamePrefix, container string) { url2 := e2eutil.GetNoContentLengthFileURL(path) // make ranged requests to invoke prefetch feature - if featureGates.Enabled(featureGateRange) { + if e2eutil.FeatureGates.Enabled(e2eutil.FeatureGateRange) { rg1, rg2 := getRandomRange(size), getRandomRange(size) downloadSingleFile(ns, pod, path, url1, size, rg1, rg1.String()) - if featureGates.Enabled(featureGateNoLength) { + if e2eutil.FeatureGates.Enabled(e2eutil.FeatureGateNoLength) { downloadSingleFile(ns, pod, path, url2, size, rg2, rg2.String()) } - if featureGates.Enabled(featureGateOpenRange) { + if e2eutil.FeatureGates.Enabled(e2eutil.FeatureGateOpenRange) { rg3, rg4 := getRandomRange(size), getRandomRange(size) // set target length rg3.Length = int64(size) - rg3.Start rg4.Length = int64(size) - rg4.Start downloadSingleFile(ns, pod, path, url1, size, rg3, fmt.Sprintf("bytes=%d-", rg3.Start)) - if featureGates.Enabled(featureGateNoLength) { + if e2eutil.FeatureGates.Enabled(e2eutil.FeatureGateNoLength) { downloadSingleFile(ns, pod, path, url2, size, rg4, fmt.Sprintf("bytes=%d-", rg4.Start)) } } @@ -151,13 +151,13 @@ func singleDfgetTest(name, ns, label, podNamePrefix, container string) { downloadSingleFile(ns, pod, path, url1, size, nil, "") - if featureGates.Enabled(featureGateNoLength) { + if e2eutil.FeatureGates.Enabled(e2eutil.FeatureGateNoLength) { downloadSingleFile(ns, pod, path, url2, size, nil, "") } } }) It(name+" - recursive with dfget", Label("download", "recursive", "dfget"), func() { - if !featureGates.Enabled(featureGateRecursive) { + if !e2eutil.FeatureGates.Enabled(e2eutil.FeatureGateRecursive) { fmt.Println("feature gate recursive is disable, skip") return } @@ -222,7 +222,7 @@ func singleDfgetTest(name, ns, label, podNamePrefix, container string) { }) It(name+" - recursive with grpc", Label("download", "recursive", "grpc"), func() { - if !featureGates.Enabled(featureGateRecursive) { + if !e2eutil.FeatureGates.Enabled(e2eutil.FeatureGateRecursive) { fmt.Println("feature gate recursive is disable, skip") return } diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index ae4004f5e58..562210a2591 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -27,59 +27,14 @@ import ( . "github.com/onsi/ginkgo/v2" //nolint . "github.com/onsi/gomega" //nolint - "k8s.io/component-base/featuregate" "d7y.io/dragonfly/v2/test/e2e/e2eutil" _ "d7y.io/dragonfly/v2/test/e2e/manager" ) -var ( - featureGates = featuregate.NewFeatureGate() - featureGatesFlag string - - featureGateRange featuregate.Feature = "dfget-range" - featureGateOpenRange featuregate.Feature = "dfget-open-range" - featureGateCommit featuregate.Feature = "dfget-commit" - featureGateNoLength featuregate.Feature = "dfget-no-length" - featureGateEmptyFile featuregate.Feature = "dfget-empty-file" - featureGateRecursive featuregate.Feature = "dfget-recursive" - - defaultFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ - featureGateCommit: { - Default: true, - LockToDefault: false, - PreRelease: featuregate.Alpha, - }, - featureGateNoLength: { - Default: true, - LockToDefault: false, - PreRelease: featuregate.Alpha, - }, - featureGateRange: { - Default: false, - LockToDefault: false, - PreRelease: featuregate.Alpha, - }, - featureGateOpenRange: { - Default: false, - LockToDefault: false, - PreRelease: featuregate.Alpha, - }, - featureGateEmptyFile: { - Default: false, - LockToDefault: false, - PreRelease: featuregate.Alpha, - }, - featureGateRecursive: { - Default: false, - LockToDefault: false, - PreRelease: featuregate.Alpha, - }, - } -) +var featureGatesFlag string func init() { - _ = featureGates.Add(defaultFeatureGates) flag.StringVar(&featureGatesFlag, "feature-gates", "", "e2e test feature gates") } @@ -152,9 +107,9 @@ var _ = AfterSuite(func() { }) var _ = BeforeSuite(func() { - err := featureGates.Set(featureGatesFlag) + err := e2eutil.FeatureGates.Set(featureGatesFlag) Expect(err).NotTo(HaveOccurred()) - fmt.Printf("feature gates: %q, flags: %q\n", featureGates.String(), featureGatesFlag) + fmt.Printf("feature gates: %q, flags: %q\n", e2eutil.FeatureGates.String(), featureGatesFlag) mode := os.Getenv("DRAGONFLY_COMPATIBILITY_E2E_TEST_MODE") imageName := os.Getenv("DRAGONFLY_COMPATIBILITY_E2E_TEST_IMAGE") @@ -187,7 +142,7 @@ var _ = BeforeSuite(func() { fmt.Printf("raw dfget version:\n%s\n", rawDfgetVersion) fmt.Printf("dfget merge commit: %s\n", dfgetGitCommit) - if !featureGates.Enabled(featureGateCommit) { + if !e2eutil.FeatureGates.Enabled(e2eutil.FeatureGateCommit) { return } if mode == dfdaemonCompatibilityTestMode { @@ -197,7 +152,7 @@ var _ = BeforeSuite(func() { Expect(gitCommit).To(Equal(dfgetGitCommit)) - if featureGates.Enabled(featureGateRange) { + if e2eutil.FeatureGates.Enabled(e2eutil.FeatureGateRange) { out, err := e2eutil.DockerCopy("/bin/", "/tmp/sha256sum-offset").CombinedOutput() if err != nil { fmt.Println(string(out)) @@ -205,7 +160,7 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) } - if featureGates.Enabled(featureGateRecursive) { + if e2eutil.FeatureGates.Enabled(e2eutil.FeatureGateRecursive) { out, err := e2eutil.DockerCopy("/bin/", "/tmp/download-grpc-test").CombinedOutput() if err != nil { fmt.Println(string(out)) diff --git a/test/e2e/e2eutil/feature_gate.go b/test/e2e/e2eutil/feature_gate.go new file mode 100644 index 00000000000..e7b6d4f88a8 --- /dev/null +++ b/test/e2e/e2eutil/feature_gate.go @@ -0,0 +1,57 @@ +package e2eutil + +import "k8s.io/component-base/featuregate" + +var ( + FeatureGates = featuregate.NewFeatureGate() + + FeatureGateRange featuregate.Feature = "dfget-range" + FeatureGateOpenRange featuregate.Feature = "dfget-open-range" + FeatureGateCommit featuregate.Feature = "dfget-commit" + FeatureGateNoLength featuregate.Feature = "dfget-no-length" + FeatureGateEmptyFile featuregate.Feature = "dfget-empty-file" + FeatureGateRecursive featuregate.Feature = "dfget-recursive" + FeatureGatePreheatMultiArchImage featuregate.Feature = "dfget-preheat-multi-arch-image" + + defaultFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ + FeatureGateCommit: { + Default: true, + LockToDefault: false, + PreRelease: featuregate.Alpha, + }, + FeatureGateNoLength: { + Default: true, + LockToDefault: false, + PreRelease: featuregate.Alpha, + }, + FeatureGateRange: { + Default: false, + LockToDefault: false, + PreRelease: featuregate.Alpha, + }, + FeatureGateOpenRange: { + Default: false, + LockToDefault: false, + PreRelease: featuregate.Alpha, + }, + FeatureGateEmptyFile: { + Default: false, + LockToDefault: false, + PreRelease: featuregate.Alpha, + }, + FeatureGateRecursive: { + Default: false, + LockToDefault: false, + PreRelease: featuregate.Alpha, + }, + FeatureGatePreheatMultiArchImage: { + Default: false, + LockToDefault: false, + PreRelease: featuregate.Alpha, + }, + } +) + +func init() { + _ = FeatureGates.Add(defaultFeatureGates) +} diff --git a/test/e2e/manager/preheat.go b/test/e2e/manager/preheat.go index 228a8b355d3..ba59bfd87e3 100644 --- a/test/e2e/manager/preheat.go +++ b/test/e2e/manager/preheat.go @@ -139,7 +139,12 @@ var _ = Describe("Preheat with manager", func() { } }) - It("preheat image for linux/amd64 platform should be ok", Label("preheat", "image", "platform", "linux/amd64"), func() { + It("preheat image for linux/amd64 platform should be ok", Label("preheat", "image"), func() { + if !e2eutil.FeatureGates.Enabled(e2eutil.FeatureGatePreheatMultiArchImage) { + fmt.Println("feature gate preheat multi arch image is disable, skip") + return + } + url := "https://index.docker.io/v2/dragonflyoss/scheduler/manifests/v2.1.0" fmt.Println("download image: " + url) @@ -196,7 +201,12 @@ var _ = Describe("Preheat with manager", func() { } }) - It("preheat image for linux/arm64 platform should be ok", Label("preheat", "image", "platform", "linux/arm64"), func() { + It("preheat image for linux/arm64 platform should be ok", Label("preheat", "image"), func() { + if !e2eutil.FeatureGates.Enabled(e2eutil.FeatureGatePreheatMultiArchImage) { + fmt.Println("feature gate preheat multi arch image is disable, skip") + return + } + url := "https://index.docker.io/v2/dragonflyoss/scheduler/manifests/v2.1.0" fmt.Println("download image: " + url)