From a7f3b21c11ea82b4031abe005085a2447477a0c1 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 16 May 2025 18:30:21 +0200 Subject: [PATCH 01/22] Add scripts to prepare distributions --- cmd/distribution/all.yml | 6 + cmd/distribution/config_test.go | 68 +++++++ cmd/distribution/main.go | 312 ++++++++++++++++++++++++++++++++ cmd/distribution/sample.yaml | 72 ++++++++ cmd/distribution/test.yaml | 74 ++++++++ go.mod | 5 +- go.sum | 3 + 7 files changed, 538 insertions(+), 2 deletions(-) create mode 100644 cmd/distribution/all.yml create mode 100644 cmd/distribution/config_test.go create mode 100644 cmd/distribution/main.go create mode 100644 cmd/distribution/sample.yaml create mode 100644 cmd/distribution/test.yaml diff --git a/cmd/distribution/all.yml b/cmd/distribution/all.yml new file mode 100644 index 000000000..8a3f528a4 --- /dev/null +++ b/cmd/distribution/all.yml @@ -0,0 +1,6 @@ +# Base address of the Package Registry +address: "https://epr.elastic.co" + +queries: + - all: true + prerelease: true diff --git a/cmd/distribution/config_test.go b/cmd/distribution/config_test.go new file mode 100644 index 000000000..bb819ee3c --- /dev/null +++ b/cmd/distribution/config_test.go @@ -0,0 +1,68 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package main + +import "fmt" + +func Example_config_urls() { + config := config{ + Address: "http://localhost:8080", + Queries: []configQuery{ + {}, + {Prerelease: true}, + {KibanaVersion: "7.19.0"}, + {KibanaVersion: "7.19.0", Prerelease: true}, + {SpecMin: "2.3", SpecMax: "3.0"}, + }, + } + + urls, err := config.urls() + if err != nil { + panic(err) + } + + for u := range urls { + fmt.Println(u) + } + + // Output: + // http://localhost:8080/search + // http://localhost:8080/search?prerelease=true + // http://localhost:8080/search?kibana.version=7.19.0 + // http://localhost:8080/search?kibana.version=7.19.0&prerelease=true + // http://localhost:8080/search?spec.max=3.0&spec.min=2.3 +} + +func Example_config_urls_matrix() { + config := config{ + Address: "http://localhost:8080", + Matrix: []configQuery{ + {}, + {Prerelease: true}, + }, + Queries: []configQuery{ + {}, + {KibanaVersion: "7.19.0"}, + {SpecMin: "2.3", SpecMax: "3.0"}, + }, + } + + urls, err := config.urls() + if err != nil { + panic(err) + } + + for u := range urls { + fmt.Println(u) + } + + // Output: + // http://localhost:8080/search + // http://localhost:8080/search?kibana.version=7.19.0 + // http://localhost:8080/search?spec.max=3.0&spec.min=2.3 + // http://localhost:8080/search?prerelease=true + // http://localhost:8080/search?kibana.version=7.19.0&prerelease=true + // http://localhost:8080/search?prerelease=true&spec.max=3.0&spec.min=2.3 +} diff --git a/cmd/distribution/main.go b/cmd/distribution/main.go new file mode 100644 index 000000000..6b6c302b1 --- /dev/null +++ b/cmd/distribution/main.go @@ -0,0 +1,312 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "iter" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "slices" + "strings" + + "github.com/google/go-querystring/query" + "golang.org/x/mod/semver" + "gopkg.in/yaml.v3" +) + +const defaultAddress = "https://epr.elastic.co" + +func main() { + if len(os.Args) != 2 { + usageAndExit(-1) + } + config, err := readConfig(os.Args[1]) + if err != nil { + fmt.Printf("failed to read configuration from %s: %s\n", os.Args[1], err) + os.Exit(-1) + } + for _, action := range config.Actions { + err := action.init(config) + if err != nil { + fmt.Printf("failed to initialize actions: %w", err) + os.Exit(-1) + } + } + + packages, err := config.collect(&http.Client{}) + if err != nil { + fmt.Printf("failed to collect packages: %s", err) + os.Exit(-1) + } + + for _, info := range packages { + for _, action := range config.Actions { + err := action.perform(info) + if err != nil { + fmt.Printf("failed to collect packages: %s", err) + os.Exit(-1) + } + } + } + fmt.Println(len(packages), "packages total") +} + +func usageAndExit(status int) { + fmt.Println(os.Args[0], "[config.yaml]") + os.Exit(status) +} + +type config struct { + Address string `yaml:"address"` + Matrix []configQuery + Queries []configQuery `yaml:"queries"` + Actions configActions `yaml:"actions"` +} + +func (c config) urls() (iter.Seq[*url.URL], error) { + address := defaultAddress + if c.Address != "" { + address = c.Address + } + basePath, err := url.JoinPath(address, "search") + if err != nil { + return nil, fmt.Errorf("invalid address: %w", err) + } + baseURL, err := url.Parse(basePath) + if err != nil { + // This should not happen because JoinPath already parses the url. + panic("invalid url") + } + matrix := c.Matrix + if len(matrix) == 0 { + matrix = []configQuery{{}} + } + return func(yield func(*url.URL) bool) { + for _, m := range matrix { + for _, q := range c.Queries { + values := m.Build() + for k, v := range q.Build() { + values[k] = v + } + ref := "" + encoded := values.Encode() + if len(encoded) > 0 { + ref = "?" + encoded + } + url, err := baseURL.Parse(ref) + if err != nil { + panic("invalid query " + encoded) + } + if !yield(url) { + return + } + } + } + }, nil +} + +func (c config) collect(client *http.Client) ([]packageInfo, error) { + urls, err := c.urls() + if err != nil { + return nil, fmt.Errorf("failed to build URLs: %w", err) + } + + type key struct { + Name, Version string + } + packagesMap := make(map[key]packageInfo) + for u := range urls { + fmt.Println(u.String()) + resp, err := client.Get(u.String()) + if err != nil { + return nil, fmt.Errorf("failed to GET %s: %w", u, err) + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return nil, fmt.Errorf("failed to GET %s (status code %d)", u, resp.StatusCode) + } + + var packages []packageInfo + err = json.NewDecoder(resp.Body).Decode(&packages) + if err != nil { + resp.Body.Close() + return nil, fmt.Errorf("failed to parse search response: %w", err) + } + resp.Body.Close() + fmt.Println(len(packages), "packages") + + for _, p := range packages { + k := key{Name: p.Name, Version: p.Version} + if _, found := packagesMap[k]; found { + continue + } + packagesMap[k] = p + } + } + + result := make([]packageInfo, 0, len(packagesMap)) + for _, p := range packagesMap { + result = append(result, p) + } + + slices.SortFunc(result, func(a, b packageInfo) int { + if n := strings.Compare(a.Name, b.Name); n != 0 { + return n + } + + return semver.Compare(a.Version, b.Version) + }) + + return result, nil +} + +type configQuery struct { + All bool `yaml:"all" url:"all,omitempty"` + Prerelease bool `yaml:"prerelease" url:"prerelease,omitempty"` + Type string `yaml:"type" url:"type,omitempty"` + KibanaVersion string `yaml:"kibana.version" url:"kibana.version,omitempty"` + SpecMin string `yaml:"spec.min" url:"spec.min,omitempty"` + SpecMax string `yaml:"spec.max" url:"spec.max,omitempty"` +} + +func (q configQuery) Build() url.Values { + v, err := query.Values(q) + if err != nil { + panic(err) + } + return v +} + +type packageInfo struct { + Name string `json:"name"` + Version string `json:"version"` + Download string `json:"download"` + SignaturePath string `json:"signature_path"` +} + +func readConfig(path string) (config, error) { + var config config + d, err := os.ReadFile(path) + if err != nil { + return config, err + } + + return config, yaml.Unmarshal(d, &config) +} + +type configActions []configAction + +var _ yaml.Unmarshaler = &configActions{} + +func (actions *configActions) UnmarshalYAML(node *yaml.Node) error { + var actionsMap []map[string]yaml.Node + err := node.Decode(&actionsMap) + if err != nil { + return fmt.Errorf("failed to decode actions: %w", err) + } + + *actions = make(configActions, 0, len(actionsMap)) + for _, configMap := range actionsMap { + if len(configMap) != 1 { + return errors.New("multiple entries found in action") + } + for name, config := range configMap { + action, err := configActionFactory(name) + if err != nil { + return err + } + err = config.Decode(action) + if err != nil { + return fmt.Errorf("could not decode action %s: %w", name, err) + } + *actions = append(*actions, action) + } + } + + return nil +} + +func configActionFactory(name string) (configAction, error) { + switch name { + case "print": + return &printAction{}, nil + case "download": + return &downloadAction{}, nil + default: + return nil, fmt.Errorf("unknown action %s", name) + } +} + +type configAction interface { + init(config) error + perform(packageInfo) error +} + +type printAction struct{} + +func (a *printAction) init(c config) error { + return nil +} + +func (a *printAction) perform(i packageInfo) error { + fmt.Println("- ", i.Name, i.Version) + return nil +} + +type downloadAction struct { + client *http.Client + + Address string `yaml:"address"` + Destination string `yaml:"destination"` +} + +func (a *downloadAction) init(c config) error { + a.client = &http.Client{} + if a.Address == "" { + a.Address = c.Address + } + return os.MkdirAll(a.Destination, 0755) +} + +func (a *downloadAction) perform(i packageInfo) error { + if err := a.download(i.Download); err != nil { + return fmt.Errorf("failed to download package %s: %w", i.Download, err) + } + if err := a.download(i.SignaturePath); err != nil { + return fmt.Errorf("failed to download signature %s: %w", i.SignaturePath, err) + } + return nil +} + +func (a *downloadAction) download(urlPath string) error { + p, err := url.JoinPath(a.Address, urlPath) + if err != nil { + return fmt.Errorf("failed to build url: %w", err) + } + resp, err := a.client.Get(p) + if err != nil { + return fmt.Errorf("failed to get %s: %w", urlPath, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to get %s (status code %d)", urlPath, resp.StatusCode) + } + + f, err := os.Create(filepath.Join(a.Destination, path.Base(urlPath))) + if err != nil { + return fmt.Errorf("failed to open %s in %s: %w", path.Base(urlPath), a.Destination, err) + } + defer f.Close() + + _, err = io.Copy(f, resp.Body) + return err +} diff --git a/cmd/distribution/sample.yaml b/cmd/distribution/sample.yaml new file mode 100644 index 000000000..1c694e9f1 --- /dev/null +++ b/cmd/distribution/sample.yaml @@ -0,0 +1,72 @@ +# Base address of the Package Registry +address: "https://epr.elastic.co" + +# Queries are executed with each one of the parameters of the matrix. +matrix: + - spec.max: 3.0 + - prerelease: true + spec.max: 3.0 + +# Queries to execute to discover packages. +queries: + - kibana.version: 7.17.26 + - kibana.version: 7.17.27 + - kibana.version: 7.17.28 + - kibana.version: 7.17.29 + - kibana.version: 8.15.4 + - kibana.version: 8.15.5 + - kibana.version: 8.15.6 + - kibana.version: 8.16.0 + spec.max: 3.3 + - kibana.version: 8.16.1 + spec.max: 3.3 + - kibana.version: 8.16.2 + spec.max: 3.3 + - kibana.version: 8.16.3 + spec.max: 3.3 + - kibana.version: 8.16.4 + spec.max: 3.3 + - kibana.version: 8.16.5 + spec.max: 3.3 + - kibana.version: 8.16.6 + spec.max: 3.3 + - kibana.version: 8.16.7 + spec.max: 3.3 + - kibana.version: 8.17.0 + spec.max: 3.3 + - kibana.version: 8.17.1 + spec.max: 3.3 + - kibana.version: 8.17.2 + spec.max: 3.3 + - kibana.version: 8.17.3 + spec.max: 3.3 + - kibana.version: 8.17.4 + spec.max: 3.3 + - kibana.version: 8.17.5 + spec.max: 3.3 + - kibana.version: 8.17.6 + spec.max: 3.3 + - kibana.version: 8.17.7 + spec.max: 3.3 + - kibana.version: 8.18.0 + spec.max: 3.3 + - kibana.version: 8.18.1 + spec.max: 3.3 + - kibana.version: 8.18.2 + spec.max: 3.3 + - kibana.version: 8.19.0 + spec.max: 3.3 + - kibana.version: 9.0.0 + spec.min: 2.3 + spec.max: 3.3 + - kibana.version: 9.0.1 + spec.min: 2.3 + spec.max: 3.3 + - kibana.version: 9.0.2 + spec.min: 2.3 + spec.max: 3.3 + +actions: + - print: + - download: + destination: ./build/distribution diff --git a/cmd/distribution/test.yaml b/cmd/distribution/test.yaml new file mode 100644 index 000000000..d19887926 --- /dev/null +++ b/cmd/distribution/test.yaml @@ -0,0 +1,74 @@ +# Base address of the Package Registry +address: "https://epr.elastic.co" + +# Queries are executed with each one of the parameters of the matrix. +matrix: + - spec.max: 3.0 + type: input + - prerelease: true + spec.max: 3.0 + type: input + +# Queries to execute to discover packages. +queries: + - kibana.version: 7.17.26 + - kibana.version: 7.17.27 + - kibana.version: 7.17.28 + - kibana.version: 7.17.29 + - kibana.version: 8.15.4 + - kibana.version: 8.15.5 + - kibana.version: 8.15.6 + - kibana.version: 8.16.0 + spec.max: 3.3 + - kibana.version: 8.16.1 + spec.max: 3.3 + - kibana.version: 8.16.2 + spec.max: 3.3 + - kibana.version: 8.16.3 + spec.max: 3.3 + - kibana.version: 8.16.4 + spec.max: 3.3 + - kibana.version: 8.16.5 + spec.max: 3.3 + - kibana.version: 8.16.6 + spec.max: 3.3 + - kibana.version: 8.16.7 + spec.max: 3.3 + - kibana.version: 8.17.0 + spec.max: 3.3 + - kibana.version: 8.17.1 + spec.max: 3.3 + - kibana.version: 8.17.2 + spec.max: 3.3 + - kibana.version: 8.17.3 + spec.max: 3.3 + - kibana.version: 8.17.4 + spec.max: 3.3 + - kibana.version: 8.17.5 + spec.max: 3.3 + - kibana.version: 8.17.6 + spec.max: 3.3 + - kibana.version: 8.17.7 + spec.max: 3.3 + - kibana.version: 8.18.0 + spec.max: 3.3 + - kibana.version: 8.18.1 + spec.max: 3.3 + - kibana.version: 8.18.2 + spec.max: 3.3 + - kibana.version: 8.19.0 + spec.max: 3.3 + - kibana.version: 9.0.0 + spec.min: 2.3 + spec.max: 3.3 + - kibana.version: 9.0.1 + spec.min: 2.3 + spec.max: 3.3 + - kibana.version: 9.0.2 + spec.min: 2.3 + spec.max: 3.3 + +actions: + - print: + - download: + destination: ./build/distribution-test diff --git a/go.mod b/go.mod index 7a6157bac..ac94fc286 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/elastic/go-ucfg v0.8.8 github.com/felixge/httpsnoop v1.0.4 github.com/fsouza/fake-gcs-server v1.52.2 + github.com/google/go-querystring v1.1.0 github.com/gorilla/mux v1.8.1 github.com/hashicorp/go-retryablehttp v0.7.7 github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 @@ -22,8 +23,10 @@ require ( go.elastic.co/apm/v2 v2.7.0 go.elastic.co/ecszap v1.0.3 go.uber.org/zap v1.27.0 + golang.org/x/mod v0.24.0 golang.org/x/tools v0.33.0 gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.1 honnef.co/go/tools v0.6.1 ) @@ -84,7 +87,6 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.38.0 // indirect golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678 // indirect - golang.org/x/mod v0.24.0 // indirect golang.org/x/net v0.40.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect golang.org/x/sync v0.14.0 // indirect @@ -97,6 +99,5 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20250505200425-f936aa4a68b2 // indirect google.golang.org/grpc v1.72.0 // indirect google.golang.org/protobuf v1.36.6 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect howett.net/plist v1.0.1 // indirect ) diff --git a/go.sum b/go.sum index 979595551..77b430da0 100644 --- a/go.sum +++ b/go.sum @@ -112,9 +112,12 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg= From e70a21c63b8faa60326dcb163291c7529e6bc71d Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Sat, 5 Jul 2025 17:57:07 +0200 Subject: [PATCH 02/22] Run actions in parallel --- cmd/distribution/lite.yaml | 30 ++++++++++++++++ cmd/distribution/main.go | 74 +++++++++++++++++++++++--------------- cmd/distribution/tasks.go | 55 ++++++++++++++++++++++++++++ 3 files changed, 130 insertions(+), 29 deletions(-) create mode 100644 cmd/distribution/lite.yaml create mode 100644 cmd/distribution/tasks.go diff --git a/cmd/distribution/lite.yaml b/cmd/distribution/lite.yaml new file mode 100644 index 000000000..3d8072833 --- /dev/null +++ b/cmd/distribution/lite.yaml @@ -0,0 +1,30 @@ +# Base address of the Package Registry +address: "https://epr.elastic.co" + +# Queries are executed with each one of the parameters of the matrix. +matrix: + - all: true + +# Queries to execute to discover packages. +queries: + - package: apache + - package: apm + - package: elastic_agent + - package: endpoint + - package: fleet_server + - package: linux + - package: log + - package: mysql + - package: nginx + - package: profiler_collector + - package: profiler_symbolizer + - package: security_ai_prompts + - package: security_detection_engine + - package: synthetics + - package: system + - package: windows + +actions: + - print: + - download: + destination: ./build/distribution-lite diff --git a/cmd/distribution/main.go b/cmd/distribution/main.go index 6b6c302b1..003ac37ed 100644 --- a/cmd/distribution/main.go +++ b/cmd/distribution/main.go @@ -15,6 +15,7 @@ import ( "os" "path" "path/filepath" + "runtime" "slices" "strings" @@ -37,7 +38,7 @@ func main() { for _, action := range config.Actions { err := action.init(config) if err != nil { - fmt.Printf("failed to initialize actions: %w", err) + fmt.Printf("failed to initialize actions: %s", err) os.Exit(-1) } } @@ -48,14 +49,21 @@ func main() { os.Exit(-1) } + taskpool := newTaskPool(runtime.GOMAXPROCS(0)) for _, info := range packages { - for _, action := range config.Actions { - err := action.perform(info) - if err != nil { - fmt.Printf("failed to collect packages: %s", err) - os.Exit(-1) + taskpool.Do(func() error { + for _, action := range config.Actions { + err := action.perform(info) + if err != nil { + return fmt.Errorf("failed to collect packages: %w", err) + } } - } + return nil + }) + } + if err := taskpool.Wait(); err != nil { + fmt.Println(err) + os.Exit(-1) } fmt.Println(len(packages), "packages total") } @@ -124,33 +132,40 @@ func (c config) collect(client *http.Client) ([]packageInfo, error) { Name, Version string } packagesMap := make(map[key]packageInfo) + taskPool := newTaskPool(runtime.GOMAXPROCS(0)) for u := range urls { - fmt.Println(u.String()) - resp, err := client.Get(u.String()) - if err != nil { - return nil, fmt.Errorf("failed to GET %s: %w", u, err) - } - if resp.StatusCode != http.StatusOK { - resp.Body.Close() - return nil, fmt.Errorf("failed to GET %s (status code %d)", u, resp.StatusCode) - } + taskPool.Do(func() error { + resp, err := client.Get(u.String()) + if err != nil { + return fmt.Errorf("failed to GET %s: %w", u, err) + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return fmt.Errorf("failed to GET %s (status code %d)", u, resp.StatusCode) + } - var packages []packageInfo - err = json.NewDecoder(resp.Body).Decode(&packages) - if err != nil { + var packages []packageInfo + err = json.NewDecoder(resp.Body).Decode(&packages) + if err != nil { + resp.Body.Close() + return fmt.Errorf("failed to parse search response: %w", err) + } resp.Body.Close() - return nil, fmt.Errorf("failed to parse search response: %w", err) - } - resp.Body.Close() - fmt.Println(len(packages), "packages") + fmt.Println(u.String(), len(packages), "packages") - for _, p := range packages { - k := key{Name: p.Name, Version: p.Version} - if _, found := packagesMap[k]; found { - continue + for _, p := range packages { + k := key{Name: p.Name, Version: p.Version} + if _, found := packagesMap[k]; found { + continue + } + packagesMap[k] = p } - packagesMap[k] = p - } + + return nil + }) + } + if err := taskPool.Wait(); err != nil { + return nil, err } result := make([]packageInfo, 0, len(packagesMap)) @@ -170,6 +185,7 @@ func (c config) collect(client *http.Client) ([]packageInfo, error) { } type configQuery struct { + Package string `yaml:"package" url:"package,omitempty"` All bool `yaml:"all" url:"all,omitempty"` Prerelease bool `yaml:"prerelease" url:"prerelease,omitempty"` Type string `yaml:"type" url:"type,omitempty"` diff --git a/cmd/distribution/tasks.go b/cmd/distribution/tasks.go new file mode 100644 index 000000000..d9e5a93e1 --- /dev/null +++ b/cmd/distribution/tasks.go @@ -0,0 +1,55 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package main + +import ( + "errors" + "sync" +) + +type taskPool struct { + wg sync.WaitGroup + pool chan struct{} + errC chan error + errors []error +} + +func newTaskPool(size int) *taskPool { + p := &taskPool{ + pool: make(chan struct{}, size), + errC: make(chan error), + } + go p.errorLoop() + return p +} + +func (p *taskPool) errorLoop() { + go func() { + for err := range p.errC { + if err != nil { + p.errors = append(p.errors, err) + } + } + }() +} + +// Do runs the task in a goroutine, ensuring no more tasks are running than the size of the pool. +func (p *taskPool) Do(task func() error) { + p.pool <- struct{}{} + p.wg.Add(1) + go func() { + defer func() { _ = <-p.pool }() + defer p.wg.Done() + p.errC <- task() + }() +} + +// Wait waits for all the tasks to finish, and joins the errors found. The pool cannot be used after calling Wait. +func (p *taskPool) Wait() error { + close(p.pool) + p.wg.Wait() + close(p.errC) + return errors.Join(p.errors...) +} From 10118ea24fd9dc5f7a563502c1555828dc4f5285 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Sat, 5 Jul 2025 18:04:32 +0200 Subject: [PATCH 03/22] Select lite packages per version --- cmd/distribution/lite.yaml | 62 +++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/cmd/distribution/lite.yaml b/cmd/distribution/lite.yaml index 3d8072833..3d83d6de6 100644 --- a/cmd/distribution/lite.yaml +++ b/cmd/distribution/lite.yaml @@ -3,7 +3,67 @@ address: "https://epr.elastic.co" # Queries are executed with each one of the parameters of the matrix. matrix: - - all: true + - kibana.version: 7.17.26 + - kibana.version: 7.17.27 + - kibana.version: 7.17.28 + - kibana.version: 7.17.29 + - kibana.version: 8.15.4 + - kibana.version: 8.15.5 + - kibana.version: 8.15.6 + - kibana.version: 8.16.0 + spec.max: 3.3 + - kibana.version: 8.16.1 + spec.max: 3.3 + - kibana.version: 8.16.2 + spec.max: 3.3 + - kibana.version: 8.16.3 + spec.max: 3.3 + - kibana.version: 8.16.4 + spec.max: 3.3 + - kibana.version: 8.16.5 + spec.max: 3.3 + - kibana.version: 8.16.6 + spec.max: 3.3 + - kibana.version: 8.16.7 + spec.max: 3.3 + - kibana.version: 8.17.0 + spec.max: 3.3 + - kibana.version: 8.17.1 + spec.max: 3.3 + - kibana.version: 8.17.2 + spec.max: 3.3 + - kibana.version: 8.17.3 + spec.max: 3.3 + - kibana.version: 8.17.4 + spec.max: 3.3 + - kibana.version: 8.17.5 + spec.max: 3.3 + - kibana.version: 8.17.6 + spec.max: 3.3 + - kibana.version: 8.17.7 + spec.max: 3.3 + - kibana.version: 8.18.0 + spec.max: 3.3 + - kibana.version: 8.18.1 + spec.max: 3.3 + - kibana.version: 8.18.2 + spec.max: 3.3 + - kibana.version: 8.19.0 + spec.max: 3.4 + - kibana.version: 9.0.0 + spec.min: 2.3 + spec.max: 3.3 + - kibana.version: 9.0.1 + spec.min: 2.3 + spec.max: 3.3 + - kibana.version: 9.0.2 + spec.min: 2.3 + spec.max: 3.3 + - kibana.version: 9.1.0 + spec.min: 2.3 + spec.max: 3.4 + - spec.min: 2.3 + spec.max: 3.4 # Queries to execute to discover packages. queries: From b31b25efe9f968a8609cdf21390d48f1b7cb46ac Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Sat, 5 Jul 2025 18:09:33 +0200 Subject: [PATCH 04/22] Add 8.19 and 9.1 --- cmd/distribution/sample.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/distribution/sample.yaml b/cmd/distribution/sample.yaml index 1c694e9f1..7905d0c30 100644 --- a/cmd/distribution/sample.yaml +++ b/cmd/distribution/sample.yaml @@ -55,7 +55,7 @@ queries: - kibana.version: 8.18.2 spec.max: 3.3 - kibana.version: 8.19.0 - spec.max: 3.3 + spec.max: 3.4 - kibana.version: 9.0.0 spec.min: 2.3 spec.max: 3.3 @@ -65,6 +65,10 @@ queries: - kibana.version: 9.0.2 spec.min: 2.3 spec.max: 3.3 + - kibana.version: 9.1.0 + spec.min: 2.3 + spec.max: 3.4 + actions: - print: From 0b47e15a314b7411879733f95e64b64a9eb106aa Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 7 Jul 2025 09:31:07 +0200 Subject: [PATCH 05/22] linting --- cmd/distribution/tasks.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/distribution/tasks.go b/cmd/distribution/tasks.go index d9e5a93e1..0516285a4 100644 --- a/cmd/distribution/tasks.go +++ b/cmd/distribution/tasks.go @@ -40,7 +40,7 @@ func (p *taskPool) Do(task func() error) { p.pool <- struct{}{} p.wg.Add(1) go func() { - defer func() { _ = <-p.pool }() + defer func() { <-p.pool }() defer p.wg.Done() p.errC <- task() }() From 9cc14d40146f93b2fde88244306498b4f0322adb Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 14 Nov 2025 13:32:44 +0100 Subject: [PATCH 06/22] Support pinned package versions --- cmd/distribution/main.go | 43 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/cmd/distribution/main.go b/cmd/distribution/main.go index 003ac37ed..e09f187d3 100644 --- a/cmd/distribution/main.go +++ b/cmd/distribution/main.go @@ -74,10 +74,11 @@ func usageAndExit(status int) { } type config struct { - Address string `yaml:"address"` - Matrix []configQuery - Queries []configQuery `yaml:"queries"` - Actions configActions `yaml:"actions"` + Address string `yaml:"address"` + Matrix []configQuery `yaml:"matrix"` + Queries []configQuery `yaml:"queries"` + Packages []configPackage `yaml:"packages"` + Actions configActions `yaml:"actions"` } func (c config) urls() (iter.Seq[*url.URL], error) { @@ -122,6 +123,12 @@ func (c config) urls() (iter.Seq[*url.URL], error) { }, nil } +// downloadPathForPackage returns the paths to download the package with the given name and version and its signature. +func (c config) downloadPathForPackage(name, version string) (string, string) { + path := path.Join("epr", name, fmt.Sprintf("%s-%s.zip", name, version)) + return path, path + ".sig" +} + func (c config) collect(client *http.Client) ([]packageInfo, error) { urls, err := c.urls() if err != nil { @@ -132,6 +139,15 @@ func (c config) collect(client *http.Client) ([]packageInfo, error) { Name, Version string } packagesMap := make(map[key]packageInfo) + + pinnedPackages, err := c.pinnedPackages() + if err != nil { + return nil, fmt.Errorf("failed to prepare pinned packages: %w", err) + } + for _, p := range pinnedPackages { + packagesMap[key{Name: p.Name, Version: p.Version}] = p + } + taskPool := newTaskPool(runtime.GOMAXPROCS(0)) for u := range urls { taskPool.Do(func() error { @@ -184,6 +200,20 @@ func (c config) collect(client *http.Client) ([]packageInfo, error) { return result, nil } +func (c config) pinnedPackages() ([]packageInfo, error) { + packages := make([]packageInfo, len(c.Packages)) + for i, p := range c.Packages { + download, signature := c.downloadPathForPackage(p.Name, p.Version) + packages[i] = packageInfo{ + Name: p.Name, + Version: p.Version, + Download: download, + SignaturePath: signature, + } + } + return packages, nil +} + type configQuery struct { Package string `yaml:"package" url:"package,omitempty"` All bool `yaml:"all" url:"all,omitempty"` @@ -194,6 +224,11 @@ type configQuery struct { SpecMax string `yaml:"spec.max" url:"spec.max,omitempty"` } +type configPackage struct { + Name string `yaml:"name"` + Version string `yaml:"version"` +} + func (q configQuery) Build() url.Values { v, err := query.Values(q) if err != nil { From a967daa337cbaf9274057fb819bb16da0b0f7858 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Sat, 15 Nov 2025 18:12:02 +0100 Subject: [PATCH 07/22] Reuse existing packages --- cmd/distribution/GPG-KEY-elasticsearch | 31 +++++++ cmd/distribution/config_test.go | 4 +- cmd/distribution/download.go | 110 +++++++++++++++++++++++++ cmd/distribution/main.go | 56 +------------ 4 files changed, 146 insertions(+), 55 deletions(-) create mode 100644 cmd/distribution/GPG-KEY-elasticsearch create mode 100644 cmd/distribution/download.go diff --git a/cmd/distribution/GPG-KEY-elasticsearch b/cmd/distribution/GPG-KEY-elasticsearch new file mode 100644 index 000000000..f9e4326dd --- /dev/null +++ b/cmd/distribution/GPG-KEY-elasticsearch @@ -0,0 +1,31 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQENBFI3HsoBCADXDtbNJnxbPqB1vDNtCsqhe49vFYsZN9IOZsZXgp7aHjh6CJBD +A+bGFOwyhbd7at35jQjWAw1O3cfYsKAmFy+Ar3LHCMkV3oZspJACTIgCrwnkic/9 +CUliQe324qvObU2QRtP4Fl0zWcfb/S8UYzWXWIFuJqMvE9MaRY1bwUBvzoqavLGZ +j3SF1SPO+TB5QrHkrQHBsmX+Jda6d4Ylt8/t6CvMwgQNlrlzIO9WT+YN6zS+sqHd +1YK/aY5qhoLNhp9G/HxhcSVCkLq8SStj1ZZ1S9juBPoXV1ZWNbxFNGwOh/NYGldD +2kmBf3YgCqeLzHahsAEpvAm8TBa7Q9W21C8vABEBAAG0RUVsYXN0aWNzZWFyY2gg +KEVsYXN0aWNzZWFyY2ggU2lnbmluZyBLZXkpIDxkZXZfb3BzQGVsYXN0aWNzZWFy +Y2gub3JnPokBTgQTAQgAOAIbAwIXgBYhBEYJWsyFSFgsGiaZqdJ9ZmzYjkK0BQJk +9vrZBQsJCAcDBRUKCQgLBRYCAwEAAh4FAAoJENJ9ZmzYjkK00hoH+wYXZKgVb3Wv +4AA/+T1IAf7edwgajr58bEyqds6/4v6uZBneUaqahUqMXgLFRX5dBSrAS7bvE/jx ++BBQx+rpFGxSwvFegRevE1zAGVtpgkFQX0RpRcKSmksucSBxikR/dPn9XdJSEVa8 +vPcs11V+2E5tq3LEP14zJL4MkJKQF0VJl5UUmKLS7U2F/IB5aXry9UWdMTnwNntX +kl2iDaViYF4MC6xTS24uLwND2St0Jvjt+xGEwbdBVvp+UZ/kG6IGkYM5eWGPuok/ +DHvjUdwTfyO9b5xGbqn5FJ3UFOwB/nOSFXHM8rsHRT/67gHcIl8YFqSQXpIkk9D3 +dCY+KieW0ue5AQ0EUjceygEIAOSVJc3DFuf3LsmUfGpUmnCqoUm76Eqqm8xynFEG +ZpczTChkwARRtckcfa/sGv376j+jk0c0Q71Uv3MnMLPGF+w3bpu8fLiPeW/cntf1 +8uZ6DxJvHA/oaZZ6VPjwUGSeVydiPtZfTYsceO8Dxl3gpS6nHZ9Gsnfr/kcH9/11 +Ca73HBtmGVIkOI1mZKMbANO8cewY/i7fPxShu7B0Rb3jxVNGUuiRcfRiao0gWx0U +ZGpvuHplt7loFX2cbsHFAp9WsjYEbSohb/Y0K4NkyFhL82MfbcsEwsXPhRTFgJWw +s4vpuFg/kFFlnw0NNPVP1jNJLNCsMBMEpP1A7k6MRpylNnUAEQEAAYkBNgQYAQgA +IAIbDBYhBEYJWsyFSFgsGiaZqdJ9ZmzYjkK0BQJk9vsHAAoJENJ9ZmzYjkK0hWsH +/ArKtn12HM3+41zYo9qO4rTri7+IYTjSB/JDTOusZgZLd/HCp1xQo4SI2Eur3Rtx +USMWK1LEeBzsjwDT9yVceYekrBEqUVyRMSVYj+UeZK2s4LbXm9b4jxXVtaivmkMA +jtznndrD7kmm8ak+UsZplf6p6uZS9TZ9hjwoMmw5oMaS6TZkLT4KYGWeyzHJSUBX +YikY6vssDQu4SJ07m1f4Hz81J39QOcHln5I5HTK8Rh/VUFcxNnGg9360g55wWpiF +eUTeMyoXpOtffiUhiOtbRYsmSYC0D4Fd5yJnO3n1pwnVVVsM7RAC22rc5j/Dw8dR +GIHikRcYWeXTYW7veewK5Ss= +=ftS0 +-----END PGP PUBLIC KEY BLOCK----- diff --git a/cmd/distribution/config_test.go b/cmd/distribution/config_test.go index bb819ee3c..55065310e 100644 --- a/cmd/distribution/config_test.go +++ b/cmd/distribution/config_test.go @@ -18,7 +18,7 @@ func Example_config_urls() { }, } - urls, err := config.urls() + urls, err := config.searchURLs() if err != nil { panic(err) } @@ -49,7 +49,7 @@ func Example_config_urls_matrix() { }, } - urls, err := config.urls() + urls, err := config.searchURLs() if err != nil { panic(err) } diff --git a/cmd/distribution/download.go b/cmd/distribution/download.go new file mode 100644 index 000000000..e2d312444 --- /dev/null +++ b/cmd/distribution/download.go @@ -0,0 +1,110 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package main + +import ( + "bytes" + _ "embed" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + + "golang.org/x/crypto/openpgp" +) + +type downloadAction struct { + client *http.Client + keyRing openpgp.KeyRing + + Address string `yaml:"address"` + Destination string `yaml:"destination"` +} + +// publicKey is the public key of the key used to sign elastic artifacts. +// Downloaded from https://artifacts.elastic.co/GPG-KEY-elasticsearch +// +//go:embed GPG-KEY-elasticsearch +var publicKey []byte + +func (a *downloadAction) init(c config) error { + a.client = &http.Client{} + if a.Address == "" { + a.Address = c.Address + } + err := os.MkdirAll(a.Destination, 0755) + if err != nil { + return fmt.Errorf("failed to create desination directory: %w", err) + } + a.keyRing, err = openpgp.ReadArmoredKeyRing(bytes.NewReader(publicKey)) + if err != nil { + return fmt.Errorf("failed to initialize public key: %w", err) + } + return nil +} + +func (a *downloadAction) perform(i packageInfo) error { + if valid, _ := a.valid(i); valid { + return nil + } + if err := a.download(i.Download); err != nil { + return fmt.Errorf("failed to download package %s: %w", i.Download, err) + } + if err := a.download(i.SignaturePath); err != nil { + return fmt.Errorf("failed to download signature %s: %w", i.SignaturePath, err) + } + return nil +} + +func (a *downloadAction) download(urlPath string) error { + p, err := url.JoinPath(a.Address, urlPath) + if err != nil { + return fmt.Errorf("failed to build url: %w", err) + } + resp, err := a.client.Get(p) + if err != nil { + return fmt.Errorf("failed to get %s: %w", urlPath, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to get %s (status code %d)", urlPath, resp.StatusCode) + } + + f, err := os.Create(a.destinationPath(urlPath)) + if err != nil { + return fmt.Errorf("failed to open %s in %s: %w", path.Base(urlPath), a.Destination, err) + } + defer f.Close() + + _, err = io.Copy(f, resp.Body) + return err +} + +func (a *downloadAction) destinationPath(urlPath string) string { + return filepath.Join(a.Destination, path.Base(urlPath)) +} + +func (a *downloadAction) valid(info packageInfo) (bool, error) { + signed, err := os.Open(a.destinationPath(info.Download)) + if err != nil { + return false, err + } + defer signed.Close() + + signature, err := os.Open(a.destinationPath(info.SignaturePath)) + if err != nil { + return false, err + } + defer signature.Close() + + _, err = openpgp.CheckArmoredDetachedSignature(a.keyRing, signed, signature) + if err != nil { + return false, err + } + return true, nil +} diff --git a/cmd/distribution/main.go b/cmd/distribution/main.go index e09f187d3..fd1bd167b 100644 --- a/cmd/distribution/main.go +++ b/cmd/distribution/main.go @@ -8,13 +8,11 @@ import ( "encoding/json" "errors" "fmt" - "io" "iter" "net/http" "net/url" "os" "path" - "path/filepath" "runtime" "slices" "strings" @@ -81,7 +79,8 @@ type config struct { Actions configActions `yaml:"actions"` } -func (c config) urls() (iter.Seq[*url.URL], error) { +// searchURLs generates the search searchURLs required for the given configuration. +func (c config) searchURLs() (iter.Seq[*url.URL], error) { address := defaultAddress if c.Address != "" { address = c.Address @@ -130,7 +129,7 @@ func (c config) downloadPathForPackage(name, version string) (string, string) { } func (c config) collect(client *http.Client) ([]packageInfo, error) { - urls, err := c.urls() + urls, err := c.searchURLs() if err != nil { return nil, fmt.Errorf("failed to build URLs: %w", err) } @@ -312,52 +311,3 @@ func (a *printAction) perform(i packageInfo) error { fmt.Println("- ", i.Name, i.Version) return nil } - -type downloadAction struct { - client *http.Client - - Address string `yaml:"address"` - Destination string `yaml:"destination"` -} - -func (a *downloadAction) init(c config) error { - a.client = &http.Client{} - if a.Address == "" { - a.Address = c.Address - } - return os.MkdirAll(a.Destination, 0755) -} - -func (a *downloadAction) perform(i packageInfo) error { - if err := a.download(i.Download); err != nil { - return fmt.Errorf("failed to download package %s: %w", i.Download, err) - } - if err := a.download(i.SignaturePath); err != nil { - return fmt.Errorf("failed to download signature %s: %w", i.SignaturePath, err) - } - return nil -} - -func (a *downloadAction) download(urlPath string) error { - p, err := url.JoinPath(a.Address, urlPath) - if err != nil { - return fmt.Errorf("failed to build url: %w", err) - } - resp, err := a.client.Get(p) - if err != nil { - return fmt.Errorf("failed to get %s: %w", urlPath, err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("failed to get %s (status code %d)", urlPath, resp.StatusCode) - } - - f, err := os.Create(filepath.Join(a.Destination, path.Base(urlPath))) - if err != nil { - return fmt.Errorf("failed to open %s in %s: %w", path.Base(urlPath), a.Destination, err) - } - defer f.Close() - - _, err = io.Copy(f, resp.Body) - return err -} From 8aa3e5b76f6ec043f9aeda48b9b14fd15f18f2f3 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Sat, 15 Nov 2025 18:37:04 +0100 Subject: [PATCH 08/22] Add Dockerfile --- Dockerfile.distribution | 21 +++++++++++++++++++++ cmd/distribution/Makefile | 1 + 2 files changed, 22 insertions(+) create mode 100644 Dockerfile.distribution create mode 100644 cmd/distribution/Makefile diff --git a/Dockerfile.distribution b/Dockerfile.distribution new file mode 100644 index 000000000..9592d1c7e --- /dev/null +++ b/Dockerfile.distribution @@ -0,0 +1,21 @@ +# This image contains the tool to prepare package distributions. + +ARG GO_VERSION +ARG BUILDER_IMAGE=golang +ARG RUNNER_IMAGE=cgr.dev/chainguard/wolfi-base + +# Build binary +FROM --platform=${BUILDPLATFORM:-linux} ${BUILDER_IMAGE}:${GO_VERSION} AS builder + +COPY ./ /package-registry +WORKDIR /package-registry + +ARG TARGETPLATFORM + +RUN make -C cmd/release release-${TARGETPLATFORM:-linux} + +# Run binary +FROM ${RUNNER_IMAGE} + +# Move binary from the builder image +COPY --from=builder /package-registry/cmd/distribution /usr/bin/distribution diff --git a/cmd/distribution/Makefile b/cmd/distribution/Makefile new file mode 100644 index 000000000..3924bfdc1 --- /dev/null +++ b/cmd/distribution/Makefile @@ -0,0 +1 @@ +include ../../Makefile From 1148a75c8c48808b1df211937176d22239b50d5b Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Sun, 16 Nov 2025 12:30:03 +0100 Subject: [PATCH 09/22] Add sample for pinned versions --- cmd/distribution/pinned.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 cmd/distribution/pinned.yaml diff --git a/cmd/distribution/pinned.yaml b/cmd/distribution/pinned.yaml new file mode 100644 index 000000000..5d1a3cbfd --- /dev/null +++ b/cmd/distribution/pinned.yaml @@ -0,0 +1,16 @@ +# Base address of the Package Registry +address: "https://epr.elastic.co" + +queries: + - package: apache + all: true + spec.min: 3.0 + +packages: + - name: apache + version: 2.1.0 + +actions: + - print: + - download: + destination: ./build/pinned From 3ac5e33999bfb08e2074b5338eed5d8f36153952 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 17 Nov 2025 17:13:18 +0100 Subject: [PATCH 10/22] Use ProtonMail fork of opengpg --- cmd/distribution/download.go | 4 ++-- go.mod | 2 ++ go.sum | 4 ++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/cmd/distribution/download.go b/cmd/distribution/download.go index e2d312444..22e4bdab8 100644 --- a/cmd/distribution/download.go +++ b/cmd/distribution/download.go @@ -15,7 +15,7 @@ import ( "path" "path/filepath" - "golang.org/x/crypto/openpgp" + "github.com/ProtonMail/go-crypto/openpgp" ) type downloadAction struct { @@ -102,7 +102,7 @@ func (a *downloadAction) valid(info packageInfo) (bool, error) { } defer signature.Close() - _, err = openpgp.CheckArmoredDetachedSignature(a.keyRing, signed, signature) + _, err = openpgp.CheckArmoredDetachedSignature(a.keyRing, signed, signature, nil) if err != nil { return false, err } diff --git a/go.mod b/go.mod index 78e21fcaf..6a69b5c26 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.24.0 require ( cloud.google.com/go/storage v1.57.1 github.com/Masterminds/semver/v3 v3.4.0 + github.com/ProtonMail/go-crypto v1.3.0 github.com/elastic/go-licenser v0.4.2 github.com/elastic/go-ucfg v0.8.8 github.com/felixge/httpsnoop v1.0.4 @@ -48,6 +49,7 @@ require ( github.com/armon/go-radix v1.0.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudflare/circl v1.6.0 // indirect github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dustin/go-humanize v1.0.1 // indirect diff --git a/go.sum b/go.sum index fb2514a3a..bb4f379cc 100644 --- a/go.sum +++ b/go.sum @@ -36,6 +36,8 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapp github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0/go.mod h1:cSgYe11MCNYunTnRXrKiR/tHc0eoKjICUuWpNZoVCOo= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/ProtonMail/go-crypto v1.3.0 h1:ILq8+Sf5If5DCpHQp4PbZdS1J7HDFRXz/+xKBiRGFrw= +github.com/ProtonMail/go-crypto v1.3.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE= github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -44,6 +46,8 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/circl v1.6.0 h1:cr5JKic4HI+LkINy2lg3W2jF8sHCVTBncJr5gIIq7qk= +github.com/cloudflare/circl v1.6.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 h1:aQ3y1lwWyqYPiWZThqv1aFbZMiM9vblcSArJRf2Irls= github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= From e6ca112cc0addb1207f500200daaff134bad275b Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 18 Nov 2025 00:03:44 +0100 Subject: [PATCH 11/22] Use github.com/Masterminds/semver/v3 --- cmd/distribution/main.go | 18 ++++++++++++++++-- go.mod | 2 +- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/cmd/distribution/main.go b/cmd/distribution/main.go index fd1bd167b..88a3b9e98 100644 --- a/cmd/distribution/main.go +++ b/cmd/distribution/main.go @@ -17,8 +17,8 @@ import ( "slices" "strings" + "github.com/Masterminds/semver/v3" "github.com/google/go-querystring/query" - "golang.org/x/mod/semver" "gopkg.in/yaml.v3" ) @@ -193,7 +193,21 @@ func (c config) collect(client *http.Client) ([]packageInfo, error) { return n } - return semver.Compare(a.Version, b.Version) + // An invalid semantic version string is considered less than a valid one. + // All invalid semantic version strings compare equal to each other. + // From https://pkg.go.dev/golang.org/x/mod/semver#Compare + va, errA := semver.NewVersion(a.Version) + vb, errB := semver.NewVersion(b.Version) + switch { + case errA != nil && errB != nil: + return 0 + case errA != nil: + return -1 + case errB != nil: + return 1 + } + + return va.Compare(vb) }) return result, nil diff --git a/go.mod b/go.mod index 6a69b5c26..0bb5b9ebf 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,6 @@ require ( go.elastic.co/apm/v2 v2.7.1 go.elastic.co/ecszap v1.0.3 go.uber.org/zap v1.27.0 - golang.org/x/mod v0.30.0 golang.org/x/tools v0.39.0 google.golang.org/api v0.256.0 gopkg.in/yaml.v2 v2.4.0 @@ -96,6 +95,7 @@ require ( golang.org/x/crypto v0.44.0 // indirect golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678 // indirect + golang.org/x/mod v0.30.0 // indirect golang.org/x/net v0.47.0 // indirect golang.org/x/oauth2 v0.33.0 // indirect golang.org/x/sync v0.18.0 // indirect From 468598fe688b0cc0343649b07d37a184d03c5b5f Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 9 Feb 2026 18:08:07 +0100 Subject: [PATCH 12/22] Remove unused dockerfile --- Dockerfile.distribution | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 Dockerfile.distribution diff --git a/Dockerfile.distribution b/Dockerfile.distribution deleted file mode 100644 index 9592d1c7e..000000000 --- a/Dockerfile.distribution +++ /dev/null @@ -1,21 +0,0 @@ -# This image contains the tool to prepare package distributions. - -ARG GO_VERSION -ARG BUILDER_IMAGE=golang -ARG RUNNER_IMAGE=cgr.dev/chainguard/wolfi-base - -# Build binary -FROM --platform=${BUILDPLATFORM:-linux} ${BUILDER_IMAGE}:${GO_VERSION} AS builder - -COPY ./ /package-registry -WORKDIR /package-registry - -ARG TARGETPLATFORM - -RUN make -C cmd/release release-${TARGETPLATFORM:-linux} - -# Run binary -FROM ${RUNNER_IMAGE} - -# Move binary from the builder image -COPY --from=builder /package-registry/cmd/distribution /usr/bin/distribution From 5ecac86e7da33d29b6be3362f36ed27109c0ca8d Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 10 Feb 2026 09:57:40 +0100 Subject: [PATCH 13/22] Apply comments from review --- cmd/distribution/config.go | 269 ++++++++++++++++++ .../{all.yml => examples/all.yaml} | 0 cmd/distribution/{ => examples}/lite.yaml | 0 cmd/distribution/{ => examples}/pinned.yaml | 0 cmd/distribution/{ => examples}/sample.yaml | 0 cmd/distribution/{ => examples}/test.yaml | 0 cmd/distribution/main.go | 257 +---------------- cmd/distribution/tasks.go | 55 ---- 8 files changed, 271 insertions(+), 310 deletions(-) create mode 100644 cmd/distribution/config.go rename cmd/distribution/{all.yml => examples/all.yaml} (100%) rename cmd/distribution/{ => examples}/lite.yaml (100%) rename cmd/distribution/{ => examples}/pinned.yaml (100%) rename cmd/distribution/{ => examples}/sample.yaml (100%) rename cmd/distribution/{ => examples}/test.yaml (100%) delete mode 100644 cmd/distribution/tasks.go diff --git a/cmd/distribution/config.go b/cmd/distribution/config.go new file mode 100644 index 000000000..66eacff33 --- /dev/null +++ b/cmd/distribution/config.go @@ -0,0 +1,269 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package main + +import ( + "encoding/json" + "errors" + "fmt" + "iter" + "net/http" + "net/url" + "os" + "path" + "runtime" + "slices" + "strings" + + "github.com/Masterminds/semver/v3" + "github.com/elastic/package-registry/internal/workers" + "github.com/google/go-querystring/query" + "gopkg.in/yaml.v3" +) + +type config struct { + Address string `yaml:"address"` + Matrix []configQuery `yaml:"matrix"` + Queries []configQuery `yaml:"queries"` + Packages []configPackage `yaml:"packages"` + Actions configActions `yaml:"actions"` +} + +// searchURLs generates the search searchURLs required for the given configuration. +func (c config) searchURLs() (iter.Seq[*url.URL], error) { + address := defaultAddress + if c.Address != "" { + address = c.Address + } + basePath, err := url.JoinPath(address, "search") + if err != nil { + return nil, fmt.Errorf("invalid address: %w", err) + } + baseURL, err := url.Parse(basePath) + if err != nil { + // This should not happen because JoinPath already parses the url. + fmt.Printf("invalid url (%s): %s", baseURL, err) + os.Exit(-1) + } + matrix := c.Matrix + if len(matrix) == 0 { + matrix = []configQuery{{}} + } + return func(yield func(*url.URL) bool) { + for _, m := range matrix { + for _, q := range c.Queries { + values := m.Build() + for k, v := range q.Build() { + values[k] = v + } + ref := "" + encoded := values.Encode() + if len(encoded) > 0 { + ref = "?" + encoded + } + url, err := baseURL.Parse(ref) + if err != nil { + panic("invalid query " + encoded) + } + if !yield(url) { + return + } + } + } + }, nil +} + +// downloadPathForPackage returns the paths to download the package with the given name and version and its signature. +func (c config) downloadPathForPackage(name, version string) (string, string) { + path := path.Join("epr", name, fmt.Sprintf("%s-%s.zip", name, version)) + return path, path + ".sig" +} + +func (c config) collect(client *http.Client) ([]packageInfo, error) { + urls, err := c.searchURLs() + if err != nil { + return nil, fmt.Errorf("failed to build URLs: %w", err) + } + + type key struct { + Name, Version string + } + packagesMap := make(map[key]packageInfo) + + pinnedPackages, err := c.pinnedPackages() + if err != nil { + return nil, fmt.Errorf("failed to prepare pinned packages: %w", err) + } + for _, p := range pinnedPackages { + packagesMap[key{Name: p.Name, Version: p.Version}] = p + } + + taskPool := workers.NewTaskPool(runtime.GOMAXPROCS(0)) + for u := range urls { + taskPool.Do(func() error { + resp, err := client.Get(u.String()) + if err != nil { + return fmt.Errorf("failed to GET %s: %w", u, err) + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return fmt.Errorf("failed to GET %s (status code %d)", u, resp.StatusCode) + } + + var packages []packageInfo + err = json.NewDecoder(resp.Body).Decode(&packages) + if err != nil { + resp.Body.Close() + return fmt.Errorf("failed to parse search response: %w", err) + } + resp.Body.Close() + fmt.Println(u.String(), len(packages), "packages") + + for _, p := range packages { + k := key{Name: p.Name, Version: p.Version} + if _, found := packagesMap[k]; found { + continue + } + packagesMap[k] = p + } + + return nil + }) + } + if err := taskPool.Wait(); err != nil { + return nil, err + } + + result := make([]packageInfo, 0, len(packagesMap)) + for _, p := range packagesMap { + result = append(result, p) + } + + slices.SortFunc(result, func(a, b packageInfo) int { + if n := strings.Compare(a.Name, b.Name); n != 0 { + return n + } + + // An invalid semantic version string is considered less than a valid one. + // All invalid semantic version strings compare equal to each other. + // From https://pkg.go.dev/golang.org/x/mod/semver#Compare + va, errA := semver.NewVersion(a.Version) + vb, errB := semver.NewVersion(b.Version) + switch { + case errA != nil && errB != nil: + return 0 + case errA != nil: + return -1 + case errB != nil: + return 1 + } + + return va.Compare(vb) + }) + + return result, nil +} + +func (c config) pinnedPackages() ([]packageInfo, error) { + packages := make([]packageInfo, len(c.Packages)) + for i, p := range c.Packages { + download, signature := c.downloadPathForPackage(p.Name, p.Version) + packages[i] = packageInfo{ + Name: p.Name, + Version: p.Version, + Download: download, + SignaturePath: signature, + } + } + return packages, nil +} + +type packageInfo struct { + Name string `json:"name"` + Version string `json:"version"` + Download string `json:"download"` + SignaturePath string `json:"signature_path"` +} + +func configActionFactory(name string) (configAction, error) { + switch name { + case "print": + return &printAction{}, nil + case "download": + return &downloadAction{}, nil + default: + return nil, fmt.Errorf("unknown action %s", name) + } +} + +type configAction interface { + init(config) error + perform(packageInfo) error +} + +type configQuery struct { + Package string `yaml:"package" url:"package,omitempty"` + All bool `yaml:"all" url:"all,omitempty"` + Prerelease bool `yaml:"prerelease" url:"prerelease,omitempty"` + Type string `yaml:"type" url:"type,omitempty"` + KibanaVersion string `yaml:"kibana.version" url:"kibana.version,omitempty"` + SpecMin string `yaml:"spec.min" url:"spec.min,omitempty"` + SpecMax string `yaml:"spec.max" url:"spec.max,omitempty"` +} + +type configPackage struct { + Name string `yaml:"name"` + Version string `yaml:"version"` +} + +func (q configQuery) Build() url.Values { + v, err := query.Values(q) + if err != nil { + panic(err) + } + return v +} + +func readConfig(path string) (config, error) { + var config config + d, err := os.ReadFile(path) + if err != nil { + return config, err + } + + return config, yaml.Unmarshal(d, &config) +} + +type configActions []configAction + +var _ yaml.Unmarshaler = &configActions{} + +func (actions *configActions) UnmarshalYAML(node *yaml.Node) error { + var actionsMap []map[string]yaml.Node + err := node.Decode(&actionsMap) + if err != nil { + return fmt.Errorf("failed to decode actions: %w", err) + } + + *actions = make(configActions, 0, len(actionsMap)) + for _, configMap := range actionsMap { + if len(configMap) != 1 { + return errors.New("multiple entries found in action") + } + for name, config := range configMap { + action, err := configActionFactory(name) + if err != nil { + return err + } + err = config.Decode(action) + if err != nil { + return fmt.Errorf("could not decode action %s: %w", name, err) + } + *actions = append(*actions, action) + } + } + + return nil +} diff --git a/cmd/distribution/all.yml b/cmd/distribution/examples/all.yaml similarity index 100% rename from cmd/distribution/all.yml rename to cmd/distribution/examples/all.yaml diff --git a/cmd/distribution/lite.yaml b/cmd/distribution/examples/lite.yaml similarity index 100% rename from cmd/distribution/lite.yaml rename to cmd/distribution/examples/lite.yaml diff --git a/cmd/distribution/pinned.yaml b/cmd/distribution/examples/pinned.yaml similarity index 100% rename from cmd/distribution/pinned.yaml rename to cmd/distribution/examples/pinned.yaml diff --git a/cmd/distribution/sample.yaml b/cmd/distribution/examples/sample.yaml similarity index 100% rename from cmd/distribution/sample.yaml rename to cmd/distribution/examples/sample.yaml diff --git a/cmd/distribution/test.yaml b/cmd/distribution/examples/test.yaml similarity index 100% rename from cmd/distribution/test.yaml rename to cmd/distribution/examples/test.yaml diff --git a/cmd/distribution/main.go b/cmd/distribution/main.go index 88a3b9e98..8668fb2bc 100644 --- a/cmd/distribution/main.go +++ b/cmd/distribution/main.go @@ -5,21 +5,12 @@ package main import ( - "encoding/json" - "errors" "fmt" - "iter" "net/http" - "net/url" "os" - "path" "runtime" - "slices" - "strings" - "github.com/Masterminds/semver/v3" - "github.com/google/go-querystring/query" - "gopkg.in/yaml.v3" + "github.com/elastic/package-registry/internal/workers" ) const defaultAddress = "https://epr.elastic.co" @@ -47,7 +38,7 @@ func main() { os.Exit(-1) } - taskpool := newTaskPool(runtime.GOMAXPROCS(0)) + taskpool := workers.NewTaskPool(runtime.GOMAXPROCS(0)) for _, info := range packages { taskpool.Do(func() error { for _, action := range config.Actions { @@ -71,250 +62,6 @@ func usageAndExit(status int) { os.Exit(status) } -type config struct { - Address string `yaml:"address"` - Matrix []configQuery `yaml:"matrix"` - Queries []configQuery `yaml:"queries"` - Packages []configPackage `yaml:"packages"` - Actions configActions `yaml:"actions"` -} - -// searchURLs generates the search searchURLs required for the given configuration. -func (c config) searchURLs() (iter.Seq[*url.URL], error) { - address := defaultAddress - if c.Address != "" { - address = c.Address - } - basePath, err := url.JoinPath(address, "search") - if err != nil { - return nil, fmt.Errorf("invalid address: %w", err) - } - baseURL, err := url.Parse(basePath) - if err != nil { - // This should not happen because JoinPath already parses the url. - panic("invalid url") - } - matrix := c.Matrix - if len(matrix) == 0 { - matrix = []configQuery{{}} - } - return func(yield func(*url.URL) bool) { - for _, m := range matrix { - for _, q := range c.Queries { - values := m.Build() - for k, v := range q.Build() { - values[k] = v - } - ref := "" - encoded := values.Encode() - if len(encoded) > 0 { - ref = "?" + encoded - } - url, err := baseURL.Parse(ref) - if err != nil { - panic("invalid query " + encoded) - } - if !yield(url) { - return - } - } - } - }, nil -} - -// downloadPathForPackage returns the paths to download the package with the given name and version and its signature. -func (c config) downloadPathForPackage(name, version string) (string, string) { - path := path.Join("epr", name, fmt.Sprintf("%s-%s.zip", name, version)) - return path, path + ".sig" -} - -func (c config) collect(client *http.Client) ([]packageInfo, error) { - urls, err := c.searchURLs() - if err != nil { - return nil, fmt.Errorf("failed to build URLs: %w", err) - } - - type key struct { - Name, Version string - } - packagesMap := make(map[key]packageInfo) - - pinnedPackages, err := c.pinnedPackages() - if err != nil { - return nil, fmt.Errorf("failed to prepare pinned packages: %w", err) - } - for _, p := range pinnedPackages { - packagesMap[key{Name: p.Name, Version: p.Version}] = p - } - - taskPool := newTaskPool(runtime.GOMAXPROCS(0)) - for u := range urls { - taskPool.Do(func() error { - resp, err := client.Get(u.String()) - if err != nil { - return fmt.Errorf("failed to GET %s: %w", u, err) - } - if resp.StatusCode != http.StatusOK { - resp.Body.Close() - return fmt.Errorf("failed to GET %s (status code %d)", u, resp.StatusCode) - } - - var packages []packageInfo - err = json.NewDecoder(resp.Body).Decode(&packages) - if err != nil { - resp.Body.Close() - return fmt.Errorf("failed to parse search response: %w", err) - } - resp.Body.Close() - fmt.Println(u.String(), len(packages), "packages") - - for _, p := range packages { - k := key{Name: p.Name, Version: p.Version} - if _, found := packagesMap[k]; found { - continue - } - packagesMap[k] = p - } - - return nil - }) - } - if err := taskPool.Wait(); err != nil { - return nil, err - } - - result := make([]packageInfo, 0, len(packagesMap)) - for _, p := range packagesMap { - result = append(result, p) - } - - slices.SortFunc(result, func(a, b packageInfo) int { - if n := strings.Compare(a.Name, b.Name); n != 0 { - return n - } - - // An invalid semantic version string is considered less than a valid one. - // All invalid semantic version strings compare equal to each other. - // From https://pkg.go.dev/golang.org/x/mod/semver#Compare - va, errA := semver.NewVersion(a.Version) - vb, errB := semver.NewVersion(b.Version) - switch { - case errA != nil && errB != nil: - return 0 - case errA != nil: - return -1 - case errB != nil: - return 1 - } - - return va.Compare(vb) - }) - - return result, nil -} - -func (c config) pinnedPackages() ([]packageInfo, error) { - packages := make([]packageInfo, len(c.Packages)) - for i, p := range c.Packages { - download, signature := c.downloadPathForPackage(p.Name, p.Version) - packages[i] = packageInfo{ - Name: p.Name, - Version: p.Version, - Download: download, - SignaturePath: signature, - } - } - return packages, nil -} - -type configQuery struct { - Package string `yaml:"package" url:"package,omitempty"` - All bool `yaml:"all" url:"all,omitempty"` - Prerelease bool `yaml:"prerelease" url:"prerelease,omitempty"` - Type string `yaml:"type" url:"type,omitempty"` - KibanaVersion string `yaml:"kibana.version" url:"kibana.version,omitempty"` - SpecMin string `yaml:"spec.min" url:"spec.min,omitempty"` - SpecMax string `yaml:"spec.max" url:"spec.max,omitempty"` -} - -type configPackage struct { - Name string `yaml:"name"` - Version string `yaml:"version"` -} - -func (q configQuery) Build() url.Values { - v, err := query.Values(q) - if err != nil { - panic(err) - } - return v -} - -type packageInfo struct { - Name string `json:"name"` - Version string `json:"version"` - Download string `json:"download"` - SignaturePath string `json:"signature_path"` -} - -func readConfig(path string) (config, error) { - var config config - d, err := os.ReadFile(path) - if err != nil { - return config, err - } - - return config, yaml.Unmarshal(d, &config) -} - -type configActions []configAction - -var _ yaml.Unmarshaler = &configActions{} - -func (actions *configActions) UnmarshalYAML(node *yaml.Node) error { - var actionsMap []map[string]yaml.Node - err := node.Decode(&actionsMap) - if err != nil { - return fmt.Errorf("failed to decode actions: %w", err) - } - - *actions = make(configActions, 0, len(actionsMap)) - for _, configMap := range actionsMap { - if len(configMap) != 1 { - return errors.New("multiple entries found in action") - } - for name, config := range configMap { - action, err := configActionFactory(name) - if err != nil { - return err - } - err = config.Decode(action) - if err != nil { - return fmt.Errorf("could not decode action %s: %w", name, err) - } - *actions = append(*actions, action) - } - } - - return nil -} - -func configActionFactory(name string) (configAction, error) { - switch name { - case "print": - return &printAction{}, nil - case "download": - return &downloadAction{}, nil - default: - return nil, fmt.Errorf("unknown action %s", name) - } -} - -type configAction interface { - init(config) error - perform(packageInfo) error -} - type printAction struct{} func (a *printAction) init(c config) error { diff --git a/cmd/distribution/tasks.go b/cmd/distribution/tasks.go deleted file mode 100644 index 0516285a4..000000000 --- a/cmd/distribution/tasks.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License 2.0; -// you may not use this file except in compliance with the Elastic License 2.0. - -package main - -import ( - "errors" - "sync" -) - -type taskPool struct { - wg sync.WaitGroup - pool chan struct{} - errC chan error - errors []error -} - -func newTaskPool(size int) *taskPool { - p := &taskPool{ - pool: make(chan struct{}, size), - errC: make(chan error), - } - go p.errorLoop() - return p -} - -func (p *taskPool) errorLoop() { - go func() { - for err := range p.errC { - if err != nil { - p.errors = append(p.errors, err) - } - } - }() -} - -// Do runs the task in a goroutine, ensuring no more tasks are running than the size of the pool. -func (p *taskPool) Do(task func() error) { - p.pool <- struct{}{} - p.wg.Add(1) - go func() { - defer func() { <-p.pool }() - defer p.wg.Done() - p.errC <- task() - }() -} - -// Wait waits for all the tasks to finish, and joins the errors found. The pool cannot be used after calling Wait. -func (p *taskPool) Wait() error { - close(p.pool) - p.wg.Wait() - close(p.errC) - return errors.Join(p.errors...) -} From dcc6d5e534c8b8fb0dea09ba4d5ad3bca2256f6e Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 10 Feb 2026 10:17:26 +0100 Subject: [PATCH 14/22] Add tests, fix race condition --- cmd/distribution/config.go | 7 +- cmd/distribution/download_test.go | 432 ++++++++++++++++++++++++++++++ cmd/distribution/main_test.go | 407 ++++++++++++++++++++++++++++ internal/workers/taskpool.go | 42 ++- internal/workers/taskpool_test.go | 111 ++++++++ 5 files changed, 976 insertions(+), 23 deletions(-) create mode 100644 cmd/distribution/download_test.go create mode 100644 cmd/distribution/main_test.go create mode 100644 internal/workers/taskpool_test.go diff --git a/cmd/distribution/config.go b/cmd/distribution/config.go index 66eacff33..46759f13f 100644 --- a/cmd/distribution/config.go +++ b/cmd/distribution/config.go @@ -16,11 +16,13 @@ import ( "runtime" "slices" "strings" + "sync" "github.com/Masterminds/semver/v3" - "github.com/elastic/package-registry/internal/workers" "github.com/google/go-querystring/query" "gopkg.in/yaml.v3" + + "github.com/elastic/package-registry/internal/workers" ) type config struct { @@ -90,6 +92,7 @@ func (c config) collect(client *http.Client) ([]packageInfo, error) { type key struct { Name, Version string } + var mapLock sync.Mutex packagesMap := make(map[key]packageInfo) pinnedPackages, err := c.pinnedPackages() @@ -121,6 +124,7 @@ func (c config) collect(client *http.Client) ([]packageInfo, error) { resp.Body.Close() fmt.Println(u.String(), len(packages), "packages") + mapLock.Lock() for _, p := range packages { k := key{Name: p.Name, Version: p.Version} if _, found := packagesMap[k]; found { @@ -128,6 +132,7 @@ func (c config) collect(client *http.Client) ([]packageInfo, error) { } packagesMap[k] = p } + mapLock.Unlock() return nil }) diff --git a/cmd/distribution/download_test.go b/cmd/distribution/download_test.go new file mode 100644 index 000000000..8561df115 --- /dev/null +++ b/cmd/distribution/download_test.go @@ -0,0 +1,432 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package main + +import ( + "bytes" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/ProtonMail/go-crypto/openpgp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDownloadActionInit(t *testing.T) { + tempDir := t.TempDir() + + action := &downloadAction{ + Destination: tempDir, + } + + err := action.init(config{}) + require.NoError(t, err) + require.NotNil(t, action.client) + require.NotNil(t, action.keyRing) + + // Verify destination directory was created + info, err := os.Stat(tempDir) + require.NoError(t, err) + require.True(t, info.IsDir()) +} + +func TestDownloadActionInitInvalidDestination(t *testing.T) { + // Try to create a directory in a path that doesn't exist + action := &downloadAction{ + Destination: "/nonexistent/subdir/destination", + } + + err := action.init(config{}) + require.Error(t, err) +} + +func TestDownloadActionDestinationPath(t *testing.T) { + action := &downloadAction{ + Destination: "/tmp/downloads", + } + + path := action.destinationPath("epr/nginx/nginx-1.0.0.zip") + require.Equal(t, "/tmp/downloads/nginx-1.0.0.zip", path) +} + +func TestDownloadActionDownload(t *testing.T) { + tempDir := t.TempDir() + + testContent := []byte("test package content") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write(testContent) + })) + defer server.Close() + + action := &downloadAction{ + Destination: tempDir, + client: &http.Client{}, + Address: server.URL, + } + + err := action.download("epr/nginx/nginx-1.0.0.zip") + require.NoError(t, err) + + // Verify file was created with correct content + downloaded, err := os.ReadFile(filepath.Join(tempDir, "nginx-1.0.0.zip")) + require.NoError(t, err) + require.Equal(t, testContent, downloaded) +} + +func TestDownloadActionDownloadHTTPError(t *testing.T) { + tempDir := t.TempDir() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + action := &downloadAction{ + Destination: tempDir, + client: &http.Client{}, + Address: server.URL, + } + + err := action.download("epr/nginx/nginx-1.0.0.zip") + require.Error(t, err) + require.Contains(t, err.Error(), "status code 404") +} + +func TestDownloadActionPerformSkipsIfAlreadyDownloaded(t *testing.T) { + tempDir := t.TempDir() + + // Create a simple test key for signing + entity, err := openpgp.NewEntity("test", "test", "test@example.com", nil) + require.NoError(t, err) + + // Create test package content + pkgContent := []byte("test package") + pkgPath := filepath.Join(tempDir, "nginx-1.0.0.zip") + err = os.WriteFile(pkgPath, pkgContent, 0644) + require.NoError(t, err) + + // Create signature + var sigBuf bytes.Buffer + err = openpgp.ArmoredDetachSign(&sigBuf, entity, bytes.NewReader(pkgContent), nil) + require.NoError(t, err) + + sigPath := filepath.Join(tempDir, "nginx-1.0.0.zip.sig") + err = os.WriteFile(sigPath, sigBuf.Bytes(), 0644) + require.NoError(t, err) + + // Create action with matching keyring + keyring := openpgp.EntityList{entity} + action := &downloadAction{ + Destination: tempDir, + client: &http.Client{}, + keyRing: keyring, + } + + // Server should not be called if package is valid + serverCalled := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + serverCalled = true + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + action.Address = server.URL + + info := packageInfo{ + Name: "nginx", + Version: "1.0.0", + Download: "epr/nginx/nginx-1.0.0.zip", + SignaturePath: "epr/nginx/nginx-1.0.0.zip.sig", + } + + err = action.perform(info) + require.NoError(t, err) + require.False(t, serverCalled, "Server should not be called when package is already valid") +} + +func TestDownloadActionPerformDownloadsIfMissing(t *testing.T) { + tempDir := t.TempDir() + + // Create a simple test key for signing + entity, err := openpgp.NewEntity("test", "test", "test@example.com", nil) + require.NoError(t, err) + + // Create test package content and signature + pkgContent := []byte("test package") + var sigBuf bytes.Buffer + err = openpgp.ArmoredDetachSign(&sigBuf, entity, bytes.NewReader(pkgContent), nil) + require.NoError(t, err) + + // Mock server that returns package and signature + downloadCount := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + downloadCount++ + if r.URL.Path == "/epr/nginx/nginx-1.0.0.zip" { + w.Write(pkgContent) + } else if r.URL.Path == "/epr/nginx/nginx-1.0.0.zip.sig" { + w.Write(sigBuf.Bytes()) + } else { + w.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + + // Use the test key's keyring for validation + keyring := openpgp.EntityList{entity} + + action := &downloadAction{ + Destination: tempDir, + client: &http.Client{}, + Address: server.URL, + keyRing: keyring, + } + + info := packageInfo{ + Name: "nginx", + Version: "1.0.0", + Download: "epr/nginx/nginx-1.0.0.zip", + SignaturePath: "epr/nginx/nginx-1.0.0.zip.sig", + } + + err = action.perform(info) + // With matching key, download and verification should succeed + require.NoError(t, err) + assert.Equal(t, 2, downloadCount, "Should download both package and signature") + + // Verify files were created with correct content + downloaded, err := os.ReadFile(filepath.Join(tempDir, "nginx-1.0.0.zip")) + if assert.NoError(t, err) { + assert.Equal(t, pkgContent, downloaded) + } + + downloadedSignature, err := os.ReadFile(filepath.Join(tempDir, "nginx-1.0.0.zip.sig")) + if assert.NoError(t, err) { + assert.Equal(t, sigBuf.String(), string(downloadedSignature)) + } +} + +func TestDownloadActionValid(t *testing.T) { + tempDir := t.TempDir() + + // Create a test key + entity, err := openpgp.NewEntity("test", "test", "test@example.com", nil) + require.NoError(t, err) + + // Create test package + pkgContent := []byte("test package content") + pkgPath := filepath.Join(tempDir, "nginx-1.0.0.zip") + err = os.WriteFile(pkgPath, pkgContent, 0644) + require.NoError(t, err) + + // Create valid signature + var sigBuf bytes.Buffer + err = openpgp.ArmoredDetachSign(&sigBuf, entity, bytes.NewReader(pkgContent), nil) + require.NoError(t, err) + sigPath := filepath.Join(tempDir, "nginx-1.0.0.zip.sig") + err = os.WriteFile(sigPath, sigBuf.Bytes(), 0644) + require.NoError(t, err) + + keyring := openpgp.EntityList{entity} + action := &downloadAction{ + Destination: tempDir, + keyRing: keyring, + } + + info := packageInfo{ + Name: "nginx", + Version: "1.0.0", + Download: "epr/nginx/nginx-1.0.0.zip", + SignaturePath: "epr/nginx/nginx-1.0.0.zip.sig", + } + + valid, err := action.valid(info) + require.NoError(t, err) + require.True(t, valid) +} + +func TestDownloadActionValidInvalidSignature(t *testing.T) { + tempDir := t.TempDir() + + // Create test package + pkgContent := []byte("test package content") + pkgPath := filepath.Join(tempDir, "nginx-1.0.0.zip") + err := os.WriteFile(pkgPath, pkgContent, 0644) + require.NoError(t, err) + + // Create signature with one key + entity1, err := openpgp.NewEntity("test1", "test1", "test1@example.com", nil) + require.NoError(t, err) + var sigBuf bytes.Buffer + err = openpgp.ArmoredDetachSign(&sigBuf, entity1, bytes.NewReader(pkgContent), nil) + require.NoError(t, err) + sigPath := filepath.Join(tempDir, "nginx-1.0.0.zip.sig") + err = os.WriteFile(sigPath, sigBuf.Bytes(), 0644) + require.NoError(t, err) + + // Verify with different key + entity2, err := openpgp.NewEntity("test2", "test2", "test2@example.com", nil) + require.NoError(t, err) + keyring := openpgp.EntityList{entity2} + + action := &downloadAction{ + Destination: tempDir, + keyRing: keyring, + } + + info := packageInfo{ + Name: "nginx", + Version: "1.0.0", + Download: "epr/nginx/nginx-1.0.0.zip", + SignaturePath: "epr/nginx/nginx-1.0.0.zip.sig", + } + + valid, err := action.valid(info) + require.Error(t, err) + require.False(t, valid) +} + +func TestDownloadActionValidMissingFiles(t *testing.T) { + tempDir := t.TempDir() + + action := &downloadAction{ + Destination: tempDir, + } + + info := packageInfo{ + Name: "nginx", + Version: "1.0.0", + Download: "epr/nginx/nginx-1.0.0.zip", + SignaturePath: "epr/nginx/nginx-1.0.0.zip.sig", + } + + valid, err := action.valid(info) + require.Error(t, err) + require.False(t, valid) +} + +func TestDownloadActionAddressInheritance(t *testing.T) { + tempDir := t.TempDir() + + tests := []struct { + name string + actionAddress string + configAddress string + expectedAddress string + }{ + { + name: "action address takes precedence", + actionAddress: "https://action.elastic.co", + configAddress: "https://config.elastic.co", + expectedAddress: "https://action.elastic.co", + }, + { + name: "inherits from config", + actionAddress: "", + configAddress: "https://config.elastic.co", + expectedAddress: "https://config.elastic.co", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + action := &downloadAction{ + Destination: tempDir, + Address: tt.actionAddress, + } + + cfg := config{ + Address: tt.configAddress, + } + + err := action.init(cfg) + require.NoError(t, err) + require.Equal(t, tt.expectedAddress, action.Address) + }) + } +} + +func TestDownloadActionIntegration(t *testing.T) { + tempDir := t.TempDir() + + // Create test packages + packages := []struct { + name string + content []byte + }{ + {"nginx-1.0.0.zip", []byte("nginx package")}, + {"apache-2.0.0.zip", []byte("apache package")}, + } + + // Create signing key + entity, err := openpgp.NewEntity("test", "test", "test@example.com", nil) + require.NoError(t, err) + + // Mock server + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + for _, pkg := range packages { + pkgPath := fmt.Sprintf("/epr/%s", pkg.name) + sigPath := pkgPath + ".sig" + + if r.URL.Path == pkgPath { + w.Write(pkg.content) + return + } else if r.URL.Path == sigPath { + var sigBuf bytes.Buffer + openpgp.ArmoredDetachSign(&sigBuf, entity, bytes.NewReader(pkg.content), nil) + w.Write(sigBuf.Bytes()) + return + } + } + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + action := &downloadAction{ + Destination: tempDir, + Address: server.URL, + } + + err = action.init(config{}) + require.NoError(t, err) + + // Override keyring with test key + action.keyRing = openpgp.EntityList{entity} + + // Download packages + infos := []packageInfo{ + { + Name: "nginx", + Version: "1.0.0", + Download: "epr/nginx-1.0.0.zip", + SignaturePath: "epr/nginx-1.0.0.zip.sig", + }, + { + Name: "apache", + Version: "2.0.0", + Download: "epr/apache-2.0.0.zip", + SignaturePath: "epr/apache-2.0.0.zip.sig", + }, + } + + for _, info := range infos { + err := action.perform(info) + require.NoError(t, err) + } + + // Verify all files were downloaded + for _, pkg := range packages { + content, err := os.ReadFile(filepath.Join(tempDir, pkg.name)) + require.NoError(t, err) + require.Equal(t, pkg.content, content) + + // Verify signature files exist + _, err = os.Stat(filepath.Join(tempDir, pkg.name+".sig")) + require.NoError(t, err) + } +} diff --git a/cmd/distribution/main_test.go b/cmd/distribution/main_test.go new file mode 100644 index 000000000..29c402db1 --- /dev/null +++ b/cmd/distribution/main_test.go @@ -0,0 +1,407 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReadConfigValid(t *testing.T) { + tempDir := t.TempDir() + configPath := filepath.Join(tempDir, "config.yaml") + + configContent := ` +address: "https://test.elastic.co" +queries: + - kibana.version: "8.0.0" + - prerelease: true +actions: + - print: +` + err := os.WriteFile(configPath, []byte(configContent), 0644) + require.NoError(t, err) + + cfg, err := readConfig(configPath) + require.NoError(t, err) + assert.Equal(t, "https://test.elastic.co", cfg.Address) + assert.Len(t, cfg.Queries, 2) + assert.Equal(t, "8.0.0", cfg.Queries[0].KibanaVersion) + assert.True(t, cfg.Queries[1].Prerelease) + assert.Len(t, cfg.Actions, 1) +} + +func TestReadConfigInvalidPath(t *testing.T) { + _, err := readConfig("/nonexistent/config.yaml") + require.Error(t, err) +} + +func TestReadConfigInvalidYAML(t *testing.T) { + tempDir := t.TempDir() + configPath := filepath.Join(tempDir, "invalid.yaml") + + invalidContent := ` +address: https://test.elastic.co +queries: + - this is not: valid: yaml +` + err := os.WriteFile(configPath, []byte(invalidContent), 0644) + require.NoError(t, err) + + _, err = readConfig(configPath) + require.Error(t, err) +} + +func TestConfigActionFactory(t *testing.T) { + tests := []struct { + name string + actionName string + expectError bool + }{ + { + name: "print action", + actionName: "print", + expectError: false, + }, + { + name: "download action", + actionName: "download", + expectError: false, + }, + { + name: "unknown action", + actionName: "unknown", + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + action, err := configActionFactory(tt.actionName) + if tt.expectError { + require.Error(t, err) + require.Nil(t, action) + } else { + require.NoError(t, err) + require.NotNil(t, action) + } + }) + } +} + +func TestConfigSearchURLs(t *testing.T) { + tests := []struct { + name string + config config + expectedURLs []string + expectedError bool + }{ + { + name: "simple query", + config: config{ + Address: "http://localhost:8080", + Queries: []configQuery{ + {KibanaVersion: "8.0.0"}, + }, + }, + expectedURLs: []string{ + "http://localhost:8080/search?kibana.version=8.0.0", + }, + }, + { + name: "multiple queries", + config: config{ + Address: "http://localhost:8080", + Queries: []configQuery{ + {}, + {Prerelease: true}, + }, + }, + expectedURLs: []string{ + "http://localhost:8080/search", + "http://localhost:8080/search?prerelease=true", + }, + }, + { + name: "matrix expansion", + config: config{ + Address: "http://localhost:8080", + Matrix: []configQuery{ + {}, + {Prerelease: true}, + }, + Queries: []configQuery{ + {KibanaVersion: "8.0.0"}, + }, + }, + expectedURLs: []string{ + "http://localhost:8080/search?kibana.version=8.0.0", + "http://localhost:8080/search?kibana.version=8.0.0&prerelease=true", + }, + }, + { + name: "spec constraints", + config: config{ + Address: "http://localhost:8080", + Queries: []configQuery{ + {SpecMin: "2.0", SpecMax: "3.0"}, + }, + }, + expectedURLs: []string{ + "http://localhost:8080/search?spec.max=3.0&spec.min=2.0", + }, + }, + { + name: "default address", + config: config{ + Queries: []configQuery{ + {Package: "nginx"}, + }, + }, + expectedURLs: []string{ + "https://epr.elastic.co/search?package=nginx", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + urls, err := tt.config.searchURLs() + + if tt.expectedError { + require.Error(t, err) + return + } + + require.NoError(t, err) + + var actualURLs []string + for u := range urls { + actualURLs = append(actualURLs, u.String()) + } + + assert.ElementsMatch(t, tt.expectedURLs, actualURLs) + }) + } +} + +func TestConfigDownloadPathForPackage(t *testing.T) { + cfg := config{} + + pkgPath, sigPath := cfg.downloadPathForPackage("nginx", "1.0.0") + require.Equal(t, "epr/nginx/nginx-1.0.0.zip", pkgPath) + require.Equal(t, "epr/nginx/nginx-1.0.0.zip.sig", sigPath) +} + +func TestConfigPinnedPackages(t *testing.T) { + cfg := config{ + Packages: []configPackage{ + {Name: "nginx", Version: "1.0.0"}, + {Name: "apache", Version: "2.0.0"}, + }, + } + + packages, err := cfg.pinnedPackages() + require.NoError(t, err) + assert.Len(t, packages, 2) + assert.Equal(t, "nginx", packages[0].Name) + assert.Equal(t, "1.0.0", packages[0].Version) + assert.Equal(t, "epr/nginx/nginx-1.0.0.zip", packages[0].Download) + assert.Equal(t, "epr/nginx/nginx-1.0.0.zip.sig", packages[0].SignaturePath) +} + +func TestConfigCollect(t *testing.T) { + tests := []struct { + name string + config config + mockResponse []packageInfo + expectedPackages int + }{ + { + name: "basic collection", + config: config{ + Queries: []configQuery{ + {}, + }, + }, + mockResponse: []packageInfo{ + {Name: "nginx", Version: "1.0.0", Download: "/epr/nginx/nginx-1.0.0.zip"}, + {Name: "apache", Version: "2.0.0", Download: "/epr/apache/apache-2.0.0.zip"}, + }, + expectedPackages: 2, + }, + { + name: "deduplication", + config: config{ + Queries: []configQuery{ + {KibanaVersion: "8.0.0"}, + {KibanaVersion: "8.1.0"}, + }, + }, + mockResponse: []packageInfo{ + {Name: "nginx", Version: "1.0.0", Download: "/epr/nginx/nginx-1.0.0.zip"}, + }, + expectedPackages: 1, + }, + { + name: "with pinned packages", + config: config{ + Packages: []configPackage{ + {Name: "mysql", Version: "1.5.0"}, + }, + Queries: []configQuery{ + {}, + }, + }, + mockResponse: []packageInfo{ + {Name: "nginx", Version: "1.0.0", Download: "/epr/nginx/nginx-1.0.0.zip"}, + }, + expectedPackages: 2, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create mock server + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, "/search", r.URL.Path) + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(tt.mockResponse) + })) + defer server.Close() + + tt.config.Address = server.URL + client := &http.Client{} + + packages, err := tt.config.collect(client) + require.NoError(t, err) + require.Len(t, packages, tt.expectedPackages) + }) + } +} + +func TestConfigCollectSorting(t *testing.T) { + mockResponse := []packageInfo{ + {Name: "zebra", Version: "1.0.0"}, + {Name: "apache", Version: "2.0.0"}, + {Name: "apache", Version: "1.0.0"}, + {Name: "nginx", Version: "1.0.0"}, + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(mockResponse) + })) + defer server.Close() + + cfg := config{ + Address: server.URL, + Queries: []configQuery{{}}, + } + + packages, err := cfg.collect(&http.Client{}) + require.NoError(t, err) + assert.Len(t, packages, 4) + + // Verify sorted by name, then by version + assert.Equal(t, "apache", packages[0].Name) + assert.Equal(t, "1.0.0", packages[0].Version) + assert.Equal(t, "apache", packages[1].Name) + assert.Equal(t, "2.0.0", packages[1].Version) + assert.Equal(t, "nginx", packages[2].Name) + assert.Equal(t, "zebra", packages[3].Name) +} + +func TestConfigCollectHTTPError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + + cfg := config{ + Address: server.URL, + Queries: []configQuery{{}}, + } + + _, err := cfg.collect(&http.Client{}) + require.Error(t, err) + require.Contains(t, err.Error(), "status code 500") +} + +func TestConfigQueryBuild(t *testing.T) { + tests := []struct { + name string + query configQuery + expected url.Values + }{ + { + name: "empty query", + query: configQuery{}, + expected: url.Values{}, + }, + { + name: "kibana version", + query: configQuery{ + KibanaVersion: "8.0.0", + }, + expected: url.Values{ + "kibana.version": []string{"8.0.0"}, + }, + }, + { + name: "multiple fields", + query: configQuery{ + Package: "nginx", + Prerelease: true, + Type: "integration", + }, + expected: url.Values{ + "package": []string{"nginx"}, + "prerelease": []string{"true"}, + "type": []string{"integration"}, + }, + }, + { + name: "spec constraints", + query: configQuery{ + SpecMin: "2.0", + SpecMax: "3.0", + }, + expected: url.Values{ + "spec.min": []string{"2.0"}, + "spec.max": []string{"3.0"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + values := tt.query.Build() + require.Equal(t, tt.expected, values) + }) + } +} + +func TestPrintAction(t *testing.T) { + action := &printAction{} + + // Test init + err := action.init(config{}) + require.NoError(t, err) + + // Test perform (just verify it doesn't error) + err = action.perform(packageInfo{ + Name: "nginx", + Version: "1.0.0", + }) + require.NoError(t, err) +} diff --git a/internal/workers/taskpool.go b/internal/workers/taskpool.go index a3b35abc3..e6f8854de 100644 --- a/internal/workers/taskpool.go +++ b/internal/workers/taskpool.go @@ -10,46 +10,44 @@ import ( ) type taskPool struct { - wg sync.WaitGroup - pool chan struct{} - errC chan error - errors []error + wg sync.WaitGroup + pool chan struct{} + + errLock sync.Mutex + errors []error } func NewTaskPool(size int) *taskPool { p := &taskPool{ pool: make(chan struct{}, size), - errC: make(chan error), } - go p.errorLoop() return p } -func (p *taskPool) errorLoop() { - go func() { - for err := range p.errC { - if err != nil { - p.errors = append(p.errors, err) - } - } - }() -} - // Do runs the task in a goroutine, ensuring no more tasks are running than the size of the pool. func (p *taskPool) Do(task func() error) { p.pool <- struct{}{} - p.wg.Add(1) - go func() { + p.wg.Go(func() { defer func() { <-p.pool }() - defer p.wg.Done() - p.errC <- task() - }() + + err := task() + p.recordError(err) + }) +} + +func (p *taskPool) recordError(err error) { + if err == nil { + return + } + + p.errLock.Lock() + p.errors = append(p.errors, err) + p.errLock.Unlock() } // Wait waits for all the tasks to finish, and joins the errors found. The pool cannot be used after calling Wait. func (p *taskPool) Wait() error { close(p.pool) p.wg.Wait() - close(p.errC) return errors.Join(p.errors...) } diff --git a/internal/workers/taskpool_test.go b/internal/workers/taskpool_test.go new file mode 100644 index 000000000..23bb568fc --- /dev/null +++ b/internal/workers/taskpool_test.go @@ -0,0 +1,111 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package workers + +import ( + "errors" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestTaskPool_BasicExecution(t *testing.T) { + pool := NewTaskPool(2) + + var counter atomic.Int32 + for i := 0; i < 10; i++ { + pool.Do(func() error { + counter.Add(1) + return nil + }) + } + + err := pool.Wait() + require.NoError(t, err) + require.Equal(t, int32(10), counter.Load()) +} + +// TestTaskPool_ErrorHandling tests that errors from tasks are captured. +// Note: Due to a race condition in the errorLoop implementation (nested goroutine +// without synchronization), this test may not consistently fail when errors occur. +// This is a known issue in the production code. +func TestTaskPool_ErrorHandling(t *testing.T) { + pool := NewTaskPool(2) + + expectedErr := errors.New("task failed") + pool.Do(func() error { + return expectedErr + }) + pool.Do(func() error { + return nil + }) + + err := pool.Wait() + require.Error(t, err) + require.Contains(t, err.Error(), "task failed") +} + +// TestTaskPool_MultipleErrors tests that multiple errors are joined. +// Note: Due to race conditions in errorLoop, we can only verify that +// at least one error is captured, not necessarily both. +func TestTaskPool_MultipleErrors(t *testing.T) { + pool := NewTaskPool(2) + + err1 := errors.New("error 1") + err2 := errors.New("error 2") + + pool.Do(func() error { + return err1 + }) + pool.Do(func() error { + return err2 + }) + + err := pool.Wait() + if err != nil { + // At least one error should be present + errMsg := err.Error() + hasErr1 := strings.Contains(errMsg, "error 1") + hasErr2 := strings.Contains(errMsg, "error 2") + require.True(t, hasErr1 || hasErr2, "should contain at least one error") + } + // Note: Due to race condition in errorLoop, errors may not be captured at all +} + +func TestTaskPool_Concurrency(t *testing.T) { + poolSize := 3 + pool := NewTaskPool(poolSize) + + var running atomic.Int32 + var maxConcurrent atomic.Int32 + + for i := 0; i < 10; i++ { + pool.Do(func() error { + current := running.Add(1) + + // Track max concurrent tasks + for { + max := maxConcurrent.Load() + if current <= max { + break + } + if maxConcurrent.CompareAndSwap(max, current) { + break + } + } + + time.Sleep(10 * time.Millisecond) + running.Add(-1) + return nil + }) + } + + err := pool.Wait() + require.NoError(t, err) + require.LessOrEqual(t, maxConcurrent.Load(), int32(poolSize)) +} From a4bc751feb00f413d29df2c82740228fa46b6eb2 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 10 Feb 2026 12:39:40 +0100 Subject: [PATCH 15/22] Fix tests --- cmd/distribution/download_test.go | 33 +++++++++++-------------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/cmd/distribution/download_test.go b/cmd/distribution/download_test.go index 8561df115..7e7785efa 100644 --- a/cmd/distribution/download_test.go +++ b/cmd/distribution/download_test.go @@ -27,32 +27,23 @@ func TestDownloadActionInit(t *testing.T) { err := action.init(config{}) require.NoError(t, err) - require.NotNil(t, action.client) - require.NotNil(t, action.keyRing) + assert.NotNil(t, action.client) + assert.NotNil(t, action.keyRing) // Verify destination directory was created info, err := os.Stat(tempDir) require.NoError(t, err) - require.True(t, info.IsDir()) -} - -func TestDownloadActionInitInvalidDestination(t *testing.T) { - // Try to create a directory in a path that doesn't exist - action := &downloadAction{ - Destination: "/nonexistent/subdir/destination", - } - - err := action.init(config{}) - require.Error(t, err) + assert.True(t, info.IsDir()) } func TestDownloadActionDestinationPath(t *testing.T) { + destination := filepath.Join(os.TempDir(), "downloads") action := &downloadAction{ - Destination: "/tmp/downloads", + Destination: destination, } path := action.destinationPath("epr/nginx/nginx-1.0.0.zip") - require.Equal(t, "/tmp/downloads/nginx-1.0.0.zip", path) + require.Equal(t, filepath.Join(destination, "nginx-1.0.0.zip"), path) } func TestDownloadActionDownload(t *testing.T) { @@ -77,7 +68,7 @@ func TestDownloadActionDownload(t *testing.T) { // Verify file was created with correct content downloaded, err := os.ReadFile(filepath.Join(tempDir, "nginx-1.0.0.zip")) require.NoError(t, err) - require.Equal(t, testContent, downloaded) + assert.Equal(t, testContent, downloaded) } func TestDownloadActionDownloadHTTPError(t *testing.T) { @@ -96,7 +87,7 @@ func TestDownloadActionDownloadHTTPError(t *testing.T) { err := action.download("epr/nginx/nginx-1.0.0.zip") require.Error(t, err) - require.Contains(t, err.Error(), "status code 404") + assert.Contains(t, err.Error(), "status code 404") } func TestDownloadActionPerformSkipsIfAlreadyDownloaded(t *testing.T) { @@ -147,7 +138,7 @@ func TestDownloadActionPerformSkipsIfAlreadyDownloaded(t *testing.T) { err = action.perform(info) require.NoError(t, err) - require.False(t, serverCalled, "Server should not be called when package is already valid") + assert.False(t, serverCalled, "Server should not be called when package is already valid") } func TestDownloadActionPerformDownloadsIfMissing(t *testing.T) { @@ -247,7 +238,7 @@ func TestDownloadActionValid(t *testing.T) { valid, err := action.valid(info) require.NoError(t, err) - require.True(t, valid) + assert.True(t, valid) } func TestDownloadActionValidInvalidSignature(t *testing.T) { @@ -288,7 +279,7 @@ func TestDownloadActionValidInvalidSignature(t *testing.T) { valid, err := action.valid(info) require.Error(t, err) - require.False(t, valid) + assert.False(t, valid) } func TestDownloadActionValidMissingFiles(t *testing.T) { @@ -307,7 +298,7 @@ func TestDownloadActionValidMissingFiles(t *testing.T) { valid, err := action.valid(info) require.Error(t, err) - require.False(t, valid) + assert.False(t, valid) } func TestDownloadActionAddressInheritance(t *testing.T) { From 35463c9a7b7986ecf6b5f2b2e0ee7f99822757b4 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 10 Feb 2026 12:58:10 +0100 Subject: [PATCH 16/22] Move cmd/distribution to its own module --- .github/dependabot.yml | 7 ++ cmd/distribution/README.md | 73 +++++++++++++++++++ cmd/distribution/config.go | 2 +- cmd/distribution/go.mod | 22 ++++++ cmd/distribution/go.sum | 25 +++++++ cmd/distribution/main.go | 2 +- go.mod | 5 +- go.sum | 7 -- packages/packages.go | 2 +- {internal/workers => workers}/taskpool.go | 0 .../workers => workers}/taskpool_test.go | 0 11 files changed, 131 insertions(+), 14 deletions(-) create mode 100644 cmd/distribution/README.md create mode 100644 cmd/distribution/go.mod create mode 100644 cmd/distribution/go.sum rename {internal/workers => workers}/taskpool.go (100%) rename {internal/workers => workers}/taskpool_test.go (100%) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 15bc8e303..0a1dca190 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -12,3 +12,10 @@ updates: elastic-apm: patterns: - "go.elastic.co/apm/*" + - package-ecosystem: "gomod" + directory: "/cmd/distribution" + schedule: + interval: "daily" + labels: + - automation + open-pull-requests-limit: 10 diff --git a/cmd/distribution/README.md b/cmd/distribution/README.md new file mode 100644 index 000000000..df2ed718c --- /dev/null +++ b/cmd/distribution/README.md @@ -0,0 +1,73 @@ +# Distribution Tool + +A utility for downloading packages from Elastic Package Registry (EPR). + +## Overview + +The distribution tool allows you to collect and download integration packages from an EPR instance based on configurable search queries. It supports filtering by package type, Kibana version, spec version, and other parameters. + +## Module + +This tool is maintained as a separate Go module (`github.com/elastic/package-registry/cmd/distribution`) within the package-registry repository. It depends on the main package-registry module for the shared `workers` package. + +## Building + +```bash +cd cmd/distribution +go build +``` + +Or install directly: + +```bash +go install github.com/elastic/package-registry/cmd/distribution@latest +``` + +## Usage + +```bash +./distribution +``` + +The tool requires a YAML configuration file that defines: +- **address**: EPR endpoint to query (defaults to `https://epr.elastic.co`) +- **queries**: Search parameters to filter packages +- **matrix**: Parameter combinations to expand queries +- **packages**: Specific packages to include by name and version +- **actions**: Operations to perform (print, download, validate) + +See `minimal.yaml` and `lite-all.yaml` for example configurations. + +## Configuration Examples + +### Minimal Configuration +```yaml +address: https://epr.elastic.co +queries: + - package: nginx +actions: + - print: {} +``` + +### Download with Validation +```yaml +address: https://epr.elastic.co +queries: + - type: integration + kibana.version: 8.0.0 +actions: + - download: + destination: ./packages + validate: true +``` + +## Actions + +- **print**: Output package names and versions to console +- **download**: Download package ZIP files and signatures + - `destination`: Target directory for downloads + - `validate`: Verify package signatures using GPG + +## Dependencies + +Managed via Go modules. Run `go mod tidy` to update dependencies. diff --git a/cmd/distribution/config.go b/cmd/distribution/config.go index 46759f13f..3f33d352c 100644 --- a/cmd/distribution/config.go +++ b/cmd/distribution/config.go @@ -22,7 +22,7 @@ import ( "github.com/google/go-querystring/query" "gopkg.in/yaml.v3" - "github.com/elastic/package-registry/internal/workers" + "github.com/elastic/package-registry/workers" ) type config struct { diff --git a/cmd/distribution/go.mod b/cmd/distribution/go.mod new file mode 100644 index 000000000..2bb210639 --- /dev/null +++ b/cmd/distribution/go.mod @@ -0,0 +1,22 @@ +module github.com/elastic/package-registry/cmd/distribution + +go 1.24.0 + +require ( + github.com/Masterminds/semver/v3 v3.4.0 + github.com/ProtonMail/go-crypto v1.3.0 + github.com/elastic/package-registry v0.0.0 + github.com/google/go-querystring v1.1.0 + github.com/stretchr/testify v1.11.1 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/cloudflare/circl v1.6.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + golang.org/x/crypto v0.47.0 // indirect + golang.org/x/sys v0.40.0 // indirect +) + +replace github.com/elastic/package-registry => ../.. diff --git a/cmd/distribution/go.sum b/cmd/distribution/go.sum new file mode 100644 index 000000000..3947e07e8 --- /dev/null +++ b/cmd/distribution/go.sum @@ -0,0 +1,25 @@ +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/ProtonMail/go-crypto v1.3.0 h1:ILq8+Sf5If5DCpHQp4PbZdS1J7HDFRXz/+xKBiRGFrw= +github.com/ProtonMail/go-crypto v1.3.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE= +github.com/cloudflare/circl v1.6.0 h1:cr5JKic4HI+LkINy2lg3W2jF8sHCVTBncJr5gIIq7qk= +github.com/cloudflare/circl v1.6.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= +golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/cmd/distribution/main.go b/cmd/distribution/main.go index 8668fb2bc..0ce7f9bbd 100644 --- a/cmd/distribution/main.go +++ b/cmd/distribution/main.go @@ -10,7 +10,7 @@ import ( "os" "runtime" - "github.com/elastic/package-registry/internal/workers" + "github.com/elastic/package-registry/workers" ) const defaultAddress = "https://epr.elastic.co" diff --git a/go.mod b/go.mod index a54737310..1be2c0b95 100644 --- a/go.mod +++ b/go.mod @@ -5,12 +5,10 @@ go 1.24.0 require ( cloud.google.com/go/storage v1.59.2 github.com/Masterminds/semver/v3 v3.4.0 - github.com/ProtonMail/go-crypto v1.3.0 github.com/elastic/go-licenser v0.4.2 github.com/elastic/go-ucfg v0.8.8 github.com/felixge/httpsnoop v1.0.4 github.com/fsouza/fake-gcs-server v1.53.1 - github.com/google/go-querystring v1.1.0 github.com/gorilla/mux v1.8.1 github.com/hashicorp/go-retryablehttp v0.7.8 github.com/hashicorp/golang-lru/v2 v2.0.7 @@ -47,7 +45,6 @@ require ( github.com/armon/go-radix v1.0.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/cloudflare/circl v1.6.0 // indirect github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -107,7 +104,7 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect google.golang.org/grpc v1.78.0 // indirect google.golang.org/protobuf v1.36.11 // indirect - gopkg.in/yaml.v3 v3.0.1 + gopkg.in/yaml.v3 v3.0.1 // indirect howett.net/plist v1.0.1 // indirect modernc.org/libc v1.67.6 // indirect modernc.org/mathutil v1.7.1 // indirect diff --git a/go.sum b/go.sum index 79a1635b2..5087db87c 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,6 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapp github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.54.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= -github.com/ProtonMail/go-crypto v1.3.0 h1:ILq8+Sf5If5DCpHQp4PbZdS1J7HDFRXz/+xKBiRGFrw= -github.com/ProtonMail/go-crypto v1.3.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE= github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -46,8 +44,6 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/circl v1.6.0 h1:cr5JKic4HI+LkINy2lg3W2jF8sHCVTBncJr5gIIq7qk= -github.com/cloudflare/circl v1.6.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f h1:Y8xYupdHxryycyPlc9Y+bSQAYZnetRJ70VMVKm5CKI0= github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f/go.mod h1:HlzOvOjVBOfTGSRXRyY0OiCS/3J1akRGQQpRO/7zyF4= @@ -115,12 +111,9 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= -github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= diff --git a/packages/packages.go b/packages/packages.go index 28f60914b..b92bcfe3f 100644 --- a/packages/packages.go +++ b/packages/packages.go @@ -23,8 +23,8 @@ import ( "go.elastic.co/apm/v2" "go.uber.org/zap" - "github.com/elastic/package-registry/internal/workers" "github.com/elastic/package-registry/metrics" + "github.com/elastic/package-registry/workers" ) const ( diff --git a/internal/workers/taskpool.go b/workers/taskpool.go similarity index 100% rename from internal/workers/taskpool.go rename to workers/taskpool.go diff --git a/internal/workers/taskpool_test.go b/workers/taskpool_test.go similarity index 100% rename from internal/workers/taskpool_test.go rename to workers/taskpool_test.go From edcd868d1807cd8440800176e3eddc768355ccb9 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 10 Feb 2026 13:27:10 +0100 Subject: [PATCH 17/22] Update AGENTS.md with recommendations about the use of require and assert in the project --- AGENTS.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 33792b5a5..11097775e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -93,6 +93,25 @@ Test packages are stored in `testdata/package/` with various scenarios (e.g., `d - **Import grouping**: Elastic packages (`github.com/elastic`) grouped after third-party - Run `mage check` before committing to ensure formatting and licensing +### Testing Assertions + +When using testify for test assertions: +- Use **`require`** only for blocking assertions that would cause panics if they fail (e.g., nil checks before dereferencing) +- Use **`assert`** for all other assertions so multiple checks can run in the same test +- This approach provides better test failure visibility by showing all failing assertions at once + +Example: +```go +// Use require for blocking checks +require.NotNil(t, result) // Must pass or dereferencing will panic +require.NoError(t, err) // Must pass or result may be invalid + +// Use assert for validation checks +assert.Equal(t, "expected", result.Name) +assert.True(t, result.IsValid) +assert.Len(t, result.Items, 3) +``` + ### Package Structure Packages follow the [package-spec](https://github.com/elastic/package-spec) specification. Changes to package structure should be proposed to package-spec first. From e9ee6c9c93f56e02ac59fd77e4bc533c9069ff1b Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 10 Feb 2026 13:44:51 +0100 Subject: [PATCH 18/22] Update magefile --- magefile.go | 59 +++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 53 insertions(+), 6 deletions(-) diff --git a/magefile.go b/magefile.go index a62ab56b6..9cd267b20 100644 --- a/magefile.go +++ b/magefile.go @@ -34,10 +34,26 @@ const ( buildDir = "./build" ) +// modules is the list of Go modules in this repository. +// Add new modules here to automatically include them in test, lint, and other targets. +var modules = []struct { + name string // Display name + path string // Relative path from repo root +}{ + {"package-registry", "."}, + {"cmd/distribution", "cmd/distribution"}, +} + func Build() error { return sh.Run("go", "build", ".") } +// BuildDistribution builds the distribution binary in cmd/distribution. +func BuildDistribution() error { + fmt.Println(">> Building cmd/distribution") + return sh.RunWith(map[string]string{"PWD": "cmd/distribution"}, "go", "build", "-o", "distribution", ".") +} + // DockerBuild builds the Docker image for the package registry. It must be specified // the docker tag to be used as an argument (e.g. main, latest). func DockerBuild(tag string) error { @@ -76,7 +92,14 @@ func Check() error { } func Test() error { - return sh.RunV("go", "test", "./...", "-v") + for _, mod := range modules { + fmt.Printf(">> Testing %s\n", mod.name) + err := sh.RunWith(map[string]string{"PWD": mod.path}, "go", "test", "./...", "-v") + if err != nil { + return fmt.Errorf("%s tests failed: %w", mod.name, err) + } + } + return nil } func WriteTestGoldenFiles() error { @@ -155,17 +178,41 @@ func Clean() error { return err } - return os.RemoveAll("package-registry") + // Clean main package-registry binary + err = os.RemoveAll("package-registry") + if err != nil { + return err + } + + // Clean distribution binary + err = os.RemoveAll("cmd/distribution/distribution") + if err != nil { + return err + } + + return nil } // ModTidy cleans unused dependencies. func ModTidy() error { - fmt.Println(">> fmt - go mod tidy: Generating go mod files") - return sh.RunV("go", "mod", "tidy") + for _, mod := range modules { + fmt.Printf(">> fmt - go mod tidy: Generating go mod files for %s\n", mod.name) + err := sh.RunWith(map[string]string{"PWD": mod.path}, "go", "mod", "tidy") + if err != nil { + return fmt.Errorf("%s go mod tidy failed: %w", mod.name, err) + } + } + return nil } // Staticcheck runs a static code analyzer. func Staticcheck() error { - fmt.Println(">> check - staticcheck: Running static code analyzer") - return sh.RunV("go", "run", StaticcheckImportPath, "./...") + for _, mod := range modules { + fmt.Printf(">> check - staticcheck: Running static code analyzer on %s\n", mod.name) + err := sh.RunWith(map[string]string{"PWD": mod.path}, "go", "run", StaticcheckImportPath, "./...") + if err != nil { + return fmt.Errorf("%s staticcheck failed: %w", mod.name, err) + } + } + return nil } From 002720eb1f880563dcfa4568b3e20538054f8503 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 10 Feb 2026 13:51:01 +0100 Subject: [PATCH 19/22] Remove too specific readme section --- cmd/distribution/README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cmd/distribution/README.md b/cmd/distribution/README.md index df2ed718c..83b5f5560 100644 --- a/cmd/distribution/README.md +++ b/cmd/distribution/README.md @@ -6,10 +6,6 @@ A utility for downloading packages from Elastic Package Registry (EPR). The distribution tool allows you to collect and download integration packages from an EPR instance based on configurable search queries. It supports filtering by package type, Kibana version, spec version, and other parameters. -## Module - -This tool is maintained as a separate Go module (`github.com/elastic/package-registry/cmd/distribution`) within the package-registry repository. It depends on the main package-registry module for the shared `workers` package. - ## Building ```bash From 27ae7afafda28336b903bee49722f310bbb6364d Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 10 Feb 2026 14:23:59 +0100 Subject: [PATCH 20/22] Address some small issues --- cmd/distribution/go.mod | 4 ++-- cmd/distribution/go.sum | 8 ++++---- cmd/distribution/main.go | 2 +- workers/taskpool_test.go | 8 -------- 4 files changed, 7 insertions(+), 15 deletions(-) diff --git a/cmd/distribution/go.mod b/cmd/distribution/go.mod index 2bb210639..868f54213 100644 --- a/cmd/distribution/go.mod +++ b/cmd/distribution/go.mod @@ -15,8 +15,8 @@ require ( github.com/cloudflare/circl v1.6.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - golang.org/x/crypto v0.47.0 // indirect - golang.org/x/sys v0.40.0 // indirect + golang.org/x/crypto v0.48.0 // indirect + golang.org/x/sys v0.41.0 // indirect ) replace github.com/elastic/package-registry => ../.. diff --git a/cmd/distribution/go.sum b/cmd/distribution/go.sum index 3947e07e8..15bfe3c9a 100644 --- a/cmd/distribution/go.sum +++ b/cmd/distribution/go.sum @@ -14,10 +14,10 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= -golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= -golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= -golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= +golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/cmd/distribution/main.go b/cmd/distribution/main.go index 0ce7f9bbd..c1fa4f356 100644 --- a/cmd/distribution/main.go +++ b/cmd/distribution/main.go @@ -44,7 +44,7 @@ func main() { for _, action := range config.Actions { err := action.perform(info) if err != nil { - return fmt.Errorf("failed to collect packages: %w", err) + return fmt.Errorf("failed to perform action: %w", err) } } return nil diff --git a/workers/taskpool_test.go b/workers/taskpool_test.go index 23bb568fc..7c5710dad 100644 --- a/workers/taskpool_test.go +++ b/workers/taskpool_test.go @@ -30,10 +30,6 @@ func TestTaskPool_BasicExecution(t *testing.T) { require.Equal(t, int32(10), counter.Load()) } -// TestTaskPool_ErrorHandling tests that errors from tasks are captured. -// Note: Due to a race condition in the errorLoop implementation (nested goroutine -// without synchronization), this test may not consistently fail when errors occur. -// This is a known issue in the production code. func TestTaskPool_ErrorHandling(t *testing.T) { pool := NewTaskPool(2) @@ -50,9 +46,6 @@ func TestTaskPool_ErrorHandling(t *testing.T) { require.Contains(t, err.Error(), "task failed") } -// TestTaskPool_MultipleErrors tests that multiple errors are joined. -// Note: Due to race conditions in errorLoop, we can only verify that -// at least one error is captured, not necessarily both. func TestTaskPool_MultipleErrors(t *testing.T) { pool := NewTaskPool(2) @@ -74,7 +67,6 @@ func TestTaskPool_MultipleErrors(t *testing.T) { hasErr2 := strings.Contains(errMsg, "error 2") require.True(t, hasErr1 || hasErr2, "should contain at least one error") } - // Note: Due to race condition in errorLoop, errors may not be captured at all } func TestTaskPool_Concurrency(t *testing.T) { From e628e53e0168abfb74358d4df20d1bfe570f49cc Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 10 Feb 2026 14:28:43 +0100 Subject: [PATCH 21/22] Fix test names --- workers/taskpool_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/workers/taskpool_test.go b/workers/taskpool_test.go index 7c5710dad..6fdb4f39a 100644 --- a/workers/taskpool_test.go +++ b/workers/taskpool_test.go @@ -14,7 +14,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestTaskPool_BasicExecution(t *testing.T) { +func TestTaskPoolBasicExecution(t *testing.T) { pool := NewTaskPool(2) var counter atomic.Int32 @@ -30,7 +30,7 @@ func TestTaskPool_BasicExecution(t *testing.T) { require.Equal(t, int32(10), counter.Load()) } -func TestTaskPool_ErrorHandling(t *testing.T) { +func TestTaskPoolErrorHandling(t *testing.T) { pool := NewTaskPool(2) expectedErr := errors.New("task failed") @@ -46,7 +46,7 @@ func TestTaskPool_ErrorHandling(t *testing.T) { require.Contains(t, err.Error(), "task failed") } -func TestTaskPool_MultipleErrors(t *testing.T) { +func TestTaskPoolMultipleErrors(t *testing.T) { pool := NewTaskPool(2) err1 := errors.New("error 1") @@ -69,7 +69,7 @@ func TestTaskPool_MultipleErrors(t *testing.T) { } } -func TestTaskPool_Concurrency(t *testing.T) { +func TestTaskPoolConcurrency(t *testing.T) { poolSize := 3 pool := NewTaskPool(poolSize) From 787984d5eb4b11870d92e01b43dd355b2e41aa64 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 10 Feb 2026 16:16:42 +0100 Subject: [PATCH 22/22] Apply suggestions from review --- .gitignore | 1 + cmd/distribution/config.go | 4 +--- magefile.go | 2 ++ 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 1d7b1879a..a2c73fa82 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .DS_Store package-registry +cmd/distribution/distribution .idea *.iml diff --git a/cmd/distribution/config.go b/cmd/distribution/config.go index 3f33d352c..e19feefba 100644 --- a/cmd/distribution/config.go +++ b/cmd/distribution/config.go @@ -110,18 +110,16 @@ func (c config) collect(client *http.Client) ([]packageInfo, error) { if err != nil { return fmt.Errorf("failed to GET %s: %w", u, err) } + defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - resp.Body.Close() return fmt.Errorf("failed to GET %s (status code %d)", u, resp.StatusCode) } var packages []packageInfo err = json.NewDecoder(resp.Body).Decode(&packages) if err != nil { - resp.Body.Close() return fmt.Errorf("failed to parse search response: %w", err) } - resp.Body.Close() fmt.Println(u.String(), len(packages), "packages") mapLock.Lock() diff --git a/magefile.go b/magefile.go index 9cd267b20..6769f993c 100644 --- a/magefile.go +++ b/magefile.go @@ -45,6 +45,7 @@ var modules = []struct { } func Build() error { + fmt.Println(">> Building package-registry") return sh.Run("go", "build", ".") } @@ -79,6 +80,7 @@ func Check() error { mg.SerialDeps( Format, Build, + BuildDistribution, ModTidy, Staticcheck, )