diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 15bc8e303..0a1dca190 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -12,3 +12,10 @@ updates: elastic-apm: patterns: - "go.elastic.co/apm/*" + - package-ecosystem: "gomod" + directory: "/cmd/distribution" + schedule: + interval: "daily" + labels: + - automation + open-pull-requests-limit: 10 diff --git a/.gitignore b/.gitignore index 1d7b1879a..a2c73fa82 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .DS_Store package-registry +cmd/distribution/distribution .idea *.iml diff --git a/AGENTS.md b/AGENTS.md index 33792b5a5..11097775e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -93,6 +93,25 @@ Test packages are stored in `testdata/package/` with various scenarios (e.g., `d - **Import grouping**: Elastic packages (`github.com/elastic`) grouped after third-party - Run `mage check` before committing to ensure formatting and licensing +### Testing Assertions + +When using testify for test assertions: +- Use **`require`** only for blocking assertions that would cause panics if they fail (e.g., nil checks before dereferencing) +- Use **`assert`** for all other assertions so multiple checks can run in the same test +- This approach provides better test failure visibility by showing all failing assertions at once + +Example: +```go +// Use require for blocking checks +require.NotNil(t, result) // Must pass or dereferencing will panic +require.NoError(t, err) // Must pass or result may be invalid + +// Use assert for validation checks +assert.Equal(t, "expected", result.Name) +assert.True(t, result.IsValid) +assert.Len(t, result.Items, 3) +``` + ### Package Structure Packages follow the [package-spec](https://github.com/elastic/package-spec) specification. Changes to package structure should be proposed to package-spec first. diff --git a/cmd/distribution/GPG-KEY-elasticsearch b/cmd/distribution/GPG-KEY-elasticsearch new file mode 100644 index 000000000..f9e4326dd --- /dev/null +++ b/cmd/distribution/GPG-KEY-elasticsearch @@ -0,0 +1,31 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQENBFI3HsoBCADXDtbNJnxbPqB1vDNtCsqhe49vFYsZN9IOZsZXgp7aHjh6CJBD +A+bGFOwyhbd7at35jQjWAw1O3cfYsKAmFy+Ar3LHCMkV3oZspJACTIgCrwnkic/9 +CUliQe324qvObU2QRtP4Fl0zWcfb/S8UYzWXWIFuJqMvE9MaRY1bwUBvzoqavLGZ +j3SF1SPO+TB5QrHkrQHBsmX+Jda6d4Ylt8/t6CvMwgQNlrlzIO9WT+YN6zS+sqHd +1YK/aY5qhoLNhp9G/HxhcSVCkLq8SStj1ZZ1S9juBPoXV1ZWNbxFNGwOh/NYGldD +2kmBf3YgCqeLzHahsAEpvAm8TBa7Q9W21C8vABEBAAG0RUVsYXN0aWNzZWFyY2gg +KEVsYXN0aWNzZWFyY2ggU2lnbmluZyBLZXkpIDxkZXZfb3BzQGVsYXN0aWNzZWFy +Y2gub3JnPokBTgQTAQgAOAIbAwIXgBYhBEYJWsyFSFgsGiaZqdJ9ZmzYjkK0BQJk +9vrZBQsJCAcDBRUKCQgLBRYCAwEAAh4FAAoJENJ9ZmzYjkK00hoH+wYXZKgVb3Wv +4AA/+T1IAf7edwgajr58bEyqds6/4v6uZBneUaqahUqMXgLFRX5dBSrAS7bvE/jx ++BBQx+rpFGxSwvFegRevE1zAGVtpgkFQX0RpRcKSmksucSBxikR/dPn9XdJSEVa8 +vPcs11V+2E5tq3LEP14zJL4MkJKQF0VJl5UUmKLS7U2F/IB5aXry9UWdMTnwNntX +kl2iDaViYF4MC6xTS24uLwND2St0Jvjt+xGEwbdBVvp+UZ/kG6IGkYM5eWGPuok/ +DHvjUdwTfyO9b5xGbqn5FJ3UFOwB/nOSFXHM8rsHRT/67gHcIl8YFqSQXpIkk9D3 +dCY+KieW0ue5AQ0EUjceygEIAOSVJc3DFuf3LsmUfGpUmnCqoUm76Eqqm8xynFEG +ZpczTChkwARRtckcfa/sGv376j+jk0c0Q71Uv3MnMLPGF+w3bpu8fLiPeW/cntf1 +8uZ6DxJvHA/oaZZ6VPjwUGSeVydiPtZfTYsceO8Dxl3gpS6nHZ9Gsnfr/kcH9/11 +Ca73HBtmGVIkOI1mZKMbANO8cewY/i7fPxShu7B0Rb3jxVNGUuiRcfRiao0gWx0U +ZGpvuHplt7loFX2cbsHFAp9WsjYEbSohb/Y0K4NkyFhL82MfbcsEwsXPhRTFgJWw +s4vpuFg/kFFlnw0NNPVP1jNJLNCsMBMEpP1A7k6MRpylNnUAEQEAAYkBNgQYAQgA +IAIbDBYhBEYJWsyFSFgsGiaZqdJ9ZmzYjkK0BQJk9vsHAAoJENJ9ZmzYjkK0hWsH +/ArKtn12HM3+41zYo9qO4rTri7+IYTjSB/JDTOusZgZLd/HCp1xQo4SI2Eur3Rtx +USMWK1LEeBzsjwDT9yVceYekrBEqUVyRMSVYj+UeZK2s4LbXm9b4jxXVtaivmkMA +jtznndrD7kmm8ak+UsZplf6p6uZS9TZ9hjwoMmw5oMaS6TZkLT4KYGWeyzHJSUBX +YikY6vssDQu4SJ07m1f4Hz81J39QOcHln5I5HTK8Rh/VUFcxNnGg9360g55wWpiF +eUTeMyoXpOtffiUhiOtbRYsmSYC0D4Fd5yJnO3n1pwnVVVsM7RAC22rc5j/Dw8dR +GIHikRcYWeXTYW7veewK5Ss= +=ftS0 +-----END PGP PUBLIC KEY BLOCK----- diff --git a/cmd/distribution/Makefile b/cmd/distribution/Makefile new file mode 100644 index 000000000..3924bfdc1 --- /dev/null +++ b/cmd/distribution/Makefile @@ -0,0 +1 @@ +include ../../Makefile diff --git a/cmd/distribution/README.md b/cmd/distribution/README.md new file mode 100644 index 000000000..83b5f5560 --- /dev/null +++ b/cmd/distribution/README.md @@ -0,0 +1,69 @@ +# Distribution Tool + +A utility for downloading packages from Elastic Package Registry (EPR). + +## Overview + +The distribution tool allows you to collect and download integration packages from an EPR instance based on configurable search queries. It supports filtering by package type, Kibana version, spec version, and other parameters. + +## Building + +```bash +cd cmd/distribution +go build +``` + +Or install directly: + +```bash +go install github.com/elastic/package-registry/cmd/distribution@latest +``` + +## Usage + +```bash +./distribution +``` + +The tool requires a YAML configuration file that defines: +- **address**: EPR endpoint to query (defaults to `https://epr.elastic.co`) +- **queries**: Search parameters to filter packages +- **matrix**: Parameter combinations to expand queries +- **packages**: Specific packages to include by name and version +- **actions**: Operations to perform (print, download, validate) + +See `minimal.yaml` and `lite-all.yaml` for example configurations. + +## Configuration Examples + +### Minimal Configuration +```yaml +address: https://epr.elastic.co +queries: + - package: nginx +actions: + - print: {} +``` + +### Download with Validation +```yaml +address: https://epr.elastic.co +queries: + - type: integration + kibana.version: 8.0.0 +actions: + - download: + destination: ./packages + validate: true +``` + +## Actions + +- **print**: Output package names and versions to console +- **download**: Download package ZIP files and signatures + - `destination`: Target directory for downloads + - `validate`: Verify package signatures using GPG + +## Dependencies + +Managed via Go modules. Run `go mod tidy` to update dependencies. diff --git a/cmd/distribution/config.go b/cmd/distribution/config.go new file mode 100644 index 000000000..e19feefba --- /dev/null +++ b/cmd/distribution/config.go @@ -0,0 +1,272 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package main + +import ( + "encoding/json" + "errors" + "fmt" + "iter" + "net/http" + "net/url" + "os" + "path" + "runtime" + "slices" + "strings" + "sync" + + "github.com/Masterminds/semver/v3" + "github.com/google/go-querystring/query" + "gopkg.in/yaml.v3" + + "github.com/elastic/package-registry/workers" +) + +type config struct { + Address string `yaml:"address"` + Matrix []configQuery `yaml:"matrix"` + Queries []configQuery `yaml:"queries"` + Packages []configPackage `yaml:"packages"` + Actions configActions `yaml:"actions"` +} + +// searchURLs generates the search searchURLs required for the given configuration. +func (c config) searchURLs() (iter.Seq[*url.URL], error) { + address := defaultAddress + if c.Address != "" { + address = c.Address + } + basePath, err := url.JoinPath(address, "search") + if err != nil { + return nil, fmt.Errorf("invalid address: %w", err) + } + baseURL, err := url.Parse(basePath) + if err != nil { + // This should not happen because JoinPath already parses the url. + fmt.Printf("invalid url (%s): %s", baseURL, err) + os.Exit(-1) + } + matrix := c.Matrix + if len(matrix) == 0 { + matrix = []configQuery{{}} + } + return func(yield func(*url.URL) bool) { + for _, m := range matrix { + for _, q := range c.Queries { + values := m.Build() + for k, v := range q.Build() { + values[k] = v + } + ref := "" + encoded := values.Encode() + if len(encoded) > 0 { + ref = "?" + encoded + } + url, err := baseURL.Parse(ref) + if err != nil { + panic("invalid query " + encoded) + } + if !yield(url) { + return + } + } + } + }, nil +} + +// downloadPathForPackage returns the paths to download the package with the given name and version and its signature. +func (c config) downloadPathForPackage(name, version string) (string, string) { + path := path.Join("epr", name, fmt.Sprintf("%s-%s.zip", name, version)) + return path, path + ".sig" +} + +func (c config) collect(client *http.Client) ([]packageInfo, error) { + urls, err := c.searchURLs() + if err != nil { + return nil, fmt.Errorf("failed to build URLs: %w", err) + } + + type key struct { + Name, Version string + } + var mapLock sync.Mutex + packagesMap := make(map[key]packageInfo) + + pinnedPackages, err := c.pinnedPackages() + if err != nil { + return nil, fmt.Errorf("failed to prepare pinned packages: %w", err) + } + for _, p := range pinnedPackages { + packagesMap[key{Name: p.Name, Version: p.Version}] = p + } + + taskPool := workers.NewTaskPool(runtime.GOMAXPROCS(0)) + for u := range urls { + taskPool.Do(func() error { + resp, err := client.Get(u.String()) + if err != nil { + return fmt.Errorf("failed to GET %s: %w", u, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to GET %s (status code %d)", u, resp.StatusCode) + } + + var packages []packageInfo + err = json.NewDecoder(resp.Body).Decode(&packages) + if err != nil { + return fmt.Errorf("failed to parse search response: %w", err) + } + fmt.Println(u.String(), len(packages), "packages") + + mapLock.Lock() + for _, p := range packages { + k := key{Name: p.Name, Version: p.Version} + if _, found := packagesMap[k]; found { + continue + } + packagesMap[k] = p + } + mapLock.Unlock() + + return nil + }) + } + if err := taskPool.Wait(); err != nil { + return nil, err + } + + result := make([]packageInfo, 0, len(packagesMap)) + for _, p := range packagesMap { + result = append(result, p) + } + + slices.SortFunc(result, func(a, b packageInfo) int { + if n := strings.Compare(a.Name, b.Name); n != 0 { + return n + } + + // An invalid semantic version string is considered less than a valid one. + // All invalid semantic version strings compare equal to each other. + // From https://pkg.go.dev/golang.org/x/mod/semver#Compare + va, errA := semver.NewVersion(a.Version) + vb, errB := semver.NewVersion(b.Version) + switch { + case errA != nil && errB != nil: + return 0 + case errA != nil: + return -1 + case errB != nil: + return 1 + } + + return va.Compare(vb) + }) + + return result, nil +} + +func (c config) pinnedPackages() ([]packageInfo, error) { + packages := make([]packageInfo, len(c.Packages)) + for i, p := range c.Packages { + download, signature := c.downloadPathForPackage(p.Name, p.Version) + packages[i] = packageInfo{ + Name: p.Name, + Version: p.Version, + Download: download, + SignaturePath: signature, + } + } + return packages, nil +} + +type packageInfo struct { + Name string `json:"name"` + Version string `json:"version"` + Download string `json:"download"` + SignaturePath string `json:"signature_path"` +} + +func configActionFactory(name string) (configAction, error) { + switch name { + case "print": + return &printAction{}, nil + case "download": + return &downloadAction{}, nil + default: + return nil, fmt.Errorf("unknown action %s", name) + } +} + +type configAction interface { + init(config) error + perform(packageInfo) error +} + +type configQuery struct { + Package string `yaml:"package" url:"package,omitempty"` + All bool `yaml:"all" url:"all,omitempty"` + Prerelease bool `yaml:"prerelease" url:"prerelease,omitempty"` + Type string `yaml:"type" url:"type,omitempty"` + KibanaVersion string `yaml:"kibana.version" url:"kibana.version,omitempty"` + SpecMin string `yaml:"spec.min" url:"spec.min,omitempty"` + SpecMax string `yaml:"spec.max" url:"spec.max,omitempty"` +} + +type configPackage struct { + Name string `yaml:"name"` + Version string `yaml:"version"` +} + +func (q configQuery) Build() url.Values { + v, err := query.Values(q) + if err != nil { + panic(err) + } + return v +} + +func readConfig(path string) (config, error) { + var config config + d, err := os.ReadFile(path) + if err != nil { + return config, err + } + + return config, yaml.Unmarshal(d, &config) +} + +type configActions []configAction + +var _ yaml.Unmarshaler = &configActions{} + +func (actions *configActions) UnmarshalYAML(node *yaml.Node) error { + var actionsMap []map[string]yaml.Node + err := node.Decode(&actionsMap) + if err != nil { + return fmt.Errorf("failed to decode actions: %w", err) + } + + *actions = make(configActions, 0, len(actionsMap)) + for _, configMap := range actionsMap { + if len(configMap) != 1 { + return errors.New("multiple entries found in action") + } + for name, config := range configMap { + action, err := configActionFactory(name) + if err != nil { + return err + } + err = config.Decode(action) + if err != nil { + return fmt.Errorf("could not decode action %s: %w", name, err) + } + *actions = append(*actions, action) + } + } + + return nil +} diff --git a/cmd/distribution/config_test.go b/cmd/distribution/config_test.go new file mode 100644 index 000000000..55065310e --- /dev/null +++ b/cmd/distribution/config_test.go @@ -0,0 +1,68 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package main + +import "fmt" + +func Example_config_urls() { + config := config{ + Address: "http://localhost:8080", + Queries: []configQuery{ + {}, + {Prerelease: true}, + {KibanaVersion: "7.19.0"}, + {KibanaVersion: "7.19.0", Prerelease: true}, + {SpecMin: "2.3", SpecMax: "3.0"}, + }, + } + + urls, err := config.searchURLs() + if err != nil { + panic(err) + } + + for u := range urls { + fmt.Println(u) + } + + // Output: + // http://localhost:8080/search + // http://localhost:8080/search?prerelease=true + // http://localhost:8080/search?kibana.version=7.19.0 + // http://localhost:8080/search?kibana.version=7.19.0&prerelease=true + // http://localhost:8080/search?spec.max=3.0&spec.min=2.3 +} + +func Example_config_urls_matrix() { + config := config{ + Address: "http://localhost:8080", + Matrix: []configQuery{ + {}, + {Prerelease: true}, + }, + Queries: []configQuery{ + {}, + {KibanaVersion: "7.19.0"}, + {SpecMin: "2.3", SpecMax: "3.0"}, + }, + } + + urls, err := config.searchURLs() + if err != nil { + panic(err) + } + + for u := range urls { + fmt.Println(u) + } + + // Output: + // http://localhost:8080/search + // http://localhost:8080/search?kibana.version=7.19.0 + // http://localhost:8080/search?spec.max=3.0&spec.min=2.3 + // http://localhost:8080/search?prerelease=true + // http://localhost:8080/search?kibana.version=7.19.0&prerelease=true + // http://localhost:8080/search?prerelease=true&spec.max=3.0&spec.min=2.3 +} diff --git a/cmd/distribution/download.go b/cmd/distribution/download.go new file mode 100644 index 000000000..22e4bdab8 --- /dev/null +++ b/cmd/distribution/download.go @@ -0,0 +1,110 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package main + +import ( + "bytes" + _ "embed" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + + "github.com/ProtonMail/go-crypto/openpgp" +) + +type downloadAction struct { + client *http.Client + keyRing openpgp.KeyRing + + Address string `yaml:"address"` + Destination string `yaml:"destination"` +} + +// publicKey is the public key of the key used to sign elastic artifacts. +// Downloaded from https://artifacts.elastic.co/GPG-KEY-elasticsearch +// +//go:embed GPG-KEY-elasticsearch +var publicKey []byte + +func (a *downloadAction) init(c config) error { + a.client = &http.Client{} + if a.Address == "" { + a.Address = c.Address + } + err := os.MkdirAll(a.Destination, 0755) + if err != nil { + return fmt.Errorf("failed to create desination directory: %w", err) + } + a.keyRing, err = openpgp.ReadArmoredKeyRing(bytes.NewReader(publicKey)) + if err != nil { + return fmt.Errorf("failed to initialize public key: %w", err) + } + return nil +} + +func (a *downloadAction) perform(i packageInfo) error { + if valid, _ := a.valid(i); valid { + return nil + } + if err := a.download(i.Download); err != nil { + return fmt.Errorf("failed to download package %s: %w", i.Download, err) + } + if err := a.download(i.SignaturePath); err != nil { + return fmt.Errorf("failed to download signature %s: %w", i.SignaturePath, err) + } + return nil +} + +func (a *downloadAction) download(urlPath string) error { + p, err := url.JoinPath(a.Address, urlPath) + if err != nil { + return fmt.Errorf("failed to build url: %w", err) + } + resp, err := a.client.Get(p) + if err != nil { + return fmt.Errorf("failed to get %s: %w", urlPath, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to get %s (status code %d)", urlPath, resp.StatusCode) + } + + f, err := os.Create(a.destinationPath(urlPath)) + if err != nil { + return fmt.Errorf("failed to open %s in %s: %w", path.Base(urlPath), a.Destination, err) + } + defer f.Close() + + _, err = io.Copy(f, resp.Body) + return err +} + +func (a *downloadAction) destinationPath(urlPath string) string { + return filepath.Join(a.Destination, path.Base(urlPath)) +} + +func (a *downloadAction) valid(info packageInfo) (bool, error) { + signed, err := os.Open(a.destinationPath(info.Download)) + if err != nil { + return false, err + } + defer signed.Close() + + signature, err := os.Open(a.destinationPath(info.SignaturePath)) + if err != nil { + return false, err + } + defer signature.Close() + + _, err = openpgp.CheckArmoredDetachedSignature(a.keyRing, signed, signature, nil) + if err != nil { + return false, err + } + return true, nil +} diff --git a/cmd/distribution/download_test.go b/cmd/distribution/download_test.go new file mode 100644 index 000000000..7e7785efa --- /dev/null +++ b/cmd/distribution/download_test.go @@ -0,0 +1,423 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package main + +import ( + "bytes" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/ProtonMail/go-crypto/openpgp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDownloadActionInit(t *testing.T) { + tempDir := t.TempDir() + + action := &downloadAction{ + Destination: tempDir, + } + + err := action.init(config{}) + require.NoError(t, err) + assert.NotNil(t, action.client) + assert.NotNil(t, action.keyRing) + + // Verify destination directory was created + info, err := os.Stat(tempDir) + require.NoError(t, err) + assert.True(t, info.IsDir()) +} + +func TestDownloadActionDestinationPath(t *testing.T) { + destination := filepath.Join(os.TempDir(), "downloads") + action := &downloadAction{ + Destination: destination, + } + + path := action.destinationPath("epr/nginx/nginx-1.0.0.zip") + require.Equal(t, filepath.Join(destination, "nginx-1.0.0.zip"), path) +} + +func TestDownloadActionDownload(t *testing.T) { + tempDir := t.TempDir() + + testContent := []byte("test package content") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write(testContent) + })) + defer server.Close() + + action := &downloadAction{ + Destination: tempDir, + client: &http.Client{}, + Address: server.URL, + } + + err := action.download("epr/nginx/nginx-1.0.0.zip") + require.NoError(t, err) + + // Verify file was created with correct content + downloaded, err := os.ReadFile(filepath.Join(tempDir, "nginx-1.0.0.zip")) + require.NoError(t, err) + assert.Equal(t, testContent, downloaded) +} + +func TestDownloadActionDownloadHTTPError(t *testing.T) { + tempDir := t.TempDir() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + action := &downloadAction{ + Destination: tempDir, + client: &http.Client{}, + Address: server.URL, + } + + err := action.download("epr/nginx/nginx-1.0.0.zip") + require.Error(t, err) + assert.Contains(t, err.Error(), "status code 404") +} + +func TestDownloadActionPerformSkipsIfAlreadyDownloaded(t *testing.T) { + tempDir := t.TempDir() + + // Create a simple test key for signing + entity, err := openpgp.NewEntity("test", "test", "test@example.com", nil) + require.NoError(t, err) + + // Create test package content + pkgContent := []byte("test package") + pkgPath := filepath.Join(tempDir, "nginx-1.0.0.zip") + err = os.WriteFile(pkgPath, pkgContent, 0644) + require.NoError(t, err) + + // Create signature + var sigBuf bytes.Buffer + err = openpgp.ArmoredDetachSign(&sigBuf, entity, bytes.NewReader(pkgContent), nil) + require.NoError(t, err) + + sigPath := filepath.Join(tempDir, "nginx-1.0.0.zip.sig") + err = os.WriteFile(sigPath, sigBuf.Bytes(), 0644) + require.NoError(t, err) + + // Create action with matching keyring + keyring := openpgp.EntityList{entity} + action := &downloadAction{ + Destination: tempDir, + client: &http.Client{}, + keyRing: keyring, + } + + // Server should not be called if package is valid + serverCalled := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + serverCalled = true + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + action.Address = server.URL + + info := packageInfo{ + Name: "nginx", + Version: "1.0.0", + Download: "epr/nginx/nginx-1.0.0.zip", + SignaturePath: "epr/nginx/nginx-1.0.0.zip.sig", + } + + err = action.perform(info) + require.NoError(t, err) + assert.False(t, serverCalled, "Server should not be called when package is already valid") +} + +func TestDownloadActionPerformDownloadsIfMissing(t *testing.T) { + tempDir := t.TempDir() + + // Create a simple test key for signing + entity, err := openpgp.NewEntity("test", "test", "test@example.com", nil) + require.NoError(t, err) + + // Create test package content and signature + pkgContent := []byte("test package") + var sigBuf bytes.Buffer + err = openpgp.ArmoredDetachSign(&sigBuf, entity, bytes.NewReader(pkgContent), nil) + require.NoError(t, err) + + // Mock server that returns package and signature + downloadCount := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + downloadCount++ + if r.URL.Path == "/epr/nginx/nginx-1.0.0.zip" { + w.Write(pkgContent) + } else if r.URL.Path == "/epr/nginx/nginx-1.0.0.zip.sig" { + w.Write(sigBuf.Bytes()) + } else { + w.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + + // Use the test key's keyring for validation + keyring := openpgp.EntityList{entity} + + action := &downloadAction{ + Destination: tempDir, + client: &http.Client{}, + Address: server.URL, + keyRing: keyring, + } + + info := packageInfo{ + Name: "nginx", + Version: "1.0.0", + Download: "epr/nginx/nginx-1.0.0.zip", + SignaturePath: "epr/nginx/nginx-1.0.0.zip.sig", + } + + err = action.perform(info) + // With matching key, download and verification should succeed + require.NoError(t, err) + assert.Equal(t, 2, downloadCount, "Should download both package and signature") + + // Verify files were created with correct content + downloaded, err := os.ReadFile(filepath.Join(tempDir, "nginx-1.0.0.zip")) + if assert.NoError(t, err) { + assert.Equal(t, pkgContent, downloaded) + } + + downloadedSignature, err := os.ReadFile(filepath.Join(tempDir, "nginx-1.0.0.zip.sig")) + if assert.NoError(t, err) { + assert.Equal(t, sigBuf.String(), string(downloadedSignature)) + } +} + +func TestDownloadActionValid(t *testing.T) { + tempDir := t.TempDir() + + // Create a test key + entity, err := openpgp.NewEntity("test", "test", "test@example.com", nil) + require.NoError(t, err) + + // Create test package + pkgContent := []byte("test package content") + pkgPath := filepath.Join(tempDir, "nginx-1.0.0.zip") + err = os.WriteFile(pkgPath, pkgContent, 0644) + require.NoError(t, err) + + // Create valid signature + var sigBuf bytes.Buffer + err = openpgp.ArmoredDetachSign(&sigBuf, entity, bytes.NewReader(pkgContent), nil) + require.NoError(t, err) + sigPath := filepath.Join(tempDir, "nginx-1.0.0.zip.sig") + err = os.WriteFile(sigPath, sigBuf.Bytes(), 0644) + require.NoError(t, err) + + keyring := openpgp.EntityList{entity} + action := &downloadAction{ + Destination: tempDir, + keyRing: keyring, + } + + info := packageInfo{ + Name: "nginx", + Version: "1.0.0", + Download: "epr/nginx/nginx-1.0.0.zip", + SignaturePath: "epr/nginx/nginx-1.0.0.zip.sig", + } + + valid, err := action.valid(info) + require.NoError(t, err) + assert.True(t, valid) +} + +func TestDownloadActionValidInvalidSignature(t *testing.T) { + tempDir := t.TempDir() + + // Create test package + pkgContent := []byte("test package content") + pkgPath := filepath.Join(tempDir, "nginx-1.0.0.zip") + err := os.WriteFile(pkgPath, pkgContent, 0644) + require.NoError(t, err) + + // Create signature with one key + entity1, err := openpgp.NewEntity("test1", "test1", "test1@example.com", nil) + require.NoError(t, err) + var sigBuf bytes.Buffer + err = openpgp.ArmoredDetachSign(&sigBuf, entity1, bytes.NewReader(pkgContent), nil) + require.NoError(t, err) + sigPath := filepath.Join(tempDir, "nginx-1.0.0.zip.sig") + err = os.WriteFile(sigPath, sigBuf.Bytes(), 0644) + require.NoError(t, err) + + // Verify with different key + entity2, err := openpgp.NewEntity("test2", "test2", "test2@example.com", nil) + require.NoError(t, err) + keyring := openpgp.EntityList{entity2} + + action := &downloadAction{ + Destination: tempDir, + keyRing: keyring, + } + + info := packageInfo{ + Name: "nginx", + Version: "1.0.0", + Download: "epr/nginx/nginx-1.0.0.zip", + SignaturePath: "epr/nginx/nginx-1.0.0.zip.sig", + } + + valid, err := action.valid(info) + require.Error(t, err) + assert.False(t, valid) +} + +func TestDownloadActionValidMissingFiles(t *testing.T) { + tempDir := t.TempDir() + + action := &downloadAction{ + Destination: tempDir, + } + + info := packageInfo{ + Name: "nginx", + Version: "1.0.0", + Download: "epr/nginx/nginx-1.0.0.zip", + SignaturePath: "epr/nginx/nginx-1.0.0.zip.sig", + } + + valid, err := action.valid(info) + require.Error(t, err) + assert.False(t, valid) +} + +func TestDownloadActionAddressInheritance(t *testing.T) { + tempDir := t.TempDir() + + tests := []struct { + name string + actionAddress string + configAddress string + expectedAddress string + }{ + { + name: "action address takes precedence", + actionAddress: "https://action.elastic.co", + configAddress: "https://config.elastic.co", + expectedAddress: "https://action.elastic.co", + }, + { + name: "inherits from config", + actionAddress: "", + configAddress: "https://config.elastic.co", + expectedAddress: "https://config.elastic.co", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + action := &downloadAction{ + Destination: tempDir, + Address: tt.actionAddress, + } + + cfg := config{ + Address: tt.configAddress, + } + + err := action.init(cfg) + require.NoError(t, err) + require.Equal(t, tt.expectedAddress, action.Address) + }) + } +} + +func TestDownloadActionIntegration(t *testing.T) { + tempDir := t.TempDir() + + // Create test packages + packages := []struct { + name string + content []byte + }{ + {"nginx-1.0.0.zip", []byte("nginx package")}, + {"apache-2.0.0.zip", []byte("apache package")}, + } + + // Create signing key + entity, err := openpgp.NewEntity("test", "test", "test@example.com", nil) + require.NoError(t, err) + + // Mock server + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + for _, pkg := range packages { + pkgPath := fmt.Sprintf("/epr/%s", pkg.name) + sigPath := pkgPath + ".sig" + + if r.URL.Path == pkgPath { + w.Write(pkg.content) + return + } else if r.URL.Path == sigPath { + var sigBuf bytes.Buffer + openpgp.ArmoredDetachSign(&sigBuf, entity, bytes.NewReader(pkg.content), nil) + w.Write(sigBuf.Bytes()) + return + } + } + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + action := &downloadAction{ + Destination: tempDir, + Address: server.URL, + } + + err = action.init(config{}) + require.NoError(t, err) + + // Override keyring with test key + action.keyRing = openpgp.EntityList{entity} + + // Download packages + infos := []packageInfo{ + { + Name: "nginx", + Version: "1.0.0", + Download: "epr/nginx-1.0.0.zip", + SignaturePath: "epr/nginx-1.0.0.zip.sig", + }, + { + Name: "apache", + Version: "2.0.0", + Download: "epr/apache-2.0.0.zip", + SignaturePath: "epr/apache-2.0.0.zip.sig", + }, + } + + for _, info := range infos { + err := action.perform(info) + require.NoError(t, err) + } + + // Verify all files were downloaded + for _, pkg := range packages { + content, err := os.ReadFile(filepath.Join(tempDir, pkg.name)) + require.NoError(t, err) + require.Equal(t, pkg.content, content) + + // Verify signature files exist + _, err = os.Stat(filepath.Join(tempDir, pkg.name+".sig")) + require.NoError(t, err) + } +} diff --git a/cmd/distribution/examples/all.yaml b/cmd/distribution/examples/all.yaml new file mode 100644 index 000000000..8a3f528a4 --- /dev/null +++ b/cmd/distribution/examples/all.yaml @@ -0,0 +1,6 @@ +# Base address of the Package Registry +address: "https://epr.elastic.co" + +queries: + - all: true + prerelease: true diff --git a/cmd/distribution/examples/lite.yaml b/cmd/distribution/examples/lite.yaml new file mode 100644 index 000000000..3d83d6de6 --- /dev/null +++ b/cmd/distribution/examples/lite.yaml @@ -0,0 +1,90 @@ +# Base address of the Package Registry +address: "https://epr.elastic.co" + +# Queries are executed with each one of the parameters of the matrix. +matrix: + - kibana.version: 7.17.26 + - kibana.version: 7.17.27 + - kibana.version: 7.17.28 + - kibana.version: 7.17.29 + - kibana.version: 8.15.4 + - kibana.version: 8.15.5 + - kibana.version: 8.15.6 + - kibana.version: 8.16.0 + spec.max: 3.3 + - kibana.version: 8.16.1 + spec.max: 3.3 + - kibana.version: 8.16.2 + spec.max: 3.3 + - kibana.version: 8.16.3 + spec.max: 3.3 + - kibana.version: 8.16.4 + spec.max: 3.3 + - kibana.version: 8.16.5 + spec.max: 3.3 + - kibana.version: 8.16.6 + spec.max: 3.3 + - kibana.version: 8.16.7 + spec.max: 3.3 + - kibana.version: 8.17.0 + spec.max: 3.3 + - kibana.version: 8.17.1 + spec.max: 3.3 + - kibana.version: 8.17.2 + spec.max: 3.3 + - kibana.version: 8.17.3 + spec.max: 3.3 + - kibana.version: 8.17.4 + spec.max: 3.3 + - kibana.version: 8.17.5 + spec.max: 3.3 + - kibana.version: 8.17.6 + spec.max: 3.3 + - kibana.version: 8.17.7 + spec.max: 3.3 + - kibana.version: 8.18.0 + spec.max: 3.3 + - kibana.version: 8.18.1 + spec.max: 3.3 + - kibana.version: 8.18.2 + spec.max: 3.3 + - kibana.version: 8.19.0 + spec.max: 3.4 + - kibana.version: 9.0.0 + spec.min: 2.3 + spec.max: 3.3 + - kibana.version: 9.0.1 + spec.min: 2.3 + spec.max: 3.3 + - kibana.version: 9.0.2 + spec.min: 2.3 + spec.max: 3.3 + - kibana.version: 9.1.0 + spec.min: 2.3 + spec.max: 3.4 + - spec.min: 2.3 + spec.max: 3.4 + +# Queries to execute to discover packages. +queries: + - package: apache + - package: apm + - package: elastic_agent + - package: endpoint + - package: fleet_server + - package: linux + - package: log + - package: mysql + - package: nginx + - package: profiler_collector + - package: profiler_symbolizer + - package: security_ai_prompts + - package: security_detection_engine + - package: synthetics + - package: system + - package: windows + +actions: + - print: + - download: + destination: ./build/distribution-lite diff --git a/cmd/distribution/examples/pinned.yaml b/cmd/distribution/examples/pinned.yaml new file mode 100644 index 000000000..5d1a3cbfd --- /dev/null +++ b/cmd/distribution/examples/pinned.yaml @@ -0,0 +1,16 @@ +# Base address of the Package Registry +address: "https://epr.elastic.co" + +queries: + - package: apache + all: true + spec.min: 3.0 + +packages: + - name: apache + version: 2.1.0 + +actions: + - print: + - download: + destination: ./build/pinned diff --git a/cmd/distribution/examples/sample.yaml b/cmd/distribution/examples/sample.yaml new file mode 100644 index 000000000..7905d0c30 --- /dev/null +++ b/cmd/distribution/examples/sample.yaml @@ -0,0 +1,76 @@ +# Base address of the Package Registry +address: "https://epr.elastic.co" + +# Queries are executed with each one of the parameters of the matrix. +matrix: + - spec.max: 3.0 + - prerelease: true + spec.max: 3.0 + +# Queries to execute to discover packages. +queries: + - kibana.version: 7.17.26 + - kibana.version: 7.17.27 + - kibana.version: 7.17.28 + - kibana.version: 7.17.29 + - kibana.version: 8.15.4 + - kibana.version: 8.15.5 + - kibana.version: 8.15.6 + - kibana.version: 8.16.0 + spec.max: 3.3 + - kibana.version: 8.16.1 + spec.max: 3.3 + - kibana.version: 8.16.2 + spec.max: 3.3 + - kibana.version: 8.16.3 + spec.max: 3.3 + - kibana.version: 8.16.4 + spec.max: 3.3 + - kibana.version: 8.16.5 + spec.max: 3.3 + - kibana.version: 8.16.6 + spec.max: 3.3 + - kibana.version: 8.16.7 + spec.max: 3.3 + - kibana.version: 8.17.0 + spec.max: 3.3 + - kibana.version: 8.17.1 + spec.max: 3.3 + - kibana.version: 8.17.2 + spec.max: 3.3 + - kibana.version: 8.17.3 + spec.max: 3.3 + - kibana.version: 8.17.4 + spec.max: 3.3 + - kibana.version: 8.17.5 + spec.max: 3.3 + - kibana.version: 8.17.6 + spec.max: 3.3 + - kibana.version: 8.17.7 + spec.max: 3.3 + - kibana.version: 8.18.0 + spec.max: 3.3 + - kibana.version: 8.18.1 + spec.max: 3.3 + - kibana.version: 8.18.2 + spec.max: 3.3 + - kibana.version: 8.19.0 + spec.max: 3.4 + - kibana.version: 9.0.0 + spec.min: 2.3 + spec.max: 3.3 + - kibana.version: 9.0.1 + spec.min: 2.3 + spec.max: 3.3 + - kibana.version: 9.0.2 + spec.min: 2.3 + spec.max: 3.3 + - kibana.version: 9.1.0 + spec.min: 2.3 + spec.max: 3.4 + + +actions: + - print: + - download: + destination: ./build/distribution diff --git a/cmd/distribution/examples/test.yaml b/cmd/distribution/examples/test.yaml new file mode 100644 index 000000000..d19887926 --- /dev/null +++ b/cmd/distribution/examples/test.yaml @@ -0,0 +1,74 @@ +# Base address of the Package Registry +address: "https://epr.elastic.co" + +# Queries are executed with each one of the parameters of the matrix. +matrix: + - spec.max: 3.0 + type: input + - prerelease: true + spec.max: 3.0 + type: input + +# Queries to execute to discover packages. +queries: + - kibana.version: 7.17.26 + - kibana.version: 7.17.27 + - kibana.version: 7.17.28 + - kibana.version: 7.17.29 + - kibana.version: 8.15.4 + - kibana.version: 8.15.5 + - kibana.version: 8.15.6 + - kibana.version: 8.16.0 + spec.max: 3.3 + - kibana.version: 8.16.1 + spec.max: 3.3 + - kibana.version: 8.16.2 + spec.max: 3.3 + - kibana.version: 8.16.3 + spec.max: 3.3 + - kibana.version: 8.16.4 + spec.max: 3.3 + - kibana.version: 8.16.5 + spec.max: 3.3 + - kibana.version: 8.16.6 + spec.max: 3.3 + - kibana.version: 8.16.7 + spec.max: 3.3 + - kibana.version: 8.17.0 + spec.max: 3.3 + - kibana.version: 8.17.1 + spec.max: 3.3 + - kibana.version: 8.17.2 + spec.max: 3.3 + - kibana.version: 8.17.3 + spec.max: 3.3 + - kibana.version: 8.17.4 + spec.max: 3.3 + - kibana.version: 8.17.5 + spec.max: 3.3 + - kibana.version: 8.17.6 + spec.max: 3.3 + - kibana.version: 8.17.7 + spec.max: 3.3 + - kibana.version: 8.18.0 + spec.max: 3.3 + - kibana.version: 8.18.1 + spec.max: 3.3 + - kibana.version: 8.18.2 + spec.max: 3.3 + - kibana.version: 8.19.0 + spec.max: 3.3 + - kibana.version: 9.0.0 + spec.min: 2.3 + spec.max: 3.3 + - kibana.version: 9.0.1 + spec.min: 2.3 + spec.max: 3.3 + - kibana.version: 9.0.2 + spec.min: 2.3 + spec.max: 3.3 + +actions: + - print: + - download: + destination: ./build/distribution-test diff --git a/cmd/distribution/go.mod b/cmd/distribution/go.mod new file mode 100644 index 000000000..868f54213 --- /dev/null +++ b/cmd/distribution/go.mod @@ -0,0 +1,22 @@ +module github.com/elastic/package-registry/cmd/distribution + +go 1.24.0 + +require ( + github.com/Masterminds/semver/v3 v3.4.0 + github.com/ProtonMail/go-crypto v1.3.0 + github.com/elastic/package-registry v0.0.0 + github.com/google/go-querystring v1.1.0 + github.com/stretchr/testify v1.11.1 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/cloudflare/circl v1.6.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + golang.org/x/crypto v0.48.0 // indirect + golang.org/x/sys v0.41.0 // indirect +) + +replace github.com/elastic/package-registry => ../.. diff --git a/cmd/distribution/go.sum b/cmd/distribution/go.sum new file mode 100644 index 000000000..15bfe3c9a --- /dev/null +++ b/cmd/distribution/go.sum @@ -0,0 +1,25 @@ +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/ProtonMail/go-crypto v1.3.0 h1:ILq8+Sf5If5DCpHQp4PbZdS1J7HDFRXz/+xKBiRGFrw= +github.com/ProtonMail/go-crypto v1.3.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE= +github.com/cloudflare/circl v1.6.0 h1:cr5JKic4HI+LkINy2lg3W2jF8sHCVTBncJr5gIIq7qk= +github.com/cloudflare/circl v1.6.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= +golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/cmd/distribution/main.go b/cmd/distribution/main.go new file mode 100644 index 000000000..c1fa4f356 --- /dev/null +++ b/cmd/distribution/main.go @@ -0,0 +1,74 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package main + +import ( + "fmt" + "net/http" + "os" + "runtime" + + "github.com/elastic/package-registry/workers" +) + +const defaultAddress = "https://epr.elastic.co" + +func main() { + if len(os.Args) != 2 { + usageAndExit(-1) + } + config, err := readConfig(os.Args[1]) + if err != nil { + fmt.Printf("failed to read configuration from %s: %s\n", os.Args[1], err) + os.Exit(-1) + } + for _, action := range config.Actions { + err := action.init(config) + if err != nil { + fmt.Printf("failed to initialize actions: %s", err) + os.Exit(-1) + } + } + + packages, err := config.collect(&http.Client{}) + if err != nil { + fmt.Printf("failed to collect packages: %s", err) + os.Exit(-1) + } + + taskpool := workers.NewTaskPool(runtime.GOMAXPROCS(0)) + for _, info := range packages { + taskpool.Do(func() error { + for _, action := range config.Actions { + err := action.perform(info) + if err != nil { + return fmt.Errorf("failed to perform action: %w", err) + } + } + return nil + }) + } + if err := taskpool.Wait(); err != nil { + fmt.Println(err) + os.Exit(-1) + } + fmt.Println(len(packages), "packages total") +} + +func usageAndExit(status int) { + fmt.Println(os.Args[0], "[config.yaml]") + os.Exit(status) +} + +type printAction struct{} + +func (a *printAction) init(c config) error { + return nil +} + +func (a *printAction) perform(i packageInfo) error { + fmt.Println("- ", i.Name, i.Version) + return nil +} diff --git a/cmd/distribution/main_test.go b/cmd/distribution/main_test.go new file mode 100644 index 000000000..29c402db1 --- /dev/null +++ b/cmd/distribution/main_test.go @@ -0,0 +1,407 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReadConfigValid(t *testing.T) { + tempDir := t.TempDir() + configPath := filepath.Join(tempDir, "config.yaml") + + configContent := ` +address: "https://test.elastic.co" +queries: + - kibana.version: "8.0.0" + - prerelease: true +actions: + - print: +` + err := os.WriteFile(configPath, []byte(configContent), 0644) + require.NoError(t, err) + + cfg, err := readConfig(configPath) + require.NoError(t, err) + assert.Equal(t, "https://test.elastic.co", cfg.Address) + assert.Len(t, cfg.Queries, 2) + assert.Equal(t, "8.0.0", cfg.Queries[0].KibanaVersion) + assert.True(t, cfg.Queries[1].Prerelease) + assert.Len(t, cfg.Actions, 1) +} + +func TestReadConfigInvalidPath(t *testing.T) { + _, err := readConfig("/nonexistent/config.yaml") + require.Error(t, err) +} + +func TestReadConfigInvalidYAML(t *testing.T) { + tempDir := t.TempDir() + configPath := filepath.Join(tempDir, "invalid.yaml") + + invalidContent := ` +address: https://test.elastic.co +queries: + - this is not: valid: yaml +` + err := os.WriteFile(configPath, []byte(invalidContent), 0644) + require.NoError(t, err) + + _, err = readConfig(configPath) + require.Error(t, err) +} + +func TestConfigActionFactory(t *testing.T) { + tests := []struct { + name string + actionName string + expectError bool + }{ + { + name: "print action", + actionName: "print", + expectError: false, + }, + { + name: "download action", + actionName: "download", + expectError: false, + }, + { + name: "unknown action", + actionName: "unknown", + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + action, err := configActionFactory(tt.actionName) + if tt.expectError { + require.Error(t, err) + require.Nil(t, action) + } else { + require.NoError(t, err) + require.NotNil(t, action) + } + }) + } +} + +func TestConfigSearchURLs(t *testing.T) { + tests := []struct { + name string + config config + expectedURLs []string + expectedError bool + }{ + { + name: "simple query", + config: config{ + Address: "http://localhost:8080", + Queries: []configQuery{ + {KibanaVersion: "8.0.0"}, + }, + }, + expectedURLs: []string{ + "http://localhost:8080/search?kibana.version=8.0.0", + }, + }, + { + name: "multiple queries", + config: config{ + Address: "http://localhost:8080", + Queries: []configQuery{ + {}, + {Prerelease: true}, + }, + }, + expectedURLs: []string{ + "http://localhost:8080/search", + "http://localhost:8080/search?prerelease=true", + }, + }, + { + name: "matrix expansion", + config: config{ + Address: "http://localhost:8080", + Matrix: []configQuery{ + {}, + {Prerelease: true}, + }, + Queries: []configQuery{ + {KibanaVersion: "8.0.0"}, + }, + }, + expectedURLs: []string{ + "http://localhost:8080/search?kibana.version=8.0.0", + "http://localhost:8080/search?kibana.version=8.0.0&prerelease=true", + }, + }, + { + name: "spec constraints", + config: config{ + Address: "http://localhost:8080", + Queries: []configQuery{ + {SpecMin: "2.0", SpecMax: "3.0"}, + }, + }, + expectedURLs: []string{ + "http://localhost:8080/search?spec.max=3.0&spec.min=2.0", + }, + }, + { + name: "default address", + config: config{ + Queries: []configQuery{ + {Package: "nginx"}, + }, + }, + expectedURLs: []string{ + "https://epr.elastic.co/search?package=nginx", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + urls, err := tt.config.searchURLs() + + if tt.expectedError { + require.Error(t, err) + return + } + + require.NoError(t, err) + + var actualURLs []string + for u := range urls { + actualURLs = append(actualURLs, u.String()) + } + + assert.ElementsMatch(t, tt.expectedURLs, actualURLs) + }) + } +} + +func TestConfigDownloadPathForPackage(t *testing.T) { + cfg := config{} + + pkgPath, sigPath := cfg.downloadPathForPackage("nginx", "1.0.0") + require.Equal(t, "epr/nginx/nginx-1.0.0.zip", pkgPath) + require.Equal(t, "epr/nginx/nginx-1.0.0.zip.sig", sigPath) +} + +func TestConfigPinnedPackages(t *testing.T) { + cfg := config{ + Packages: []configPackage{ + {Name: "nginx", Version: "1.0.0"}, + {Name: "apache", Version: "2.0.0"}, + }, + } + + packages, err := cfg.pinnedPackages() + require.NoError(t, err) + assert.Len(t, packages, 2) + assert.Equal(t, "nginx", packages[0].Name) + assert.Equal(t, "1.0.0", packages[0].Version) + assert.Equal(t, "epr/nginx/nginx-1.0.0.zip", packages[0].Download) + assert.Equal(t, "epr/nginx/nginx-1.0.0.zip.sig", packages[0].SignaturePath) +} + +func TestConfigCollect(t *testing.T) { + tests := []struct { + name string + config config + mockResponse []packageInfo + expectedPackages int + }{ + { + name: "basic collection", + config: config{ + Queries: []configQuery{ + {}, + }, + }, + mockResponse: []packageInfo{ + {Name: "nginx", Version: "1.0.0", Download: "/epr/nginx/nginx-1.0.0.zip"}, + {Name: "apache", Version: "2.0.0", Download: "/epr/apache/apache-2.0.0.zip"}, + }, + expectedPackages: 2, + }, + { + name: "deduplication", + config: config{ + Queries: []configQuery{ + {KibanaVersion: "8.0.0"}, + {KibanaVersion: "8.1.0"}, + }, + }, + mockResponse: []packageInfo{ + {Name: "nginx", Version: "1.0.0", Download: "/epr/nginx/nginx-1.0.0.zip"}, + }, + expectedPackages: 1, + }, + { + name: "with pinned packages", + config: config{ + Packages: []configPackage{ + {Name: "mysql", Version: "1.5.0"}, + }, + Queries: []configQuery{ + {}, + }, + }, + mockResponse: []packageInfo{ + {Name: "nginx", Version: "1.0.0", Download: "/epr/nginx/nginx-1.0.0.zip"}, + }, + expectedPackages: 2, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create mock server + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, "/search", r.URL.Path) + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(tt.mockResponse) + })) + defer server.Close() + + tt.config.Address = server.URL + client := &http.Client{} + + packages, err := tt.config.collect(client) + require.NoError(t, err) + require.Len(t, packages, tt.expectedPackages) + }) + } +} + +func TestConfigCollectSorting(t *testing.T) { + mockResponse := []packageInfo{ + {Name: "zebra", Version: "1.0.0"}, + {Name: "apache", Version: "2.0.0"}, + {Name: "apache", Version: "1.0.0"}, + {Name: "nginx", Version: "1.0.0"}, + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(mockResponse) + })) + defer server.Close() + + cfg := config{ + Address: server.URL, + Queries: []configQuery{{}}, + } + + packages, err := cfg.collect(&http.Client{}) + require.NoError(t, err) + assert.Len(t, packages, 4) + + // Verify sorted by name, then by version + assert.Equal(t, "apache", packages[0].Name) + assert.Equal(t, "1.0.0", packages[0].Version) + assert.Equal(t, "apache", packages[1].Name) + assert.Equal(t, "2.0.0", packages[1].Version) + assert.Equal(t, "nginx", packages[2].Name) + assert.Equal(t, "zebra", packages[3].Name) +} + +func TestConfigCollectHTTPError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + + cfg := config{ + Address: server.URL, + Queries: []configQuery{{}}, + } + + _, err := cfg.collect(&http.Client{}) + require.Error(t, err) + require.Contains(t, err.Error(), "status code 500") +} + +func TestConfigQueryBuild(t *testing.T) { + tests := []struct { + name string + query configQuery + expected url.Values + }{ + { + name: "empty query", + query: configQuery{}, + expected: url.Values{}, + }, + { + name: "kibana version", + query: configQuery{ + KibanaVersion: "8.0.0", + }, + expected: url.Values{ + "kibana.version": []string{"8.0.0"}, + }, + }, + { + name: "multiple fields", + query: configQuery{ + Package: "nginx", + Prerelease: true, + Type: "integration", + }, + expected: url.Values{ + "package": []string{"nginx"}, + "prerelease": []string{"true"}, + "type": []string{"integration"}, + }, + }, + { + name: "spec constraints", + query: configQuery{ + SpecMin: "2.0", + SpecMax: "3.0", + }, + expected: url.Values{ + "spec.min": []string{"2.0"}, + "spec.max": []string{"3.0"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + values := tt.query.Build() + require.Equal(t, tt.expected, values) + }) + } +} + +func TestPrintAction(t *testing.T) { + action := &printAction{} + + // Test init + err := action.init(config{}) + require.NoError(t, err) + + // Test perform (just verify it doesn't error) + err = action.perform(packageInfo{ + Name: "nginx", + Version: "1.0.0", + }) + require.NoError(t, err) +} diff --git a/magefile.go b/magefile.go index a62ab56b6..6769f993c 100644 --- a/magefile.go +++ b/magefile.go @@ -34,10 +34,27 @@ const ( buildDir = "./build" ) +// modules is the list of Go modules in this repository. +// Add new modules here to automatically include them in test, lint, and other targets. +var modules = []struct { + name string // Display name + path string // Relative path from repo root +}{ + {"package-registry", "."}, + {"cmd/distribution", "cmd/distribution"}, +} + func Build() error { + fmt.Println(">> Building package-registry") return sh.Run("go", "build", ".") } +// BuildDistribution builds the distribution binary in cmd/distribution. +func BuildDistribution() error { + fmt.Println(">> Building cmd/distribution") + return sh.RunWith(map[string]string{"PWD": "cmd/distribution"}, "go", "build", "-o", "distribution", ".") +} + // DockerBuild builds the Docker image for the package registry. It must be specified // the docker tag to be used as an argument (e.g. main, latest). func DockerBuild(tag string) error { @@ -63,6 +80,7 @@ func Check() error { mg.SerialDeps( Format, Build, + BuildDistribution, ModTidy, Staticcheck, ) @@ -76,7 +94,14 @@ func Check() error { } func Test() error { - return sh.RunV("go", "test", "./...", "-v") + for _, mod := range modules { + fmt.Printf(">> Testing %s\n", mod.name) + err := sh.RunWith(map[string]string{"PWD": mod.path}, "go", "test", "./...", "-v") + if err != nil { + return fmt.Errorf("%s tests failed: %w", mod.name, err) + } + } + return nil } func WriteTestGoldenFiles() error { @@ -155,17 +180,41 @@ func Clean() error { return err } - return os.RemoveAll("package-registry") + // Clean main package-registry binary + err = os.RemoveAll("package-registry") + if err != nil { + return err + } + + // Clean distribution binary + err = os.RemoveAll("cmd/distribution/distribution") + if err != nil { + return err + } + + return nil } // ModTidy cleans unused dependencies. func ModTidy() error { - fmt.Println(">> fmt - go mod tidy: Generating go mod files") - return sh.RunV("go", "mod", "tidy") + for _, mod := range modules { + fmt.Printf(">> fmt - go mod tidy: Generating go mod files for %s\n", mod.name) + err := sh.RunWith(map[string]string{"PWD": mod.path}, "go", "mod", "tidy") + if err != nil { + return fmt.Errorf("%s go mod tidy failed: %w", mod.name, err) + } + } + return nil } // Staticcheck runs a static code analyzer. func Staticcheck() error { - fmt.Println(">> check - staticcheck: Running static code analyzer") - return sh.RunV("go", "run", StaticcheckImportPath, "./...") + for _, mod := range modules { + fmt.Printf(">> check - staticcheck: Running static code analyzer on %s\n", mod.name) + err := sh.RunWith(map[string]string{"PWD": mod.path}, "go", "run", StaticcheckImportPath, "./...") + if err != nil { + return fmt.Errorf("%s staticcheck failed: %w", mod.name, err) + } + } + return nil } diff --git a/packages/packages.go b/packages/packages.go index 28f60914b..b92bcfe3f 100644 --- a/packages/packages.go +++ b/packages/packages.go @@ -23,8 +23,8 @@ import ( "go.elastic.co/apm/v2" "go.uber.org/zap" - "github.com/elastic/package-registry/internal/workers" "github.com/elastic/package-registry/metrics" + "github.com/elastic/package-registry/workers" ) const ( diff --git a/internal/workers/taskpool.go b/workers/taskpool.go similarity index 70% rename from internal/workers/taskpool.go rename to workers/taskpool.go index a3b35abc3..e6f8854de 100644 --- a/internal/workers/taskpool.go +++ b/workers/taskpool.go @@ -10,46 +10,44 @@ import ( ) type taskPool struct { - wg sync.WaitGroup - pool chan struct{} - errC chan error - errors []error + wg sync.WaitGroup + pool chan struct{} + + errLock sync.Mutex + errors []error } func NewTaskPool(size int) *taskPool { p := &taskPool{ pool: make(chan struct{}, size), - errC: make(chan error), } - go p.errorLoop() return p } -func (p *taskPool) errorLoop() { - go func() { - for err := range p.errC { - if err != nil { - p.errors = append(p.errors, err) - } - } - }() -} - // Do runs the task in a goroutine, ensuring no more tasks are running than the size of the pool. func (p *taskPool) Do(task func() error) { p.pool <- struct{}{} - p.wg.Add(1) - go func() { + p.wg.Go(func() { defer func() { <-p.pool }() - defer p.wg.Done() - p.errC <- task() - }() + + err := task() + p.recordError(err) + }) +} + +func (p *taskPool) recordError(err error) { + if err == nil { + return + } + + p.errLock.Lock() + p.errors = append(p.errors, err) + p.errLock.Unlock() } // Wait waits for all the tasks to finish, and joins the errors found. The pool cannot be used after calling Wait. func (p *taskPool) Wait() error { close(p.pool) p.wg.Wait() - close(p.errC) return errors.Join(p.errors...) } diff --git a/workers/taskpool_test.go b/workers/taskpool_test.go new file mode 100644 index 000000000..6fdb4f39a --- /dev/null +++ b/workers/taskpool_test.go @@ -0,0 +1,103 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package workers + +import ( + "errors" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestTaskPoolBasicExecution(t *testing.T) { + pool := NewTaskPool(2) + + var counter atomic.Int32 + for i := 0; i < 10; i++ { + pool.Do(func() error { + counter.Add(1) + return nil + }) + } + + err := pool.Wait() + require.NoError(t, err) + require.Equal(t, int32(10), counter.Load()) +} + +func TestTaskPoolErrorHandling(t *testing.T) { + pool := NewTaskPool(2) + + expectedErr := errors.New("task failed") + pool.Do(func() error { + return expectedErr + }) + pool.Do(func() error { + return nil + }) + + err := pool.Wait() + require.Error(t, err) + require.Contains(t, err.Error(), "task failed") +} + +func TestTaskPoolMultipleErrors(t *testing.T) { + pool := NewTaskPool(2) + + err1 := errors.New("error 1") + err2 := errors.New("error 2") + + pool.Do(func() error { + return err1 + }) + pool.Do(func() error { + return err2 + }) + + err := pool.Wait() + if err != nil { + // At least one error should be present + errMsg := err.Error() + hasErr1 := strings.Contains(errMsg, "error 1") + hasErr2 := strings.Contains(errMsg, "error 2") + require.True(t, hasErr1 || hasErr2, "should contain at least one error") + } +} + +func TestTaskPoolConcurrency(t *testing.T) { + poolSize := 3 + pool := NewTaskPool(poolSize) + + var running atomic.Int32 + var maxConcurrent atomic.Int32 + + for i := 0; i < 10; i++ { + pool.Do(func() error { + current := running.Add(1) + + // Track max concurrent tasks + for { + max := maxConcurrent.Load() + if current <= max { + break + } + if maxConcurrent.CompareAndSwap(max, current) { + break + } + } + + time.Sleep(10 * time.Millisecond) + running.Add(-1) + return nil + }) + } + + err := pool.Wait() + require.NoError(t, err) + require.LessOrEqual(t, maxConcurrent.Load(), int32(poolSize)) +}