diff --git a/release/Makefile b/release/Makefile index 85214f9142f..861bbc52463 100644 --- a/release/Makefile +++ b/release/Makefile @@ -21,7 +21,7 @@ clean: @find ../ -name '*release*.log' -delete @find . -name '*.received.*' -delete -bin/release: $(shell find . -name "*.go") +bin/release: $(shell find . -name "*.go" -or -name "*.gotmpl") @mkdir -p bin && \ $(call build_binary, ./cmd, bin/release) diff --git a/release/internal/aptrepo/apt.go b/release/internal/aptrepo/apt.go new file mode 100644 index 00000000000..f9879d6237e --- /dev/null +++ b/release/internal/aptrepo/apt.go @@ -0,0 +1,208 @@ +// Package aptrepo contains functionality for creating and managing apt repositories +package aptrepo + +import ( + "bytes" + _ "embed" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "text/template" + + "github.com/sirupsen/logrus" +) + +// A brief note on Ubuntu/Debian/apt repo terminology: +// +// For Ubuntu and Debian, releases are numbered and codenamed; Ubuntu releases follow +// a fixed schedule and are numbered by release year and month, e.g. 24.04 was released +// in April 2024. Debian releases do not follow a fixed schedule and are numbered +// sequentially, e.g. 12.10, 12.11, etc. +// +// The 'codename' is the one-word name of the release, such as 'noble', 'trixie', +// etc. If you run `lsb_release -a` it will give you these names in the 'Codename' +// field. +// +// Here is some terminology and how it's used for these distros and how that relates to +// use in apt repositories. +// +// Suite +// In Debian, the 'suite' refers to a category of release 'oldstable', 'stable', 'testing', +// etc., and allows users to float their version to the current 'stable' or 'testing' release +// for example; when a release is promoted to 'stable' then 'stable' refers to that new +// release (whichever it is) and users will now start to get packages from that new release; +// 'oldstable' now refers to the former 'stable'. +// +// In apt, however, this distinction is not made, and the 'suite' field can contain the codename +// of the Debian or Ubuntu release, such as 'noble', 'bookworm', etc., or the Debian 'suite' +// such as 'stable' or 'testing'. +// +// Some third party repositories will create a separate suite for their own releases; for example, +// LLVM has suites for 'llvm-toolchain-noble-18', 'llvm-toolchain-noble-19', etc. We may consider +// doing something similar, e.g. 'calico-enterprise-v3.23-noble'. +// +// Component +// Which 'part' of the release it is. Most common in Ubuntu are 'main', 'restricted', +// 'universe', and 'multiverse'; for Debian the equivalents are 'main', 'non-free-firmware', +// 'contrib', and 'non-free'. +// +// While these terms have specific meaning for these releases, we can just use 'main' +// for everything. +// +// Hopefully this explains why 'suite' and 'codename' are used mostly interchangeably in +// this code depending on what they're actually being used for! + +type aptSourcesData struct { + // RepoName is the name of the repository as might be shown by repolib (e.g. in a UI) + RepoName string + // RepoURL is the base URL of the repository (i.e. where pool/ and dists/ are) + RepoURL string + // Suite is the 'suite' field, e.g. noble, bookworm, etc. + Suite string + // GpgKey is the ascii-armored GPG public key + GpgKey string + // Architectures is the list of architectures this sources file will claim support for + Architectures []string +} + +//go:embed templates/repo.sources.gotmpl +var aptSourcesTemplate string + +// writeAptSourcesFile creates a deb822-style sources file for a given set +// of parameters, and writes it to .sources under +// For more info on the format: https://repolib.readthedocs.io/en/latest/deb822-format.html +func (asd *aptSourcesData) writeAptSourcesFile(rootPath string) error { + logrus.WithField("suite", asd.Suite).Info("Generating apt .sources file") + sourcesFilePath := filepath.Join(rootPath, fmt.Sprintf("%s.sources", asd.Suite)) + sourcesFile, err := os.OpenFile(sourcesFilePath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + return fmt.Errorf("opening %s: %w", sourcesFilePath, err) + } + defer func() { _ = sourcesFile.Close() }() + + funcMap := template.FuncMap{ + "join": strings.Join, + } + + tmpl, err := template.New("apt.sources").Funcs(funcMap).Parse(aptSourcesTemplate) + if err != nil { + return fmt.Errorf("failed to parse apt sources template: %w", err) + } + + if err := tmpl.Execute(sourcesFile, asd); err != nil { + logrus.WithField("suite", asd.Suite).WithError(err).Error("failed to write apt sources file") + return fmt.Errorf("failed to write apt sources file: %w", err) + } + + logrus.WithField("file", sourcesFilePath).Info("Wrote apt .sources file") + + return nil +} + +func getVersionFromDebfile(debfilePath string) (string, error) { + logrus.WithField("debfile", debfilePath).Debug("Getting version information from debian package") + cmd := exec.Command("dpkg-deb", "--show", "--showformat", "${Version}", "--", debfilePath) + out, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("getting version for %s: %w", debfilePath, err) + } + return string(out), nil +} + +func getComponentNameFromVersion(version string) (string, error) { + if lastIdx := strings.LastIndex(version, "~"); lastIdx != -1 { + return version[lastIdx+1:], nil + } + return "", fmt.Errorf("version %s does not contain a tilde separator", version) +} + +func getSuiteNameFromDebFile(debfilePath string) (string, error) { + version, err := getVersionFromDebfile(debfilePath) + if err != nil { + return "", fmt.Errorf("getting version for %s: %w", debfilePath, err) + } + + suite, err := getComponentNameFromVersion(version) + if err != nil { + return "", fmt.Errorf("getting component name for %s: %w", debfilePath, err) + } + + return suite, nil +} + +// formatGPGKeyForSourcesFile formats a GPG public key into a format suitable to +// be appended into a sources file template (indented one space, blank +// lines replaced with '.') +func formatGPGKeyForSourcesFile(gpgKey string) string { + // To make it easier to insert the GPG key into the sources file, we want to + // 1. Replace every blank line (there should only be one) with a '.' + // 2. Indent each line with a single space + var processedKey bytes.Buffer + lines := strings.Split(gpgKey, "\n") + // The split might result in a trailing empty string if the output ends in newline, which is typical. + // We should be careful not to add extra newlines if not present, but `gpg` output usually has a trailing newline. + if len(lines) > 0 && lines[len(lines)-1] == "" { + lines = lines[:len(lines)-1] + } + + for _, line := range lines { + if line == "" { + line = "." + } + processedKey.WriteString(" " + line + "\n") + } + + return processedKey.String() +} + +func getRecursiveDebsBySuite(searchPaths []string) (map[string][]string, error) { + debsBySuite := make(map[string][]string, 0) + + files, err := getRecursiveDebs(searchPaths) + if err != nil { + return map[string][]string{}, err + } + + logrus.Debugf("Found %d debian package files to process", len(files)) + for _, debFile := range files { + suite, err := getSuiteNameFromDebFile(debFile) + if err != nil { + return map[string][]string{}, fmt.Errorf("getting suite name for %s: %w", debFile, err) + } + debsBySuite[suite] = append(debsBySuite[suite], debFile) + } + + return debsBySuite, nil +} + +func getRecursiveDebs(searchPaths []string) ([]string, error) { + // Find .deb and .ddeb files + var files []string + for _, searchPath := range searchPaths { + logrus.Infof("Scanning for debian packages in %s", searchPath) + err := filepath.WalkDir(searchPath, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + // Avoid walking into .git or .aptly to save time/confusion, + // though bash script doesn't explicitly exclude them (it relies on glob). + if d.Name() == ".git" || d.Name() == "pool" { + return filepath.SkipDir + } + return nil + } + if strings.HasSuffix(path, ".deb") || strings.HasSuffix(path, ".ddeb") { + logrus.Debug(fmt.Sprintf("Found debian package %s", path)) + files = append(files, path) + } + return nil + }) + if err != nil { + return []string{}, fmt.Errorf("walking directory: %w", err) + } + } + return files, nil +} diff --git a/release/internal/aptrepo/repo.go b/release/internal/aptrepo/repo.go new file mode 100644 index 00000000000..2bb434706c0 --- /dev/null +++ b/release/internal/aptrepo/repo.go @@ -0,0 +1,257 @@ +package aptrepo + +import ( + _ "embed" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "text/template" + + "github.com/sirupsen/logrus" + + "github.com/projectcalico/calico/release/internal/command" + "github.com/projectcalico/calico/release/internal/utils" +) + +// Reprepro is a terrible name but it's what we have + +// RepoConfig is the information we'll use to generate Reprepro's 'distributions' configuration file +type RepoConfig struct { + // Architectures is the list of architectures we'll publish + Architectures []string + // Origin is a freeform text field that admins can use to filter on; probably should be 'Tigera' + Origin string + // Label is another freeform text field to filter on; probably should be 'Calico Enterprise' + Label string + // Components is the list of 'components' (releases) we intend to publish, e.g. noble, jammy, bookworm + Components []string + // ProductName is the full name of our product that will show in the description field of the repo; e.g. + // "Calico Enterprise v3.21", or maybe "Calico Enterprise v3.21 hashrelease" + ProductName string + // GPGKeyID is the GPG key ID that we'll sign the repository with + GPGKeyID string +} + +// Repo defines the core information about a local (on-disk) repo that we want to create/manipulate +type Repo struct { + // TempDir is where we're going to store our files while we do our generation + TempDir string + // BaseDirectory is the absolute path to the repo base (where our configs and db are stored) + BaseDirectory string + // OutputDirectory is the absolute path to the output directory, where our pool and dists will be stored) + OutputDirectory string + // RepoConfig is the RepoConfig object representing the information about the repo we'll be publishing + Config RepoConfig + // PublishingURL is the full URL to the root of the published repository, e.g. https://host.com/ubuntu + PublishingURL string +} + +//go:embed templates/reprepro-conf.gotmpl +var repoDistributionsTemplate string + +// NewRepo creates a new Repo instance with the appropriate fields populated +func NewRepo(tempDir, outputDir string, repoConfig RepoConfig, url string) (*Repo, error) { + if outputDir == "" { + outputDir = filepath.Join(tempDir, "_apt_output_dir") + } + repo := Repo{ + TempDir: tempDir, + BaseDirectory: filepath.Join(tempDir, "_apt_repo_conf"), + OutputDirectory: outputDir, + Config: repoConfig, + PublishingURL: url, + } + return &repo, nil +} + +// RepositoryDBExists checks to see if the configured repository path already has +// a repo database; we need this to update existing remote repositories. +func (repo *Repo) RepositoryDBExists() (bool, error) { + dbPath := filepath.Join(repo.BaseDirectory, "db") + exist, err := utils.DirExists(dbPath) + if err != nil || !exist { + return exist, err + } + entries, err := os.ReadDir(dbPath) + if err != nil || len(entries) == 0 { + return false, err + } + return true, nil +} + +// exec 'wrapper' commands that we can use for later + +// exec runs a reprepro command but discards the output +func (repo *Repo) exec(args ...string) error { + _, err := repo.execWithOutput(args...) + return err +} + +// execWithOutput executes a reprepro command using the existing configuration and returns the output +func (repo *Repo) execWithOutput(args ...string) (string, error) { + cmdArgs := []string{ + "--basedir", + repo.BaseDirectory, + "--outdir", + repo.OutputDirectory, + "--ignore=extension", + } + cmdArgs = append(cmdArgs, args...) + logrus.Debugf("running reprepro command %s", strings.Join(cmdArgs, " ")) + out, err := command.Run("reprepro", cmdArgs) + if err != nil { + logrus.Error(out) + return "", fmt.Errorf("running 'reprepro %s': %w", strings.Join(args, " "), err) + } + return out, nil +} + +// Functions that handle configuration, setup, etc. + +// configDirPath returns the path to the configuration directory; does not guarantee it exists +func (repo *Repo) configDirPath() string { + return filepath.Join(repo.BaseDirectory, "conf") +} + +// configFilePath returns the path to the configuration file; does not guarantee it exists +func (repo *Repo) configFilePath() string { + return filepath.Join(repo.configDirPath(), "distributions") +} + +// CleanBaseDir removes the repo's configured base directory +func (repo *Repo) cleanBaseDir() error { + logrus.Debugf("removing repo base directory %s", repo.BaseDirectory) + if err := os.RemoveAll(repo.BaseDirectory); err != nil { + return fmt.Errorf("could not clean repo base directory %s: %w", repo.BaseDirectory, err) + } + return nil +} + +// CleanOutputDir removes the repo's configured output directory +func (repo *Repo) cleanOutputDir() error { + logrus.Debugf("removing repo output directory %s", repo.OutputDirectory) + if err := os.RemoveAll(repo.OutputDirectory); err != nil { + return fmt.Errorf("could not clean repo output directory %s: %w", repo.OutputDirectory, err) + } + return nil +} + +// Clean removes the configured base and output directories. Must be run before creating +// the repository configuration with Repo.WriteRepoConfig() +func (repo *Repo) clean() error { + return errors.Join( + repo.cleanBaseDir(), + repo.cleanOutputDir(), + ) +} + +// PrepareForBuild sets up the configured paths to be ready to build an +// apt repo. If we're building a repository from packages, this should +// be run before we start touching the filesystem. +func (repo *Repo) PrepareForBuild() error { + // We need to run clean() to ensure that we don't have leftover files + // from a previous build, leftover repo configuration or database, etc. + return repo.clean() +} + +// WriteRepoConfig generates and writes the config file for the repo software to the appropriate path +func (repo *Repo) WriteRepoConfig() error { + if err := os.MkdirAll(repo.configDirPath(), utils.DirPerms); err != nil { + return fmt.Errorf("failed to create config dir: %w", err) + } + repoConfigFile, err := os.OpenFile(repo.configFilePath(), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) + if err != nil { + return fmt.Errorf("failed to create config file: %w", err) + } + defer func() { _ = repoConfigFile.Close() }() + + funcMap := template.FuncMap{ + "join": strings.Join, + } + tmpl, err := template.New("repo/config/distributions").Funcs(funcMap).Parse(repoDistributionsTemplate) + if err != nil { + return fmt.Errorf("failed to parse repo's distributions template: %w", err) + } + + if err := tmpl.Execute(repoConfigFile, repo.Config); err != nil { + return fmt.Errorf("failed to write repo distributions file: %w", err) + } + + return nil +} + +// Functions that expose reprepro's functionality (e.g. adding debian packages) + +// IncludeDeb adds a specified debian file to the specified component in the repo +func (repo *Repo) IncludeDeb(component, debFile string) error { + if !slices.Contains(repo.Config.Components, component) { + return fmt.Errorf("specified component %s not present in configured components list %s", component, strings.Join(repo.Config.Components, ", ")) + } + + err := repo.exec("includedeb", component, debFile) + if err != nil { + return fmt.Errorf("could not add file %s to component %s: %w", debFile, component, err) + } + return nil +} + +// RecursiveAddDebsFromDirectories takes a list of paths to search and finds all debian packages +// under those paths, gets their suite/component name, and adds them to the repo +func (repo *Repo) RecursiveAddDebsFromDirectories(searchPaths []string) error { + debsBySuite, err := getRecursiveDebsBySuite(searchPaths) + if err != nil { + return fmt.Errorf("could not scan for debian packages: %w", err) + } + + var publishingErrors []error + + for suite, filesList := range debsBySuite { + for _, filename := range filesList { + if err := repo.IncludeDeb(suite, filename); err != nil { + publishingErrors = append(publishingErrors, err) + } + } + } + if err := errors.Join(publishingErrors...); err != nil { + return fmt.Errorf("encountered errors publishing Apt repository: %w", err) + } + return nil +} + +// WriteSourcesFile writes out the .sources file for a given codename to the repo's output directory +func (repo *Repo) WriteSourcesFile(codename string) error { + if !slices.Contains(repo.Config.Components, codename) { + return fmt.Errorf("specified codename %s does not exist in defined codenames (%s)", codename, strings.Join(repo.Config.Components, ", ")) + } + gpgPubKey, err := utils.GetGPGPubKey(repo.Config.GPGKeyID) + if err != nil { + return fmt.Errorf("could not fetch GPG key %s: %w", repo.Config.GPGKeyID, err) + } + gpgPubKeyFormatted := formatGPGKeyForSourcesFile(gpgPubKey) + + sourcesFields := aptSourcesData{ + RepoName: repo.Config.ProductName, + RepoURL: repo.PublishingURL, + Suite: codename, + GpgKey: gpgPubKeyFormatted, + Architectures: repo.Config.Architectures, + } + + if err := sourcesFields.writeAptSourcesFile(repo.OutputDirectory); err != nil { + return fmt.Errorf("unable to write sources file for %s: %w", codename, err) + } + return nil +} + +// WriteAllSourcesFiles creates a .sources in the repo's output directory for +// each configured codename/suite. +func (repo *Repo) WriteAllSourcesFiles() error { + var errs []error + for _, codename := range repo.Config.Components { + errs = append(errs, repo.WriteSourcesFile(codename)) + } + return errors.Join(errs...) +} diff --git a/release/internal/aptrepo/templates/repo.sources.gotmpl b/release/internal/aptrepo/templates/repo.sources.gotmpl new file mode 100644 index 00000000000..64d6490489b --- /dev/null +++ b/release/internal/aptrepo/templates/repo.sources.gotmpl @@ -0,0 +1,7 @@ +X-Repolib-Name: {{.RepoName}} +Types: deb +URIs: {{.RepoURL}} +Suites: {{.Suite}} +Components: main +Architectures: {{ join .Architectures " " }} +Signed-By: {{.GpgKey}} \ No newline at end of file diff --git a/release/internal/aptrepo/templates/reprepro-conf.gotmpl b/release/internal/aptrepo/templates/reprepro-conf.gotmpl new file mode 100644 index 00000000000..15d79a0b1f8 --- /dev/null +++ b/release/internal/aptrepo/templates/reprepro-conf.gotmpl @@ -0,0 +1,12 @@ +{{ $config := . }} +{{ range $codename := .Components }} +Origin: {{ $config.Origin }} +Label: {{ $config.Label }} +Codename: {{ $codename }} +Suite: {{ $codename }} +Architectures: {{ join $config.Architectures " "}} +Components: main +Description: {{ $config.ProductName }} packages for Debian and Ubuntu systems +SignWith: {{ $config.GPGKeyID }} +Contents: +{{ end }} \ No newline at end of file diff --git a/release/internal/utils/files.go b/release/internal/utils/files.go index 86ded5e673e..d39d153d95e 100644 --- a/release/internal/utils/files.go +++ b/release/internal/utils/files.go @@ -15,9 +15,14 @@ package utils import ( + "errors" "fmt" + "io/fs" "os" + "os/exec" "path/filepath" + + "github.com/sirupsen/logrus" ) const ( @@ -58,3 +63,40 @@ func CopyFile(src, dst string) error { } return nil } + +// PathExists validates if a given (relative or absolute) path exists +func PathExists(path string) (bool, error) { + _, err := os.Stat(path) + if err == nil { + return true, nil + } + if errors.Is(err, fs.ErrNotExist) { + return false, nil + } + return false, err +} + +// DirExists validates if a given (relative or absolute) path exists and +// is a directory (or as symlink to one) +func DirExists(path string) (bool, error) { + stat, err := os.Stat(path) + if err == nil { + return stat.IsDir(), nil + } + if errors.Is(err, fs.ErrNotExist) { + return false, nil + } + return false, err +} + +// CheckBinary searches the current PATH for a binary and returns an error if it's not found +func CheckBinary(binaryName, neededFor string) error { + if path, err := exec.LookPath(binaryName); err != nil { + logrus.WithError(err).Errorf("Error trying to find %s in PATH (needed for %s)", binaryName, neededFor) + return fmt.Errorf("unable to find %s in PATH (needed for %s)", binaryName, neededFor) + } else if path == "" { + logrus.Errorf("%s not found in PATH (needed for %s)", binaryName, neededFor) + return fmt.Errorf("%s not found in PATH (needed for %s)", binaryName, neededFor) + } + return nil +} diff --git a/release/internal/utils/gpg.go b/release/internal/utils/gpg.go new file mode 100644 index 00000000000..a31dfe821b7 --- /dev/null +++ b/release/internal/utils/gpg.go @@ -0,0 +1,37 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "fmt" + "strings" + + "github.com/sirupsen/logrus" + + "github.com/projectcalico/calico/release/internal/command" +) + +// GetGPGPubKey takes a GPG key ID and fetches the ascii-armored GPG public key +func GetGPGPubKey(gpgKeyID string) (string, error) { + logrus.Debugf("Getting ascii-armored public key for GPG key %s", gpgKeyID) + + cmdArgs := []string{"--armor", "--export", gpgKeyID} + logrus.Debugf("running gpg with args %s", strings.Join(cmdArgs, " ")) + gpgOut, err := command.Run("gpg", cmdArgs) + if err != nil { + return "", fmt.Errorf("exporting gpg key: %w", err) + } + return string(gpgOut), nil +} diff --git a/release/internal/utils/utils.go b/release/internal/utils/utils.go index 11112047ded..768ebfe1a76 100644 --- a/release/internal/utils/utils.go +++ b/release/internal/utils/utils.go @@ -52,6 +52,9 @@ const ( // TigeraOrg is the name of the Tigera organization. TigeraOrg = "tigera" + // TigeraCompany is the short-form human-facing name of the company, for freeform text fields or branding + TigeraCompany = "Tigera" + // TigeraOperatorChart is the name of the Tigera Operator Helm chart. TigeraOperatorChart = "tigera-operator"