From c0b9a190b22ae0251f991039e65e32d8cd1455e8 Mon Sep 17 00:00:00 2001 From: hasbro17 Date: Thu, 6 Feb 2020 22:41:41 -0800 Subject: [PATCH 01/16] generate csv: replace config with CSV and package manifest generators internal/generate/olm-catalog: implement CSV and package manifest generators, moving most of the code from internal/scaffold/olm-catalog verbatim. Also removed CSV config doc: remove CSVConfig documentation, update CLI docs Co-authored-by: Eric Stroczynski --- CHANGELOG.md | 1 + cmd/operator-sdk/bundle/create.go | 2 +- cmd/operator-sdk/generate/csv.go | 124 ++--- cmd/operator-sdk/run/cmd.go | 2 +- doc/cli/operator-sdk_generate_csv.md | 5 +- doc/user/olm-catalog/generating-a-csv.md | 28 +- internal/generate/olm-catalog/csv.go | 405 ++++++++++++++++- internal/generate/olm-catalog/csv_go_test.go | 249 ++++++++++ .../olm-catalog/csv_updaters.go | 64 ++- .../olm-catalog/descriptor/descriptor.go | 0 .../olm-catalog/descriptor/descriptor_test.go | 37 +- .../olm-catalog/descriptor/parse.go | 0 .../olm-catalog/descriptor/parse_test.go | 0 .../olm-catalog/descriptor/search.go | 0 .../generate/olm-catalog/package_manifest.go | 12 +- .../olm-catalog/package_manifest_test.go | 10 - .../pkg/apis/cache/v1alpha1/dummy_types.go | 2 +- internal/scaffold/olm-catalog/config.go | 148 ------ internal/scaffold/olm-catalog/config_test.go | 55 --- internal/scaffold/olm-catalog/csv.go | 430 ------------------ internal/util/k8sutil/crd.go | 14 +- ...operator.v0.0.2.clusterserviceversion.yaml | 199 ++++---- ...operator.v0.0.3.clusterserviceversion.yaml | 34 +- 23 files changed, 945 insertions(+), 876 deletions(-) create mode 100644 internal/generate/olm-catalog/csv_go_test.go rename internal/{scaffold => generate}/olm-catalog/csv_updaters.go (82%) rename internal/{scaffold => generate}/olm-catalog/descriptor/descriptor.go (100%) rename internal/{scaffold => generate}/olm-catalog/descriptor/descriptor_test.go (89%) rename internal/{scaffold => generate}/olm-catalog/descriptor/parse.go (100%) rename internal/{scaffold => generate}/olm-catalog/descriptor/parse_test.go (100%) rename internal/{scaffold => generate}/olm-catalog/descriptor/search.go (100%) rename {test/test-framework => internal/generate/testdata/go}/pkg/apis/cache/v1alpha1/dummy_types.go (99%) delete mode 100644 internal/scaffold/olm-catalog/config.go delete mode 100644 internal/scaffold/olm-catalog/config_test.go delete mode 100644 internal/scaffold/olm-catalog/csv.go diff --git a/CHANGELOG.md b/CHANGELOG.md index d33a7f61221..ddf4ae4002a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,6 +82,7 @@ ### Removed - **Breaking Change:** The additional Ansible sidecar container. ([#2586](https://github.com/operator-framework/operator-sdk/pull/2586)) +- **Breaking Change:** Removed CSV configuration file support (defaulting to deploy/olm-catalog/csv-config.yaml) in favor of including files as input to the generator using [`generate csv --include`](doc/cli/operator-sdk_generate_csv.md#options), defaulting to the `deploy/` directory. ([#2249](https://github.com/operator-framework/operator-sdk/pull/2249)) ### Bug Fixes diff --git a/cmd/operator-sdk/bundle/create.go b/cmd/operator-sdk/bundle/create.go index d3e6ddf8e7f..517fe7c2566 100644 --- a/cmd/operator-sdk/bundle/create.go +++ b/cmd/operator-sdk/bundle/create.go @@ -22,7 +22,7 @@ import ( "os" "path/filepath" - catalog "github.com/operator-framework/operator-sdk/internal/scaffold/olm-catalog" + catalog "github.com/operator-framework/operator-sdk/internal/generate/olm-catalog" "github.com/operator-framework/operator-sdk/internal/util/projutil" "github.com/operator-framework/operator-registry/pkg/lib/bundle" diff --git a/cmd/operator-sdk/generate/csv.go b/cmd/operator-sdk/generate/csv.go index 70b5c9cb69c..51df1b72f10 100644 --- a/cmd/operator-sdk/generate/csv.go +++ b/cmd/operator-sdk/generate/csv.go @@ -17,13 +17,12 @@ package generate import ( "fmt" "io/ioutil" + "os" "path/filepath" "github.com/operator-framework/operator-sdk/internal/generate/gen" gencatalog "github.com/operator-framework/operator-sdk/internal/generate/olm-catalog" "github.com/operator-framework/operator-sdk/internal/scaffold" - "github.com/operator-framework/operator-sdk/internal/scaffold/input" - catalog "github.com/operator-framework/operator-sdk/internal/scaffold/olm-catalog" "github.com/operator-framework/operator-sdk/internal/util/fileutil" "github.com/operator-framework/operator-sdk/internal/util/k8sutil" "github.com/operator-framework/operator-sdk/internal/util/projutil" @@ -37,8 +36,9 @@ type csvCmd struct { csvVersion string csvChannel string fromVersion string - csvConfigPath string operatorName string + outputDir string + includePaths []string updateCRDs bool defaultChannel bool } @@ -53,9 +53,8 @@ for the operator. This file is used to publish the operator to the OLM Catalog. A CSV semantic version is supplied via the --csv-version flag. If your operator has already generated a CSV manifest you want to use as a base, supply its -version to --from-version. Otherwise the SDK will scaffold a new CSV manifest. +version to --from-version. Otherwise the SDK will scaffold a new CSV manifest.`, -Configure CSV generation by writing a config file 'deploy/olm-catalog/csv-config.yaml`, RunE: func(cmd *cobra.Command, args []string) error { // The CSV generator assumes that the deploy and pkg directories are // present at runtime, so this command must be run in a project's root. @@ -74,14 +73,20 @@ Configure CSV generation by writing a config file 'deploy/olm-catalog/csv-config }, } - cmd.Flags().StringVar(&c.csvVersion, "csv-version", "", "Semantic version of the CSV") + cmd.Flags().StringVar(&c.csvVersion, "csv-version", "", + "Semantic version of the CSV") if err := cmd.MarkFlagRequired("csv-version"); err != nil { log.Fatalf("Failed to mark `csv-version` flag for `generate csv` subcommand as required: %v", err) } cmd.Flags().StringVar(&c.fromVersion, "from-version", "", "Semantic version of an existing CSV to use as a base") - cmd.Flags().StringVar(&c.csvConfigPath, "csv-config", "", - "Path to CSV config file. Defaults to deploy/olm-catalog/csv-config.yaml") + cmd.Flags().StringSliceVar(&c.includePaths, "include", []string{scaffold.DeployDir}, + "Paths to include in CSV generation, ex. \"deploy/prod,deploy/test\". "+ + "If this flag is set and you want to enable default behavior, "+ + "you must include \"deploy/\" in the argument list") + cmd.Flags().StringVar(&c.outputDir, "output-dir", scaffold.DeployDir, + "Base directory to output generated CSV. The resulting CSV bundle directory"+ + "will be \"/olm-catalog//\"") cmd.Flags().BoolVar(&c.updateCRDs, "update-crds", false, "Update CRD manifests in deploy/{operator-name}/{csv-version} the using latest API's") cmd.Flags().StringVar(&c.operatorName, "operator-name", "", @@ -97,60 +102,42 @@ Configure CSV generation by writing a config file 'deploy/olm-catalog/csv-config func (c csvCmd) run() error { - absProjectPath := projutil.MustGetwd() - cfg := &input.Config{ - AbsProjectPath: absProjectPath, - ProjectName: filepath.Base(absProjectPath), - } - if projutil.IsOperatorGo() { - cfg.Repo = projutil.GetGoPkg() - } - log.Infof("Generating CSV manifest version %s", c.csvVersion) - csvCfg, err := catalog.GetCSVConfig(c.csvConfigPath) - if err != nil { - return err - } if c.operatorName == "" { - // Use config operator name if not set by CLI, i.e. prefer CLI value over - // config value. - if c.operatorName = csvCfg.OperatorName; c.operatorName == "" { - // Default to using project name if both are empty. - c.operatorName = filepath.Base(absProjectPath) - } + c.operatorName = filepath.Base(projutil.MustGetwd()) } - - s := &scaffold.Scaffold{} - csv := &catalog.CSV{ - CSVVersion: c.csvVersion, - FromVersion: c.fromVersion, - ConfigFilePath: c.csvConfigPath, - OperatorName: c.operatorName, - } - err = s.Execute(cfg, csv) - if err != nil { - return fmt.Errorf("catalog scaffold failed: %v", err) + cfg := gen.Config{ + OperatorName: c.operatorName, + OutputDir: c.outputDir, + Filters: gen.MakeFilters(c.includePaths...), } - gcfg := gen.Config{ - OperatorName: c.operatorName, - OutputDir: filepath.Join(gencatalog.OLMCatalogDir, c.operatorName), + csv := gencatalog.NewCSV(cfg, c.csvVersion, c.fromVersion) + if err := csv.Generate(); err != nil { + return fmt.Errorf("error generating CSV: %v", err) } - pkg := gencatalog.NewPackageManifest(gcfg, c.csvVersion, c.csvChannel, c.defaultChannel) + pkg := gencatalog.NewPackageManifest(cfg, c.csvVersion, c.csvChannel, c.defaultChannel) if err := pkg.Generate(); err != nil { return fmt.Errorf("error generating package manifest: %v", err) } // Write CRD's to the new or updated CSV package dir. if c.updateCRDs { - input, err := csv.GetInput() + crdManifestSet, err := findCRDs(c.includePaths...) if err != nil { return err } - err = writeCRDsToDir(csvCfg.CRDCRPaths, filepath.Dir(input.Path)) - if err != nil { - return err + baseDir := c.outputDir + if baseDir == "" { + baseDir = gencatalog.OLMCatalogDir + } + bundleDir := filepath.Join(baseDir, c.operatorName, c.csvVersion) + for path, b := range crdManifestSet { + path = filepath.Join(bundleDir, path) + if err = ioutil.WriteFile(path, b, fileutil.DefaultFileMode); err != nil { + return err + } } } @@ -191,25 +178,44 @@ func validateVersion(version string) error { return nil } -func writeCRDsToDir(crdPaths []string, toDir string) error { - for _, p := range crdPaths { - b, err := ioutil.ReadFile(p) - if err != nil { - return err - } - typeMeta, err := k8sutil.GetTypeMetaFromBytes(b) +// findCRDs searches directories and files in paths for CRD manifest paths, +// returning a map of paths to file contents. +func findCRDs(paths ...string) (map[string][]byte, error) { + crdFileSet := map[string][]byte{} + for _, path := range paths { + info, err := os.Stat(path) if err != nil { return fmt.Errorf("error in %s : %v", p, err) } if typeMeta.Kind != "CustomResourceDefinition" { continue } - - path := filepath.Join(toDir, filepath.Base(p)) - err = ioutil.WriteFile(path, b, fileutil.DefaultFileMode) - if err != nil { - return err + if info.IsDir() { + subsetPaths, err := k8sutil.GetCRDManifestPaths(path) + if err != nil { + return nil, err + } + for _, crdPath := range subsetPaths { + b, err := ioutil.ReadFile(crdPath) + if err != nil { + return nil, err + } + crdFileSet[filepath.Base(crdPath)] = b + } + } else { + b, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + typeMeta, err := k8sutil.GetTypeMetaFromBytes(b) + if err != nil { + log.Infof("Skipping non-manifest file %s", path) + continue + } + if typeMeta.Kind == "CustomResourceDefinition" { + crdFileSet[filepath.Base(path)] = b + } } } - return nil + return crdFileSet, nil } diff --git a/cmd/operator-sdk/run/cmd.go b/cmd/operator-sdk/run/cmd.go index 0691737a6e8..86069455618 100644 --- a/cmd/operator-sdk/run/cmd.go +++ b/cmd/operator-sdk/run/cmd.go @@ -19,8 +19,8 @@ import ( "fmt" "path/filepath" + olmcatalog "github.com/operator-framework/operator-sdk/internal/generate/olm-catalog" olmoperator "github.com/operator-framework/operator-sdk/internal/olm/operator" - olmcatalog "github.com/operator-framework/operator-sdk/internal/scaffold/olm-catalog" k8sinternal "github.com/operator-framework/operator-sdk/internal/util/k8sutil" "github.com/operator-framework/operator-sdk/internal/util/projutil" aoflags "github.com/operator-framework/operator-sdk/pkg/ansible/flags" diff --git a/doc/cli/operator-sdk_generate_csv.md b/doc/cli/operator-sdk_generate_csv.md index 1b20934ca18..ad9116dbd66 100644 --- a/doc/cli/operator-sdk_generate_csv.md +++ b/doc/cli/operator-sdk_generate_csv.md @@ -11,8 +11,6 @@ A CSV semantic version is supplied via the --csv-version flag. If your operator has already generated a CSV manifest you want to use as a base, supply its version to --from-version. Otherwise the SDK will scaffold a new CSV manifest. -Configure CSV generation by writing a config file 'deploy/olm-catalog/csv-config.yaml - ``` operator-sdk generate csv [flags] ``` @@ -21,12 +19,13 @@ operator-sdk generate csv [flags] ``` --csv-channel string Channel the CSV should be registered under in the package manifest - --csv-config string Path to CSV config file. Defaults to deploy/olm-catalog/csv-config.yaml --csv-version string Semantic version of the CSV --default-channel Use the channel passed to --csv-channel as the package manifests' default channel. Only valid when --csv-channel is set --from-version string Semantic version of an existing CSV to use as a base -h, --help help for csv + --include strings Paths to include in CSV generation, ex. "deploy/prod,deploy/test". If this flag is set and you want to enable default behavior, you must include "deploy/" in the argument list (default [deploy]) --operator-name string Operator name to use while generating CSV + --output-dir string Base directory to output generated CSV. The resulting CSV bundle directory will be "/olm-catalog//" (default "deploy") --update-crds Update CRD manifests in deploy/{operator-name}/{csv-version} the using latest API's ``` diff --git a/doc/user/olm-catalog/generating-a-csv.md b/doc/user/olm-catalog/generating-a-csv.md index 4db5492e0d8..4c42508cf74 100644 --- a/doc/user/olm-catalog/generating-a-csv.md +++ b/doc/user/olm-catalog/generating-a-csv.md @@ -13,32 +13,12 @@ This document describes how to manage the following lifecycle for your Operator Operator SDK projects have an expected [project layout][doc-project-layout]. In particular, a few manifests are expected to be present in the `deploy` directory: * Roles: `role.yaml` +* ClusterRoles: `cluster_role.yaml` * Deployments: `operator.yaml` * Custom Resources (CR's): `crds/___cr.yaml` -* Custom Resource Definitions (CRD's): `crds/__crd.yaml`. +* CustomResourceDefinitions (CRD's): `crds/__crd.yaml`. -`generate csv` reads these files and adds their data to a CSV in an alternate form. - -The following example config containing default values should be copied and written to `deploy/olm-catalog/csv-config.yaml`: - -```yaml -crd-cr-paths: -- deploy/crds -operator-path: deploy/operator.yaml -role-paths: -- deploy/role.yaml -``` - -Explanation of all config fields: - -- `crd-cr-paths`: list of strings - a list of CRD and CR manifest file/directory paths. Defaults to `[deploy/crds]`. -- `operator-path`: string - the operator `Deployment` manifest file path. Defaults to `deploy/operator.yaml`. -- `role-paths`: list of strings - Role and ClusterRole manifest file paths. Defaults to `[deploy/role.yaml]`. -- `operator-name`: string - the name used to create the CSV and manifest file names. Defaults to the project's name. - -**Note**: The [design doc][doc-csv-design] has outdated field information which should not be referenced. - -Fields in this config file can be modified to point towards alternate manifest locations, and passed to `generate csv --csv-config=` to configure CSV generation. For example, if I have one set of production CR/CRD manifests under `deploy/crds/production`, and a set of test manifests under `deploy/crds/test`, and I only want to include production manifests in my CSV, I can set `crd-cr-paths: [deploy/crds/production]`. `generate csv` will then ignore `deploy/crds/test` when getting CR/CRD data. +`generate csv` extracts manifests from files in `deploy/` by default that match the kinds above and adds them to the CSV. If your manifest files are not in `deploy/`, you can use the `--include=[list of paths]` option to instruct the command to extract manifests from files at those paths, ex. `--include="deploy/prod,deploy/test"`. Setting `--include` overrides default behavior; if you still want default behavior, you must append `deploy/` to the list of paths passed to `--include`. ## Versioning @@ -46,7 +26,7 @@ CSV's are versioned in path, file name, and in their `metadata.name` field. For `generate csv` allows you to upgrade your CSV using the `--from-version` flag. If you have an existing CSV with version `0.0.1` and want to write a new version `0.0.2`, you can run `operator-sdk generate csv --csv-version 0.0.2 --from-version 0.0.1`. This will write a new CSV manifest to `deploy/olm-catalog//0.0.2/.v0.0.2.clusterserviceversion.yaml` containing user-defined data from `0.0.1` and any modifications you've made to `roles.yaml`, `operator.yaml`, CR's, or CRD's. -The SDK can manage CRD's in your Operator bundle as well. You can pass the `--update-crds` flag to `generate csv` to add or update your CRD's in your bundle by copying manifests pointed to by `crd-cr-paths` in your config. CRD's in a bundle are not updated by default. +The SDK can manage CRD's in your Operator bundle as well. You can pass the `--update-crds` flag to `generate csv` to add or update your CRD's in your bundle by copying manifests in `deploy/crds` to your bundle. CRD's in a bundle are not updated by default. ## First Generation diff --git a/internal/generate/olm-catalog/csv.go b/internal/generate/olm-catalog/csv.go index 5a70c3fd951..c2665afdd74 100644 --- a/internal/generate/olm-catalog/csv.go +++ b/internal/generate/olm-catalog/csv.go @@ -15,15 +15,418 @@ package olmcatalog import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" "path/filepath" + "regexp" + "sort" + "strings" + "github.com/operator-framework/operator-sdk/internal/generate/gen" "github.com/operator-framework/operator-sdk/internal/scaffold" + "github.com/operator-framework/operator-sdk/internal/util/fileutil" + "github.com/operator-framework/operator-sdk/internal/util/k8sutil" + "github.com/operator-framework/operator-sdk/internal/util/projutil" + "github.com/operator-framework/operator-sdk/internal/util/yamlutil" + + "github.com/blang/semver" + "github.com/ghodss/yaml" + olmapiv1alpha1 "github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1" + olminstall "github.com/operator-framework/operator-lifecycle-manager/pkg/controller/install" + olmversion "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/version" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" ) const ( - OLMCatalogDir = scaffold.DeployDir + string(filepath.Separator) + "olm-catalog" + olmCatalogChildDir = "olm-catalog" + OLMCatalogDir = scaffold.DeployDir + string(filepath.Separator) + olmCatalogChildDir + CSVYamlFileExt = ".clusterserviceversion.yaml" + + BundleDirKey = "bundle" + DeployDirKey = "deploy" + APIsDirKey = "apis" ) +type csvGenerator struct { + gen.Config + // csvVersion is the CSV current version. + csvVersion string + // fromVersion is the CSV version from which to build a new CSV. A CSV + // manifest with this version should exist at: + // deploy/olm-catalog/{from_version}/operator-name.v{from_version}.{CSVYamlFileExt} + fromVersion string +} + +func NewCSV(cfg gen.Config, csvVersion, fromVersion string) gen.Generator { + g := csvGenerator{ + Config: cfg, + csvVersion: csvVersion, + fromVersion: fromVersion, + } + if g.Inputs == nil { + g.Inputs = map[string]string{} + } + // The olm-catalog directory location depends on where the deploy (operator + // manifests) directory is. + olmCatalogDir := OLMCatalogDir + if deployDir, ok := g.Inputs[DeployDirKey]; !ok || deployDir == "" { + g.Inputs[DeployDirKey] = scaffold.DeployDir + } else { + // Set to non-standard location. + olmCatalogDir = filepath.Join(g.Inputs[DeployDirKey], olmCatalogChildDir) + } + if bundle, ok := g.Inputs[BundleDirKey]; !ok || bundle == "" { + // Initialize so we don't have to check for key existence elsewhere. + g.Inputs[BundleDirKey] = "" + parentDir := filepath.Join(olmCatalogDir, g.OperatorName) + if isBundleDirExist(parentDir, g.fromVersion) { + g.Inputs[BundleDirKey] = filepath.Join(olmCatalogDir, g.OperatorName, g.fromVersion) + } else if isBundleDirExist(parentDir, g.csvVersion) { + g.Inputs[BundleDirKey] = filepath.Join(olmCatalogDir, g.OperatorName, g.csvVersion) + } + } + if apisDir, ok := g.Inputs[APIsDirKey]; !ok || apisDir == "" { + g.Inputs[APIsDirKey] = scaffold.ApisDir + } + if g.OutputDir == "" { + g.OutputDir = filepath.Join(olmCatalogDir, g.OperatorName, g.csvVersion) + } + return g +} + +func isBundleDirExist(parentDir, version string) bool { + // Ensure full path is constructed. + if parentDir == "" || version == "" { + return false + } + _, err := os.Stat(filepath.Join(parentDir, version)) + return err == nil || os.IsExist(err) +} + func getCSVName(name, version string) string { return name + ".v" + version } + +func getCSVFileName(name, version string) string { + return getCSVName(strings.ToLower(name), version) + CSVYamlFileExt +} + +// Generate allows a CSV to be written by marshalling +// olmapiv1alpha1.ClusterServiceVersion instead of writing to a template. +func (g csvGenerator) Generate() error { + fileMap, err := g.generate() + if err != nil { + return err + } + if len(fileMap) == 0 { + return errors.New("error generating CSV manifest: no generated file found") + } + if err = os.MkdirAll(g.OutputDir, fileutil.DefaultDirFileMode); err != nil { + return errors.Wrapf(err, "error mkdir %s", g.OutputDir) + } + for fileName, b := range fileMap { + path := filepath.Join(g.OutputDir, fileName) + if err = ioutil.WriteFile(path, b, fileutil.DefaultFileMode); err != nil { + return err + } + } + return nil +} + +func (g csvGenerator) generate() (fileMap map[string][]byte, err error) { + bundle := g.Inputs[BundleDirKey] + // Get current CSV to update, otherwise start with a fresh CSV. + var csv *olmapiv1alpha1.ClusterServiceVersion + if bundle != "" { + if csv, err = getCSVFromDir(bundle); err != nil { + return nil, err + } + // TODO: validate existing CSV. + if err = g.updateCSVVersions(csv); err != nil { + return nil, err + } + } else { + if csv, err = newCSV(g.OperatorName, g.csvVersion); err != nil { + return nil, err + } + } + + if err = g.updateCSVFromManifests(csv); err != nil { + return nil, err + } + + path := getCSVFileName(g.OperatorName, g.csvVersion) + if fields := getEmptyRequiredCSVFields(csv); len(fields) != 0 { + if bundle != "" { + log.Warnf("Required csv fields not filled in file %s:%s\n", path, joinFields(fields)) + } else { + // A new csv won't have several required fields populated. + // Report required fields to user informationally. + log.Infof("Fill in the following required fields in file %s:%s\n", path, joinFields(fields)) + } + } + + b, err := k8sutil.GetObjectBytes(csv, yaml.Marshal) + if err != nil { + return nil, err + } + fileMap = map[string][]byte{ + path: b, + } + return fileMap, nil +} + +func getCSVFromDir(dir string) (*olmapiv1alpha1.ClusterServiceVersion, error) { + infos, err := ioutil.ReadDir(dir) + if err != nil { + return nil, err + } + for _, info := range infos { + path := filepath.Join(dir, info.Name()) + info, err := os.Stat(path) + if err != nil || info.IsDir() { + // Skip any directories or files accessed in error. + continue + } + b, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + typeMeta, err := k8sutil.GetTypeMetaFromBytes(b) + if err != nil { + return nil, err + } + if typeMeta.Kind != olmapiv1alpha1.ClusterServiceVersionKind { + continue + } + csv := &olmapiv1alpha1.ClusterServiceVersion{} + if err := yaml.Unmarshal(b, csv); err != nil { + return nil, errors.Wrapf(err, "error unmarshalling CSV %s", path) + } + return csv, nil + } + return nil, fmt.Errorf("no CSV manifest in %s", dir) +} + +// newCSV sets all csv fields that should be populated by a user +// to sane defaults. +func newCSV(name, version string) (*olmapiv1alpha1.ClusterServiceVersion, error) { + ver, err := semver.Parse(version) + if err != nil { + return nil, err + } + return &olmapiv1alpha1.ClusterServiceVersion{ + TypeMeta: metav1.TypeMeta{ + APIVersion: olmapiv1alpha1.ClusterServiceVersionAPIVersion, + Kind: olmapiv1alpha1.ClusterServiceVersionKind, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: getCSVName(name, version), + Namespace: "placeholder", + Annotations: map[string]string{ + "capabilities": "Basic Install", + }, + }, + Spec: olmapiv1alpha1.ClusterServiceVersionSpec{ + DisplayName: k8sutil.GetDisplayName(name), + Description: "", + Provider: olmapiv1alpha1.AppLink{}, + Maintainers: make([]olmapiv1alpha1.Maintainer, 1), + Links: []olmapiv1alpha1.AppLink{}, + Maturity: "alpha", + Version: olmversion.OperatorVersion{Version: ver}, + Icon: make([]olmapiv1alpha1.Icon, 1), + Keywords: []string{""}, + InstallModes: []olmapiv1alpha1.InstallMode{ + {Type: olmapiv1alpha1.InstallModeTypeOwnNamespace, Supported: true}, + {Type: olmapiv1alpha1.InstallModeTypeSingleNamespace, Supported: true}, + {Type: olmapiv1alpha1.InstallModeTypeMultiNamespace, Supported: false}, + {Type: olmapiv1alpha1.InstallModeTypeAllNamespaces, Supported: true}, + }, + InstallStrategy: olmapiv1alpha1.NamedInstallStrategy{ + StrategyName: olminstall.InstallStrategyNameDeployment, + StrategySpecRaw: json.RawMessage("{}"), + }, + }, + }, nil +} + +// TODO: replace with validation library. +func getEmptyRequiredCSVFields(csv *olmapiv1alpha1.ClusterServiceVersion) (fields []string) { + // Metadata + if csv.TypeMeta.APIVersion != olmapiv1alpha1.ClusterServiceVersionAPIVersion { + fields = append(fields, "apiVersion") + } + if csv.TypeMeta.Kind != olmapiv1alpha1.ClusterServiceVersionKind { + fields = append(fields, "kind") + } + if csv.ObjectMeta.Name == "" { + fields = append(fields, "metadata.name") + } + // Spec fields + if csv.Spec.Version.String() == "" { + fields = append(fields, "spec.version") + } + if csv.Spec.DisplayName == "" { + fields = append(fields, "spec.displayName") + } + if csv.Spec.Description == "" { + fields = append(fields, "spec.description") + } + if len(csv.Spec.Keywords) == 0 || len(csv.Spec.Keywords[0]) == 0 { + fields = append(fields, "spec.keywords") + } + if len(csv.Spec.Maintainers) == 0 { + fields = append(fields, "spec.maintainers") + } + if csv.Spec.Provider == (olmapiv1alpha1.AppLink{}) { + fields = append(fields, "spec.provider") + } + if csv.Spec.Maturity == "" { + fields = append(fields, "spec.maturity") + } + + return fields +} + +func joinFields(fields []string) string { + sb := &strings.Builder{} + for _, f := range fields { + sb.WriteString("\n\t" + f) + } + return sb.String() +} + +// updateCSVVersions updates csv's version and data involving the version, +// ex. ObjectMeta.Name, and place the old version in the `replaces` object, +// if there is an old version to replace. +func (g csvGenerator) updateCSVVersions(csv *olmapiv1alpha1.ClusterServiceVersion) error { + + // Old csv version to replace, and updated csv version. + oldVer, newVer := csv.Spec.Version.String(), g.csvVersion + if oldVer == newVer { + return nil + } + + // Replace all references to the old operator name. + oldCSVName := getCSVName(g.OperatorName, oldVer) + oldRe, err := regexp.Compile(fmt.Sprintf("\\b%s\\b", regexp.QuoteMeta(oldCSVName))) + if err != nil { + return errors.Wrapf(err, "error compiling CSV name regexp %s", oldRe.String()) + } + b, err := yaml.Marshal(csv) + if err != nil { + return err + } + newCSVName := getCSVName(g.OperatorName, newVer) + b = oldRe.ReplaceAll(b, []byte(newCSVName)) + *csv = olmapiv1alpha1.ClusterServiceVersion{} + if err = yaml.Unmarshal(b, csv); err != nil { + return errors.Wrapf(err, "error unmarshalling CSV %s after replacing old CSV name", csv.GetName()) + } + + ver, err := semver.Parse(g.csvVersion) + if err != nil { + return err + } + csv.Spec.Version = olmversion.OperatorVersion{Version: ver} + csv.Spec.Replaces = oldCSVName + return nil +} + +// updateCSVFromManifests gathers relevant data from generated and +// user-defined manifests and updates csv. +func (g csvGenerator) updateCSVFromManifests(csv *olmapiv1alpha1.ClusterServiceVersion) (err error) { + kindManifestMap := map[schema.GroupVersionKind][][]byte{} + crGVKSet := map[schema.GroupVersionKind]struct{}{} + err = filepath.Walk(g.Inputs[DeployDirKey], func(path string, info os.FileInfo, werr error) error { + if werr != nil || info.IsDir() { + return werr + } + if !g.Filters.SatisfiesAny(path) { + return nil + } + b, err := ioutil.ReadFile(path) + if err != nil { + return err + } + scanner := yamlutil.NewYAMLScanner(b) + for scanner.Scan() { + manifest := scanner.Bytes() + typeMeta, err := k8sutil.GetTypeMetaFromBytes(manifest) + if err != nil { + log.Infof("No TypeMeta in %s, skipping file", path) + continue + } + gvk := typeMeta.GroupVersionKind() + kindManifestMap[gvk] = append(kindManifestMap[gvk], manifest) + switch typeMeta.Kind { + case "CustomResourceDefinition": + // Collect CRD kinds to filter them out from unsupported manifest types. + // The CRD version type doesn't matter as long as it has a group, kind, + // and versions in the expected fields. + crd := v1beta1.CustomResourceDefinition{} + if err = yaml.Unmarshal(manifest, &crd); err != nil { + return err + } + for _, ver := range crd.Spec.Versions { + crGVK := schema.GroupVersionKind{ + Group: crd.Spec.Group, + Version: ver.Name, + Kind: crd.Spec.Names.Kind, + } + crGVKSet[crGVK] = struct{}{} + } + } + } + return scanner.Err() + }) + + crUpdaters := crs{} + for gvk, manifests := range kindManifestMap { + // We don't necessarily care about sorting by a field value, more about + // consistent ordering. + sort.Slice(manifests, func(i int, j int) bool { + return string(manifests[i]) < string(manifests[j]) + }) + switch gvk.Kind { + case "Role": + err = roles(manifests).apply(csv) + case "ClusterRole": + err = clusterRoles(manifests).apply(csv) + case "Deployment": + err = deployments(manifests).apply(csv) + case "CustomResourceDefinition": + err = crds(manifests).apply(csv) + default: + if _, ok := crGVKSet[gvk]; ok { + crUpdaters = append(crUpdaters, crs(manifests)...) + } else { + log.Infof("Skipping manifest %s", gvk) + } + } + if err != nil { + return err + } + } + err = updateDescriptions(csv, g.Inputs[APIsDirKey], projutil.GetOperatorType()) + if err != nil { + return fmt.Errorf("error updating CSV customresourcedefinitions: %w", err) + } + // Re-sort CR's since they are appended in random order. + if len(crUpdaters) != 0 { + sort.Slice(crUpdaters, func(i int, j int) bool { + return string(crUpdaters[i]) < string(crUpdaters[j]) + }) + if err = crUpdaters.apply(csv); err != nil { + return err + } + } + return nil +} diff --git a/internal/generate/olm-catalog/csv_go_test.go b/internal/generate/olm-catalog/csv_go_test.go new file mode 100644 index 00000000000..88779dd12c0 --- /dev/null +++ b/internal/generate/olm-catalog/csv_go_test.go @@ -0,0 +1,249 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package olmcatalog + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + + gen "github.com/operator-framework/operator-sdk/internal/generate/gen" + "github.com/operator-framework/operator-sdk/internal/scaffold" + internalk8sutil "github.com/operator-framework/operator-sdk/internal/util/k8sutil" + "github.com/operator-framework/operator-sdk/pkg/k8sutil" + + "github.com/blang/semver" + "github.com/ghodss/yaml" + "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" +) + +const ( + sdkCmd = "operator-sdk" + + testProjectName = "memcached-operator" + csvVersion = "0.0.3" + fromVersion = "0.0.2" + notExistVersion = "1.0.0" + scratchBundleDir = "scratch" + testGroup = "cache.example.com" + testKind1 = "Memcached" + testVersion1 = "v1alpha1" +) + +var ( + testDataDir = filepath.Join("..", "testdata") + testGoDataDir = filepath.Join(testDataDir, "go") +) + +func setupTestEnvWithCleanup(t *testing.T, dataDir string) (cleanupFuncs []func()) { + wd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + if err := os.Chdir(dataDir); err != nil { + t.Fatal(err) + } + cleanupFuncs = append(cleanupFuncs, func() { + if err := os.Chdir(wd); err != nil { + t.Fatal(err) + } + }) + return cleanupFuncs +} + +func TestGoCSVGoNew(t *testing.T) { + for _, cleanupFunc := range setupTestEnvWithCleanup(t, testGoDataDir) { + defer cleanupFunc() + } + + cfg := gen.Config{ + OperatorName: testProjectName, + Filters: gen.MakeFilters(scaffold.DeployDir), + } + g := NewCSV(cfg, csvVersion, "") + fileMap, err := g.(csvGenerator).generate() + if err != nil { + t.Fatalf("Failed to execute CSV generator: %v", err) + } + + csvExpFile := getCSVFileName(testProjectName, csvVersion) + csvExpBytes, err := ioutil.ReadFile(filepath.Join(OLMCatalogDir, testProjectName, csvVersion, csvExpFile)) + if err != nil { + t.Fatalf("Failed to read expected CSV file: %v", err) + } + csvExp := string(csvExpBytes) + // Replace image tag, which is retrieved from the deployment and is + // different than that in the expected CSV, but doesn't matter for this test. + csvExp = strings.Replace(csvExp, + "image: quay.io/example/memcached-operator:v0.0.2", + "image: quay.io/example/memcached-operator:v0.0.3", + -1) + if b, ok := fileMap[csvExpFile]; !ok { + t.Errorf("Failed to generate CSV for version %s", csvVersion) + } else { + assert.Equal(t, csvExp, string(b)) + } +} + +func TestGoCSVFromOld(t *testing.T) { + for _, cleanupFunc := range setupTestEnvWithCleanup(t, testGoDataDir) { + defer cleanupFunc() + } + + cfg := gen.Config{ + OperatorName: testProjectName, + Filters: gen.MakeFilters(scaffold.DeployDir), + } + g := NewCSV(cfg, csvVersion, fromVersion) + fileMap, err := g.(csvGenerator).generate() + if err != nil { + t.Fatalf("Failed to execute CSV generator: %v", err) + } + + csvExpFile := getCSVFileName(testProjectName, csvVersion) + csvExpBytes, err := ioutil.ReadFile(filepath.Join(OLMCatalogDir, testProjectName, csvVersion, csvExpFile)) + if err != nil { + t.Fatalf("Failed to read expected CSV file: %v", err) + } + csvExp := string(csvExpBytes) + if b, ok := fileMap[csvExpFile]; !ok { + t.Errorf("Failed to generate CSV for version %s", csvVersion) + } else { + assert.Equal(t, csvExp, string(b)) + } +} + +func TestGoCSVIncludeAll(t *testing.T) { + cfg := gen.Config{OperatorName: testProjectName} + g := NewCSV(cfg, notExistVersion, "") + fileMap, err := g.(csvGenerator).generate() + if err != nil { + t.Fatalf("Failed to execute CSV generator: %v", err) + } + + // Create an empty CSV. + csv, err := newCSV(testProjectName, notExistVersion) + if err != nil { + t.Fatal(err) + } + csvExpBytes, err := internalk8sutil.GetObjectBytes(csv, yaml.Marshal) + if err != nil { + t.Fatal(err) + } + csvExpFile := getCSVFileName(testProjectName, notExistVersion) + if b, ok := fileMap[csvExpFile]; !ok { + t.Errorf("Failed to generate CSV for version %s", notExistVersion) + } else { + assert.Equal(t, string(csvExpBytes), string(b)) + } +} + +func getTestCRDFile(g, k string) string { + return fmt.Sprintf("%s_%s_crd.yaml", g, strings.ToLower(k)+"s") +} + +func getTestCRFile(g, v, k string) string { + return fmt.Sprintf("%s_%s_%s_cr.yaml", g, v, strings.ToLower(k)) +} + +func TestUpdateVersion(t *testing.T) { + csv, err := getCSVFromDir(filepath.Join(testGoDataDir, OLMCatalogDir, testProjectName, fromVersion)) + if err != nil { + t.Fatal("Failed to get new CSV") + } + + cfg := gen.Config{OperatorName: testProjectName} + g := NewCSV(cfg, csvVersion, fromVersion) + if err := g.(csvGenerator).updateCSVVersions(csv); err != nil { + t.Fatalf("Failed to update csv with version %s: (%v)", csvVersion, err) + } + + wantedSemver, err := semver.Parse(csvVersion) + if err != nil { + t.Errorf("Failed to parse %s: %v", csvVersion, err) + } + if !csv.Spec.Version.Equals(wantedSemver) { + t.Errorf("Wanted csv version %v, got %v", wantedSemver, csv.Spec.Version) + } + wantedName := getCSVName(testProjectName, csvVersion) + if csv.ObjectMeta.Name != wantedName { + t.Errorf("Wanted csv name %s, got %s", wantedName, csv.ObjectMeta.Name) + } + + csvDepSpecs := csv.Spec.InstallStrategy.StrategySpec.DeploymentSpecs + if len(csvDepSpecs) != 1 { + t.Fatal("No deployment specs in CSV") + } + csvPodImage := csvDepSpecs[0].Spec.Template.Spec.Containers[0].Image + if len(csvDepSpecs[0].Spec.Template.Spec.Containers) != 1 { + t.Fatal("No containers in CSV deployment spec") + } + // updateCSVVersions should not update podspec image. + wantedImage := "quay.io/example/memcached-operator:v0.0.2" + if csvPodImage != wantedImage { + t.Errorf("Podspec image changed from %s to %s", wantedImage, csvPodImage) + } + + wantedReplaces := getCSVName(testProjectName, fromVersion) + if csv.Spec.Replaces != wantedReplaces { + t.Errorf("Wanted csv replaces %s, got %s", wantedReplaces, csv.Spec.Replaces) + } +} + +func TestSetAndCheckOLMNamespaces(t *testing.T) { + depBytes, err := ioutil.ReadFile(filepath.Join(testGoDataDir, scaffold.DeployDir, "operator.yaml")) + if err != nil { + t.Fatalf("Failed to read Deployment bytes: %v", err) + } + + // The test operator.yaml doesn't have "olm.targetNamespaces", so first + // check that depHasOLMNamespaces() returns false. + dep := appsv1.Deployment{} + if err := yaml.Unmarshal(depBytes, &dep); err != nil { + t.Fatalf("Failed to unmarshal Deployment bytes: %v", err) + } + if depHasOLMNamespaces(dep) { + t.Error("Expected depHasOLMNamespaces to return false, got true") + } + + // Insert "olm.targetNamespaces" into WATCH_NAMESPACE and check that + // depHasOLMNamespaces() returns true. + setWatchNamespacesEnv(&dep) + if !depHasOLMNamespaces(dep) { + t.Error("Expected depHasOLMNamespaces to return true, got false") + } + + // Overwrite WATCH_NAMESPACE and check that depHasOLMNamespaces() returns + // false. + overwriteContainerEnvVar(&dep, k8sutil.WatchNamespaceEnvVar, newEnvVar("FOO", "bar")) + if depHasOLMNamespaces(dep) { + t.Error("Expected depHasOLMNamespaces to return false, got true") + } + + // Insert "olm.targetNamespaces" elsewhere in the deployment pod spec + // and check that depHasOLMNamespaces() returns true. + dep = appsv1.Deployment{} + if err := yaml.Unmarshal(depBytes, &dep); err != nil { + t.Fatalf("Failed to unmarshal Deployment bytes: %v", err) + } + dep.Spec.Template.ObjectMeta.Labels["namespace"] = olmTNMeta + if !depHasOLMNamespaces(dep) { + t.Error("Expected depHasOLMNamespaces to return true, got false") + } +} diff --git a/internal/scaffold/olm-catalog/csv_updaters.go b/internal/generate/olm-catalog/csv_updaters.go similarity index 82% rename from internal/scaffold/olm-catalog/csv_updaters.go rename to internal/generate/olm-catalog/csv_updaters.go index da9cb3a5180..ede307cc810 100644 --- a/internal/scaffold/olm-catalog/csv_updaters.go +++ b/internal/generate/olm-catalog/csv_updaters.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package catalog +package olmcatalog import ( "bytes" @@ -20,9 +20,11 @@ import ( goerrors "errors" "fmt" "sort" + "strings" - "github.com/operator-framework/operator-sdk/internal/scaffold" - "github.com/operator-framework/operator-sdk/internal/scaffold/olm-catalog/descriptor" + "github.com/operator-framework/operator-registry/pkg/registry" + "github.com/operator-framework/operator-sdk/internal/generate/olm-catalog/descriptor" + "github.com/operator-framework/operator-sdk/internal/util/projutil" "github.com/operator-framework/operator-sdk/pkg/k8sutil" "github.com/ghodss/yaml" @@ -202,20 +204,62 @@ var _ csvUpdater = crds{} // apply will only make a new spec.customresourcedefinitions.owned element for // a type if an annotation is present on that type's declaration. func (us crds) apply(csv *olmapiv1alpha1.ClusterServiceVersion) error { - ownedCRDs := []olmapiv1alpha1.CRDDescription{} + ownedDescs := []olmapiv1alpha1.CRDDescription{} + descMap := map[registry.DefinitionKey]olmapiv1alpha1.CRDDescription{} + for _, owned := range csv.Spec.CustomResourceDefinitions.Owned { + defKey := registry.DefinitionKey{ + Name: owned.Name, + Version: owned.Version, + Kind: owned.Kind, + } + descMap[defKey] = owned + } for _, u := range us { crd := apiextv1beta1.CustomResourceDefinition{} if err := yaml.Unmarshal(u, &crd); err != nil { return err } for _, ver := range crd.Spec.Versions { - // Parse CRD descriptors from source code comments and annotations. - gvk := schema.GroupVersionKind{ - Group: crd.Spec.Group, + defKey := registry.DefinitionKey{ + Name: crd.GetName(), Version: ver.Name, Kind: crd.Spec.Names.Kind, } - newCRDDesc, err := descriptor.GetCRDDescriptionForGVK(scaffold.ApisDir, gvk) + if owned, ownedExists := descMap[defKey]; ownedExists { + ownedDescs = append(ownedDescs, owned) + } else { + ownedDescs = append(ownedDescs, olmapiv1alpha1.CRDDescription{ + Name: defKey.Name, + Version: defKey.Version, + Kind: defKey.Kind, + }) + } + } + } + csv.Spec.CustomResourceDefinitions.Owned = ownedDescs + sort.Sort(descSorter(csv.Spec.CustomResourceDefinitions.Owned)) + sort.Sort(descSorter(csv.Spec.CustomResourceDefinitions.Required)) + return nil +} + +func updateDescriptions(csv *olmapiv1alpha1.ClusterServiceVersion, searchDir string, opType projutil.OperatorType) error { + ownedCRDs := []olmapiv1alpha1.CRDDescription{} + for _, ownedCRD := range csv.Spec.CustomResourceDefinitions.Owned { + name := ownedCRD.Name + group := name + if split := strings.Split(name, "."); len(split) > 1 { + group = strings.Join(split[1:], ".") + } + // Parse CRD descriptors from source code comments and annotations. + gvk := schema.GroupVersionKind{ + Group: group, + Version: ownedCRD.Version, + Kind: ownedCRD.Kind, + } + var err error + switch opType { + case projutil.OperatorTypeGo: + ownedCRD, err = descriptor.GetCRDDescriptionForGVK(searchDir, gvk) if err != nil { if goerrors.Is(err, descriptor.ErrAPIDirNotExist) { log.Infof("Directory for API %s does not exist. Skipping CSV annotation parsing for API.", gvk) @@ -227,9 +271,9 @@ func (us crds) apply(csv *olmapiv1alpha1.ClusterServiceVersion) error { continue } // Only set the name if no error was returned. - newCRDDesc.Name = crd.GetName() - ownedCRDs = append(ownedCRDs, newCRDDesc) + ownedCRD.Name = name } + ownedCRDs = append(ownedCRDs, ownedCRD) } csv.Spec.CustomResourceDefinitions.Owned = ownedCRDs sort.Sort(descSorter(csv.Spec.CustomResourceDefinitions.Owned)) diff --git a/internal/scaffold/olm-catalog/descriptor/descriptor.go b/internal/generate/olm-catalog/descriptor/descriptor.go similarity index 100% rename from internal/scaffold/olm-catalog/descriptor/descriptor.go rename to internal/generate/olm-catalog/descriptor/descriptor.go diff --git a/internal/scaffold/olm-catalog/descriptor/descriptor_test.go b/internal/generate/olm-catalog/descriptor/descriptor_test.go similarity index 89% rename from internal/scaffold/olm-catalog/descriptor/descriptor_test.go rename to internal/generate/olm-catalog/descriptor/descriptor_test.go index 07744be74f7..5b82a0cb356 100644 --- a/internal/scaffold/olm-catalog/descriptor/descriptor_test.go +++ b/internal/generate/olm-catalog/descriptor/descriptor_test.go @@ -18,38 +18,25 @@ import ( "os" "path/filepath" "reflect" - "strings" "testing" "github.com/operator-framework/operator-sdk/internal/util/diffutil" + "github.com/operator-framework/operator-sdk/internal/util/fileutil" "github.com/ghodss/yaml" olmapiv1alpha1 "github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1" "k8s.io/apimachinery/pkg/runtime/schema" ) -const testFrameworkPackage = "github.com/operator-framework/operator-sdk/test/test-framework" - -func getTestFrameworkDir(t *testing.T) string { - t.Helper() - absPath, err := os.Getwd() - if err != nil { - t.Fatal(err) - } - sdkPath := absPath[:strings.Index(absPath, "internal")] - tfDir := filepath.Join(sdkPath, "test", "test-framework") - // parser.AddDirRecursive doesn't like absolute paths. - relPath, err := filepath.Rel(absPath, tfDir) - if err != nil { - t.Fatal(err) - } - return relPath -} +var ( + testDataDir = filepath.Join("..", "..", "testdata") + testGoDataDir = filepath.Join(testDataDir, "go") +) func TestGetKindTypeForAPI(t *testing.T) { cases := []struct { description string - pkg, kind string + kind string numPkgTypes int wantNil bool }{ @@ -66,8 +53,7 @@ func TestGetKindTypeForAPI(t *testing.T) { if err != nil { t.Fatal(err) } - tfDir := getTestFrameworkDir(t) - if err := os.Chdir(tfDir); err != nil { + if err := os.Chdir(testGoDataDir); err != nil { t.Fatal(err) } defer func() { @@ -75,14 +61,14 @@ func TestGetKindTypeForAPI(t *testing.T) { t.Fatal(err) } }() - tfAPIDir := filepath.Join("pkg", "apis", "cache", "v1alpha1") - universe, err := getTypesFromDir(tfAPIDir) + testAPIDir := filepath.Join("pkg", "apis", "cache", "v1alpha1") + universe, err := getTypesFromDir(testAPIDir) if err != nil { t.Fatal(err) } for _, c := range cases { - pkgTypes, err := getTypesForPkg(c.pkg, universe) + pkgTypes, err := getTypesForPkg(fileutil.DotPath(testAPIDir), universe) if err != nil { t.Fatal(err) } @@ -107,8 +93,7 @@ func TestGetCRDDescriptionForGVK(t *testing.T) { if err != nil { t.Fatal(err) } - tfDir := getTestFrameworkDir(t) - if err := os.Chdir(tfDir); err != nil { + if err := os.Chdir(testGoDataDir); err != nil { t.Fatal(err) } defer func() { diff --git a/internal/scaffold/olm-catalog/descriptor/parse.go b/internal/generate/olm-catalog/descriptor/parse.go similarity index 100% rename from internal/scaffold/olm-catalog/descriptor/parse.go rename to internal/generate/olm-catalog/descriptor/parse.go diff --git a/internal/scaffold/olm-catalog/descriptor/parse_test.go b/internal/generate/olm-catalog/descriptor/parse_test.go similarity index 100% rename from internal/scaffold/olm-catalog/descriptor/parse_test.go rename to internal/generate/olm-catalog/descriptor/parse_test.go diff --git a/internal/scaffold/olm-catalog/descriptor/search.go b/internal/generate/olm-catalog/descriptor/search.go similarity index 100% rename from internal/scaffold/olm-catalog/descriptor/search.go rename to internal/generate/olm-catalog/descriptor/search.go diff --git a/internal/generate/olm-catalog/package_manifest.go b/internal/generate/olm-catalog/package_manifest.go index ae3c0aecda3..8c875abc089 100644 --- a/internal/generate/olm-catalog/package_manifest.go +++ b/internal/generate/olm-catalog/package_manifest.go @@ -63,11 +63,19 @@ func NewPackageManifest(cfg gen.Config, csvVersion, channel string, isDefault bo if g.Inputs == nil { g.Inputs = map[string]string{} } + // The olm-catalog directory location depends on where the deploy (operator + // manifests) directory is. + olmCatalogDir := OLMCatalogDir + if deployDir, ok := g.Inputs[DeployDirKey]; ok && deployDir != "" { + // Set to non-standard location. + olmCatalogDir = filepath.Join(g.Inputs[DeployDirKey], olmCatalogChildDir) + } + if manifests, ok := g.Inputs[ManifestsDirKey]; !ok || manifests == "" { - g.Inputs[ManifestsDirKey] = filepath.Join(OLMCatalogDir, g.OperatorName) + g.Inputs[ManifestsDirKey] = filepath.Join(olmCatalogDir, g.OperatorName) } if g.OutputDir == "" { - g.OutputDir = filepath.Join(OLMCatalogDir, g.OperatorName) + g.OutputDir = filepath.Join(olmCatalogDir, g.OperatorName) } return g } diff --git a/internal/generate/olm-catalog/package_manifest_test.go b/internal/generate/olm-catalog/package_manifest_test.go index 438f4125d02..4ea8dded268 100644 --- a/internal/generate/olm-catalog/package_manifest_test.go +++ b/internal/generate/olm-catalog/package_manifest_test.go @@ -25,16 +25,6 @@ import ( "github.com/stretchr/testify/assert" ) -const ( - testProjectName = "memcached-operator" - csvVersion = "0.0.3" -) - -var ( - testDataDir = filepath.Join("..", "testdata") - testGoDataDir = filepath.Join(testDataDir, "go") -) - // newTestPackageManifestGenerator returns a package manifest Generator populated with test values. func newTestPackageManifestGenerator() gen.Generator { inputDir := filepath.Join(testGoDataDir, OLMCatalogDir, testProjectName) diff --git a/test/test-framework/pkg/apis/cache/v1alpha1/dummy_types.go b/internal/generate/testdata/go/pkg/apis/cache/v1alpha1/dummy_types.go similarity index 99% rename from test/test-framework/pkg/apis/cache/v1alpha1/dummy_types.go rename to internal/generate/testdata/go/pkg/apis/cache/v1alpha1/dummy_types.go index b255b336ad8..e6ee1101a7c 100644 --- a/test/test-framework/pkg/apis/cache/v1alpha1/dummy_types.go +++ b/internal/generate/testdata/go/pkg/apis/cache/v1alpha1/dummy_types.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Operator-SDK Authors +// Copyright 2020 The Operator-SDK Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/internal/scaffold/olm-catalog/config.go b/internal/scaffold/olm-catalog/config.go deleted file mode 100644 index acddbdbd5b6..00000000000 --- a/internal/scaffold/olm-catalog/config.go +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright 2018 The Operator-SDK Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package catalog - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - - "github.com/operator-framework/operator-sdk/internal/scaffold" - - "github.com/ghodss/yaml" - log "github.com/sirupsen/logrus" -) - -// CSVConfig is a configuration file for CSV composition. Its fields contain -// file path information. -// TODO(estroz): define field for path to write CSV bundle. -// TODO(estroz): make CSVConfig a viper.Config -type CSVConfig struct { - // The operator manifest file path. Defaults to deploy/operator.yaml. - OperatorPath string `json:"operator-path,omitempty"` - // Role and ClusterRole manifest file paths. Defaults to [deploy/role.yaml]. - RolePaths []string `json:"role-paths,omitempty"` - // A list of CRD and CR manifest file paths. Defaults to [deploy/crds]. - CRDCRPaths []string `json:"crd-cr-paths,omitempty"` - // OperatorName is the name used to create the CSV and manifest file names. - // Defaults to the project's name. - OperatorName string `json:"operator-name,omitempty"` -} - -// TODO: discuss case of no config file at default path: write new file or not. -func GetCSVConfig(cfgFile string) (*CSVConfig, error) { - cfg := &CSVConfig{} - if _, err := os.Stat(cfgFile); err == nil { - cfgData, err := ioutil.ReadFile(cfgFile) - if err != nil { - return nil, err - } - if err = yaml.Unmarshal(cfgData, cfg); err != nil { - return nil, err - } - } else if !os.IsNotExist(err) { - return nil, err - } - - if err := cfg.setFields(); err != nil { - return nil, err - } - return cfg, nil -} - -const yamlExt = ".yaml" - -func (c *CSVConfig) setFields() error { - if c.OperatorPath == "" { - info, err := (&scaffold.Operator{}).GetInput() - if err != nil { - return err - } - c.OperatorPath = info.Path - } - - if len(c.RolePaths) == 0 { - info, err := (&scaffold.Role{}).GetInput() - if err != nil { - return err - } - c.RolePaths = []string{info.Path} - } - - if len(c.CRDCRPaths) == 0 { - paths, err := getManifestPathsFromDir(scaffold.CRDsDir) - if err != nil && !os.IsNotExist(err) { - return err - } - if os.IsNotExist(err) { - log.Infof("Default CRDs dir %s does not exist. Omitting field spec.customresourcedefinitions.owned"+ - " from CSV.", scaffold.CRDsDir) - } else if len(paths) == 0 { - log.Infof("Default CRDs dir %s is empty. Omitting field spec.customresourcedefinitions.owned"+ - " from CSV.", scaffold.CRDsDir) - } else { - c.CRDCRPaths = paths - } - } else { - // Allow user to specify a list of dirs to search. Avoid duplicate files. - paths, seen := make([]string, 0), make(map[string]struct{}) - for _, path := range c.CRDCRPaths { - info, err := os.Stat(path) - if err != nil { - return err - } - if info.IsDir() { - tmpPaths, err := getManifestPathsFromDir(path) - if err != nil { - return err - } - for _, p := range tmpPaths { - if _, ok := seen[p]; !ok { - paths = append(paths, p) - seen[p] = struct{}{} - } - } - } else if filepath.Ext(path) == yamlExt { - if _, ok := seen[path]; !ok { - paths = append(paths, path) - seen[path] = struct{}{} - } - } - } - c.CRDCRPaths = paths - } - - return nil -} - -func getManifestPathsFromDir(dir string) (paths []string, err error) { - err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info == nil { - return fmt.Errorf("file info for %s was nil", path) - } - if !info.IsDir() && filepath.Ext(path) == yamlExt { - paths = append(paths, path) - } - return nil - }) - if err != nil { - return nil, err - } - return paths, nil -} diff --git a/internal/scaffold/olm-catalog/config_test.go b/internal/scaffold/olm-catalog/config_test.go deleted file mode 100644 index 4699006acaa..00000000000 --- a/internal/scaffold/olm-catalog/config_test.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2018 The Operator-SDK Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package catalog - -import ( - "path/filepath" - "reflect" - "sort" - "testing" - - "github.com/operator-framework/operator-sdk/internal/scaffold" -) - -func TestConfig(t *testing.T) { - crdsDir := filepath.Join(testDataDir, scaffold.CRDsDir) - - cfg := &CSVConfig{ - CRDCRPaths: []string{crdsDir}, - } - if err := cfg.setFields(); err != nil { - t.Errorf("Set fields crd-cr paths dir only: (%v)", err) - } - if len(cfg.CRDCRPaths) != 3 { - t.Errorf("Wanted 3 crd/cr files, got: %v", cfg.CRDCRPaths) - } - - cfg = &CSVConfig{ - CRDCRPaths: []string{crdsDir, filepath.Join(crdsDir, "app.example.com_appservices_crd.yaml")}, - } - if err := cfg.setFields(); err != nil { - t.Errorf("Set fields crd-cr paths dir file mix: (%v)", err) - } - want := []string{ - filepath.Join(crdsDir, "app.example.com_v1alpha1_appservice_cr.yaml"), - filepath.Join(crdsDir, "app.example.com_appservices_crd.yaml"), - filepath.Join(crdsDir, "app.example.com_appservices2_crd.yaml"), - } - sort.Strings(want) - sort.Strings(cfg.CRDCRPaths) - if !reflect.DeepEqual(want, cfg.CRDCRPaths) { - t.Errorf("Files in crd-cr-paths do not match expected:\nwanted: %+q\ngot: %+q", want, cfg.CRDCRPaths) - } -} diff --git a/internal/scaffold/olm-catalog/csv.go b/internal/scaffold/olm-catalog/csv.go deleted file mode 100644 index c8873b19615..00000000000 --- a/internal/scaffold/olm-catalog/csv.go +++ /dev/null @@ -1,430 +0,0 @@ -// Copyright 2018 The Operator-SDK Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package catalog - -import ( - "errors" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "regexp" - "sort" - "strings" - "sync" - - "github.com/operator-framework/operator-sdk/internal/scaffold" - "github.com/operator-framework/operator-sdk/internal/scaffold/input" - "github.com/operator-framework/operator-sdk/internal/util/k8sutil" - "github.com/operator-framework/operator-sdk/internal/util/projutil" - "github.com/operator-framework/operator-sdk/internal/util/yamlutil" - - "github.com/blang/semver" - "github.com/ghodss/yaml" - olmapiv1alpha1 "github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1" - olmversion "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/version" - log "github.com/sirupsen/logrus" - "github.com/spf13/afero" - apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - "k8s.io/apimachinery/pkg/runtime/schema" -) - -const ( - OLMCatalogDir = scaffold.DeployDir + string(filepath.Separator) + "olm-catalog" - CSVYamlFileExt = ".clusterserviceversion.yaml" - CSVConfigYamlFile = "csv-config.yaml" -) - -var ErrNoCSVVersion = errors.New("no CSV version supplied") - -type CSV struct { - input.Input - - // ConfigFilePath is the location of a configuration file path for this - // projects' CSV file. - ConfigFilePath string - // CSVVersion is the CSV current version. - CSVVersion string - // FromVersion is the CSV version from which to build a new CSV. A CSV - // manifest with this version should exist at: - // deploy/olm-catalog/{from_version}/operator-name.v{from_version}.{CSVYamlFileExt} - FromVersion string - // OperatorName is the operator's name, ex. app-operator - OperatorName string - - once sync.Once - fs afero.Fs // For testing, ex. afero.NewMemMapFs() - pathPrefix string // For testing, ex. testdata/deploy/olm-catalog -} - -func (s *CSV) initFS(fs afero.Fs) { - s.once.Do(func() { - s.fs = fs - }) -} - -func (s *CSV) getFS() afero.Fs { - s.initFS(afero.NewOsFs()) - return s.fs -} - -func (s *CSV) GetInput() (input.Input, error) { - // A CSV version is required. - if s.CSVVersion == "" { - return input.Input{}, ErrNoCSVVersion - } - if s.Path == "" { - operatorName := strings.ToLower(s.OperatorName) - // Path is what the operator-registry expects: - // {manifests -> olm-catalog}/{operator_name}/{semver}/{operator_name}.v{semver}.clusterserviceversion.yaml - s.Path = filepath.Join(s.pathPrefix, - OLMCatalogDir, - operatorName, - s.CSVVersion, - getCSVFileName(operatorName, s.CSVVersion), - ) - } - if s.ConfigFilePath == "" { - s.ConfigFilePath = filepath.Join(s.pathPrefix, OLMCatalogDir, CSVConfigYamlFile) - } - return s.Input, nil -} - -func (s *CSV) SetFS(fs afero.Fs) { s.initFS(fs) } - -// CustomRender allows a CSV to be written by marshalling -// olmapiv1alpha1.ClusterServiceVersion instead of writing to a template. -func (s *CSV) CustomRender() ([]byte, error) { - s.initFS(afero.NewOsFs()) - - // Get current CSV to update. - csv, exists, err := s.getBaseCSVIfExists() - if err != nil { - return nil, err - } - if !exists { - csv = &olmapiv1alpha1.ClusterServiceVersion{} - } - - cfg, err := GetCSVConfig(s.ConfigFilePath) - if err != nil { - return nil, err - } - - if err = s.updateCSVVersions(csv); err != nil { - return nil, err - } - if err = s.updateCSVFromManifests(cfg, csv); err != nil { - return nil, err - } - s.setCSVDefaultFields(csv) - - if fields := getEmptyRequiredCSVFields(csv); len(fields) != 0 { - if exists { - log.Warnf("Required csv fields not filled in file %s:%s\n", s.Path, joinFields(fields)) - } else { - // A new csv won't have several required fields populated. - // Report required fields to user informationally. - log.Infof("Fill in the following required fields in file %s:%s\n", s.Path, joinFields(fields)) - } - } - - return k8sutil.GetObjectBytes(csv, yaml.Marshal) -} - -func (s *CSV) getBaseCSVIfExists() (*olmapiv1alpha1.ClusterServiceVersion, bool, error) { - verToGet := s.CSVVersion - if s.FromVersion != "" { - verToGet = s.FromVersion - } - csv, exists, err := getCSVFromFSIfExists(s.getFS(), s.getCSVPath(verToGet)) - if err != nil { - return nil, false, err - } - if !exists && s.FromVersion != "" { - log.Warnf("FromVersion set (%s) but CSV does not exist", s.FromVersion) - } - return csv, exists, nil -} - -func getCSVFromFSIfExists(fs afero.Fs, path string) (*olmapiv1alpha1.ClusterServiceVersion, bool, error) { - csvBytes, err := afero.ReadFile(fs, path) - if err != nil { - if os.IsNotExist(err) { - return nil, false, nil - } - return nil, false, err - } - if len(csvBytes) == 0 { - return nil, false, nil - } - - csv := &olmapiv1alpha1.ClusterServiceVersion{} - if err := yaml.Unmarshal(csvBytes, csv); err != nil { - return nil, false, fmt.Errorf("error unmarshalling CSV %s: %v", path, err) - } - - return csv, true, nil -} - -func getCSVName(name, version string) string { - return name + ".v" + version -} - -func getCSVFileName(name, version string) string { - return getCSVName(name, version) + CSVYamlFileExt -} - -func (s *CSV) getCSVPath(ver string) string { - lowerOperatorName := strings.ToLower(s.OperatorName) - name := getCSVFileName(lowerOperatorName, ver) - return filepath.Join(s.pathPrefix, OLMCatalogDir, lowerOperatorName, ver, name) -} - -// setCSVDefaultFields sets all csv fields that should be populated by a user -// to sane defaults. -func (s *CSV) setCSVDefaultFields(csv *olmapiv1alpha1.ClusterServiceVersion) { - // These fields have well-defined required values. - csv.TypeMeta.APIVersion = olmapiv1alpha1.ClusterServiceVersionAPIVersion - csv.TypeMeta.Kind = olmapiv1alpha1.ClusterServiceVersionKind - csv.SetName(getCSVName(strings.ToLower(s.OperatorName), s.CSVVersion)) - - // Set if empty. - if csv.GetNamespace() == "" { - csv.SetNamespace("placeholder") - } - if csv.GetAnnotations() == nil { - csv.SetAnnotations(map[string]string{}) - } - if caps, ok := csv.GetAnnotations()["capabilities"]; !ok || caps == "" { - csv.GetAnnotations()["capabilities"] = "Basic Install" - } - if csv.Spec.Provider == (olmapiv1alpha1.AppLink{}) { - csv.Spec.Provider = olmapiv1alpha1.AppLink{} - } - if len(csv.Spec.Maintainers) == 0 { - csv.Spec.Maintainers = []olmapiv1alpha1.Maintainer{} - } - if len(csv.Spec.Links) == 0 { - csv.Spec.Links = []olmapiv1alpha1.AppLink{} - } - if csv.Spec.DisplayName == "" { - csv.Spec.DisplayName = k8sutil.GetDisplayName(s.OperatorName) - } - if csv.Spec.Description == "" { - csv.Spec.Description = "Placeholder description" - } - if csv.Spec.Maturity == "" { - csv.Spec.Maturity = "alpha" - } - if len(csv.Spec.InstallModes) == 0 { - csv.Spec.InstallModes = []olmapiv1alpha1.InstallMode{ - {Type: olmapiv1alpha1.InstallModeTypeOwnNamespace, Supported: true}, - {Type: olmapiv1alpha1.InstallModeTypeSingleNamespace, Supported: true}, - {Type: olmapiv1alpha1.InstallModeTypeMultiNamespace, Supported: false}, - {Type: olmapiv1alpha1.InstallModeTypeAllNamespaces, Supported: true}, - } - } - if len(csv.Spec.Icon) == 0 { - csv.Spec.Icon = make([]olmapiv1alpha1.Icon, 1) - } - if len(csv.Spec.Keywords) == 0 { - csv.Spec.Keywords = []string{""} - } - if len(csv.Spec.Maintainers) == 0 { - csv.Spec.Maintainers = make([]olmapiv1alpha1.Maintainer, 1) - } - -} - -// TODO: validate that all fields from files are populated as expected -// ex. add `resources` to a CRD - -func getEmptyRequiredCSVFields(csv *olmapiv1alpha1.ClusterServiceVersion) (fields []string) { - // Metadata - if csv.TypeMeta.APIVersion != olmapiv1alpha1.ClusterServiceVersionAPIVersion { - fields = append(fields, "apiVersion") - } - if csv.TypeMeta.Kind != olmapiv1alpha1.ClusterServiceVersionKind { - fields = append(fields, "kind") - } - if csv.ObjectMeta.Name == "" { - fields = append(fields, "metadata.name") - } - // Spec fields - if csv.Spec.Version.String() == "" { - fields = append(fields, "spec.version") - } - if csv.Spec.DisplayName == "" { - fields = append(fields, "spec.displayName") - } - if csv.Spec.Description == "" { - fields = append(fields, "spec.description") - } - if len(csv.Spec.Keywords) == 0 || len(csv.Spec.Keywords[0]) == 0 { - fields = append(fields, "spec.keywords") - } - if len(csv.Spec.Maintainers) == 0 { - fields = append(fields, "spec.maintainers") - } - if csv.Spec.Provider == (olmapiv1alpha1.AppLink{}) { - fields = append(fields, "spec.provider") - } - if csv.Spec.Maturity == "" { - fields = append(fields, "spec.maturity") - } - - return fields -} - -func joinFields(fields []string) string { - sb := &strings.Builder{} - for _, f := range fields { - sb.WriteString("\n\t" + f) - } - return sb.String() -} - -// updateCSVVersions updates csv's version and data involving the version, -// ex. ObjectMeta.Name, and place the old version in the `replaces` object, -// if there is an old version to replace. -func (s *CSV) updateCSVVersions(csv *olmapiv1alpha1.ClusterServiceVersion) error { - - // Old csv version to replace, and updated csv version. - oldVer, newVer := csv.Spec.Version.String(), s.CSVVersion - if oldVer == newVer { - return nil - } - - // Replace all references to the old operator name. - lowerOperatorName := strings.ToLower(s.OperatorName) - oldCSVName := getCSVName(lowerOperatorName, oldVer) - oldRe, err := regexp.Compile(fmt.Sprintf("\\b%s\\b", regexp.QuoteMeta(oldCSVName))) - if err != nil { - return fmt.Errorf("error compiling CSV name regexp %s: %v", oldRe.String(), err) - } - b, err := yaml.Marshal(csv) - if err != nil { - return err - } - newCSVName := getCSVName(lowerOperatorName, newVer) - b = oldRe.ReplaceAll(b, []byte(newCSVName)) - *csv = olmapiv1alpha1.ClusterServiceVersion{} - if err = yaml.Unmarshal(b, csv); err != nil { - return fmt.Errorf("error unmarshalling CSV %s after replacing old CSV name: %v", csv.GetName(), err) - } - - ver, err := semver.Parse(s.CSVVersion) - if err != nil { - return err - } - csv.Spec.Version = olmversion.OperatorVersion{Version: ver} - csv.Spec.Replaces = oldCSVName - return nil -} - -// updateCSVFromManifestFiles gathers relevant data from generated and -// user-defined manifests and updates csv. -func (s *CSV) updateCSVFromManifests(cfg *CSVConfig, csv *olmapiv1alpha1.ClusterServiceVersion) (err error) { - paths := append(cfg.CRDCRPaths, cfg.OperatorPath) - paths = append(paths, cfg.RolePaths...) - manifestGVKMap := map[schema.GroupVersionKind][][]byte{} - crGVKSet := map[schema.GroupVersionKind]struct{}{} - for _, path := range paths { - info, err := s.getFS().Stat(path) - if err != nil { - return err - } - if info.IsDir() { - continue - } - b, err := ioutil.ReadFile(path) - if err != nil { - return err - } - scanner := yamlutil.NewYAMLScanner(b) - for scanner.Scan() { - manifest := scanner.Bytes() - typeMeta, err := k8sutil.GetTypeMetaFromBytes(manifest) - if err != nil { - log.Infof("No TypeMeta in %s, skipping file", path) - continue - } - gvk := typeMeta.GroupVersionKind() - manifestGVKMap[gvk] = append(manifestGVKMap[gvk], manifest) - switch typeMeta.Kind { - case "CustomResourceDefinition": - // Collect CRD kinds to filter them out from unsupported manifest types. - // The CRD type version doesn't matter as long as it has a group, kind, - // and versions in the expected fields. - crd := apiextv1beta1.CustomResourceDefinition{} - if err = yaml.Unmarshal(manifest, &crd); err != nil { - return err - } - for _, ver := range crd.Spec.Versions { - crGVK := schema.GroupVersionKind{ - Group: crd.Spec.Group, - Version: ver.Name, - Kind: crd.Spec.Names.Kind, - } - crGVKSet[crGVK] = struct{}{} - } - } - } - if err = scanner.Err(); err != nil { - return err - } - } - - crUpdaters := crs{} - for gvk, manifests := range manifestGVKMap { - // We don't necessarily care about sorting by a field value, more about - // consistent ordering. - sort.Slice(manifests, func(i int, j int) bool { - return string(manifests[i]) < string(manifests[j]) - }) - switch gvk.Kind { - case "Role": - err = roles(manifests).apply(csv) - case "ClusterRole": - err = clusterRoles(manifests).apply(csv) - case "Deployment": - err = deployments(manifests).apply(csv) - case "CustomResourceDefinition": - // TODO(estroz): customresourcedefinition should not be updated for - // Ansible and Helm CSV's until annotated updates are implemented. - if projutil.IsOperatorGo() { - err = crds(manifests).apply(csv) - } - default: - if _, ok := crGVKSet[gvk]; ok { - crUpdaters = append(crUpdaters, manifests...) - } else { - log.Infof("Skipping manifest %s", gvk) - } - } - if err != nil { - return err - } - } - // Re-sort CR's since they are appended in random order. - sort.Slice(crUpdaters, func(i int, j int) bool { - return string(crUpdaters[i]) < string(crUpdaters[j]) - }) - if err = crUpdaters.apply(csv); err != nil { - return err - } - return nil -} diff --git a/internal/util/k8sutil/crd.go b/internal/util/k8sutil/crd.go index 32a521fbca0..d42df2ad6b6 100644 --- a/internal/util/k8sutil/crd.go +++ b/internal/util/k8sutil/crd.go @@ -48,7 +48,7 @@ func GetCRDs(crdsDir string) ([]*apiextv1beta1.CustomResourceDefinition, error) return crds, nil } -// GetCRDManifestPaths gets all CRD manifest paths in crdsDir and subdirs. +// GetCRDManifestPaths returns all CRD manifest paths in crdsDir and subdirs. func GetCRDManifestPaths(crdsDir string) (crdPaths []string, err error) { err = filepath.Walk(crdsDir, func(path string, info os.FileInfo, werr error) error { if werr != nil { @@ -62,12 +62,12 @@ func GetCRDManifestPaths(crdsDir string) (crdPaths []string, err error) { if err != nil { return fmt.Errorf("error reading manifest %s: %v", path, err) } - typeMeta, err := GetTypeMetaFromBytes(b) - if err != nil { - return fmt.Errorf("error getting kind from manifest %s: %v", path, err) - } - if typeMeta.Kind == "CustomResourceDefinition" { - crdPaths = append(crdPaths, path) + // Skip files in crdsDir that aren't k8s manifests since we do not know + // what other files are in crdsDir. + if typeMeta, err := GetTypeMetaFromBytes(b); err == nil { + if typeMeta.Kind == "CustomResourceDefinition" { + crdPaths = append(crdPaths, path) + } } } return nil diff --git a/test/test-framework/deploy/olm-catalog/memcached-operator/0.0.2/memcached-operator.v0.0.2.clusterserviceversion.yaml b/test/test-framework/deploy/olm-catalog/memcached-operator/0.0.2/memcached-operator.v0.0.2.clusterserviceversion.yaml index d2c48a2ddf3..3633c973d7b 100644 --- a/test/test-framework/deploy/olm-catalog/memcached-operator/0.0.2/memcached-operator.v0.0.2.clusterserviceversion.yaml +++ b/test/test-framework/deploy/olm-catalog/memcached-operator/0.0.2/memcached-operator.v0.0.2.clusterserviceversion.yaml @@ -1,30 +1,109 @@ -# This file defines the ClusterServiceVersion (CSV) to tell the catalog how to display, create and -# manage the application as a whole. If changes are made to the CRD for this application kind, -# make sure to replace those references below as well. apiVersion: operators.coreos.com/v1alpha1 kind: ClusterServiceVersion metadata: annotations: - alm-examples: '[{"apiVersion":"cache.example.com/v1alpha1","kind":"Memcached","metadata":{"name":"example-memcached"},"spec":{"size":3}}]' + alm-examples: |- + [ + { + "apiVersion": "cache.example.com/v1alpha1", + "kind": "Memcached", + "metadata": { + "name": "example-memcached" + }, + "spec": { + "size": 3 + } + }, + { + "apiVersion": "cache.example.com/v1alpha1", + "kind": "MemcachedRS", + "metadata": { + "name": "example-memcachedrs" + }, + "spec": { + "numNodes": 4 + } + } + ] capabilities: Basic Install name: memcached-operator.v0.0.2 namespace: placeholder spec: - installModes: - - type: OwnNamespace - supported: true - - type: SingleNamespace - supported: true - - type: MultiNamespace - supported: false - - type: AllNamespaces - supported: true + apiservicedefinitions: {} + customresourcedefinitions: + owned: + - kind: MemcachedRS + name: memcachedrs.cache.example.com + version: v1alpha1 + - description: Represents a cluster of Memcached apps + displayName: Memcached App + kind: Memcached + name: memcacheds.cache.example.com + resources: + - kind: Deployment + name: "" + version: v1 + - kind: ReplicaSet + name: "" + version: v1 + - kind: Pod + name: "" + version: v1 + specDescriptors: + - description: The desired number of member Pods for the deployment. + displayName: Size + path: size + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:podCount + statusDescriptors: + - description: The current status of the application. + displayName: Status + path: phase + x-descriptors: + - urn:alm:descriptor:io.kubernetes.phase + - description: Explanation for the current status of the application. + displayName: Status Details + path: reason + x-descriptors: + - urn:alm:descriptor:io.kubernetes.phase:reason + version: v1alpha1 + description: Main enterprise application providing business critical features with + high availability and no manual intervention. + displayName: Memcached Application install: - strategy: deployment spec: + deployments: + - name: memcached-operator + spec: + replicas: 1 + selector: + matchLabels: + name: memcached-operator + template: + metadata: + labels: + name: memcached-operator + spec: + containers: + - command: + - memcached-operator + env: + - name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.annotations['olm.targetNamespaces'] + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPERATOR_NAME + value: memcached-operator + image: quay.io/coreos/operator-sdk-dev:test-framework-operator + imagePullPolicy: Always + name: memcached-operator + serviceAccountName: memcached-operator permissions: - - serviceAccountName: memcached-operator - rules: + - rules: - apiGroups: - "" resources: @@ -55,83 +134,31 @@ spec: - '*' - apiGroups: - apps - resources: - - deployments/finalizers resourceNames: - memcached-operator + resources: + - deployments/finalizers verbs: - - "update" - deployments: - - name: memcached-operator - spec: - replicas: 1 - selector: - matchLabels: - name: memcached-operator - template: - metadata: - labels: - name: memcached-operator - spec: - serviceAccountName: memcached-operator - containers: - - name: memcached-operator - image: quay.io/coreos/operator-sdk-dev:test-framework-operator - command: - - memcached-operator - imagePullPolicy: Always - env: - - name: WATCH_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.annotations['olm.targetNamespaces'] - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: OPERATOR_NAME - value: "memcached-operator" - customresourcedefinitions: - owned: - - description: Represents a cluster of Memcached apps - displayName: Memcached App - kind: Memcached - name: memcacheds.cache.example.com - version: v1alpha1 - resources: - - kind: Deployment - version: v1 - - kind: ReplicaSet - version: v1beta2 - - kind: Pod - version: v1 - specDescriptors: - - description: The desired number of member Pods for the deployment. - displayName: Size - path: size - x-descriptors: - - 'urn:alm:descriptor:com.tectonic.ui:podCount' - statusDescriptors: - - description: The current status of the application. - displayName: Status - path: phase - x-descriptors: - - 'urn:alm:descriptor:io.kubernetes.phase' - - description: Explanation for the current status of the application. - displayName: Status Details - path: reason - x-descriptors: - - 'urn:alm:descriptor:io.kubernetes.phase:reason' + - update + serviceAccountName: memcached-operator + strategy: deployment + installModes: + - supported: true + type: OwnNamespace + - supported: true + type: SingleNamespace + - supported: false + type: MultiNamespace + - supported: true + type: AllNamespaces keywords: - memcached - app - displayName: Memcached Application + maintainers: + - email: corp@example.com + name: Some Corp + maturity: alpha provider: name: Example url: www.example.com - maturity: alpha version: 0.0.2 - maintainers: - - email: corp@example.com - name: Some Corp - description: Main enterprise application providing business critical features with high availability and no manual intervention. diff --git a/test/test-framework/deploy/olm-catalog/memcached-operator/0.0.3/memcached-operator.v0.0.3.clusterserviceversion.yaml b/test/test-framework/deploy/olm-catalog/memcached-operator/0.0.3/memcached-operator.v0.0.3.clusterserviceversion.yaml index c331fa07de4..e2e61f4a81b 100644 --- a/test/test-framework/deploy/olm-catalog/memcached-operator/0.0.3/memcached-operator.v0.0.3.clusterserviceversion.yaml +++ b/test/test-framework/deploy/olm-catalog/memcached-operator/0.0.3/memcached-operator.v0.0.3.clusterserviceversion.yaml @@ -13,6 +13,16 @@ metadata: "spec": { "size": 3 } + }, + { + "apiVersion": "cache.example.com/v1alpha1", + "kind": "MemcachedRS", + "metadata": { + "name": "example-memcachedrs" + }, + "spec": { + "numNodes": 4 + } } ] capabilities: Basic Install @@ -22,6 +32,18 @@ spec: apiservicedefinitions: {} customresourcedefinitions: owned: + - description: Represents a cluster of MemcachedRS apps + displayName: MemcachedRS App + kind: MemcachedRS + name: memcachedrs.cache.example.com + statusDescriptors: + - description: List of the pod names running Memcached in the cluster + displayName: Nodes + path: nodeList + - description: A useless testing variable + displayName: Test + path: test + version: v1alpha1 - description: Represents a cluster of Memcached apps displayName: Memcached App kind: Memcached @@ -54,18 +76,6 @@ spec: x-descriptors: - urn:alm:descriptor:io.kubernetes.phase:reason version: v1alpha1 - - description: Represents a cluster of MemcachedRS apps - displayName: MemcachedRS App - kind: MemcachedRS - name: memcachedrs.cache.example.com - version: v1alpha1 - statusDescriptors: - - description: List of the pod names running Memcached in the cluster - displayName: Nodes - path: nodeList - - description: A useless testing variable - displayName: Test - path: test description: Main enterprise application providing business critical features with high availability and no manual intervention. displayName: Memcached Application From 0372379a9aa3ab5d81a94ec613f3081cc3b20106 Mon Sep 17 00:00:00 2001 From: hasbro17 Date: Mon, 10 Feb 2020 22:06:31 -0800 Subject: [PATCH 02/16] refactor Package and CSV generators to use Inputs and OutputDir --- cmd/operator-sdk/generate/csv.go | 83 +++++++++---------- internal/generate/gen/config.go | 9 +- internal/generate/olm-catalog/csv.go | 50 ++++++----- internal/generate/olm-catalog/csv_go_test.go | 2 - .../olm-catalog/descriptor/descriptor.go | 3 + .../generate/olm-catalog/package_manifest.go | 28 ++++--- .../olm-catalog/package_manifest_test.go | 2 +- internal/util/k8sutil/crd.go | 14 +++- 8 files changed, 102 insertions(+), 89 deletions(-) diff --git a/cmd/operator-sdk/generate/csv.go b/cmd/operator-sdk/generate/csv.go index 51df1b72f10..3f1578a15a6 100644 --- a/cmd/operator-sdk/generate/csv.go +++ b/cmd/operator-sdk/generate/csv.go @@ -38,7 +38,7 @@ type csvCmd struct { fromVersion string operatorName string outputDir string - includePaths []string + inputs map[string]string updateCRDs bool defaultChannel bool } @@ -80,10 +80,18 @@ version to --from-version. Otherwise the SDK will scaffold a new CSV manifest.`, } cmd.Flags().StringVar(&c.fromVersion, "from-version", "", "Semantic version of an existing CSV to use as a base") - cmd.Flags().StringSliceVar(&c.includePaths, "include", []string{scaffold.DeployDir}, - "Paths to include in CSV generation, ex. \"deploy/prod,deploy/test\". "+ - "If this flag is set and you want to enable default behavior, "+ - "you must include \"deploy/\" in the argument list") + cmd.Flags().StringToStringVar(&c.inputs, "inputs", + map[string]string{ + gencatalog.DeployDirKey: "deploy", + gencatalog.APIsDirKey: "pkg/apis", + }, + `Key value input paths used in CSV generation. +Use this to set custom paths for operator manifests and API type definitions +E.g: --inputs deploy=config/production,apis=pkg/myapp/apis +Supported input keys: + - deploy= + - apis= +`) cmd.Flags().StringVar(&c.outputDir, "output-dir", scaffold.DeployDir, "Base directory to output generated CSV. The resulting CSV bundle directory"+ "will be \"/olm-catalog//\"") @@ -109,8 +117,8 @@ func (c csvCmd) run() error { } cfg := gen.Config{ OperatorName: c.operatorName, + Inputs: c.inputs, OutputDir: c.outputDir, - Filters: gen.MakeFilters(c.includePaths...), } csv := gencatalog.NewCSV(cfg, c.csvVersion, c.fromVersion) @@ -124,19 +132,19 @@ func (c csvCmd) run() error { // Write CRD's to the new or updated CSV package dir. if c.updateCRDs { - crdManifestSet, err := findCRDs(c.includePaths...) + // TODO: Any requirement to update CRD's from any place other + // than the input "deploy" directory? + // Can CRD manifests be present outside of the "deploy" directory? + crdManifestSet, err := findCRDFileSet(c.inputs[gencatalog.DeployDirKey]) if err != nil { - return err + return fmt.Errorf("failed to update CRD's: %v", err) } - baseDir := c.outputDir - if baseDir == "" { - baseDir = gencatalog.OLMCatalogDir - } - bundleDir := filepath.Join(baseDir, c.operatorName, c.csvVersion) + // TODO: This path should come from the CSV generator + bundleDir := filepath.Join(c.outputDir, gencatalog.OLMCatalogChildDir, c.operatorName, c.csvVersion) for path, b := range crdManifestSet { path = filepath.Join(bundleDir, path) if err = ioutil.WriteFile(path, b, fileutil.DefaultFileMode); err != nil { - return err + return fmt.Errorf("failed to update CRD's: %v", err) } } } @@ -178,44 +186,31 @@ func validateVersion(version string) error { return nil } -// findCRDs searches directories and files in paths for CRD manifest paths, +// findCRDFileSet searches all directories and files in path for CRD manifests, // returning a map of paths to file contents. -func findCRDs(paths ...string) (map[string][]byte, error) { +func findCRDFileSet(path string) (map[string][]byte, error) { crdFileSet := map[string][]byte{} - for _, path := range paths { - info, err := os.Stat(path) + info, err := os.Stat(path) + if err != nil { + return nil, err + } + if !info.IsDir() { + return nil, fmt.Errorf("CRD's must be read from a directory. %s is a file", path) + } + + crdPaths, err := k8sutil.GetCRDManifestPaths(path) + if err != nil { + return nil, err + } + for _, crdPath := range crdPaths { + b, err := ioutil.ReadFile(crdPath) if err != nil { return fmt.Errorf("error in %s : %v", p, err) } if typeMeta.Kind != "CustomResourceDefinition" { continue } - if info.IsDir() { - subsetPaths, err := k8sutil.GetCRDManifestPaths(path) - if err != nil { - return nil, err - } - for _, crdPath := range subsetPaths { - b, err := ioutil.ReadFile(crdPath) - if err != nil { - return nil, err - } - crdFileSet[filepath.Base(crdPath)] = b - } - } else { - b, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - typeMeta, err := k8sutil.GetTypeMetaFromBytes(b) - if err != nil { - log.Infof("Skipping non-manifest file %s", path) - continue - } - if typeMeta.Kind == "CustomResourceDefinition" { - crdFileSet[filepath.Base(path)] = b - } - } + crdFileSet[filepath.Base(crdPath)] = b } return crdFileSet, nil } diff --git a/internal/generate/gen/config.go b/internal/generate/gen/config.go index 5ff046368df..d7bc4f6df81 100644 --- a/internal/generate/gen/config.go +++ b/internal/generate/gen/config.go @@ -31,14 +31,9 @@ type Config struct { // on-disk inputs are required. If not set, a default is used on a // per-generator basis. Inputs map[string]string - // OutputDir is a dir in which to generate output files. If not set, a - // default is used on a per-generator basis. + // OutputDir is the root directory where the output files will be generated. + // If not set, a default is used on a per-generator basis. OutputDir string - // Filters is a set of functional filters for paths that a generator may - // encounter while gathering data for generation. Filters provides - // fine-grained control over Inputs, since often those paths are often - // top-level directories. - Filters FilterFuncs } // FilterFuncs is a slice of filter funcs. diff --git a/internal/generate/olm-catalog/csv.go b/internal/generate/olm-catalog/csv.go index c2665afdd74..0a2030a5f5f 100644 --- a/internal/generate/olm-catalog/csv.go +++ b/internal/generate/olm-catalog/csv.go @@ -44,13 +44,20 @@ import ( ) const ( - olmCatalogChildDir = "olm-catalog" - OLMCatalogDir = scaffold.DeployDir + string(filepath.Separator) + olmCatalogChildDir - CSVYamlFileExt = ".clusterserviceversion.yaml" + OLMCatalogChildDir = "olm-catalog" + // OLMCatalogDir is the default location for OLM catalog directory. + OLMCatalogDir = scaffold.DeployDir + string(filepath.Separator) + OLMCatalogChildDir + csvYamlFileExt = ".clusterserviceversion.yaml" - BundleDirKey = "bundle" + // Input keys for CSV generator whose values are the filepaths for the respective input directories + + // DEBUG: Why is this an input key? Users don't need to configure this. + // bundleDirKey is for the location of the bundle manifests directory + bundleDirKey = "bundle" + // DeployDirKey is for the location of the operator manifests directory e.g "deploy/production" DeployDirKey = "deploy" - APIsDirKey = "apis" + // APIsDirKey is for the location of the API types directory e.g "pkg/apis" + APIsDirKey = "apis" ) type csvGenerator struct { @@ -59,7 +66,7 @@ type csvGenerator struct { csvVersion string // fromVersion is the CSV version from which to build a new CSV. A CSV // manifest with this version should exist at: - // deploy/olm-catalog/{from_version}/operator-name.v{from_version}.{CSVYamlFileExt} + // deploy/olm-catalog/{from_version}/operator-name.v{from_version}.{csvYamlFileExt} fromVersion string } @@ -79,23 +86,23 @@ func NewCSV(cfg gen.Config, csvVersion, fromVersion string) gen.Generator { g.Inputs[DeployDirKey] = scaffold.DeployDir } else { // Set to non-standard location. - olmCatalogDir = filepath.Join(g.Inputs[DeployDirKey], olmCatalogChildDir) + olmCatalogDir = filepath.Join(g.Inputs[DeployDirKey], OLMCatalogChildDir) } - if bundle, ok := g.Inputs[BundleDirKey]; !ok || bundle == "" { + if bundle, ok := g.Inputs[bundleDirKey]; !ok || bundle == "" { // Initialize so we don't have to check for key existence elsewhere. - g.Inputs[BundleDirKey] = "" + g.Inputs[bundleDirKey] = "" parentDir := filepath.Join(olmCatalogDir, g.OperatorName) if isBundleDirExist(parentDir, g.fromVersion) { - g.Inputs[BundleDirKey] = filepath.Join(olmCatalogDir, g.OperatorName, g.fromVersion) + g.Inputs[bundleDirKey] = filepath.Join(olmCatalogDir, g.OperatorName, g.fromVersion) } else if isBundleDirExist(parentDir, g.csvVersion) { - g.Inputs[BundleDirKey] = filepath.Join(olmCatalogDir, g.OperatorName, g.csvVersion) + g.Inputs[bundleDirKey] = filepath.Join(olmCatalogDir, g.OperatorName, g.csvVersion) } } if apisDir, ok := g.Inputs[APIsDirKey]; !ok || apisDir == "" { g.Inputs[APIsDirKey] = scaffold.ApisDir } if g.OutputDir == "" { - g.OutputDir = filepath.Join(olmCatalogDir, g.OperatorName, g.csvVersion) + g.OutputDir = scaffold.DeployDir } return g } @@ -114,7 +121,7 @@ func getCSVName(name, version string) string { } func getCSVFileName(name, version string) string { - return getCSVName(strings.ToLower(name), version) + CSVYamlFileExt + return getCSVName(strings.ToLower(name), version) + csvYamlFileExt } // Generate allows a CSV to be written by marshalling @@ -127,11 +134,14 @@ func (g csvGenerator) Generate() error { if len(fileMap) == 0 { return errors.New("error generating CSV manifest: no generated file found") } - if err = os.MkdirAll(g.OutputDir, fileutil.DefaultDirFileMode); err != nil { - return errors.Wrapf(err, "error mkdir %s", g.OutputDir) + + csvOutputDir := filepath.Join(g.OutputDir, OLMCatalogChildDir, g.OperatorName, g.csvVersion) + if err = os.MkdirAll(csvOutputDir, fileutil.DefaultDirFileMode); err != nil { + return errors.Wrapf(err, "error mkdir %s", csvOutputDir) } for fileName, b := range fileMap { - path := filepath.Join(g.OutputDir, fileName) + path := filepath.Join(csvOutputDir, fileName) + log.Debugf("CSV generator writing %s", path) if err = ioutil.WriteFile(path, b, fileutil.DefaultFileMode); err != nil { return err } @@ -140,7 +150,7 @@ func (g csvGenerator) Generate() error { } func (g csvGenerator) generate() (fileMap map[string][]byte, err error) { - bundle := g.Inputs[BundleDirKey] + bundle := g.Inputs[bundleDirKey] // Get current CSV to update, otherwise start with a fresh CSV. var csv *olmapiv1alpha1.ClusterServiceVersion if bundle != "" { @@ -176,6 +186,7 @@ func (g csvGenerator) generate() (fileMap map[string][]byte, err error) { if err != nil { return nil, err } + // TODO: Why is this a file map when we only generate one CSV file? fileMap = map[string][]byte{ path: b, } @@ -346,12 +357,11 @@ func (g csvGenerator) updateCSVFromManifests(csv *olmapiv1alpha1.ClusterServiceV kindManifestMap := map[schema.GroupVersionKind][][]byte{} crGVKSet := map[schema.GroupVersionKind]struct{}{} err = filepath.Walk(g.Inputs[DeployDirKey], func(path string, info os.FileInfo, werr error) error { + // DEBUG: How is this walking to read deploy/crds/... manifests + // if it's not walking directories? if werr != nil || info.IsDir() { return werr } - if !g.Filters.SatisfiesAny(path) { - return nil - } b, err := ioutil.ReadFile(path) if err != nil { return err diff --git a/internal/generate/olm-catalog/csv_go_test.go b/internal/generate/olm-catalog/csv_go_test.go index 88779dd12c0..4bf1481ccbb 100644 --- a/internal/generate/olm-catalog/csv_go_test.go +++ b/internal/generate/olm-catalog/csv_go_test.go @@ -74,7 +74,6 @@ func TestGoCSVGoNew(t *testing.T) { cfg := gen.Config{ OperatorName: testProjectName, - Filters: gen.MakeFilters(scaffold.DeployDir), } g := NewCSV(cfg, csvVersion, "") fileMap, err := g.(csvGenerator).generate() @@ -108,7 +107,6 @@ func TestGoCSVFromOld(t *testing.T) { cfg := gen.Config{ OperatorName: testProjectName, - Filters: gen.MakeFilters(scaffold.DeployDir), } g := NewCSV(cfg, csvVersion, fromVersion) fileMap, err := g.(csvGenerator).generate() diff --git a/internal/generate/olm-catalog/descriptor/descriptor.go b/internal/generate/olm-catalog/descriptor/descriptor.go index 4a8380407ae..5c0f03c7f2f 100644 --- a/internal/generate/olm-catalog/descriptor/descriptor.go +++ b/internal/generate/olm-catalog/descriptor/descriptor.go @@ -51,6 +51,9 @@ func GetCRDDescriptionForGVK(apisDir string, gvk schema.GroupVersionKind) (olmap if strings.Contains(group, ".") { group = strings.Split(group, ".")[0] } + // TODO(hasbro17): Lookup for API directory should not be configured to + // // + // Kubebuilder's layout is different. apiDir := filepath.Join(apisDir, group, gvk.Version) universe, err := getTypesFromDir(apiDir) if err != nil { diff --git a/internal/generate/olm-catalog/package_manifest.go b/internal/generate/olm-catalog/package_manifest.go index 8c875abc089..e7580f56890 100644 --- a/internal/generate/olm-catalog/package_manifest.go +++ b/internal/generate/olm-catalog/package_manifest.go @@ -26,6 +26,7 @@ import ( "github.com/operator-framework/api/pkg/validation" "github.com/operator-framework/operator-registry/pkg/registry" "github.com/operator-framework/operator-sdk/internal/generate/gen" + "github.com/operator-framework/operator-sdk/internal/scaffold" "github.com/operator-framework/operator-sdk/internal/util/fileutil" "github.com/ghodss/yaml" @@ -33,9 +34,10 @@ import ( ) const ( - PackageManifestFileExt = ".package.yaml" + packageManifestFileExt = ".package.yaml" - ManifestsDirKey = "manifests" + // TODO: Is this an input that needs to be set via the csv generator inputs flag? + manifestsDirKey = "manifests" ) type pkgGenerator struct { @@ -68,21 +70,22 @@ func NewPackageManifest(cfg gen.Config, csvVersion, channel string, isDefault bo olmCatalogDir := OLMCatalogDir if deployDir, ok := g.Inputs[DeployDirKey]; ok && deployDir != "" { // Set to non-standard location. - olmCatalogDir = filepath.Join(g.Inputs[DeployDirKey], olmCatalogChildDir) + olmCatalogDir = filepath.Join(g.Inputs[DeployDirKey], OLMCatalogChildDir) } - if manifests, ok := g.Inputs[ManifestsDirKey]; !ok || manifests == "" { - g.Inputs[ManifestsDirKey] = filepath.Join(olmCatalogDir, g.OperatorName) + if manifests, ok := g.Inputs[manifestsDirKey]; !ok || manifests == "" { + g.Inputs[manifestsDirKey] = filepath.Join(olmCatalogDir, g.OperatorName) } + if g.OutputDir == "" { - g.OutputDir = filepath.Join(olmCatalogDir, g.OperatorName) + g.OutputDir = scaffold.DeployDir } return g } // getPkgFileName will return the name of the PackageManifestFile func getPkgFileName(operatorName string) string { - return strings.ToLower(operatorName) + PackageManifestFileExt + return strings.ToLower(operatorName) + packageManifestFileExt } func (g pkgGenerator) Generate() error { @@ -93,11 +96,13 @@ func (g pkgGenerator) Generate() error { if len(fileMap) == 0 { return errors.New("error generating package manifest: no generated file found") } - if err = os.MkdirAll(g.OutputDir, fileutil.DefaultDirFileMode); err != nil { - return fmt.Errorf("error mkdir %s: %v", g.OutputDir, err) + pkgManifestOutputDir := filepath.Join(g.OutputDir, OLMCatalogChildDir, g.OperatorName) + if err = os.MkdirAll(pkgManifestOutputDir, fileutil.DefaultDirFileMode); err != nil { + return fmt.Errorf("error mkdir %s: %v", pkgManifestOutputDir, err) } for fileName, b := range fileMap { - path := filepath.Join(g.OutputDir, fileName) + path := filepath.Join(pkgManifestOutputDir, fileName) + log.Debugf("Package manifest generator writing %s", path) if err = ioutil.WriteFile(path, b, fileutil.DefaultFileMode); err != nil { return err } @@ -126,6 +131,7 @@ func (g pkgGenerator) generate() (map[string][]byte, error) { return nil, err } + // TODO: Why is this a file map? fileMap := map[string][]byte{ g.fileName: b, } @@ -136,7 +142,7 @@ func (g pkgGenerator) generate() (map[string][]byte, error) { // an existing one if found at the expected path. func (g pkgGenerator) buildPackageManifest() (registry.PackageManifest, error) { pkg := registry.PackageManifest{} - path := filepath.Join(g.Inputs[ManifestsDirKey], g.fileName) + path := filepath.Join(g.Inputs[manifestsDirKey], g.fileName) if _, err := os.Stat(path); err == nil { b, err := ioutil.ReadFile(path) if err != nil { diff --git a/internal/generate/olm-catalog/package_manifest_test.go b/internal/generate/olm-catalog/package_manifest_test.go index 4ea8dded268..ff355da69f6 100644 --- a/internal/generate/olm-catalog/package_manifest_test.go +++ b/internal/generate/olm-catalog/package_manifest_test.go @@ -30,7 +30,7 @@ func newTestPackageManifestGenerator() gen.Generator { inputDir := filepath.Join(testGoDataDir, OLMCatalogDir, testProjectName) cfg := gen.Config{ OperatorName: testProjectName, - Inputs: map[string]string{ManifestsDirKey: inputDir}, + Inputs: map[string]string{manifestsDirKey: inputDir}, } g := NewPackageManifest(cfg, csvVersion, "stable", true) return g diff --git a/internal/util/k8sutil/crd.go b/internal/util/k8sutil/crd.go index d42df2ad6b6..bb5daec70f6 100644 --- a/internal/util/k8sutil/crd.go +++ b/internal/util/k8sutil/crd.go @@ -23,6 +23,7 @@ import ( "regexp" yaml "github.com/ghodss/yaml" + log "github.com/sirupsen/logrus" apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" "k8s.io/apimachinery/pkg/version" ) @@ -64,11 +65,16 @@ func GetCRDManifestPaths(crdsDir string) (crdPaths []string, err error) { } // Skip files in crdsDir that aren't k8s manifests since we do not know // what other files are in crdsDir. - if typeMeta, err := GetTypeMetaFromBytes(b); err == nil { - if typeMeta.Kind == "CustomResourceDefinition" { - crdPaths = append(crdPaths, path) - } + typeMeta, err := GetTypeMetaFromBytes(b) + if err != nil { + log.Debugf("Skipping non-manifest file %s: %v", path, err) + return nil + } + if typeMeta.Kind != "CustomResourceDefinition" { + log.Debugf("Skipping non CRD manifest %s", path) + return nil } + crdPaths = append(crdPaths, path) } return nil }) From 9eab63883973280c15de4b5d8e390e83c32965aa Mon Sep 17 00:00:00 2001 From: hasbro17 Date: Wed, 12 Feb 2020 00:04:39 -0800 Subject: [PATCH 03/16] Replace input keys bundleDirKey and manifestsDirKey with private fields bundleDirKey and manifestsDirKey were not suited to be generator input keys. Much simpler to have them as private fields. Plus some linter and cli docs cleanup. --- CHANGELOG.md | 2 +- cmd/operator-sdk/generate/csv.go | 8 +-- doc/cli/operator-sdk_generate_csv.md | 24 +++++--- internal/generate/olm-catalog/csv.go | 56 +++++++++++-------- internal/generate/olm-catalog/csv_go_test.go | 27 +++------ internal/generate/olm-catalog/csv_updaters.go | 4 +- .../generate/olm-catalog/package_manifest.go | 39 ++++++++----- .../olm-catalog/package_manifest_test.go | 12 +++- 8 files changed, 96 insertions(+), 76 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ddf4ae4002a..3f0955aa777 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,7 +82,7 @@ ### Removed - **Breaking Change:** The additional Ansible sidecar container. ([#2586](https://github.com/operator-framework/operator-sdk/pull/2586)) -- **Breaking Change:** Removed CSV configuration file support (defaulting to deploy/olm-catalog/csv-config.yaml) in favor of including files as input to the generator using [`generate csv --include`](doc/cli/operator-sdk_generate_csv.md#options), defaulting to the `deploy/` directory. ([#2249](https://github.com/operator-framework/operator-sdk/pull/2249)) +- **Breaking Change:** Removed CSV configuration file support (defaulting to deploy/olm-catalog/csv-config.yaml) in favor of specifying inputs to the generator via [`generate csv --input`](doc/cli/operator-sdk_generate_csv.md#options), and configuring output locations via [`generate csv --outputDir`](doc/cli/operator-sdk_generate_csv.md#options). ([#2511](https://github.com/operator-framework/operator-sdk/pull/2511)) ### Bug Fixes diff --git a/cmd/operator-sdk/generate/csv.go b/cmd/operator-sdk/generate/csv.go index 3f1578a15a6..ab268adb3a0 100644 --- a/cmd/operator-sdk/generate/csv.go +++ b/cmd/operator-sdk/generate/csv.go @@ -89,8 +89,8 @@ version to --from-version. Otherwise the SDK will scaffold a new CSV manifest.`, Use this to set custom paths for operator manifests and API type definitions E.g: --inputs deploy=config/production,apis=pkg/myapp/apis Supported input keys: - - deploy= - - apis= + - deploy= + - apis= `) cmd.Flags().StringVar(&c.outputDir, "output-dir", scaffold.DeployDir, "Base directory to output generated CSV. The resulting CSV bundle directory"+ @@ -139,7 +139,7 @@ func (c csvCmd) run() error { if err != nil { return fmt.Errorf("failed to update CRD's: %v", err) } - // TODO: This path should come from the CSV generator + // TODO: This path should come from the CSV generator field csvOutputDir bundleDir := filepath.Join(c.outputDir, gencatalog.OLMCatalogChildDir, c.operatorName, c.csvVersion) for path, b := range crdManifestSet { path = filepath.Join(bundleDir, path) @@ -195,7 +195,7 @@ func findCRDFileSet(path string) (map[string][]byte, error) { return nil, err } if !info.IsDir() { - return nil, fmt.Errorf("CRD's must be read from a directory. %s is a file", path) + return nil, fmt.Errorf("crd's must be read from a directory. %s is a file", path) } crdPaths, err := k8sutil.GetCRDManifestPaths(path) diff --git a/doc/cli/operator-sdk_generate_csv.md b/doc/cli/operator-sdk_generate_csv.md index ad9116dbd66..c1df7744eb4 100644 --- a/doc/cli/operator-sdk_generate_csv.md +++ b/doc/cli/operator-sdk_generate_csv.md @@ -18,15 +18,21 @@ operator-sdk generate csv [flags] ### Options ``` - --csv-channel string Channel the CSV should be registered under in the package manifest - --csv-version string Semantic version of the CSV - --default-channel Use the channel passed to --csv-channel as the package manifests' default channel. Only valid when --csv-channel is set - --from-version string Semantic version of an existing CSV to use as a base - -h, --help help for csv - --include strings Paths to include in CSV generation, ex. "deploy/prod,deploy/test". If this flag is set and you want to enable default behavior, you must include "deploy/" in the argument list (default [deploy]) - --operator-name string Operator name to use while generating CSV - --output-dir string Base directory to output generated CSV. The resulting CSV bundle directory will be "/olm-catalog//" (default "deploy") - --update-crds Update CRD manifests in deploy/{operator-name}/{csv-version} the using latest API's + --csv-channel string Channel the CSV should be registered under in the package manifest + --csv-version string Semantic version of the CSV + --default-channel Use the channel passed to --csv-channel as the package manifests' default channel. Only valid when --csv-channel is set + --from-version string Semantic version of an existing CSV to use as a base + -h, --help help for csv + --inputs stringToString Key value input paths used in CSV generation. + Use this to set custom paths for operator manifests and API type definitions + E.g: --inputs deploy=config/production,apis=pkg/myapp/apis + Supported input keys: + - deploy= + - apis= + (default [apis=pkg/apis,deploy=deploy]) + --operator-name string Operator name to use while generating CSV + --output-dir string Base directory to output generated CSV. The resulting CSV bundle directorywill be "/olm-catalog//" (default "deploy") + --update-crds Update CRD manifests in deploy/{operator-name}/{csv-version} the using latest API's ``` ### SEE ALSO diff --git a/internal/generate/olm-catalog/csv.go b/internal/generate/olm-catalog/csv.go index 0a2030a5f5f..2641546cbf8 100644 --- a/internal/generate/olm-catalog/csv.go +++ b/internal/generate/olm-catalog/csv.go @@ -51,9 +51,6 @@ const ( // Input keys for CSV generator whose values are the filepaths for the respective input directories - // DEBUG: Why is this an input key? Users don't need to configure this. - // bundleDirKey is for the location of the bundle manifests directory - bundleDirKey = "bundle" // DeployDirKey is for the location of the operator manifests directory e.g "deploy/production" DeployDirKey = "deploy" // APIsDirKey is for the location of the API types directory e.g "pkg/apis" @@ -68,6 +65,12 @@ type csvGenerator struct { // manifest with this version should exist at: // deploy/olm-catalog/{from_version}/operator-name.v{from_version}.{csvYamlFileExt} fromVersion string + // existingCSVBundleDir is set if the generator needs to update from + // an existing CSV bundle directory + existingCSVBundleDir string + // csvOutputDir is the bundle directory filepath where the CSV will be generated + // This is set according to the generator's OutputDir + csvOutputDir string } func NewCSV(cfg gen.Config, csvVersion, fromVersion string) gen.Generator { @@ -88,22 +91,23 @@ func NewCSV(cfg gen.Config, csvVersion, fromVersion string) gen.Generator { // Set to non-standard location. olmCatalogDir = filepath.Join(g.Inputs[DeployDirKey], OLMCatalogChildDir) } - if bundle, ok := g.Inputs[bundleDirKey]; !ok || bundle == "" { - // Initialize so we don't have to check for key existence elsewhere. - g.Inputs[bundleDirKey] = "" - parentDir := filepath.Join(olmCatalogDir, g.OperatorName) - if isBundleDirExist(parentDir, g.fromVersion) { - g.Inputs[bundleDirKey] = filepath.Join(olmCatalogDir, g.OperatorName, g.fromVersion) - } else if isBundleDirExist(parentDir, g.csvVersion) { - g.Inputs[bundleDirKey] = filepath.Join(olmCatalogDir, g.OperatorName, g.csvVersion) - } + + parentDir := filepath.Join(olmCatalogDir, g.OperatorName) + if isBundleDirExist(parentDir, g.fromVersion) { + g.existingCSVBundleDir = filepath.Join(olmCatalogDir, g.OperatorName, g.fromVersion) + } else if isBundleDirExist(parentDir, g.csvVersion) { + g.existingCSVBundleDir = filepath.Join(olmCatalogDir, g.OperatorName, g.csvVersion) } + if apisDir, ok := g.Inputs[APIsDirKey]; !ok || apisDir == "" { g.Inputs[APIsDirKey] = scaffold.ApisDir } if g.OutputDir == "" { g.OutputDir = scaffold.DeployDir } + + // Set the CSV bundle dir output path under the generator's OutputDir + g.csvOutputDir = filepath.Join(g.OutputDir, OLMCatalogChildDir, g.OperatorName, g.csvVersion) return g } @@ -112,8 +116,13 @@ func isBundleDirExist(parentDir, version string) bool { if parentDir == "" || version == "" { return false } - _, err := os.Stat(filepath.Join(parentDir, version)) - return err == nil || os.IsExist(err) + bundleDir := filepath.Join(parentDir, version) + _, err := os.Stat(bundleDir) + if !os.IsNotExist(err) { + // TODO: return and handle this error + log.Fatalf("Failed to stat existing bundle directory %s: %v", bundleDir, err) + } + return err == nil } func getCSVName(name, version string) string { @@ -135,12 +144,11 @@ func (g csvGenerator) Generate() error { return errors.New("error generating CSV manifest: no generated file found") } - csvOutputDir := filepath.Join(g.OutputDir, OLMCatalogChildDir, g.OperatorName, g.csvVersion) - if err = os.MkdirAll(csvOutputDir, fileutil.DefaultDirFileMode); err != nil { - return errors.Wrapf(err, "error mkdir %s", csvOutputDir) + if err = os.MkdirAll(g.csvOutputDir, fileutil.DefaultDirFileMode); err != nil { + return errors.Wrapf(err, "error mkdir %s", g.csvOutputDir) } for fileName, b := range fileMap { - path := filepath.Join(csvOutputDir, fileName) + path := filepath.Join(g.csvOutputDir, fileName) log.Debugf("CSV generator writing %s", path) if err = ioutil.WriteFile(path, b, fileutil.DefaultFileMode); err != nil { return err @@ -150,11 +158,12 @@ func (g csvGenerator) Generate() error { } func (g csvGenerator) generate() (fileMap map[string][]byte, err error) { - bundle := g.Inputs[bundleDirKey] // Get current CSV to update, otherwise start with a fresh CSV. var csv *olmapiv1alpha1.ClusterServiceVersion - if bundle != "" { - if csv, err = getCSVFromDir(bundle); err != nil { + if g.existingCSVBundleDir != "" { + // TODO: If bundle dir exists, but the CSV file does not + // then we should create a new one and not return an error. + if csv, err = getCSVFromDir(g.existingCSVBundleDir); err != nil { return nil, err } // TODO: validate existing CSV. @@ -173,7 +182,8 @@ func (g csvGenerator) generate() (fileMap map[string][]byte, err error) { path := getCSVFileName(g.OperatorName, g.csvVersion) if fields := getEmptyRequiredCSVFields(csv); len(fields) != 0 { - if bundle != "" { + if g.existingCSVBundleDir != "" { + // An existing csv should have several required fields populated. log.Warnf("Required csv fields not filled in file %s:%s\n", path, joinFields(fields)) } else { // A new csv won't have several required fields populated. @@ -357,8 +367,6 @@ func (g csvGenerator) updateCSVFromManifests(csv *olmapiv1alpha1.ClusterServiceV kindManifestMap := map[schema.GroupVersionKind][][]byte{} crGVKSet := map[schema.GroupVersionKind]struct{}{} err = filepath.Walk(g.Inputs[DeployDirKey], func(path string, info os.FileInfo, werr error) error { - // DEBUG: How is this walking to read deploy/crds/... manifests - // if it's not walking directories? if werr != nil || info.IsDir() { return werr } diff --git a/internal/generate/olm-catalog/csv_go_test.go b/internal/generate/olm-catalog/csv_go_test.go index 4bf1481ccbb..471ecc8dcd7 100644 --- a/internal/generate/olm-catalog/csv_go_test.go +++ b/internal/generate/olm-catalog/csv_go_test.go @@ -15,7 +15,6 @@ package olmcatalog import ( - "fmt" "io/ioutil" "os" "path/filepath" @@ -34,16 +33,10 @@ import ( ) const ( - sdkCmd = "operator-sdk" - - testProjectName = "memcached-operator" - csvVersion = "0.0.3" - fromVersion = "0.0.2" - notExistVersion = "1.0.0" - scratchBundleDir = "scratch" - testGroup = "cache.example.com" - testKind1 = "Memcached" - testVersion1 = "v1alpha1" + testProjectName = "memcached-operator" + csvVersion = "0.0.3" + fromVersion = "0.0.2" + notExistVersion = "1.0.0" ) var ( @@ -67,7 +60,7 @@ func setupTestEnvWithCleanup(t *testing.T, dataDir string) (cleanupFuncs []func( return cleanupFuncs } -func TestGoCSVGoNew(t *testing.T) { +func TestGoCSVFromNew(t *testing.T) { for _, cleanupFunc := range setupTestEnvWithCleanup(t, testGoDataDir) { defer cleanupFunc() } @@ -76,6 +69,8 @@ func TestGoCSVGoNew(t *testing.T) { OperatorName: testProjectName, } g := NewCSV(cfg, csvVersion, "") + // TODO: Expand this test to test the g.Generate() method + // so we can test the CSV generator's input/out paths. fileMap, err := g.(csvGenerator).generate() if err != nil { t.Fatalf("Failed to execute CSV generator: %v", err) @@ -152,14 +147,6 @@ func TestGoCSVIncludeAll(t *testing.T) { } } -func getTestCRDFile(g, k string) string { - return fmt.Sprintf("%s_%s_crd.yaml", g, strings.ToLower(k)+"s") -} - -func getTestCRFile(g, v, k string) string { - return fmt.Sprintf("%s_%s_%s_cr.yaml", g, v, strings.ToLower(k)) -} - func TestUpdateVersion(t *testing.T) { csv, err := getCSVFromDir(filepath.Join(testGoDataDir, OLMCatalogDir, testProjectName, fromVersion)) if err != nil { diff --git a/internal/generate/olm-catalog/csv_updaters.go b/internal/generate/olm-catalog/csv_updaters.go index ede307cc810..0a3d835dac2 100644 --- a/internal/generate/olm-catalog/csv_updaters.go +++ b/internal/generate/olm-catalog/csv_updaters.go @@ -242,7 +242,9 @@ func (us crds) apply(csv *olmapiv1alpha1.ClusterServiceVersion) error { return nil } -func updateDescriptions(csv *olmapiv1alpha1.ClusterServiceVersion, searchDir string, opType projutil.OperatorType) error { +func updateDescriptions(csv *olmapiv1alpha1.ClusterServiceVersion, + searchDir string, + opType projutil.OperatorType) error { ownedCRDs := []olmapiv1alpha1.CRDDescription{} for _, ownedCRD := range csv.Spec.CustomResourceDefinitions.Owned { name := ownedCRD.Name diff --git a/internal/generate/olm-catalog/package_manifest.go b/internal/generate/olm-catalog/package_manifest.go index e7580f56890..1e91e71520a 100644 --- a/internal/generate/olm-catalog/package_manifest.go +++ b/internal/generate/olm-catalog/package_manifest.go @@ -35,9 +35,6 @@ import ( const ( packageManifestFileExt = ".package.yaml" - - // TODO: Is this an input that needs to be set via the csv generator inputs flag? - manifestsDirKey = "manifests" ) type pkgGenerator struct { @@ -52,6 +49,10 @@ type pkgGenerator struct { channelIsDefault bool // PackageManifest file name fileName string + + // existingPkgManifestDir is set to the package manifest root directory + // if the generator needs to update from an existing package manifest file. + existingPkgManifestDir string } func NewPackageManifest(cfg gen.Config, csvVersion, channel string, isDefault bool) gen.Generator { @@ -73,8 +74,11 @@ func NewPackageManifest(cfg gen.Config, csvVersion, channel string, isDefault bo olmCatalogDir = filepath.Join(g.Inputs[DeployDirKey], OLMCatalogChildDir) } - if manifests, ok := g.Inputs[manifestsDirKey]; !ok || manifests == "" { - g.Inputs[manifestsDirKey] = filepath.Join(olmCatalogDir, g.OperatorName) + // Check if the generator should update from an existing package manifest file + pkgManifestDirPath := filepath.Join(olmCatalogDir, g.OperatorName) + pkgManifestFilePath := filepath.Join(pkgManifestDirPath, g.fileName) + if isFileExist(pkgManifestFilePath) { + g.existingPkgManifestDir = pkgManifestDirPath } if g.OutputDir == "" { @@ -83,6 +87,15 @@ func NewPackageManifest(cfg gen.Config, csvVersion, channel string, isDefault bo return g } +func isFileExist(path string) bool { + _, err := os.Stat(path) + if !os.IsNotExist(err) { + // TODO: return and handle this error + log.Fatalf("Failed to stat %s: %v", path, err) + } + return err == nil +} + // getPkgFileName will return the name of the PackageManifestFile func getPkgFileName(operatorName string) string { return strings.ToLower(operatorName) + packageManifestFileExt @@ -138,23 +151,21 @@ func (g pkgGenerator) generate() (map[string][]byte, error) { return fileMap, nil } -// buildPackageManifest will create a registry.PackageManifest from scratch, or modify +// buildPackageManifest will create a registry.PackageManifest from scratch, or reads // an existing one if found at the expected path. func (g pkgGenerator) buildPackageManifest() (registry.PackageManifest, error) { pkg := registry.PackageManifest{} - path := filepath.Join(g.Inputs[manifestsDirKey], g.fileName) - if _, err := os.Stat(path); err == nil { - b, err := ioutil.ReadFile(path) + if g.existingPkgManifestDir != "" { + existingPkgManifest := filepath.Join(g.existingPkgManifestDir, g.fileName) + b, err := ioutil.ReadFile(existingPkgManifest) if err != nil { - return pkg, fmt.Errorf("failed to read package manifest %s: %v", path, err) + return pkg, fmt.Errorf("failed to read package manifest %s: %v", existingPkgManifest, err) } if err = yaml.Unmarshal(b, &pkg); err != nil { - return pkg, fmt.Errorf("failed to unmarshal package manifest %s: %v", path, err) + return pkg, fmt.Errorf("failed to unmarshal package manifest %s: %v", existingPkgManifest, err) } - } else if os.IsNotExist(err) { - pkg = newPackageManifest(g.OperatorName, g.channel, g.csvVersion) } else { - return pkg, fmt.Errorf("error reading package manifest %s: %v", path, err) + pkg = newPackageManifest(g.OperatorName, g.channel, g.csvVersion) } return pkg, nil } diff --git a/internal/generate/olm-catalog/package_manifest_test.go b/internal/generate/olm-catalog/package_manifest_test.go index ff355da69f6..68e97ea85b2 100644 --- a/internal/generate/olm-catalog/package_manifest_test.go +++ b/internal/generate/olm-catalog/package_manifest_test.go @@ -27,13 +27,19 @@ import ( // newTestPackageManifestGenerator returns a package manifest Generator populated with test values. func newTestPackageManifestGenerator() gen.Generator { - inputDir := filepath.Join(testGoDataDir, OLMCatalogDir, testProjectName) cfg := gen.Config{ OperatorName: testProjectName, - Inputs: map[string]string{manifestsDirKey: inputDir}, } g := NewPackageManifest(cfg, csvVersion, "stable", true) - return g + + // TODO: The deploy dir input should be the way to point the generator to test data + // E.g: Inputs: map[string]string{DeployDirKey: testGoDataDir} + + // Override the generator to point to the package manifest in testGoDataDir + testPkgManifestDir := filepath.Join(testGoDataDir, OLMCatalogDir, testProjectName) + pkgGen := g.(pkgGenerator) + pkgGen.existingPkgManifestDir = testPkgManifestDir + return pkgGen } func TestGeneratePackageManifest(t *testing.T) { From eecbeac1418647f133694af375589f216e8a0738 Mon Sep 17 00:00:00 2001 From: hasbro17 Date: Wed, 12 Feb 2020 16:19:39 -0800 Subject: [PATCH 04/16] address review comments plus cleanup --- cmd/operator-sdk/generate/csv.go | 8 +-- doc/cli/operator-sdk_generate_csv.md | 2 +- internal/generate/gen/config.go | 43 +------------ internal/generate/gen/config_test.go | 63 ------------------- internal/generate/olm-catalog/csv.go | 8 ++- .../generate/olm-catalog/package_manifest.go | 8 ++- 6 files changed, 17 insertions(+), 115 deletions(-) delete mode 100644 internal/generate/gen/config_test.go diff --git a/cmd/operator-sdk/generate/csv.go b/cmd/operator-sdk/generate/csv.go index ab268adb3a0..3a50e541fc7 100644 --- a/cmd/operator-sdk/generate/csv.go +++ b/cmd/operator-sdk/generate/csv.go @@ -83,7 +83,7 @@ version to --from-version. Otherwise the SDK will scaffold a new CSV manifest.`, cmd.Flags().StringToStringVar(&c.inputs, "inputs", map[string]string{ gencatalog.DeployDirKey: "deploy", - gencatalog.APIsDirKey: "pkg/apis", + gencatalog.APIsDirKey: filepath.Join("pkg", "apis"), }, `Key value input paths used in CSV generation. Use this to set custom paths for operator manifests and API type definitions @@ -132,9 +132,9 @@ func (c csvCmd) run() error { // Write CRD's to the new or updated CSV package dir. if c.updateCRDs { - // TODO: Any requirement to update CRD's from any place other - // than the input "deploy" directory? - // Can CRD manifests be present outside of the "deploy" directory? + // TODO: Add a "crd" key to the --inputs flag to allow + // configuring the location of the CRD manifests directory + // from which to find and update CRD's. crdManifestSet, err := findCRDFileSet(c.inputs[gencatalog.DeployDirKey]) if err != nil { return fmt.Errorf("failed to update CRD's: %v", err) diff --git a/doc/cli/operator-sdk_generate_csv.md b/doc/cli/operator-sdk_generate_csv.md index c1df7744eb4..e80d5d10465 100644 --- a/doc/cli/operator-sdk_generate_csv.md +++ b/doc/cli/operator-sdk_generate_csv.md @@ -29,7 +29,7 @@ operator-sdk generate csv [flags] Supported input keys: - deploy= - apis= - (default [apis=pkg/apis,deploy=deploy]) + (default [deploy=deploy,apis=pkg/apis]) --operator-name string Operator name to use while generating CSV --output-dir string Base directory to output generated CSV. The resulting CSV bundle directorywill be "/olm-catalog//" (default "deploy") --update-crds Update CRD manifests in deploy/{operator-name}/{csv-version} the using latest API's diff --git a/internal/generate/gen/config.go b/internal/generate/gen/config.go index d7bc4f6df81..d249f80a4a3 100644 --- a/internal/generate/gen/config.go +++ b/internal/generate/gen/config.go @@ -14,13 +14,8 @@ package gen -import ( - "path/filepath" - "strings" - - "github.com/operator-framework/operator-sdk/internal/util/projutil" -) - +// TODO(hasbro17/estroz): Remove the generator config in favor of generator +// specific option structs configured with Inputs and OutputDir. // Config configures a generator with common operator project information. type Config struct { // OperatorName is the operator's name, ex. app-operator @@ -35,37 +30,3 @@ type Config struct { // If not set, a default is used on a per-generator basis. OutputDir string } - -// FilterFuncs is a slice of filter funcs. -type FilterFuncs []func(string) bool - -// MakeFilters creates a set of closures around each path in paths. -// If the argument to a closure has a prefix of path, it returns true. -func MakeFilters(paths ...string) (filters FilterFuncs) { - pathSet := map[string]struct{}{} - for _, path := range paths { - pathSet[filepath.Clean(path)] = struct{}{} - } - wd := projutil.MustGetwd() + string(filepath.Separator) - for path := range pathSet { - // Copy the string for the closure. - pb := strings.Builder{} - pb.WriteString(path) - filters = append(filters, func(p string) bool { - // Handle absolute paths referencing the project directory. - p = strings.TrimPrefix(p, wd) - return strings.HasPrefix(filepath.Clean(p), pb.String()) - }) - } - return filters -} - -// SatisfiesAny returns true if path passes any filter in funcs. -func (funcs FilterFuncs) SatisfiesAny(path string) bool { - for _, f := range funcs { - if f(path) { - return true - } - } - return false -} diff --git a/internal/generate/gen/config_test.go b/internal/generate/gen/config_test.go deleted file mode 100644 index 2ad25969880..00000000000 --- a/internal/generate/gen/config_test.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2020 The Operator-SDK Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package gen - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestFilterFuncs(t *testing.T) { - cases := []struct { - name string - paths []string - wantedSatPaths map[string]bool - }{ - { - "empty filters with one path", - nil, - map[string]bool{"notexist": false}, - }, - { - "two filters with no matching paths", - []string{"key1", "key2"}, - map[string]bool{"notexist": false}, - }, - { - "multiple pathed filters with multiple matching paths", - []string{"key1/key2", "key3", "/abs/path/to/something"}, - map[string]bool{ - "key3": true, - "/abs/path/to/something/else": true, - "key2": false, - "path/to": false, - }, - }, - } - for _, c := range cases { - filters := MakeFilters(c.paths...) - for path, wantedSat := range c.wantedSatPaths { - t.Run(c.name+": "+path, func(t *testing.T) { - isSat := filters.SatisfiesAny(path) - if wantedSat { - assert.True(t, isSat) - } else { - assert.False(t, isSat) - } - }) - } - } -} diff --git a/internal/generate/olm-catalog/csv.go b/internal/generate/olm-catalog/csv.go index 2641546cbf8..6c165efd6fc 100644 --- a/internal/generate/olm-catalog/csv.go +++ b/internal/generate/olm-catalog/csv.go @@ -118,11 +118,14 @@ func isBundleDirExist(parentDir, version string) bool { } bundleDir := filepath.Join(parentDir, version) _, err := os.Stat(bundleDir) - if !os.IsNotExist(err) { + if err != nil { + if os.IsNotExist(err) { + return false + } // TODO: return and handle this error log.Fatalf("Failed to stat existing bundle directory %s: %v", bundleDir, err) } - return err == nil + return true } func getCSVName(name, version string) string { @@ -196,7 +199,6 @@ func (g csvGenerator) generate() (fileMap map[string][]byte, err error) { if err != nil { return nil, err } - // TODO: Why is this a file map when we only generate one CSV file? fileMap = map[string][]byte{ path: b, } diff --git a/internal/generate/olm-catalog/package_manifest.go b/internal/generate/olm-catalog/package_manifest.go index 1e91e71520a..de3813560b6 100644 --- a/internal/generate/olm-catalog/package_manifest.go +++ b/internal/generate/olm-catalog/package_manifest.go @@ -89,11 +89,14 @@ func NewPackageManifest(cfg gen.Config, csvVersion, channel string, isDefault bo func isFileExist(path string) bool { _, err := os.Stat(path) - if !os.IsNotExist(err) { + if err != nil { + if os.IsNotExist(err) { + return false + } // TODO: return and handle this error log.Fatalf("Failed to stat %s: %v", path, err) } - return err == nil + return true } // getPkgFileName will return the name of the PackageManifestFile @@ -144,7 +147,6 @@ func (g pkgGenerator) generate() (map[string][]byte, error) { return nil, err } - // TODO: Why is this a file map? fileMap := map[string][]byte{ g.fileName: b, } From 3f7ff47001340b7b210fed09338730d35985b672 Mon Sep 17 00:00:00 2001 From: hasbro17 Date: Wed, 12 Feb 2020 23:24:09 -0800 Subject: [PATCH 05/16] Add tests for CSV and package manifest generation with inputs and output dir configured Also relax the requirement for running generate csv in project root --- cmd/operator-sdk/generate/csv.go | 4 - internal/generate/olm-catalog/csv_go_test.go | 63 ++++++- .../olm-catalog/package_manifest_test.go | 58 ++++++ .../api/cache/v1alpha1/doc.go | 18 ++ .../api/cache/v1alpha1/dummy_types.go | 168 ++++++++++++++++++ .../api/cache/v1alpha1/memcached_types.go | 57 ++++++ .../api/cache/v1alpha1/memcachedrs_types.go | 55 ++++++ .../cache.example.com_memcachedrs_crd.yaml | 58 ++++++ .../cache.example.com_memcacheds_crd.yaml | 57 ++++++ ...che.example.com_v1alpha1_memcached_cr.yaml | 7 + ...e.example.com_v1alpha1_memcachedrs_cr.yaml | 6 + .../non-standard-layout/config/operator.yaml | 32 ++++ .../non-standard-layout/config/role.yaml | 61 +++++++ .../config/role_binding.yaml | 11 ++ .../config/service_account.yaml | 4 + ...operator.v0.0.3.clusterserviceversion.yaml | 154 ++++++++++++++++ .../memcached-operator.package.yaml | 5 + 17 files changed, 812 insertions(+), 6 deletions(-) create mode 100644 internal/generate/testdata/non-standard-layout/api/cache/v1alpha1/doc.go create mode 100644 internal/generate/testdata/non-standard-layout/api/cache/v1alpha1/dummy_types.go create mode 100644 internal/generate/testdata/non-standard-layout/api/cache/v1alpha1/memcached_types.go create mode 100644 internal/generate/testdata/non-standard-layout/api/cache/v1alpha1/memcachedrs_types.go create mode 100644 internal/generate/testdata/non-standard-layout/config/crds/cache.example.com_memcachedrs_crd.yaml create mode 100644 internal/generate/testdata/non-standard-layout/config/crds/cache.example.com_memcacheds_crd.yaml create mode 100644 internal/generate/testdata/non-standard-layout/config/crds/cache.example.com_v1alpha1_memcached_cr.yaml create mode 100644 internal/generate/testdata/non-standard-layout/config/crds/cache.example.com_v1alpha1_memcachedrs_cr.yaml create mode 100644 internal/generate/testdata/non-standard-layout/config/operator.yaml create mode 100644 internal/generate/testdata/non-standard-layout/config/role.yaml create mode 100644 internal/generate/testdata/non-standard-layout/config/role_binding.yaml create mode 100644 internal/generate/testdata/non-standard-layout/config/service_account.yaml create mode 100644 internal/generate/testdata/non-standard-layout/expected-catalog/olm-catalog/memcached-operator/0.0.3/memcached-operator.v0.0.3.clusterserviceversion.yaml create mode 100644 internal/generate/testdata/non-standard-layout/expected-catalog/olm-catalog/memcached-operator/memcached-operator.package.yaml diff --git a/cmd/operator-sdk/generate/csv.go b/cmd/operator-sdk/generate/csv.go index 3a50e541fc7..14736b63e09 100644 --- a/cmd/operator-sdk/generate/csv.go +++ b/cmd/operator-sdk/generate/csv.go @@ -56,10 +56,6 @@ has already generated a CSV manifest you want to use as a base, supply its version to --from-version. Otherwise the SDK will scaffold a new CSV manifest.`, RunE: func(cmd *cobra.Command, args []string) error { - // The CSV generator assumes that the deploy and pkg directories are - // present at runtime, so this command must be run in a project's root. - projutil.MustInProjectRoot() - if len(args) != 0 { return fmt.Errorf("command %s doesn't accept any arguments", cmd.CommandPath()) } diff --git a/internal/generate/olm-catalog/csv_go_test.go b/internal/generate/olm-catalog/csv_go_test.go index 471ecc8dcd7..8ea6823f01a 100644 --- a/internal/generate/olm-catalog/csv_go_test.go +++ b/internal/generate/olm-catalog/csv_go_test.go @@ -16,6 +16,7 @@ package olmcatalog import ( "io/ioutil" + "log" "os" "path/filepath" "strings" @@ -60,6 +61,66 @@ func setupTestEnvWithCleanup(t *testing.T, dataDir string) (cleanupFuncs []func( return cleanupFuncs } +func TestGoCSVWithInputsAndOutput(t *testing.T) { + // Move to testdata/non-standard to test on the non-standard project layout + // TODO: Refactor to make the chdir logic more readable + nonStandardTestDataDir := filepath.Join(testDataDir, "non-standard-layout") + for _, cleanupFunc := range setupTestEnvWithCleanup(t, nonStandardTestDataDir) { + defer cleanupFunc() + } + + // Temporary output dir for generating catalog bundle + outputDir, err := ioutil.TempDir("", t.Name()+"-output-catalog") + if err != nil { + log.Fatal(err) + } + // Clean up output catalog dir + defer func() { + if err := os.RemoveAll(outputDir); err != nil && !os.IsNotExist(err) { + // Not a test failure since files in /tmp will eventually get deleted + t.Logf("Failed to remove tmp generated catalog directory (%s): %v", outputDir, err) + } + }() + + cfg := gen.Config{ + OperatorName: testProjectName, + Inputs: map[string]string{ + DeployDirKey: "config", + APIsDirKey: "api", + }, + OutputDir: outputDir, + } + g := NewCSV(cfg, csvVersion, "") + + if err := g.Generate(); err != nil { + t.Fatalf("Failed to execute CSV generator: %v", err) + } + + csvFileName := getCSVFileName(testProjectName, csvVersion) + + // Read expected CSV + expCatalogDir := filepath.Join("expected-catalog", OLMCatalogChildDir) + csvExpBytes, err := ioutil.ReadFile(filepath.Join(expCatalogDir, testProjectName, csvVersion, csvFileName)) + if err != nil { + t.Fatalf("Failed to read expected CSV file: %v", err) + } + csvExp := string(csvExpBytes) + + // Read generated CSV from OutputDir/olm-catalog + outputCatalogDir := filepath.Join(cfg.OutputDir, OLMCatalogChildDir) + csvOutputBytes, err := ioutil.ReadFile(filepath.Join(outputCatalogDir, testProjectName, csvVersion, csvFileName)) + if err != nil { + t.Fatalf("Failed to read output CSV file: %v", err) + } + csvOutput := string(csvOutputBytes) + + assert.Equal(t, csvExp, csvOutput) +} + +// TODO: This test is only updating the existing CSV +// deploy/olm-catalog/memcached-operator/0.0.3/memcached-operator.v0.0.3.clusterserviceversion.yaml +// present in testdata/go +// Fix to generate a new CSV rather than only update an existing one func TestGoCSVFromNew(t *testing.T) { for _, cleanupFunc := range setupTestEnvWithCleanup(t, testGoDataDir) { defer cleanupFunc() @@ -69,8 +130,6 @@ func TestGoCSVFromNew(t *testing.T) { OperatorName: testProjectName, } g := NewCSV(cfg, csvVersion, "") - // TODO: Expand this test to test the g.Generate() method - // so we can test the CSV generator's input/out paths. fileMap, err := g.(csvGenerator).generate() if err != nil { t.Fatalf("Failed to execute CSV generator: %v", err) diff --git a/internal/generate/olm-catalog/package_manifest_test.go b/internal/generate/olm-catalog/package_manifest_test.go index 68e97ea85b2..1d994fb3b4d 100644 --- a/internal/generate/olm-catalog/package_manifest_test.go +++ b/internal/generate/olm-catalog/package_manifest_test.go @@ -15,6 +15,9 @@ package olmcatalog import ( + "io/ioutil" + "log" + "os" "path/filepath" "reflect" "testing" @@ -42,6 +45,61 @@ func newTestPackageManifestGenerator() gen.Generator { return pkgGen } +func TestNewPkgManifestWithInputsAndOutput(t *testing.T) { + nonStandardTestDataDir := filepath.Join(testDataDir, "non-standard-layout") + for _, cleanupFunc := range setupTestEnvWithCleanup(t, nonStandardTestDataDir) { + defer cleanupFunc() + } + + // Temporary output dir for generating catalog bundle + outputDir, err := ioutil.TempDir("", t.Name()+"-output-catalog") + if err != nil { + log.Fatal(err) + } + // Clean up output catalog dir + defer func() { + if err := os.RemoveAll(outputDir); err != nil && !os.IsNotExist(err) { + // Not a test failure since files in /tmp will eventually get deleted + t.Logf("Failed to remove tmp generated catalog directory (%s): %v", outputDir, err) + } + }() + + cfg := gen.Config{ + OperatorName: testProjectName, + Inputs: map[string]string{ + DeployDirKey: "config", + APIsDirKey: "api", + }, + OutputDir: outputDir, + } + + g := NewPackageManifest(cfg, csvVersion, "stable", true) + if err := g.Generate(); err != nil { + t.Fatalf("Failed to execute package manifest generator: %v", err) + } + + pkgManFileName := getPkgFileName(testProjectName) + + // Read expected Package Manifest + expCatalogDir := filepath.Join("expected-catalog", OLMCatalogChildDir) + pkgManExpBytes, err := ioutil.ReadFile(filepath.Join(expCatalogDir, testProjectName, pkgManFileName)) + if err != nil { + t.Fatalf("Failed to read expected package manifest file: %v", err) + } + pkgManExp := string(pkgManExpBytes) + + // Read generated Package Manifest from OutputDir/olm-catalog + outputCatalogDir := filepath.Join(cfg.OutputDir, OLMCatalogChildDir) + pkgManOutputBytes, err := ioutil.ReadFile(filepath.Join(outputCatalogDir, testProjectName, pkgManFileName)) + if err != nil { + t.Fatalf("Failed to read output package manifest file: %v", err) + } + pkgManOutput := string(pkgManOutputBytes) + + assert.Equal(t, pkgManExp, pkgManOutput) + +} + func TestGeneratePackageManifest(t *testing.T) { g := newTestPackageManifestGenerator() fileMap, err := g.(pkgGenerator).generate() diff --git a/internal/generate/testdata/non-standard-layout/api/cache/v1alpha1/doc.go b/internal/generate/testdata/non-standard-layout/api/cache/v1alpha1/doc.go new file mode 100644 index 00000000000..d5094b45a66 --- /dev/null +++ b/internal/generate/testdata/non-standard-layout/api/cache/v1alpha1/doc.go @@ -0,0 +1,18 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package v1alpha1 contains API Schema definitions for the cache v1alpha1 API group +// +k8s:deepcopy-gen=package,register +// +groupName=cache.example.com +package v1alpha1 diff --git a/internal/generate/testdata/non-standard-layout/api/cache/v1alpha1/dummy_types.go b/internal/generate/testdata/non-standard-layout/api/cache/v1alpha1/dummy_types.go new file mode 100644 index 00000000000..6499dade39a --- /dev/null +++ b/internal/generate/testdata/non-standard-layout/api/cache/v1alpha1/dummy_types.go @@ -0,0 +1,168 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +k8s:deepcopy-gen=false +// +k8s:openapi-gen=false +type NoKindSpec struct { + // Not included in anything, no kind type + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true + // +operator-sdk:gen-csv:customresourcedefinitions.statusDescriptors=true + Size int32 `json:"size"` + // Not included in anything, no kind type + Boss Hog `json:"hog"` +} + +// +k8s:deepcopy-gen=false +// +k8s:openapi-gen=false +type NoKindStatus struct { + // Not included in anything, no kind type + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true + // +operator-sdk:gen-csv:customresourcedefinitions.statusDescriptors=true + Nodes []string `json:"nodes"` +} + +// +k8s:deepcopy-gen=false +// +k8s:openapi-gen=false +type DummySpec struct { + // Should be in spec + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="dummy-pods" + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:podCount" + Size int32 `json:"size"` +} + +// +k8s:deepcopy-gen=false +// +k8s:openapi-gen=false +type DummyStatus struct { + // Should be in status but not spec, since DummyStatus isn't in DummySpec + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true + // +operator-sdk:gen-csv:customresourcedefinitions.statusDescriptors=true + Nodes []string `json:"nodes"` + // Not included in status but children should be + Boss Hog `json:"hog"` +} + +// +k8s:deepcopy-gen=false +// +k8s:openapi-gen=false +type Hog struct { + // Should be in status but not spec, since Hog isn't in DummySpec + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true + // +operator-sdk:gen-csv:customresourcedefinitions.statusDescriptors=true + // +operator-sdk:gen-csv:customresourcedefinitions.statusDescriptors.displayName="boss-hog-engine" + Engine Engine `json:"engine"` + // Not in spec or status, no boolean annotation + // +operator-sdk:gen-csv:customresourcedefinitions.statusDescriptors.displayName="doesnt-matter" + Brand string `json:"brand"` + // Not in spec or status + Helmet string `json:"helmet"` + // Fields should be inlined + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true + // +operator-sdk:gen-csv:customresourcedefinitions.statusDescriptors=true + Inlined InlinedComponent `json:",inline"` + // Fields should be inlined + InlinedComponent `json:",inline"` + // Should be ignored + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true + // +operator-sdk:gen-csv:customresourcedefinitions.statusDescriptors=true + Ignored IgnoredComponent `json:"-"` + // Should be ignored, but exported children should not be + notExported `json:",inline"` +} + +type notExported struct { + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true + // +operator-sdk:gen-csv:customresourcedefinitions.statusDescriptors=true + Public string `json:"foo"` + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true + // +operator-sdk:gen-csv:customresourcedefinitions.statusDescriptors=true + private string `json:"-"` +} + +// +k8s:deepcopy-gen=false +// +k8s:openapi-gen=false +type Engine struct { + // Should not be included, no annotations. + Pistons []string `json:"pistons"` +} + +// +k8s:deepcopy-gen=false +// +k8s:openapi-gen=false +type InlinedComponent struct { + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true + // +operator-sdk:gen-csv:customresourcedefinitions.statusDescriptors=true + SeatMaterial string `json:"seatMaterial"` +} + +// +k8s:deepcopy-gen=false +// +k8s:openapi-gen=false +type IgnoredComponent struct { + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true + // +operator-sdk:gen-csv:customresourcedefinitions.statusDescriptors=true + TrunkSpace string `json:"trunkSpace"` +} + +// +k8s:deepcopy-gen=false +// +k8s:openapi-gen=false +type OtherDummyStatus struct { + // Should be in status but not spec, since this isn't a spec type + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true + // +operator-sdk:gen-csv:customresourcedefinitions.statusDescriptors=true + Nothing string `json:"nothing"` +} + +// Dummy is the Schema for the dummy API +// +k8s:deepcopy-gen=false +// +k8s:openapi-gen=false +// +kubebuilder:subresource:status +// +kubebuilder:resource:path=dummys,scope=Namespaced +// +operator-sdk:gen-csv:customresourcedefinitions.displayName="Dummy App" +// +operator-sdk:gen-csv:customresourcedefinitions.resources="Deployment,v1,\"dummy-deployment\"" +// +operator-sdk:gen-csv:customresourcedefinitions.resources="ReplicaSet,v1beta2,\"dummy-replicaset\"" +// +operator-sdk:gen-csv:customresourcedefinitions.resources="Pod,v1,\"dummy-pod\"" +type Dummy struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec DummySpec `json:"spec,omitempty"` + Status DummyStatus `json:"status,omitempty"` +} + +// OtherDummy is the Schema for the other dummy API +// +k8s:deepcopy-gen=false +// +k8s:openapi-gen=false +// +operator-sdk:gen-csv:customresourcedefinitions.displayName="Other Dummy App" +// +operator-sdk:gen-csv:customresourcedefinitions.resources="Service,v1,\"other-dummy-service\"" +// +operator-sdk:gen-csv:customresourcedefinitions.resources="Pod,v1,\"other-dummy-pod\"" +type OtherDummy struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec Hog `json:"spec,omitempty"` + Status OtherDummyStatus `json:"status,omitempty"` +} + +// DummyList contains a list of Dummy +// +k8s:deepcopy-gen=false +// +k8s:openapi-gen=false +type DummyList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Dummy `json:"items"` +} diff --git a/internal/generate/testdata/non-standard-layout/api/cache/v1alpha1/memcached_types.go b/internal/generate/testdata/non-standard-layout/api/cache/v1alpha1/memcached_types.go new file mode 100644 index 00000000000..985292f393e --- /dev/null +++ b/internal/generate/testdata/non-standard-layout/api/cache/v1alpha1/memcached_types.go @@ -0,0 +1,57 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// MemcachedSpec defines the desired state of Memcached +type MemcachedSpec struct { + // Size is the size of the memcached deployment + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true + Size int32 `json:"size"` +} + +// MemcachedStatus defines the observed state of Memcached +type MemcachedStatus struct { + // Nodes are the names of the memcached pods + // +operator-sdk:gen-csv:customresourcedefinitions.statusDescriptors=true + Nodes []string `json:"nodes"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// Memcached is the Schema for the memcacheds API +// +kubebuilder:subresource:status +// +kubebuilder:resource:path=memcacheds,scope=Namespaced +// +kubebuilder:storageversion +// +operator-sdk:gen-csv:customresourcedefinitions.displayName="Memcached App" +type Memcached struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec MemcachedSpec `json:"spec,omitempty"` + Status MemcachedStatus `json:"status,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// MemcachedList contains a list of Memcached +type MemcachedList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Memcached `json:"items"` +} diff --git a/internal/generate/testdata/non-standard-layout/api/cache/v1alpha1/memcachedrs_types.go b/internal/generate/testdata/non-standard-layout/api/cache/v1alpha1/memcachedrs_types.go new file mode 100644 index 00000000000..ec8dde33bfc --- /dev/null +++ b/internal/generate/testdata/non-standard-layout/api/cache/v1alpha1/memcachedrs_types.go @@ -0,0 +1,55 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// MemcachedRSSpec defines the desired state of MemcachedRS +type MemcachedRSSpec struct { + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true + NumNodes int32 `json:"numNodes"` +} + +// MemcachedRSStatus defines the observed state of MemcachedRS +type MemcachedRSStatus struct { + // +operator-sdk:gen-csv:customresourcedefinitions.statusDescriptors=true + NodeList []string `json:"nodeList"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// MemcachedRS is the Schema for the memcachedrs API +// +kubebuilder:subresource:status +// +kubebuilder:resource:path=memcachedrs,scope=Namespaced +// +kubebuilder:storageversion +// +operator-sdk:gen-csv:customresourcedefinitions.displayName="MemcachedRS App" +type MemcachedRS struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec MemcachedRSSpec `json:"spec,omitempty"` + Status MemcachedRSStatus `json:"status,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// MemcachedRSList contains a list of MemcachedRS +type MemcachedRSList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []MemcachedRS `json:"items"` +} diff --git a/internal/generate/testdata/non-standard-layout/config/crds/cache.example.com_memcachedrs_crd.yaml b/internal/generate/testdata/non-standard-layout/config/crds/cache.example.com_memcachedrs_crd.yaml new file mode 100644 index 00000000000..9ffb3fab46d --- /dev/null +++ b/internal/generate/testdata/non-standard-layout/config/crds/cache.example.com_memcachedrs_crd.yaml @@ -0,0 +1,58 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: memcachedrs.cache.example.com +spec: + group: cache.example.com + names: + kind: MemcachedRS + listKind: MemcachedRSList + plural: memcachedrs + singular: memcachedrs + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + description: MemcachedRS is the Schema for the memcachedrs API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: MemcachedRSSpec defines the desired state of MemcachedRS + properties: + numNodes: + format: int32 + type: integer + required: + - numNodes + type: object + status: + description: MemcachedRSStatus defines the observed state of MemcachedRS + properties: + nodeList: + items: + type: string + type: array + test: + type: boolean + required: + - nodeList + - test + type: object + type: object + version: v1alpha1 + versions: + - name: v1alpha1 + served: true + storage: true diff --git a/internal/generate/testdata/non-standard-layout/config/crds/cache.example.com_memcacheds_crd.yaml b/internal/generate/testdata/non-standard-layout/config/crds/cache.example.com_memcacheds_crd.yaml new file mode 100644 index 00000000000..0109edbab9a --- /dev/null +++ b/internal/generate/testdata/non-standard-layout/config/crds/cache.example.com_memcacheds_crd.yaml @@ -0,0 +1,57 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: memcacheds.cache.example.com +spec: + group: cache.example.com + names: + kind: Memcached + listKind: MemcachedList + plural: memcacheds + singular: memcached + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + description: Memcached is the Schema for the memcacheds API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: MemcachedSpec defines the desired state of Memcached + properties: + size: + description: Size is the size of the memcached deployment + format: int32 + type: integer + required: + - size + type: object + status: + description: MemcachedStatus defines the observed state of Memcached + properties: + nodes: + description: Nodes are the names of the memcached pods + items: + type: string + type: array + required: + - nodes + type: object + type: object + version: v1alpha1 + versions: + - name: v1alpha1 + served: true + storage: true diff --git a/internal/generate/testdata/non-standard-layout/config/crds/cache.example.com_v1alpha1_memcached_cr.yaml b/internal/generate/testdata/non-standard-layout/config/crds/cache.example.com_v1alpha1_memcached_cr.yaml new file mode 100644 index 00000000000..2b8f17c3998 --- /dev/null +++ b/internal/generate/testdata/non-standard-layout/config/crds/cache.example.com_v1alpha1_memcached_cr.yaml @@ -0,0 +1,7 @@ +apiVersion: cache.example.com/v1alpha1 +kind: Memcached +metadata: + name: example-memcached +spec: + # Add fields here + size: 3 diff --git a/internal/generate/testdata/non-standard-layout/config/crds/cache.example.com_v1alpha1_memcachedrs_cr.yaml b/internal/generate/testdata/non-standard-layout/config/crds/cache.example.com_v1alpha1_memcachedrs_cr.yaml new file mode 100644 index 00000000000..92861ed5fe5 --- /dev/null +++ b/internal/generate/testdata/non-standard-layout/config/crds/cache.example.com_v1alpha1_memcachedrs_cr.yaml @@ -0,0 +1,6 @@ +apiVersion: cache.example.com/v1alpha1 +kind: MemcachedRS +metadata: + name: example-memcachedrs +spec: + numNodes: 4 diff --git a/internal/generate/testdata/non-standard-layout/config/operator.yaml b/internal/generate/testdata/non-standard-layout/config/operator.yaml new file mode 100644 index 00000000000..8a00f16ad68 --- /dev/null +++ b/internal/generate/testdata/non-standard-layout/config/operator.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: memcached-operator +spec: + replicas: 1 + selector: + matchLabels: + name: memcached-operator + template: + metadata: + labels: + name: memcached-operator + spec: + serviceAccountName: memcached-operator + containers: + - name: memcached-operator + image: quay.io/example/memcached-operator:v0.0.3 + command: + - memcached-operator + imagePullPolicy: Never + env: + - name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPERATOR_NAME + value: memcached-operator diff --git a/internal/generate/testdata/non-standard-layout/config/role.yaml b/internal/generate/testdata/non-standard-layout/config/role.yaml new file mode 100644 index 00000000000..d1a2a04d0d0 --- /dev/null +++ b/internal/generate/testdata/non-standard-layout/config/role.yaml @@ -0,0 +1,61 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: memcached-operator +rules: +- apiGroups: + - "" + resources: + - pods + - services + - services/finalizers + - endpoints + - persistentvolumeclaims + - events + - configmaps + - secrets + verbs: + - '*' +- apiGroups: + - apps + resources: + - deployments + - daemonsets + - replicasets + - statefulsets + verbs: + - '*' +- apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + verbs: + - get + - create +- apiGroups: + - apps + resourceNames: + - memcached-operator + resources: + - deployments/finalizers + verbs: + - update +- apiGroups: + - "" + resources: + - pods + verbs: + - get +- apiGroups: + - apps + resources: + - replicasets + - deployments + verbs: + - get +- apiGroups: + - cache.example.com + resources: + - '*' + verbs: + - '*' diff --git a/internal/generate/testdata/non-standard-layout/config/role_binding.yaml b/internal/generate/testdata/non-standard-layout/config/role_binding.yaml new file mode 100644 index 00000000000..322ecc9e6ac --- /dev/null +++ b/internal/generate/testdata/non-standard-layout/config/role_binding.yaml @@ -0,0 +1,11 @@ +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: memcached-operator +subjects: +- kind: ServiceAccount + name: memcached-operator +roleRef: + kind: Role + name: memcached-operator + apiGroup: rbac.authorization.k8s.io diff --git a/internal/generate/testdata/non-standard-layout/config/service_account.yaml b/internal/generate/testdata/non-standard-layout/config/service_account.yaml new file mode 100644 index 00000000000..8d58bc78322 --- /dev/null +++ b/internal/generate/testdata/non-standard-layout/config/service_account.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: memcached-operator diff --git a/internal/generate/testdata/non-standard-layout/expected-catalog/olm-catalog/memcached-operator/0.0.3/memcached-operator.v0.0.3.clusterserviceversion.yaml b/internal/generate/testdata/non-standard-layout/expected-catalog/olm-catalog/memcached-operator/0.0.3/memcached-operator.v0.0.3.clusterserviceversion.yaml new file mode 100644 index 00000000000..c68ff8d8ad1 --- /dev/null +++ b/internal/generate/testdata/non-standard-layout/expected-catalog/olm-catalog/memcached-operator/0.0.3/memcached-operator.v0.0.3.clusterserviceversion.yaml @@ -0,0 +1,154 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: ClusterServiceVersion +metadata: + annotations: + alm-examples: |- + [ + { + "apiVersion": "cache.example.com/v1alpha1", + "kind": "Memcached", + "metadata": { + "name": "example-memcached" + }, + "spec": { + "size": 3 + } + }, + { + "apiVersion": "cache.example.com/v1alpha1", + "kind": "MemcachedRS", + "metadata": { + "name": "example-memcachedrs" + }, + "spec": { + "numNodes": 4 + } + } + ] + capabilities: Basic Install + name: memcached-operator.v0.0.3 + namespace: placeholder +spec: + apiservicedefinitions: {} + customresourcedefinitions: + owned: + - kind: MemcachedRS + name: memcachedrs.cache.example.com + version: v1alpha1 + - kind: Memcached + name: memcacheds.cache.example.com + version: v1alpha1 + displayName: Memcached Operator + icon: + - base64data: "" + mediatype: "" + install: + spec: + deployments: + - name: memcached-operator + spec: + replicas: 1 + selector: + matchLabels: + name: memcached-operator + strategy: {} + template: + metadata: + labels: + name: memcached-operator + spec: + containers: + - command: + - memcached-operator + env: + - name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.annotations['olm.targetNamespaces'] + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPERATOR_NAME + value: memcached-operator + image: quay.io/example/memcached-operator:v0.0.3 + imagePullPolicy: Never + name: memcached-operator + resources: {} + serviceAccountName: memcached-operator + permissions: + - rules: + - apiGroups: + - "" + resources: + - pods + - services + - services/finalizers + - endpoints + - persistentvolumeclaims + - events + - configmaps + - secrets + verbs: + - '*' + - apiGroups: + - apps + resources: + - deployments + - daemonsets + - replicasets + - statefulsets + verbs: + - '*' + - apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + verbs: + - get + - create + - apiGroups: + - apps + resourceNames: + - memcached-operator + resources: + - deployments/finalizers + verbs: + - update + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - apiGroups: + - apps + resources: + - replicasets + - deployments + verbs: + - get + - apiGroups: + - cache.example.com + resources: + - '*' + verbs: + - '*' + serviceAccountName: memcached-operator + strategy: deployment + installModes: + - supported: true + type: OwnNamespace + - supported: true + type: SingleNamespace + - supported: false + type: MultiNamespace + - supported: true + type: AllNamespaces + keywords: + - "" + maintainers: + - {} + maturity: alpha + provider: {} + version: 0.0.3 diff --git a/internal/generate/testdata/non-standard-layout/expected-catalog/olm-catalog/memcached-operator/memcached-operator.package.yaml b/internal/generate/testdata/non-standard-layout/expected-catalog/olm-catalog/memcached-operator/memcached-operator.package.yaml new file mode 100644 index 00000000000..64d34201345 --- /dev/null +++ b/internal/generate/testdata/non-standard-layout/expected-catalog/olm-catalog/memcached-operator/memcached-operator.package.yaml @@ -0,0 +1,5 @@ +channels: +- currentCSV: memcached-operator.v0.0.3 + name: stable +defaultChannel: stable +packageName: memcached-operator From 09e3e4b7830779cd50f46652b0fb423d0489e213 Mon Sep 17 00:00:00 2001 From: hasbro17 Date: Fri, 14 Feb 2020 17:25:18 -0800 Subject: [PATCH 06/16] Bugfix for using OutputDir to look for olm-catalog dir in the generators --- internal/generate/olm-catalog/csv.go | 36 +++++++++---------- internal/generate/olm-catalog/csv_go_test.go | 6 +++- .../generate/olm-catalog/package_manifest.go | 20 +++++------ .../olm-catalog/package_manifest_test.go | 23 ++++++------ 4 files changed, 43 insertions(+), 42 deletions(-) diff --git a/internal/generate/olm-catalog/csv.go b/internal/generate/olm-catalog/csv.go index 6c165efd6fc..161f2bce35c 100644 --- a/internal/generate/olm-catalog/csv.go +++ b/internal/generate/olm-catalog/csv.go @@ -82,32 +82,32 @@ func NewCSV(cfg gen.Config, csvVersion, fromVersion string) gen.Generator { if g.Inputs == nil { g.Inputs = map[string]string{} } - // The olm-catalog directory location depends on where the deploy (operator - // manifests) directory is. - olmCatalogDir := OLMCatalogDir - if deployDir, ok := g.Inputs[DeployDirKey]; !ok || deployDir == "" { - g.Inputs[DeployDirKey] = scaffold.DeployDir - } else { - // Set to non-standard location. - olmCatalogDir = filepath.Join(g.Inputs[DeployDirKey], OLMCatalogChildDir) + + // The olm-catalog directory location depends on where the output directory is set. + if g.OutputDir == "" { + g.OutputDir = scaffold.DeployDir } + // Set the CSV bundle dir output path under the generator's OutputDir + olmCatalogDir := filepath.Join(g.OutputDir, OLMCatalogChildDir) + g.csvOutputDir = filepath.Join(olmCatalogDir, g.OperatorName, g.csvVersion) - parentDir := filepath.Join(olmCatalogDir, g.OperatorName) - if isBundleDirExist(parentDir, g.fromVersion) { - g.existingCSVBundleDir = filepath.Join(olmCatalogDir, g.OperatorName, g.fromVersion) - } else if isBundleDirExist(parentDir, g.csvVersion) { - g.existingCSVBundleDir = filepath.Join(olmCatalogDir, g.OperatorName, g.csvVersion) + bundleParentDir := filepath.Join(olmCatalogDir, g.OperatorName) + if isBundleDirExist(bundleParentDir, g.fromVersion) { + // Upgrading a new CSV from previous CSV version + g.existingCSVBundleDir = filepath.Join(bundleParentDir, g.fromVersion) + } else if isBundleDirExist(bundleParentDir, g.csvVersion) { + // Updating an existing CSV version + g.existingCSVBundleDir = filepath.Join(bundleParentDir, g.csvVersion) + } + + if deployDir, ok := g.Inputs[DeployDirKey]; !ok || deployDir == "" { + g.Inputs[DeployDirKey] = scaffold.DeployDir } if apisDir, ok := g.Inputs[APIsDirKey]; !ok || apisDir == "" { g.Inputs[APIsDirKey] = scaffold.ApisDir } - if g.OutputDir == "" { - g.OutputDir = scaffold.DeployDir - } - // Set the CSV bundle dir output path under the generator's OutputDir - g.csvOutputDir = filepath.Join(g.OutputDir, OLMCatalogChildDir, g.OperatorName, g.csvVersion) return g } diff --git a/internal/generate/olm-catalog/csv_go_test.go b/internal/generate/olm-catalog/csv_go_test.go index 8ea6823f01a..8f138769794 100644 --- a/internal/generate/olm-catalog/csv_go_test.go +++ b/internal/generate/olm-catalog/csv_go_test.go @@ -61,7 +61,7 @@ func setupTestEnvWithCleanup(t *testing.T, dataDir string) (cleanupFuncs []func( return cleanupFuncs } -func TestGoCSVWithInputsAndOutput(t *testing.T) { +func TestGenerateNewCSVWithInputsToOutput(t *testing.T) { // Move to testdata/non-standard to test on the non-standard project layout // TODO: Refactor to make the chdir logic more readable nonStandardTestDataDir := filepath.Join(testDataDir, "non-standard-layout") @@ -117,6 +117,10 @@ func TestGoCSVWithInputsAndOutput(t *testing.T) { assert.Equal(t, csvExp, csvOutput) } +func TestUpgradeFromExistingCSVWithInputsToOutput() { + // TODO +} + // TODO: This test is only updating the existing CSV // deploy/olm-catalog/memcached-operator/0.0.3/memcached-operator.v0.0.3.clusterserviceversion.yaml // present in testdata/go diff --git a/internal/generate/olm-catalog/package_manifest.go b/internal/generate/olm-catalog/package_manifest.go index de3813560b6..2d9d73b2724 100644 --- a/internal/generate/olm-catalog/package_manifest.go +++ b/internal/generate/olm-catalog/package_manifest.go @@ -63,16 +63,15 @@ func NewPackageManifest(cfg gen.Config, csvVersion, channel string, isDefault bo channelIsDefault: isDefault, fileName: getPkgFileName(cfg.OperatorName), } - if g.Inputs == nil { - g.Inputs = map[string]string{} - } - // The olm-catalog directory location depends on where the deploy (operator - // manifests) directory is. - olmCatalogDir := OLMCatalogDir - if deployDir, ok := g.Inputs[DeployDirKey]; ok && deployDir != "" { - // Set to non-standard location. - olmCatalogDir = filepath.Join(g.Inputs[DeployDirKey], OLMCatalogChildDir) + + // Pkg manifest generator has no defined inputs + g.Inputs = map[string]string{} + + // The olm-catalog directory location depends on where the output directory is set. + if g.OutputDir == "" { + g.OutputDir = scaffold.DeployDir } + olmCatalogDir := filepath.Join(g.OutputDir, OLMCatalogChildDir) // Check if the generator should update from an existing package manifest file pkgManifestDirPath := filepath.Join(olmCatalogDir, g.OperatorName) @@ -81,9 +80,6 @@ func NewPackageManifest(cfg gen.Config, csvVersion, channel string, isDefault bo g.existingPkgManifestDir = pkgManifestDirPath } - if g.OutputDir == "" { - g.OutputDir = scaffold.DeployDir - } return g } diff --git a/internal/generate/olm-catalog/package_manifest_test.go b/internal/generate/olm-catalog/package_manifest_test.go index 1d994fb3b4d..cea138907b3 100644 --- a/internal/generate/olm-catalog/package_manifest_test.go +++ b/internal/generate/olm-catalog/package_manifest_test.go @@ -32,20 +32,21 @@ import ( func newTestPackageManifestGenerator() gen.Generator { cfg := gen.Config{ OperatorName: testProjectName, + OutputDir: testGoDataDir, } - g := NewPackageManifest(cfg, csvVersion, "stable", true) + return NewPackageManifest(cfg, csvVersion, "stable", true) // TODO: The deploy dir input should be the way to point the generator to test data // E.g: Inputs: map[string]string{DeployDirKey: testGoDataDir} // Override the generator to point to the package manifest in testGoDataDir - testPkgManifestDir := filepath.Join(testGoDataDir, OLMCatalogDir, testProjectName) - pkgGen := g.(pkgGenerator) - pkgGen.existingPkgManifestDir = testPkgManifestDir - return pkgGen + // testPkgManifestDir := filepath.Join(testGoDataDir, OLMCatalogDir, testProjectName) + // pkgGen := g.(pkgGenerator) + // pkgGen.existingPkgManifestDir = testPkgManifestDir + // return pkgGen } -func TestNewPkgManifestWithInputsAndOutput(t *testing.T) { +func TestGeneratePkgManifestToOutput(t *testing.T) { nonStandardTestDataDir := filepath.Join(testDataDir, "non-standard-layout") for _, cleanupFunc := range setupTestEnvWithCleanup(t, nonStandardTestDataDir) { defer cleanupFunc() @@ -66,11 +67,7 @@ func TestNewPkgManifestWithInputsAndOutput(t *testing.T) { cfg := gen.Config{ OperatorName: testProjectName, - Inputs: map[string]string{ - DeployDirKey: "config", - APIsDirKey: "api", - }, - OutputDir: outputDir, + OutputDir: outputDir, } g := NewPackageManifest(cfg, csvVersion, "stable", true) @@ -100,6 +97,10 @@ func TestNewPkgManifestWithInputsAndOutput(t *testing.T) { } +func TestUpdatePkgManifestToOutput(t *testing.T) { + // TODO +} + func TestGeneratePackageManifest(t *testing.T) { g := newTestPackageManifestGenerator() fileMap, err := g.(pkgGenerator).generate() From b111036ef1527da437bcac3e34cfd26f13db050f Mon Sep 17 00:00:00 2001 From: hasbro17 Date: Thu, 5 Mar 2020 23:51:52 -0800 Subject: [PATCH 07/16] Refactor GetCRDDescriptionForGVK(), add unit tests, replace --inputs with individual flags, add caveats and TODOs --- CHANGELOG.md | 1 - cmd/operator-sdk/generate/csv.go | 64 ++++-- doc/cli/operator-sdk_generate_csv.md | 44 +++-- internal/generate/olm-catalog/csv.go | 15 +- internal/generate/olm-catalog/csv_go_test.go | 185 +++++++++++++++--- internal/generate/olm-catalog/csv_updaters.go | 2 - .../olm-catalog/descriptor/descriptor.go | 129 ++++++++++-- .../olm-catalog/descriptor/descriptor_test.go | 161 +++++++++++---- .../olm-catalog/package_manifest_test.go | 48 ++--- .../api/v1alpha1/memcached_types.go} | 32 +-- .../api/cache/v1alpha1/dummy_types.go | 168 ---------------- .../api/cache/v1alpha1/memcached_types.go | 2 +- .../cache.example.com_memcachedrs_crd.yaml | 58 ------ ...e.example.com_v1alpha1_memcachedrs_cr.yaml | 6 - ...operator.v0.0.1.clusterserviceversion.yaml | 153 +++++++++++++++ ...operator.v0.0.3.clusterserviceversion.yaml | 30 +-- ...operator.v0.0.4.clusterserviceversion.yaml | 155 +++++++++++++++ internal/util/k8sutil/crd.go | 46 +++-- internal/util/projutil/project_util.go | 19 ++ 19 files changed, 879 insertions(+), 439 deletions(-) rename internal/generate/testdata/{non-standard-layout/api/cache/v1alpha1/memcachedrs_types.go => go/api/v1alpha1/memcached_types.go} (64%) delete mode 100644 internal/generate/testdata/non-standard-layout/api/cache/v1alpha1/dummy_types.go delete mode 100644 internal/generate/testdata/non-standard-layout/config/crds/cache.example.com_memcachedrs_crd.yaml delete mode 100644 internal/generate/testdata/non-standard-layout/config/crds/cache.example.com_v1alpha1_memcachedrs_cr.yaml create mode 100644 internal/generate/testdata/non-standard-layout/expected-catalog/olm-catalog/memcached-operator/0.0.1/memcached-operator.v0.0.1.clusterserviceversion.yaml create mode 100644 internal/generate/testdata/non-standard-layout/expected-catalog/olm-catalog/memcached-operator/0.0.4/memcached-operator.v0.0.4.clusterserviceversion.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f0955aa777..d33a7f61221 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,7 +82,6 @@ ### Removed - **Breaking Change:** The additional Ansible sidecar container. ([#2586](https://github.com/operator-framework/operator-sdk/pull/2586)) -- **Breaking Change:** Removed CSV configuration file support (defaulting to deploy/olm-catalog/csv-config.yaml) in favor of specifying inputs to the generator via [`generate csv --input`](doc/cli/operator-sdk_generate_csv.md#options), and configuring output locations via [`generate csv --outputDir`](doc/cli/operator-sdk_generate_csv.md#options). ([#2511](https://github.com/operator-framework/operator-sdk/pull/2511)) ### Bug Fixes diff --git a/cmd/operator-sdk/generate/csv.go b/cmd/operator-sdk/generate/csv.go index 14736b63e09..769add8a7a6 100644 --- a/cmd/operator-sdk/generate/csv.go +++ b/cmd/operator-sdk/generate/csv.go @@ -38,7 +38,9 @@ type csvCmd struct { fromVersion string operatorName string outputDir string - inputs map[string]string + deployDir string + apisDir string + crdDir string updateCRDs bool defaultChannel bool } @@ -54,6 +56,9 @@ for the operator. This file is used to publish the operator to the OLM Catalog. A CSV semantic version is supplied via the --csv-version flag. If your operator has already generated a CSV manifest you want to use as a base, supply its version to --from-version. Otherwise the SDK will scaffold a new CSV manifest.`, + Example: ` +TODO + `, RunE: func(cmd *cobra.Command, args []string) error { if len(args) != 0 { @@ -62,6 +67,10 @@ version to --from-version. Otherwise the SDK will scaffold a new CSV manifest.`, if err := c.validate(); err != nil { return fmt.Errorf("error validating command flags: %v", err) } + // Default for crd dir if unset + if c.crdDir == "" { + c.crdDir = c.deployDir + } if err := c.run(); err != nil { log.Fatal(err) } @@ -76,18 +85,25 @@ version to --from-version. Otherwise the SDK will scaffold a new CSV manifest.`, } cmd.Flags().StringVar(&c.fromVersion, "from-version", "", "Semantic version of an existing CSV to use as a base") - cmd.Flags().StringToStringVar(&c.inputs, "inputs", - map[string]string{ - gencatalog.DeployDirKey: "deploy", - gencatalog.APIsDirKey: filepath.Join("pkg", "apis"), - }, - `Key value input paths used in CSV generation. -Use this to set custom paths for operator manifests and API type definitions -E.g: --inputs deploy=config/production,apis=pkg/myapp/apis -Supported input keys: - - deploy= - - apis= + + cmd.Flags().StringVar(&c.deployDir, "deploy-dir", "deploy", + `Project relative path to root directory for operator manifests (Deployment, RBAC, CRDs). +The CSV file contents will be generated from the manifests present in this directory. +`) + cmd.Flags().StringVar(&c.apisDir, "apis-dir", filepath.Join("pkg", "apis"), + `Project relative path to root directory for API type defintions. +The CSV annotation comments will be parsed from the Go types under this path to +fill out metadata for owned APIs in spec.customresourcedefinitions.owned. `) + cmd.Flags().StringVar(&c.crdDir, "crd-dir", "", + `Project relative path to root directory for for CRD manifests. +Used when --update-crds is set to copy over CRD manifests to the CSV bundle directory. +Note: The CSV generator only uses this to copy the CRD manifests. +The CSV contents for spec.customresourcedefinitions.owned will still be updated +from the CRD manifests in the deploy directory specified by --deploy-dir. +If unset, it defaults to the same value as --deploy-dir. +`) + cmd.Flags().StringVar(&c.outputDir, "output-dir", scaffold.DeployDir, "Base directory to output generated CSV. The resulting CSV bundle directory"+ "will be \"/olm-catalog//\"") @@ -105,7 +121,6 @@ Supported input keys: } func (c csvCmd) run() error { - log.Infof("Generating CSV manifest version %s", c.csvVersion) if c.operatorName == "" { @@ -113,8 +128,13 @@ func (c csvCmd) run() error { } cfg := gen.Config{ OperatorName: c.operatorName, - Inputs: c.inputs, - OutputDir: c.outputDir, + // TODO(hasbro17): Remove the Input key map when the Generator input keys + // are removed in favour of config fields in the csvGenerator + Inputs: map[string]string{ + gencatalog.DeployDirKey: c.deployDir, + gencatalog.APIsDirKey: c.apisDir, + }, + OutputDir: c.outputDir, } csv := gencatalog.NewCSV(cfg, c.csvVersion, c.fromVersion) @@ -128,10 +148,12 @@ func (c csvCmd) run() error { // Write CRD's to the new or updated CSV package dir. if c.updateCRDs { - // TODO: Add a "crd" key to the --inputs flag to allow - // configuring the location of the CRD manifests directory - // from which to find and update CRD's. - crdManifestSet, err := findCRDFileSet(c.inputs[gencatalog.DeployDirKey]) + // TODO(hasbro17): Reconsider the crd-dir flag since it only lets you control + // where the CRD manifests are copied from but the CSV generator above + // will uses the CRD manifests in deploy dir to build the CSV contents for + // spec.customresourcedefinitions.owned + // Need to reconcile this disparity. + crdManifestSet, err := findCRDFileSet(c.crdDir) if err != nil { return fmt.Errorf("failed to update CRD's: %v", err) } @@ -194,7 +216,9 @@ func findCRDFileSet(path string) (map[string][]byte, error) { return nil, fmt.Errorf("crd's must be read from a directory. %s is a file", path) } - crdPaths, err := k8sutil.GetCRDManifestPaths(path) + // Get CRD manifest paths from path recursively but ignore olm-catalog subdir + // if it is present in the search directory + crdPaths, err := k8sutil.GetCRDManifestPaths(path, gencatalog.OLMCatalogChildDir) if err != nil { return nil, err } diff --git a/doc/cli/operator-sdk_generate_csv.md b/doc/cli/operator-sdk_generate_csv.md index e80d5d10465..e3fb3a22778 100644 --- a/doc/cli/operator-sdk_generate_csv.md +++ b/doc/cli/operator-sdk_generate_csv.md @@ -15,24 +15,38 @@ version to --from-version. Otherwise the SDK will scaffold a new CSV manifest. operator-sdk generate csv [flags] ``` +### Examples + +``` + +TODO + +``` + ### Options ``` - --csv-channel string Channel the CSV should be registered under in the package manifest - --csv-version string Semantic version of the CSV - --default-channel Use the channel passed to --csv-channel as the package manifests' default channel. Only valid when --csv-channel is set - --from-version string Semantic version of an existing CSV to use as a base - -h, --help help for csv - --inputs stringToString Key value input paths used in CSV generation. - Use this to set custom paths for operator manifests and API type definitions - E.g: --inputs deploy=config/production,apis=pkg/myapp/apis - Supported input keys: - - deploy= - - apis= - (default [deploy=deploy,apis=pkg/apis]) - --operator-name string Operator name to use while generating CSV - --output-dir string Base directory to output generated CSV. The resulting CSV bundle directorywill be "/olm-catalog//" (default "deploy") - --update-crds Update CRD manifests in deploy/{operator-name}/{csv-version} the using latest API's + --apis-dir string Project relative path to root directory for API type defintions. + The CSV annotation comments will be parsed from the Go types under this path to + fill out metadata for owned APIs in spec.customresourcedefinitions.owned. + (default "pkg/apis") + --crd-dir string Project relative path to root directory for for CRD manifests. + Used when --update-crds is set to copy over CRD manifests to the CSV bundle directory. + Note: The CSV generator only uses this to copy the CRD manifests. + The CSV contents for spec.customresourcedefinitions.owned will still be updated + from the CRD manifests in the deploy directory specified by --deploy-dir. + (default "deploy") + --csv-channel string Channel the CSV should be registered under in the package manifest + --csv-version string Semantic version of the CSV + --default-channel Use the channel passed to --csv-channel as the package manifests' default channel. Only valid when --csv-channel is set + --deploy-dir string Project relative path to root directory for operator manifests (Deployment, RBAC, CRDs). + The CSV file contents will be generated from the manifests present in this directory. + (default "deploy") + --from-version string Semantic version of an existing CSV to use as a base + -h, --help help for csv + --operator-name string Operator name to use while generating CSV + --output-dir string Base directory to output generated CSV. The resulting CSV bundle directorywill be "/olm-catalog//" (default "deploy") + --update-crds Update CRD manifests in deploy/{operator-name}/{csv-version} the using latest API's ``` ### SEE ALSO diff --git a/internal/generate/olm-catalog/csv.go b/internal/generate/olm-catalog/csv.go index 161f2bce35c..8afc6c03db0 100644 --- a/internal/generate/olm-catalog/csv.go +++ b/internal/generate/olm-catalog/csv.go @@ -369,9 +369,19 @@ func (g csvGenerator) updateCSVFromManifests(csv *olmapiv1alpha1.ClusterServiceV kindManifestMap := map[schema.GroupVersionKind][][]byte{} crGVKSet := map[schema.GroupVersionKind]struct{}{} err = filepath.Walk(g.Inputs[DeployDirKey], func(path string, info os.FileInfo, werr error) error { - if werr != nil || info.IsDir() { + if werr != nil { + log.Debugf("Failed to walk dir: %v", werr) return werr } + // Only read manifest from files, not directories + if info.IsDir() { + // Skip walking olm-catalog dir if it's present in the deploy directory + if info.Name() == OLMCatalogChildDir { + return filepath.SkipDir + } + return nil + } + b, err := ioutil.ReadFile(path) if err != nil { return err @@ -407,6 +417,9 @@ func (g csvGenerator) updateCSVFromManifests(csv *olmapiv1alpha1.ClusterServiceV } return scanner.Err() }) + if err != nil { + return fmt.Errorf("failed to walk manifests directory for CSV updates: %v", err) + } crUpdaters := crs{} for gvk, manifests := range kindManifestMap { diff --git a/internal/generate/olm-catalog/csv_go_test.go b/internal/generate/olm-catalog/csv_go_test.go index 8f138769794..32b57c7e9d6 100644 --- a/internal/generate/olm-catalog/csv_go_test.go +++ b/internal/generate/olm-catalog/csv_go_test.go @@ -18,13 +18,16 @@ import ( "io/ioutil" "log" "os" + "os/exec" "path/filepath" "strings" "testing" gen "github.com/operator-framework/operator-sdk/internal/generate/gen" "github.com/operator-framework/operator-sdk/internal/scaffold" + "github.com/operator-framework/operator-sdk/internal/util/fileutil" internalk8sutil "github.com/operator-framework/operator-sdk/internal/util/k8sutil" + "github.com/operator-framework/operator-sdk/internal/util/projutil" "github.com/operator-framework/operator-sdk/pkg/k8sutil" "github.com/blang/semver" @@ -41,11 +44,18 @@ const ( ) var ( - testDataDir = filepath.Join("..", "testdata") - testGoDataDir = filepath.Join(testDataDir, "go") + // Used to change directory to project root so the CSV generator's descriptor updates can form + // the correct pkg import paths to the API types directory + // See limitation with projutil.GetGoPkg() + relativeProjectRootDir = filepath.Join("..", "..", "..") + // Used to form all relative paths from the SDK root dir to + // the API, CRD, and deploy test dirs + testDataRelativePathFromRoot = filepath.Join("internal", "generate", "testdata") + testGoDataDir = filepath.Join(testDataRelativePathFromRoot, "go") + testNonStandardLayoutDataDir = filepath.Join(testDataRelativePathFromRoot, "non-standard-layout") ) -func setupTestEnvWithCleanup(t *testing.T, dataDir string) (cleanupFuncs []func()) { +func chDirWithCleanup(t *testing.T, dataDir string) func() { wd, err := os.Getwd() if err != nil { t.Fatal(err) @@ -53,21 +63,19 @@ func setupTestEnvWithCleanup(t *testing.T, dataDir string) (cleanupFuncs []func( if err := os.Chdir(dataDir); err != nil { t.Fatal(err) } - cleanupFuncs = append(cleanupFuncs, func() { + chDirCleanupFunc := func() { if err := os.Chdir(wd); err != nil { t.Fatal(err) } - }) - return cleanupFuncs + } + return chDirCleanupFunc } +// TODO: Change to table driven subtests to test out different Inputs/Output for the generator func TestGenerateNewCSVWithInputsToOutput(t *testing.T) { - // Move to testdata/non-standard to test on the non-standard project layout - // TODO: Refactor to make the chdir logic more readable - nonStandardTestDataDir := filepath.Join(testDataDir, "non-standard-layout") - for _, cleanupFunc := range setupTestEnvWithCleanup(t, nonStandardTestDataDir) { - defer cleanupFunc() - } + // Change directory to project root so the test cases can form the correct pkg imports + cleanupFunc := chDirWithCleanup(t, relativeProjectRootDir) + defer cleanupFunc() // Temporary output dir for generating catalog bundle outputDir, err := ioutil.TempDir("", t.Name()+"-output-catalog") @@ -85,11 +93,12 @@ func TestGenerateNewCSVWithInputsToOutput(t *testing.T) { cfg := gen.Config{ OperatorName: testProjectName, Inputs: map[string]string{ - DeployDirKey: "config", - APIsDirKey: "api", + DeployDirKey: filepath.Join(testNonStandardLayoutDataDir, "config"), + APIsDirKey: filepath.Join(testNonStandardLayoutDataDir, "api"), }, OutputDir: outputDir, } + csvVersion := "0.0.1" g := NewCSV(cfg, csvVersion, "") if err := g.Generate(); err != nil { @@ -99,7 +108,7 @@ func TestGenerateNewCSVWithInputsToOutput(t *testing.T) { csvFileName := getCSVFileName(testProjectName, csvVersion) // Read expected CSV - expCatalogDir := filepath.Join("expected-catalog", OLMCatalogChildDir) + expCatalogDir := filepath.Join(testNonStandardLayoutDataDir, "expected-catalog", OLMCatalogChildDir) csvExpBytes, err := ioutil.ReadFile(filepath.Join(expCatalogDir, testProjectName, csvVersion, csvFileName)) if err != nil { t.Fatalf("Failed to read expected CSV file: %v", err) @@ -117,8 +126,73 @@ func TestGenerateNewCSVWithInputsToOutput(t *testing.T) { assert.Equal(t, csvExp, csvOutput) } -func TestUpgradeFromExistingCSVWithInputsToOutput() { - // TODO +func TestUpgradeFromExistingCSVWithInputsToOutput(t *testing.T) { + // Change directory to project root so the test cases can form the correct pkg imports + cleanupFunc := chDirWithCleanup(t, relativeProjectRootDir) + defer cleanupFunc() + + // Temporary output dir for generating catalog bundle + outputDir, err := ioutil.TempDir("", t.Name()+"-output-catalog") + if err != nil { + log.Fatal(err) + } + // Clean up output catalog dir + defer func() { + if err := os.RemoveAll(outputDir); err != nil && !os.IsNotExist(err) { + // Not a test failure since files in /tmp will eventually get deleted + t.Logf("Failed to remove tmp generated catalog directory (%s): %v", outputDir, err) + } + }() + + cfg := gen.Config{ + OperatorName: testProjectName, + Inputs: map[string]string{ + DeployDirKey: filepath.Join(testNonStandardLayoutDataDir, "config"), + APIsDirKey: filepath.Join(testNonStandardLayoutDataDir, "api"), + }, + OutputDir: outputDir, + } + fromVersion := "0.0.3" + csvVersion := "0.0.4" + + // Copy over expected fromVersion CSV bundle directory to the output dir + // so the test can upgrade from it + outputFromCSVDir := filepath.Join(outputDir, OLMCatalogChildDir, testProjectName) + if err := os.MkdirAll(outputFromCSVDir, os.FileMode(fileutil.DefaultDirFileMode)); err != nil { + t.Fatalf("Failed to create CSV bundle dir (%s) for fromVersion (%s): %v", outputFromCSVDir, fromVersion, err) + } + expCatalogDir := filepath.Join(testNonStandardLayoutDataDir, "expected-catalog", OLMCatalogChildDir) + expFromCSVDir := filepath.Join(expCatalogDir, testProjectName, fromVersion) + cmd := exec.Command("cp", "-r", expFromCSVDir, outputFromCSVDir) + t.Logf("Copying expected fromVersion CSV manifest dir %#v", cmd.Args) + if err := projutil.ExecCmd(cmd); err != nil { + t.Fatalf("Failed to copy expected CSV bundle dir (%s) to output dir (%s): %v", expFromCSVDir, outputFromCSVDir, err) + } + + // Upgrade new CSV from old + g := NewCSV(cfg, csvVersion, fromVersion) + if err := g.Generate(); err != nil { + t.Fatalf("Failed to execute CSV generator: %v", err) + } + csvFileName := getCSVFileName(testProjectName, csvVersion) + + // Read expected CSV + expCsvFile := filepath.Join(expCatalogDir, testProjectName, csvVersion, csvFileName) + csvExpBytes, err := ioutil.ReadFile(expCsvFile) + if err != nil { + t.Fatalf("Failed to read expected CSV file: %v", err) + } + csvExp := string(csvExpBytes) + + // Read generated CSV from OutputDir/olm-catalog + outputCatalogDir := filepath.Join(cfg.OutputDir, OLMCatalogChildDir) + csvOutputBytes, err := ioutil.ReadFile(filepath.Join(outputCatalogDir, testProjectName, csvVersion, csvFileName)) + if err != nil { + t.Fatalf("Failed to read output CSV file: %v", err) + } + csvOutput := string(csvOutputBytes) + + assert.Equal(t, csvExp, csvOutput) } // TODO: This test is only updating the existing CSV @@ -126,12 +200,16 @@ func TestUpgradeFromExistingCSVWithInputsToOutput() { // present in testdata/go // Fix to generate a new CSV rather than only update an existing one func TestGoCSVFromNew(t *testing.T) { - for _, cleanupFunc := range setupTestEnvWithCleanup(t, testGoDataDir) { - defer cleanupFunc() - } + cleanupFunc := chDirWithCleanup(t, relativeProjectRootDir) + defer cleanupFunc() cfg := gen.Config{ OperatorName: testProjectName, + Inputs: map[string]string{ + DeployDirKey: filepath.Join(testGoDataDir, "deploy"), + APIsDirKey: filepath.Join(testGoDataDir, filepath.Join("pkg", "apis")), + }, + OutputDir: filepath.Join(testGoDataDir, "deploy"), } g := NewCSV(cfg, csvVersion, "") fileMap, err := g.(csvGenerator).generate() @@ -140,7 +218,8 @@ func TestGoCSVFromNew(t *testing.T) { } csvExpFile := getCSVFileName(testProjectName, csvVersion) - csvExpBytes, err := ioutil.ReadFile(filepath.Join(OLMCatalogDir, testProjectName, csvVersion, csvExpFile)) + csvExpBytes, err := ioutil.ReadFile(filepath.Join(testGoDataDir, + OLMCatalogDir, testProjectName, csvVersion, csvExpFile)) if err != nil { t.Fatalf("Failed to read expected CSV file: %v", err) } @@ -159,12 +238,16 @@ func TestGoCSVFromNew(t *testing.T) { } func TestGoCSVFromOld(t *testing.T) { - for _, cleanupFunc := range setupTestEnvWithCleanup(t, testGoDataDir) { - defer cleanupFunc() - } + cleanupFunc := chDirWithCleanup(t, relativeProjectRootDir) + defer cleanupFunc() cfg := gen.Config{ OperatorName: testProjectName, + Inputs: map[string]string{ + DeployDirKey: filepath.Join(testGoDataDir, "deploy"), + APIsDirKey: filepath.Join(testGoDataDir, filepath.Join("pkg", "apis")), + }, + OutputDir: filepath.Join(testGoDataDir, "deploy"), } g := NewCSV(cfg, csvVersion, fromVersion) fileMap, err := g.(csvGenerator).generate() @@ -173,7 +256,8 @@ func TestGoCSVFromOld(t *testing.T) { } csvExpFile := getCSVFileName(testProjectName, csvVersion) - csvExpBytes, err := ioutil.ReadFile(filepath.Join(OLMCatalogDir, testProjectName, csvVersion, csvExpFile)) + csvExpBytes, err := ioutil.ReadFile(filepath.Join(testGoDataDir, + OLMCatalogDir, testProjectName, csvVersion, csvExpFile)) if err != nil { t.Fatalf("Failed to read expected CSV file: %v", err) } @@ -185,8 +269,40 @@ func TestGoCSVFromOld(t *testing.T) { } } -func TestGoCSVIncludeAll(t *testing.T) { - cfg := gen.Config{OperatorName: testProjectName} +func TestGoCSVWithInvalidManifestsDir(t *testing.T) { + cleanupFunc := chDirWithCleanup(t, relativeProjectRootDir) + defer cleanupFunc() + + cfg := gen.Config{ + OperatorName: testProjectName, + Inputs: map[string]string{ + DeployDirKey: filepath.Join(testGoDataDir, "notExist"), + APIsDirKey: filepath.Join(testGoDataDir, filepath.Join("pkg", "apis")), + }, + OutputDir: filepath.Join(testGoDataDir, "deploy"), + } + + g := NewCSV(cfg, notExistVersion, "") + _, err := g.(csvGenerator).generate() + if err == nil { + t.Fatalf("Failed to get error for running CSV generator"+ + "on non-existent manifests directory: %s", cfg.Inputs[DeployDirKey]) + } +} + +func TestGoCSVWithEmptyManifestsDir(t *testing.T) { + cleanupFunc := chDirWithCleanup(t, relativeProjectRootDir) + defer cleanupFunc() + + cfg := gen.Config{ + OperatorName: testProjectName, + Inputs: map[string]string{ + DeployDirKey: filepath.Join(testGoDataDir, "emptydir"), + APIsDirKey: filepath.Join(testGoDataDir, filepath.Join("pkg", "apis")), + }, + OutputDir: filepath.Join(testGoDataDir, "emptydir"), + } + g := NewCSV(cfg, notExistVersion, "") fileMap, err := g.(csvGenerator).generate() if err != nil { @@ -211,12 +327,22 @@ func TestGoCSVIncludeAll(t *testing.T) { } func TestUpdateVersion(t *testing.T) { + cleanupFunc := chDirWithCleanup(t, relativeProjectRootDir) + defer cleanupFunc() + csv, err := getCSVFromDir(filepath.Join(testGoDataDir, OLMCatalogDir, testProjectName, fromVersion)) if err != nil { t.Fatal("Failed to get new CSV") } - cfg := gen.Config{OperatorName: testProjectName} + cfg := gen.Config{ + OperatorName: testProjectName, + Inputs: map[string]string{ + DeployDirKey: filepath.Join(testGoDataDir, "deploy"), + APIsDirKey: filepath.Join(testGoDataDir, filepath.Join("pkg", "apis")), + }, + OutputDir: filepath.Join(testGoDataDir, "deploy"), + } g := NewCSV(cfg, csvVersion, fromVersion) if err := g.(csvGenerator).updateCSVVersions(csv); err != nil { t.Fatalf("Failed to update csv with version %s: (%v)", csvVersion, err) @@ -255,6 +381,9 @@ func TestUpdateVersion(t *testing.T) { } func TestSetAndCheckOLMNamespaces(t *testing.T) { + cleanupFunc := chDirWithCleanup(t, relativeProjectRootDir) + defer cleanupFunc() + depBytes, err := ioutil.ReadFile(filepath.Join(testGoDataDir, scaffold.DeployDir, "operator.yaml")) if err != nil { t.Fatalf("Failed to read Deployment bytes: %v", err) diff --git a/internal/generate/olm-catalog/csv_updaters.go b/internal/generate/olm-catalog/csv_updaters.go index 0a3d835dac2..9b7de7ba7be 100644 --- a/internal/generate/olm-catalog/csv_updaters.go +++ b/internal/generate/olm-catalog/csv_updaters.go @@ -201,8 +201,6 @@ var _ csvUpdater = crds{} // apply updates csv's "owned" CRDDescriptions. "required" CRDDescriptions are // left as-is, since they are user-defined values. -// apply will only make a new spec.customresourcedefinitions.owned element for -// a type if an annotation is present on that type's declaration. func (us crds) apply(csv *olmapiv1alpha1.ClusterServiceVersion) error { ownedDescs := []olmapiv1alpha1.CRDDescription{} descMap := map[registry.DefinitionKey]olmapiv1alpha1.CRDDescription{} diff --git a/internal/generate/olm-catalog/descriptor/descriptor.go b/internal/generate/olm-catalog/descriptor/descriptor.go index 5c0f03c7f2f..4fe2fdf0a45 100644 --- a/internal/generate/olm-catalog/descriptor/descriptor.go +++ b/internal/generate/olm-catalog/descriptor/descriptor.go @@ -51,22 +51,50 @@ func GetCRDDescriptionForGVK(apisDir string, gvk schema.GroupVersionKind) (olmap if strings.Contains(group, ".") { group = strings.Split(group, ".")[0] } - // TODO(hasbro17): Lookup for API directory should not be configured to - // // - // Kubebuilder's layout is different. - apiDir := filepath.Join(apisDir, group, gvk.Version) - universe, err := getTypesFromDir(apiDir) - if err != nil { - if os.IsNotExist(err) { - return olmapiv1alpha1.CRDDescription{}, ErrAPIDirNotExist - } - return olmapiv1alpha1.CRDDescription{}, err + + // Check if apisDir exists + if exists, _ := isDirExist(apisDir); !exists { + return olmapiv1alpha1.CRDDescription{}, ErrAPIDirNotExist } - apiPkg := path.Join(projutil.GetGoPkg(), filepath.ToSlash(apiDir)) - pkgTypes, err := getTypesForPkg(apiPkg, universe) + + // Check if the kind pkg is at the expected layout + // multi-group layout: // + // single-group layout: / + expectedPkgPath, err := getExpectedPkgLayout(apisDir, group, gvk.Version) if err != nil { return olmapiv1alpha1.CRDDescription{}, err } + + // Get pkg types for the given GVK + var pkgTypes []*types.Type + if expectedPkgPath != "" { + // Look for the pkg types at the expected single or multi group import path + universe, err := getTypesFromDirRecursive(expectedPkgPath) + if err != nil { + return olmapiv1alpha1.CRDDescription{}, err + } + pkgTypes, err = getTypesForPkgPath(expectedPkgPath, universe) + if err != nil { + return olmapiv1alpha1.CRDDescription{}, err + } + } else { + // Unknown apis directory layout: /.../ + // Look in recursively for expected pkg name + + // TODO: gengo.parse.AddDirRecursive() will (sometimes?) fail if the + // root apisDir has no .go files. + // Workaround for this is to have a doc.go file in the package. + // Move away from using gengo in the future if possible. + universe, err := getTypesFromDirRecursive(apisDir) + if err != nil { + return olmapiv1alpha1.CRDDescription{}, err + } + pkgTypes, err = getTypesForPkgName(gvk.Version, universe) + if err != nil { + return olmapiv1alpha1.CRDDescription{}, err + } + } + kindType := findKindType(gvk.Kind, pkgTypes) if kindType == nil { return olmapiv1alpha1.CRDDescription{}, ErrAPITypeNotFound @@ -122,16 +150,57 @@ func GetCRDDescriptionForGVK(apisDir string, gvk schema.GroupVersionKind) (olmap return crdDesc, nil } -// getTypesFromDir gets all Go types from dir. -func getTypesFromDir(dir string) (types.Universe, error) { +func isDirExist(path string) (bool, error) { + fileInfo, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + return fileInfo.IsDir(), nil +} + +// getExpectedPkgLayout checks the directory layout in apisDir +// for single and multi group layouts and returns the expected pkg path +// for the group and version. +// Returns empty string if neither single or multi group layout is detected +// multi group path: // +// single group path: / +func getExpectedPkgLayout(apisDir, group, version string) (expectedPkgPath string, err error) { + groupVersionDir := filepath.Join(apisDir, group, version) + if isMultiGroupLayout, err := isDirExist(groupVersionDir); isMultiGroupLayout { + if err != nil { + return "", err + } + return groupVersionDir, nil + } + versionDir := filepath.Join(apisDir, version) + if isSingleGroupLayout, err := isDirExist(versionDir); isSingleGroupLayout { + if err != nil { + return "", err + } + return versionDir, nil + } + // Neither multi nor single group layout + return "", nil +} + +// getTypesFromDir gets all Go types from dir and recursively its sub directories. +// dir must be the project relative path to the pkg directory +func getTypesFromDirRecursive(dir string) (types.Universe, error) { if _, err := os.Stat(dir); err != nil { return nil, err } - if !filepath.IsAbs(dir) && !strings.HasPrefix(dir, ".") { - dir = fmt.Sprintf(".%s%s", string(filepath.Separator), dir) - } + + // Gengo's AddDirRecursive fails to load subdir pkgs if the root dir + // isn't the full pkg import path. + dirImportPath := path.Join(projutil.GetGoPkg(), filepath.ToSlash(dir)) p := parser.New() - if err := p.AddDirRecursive(dir); err != nil { + // TODO(hasbro17): AddDirRecursive can be noisy with klog warnings + // when it skips directories with no .go files. + // Silence those warnings unless in debug mode. + if err := p.AddDirRecursive(dirImportPath); err != nil { return nil, err } universe, err := p.FindTypes() @@ -141,10 +210,13 @@ func getTypesFromDir(dir string) (types.Universe, error) { return universe, nil } -func getTypesForPkg(pkgPath string, universe types.Universe) (pkgTypes []*types.Type, err error) { +// getTypesForPkgPath find the pkg with the given path in universe +// Note that pkg path must be relative to the project root directory. +func getTypesForPkgPath(projectRelativePkgPath string, universe types.Universe) (pkgTypes []*types.Type, err error) { + pkgPath := path.Join(projutil.GetGoPkg(), projectRelativePkgPath) var pkg *types.Package for _, upkg := range universe { - if strings.HasPrefix(upkg.Path, pkgPath) || strings.HasPrefix(upkg.Path, "."+string(filepath.Separator)) { + if upkg.Path == pkgPath { pkg = upkg break } @@ -158,6 +230,23 @@ func getTypesForPkg(pkgPath string, universe types.Universe) (pkgTypes []*types. return pkgTypes, nil } +func getTypesForPkgName(pkgName string, universe types.Universe) (pkgTypes []*types.Type, err error) { + var pkg *types.Package + for _, upkg := range universe { + if upkg.Name == pkgName { + pkg = upkg + break + } + } + if pkg == nil { + return nil, fmt.Errorf("no package found for %s", pkgName) + } + for _, t := range pkg.Types { + pkgTypes = append(pkgTypes, t) + } + return pkgTypes, nil +} + func findKindType(kind string, pkgTypes []*types.Type) *types.Type { for _, t := range pkgTypes { if t.Name.Name == kind { diff --git a/internal/generate/olm-catalog/descriptor/descriptor_test.go b/internal/generate/olm-catalog/descriptor/descriptor_test.go index 5b82a0cb356..bea00d46d1f 100644 --- a/internal/generate/olm-catalog/descriptor/descriptor_test.go +++ b/internal/generate/olm-catalog/descriptor/descriptor_test.go @@ -18,42 +18,95 @@ import ( "os" "path/filepath" "reflect" + "strings" "testing" "github.com/operator-framework/operator-sdk/internal/util/diffutil" - "github.com/operator-framework/operator-sdk/internal/util/fileutil" "github.com/ghodss/yaml" olmapiv1alpha1 "github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/gengo/types" ) var ( - testDataDir = filepath.Join("..", "..", "testdata") - testGoDataDir = filepath.Join(testDataDir, "go") + relativeProjectRootDir = filepath.Join("..", "..", "..", "..") + testDataRelativePathFromRoot = filepath.Join("internal", "generate", "testdata", "go") ) func TestGetKindTypeForAPI(t *testing.T) { - cases := []struct { + multiAPIRootDir := filepath.Join("pkg", "apis") + singleAPIRootDir := filepath.Join("api") + group := "cache" + version := "v1alpha1" + + subTests := []struct { description string - kind string - numPkgTypes int - wantNil bool + // True if apis dir in the expected single or multi group layout + isExpectedLayout bool + // path to apis types root dir e.g pkg/apis + apisDir string + // path to kind api pkg e.g pkg/apis/cache/v1alpha1 + expectedPkgPath string + group string + version string + kind string + numPkgTypes int + wantNil bool }{ { - "Find types successfully", - testFrameworkPackage, "Dummy", 22, false, + "Must Succeed: Find types for Kind from multi APIs root directory", + true, + multiAPIRootDir, + filepath.Join(multiAPIRootDir, group, version), + group, + version, + "Dummy", + 22, + false, + }, + { + "Must Fail: Find types for non-existing Kind from multi APIs root directory", + true, + multiAPIRootDir, + filepath.Join(multiAPIRootDir, group, version), + group, + version, + "NotFound", + 22, + true, + }, + { + "Must Succeed: Find types for Kind from single APIs root directory", + true, + singleAPIRootDir, + filepath.Join(singleAPIRootDir, version), + group, + version, + "Memcached", + 4, + false, }, { - "Find types with error from wrong kind", - testFrameworkPackage, "NotFound", 22, true, + "Must Fail: Find types for non-existing Kind from single APIs root directory", + true, + singleAPIRootDir, + filepath.Join(singleAPIRootDir, version), + group, + version, + "NotFound", + 4, + true, }, + // TODO: Add cases for non-standard api dir layouts: pkg/apis///version } + + // Change directory to project root so the test cases can form the correct pkg imports wd, err := os.Getwd() if err != nil { t.Fatal(err) } - if err := os.Chdir(testGoDataDir); err != nil { + if err := os.Chdir(relativeProjectRootDir); err != nil { t.Fatal(err) } defer func() { @@ -61,39 +114,66 @@ func TestGetKindTypeForAPI(t *testing.T) { t.Fatal(err) } }() - testAPIDir := filepath.Join("pkg", "apis", "cache", "v1alpha1") - universe, err := getTypesFromDir(testAPIDir) - if err != nil { - t.Fatal(err) - } - for _, c := range cases { - pkgTypes, err := getTypesForPkg(fileutil.DotPath(testAPIDir), universe) - if err != nil { - t.Fatal(err) - } - if n := len(pkgTypes); n != c.numPkgTypes { - t.Errorf("%s: expected %d package types, got %d", c.description, c.numPkgTypes, n) - } - kindType := findKindType(c.kind, pkgTypes) - if c.wantNil && kindType != nil { - t.Errorf("%s: expected type %q to not be found", c.description, kindType.Name) - } - if !c.wantNil && kindType == nil { - t.Errorf("%s: expected type %q to be found", c.description, c.kind) - } - if !c.wantNil && kindType != nil && kindType.Name.Name != c.kind { - t.Errorf("%s: expected type %q to have type name %q", c.description, kindType.Name, c.kind) - } + for _, st := range subTests { + t.Run(st.description, func(t *testing.T) { + testAPIRootDir := filepath.Join(testDataRelativePathFromRoot, st.apisDir) + expectedPkgPath, err := getExpectedPkgLayout(testAPIRootDir, st.group, st.version) + if err != nil { + t.Fatalf("Failed to getExpectedPkgLayout(%s, %s, %s): %v", testAPIRootDir, st.group, st.version, err) + } + if st.isExpectedLayout { + if expectedPkgPath == "" || !strings.HasSuffix(expectedPkgPath, st.expectedPkgPath) { + t.Fatalf("Expected (%s) as suffix to expected pkg path (%s)", st.expectedPkgPath, expectedPkgPath) + } + } + + var pkgTypes []*types.Type + if st.isExpectedLayout { + universe, err := getTypesFromDirRecursive(expectedPkgPath) + if err != nil { + t.Fatalf("Failed to get universe of types from API root directory (%s): %v)", st.apisDir, err) + } + pkgTypes, err = getTypesForPkgPath(expectedPkgPath, universe) + if err != nil { + t.Fatalf("Failed to get types of pkg path (%s) from API root directory(%s): %v)", + expectedPkgPath, testAPIRootDir, err) + } + } else { + universe, err := getTypesFromDirRecursive(testAPIRootDir) + if err != nil { + t.Fatalf("Failed to get universe of types from API root directory (%s): %v)", st.apisDir, err) + } + pkgTypes, err = getTypesForPkgName(st.version, universe) + if err != nil { + t.Fatalf("Failed to get types of pkg name (%s) from API root directory(%s): %v)", st.version, testAPIRootDir, err) + } + } + + if n := len(pkgTypes); n != st.numPkgTypes { + t.Errorf("expected %d package types, got %d", st.numPkgTypes, n) + } + kindType := findKindType(st.kind, pkgTypes) + if st.wantNil && kindType != nil { + t.Errorf("expected type %q to not be found", kindType.Name) + } + if !st.wantNil && kindType == nil { + t.Errorf("expected type %q to be found", st.kind) + } + if !st.wantNil && kindType != nil && kindType.Name.Name != st.kind { + t.Errorf("expected type %q to have type name %q", kindType.Name, st.kind) + } + }) } } func TestGetCRDDescriptionForGVK(t *testing.T) { + wd, err := os.Getwd() if err != nil { t.Fatal(err) } - if err := os.Chdir(testGoDataDir); err != nil { + if err := os.Chdir(relativeProjectRootDir); err != nil { t.Fatal(err) } defer func() { @@ -108,6 +188,7 @@ func TestGetCRDDescriptionForGVK(t *testing.T) { return xdescs } + // TODO(hasbro17): Change to run as subtests cases := []struct { description string apisDir string @@ -118,7 +199,7 @@ func TestGetCRDDescriptionForGVK(t *testing.T) { }{ { "Populate CRDDescription successfully", - filepath.Join("pkg", "apis"), + filepath.Join(testDataRelativePathFromRoot, "pkg", "apis"), schema.GroupVersionKind{Group: "cache.example.com", Version: "v1alpha1", Kind: "Dummy"}, olmapiv1alpha1.CRDDescription{ Kind: "Dummy", @@ -157,7 +238,7 @@ func TestGetCRDDescriptionForGVK(t *testing.T) { }, { "Populate CRDDescription with non-standard spec type successfully", - filepath.Join("pkg", "apis"), + filepath.Join(testDataRelativePathFromRoot, "pkg", "apis"), schema.GroupVersionKind{Group: "cache.example.com", Version: "v1alpha1", Kind: "OtherDummy"}, olmapiv1alpha1.CRDDescription{ Kind: "OtherDummy", @@ -185,7 +266,7 @@ func TestGetCRDDescriptionForGVK(t *testing.T) { }, { "Fail to populate CRDDescription with skip on dir not exist", - filepath.Join("pkg", "notexist"), + filepath.Join(testDataRelativePathFromRoot, "pkg", "notexist"), schema.GroupVersionKind{Group: "cache.example.com", Version: "v1alpha1", Kind: "Dummy"}, olmapiv1alpha1.CRDDescription{}, true, @@ -193,7 +274,7 @@ func TestGetCRDDescriptionForGVK(t *testing.T) { }, { "Fail to populate CRDDescription with skip on type", - filepath.Join("pkg", "apis"), + filepath.Join(testDataRelativePathFromRoot, "pkg", "apis"), schema.GroupVersionKind{Group: "cache.example.com", Version: "v1alpha1", Kind: "NoKind"}, olmapiv1alpha1.CRDDescription{}, true, diff --git a/internal/generate/olm-catalog/package_manifest_test.go b/internal/generate/olm-catalog/package_manifest_test.go index cea138907b3..e6562947e62 100644 --- a/internal/generate/olm-catalog/package_manifest_test.go +++ b/internal/generate/olm-catalog/package_manifest_test.go @@ -28,29 +28,9 @@ import ( "github.com/stretchr/testify/assert" ) -// newTestPackageManifestGenerator returns a package manifest Generator populated with test values. -func newTestPackageManifestGenerator() gen.Generator { - cfg := gen.Config{ - OperatorName: testProjectName, - OutputDir: testGoDataDir, - } - return NewPackageManifest(cfg, csvVersion, "stable", true) - - // TODO: The deploy dir input should be the way to point the generator to test data - // E.g: Inputs: map[string]string{DeployDirKey: testGoDataDir} - - // Override the generator to point to the package manifest in testGoDataDir - // testPkgManifestDir := filepath.Join(testGoDataDir, OLMCatalogDir, testProjectName) - // pkgGen := g.(pkgGenerator) - // pkgGen.existingPkgManifestDir = testPkgManifestDir - // return pkgGen -} - func TestGeneratePkgManifestToOutput(t *testing.T) { - nonStandardTestDataDir := filepath.Join(testDataDir, "non-standard-layout") - for _, cleanupFunc := range setupTestEnvWithCleanup(t, nonStandardTestDataDir) { - defer cleanupFunc() - } + cleanupFunc := chDirWithCleanup(t, relativeProjectRootDir) + defer cleanupFunc() // Temporary output dir for generating catalog bundle outputDir, err := ioutil.TempDir("", t.Name()+"-output-catalog") @@ -78,7 +58,7 @@ func TestGeneratePkgManifestToOutput(t *testing.T) { pkgManFileName := getPkgFileName(testProjectName) // Read expected Package Manifest - expCatalogDir := filepath.Join("expected-catalog", OLMCatalogChildDir) + expCatalogDir := filepath.Join(testNonStandardLayoutDataDir, "expected-catalog", OLMCatalogChildDir) pkgManExpBytes, err := ioutil.ReadFile(filepath.Join(expCatalogDir, testProjectName, pkgManFileName)) if err != nil { t.Fatalf("Failed to read expected package manifest file: %v", err) @@ -97,12 +77,15 @@ func TestGeneratePkgManifestToOutput(t *testing.T) { } -func TestUpdatePkgManifestToOutput(t *testing.T) { - // TODO -} - func TestGeneratePackageManifest(t *testing.T) { - g := newTestPackageManifestGenerator() + cleanupFunc := chDirWithCleanup(t, relativeProjectRootDir) + defer cleanupFunc() + + cfg := gen.Config{ + OperatorName: testProjectName, + OutputDir: filepath.Join(testGoDataDir, "deploy"), + } + g := NewPackageManifest(cfg, csvVersion, "stable", true) fileMap, err := g.(pkgGenerator).generate() if err != nil { t.Fatalf("Failed to execute package manifest generator: %v", err) @@ -116,7 +99,14 @@ func TestGeneratePackageManifest(t *testing.T) { } func TestValidatePackageManifest(t *testing.T) { - g := newTestPackageManifestGenerator() + cleanupFunc := chDirWithCleanup(t, relativeProjectRootDir) + defer cleanupFunc() + + cfg := gen.Config{ + OperatorName: testProjectName, + OutputDir: filepath.Join(testGoDataDir, "deploy"), + } + g := NewPackageManifest(cfg, csvVersion, "stable", true) // pkg is a basic, valid package manifest. pkg, err := g.(pkgGenerator).buildPackageManifest() diff --git a/internal/generate/testdata/non-standard-layout/api/cache/v1alpha1/memcachedrs_types.go b/internal/generate/testdata/go/api/v1alpha1/memcached_types.go similarity index 64% rename from internal/generate/testdata/non-standard-layout/api/cache/v1alpha1/memcachedrs_types.go rename to internal/generate/testdata/go/api/v1alpha1/memcached_types.go index ec8dde33bfc..985292f393e 100644 --- a/internal/generate/testdata/non-standard-layout/api/cache/v1alpha1/memcachedrs_types.go +++ b/internal/generate/testdata/go/api/v1alpha1/memcached_types.go @@ -18,38 +18,40 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// MemcachedRSSpec defines the desired state of MemcachedRS -type MemcachedRSSpec struct { +// MemcachedSpec defines the desired state of Memcached +type MemcachedSpec struct { + // Size is the size of the memcached deployment // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true - NumNodes int32 `json:"numNodes"` + Size int32 `json:"size"` } -// MemcachedRSStatus defines the observed state of MemcachedRS -type MemcachedRSStatus struct { +// MemcachedStatus defines the observed state of Memcached +type MemcachedStatus struct { + // Nodes are the names of the memcached pods // +operator-sdk:gen-csv:customresourcedefinitions.statusDescriptors=true - NodeList []string `json:"nodeList"` + Nodes []string `json:"nodes"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -// MemcachedRS is the Schema for the memcachedrs API +// Memcached is the Schema for the memcacheds API // +kubebuilder:subresource:status -// +kubebuilder:resource:path=memcachedrs,scope=Namespaced +// +kubebuilder:resource:path=memcacheds,scope=Namespaced // +kubebuilder:storageversion -// +operator-sdk:gen-csv:customresourcedefinitions.displayName="MemcachedRS App" -type MemcachedRS struct { +// +operator-sdk:gen-csv:customresourcedefinitions.displayName="Memcached App" +type Memcached struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` - Spec MemcachedRSSpec `json:"spec,omitempty"` - Status MemcachedRSStatus `json:"status,omitempty"` + Spec MemcachedSpec `json:"spec,omitempty"` + Status MemcachedStatus `json:"status,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -// MemcachedRSList contains a list of MemcachedRS -type MemcachedRSList struct { +// MemcachedList contains a list of Memcached +type MemcachedList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` - Items []MemcachedRS `json:"items"` + Items []Memcached `json:"items"` } diff --git a/internal/generate/testdata/non-standard-layout/api/cache/v1alpha1/dummy_types.go b/internal/generate/testdata/non-standard-layout/api/cache/v1alpha1/dummy_types.go deleted file mode 100644 index 6499dade39a..00000000000 --- a/internal/generate/testdata/non-standard-layout/api/cache/v1alpha1/dummy_types.go +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright 2020 The Operator-SDK Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package v1alpha1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// +k8s:deepcopy-gen=false -// +k8s:openapi-gen=false -type NoKindSpec struct { - // Not included in anything, no kind type - // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true - // +operator-sdk:gen-csv:customresourcedefinitions.statusDescriptors=true - Size int32 `json:"size"` - // Not included in anything, no kind type - Boss Hog `json:"hog"` -} - -// +k8s:deepcopy-gen=false -// +k8s:openapi-gen=false -type NoKindStatus struct { - // Not included in anything, no kind type - // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true - // +operator-sdk:gen-csv:customresourcedefinitions.statusDescriptors=true - Nodes []string `json:"nodes"` -} - -// +k8s:deepcopy-gen=false -// +k8s:openapi-gen=false -type DummySpec struct { - // Should be in spec - // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true - // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="dummy-pods" - // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:podCount" - Size int32 `json:"size"` -} - -// +k8s:deepcopy-gen=false -// +k8s:openapi-gen=false -type DummyStatus struct { - // Should be in status but not spec, since DummyStatus isn't in DummySpec - // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true - // +operator-sdk:gen-csv:customresourcedefinitions.statusDescriptors=true - Nodes []string `json:"nodes"` - // Not included in status but children should be - Boss Hog `json:"hog"` -} - -// +k8s:deepcopy-gen=false -// +k8s:openapi-gen=false -type Hog struct { - // Should be in status but not spec, since Hog isn't in DummySpec - // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true - // +operator-sdk:gen-csv:customresourcedefinitions.statusDescriptors=true - // +operator-sdk:gen-csv:customresourcedefinitions.statusDescriptors.displayName="boss-hog-engine" - Engine Engine `json:"engine"` - // Not in spec or status, no boolean annotation - // +operator-sdk:gen-csv:customresourcedefinitions.statusDescriptors.displayName="doesnt-matter" - Brand string `json:"brand"` - // Not in spec or status - Helmet string `json:"helmet"` - // Fields should be inlined - // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true - // +operator-sdk:gen-csv:customresourcedefinitions.statusDescriptors=true - Inlined InlinedComponent `json:",inline"` - // Fields should be inlined - InlinedComponent `json:",inline"` - // Should be ignored - // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true - // +operator-sdk:gen-csv:customresourcedefinitions.statusDescriptors=true - Ignored IgnoredComponent `json:"-"` - // Should be ignored, but exported children should not be - notExported `json:",inline"` -} - -type notExported struct { - // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true - // +operator-sdk:gen-csv:customresourcedefinitions.statusDescriptors=true - Public string `json:"foo"` - // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true - // +operator-sdk:gen-csv:customresourcedefinitions.statusDescriptors=true - private string `json:"-"` -} - -// +k8s:deepcopy-gen=false -// +k8s:openapi-gen=false -type Engine struct { - // Should not be included, no annotations. - Pistons []string `json:"pistons"` -} - -// +k8s:deepcopy-gen=false -// +k8s:openapi-gen=false -type InlinedComponent struct { - // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true - // +operator-sdk:gen-csv:customresourcedefinitions.statusDescriptors=true - SeatMaterial string `json:"seatMaterial"` -} - -// +k8s:deepcopy-gen=false -// +k8s:openapi-gen=false -type IgnoredComponent struct { - // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true - // +operator-sdk:gen-csv:customresourcedefinitions.statusDescriptors=true - TrunkSpace string `json:"trunkSpace"` -} - -// +k8s:deepcopy-gen=false -// +k8s:openapi-gen=false -type OtherDummyStatus struct { - // Should be in status but not spec, since this isn't a spec type - // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true - // +operator-sdk:gen-csv:customresourcedefinitions.statusDescriptors=true - Nothing string `json:"nothing"` -} - -// Dummy is the Schema for the dummy API -// +k8s:deepcopy-gen=false -// +k8s:openapi-gen=false -// +kubebuilder:subresource:status -// +kubebuilder:resource:path=dummys,scope=Namespaced -// +operator-sdk:gen-csv:customresourcedefinitions.displayName="Dummy App" -// +operator-sdk:gen-csv:customresourcedefinitions.resources="Deployment,v1,\"dummy-deployment\"" -// +operator-sdk:gen-csv:customresourcedefinitions.resources="ReplicaSet,v1beta2,\"dummy-replicaset\"" -// +operator-sdk:gen-csv:customresourcedefinitions.resources="Pod,v1,\"dummy-pod\"" -type Dummy struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec DummySpec `json:"spec,omitempty"` - Status DummyStatus `json:"status,omitempty"` -} - -// OtherDummy is the Schema for the other dummy API -// +k8s:deepcopy-gen=false -// +k8s:openapi-gen=false -// +operator-sdk:gen-csv:customresourcedefinitions.displayName="Other Dummy App" -// +operator-sdk:gen-csv:customresourcedefinitions.resources="Service,v1,\"other-dummy-service\"" -// +operator-sdk:gen-csv:customresourcedefinitions.resources="Pod,v1,\"other-dummy-pod\"" -type OtherDummy struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec Hog `json:"spec,omitempty"` - Status OtherDummyStatus `json:"status,omitempty"` -} - -// DummyList contains a list of Dummy -// +k8s:deepcopy-gen=false -// +k8s:openapi-gen=false -type DummyList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []Dummy `json:"items"` -} diff --git a/internal/generate/testdata/non-standard-layout/api/cache/v1alpha1/memcached_types.go b/internal/generate/testdata/non-standard-layout/api/cache/v1alpha1/memcached_types.go index 985292f393e..156d2bb86b5 100644 --- a/internal/generate/testdata/non-standard-layout/api/cache/v1alpha1/memcached_types.go +++ b/internal/generate/testdata/non-standard-layout/api/cache/v1alpha1/memcached_types.go @@ -38,7 +38,7 @@ type MemcachedStatus struct { // +kubebuilder:subresource:status // +kubebuilder:resource:path=memcacheds,scope=Namespaced // +kubebuilder:storageversion -// +operator-sdk:gen-csv:customresourcedefinitions.displayName="Memcached App" +// +operator-sdk:gen-csv:customresourcedefinitions.displayName="Memcached App Display Name" type Memcached struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/internal/generate/testdata/non-standard-layout/config/crds/cache.example.com_memcachedrs_crd.yaml b/internal/generate/testdata/non-standard-layout/config/crds/cache.example.com_memcachedrs_crd.yaml deleted file mode 100644 index 9ffb3fab46d..00000000000 --- a/internal/generate/testdata/non-standard-layout/config/crds/cache.example.com_memcachedrs_crd.yaml +++ /dev/null @@ -1,58 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: memcachedrs.cache.example.com -spec: - group: cache.example.com - names: - kind: MemcachedRS - listKind: MemcachedRSList - plural: memcachedrs - singular: memcachedrs - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - description: MemcachedRS is the Schema for the memcachedrs API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: MemcachedRSSpec defines the desired state of MemcachedRS - properties: - numNodes: - format: int32 - type: integer - required: - - numNodes - type: object - status: - description: MemcachedRSStatus defines the observed state of MemcachedRS - properties: - nodeList: - items: - type: string - type: array - test: - type: boolean - required: - - nodeList - - test - type: object - type: object - version: v1alpha1 - versions: - - name: v1alpha1 - served: true - storage: true diff --git a/internal/generate/testdata/non-standard-layout/config/crds/cache.example.com_v1alpha1_memcachedrs_cr.yaml b/internal/generate/testdata/non-standard-layout/config/crds/cache.example.com_v1alpha1_memcachedrs_cr.yaml deleted file mode 100644 index 92861ed5fe5..00000000000 --- a/internal/generate/testdata/non-standard-layout/config/crds/cache.example.com_v1alpha1_memcachedrs_cr.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: cache.example.com/v1alpha1 -kind: MemcachedRS -metadata: - name: example-memcachedrs -spec: - numNodes: 4 diff --git a/internal/generate/testdata/non-standard-layout/expected-catalog/olm-catalog/memcached-operator/0.0.1/memcached-operator.v0.0.1.clusterserviceversion.yaml b/internal/generate/testdata/non-standard-layout/expected-catalog/olm-catalog/memcached-operator/0.0.1/memcached-operator.v0.0.1.clusterserviceversion.yaml new file mode 100644 index 00000000000..65825e9192f --- /dev/null +++ b/internal/generate/testdata/non-standard-layout/expected-catalog/olm-catalog/memcached-operator/0.0.1/memcached-operator.v0.0.1.clusterserviceversion.yaml @@ -0,0 +1,153 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: ClusterServiceVersion +metadata: + annotations: + alm-examples: |- + [ + { + "apiVersion": "cache.example.com/v1alpha1", + "kind": "Memcached", + "metadata": { + "name": "example-memcached" + }, + "spec": { + "size": 3 + } + } + ] + capabilities: Basic Install + name: memcached-operator.v0.0.1 + namespace: placeholder +spec: + apiservicedefinitions: {} + customresourcedefinitions: + owned: + - description: Memcached is the Schema for the memcacheds API + displayName: Memcached App Display Name + kind: Memcached + name: memcacheds.cache.example.com + specDescriptors: + - description: Size is the size of the memcached deployment + displayName: Size + path: size + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:podCount + statusDescriptors: + - description: Nodes are the names of the memcached pods + displayName: Nodes + path: nodes + version: v1alpha1 + displayName: Memcached Operator + icon: + - base64data: "" + mediatype: "" + install: + spec: + deployments: + - name: memcached-operator + spec: + replicas: 1 + selector: + matchLabels: + name: memcached-operator + strategy: {} + template: + metadata: + labels: + name: memcached-operator + spec: + containers: + - command: + - memcached-operator + env: + - name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.annotations['olm.targetNamespaces'] + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPERATOR_NAME + value: memcached-operator + image: quay.io/example/memcached-operator:v0.0.3 + imagePullPolicy: Never + name: memcached-operator + resources: {} + serviceAccountName: memcached-operator + permissions: + - rules: + - apiGroups: + - "" + resources: + - pods + - services + - services/finalizers + - endpoints + - persistentvolumeclaims + - events + - configmaps + - secrets + verbs: + - '*' + - apiGroups: + - apps + resources: + - deployments + - daemonsets + - replicasets + - statefulsets + verbs: + - '*' + - apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + verbs: + - get + - create + - apiGroups: + - apps + resourceNames: + - memcached-operator + resources: + - deployments/finalizers + verbs: + - update + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - apiGroups: + - apps + resources: + - replicasets + - deployments + verbs: + - get + - apiGroups: + - cache.example.com + resources: + - '*' + verbs: + - '*' + serviceAccountName: memcached-operator + strategy: deployment + installModes: + - supported: true + type: OwnNamespace + - supported: true + type: SingleNamespace + - supported: false + type: MultiNamespace + - supported: true + type: AllNamespaces + keywords: + - "" + maintainers: + - {} + maturity: alpha + provider: {} + version: 0.0.1 diff --git a/internal/generate/testdata/non-standard-layout/expected-catalog/olm-catalog/memcached-operator/0.0.3/memcached-operator.v0.0.3.clusterserviceversion.yaml b/internal/generate/testdata/non-standard-layout/expected-catalog/olm-catalog/memcached-operator/0.0.3/memcached-operator.v0.0.3.clusterserviceversion.yaml index c68ff8d8ad1..3e3d7f26f8f 100644 --- a/internal/generate/testdata/non-standard-layout/expected-catalog/olm-catalog/memcached-operator/0.0.3/memcached-operator.v0.0.3.clusterserviceversion.yaml +++ b/internal/generate/testdata/non-standard-layout/expected-catalog/olm-catalog/memcached-operator/0.0.3/memcached-operator.v0.0.3.clusterserviceversion.yaml @@ -13,16 +13,6 @@ metadata: "spec": { "size": 3 } - }, - { - "apiVersion": "cache.example.com/v1alpha1", - "kind": "MemcachedRS", - "metadata": { - "name": "example-memcachedrs" - }, - "spec": { - "numNodes": 4 - } } ] capabilities: Basic Install @@ -32,11 +22,20 @@ spec: apiservicedefinitions: {} customresourcedefinitions: owned: - - kind: MemcachedRS - name: memcachedrs.cache.example.com - version: v1alpha1 - - kind: Memcached + - description: Memcached is the Schema for the memcacheds API + displayName: Memcached App Display Name + kind: Memcached name: memcacheds.cache.example.com + specDescriptors: + - description: Size is the size of the memcached deployment + displayName: Size + path: size + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:podCount + statusDescriptors: + - description: Nodes are the names of the memcached pods + displayName: Nodes + path: nodes version: v1alpha1 displayName: Memcached Operator icon: @@ -146,7 +145,8 @@ spec: - supported: true type: AllNamespaces keywords: - - "" + - "FooBar" + - "These keywords must be preserved in the CSV update tests from 0.0.3 to 0.0.4" maintainers: - {} maturity: alpha diff --git a/internal/generate/testdata/non-standard-layout/expected-catalog/olm-catalog/memcached-operator/0.0.4/memcached-operator.v0.0.4.clusterserviceversion.yaml b/internal/generate/testdata/non-standard-layout/expected-catalog/olm-catalog/memcached-operator/0.0.4/memcached-operator.v0.0.4.clusterserviceversion.yaml new file mode 100644 index 00000000000..d40de55874c --- /dev/null +++ b/internal/generate/testdata/non-standard-layout/expected-catalog/olm-catalog/memcached-operator/0.0.4/memcached-operator.v0.0.4.clusterserviceversion.yaml @@ -0,0 +1,155 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: ClusterServiceVersion +metadata: + annotations: + alm-examples: |- + [ + { + "apiVersion": "cache.example.com/v1alpha1", + "kind": "Memcached", + "metadata": { + "name": "example-memcached" + }, + "spec": { + "size": 3 + } + } + ] + capabilities: Basic Install + name: memcached-operator.v0.0.4 + namespace: placeholder +spec: + apiservicedefinitions: {} + customresourcedefinitions: + owned: + - description: Memcached is the Schema for the memcacheds API + displayName: Memcached App Display Name + kind: Memcached + name: memcacheds.cache.example.com + specDescriptors: + - description: Size is the size of the memcached deployment + displayName: Size + path: size + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:podCount + statusDescriptors: + - description: Nodes are the names of the memcached pods + displayName: Nodes + path: nodes + version: v1alpha1 + displayName: Memcached Operator + icon: + - base64data: "" + mediatype: "" + install: + spec: + deployments: + - name: memcached-operator + spec: + replicas: 1 + selector: + matchLabels: + name: memcached-operator + strategy: {} + template: + metadata: + labels: + name: memcached-operator + spec: + containers: + - command: + - memcached-operator + env: + - name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.annotations['olm.targetNamespaces'] + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPERATOR_NAME + value: memcached-operator + image: quay.io/example/memcached-operator:v0.0.3 + imagePullPolicy: Never + name: memcached-operator + resources: {} + serviceAccountName: memcached-operator + permissions: + - rules: + - apiGroups: + - "" + resources: + - pods + - services + - services/finalizers + - endpoints + - persistentvolumeclaims + - events + - configmaps + - secrets + verbs: + - '*' + - apiGroups: + - apps + resources: + - deployments + - daemonsets + - replicasets + - statefulsets + verbs: + - '*' + - apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + verbs: + - get + - create + - apiGroups: + - apps + resourceNames: + - memcached-operator + resources: + - deployments/finalizers + verbs: + - update + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - apiGroups: + - apps + resources: + - replicasets + - deployments + verbs: + - get + - apiGroups: + - cache.example.com + resources: + - '*' + verbs: + - '*' + serviceAccountName: memcached-operator + strategy: deployment + installModes: + - supported: true + type: OwnNamespace + - supported: true + type: SingleNamespace + - supported: false + type: MultiNamespace + - supported: true + type: AllNamespaces + keywords: + - FooBar + - These keywords must be preserved in the CSV update tests from 0.0.3 to 0.0.4 + maintainers: + - {} + maturity: alpha + provider: {} + replaces: memcached-operator.v0.0.3 + version: 0.0.4 diff --git a/internal/util/k8sutil/crd.go b/internal/util/k8sutil/crd.go index bb5daec70f6..feb8af9719a 100644 --- a/internal/util/k8sutil/crd.go +++ b/internal/util/k8sutil/crd.go @@ -30,7 +30,7 @@ import ( // GetCRDs parses all CRD manifests in the directory crdsDir and all of its subdirectories. func GetCRDs(crdsDir string) ([]*apiextv1beta1.CustomResourceDefinition, error) { - manifests, err := GetCRDManifestPaths(crdsDir) + manifests, err := GetCRDManifestPaths(crdsDir, "") if err != nil { return nil, fmt.Errorf("failed to get CRD's from %s: %v", crdsDir, err) } @@ -50,32 +50,38 @@ func GetCRDs(crdsDir string) ([]*apiextv1beta1.CustomResourceDefinition, error) } // GetCRDManifestPaths returns all CRD manifest paths in crdsDir and subdirs. -func GetCRDManifestPaths(crdsDir string) (crdPaths []string, err error) { +// The directory ignoreSubDir will be ignored +func GetCRDManifestPaths(crdsDir, ignoreSubDir string) (crdPaths []string, err error) { err = filepath.Walk(crdsDir, func(path string, info os.FileInfo, werr error) error { if werr != nil { return werr } - if info == nil { + + // Only read manifest from files, not directories + if info.IsDir() { + // Skip walking ignored directory + if ignoreSubDir != "" && info.Name() == ignoreSubDir { + return filepath.SkipDir + } return nil } - if !info.IsDir() { - b, err := ioutil.ReadFile(path) - if err != nil { - return fmt.Errorf("error reading manifest %s: %v", path, err) - } - // Skip files in crdsDir that aren't k8s manifests since we do not know - // what other files are in crdsDir. - typeMeta, err := GetTypeMetaFromBytes(b) - if err != nil { - log.Debugf("Skipping non-manifest file %s: %v", path, err) - return nil - } - if typeMeta.Kind != "CustomResourceDefinition" { - log.Debugf("Skipping non CRD manifest %s", path) - return nil - } - crdPaths = append(crdPaths, path) + + b, err := ioutil.ReadFile(path) + if err != nil { + return fmt.Errorf("error reading manifest %s: %v", path, err) + } + // Skip files in crdsDir that aren't k8s manifests since we do not know + // what other files are in crdsDir. + typeMeta, err := GetTypeMetaFromBytes(b) + if err != nil { + log.Debugf("Skipping non-manifest file %s: %v", path, err) + return nil + } + if typeMeta.Kind != "CustomResourceDefinition" { + log.Debugf("Skipping non CRD manifest %s", path) + return nil } + crdPaths = append(crdPaths, path) return nil }) return crdPaths, err diff --git a/internal/util/projutil/project_util.go b/internal/util/projutil/project_util.go index 3b47d0dbc94..5e5c07533d5 100644 --- a/internal/util/projutil/project_util.go +++ b/internal/util/projutil/project_util.go @@ -113,9 +113,19 @@ func getHomeDir() (string, error) { return homedir.Expand(hd) } +// TODO(hasbro17): If this function is called in the subdir of +// a module project it will fail to parse go.mod and return +// the correct import path. +// This needs to be fixed to return the pkg import path for any subdir +// in order for `generate csv` to correctly form pkg imports +// for API pkg paths that are not relative to the root dir. +// This might not be fixable since there is no good way to +// get the project root from inside the subdir of a module project. +// // GetGoPkg returns the current directory's import path by parsing it from // wd if this project's repository path is rooted under $GOPATH/src, or // from go.mod the project uses Go modules to manage dependencies. +// If the project has a go.mod then wd must be the project root. // // Example: "github.com/example-inc/app-operator" func GetGoPkg() string { @@ -176,10 +186,19 @@ func GetOperatorType() OperatorType { return OperatorTypeAnsible case IsOperatorHelm(): return OperatorTypeHelm + // Additional case for when tests run cmds which have this check + // from the SDK root + case hasGoPkgFile(): + return OperatorTypeGo } return OperatorTypeUnknown } +func hasGoPkgFile() bool { + _, err := os.Stat("go.mod") + return err == nil +} + func IsOperatorGo() bool { _, err := os.Stat(mainFile) return err == nil From 6df6024dc18967e1d0cd23723669a0bb7ed75044 Mon Sep 17 00:00:00 2001 From: hasbro17 Date: Sun, 15 Mar 2020 17:56:00 -0700 Subject: [PATCH 08/16] Don't skip CRD description updates for non-Go project type silently --- internal/generate/olm-catalog/csv.go | 3 +- internal/generate/olm-catalog/csv_updaters.go | 59 ++++++++++--------- 2 files changed, 33 insertions(+), 29 deletions(-) diff --git a/internal/generate/olm-catalog/csv.go b/internal/generate/olm-catalog/csv.go index 8afc6c03db0..f94fbf2fec3 100644 --- a/internal/generate/olm-catalog/csv.go +++ b/internal/generate/olm-catalog/csv.go @@ -28,7 +28,6 @@ import ( "github.com/operator-framework/operator-sdk/internal/scaffold" "github.com/operator-framework/operator-sdk/internal/util/fileutil" "github.com/operator-framework/operator-sdk/internal/util/k8sutil" - "github.com/operator-framework/operator-sdk/internal/util/projutil" "github.com/operator-framework/operator-sdk/internal/util/yamlutil" "github.com/blang/semver" @@ -448,7 +447,7 @@ func (g csvGenerator) updateCSVFromManifests(csv *olmapiv1alpha1.ClusterServiceV return err } } - err = updateDescriptions(csv, g.Inputs[APIsDirKey], projutil.GetOperatorType()) + err = updateDescriptions(csv, g.Inputs[APIsDirKey]) if err != nil { return fmt.Errorf("error updating CSV customresourcedefinitions: %w", err) } diff --git a/internal/generate/olm-catalog/csv_updaters.go b/internal/generate/olm-catalog/csv_updaters.go index 9b7de7ba7be..3627fad7d52 100644 --- a/internal/generate/olm-catalog/csv_updaters.go +++ b/internal/generate/olm-catalog/csv_updaters.go @@ -240,42 +240,47 @@ func (us crds) apply(csv *olmapiv1alpha1.ClusterServiceVersion) error { return nil } -func updateDescriptions(csv *olmapiv1alpha1.ClusterServiceVersion, - searchDir string, - opType projutil.OperatorType) error { - ownedCRDs := []olmapiv1alpha1.CRDDescription{} - for _, ownedCRD := range csv.Spec.CustomResourceDefinitions.Owned { - name := ownedCRD.Name - group := name - if split := strings.Split(name, "."); len(split) > 1 { +func updateDescriptions(csv *olmapiv1alpha1.ClusterServiceVersion, searchDir string) error { + // TODO: Should this skip generating descriptions if API directory exists + // but project type can't be detected due to layout differences. + if opType := projutil.GetOperatorType(); opType != projutil.OperatorTypeGo { + log.Infof("Skipping generation of owned CRDDescriptions from CSV annotations in %s."+ + " Could not detect project to be of Go type.", searchDir) + return nil + } + + updatedDescriptions := []olmapiv1alpha1.CRDDescription{} + for _, currDescription := range csv.Spec.CustomResourceDefinitions.Owned { + group := currDescription.Name + if split := strings.Split(currDescription.Name, "."); len(split) > 1 { group = strings.Join(split[1:], ".") } // Parse CRD descriptors from source code comments and annotations. gvk := schema.GroupVersionKind{ Group: group, - Version: ownedCRD.Version, - Kind: ownedCRD.Kind, + Version: currDescription.Version, + Kind: currDescription.Kind, } - var err error - switch opType { - case projutil.OperatorTypeGo: - ownedCRD, err = descriptor.GetCRDDescriptionForGVK(searchDir, gvk) - if err != nil { - if goerrors.Is(err, descriptor.ErrAPIDirNotExist) { - log.Infof("Directory for API %s does not exist. Skipping CSV annotation parsing for API.", gvk) - } else if goerrors.Is(err, descriptor.ErrAPITypeNotFound) { - log.Infof("No kind type found for API %s. Skipping CSV annotation parsing for API.", gvk) - } else { - return fmt.Errorf("failed to set CRD descriptors for %s: %v", gvk, err) - } - continue + newDescription, err := descriptor.GetCRDDescriptionForGVK(searchDir, gvk) + if err != nil { + if goerrors.Is(err, descriptor.ErrAPIDirNotExist) { + log.Infof("Directory for API %s does not exist. Skipping CSV annotation parsing for API.", gvk) + } else if goerrors.Is(err, descriptor.ErrAPITypeNotFound) { + log.Infof("No kind type found for API %s. Skipping CSV annotation parsing for API.", gvk) + } else { + // TODO: Should we ignore all CSV annotation parsing errors and simply log the error + // like we do for the above cases. + return fmt.Errorf("failed to set CRD descriptors for %s: %v", gvk, err) } - // Only set the name if no error was returned. - ownedCRD.Name = name + // Keep the existing description and don't update on error + updatedDescriptions = append(updatedDescriptions, currDescription) + } else { + // Replace the existing description with the newly parsed one + newDescription.Name = currDescription.Name + updatedDescriptions = append(updatedDescriptions, newDescription) } - ownedCRDs = append(ownedCRDs, ownedCRD) } - csv.Spec.CustomResourceDefinitions.Owned = ownedCRDs + csv.Spec.CustomResourceDefinitions.Owned = updatedDescriptions sort.Sort(descSorter(csv.Spec.CustomResourceDefinitions.Owned)) sort.Sort(descSorter(csv.Spec.CustomResourceDefinitions.Required)) return nil From 52fa426e4e248f13f3bab37f690125fc74005069 Mon Sep 17 00:00:00 2001 From: hasbro17 Date: Sun, 15 Mar 2020 18:01:33 -0700 Subject: [PATCH 09/16] Use dot relative pkg paths to parse API pkgs --- internal/generate/olm-catalog/csv_go_test.go | 79 ++++++++----------- .../olm-catalog/descriptor/descriptor.go | 27 ++++--- .../olm-catalog/descriptor/descriptor_test.go | 42 +++++----- .../generate/olm-catalog/package_manifest.go | 13 +-- .../olm-catalog/package_manifest_test.go | 12 +-- .../generate/testdata/go/build/Dockerfile | 0 internal/generate/testdata/go/go.mod | 1 + .../testdata/non-standard-layout/go.mod | 1 + 8 files changed, 81 insertions(+), 94 deletions(-) delete mode 100644 internal/generate/testdata/go/build/Dockerfile create mode 100644 internal/generate/testdata/go/go.mod create mode 100644 internal/generate/testdata/non-standard-layout/go.mod diff --git a/internal/generate/olm-catalog/csv_go_test.go b/internal/generate/olm-catalog/csv_go_test.go index 32b57c7e9d6..489b2d88225 100644 --- a/internal/generate/olm-catalog/csv_go_test.go +++ b/internal/generate/olm-catalog/csv_go_test.go @@ -44,15 +44,8 @@ const ( ) var ( - // Used to change directory to project root so the CSV generator's descriptor updates can form - // the correct pkg import paths to the API types directory - // See limitation with projutil.GetGoPkg() - relativeProjectRootDir = filepath.Join("..", "..", "..") - // Used to form all relative paths from the SDK root dir to - // the API, CRD, and deploy test dirs - testDataRelativePathFromRoot = filepath.Join("internal", "generate", "testdata") - testGoDataDir = filepath.Join(testDataRelativePathFromRoot, "go") - testNonStandardLayoutDataDir = filepath.Join(testDataRelativePathFromRoot, "non-standard-layout") + testGoDataDir = filepath.Join("..", "testdata", "go") + testNonStandardLayoutDataDir = filepath.Join("..", "testdata", "non-standard-layout") ) func chDirWithCleanup(t *testing.T, dataDir string) func() { @@ -74,7 +67,7 @@ func chDirWithCleanup(t *testing.T, dataDir string) func() { // TODO: Change to table driven subtests to test out different Inputs/Output for the generator func TestGenerateNewCSVWithInputsToOutput(t *testing.T) { // Change directory to project root so the test cases can form the correct pkg imports - cleanupFunc := chDirWithCleanup(t, relativeProjectRootDir) + cleanupFunc := chDirWithCleanup(t, testNonStandardLayoutDataDir) defer cleanupFunc() // Temporary output dir for generating catalog bundle @@ -93,8 +86,8 @@ func TestGenerateNewCSVWithInputsToOutput(t *testing.T) { cfg := gen.Config{ OperatorName: testProjectName, Inputs: map[string]string{ - DeployDirKey: filepath.Join(testNonStandardLayoutDataDir, "config"), - APIsDirKey: filepath.Join(testNonStandardLayoutDataDir, "api"), + DeployDirKey: "config", + APIsDirKey: "api", }, OutputDir: outputDir, } @@ -108,7 +101,7 @@ func TestGenerateNewCSVWithInputsToOutput(t *testing.T) { csvFileName := getCSVFileName(testProjectName, csvVersion) // Read expected CSV - expCatalogDir := filepath.Join(testNonStandardLayoutDataDir, "expected-catalog", OLMCatalogChildDir) + expCatalogDir := filepath.Join("expected-catalog", OLMCatalogChildDir) csvExpBytes, err := ioutil.ReadFile(filepath.Join(expCatalogDir, testProjectName, csvVersion, csvFileName)) if err != nil { t.Fatalf("Failed to read expected CSV file: %v", err) @@ -128,7 +121,7 @@ func TestGenerateNewCSVWithInputsToOutput(t *testing.T) { func TestUpgradeFromExistingCSVWithInputsToOutput(t *testing.T) { // Change directory to project root so the test cases can form the correct pkg imports - cleanupFunc := chDirWithCleanup(t, relativeProjectRootDir) + cleanupFunc := chDirWithCleanup(t, testNonStandardLayoutDataDir) defer cleanupFunc() // Temporary output dir for generating catalog bundle @@ -147,8 +140,8 @@ func TestUpgradeFromExistingCSVWithInputsToOutput(t *testing.T) { cfg := gen.Config{ OperatorName: testProjectName, Inputs: map[string]string{ - DeployDirKey: filepath.Join(testNonStandardLayoutDataDir, "config"), - APIsDirKey: filepath.Join(testNonStandardLayoutDataDir, "api"), + DeployDirKey: "config", + APIsDirKey: "api", }, OutputDir: outputDir, } @@ -161,7 +154,7 @@ func TestUpgradeFromExistingCSVWithInputsToOutput(t *testing.T) { if err := os.MkdirAll(outputFromCSVDir, os.FileMode(fileutil.DefaultDirFileMode)); err != nil { t.Fatalf("Failed to create CSV bundle dir (%s) for fromVersion (%s): %v", outputFromCSVDir, fromVersion, err) } - expCatalogDir := filepath.Join(testNonStandardLayoutDataDir, "expected-catalog", OLMCatalogChildDir) + expCatalogDir := filepath.Join("expected-catalog", OLMCatalogChildDir) expFromCSVDir := filepath.Join(expCatalogDir, testProjectName, fromVersion) cmd := exec.Command("cp", "-r", expFromCSVDir, outputFromCSVDir) t.Logf("Copying expected fromVersion CSV manifest dir %#v", cmd.Args) @@ -200,16 +193,16 @@ func TestUpgradeFromExistingCSVWithInputsToOutput(t *testing.T) { // present in testdata/go // Fix to generate a new CSV rather than only update an existing one func TestGoCSVFromNew(t *testing.T) { - cleanupFunc := chDirWithCleanup(t, relativeProjectRootDir) + cleanupFunc := chDirWithCleanup(t, testGoDataDir) defer cleanupFunc() cfg := gen.Config{ OperatorName: testProjectName, Inputs: map[string]string{ - DeployDirKey: filepath.Join(testGoDataDir, "deploy"), - APIsDirKey: filepath.Join(testGoDataDir, filepath.Join("pkg", "apis")), + DeployDirKey: "deploy", + APIsDirKey: filepath.Join("pkg", "apis"), }, - OutputDir: filepath.Join(testGoDataDir, "deploy"), + OutputDir: "deploy", } g := NewCSV(cfg, csvVersion, "") fileMap, err := g.(csvGenerator).generate() @@ -218,8 +211,7 @@ func TestGoCSVFromNew(t *testing.T) { } csvExpFile := getCSVFileName(testProjectName, csvVersion) - csvExpBytes, err := ioutil.ReadFile(filepath.Join(testGoDataDir, - OLMCatalogDir, testProjectName, csvVersion, csvExpFile)) + csvExpBytes, err := ioutil.ReadFile(filepath.Join(OLMCatalogDir, testProjectName, csvVersion, csvExpFile)) if err != nil { t.Fatalf("Failed to read expected CSV file: %v", err) } @@ -238,16 +230,16 @@ func TestGoCSVFromNew(t *testing.T) { } func TestGoCSVFromOld(t *testing.T) { - cleanupFunc := chDirWithCleanup(t, relativeProjectRootDir) + cleanupFunc := chDirWithCleanup(t, testGoDataDir) defer cleanupFunc() cfg := gen.Config{ OperatorName: testProjectName, Inputs: map[string]string{ - DeployDirKey: filepath.Join(testGoDataDir, "deploy"), - APIsDirKey: filepath.Join(testGoDataDir, filepath.Join("pkg", "apis")), + DeployDirKey: "deploy", + APIsDirKey: filepath.Join("pkg", "apis"), }, - OutputDir: filepath.Join(testGoDataDir, "deploy"), + OutputDir: "deploy", } g := NewCSV(cfg, csvVersion, fromVersion) fileMap, err := g.(csvGenerator).generate() @@ -256,8 +248,7 @@ func TestGoCSVFromOld(t *testing.T) { } csvExpFile := getCSVFileName(testProjectName, csvVersion) - csvExpBytes, err := ioutil.ReadFile(filepath.Join(testGoDataDir, - OLMCatalogDir, testProjectName, csvVersion, csvExpFile)) + csvExpBytes, err := ioutil.ReadFile(filepath.Join(OLMCatalogDir, testProjectName, csvVersion, csvExpFile)) if err != nil { t.Fatalf("Failed to read expected CSV file: %v", err) } @@ -270,16 +261,16 @@ func TestGoCSVFromOld(t *testing.T) { } func TestGoCSVWithInvalidManifestsDir(t *testing.T) { - cleanupFunc := chDirWithCleanup(t, relativeProjectRootDir) + cleanupFunc := chDirWithCleanup(t, testGoDataDir) defer cleanupFunc() cfg := gen.Config{ OperatorName: testProjectName, Inputs: map[string]string{ - DeployDirKey: filepath.Join(testGoDataDir, "notExist"), - APIsDirKey: filepath.Join(testGoDataDir, filepath.Join("pkg", "apis")), + DeployDirKey: "notExist", + APIsDirKey: filepath.Join("pkg", "apis"), }, - OutputDir: filepath.Join(testGoDataDir, "deploy"), + OutputDir: "deploy", } g := NewCSV(cfg, notExistVersion, "") @@ -291,16 +282,16 @@ func TestGoCSVWithInvalidManifestsDir(t *testing.T) { } func TestGoCSVWithEmptyManifestsDir(t *testing.T) { - cleanupFunc := chDirWithCleanup(t, relativeProjectRootDir) + cleanupFunc := chDirWithCleanup(t, testGoDataDir) defer cleanupFunc() cfg := gen.Config{ OperatorName: testProjectName, Inputs: map[string]string{ - DeployDirKey: filepath.Join(testGoDataDir, "emptydir"), - APIsDirKey: filepath.Join(testGoDataDir, filepath.Join("pkg", "apis")), + DeployDirKey: "emptydir", + APIsDirKey: filepath.Join("pkg", "apis"), }, - OutputDir: filepath.Join(testGoDataDir, "emptydir"), + OutputDir: "emptydir", } g := NewCSV(cfg, notExistVersion, "") @@ -327,10 +318,10 @@ func TestGoCSVWithEmptyManifestsDir(t *testing.T) { } func TestUpdateVersion(t *testing.T) { - cleanupFunc := chDirWithCleanup(t, relativeProjectRootDir) + cleanupFunc := chDirWithCleanup(t, testGoDataDir) defer cleanupFunc() - csv, err := getCSVFromDir(filepath.Join(testGoDataDir, OLMCatalogDir, testProjectName, fromVersion)) + csv, err := getCSVFromDir(filepath.Join(OLMCatalogDir, testProjectName, fromVersion)) if err != nil { t.Fatal("Failed to get new CSV") } @@ -338,10 +329,10 @@ func TestUpdateVersion(t *testing.T) { cfg := gen.Config{ OperatorName: testProjectName, Inputs: map[string]string{ - DeployDirKey: filepath.Join(testGoDataDir, "deploy"), - APIsDirKey: filepath.Join(testGoDataDir, filepath.Join("pkg", "apis")), + DeployDirKey: "deploy", + APIsDirKey: filepath.Join("pkg", "apis"), }, - OutputDir: filepath.Join(testGoDataDir, "deploy"), + OutputDir: "deploy", } g := NewCSV(cfg, csvVersion, fromVersion) if err := g.(csvGenerator).updateCSVVersions(csv); err != nil { @@ -381,10 +372,10 @@ func TestUpdateVersion(t *testing.T) { } func TestSetAndCheckOLMNamespaces(t *testing.T) { - cleanupFunc := chDirWithCleanup(t, relativeProjectRootDir) + cleanupFunc := chDirWithCleanup(t, testGoDataDir) defer cleanupFunc() - depBytes, err := ioutil.ReadFile(filepath.Join(testGoDataDir, scaffold.DeployDir, "operator.yaml")) + depBytes, err := ioutil.ReadFile(filepath.Join(scaffold.DeployDir, "operator.yaml")) if err != nil { t.Fatalf("Failed to read Deployment bytes: %v", err) } diff --git a/internal/generate/olm-catalog/descriptor/descriptor.go b/internal/generate/olm-catalog/descriptor/descriptor.go index 4fe2fdf0a45..16cdbddd589 100644 --- a/internal/generate/olm-catalog/descriptor/descriptor.go +++ b/internal/generate/olm-catalog/descriptor/descriptor.go @@ -18,12 +18,9 @@ import ( "errors" "fmt" "os" - "path" "path/filepath" "strings" - "github.com/operator-framework/operator-sdk/internal/util/projutil" - olmapiv1alpha1 "github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/gengo/parser" @@ -53,7 +50,11 @@ func GetCRDDescriptionForGVK(apisDir string, gvk schema.GroupVersionKind) (olmap } // Check if apisDir exists - if exists, _ := isDirExist(apisDir); !exists { + exists, err := isDirExist(apisDir) + if err != nil { + return olmapiv1alpha1.CRDDescription{}, err + } + if !exists { return olmapiv1alpha1.CRDDescription{}, ErrAPIDirNotExist } @@ -192,15 +193,18 @@ func getTypesFromDirRecursive(dir string) (types.Universe, error) { if _, err := os.Stat(dir); err != nil { return nil, err } - - // Gengo's AddDirRecursive fails to load subdir pkgs if the root dir - // isn't the full pkg import path. - dirImportPath := path.Join(projutil.GetGoPkg(), filepath.ToSlash(dir)) p := parser.New() + // Gengo's AddDirRecursive fails to load subdir pkgs if the root dir + // isn't the full pkg import path, or begins with ./ + // Use path relative to current dir + // TODO: Turn abs path into ./... relative path as well + if !filepath.IsAbs(dir) && !strings.HasPrefix(dir, ".") { + dir = fmt.Sprintf(".%s%s", string(filepath.Separator), dir) + } // TODO(hasbro17): AddDirRecursive can be noisy with klog warnings // when it skips directories with no .go files. // Silence those warnings unless in debug mode. - if err := p.AddDirRecursive(dirImportPath); err != nil { + if err := p.AddDirRecursive(dir); err != nil { return nil, err } universe, err := p.FindTypes() @@ -212,11 +216,10 @@ func getTypesFromDirRecursive(dir string) (types.Universe, error) { // getTypesForPkgPath find the pkg with the given path in universe // Note that pkg path must be relative to the project root directory. -func getTypesForPkgPath(projectRelativePkgPath string, universe types.Universe) (pkgTypes []*types.Type, err error) { - pkgPath := path.Join(projutil.GetGoPkg(), projectRelativePkgPath) +func getTypesForPkgPath(pkgPath string, universe types.Universe) (pkgTypes []*types.Type, err error) { var pkg *types.Package for _, upkg := range universe { - if upkg.Path == pkgPath { + if strings.HasSuffix(upkg.Path, pkgPath) { pkg = upkg break } diff --git a/internal/generate/olm-catalog/descriptor/descriptor_test.go b/internal/generate/olm-catalog/descriptor/descriptor_test.go index bea00d46d1f..444abd7b380 100644 --- a/internal/generate/olm-catalog/descriptor/descriptor_test.go +++ b/internal/generate/olm-catalog/descriptor/descriptor_test.go @@ -30,20 +30,17 @@ import ( ) var ( - relativeProjectRootDir = filepath.Join("..", "..", "..", "..") - testDataRelativePathFromRoot = filepath.Join("internal", "generate", "testdata", "go") + testDataDir = filepath.Join("..", "..", "testdata", "go") ) func TestGetKindTypeForAPI(t *testing.T) { multiAPIRootDir := filepath.Join("pkg", "apis") - singleAPIRootDir := filepath.Join("api") + singleAPIRootDir := "api" group := "cache" version := "v1alpha1" subTests := []struct { description string - // True if apis dir in the expected single or multi group layout - isExpectedLayout bool // path to apis types root dir e.g pkg/apis apisDir string // path to kind api pkg e.g pkg/apis/cache/v1alpha1 @@ -53,10 +50,11 @@ func TestGetKindTypeForAPI(t *testing.T) { kind string numPkgTypes int wantNil bool + // True if apis dir in the expected single or multi group layout + isExpectedLayout bool }{ { "Must Succeed: Find types for Kind from multi APIs root directory", - true, multiAPIRootDir, filepath.Join(multiAPIRootDir, group, version), group, @@ -64,10 +62,10 @@ func TestGetKindTypeForAPI(t *testing.T) { "Dummy", 22, false, + true, }, { "Must Fail: Find types for non-existing Kind from multi APIs root directory", - true, multiAPIRootDir, filepath.Join(multiAPIRootDir, group, version), group, @@ -75,10 +73,10 @@ func TestGetKindTypeForAPI(t *testing.T) { "NotFound", 22, true, + true, }, { "Must Succeed: Find types for Kind from single APIs root directory", - true, singleAPIRootDir, filepath.Join(singleAPIRootDir, version), group, @@ -86,10 +84,10 @@ func TestGetKindTypeForAPI(t *testing.T) { "Memcached", 4, false, + true, }, { "Must Fail: Find types for non-existing Kind from single APIs root directory", - true, singleAPIRootDir, filepath.Join(singleAPIRootDir, version), group, @@ -97,16 +95,17 @@ func TestGetKindTypeForAPI(t *testing.T) { "NotFound", 4, true, + true, }, // TODO: Add cases for non-standard api dir layouts: pkg/apis///version } - // Change directory to project root so the test cases can form the correct pkg imports + // Change directory to test data dir so the test cases can form the correct pkg imports wd, err := os.Getwd() if err != nil { t.Fatal(err) } - if err := os.Chdir(relativeProjectRootDir); err != nil { + if err := os.Chdir(testDataDir); err != nil { t.Fatal(err) } defer func() { @@ -117,10 +116,9 @@ func TestGetKindTypeForAPI(t *testing.T) { for _, st := range subTests { t.Run(st.description, func(t *testing.T) { - testAPIRootDir := filepath.Join(testDataRelativePathFromRoot, st.apisDir) - expectedPkgPath, err := getExpectedPkgLayout(testAPIRootDir, st.group, st.version) + expectedPkgPath, err := getExpectedPkgLayout(st.apisDir, st.group, st.version) if err != nil { - t.Fatalf("Failed to getExpectedPkgLayout(%s, %s, %s): %v", testAPIRootDir, st.group, st.version, err) + t.Fatalf("Failed to getExpectedPkgLayout(%s, %s, %s): %v", st.apisDir, st.group, st.version, err) } if st.isExpectedLayout { if expectedPkgPath == "" || !strings.HasSuffix(expectedPkgPath, st.expectedPkgPath) { @@ -137,16 +135,16 @@ func TestGetKindTypeForAPI(t *testing.T) { pkgTypes, err = getTypesForPkgPath(expectedPkgPath, universe) if err != nil { t.Fatalf("Failed to get types of pkg path (%s) from API root directory(%s): %v)", - expectedPkgPath, testAPIRootDir, err) + expectedPkgPath, st.apisDir, err) } } else { - universe, err := getTypesFromDirRecursive(testAPIRootDir) + universe, err := getTypesFromDirRecursive(st.apisDir) if err != nil { t.Fatalf("Failed to get universe of types from API root directory (%s): %v)", st.apisDir, err) } pkgTypes, err = getTypesForPkgName(st.version, universe) if err != nil { - t.Fatalf("Failed to get types of pkg name (%s) from API root directory(%s): %v)", st.version, testAPIRootDir, err) + t.Fatalf("Failed to get types of pkg name (%s) from API root directory(%s): %v)", st.version, st.apisDir, err) } } @@ -173,7 +171,7 @@ func TestGetCRDDescriptionForGVK(t *testing.T) { if err != nil { t.Fatal(err) } - if err := os.Chdir(relativeProjectRootDir); err != nil { + if err := os.Chdir(testDataDir); err != nil { t.Fatal(err) } defer func() { @@ -199,7 +197,7 @@ func TestGetCRDDescriptionForGVK(t *testing.T) { }{ { "Populate CRDDescription successfully", - filepath.Join(testDataRelativePathFromRoot, "pkg", "apis"), + filepath.Join("pkg", "apis"), schema.GroupVersionKind{Group: "cache.example.com", Version: "v1alpha1", Kind: "Dummy"}, olmapiv1alpha1.CRDDescription{ Kind: "Dummy", @@ -238,7 +236,7 @@ func TestGetCRDDescriptionForGVK(t *testing.T) { }, { "Populate CRDDescription with non-standard spec type successfully", - filepath.Join(testDataRelativePathFromRoot, "pkg", "apis"), + filepath.Join("pkg", "apis"), schema.GroupVersionKind{Group: "cache.example.com", Version: "v1alpha1", Kind: "OtherDummy"}, olmapiv1alpha1.CRDDescription{ Kind: "OtherDummy", @@ -266,7 +264,7 @@ func TestGetCRDDescriptionForGVK(t *testing.T) { }, { "Fail to populate CRDDescription with skip on dir not exist", - filepath.Join(testDataRelativePathFromRoot, "pkg", "notexist"), + filepath.Join("pkg", "notexist"), schema.GroupVersionKind{Group: "cache.example.com", Version: "v1alpha1", Kind: "Dummy"}, olmapiv1alpha1.CRDDescription{}, true, @@ -274,7 +272,7 @@ func TestGetCRDDescriptionForGVK(t *testing.T) { }, { "Fail to populate CRDDescription with skip on type", - filepath.Join(testDataRelativePathFromRoot, "pkg", "apis"), + filepath.Join("pkg", "apis"), schema.GroupVersionKind{Group: "cache.example.com", Version: "v1alpha1", Kind: "NoKind"}, olmapiv1alpha1.CRDDescription{}, true, diff --git a/internal/generate/olm-catalog/package_manifest.go b/internal/generate/olm-catalog/package_manifest.go index 2d9d73b2724..82cca199dca 100644 --- a/internal/generate/olm-catalog/package_manifest.go +++ b/internal/generate/olm-catalog/package_manifest.go @@ -71,14 +71,6 @@ func NewPackageManifest(cfg gen.Config, csvVersion, channel string, isDefault bo if g.OutputDir == "" { g.OutputDir = scaffold.DeployDir } - olmCatalogDir := filepath.Join(g.OutputDir, OLMCatalogChildDir) - - // Check if the generator should update from an existing package manifest file - pkgManifestDirPath := filepath.Join(olmCatalogDir, g.OperatorName) - pkgManifestFilePath := filepath.Join(pkgManifestDirPath, g.fileName) - if isFileExist(pkgManifestFilePath) { - g.existingPkgManifestDir = pkgManifestDirPath - } return g } @@ -153,8 +145,9 @@ func (g pkgGenerator) generate() (map[string][]byte, error) { // an existing one if found at the expected path. func (g pkgGenerator) buildPackageManifest() (registry.PackageManifest, error) { pkg := registry.PackageManifest{} - if g.existingPkgManifestDir != "" { - existingPkgManifest := filepath.Join(g.existingPkgManifestDir, g.fileName) + olmCatalogDir := filepath.Join(g.OutputDir, OLMCatalogChildDir) + existingPkgManifest := filepath.Join(olmCatalogDir, g.OperatorName, g.fileName) + if isFileExist(existingPkgManifest) { b, err := ioutil.ReadFile(existingPkgManifest) if err != nil { return pkg, fmt.Errorf("failed to read package manifest %s: %v", existingPkgManifest, err) diff --git a/internal/generate/olm-catalog/package_manifest_test.go b/internal/generate/olm-catalog/package_manifest_test.go index e6562947e62..466b2df724e 100644 --- a/internal/generate/olm-catalog/package_manifest_test.go +++ b/internal/generate/olm-catalog/package_manifest_test.go @@ -29,7 +29,7 @@ import ( ) func TestGeneratePkgManifestToOutput(t *testing.T) { - cleanupFunc := chDirWithCleanup(t, relativeProjectRootDir) + cleanupFunc := chDirWithCleanup(t, testNonStandardLayoutDataDir) defer cleanupFunc() // Temporary output dir for generating catalog bundle @@ -58,7 +58,7 @@ func TestGeneratePkgManifestToOutput(t *testing.T) { pkgManFileName := getPkgFileName(testProjectName) // Read expected Package Manifest - expCatalogDir := filepath.Join(testNonStandardLayoutDataDir, "expected-catalog", OLMCatalogChildDir) + expCatalogDir := filepath.Join("expected-catalog", OLMCatalogChildDir) pkgManExpBytes, err := ioutil.ReadFile(filepath.Join(expCatalogDir, testProjectName, pkgManFileName)) if err != nil { t.Fatalf("Failed to read expected package manifest file: %v", err) @@ -78,12 +78,12 @@ func TestGeneratePkgManifestToOutput(t *testing.T) { } func TestGeneratePackageManifest(t *testing.T) { - cleanupFunc := chDirWithCleanup(t, relativeProjectRootDir) + cleanupFunc := chDirWithCleanup(t, testGoDataDir) defer cleanupFunc() cfg := gen.Config{ OperatorName: testProjectName, - OutputDir: filepath.Join(testGoDataDir, "deploy"), + OutputDir: "deploy", } g := NewPackageManifest(cfg, csvVersion, "stable", true) fileMap, err := g.(pkgGenerator).generate() @@ -99,12 +99,12 @@ func TestGeneratePackageManifest(t *testing.T) { } func TestValidatePackageManifest(t *testing.T) { - cleanupFunc := chDirWithCleanup(t, relativeProjectRootDir) + cleanupFunc := chDirWithCleanup(t, testGoDataDir) defer cleanupFunc() cfg := gen.Config{ OperatorName: testProjectName, - OutputDir: filepath.Join(testGoDataDir, "deploy"), + OutputDir: "deploy", } g := NewPackageManifest(cfg, csvVersion, "stable", true) diff --git a/internal/generate/testdata/go/build/Dockerfile b/internal/generate/testdata/go/build/Dockerfile deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/internal/generate/testdata/go/go.mod b/internal/generate/testdata/go/go.mod new file mode 100644 index 00000000000..6f55c957e4b --- /dev/null +++ b/internal/generate/testdata/go/go.mod @@ -0,0 +1 @@ +// Only here so "generate csv" can detect the testdata directory as a Go type project \ No newline at end of file diff --git a/internal/generate/testdata/non-standard-layout/go.mod b/internal/generate/testdata/non-standard-layout/go.mod new file mode 100644 index 00000000000..6f55c957e4b --- /dev/null +++ b/internal/generate/testdata/non-standard-layout/go.mod @@ -0,0 +1 @@ +// Only here so "generate csv" can detect the testdata directory as a Go type project \ No newline at end of file From a926ed494620b72faf52cdf626b7c17599074444 Mon Sep 17 00:00:00 2001 From: hasbro17 Date: Sun, 15 Mar 2020 18:22:33 -0700 Subject: [PATCH 10/16] fix error from rebase --- cmd/operator-sdk/generate/csv.go | 5 +---- doc/cli/operator-sdk_generate_csv.md | 3 ++- .../generate/olm-catalog/descriptor/descriptor_test.go | 8 ++++---- internal/generate/olm-catalog/package_manifest.go | 4 ---- 4 files changed, 7 insertions(+), 13 deletions(-) diff --git a/cmd/operator-sdk/generate/csv.go b/cmd/operator-sdk/generate/csv.go index 769add8a7a6..d297cb9f918 100644 --- a/cmd/operator-sdk/generate/csv.go +++ b/cmd/operator-sdk/generate/csv.go @@ -225,10 +225,7 @@ func findCRDFileSet(path string) (map[string][]byte, error) { for _, crdPath := range crdPaths { b, err := ioutil.ReadFile(crdPath) if err != nil { - return fmt.Errorf("error in %s : %v", p, err) - } - if typeMeta.Kind != "CustomResourceDefinition" { - continue + return nil, err } crdFileSet[filepath.Base(crdPath)] = b } diff --git a/doc/cli/operator-sdk_generate_csv.md b/doc/cli/operator-sdk_generate_csv.md index e3fb3a22778..1e807516df3 100644 --- a/doc/cli/operator-sdk_generate_csv.md +++ b/doc/cli/operator-sdk_generate_csv.md @@ -35,7 +35,8 @@ TODO Note: The CSV generator only uses this to copy the CRD manifests. The CSV contents for spec.customresourcedefinitions.owned will still be updated from the CRD manifests in the deploy directory specified by --deploy-dir. - (default "deploy") + If unset, it defaults to the same value as --deploy-dir. + --csv-channel string Channel the CSV should be registered under in the package manifest --csv-version string Semantic version of the CSV --default-channel Use the channel passed to --csv-channel as the package manifests' default channel. Only valid when --csv-channel is set diff --git a/internal/generate/olm-catalog/descriptor/descriptor_test.go b/internal/generate/olm-catalog/descriptor/descriptor_test.go index 444abd7b380..fc926cf378b 100644 --- a/internal/generate/olm-catalog/descriptor/descriptor_test.go +++ b/internal/generate/olm-catalog/descriptor/descriptor_test.go @@ -149,17 +149,17 @@ func TestGetKindTypeForAPI(t *testing.T) { } if n := len(pkgTypes); n != st.numPkgTypes { - t.Errorf("expected %d package types, got %d", st.numPkgTypes, n) + t.Errorf("Expected %d package types, got %d", st.numPkgTypes, n) } kindType := findKindType(st.kind, pkgTypes) if st.wantNil && kindType != nil { - t.Errorf("expected type %q to not be found", kindType.Name) + t.Errorf("Expected type %q to not be found", kindType.Name) } if !st.wantNil && kindType == nil { - t.Errorf("expected type %q to be found", st.kind) + t.Errorf("Expected type %q to be found", st.kind) } if !st.wantNil && kindType != nil && kindType.Name.Name != st.kind { - t.Errorf("expected type %q to have type name %q", kindType.Name, st.kind) + t.Errorf("Expected type %q to have type name %q", kindType.Name, st.kind) } }) } diff --git a/internal/generate/olm-catalog/package_manifest.go b/internal/generate/olm-catalog/package_manifest.go index 82cca199dca..bb65626954d 100644 --- a/internal/generate/olm-catalog/package_manifest.go +++ b/internal/generate/olm-catalog/package_manifest.go @@ -49,10 +49,6 @@ type pkgGenerator struct { channelIsDefault bool // PackageManifest file name fileName string - - // existingPkgManifestDir is set to the package manifest root directory - // if the generator needs to update from an existing package manifest file. - existingPkgManifestDir string } func NewPackageManifest(cfg gen.Config, csvVersion, channel string, isDefault bool) gen.Generator { From 4aa680b550c1a7707afbdb71a7ec8b860ce0e82e Mon Sep 17 00:00:00 2001 From: hasbro17 Date: Mon, 16 Mar 2020 17:57:30 -0700 Subject: [PATCH 11/16] Remove check to skip CSV annotation for Go types Instead only skip when API types directory is missing. --- internal/generate/olm-catalog/csv_updaters.go | 9 --------- internal/generate/olm-catalog/descriptor/descriptor.go | 2 ++ internal/generate/testdata/go/go.mod | 1 - internal/generate/testdata/non-standard-layout/go.mod | 1 - internal/util/projutil/project_util.go | 4 ---- 5 files changed, 2 insertions(+), 15 deletions(-) delete mode 100644 internal/generate/testdata/go/go.mod delete mode 100644 internal/generate/testdata/non-standard-layout/go.mod diff --git a/internal/generate/olm-catalog/csv_updaters.go b/internal/generate/olm-catalog/csv_updaters.go index 3627fad7d52..1f425360855 100644 --- a/internal/generate/olm-catalog/csv_updaters.go +++ b/internal/generate/olm-catalog/csv_updaters.go @@ -24,7 +24,6 @@ import ( "github.com/operator-framework/operator-registry/pkg/registry" "github.com/operator-framework/operator-sdk/internal/generate/olm-catalog/descriptor" - "github.com/operator-framework/operator-sdk/internal/util/projutil" "github.com/operator-framework/operator-sdk/pkg/k8sutil" "github.com/ghodss/yaml" @@ -241,14 +240,6 @@ func (us crds) apply(csv *olmapiv1alpha1.ClusterServiceVersion) error { } func updateDescriptions(csv *olmapiv1alpha1.ClusterServiceVersion, searchDir string) error { - // TODO: Should this skip generating descriptions if API directory exists - // but project type can't be detected due to layout differences. - if opType := projutil.GetOperatorType(); opType != projutil.OperatorTypeGo { - log.Infof("Skipping generation of owned CRDDescriptions from CSV annotations in %s."+ - " Could not detect project to be of Go type.", searchDir) - return nil - } - updatedDescriptions := []olmapiv1alpha1.CRDDescription{} for _, currDescription := range csv.Spec.CustomResourceDefinitions.Owned { group := currDescription.Name diff --git a/internal/generate/olm-catalog/descriptor/descriptor.go b/internal/generate/olm-catalog/descriptor/descriptor.go index 16cdbddd589..139778b34d2 100644 --- a/internal/generate/olm-catalog/descriptor/descriptor.go +++ b/internal/generate/olm-catalog/descriptor/descriptor.go @@ -22,6 +22,7 @@ import ( "strings" olmapiv1alpha1 "github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1" + log "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/gengo/parser" "k8s.io/gengo/types" @@ -55,6 +56,7 @@ func GetCRDDescriptionForGVK(apisDir string, gvk schema.GroupVersionKind) (olmap return olmapiv1alpha1.CRDDescription{}, err } if !exists { + log.Debugf("Could not find API types directory: %s", apisDir) return olmapiv1alpha1.CRDDescription{}, ErrAPIDirNotExist } diff --git a/internal/generate/testdata/go/go.mod b/internal/generate/testdata/go/go.mod deleted file mode 100644 index 6f55c957e4b..00000000000 --- a/internal/generate/testdata/go/go.mod +++ /dev/null @@ -1 +0,0 @@ -// Only here so "generate csv" can detect the testdata directory as a Go type project \ No newline at end of file diff --git a/internal/generate/testdata/non-standard-layout/go.mod b/internal/generate/testdata/non-standard-layout/go.mod deleted file mode 100644 index 6f55c957e4b..00000000000 --- a/internal/generate/testdata/non-standard-layout/go.mod +++ /dev/null @@ -1 +0,0 @@ -// Only here so "generate csv" can detect the testdata directory as a Go type project \ No newline at end of file diff --git a/internal/util/projutil/project_util.go b/internal/util/projutil/project_util.go index 5e5c07533d5..a56a0e74a1a 100644 --- a/internal/util/projutil/project_util.go +++ b/internal/util/projutil/project_util.go @@ -186,10 +186,6 @@ func GetOperatorType() OperatorType { return OperatorTypeAnsible case IsOperatorHelm(): return OperatorTypeHelm - // Additional case for when tests run cmds which have this check - // from the SDK root - case hasGoPkgFile(): - return OperatorTypeGo } return OperatorTypeUnknown } From a01de386ee9fb22db4269ab19397871caaa4f95a Mon Sep 17 00:00:00 2001 From: hasbro17 Date: Sun, 22 Mar 2020 22:29:06 -0700 Subject: [PATCH 12/16] Add example, long help text, and rename getTypesFromDirRecursive --- CHANGELOG.md | 3 + cmd/operator-sdk/generate/csv.go | 107 +++++++++++++++--- doc/cli/operator-sdk_generate_csv.md | 99 +++++++++++++--- .../olm-catalog/descriptor/descriptor.go | 9 +- .../olm-catalog/descriptor/descriptor_test.go | 4 +- internal/util/projutil/project_util.go | 7 +- 6 files changed, 184 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d33a7f61221..d0743c24dfd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ### Added +- Added the [`generate csv --deploy-dir --apis-dir --crd-dir`](doc/cli/operator-sdk_generate_csv.md#options) flags to allow configuring input locations for operator manifests and API types directories to the CSV generator in lieu of a config. See the CLI reference doc or `generate csv -h` help text for more details. ([#2511](https://github.com/operator-framework/operator-sdk/pull/2511)) +- Added the [`generate csv --output-dir`](doc/cli/operator-sdk_generate_csv.md#options) flag to allow configuring the output location for the catalog directory. ([#2511](https://github.com/operator-framework/operator-sdk/pull/2511)) - The flag `--watch-namespace` and `--operator-namespace` was added to `operator-sdk run --local`, `operator-sdk test --local` and `operator-sdk cleanup` commands in order to replace the flag `--namespace` which was deprecated.([#2617](https://github.com/operator-framework/operator-sdk/pull/2617)) - The methods `ctx.GetOperatorNamespace()` and `ctx.GetWatchNamespace()` was added `pkg/test` in order to replace `ctx.GetNamespace()` which is deprecated. ([#2617](https://github.com/operator-framework/operator-sdk/pull/2617)) - The `--crd-version` flag was added to the `new`, `add api`, `add crd`, and `generate crds` commands so that users can opt-in to `v1` CRDs. ([#2684](https://github.com/operator-framework/operator-sdk/pull/2684)) @@ -29,6 +31,7 @@ - **Breaking Change:** remove `pkg/restmapper` which was deprecated in `v0.14.0`. Projects that use this package must switch to the `DynamicRESTMapper` implementation in [controller-runtime](https://godoc.org/github.com/kubernetes-sigs/controller-runtime/pkg/client/apiutil#NewDynamicRESTMapper). ([#2544](https://github.com/operator-framework/operator-sdk/pull/2544)) - **Breaking Change:** remove deprecated `operator-sdk generate openapi` subcommand. ([#2740](https://github.com/operator-framework/operator-sdk/pull/2740)) +- **Breaking Change:** Removed CSV configuration file support (defaulting to deploy/olm-catalog/csv-config.yaml) in favor of specifying inputs to the generator via [`generate csv --deploy-dir --apis-dir --crd-dir`](doc/cli/operator-sdk_generate_csv.md#options), and configuring output locations via [`generate csv --output-dir`](doc/cli/operator-sdk_generate_csv.md#options). ([#2511](https://github.com/operator-framework/operator-sdk/pull/2511)) ### Bug Fixes diff --git a/cmd/operator-sdk/generate/csv.go b/cmd/operator-sdk/generate/csv.go index d297cb9f918..b705de36c20 100644 --- a/cmd/operator-sdk/generate/csv.go +++ b/cmd/operator-sdk/generate/csv.go @@ -55,10 +55,88 @@ for the operator. This file is used to publish the operator to the OLM Catalog. A CSV semantic version is supplied via the --csv-version flag. If your operator has already generated a CSV manifest you want to use as a base, supply its -version to --from-version. Otherwise the SDK will scaffold a new CSV manifest.`, +version to --from-version. Otherwise the SDK will scaffold a new CSV manifest. + +CSV input flags: + --deploy-dir: The CSV file contents will be generated from the operator manifests + present in this directory. + + --apis-dir: The CSV annotation comments will be parsed from the Go types under this path to + fill out metadata for owned APIs in spec.customresourcedefinitions.owned. + + --crd-dir: The CRD manifests are updated from this path to the CSV bundle directory. + Note: The CSV generator only uses this to copy the CRD manifests. + The CSV contents for spec.customresourcedefinitions.owned will still be generated + from the CRD manifests in the deploy directory specified by --deploy-dir. + If unset, it defaults to the same value as --deploy-dir. + +`, Example: ` -TODO - `, + ##### Generate CSV from default input paths ##### + $ tree pkg/apis/ deploy/ + pkg/apis/ + ├── ... + └── cache + ├── group.go + └── v1alpha1 + ├── ... + └── memcached_types.go + deploy/ + ├── crds + │   ├── cache.example.com_memcacheds_crd.yaml + │   └── cache.example.com_v1alpha1_memcached_cr.yaml + ├── operator.yaml + ├── role.yaml + ├── role_binding.yaml + └── service_account.yaml + + $ operator-sdk generate csv --csv-version=0.0.1 --update-crds + INFO[0000] Generating CSV manifest version 0.0.1 + ... + + $ tree deploy/ + deploy/ + ... + ├── olm-catalog + │   └── memcached-operator + │   ├── 0.0.1 + │   │   ├── cache.example.com_memcacheds_crd.yaml + │   │   └── memcached-operator.v0.0.1.clusterserviceversion.yaml + │   └── memcached-operator.package.yaml + ... + + + + ##### Generate CSV from custom input paths ##### + $ operator-sdk generate csv --csv-version=0.0.1 --update-crds \ + --deploy-dir=config --apis-dir=api --output-dir=production + INFO[0000] Generating CSV manifest version 0.0.1 + ... + + $ tree config/ api/ production/ + config/ + ├── crds + │   ├── cache.example.com_memcacheds_crd.yaml + │   └── cache.example.com_v1alpha1_memcached_cr.yaml + ├── operator.yaml + ├── role.yaml + ├── role_binding.yaml + └── service_account.yaml + api/ + ├── ... + └── cache + ├── group.go + └── v1alpha1 + ├── ... + └── memcached_types.go + production/ + └── olm-catalog + └── memcached-operator + ├── 0.0.1 + │   ├── cache.example.com_memcacheds_crd.yaml + │   └── memcached-operator.v0.0.1.clusterserviceversion.yaml + └── memcached-operator.package.yaml +`, RunE: func(cmd *cobra.Command, args []string) error { if len(args) != 0 { @@ -67,6 +145,12 @@ TODO if err := c.validate(); err != nil { return fmt.Errorf("error validating command flags: %v", err) } + + if err := projutil.CheckProjectRoot(); err != nil { + log.Warn("Could not detect project root. Ensure that this command " + + "runs from the project root directory.") + } + // Default for crd dir if unset if c.crdDir == "" { c.crdDir = c.deployDir @@ -87,22 +171,11 @@ TODO "Semantic version of an existing CSV to use as a base") cmd.Flags().StringVar(&c.deployDir, "deploy-dir", "deploy", - `Project relative path to root directory for operator manifests (Deployment, RBAC, CRDs). -The CSV file contents will be generated from the manifests present in this directory. -`) + `Project relative path to root directory for operator manifests (Deployment, RBAC, CRDs)`) cmd.Flags().StringVar(&c.apisDir, "apis-dir", filepath.Join("pkg", "apis"), - `Project relative path to root directory for API type defintions. -The CSV annotation comments will be parsed from the Go types under this path to -fill out metadata for owned APIs in spec.customresourcedefinitions.owned. -`) + `Project relative path to root directory for API type defintions`) cmd.Flags().StringVar(&c.crdDir, "crd-dir", "", - `Project relative path to root directory for for CRD manifests. -Used when --update-crds is set to copy over CRD manifests to the CSV bundle directory. -Note: The CSV generator only uses this to copy the CRD manifests. -The CSV contents for spec.customresourcedefinitions.owned will still be updated -from the CRD manifests in the deploy directory specified by --deploy-dir. -If unset, it defaults to the same value as --deploy-dir. -`) + `Project relative path to root directory for for CRD manifests`) cmd.Flags().StringVar(&c.outputDir, "output-dir", scaffold.DeployDir, "Base directory to output generated CSV. The resulting CSV bundle directory"+ diff --git a/doc/cli/operator-sdk_generate_csv.md b/doc/cli/operator-sdk_generate_csv.md index 1e807516df3..935cf6b046b 100644 --- a/doc/cli/operator-sdk_generate_csv.md +++ b/doc/cli/operator-sdk_generate_csv.md @@ -11,6 +11,21 @@ A CSV semantic version is supplied via the --csv-version flag. If your operator has already generated a CSV manifest you want to use as a base, supply its version to --from-version. Otherwise the SDK will scaffold a new CSV manifest. +CSV input flags: + --deploy-dir: The CSV file contents will be generated from the operator manifests + present in this directory. + + --apis-dir: The CSV annotation comments will be parsed from the Go types under this path to + fill out metadata for owned APIs in spec.customresourcedefinitions.owned. + + --crd-dir: The CRD manifests are updated from this path to the CSV bundle directory. + Note: The CSV generator only uses this to copy the CRD manifests. + The CSV contents for spec.customresourcedefinitions.owned will still be generated + from the CRD manifests in the deploy directory specified by --deploy-dir. + If unset, it defaults to the same value as --deploy-dir. + + + ``` operator-sdk generate csv [flags] ``` @@ -19,30 +34,82 @@ operator-sdk generate csv [flags] ``` -TODO - + ##### Generate CSV from default input paths ##### + $ tree pkg/apis/ deploy/ + pkg/apis/ + ├── ... + └── cache + ├── group.go + └── v1alpha1 + ├── ... + └── memcached_types.go + deploy/ + ├── crds + │   ├── cache.example.com_memcacheds_crd.yaml + │   └── cache.example.com_v1alpha1_memcached_cr.yaml + ├── operator.yaml + ├── role.yaml + ├── role_binding.yaml + └── service_account.yaml + + $ operator-sdk generate csv --csv-version=0.0.1 --update-crds + INFO[0000] Generating CSV manifest version 0.0.1 + ... + + $ tree deploy/ + deploy/ + ... + ├── olm-catalog + │   └── memcached-operator + │   ├── 0.0.1 + │   │   ├── cache.example.com_memcacheds_crd.yaml + │   │   └── memcached-operator.v0.0.1.clusterserviceversion.yaml + │   └── memcached-operator.package.yaml + ... + + + + ##### Generate CSV from custom input paths ##### + $ operator-sdk generate csv --csv-version=0.0.1 --update-crds \ + --deploy-dir=config --apis-dir=api --output-dir=production + INFO[0000] Generating CSV manifest version 0.0.1 + ... + + $ tree config/ api/ production/ + config/ + ├── crds + │   ├── cache.example.com_memcacheds_crd.yaml + │   └── cache.example.com_v1alpha1_memcached_cr.yaml + ├── operator.yaml + ├── role.yaml + ├── role_binding.yaml + └── service_account.yaml + api/ + ├── ... + └── cache + ├── group.go + └── v1alpha1 + ├── ... + └── memcached_types.go + production/ + └── olm-catalog + └── memcached-operator + ├── 0.0.1 + │   ├── cache.example.com_memcacheds_crd.yaml + │   └── memcached-operator.v0.0.1.clusterserviceversion.yaml + └── memcached-operator.package.yaml + ``` ### Options ``` - --apis-dir string Project relative path to root directory for API type defintions. - The CSV annotation comments will be parsed from the Go types under this path to - fill out metadata for owned APIs in spec.customresourcedefinitions.owned. - (default "pkg/apis") - --crd-dir string Project relative path to root directory for for CRD manifests. - Used when --update-crds is set to copy over CRD manifests to the CSV bundle directory. - Note: The CSV generator only uses this to copy the CRD manifests. - The CSV contents for spec.customresourcedefinitions.owned will still be updated - from the CRD manifests in the deploy directory specified by --deploy-dir. - If unset, it defaults to the same value as --deploy-dir. - + --apis-dir string Project relative path to root directory for API type defintions (default "pkg/apis") + --crd-dir string Project relative path to root directory for for CRD manifests --csv-channel string Channel the CSV should be registered under in the package manifest --csv-version string Semantic version of the CSV --default-channel Use the channel passed to --csv-channel as the package manifests' default channel. Only valid when --csv-channel is set - --deploy-dir string Project relative path to root directory for operator manifests (Deployment, RBAC, CRDs). - The CSV file contents will be generated from the manifests present in this directory. - (default "deploy") + --deploy-dir string Project relative path to root directory for operator manifests (Deployment, RBAC, CRDs) (default "deploy") --from-version string Semantic version of an existing CSV to use as a base -h, --help help for csv --operator-name string Operator name to use while generating CSV diff --git a/internal/generate/olm-catalog/descriptor/descriptor.go b/internal/generate/olm-catalog/descriptor/descriptor.go index 139778b34d2..4b2ea6e9a36 100644 --- a/internal/generate/olm-catalog/descriptor/descriptor.go +++ b/internal/generate/olm-catalog/descriptor/descriptor.go @@ -72,7 +72,7 @@ func GetCRDDescriptionForGVK(apisDir string, gvk schema.GroupVersionKind) (olmap var pkgTypes []*types.Type if expectedPkgPath != "" { // Look for the pkg types at the expected single or multi group import path - universe, err := getTypesFromDirRecursive(expectedPkgPath) + universe, err := getPkgsFromDirRecursive(expectedPkgPath) if err != nil { return olmapiv1alpha1.CRDDescription{}, err } @@ -88,7 +88,7 @@ func GetCRDDescriptionForGVK(apisDir string, gvk schema.GroupVersionKind) (olmap // root apisDir has no .go files. // Workaround for this is to have a doc.go file in the package. // Move away from using gengo in the future if possible. - universe, err := getTypesFromDirRecursive(apisDir) + universe, err := getPkgsFromDirRecursive(apisDir) if err != nil { return olmapiv1alpha1.CRDDescription{}, err } @@ -189,9 +189,9 @@ func getExpectedPkgLayout(apisDir, group, version string) (expectedPkgPath strin return "", nil } -// getTypesFromDir gets all Go types from dir and recursively its sub directories. +// getPkgsFromDirRecursive gets all Go types from dir and recursively its sub directories. // dir must be the project relative path to the pkg directory -func getTypesFromDirRecursive(dir string) (types.Universe, error) { +func getPkgsFromDirRecursive(dir string) (types.Universe, error) { if _, err := os.Stat(dir); err != nil { return nil, err } @@ -217,7 +217,6 @@ func getTypesFromDirRecursive(dir string) (types.Universe, error) { } // getTypesForPkgPath find the pkg with the given path in universe -// Note that pkg path must be relative to the project root directory. func getTypesForPkgPath(pkgPath string, universe types.Universe) (pkgTypes []*types.Type, err error) { var pkg *types.Package for _, upkg := range universe { diff --git a/internal/generate/olm-catalog/descriptor/descriptor_test.go b/internal/generate/olm-catalog/descriptor/descriptor_test.go index fc926cf378b..4f8c4633866 100644 --- a/internal/generate/olm-catalog/descriptor/descriptor_test.go +++ b/internal/generate/olm-catalog/descriptor/descriptor_test.go @@ -128,7 +128,7 @@ func TestGetKindTypeForAPI(t *testing.T) { var pkgTypes []*types.Type if st.isExpectedLayout { - universe, err := getTypesFromDirRecursive(expectedPkgPath) + universe, err := getPkgsFromDirRecursive(expectedPkgPath) if err != nil { t.Fatalf("Failed to get universe of types from API root directory (%s): %v)", st.apisDir, err) } @@ -138,7 +138,7 @@ func TestGetKindTypeForAPI(t *testing.T) { expectedPkgPath, st.apisDir, err) } } else { - universe, err := getTypesFromDirRecursive(st.apisDir) + universe, err := getPkgsFromDirRecursive(st.apisDir) if err != nil { t.Fatalf("Failed to get universe of types from API root directory (%s): %v)", st.apisDir, err) } diff --git a/internal/util/projutil/project_util.go b/internal/util/projutil/project_util.go index a56a0e74a1a..4ab7c609a12 100644 --- a/internal/util/projutil/project_util.go +++ b/internal/util/projutil/project_util.go @@ -77,6 +77,8 @@ func MustInProjectRoot() { // CheckProjectRoot checks if the current dir is the project root, and returns // an error if not. +// TODO(hasbro17): Change this to check for go.mod +// "build/Dockerfile" may not be present in all projects func CheckProjectRoot() error { // If the current directory has a "build/Dockerfile", then it is safe to say // we are at the project root. @@ -190,11 +192,6 @@ func GetOperatorType() OperatorType { return OperatorTypeUnknown } -func hasGoPkgFile() bool { - _, err := os.Stat("go.mod") - return err == nil -} - func IsOperatorGo() bool { _, err := os.Stat(mainFile) return err == nil From aaa247530bde42f92e91c61e2608b4df0a412d2d Mon Sep 17 00:00:00 2001 From: hasbro17 Date: Tue, 24 Mar 2020 11:59:53 -0700 Subject: [PATCH 13/16] doc updates --- cmd/operator-sdk/generate/csv.go | 4 +- doc/cli/operator-sdk_generate_csv.md | 2 +- doc/user/olm-catalog/generating-a-csv.md | 27 ++++-- .../en/docs/cli/operator-sdk_generate_csv.md | 91 ++++++++++++++++++- .../golang/olm-catalog/generating-a-csv.md | 45 ++++----- 5 files changed, 125 insertions(+), 44 deletions(-) diff --git a/cmd/operator-sdk/generate/csv.go b/cmd/operator-sdk/generate/csv.go index b705de36c20..bb2822cb3e7 100644 --- a/cmd/operator-sdk/generate/csv.go +++ b/cmd/operator-sdk/generate/csv.go @@ -178,8 +178,8 @@ CSV input flags: `Project relative path to root directory for for CRD manifests`) cmd.Flags().StringVar(&c.outputDir, "output-dir", scaffold.DeployDir, - "Base directory to output generated CSV. The resulting CSV bundle directory"+ - "will be \"/olm-catalog//\"") + "Base directory to output generated CSV. The resulting CSV bundle directory "+ + "will be \"/olm-catalog//\".") cmd.Flags().BoolVar(&c.updateCRDs, "update-crds", false, "Update CRD manifests in deploy/{operator-name}/{csv-version} the using latest API's") cmd.Flags().StringVar(&c.operatorName, "operator-name", "", diff --git a/doc/cli/operator-sdk_generate_csv.md b/doc/cli/operator-sdk_generate_csv.md index 935cf6b046b..e694b0f6662 100644 --- a/doc/cli/operator-sdk_generate_csv.md +++ b/doc/cli/operator-sdk_generate_csv.md @@ -113,7 +113,7 @@ operator-sdk generate csv [flags] --from-version string Semantic version of an existing CSV to use as a base -h, --help help for csv --operator-name string Operator name to use while generating CSV - --output-dir string Base directory to output generated CSV. The resulting CSV bundle directorywill be "/olm-catalog//" (default "deploy") + --output-dir string Base directory to output generated CSV. The resulting CSV bundle directory will be "/olm-catalog//". (default "deploy") --update-crds Update CRD manifests in deploy/{operator-name}/{csv-version} the using latest API's ``` diff --git a/doc/user/olm-catalog/generating-a-csv.md b/doc/user/olm-catalog/generating-a-csv.md index 4c42508cf74..3f6782a25a1 100644 --- a/doc/user/olm-catalog/generating-a-csv.md +++ b/doc/user/olm-catalog/generating-a-csv.md @@ -10,15 +10,23 @@ This document describes how to manage the following lifecycle for your Operator ## Configuration -Operator SDK projects have an expected [project layout][doc-project-layout]. In particular, a few manifests are expected to be present in the `deploy` directory: +### Inputs -* Roles: `role.yaml` -* ClusterRoles: `cluster_role.yaml` -* Deployments: `operator.yaml` -* Custom Resources (CR's): `crds/___cr.yaml` -* CustomResourceDefinitions (CRD's): `crds/__crd.yaml`. +The CSV generator requires certain inputs to construct a CSV manifest. -`generate csv` extracts manifests from files in `deploy/` by default that match the kinds above and adds them to the CSV. If your manifest files are not in `deploy/`, you can use the `--include=[list of paths]` option to instruct the command to extract manifests from files at those paths, ex. `--include="deploy/prod,deploy/test"`. Setting `--include` overrides default behavior; if you still want default behavior, you must append `deploy/` to the list of paths passed to `--include`. +1. Path to the operator manifests root directory. By default `generate csv` extracts manifests from files in `deploy/` for the following kinds and adds them to the CSV. Use the `--deploy-dir` flag to change this path. + * Roles: `role.yaml` + * ClusterRoles: `cluster_role.yaml` + * Deployments: `operator.yaml` + * Custom Resources (CR's): `crds/___cr.yaml` + * CustomResourceDefinitions (CRD's): `crds/__crd.yaml` +2. Path to API types root directory. The CSV generator also parses the [CSV annotations][csv-annotations] from the API type definitions to populate certain CSV fields. By default the API types directory is `pkg/apis/`. Use the `--apis-dir` flag to change this path. The CSV generator expects either of the following layouyts for the API types directory + * Mulitple groups: `///` + * Single groups: `//` + +### Output + +By default `generate csv` will generate the catalog bundle directory `olm-catalog/...` under `deploy/`. To change where the CSV bundle directory is generated use the `--ouput-dir` flag. ## Versioning @@ -66,7 +74,7 @@ Be sure to include the `--update-crds` flag if you want to add CRD's to your bun Below are two lists of fields: the first is a list of all fields the SDK and OLM expect in a CSV, and the second are optional. -Several fields require user input (labeled _user_) or a [code annotation][code-annotations] (labeled _annotation_). This list may change as the SDK becomes better at generating CSV's. +Several fields require user input (labeled _user_) or a [CSV annotation][csv-annotations] (labeled _annotation_). This list may change as the SDK becomes better at generating CSV's. Required: @@ -110,10 +118,9 @@ Optional: [doc-csv]:https://github.com/operator-framework/operator-lifecycle-manager/blob/4197455/Documentation/design/building-your-csv.md [olm]:https://github.com/operator-framework/operator-lifecycle-manager [generate-csv-cli]:../../cli/operator-sdk_generate_csv.md -[doc-project-layout]:../../project_layout.md [doc-csv-design]:../../design/milestone-0.2.0/csv-generation.md [doc-bundle]:https://github.com/operator-framework/operator-registry/blob/6893d19/README.md#manifest-format [x-desc-list]:https://github.com/openshift/console/blob/70bccfe/frontend/public/components/operator-lifecycle-manager/descriptors/types.ts#L3-L35 [install-modes]:https://github.com/operator-framework/operator-lifecycle-manager/blob/4197455/Documentation/design/building-your-csv.md#operator-metadata [olm-capabilities]:../../images/operator-capability-level.png -[code-annotations]:../../proposals/sdk-code-annotations.md +[csv-annotations]: ./csv-annotations.md diff --git a/website/content/en/docs/cli/operator-sdk_generate_csv.md b/website/content/en/docs/cli/operator-sdk_generate_csv.md index 1b20934ca18..e694b0f6662 100644 --- a/website/content/en/docs/cli/operator-sdk_generate_csv.md +++ b/website/content/en/docs/cli/operator-sdk_generate_csv.md @@ -11,22 +11,109 @@ A CSV semantic version is supplied via the --csv-version flag. If your operator has already generated a CSV manifest you want to use as a base, supply its version to --from-version. Otherwise the SDK will scaffold a new CSV manifest. -Configure CSV generation by writing a config file 'deploy/olm-catalog/csv-config.yaml +CSV input flags: + --deploy-dir: The CSV file contents will be generated from the operator manifests + present in this directory. + + --apis-dir: The CSV annotation comments will be parsed from the Go types under this path to + fill out metadata for owned APIs in spec.customresourcedefinitions.owned. + + --crd-dir: The CRD manifests are updated from this path to the CSV bundle directory. + Note: The CSV generator only uses this to copy the CRD manifests. + The CSV contents for spec.customresourcedefinitions.owned will still be generated + from the CRD manifests in the deploy directory specified by --deploy-dir. + If unset, it defaults to the same value as --deploy-dir. + + ``` operator-sdk generate csv [flags] ``` +### Examples + +``` + + ##### Generate CSV from default input paths ##### + $ tree pkg/apis/ deploy/ + pkg/apis/ + ├── ... + └── cache + ├── group.go + └── v1alpha1 + ├── ... + └── memcached_types.go + deploy/ + ├── crds + │   ├── cache.example.com_memcacheds_crd.yaml + │   └── cache.example.com_v1alpha1_memcached_cr.yaml + ├── operator.yaml + ├── role.yaml + ├── role_binding.yaml + └── service_account.yaml + + $ operator-sdk generate csv --csv-version=0.0.1 --update-crds + INFO[0000] Generating CSV manifest version 0.0.1 + ... + + $ tree deploy/ + deploy/ + ... + ├── olm-catalog + │   └── memcached-operator + │   ├── 0.0.1 + │   │   ├── cache.example.com_memcacheds_crd.yaml + │   │   └── memcached-operator.v0.0.1.clusterserviceversion.yaml + │   └── memcached-operator.package.yaml + ... + + + + ##### Generate CSV from custom input paths ##### + $ operator-sdk generate csv --csv-version=0.0.1 --update-crds \ + --deploy-dir=config --apis-dir=api --output-dir=production + INFO[0000] Generating CSV manifest version 0.0.1 + ... + + $ tree config/ api/ production/ + config/ + ├── crds + │   ├── cache.example.com_memcacheds_crd.yaml + │   └── cache.example.com_v1alpha1_memcached_cr.yaml + ├── operator.yaml + ├── role.yaml + ├── role_binding.yaml + └── service_account.yaml + api/ + ├── ... + └── cache + ├── group.go + └── v1alpha1 + ├── ... + └── memcached_types.go + production/ + └── olm-catalog + └── memcached-operator + ├── 0.0.1 + │   ├── cache.example.com_memcacheds_crd.yaml + │   └── memcached-operator.v0.0.1.clusterserviceversion.yaml + └── memcached-operator.package.yaml + +``` + ### Options ``` + --apis-dir string Project relative path to root directory for API type defintions (default "pkg/apis") + --crd-dir string Project relative path to root directory for for CRD manifests --csv-channel string Channel the CSV should be registered under in the package manifest - --csv-config string Path to CSV config file. Defaults to deploy/olm-catalog/csv-config.yaml --csv-version string Semantic version of the CSV --default-channel Use the channel passed to --csv-channel as the package manifests' default channel. Only valid when --csv-channel is set + --deploy-dir string Project relative path to root directory for operator manifests (Deployment, RBAC, CRDs) (default "deploy") --from-version string Semantic version of an existing CSV to use as a base -h, --help help for csv --operator-name string Operator name to use while generating CSV + --output-dir string Base directory to output generated CSV. The resulting CSV bundle directory will be "/olm-catalog//". (default "deploy") --update-crds Update CRD manifests in deploy/{operator-name}/{csv-version} the using latest API's ``` diff --git a/website/content/en/docs/golang/olm-catalog/generating-a-csv.md b/website/content/en/docs/golang/olm-catalog/generating-a-csv.md index 4db5492e0d8..3f6782a25a1 100644 --- a/website/content/en/docs/golang/olm-catalog/generating-a-csv.md +++ b/website/content/en/docs/golang/olm-catalog/generating-a-csv.md @@ -10,35 +10,23 @@ This document describes how to manage the following lifecycle for your Operator ## Configuration -Operator SDK projects have an expected [project layout][doc-project-layout]. In particular, a few manifests are expected to be present in the `deploy` directory: +### Inputs -* Roles: `role.yaml` -* Deployments: `operator.yaml` -* Custom Resources (CR's): `crds/___cr.yaml` -* Custom Resource Definitions (CRD's): `crds/__crd.yaml`. +The CSV generator requires certain inputs to construct a CSV manifest. -`generate csv` reads these files and adds their data to a CSV in an alternate form. +1. Path to the operator manifests root directory. By default `generate csv` extracts manifests from files in `deploy/` for the following kinds and adds them to the CSV. Use the `--deploy-dir` flag to change this path. + * Roles: `role.yaml` + * ClusterRoles: `cluster_role.yaml` + * Deployments: `operator.yaml` + * Custom Resources (CR's): `crds/___cr.yaml` + * CustomResourceDefinitions (CRD's): `crds/__crd.yaml` +2. Path to API types root directory. The CSV generator also parses the [CSV annotations][csv-annotations] from the API type definitions to populate certain CSV fields. By default the API types directory is `pkg/apis/`. Use the `--apis-dir` flag to change this path. The CSV generator expects either of the following layouyts for the API types directory + * Mulitple groups: `///` + * Single groups: `//` -The following example config containing default values should be copied and written to `deploy/olm-catalog/csv-config.yaml`: +### Output -```yaml -crd-cr-paths: -- deploy/crds -operator-path: deploy/operator.yaml -role-paths: -- deploy/role.yaml -``` - -Explanation of all config fields: - -- `crd-cr-paths`: list of strings - a list of CRD and CR manifest file/directory paths. Defaults to `[deploy/crds]`. -- `operator-path`: string - the operator `Deployment` manifest file path. Defaults to `deploy/operator.yaml`. -- `role-paths`: list of strings - Role and ClusterRole manifest file paths. Defaults to `[deploy/role.yaml]`. -- `operator-name`: string - the name used to create the CSV and manifest file names. Defaults to the project's name. - -**Note**: The [design doc][doc-csv-design] has outdated field information which should not be referenced. - -Fields in this config file can be modified to point towards alternate manifest locations, and passed to `generate csv --csv-config=` to configure CSV generation. For example, if I have one set of production CR/CRD manifests under `deploy/crds/production`, and a set of test manifests under `deploy/crds/test`, and I only want to include production manifests in my CSV, I can set `crd-cr-paths: [deploy/crds/production]`. `generate csv` will then ignore `deploy/crds/test` when getting CR/CRD data. +By default `generate csv` will generate the catalog bundle directory `olm-catalog/...` under `deploy/`. To change where the CSV bundle directory is generated use the `--ouput-dir` flag. ## Versioning @@ -46,7 +34,7 @@ CSV's are versioned in path, file name, and in their `metadata.name` field. For `generate csv` allows you to upgrade your CSV using the `--from-version` flag. If you have an existing CSV with version `0.0.1` and want to write a new version `0.0.2`, you can run `operator-sdk generate csv --csv-version 0.0.2 --from-version 0.0.1`. This will write a new CSV manifest to `deploy/olm-catalog//0.0.2/.v0.0.2.clusterserviceversion.yaml` containing user-defined data from `0.0.1` and any modifications you've made to `roles.yaml`, `operator.yaml`, CR's, or CRD's. -The SDK can manage CRD's in your Operator bundle as well. You can pass the `--update-crds` flag to `generate csv` to add or update your CRD's in your bundle by copying manifests pointed to by `crd-cr-paths` in your config. CRD's in a bundle are not updated by default. +The SDK can manage CRD's in your Operator bundle as well. You can pass the `--update-crds` flag to `generate csv` to add or update your CRD's in your bundle by copying manifests in `deploy/crds` to your bundle. CRD's in a bundle are not updated by default. ## First Generation @@ -86,7 +74,7 @@ Be sure to include the `--update-crds` flag if you want to add CRD's to your bun Below are two lists of fields: the first is a list of all fields the SDK and OLM expect in a CSV, and the second are optional. -Several fields require user input (labeled _user_) or a [code annotation][code-annotations] (labeled _annotation_). This list may change as the SDK becomes better at generating CSV's. +Several fields require user input (labeled _user_) or a [CSV annotation][csv-annotations] (labeled _annotation_). This list may change as the SDK becomes better at generating CSV's. Required: @@ -130,10 +118,9 @@ Optional: [doc-csv]:https://github.com/operator-framework/operator-lifecycle-manager/blob/4197455/Documentation/design/building-your-csv.md [olm]:https://github.com/operator-framework/operator-lifecycle-manager [generate-csv-cli]:../../cli/operator-sdk_generate_csv.md -[doc-project-layout]:../../project_layout.md [doc-csv-design]:../../design/milestone-0.2.0/csv-generation.md [doc-bundle]:https://github.com/operator-framework/operator-registry/blob/6893d19/README.md#manifest-format [x-desc-list]:https://github.com/openshift/console/blob/70bccfe/frontend/public/components/operator-lifecycle-manager/descriptors/types.ts#L3-L35 [install-modes]:https://github.com/operator-framework/operator-lifecycle-manager/blob/4197455/Documentation/design/building-your-csv.md#operator-metadata [olm-capabilities]:../../images/operator-capability-level.png -[code-annotations]:../../proposals/sdk-code-annotations.md +[csv-annotations]: ./csv-annotations.md From 8774de6bf4063b9ab9ba2e7097f0e8822d2bfa18 Mon Sep 17 00:00:00 2001 From: hasbro17 Date: Thu, 26 Mar 2020 23:07:13 -0700 Subject: [PATCH 14/16] Use --crd-dir path to read and parse owned CRDs --- cmd/operator-sdk/generate/csv.go | 75 +++++---- doc/cli/operator-sdk_generate_csv.md | 24 +-- internal/generate/olm-catalog/csv.go | 142 +++++++++++------- internal/generate/olm-catalog/csv_go_test.go | 7 + internal/generate/testdata/go/emptydir/.keep | 0 internal/util/k8sutil/crd.go | 9 +- .../en/docs/cli/operator-sdk_generate_csv.md | 24 +-- 7 files changed, 169 insertions(+), 112 deletions(-) create mode 100644 internal/generate/testdata/go/emptydir/.keep diff --git a/cmd/operator-sdk/generate/csv.go b/cmd/operator-sdk/generate/csv.go index bb2822cb3e7..fd970de1bd6 100644 --- a/cmd/operator-sdk/generate/csv.go +++ b/cmd/operator-sdk/generate/csv.go @@ -58,18 +58,18 @@ has already generated a CSV manifest you want to use as a base, supply its version to --from-version. Otherwise the SDK will scaffold a new CSV manifest. CSV input flags: - --deploy-dir: The CSV file contents will be generated from the operator manifests - present in this directory. - - --apis-dir: The CSV annotation comments will be parsed from the Go types under this path to - fill out metadata for owned APIs in spec.customresourcedefinitions.owned. - - --crd-dir: The CRD manifests are updated from this path to the CSV bundle directory. - Note: The CSV generator only uses this to copy the CRD manifests. - The CSV contents for spec.customresourcedefinitions.owned will still be generated - from the CRD manifests in the deploy directory specified by --deploy-dir. - If unset, it defaults to the same value as --deploy-dir. - + --deploy-dir: + The CSV's install strategy and permissions will be generated from the operator manifests + (Deployment and Role/ClusterRole) present in this directory. + + --apis-dir: + The CSV annotation comments will be parsed from the Go types under this path to + fill out metadata for owned APIs in spec.customresourcedefinitions.owned. + + --crd-dir: + The CSV's spec.customresourcedefinitions.owned field is generated from the CRD manifests + in this path.These CRD manifests are also copied over to the bundle directory if --update-crds is set. + Additionally the CR manifests will be used to populate the CSV example CRs. `, Example: ` ##### Generate CSV from default input paths ##### @@ -170,12 +170,16 @@ CSV input flags: cmd.Flags().StringVar(&c.fromVersion, "from-version", "", "Semantic version of an existing CSV to use as a base") + // TODO: Allow multiple paths + // Deployment and RBAC manifests might be in different dirs e.g kubebuilder cmd.Flags().StringVar(&c.deployDir, "deploy-dir", "deploy", - `Project relative path to root directory for operator manifests (Deployment, RBAC, CRDs)`) + `Project relative path to root directory for operator manifests (Deployment and RBAC)`) cmd.Flags().StringVar(&c.apisDir, "apis-dir", filepath.Join("pkg", "apis"), `Project relative path to root directory for API type defintions`) - cmd.Flags().StringVar(&c.crdDir, "crd-dir", "", - `Project relative path to root directory for for CRD manifests`) + // TODO: Allow multiple paths + // CRD and CR manifests might be in different dirs e.g kubebuilder + cmd.Flags().StringVar(&c.crdDir, "crd-dir", filepath.Join("deploy", "crds"), + `Project relative path to root directory for CRD and CR manifests`) cmd.Flags().StringVar(&c.outputDir, "output-dir", scaffold.DeployDir, "Base directory to output generated CSV. The resulting CSV bundle directory "+ @@ -206,6 +210,7 @@ func (c csvCmd) run() error { Inputs: map[string]string{ gencatalog.DeployDirKey: c.deployDir, gencatalog.APIsDirKey: c.apisDir, + gencatalog.CRDsDirKey: c.crdDir, }, OutputDir: c.outputDir, } @@ -221,11 +226,6 @@ func (c csvCmd) run() error { // Write CRD's to the new or updated CSV package dir. if c.updateCRDs { - // TODO(hasbro17): Reconsider the crd-dir flag since it only lets you control - // where the CRD manifests are copied from but the CSV generator above - // will uses the CRD manifests in deploy dir to build the CSV contents for - // spec.customresourcedefinitions.owned - // Need to reconcile this disparity. crdManifestSet, err := findCRDFileSet(c.crdDir) if err != nil { return fmt.Errorf("failed to update CRD's: %v", err) @@ -277,28 +277,43 @@ func validateVersion(version string) error { return nil } -// findCRDFileSet searches all directories and files in path for CRD manifests, +// findCRDFileSet searches files in the given directory path for CRD manifests, // returning a map of paths to file contents. -func findCRDFileSet(path string) (map[string][]byte, error) { +func findCRDFileSet(crdDir string) (map[string][]byte, error) { crdFileSet := map[string][]byte{} - info, err := os.Stat(path) + info, err := os.Stat(crdDir) if err != nil { return nil, err } if !info.IsDir() { - return nil, fmt.Errorf("crd's must be read from a directory. %s is a file", path) + return nil, fmt.Errorf("crd's must be read from a directory. %s is a file", crdDir) } - - // Get CRD manifest paths from path recursively but ignore olm-catalog subdir - // if it is present in the search directory - crdPaths, err := k8sutil.GetCRDManifestPaths(path, gencatalog.OLMCatalogChildDir) + files, err := ioutil.ReadDir(crdDir) if err != nil { return nil, err } - for _, crdPath := range crdPaths { + + wd := projutil.MustGetwd() + for _, f := range files { + if f.IsDir() { + continue + } + + crdPath := filepath.Join(wd, crdDir, f.Name()) b, err := ioutil.ReadFile(crdPath) if err != nil { - return nil, err + return nil, fmt.Errorf("error reading manifest %s: %v", crdPath, err) + } + // Skip files in crdsDir that aren't k8s manifests since we do not know + // what other files are in crdsDir. + typeMeta, err := k8sutil.GetTypeMetaFromBytes(b) + if err != nil { + log.Debugf("Skipping non-manifest file %s: %v", crdPath, err) + continue + } + if typeMeta.Kind != "CustomResourceDefinition" { + log.Debugf("Skipping non CRD manifest %s", crdPath) + continue } crdFileSet[filepath.Base(crdPath)] = b } diff --git a/doc/cli/operator-sdk_generate_csv.md b/doc/cli/operator-sdk_generate_csv.md index e694b0f6662..57d80b6d193 100644 --- a/doc/cli/operator-sdk_generate_csv.md +++ b/doc/cli/operator-sdk_generate_csv.md @@ -12,18 +12,18 @@ has already generated a CSV manifest you want to use as a base, supply its version to --from-version. Otherwise the SDK will scaffold a new CSV manifest. CSV input flags: - --deploy-dir: The CSV file contents will be generated from the operator manifests - present in this directory. + --deploy-dir: + The CSV's install strategy and permissions will be generated from the operator manifests + (Deployment and Role/ClusterRole) present in this directory. - --apis-dir: The CSV annotation comments will be parsed from the Go types under this path to - fill out metadata for owned APIs in spec.customresourcedefinitions.owned. - - --crd-dir: The CRD manifests are updated from this path to the CSV bundle directory. - Note: The CSV generator only uses this to copy the CRD manifests. - The CSV contents for spec.customresourcedefinitions.owned will still be generated - from the CRD manifests in the deploy directory specified by --deploy-dir. - If unset, it defaults to the same value as --deploy-dir. + --apis-dir: + The CSV annotation comments will be parsed from the Go types under this path to + fill out metadata for owned APIs in spec.customresourcedefinitions.owned. + --crd-dir: + The CSV's spec.customresourcedefinitions.owned field is generated from the CRD manifests + in this path.These CRD manifests are also copied over to the bundle directory if --update-crds is set. + Additionally the CR manifests will be used to populate the CSV example CRs. ``` @@ -105,11 +105,11 @@ operator-sdk generate csv [flags] ``` --apis-dir string Project relative path to root directory for API type defintions (default "pkg/apis") - --crd-dir string Project relative path to root directory for for CRD manifests + --crd-dir string Project relative path to root directory for CRD and CR manifests (default "deploy/crds") --csv-channel string Channel the CSV should be registered under in the package manifest --csv-version string Semantic version of the CSV --default-channel Use the channel passed to --csv-channel as the package manifests' default channel. Only valid when --csv-channel is set - --deploy-dir string Project relative path to root directory for operator manifests (Deployment, RBAC, CRDs) (default "deploy") + --deploy-dir string Project relative path to root directory for operator manifests (Deployment and RBAC) (default "deploy") --from-version string Semantic version of an existing CSV to use as a base -h, --help help for csv --operator-name string Operator name to use while generating CSV diff --git a/internal/generate/olm-catalog/csv.go b/internal/generate/olm-catalog/csv.go index f94fbf2fec3..004900259b1 100644 --- a/internal/generate/olm-catalog/csv.go +++ b/internal/generate/olm-catalog/csv.go @@ -28,6 +28,7 @@ import ( "github.com/operator-framework/operator-sdk/internal/scaffold" "github.com/operator-framework/operator-sdk/internal/util/fileutil" "github.com/operator-framework/operator-sdk/internal/util/k8sutil" + "github.com/operator-framework/operator-sdk/internal/util/projutil" "github.com/operator-framework/operator-sdk/internal/util/yamlutil" "github.com/blang/semver" @@ -51,9 +52,17 @@ const ( // Input keys for CSV generator whose values are the filepaths for the respective input directories // DeployDirKey is for the location of the operator manifests directory e.g "deploy/production" + // The Deployment and RBAC manifests from this directory will be used to populate the CSV + // install strategy: spec.install DeployDirKey = "deploy" // APIsDirKey is for the location of the API types directory e.g "pkg/apis" + // The CSV annotation comments will be parsed from the types under this path. APIsDirKey = "apis" + // CRDsDirKey is for the location of the CRD manifests directory e.g "deploy/crds" + // Both the CRD and CR manifests from this path will be used to populate CSV fields + // metadata.annotations.alm-examples for CR examples + // and spec.customresourcedefinitions.owned for owned CRDs + CRDsDirKey = "crds" ) type csvGenerator struct { @@ -107,6 +116,10 @@ func NewCSV(cfg gen.Config, csvVersion, fromVersion string) gen.Generator { g.Inputs[APIsDirKey] = scaffold.ApisDir } + if crdsDir, ok := g.Inputs[CRDsDirKey]; !ok || crdsDir == "" { + g.Inputs[CRDsDirKey] = filepath.Join(g.Inputs[DeployDirKey], "crds") + } + return g } @@ -366,60 +379,24 @@ func (g csvGenerator) updateCSVVersions(csv *olmapiv1alpha1.ClusterServiceVersio // user-defined manifests and updates csv. func (g csvGenerator) updateCSVFromManifests(csv *olmapiv1alpha1.ClusterServiceVersion) (err error) { kindManifestMap := map[schema.GroupVersionKind][][]byte{} - crGVKSet := map[schema.GroupVersionKind]struct{}{} - err = filepath.Walk(g.Inputs[DeployDirKey], func(path string, info os.FileInfo, werr error) error { - if werr != nil { - log.Debugf("Failed to walk dir: %v", werr) - return werr - } - // Only read manifest from files, not directories - if info.IsDir() { - // Skip walking olm-catalog dir if it's present in the deploy directory - if info.Name() == OLMCatalogChildDir { - return filepath.SkipDir - } - return nil - } - b, err := ioutil.ReadFile(path) - if err != nil { - return err - } - scanner := yamlutil.NewYAMLScanner(b) - for scanner.Scan() { - manifest := scanner.Bytes() - typeMeta, err := k8sutil.GetTypeMetaFromBytes(manifest) - if err != nil { - log.Infof("No TypeMeta in %s, skipping file", path) - continue - } - gvk := typeMeta.GroupVersionKind() - kindManifestMap[gvk] = append(kindManifestMap[gvk], manifest) - switch typeMeta.Kind { - case "CustomResourceDefinition": - // Collect CRD kinds to filter them out from unsupported manifest types. - // The CRD version type doesn't matter as long as it has a group, kind, - // and versions in the expected fields. - crd := v1beta1.CustomResourceDefinition{} - if err = yaml.Unmarshal(manifest, &crd); err != nil { - return err - } - for _, ver := range crd.Spec.Versions { - crGVK := schema.GroupVersionKind{ - Group: crd.Spec.Group, - Version: ver.Name, - Kind: crd.Spec.Names.Kind, - } - crGVKSet[crGVK] = struct{}{} - } - } - } - return scanner.Err() - }) + // Read CRD and CR manifests from CRD dir + if err := updateFromManifests(g.Inputs[CRDsDirKey], kindManifestMap); err != nil { + return err + } + + // Get owned CRDs from CRD manifests + ownedCRDs, err := getOwnedCRDs(kindManifestMap) if err != nil { - return fmt.Errorf("failed to walk manifests directory for CSV updates: %v", err) + return err + } + + // Read Deployment and RBAC manifests from Deploy dir + if err := updateFromManifests(g.Inputs[DeployDirKey], kindManifestMap); err != nil { + return err } + // Update CSV from all manifest types crUpdaters := crs{} for gvk, manifests := range kindManifestMap { // We don't necessarily care about sorting by a field value, more about @@ -437,7 +414,8 @@ func (g csvGenerator) updateCSVFromManifests(csv *olmapiv1alpha1.ClusterServiceV case "CustomResourceDefinition": err = crds(manifests).apply(csv) default: - if _, ok := crGVKSet[gvk]; ok { + // Only update CR examples for owned CRD types + if _, ok := ownedCRDs[gvk]; ok { crUpdaters = append(crUpdaters, crs(manifests)...) } else { log.Infof("Skipping manifest %s", gvk) @@ -462,3 +440,65 @@ func (g csvGenerator) updateCSVFromManifests(csv *olmapiv1alpha1.ClusterServiceV } return nil } + +func updateFromManifests(dir string, kindManifestMap map[schema.GroupVersionKind][][]byte) error { + files, err := ioutil.ReadDir(dir) + if err != nil { + return err + } + // Read and scan all files into kindManifestMap + wd := projutil.MustGetwd() + for _, f := range files { + if f.IsDir() { + continue + } + path := filepath.Join(wd, dir, f.Name()) + b, err := ioutil.ReadFile(path) + if err != nil { + return err + } + scanner := yamlutil.NewYAMLScanner(b) + for scanner.Scan() { + manifest := scanner.Bytes() + typeMeta, err := k8sutil.GetTypeMetaFromBytes(manifest) + if err != nil { + log.Infof("No TypeMeta in %s, skipping file", path) + continue + } + + gvk := typeMeta.GroupVersionKind() + kindManifestMap[gvk] = append(kindManifestMap[gvk], manifest) + } + if scanner.Err() != nil { + return scanner.Err() + } + } + return nil +} + +func getOwnedCRDs(kindManifestMap map[schema.GroupVersionKind][][]byte) (map[schema.GroupVersionKind]struct{}, error) { + ownedCRDs := map[schema.GroupVersionKind]struct{}{} + for gvk, manifests := range kindManifestMap { + if gvk.Kind != "CustomResourceDefinition" { + continue + } + // Collect CRD kinds to filter them out from unsupported manifest types. + // The CRD version type doesn't matter as long as it has a group, kind, + // and versions in the expected fields. + for _, manifest := range manifests { + crd := v1beta1.CustomResourceDefinition{} + if err := yaml.Unmarshal(manifest, &crd); err != nil { + return ownedCRDs, err + } + for _, ver := range crd.Spec.Versions { + crGVK := schema.GroupVersionKind{ + Group: crd.Spec.Group, + Version: ver.Name, + Kind: crd.Spec.Names.Kind, + } + ownedCRDs[crGVK] = struct{}{} + } + } + } + return ownedCRDs, nil +} diff --git a/internal/generate/olm-catalog/csv_go_test.go b/internal/generate/olm-catalog/csv_go_test.go index 489b2d88225..c139f957984 100644 --- a/internal/generate/olm-catalog/csv_go_test.go +++ b/internal/generate/olm-catalog/csv_go_test.go @@ -88,6 +88,7 @@ func TestGenerateNewCSVWithInputsToOutput(t *testing.T) { Inputs: map[string]string{ DeployDirKey: "config", APIsDirKey: "api", + CRDsDirKey: filepath.Join("config", "crds"), }, OutputDir: outputDir, } @@ -142,6 +143,7 @@ func TestUpgradeFromExistingCSVWithInputsToOutput(t *testing.T) { Inputs: map[string]string{ DeployDirKey: "config", APIsDirKey: "api", + CRDsDirKey: filepath.Join("config", "crds"), }, OutputDir: outputDir, } @@ -201,6 +203,7 @@ func TestGoCSVFromNew(t *testing.T) { Inputs: map[string]string{ DeployDirKey: "deploy", APIsDirKey: filepath.Join("pkg", "apis"), + CRDsDirKey: filepath.Join("deploy", "crds_v1beta1"), }, OutputDir: "deploy", } @@ -238,6 +241,7 @@ func TestGoCSVFromOld(t *testing.T) { Inputs: map[string]string{ DeployDirKey: "deploy", APIsDirKey: filepath.Join("pkg", "apis"), + CRDsDirKey: filepath.Join("deploy", "crds_v1beta1"), }, OutputDir: "deploy", } @@ -269,6 +273,7 @@ func TestGoCSVWithInvalidManifestsDir(t *testing.T) { Inputs: map[string]string{ DeployDirKey: "notExist", APIsDirKey: filepath.Join("pkg", "apis"), + CRDsDirKey: "notExist", }, OutputDir: "deploy", } @@ -290,6 +295,7 @@ func TestGoCSVWithEmptyManifestsDir(t *testing.T) { Inputs: map[string]string{ DeployDirKey: "emptydir", APIsDirKey: filepath.Join("pkg", "apis"), + CRDsDirKey: "emptydir", }, OutputDir: "emptydir", } @@ -331,6 +337,7 @@ func TestUpdateVersion(t *testing.T) { Inputs: map[string]string{ DeployDirKey: "deploy", APIsDirKey: filepath.Join("pkg", "apis"), + CRDsDirKey: filepath.Join("deploy", "crds_v1beta1"), }, OutputDir: "deploy", } diff --git a/internal/generate/testdata/go/emptydir/.keep b/internal/generate/testdata/go/emptydir/.keep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/internal/util/k8sutil/crd.go b/internal/util/k8sutil/crd.go index feb8af9719a..2d974a0edc2 100644 --- a/internal/util/k8sutil/crd.go +++ b/internal/util/k8sutil/crd.go @@ -30,7 +30,7 @@ import ( // GetCRDs parses all CRD manifests in the directory crdsDir and all of its subdirectories. func GetCRDs(crdsDir string) ([]*apiextv1beta1.CustomResourceDefinition, error) { - manifests, err := GetCRDManifestPaths(crdsDir, "") + manifests, err := GetCRDManifestPaths(crdsDir) if err != nil { return nil, fmt.Errorf("failed to get CRD's from %s: %v", crdsDir, err) } @@ -50,8 +50,7 @@ func GetCRDs(crdsDir string) ([]*apiextv1beta1.CustomResourceDefinition, error) } // GetCRDManifestPaths returns all CRD manifest paths in crdsDir and subdirs. -// The directory ignoreSubDir will be ignored -func GetCRDManifestPaths(crdsDir, ignoreSubDir string) (crdPaths []string, err error) { +func GetCRDManifestPaths(crdsDir string) (crdPaths []string, err error) { err = filepath.Walk(crdsDir, func(path string, info os.FileInfo, werr error) error { if werr != nil { return werr @@ -59,10 +58,6 @@ func GetCRDManifestPaths(crdsDir, ignoreSubDir string) (crdPaths []string, err e // Only read manifest from files, not directories if info.IsDir() { - // Skip walking ignored directory - if ignoreSubDir != "" && info.Name() == ignoreSubDir { - return filepath.SkipDir - } return nil } diff --git a/website/content/en/docs/cli/operator-sdk_generate_csv.md b/website/content/en/docs/cli/operator-sdk_generate_csv.md index e694b0f6662..57d80b6d193 100644 --- a/website/content/en/docs/cli/operator-sdk_generate_csv.md +++ b/website/content/en/docs/cli/operator-sdk_generate_csv.md @@ -12,18 +12,18 @@ has already generated a CSV manifest you want to use as a base, supply its version to --from-version. Otherwise the SDK will scaffold a new CSV manifest. CSV input flags: - --deploy-dir: The CSV file contents will be generated from the operator manifests - present in this directory. + --deploy-dir: + The CSV's install strategy and permissions will be generated from the operator manifests + (Deployment and Role/ClusterRole) present in this directory. - --apis-dir: The CSV annotation comments will be parsed from the Go types under this path to - fill out metadata for owned APIs in spec.customresourcedefinitions.owned. - - --crd-dir: The CRD manifests are updated from this path to the CSV bundle directory. - Note: The CSV generator only uses this to copy the CRD manifests. - The CSV contents for spec.customresourcedefinitions.owned will still be generated - from the CRD manifests in the deploy directory specified by --deploy-dir. - If unset, it defaults to the same value as --deploy-dir. + --apis-dir: + The CSV annotation comments will be parsed from the Go types under this path to + fill out metadata for owned APIs in spec.customresourcedefinitions.owned. + --crd-dir: + The CSV's spec.customresourcedefinitions.owned field is generated from the CRD manifests + in this path.These CRD manifests are also copied over to the bundle directory if --update-crds is set. + Additionally the CR manifests will be used to populate the CSV example CRs. ``` @@ -105,11 +105,11 @@ operator-sdk generate csv [flags] ``` --apis-dir string Project relative path to root directory for API type defintions (default "pkg/apis") - --crd-dir string Project relative path to root directory for for CRD manifests + --crd-dir string Project relative path to root directory for CRD and CR manifests (default "deploy/crds") --csv-channel string Channel the CSV should be registered under in the package manifest --csv-version string Semantic version of the CSV --default-channel Use the channel passed to --csv-channel as the package manifests' default channel. Only valid when --csv-channel is set - --deploy-dir string Project relative path to root directory for operator manifests (Deployment, RBAC, CRDs) (default "deploy") + --deploy-dir string Project relative path to root directory for operator manifests (Deployment and RBAC) (default "deploy") --from-version string Semantic version of an existing CSV to use as a base -h, --help help for csv --operator-name string Operator name to use while generating CSV From 3708227022e2acf19f6e18a7caf9b75de71a784d Mon Sep 17 00:00:00 2001 From: hasbro17 Date: Tue, 31 Mar 2020 18:37:03 -0700 Subject: [PATCH 15/16] fix olm pkg dependencies after rebase --- internal/generate/olm-catalog/csv.go | 6 +- internal/scaffold/olm-catalog/csv_test.go | 207 ---------------------- 2 files changed, 2 insertions(+), 211 deletions(-) delete mode 100644 internal/scaffold/olm-catalog/csv_test.go diff --git a/internal/generate/olm-catalog/csv.go b/internal/generate/olm-catalog/csv.go index 004900259b1..9d0052d5958 100644 --- a/internal/generate/olm-catalog/csv.go +++ b/internal/generate/olm-catalog/csv.go @@ -15,7 +15,6 @@ package olmcatalog import ( - "encoding/json" "fmt" "io/ioutil" "os" @@ -34,7 +33,6 @@ import ( "github.com/blang/semver" "github.com/ghodss/yaml" olmapiv1alpha1 "github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1" - olminstall "github.com/operator-framework/operator-lifecycle-manager/pkg/controller/install" olmversion "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/version" "github.com/pkg/errors" log "github.com/sirupsen/logrus" @@ -285,8 +283,8 @@ func newCSV(name, version string) (*olmapiv1alpha1.ClusterServiceVersion, error) {Type: olmapiv1alpha1.InstallModeTypeAllNamespaces, Supported: true}, }, InstallStrategy: olmapiv1alpha1.NamedInstallStrategy{ - StrategyName: olminstall.InstallStrategyNameDeployment, - StrategySpecRaw: json.RawMessage("{}"), + StrategyName: olmapiv1alpha1.InstallStrategyNameDeployment, + StrategySpec: olmapiv1alpha1.StrategyDetailsDeployment{}, }, }, }, nil diff --git a/internal/scaffold/olm-catalog/csv_test.go b/internal/scaffold/olm-catalog/csv_test.go deleted file mode 100644 index 46a00118a70..00000000000 --- a/internal/scaffold/olm-catalog/csv_test.go +++ /dev/null @@ -1,207 +0,0 @@ -// Copyright 2018 The Operator-SDK Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package catalog - -import ( - "bytes" - "io" - "io/ioutil" - "os" - "path/filepath" - "testing" - - "github.com/operator-framework/operator-sdk/internal/scaffold" - "github.com/operator-framework/operator-sdk/internal/scaffold/input" - testutil "github.com/operator-framework/operator-sdk/internal/scaffold/internal/testutil" - "github.com/operator-framework/operator-sdk/internal/util/diffutil" - "github.com/operator-framework/operator-sdk/pkg/k8sutil" - - "github.com/blang/semver" - "github.com/ghodss/yaml" - olmapiv1alpha1 "github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1" - "github.com/spf13/afero" - appsv1 "k8s.io/api/apps/v1" -) - -const ( - testDataDir = "testdata" - projectName = "app-operator-dir" - operatorName = "app-operator" - oldCSVVer = "0.1.0" - newCSVVer = "0.2.0" - csvVer = "0.1.0" -) - -var testDeployDir = filepath.Join(testDataDir, scaffold.DeployDir) - -func TestCSVNew(t *testing.T) { - buf := &bytes.Buffer{} - s := &scaffold.Scaffold{ - GetWriter: func(_ string, _ os.FileMode) (io.Writer, error) { - return buf, nil - }, - } - - sc := &CSV{CSVVersion: csvVer, pathPrefix: testDataDir, OperatorName: operatorName} - err := s.Execute(&input.Config{ProjectName: projectName}, sc) - if err != nil { - t.Fatalf("Failed to execute the scaffold: (%v)", err) - } - - // Get the expected CSV manifest from test data dir. - csvExpBytes, err := afero.ReadFile(s.Fs, sc.getCSVPath(csvVer)) - if err != nil { - t.Fatal(err) - } - csvExp := string(csvExpBytes) - if csvExp != buf.String() { - diffs := diffutil.Diff(csvExp, buf.String()) - t.Errorf("Expected vs actual differs.\n%v", diffs) - } -} - -func TestCSVFromOld(t *testing.T) { - s := &scaffold.Scaffold{Fs: afero.NewMemMapFs()} - - // Write all files in testdata/deploy to fs so manifests are present when - // writing a new CSV. - if err := testutil.WriteOSPathToFS(afero.NewOsFs(), s.Fs, testDeployDir); err != nil { - t.Fatalf("Failed to write %s to in-memory test fs: (%v)", testDeployDir, err) - } - - sc := &CSV{ - CSVVersion: newCSVVer, - FromVersion: oldCSVVer, - pathPrefix: testDataDir, - OperatorName: operatorName, - } - err := s.Execute(&input.Config{ProjectName: projectName}, sc) - if err != nil { - t.Fatalf("Failed to execute the scaffold: (%v)", err) - } - - // Check if a new file was written at the expected path. - newCSVPath := sc.getCSVPath(newCSVVer) - newCSV, newExists, err := getCSVFromFSIfExists(s.Fs, newCSVPath) - if err != nil { - t.Fatalf("Failed to get new CSV %s: (%v)", newCSVPath, err) - } - if !newExists { - t.Fatalf("New CSV does not exist at %s", newCSVPath) - } - - expName := getCSVName(operatorName, newCSVVer) - if newCSV.ObjectMeta.Name != expName { - t.Errorf("Expected CSV metadata.name %s, got %s", expName, newCSV.ObjectMeta.Name) - } - expReplaces := getCSVName(operatorName, oldCSVVer) - if newCSV.Spec.Replaces != expReplaces { - t.Errorf("Expected CSV spec.replaces %s, got %s", expReplaces, newCSV.Spec.Replaces) - } -} - -func TestUpdateVersion(t *testing.T) { - sc := &CSV{ - Input: input.Input{ProjectName: projectName}, - CSVVersion: newCSVVer, - pathPrefix: testDataDir, - OperatorName: operatorName, - } - csvExpBytes, err := ioutil.ReadFile(sc.getCSVPath(oldCSVVer)) - if err != nil { - t.Fatal(err) - } - csv := &olmapiv1alpha1.ClusterServiceVersion{} - if err := yaml.Unmarshal(csvExpBytes, csv); err != nil { - t.Fatal(err) - } - - if err := sc.updateCSVVersions(csv); err != nil { - t.Fatalf("Failed to update csv with version %s: (%v)", newCSVVer, err) - } - - wantedSemver, err := semver.Parse(newCSVVer) - if err != nil { - t.Errorf("Failed to parse %s: %v", newCSVVer, err) - } - if !csv.Spec.Version.Equals(wantedSemver) { - t.Errorf("Wanted csv version %v, got %v", wantedSemver, csv.Spec.Version) - } - wantedName := getCSVName(operatorName, newCSVVer) - if csv.ObjectMeta.Name != wantedName { - t.Errorf("Wanted csv name %s, got %s", wantedName, csv.ObjectMeta.Name) - } - - csvDepSpecs := csv.Spec.InstallStrategy.StrategySpec.DeploymentSpecs - if len(csvDepSpecs) != 1 { - t.Fatal("No deployment specs in CSV") - } - csvPodImage := csvDepSpecs[0].Spec.Template.Spec.Containers[0].Image - if len(csvDepSpecs[0].Spec.Template.Spec.Containers) != 1 { - t.Fatal("No containers in CSV deployment spec") - } - // updateCSVVersions should not update podspec image. - wantedImage := "quay.io/example-inc/operator:v0.1.0" - if csvPodImage != wantedImage { - t.Errorf("Podspec image changed from %s to %s", wantedImage, csvPodImage) - } - - wantedReplaces := getCSVName(operatorName, "0.1.0") - if csv.Spec.Replaces != wantedReplaces { - t.Errorf("Wanted csv replaces %s, got %s", wantedReplaces, csv.Spec.Replaces) - } -} - -func TestSetAndCheckOLMNamespaces(t *testing.T) { - depBytes, err := ioutil.ReadFile(filepath.Join(testDeployDir, "operator.yaml")) - if err != nil { - t.Fatalf("Failed to read Deployment bytes: %v", err) - } - - // The test operator.yaml doesn't have "olm.targetNamespaces", so first - // check that depHasOLMNamespaces() returns false. - dep := appsv1.Deployment{} - if err := yaml.Unmarshal(depBytes, &dep); err != nil { - t.Fatalf("Failed to unmarshal Deployment bytes: %v", err) - } - if depHasOLMNamespaces(dep) { - t.Error("Expected depHasOLMNamespaces to return false, got true") - } - - // Insert "olm.targetNamespaces" into WATCH_NAMESPACE and check that - // depHasOLMNamespaces() returns true. - setWatchNamespacesEnv(&dep) - if !depHasOLMNamespaces(dep) { - t.Error("Expected depHasOLMNamespaces to return true, got false") - } - - // Overwrite WATCH_NAMESPACE and check that depHasOLMNamespaces() returns - // false. - overwriteContainerEnvVar(&dep, k8sutil.WatchNamespaceEnvVar, newEnvVar("FOO", "bar")) - if depHasOLMNamespaces(dep) { - t.Error("Expected depHasOLMNamespaces to return false, got true") - } - - // Insert "olm.targetNamespaces" elsewhere in the deployment pod spec - // and check that depHasOLMNamespaces() returns true. - dep = appsv1.Deployment{} - if err := yaml.Unmarshal(depBytes, &dep); err != nil { - t.Fatalf("Failed to unmarshal Deployment bytes: %v", err) - } - dep.Spec.Template.ObjectMeta.Labels["namespace"] = olmTNMeta - if !depHasOLMNamespaces(dep) { - t.Error("Expected depHasOLMNamespaces to return true, got false") - } -} From 6b41760853a26a3093c817967a61d28c6d6cdd09 Mon Sep 17 00:00:00 2001 From: Eric Stroczynski Date: Tue, 31 Mar 2020 23:50:30 -0400 Subject: [PATCH 16/16] Update test/test-framework/deploy/olm-catalog/memcached-operator/0.0.2/memcached-operator.v0.0.2.clusterserviceversion.yaml --- .../0.0.2/memcached-operator.v0.0.2.clusterserviceversion.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/test-framework/deploy/olm-catalog/memcached-operator/0.0.2/memcached-operator.v0.0.2.clusterserviceversion.yaml b/test/test-framework/deploy/olm-catalog/memcached-operator/0.0.2/memcached-operator.v0.0.2.clusterserviceversion.yaml index 3633c973d7b..663ab2647d4 100644 --- a/test/test-framework/deploy/olm-catalog/memcached-operator/0.0.2/memcached-operator.v0.0.2.clusterserviceversion.yaml +++ b/test/test-framework/deploy/olm-catalog/memcached-operator/0.0.2/memcached-operator.v0.0.2.clusterserviceversion.yaml @@ -67,6 +67,9 @@ spec: x-descriptors: - urn:alm:descriptor:io.kubernetes.phase:reason version: v1alpha1 + - kind: NotExistKind + name: notexist.example.com + version: v1alpha1 description: Main enterprise application providing business critical features with high availability and no manual intervention. displayName: Memcached Application