diff --git a/internal/generate/olm-catalog/csv.go b/internal/generate/olm-catalog/csv.go index c90f5dcd5ff..aca359ef137 100644 --- a/internal/generate/olm-catalog/csv.go +++ b/internal/generate/olm-catalog/csv.go @@ -20,7 +20,6 @@ import ( "os" "path/filepath" "regexp" - "sort" "strings" "github.com/operator-framework/operator-sdk/internal/generate/gen" @@ -35,9 +34,7 @@ import ( olmversion "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/version" "github.com/pkg/errors" log "github.com/sirupsen/logrus" - "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" ) const ( @@ -192,6 +189,7 @@ func (g csvGenerator) generate() (fileMap map[string][]byte, err error) { return nil, err } + // TODO(estroz): replace with CSV validator from API library. path := getCSVFileName(g.OperatorName, g.csvVersion) if fields := getEmptyRequiredCSVFields(csv); len(fields) != 0 { if g.existingCSVBundleDir != "" { @@ -221,27 +219,19 @@ func getCSVFromDir(dir string) (*olmapiv1alpha1.ClusterServiceVersion, error) { } for _, info := range infos { path := filepath.Join(dir, info.Name()) - info, err := os.Stat(path) - if err != nil || info.IsDir() { - // Skip any directories or files accessed in error. - continue - } - b, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - typeMeta, err := k8sutil.GetTypeMetaFromBytes(b) - if err != nil { - return nil, err - } - if typeMeta.Kind != olmapiv1alpha1.ClusterServiceVersionKind { - continue - } - csv := &olmapiv1alpha1.ClusterServiceVersion{} - if err := yaml.Unmarshal(b, csv); err != nil { - return nil, errors.Wrapf(err, "error unmarshalling CSV %s", path) + // Only read manifest from files, not directories + if !info.IsDir() { + b, err := ioutil.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("error reading manifest %s: %v", path, err) + } + csv := &olmapiv1alpha1.ClusterServiceVersion{} + if err := yaml.Unmarshal(b, csv); err != nil { + log.Debugf("Skipping manifest %s: %v", path, err) + continue + } + return csv, nil } - return csv, nil } return nil, fmt.Errorf("no CSV manifest in %s", dir) } @@ -249,11 +239,7 @@ func getCSVFromDir(dir string) (*olmapiv1alpha1.ClusterServiceVersion, error) { // newCSV sets all csv fields that should be populated by a user // to sane defaults. func newCSV(name, version string) (*olmapiv1alpha1.ClusterServiceVersion, error) { - ver, err := semver.Parse(version) - if err != nil { - return nil, err - } - return &olmapiv1alpha1.ClusterServiceVersion{ + csv := &olmapiv1alpha1.ClusterServiceVersion{ TypeMeta: metav1.TypeMeta{ APIVersion: olmapiv1alpha1.ClusterServiceVersionAPIVersion, Kind: olmapiv1alpha1.ClusterServiceVersionKind, @@ -263,18 +249,17 @@ func newCSV(name, version string) (*olmapiv1alpha1.ClusterServiceVersion, error) Namespace: "placeholder", Annotations: map[string]string{ "capabilities": "Basic Install", + "alm-examples": "[]", }, }, Spec: olmapiv1alpha1.ClusterServiceVersionSpec{ DisplayName: k8sutil.GetDisplayName(name), - Description: "", Provider: olmapiv1alpha1.AppLink{}, Maintainers: make([]olmapiv1alpha1.Maintainer, 1), Links: []olmapiv1alpha1.AppLink{}, Maturity: "alpha", - Version: olmversion.OperatorVersion{Version: ver}, Icon: make([]olmapiv1alpha1.Icon, 1), - Keywords: []string{""}, + Keywords: make([]string, 1), InstallModes: []olmapiv1alpha1.InstallMode{ {Type: olmapiv1alpha1.InstallModeTypeOwnNamespace, Supported: true}, {Type: olmapiv1alpha1.InstallModeTypeSingleNamespace, Supported: true}, @@ -283,10 +268,25 @@ func newCSV(name, version string) (*olmapiv1alpha1.ClusterServiceVersion, error) }, InstallStrategy: olmapiv1alpha1.NamedInstallStrategy{ StrategyName: olmapiv1alpha1.InstallStrategyNameDeployment, - StrategySpec: olmapiv1alpha1.StrategyDetailsDeployment{}, + StrategySpec: olmapiv1alpha1.StrategyDetailsDeployment{ + Permissions: []olmapiv1alpha1.StrategyDeploymentPermissions{}, + ClusterPermissions: []olmapiv1alpha1.StrategyDeploymentPermissions{}, + DeploymentSpecs: []olmapiv1alpha1.StrategyDeploymentSpec{}, + }, }, }, - }, nil + } + + // An empty version string will evaluate to "v0.0.0". + if version != "" { + ver, err := semver.Parse(version) + if err != nil { + return nil, err + } + csv.Spec.Version = olmversion.OperatorVersion{Version: ver} + } + + return csv, nil } // TODO: replace with validation library. @@ -369,87 +369,24 @@ func (g csvGenerator) updateCSVVersions(csv *olmapiv1alpha1.ClusterServiceVersio } csv.Spec.Version = olmversion.OperatorVersion{Version: ver} csv.Spec.Replaces = oldCSVName + return nil } // updateCSVFromManifests gathers relevant data from generated and // user-defined manifests and updates csv. func (g csvGenerator) updateCSVFromManifests(csv *olmapiv1alpha1.ClusterServiceVersion) (err error) { - kindManifestMap := map[schema.GroupVersionKind][][]byte{} - - // Read CRD and CR manifests from CRD dir - if err := updateFromManifests(g.Inputs[CRDsDirKey], kindManifestMap); err != nil { - return err - } - - // Get owned CRDs from CRD manifests - ownedCRDs, err := getOwnedCRDs(kindManifestMap) - if err != nil { - return err - } - - // Read Deployment and RBAC manifests from Deploy dir - if err := updateFromManifests(g.Inputs[DeployDirKey], kindManifestMap); err != nil { - return err - } - - // Update CSV from all manifest types - crUpdaters := crs{} - for gvk, manifests := range kindManifestMap { - // We don't necessarily care about sorting by a field value, more about - // consistent ordering. - sort.Slice(manifests, func(i int, j int) bool { - return string(manifests[i]) < string(manifests[j]) - }) - switch gvk.Kind { - case "Role": - err = roles(manifests).apply(csv) - case "ClusterRole": - err = clusterRoles(manifests).apply(csv) - case "Deployment": - err = deployments(manifests).apply(csv) - case "CustomResourceDefinition": - err = crds(manifests).apply(csv) - default: - // Only update CR examples for owned CRD types - if _, ok := ownedCRDs[gvk]; ok { - crUpdaters = append(crUpdaters, crs(manifests)...) - } else { - log.Infof("Skipping manifest %s", gvk) - } - } + // Collect all manifests in paths. + collection := manifestCollection{} + err = filepath.Walk(g.Inputs[DeployDirKey], func(path string, info os.FileInfo, err error) error { if err != nil { return err } - } - err = updateDescriptions(csv, g.Inputs[APIsDirKey]) - if err != nil { - return fmt.Errorf("error updating CSV customresourcedefinitions: %w", err) - } - // Re-sort CR's since they are appended in random order. - if len(crUpdaters) != 0 { - sort.Slice(crUpdaters, func(i int, j int) bool { - return string(crUpdaters[i]) < string(crUpdaters[j]) - }) - if err = crUpdaters.apply(csv); err != nil { - return err + // Only read manifest from files, not directories + if info.IsDir() { + return nil } - } - return nil -} -func updateFromManifests(dir string, kindManifestMap map[schema.GroupVersionKind][][]byte) error { - files, err := ioutil.ReadDir(dir) - if err != nil { - return err - } - // Read and scan all files into kindManifestMap - wd := projutil.MustGetwd() - for _, f := range files { - if f.IsDir() { - continue - } - path := filepath.Join(wd, dir, f.Name()) b, err := ioutil.ReadFile(path) if err != nil { return err @@ -459,43 +396,65 @@ func updateFromManifests(dir string, kindManifestMap map[schema.GroupVersionKind manifest := scanner.Bytes() typeMeta, err := k8sutil.GetTypeMetaFromBytes(manifest) if err != nil { - log.Infof("No TypeMeta in %s, skipping file", path) + log.Debugf("No TypeMeta in %s, skipping file", path) continue } - - gvk := typeMeta.GroupVersionKind() - kindManifestMap[gvk] = append(kindManifestMap[gvk], manifest) - } - if scanner.Err() != nil { - return scanner.Err() + switch typeMeta.GroupVersionKind().Kind { + case "Role": + err = collection.addRoles(manifest) + case "ClusterRole": + err = collection.addClusterRoles(manifest) + case "Deployment": + err = collection.addDeployments(manifest) + case "CustomResourceDefinition": + // Skip for now and add explicitly from CRDsDir input. + default: + err = collection.addOthers(manifest) + } + if err != nil { + return err + } } + return scanner.Err() + }) + if err != nil { + return fmt.Errorf("failed to walk manifests directory for CSV updates: %v", err) } - return nil -} -func getOwnedCRDs(kindManifestMap map[schema.GroupVersionKind][][]byte) (map[schema.GroupVersionKind]struct{}, error) { - ownedCRDs := map[schema.GroupVersionKind]struct{}{} - for gvk, manifests := range kindManifestMap { - if gvk.Kind != "CustomResourceDefinition" { - continue + // Add CRDs from input. + crdsDir := g.Inputs[CRDsDirKey] + if _, err := os.Stat(crdsDir); err == nil || os.IsExist(err) { + collection.CustomResourceDefinitions, err = k8sutil.GetCustomResourceDefinitions(crdsDir) + if err != nil { + return err } - // Collect CRD kinds to filter them out from unsupported manifest types. - // The CRD version type doesn't matter as long as it has a group, kind, - // and versions in the expected fields. - for _, manifest := range manifests { - crd := v1beta1.CustomResourceDefinition{} - if err := yaml.Unmarshal(manifest, &crd); err != nil { - return ownedCRDs, err - } - for _, ver := range crd.Spec.Versions { - crGVK := schema.GroupVersionKind{ - Group: crd.Spec.Group, - Version: ver.Name, - Kind: crd.Spec.Names.Kind, - } - ownedCRDs[crGVK] = struct{}{} - } + } + + // Filter the collection based on data collected. + collection.filter() + + // Remove duplicate manifests. + if err = collection.deduplicate(); err != nil { + return fmt.Errorf("error removing duplicate manifests: %v", err) + } + + // Apply manifests to the CSV object. + if err = collection.apply(csv); err != nil { + return fmt.Errorf("error building CSV: %v", err) + } + + // Update descriptions from the APIs dir. + // FEAT(estroz): customresourcedefinition should not be updated for + // Ansible and Helm CSV's until annotated updates are implemented. + if projutil.IsOperatorGo() { + err = updateDescriptions(csv, g.Inputs[APIsDirKey]) + if err != nil { + return fmt.Errorf("error updating CSV customresourcedefinitions: %w", err) } } - return ownedCRDs, nil + + // Finally sort all updated fields. + sortUpdates(csv) + + return nil } diff --git a/internal/generate/olm-catalog/csv_go_test.go b/internal/generate/olm-catalog/csv_go_test.go index c139f957984..944e8e98ebe 100644 --- a/internal/generate/olm-catalog/csv_go_test.go +++ b/internal/generate/olm-catalog/csv_go_test.go @@ -24,16 +24,13 @@ import ( "testing" gen "github.com/operator-framework/operator-sdk/internal/generate/gen" - "github.com/operator-framework/operator-sdk/internal/scaffold" "github.com/operator-framework/operator-sdk/internal/util/fileutil" internalk8sutil "github.com/operator-framework/operator-sdk/internal/util/k8sutil" "github.com/operator-framework/operator-sdk/internal/util/projutil" - "github.com/operator-framework/operator-sdk/pkg/k8sutil" "github.com/blang/semver" "github.com/ghodss/yaml" "github.com/stretchr/testify/assert" - appsv1 "k8s.io/api/apps/v1" ) const ( @@ -377,48 +374,3 @@ func TestUpdateVersion(t *testing.T) { t.Errorf("Wanted csv replaces %s, got %s", wantedReplaces, csv.Spec.Replaces) } } - -func TestSetAndCheckOLMNamespaces(t *testing.T) { - cleanupFunc := chDirWithCleanup(t, testGoDataDir) - defer cleanupFunc() - - depBytes, err := ioutil.ReadFile(filepath.Join(scaffold.DeployDir, "operator.yaml")) - if err != nil { - t.Fatalf("Failed to read Deployment bytes: %v", err) - } - - // The test operator.yaml doesn't have "olm.targetNamespaces", so first - // check that depHasOLMNamespaces() returns false. - dep := appsv1.Deployment{} - if err := yaml.Unmarshal(depBytes, &dep); err != nil { - t.Fatalf("Failed to unmarshal Deployment bytes: %v", err) - } - if depHasOLMNamespaces(dep) { - t.Error("Expected depHasOLMNamespaces to return false, got true") - } - - // Insert "olm.targetNamespaces" into WATCH_NAMESPACE and check that - // depHasOLMNamespaces() returns true. - setWatchNamespacesEnv(&dep) - if !depHasOLMNamespaces(dep) { - t.Error("Expected depHasOLMNamespaces to return true, got false") - } - - // Overwrite WATCH_NAMESPACE and check that depHasOLMNamespaces() returns - // false. - overwriteContainerEnvVar(&dep, k8sutil.WatchNamespaceEnvVar, newEnvVar("FOO", "bar")) - if depHasOLMNamespaces(dep) { - t.Error("Expected depHasOLMNamespaces to return false, got true") - } - - // Insert "olm.targetNamespaces" elsewhere in the deployment pod spec - // and check that depHasOLMNamespaces() returns true. - dep = appsv1.Deployment{} - if err := yaml.Unmarshal(depBytes, &dep); err != nil { - t.Fatalf("Failed to unmarshal Deployment bytes: %v", err) - } - dep.Spec.Template.ObjectMeta.Labels["namespace"] = olmTNMeta - if !depHasOLMNamespaces(dep) { - t.Error("Expected depHasOLMNamespaces to return true, got false") - } -} diff --git a/internal/generate/olm-catalog/csv_updaters.go b/internal/generate/olm-catalog/csv_updaters.go index 1f425360855..c1603c93bfb 100644 --- a/internal/generate/olm-catalog/csv_updaters.go +++ b/internal/generate/olm-catalog/csv_updaters.go @@ -16,6 +16,7 @@ package olmcatalog import ( "bytes" + "crypto/sha256" "encoding/json" goerrors "errors" "fmt" @@ -27,113 +28,179 @@ import ( "github.com/operator-framework/operator-sdk/pkg/k8sutil" "github.com/ghodss/yaml" - olmapiv1alpha1 "github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1" + operatorsv1alpha1 "github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1" log "github.com/sirupsen/logrus" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/version" ) -// csvUpdater is an interface for any data that can be in a CSV, which will be -// set to the corresponding field on apply(). -type csvUpdater interface { - // apply applies a data update to a CSV argument. - apply(*olmapiv1alpha1.ClusterServiceVersion) error +// manifestCollection holds a collection of all manifests relevant to CSV updates. +type manifestCollection struct { + Roles []rbacv1.Role + ClusterRoles []rbacv1.ClusterRole + Deployments []appsv1.Deployment + CustomResourceDefinitions []apiextv1beta1.CustomResourceDefinition + CustomResources []unstructured.Unstructured + Others []unstructured.Unstructured } -// Get install strategy from csv. -func getCSVInstallStrategy(csv *olmapiv1alpha1.ClusterServiceVersion) olmapiv1alpha1.NamedInstallStrategy { - // Default to a deployment strategy if none found. - if csv.Spec.InstallStrategy.StrategyName == "" { - csv.Spec.InstallStrategy.StrategyName = olmapiv1alpha1.InstallStrategyNameDeployment +// addRoles assumes add manifest data in rawManifests are Roles and adds them +// to the collection. +func (c *manifestCollection) addRoles(rawManifests ...[]byte) error { + for _, rawManifest := range rawManifests { + role := rbacv1.Role{} + if err := yaml.Unmarshal(rawManifest, &role); err != nil { + return fmt.Errorf("error adding Role to manifest collection: %v", err) + } + c.Roles = append(c.Roles, role) } - return csv.Spec.InstallStrategy + return nil } -type roles [][]byte - -var _ csvUpdater = roles{} +// addClusterRoles assumes add manifest data in rawManifests are ClusterRoles +// and adds them to the collection. +func (c *manifestCollection) addClusterRoles(rawManifests ...[]byte) error { + for _, rawManifest := range rawManifests { + role := rbacv1.ClusterRole{} + if err := yaml.Unmarshal(rawManifest, &role); err != nil { + return fmt.Errorf("error adding ClusterRole to manifest collection: %v", err) + } + c.ClusterRoles = append(c.ClusterRoles, role) + } + return nil +} -func (us roles) apply(csv *olmapiv1alpha1.ClusterServiceVersion) (err error) { - strategy := getCSVInstallStrategy(csv) - switch csv.Spec.InstallStrategy.StrategyName { - case olmapiv1alpha1.InstallStrategyNameDeployment: - perms := []olmapiv1alpha1.StrategyDeploymentPermissions{} - for _, u := range us { - role := rbacv1.Role{} - if err := yaml.Unmarshal(u, &role); err != nil { - return err - } - perms = append(perms, olmapiv1alpha1.StrategyDeploymentPermissions{ - ServiceAccountName: role.GetName(), - Rules: role.Rules, - }) +// addDeployments assumes add manifest data in rawManifests are Deployments +// and adds them to the collection. +func (c *manifestCollection) addDeployments(rawManifests ...[]byte) error { + for _, rawManifest := range rawManifests { + dep := appsv1.Deployment{} + if err := yaml.Unmarshal(rawManifest, &dep); err != nil { + return fmt.Errorf("error adding Deployment to manifest collection: %v", err) } - strategy.StrategySpec.Permissions = perms + c.Deployments = append(c.Deployments, dep) } - csv.Spec.InstallStrategy = strategy return nil } -type clusterRoles [][]byte +// addOthers assumes add manifest data in rawManifests are able to be +// unmarshalled into an Unstructured object and adds them to the collection. +func (c *manifestCollection) addOthers(rawManifests ...[]byte) error { + for _, rawManifest := range rawManifests { + u := unstructured.Unstructured{} + if err := yaml.Unmarshal(rawManifest, &u); err != nil { + return fmt.Errorf("error adding manifest collection: %v", err) + } + c.Others = append(c.Others, u) + } + return nil +} -var _ csvUpdater = clusterRoles{} +// filter applies filtering rules to certain manifest types in a collection. +func (c *manifestCollection) filter() { + c.filterCustomResources() +} -func (us clusterRoles) apply(csv *olmapiv1alpha1.ClusterServiceVersion) (err error) { - strategy := getCSVInstallStrategy(csv) - switch csv.Spec.InstallStrategy.StrategyName { - case olmapiv1alpha1.InstallStrategyNameDeployment: - perms := []olmapiv1alpha1.StrategyDeploymentPermissions{} - for _, u := range us { - clusterRole := rbacv1.ClusterRole{} - if err := yaml.Unmarshal(u, &clusterRole); err != nil { - return err +// filterCustomResources filters "other" objects, which contain likely +// Custom Resources corresponding to a CustomResourceDefinition, by GVK. +func (c *manifestCollection) filterCustomResources() { + crdGVKSet := make(map[schema.GroupVersionKind]struct{}) + for _, crd := range c.CustomResourceDefinitions { + for _, version := range crd.Spec.Versions { + gvk := schema.GroupVersionKind{ + Group: crd.Spec.Group, + Version: version.Name, + Kind: crd.Spec.Names.Kind, } - perms = append(perms, olmapiv1alpha1.StrategyDeploymentPermissions{ - ServiceAccountName: clusterRole.GetName(), - Rules: clusterRole.Rules, - }) + crdGVKSet[gvk] = struct{}{} } - strategy.StrategySpec.ClusterPermissions = perms + } + + customResources := []unstructured.Unstructured{} + for _, other := range c.Others { + if _, gvkMatches := crdGVKSet[other.GroupVersionKind()]; gvkMatches { + customResources = append(customResources, other) + } + } + c.CustomResources = customResources +} + +// apply applies the manifests in the collection to csv. +func (c manifestCollection) apply(csv *operatorsv1alpha1.ClusterServiceVersion) error { + strategy := getCSVInstallStrategy(csv) + switch strategy.StrategyName { + case operatorsv1alpha1.InstallStrategyNameDeployment: + c.applyRoles(&strategy.StrategySpec) + c.applyClusterRoles(&strategy.StrategySpec) + c.applyDeployments(&strategy.StrategySpec) } csv.Spec.InstallStrategy = strategy + + c.applyCustomResourceDefinitions(csv) + if err := c.applyCustomResources(csv); err != nil { + return fmt.Errorf("error applying Custom Resource: %v", err) + } return nil } -type deployments [][]byte +// Get install strategy from csv. +func getCSVInstallStrategy(csv *operatorsv1alpha1.ClusterServiceVersion) operatorsv1alpha1.NamedInstallStrategy { + // Default to a deployment strategy if none found. + if csv.Spec.InstallStrategy.StrategyName == "" { + csv.Spec.InstallStrategy.StrategyName = operatorsv1alpha1.InstallStrategyNameDeployment + } + return csv.Spec.InstallStrategy +} + +// applyRoles updates strategy's permissions with the Roles in the collection. +func (c manifestCollection) applyRoles(strategy *operatorsv1alpha1.StrategyDetailsDeployment) { + perms := []operatorsv1alpha1.StrategyDeploymentPermissions{} + for _, role := range c.Roles { + perms = append(perms, operatorsv1alpha1.StrategyDeploymentPermissions{ + ServiceAccountName: role.GetName(), + Rules: role.Rules, + }) + } + strategy.Permissions = perms +} -var _ csvUpdater = deployments{} +// applyClusterRoles updates strategy's cluserPermissions with the ClusterRoles +// in the collection. +func (c manifestCollection) applyClusterRoles(strategy *operatorsv1alpha1.StrategyDetailsDeployment) { + perms := []operatorsv1alpha1.StrategyDeploymentPermissions{} + for _, role := range c.ClusterRoles { + perms = append(perms, operatorsv1alpha1.StrategyDeploymentPermissions{ + ServiceAccountName: role.GetName(), + Rules: role.Rules, + }) + } + strategy.ClusterPermissions = perms +} -func (us deployments) apply(csv *olmapiv1alpha1.ClusterServiceVersion) (err error) { - strategy := getCSVInstallStrategy(csv) - switch csv.Spec.InstallStrategy.StrategyName { - case olmapiv1alpha1.InstallStrategyNameDeployment: - depSpecs := []olmapiv1alpha1.StrategyDeploymentSpec{} - for _, u := range us { - dep := appsv1.Deployment{} - if err := yaml.Unmarshal(u, &dep); err != nil { - return err - } - setWatchNamespacesEnv(&dep) - // Make sure "olm.targetNamespaces" is referenced somewhere in dep, - // and emit a warning of not. - if !depHasOLMNamespaces(dep) { - log.Warnf(`No WATCH_NAMESPACE environment variable nor reference to "%s"`+ - ` detected in operator Deployment. For OLM compatibility, your operator`+ - ` MUST watch namespaces defined in "%s"`, olmTNMeta, olmTNMeta) - } - depSpecs = append(depSpecs, olmapiv1alpha1.StrategyDeploymentSpec{ - Name: dep.GetName(), - Spec: dep.Spec, - }) +// applyDeployments updates strategy's deployments with the Deployments +// in the collection. +func (c manifestCollection) applyDeployments(strategy *operatorsv1alpha1.StrategyDetailsDeployment) { + depSpecs := []operatorsv1alpha1.StrategyDeploymentSpec{} + for _, dep := range c.Deployments { + setWatchNamespacesEnv(&dep) + // Make sure "olm.targetNamespaces" is referenced somewhere in dep, + // and emit a warning of not. + if !depHasOLMNamespaces(dep) { + log.Warnf(`No WATCH_NAMESPACE environment variable nor reference to "%s"`+ + ` detected in operator Deployment. For OLM compatibility, your operator`+ + ` MUST watch namespaces defined in "%s"`, olmTNMeta, olmTNMeta) } - strategy.StrategySpec.DeploymentSpecs = depSpecs + depSpecs = append(depSpecs, operatorsv1alpha1.StrategyDeploymentSpec{ + Name: dep.GetName(), + Spec: dep.Spec, + }) } - csv.Spec.InstallStrategy = strategy - return nil + strategy.DeploymentSpecs = depSpecs } const olmTNMeta = "metadata.annotations['olm.targetNamespaces']" @@ -180,29 +247,13 @@ func depHasOLMNamespaces(dep appsv1.Deployment) bool { return bytes.Contains(b, []byte(olmTNMeta)) } -type descSorter []olmapiv1alpha1.CRDDescription - -func (descs descSorter) Len() int { return len(descs) } -func (descs descSorter) Less(i, j int) bool { - if descs[i].Name == descs[j].Name { - if descs[i].Kind == descs[j].Kind { - return version.CompareKubeAwareVersionStrings(descs[i].Version, descs[j].Version) > 0 - } - return descs[i].Kind < descs[j].Kind - } - return descs[i].Name < descs[j].Name -} -func (descs descSorter) Swap(i, j int) { descs[i], descs[j] = descs[j], descs[i] } - -type crds [][]byte - -var _ csvUpdater = crds{} - -// apply updates csv's "owned" CRDDescriptions. "required" CRDDescriptions are -// left as-is, since they are user-defined values. -func (us crds) apply(csv *olmapiv1alpha1.ClusterServiceVersion) error { - ownedDescs := []olmapiv1alpha1.CRDDescription{} - descMap := map[registry.DefinitionKey]olmapiv1alpha1.CRDDescription{} +// applyCustomResourceDefinitions updates csv's customresourcedefinitions.owned +// with CustomResourceDefinitions in the collection. +// customresourcedefinitions.required are left as-is, since they are +// manually-defined values. +func (c manifestCollection) applyCustomResourceDefinitions(csv *operatorsv1alpha1.ClusterServiceVersion) { + ownedDescs := []operatorsv1alpha1.CRDDescription{} + descMap := map[registry.DefinitionKey]operatorsv1alpha1.CRDDescription{} for _, owned := range csv.Spec.CustomResourceDefinitions.Owned { defKey := registry.DefinitionKey{ Name: owned.Name, @@ -211,11 +262,7 @@ func (us crds) apply(csv *olmapiv1alpha1.ClusterServiceVersion) error { } descMap[defKey] = owned } - for _, u := range us { - crd := apiextv1beta1.CustomResourceDefinition{} - if err := yaml.Unmarshal(u, &crd); err != nil { - return err - } + for _, crd := range c.CustomResourceDefinitions { for _, ver := range crd.Spec.Versions { defKey := registry.DefinitionKey{ Name: crd.GetName(), @@ -225,7 +272,7 @@ func (us crds) apply(csv *olmapiv1alpha1.ClusterServiceVersion) error { if owned, ownedExists := descMap[defKey]; ownedExists { ownedDescs = append(ownedDescs, owned) } else { - ownedDescs = append(ownedDescs, olmapiv1alpha1.CRDDescription{ + ownedDescs = append(ownedDescs, operatorsv1alpha1.CRDDescription{ Name: defKey.Name, Version: defKey.Version, Kind: defKey.Kind, @@ -234,13 +281,13 @@ func (us crds) apply(csv *olmapiv1alpha1.ClusterServiceVersion) error { } } csv.Spec.CustomResourceDefinitions.Owned = ownedDescs - sort.Sort(descSorter(csv.Spec.CustomResourceDefinitions.Owned)) - sort.Sort(descSorter(csv.Spec.CustomResourceDefinitions.Required)) - return nil } -func updateDescriptions(csv *olmapiv1alpha1.ClusterServiceVersion, searchDir string) error { - updatedDescriptions := []olmapiv1alpha1.CRDDescription{} +// updateDescriptions parses APIs in apisDir for code and annotations that +// can build a verbose crdDescription and updates existing crdDescriptions in +// csv. If no code/annotations are found, the crdDescription is appended as-is. +func updateDescriptions(csv *operatorsv1alpha1.ClusterServiceVersion, apisDir string) error { + updatedDescriptions := []operatorsv1alpha1.CRDDescription{} for _, currDescription := range csv.Spec.CustomResourceDefinitions.Owned { group := currDescription.Name if split := strings.Split(currDescription.Name, "."); len(split) > 1 { @@ -252,7 +299,7 @@ func updateDescriptions(csv *olmapiv1alpha1.ClusterServiceVersion, searchDir str Version: currDescription.Version, Kind: currDescription.Kind, } - newDescription, err := descriptor.GetCRDDescriptionForGVK(searchDir, gvk) + newDescription, err := descriptor.GetCRDDescriptionForGVK(apisDir, gvk) if err != nil { if goerrors.Is(err, descriptor.ErrAPIDirNotExist) { log.Infof("Directory for API %s does not exist. Skipping CSV annotation parsing for API.", gvk) @@ -272,19 +319,15 @@ func updateDescriptions(csv *olmapiv1alpha1.ClusterServiceVersion, searchDir str } } csv.Spec.CustomResourceDefinitions.Owned = updatedDescriptions - sort.Sort(descSorter(csv.Spec.CustomResourceDefinitions.Owned)) - sort.Sort(descSorter(csv.Spec.CustomResourceDefinitions.Required)) return nil } -type crs [][]byte - -var _ csvUpdater = crs{} - -func (us crs) apply(csv *olmapiv1alpha1.ClusterServiceVersion) error { +// applyCustomResources updates csv's "alm-examples" annotation with the +// Custom Resources in the collection. +func (c manifestCollection) applyCustomResources(csv *operatorsv1alpha1.ClusterServiceVersion) error { examples := []json.RawMessage{} - for _, u := range us { - crBytes, err := yaml.YAMLToJSON(u) + for _, cr := range c.CustomResources { + crBytes, err := cr.MarshalJSON() if err != nil { return err } @@ -311,3 +354,125 @@ func prettifyJSON(b []byte) ([]byte, error) { err := json.Indent(&out, b, "", " ") return out.Bytes(), err } + +// deduplicate removes duplicate objects from the collection, since we are +// collecting an arbitrary list of manifests. +func (c *manifestCollection) deduplicate() error { + hashes := make(map[string]struct{}) + + roles := []rbacv1.Role{} + for _, role := range c.Roles { + hasHash, err := addToHashes(&role, hashes) + if err != nil { + return err + } + if !hasHash { + roles = append(roles, role) + } + } + c.Roles = roles + + clusterRoles := []rbacv1.ClusterRole{} + for _, clusterRole := range c.ClusterRoles { + hasHash, err := addToHashes(&clusterRole, hashes) + if err != nil { + return err + } + if !hasHash { + clusterRoles = append(clusterRoles, clusterRole) + } + } + c.ClusterRoles = clusterRoles + + deps := []appsv1.Deployment{} + for _, dep := range c.Deployments { + hasHash, err := addToHashes(&dep, hashes) + if err != nil { + return err + } + if !hasHash { + deps = append(deps, dep) + } + } + c.Deployments = deps + + crds := []apiextv1beta1.CustomResourceDefinition{} + for _, crd := range c.CustomResourceDefinitions { + hasHash, err := addToHashes(&crd, hashes) + if err != nil { + return err + } + if !hasHash { + crds = append(crds, crd) + } + } + c.CustomResourceDefinitions = crds + + crs := []unstructured.Unstructured{} + for _, cr := range c.CustomResources { + b, err := cr.MarshalJSON() + if err != nil { + return err + } + hash := hashContents(b) + if _, hasHash := hashes[hash]; !hasHash { + crs = append(crs, cr) + hashes[hash] = struct{}{} + } + } + c.CustomResources = crs + + return nil +} + +// marshaller is an interface used to generalize hashing for deduplication. +type marshaller interface { + Marshal() ([]byte, error) +} + +// addToHashes calls m.Marshal(), hashes the returned bytes, and adds the +// hash to hashes if it does not exist. addToHashes returns true if m's hash +// was not in hashes. +func addToHashes(m marshaller, hashes map[string]struct{}) (bool, error) { + b, err := m.Marshal() + if err != nil { + return false, err + } + hash := hashContents(b) + _, hasHash := hashes[hash] + if !hasHash { + hashes[hash] = struct{}{} + } + return hasHash, nil +} + +// hashContents creates a sha256 md5 digest of b's bytes. +func hashContents(b []byte) string { + h := sha256.New() + _, _ = h.Write(b) + return string(h.Sum(nil)) +} + +// sortUpdates sorts all fields updated in csv. +// TODO(estroz): sort other modified fields. +func sortUpdates(csv *operatorsv1alpha1.ClusterServiceVersion) { + sort.Sort(descSorter(csv.Spec.CustomResourceDefinitions.Owned)) + sort.Sort(descSorter(csv.Spec.CustomResourceDefinitions.Required)) +} + +// descSorter sorts a set of crdDescriptions. +type descSorter []operatorsv1alpha1.CRDDescription + +var _ sort.Interface = descSorter{} + +func (descs descSorter) Len() int { return len(descs) } +func (descs descSorter) Less(i, j int) bool { + if descs[i].Name == descs[j].Name { + if descs[i].Kind == descs[j].Kind { + return version.CompareKubeAwareVersionStrings(descs[i].Version, descs[j].Version) > 0 + } + return descs[i].Kind < descs[j].Kind + } + return descs[i].Name < descs[j].Name +} +func (descs descSorter) Swap(i, j int) { descs[i], descs[j] = descs[j], descs[i] } diff --git a/internal/generate/olm-catalog/csv_updaters_test.go b/internal/generate/olm-catalog/csv_updaters_test.go new file mode 100644 index 00000000000..282ad1861e8 --- /dev/null +++ b/internal/generate/olm-catalog/csv_updaters_test.go @@ -0,0 +1,72 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package olmcatalog + +import ( + "io/ioutil" + "path/filepath" + "testing" + + "github.com/operator-framework/operator-sdk/internal/scaffold" + "github.com/operator-framework/operator-sdk/pkg/k8sutil" + + "github.com/ghodss/yaml" + appsv1 "k8s.io/api/apps/v1" +) + +func TestSetAndCheckOLMNamespaces(t *testing.T) { + cleanupFunc := chDirWithCleanup(t, testGoDataDir) + defer cleanupFunc() + + depBytes, err := ioutil.ReadFile(filepath.Join(scaffold.DeployDir, "operator.yaml")) + if err != nil { + t.Fatalf("Failed to read Deployment bytes: %v", err) + } + + // The test operator.yaml doesn't have "olm.targetNamespaces", so first + // check that depHasOLMNamespaces() returns false. + dep := appsv1.Deployment{} + if err := yaml.Unmarshal(depBytes, &dep); err != nil { + t.Fatalf("Failed to unmarshal Deployment bytes: %v", err) + } + if depHasOLMNamespaces(dep) { + t.Error("Expected depHasOLMNamespaces to return false, got true") + } + + // Insert "olm.targetNamespaces" into WATCH_NAMESPACE and check that + // depHasOLMNamespaces() returns true. + setWatchNamespacesEnv(&dep) + if !depHasOLMNamespaces(dep) { + t.Error("Expected depHasOLMNamespaces to return true, got false") + } + + // Overwrite WATCH_NAMESPACE and check that depHasOLMNamespaces() returns + // false. + overwriteContainerEnvVar(&dep, k8sutil.WatchNamespaceEnvVar, newEnvVar("FOO", "bar")) + if depHasOLMNamespaces(dep) { + t.Error("Expected depHasOLMNamespaces to return false, got true") + } + + // Insert "olm.targetNamespaces" elsewhere in the deployment pod spec + // and check that depHasOLMNamespaces() returns true. + dep = appsv1.Deployment{} + if err := yaml.Unmarshal(depBytes, &dep); err != nil { + t.Fatalf("Failed to unmarshal Deployment bytes: %v", err) + } + dep.Spec.Template.ObjectMeta.Labels["namespace"] = olmTNMeta + if !depHasOLMNamespaces(dep) { + t.Error("Expected depHasOLMNamespaces to return true, got false") + } +} diff --git a/internal/generate/testdata/non-standard-layout/main.go b/internal/generate/testdata/non-standard-layout/main.go new file mode 100644 index 00000000000..38dd16da61a --- /dev/null +++ b/internal/generate/testdata/non-standard-layout/main.go @@ -0,0 +1,3 @@ +package main + +func main() {} diff --git a/internal/scaffold/olm-catalog/testdata/deploy/crds/app.example.com_appservices2_crd.yaml b/internal/scaffold/olm-catalog/testdata/deploy/crds/app.example.com_appservices2_crd.yaml deleted file mode 100644 index ec850024040..00000000000 --- a/internal/scaffold/olm-catalog/testdata/deploy/crds/app.example.com_appservices2_crd.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: appservice2.example.com -spec: - group: app.example.com - names: - kind: AppService2 - listKind: AppService2List - plural: appservices2 - singular: appservice2 - scope: Namespaced - version: v1alpha2 - versions: - - name: v1alpha2 - served: true - storage: true diff --git a/internal/scaffold/olm-catalog/testdata/deploy/crds/app.example.com_appservices_crd.yaml b/internal/scaffold/olm-catalog/testdata/deploy/crds/app.example.com_appservices_crd.yaml deleted file mode 100644 index 3f52bf07d92..00000000000 --- a/internal/scaffold/olm-catalog/testdata/deploy/crds/app.example.com_appservices_crd.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: appservice.example.com -spec: - group: app.example.com - names: - kind: AppService - listKind: AppServiceList - plural: appservices - singular: appservice - scope: Namespaced - version: v1alpha1 - versions: - - name: v1alpha1 - served: true - storage: true diff --git a/internal/scaffold/olm-catalog/testdata/deploy/crds/app.example.com_v1alpha1_appservice_cr.yaml b/internal/scaffold/olm-catalog/testdata/deploy/crds/app.example.com_v1alpha1_appservice_cr.yaml deleted file mode 100644 index 1076adecba0..00000000000 --- a/internal/scaffold/olm-catalog/testdata/deploy/crds/app.example.com_v1alpha1_appservice_cr.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: app.example.com/v1alpha1 -kind: AppService -metadata: - name: example-app -spec: - # Add fields here - size: 3 diff --git a/internal/scaffold/olm-catalog/testdata/deploy/olm-catalog/app-operator/0.1.0/app-operator.v0.1.0.clusterserviceversion.yaml b/internal/scaffold/olm-catalog/testdata/deploy/olm-catalog/app-operator/0.1.0/app-operator.v0.1.0.clusterserviceversion.yaml deleted file mode 100644 index 37b76c1b74a..00000000000 --- a/internal/scaffold/olm-catalog/testdata/deploy/olm-catalog/app-operator/0.1.0/app-operator.v0.1.0.clusterserviceversion.yaml +++ /dev/null @@ -1,119 +0,0 @@ -apiVersion: operators.coreos.com/v1alpha1 -kind: ClusterServiceVersion -metadata: - annotations: - alm-examples: |- - [ - { - "apiVersion": "app.example.com/v1alpha1", - "kind": "AppService", - "metadata": { - "name": "example-app" - }, - "spec": { - "size": 3 - } - } - ] - capabilities: Basic Install - name: app-operator.v0.1.0 - namespace: placeholder -spec: - apiservicedefinitions: {} - customresourcedefinitions: - required: - - description: Represents a cluster of etcd nodes. - displayName: etcd Cluster - kind: EtcdCluster - name: etcdclusters.etcd.database.coreos.com - version: v1beta2 - description: Placeholder description - displayName: App Operator - icon: - - base64data: "" - mediatype: "" - install: - spec: - deployments: - - name: app-operator - spec: - replicas: 1 - selector: - matchLabels: - name: app-operator - strategy: {} - template: - metadata: - labels: - name: app-operator - spec: - containers: - - command: - - app-operator - env: - - name: WATCH_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.annotations['olm.targetNamespaces'] - - name: OPERATOR_NAME - value: app-operator - image: quay.io/example-inc/operator:v0.1.0 - imagePullPolicy: Always - name: app-operator - resources: {} - serviceAccountName: app-operator - permissions: - - rules: - - apiGroups: - - "" - resources: - - pods - - services - - endpoints - - persistentvolumeclaims - - events - - configmaps - - secrets - verbs: - - '*' - - apiGroups: - - apps - resources: - - deployments - - daemonsets - - replicasets - - statefulsets - verbs: - - '*' - - apiGroups: - - app.example.com - resources: - - '*' - verbs: - - '*' - - apiGroups: - - apps - resourceNames: - - app-operator - resources: - - deployments/finalizers - verbs: - - update - serviceAccountName: app-operator - strategy: deployment - installModes: - - supported: true - type: OwnNamespace - - supported: true - type: SingleNamespace - - supported: false - type: MultiNamespace - - supported: true - type: AllNamespace - keywords: - - "" - maintainers: - - {} - maturity: alpha - provider: {} - version: 0.1.0 diff --git a/internal/scaffold/olm-catalog/testdata/deploy/olm-catalog/app-operator/app-operator.package.yaml b/internal/scaffold/olm-catalog/testdata/deploy/olm-catalog/app-operator/app-operator.package.yaml deleted file mode 100644 index 01b3cc9a7bf..00000000000 --- a/internal/scaffold/olm-catalog/testdata/deploy/olm-catalog/app-operator/app-operator.package.yaml +++ /dev/null @@ -1,5 +0,0 @@ -channels: -- currentCSV: app-operator.v0.1.0 - name: beta -defaultChannel: beta -packageName: app-operator diff --git a/internal/scaffold/olm-catalog/testdata/deploy/olm-catalog/csv-config.yaml b/internal/scaffold/olm-catalog/testdata/deploy/olm-catalog/csv-config.yaml deleted file mode 100644 index c8281c290dc..00000000000 --- a/internal/scaffold/olm-catalog/testdata/deploy/olm-catalog/csv-config.yaml +++ /dev/null @@ -1,6 +0,0 @@ -operator-path: "testdata/deploy/operator.yaml" -crd-cr-paths: - - "testdata/deploy/crds" - - "testdata/deploy/crds/app.example.com_appservices2_crd.yaml" -role-paths: - - "testdata/deploy/role.yaml" diff --git a/internal/scaffold/olm-catalog/testdata/deploy/operator.yaml b/internal/scaffold/olm-catalog/testdata/deploy/operator.yaml deleted file mode 100644 index a663dccd433..00000000000 --- a/internal/scaffold/olm-catalog/testdata/deploy/operator.yaml +++ /dev/null @@ -1,28 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: app-operator -spec: - replicas: 1 - selector: - matchLabels: - name: app-operator - template: - metadata: - labels: - name: app-operator - spec: - serviceAccountName: app-operator - containers: - - name: app-operator - image: quay.io/example-inc/operator:v0.1.0 - command: - - app-operator - imagePullPolicy: Always - env: - - name: WATCH_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: OPERATOR_NAME - value: "app-operator" diff --git a/internal/scaffold/olm-catalog/testdata/deploy/role.yaml b/internal/scaffold/olm-catalog/testdata/deploy/role.yaml deleted file mode 100644 index 03a7418cfd9..00000000000 --- a/internal/scaffold/olm-catalog/testdata/deploy/role.yaml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: app-operator -rules: -- apiGroups: - - "" - resources: - - pods - - services - - endpoints - - persistentvolumeclaims - - events - - configmaps - - secrets - verbs: - - '*' -- apiGroups: - - apps - resources: - - deployments - - daemonsets - - replicasets - - statefulsets - verbs: - - '*' -- apiGroups: - - app.example.com - resources: - - '*' - verbs: - - '*' -- apiGroups: - - apps - resources: - - deployments/finalizers - resourceNames: - - app-operator - verbs: - - "update" -serviceAccountName: app-operator diff --git a/internal/scaffold/olm-catalog/testdata/deploy/role_binding.yaml b/internal/scaffold/olm-catalog/testdata/deploy/role_binding.yaml deleted file mode 100644 index 22a1c7d17c2..00000000000 --- a/internal/scaffold/olm-catalog/testdata/deploy/role_binding.yaml +++ /dev/null @@ -1,11 +0,0 @@ -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: app-operator -subjects: -- kind: ServiceAccount - name: app-operator -roleRef: - kind: Role - name: app-operator - apiGroup: rbac.authorization.k8s.io diff --git a/internal/scaffold/olm-catalog/testdata/deploy/service_account.yaml b/internal/scaffold/olm-catalog/testdata/deploy/service_account.yaml deleted file mode 100644 index 2ab14f471a5..00000000000 --- a/internal/scaffold/olm-catalog/testdata/deploy/service_account.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: app-operator diff --git a/internal/util/projutil/project_util.go b/internal/util/projutil/project_util.go index f1e0112deee..f1d5ec5e570 100644 --- a/internal/util/projutil/project_util.go +++ b/internal/util/projutil/project_util.go @@ -35,7 +35,8 @@ const ( SrcDir = "src" fsep = string(filepath.Separator) - mainFile = "cmd" + fsep + "manager" + fsep + "main.go" + mainFile = "main.go" + managerMainFile = "cmd" + fsep + "manager" + fsep + mainFile buildDockerfile = "build" + fsep + "Dockerfile" rolesDir = "roles" helmChartsDir = "helm-charts" @@ -98,7 +99,8 @@ func CheckGoProjectCmd(cmd *cobra.Command) error { if IsOperatorGo() { return nil } - return fmt.Errorf("'%s' can only be run for Go operators; %s does not exist", cmd.CommandPath(), mainFile) + return fmt.Errorf("'%s' can only be run for Go operators; %s or %s do not exist", + cmd.CommandPath(), managerMainFile, mainFile) } func MustGetwd() string { @@ -195,18 +197,23 @@ func GetOperatorType() OperatorType { } func IsOperatorGo() bool { - _, err := os.Stat(mainFile) - return err == nil + _, err := os.Stat(managerMainFile) + if err == nil || os.IsExist(err) { + return true + } + // Aware of an alternative location for main.go. + _, err = os.Stat(mainFile) + return err == nil || os.IsExist(err) } func IsOperatorAnsible() bool { stat, err := os.Stat(rolesDir) - return err == nil && stat.IsDir() + return (err == nil && stat.IsDir()) || os.IsExist(err) } func IsOperatorHelm() bool { stat, err := os.Stat(helmChartsDir) - return err == nil && stat.IsDir() + return (err == nil && stat.IsDir()) || os.IsExist(err) } // MustGetGopath gets GOPATH and ensures it is set and non-empty. If GOPATH