move to new directory

cloudnetwork/v1/manual-override-crd-manifests/cloudprivateipconfigs.cloud.network.openshift.io/AAA_ungated.yaml

@@ -3,9 +3,6 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     feature-gate.release.openshift.io/: "true"
-    include.release.openshift.io/ibm-cloud-managed-: ""
-    include.release.openshift.io/self-managed-high-availability-: ""
-    include.release.openshift.io/single-node-developer-: ""
   name: cloudprivateipconfigs.cloud.network.openshift.io
 spec:
   versions:

cloudnetwork/v1/001-cloudprivateipconfig.crd.yaml → cloudnetwork/v1/zz_generated.crd-manifests/001-cloudprivateipconfig.crd.yaml

@@ -4,6 +4,9 @@ metadata:
   annotations:
     api-approved.openshift.io: https://github.com/openshift/api/pull/859
     api.openshift.io/merged-by-featuregates: "true"
+    include.release.openshift.io/ibm-cloud-managed: "true"
+    include.release.openshift.io/self-managed-high-availability: "true"
+    include.release.openshift.io/single-node-developer: "true"
   name: cloudprivateipconfigs.cloud.network.openshift.io
 spec:
   group: cloud.network.openshift.io

config/v1/types_infrastructure_test.go

@@ -2,6 +2,7 @@ package v1
 
 import (
 	"io/ioutil"
+	"path"
 	"testing"
 
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -23,7 +24,7 @@ func TestInfrastructureStatusDefault(t *testing.T) {
 	filePaths := []string{infraCRDDefaultFilePath, infraCRDTestPreviewFilePath}
 
 	for _, filepath := range filePaths {
-		infraCRDBytes, err := ioutil.ReadFile(filepath)
+		infraCRDBytes, err := ioutil.ReadFile(path.Join("zz_generated.crd-manifests", filepath))
 		if err != nil {
 			t.Fatalf("failed to read infrastructure CRD file %q: %v", filepath, err)
 		}

hack/update-payload-crds.sh

@@ -3,13 +3,13 @@
 source "$(dirname "${BASH_SOURCE}")/lib/init.sh"
 
 crd_globs="\
-    config/v1/*_config-operator_*.crd*yaml\
-    quota/v1/*.crd*yaml\
-    security/v1/*.crd*yaml\
-    securityinternal/v1/*.crd*yaml\
-    authorization/v1/*.crd*yaml\
-    operator/v1alpha1/0000_10_config-operator_01_imagecontentsourcepolicy.crd*yaml\
-    operator/v1/0000_10_config-operator_*.yaml
+    config/v1/zz_generated.crd-manifests/*_config-operator_*.crd*yaml\
+    quota/v1/zz_generated.crd-manifests/*.crd*yaml\
+    security/v1/zz_generated.crd-manifests/*.crd*yaml\
+    securityinternal/v1/zz_generated.crd-manifests/*.crd*yaml\
+    authorization/v1/zz_generated.crd-manifests/*.crd*yaml\
+    operator/v1alpha1/zz_generated.crd-manifests/0000_10_config-operator_01_imagecontentsourcepolicy.crd*yaml\
+    operator/v1/zz_generated.crd-manifests/0000_10_config-operator_*.yaml
     "
 
 # To allow the crd_globs to be sourced in the verify script,

hack/verify-integration-tests.sh

@@ -14,7 +14,7 @@ fi
 validate_suite_files() {
   FOLDER=$1
 
-  for file in ${FOLDER}/*.yaml; do
+  for file in ${FOLDER}/zz_generated.crd-manifests/*.yaml; do
     if [ ! -f $file ]; then
       # It's likely the bash expansion didn't find any yaml files.
       continue
@@ -53,13 +53,13 @@ validate_suite_files() {
     SUITE_FILE=${FOLDER}/${FILE_BASE}.${SINGULAR}.testsuite.yaml
 
     if [ ! -f ${SUITE_FILE} ]; then
-      echo "No test suite file found for CRD ${file}"
+      echo "No test suite file found for CRD ${file}, expected ${SUITE_FILE}"
       exit 1
     fi
   done
 }
 
 for groupVersion in ${API_GROUP_VERSIONS}; do
-  echo "Validating integration tests for ${groupVersion}"
+  echo "Validating integration tests for ${groupVersion}/zz_generated.crd-manifests"
   validate_suite_files ${SCRIPT_ROOT}/${groupVersion}
 done

route/v1/manual-override-crd-manifests/routes.route.openshift.io/AAA_ungated.yaml

@@ -3,9 +3,6 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     feature-gate.release.openshift.io/: "true"
-    include.release.openshift.io/ibm-cloud-managed-: ""
-    include.release.openshift.io/self-managed-high-availability-: ""
-    include.release.openshift.io/single-node-developer-: ""
   name: routes.route.openshift.io
 spec:
   versions:

route/v1/route-CustomNoUpgrade.crd.yaml → route/v1/zz_generated.crd-manifests/route-CustomNoUpgrade.crd.yaml

@@ -4,6 +4,9 @@ metadata:
   annotations:
     api-approved.openshift.io: https://github.com/openshift/api/pull/1228
     api.openshift.io/merged-by-featuregates: "true"
+    include.release.openshift.io/ibm-cloud-managed: "true"
+    include.release.openshift.io/self-managed-high-availability: "true"
+    include.release.openshift.io/single-node-developer: "true"
     release.openshift.io/feature-set: CustomNoUpgrade
   name: routes.route.openshift.io
 spec:

route/v1/route-Default.crd.yaml → route/v1/zz_generated.crd-manifests/route-Default.crd.yaml

@@ -4,6 +4,9 @@ metadata:
   annotations:
     api-approved.openshift.io: https://github.com/openshift/api/pull/1228
     api.openshift.io/merged-by-featuregates: "true"
+    include.release.openshift.io/ibm-cloud-managed: "true"
+    include.release.openshift.io/self-managed-high-availability: "true"
+    include.release.openshift.io/single-node-developer: "true"
     release.openshift.io/feature-set: Default
   name: routes.route.openshift.io
 spec:

route/v1/route-TechPreviewNoUpgrade.crd.yaml → route/v1/zz_generated.crd-manifests/route-TechPreviewNoUpgrade.crd.yaml

@@ -4,6 +4,9 @@ metadata:
   annotations:
     api-approved.openshift.io: https://github.com/openshift/api/pull/1228
     api.openshift.io/merged-by-featuregates: "true"
+    include.release.openshift.io/ibm-cloud-managed: "true"
+    include.release.openshift.io/self-managed-high-availability: "true"
+    include.release.openshift.io/single-node-developer: "true"
     release.openshift.io/feature-set: TechPreviewNoUpgrade
   name: routes.route.openshift.io
 spec:

tests/generator.go

@@ -99,7 +99,8 @@ func setAbsolutePath(suitePath string, path *string) error {
 		return nil
 	}
 	dir := filepath.Dir(suitePath)
-	relPath := filepath.Join(dir, *path)
+	// account for the generated file move.
+	relPath := filepath.Join(dir, "zz_generated.crd-manifests", *path)
 
 	absPath, err := filepath.Abs(relPath)
 	if err != nil {
@@ -446,6 +447,6 @@ func getSuiteSpecTestVersion(suiteSpec SuiteSpec) (string, error) {
 
 	// When there's multiple versions we fall back to an educated guess based
 	// on the directory structure. Normally the folder name is the version.
-	packageDir := filepath.Dir(suiteSpec.CRD)
+	packageDir := filepath.Dir(filepath.Dir(suiteSpec.CRD))
 	return filepath.Base(packageDir), nil
 }

tools/codegen/pkg/manifestmerge/generator.go

@@ -141,6 +141,8 @@ func (g *generator) genGroupVersion(group string, version generation.APIVersionC
 
 		manualCRDOverridesPath := filepath.Join(versionPath, "manual-override-crd-manifests")
 		byFeatureGatePath := filepath.Join(versionPath, "zz_generated.featuregated-crd-manifests")
+		generatedOutputPath := filepath.Join(versionPath, "zz_generated.crd-manifests")
+
 		possibleResources, err := os.ReadDir(byFeatureGatePath)
 		if os.IsNotExist(err) {
 			continue
@@ -154,6 +156,8 @@ func (g *generator) genGroupVersion(group string, version generation.APIVersionC
 			}
 		}
 
+		allCRDsToRender := []crdForFeatureSet{}
+
 		for _, resourcePath := range resourcePaths {
 			// at this point we have a few paths.  In the end, we want to generate manifests for every (crd, clusterprofile, featureset) tuple.
 			// a prefix can be specified to associate with an image that creates these, but we create one file each.
@@ -258,32 +262,70 @@ func (g *generator) genGroupVersion(group string, version generation.APIVersionC
 			}
 
 			// check to see if all the resultingCRDs are the same
-			crdsToRender := getCRDsToRender(resultingCRDs, crdFilenamePattern, versionPath)
+			crdsToRender := getCRDsToRender(resultingCRDs, crdFilenamePattern, generatedOutputPath)
+			allCRDsToRender = append(allCRDsToRender, crdsToRender...)
+		}
+
+		if !g.verify {
+			if err := os.MkdirAll(generatedOutputPath, 0755); err != nil {
+				errs = append(errs, fmt.Errorf("failed creating directory: %w", err))
+				continue
+			}
+		}
+
+		for _, resultingCRD := range allCRDsToRender {
+			manifestData, err := kyaml.Marshal(resultingCRD.crd.Object)
+			if err != nil {
+				errs = append(errs, fmt.Errorf("could not encode file %s: %v", resultingCRD.outputFile, err))
+				continue
+			}
 
-			for _, resultingCRD := range crdsToRender {
-				manifestData, err := kyaml.Marshal(resultingCRD.crd.Object)
+			if g.verify {
+				existingBytes, err := os.ReadFile(resultingCRD.outputFile)
 				if err != nil {
-					errs = append(errs, fmt.Errorf("could not encode file %s: %v", resultingCRD.outputFile, err))
+					errs = append(errs, fmt.Errorf("could not read file %s: %v", resultingCRD.outputFile, err))
 					continue
 				}
+				if !bytes.Equal(manifestData, existingBytes) {
+					diff := utils.Diff(existingBytes, manifestData, resultingCRD.outputFile)
 
-				if g.verify {
-					existingBytes, err := os.ReadFile(resultingCRD.outputFile)
-					if err != nil {
-						errs = append(errs, fmt.Errorf("could not read file %s: %v", resultingCRD.outputFile, err))
-						continue
-					}
-					if !bytes.Equal(manifestData, existingBytes) {
-						diff := utils.Diff(existingBytes, manifestData, resultingCRD.outputFile)
+					return fmt.Errorf("API schema for %s is out of date, please regenerate the API schema:\n%s", resultingCRD.outputFile, diff)
+				}
 
-						return fmt.Errorf("API schema for %s is out of date, please regenerate the API schema:\n%s", resultingCRD.outputFile, diff)
-					}
+				continue
+			}
 
-					continue
-				}
+			if err := os.WriteFile(resultingCRD.outputFile, manifestData, 0644); err != nil {
+				return fmt.Errorf("could not write manifest %s: %w", resultingCRD.outputFile, err)
+			}
+		}
 
-				if err := os.WriteFile(resultingCRD.outputFile, manifestData, 0644); err != nil {
-					return fmt.Errorf("could not write manifest %s: %w", resultingCRD.outputFile, err)
+		// remove extra content.
+		outputResources, err := os.ReadDir(generatedOutputPath)
+		switch {
+		case g.verify && os.IsNotExist(err):
+			// do nothing, may not be a failure if there's nothing to put here
+		case err != nil:
+			errs = append(errs, fmt.Errorf("failed to read generated output: %w", err))
+		}
+		for _, curr := range outputResources {
+			if curr.IsDir() {
+				errs = append(errs, fmt.Errorf("unexpected directory: %q", curr.Name()))
+			}
+			found := false
+			for _, expectedCRD := range allCRDsToRender {
+				filename := filepath.Base(expectedCRD.outputFile)
+				if curr.Name() == filename {
+					found = true
+					break
+				}
+			}
+			switch {
+			case !found && g.verify:
+				errs = append(errs, fmt.Errorf("need to remove: %q", curr.Name()))
+			case !found && !g.verify:
+				if err := os.Remove(filepath.Join(generatedOutputPath, curr.Name())); err != nil {
+					errs = append(errs, fmt.Errorf("failed to remove: %q", curr.Name()))
 				}
 			}
 		}
@@ -302,13 +344,13 @@ func hasClusterProfilePreference(annotations map[string]string) bool {
 	return false
 }
 
-func getCRDsToRender(resultingCRDs []crdForFeatureSet, crdFilenamePattern, versionPath string) []crdForFeatureSet {
+func getCRDsToRender(resultingCRDs []crdForFeatureSet, crdFilenamePattern, outputPath string) []crdForFeatureSet {
 	allCRDsWithData := filterCRDs(resultingCRDs, &HasData{})
 	sameSchemaInAllCRDs := areCRDsTheSame(allCRDsWithData)
 	hasAllFeatureSets := featureSetsFromCRDs(allCRDsWithData).Equal(sets.NewString(allFeatureSets...))
 	if sameSchemaInAllCRDs && hasAllFeatureSets {
 		crdFilename := strings.ReplaceAll(crdFilenamePattern, "MARKERS", "")
-		crdFullPath := filepath.Join(versionPath, crdFilename)
+		crdFullPath := filepath.Join(outputPath, crdFilename)
 		crdToWrite := allCRDsWithData[0].crd.DeepCopy()
 
 		clusterProfilesToAdd := clusterProfilesFromCRDs(allCRDsWithData)
@@ -363,7 +405,7 @@ func getCRDsToRender(resultingCRDs []crdForFeatureSet, crdFilenamePattern, versi
 			}
 
 			crdFilename := strings.ReplaceAll(crdFilenamePattern, "MARKERS", fmt.Sprintf("-%s", featureSet))
-			crdFullPath := filepath.Join(versionPath, crdFilename)
+			crdFullPath := filepath.Join(outputPath, crdFilename)
 			crdToWrite := filteredCRDs[0].crd.DeepCopy()
 
 			clusterProfilesToAdd := clusterProfilesFromCRDs(filteredCRDs)
@@ -402,7 +444,7 @@ func getCRDsToRender(resultingCRDs []crdForFeatureSet, crdFilenamePattern, versi
 		}
 
 		crdFilename := strings.ReplaceAll(crdFilenamePattern, "MARKERS", fmt.Sprintf("-%s", clusterProfileToShortName[clusterProfile]))
-		crdFullPath := filepath.Join(versionPath, crdFilename)
+		crdFullPath := filepath.Join(outputPath, crdFilename)
 		crdToWrite := filteredCRDs[0].crd.DeepCopy()
 
 		annotations := crdToWrite.GetAnnotations()
@@ -427,7 +469,7 @@ func getCRDsToRender(resultingCRDs []crdForFeatureSet, crdFilenamePattern, versi
 			continue
 		}
 		crdFilename := strings.ReplaceAll(crdFilenamePattern, "MARKERS", fmt.Sprintf("-%s-%s", clusterProfileToShortName[curr.clusterProfile], curr.featureSet))
-		crdFullPath := filepath.Join(versionPath, crdFilename)
+		crdFullPath := filepath.Join(outputPath, crdFilename)
 
 		crdToWrite := notHandled[i].crd.DeepCopy()
 		annotations := crdToWrite.GetAnnotations()