From 50f72e8c1a8d2150fb189319fc7f9ef2118dbca6 Mon Sep 17 00:00:00 2001 From: Daniel Hawton Date: Mon, 5 Jun 2023 09:30:35 -0600 Subject: [PATCH] CRD List Generation Running `make generate-crd-docs` in the root project or `make update-crdlist` in Documentation will run the crdlistgen tool. This tool rebuilds the Documentation/crdlist.rst file by looking at the pkg/k8s/apis/cilium.io/client package's CRD List. It will also scan the rst files of Documentation to see if the CRD is being used as a header anywhere to generate a link to it. Signed-off-by: Daniel Hawton --- .gitattributes | 1 + CODEOWNERS | 1 + Documentation/Makefile | 9 +- Documentation/check-crdlist.sh | 14 +++ Documentation/crdlist.rst | 17 +++ Documentation/internals/cilium_operator.rst | 16 +-- Makefile | 3 + pkg/k8s/apis/cilium.io/client/register.go | 110 +++++++++++++---- tools/crdlistgen/main.go | 124 ++++++++++++++++++++ 9 files changed, 254 insertions(+), 41 deletions(-) create mode 100755 Documentation/check-crdlist.sh create mode 100644 Documentation/crdlist.rst create mode 100644 tools/crdlistgen/main.go diff --git a/.gitattributes b/.gitattributes index 3b78eae5c9741..624b1563dad91 100644 --- a/.gitattributes +++ b/.gitattributes @@ -8,6 +8,7 @@ pkg/k8s/apis/cilium.io/v2/client/crds/*.yaml linguist-generated test/controlplane/**/v1.[0-9][0-9]/*.yaml linguist-generated test/controlplane/services/graceful-termination/*.yaml linguist-generated Documentation/cmdref/** linguist-generated +Documentation/crdlist.rst linguist-generated Documentation/helm-values.rst linguist-generated Documentation/codeowners.rst linguist-generated Documentation/_static/* -diff diff --git a/CODEOWNERS b/CODEOWNERS index 54421f310214a..ca254f33d9580 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -292,6 +292,7 @@ Makefile* @cilium/build /Documentation/conf.py @cilium/docs-structure /Documentation/configuration/index.rst @cilium/docs-structure /Documentation/contributing/ @cilium/contributing @cilium/docs-structure +/Documentation/crdlist.rst /Documentation/Dockerfile @cilium/docs-structure /Documentation/gettingstarted/demo.rst @cilium/docs-structure /Documentation/gettingstarted/gettinghelp.rst @cilium/contributing @cilium/docs-structure diff --git a/Documentation/Makefile b/Documentation/Makefile index 20ecf495647df..9abac9a0a8209 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -56,6 +56,11 @@ update-cmdref: builder-image cilium-build ## Update the command reference docume -$(QUIET)rm -rf cmdref/cilium*.md $(QUIET)$(DOCKER_RUN) ./update-cmdref.sh +.PHONY: update-crdlist +update-crdlist: + @$(ECHO_GEN)crdlist + make -C ../ generate-crd-docs + codeowners.rst: $(ROOT_DIR)/CODEOWNERS @$(ECHO_GEN)$@ $(QUIET)$(DOCKER_RUN) ./update-codeowners.sh @@ -63,7 +68,7 @@ codeowners.rst: $(ROOT_DIR)/CODEOWNERS .PHONY: update-codeowners update-codeowners: codeowners.rst -check: builder-image api-flaggen update-cmdref update-helm-values update-codeowners ## Validate command and Helm references, as well as policy examples. +check: builder-image api-flaggen update-cmdref update-crdlist update-helm-values update-codeowners ## Validate command and Helm references, as well as policy examples. @$(ECHO_CHECK) cmdref $(QUIET) ./check-cmdref.sh @$(ECHO_CHECK) $(HELM_VALUES) @@ -74,6 +79,8 @@ check: builder-image api-flaggen update-cmdref update-helm-values update-codeown $(QUIET) ./check-codeowners.sh @$(ECHO_CHECK) configuration/api-restrictions-table.rst $(QUIET) ./check-flaggen.sh + @$(ECHO_CHECK) crdlist.rst + $(QUIET) ./check-crdlist.sh ifeq ($(V),0) SPHINX_OPTS += -q diff --git a/Documentation/check-crdlist.sh b/Documentation/check-crdlist.sh new file mode 100755 index 0000000000000..438b8a0f3e399 --- /dev/null +++ b/Documentation/check-crdlist.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +target_file="${script_dir}/crdlist.rst" + +if ! git diff --quiet -- "$target_file" ; then + git --no-pager diff -- "$target_file" + echo "HINT: to fix this, run 'make -C Documentation update-crdlist'" + exit 1 +fi diff --git a/Documentation/crdlist.rst b/Documentation/crdlist.rst new file mode 100644 index 0000000000000..77e9ae5ac6378 --- /dev/null +++ b/Documentation/crdlist.rst @@ -0,0 +1,17 @@ +- CiliumBGPPeeringPolicy +- CiliumCIDRGroup +- CiliumClusterwideEnvoyConfig +- :ref:`CiliumClusterwideNetworkPolicy` +- CiliumEgressGatewayPolicy +- :ref:`CiliumEndpoint` +- :ref:`CiliumEndpointSlice` +- CiliumEnvoyConfig +- CiliumExternalWorkload +- CiliumIdentity +- CiliumL2AnnouncementPolicy +- CiliumLoadBalancerIPPool +- CiliumLocalRedirectPolicy +- :ref:`CiliumNetworkPolicy` +- CiliumNode +- CiliumNodeConfig +- CiliumPodIPPool diff --git a/Documentation/internals/cilium_operator.rst b/Documentation/internals/cilium_operator.rst index a4515e2d483af..32aa180309bf3 100644 --- a/Documentation/internals/cilium_operator.rst +++ b/Documentation/internals/cilium_operator.rst @@ -43,21 +43,7 @@ CRD Registration The default behavior of the Cilium Operator is to register the CRDs used by Cilium. The following custom resources are registered by the Cilium Operator: -- :ref:`CiliumNetworkPolicy` -- :ref:`CiliumClusterwideNetworkPolicy` -- :ref:`CiliumEndpoint ` -- CiliumNode -- CiliumExternalWorkload -- CiliumIdentity -- CiliumLocalRedirectPolicy -- CiliumEgressGatewayPolicy -- CiliumEndpointSlice -- CiliumClusterwideEnvoyConfig -- CiliumEnvoyConfig -- CiliumBGPPeeringPolicy -- CiliumLoadBalancerIPPool -- CiliumNodeConfig -- CiliumCIDRGroup +.. include:: ../crdlist.rst IPAM ~~~~ diff --git a/Makefile b/Makefile index 78ff83dab5290..7a21c650483ac 100644 --- a/Makefile +++ b/Makefile @@ -640,6 +640,9 @@ update-authors: ## Update AUTHORS file for Cilium repository. @contrib/scripts/extract_authors.sh >> AUTHORS @cat .authors.aux >> AUTHORS +generate-crd-docs: ## Generate CRD List for documentation + $(QUIET)$(GO) run ./tools/crdlistgen + test-docs: ## Build HTML documentation. $(MAKE) -C Documentation html diff --git a/pkg/k8s/apis/cilium.io/client/register.go b/pkg/k8s/apis/cilium.io/client/register.go index 367cf36e73458..f7eb19e1c3b9a 100644 --- a/pkg/k8s/apis/cilium.io/client/register.go +++ b/pkg/k8s/apis/cilium.io/client/register.go @@ -93,40 +93,100 @@ var ( comparableCRDSchemaVersion = versioncheck.MustVersion(k8sconst.CustomResourceDefinitionSchemaVersion) ) -type crdCreationFn func(clientset apiextensionsclient.Interface) error +type CRDList struct { + Name string + FullName string +} + +// Returns a map of CRDs +func CustomResourceDefinitionList() map[string]*CRDList { + return map[string]*CRDList{ + synced.CRDResourceName(k8sconstv2.CNPName): { + Name: CNPCRDName, + FullName: k8sconstv2.CNPName, + }, + synced.CRDResourceName(k8sconstv2.CCNPName): { + Name: CCNPCRDName, + FullName: k8sconstv2.CCNPName, + }, + synced.CRDResourceName(k8sconstv2.CNName): { + Name: CNCRDName, + FullName: k8sconstv2.CNName, + }, + synced.CRDResourceName(k8sconstv2.CIDName): { + Name: CIDCRDName, + FullName: k8sconstv2.CIDName, + }, + synced.CRDResourceName(k8sconstv2.CEPName): { + Name: CEPCRDName, + FullName: k8sconstv2.CEPName, + }, + synced.CRDResourceName(k8sconstv2.CEWName): { + Name: CEWCRDName, + FullName: k8sconstv2.CEWName, + }, + synced.CRDResourceName(k8sconstv2.CLRPName): { + Name: CLRPCRDName, + FullName: k8sconstv2.CLRPName, + }, + synced.CRDResourceName(k8sconstv2.CEGPName): { + Name: CEGPCRDName, + FullName: k8sconstv2.CEGPName, + }, + synced.CRDResourceName(k8sconstv2alpha1.CESName): { + Name: CESCRDName, + FullName: k8sconstv2alpha1.CESName, + }, + synced.CRDResourceName(k8sconstv2.CCECName): { + Name: CCECCRDName, + FullName: k8sconstv2.CCECName, + }, + synced.CRDResourceName(k8sconstv2.CECName): { + Name: CECCRDName, + FullName: k8sconstv2.CECName, + }, + synced.CRDResourceName(k8sconstv2alpha1.BGPPName): { + Name: BGPPCRDName, + FullName: k8sconstv2alpha1.BGPPName, + }, + synced.CRDResourceName(k8sconstv2alpha1.LBIPPoolName): { + Name: LBIPPoolCRDName, + FullName: k8sconstv2alpha1.LBIPPoolName, + }, + synced.CRDResourceName(k8sconstv2alpha1.CNCName): { + Name: CNCCRDName, + FullName: k8sconstv2alpha1.CNCName, + }, + synced.CRDResourceName(k8sconstv2alpha1.CCGName): { + Name: CCGCRDName, + FullName: k8sconstv2alpha1.CCGName, + }, + synced.CRDResourceName(k8sconstv2alpha1.L2AnnouncementName): { + Name: L2AnnouncementCRDName, + FullName: k8sconstv2alpha1.L2AnnouncementName, + }, + synced.CRDResourceName(k8sconstv2alpha1.CPIPName): { + Name: CPIPCRDName, + FullName: k8sconstv2alpha1.CPIPName, + }, + } +} // CreateCustomResourceDefinitions creates our CRD objects in the Kubernetes // cluster. func CreateCustomResourceDefinitions(clientset apiextensionsclient.Interface) error { g, _ := errgroup.WithContext(context.Background()) - resourceToCreateFnMapping := map[string]crdCreationFn{ - synced.CRDResourceName(k8sconstv2.CNPName): createCRD(CNPCRDName, k8sconstv2.CNPName), - synced.CRDResourceName(k8sconstv2.CCNPName): createCRD(CCNPCRDName, k8sconstv2.CCNPName), - synced.CRDResourceName(k8sconstv2.CNName): createCRD(CNCRDName, k8sconstv2.CNName), - synced.CRDResourceName(k8sconstv2.CIDName): createCRD(CIDCRDName, k8sconstv2.CIDName), - synced.CRDResourceName(k8sconstv2.CEPName): createCRD(CEPCRDName, k8sconstv2.CEPName), - synced.CRDResourceName(k8sconstv2.CEWName): createCRD(CEWCRDName, k8sconstv2.CEWName), - synced.CRDResourceName(k8sconstv2.CLRPName): createCRD(CLRPCRDName, k8sconstv2.CLRPName), - synced.CRDResourceName(k8sconstv2.CEGPName): createCRD(CEGPCRDName, k8sconstv2.CEGPName), - synced.CRDResourceName(k8sconstv2alpha1.CESName): createCRD(CESCRDName, k8sconstv2alpha1.CESName), - synced.CRDResourceName(k8sconstv2.CCECName): createCRD(CCECCRDName, k8sconstv2.CCECName), - synced.CRDResourceName(k8sconstv2.CECName): createCRD(CECCRDName, k8sconstv2.CECName), - synced.CRDResourceName(k8sconstv2alpha1.BGPPName): createCRD(BGPPCRDName, k8sconstv2alpha1.BGPPName), - synced.CRDResourceName(k8sconstv2alpha1.LBIPPoolName): createCRD(LBIPPoolCRDName, k8sconstv2alpha1.LBIPPoolName), - synced.CRDResourceName(k8sconstv2alpha1.CNCName): createCRD(CNCCRDName, k8sconstv2alpha1.CNCName), - synced.CRDResourceName(k8sconstv2alpha1.CCGName): createCRD(CCGCRDName, k8sconstv2alpha1.CCGName), - synced.CRDResourceName(k8sconstv2alpha1.L2AnnouncementName): createCRD(L2AnnouncementCRDName, k8sconstv2alpha1.L2AnnouncementName), - synced.CRDResourceName(k8sconstv2alpha1.CPIPName): createCRD(CPIPCRDName, k8sconstv2alpha1.CPIPName), - } + crds := CustomResourceDefinitionList() + for _, r := range synced.AllCiliumCRDResourceNames() { - fn, ok := resourceToCreateFnMapping[r] - if !ok { + if crd, ok := crds[r]; ok { + g.Go(func() error { + return createCRD(crd.Name, crd.FullName)(clientset) + }) + } else { log.Fatalf("Unknown resource %s. Please update pkg/k8s/apis/cilium.io/client to understand this type.", r) } - g.Go(func() error { - return fn(clientset) - }) } return g.Wait() diff --git a/tools/crdlistgen/main.go b/tools/crdlistgen/main.go new file mode 100644 index 0000000000000..ee90511b45870 --- /dev/null +++ b/tools/crdlistgen/main.go @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Authors of Cilium + +package main + +import ( + "bufio" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + + operatorServer "github.com/cilium/cilium/api/v1/operator/server" + "github.com/cilium/cilium/pkg/hive" + "github.com/cilium/cilium/pkg/hive/cell" + "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/client" +) + +var ( + MainCell = cell.Module( + "main", + "Main module for generating CRD Lists", + cell.Invoke(printCRDList), + ) + + Hive = hive.New( + operatorServer.SpecCell, + MainCell, + ) +) + +var ErrorBreakEarly = fmt.Errorf("break early") + +func printCRDList( + opSpec *operatorServer.Spec, + shutdown hive.Shutdowner, +) error { + list := client.CustomResourceDefinitionList() + + crdlist := []string{} + + for _, crd := range list { + crdlist = append(crdlist, cleanupCRDName(crd.Name)) + } + + // Sort the list + sort.Strings(crdlist) + + for idx, name := range crdlist { + // We need to walk ../../Documentation rst files to look and see if the CRD name is a header in the format of `.. _ :`, if so + // add `ref:` to the name so it will link to the CRD in the docs. + err := filepath.WalkDir("./Documentation", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + if filepath.Ext(path) == ".rst" { + match, err := grepFile(path, ".. _"+name+":") + if err != nil { + return err + } + // We can stop walking the documentation as we already know there is a match, so we send an ErrorBreakEarly and ignore that on the other side + // as WalkDir will keep running until it returns an error or there are no more files to walk. + if match { + // Change the name to a ref also specifically override the link text, as a couple headers add " CRD" to the text which causes the link to not be uniform. + crdlist[idx] = ":ref:`" + name + "<" + name + ">`" + return ErrorBreakEarly + } + } + + return nil + }) + if err != nil && !errors.Is(err, ErrorBreakEarly) { + return err + } + } + + f, err := os.Create("Documentation/crdlist.rst") + if err != nil { + return err + } + defer f.Close() + + for _, name := range crdlist { + _, err := f.WriteString(fmt.Sprintf("- %s\n", name)) + if err != nil { + return err + } + } + + shutdown.Shutdown() + return nil +} + +// Scan file for string +func grepFile(path, search string) (bool, error) { + //fmt.Printf("searching %s for %s\n", path, search) + f, err := os.Open(path) + if err != nil { + return false, err + } + defer f.Close() + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + if strings.Contains(scanner.Text(), search) { + return true, nil + } + } + + return false, scanner.Err() +} + +// Remove the /(version) portion from the CRD Name +func cleanupCRDName(name string) string { + return strings.Split(name, "/")[0] +} + +func main() { + Hive.Run() +}