CNTRLPLANE-3610: Decouple CAPI provider Go module dependencies

.gitignore

@@ -38,6 +38,15 @@ tools/bin
 /contrib/cleanroles/cleanroles
 /contrib/admission-tracer/admission-tracer
 
+# CAPI provider vendor directories (downloaded on demand by make capi-sync)
+hack/capi-vendor/*/vendor/
+
+# CAPI sync stamps (tracks whether pkg/capi types are up-to-date)
+pkg/capi/*/.synced
+
+# CAPI provider go.sum files (generated by Go tooling from go.mod)
+pkg/capi/*/go.sum
+
 # SNYK CLI cache data
 .dccache
 

.golangci.yml

@@ -42,6 +42,10 @@ linters:
       - linters:
           - unparam
         path: 'support/thirdparty/'
+      - linters:
+          - gocyclo
+          - unparam
+        path: 'hack/copy-capi-types/'
       - linters:
           - staticcheck
         text: 'SA1019: hcp.Status.Version is deprecated: Use versionStatus.desired.version instead.'
@@ -90,6 +94,7 @@ linters:
       - third_party$
       - builtin$
       - examples$
+      - pkg/capi/
 severity:
   default: error
 formatters:
@@ -115,3 +120,4 @@ formatters:
       - third_party$
       - builtin$
       - examples$
+      - pkg/capi/

DEVELOPMENT.md

@@ -65,7 +65,10 @@ make api                      # Regenerate all CRDs, deepcopy, clients
 make api-lint-fix             # Run API linter and auto-fix violations
 make generate                 # Run go generate (cleans stale *_mock.go files first)
 make clients                  # Update generated clients
-make update                   # Full update (api-deps, workspace-sync, deps, api, api-docs, clients, docs-aggregate)
+make update                   # Full update (api-deps, workspace-sync, capi-sync, deps, api, ...)
+make capi-sync                # Sync CAPI provider types (incremental)
+make capi-sync-force          # Force re-sync all CAPI providers
+make verify-capi-sync         # Verify CAPI sync is clean (CI target)
 ```
 
 ## Development Patterns
@@ -92,11 +95,37 @@ This repository contains **multiple Go modules**. The `api/` directory is a **se
 This means:
 
 - Edits to files under `api/` (e.g. `api/hypershift/v1beta1/`) are **not visible** to the main module until the vendored copy is updated.
-- After modifying any types, constants, or functions in `api/`, you **must** run `make update` to regenerate CRDs, revendor dependencies, and sync everything. `make update` runs the full sequence: `api-deps` → `workspace-sync` → `deps` → `api` → `api-docs` → `clients` → `docs-aggregate`. Without this, the main module build will fail with `undefined` errors for any new symbols added in `api/`.
+- After modifying any types, constants, or functions in `api/`, you **must** run `make update` to regenerate CRDs, revendor dependencies, and sync everything. `make update` runs the full sequence: `api-deps` → `workspace-sync` → `capi-sync` → `deps` → `api` → `api-docs` → `clients` → `docs-aggregate`. Without this, the main module build will fail with `undefined` errors for any new symbols added in `api/`.
 - **Do not modify `vendor/` directories directly.** The `vendor/` directories are managed by `go mod vendor` (via `make deps` and `make api-deps`). Always use `make update` to keep them in sync.
 - Running `go build ./...` or `go vet ./...` from the repository root will **not** compile the `api/` module — it is a separate module. To build/vet the API module, run commands from within the `api/` directory.
 - The `hack/workspace/` directory contains a Go workspace configuration (`go.work`) that can be used for local development across both modules.
 
+### CAPI Provider Types
+
+Upstream CAPI infrastructure provider types (e.g. `sigs.k8s.io/cluster-api-provider-aws/v2`) are **not vendored directly**. Instead, they are copied into local `pkg/capi/<provider>/` modules using an AST-based tool that strips declarations depending on banned imports. This decouples HyperShift's `go.mod` from the full transitive dependency tree of each CAPI provider.
+
+Each `pkg/capi/<provider>/` directory is a separate Go module with its own `go.mod`. The main module consumes them via `replace` directives in `go.mod`.
+
+Key commands:
+
+```bash
+make capi-sync                # Sync all CAPI provider types (incremental, stamp-based)
+make capi-sync-force          # Force re-sync all providers from scratch
+make cluster-api-provider-aws # Generate CRDs for a specific CAPI provider
+```
+
+The sync process (`hack/capi-sync-provider.sh`) for each provider:
+1. Downloads the upstream module into the Go module cache via `go mod download`
+2. Copies types from the cached module into `pkg/capi/<provider>/` using `hack/copy-capi-types/main.go`, which strips functions and types that reference banned imports (e.g. controller-runtime, cloud SDKs)
+3. Generates deepcopy functions using `controller-gen` inside a temporary Go workspace
+
+CRD generation (`hack/capi-workspace-run.sh`) similarly creates a temporary `go.work` on the fly to resolve types across the main module and `pkg/capi/<provider>` modules. No committed workspace file is required.
+
+To add a new CAPI provider or update an existing one:
+1. Create or update `hack/capi-vendor/<provider>/go.mod` with the desired upstream version
+2. Run `make capi-sync-force` to download the module and regenerate all local copies
+3. Run `make verify-capi-sync` to confirm the sync is clean
+
 ## Pre-PR Gate
 
 Run `make pre-commit` before submitting a PR. It executes the full sequence: build, e2e compile, verify (formatting, linting, gitlint), and unit tests. This is the single command that catches most CI failures locally.

Makefile

@@ -92,8 +92,10 @@ pre-commit: all verify test
 
 build: hypershift-operator control-plane-operator control-plane-pki-operator karpenter-operator hypershift product-cli
 
+CAPI_PROVIDERS := agent aws azure gcp ibmcloud kubevirt openstack
+
 .PHONY: update
-update: api-deps workspace-sync deps api api-docs clients docs-aggregate
+update: api-deps workspace-sync capi-sync deps api api-docs clients docs-aggregate
 
 GOLANGCI_LINT := $(abspath $(TOOLS_BIN_DIR)/golangci-lint)
 $(GOLANGCI_LINT): $(TOOLS_DIR)/go.mod # Build golangci-lint from tools folder.
@@ -152,7 +154,7 @@ verify-crd-schema: $(CRD_SCHEMA_CHECK) ## Verify CRD schemas for breaking change
 		--crd-dir=karpenter-operator/controllers/karpenter/assets/zz_generated.crd-manifests
 
 .PHONY: verify-parallel
-verify-parallel: verify-codespell verify-codecov verify-api-deps verify-crd-schema lint cpo-container-sync run-gitlint verify-docs-nav
+verify-parallel: verify-codespell verify-codecov verify-api-deps verify-crd-schema verify-capi-sync lint cpo-container-sync run-gitlint verify-docs-nav
 
 .PHONY: verify
 verify: generate update staticcheck fmt vet
@@ -290,12 +292,14 @@ cluster-api: $(CONTROLLER_GEN)
 	$(CONTROLLER_GEN) $(CRD_OPTIONS) paths="./vendor/sigs.k8s.io/cluster-api/api/ipam/..." output:crd:artifacts:config=cmd/install/assets/crds/cluster-api
 	$(CONTROLLER_GEN) $(CRD_OPTIONS) paths="./vendor/sigs.k8s.io/cluster-api/api/addons/..." output:crd:artifacts:config=cmd/install/assets/crds/cluster-api
 
+# CAPI provider CRD targets run controller-gen inside a temporary Go workspace
+# because each pkg/capi/<provider> is a separate Go module with a minimal go.mod.
+CAPI_CRD_GEN = $(DIR)/hack/capi-workspace-run.sh $(CONTROLLER_GEN) $(CRD_OPTIONS)
+
 .PHONY: cluster-api-provider-aws
 cluster-api-provider-aws: $(CONTROLLER_GEN)
 	rm -rf cmd/install/assets/crds/cluster-api-provider-aws/*.yaml
-	$(CONTROLLER_GEN) $(CRD_OPTIONS) paths="./vendor/sigs.k8s.io/cluster-api-provider-aws/v2/api/..." output:crd:artifacts:config=cmd/install/assets/crds/cluster-api-provider-aws
-	$(CONTROLLER_GEN) $(CRD_OPTIONS) paths="./vendor/sigs.k8s.io/cluster-api-provider-aws/v2/exp/api/..." output:crd:artifacts:config=cmd/install/assets/crds/cluster-api-provider-aws
-
+	$(CAPI_CRD_GEN) paths="sigs.k8s.io/cluster-api-provider-aws/v2/..." output:crd:artifacts:config=$(DIR)/cmd/install/assets/crds/cluster-api-provider-aws
 # remove ROSA CRDs
 	rm -rf cmd/install/assets/crds/cluster-api-provider-aws/infrastructure.cluster.x-k8s.io_rosa*.yaml
 # remove EKS CRDs
@@ -305,34 +309,34 @@ cluster-api-provider-aws: $(CONTROLLER_GEN)
 .PHONY: cluster-api-provider-gcp
 cluster-api-provider-gcp: $(CONTROLLER_GEN)
 	rm -rf cmd/install/assets/crds/cluster-api-provider-gcp/*.yaml
-	$(CONTROLLER_GEN) $(CRD_OPTIONS) paths="./vendor/sigs.k8s.io/cluster-api-provider-gcp/api/..." output:crd:artifacts:config=cmd/install/assets/crds/cluster-api-provider-gcp
+	$(CAPI_CRD_GEN) paths="sigs.k8s.io/cluster-api-provider-gcp/..." output:crd:artifacts:config=$(DIR)/cmd/install/assets/crds/cluster-api-provider-gcp
 
 .PHONY: cluster-api-provider-ibmcloud
 cluster-api-provider-ibmcloud: $(CONTROLLER_GEN)
 	rm -rf cmd/install/assets/crds/cluster-api-provider-ibmcloud/*.yaml
-	$(CONTROLLER_GEN) $(CRD_OPTIONS) paths="./vendor/sigs.k8s.io/cluster-api-provider-ibmcloud/api/..." output:crd:artifacts:config=cmd/install/assets/crds/cluster-api-provider-ibmcloud
+	$(CAPI_CRD_GEN) paths="sigs.k8s.io/cluster-api-provider-ibmcloud/..." output:crd:artifacts:config=$(DIR)/cmd/install/assets/crds/cluster-api-provider-ibmcloud
 
 .PHONY: cluster-api-provider-kubevirt
 cluster-api-provider-kubevirt: $(CONTROLLER_GEN)
 	rm -rf cmd/install/assets/crds/cluster-api-provider-kubevirt/*.yaml
-	$(CONTROLLER_GEN) $(CRD_OPTIONS) paths="./vendor/sigs.k8s.io/cluster-api-provider-kubevirt/api/v1alpha1" output:crd:artifacts:config=cmd/install/assets/crds/cluster-api-provider-kubevirt
+	$(CAPI_CRD_GEN) paths="sigs.k8s.io/cluster-api-provider-kubevirt/..." output:crd:artifacts:config=$(DIR)/cmd/install/assets/crds/cluster-api-provider-kubevirt
 
 .PHONY: cluster-api-provider-agent
 cluster-api-provider-agent: $(CONTROLLER_GEN)
 	rm -rf cmd/install/assets/crds/cluster-api-provider-agent/*.yaml
-	$(CONTROLLER_GEN) $(CRD_OPTIONS) paths="./vendor/github.com/openshift/cluster-api-provider-agent/api/..." output:crd:artifacts:config=cmd/install/assets/crds/cluster-api-provider-agent
+	$(CAPI_CRD_GEN) paths="github.com/openshift/cluster-api-provider-agent/api/..." output:crd:artifacts:config=$(DIR)/cmd/install/assets/crds/cluster-api-provider-agent
 
 .PHONY: cluster-api-provider-azure
 cluster-api-provider-azure: $(CONTROLLER_GEN)
 	rm -rf cmd/install/assets/crds/cluster-api-provider-azure/*.yaml
-	$(CONTROLLER_GEN) $(CRD_OPTIONS) paths="./vendor/sigs.k8s.io/cluster-api-provider-azure/api/..." output:crd:artifacts:config=cmd/install/assets/crds/cluster-api-provider-azure
+	$(CAPI_CRD_GEN) paths="sigs.k8s.io/cluster-api-provider-azure/..." output:crd:artifacts:config=$(DIR)/cmd/install/assets/crds/cluster-api-provider-azure
 # remove CAPZ managed CRDS
 	rm -rf cmd/install/assets/crds/cluster-api-provider-azure/infrastructure.cluster.x-k8s.io_azuremanaged*.yaml
 
 .PHONY: cluster-api-provider-openstack
 cluster-api-provider-openstack: $(CONTROLLER_GEN)
 	rm -rf cmd/install/assets/crds/cluster-api-provider-openstack/*.yaml
-	$(CONTROLLER_GEN) $(CRD_OPTIONS) paths="./vendor/sigs.k8s.io/cluster-api-provider-openstack/api/..." output:crd:artifacts:config=cmd/install/assets/crds/cluster-api-provider-openstack
+	$(CAPI_CRD_GEN) paths="sigs.k8s.io/cluster-api-provider-openstack/..." output:crd:artifacts:config=$(DIR)/cmd/install/assets/crds/cluster-api-provider-openstack
 	$(CONTROLLER_GEN) $(CRD_OPTIONS) paths="./vendor/github.com/k-orc/openstack-resource-controller/..." output:crd:artifacts:config=cmd/install/assets/crds/cluster-api-provider-openstack
 
 .PHONY: api-docs
@@ -559,6 +563,53 @@ api-deps:
 	  $(GO) mod verify && \
 	  $(GO) list -m -mod=readonly -json all > /dev/null
 
+CAPI_SYNC_STAMPS := $(foreach p,$(CAPI_PROVIDERS),$(DIR)/pkg/capi/$(p)/.synced)
+
+# Per-provider sync: re-run when any sync input changes.
+$(DIR)/pkg/capi/%/.synced: \
+	$(DIR)/hack/capi-vendor/%/go.mod \
+	$(DIR)/hack/capi-vendor/%/vendor_imports.go \
+	$(DIR)/hack/capi-sync-provider.sh \
+	$(DIR)/hack/copy-capi-types/main.go \
+	$(CONTROLLER_GEN)
+	@echo "Resolving CAPI provider deps: $*"
+	cd $(DIR)/hack/capi-vendor/$* && $(GO) mod tidy && $(GO) mod download
+	@echo "Syncing CAPI provider: $*"
+	CONTROLLER_GEN=$(CONTROLLER_GEN) $(DIR)/hack/capi-sync-provider.sh $*
+	@touch $@
+
+.PHONY: capi-deps
+capi-deps:
+	@for p in $(CAPI_PROVIDERS); do \
+	  echo "Resolving CAPI provider deps: $$p"; \
+	  (cd $(DIR)/hack/capi-vendor/$$p && $(GO) mod tidy && $(GO) mod download) || exit 1; \
+	done
+
+.PHONY: capi-sync
+capi-sync: $(CAPI_SYNC_STAMPS)
+
+# Per-provider convenience target: make capi-sync-aws, make capi-sync-azure, etc.
+.PHONY: $(foreach p,$(CAPI_PROVIDERS),capi-sync-$(p))
+$(foreach p,$(CAPI_PROVIDERS),capi-sync-$(p)): capi-sync-%:
+	@rm -f $(DIR)/pkg/capi/$*/.synced
+	@$(MAKE) $(DIR)/pkg/capi/$*/.synced
+
+.PHONY: capi-sync-force
+capi-sync-force:
+	@rm -f $(CAPI_SYNC_STAMPS)
+	@$(MAKE) capi-sync
+
+.PHONY: verify-capi-sync
+verify-capi-sync:
+	@rm -f $(CAPI_SYNC_STAMPS)
+	@$(MAKE) capi-sync
+	@if [ -n "$$(git diff --name-only -- pkg/capi/)" ] || [ -n "$$(git ls-files --others --exclude-standard -- pkg/capi/)" ]; then \
+	  echo "ERROR: pkg/capi/ is out of sync with hack/capi-vendor/. Run 'make capi-sync-force' and commit the result." >&2; \
+	  git diff --stat -- pkg/capi/; \
+	  git ls-files --others --exclude-standard -- pkg/capi/; \
+	  exit 1; \
+	fi
+
 .PHONY: workspace-sync
 workspace-sync:
 	cd hack/workspace && \
@@ -611,7 +662,7 @@ verify-docs-nav: $(PYYAML_STAMP) ## Verify docs nav entries are sorted alphabeti
 
 .PHONY: verify-codespell
 verify-codespell: codespell ## Verify codespell.
-	@$(CODESPELL) --count --ignore-words=./.codespellignore --skip="./hack/tools/bin/codespell_dist,./docs/site/*,./vendor/*,./api/vendor/*,./hack/tools/vendor/*,./api/hypershift/v1alpha1/*,./support/thirdparty/*,./docs/content/reference/*,./hack/tools/bin/*,./cmd/install/assets/*,./go.sum,./api/go.sum,./hack/workspace/go.work.sum,./api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests,./hack/tools/go.mod,./hack/tools/go.sum,./karpenter-operator/controllers/karpenter/assets/*.yaml,./dev/*"
+	@$(CODESPELL) --count --ignore-words=./.codespellignore --skip="./hack/tools/bin/codespell_dist,./docs/site/*,./vendor/*,./api/vendor/*,./hack/tools/vendor/*,./api/hypershift/v1alpha1/*,./support/thirdparty/*,./docs/content/reference/*,./hack/tools/bin/*,./cmd/install/assets/*,./go.sum,./api/go.sum,./hack/workspace/go.work.sum,./api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests,./hack/tools/go.mod,./hack/tools/go.sum,./karpenter-operator/controllers/karpenter/assets/*.yaml,./dev/*,./pkg/capi/*,./hack/capi-vendor/*"
 
 .PHONY: verify-api-deps
 verify-api-deps: $(VERIFY_API_DEPS) ## Verify API dependencies against allowlist.

cmd/install/assets/crds/assets.go

@@ -81,7 +81,6 @@ var capiResources = map[string]string{
 	"cluster-api-provider-openstack/infrastructure.cluster.x-k8s.io_openstackmachines.yaml":                       "v1beta1",
 	"cluster-api-provider-openstack/infrastructure.cluster.x-k8s.io_openstackmachinetemplates.yaml":               "v1beta1",
 	"cluster-api-provider-openstack/infrastructure.cluster.x-k8s.io_openstackfloatingippools.yaml":                "v1alpha1",
-	"cluster-api-provider-openstack/infrastructure.cluster.x-k8s.io_openstackclusteridentities.yaml":              "v1alpha1",
 	"cluster-api-provider-openstack/infrastructure.cluster.x-k8s.io_openstackservers.yaml":                        "v1alpha1",
 }