diff --git a/.github/workflows/brokenlinks.yml b/.github/workflows/brokenlinks.yml index 9cfe55cf5..ce0e0addf 100644 --- a/.github/workflows/brokenlinks.yml +++ b/.github/workflows/brokenlinks.yml @@ -5,6 +5,7 @@ on: schedule: # Run this job every sunday at midnight - cron: "0 0 * * 0" + workflow_dispatch: permissions: {} @@ -20,11 +21,11 @@ jobs: # Checks the status of hyperlinks in .md files - name: Check links - uses: gaurav-nelson/github-action-markdown-link-check@v1 + uses: tcort/github-action-markdown-link-check@v1 with: use-verbose-mode: 'yes' config-file: ".mdlinkcheck.json" - folder-path: "docs/common, docs/multicluster" + folder-path: "docs/addons, docs/common, docs/deployment-models, docs/dual-stack, docs/general, docs/guidelines, docs/update-strategy" - name: Raise an Issue to report broken links if: ${{ failure() }} diff --git a/.github/workflows/nightly-images.yaml b/.github/workflows/nightly-images.yaml index fb001bef8..5817e12fd 100644 --- a/.github/workflows/nightly-images.yaml +++ b/.github/workflows/nightly-images.yaml @@ -3,7 +3,6 @@ name: Nightly image build workflow on: schedule: - cron: "0 3 * * *" # everyday at 3AM UTC - main branch - - cron: "0 2 * * *" # everyday at 2AM UTC - release-1.0 branch run-name: nightly-images @@ -28,11 +27,6 @@ jobs: with: ref: main - - uses: actions/checkout@v4 - if: github.event.schedule == '0 2 * * *' - with: - ref: release-1.0 - - name: Build and push nightly operator image run: | make docker-buildx \ diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index da8135566..542726677 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -3,13 +3,9 @@ name: Release workflow on: workflow_dispatch: inputs: - release_version: - description: "Release version" + run_name: + description: "Run name - short description for the workflow run" required: true - bundle_channels: - description: "Bundle channels" - required: true - default: "stable,stable-3.0" is_draft_release: description: "Draft release" type: boolean @@ -21,12 +17,11 @@ on: required: false default: false -run-name: Release ${{ inputs.release_version }} +run-name: ${{ inputs.run_name }} env: GIT_USER: ${{ secrets.GIT_USER }} GITHUB_TOKEN: ${{ secrets.GIT_TOKEN }} - VERSION: ${{ inputs.release_version }} jobs: release: @@ -44,22 +39,18 @@ jobs: - name: Build and push operator image run: | - make docker-buildx \ - -e TAG=$VERSION + make docker-buildx - name: Generate bundle metadata run: | - make bundle \ - -e CHANNELS=$CHANNELS - env: - CHANNELS: ${{ inputs.bundle_channels }} + make bundle - name: Publish bundle in operatorhub.io + continue-on-error: true run: | make bundle-publish \ -e GIT_CONFIG_USER_NAME="$GIT_CONFIG_USER_NAME" \ -e GIT_CONFIG_USER_EMAIL="$GIT_CONFIG_USER_EMAIL" \ - -e OPERATOR_VERSION=$VERSION \ -e OPERATOR_HUB=community-operators \ -e OWNER=k8s-operatorhub \ -e FORK=maistra @@ -72,7 +63,6 @@ jobs: make bundle-publish \ -e GIT_CONFIG_USER_NAME="$GIT_CONFIG_USER_NAME" \ -e GIT_CONFIG_USER_EMAIL="$GIT_CONFIG_USER_EMAIL" \ - -e OPERATOR_VERSION=$VERSION \ -e OWNER=redhat-openshift-ecosystem \ -e FORK=maistra env: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 558373def..330156dca 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -40,4 +40,4 @@ The Sail Operator project has a weekly contributor call every Thursday at 4PM CE If you find a security issue in the Sail Operator project, please refer to the [Security Policy](https://github.com/istio-ecosystem/sail-operator/security/policy) for more information on how to report security issues. Please do not report security issues in the public GitHub repository. ## Writing Documentation -If you want to add new documentation or examples to our existing documentation please take a look at the [documentation guidelines](/docs/guidelines/guidelines.md) to help you get started. Take into account that we run automation test over the documentation examples, so please ensure that the examples are correct and follows the guidelines to ensure it will be properly tested. +If you want to add new documentation or examples to our existing documentation please take a look at the [documentation guidelines](docs/guidelines/guidelines.md) to help you get started. Take into account that we run automation test over the documentation examples, so please ensure that the examples are correct and follows the guidelines to ensure it will be properly tested. diff --git a/Dockerfile b/Dockerfile index 7f249db94..61017438e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM registry.access.redhat.com/ubi9/ubi-minimal:latest +FROM registry.access.redhat.com/ubi10/ubi-micro:latest ARG TARGETOS TARGETARCH diff --git a/ISSUE-MANAGEMENT.md b/ISSUE-MANAGEMENT.md index 4336d3dd3..351f47fc1 100644 --- a/ISSUE-MANAGEMENT.md +++ b/ISSUE-MANAGEMENT.md @@ -13,4 +13,4 @@ If you want to contribute to the Sail Operator project, please refer to the [CON For contributing to Istio, check out the [CONTRIBUTING.md](https://github.com/istio/community/blob/master/CONTRIBUTING.md) file, which contains all the information you need to get started. ## Did you find an Istio bug? -If you found a bug in Istio, please refer to the [Istio GitHub repository](https://github.com/istio-ecosystem/sail-operator/blob/main/CONTRIBUTING-SAIL-PROJECT.md) to check the guidelines on how to report a bug to the Sail team. \ No newline at end of file +If you found a bug in Istio, please refer to the [Istio GitHub repository](CONTRIBUTING.md) to check the guidelines on how to report a bug to the Sail team. diff --git a/Makefile.core.mk b/Makefile.core.mk index 29f1993bb..106df67e5 100644 --- a/Makefile.core.mk +++ b/Makefile.core.mk @@ -22,6 +22,16 @@ OLD_VARS := $(.VARIABLES) VERSION ?= 1.27.0 MINOR_VERSION := $(shell echo "${VERSION}" | cut -f1,2 -d'.') +# This version will be used to generate the OLM upgrade graph in the FBC as a version to be replaced by the new operator version defined in $VERSION. +# This applies for stable releases, for nightly releases we are getting previous version directly from the FBC. +# Currently we are pushing the operator to two operator hubs https://github.com/k8s-operatorhub/community-operators and +# https://github.com/redhat-openshift-ecosystem/community-operators-prod. Nightly builds go only to community-operators-prod which already +# supports FBC. FBC yaml files and kept in community-operators-prod repo and we only generate a PR with changes using make targets from this Makefile. +# There are also GH workflows defined to release nightly and stable operators. +# There is no need to define `replaces` and `skipRange` fields in the CSV as those fields are defined in the FBC and CSV values are ignored. +# FBC is source of truth for OLM upgrade graph. +PREVIOUS_VERSION ?= 1.26.0 + OPERATOR_NAME ?= sailoperator VERSIONS_YAML_DIR ?= pkg/istioversion VERSIONS_YAML_FILE ?= versions.yaml @@ -90,6 +100,12 @@ DOCKER_BUILD_FLAGS ?= "--platform=$(TARGET_OS)/$(TARGET_ARCH)" GOTEST_FLAGS := $(if $(VERBOSE),-v) $(if $(COVERAGE),-coverprofile=$(REPO_ROOT)/out/coverage-unit.out) GINKGO_FLAGS ?= $(if $(VERBOSE),-v) $(if $(CI),--no-color) $(if $(COVERAGE),-coverprofile=coverage-integration.out -coverpkg=./... --output-dir=out) +# Fail fast when keeping the environment on failure, to make sure we don't contaminate it with other resources. Also make sure to skip cleanup so it won't be deleted. +ifeq ($(KEEP_ON_FAILURE),true) +GINKGO_FLAGS += --fail-fast +SKIP_CLEANUP = true +endif + # CHANNELS define the bundle channels used in the bundle. # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") # To re-generate a bundle for other specific channels without changing the standard setup, you can: @@ -192,33 +208,17 @@ test.e2e.kind: istioctl ## Deploy a KinD cluster and run the end-to-end tests ag .PHONY: test.e2e.describe test.e2e.describe: ## Runs ginkgo outline -format indent over the e2e test to show in BDD style the steps and test structure GINKGO_FLAGS="$(GINKGO_FLAGS)" ${SOURCE_DIR}/tests/e2e/common-operator-integ-suite.sh --describe -##@ Build - -.PHONY: runme $(RUNME) -runme: OS=$(shell go env GOOS) -runme: ARCH=$(shell go env GOARCH) -runme: $(RUNME) ## Download runme to bin directory. If wrong version is installed, it will be overwritten. - @test -s $(LOCALBIN)/runme || { \ - GOBIN=$(LOCALBIN) GO111MODULE=on go install github.com/runmedev/runme/v3@v$(RUNME_VERSION) > /dev/stderr; \ - echo "runme has been downloaded and placed in $(LOCALBIN)"; \ - } - -.PHONY: update-docs-examples -update-docs-examples: ## Copy the documentation files and generate the resulting md files to be executed by the runme tool. - @echo "Executing copy script to generate the documentation examples md files" - @echo "The script will copy the files from the source folder to the destination folder and add the runme suffix to the file names" - @tests/documentation_tests/scripts/update-docs-examples.sh - @echo "Documentation examples updated successfully" - .PHONE: test.docs -test.docs: runme istioctl update-docs-examples +test.docs: runme istioctl ## Run the documentation examples tests. ## test.docs use runme to test the documentation examples. ## Check the specific documentation to understand the use of the tool - @echo "Running runme test on the documentation examples, the location of the tests is in the tests/documentation_test folder" + @echo "Running runme test on the documentation examples." @PATH=$(LOCALBIN):$$PATH tests/documentation_tests/scripts/run-docs-examples.sh @echo "Documentation examples tested successfully" +##@ Build + .PHONY: build build: build-$(TARGET_ARCH) ## Build the sail-operator binary. @@ -448,7 +448,7 @@ gen-charts: ## Pull charts from istio repository. gen: gen-all-except-bundle bundle ## Generate everything. .PHONY: gen-all-except-bundle -gen-all-except-bundle: operator-name operator-chart controller-gen gen-api gen-charts gen-manifests gen-code gen-api-docs github-workflow update-docs-examples mirror-licenses +gen-all-except-bundle: operator-name operator-chart controller-gen gen-api gen-charts gen-manifests gen-code gen-api-docs mirror-licenses .PHONY: gen-check gen-check: gen restore-manifest-dates check-clean-repo ## Verify that changes in generated resources have been checked in. @@ -493,9 +493,6 @@ operator-chart: sed -i -e "s|^\(image: \).*$$|\1${IMAGE}|g" \ -e "s/^\( version: \).*$$/\1${VERSION}/g" chart/values.yaml -github-workflow: - sed -i -e '1,/default:/ s/^\(.*default:\).*$$/\1 ${CHANNELS}/' .github/workflows/release.yaml - .PHONY: update-istio update-istio: ## Update the Istio commit hash in the 'latest' entry in versions.yaml to the latest commit in the branch. @hack/update-istio.sh @@ -523,6 +520,7 @@ GITLEAKS ?= $(LOCALBIN)/gitleaks OPM ?= $(LOCALBIN)/opm ISTIOCTL ?= $(LOCALBIN)/istioctl RUNME ?= $(LOCALBIN)/runme +MISSPELL ?= $(LOCALBIN)/misspell ## Tool Versions OPERATOR_SDK_VERSION ?= v1.39.2 @@ -534,6 +532,7 @@ OLM_VERSION ?= v0.31.0 GITLEAKS_VERSION ?= v8.26.0 ISTIOCTL_VERSION ?= 1.26.0 RUNME_VERSION ?= 3.13.2 +MISSPELL_VERSION ?= v0.3.4 # GENERATE_RELATED_IMAGES defines whether `spec.relatedImages` is going to be generated or not # To disable set flag to false @@ -586,6 +585,15 @@ $(ISTIOCTL): $(LOCALBIN) echo "istioctl has been downloaded and placed in $(LOCALBIN)"; \ } +.PHONY: runme $(RUNME) +runme: OS=$(shell go env GOOS) +runme: ARCH=$(shell go env GOARCH) +runme: $(RUNME) ## Download runme to bin directory. If wrong version is installed, it will be overwritten. + @test -s $(LOCALBIN)/runme || { \ + GOBIN=$(LOCALBIN) GO111MODULE=on go install github.com/runmedev/runme/v3@v$(RUNME_VERSION) > /dev/stderr; \ + echo "runme has been downloaded and placed in $(LOCALBIN)"; \ + } + .PHONY: controller-gen controller-gen: $(LOCALBIN) ## Download controller-gen to bin directory. If wrong version is installed, it will be overwritten. @test -s $(LOCALBIN)/controller-gen && $(LOCALBIN)/controller-gen --version | grep -q $(CONTROLLER_TOOLS_VERSION) || \ @@ -648,19 +656,24 @@ bundle-push: ## Push the bundle image. bundle-publish: ## Create a PR for publishing in OperatorHub. @export GIT_USER=$(GITHUB_USER); \ export GITHUB_TOKEN=$(GITHUB_TOKEN); \ - export OPERATOR_VERSION=$(OPERATOR_VERSION); \ + export OPERATOR_VERSION=$(VERSION); \ export OPERATOR_NAME=$(OPERATOR_NAME); \ + export CHANNELS="$(CHANNELS)"; \ + export PREVIOUS_VERSION=$(PREVIOUS_VERSION); \ ./hack/operatorhub/publish-bundle.sh +## Generate nightly bundle. .PHONY: bundle-nightly -bundle-nightly: VERSION:=$(VERSION)-nightly-$(TODAY) ## Generate nightly bundle. +bundle-nightly: VERSION:=$(VERSION)-nightly-$(TODAY) bundle-nightly: CHANNELS:=$(MINOR_VERSION)-nightly bundle-nightly: TAG=$(MINOR_VERSION)-nightly-$(TODAY) bundle-nightly: bundle +## Publish nightly bundle. .PHONY: bundle-publish-nightly -bundle-publish-nightly: OPERATOR_VERSION=$(VERSION)-nightly-$(TODAY) ## Publish nightly bundle. +bundle-publish-nightly: VERSION:=$(VERSION)-nightly-$(TODAY) bundle-publish-nightly: TAG=$(MINOR_VERSION)-nightly-$(TODAY) +bundle-publish-nightly: CHANNELS:=$(MINOR_VERSION)-nightly bundle-publish-nightly: bundle-nightly bundle-publish .PHONY: helm-artifacts-publish @@ -720,8 +733,19 @@ lint-watches: ## Checks if the operator watches all resource kinds present in He lint-secrets: gitleaks ## Checks whether any secrets are present in the repository. @${GITLEAKS} detect --no-git --redact -v +.PHONY: lint-spell ## Run spell checker on the documentation files. Skipping sailoperator.io.md file. +lint-spell: misspell + @echo "Get misspell from $(LOCALBIN)" + @echo "Running misspell on the documentation files" + @find $(REPO_ROOT)/docs -type f \( \( -name "*.md" -o -name "*.asciidoc" \) ! -name "*sailoperator.io.md" \) \ + | xargs $(LOCALBIN)/misspell -error -locale US + +.PHONY: misspell +misspell: $(LOCALBIN) ## Download misspell to bin directory. + @test -s $(LOCALBIN)/misspell || GOBIN=$(LOCALBIN) go install github.com/client9/misspell/cmd/misspell@$(MISSPELL_VERSION) + .PHONY: lint -lint: lint-scripts lint-licenses lint-copyright-banner lint-go lint-yaml lint-helm lint-bundle lint-watches lint-secrets ## Run all linters. +lint: lint-scripts lint-licenses lint-copyright-banner lint-go lint-yaml lint-helm lint-bundle lint-watches lint-secrets lint-spell ## Run all linters. .PHONY: format format: format-go tidy-go ## Auto-format all code. This should be run before sending a PR. @@ -733,7 +757,7 @@ git-hook: gitleaks ## Installs gitleaks as a git pre-commit hook. chmod +x .git/hooks/pre-commit; \ fi -.SILENT: helm $(HELM) $(LOCALBIN) deploy-yaml gen-api operator-name operator-chart github-workflow +.SILENT: helm $(HELM) $(LOCALBIN) deploy-yaml gen-api operator-name operator-chart COMMON_IMPORTS ?= mirror-licenses dump-licenses lint-all lint-licenses lint-scripts lint-copyright-banner lint-go lint-yaml lint-helm format-go tidy-go check-clean-repo update-common .PHONY: $(COMMON_IMPORTS) diff --git a/README.md b/README.md index eb6c4982e..9fd9aa6fe 100644 --- a/README.md +++ b/README.md @@ -242,7 +242,7 @@ As you might know, the Sail Operator project serves as the community upstream fo #### versions.yaml -The name of the versions.yaml file can be overwritten using the VERSIONS_YAML_FILE environment variable. This way, downstream vendors can point to a custom list of supported versions. +The name of the versions.yaml file can be overwritten using the VERSIONS_YAML_FILE environment variable. This way, downstream vendors can point to a custom list of supported versions. Note that this file should be located in the `pkg/istioversion` directory, with the default value being `versions.yaml`. #### vendor_defaults.yaml diff --git a/bundle/README.md b/bundle/README.md index d16a6112d..a2bc28245 100644 --- a/bundle/README.md +++ b/bundle/README.md @@ -85,7 +85,7 @@ Repeat the process to create a project named `istio-cni`. The `version` field of the `Istio` and `IstioCNI` resource defines which version of each component should be deployed. This can be set using the `Istio Version` drop down menu when creating a new `Istio` with the OpenShift Container Platform -web console. For a list of available versions, see the [versions.yaml](/pkg/istioversion/versions.yaml) file +web console. For a list of available versions, see the [versions.yaml](../pkg/istioversion/versions.yaml) file or use the command: ```sh diff --git a/controllers/istiocni/istiocni_controller.go b/controllers/istiocni/istiocni_controller.go index 88a8ed073..81b7f6660 100644 --- a/controllers/istiocni/istiocni_controller.go +++ b/controllers/istiocni/istiocni_controller.go @@ -153,6 +153,12 @@ func (r *Reconciler) installHelmChart(ctx context.Context, cni *v1.IstioCNI) err // apply image digests from configuration, if not already set by user userValues = applyImageDigests(version, userValues, config.Config) + // apply vendor-specific default values + userValues, err = istiovalues.ApplyIstioCNIVendorDefaults(version, userValues) + if err != nil { + return fmt.Errorf("failed to apply vendor defaults: %w", err) + } + // apply userValues on top of defaultValues from profiles mergedHelmValues, err := istiovalues.ApplyProfilesAndPlatform( r.Config.ResourceDirectory, version, r.Config.Platform, r.Config.DefaultProfile, cni.Spec.Profile, helm.FromValues(userValues)) diff --git a/controllers/istiocni/istiocni_controller_test.go b/controllers/istiocni/istiocni_controller_test.go index 5836b4ae9..57d9ac09a 100644 --- a/controllers/istiocni/istiocni_controller_test.go +++ b/controllers/istiocni/istiocni_controller_test.go @@ -22,6 +22,7 @@ import ( "github.com/google/go-cmp/cmp" v1 "github.com/istio-ecosystem/sail-operator/api/v1" "github.com/istio-ecosystem/sail-operator/pkg/config" + "github.com/istio-ecosystem/sail-operator/pkg/istiovalues" "github.com/istio-ecosystem/sail-operator/pkg/istioversion" "github.com/istio-ecosystem/sail-operator/pkg/scheme" . "github.com/onsi/gomega" @@ -452,6 +453,153 @@ func TestApplyImageDigests(t *testing.T) { } } +func TestIstioCNIvendorDefaults(t *testing.T) { + tests := []struct { + name string + version string + userValues *v1.CNIValues + expected *v1.CNIValues + vendorDefaultsYAML string + expectError bool + }{ + { + name: "user values and no vendor defaults", + version: "v1.24.2", + userValues: &v1.CNIValues{ + Cni: &v1.CNIConfig{ + CniConfDir: StringPtr("example/path"), + }, + }, + vendorDefaultsYAML: ` +`, // No vendor defaults provided + expected: &v1.CNIValues{ + Cni: &v1.CNIConfig{ + CniConfDir: StringPtr("example/path"), + }, + }, + expectError: false, + }, + { + name: "vendor default not override user values", + version: "v1.24.2", + userValues: &v1.CNIValues{ + Cni: &v1.CNIConfig{ + CniConfDir: StringPtr("example/path"), + Image: StringPtr("custom/cni-image"), + }, + }, + vendorDefaultsYAML: ` +v1.24.2: + istiocni: + cni: + cniConfDir: example/path/vendor/path +`, // Vendor defaults provided but should not override user values + expected: &v1.CNIValues{ + Cni: &v1.CNIConfig{ + CniConfDir: StringPtr("example/path"), + Image: StringPtr("custom/cni-image"), + }, + }, + expectError: false, + }, + { + name: "merge vendor defaults with user values", + version: "v1.24.2", + userValues: &v1.CNIValues{ + Cni: &v1.CNIConfig{ + Image: StringPtr("custom/cni-image"), + }, + }, + vendorDefaultsYAML: ` +v1.24.2: + istiocni: + cni: + cniConfDir: example/path/vendor/path + image: vendor/cni-image +`, + expected: &v1.CNIValues{ + Cni: &v1.CNIConfig{ + CniConfDir: StringPtr("example/path/vendor/path"), + Image: StringPtr("custom/cni-image"), + }, + }, + expectError: false, + }, + { + name: "apply vendor defaults with no user values", + version: "v1.24.2", + userValues: &v1.CNIValues{}, + vendorDefaultsYAML: ` +v1.24.2: + istiocni: + cni: + cniConfDir: example/path/vendor/path + image: vendor/cni-image +`, + expected: &v1.CNIValues{ + Cni: &v1.CNIConfig{ + CniConfDir: StringPtr("example/path/vendor/path"), + Image: StringPtr("vendor/cni-image"), + }, + }, + expectError: false, + }, + { + name: "non existing field", + version: "v1.24.2", + userValues: &v1.CNIValues{}, + vendorDefaultsYAML: ` +v1.24.2: + istiocni: + cni: + nonExistingField: example/path/vendor/path +`, // A non-existing field in vendor defaults + expected: &v1.CNIValues{ + Cni: &v1.CNIConfig{}, + }, // Should not cause an error, but the field should not be present in the result + expectError: false, + }, + { + name: "malformed vendor defaults", + version: "v1.24.2", + userValues: &v1.CNIValues{}, + vendorDefaultsYAML: ` +v1.24.2: + istiocni: + cni: "" +`, // Malformed vendor defaults (cni should be a map, not a string) + expected: nil, // Expect an error due to malformed vendor defaults + expectError: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + vendorDefaults := istiovalues.MustParseVendorDefaultsYAML([]byte(tt.vendorDefaultsYAML)) + + // Apply vendor defaults + istiovalues.OverrideVendorDefaults(vendorDefaults) + result, err := istiovalues.ApplyIstioCNIVendorDefaults(tt.version, tt.userValues) + if tt.expectError { + g.Expect(err).To(HaveOccurred()) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + + // Check if the result matches the expected values + if diff := cmp.Diff(tt.expected, result); diff != "" { + t.Errorf("unexpected merge result; diff (-expected, +actual):\n%v", diff) + } + }) + } +} + +// StringPtr returns a pointer to a string literal. +func StringPtr(s string) *string { + return &s +} + func TestDetermineStatus(t *testing.T) { cfg := newReconcilerTestConfig(t) diff --git a/controllers/ztunnel/ztunnel_controller.go b/controllers/ztunnel/ztunnel_controller.go index 0d11591f9..0c3bd8fed 100644 --- a/controllers/ztunnel/ztunnel_controller.go +++ b/controllers/ztunnel/ztunnel_controller.go @@ -144,16 +144,7 @@ func (r *Reconciler) installHelmChart(ctx context.Context, ztunnel *v1alpha1.ZTu userValues := ztunnel.Spec.Values // apply image digests from configuration, if not already set by user - // TODO: Revisit once we support ImageOverrides for ztunnel - // userValues = applyImageDigests(ztunnel, userValues, config.Config) - - if userValues == nil { - userValues = &v1.ZTunnelValues{} - } - - if userValues.ZTunnel == nil { - userValues.ZTunnel = &v1.ZTunnelConfig{} - } + userValues = applyImageDigests(version, userValues, config.Config) // apply userValues on top of defaultValues from profiles mergedHelmValues, err := istiovalues.ApplyProfilesAndPlatform( diff --git a/controllers/ztunnel/ztunnel_controller_test.go b/controllers/ztunnel/ztunnel_controller_test.go index ae6fe3aeb..d973a82a1 100644 --- a/controllers/ztunnel/ztunnel_controller_test.go +++ b/controllers/ztunnel/ztunnel_controller_test.go @@ -339,8 +339,6 @@ func TestDetermineReadyCondition(t *testing.T) { } func TestApplyImageDigests(t *testing.T) { - t.Skip("https://github.com/istio-ecosystem/sail-operator/issues/581") - testCases := []struct { name string config config.OperatorConfig diff --git a/docs/README.md b/docs/README.md index 9f10efb2e..429cb1a31 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,11 +1,14 @@ [Return to Project Root](../) -*Note*: To add new topics to this documentation, please follow the guidelines in the [guidelines](../docs/guidelines/guidelines.md) doc. +*Note*: To add new topics to this documentation, please follow the guidelines in the [guidelines](../../docs/guidelines/guidelines.md) doc. # Table of Contents - [User Documentation](#user-documentation) - [Concepts](#concepts) - [Istio resource](#istio-resource) + - [Istiod in HA mode](general/istiod-ha.md#Running–Istiod-in-HA-mode) + - [Setting up Istiod in HA mode: using fixed replicas](general/istiod-ha.md#Setting-up-Istiod-in-HA-mode:-increasing-replicaCount) + - [Setting up Istiod in HA mode: using autoscaling](general/istiod-ha.md#Setting-up-Istiod-in-HA-mode:-using-autoscaling) - [IstioRevision resource](#istiorevision-resource) - [IstioRevisionTag resource](#istiorevisiontag-resource) - [IstioCNI resource](#istiocni-resource) @@ -13,58 +16,76 @@ - [Resource Status](#resource-status) - [InUse Detection](#inuse-detection) - [API Reference documentation](#api-reference-documentation) -- [Getting Started](#getting-started) - - [Installation on OpenShift](#installation-on-openshift) - - [Installing through the web console](#installing-through-the-web-console) - - [Installing using the CLI](#installing-using-the-cli) - - [Installation from Source](#installation-from-source) -- [Migrating from Istio in-cluster Operator](#migrating-from-istio-in-cluster-operator) -- [Gateways](#gateways) -- [Update Strategy](#update-strategy) - - [InPlace](#inplace) - - [Example using the InPlace strategy](#example-using-the-inplace-strategy) - - [RevisionBased](#revisionbased) - - [Example using the RevisionBased strategy](#example-using-the-revisionbased-strategy) - - [Example using the RevisionBased strategy and an IstioRevisionTag](#example-using-the-revisionbased-strategy-and-an-istiorevisiontag) -- [Multiple meshes on a single cluster](#multiple-meshes-on-a-single-cluster) - - [Prerequisites](#prerequisites) - - [Installation Steps](#installation-steps) - - [Deploying the control planes](#deploying-the-control-planes) - - [Deploying the applications](#deploying-the-applications) - - [Validation](#validation) - - [Checking application to control plane mapping](#checking-application-to-control-plane-mapping) - - [Checking application connectivity](#checking-application-connectivity) -- [Multi-cluster](#multi-cluster) - - [Prerequisites](#prerequisites) - - [Common Setup](#common-setup) - - [Multi-Primary](#multi-primary---multi-network) - - [Primary-Remote](#primary-remote---multi-network) - - [External Control Plane](#external-control-plane) -- [Dual-stack Support](#dual-stack-support) - - [Prerequisites](#prerequisites-1) - - [Installation Steps](#installation-steps) - - [Validation](#validation) -- [Ambient mode](common/istio-ambient-mode.md) +- [Getting Started](general/getting-started.md#getting-started) + - [Installation on OpenShift](general/getting-started.md#installation-on-openshift) + - [Installation from Source](general/getting-started.md#installation-from-source) + - [Migrating from Istio in-cluster Operator](general/getting-started.md#migrating-from-istio-in-cluster-operator) + - [Setting environments variables for Istiod](general/getting-started.md#setting-environments-variables-for-istiod) + - [Components field](general/getting-started.md#components-field) + - [CNI lifecycle management](general/getting-started.md#cni-lifecycle-management) + - [Converter Script to Migrate Istio in-cluster Operator Configuration to Sail Operator](general/getting-started.md#converter-script-to-migrate-istio-in-cluster-operator-configuration-to-sail-operator) +- [Creating and Configuring Gateways](common/create-and-configure-gateways.md#creating-and-configuring-gateways) + - [Option 1: Istio Gateway Injection](common/create-and-configure-gateways.md#option-1-istio-gateway-injection) + - [Option 2: Kubernetes Gateway API](common/create-and-configure-gateways.md#option-2-kubernetes-gateway-api) +- [Update Strategy](update-strategy/update-strategy.md#update-strategy) + - [InPlace](update-strategy/update-strategy.md#inplace) + - [Example using the InPlace strategy](update-strategy/update-strategy.md#example-using-the-inplace-strategy) + - [Recommendations for InPlace strategy](update-strategy/update-strategy.md#recommendations-for-inplace-strategy) + - [RevisionBased](update-strategy/update-strategy.md#revisionbased) + - [Example using the RevisionBased strategy](update-strategy/update-strategy.md#example-using-the-revisionbased-strategy) + - [Example using the RevisionBased strategy and an IstioRevisionTag](update-strategy/update-strategy.md#example-using-the-revisionbased-strategy-and-an-istiorevisiontag) +- [Multiple meshes on a single cluster](deployment-models/multiple-mesh.md#multiple-meshes-on-a-single-cluster) + - [Prerequisites](deployment-models/multiple-mesh.md#prerequisites) + - [Installation Steps](deployment-models/multiple-mesh.md#installation-steps) + - [Deploying the control planes](deployment-models/multiple-mesh.md#deploying-the-control-planes) + - [Deploying the applications](deployment-models/multiple-mesh.md#deploying-the-applications) + - [Validation](deployment-models/multiple-mesh.md#validation) + - [Checking application to control plane mapping](deployment-models/multiple-mesh.md#checking-application-to-control-plane-mapping) + - [Checking application connectivity](deployment-models/multiple-mesh.md#checking-application-connectivity) +- [Multi-cluster](deployment-models/multicluster.md#multi-cluster) + - [Prerequisites](deployment-models/multicluster.md#prerequisites) + - [Common Setup](deployment-models/multicluster.md#common-setup) + - [Multi-Primary](deployment-models/multicluster.md#multi-primary---multi-network) + - [Primary-Remote](deployment-models/multicluster.md#primary-remote---multi-network) + - [External Control Plane](deployment-models/multicluster.md#external-control-plane) +- [Dual-stack Support](dual-stack/dual-stack.md#dual-stack-support) + - [Prerequisites](dual-stack/dual-stack.md#prerequisites) + - [Installation Steps](dual-stack/dual-stack.md#installation-steps) + - [Validation](dual-stack/dual-stack.md#validation) +- [Introduction to Istio Ambient mode](common/istio-ambient-mode.md#introduction-to-istio-ambient-mode) + - [Component version](common/istio-ambient-mode.md#component-version) + - [Concepts](common/istio-ambient-mode.md#concepts) + - [ZTunnel resource](common/istio-ambient-mode.md#ztunnel-resource) + - [API Reference documentation](common/istio-ambient-mode.md#api-reference-documentation) + - [Core features](common/istio-ambient-mode.md#core-features) - [Getting Started](common/istio-ambient-mode.md#getting-started) - - [Use Waypoint Proxies](common/istio-ambient-waypoint.md) -- [Addons](#addons) - - [Deploy Prometheus and Jaeger addons](#deploy-prometheus-and-jaeger-addons) - - [Deploy Kiali addon](#deploy-kiali-addon) - - [Deploy Gateway and Bookinfo](#deploy-gateway-and-bookinfo) - - [Generate traffic and visualize your mesh](#generate-traffic-and-visualize-your-mesh) -- [Observability Integrations](#observability-integrations) - - [Scraping metrics using the OpenShift monitoring stack](#scraping-metrics-using-the-openshift-monitoring-stack) - - [Configure Tracing with OpenShift distributed tracing](#configure-tracing-with-openshift-distributed-tracing) - - [Integrating with Kiali](#integrating-with-kiali) - - [Integrating Kiali with the OpenShift monitoring stack](#integrating-kiali-with-the-openshift-monitoring-stack) - - [Integrating Kiali with OpenShift distributed tracing](#integrating-kiali-with-openshift-distributed-tracing) -- [Uninstalling](#uninstalling) - - [Deleting Istio](#deleting-istio) - - [Deleting IstioCNI](#deleting-istiocni) - - [Deleting the Sail Operator](#deleting-the-sail-operator) - - [Deleting the istio-system and istio-cni Projects](#deleting-the-istio-system-and-istiocni-projects) - - [Decide whether you want to delete the CRDs as well](#decide-whether-you-want-to-delete-the-crds-as-well) - + - [Visualize the application using Kiali dashboard](common/istio-ambient-mode.md#visualize-the-application-using-kiali-dashboard) + - [Troubleshoot issues](common/istio-ambient-mode.md#troubleshoot-issues) + - [Cleanup](common/istio-ambient-mode.md#cleanup) +- [Introduction to Istio Waypoint Proxy](common/istio-ambient-waypoint.md#introduction-to-istio-waypoint-proxy) + - [Core features](common/istio-ambient-waypoint.md#core-features) + - [Getting Started](common/istio-ambient-waypoint.md#getting-started) + - [Layer 7 Features in Ambient Mode](common/istio-ambient-waypoint.md#layer-7-features-in-ambient-mode) + - [Troubleshoot issues](common/istio-ambient-waypoint.md#troubleshoot-issues) + - [Cleanup](common/istio-ambient-waypoint.md#cleanup) +- [Addons](addons/addons.md#addons) + - [Deploy Prometheus and Jaeger addons](addons/addons.md#deploy-prometheus-and-jaeger-addons) + - [Deploy Kiali addon](addons/addons.md#deploy-kiali-addon) + - [Find the active revision of your Istio instance. In our case it is `test`.](addons/addons.md#find-the-active-revision-of-your-istio-instance) + - [Deploy Gateway and Bookinfo](addons/addons.md#deploy-gateway-and-bookinfo) + - [Generate traffic and visualize your mesh](addons/addons.md#generate-traffic-and-visualize-your-mesh) +- [Observability Integrations](addons/observability.md#observability-integrations) + - [Scraping metrics using the OpenShift monitoring stack](addons/observability.md#scraping-metrics-using-the-openshift-monitoring-stack) + - [Configure tracing with OpenShift distributed tracing](addons/observability.md#configure-tracing-with-openshift-distributed-tracing) + - [Integrating with Kiali](addons/observability.md#integrating-with-kiali) + - [Integrating Kiali with the OpenShift monitoring stack](addons/observability.md#integrating-kiali-with-the-openshift-monitoring-stack) + - [Integrating Kiali with OpenShift Distributed Tracing](addons/observability.md#integrating-kiali-with-openshift-distributed-tracing) +- [Uninstalling](general/getting-started.md#uninstalling) + - [Deleting Istio](general/getting-started.md#deleting-istio) + - [Deleting IstioCNI](general/getting-started.md#deleting-istiocni) + - [Deleting the Sail Operator](general/getting-started.md#deleting-the-sail-operator) + - [Deleting the istio-system and istio-cni Projects](general/getting-started.md#deleting-the-istio-system-and-istio-cni-projects) + - [Decide whether you want to delete the CRDs as well](general/getting-started.md#decide-whether-you-want-to-delete-the-crds-as-well) # User Documentation Sail Operator manages the lifecycle of your Istio control planes. Instead of creating a new configuration schema, Sail Operator APIs are built around Istio's helm chart APIs. All installation and configuration options that are exposed by Istio's helm charts are available through the Sail Operator CRDs' `values` fields. @@ -129,7 +150,8 @@ spec: As you can see in the YAML above, `IstioRevisionTag` really only has one field in its spec: `targetRef`. With this field, you can reference an `Istio` or `IstioRevision` resource. So after deploying this, you will be able to use both the `istio.io/rev=default` and also `istio-injection=enabled` labels to inject proxies into your workloads. The `istio-injection` label can only be used for revisions and revision tags named `default`, like the `IstioRevisionTag` in the above example. ### IstioCNI resource -The lifecycle of Istio's CNI plugin is managed separately when using Sail Operator. To install it, you can create an `IstioCNI` resource. The `IstioCNI` resource is a cluster-wide resource as it will install a `DaemonSet` that will be operating on all nodes of your cluster. You can select a version by setting the `spec.version` field, as you can see in the sample below: + +The lifecycle of Istio's CNI plugin is managed separately when using Sail Operator. To install it, you can create an `IstioCNI` resource. The `IstioCNI` resource is a cluster-wide resource as it will install a `DaemonSet` that will be operating on all nodes of your cluster. ```yaml apiVersion: sailoperator.io/v1 @@ -145,7 +167,8 @@ spec: - kube-system ``` -Note: If you need a specific Istio version, you can explicitly set it using `spec.version`. If not specified, the Operator will install the latest supported version. +> [!NOTE] +> If you need a specific Istio version, you can explicitly set it using `spec.version`. If not specified, the Operator will install the latest supported version. #### Updating the IstioCNI resource Updates for the `IstioCNI` resource are `Inplace` updates, this means that the `DaemonSet` will be updated with the new version of the CNI plugin once the resource is updated and the `istio-cni-node` pods are going to be replaced with the new version. @@ -169,11 +192,13 @@ To update the CNI plugin, just change the `version` field to the version you wan - kube-system EOF ``` - +``` +--> 2. Confirm the installation and version of the CNI plugin. ```console $ kubectl get istiocni -n istio-cni @@ -184,21 +209,25 @@ with_retries resource_version_equal "istiocni" "default" "v1.24.2" istio-cni-node-hd9zf 1/1 Running 0 90m ``` - +``` +--> 3. Update the CNI plugin version. ```bash { name=update-cni tag=cni-update} kubectl patch istiocni default -n istio-cni --type='merge' -p '{"spec":{"version":"v1.24.3"}}' ``` - +``` +--> 4. Confirm the CNI plugin version was updated. ```console @@ -210,10 +239,12 @@ wait_cni_ready "istio-cni" istio-cni-node-jz4lg 1/1 Running 0 44s ``` - +``` +--> > [!NOTE] > The CNI plugin at version `1.x` is compatible with `Istio` at version `1.x-1`, `1.x` and `1.x+1`. @@ -221,7 +252,7 @@ print_cni_info All of the Sail Operator API resources have a `status` subresource that contains information about their current state in the Kubernetes cluster. #### Conditions -All resources have a `Ready` condition which is set to `true` as soon as all child resource have been created and are deemed Ready by their respective controllers. To see additional conditions for each of the resources, check the [API reference documentation](https://github.com/istio-ecosystem/sail-operator/tree/main/docs/api-reference/sailoperator.io.md). +All resources have a `Ready` condition which is set to `true` as soon as all child resource have been created and are deemed Ready by their respective controllers. To see additional conditions for each of the resources, check the [API reference documentation](api-reference/sailoperator.io.md). #### InUse Detection The Sail Operator uses InUse detection to determine whether an object is referenced. This is currently present on all resources apart from `IstioCNI`. On the `Istio` resource, it is a counter as it only aggregates the `InUse` conditions on its child `IstioRevisions`. @@ -233,2332 +264,4 @@ The Sail Operator uses InUse detection to determine whether an object is referen |IstioRevisionTag |Condition |Status.Conditions[type="InUse']|Set to `true` if the `IstioRevisionTag` is referenced by a namespace or workload. ## API Reference documentation -The Sail Operator API reference documentation can be found [here](https://github.com/istio-ecosystem/sail-operator/tree/main/docs/api-reference/sailoperator.io.md). - -## Getting Started - -### Installation on OpenShift - -#### Installing through the web console - -1. In the OpenShift Console, navigate to the OperatorHub by clicking **Operator** -> **Operator Hub** in the left side-pane. - -1. Search for "sail". - -1. Locate the Sail Operator, and click to select it. - -1. When the prompt that discusses the community operator appears, click **Continue**, then click **Install**. - -1. Use the default installation settings presented, and click **Install** to continue. - -1. Click **Operators** -> **Installed Operators** to verify that the Sail Operator -is installed. `Succeeded` should appear in the **Status** column. - -#### Installing using the CLI - -*Prerequisites* - -* You have access to the cluster as a user with the `cluster-admin` cluster role. - -*Steps* - -1. Create the `openshift-operators` namespace (if it does not already exist). - - ```bash - kubectl create namespace openshift-operators - ``` - -1. Create the `Subscription` object with the desired `spec.channel`. - - ```bash - kubectl apply -f - <.enabled: true/false`. Other functionality exposed through `spec.components` like the k8s overlays is not currently available. - -### Converter Script -This script is used to convert an Istio in-cluster operator configuration to a Sail Operator configuration. Upon execution, the script takes an input YAML file and istio version and generates a sail operator configuration file. - -#### Usage -To run the configuration-converter.sh script, you need to provide four arguments, only input file is required other arguments are optional: - -1. Input file path (): The path to your Istio operator configuration YAML file (required). -2. Output file path (): The path where the converted Sail configuration will be saved. If not provided, the script will save the output with -sail.yaml appended to the input file name. -3. Namespace (-n ): The Kubernetes namespace for the Istio deployment. Defaults to istio-system if not provided. -4. Version (-v ): The version of Istio to be used. If not provided, the `spec.version` field will be omitted from the output file and the operator will deploy the latest version when the YAML manifest is applied. - -```bash -./tools/configuration-converter.sh [/path/to/output.yaml] [-n namespace] [-v version] -``` - -##### Sample command only with input file: - -```bash -./tools/configuration-converter.sh /home/user/istio_config.yaml -``` - -##### Sample command with custom output, namespace, and version: - -```bash -./tools/configuration-converter.sh /home/user/input/istio_config.yaml /home/user/output/output.yaml -n custom-namespace -v v1.24.3 -``` - -> [!WARNING] -> This script is still under development. -> Please verify the resulting configuration carefully after conversion to ensure it meets your expectations and requirements. - -### CNI - -The CNI plugin's lifecycle is managed separately from the control plane. You will have to create a [IstioCNI resource](#istiocni-resource) to use CNI. - -## Gateways - -[Gateways in Istio](https://istio.io/latest/docs/concepts/traffic-management/#gateways) are used to manage inbound and outbound traffic for the mesh. The Sail Operator does not deploy or manage Gateways. You can deploy a gateway either through [gateway-api](https://istio.io/latest/docs/tasks/traffic-management/ingress/gateway-api/) or through [gateway injection](https://istio.io/latest/docs/setup/additional-setup/gateway/#deploying-a-gateway). As you are following the gateway installation instructions, skip the step to install Istio since this is handled by the Sail Operator. - -**Note:** The `IstioOperator` / `istioctl` example is separate from the Sail Operator. Setting `spec.components` or `spec.values.gateways` on your Sail Operator `Istio` resource **will not work**. - -For examples installing Gateways on OpenShift, see the [Gateways](common/create-and-configure-gateways.md) page. - -## Update Strategy - -The Sail Operator supports two update strategies to update the version of the Istio control plane: `InPlace` and `RevisionBased`. The default strategy is `InPlace`. - -### InPlace -When the `InPlace` strategy is used, the existing Istio control plane is replaced with a new version. The workload sidecars immediately connect to the new control plane. The workloads therefore don't need to be moved from one control plane instance to another. - -#### Example using the InPlace strategy - -Prerequisites: -* Sail Operator is installed. -* `istioctl` is [installed](common/install-istioctl-tool.md). - -Steps: -1. Create the `istio-system` namespace. - - ```bash { name=create-istio-ns tag=inplace-update} - kubectl create namespace istio-system - ``` - -2. Create the `Istio` resource. - - ```bash { name=create-istio-resource tag=inplace-update} - cat < -3. Confirm the installation and version of the control plane. - - ```console - $ kubectl get istio -n istio-system - NAME REVISIONS READY IN USE ACTIVE REVISION STATUS VERSION AGE - default 1 1 0 default Healthy v1.25.3 23s - ``` - Note: `IN USE` field shows as 0, as `Istio` has just been installed and there are no workloads using it. - -4. Create namespace `bookinfo` and deploy bookinfo application. - - ```bash { name=deploy-bookinfo tag=inplace-update} - kubectl create namespace bookinfo - kubectl label namespace bookinfo istio-injection=enabled - kubectl apply -n bookinfo -f https://raw.githubusercontent.com/istio/istio/release-1.22/samples/bookinfo/platform/kube/bookinfo.yaml - ``` - Note: if the `Istio` resource name is other than `default`, you need to set the `istio.io/rev` label to the name of the `Istio` resource instead of adding the `istio-injection=enabled` label. - -5. Review the `Istio` resource after application deployment. - - ```console - $ kubectl get istio -n istio-system - NAME REVISIONS READY IN USE ACTIVE REVISION STATUS VERSION AGE - default 1 1 1 default Healthy v1.25.3 115s - ``` - Note: `IN USE` field shows as 1, as the namespace label and the injected proxies reference the IstioRevision. - -6. Perform the update of the control plane by changing the version in the Istio resource. - - ```bash - kubectl patch istio default -n istio-system --type='merge' -p '{"spec":{"version":"v1.26.0"}}' - ``` - - -7. Confirm the `Istio` resource version was updated. - - ```console - $ kubectl get istio -n istio-system - NAME REVISIONS READY IN USE ACTIVE REVISION STATUS VERSION AGE - default 1 1 1 default Healthy v1.26.0 4m50s - ``` - -8. Delete `bookinfo` pods to trigger sidecar injection with the new version. - - ```bash - kubectl rollout restart deployment -n bookinfo - ``` - - -9. Confirm that the new version is used in the sidecar. - - ```bash { name=print-istio-version tag=inplace-update} - istioctl proxy-status - ``` - The column `VERSION` should match the new control plane version. - - -### RevisionBased -When the `RevisionBased` strategy is used, a new Istio control plane instance is created for every change to the `Istio.spec.version` field. The old control plane remains in place until all workloads have been moved to the new control plane instance. This needs to be done by the user by updating the namespace label and restarting all the pods. The old control plane will be deleted after the grace period specified in the `Istio` resource field `spec.updateStrategy.inactiveRevisionDeletionGracePeriodSeconds`. - -#### Example using the RevisionBased strategy - -Prerequisites: -* Sail Operator is installed. -* `istioctl` is [installed](common/install-istioctl-tool.md). - -Steps: - -1. Create the `istio-system` namespace. - - ```bash { name=create-ns tag=revision-based-update} - kubectl create namespace istio-system - ``` - -2. Create the `Istio` resource. - - ```bash { name=create-istio tag=revision-based-update} - cat < - -3. Confirm the control plane is installed and is using the desired version. - - ```console - $ kubectl get istio -n istio-system - NAME REVISIONS READY IN USE ACTIVE REVISION STATUS VERSION AGE - default 1 1 0 default-v1-25-3 Healthy v1.25.3 52s - ``` - Note: `IN USE` field shows as 0, as the control plane has just been installed and there are no workloads using it. - -4. Get the `IstioRevision` name. - - ```console - $ kubectl get istiorevision -n istio-system - NAME TYPE READY STATUS IN USE VERSION AGE - default-v1-25-3 Local True Healthy False v1.25.3 3m4s - ``` - Note: `IstioRevision` name is in the format `-`. - - -5. Create `bookinfo` namespace and label it with the revision name. - - ```bash { name=create-bookinfo-ns tag=revision-based-update} - kubectl create namespace bookinfo - kubectl label namespace bookinfo istio.io/rev=default-v1-25-3 - ``` - -6. Deploy bookinfo application. - - ```bash { name=deploy-bookinfo tag=revision-based-update} - kubectl apply -n bookinfo -f https://raw.githubusercontent.com/istio/istio/release-1.22/samples/bookinfo/platform/kube/bookinfo.yaml - ``` - -7. Review the `Istio` resource after application deployment. - - ```console - $ kubectl get istio -n istio-system - NAME REVISIONS READY IN USE ACTIVE REVISION STATUS VERSION AGE - default 1 1 1 default-v1-25-3 Healthy v1.25.3 5m13s - ``` - Note: `IN USE` field shows as 1, after application being deployed. - -8. Confirm that the proxy version matches the control plane version. - - ```bash - istioctl proxy-status - ``` - The column `VERSION` should match the control plane version. - -9. Update the control plane to a new version. - - ```bash - kubectl patch istio default -n istio-system --type='merge' -p '{"spec":{"version":"v1.26.0"}}' - ``` - -10. Verify the `Istio` and `IstioRevision` resources. There will be a new revision created with the new version. - - ```console - $ kubectl get istio - NAME REVISIONS READY IN USE ACTIVE REVISION STATUS VERSION AGE - default 2 2 1 default-v1-26-0 Healthy v1.26.0 9m23s - - $ kubectl get istiorevision - NAME TYPE READY STATUS IN USE VERSION AGE - default-v1-25-3 Local True Healthy True v1.25.3 10m - default-v1-26-0 Local True Healthy False v1.26.0 66s - ``` - -11. Confirm there are two control plane pods running, one for each revision. - - ```console - $ kubectl get pods -n istio-system - NAME READY STATUS RESTARTS AGE - istiod-default-v1-25-3-c98fd9675-r7bfw 1/1 Running 0 10m - istiod-default-v1-26-0-7495cdc7bf-v8t4g 1/1 Running 0 113s - ``` - -12. Confirm the proxy sidecar version remains the same: - - ```bash - istioctl proxy-status - ``` - The column `VERSION` should still match the old control plane version. - -13. Change the label of the `bookinfo` namespace to use the new revision. - - ```bash { name=update-bookinfo-ns-revision tag=revision-based-update} - kubectl label namespace bookinfo istio.io/rev=default-v1-26-0 --overwrite - ``` - The existing workload sidecars will continue to run and will remain connected to the old control plane instance. They will not be replaced with a new version until the pods are deleted and recreated. - -14. Restart all Deplyments in the `bookinfo` namespace. - - ```bash - kubectl rollout restart deployment -n bookinfo - ``` - -15. Confirm the new version is used in the sidecars. - - ```bash - istioctl proxy-status - ``` - The column `VERSION` should match the updated control plane version. - -16. Confirm the deletion of the old control plane and IstioRevision. - - ```console - $ kubectl get pods -n istio-system - NAME READY STATUS RESTARTS AGE - istiod-default-v1-26-0-7495cdc7bf-v8t4g 1/1 Running 0 4m40s - - $ kubectl get istio - NAME REVISIONS READY IN USE ACTIVE REVISION STATUS VERSION AGE - default 1 1 1 default-v1-26-0 Healthy v1.26.0 5m - - $ kubectl get istiorevision - NAME TYPE READY STATUS IN USE VERSION AGE - default-v1-26-0 Local True Healthy True v1.26.0 5m31s - ``` - The old `IstioRevision` resource and the old control plane will be deleted when the grace period specified in the `Istio` resource field `spec.updateStrategy.inactiveRevisionDeletionGracePeriodSeconds` expires. - -#### Example using the RevisionBased strategy and an IstioRevisionTag - -Prerequisites: -* Sail Operator is installed. -* `istioctl` is [installed](common/install-istioctl-tool.md). - -Steps: - -1. Create the `istio-system` namespace. - - ```bash { name=create-ns tag=istiorevisiontag} - kubectl create namespace istio-system - ``` - -2. Create the `Istio` and `IstioRevisionTag` resources. - - ```bash { name=create-istio-and-revision-tag tag=istiorevisiontag} - cat < -3. Confirm the control plane is installed and is using the desired version. - - ```console - $ kubectl get istio - NAME REVISIONS READY IN USE ACTIVE REVISION STATUS VERSION AGE - default 1 1 1 default-v1-25-3 Healthy v1.25.3 52s - ``` - Note: `IN USE` field shows as 1, even though no workloads are using the control plane. This is because the `IstioRevisionTag` is referencing it. - -4. Inspect the `IstioRevisionTag`. - - ```console - $ kubectl get istiorevisiontags - NAME STATUS IN USE REVISION AGE - default NotReferencedByAnything False default-v1-25-3 52s - ``` - - Note: `IN USE` field shows as `False`, as the tag is not referenced by any workloads or namespaces. -5. Create `bookinfo` namespace and label it to mark it for injection. - - ```bash { name=create-bookinfo-ns tag=istiorevisiontag} - kubectl create namespace bookinfo - kubectl label namespace bookinfo istio-injection=enabled - ``` - -6. Deploy bookinfo application. - - ```bash { name=deploy-bookinfo tag=istiorevisiontag} - kubectl apply -n bookinfo -f https://raw.githubusercontent.com/istio/istio/release-1.23/samples/bookinfo/platform/kube/bookinfo.yaml - ``` - -7. Review the `IstioRevisionTag` resource after application deployment. - - ```console - $ kubectl get istiorevisiontag - NAME STATUS IN USE REVISION AGE - default Healthy True default-v1-25-3 2m46s - ``` - Note: `IN USE` field shows 'True', as the tag is now referenced by both active workloads and the bookinfo namespace. - -8. Confirm that the proxy version matches the control plane version. - - ```bash - istioctl proxy-status - ``` - The column `VERSION` should match the control plane version. - -9. Update the control plane to a new version. - - ```bash { name=update-istio-version tag=istiorevisiontag} - kubectl patch istio default -n istio-system --type='merge' -p '{"spec":{"version":"v1.26.0"}}' - ``` - -10. Verify the `Istio`, `IstioRevision` and `IstioRevisionTag` resources. There will be a new revision created with the new version. - - ```console - $ kubectl get istio - NAME REVISIONS READY IN USE ACTIVE REVISION STATUS VERSION AGE - default 2 2 1 default-v1-26-0 Healthy v1.26.0 9m23s - - $ kubectl get istiorevision - NAME TYPE READY STATUS IN USE VERSION AGE - default-v1-25-3 Local True Healthy True v1.25.3 10m - default-v1-26-0 Local True Healthy True v1.26.0 66s - - $ kubectl get istiorevisiontag - NAME STATUS IN USE REVISION AGE - default Healthy True default-v1-26-0 10m44s - ``` - Now, both our IstioRevisions and the IstioRevisionTag are considered in use. The old revision default-v1-25-3 because it is being used by proxies, the new revision default-v1-26-0 because it is referenced by the tag, and lastly the tag because it is referenced by the bookinfo namespace. - -11. Confirm there are two control plane pods running, one for each revision. - - ```console - $ kubectl get pods -n istio-system - NAME READY STATUS RESTARTS AGE - istiod-default-v1-25-3-c98fd9675-r7bfw 1/1 Running 0 10m - istiod-default-v1-26-0-7495cdc7bf-v8t4g 1/1 Running 0 113s - ``` - -12. Confirm the proxy sidecar version remains the same: - - ```bash { name=validation-istio-proxy-version tag=istiorevisiontag} - istioctl proxy-status - ``` - The column `VERSION` should still match the old control plane version. - -13. Restart all the Deployments in the `bookinfo` namespace. - - ```bash - kubectl rollout restart deployment -n bookinfo - ``` - -14. Confirm the new version is used in the sidecars. Note that it might take a few seconds for the restarts to complete. - - ```bash - istioctl proxy-status - ``` - The column `VERSION` should match the updated control plane version. - -16. Confirm the deletion of the old control plane and IstioRevision. - - ```console - $ kubectl get pods -n istio-system - NAME READY STATUS RESTARTS AGE - istiod-default-v1-26-0-7495cdc7bf-v8t4g 1/1 Running 0 4m40s - - $ kubectl get istio -n istio-system - NAME REVISIONS READY IN USE ACTIVE REVISION STATUS VERSION AGE - default 1 1 1 default-v1-26-0 Healthy v1.26.0 5m - - $ kubectl get istiorevision -n istio-system - NAME TYPE READY STATUS IN USE VERSION AGE - default-v1-26-0 Local True Healthy True v1.26.0 5m31s - ``` - The old `IstioRevision` resource and the old control plane will be deleted when the grace period specified in the `Istio` resource field `spec.updateStrategy.inactiveRevisionDeletionGracePeriodSeconds` expires. - -## Multiple meshes on a single cluster - -The Sail Operator supports running multiple meshes on a single cluster and associating each workload with a specific mesh. -Each mesh is managed by a separate control plane. - -Applications are installed in multiple namespaces, and each namespace is associated with one of the control planes through its labels. -The `istio.io/rev` label determines which control plane injects the sidecar proxy into the application pods. -Additional namespace labels determine whether the control plane discovers and manages the resources in the namespace. -A control plane will discover and manage only those namespaces that match the discovery selectors configured on the control plane. -Additionally, discovery selectors determine which control plane creates the `istio-ca-root-cert` ConfigMap in which namespace. - -Currently, discovery selectors in multiple control planes must be configured so that they don't overlap (i.e. the discovery selectors of two control planes don't match the same namespace). -Each control plane must be deployed in a separate Kubernetes namespace. - -This guide explains how to set up two meshes: `mesh1` and `mesh2` in namespaces `istio-system1` and `istio-system2`, respectively, and three application namespaces: `app1`, `app2a`, and `app2b`. -Mesh 1 will manage namespace `app1`, and Mesh 2 will manage namespaces `app2a` and `app2b`. -Because each mesh will use its own root certificate authority and configured to use a peer authentication policy with the `STRICT` mTLS mode, the communication between the two meshes will not be allowed. - -### Prerequisites - -- Install [istioctl](common/install-istioctl-tool.md). -- Kubernetes 1.23 cluster. -- kubeconfig file with a context for the Kubernetes cluster. -- Install the Sail Operator and the Sail CRDs to the cluster. - -### Installation Steps - -#### Deploying the control planes - -1. Create the system namespace `istio-system1` and deploy the `mesh1` control plane in it. - ```bash { name=deploy-mesh1 tag=multiple-meshes} - kubectl create namespace istio-system1 - kubectl label ns istio-system1 mesh=mesh1 - kubectl apply -f - < -2. Create the system namespace `istio-system2` and deploy the `mesh2` control plane in it. - ```bash { name=deploy-mesh2 tag=multiple-meshes} - kubectl create namespace istio-system2 - kubectl label ns istio-system2 mesh=mesh2 - kubectl apply -f - < -3. Create a peer authentication policy that only allows mTLS communication within each mesh. - ```bash { name=peer-authentication tag=multiple-meshes} - kubectl apply -f - < -2. Check the control planes are `Healthy`: - ```console - $ kubectl get istios - NAME REVISIONS READY IN USE ACTIVE REVISION STATUS VERSION AGE - mesh1 1 1 0 mesh1 Healthy v1.24.0 84s - mesh2 1 1 0 mesh2 Healthy v1.24.0 77s - ``` - -3. Confirm that the validation and mutation webhook configurations exist for both meshes: - ```console - $ kubectl get validatingwebhookconfigurations - NAME WEBHOOKS AGE - istio-validator-mesh1-istio-system1 1 2m45s - istio-validator-mesh2-istio-system2 1 2m38s - - $ kubectl get mutatingwebhookconfigurations - NAME WEBHOOKS AGE - istio-sidecar-injector-mesh1-istio-system1 2 5m55s - istio-sidecar-injector-mesh2-istio-system2 2 5m48s - ``` - -#### Deploying the applications - -1. Create three application namespaces: - ```bash { name=create-app-namespaces tag=multiple-meshes} - kubectl create ns app1 - kubectl create ns app2a - kubectl create ns app2b - ``` - -2. Label each namespace to enable discovery by the corresponding control plane: - ```bash { name=label-app-namespaces tag=multiple-meshes} - kubectl label ns app1 mesh=mesh1 - kubectl label ns app2a mesh=mesh2 - kubectl label ns app2b mesh=mesh2 - ``` - -3. Label each namespace to enable injection by the corresponding control plane: - ```bash { name=label-rev-app-namespaces tag=multiple-meshes} - kubectl label ns app1 istio.io/rev=mesh1 - kubectl label ns app2a istio.io/rev=mesh2 - kubectl label ns app2b istio.io/rev=mesh2 - ``` - -4. Deploy the `curl` and `httpbin` sample applications in each namespace: - ```bash { name=deploy-apps tag=multiple-meshes} - # Deploy curl and httpbin in app1 - kubectl -n app1 apply -f https://raw.githubusercontent.com/istio/istio/refs/heads/master/samples/curl/curl.yaml - kubectl -n app1 apply -f https://raw.githubusercontent.com/istio/istio/refs/heads/master/samples/httpbin/httpbin.yaml - # Deploy curl and httpbin in app2a and app2b - kubectl -n app2a apply -f https://raw.githubusercontent.com/istio/istio/refs/heads/master/samples/curl/curl.yaml - kubectl -n app2a apply -f https://raw.githubusercontent.com/istio/istio/refs/heads/master/samples/httpbin/httpbin.yaml - # Deploy curl and httpbin in app2b - kubectl -n app2b apply -f https://raw.githubusercontent.com/istio/istio/refs/heads/master/samples/curl/curl.yaml - kubectl -n app2b apply -f https://raw.githubusercontent.com/istio/istio/refs/heads/master/samples/httpbin/httpbin.yaml - ``` - -5. Confirm that a sidecar has been injected into each of the application pods. The value `2/2` should be displayed in the `READY` column for each pod, as in the following example: - ```console - $ kubectl get pods -n app1 - NAME READY STATUS RESTARTS AGE - curl-5b549b49b8-mg7nl 2/2 Running 0 102s - httpbin-7b549f7859-h6hnk 2/2 Running 0 89s - - $ kubectl get pods -n app2a - NAME READY STATUS RESTARTS AGE - curl-5b549b49b8-2hlvm 2/2 Running 0 2m3s - httpbin-7b549f7859-bgblg 2/2 Running 0 110s - - $ kubectl get pods -n app2b - NAME READY STATUS RESTARTS AGE - curl-5b549b49b8-xnzzk 2/2 Running 0 2m9s - httpbin-7b549f7859-7k5gf 2/2 Running 0 118s - ``` - -### Validation - -#### Checking application to control plane mapping - -Use the `istioctl ps` command to confirm that the application pods are connected to the correct control plane. - -The `curl` and `httpbin` pods in namespace `app1` should be connected to the control plane in namespace `istio-system1`, as shown in the following example (note the `.app1` suffix in the `NAME` column): - -```console -$ istioctl ps -i istio-system1 -NAME CLUSTER CDS LDS EDS RDS ECDS ISTIOD VERSION -curl-5b549b49b8-mg7nl.app1 Kubernetes SYNCED (4m40s) SYNCED (4m40s) SYNCED (4m31s) SYNCED (4m40s) IGNORED istiod-mesh1-5df45b97dd-tf2wl 1.24.0 -httpbin-7b549f7859-h6hnk.app1 Kubernetes SYNCED (4m31s) SYNCED (4m31s) SYNCED (4m31s) SYNCED (4m31s) IGNORED istiod-mesh1-5df45b97dd-tf2wl 1.24.0 -``` - -The pods in namespaces `app2a` and `app2b` should be connected to the control plane in namespace `istio-system2`: - -```console -$ istioctl ps -i istio-system2 -NAME CLUSTER CDS LDS EDS RDS ECDS ISTIOD VERSION -curl-5b549b49b8-2hlvm.app2a Kubernetes SYNCED (4m37s) SYNCED (4m37s) SYNCED (4m31s) SYNCED (4m37s) IGNORED istiod-mesh2-59f6b874fb-mzxqw 1.24.0 -curl-5b549b49b8-xnzzk.app2b Kubernetes SYNCED (4m37s) SYNCED (4m37s) SYNCED (4m31s) SYNCED (4m37s) IGNORED istiod-mesh2-59f6b874fb-mzxqw 1.24.0 -httpbin-7b549f7859-7k5gf.app2b Kubernetes SYNCED (4m31s) SYNCED (4m31s) SYNCED (4m31s) SYNCED (4m31s) IGNORED istiod-mesh2-59f6b874fb-mzxqw 1.24.0 -httpbin-7b549f7859-bgblg.app2a Kubernetes SYNCED (4m32s) SYNCED (4m32s) SYNCED (4m31s) SYNCED (4m32s) IGNORED istiod-mesh2-59f6b874fb-mzxqw 1.24.0 -``` - -#### Checking application connectivity - -As both meshes are configured to use the `STRICT` mTLS peer authentication mode, the applications in namespace `app1` should not be able to communicate with the applications in namespaces `app2a` and `app2b`, and vice versa. -To test whether the `curl` pod in namespace `app2a` can connect to the `httpbin` service in namespace `app1`, run the following commands: - -```console -$ kubectl -n app2a exec deploy/curl -c curl -- curl -sIL http://httpbin.app1:8000 -HTTP/1.1 503 Service Unavailable -content-length: 95 -content-type: text/plain -date: Fri, 29 Nov 2024 08:58:28 GMT -server: envoy -``` - -As expected, the response indicates that the connection was not successful. -In contrast, the same pod should be able to connect to the `httpbin` service in namespace `app2b`, because they are part of the same mesh: - -```console -$ kubectl -n app2a exec deploy/curl -c curl -- curl -sIL http://httpbin.app2b:8000 -HTTP/1.1 200 OK -access-control-allow-credentials: true -access-control-allow-origin: * -content-security-policy: default-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' camo.githubusercontent.com -content-type: text/html; charset=utf-8 -date: Fri, 29 Nov 2024 08:57:52 GMT -x-envoy-upstream-service-time: 0 -server: envoy -transfer-encoding: chunked -``` - -### Cleanup - -To clean up the resources created in this guide, delete the `Istio` resources and the namespaces: - -```bash - kubectl delete istio mesh1 mesh2 - kubectl delete ns istio-system1 istio-system2 app1 app2a app2b -``` - - -## Multi-cluster - -You can use the Sail Operator and the Sail CRDs to manage a multi-cluster Istio deployment. The following instructions are adapted from the [Istio multi-cluster documentation](https://istio.io/latest/docs/setup/install/multicluster/) to demonstrate how you can setup the various deployment models with Sail. Please familiarize yourself with the different [deployment models](https://istio.io/latest/docs/ops/deployment/deployment-models/) before starting. - -### Prerequisites - -- Install [istioctl](common/install-istioctl-tool.md). -- Two kubernetes clusters with external lb support. (If using kind, `cloud-provider-kind` is running in the background) -- kubeconfig file with a context for each cluster. -- Install the Sail Operator and the Sail CRDs to every cluster. - -### Common Setup - -These steps are common to every multi-cluster deployment and should be completed *after* meeting the prerequisites but *before* starting on a specific deployment model. - -1. Setup environment variables. - - ```bash - export CTX_CLUSTER1= - export CTX_CLUSTER2= - export ISTIO_VERSION=1.26.0 - ``` - -2. Create `istio-system` namespace on each cluster. - - ```bash - kubectl get ns istio-system --context "${CTX_CLUSTER1}" || kubectl create namespace istio-system --context "${CTX_CLUSTER1}" - kubectl get ns istio-system --context "${CTX_CLUSTER2}" || kubectl create namespace istio-system --context "${CTX_CLUSTER2}" - ``` - -4. Create a shared root certificate. - - If you have [established trust](https://istio.io/latest/docs/setup/install/multicluster/before-you-begin/#configure-trust) between your clusters already you can skip this and the following steps. - - ```bash - openssl genrsa -out root-key.pem 4096 - cat < root-ca.conf - [ req ] - encrypt_key = no - prompt = no - utf8 = yes - default_md = sha256 - default_bits = 4096 - req_extensions = req_ext - x509_extensions = req_ext - distinguished_name = req_dn - [ req_ext ] - subjectKeyIdentifier = hash - basicConstraints = critical, CA:true - keyUsage = critical, digitalSignature, nonRepudiation, keyEncipherment, keyCertSign - [ req_dn ] - O = Istio - CN = Root CA - EOF - - openssl req -sha256 -new -key root-key.pem \ - -config root-ca.conf \ - -out root-cert.csr - - openssl x509 -req -sha256 -days 3650 \ - -signkey root-key.pem \ - -extensions req_ext -extfile root-ca.conf \ - -in root-cert.csr \ - -out root-cert.pem - ``` -5. Create intermediate certificates. - - ```bash - for cluster in west east; do - mkdir $cluster - - openssl genrsa -out ${cluster}/ca-key.pem 4096 - cat < ${cluster}/intermediate.conf - [ req ] - encrypt_key = no - prompt = no - utf8 = yes - default_md = sha256 - default_bits = 4096 - req_extensions = req_ext - x509_extensions = req_ext - distinguished_name = req_dn - [ req_ext ] - subjectKeyIdentifier = hash - basicConstraints = critical, CA:true, pathlen:0 - keyUsage = critical, digitalSignature, nonRepudiation, keyEncipherment, keyCertSign - subjectAltName=@san - [ san ] - DNS.1 = istiod.istio-system.svc - [ req_dn ] - O = Istio - CN = Intermediate CA - L = $cluster - EOF - - openssl req -new -config ${cluster}/intermediate.conf \ - -key ${cluster}/ca-key.pem \ - -out ${cluster}/cluster-ca.csr - - openssl x509 -req -sha256 -days 3650 \ - -CA root-cert.pem \ - -CAkey root-key.pem -CAcreateserial \ - -extensions req_ext -extfile ${cluster}/intermediate.conf \ - -in ${cluster}/cluster-ca.csr \ - -out ${cluster}/ca-cert.pem - - cat ${cluster}/ca-cert.pem root-cert.pem \ - > ${cluster}/cert-chain.pem - cp root-cert.pem ${cluster} - done - ``` - -6. Push the intermediate CAs to each cluster. - ```bash - kubectl --context "${CTX_CLUSTER1}" label namespace istio-system topology.istio.io/network=network1 - kubectl get secret -n istio-system --context "${CTX_CLUSTER1}" cacerts || kubectl create secret generic cacerts -n istio-system --context "${CTX_CLUSTER1}" \ - --from-file=east/ca-cert.pem \ - --from-file=east/ca-key.pem \ - --from-file=east/root-cert.pem \ - --from-file=east/cert-chain.pem - kubectl --context "${CTX_CLUSTER2}" label namespace istio-system topology.istio.io/network=network2 - kubectl get secret -n istio-system --context "${CTX_CLUSTER2}" cacerts || kubectl create secret generic cacerts -n istio-system --context "${CTX_CLUSTER2}" \ - --from-file=west/ca-cert.pem \ - --from-file=west/ca-key.pem \ - --from-file=west/root-cert.pem \ - --from-file=west/cert-chain.pem - ``` - -### Multi-Primary - Multi-Network - -These instructions install a [multi-primary/multi-network](https://istio.io/latest/docs/setup/install/multicluster/multi-primary_multi-network/) Istio deployment using the Sail Operator and Sail CRDs. **Before you begin**, ensure you complete the [common setup](#common-setup). - -You can follow the steps below to install manually or you can run [this script](multicluster/setup-multi-primary.sh) which will setup a local environment for you with kind. Before running the setup script, you must install [kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) and [cloud-provider-kind](https://kind.sigs.k8s.io/docs/user/loadbalancer/#installing-cloud-provider-kind) then ensure the `cloud-provider-kind` binary is running in the background. - -These installation instructions are adapted from: https://istio.io/latest/docs/setup/install/multicluster/multi-primary_multi-network/. - -1. Create an `Istio` resource on `cluster1`. - - ```bash - kubectl apply --context "${CTX_CLUSTER1}" -f - < /dev/null || \ - { kubectl kustomize "github.com/kubernetes-sigs/gateway-api/config/crd?ref=v1.1.0" | kubectl apply -f - --context="${CTX_CLUSTER2}"; } - ``` - - Expose `helloworld` through the ingress gateway. - ```bash - kubectl apply -f "https://raw.githubusercontent.com/istio/istio/${ISTIO_VERSION}/samples/helloworld/gateway-api/helloworld-gateway.yaml" -n sample --context="${CTX_CLUSTER2}" - kubectl -n sample --context="${CTX_CLUSTER2}" wait --for=condition=programmed gtw helloworld-gateway - ``` - - Confirm you can access the `helloworld` application through the ingress gateway created in the Remote cluster. - ```bash - curl -s "http://$(kubectl -n sample --context="${CTX_CLUSTER2}" get gtw helloworld-gateway -o jsonpath='{.status.addresses[0].value}'):80/hello" - ``` - You should see a response from the `helloworld` application: - ```bash - Hello version: v1, instance: helloworld-v1-6d65866976-jb6qc - ``` - -15. Cleanup - - ```bash - kubectl delete istios default --context="${CTX_CLUSTER1}" - kubectl delete ns istio-system --context="${CTX_CLUSTER1}" - kubectl delete istios external-istiod --context="${CTX_CLUSTER1}" - kubectl delete ns external-istiod --context="${CTX_CLUSTER1}" - kubectl delete istios external-istiod --context="${CTX_CLUSTER2}" - kubectl delete ns external-istiod --context="${CTX_CLUSTER2}" - kubectl delete ns sample --context="${CTX_CLUSTER2}" - ``` - -## Dual-stack Support - -Kubernetes supports dual-stack networking as a stable feature starting from -[v1.23](https://kubernetes.io/docs/concepts/services-networking/dual-stack/), allowing clusters to handle both -IPv4 and IPv6 traffic. With many cloud providers also beginning to offer dual-stack Kubernetes clusters, it's easier -than ever to run services that function across both address types. Istio introduced dual-stack as an experimental -feature in version 1.17, and promoted it to [Alpha](https://istio.io/latest/news/releases/1.24.x/announcing-1.24/change-notes/) in -version 1.24. With Istio in dual-stack mode, services can communicate over both IPv4 and IPv6 endpoints, which helps -organizations transition to IPv6 while still maintaining compatibility with their existing IPv4 infrastructure. - -When Kubernetes is configured for dual-stack, it automatically assigns an IPv4 and an IPv6 address to each pod, -enabling them to communicate over both IP families. For services, however, you can control how they behave using -the `ipFamilyPolicy` setting. - -Service.Spec.ipFamilyPolicy can take the following values -- SingleStack: Only one IP family is configured for the service, which can be either IPv4 or IPv6. -- PreferDualStack: Both IPv4 and IPv6 cluster IPs are assigned to the Service when dual-stack is enabled. - However, if dual-stack is not enabled or supported, it falls back to singleStack behavior. -- RequireDualStack: The service will be created only if both IPv4 and IPv6 addresses can be assigned. - -This allows you to specify the type of service, providing flexibility in managing your network configuration. -For more details, you can refer to the Kubernetes [documentation](https://kubernetes.io/docs/concepts/services-networking/dual-stack/#services). - -### Prerequisites - -- Kubernetes 1.23 or later configured with dual-stack support. -- Sail Operator is installed. - -### Installation Steps - -You can use any existing Kind cluster that supports dual-stack networking or, alternatively, install one using the following command. - -```bash -kind create cluster --name istio-ds --config - < -5. Verify that sleep pod is able to reach the dual-stack pods. - ```console - $ kubectl exec -n sleep "$(kubectl get pod -n sleep -l app=sleep -o jsonpath='{.items[0].metadata.name}')" -- sh -c "echo dualstack | nc tcp-echo.dual-stack 9000" - hello dualstack - ``` - -6. Similarly verify that sleep pod is able to reach both ipv4 pods as well as ipv6 pods. - ```console - $ kubectl exec -n sleep "$(kubectl get pod -n sleep -l app=sleep -o jsonpath='{.items[0].metadata.name}')" -- sh -c "echo ipv4 | nc tcp-echo.ipv4 9000" - hello ipv4 - ``` - - ```console - $ kubectl exec -n sleep "$(kubectl get pod -n sleep -l app=sleep -o jsonpath='{.items[0].metadata.name}')" -- sh -c "echo ipv6 | nc tcp-echo.ipv6 9000" - hello ipv6 - ``` - -7. Cleanup - ```bash - kubectl delete istios default - kubectl delete ns istio-system - kubectl delete istiocni default - kubectl delete ns istio-cni - kubectl delete ns dual-stack ipv4 ipv6 sleep - ``` - -## Addons - -Addons are managed separately from the Sail Operator. You can follow the [istio documentation](https://istio.io/latest/docs/ops/integrations/) for how to install addons. Below is an example of how to install some addons for Istio. - -The sample will deploy: - -- Prometheus -- Jaeger -- Kiali -- Bookinfo demo app - -*Prerequisites* - -- Sail operator is installed. -- Control Plane is installed via the Sail Operator. - -### Deploy Prometheus and Jaeger addons - -```bash -kubectl apply -f https://raw.githubusercontent.com/istio/istio/master/samples/addons/prometheus.yaml -kubectl apply -f https://raw.githubusercontent.com/istio/istio/master/samples/addons/jaeger.yaml -``` - -### Deploy Kiali addon - -Install the kiali operator. - -You can install the kiali operator through OLM if running on Openshift, otherwise you can use helm: - -```bash -helm install --namespace kiali-operator --create-namespace kiali-operator kiali/kiali-operator -``` - -Create a Kiali resource. We're enabling tracing and disabling grafana here since tracing is disabled by default and grafana is not part of this example. - -```bash -kubectl apply -f - < /dev/null || \ - { kubectl kustomize "github.com/kubernetes-sigs/gateway-api/config/crd?ref=v1.1.0" | kubectl apply -f -; } -``` - -Create bookinfo gateway. - -```bash -kubectl apply -n bookinfo -f https://raw.githubusercontent.com/istio/istio/master/samples/bookinfo/gateway-api/bookinfo-gateway.yaml -kubectl wait -n bookinfo --for=condition=programmed gtw bookinfo-gateway -``` - -### Generate traffic and visualize your mesh - -Send traffic to the productpage service. Note that this command will run until cancelled. - -```bash -export INGRESS_HOST=$(kubectl get gtw bookinfo-gateway -n bookinfo -o jsonpath='{.status.addresses[0].value}') -export INGRESS_PORT=$(kubectl get gtw bookinfo-gateway -n bookinfo -o jsonpath='{.spec.listeners[?(@.name=="http")].port}') -export GATEWAY_URL=$INGRESS_HOST:$INGRESS_PORT -watch curl http://${GATEWAY_URL}/productpage &> /dev/null -``` - -In a separate terminal, open Kiali to visualize your mesh. - -If using Openshift, open the Kiali route: - -```bash -echo https://$(kubectl get routes -n istio-system kiali -o jsonpath='{.spec.host}') -``` - -Otherwise, port forward to the kiali pod directly: - -```bash -kubectl port-forward -n istio-system svc/kiali 20001:20001 -``` - -You can view Kiali dashboard at: http://localhost:20001 - -## Observability Integrations - -### Scraping metrics using the OpenShift monitoring stack -The easiest way to get started with production-grade metrics collection is to use OpenShift's user-workload monitoring stack. The following steps assume that you installed Istio into the `istio-system` namespace. Note that these steps are not specific to the Sail Operator, but describe how to configure user-workload monitoring for Istio in general. - -*Prerequisites* -* User Workload monitoring is [enabled](https://docs.openshift.com/container-platform/latest/observability/monitoring/enabling-monitoring-for-user-defined-projects.html) - -*Steps* -1. Create a ServiceMonitor for istiod. - - ```yaml - apiVersion: monitoring.coreos.com/v1 - kind: ServiceMonitor - metadata: - name: istiod-monitor - namespace: istio-system - spec: - targetLabels: - - app - selector: - matchLabels: - istio: pilot - endpoints: - - port: http-monitoring - interval: 30s - ``` -1. Create a PodMonitor to scrape metrics from the istio-proxy containers. Note that *this resource has to be created in all namespaces where you are running sidecars*. - - ```yaml - apiVersion: monitoring.coreos.com/v1 - kind: PodMonitor - metadata: - name: istio-proxies-monitor - namespace: istio-system - spec: - selector: - matchExpressions: - - key: istio-prometheus-ignore - operator: DoesNotExist - podMetricsEndpoints: - - path: /stats/prometheus - interval: 30s - relabelings: - - action: keep - sourceLabels: ["__meta_kubernetes_pod_container_name"] - regex: "istio-proxy" - - action: keep - sourceLabels: ["__meta_kubernetes_pod_annotationpresent_prometheus_io_scrape"] - - action: replace - regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4}) - replacement: "[$2]:$1" - sourceLabels: ["__meta_kubernetes_pod_annotation_prometheus_io_port","__meta_kubernetes_pod_ip"] - targetLabel: "__address__" - - action: replace - regex: (\d+);((([0-9]+?)(\.|$)){4}) - replacement: "$2:$1" - sourceLabels: ["__meta_kubernetes_pod_annotation_prometheus_io_port","__meta_kubernetes_pod_ip"] - targetLabel: "__address__" - - action: labeldrop - regex: "__meta_kubernetes_pod_label_(.+)" - - sourceLabels: ["__meta_kubernetes_namespace"] - action: replace - targetLabel: namespace - - sourceLabels: ["__meta_kubernetes_pod_name"] - action: replace - targetLabel: pod_name - ``` - -Congratulations! You should now be able to see your control plane and data plane metrics in the OpenShift Console. Just go to Observe -> Metrics and try the query `istio_requests_total`. - -### Configure tracing with OpenShift distributed tracing -This section describes how to setup Istio with OpenShift Distributed Tracing to send distributed traces. - -*Prerequisites* -* A Tempo stack is installed and configured -* An instance of an OpenTelemetry collector is already configured in the istio-system namespace -* An Istio instance is created with the `openshift` profile -* An Istio CNI instance is created with the `openshift` profile - -*Steps* -1. Configure Istio to enable tracing and include the OpenTelemetry settings: - ```yaml - meshConfig: - enableTracing: true - extensionProviders: - - name: otel-tracing - opentelemetry: - port: 4317 - service: otel-collector.istio-system.svc.cluster.local - ``` -The *service* field is the OpenTelemetry collector service in the `istio-system` namespace. - -2. Create an Istio telemetry resource to active the OpenTelemetry tracer - ```yaml - apiVersion: telemetry.istio.io/v1 - kind: Telemetry - metadata: - name: otel-demo - namespace: istio-system - spec: - tracing: - - providers: - - name: otel-tracing - randomSamplingPercentage: 100 - ``` - -3. Validate the integration: Generate some traffic - -We can [Deploy Bookinfo](#deploy-gateway-and-bookinfo) and generate some traffic. - -4. Validate the integration: See the traces in the UI - -```bash -kubectl get routes -n tempo tempo-sample-query-frontend-tempo -``` - -If you [configure Kiali with OpenShift distributed tracing](#integrating-kiali-with-openshift-distributed-tracing) you can verify from there. - -### Integrating with Kiali -Integration with Kiali really depends on how you collect your metrics and traces. Note that Kiali is a separate project which for the purpose of this document we'll expect is installed using the Kiali operator. The steps here are not specific to Sail Operator, but describe how to configure Kiali for use with Istio in general. - -#### Integrating Kiali with the OpenShift monitoring stack -If you followed [Scraping metrics using the OpenShift monitoring stack](#scraping-metrics-using-the-openshift-monitoring-stack), you can set up Kiali to retrieve metrics from there. - -*Prerequisites* -* User Workload monitoring is [enabled](https://docs.openshift.com/container-platform/latest/observability/monitoring/enabling-monitoring-for-user-defined-projects.html) and [configured](#scraping-metrics-using-the-openshift-monitoring-stack) -* Kiali Operator is installed - -*Steps* -1. Create a ClusterRoleBinding for Kiali, so it can view metrics from user-workload monitoring - - ```yaml - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - name: kiali-monitoring-rbac - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-monitoring-view - subjects: - - kind: ServiceAccount - name: kiali-service-account - namespace: istio-system - ``` -1. Find out the revision name of your Istio instance. In our case it is `test`. - - ```console - $ kubectl get istiorevisions.sailoperator.io - NAME READY STATUS IN USE VERSION AGE - test True Healthy True v1.21.0 119m - ``` -1. Create a Kiali resource and point it to your Istio instance. Make sure to replace `test` with your revision name in the fields `config_map_name`, `istio_sidecar_injector_config_map_name`, `istiod_deployment_name` and `url_service_version`. - - ```yaml - apiVersion: kiali.io/v1alpha1 - kind: Kiali - metadata: - name: kiali-user-workload-monitoring - namespace: istio-system - spec: - external_services: - istio: - config_map_name: istio-test - istio_sidecar_injector_config_map_name: istio-sidecar-injector-test - istiod_deployment_name: istiod-test - url_service_version: 'http://istiod-test.istio-system:15014/version' - prometheus: - auth: - type: bearer - use_kiali_token: true - thanos_proxy: - enabled: true - url: https://thanos-querier.openshift-monitoring.svc.cluster.local:9091 - ``` -#### Integrating Kiali with OpenShift Distributed Tracing -This section describes how to setup Kiali with OpenShift Distributed Tracing to read the distributed traces. - -*Prerequisites* -* Istio tracing is [Configured with OpenShift distributed tracing](#configure-tracing-with-openshift-distributed-tracing) - -*Steps* -1. Setup Kiali to access traces from the Tempo frontend: - ```yaml - external_services: - grafana: - enabled: true - url: "http://grafana-istio-system.apps-crc.testing/" - tracing: - enabled: true - provider: tempo - use_grpc: false - in_cluster_url: http://tempo-sample-query-frontend.tempo:3200 - url: 'https://tempo-sample-query-frontend-tempo.apps-crc.testing' - tempo_config: - org_id: "1" - datasource_uid: "a8d2ef1c-d31c-4de5-a90b-e7bc5252cd00" - ``` - -Where: -* `external_services.grafana` section: Is just needed to see the "View in Tracing" link from the Traces tab -* `external_services.tracing.tempo_config`: Is just needed to see the "View in Tracing" link from the Traces tab and redirect to the proper Tempo datasource - -Now, we should be able to see traces from Kiali. For this, you can: -1. Select a Workload/Service/App -2. Click in the "Traces" tab - -## Uninstalling - -### Deleting Istio -1. In the OpenShift Container Platform web console, click **Operators** -> **Installed Operators**. -1. Click **Istio** in the **Provided APIs** column. -1. Click the Options menu, and select **Delete Istio**. -1. At the prompt to confirm the action, click **Delete**. - -### Deleting IstioCNI -1. In the OpenShift Container Platform web console, click **Operators** -> **Installed Operators**. -1. Click **IstioCNI** in the **Provided APIs** column. -1. Click the Options menu, and select **Delete IstioCNI**. -1. At the prompt to confirm the action, click **Delete**. - -### Deleting the Sail Operator -1. In the OpenShift Container Platform web console, click **Operators** -> **Installed Operators**. -1. Locate the Sail Operator. Click the Options menu, and select **Uninstall Operator**. -1. At the prompt to confirm the action, click **Uninstall**. - -### Deleting the istio-system and istio-cni Projects -1. In the OpenShift Container Platform web console, click **Home** -> **Projects**. -1. Locate the name of the project and click the Options menu. -1. Click **Delete Project**. -1. At the prompt to confirm the action, enter the name of the project. -1. Click **Delete**. - -### Decide whether you want to delete the CRDs as well -OLM leaves this [decision](https://olm.operatorframework.io/docs/tasks/uninstall-operator/#step-4-deciding-whether-or-not-to-delete-the-crds-and-apiservices) to the users. -If you want to delete the Istio CRDs, you can use the following command. -```bash -kubectl get crds -oname | grep istio.io | xargs kubectl delete -``` +The Sail Operator API reference documentation can be found [here](api-reference/sailoperator.io.md#api-reference). diff --git a/docs/addons/addons.md b/docs/addons/addons.md new file mode 100644 index 000000000..f0d4a8d16 --- /dev/null +++ b/docs/addons/addons.md @@ -0,0 +1,127 @@ +[Return to Project Root](../README.md) + +# Table of Contents + +- [Addons](#addons) + - [Deploy Prometheus and Jaeger addons](#deploy-prometheus-and-jaeger-addons) + - [Deploy Kiali addon](#deploy-kiali-addon) + - [Find the active revision of your Istio instance. In our case it is `test`.](#find-the-active-revision-of-your-istio-instance) + - [Deploy Gateway and Bookinfo](#deploy-gateway-and-bookinfo) + - [Generate traffic and visualize your mesh](#generate-traffic-and-visualize-your-mesh) + +## Addons + +Addons are managed separately from the Sail Operator. You can follow the [istio documentation](https://istio.io/latest/docs/ops/integrations/) for how to install addons. Below is an example of how to install some addons for Istio. + +The sample will deploy: + +- Prometheus +- Jaeger +- Kiali +- Bookinfo demo app + +*Prerequisites* + +- Sail operator is installed. +- Control Plane is installed via the Sail Operator. + +### Deploy Prometheus and Jaeger addons + +```bash +kubectl apply -f https://raw.githubusercontent.com/istio/istio/master/samples/addons/prometheus.yaml +kubectl apply -f https://raw.githubusercontent.com/istio/istio/master/samples/addons/jaeger.yaml +``` + +### Deploy Kiali addon + +Install the kiali operator. + +You can install the kiali operator through OLM if running on Openshift, otherwise you can use helm: + +```bash +helm install --namespace kiali-operator --create-namespace kiali-operator kiali/kiali-operator +``` + +Create a Kiali resource. We're enabling tracing and disabling grafana here since tracing is disabled by default and grafana is not part of this example. + +```bash +kubectl apply -f - < /dev/null || \ + { kubectl kustomize "github.com/kubernetes-sigs/gateway-api/config/crd?ref=v1.1.0" | kubectl apply -f -; } +``` + +Create bookinfo gateway. + +```bash +kubectl apply -n bookinfo -f https://raw.githubusercontent.com/istio/istio/master/samples/bookinfo/gateway-api/bookinfo-gateway.yaml +kubectl wait -n bookinfo --for=condition=programmed gtw bookinfo-gateway +``` + +### Generate traffic and visualize your mesh + +Send traffic to the productpage service. Note that this command will run until canceled. + +```bash +export INGRESS_HOST=$(kubectl get gtw bookinfo-gateway -n bookinfo -o jsonpath='{.status.addresses[0].value}') +export INGRESS_PORT=$(kubectl get gtw bookinfo-gateway -n bookinfo -o jsonpath='{.spec.listeners[?(@.name=="http")].port}') +export GATEWAY_URL=$INGRESS_HOST:$INGRESS_PORT +watch curl http://${GATEWAY_URL}/productpage &> /dev/null +``` + +In a separate terminal, open Kiali to visualize your mesh. + +If using Openshift, open the Kiali route: + +```bash +echo https://$(kubectl get routes -n istio-system kiali -o jsonpath='{.spec.host}') +``` + +Otherwise, port forward to the kiali pod directly: + +```bash +kubectl port-forward -n istio-system svc/kiali 20001:20001 +``` + +You can view Kiali dashboard at: http://localhost:20001 diff --git a/docs/addons/observability.md b/docs/addons/observability.md new file mode 100644 index 000000000..6062d0654 --- /dev/null +++ b/docs/addons/observability.md @@ -0,0 +1,218 @@ +[Return to Project Root](../README.md) + +# Table of Contents + +- [Observability Integrations](#observability-integrations) + - [Scraping metrics using the OpenShift monitoring stack](#scraping-metrics-using-the-openshift-monitoring-stack) + - [Configure tracing with OpenShift distributed tracing](#configure-tracing-with-openshift-distributed-tracing) + - [Integrating with Kiali](#integrating-with-kiali) + - [Integrating Kiali with the OpenShift monitoring stack](#integrating-kiali-with-the-openshift-monitoring-stack) + - [Integrating Kiali with OpenShift Distributed Tracing](#integrating-kiali-with-openshift-distributed-tracing) + +## Observability Integrations + +### Scraping metrics using the OpenShift monitoring stack +The easiest way to get started with production-grade metrics collection is to use OpenShift's user-workload monitoring stack. The following steps assume that you installed Istio into the `istio-system` namespace. Note that these steps are not specific to the Sail Operator, but describe how to configure user-workload monitoring for Istio in general. + +*Prerequisites* +* User Workload monitoring is [enabled](https://docs.openshift.com/container-platform/latest/observability/monitoring/enabling-monitoring-for-user-defined-projects.html) + +*Steps* +1. Create a ServiceMonitor for istiod. + + ```yaml + apiVersion: monitoring.coreos.com/v1 + kind: ServiceMonitor + metadata: + name: istiod-monitor + namespace: istio-system + spec: + targetLabels: + - app + selector: + matchLabels: + istio: pilot + endpoints: + - port: http-monitoring + interval: 30s + ``` +1. Create a PodMonitor to scrape metrics from the istio-proxy containers. Note that *this resource has to be created in all namespaces where you are running sidecars*. + + ```yaml + apiVersion: monitoring.coreos.com/v1 + kind: PodMonitor + metadata: + name: istio-proxies-monitor + namespace: istio-system + spec: + selector: + matchExpressions: + - key: istio-prometheus-ignore + operator: DoesNotExist + podMetricsEndpoints: + - path: /stats/prometheus + interval: 30s + relabelings: + - action: keep + sourceLabels: ["__meta_kubernetes_pod_container_name"] + regex: "istio-proxy" + - action: keep + sourceLabels: ["__meta_kubernetes_pod_annotationpresent_prometheus_io_scrape"] + - action: replace + regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4}) + replacement: "[$2]:$1" + sourceLabels: ["__meta_kubernetes_pod_annotation_prometheus_io_port","__meta_kubernetes_pod_ip"] + targetLabel: "__address__" + - action: replace + regex: (\d+);((([0-9]+?)(\.|$)){4}) + replacement: "$2:$1" + sourceLabels: ["__meta_kubernetes_pod_annotation_prometheus_io_port","__meta_kubernetes_pod_ip"] + targetLabel: "__address__" + - action: labeldrop + regex: "__meta_kubernetes_pod_label_(.+)" + - sourceLabels: ["__meta_kubernetes_namespace"] + action: replace + targetLabel: namespace + - sourceLabels: ["__meta_kubernetes_pod_name"] + action: replace + targetLabel: pod_name + ``` + +Congratulations! You should now be able to see your control plane and data plane metrics in the OpenShift Console. Just go to Observe -> Metrics and try the query `istio_requests_total`. + +### Configure tracing with OpenShift distributed tracing +This section describes how to setup Istio with OpenShift Distributed Tracing to send distributed traces. + +*Prerequisites* +* A Tempo stack is installed and configured +* An instance of an OpenTelemetry collector is already configured in the istio-system namespace +* An Istio instance is created with the `openshift` profile +* An Istio CNI instance is created with the `openshift` profile + +*Steps* +1. Configure Istio to enable tracing and include the OpenTelemetry settings: + ```yaml + meshConfig: + enableTracing: true + extensionProviders: + - name: otel-tracing + opentelemetry: + port: 4317 + service: otel-collector.istio-system.svc.cluster.local + ``` +The *service* field is the OpenTelemetry collector service in the `istio-system` namespace. + +2. Create an Istio telemetry resource to active the OpenTelemetry tracer + ```yaml + apiVersion: telemetry.istio.io/v1 + kind: Telemetry + metadata: + name: otel-demo + namespace: istio-system + spec: + tracing: + - providers: + - name: otel-tracing + randomSamplingPercentage: 100 + ``` + +3. Validate the integration: Generate some traffic + +We can [Deploy Bookinfo](addons.md#deploy-gateway-and-bookinfo) and generate some traffic. + +4. Validate the integration: See the traces in the UI + +```bash +kubectl get routes -n tempo tempo-sample-query-frontend-tempo +``` + +If you [configure Kiali with OpenShift distributed tracing](#integrating-kiali-with-openshift-distributed-tracing) you can verify from there. + +### Integrating with Kiali +Integration with Kiali really depends on how you collect your metrics and traces. Note that Kiali is a separate project which for the purpose of this document we'll expect is installed using the Kiali operator. The steps here are not specific to Sail Operator, but describe how to configure Kiali for use with Istio in general. + +#### Integrating Kiali with the OpenShift monitoring stack +If you followed [Scraping metrics using the OpenShift monitoring stack](#scraping-metrics-using-the-openshift-monitoring-stack), you can set up Kiali to retrieve metrics from there. + +*Prerequisites* +* User Workload monitoring is [enabled](https://docs.openshift.com/container-platform/latest/observability/monitoring/enabling-monitoring-for-user-defined-projects.html) and [configured](#scraping-metrics-using-the-openshift-monitoring-stack) +* Kiali Operator is installed + +*Steps* +1. Create a ClusterRoleBinding for Kiali, so it can view metrics from user-workload monitoring + + ```yaml + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: kiali-monitoring-rbac + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-monitoring-view + subjects: + - kind: ServiceAccount + name: kiali-service-account + namespace: istio-system + ``` +1. Find out the revision name of your Istio instance. In our case it is `test`. + + ```console + $ kubectl get istiorevisions.sailoperator.io + NAME READY STATUS IN USE VERSION AGE + test True Healthy True v1.21.0 119m + ``` +1. Create a Kiali resource and point it to your Istio instance. Make sure to replace `test` with your revision name in the fields `config_map_name`, `istio_sidecar_injector_config_map_name`, `istiod_deployment_name` and `url_service_version`. + + ```yaml + apiVersion: kiali.io/v1alpha1 + kind: Kiali + metadata: + name: kiali-user-workload-monitoring + namespace: istio-system + spec: + external_services: + istio: + config_map_name: istio-test + istio_sidecar_injector_config_map_name: istio-sidecar-injector-test + istiod_deployment_name: istiod-test + url_service_version: 'http://istiod-test.istio-system:15014/version' + prometheus: + auth: + type: bearer + use_kiali_token: true + thanos_proxy: + enabled: true + url: https://thanos-querier.openshift-monitoring.svc.cluster.local:9091 + ``` +#### Integrating Kiali with OpenShift Distributed Tracing +This section describes how to setup Kiali with OpenShift Distributed Tracing to read the distributed traces. + +*Prerequisites* +* Istio tracing is [Configured with OpenShift distributed tracing](#configure-tracing-with-openshift-distributed-tracing) + +*Steps* +1. Setup Kiali to access traces from the Tempo frontend: + ```yaml + external_services: + grafana: + enabled: true + url: "http://grafana-istio-system.apps-crc.testing/" + tracing: + enabled: true + provider: tempo + use_grpc: false + in_cluster_url: http://tempo-sample-query-frontend.tempo:3200 + url: 'https://tempo-sample-query-frontend-tempo.apps-crc.testing' + tempo_config: + org_id: "1" + datasource_uid: "a8d2ef1c-d31c-4de5-a90b-e7bc5252cd00" + ``` + +Where: +* `external_services.grafana` section: Is just needed to see the "View in Tracing" link from the Traces tab +* `external_services.tracing.tempo_config`: Is just needed to see the "View in Tracing" link from the Traces tab and redirect to the proper Tempo datasource + +Now, we should be able to see traces from Kiali. For this, you can: +1. Select a Workload/Service/App +2. Click in the "Traces" tab \ No newline at end of file diff --git a/docs/common/create-and-configure-gateways.md b/docs/common/create-and-configure-gateways.md index c3db3523d..390629109 100644 --- a/docs/common/create-and-configure-gateways.md +++ b/docs/common/create-and-configure-gateways.md @@ -1,12 +1,16 @@ -## Creating and Configuring Gateways +[Return to Project Root](../README.md) + +# Table of Contents -The Sail Operator does not deploy Ingress or Egress Gateways. Gateways are not -part of the control plane. As a security best-practice, Ingress and Egress -Gateways should be deployed in a different namespace than the namespace that -contains the control plane. +- [Creating and Configuring Gateways](#creating-and-configuring-gateways) + - [Option 1: Istio Gateway Injection](#option-1-istio-gateway-injection) + - [Option 2: Kubernetes Gateway API](#option-2-kubernetes-gateway-api) + +## Creating and Configuring Gateways -You can deploy gateways using either the Gateway API or Gateway Injection methods. +[Gateways in Istio](https://istio.io/latest/docs/concepts/traffic-management/#gateways) are used to manage inbound and outbound traffic for the mesh. The Sail Operator does not deploy or manage Gateways. You can deploy a gateway either through [gateway-api](https://istio.io/latest/docs/tasks/traffic-management/ingress/gateway-api/) or through [gateway injection](https://istio.io/latest/docs/setup/additional-setup/gateway/#deploying-a-gateway). As you are following the gateway installation instructions, skip the step to install Istio since this is handled by the Sail Operator. +**Note:** The `IstioOperator` / `istioctl` example is separate from the Sail Operator. Setting `spec.components` or `spec.values.gateways` on your Sail Operator `Istio` resource **will not work**. ### Option 1: Istio Gateway Injection @@ -16,7 +20,7 @@ that can be made accessible from outside the cluster. For more information, see [Installing Gateways](https://istio.io/latest/docs/setup/additional-setup/gateway/#deploying-a-gateway). To configure gateway injection with the `bookinfo` application, we have provided -a [sample gateway configuration](../../chart/samples/ingress-gateway.yaml?raw=1) that should be applied in the namespace +a [sample gateway configuration](../../chart/samples/ingress-gateway.yaml) that should be applied in the namespace where the application is installed: 1. Create the `istio-ingressgateway` deployment and service: diff --git a/docs/common/install-bookinfo-app.md b/docs/common/install-bookinfo-app.md index 0026b4d8e..f438df7cc 100644 --- a/docs/common/install-bookinfo-app.md +++ b/docs/common/install-bookinfo-app.md @@ -1,3 +1,5 @@ +[Return to Project Root](../README.md) + ## Installing the Bookinfo Application You can use the `bookinfo` example application to explore service mesh features. diff --git a/docs/common/install-istioctl-tool.md b/docs/common/install-istioctl-tool.md index 68fa0dcf3..1cbc14f0f 100644 --- a/docs/common/install-istioctl-tool.md +++ b/docs/common/install-istioctl-tool.md @@ -1,3 +1,11 @@ +[Return to Project Root](../README.md) + +# Table of Contents + +- [Installing the istioctl tool](#installing-the-istioctl-tool) + - [Prerequisites](#prerequisites) + - [Procedure](#procedure) + ## Installing the istioctl tool The `istioctl` tool is a configuration command line utility that allows service diff --git a/docs/common/istio-addons-integrations.md b/docs/common/istio-addons-integrations.md index 36f0ee1da..5dfebf2d1 100644 --- a/docs/common/istio-addons-integrations.md +++ b/docs/common/istio-addons-integrations.md @@ -1,3 +1,13 @@ +[Return to Project Root](../README.md) + +# Table of Contents + +- [Istio Addons Integrations](#istio-addons-integrations) + - [Prometheus](#prometheus) + - [Grafana](#grafana) + - [Jaeger](#jaeger) + - [Kiali](#kiali) + ## Istio Addons Integrations Istio can be integrated with other software to provide additional functionality diff --git a/docs/common/istio-ambient-mode.md b/docs/common/istio-ambient-mode.md index 6695caa97..f857254cf 100644 --- a/docs/common/istio-ambient-mode.md +++ b/docs/common/istio-ambient-mode.md @@ -1,4 +1,21 @@ +[Return to Project Root](../README.md) + +# Table of Contents + +- [Introduction to Istio Ambient mode](#introduction-to-istio-ambient-mode) + - [Component version](#component-version) + - [Concepts](#concepts) + - [ZTunnel resource](#ztunnel-resource) + - [API Reference documentation](#api-reference-documentation) + - [Core features](#core-features) + - [Getting Started](#getting-started) + - [Installation on OpenShift](#installation-on-openshift) + - [Deploy a sample application](#deploy-a-sample-application) + - [Visualize the application using Kiali dashboard](#visualize-the-application-using-kiali-dashboard) + - [Troubleshoot issues](#troubleshoot-issues) + - [Cleanup](#cleanup) + ## Introduction to Istio Ambient mode Ambient mesh was announced in September 2022 and reached a stable state in the Istio 1.24 release. It's core innovation is the separation of Layer 4 (L4) and Layer 7 (L7) processing into two distinct layers. It uses lightweight, shared L4 node proxies and optional L7 proxies, eliminating the need for traditional sidecar proxies in the data plane. @@ -36,7 +53,7 @@ Note: If you need a specific Istio version, you can explicitly set it using `spe ### API Reference documentation -The ZTunnel resource API reference documentation can be found [here](https://github.com/istio-ecosystem/sail-operator/blob/main/docs/api-reference/sailoperator.io.md#ztunnel). +The ZTunnel resource API reference documentation can be found [here](../api-reference/sailoperator.io.md#ztunnel). ## Core features @@ -56,7 +73,7 @@ The ZTunnel resource API reference documentation can be found [here](https://git *Steps* -1. Install the Sail Operator using the CLI or through the web console. The steps can be found [here](https://github.com/istio-ecosystem/sail-operator/blob/main/docs/README.md#installation-on-openshift). +1. Install the Sail Operator using the CLI or through the web console. The steps can be found [here](../general/getting-started.md#installation-on-openshift). 2. Create the `istio-system` namespace and add a label `istio-discovery=enabled`. diff --git a/docs/common/istio-ambient-waypoint.md b/docs/common/istio-ambient-waypoint.md index 9b468eee4..5a634531c 100644 --- a/docs/common/istio-ambient-waypoint.md +++ b/docs/common/istio-ambient-waypoint.md @@ -1,5 +1,25 @@ - -## Introduction to Istio Waypoint Proxy +[Return to Project Root](../README.md) + +# Table of Contents +- [Introduction to Istio Waypoint Proxy](#introduction-to-istio-waypoint-proxy) + - [Core features](#core-features) + - [Getting Started](#getting-started) + - [Prerequisites](#prerequisites) + - [Set up Istio Ambient Mode Resources and a Sample Application](#set-up-istio-ambient-mode-resources-and-a-sample-application) + - [Deploy a Waypoint Proxy](#deploy-a-waypoint-proxy) + - [Cross-namespace Waypoint](#cross-namespace-waypoint) + - [Layer 7 Features in Ambient Mode](#layer-7-features-in-ambient-mode) + - [Traffic Routing](#traffic-routing) + - [Security Authorization](#security-authorization) + - [Security Authentication](#security-authentication) + - [Troubleshoot issues](#troubleshoot-issues) + - [Cleanup](#cleanup) + - [Remove Istio and Gateway Resources](#remove-istio-and-gateway-resources) + - [Remove the namespace from the ambient data plane](#remove-the-namespace-from-the-ambient-data-plane) + - [Remove the Bookinfo applications](#remove-the-bookinfo-applications) + - [Remove the Kubernetes Gateway API CRDs](#remove-the-kubernetes-gateway-api-crds) + +# Introduction to Istio Waypoint Proxy Ambient mesh splits Istio's functionality into two distinct layers, a secure overlay layer 4 (L4) and a Layer 7 (L7). The waypoint proxy is an optional component that is Envoy-based and handles L7 processing for workloads it manages. It acts as a gateway to a resource (a namespace, service or pod). Waypoint proxies are installed, upgraded and scaled independently from applications. They can be configured using the Kubernetes [Gateway API](https://gateway-api.sigs.k8s.io/). @@ -13,7 +33,7 @@ If your applications require any of the following L7 mesh functions, you will ne ## Getting Started -*Prerequisites* +### Prerequisites Waypoint proxies are deployed using Kubernetes Gateway resources. As of Kubernetes 1.30 and OpenShift 4.17, the Kubernetes Gateway API CRDs are not available by default and must be installed to be used. This can be done with the following command: @@ -24,9 +44,9 @@ kubectl get crd gateways.gateway.networking.k8s.io &> /dev/null || \ ### Set up Istio Ambient Mode Resources and a Sample Application -1. Install the Sail Operator along with Istio in ambient mode using the [following](https://github.com/istio-ecosystem/sail-operator/blob/main/docs/common/istio-ambient-mode.md#installation-on-openshift) steps. +1. Install the Sail Operator along with Istio in ambient mode using the [following](istio-ambient-mode.md#installation-on-openshift) steps. -2. Deploy the sample Bookinfo applications. The steps can be found [here](https://github.com/istio-ecosystem/sail-operator/blob/main/docs/common/istio-ambient-mode.md#deploy-a-sample-application). +2. Deploy the sample Bookinfo applications. The steps can be found [here](istio-ambient-mode.md#deploy-a-sample-application). Before you deploy a waypoint proxy in the application namespace, confirm the namespace is labeled with `istio.io/dataplane-mode: ambient`. @@ -62,7 +82,7 @@ kubectl label ns bookinfo istio.io/use-waypoint=waypoint After a namespace is enrolled to use a waypoint, any requests from any pods using the ambient data plane mode, to any service running in the `bookinfo` namespace, will be routed through the waypoint for L7 processing and policy enforcement. -If you prefer more granularity than using a waypoint for an entire namespace, you can enroll only a specific service or pod to use a waypoint by labelling the respective service or the pod. When enrolling a pod explicitly, you must also add the `istio.io/waypoint-for: workload` label to the Gateway resource. +If you prefer more granularity than using a waypoint for an entire namespace, you can enroll only a specific service or pod to use a waypoint by labeling the respective service or the pod. When enrolling a pod explicitly, you must also add the `istio.io/waypoint-for: workload` label to the Gateway resource. #### Cross-namespace Waypoint By default, a waypoint is usable only within the same namespace, but it also supports cross-namespace usage. The following Gateway allows resources in the `bookinfo` namespace to use `waypoint-foo` from the `foo` namespace: diff --git a/docs/deployment-models/multicluster.md b/docs/deployment-models/multicluster.md new file mode 100644 index 000000000..1c8010521 --- /dev/null +++ b/docs/deployment-models/multicluster.md @@ -0,0 +1,778 @@ +[Return to Project Root](../README.md) + +# Table of Contents + +- [Multi-cluster](#multi-cluster) + - [Prerequisites](#prerequisites) + - [Common Setup](#common-setup) + - [Multi-Primary - Multi-Network](#multi-primary---multi-network) + - [Primary-Remote - Multi-Network](#primary-remote---multi-network) + - [External Control Plane](#external-control-plane) + +## Multi-cluster + +You can use the Sail Operator and the Sail CRDs to manage a multi-cluster Istio deployment. The following instructions are adapted from the [Istio multi-cluster documentation](https://istio.io/latest/docs/setup/install/multicluster/) to demonstrate how you can setup the various deployment models with Sail. Please familiarize yourself with the different [deployment models](https://istio.io/latest/docs/ops/deployment/deployment-models/) before starting. + +### Prerequisites + +- Install [istioctl](../common/install-istioctl-tool.md). +- Two kubernetes clusters with external lb support. (If using kind, `cloud-provider-kind` is running in the background) +- kubeconfig file with a context for each cluster. +- Install the Sail Operator and the Sail CRDs to every cluster. + +### Common Setup + +These steps are common to every multi-cluster deployment and should be completed *after* meeting the prerequisites but *before* starting on a specific deployment model. + +1. Setup environment variables. + + ```bash + export CTX_CLUSTER1= + export CTX_CLUSTER2= + export ISTIO_VERSION=1.26.0 + ``` + +2. Create `istio-system` namespace on each cluster. + + ```bash + kubectl get ns istio-system --context "${CTX_CLUSTER1}" || kubectl create namespace istio-system --context "${CTX_CLUSTER1}" + kubectl get ns istio-system --context "${CTX_CLUSTER2}" || kubectl create namespace istio-system --context "${CTX_CLUSTER2}" + ``` + +4. Create a shared root certificate. + + If you have [established trust](https://istio.io/latest/docs/setup/install/multicluster/before-you-begin/#configure-trust) between your clusters already you can skip this and the following steps. + + ```bash + openssl genrsa -out root-key.pem 4096 + cat < root-ca.conf + [ req ] + encrypt_key = no + prompt = no + utf8 = yes + default_md = sha256 + default_bits = 4096 + req_extensions = req_ext + x509_extensions = req_ext + distinguished_name = req_dn + [ req_ext ] + subjectKeyIdentifier = hash + basicConstraints = critical, CA:true + keyUsage = critical, digitalSignature, nonRepudiation, keyEncipherment, keyCertSign + [ req_dn ] + O = Istio + CN = Root CA + EOF + + openssl req -sha256 -new -key root-key.pem \ + -config root-ca.conf \ + -out root-cert.csr + + openssl x509 -req -sha256 -days 3650 \ + -signkey root-key.pem \ + -extensions req_ext -extfile root-ca.conf \ + -in root-cert.csr \ + -out root-cert.pem + ``` +5. Create intermediate certificates. + + ```bash + for cluster in west east; do + mkdir $cluster + + openssl genrsa -out ${cluster}/ca-key.pem 4096 + cat < ${cluster}/intermediate.conf + [ req ] + encrypt_key = no + prompt = no + utf8 = yes + default_md = sha256 + default_bits = 4096 + req_extensions = req_ext + x509_extensions = req_ext + distinguished_name = req_dn + [ req_ext ] + subjectKeyIdentifier = hash + basicConstraints = critical, CA:true, pathlen:0 + keyUsage = critical, digitalSignature, nonRepudiation, keyEncipherment, keyCertSign + subjectAltName=@san + [ san ] + DNS.1 = istiod.istio-system.svc + [ req_dn ] + O = Istio + CN = Intermediate CA + L = $cluster + EOF + + openssl req -new -config ${cluster}/intermediate.conf \ + -key ${cluster}/ca-key.pem \ + -out ${cluster}/cluster-ca.csr + + openssl x509 -req -sha256 -days 3650 \ + -CA root-cert.pem \ + -CAkey root-key.pem -CAcreateserial \ + -extensions req_ext -extfile ${cluster}/intermediate.conf \ + -in ${cluster}/cluster-ca.csr \ + -out ${cluster}/ca-cert.pem + + cat ${cluster}/ca-cert.pem root-cert.pem \ + > ${cluster}/cert-chain.pem + cp root-cert.pem ${cluster} + done + ``` + +6. Push the intermediate CAs to each cluster. + ```bash + kubectl --context "${CTX_CLUSTER1}" label namespace istio-system topology.istio.io/network=network1 + kubectl get secret -n istio-system --context "${CTX_CLUSTER1}" cacerts || kubectl create secret generic cacerts -n istio-system --context "${CTX_CLUSTER1}" \ + --from-file=east/ca-cert.pem \ + --from-file=east/ca-key.pem \ + --from-file=east/root-cert.pem \ + --from-file=east/cert-chain.pem + kubectl --context "${CTX_CLUSTER2}" label namespace istio-system topology.istio.io/network=network2 + kubectl get secret -n istio-system --context "${CTX_CLUSTER2}" cacerts || kubectl create secret generic cacerts -n istio-system --context "${CTX_CLUSTER2}" \ + --from-file=west/ca-cert.pem \ + --from-file=west/ca-key.pem \ + --from-file=west/root-cert.pem \ + --from-file=west/cert-chain.pem + ``` + +### Multi-Primary - Multi-Network + +These instructions install a [multi-primary/multi-network](https://istio.io/latest/docs/setup/install/multicluster/multi-primary_multi-network/) Istio deployment using the Sail Operator and Sail CRDs. **Before you begin**, ensure you complete the [common setup](#common-setup). + +You can follow the steps below to install manually or you can run [this script](resources/setup-multi-primary.sh) which will setup a local environment for you with kind. Before running the setup script, you must install [kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) and [cloud-provider-kind](https://kind.sigs.k8s.io/docs/user/loadbalancer/#installing-cloud-provider-kind) then ensure the `cloud-provider-kind` binary is running in the background. + +These installation instructions are adapted from: https://istio.io/latest/docs/setup/install/multicluster/multi-primary_multi-network/. + +1. Create an `Istio` resource on `cluster1`. + + ```bash + kubectl apply --context "${CTX_CLUSTER1}" -f - < /dev/null || \ + { kubectl kustomize "github.com/kubernetes-sigs/gateway-api/config/crd?ref=v1.1.0" | kubectl apply -f - --context="${CTX_CLUSTER2}"; } + ``` + + Expose `helloworld` through the ingress gateway. + ```bash + kubectl apply -f "https://raw.githubusercontent.com/istio/istio/${ISTIO_VERSION}/samples/helloworld/gateway-api/helloworld-gateway.yaml" -n sample --context="${CTX_CLUSTER2}" + kubectl -n sample --context="${CTX_CLUSTER2}" wait --for=condition=programmed gtw helloworld-gateway + ``` + + Confirm you can access the `helloworld` application through the ingress gateway created in the Remote cluster. + ```bash + curl -s "http://$(kubectl -n sample --context="${CTX_CLUSTER2}" get gtw helloworld-gateway -o jsonpath='{.status.addresses[0].value}'):80/hello" + ``` + You should see a response from the `helloworld` application: + ```bash + Hello version: v1, instance: helloworld-v1-6d65866976-jb6qc + ``` + +15. Cleanup + + ```bash + kubectl delete istios default --context="${CTX_CLUSTER1}" + kubectl delete ns istio-system --context="${CTX_CLUSTER1}" + kubectl delete istios external-istiod --context="${CTX_CLUSTER1}" + kubectl delete ns external-istiod --context="${CTX_CLUSTER1}" + kubectl delete istios external-istiod --context="${CTX_CLUSTER2}" + kubectl delete ns external-istiod --context="${CTX_CLUSTER2}" + kubectl delete ns sample --context="${CTX_CLUSTER2}" + ``` diff --git a/docs/deployment-models/multiple-mesh.md b/docs/deployment-models/multiple-mesh.md new file mode 100644 index 000000000..4435241c2 --- /dev/null +++ b/docs/deployment-models/multiple-mesh.md @@ -0,0 +1,324 @@ +[Return to Project Root](../../README.md) + +# Table of Contents +- [Multiple meshes on a single cluster](#multiple-meshes-on-a-single-cluster) + - [Prerequisites](#prerequisites) + - [Installation Steps](#installation-steps) + - [Deploying the control planes](#deploying-the-control-planes) + - [Verifying the control planes](#verifying-the-control-planes) + - [Deploying the applications](#deploying-the-applications) + - [Validation](#validation) + - [Checking application to control plane mapping](#checking-application-to-control-plane-mapping) + - [Checking application connectivity](#checking-application-connectivity) + - [Cleanup](#cleanup) + +## Multiple meshes on a single cluster + +The Sail Operator supports running multiple meshes on a single cluster and associating each workload with a specific mesh. +Each mesh is managed by a separate control plane. + +Applications are installed in multiple namespaces, and each namespace is associated with one of the control planes through its labels. +The `istio.io/rev` label determines which control plane injects the sidecar proxy into the application pods. +Additional namespace labels determine whether the control plane discovers and manages the resources in the namespace. +A control plane will discover and manage only those namespaces that match the discovery selectors configured on the control plane. +Additionally, discovery selectors determine which control plane creates the `istio-ca-root-cert` ConfigMap in which namespace. + +Currently, discovery selectors in multiple control planes must be configured so that they don't overlap (i.e. the discovery selectors of two control planes don't match the same namespace). +Each control plane must be deployed in a separate Kubernetes namespace. + +This guide explains how to set up two meshes: `mesh1` and `mesh2` in namespaces `istio-system1` and `istio-system2`, respectively, and three application namespaces: `app1`, `app2a`, and `app2b`. +Mesh 1 will manage namespace `app1`, and Mesh 2 will manage namespaces `app2a` and `app2b`. +Because each mesh will use its own root certificate authority and configured to use a peer authentication policy with the `STRICT` mTLS mode, the communication between the two meshes will not be allowed. + +### Prerequisites + +- Install [istioctl](../../docs/common/install-istioctl-tool.md). +- Kubernetes 1.23 cluster. +- kubeconfig file with a context for the Kubernetes cluster. +- Install the Sail Operator and the Sail CRDs to the cluster. + +### Installation Steps + +#### Deploying the control planes + +1. Create the system namespace `istio-system1` and deploy the `mesh1` control plane in it. + ```bash { name=deploy-mesh1 tag=multiple-meshes} + kubectl create namespace istio-system1 + kubectl label ns istio-system1 mesh=mesh1 + kubectl apply -f - < +2. Create the system namespace `istio-system2` and deploy the `mesh2` control plane in it. + ```bash { name=deploy-mesh2 tag=multiple-meshes} + kubectl create namespace istio-system2 + kubectl label ns istio-system2 mesh=mesh2 + kubectl apply -f - < +3. Create a peer authentication policy that only allows mTLS communication within each mesh. + ```bash { name=peer-authentication tag=multiple-meshes} + kubectl apply -f - < +2. Check the control planes are `Healthy`: + ```console + $ kubectl get istios + NAME REVISIONS READY IN USE ACTIVE REVISION STATUS VERSION AGE + mesh1 1 1 0 mesh1 Healthy v1.24.0 84s + mesh2 1 1 0 mesh2 Healthy v1.24.0 77s + ``` + +3. Confirm that the validation and mutation webhook configurations exist for both meshes: + ```console + $ kubectl get validatingwebhookconfigurations + NAME WEBHOOKS AGE + istio-validator-mesh1-istio-system1 1 2m45s + istio-validator-mesh2-istio-system2 1 2m38s + + $ kubectl get mutatingwebhookconfigurations + NAME WEBHOOKS AGE + istio-sidecar-injector-mesh1-istio-system1 2 5m55s + istio-sidecar-injector-mesh2-istio-system2 2 5m48s + ``` + +#### Deploying the applications + +1. Create three application namespaces: + ```bash { name=create-app-namespaces tag=multiple-meshes} + kubectl create ns app1 + kubectl create ns app2a + kubectl create ns app2b + ``` + +2. Label each namespace to enable discovery by the corresponding control plane: + ```bash { name=label-app-namespaces tag=multiple-meshes} + kubectl label ns app1 mesh=mesh1 + kubectl label ns app2a mesh=mesh2 + kubectl label ns app2b mesh=mesh2 + ``` + +3. Label each namespace to enable injection by the corresponding control plane: + ```bash { name=label-rev-app-namespaces tag=multiple-meshes} + kubectl label ns app1 istio.io/rev=mesh1 + kubectl label ns app2a istio.io/rev=mesh2 + kubectl label ns app2b istio.io/rev=mesh2 + ``` + +4. Deploy the `curl` and `httpbin` sample applications in each namespace: + ```bash { name=deploy-apps tag=multiple-meshes} + # Deploy curl and httpbin in app1 + kubectl -n app1 apply -f https://raw.githubusercontent.com/istio/istio/refs/heads/master/samples/curl/curl.yaml + kubectl -n app1 apply -f https://raw.githubusercontent.com/istio/istio/refs/heads/master/samples/httpbin/httpbin.yaml + # Deploy curl and httpbin in app2a and app2b + kubectl -n app2a apply -f https://raw.githubusercontent.com/istio/istio/refs/heads/master/samples/curl/curl.yaml + kubectl -n app2a apply -f https://raw.githubusercontent.com/istio/istio/refs/heads/master/samples/httpbin/httpbin.yaml + # Deploy curl and httpbin in app2b + kubectl -n app2b apply -f https://raw.githubusercontent.com/istio/istio/refs/heads/master/samples/curl/curl.yaml + kubectl -n app2b apply -f https://raw.githubusercontent.com/istio/istio/refs/heads/master/samples/httpbin/httpbin.yaml + ``` + +5. Confirm that a sidecar has been injected into each of the application pods. The value `2/2` should be displayed in the `READY` column for each pod, as in the following example: + ```console + $ kubectl get pods -n app1 + NAME READY STATUS RESTARTS AGE + curl-5b549b49b8-mg7nl 2/2 Running 0 102s + httpbin-7b549f7859-h6hnk 2/2 Running 0 89s + + $ kubectl get pods -n app2a + NAME READY STATUS RESTARTS AGE + curl-5b549b49b8-2hlvm 2/2 Running 0 2m3s + httpbin-7b549f7859-bgblg 2/2 Running 0 110s + + $ kubectl get pods -n app2b + NAME READY STATUS RESTARTS AGE + curl-5b549b49b8-xnzzk 2/2 Running 0 2m9s + httpbin-7b549f7859-7k5gf 2/2 Running 0 118s + ``` + +### Validation + +#### Checking application to control plane mapping + +Use the `istioctl ps` command to confirm that the application pods are connected to the correct control plane. + +The `curl` and `httpbin` pods in namespace `app1` should be connected to the control plane in namespace `istio-system1`, as shown in the following example (note the `.app1` suffix in the `NAME` column): + +```console +$ istioctl ps -i istio-system1 +NAME CLUSTER CDS LDS EDS RDS ECDS ISTIOD VERSION +curl-5b549b49b8-mg7nl.app1 Kubernetes SYNCED (4m40s) SYNCED (4m40s) SYNCED (4m31s) SYNCED (4m40s) IGNORED istiod-mesh1-5df45b97dd-tf2wl 1.24.0 +httpbin-7b549f7859-h6hnk.app1 Kubernetes SYNCED (4m31s) SYNCED (4m31s) SYNCED (4m31s) SYNCED (4m31s) IGNORED istiod-mesh1-5df45b97dd-tf2wl 1.24.0 +``` + +The pods in namespaces `app2a` and `app2b` should be connected to the control plane in namespace `istio-system2`: + +```console +$ istioctl ps -i istio-system2 +NAME CLUSTER CDS LDS EDS RDS ECDS ISTIOD VERSION +curl-5b549b49b8-2hlvm.app2a Kubernetes SYNCED (4m37s) SYNCED (4m37s) SYNCED (4m31s) SYNCED (4m37s) IGNORED istiod-mesh2-59f6b874fb-mzxqw 1.24.0 +curl-5b549b49b8-xnzzk.app2b Kubernetes SYNCED (4m37s) SYNCED (4m37s) SYNCED (4m31s) SYNCED (4m37s) IGNORED istiod-mesh2-59f6b874fb-mzxqw 1.24.0 +httpbin-7b549f7859-7k5gf.app2b Kubernetes SYNCED (4m31s) SYNCED (4m31s) SYNCED (4m31s) SYNCED (4m31s) IGNORED istiod-mesh2-59f6b874fb-mzxqw 1.24.0 +httpbin-7b549f7859-bgblg.app2a Kubernetes SYNCED (4m32s) SYNCED (4m32s) SYNCED (4m31s) SYNCED (4m32s) IGNORED istiod-mesh2-59f6b874fb-mzxqw 1.24.0 +``` + +#### Checking application connectivity + +As both meshes are configured to use the `STRICT` mTLS peer authentication mode, the applications in namespace `app1` should not be able to communicate with the applications in namespaces `app2a` and `app2b`, and vice versa. +To test whether the `curl` pod in namespace `app2a` can connect to the `httpbin` service in namespace `app1`, run the following commands: + +```console +$ kubectl -n app2a exec deploy/curl -c curl -- curl -sIL http://httpbin.app1:8000 +HTTP/1.1 503 Service Unavailable +content-length: 95 +content-type: text/plain +date: Fri, 29 Nov 2024 08:58:28 GMT +server: envoy +``` + +As expected, the response indicates that the connection was not successful. +In contrast, the same pod should be able to connect to the `httpbin` service in namespace `app2b`, because they are part of the same mesh: + +```console +$ kubectl -n app2a exec deploy/curl -c curl -- curl -sIL http://httpbin.app2b:8000 +HTTP/1.1 200 OK +access-control-allow-credentials: true +access-control-allow-origin: * +content-security-policy: default-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' camo.githubusercontent.com +content-type: text/html; charset=utf-8 +date: Fri, 29 Nov 2024 08:57:52 GMT +x-envoy-upstream-service-time: 0 +server: envoy +transfer-encoding: chunked +``` + +### Cleanup + +To clean up the resources created in this guide, delete the `Istio` resources and the namespaces: + +```bash + kubectl delete istio mesh1 mesh2 + kubectl delete ns istio-system1 istio-system2 app1 app2a app2b +``` diff --git a/docs/multicluster/controlplane-gateway.yaml b/docs/deployment-models/resources/controlplane-gateway.yaml similarity index 100% rename from docs/multicluster/controlplane-gateway.yaml rename to docs/deployment-models/resources/controlplane-gateway.yaml diff --git a/docs/multicluster/east-west-gateway-net1.yaml b/docs/deployment-models/resources/east-west-gateway-net1.yaml similarity index 100% rename from docs/multicluster/east-west-gateway-net1.yaml rename to docs/deployment-models/resources/east-west-gateway-net1.yaml diff --git a/docs/multicluster/east-west-gateway-net2.yaml b/docs/deployment-models/resources/east-west-gateway-net2.yaml similarity index 100% rename from docs/multicluster/east-west-gateway-net2.yaml rename to docs/deployment-models/resources/east-west-gateway-net2.yaml diff --git a/docs/multicluster/expose-istiod.yaml b/docs/deployment-models/resources/expose-istiod.yaml similarity index 100% rename from docs/multicluster/expose-istiod.yaml rename to docs/deployment-models/resources/expose-istiod.yaml diff --git a/docs/multicluster/expose-services.yaml b/docs/deployment-models/resources/expose-services.yaml similarity index 100% rename from docs/multicluster/expose-services.yaml rename to docs/deployment-models/resources/expose-services.yaml diff --git a/docs/multicluster/setup-multi-primary.sh b/docs/deployment-models/resources/setup-multi-primary.sh similarity index 100% rename from docs/multicluster/setup-multi-primary.sh rename to docs/deployment-models/resources/setup-multi-primary.sh diff --git a/docs/dual-stack/dual-stack.md b/docs/dual-stack/dual-stack.md new file mode 100644 index 000000000..447b05e13 --- /dev/null +++ b/docs/dual-stack/dual-stack.md @@ -0,0 +1,195 @@ +[Return to Project Root](../../README.md) + +# Table of Contents + +- [Dual-stack Support](#dual-stack-support) + - [Prerequisites](#prerequisites) + - [Installation Steps](#installation-steps) + - [Validation](#validation) + - [Cleanup](#cleanup) + +## Dual-stack Support + +Kubernetes supports dual-stack networking as a stable feature starting from +[v1.23](https://kubernetes.io/docs/concepts/services-networking/dual-stack/), allowing clusters to handle both +IPv4 and IPv6 traffic. With many cloud providers also beginning to offer dual-stack Kubernetes clusters, it's easier +than ever to run services that function across both address types. Istio introduced dual-stack as an experimental +feature in version 1.17, and promoted it to [Alpha](https://istio.io/latest/news/releases/1.24.x/announcing-1.24/change-notes/) in +version 1.24. With Istio in dual-stack mode, services can communicate over both IPv4 and IPv6 endpoints, which helps +organizations transition to IPv6 while still maintaining compatibility with their existing IPv4 infrastructure. + +When Kubernetes is configured for dual-stack, it automatically assigns an IPv4 and an IPv6 address to each pod, +enabling them to communicate over both IP families. For services, however, you can control how they behave using +the `ipFamilyPolicy` setting. + +Service.Spec.ipFamilyPolicy can take the following values +- SingleStack: Only one IP family is configured for the service, which can be either IPv4 or IPv6. +- PreferDualStack: Both IPv4 and IPv6 cluster IPs are assigned to the Service when dual-stack is enabled. + However, if dual-stack is not enabled or supported, it falls back to singleStack behavior. +- RequireDualStack: The service will be created only if both IPv4 and IPv6 addresses can be assigned. + +This allows you to specify the type of service, providing flexibility in managing your network configuration. +For more details, you can refer to the Kubernetes [documentation](https://kubernetes.io/docs/concepts/services-networking/dual-stack/#services). + +### Prerequisites + +- Kubernetes 1.23 or later configured with dual-stack support. +- Sail Operator is installed. + +### Installation Steps + +You can use any existing Kind cluster that supports dual-stack networking or, alternatively, install one using the following command. + +```bash +kind create cluster --name istio-ds --config - < +5. Verify that sleep pod is able to reach the dual-stack pods. + ```console + $ kubectl exec -n sleep "$(kubectl get pod -n sleep -l app=sleep -o jsonpath='{.items[0].metadata.name}')" -- sh -c "echo dualstack | nc tcp-echo.dual-stack 9000" + hello dualstack + ``` + +6. Similarly verify that sleep pod is able to reach both ipv4 pods as well as ipv6 pods. + ```console + $ kubectl exec -n sleep "$(kubectl get pod -n sleep -l app=sleep -o jsonpath='{.items[0].metadata.name}')" -- sh -c "echo ipv4 | nc tcp-echo.ipv4 9000" + hello ipv4 + ``` + + ```console + $ kubectl exec -n sleep "$(kubectl get pod -n sleep -l app=sleep -o jsonpath='{.items[0].metadata.name}')" -- sh -c "echo ipv6 | nc tcp-echo.ipv6 9000" + hello ipv6 + ``` + +### Cleanup +To clean up the resources created during this example, you can run the following commands: + ```bash + kubectl delete istios default + kubectl delete ns istio-system + kubectl delete istiocni default + kubectl delete ns istio-cni + kubectl delete ns dual-stack ipv4 ipv6 sleep + ``` diff --git a/docs/general/getting-started.md b/docs/general/getting-started.md new file mode 100644 index 000000000..892f77544 --- /dev/null +++ b/docs/general/getting-started.md @@ -0,0 +1,235 @@ +[Return to Project Root](../README.md) + +# Table of Contents + +- [Getting Started](#getting-started) + - [Installation on OpenShift](#installation-on-openshift) + - [Installing through the web console](#installing-through-the-web-console) + - [Installing using the CLI](#installing-using-the-cli) + - [Installation from Source](#installation-from-source) + - [Migrating from Istio in-cluster Operator](#migrating-from-istio-in-cluster-operator) + - [Converter Script to Migrate Istio in-cluster Operator Configuration to Sail Operator](#converter-script-to-migrate-istio-in-cluster-operator-configuration-to-sail-operator) + - [Usage](#usage) + - [Sample command only with input file:](#sample-command-only-with-input-file) + - [Sample command with custom output, namespace, and version:](#sample-command-with-custom-output-namespace-and-version) + - [Uninstalling](#uninstalling) + - [Deleting Istio](#deleting-istio) + - [Deleting IstioCNI](#deleting-istiocni) + - [Deleting the Sail Operator](#deleting-the-sail-operator) + - [Deleting the istio-system and istio-cni Projects](#deleting-the-istio-system-and-istio-cni-projects) + - [Decide whether you want to delete the CRDs as well](#decide-whether-you-want-to-delete-the-crds-as-well) + +## Getting Started + +## Installation on OpenShift + +### Installing through the web console + +1. In the OpenShift Console, navigate to the OperatorHub by clicking **Operator** -> **Operator Hub** in the left side-pane. + +1. Search for "sail". + +1. Locate the Sail Operator, and click to select it. + +1. When the prompt that discusses the community operator appears, click **Continue**, then click **Install**. + +1. Use the default installation settings presented, and click **Install** to continue. + +1. Click **Operators** -> **Installed Operators** to verify that the Sail Operator +is installed. `Succeeded` should appear in the **Status** column. + +### Installing using the CLI + +*Prerequisites* + +* You have access to the cluster as a user with the `cluster-admin` cluster role. + +*Steps* + +1. Create the `openshift-operators` namespace (if it does not already exist). + + ```bash + kubectl create namespace openshift-operators + ``` + +1. Create the `Subscription` object with the desired `spec.channel`. + + ```bash + kubectl apply -f - <.enabled: true/false`. Other functionality exposed through `spec.components` like the k8s overlays is not currently available. + +### CNI lifecycle management + +The CNI plugin's lifecycle is managed separately from the control plane. You will have to create a [IstioCNI resource](../README.md#istiocni-resource) to use CNI. + +### Converter Script to Migrate Istio in-cluster Operator Configuration to Sail Operator + +This script is used to convert an Istio in-cluster operator configuration to a Sail Operator configuration. Upon execution, the script takes an input YAML file and istio version and generates a sail operator configuration file. + +#### Usage +To run the configuration-converter.sh script, you need to provide four arguments, only input file is required other arguments are optional: + +1. Input file path (): The path to your Istio operator configuration YAML file (required). +2. Output file path (): The path where the converted Sail configuration will be saved. If not provided, the script will save the output with -sail.yaml appended to the input file name. +3. Namespace (-n ): The Kubernetes namespace for the Istio deployment. Defaults to istio-system if not provided. +4. Version (-v ): The version of Istio to be used. If not provided, the `spec.version` field will be omitted from the output file and the operator will deploy the latest version when the YAML manifest is applied. + +```bash +./tools/configuration-converter.sh [/path/to/output.yaml] [-n namespace] [-v version] +``` + +##### Sample command only with input file: + +```bash +./tools/configuration-converter.sh /home/user/istio_config.yaml +``` + +##### Sample command with custom output, namespace, and version: + +```bash +./tools/configuration-converter.sh /home/user/input/istio_config.yaml /home/user/output/output.yaml -n custom-namespace -v v1.24.3 +``` + +> [!WARNING] +> This script is still under development. +> Please verify the resulting configuration carefully after conversion to ensure it meets your expectations and requirements. + +## Uninstalling + +### Deleting Istio +1. In the OpenShift Container Platform web console, click **Operators** -> **Installed Operators**. +1. Click **Istio** in the **Provided APIs** column. +1. Click the Options menu, and select **Delete Istio**. +1. At the prompt to confirm the action, click **Delete**. + +### Deleting IstioCNI +1. In the OpenShift Container Platform web console, click **Operators** -> **Installed Operators**. +1. Click **IstioCNI** in the **Provided APIs** column. +1. Click the Options menu, and select **Delete IstioCNI**. +1. At the prompt to confirm the action, click **Delete**. + +### Deleting the Sail Operator +1. In the OpenShift Container Platform web console, click **Operators** -> **Installed Operators**. +1. Locate the Sail Operator. Click the Options menu, and select **Uninstall Operator**. +1. At the prompt to confirm the action, click **Uninstall**. + +### Deleting the istio-system and istio-cni Projects +1. In the OpenShift Container Platform web console, click **Home** -> **Projects**. +1. Locate the name of the project and click the Options menu. +1. Click **Delete Project**. +1. At the prompt to confirm the action, enter the name of the project. +1. Click **Delete**. + +### Decide whether you want to delete the CRDs as well +OLM leaves this [decision](https://olm.operatorframework.io/docs/tasks/uninstall-operator/#step-4-deciding-whether-or-not-to-delete-the-crds-and-apiservices) to the users. +If you want to delete the Istio CRDs, you can use the following command. +```bash +kubectl get crds -oname | grep istio.io | xargs kubectl delete +``` diff --git a/docs/general/istiod-ha.md b/docs/general/istiod-ha.md new file mode 100644 index 000000000..59e38dcaa --- /dev/null +++ b/docs/general/istiod-ha.md @@ -0,0 +1,153 @@ +[Return to Project Root](../../README.md) + +# Table of Contents + +- [Running Istiod in HA mode](#running-istiod-in-ha-mode) + - [Prerequisites](#prerequisites) + - [Setting up Istiod in HA mode: increasing replicaCount](#setting-up-istiod-in-ha-mode-increasing-replicacount) + - [Setting up Istiod in HA mode: using autoscaling](#setting-up-istiod-in-ha-mode-using-autoscaling) + - [Considerations for Single-Node Clusters](#considerations-for-single-node-clusters) + +# Running Istiod in HA mode +By default, istiod is deployed with replica count set to 1, to be able to run it in HA mode, you can achieve it in two different ways: +* Setting `replicaCount` to 2 or more in Istio resource and disabling autoscale (by default enabled). +* Setting `autoscaleMin` to 2 or more in Istio resource and keeping `autoscaleMax` to 2 or more. + +Pros and Cons of each approach: +- **Setting `replicaCount` to 2 or more**: + - Pros: Simplicity, easy to understand and manage. + - Cons: Fixed number of replicas, no autoscaling based on load. For single-node clusters, you may need to disable the default Pod Disruption Budget (PDB) as outlined in the [Considerations for Single-Node Clusters ](#considerations-for-single-node-clusters) section. +- **Setting `autoscaleMin` to 2 or more**: + - Pros: Autoscaling based on load, can handle increased traffic without manual intervention, more efficient resource usage. + - Cons: Requires monitoring to ensure proper scaling. + +Now, let's see how to achieve this in Sail. + +# Prerequisites +- Sail Operator installed and running in your cluster. +- kubernetes client configured to access your cluster. + +## Setting up Istiod in HA mode: increasing replicaCount +To set up Istiod in HA mode by increasing the `replicaCount`, you can create/modify the Istio resource: +```yaml +apiVersion: sailoperator.io/v1 +kind: Istio +metadata: + name: default +spec: + namespace: istio-system + values: + pilot: + autoscaleEnabled: false # <-- disable autoscaling + replicaCount: 2 # <-- number of desired replicas +``` + + +After applying this configuration, you can check the status of the Istiod pods: +```bash +kubectl get pods -n istio-system -l app=istiod +``` +You should see two pods running, indicating that Istiod is now in HA mode. +```console +NAME READY STATUS RESTARTS AGE +istiod-7c5947b8d7-88z7m 1/1 Running 0 14m +istiod-7c5947b8d7-ssnmt 1/1 Running 0 54m +``` + + +Let's break down the configuration: +- `spec.values.pilot.replicaCount: 2`: This sets the number of Istiod replicas to 2 (or the desired value), enabling HA mode. +- `spec.values.pilot.autoscaleEnabled: false`: This disables autoscaling, ensuring that the number of replicas remains fixed at 2 (or the desired value). + +## Setting up Istiod in HA mode: using autoscaling +To set up Istiod in HA mode using autoscaling, you can create/modify the Istio resource as follows: +```yaml +apiVersion: sailoperator.io/v1 +kind: Istio +metadata: + name: default +spec: + namespace: istio-system + values: + pilot: + autoscaleMin: 2 # <-- number of desired min replicas + autoscaleMax: 5 # <-- number of desired max replicas +``` + + +After applying this configuration, you can check the status of the Istiod pods: +```bash +kubectl get pods -n istio-system -l app=istiod +``` +You should see at least two pods running, indicating that Istiod is now in HA mode. +```console +NAME READY STATUS RESTARTS AGE +istiod-7c7b6564c9-nwhsg 1/1 Running 0 70s +istiod-7c7b6564c9-xkmsl 1/1 Running 0 85s +``` + +Let's break down the configuration: +- `spec.values.pilot.autoscaleMin: 2`: This sets the minimum number of Istiod replicas to 2, ensuring that there are always at least 2 replicas running. +- `spec.values.pilot.autoscaleMax: 5`: This sets the maximum number of Istiod replicas to 5, allowing for scaling based on load. + +## Considerations for Single-Node Clusters +For single-node clusters, it is crucial to disable the default Pod Disruption Budget (PDB) to prevent issues during node operations (e.g., draining) or scaling in HA mode. You can do this by adding the following configuration to your Istio resource: +```yaml +apiVersion: sailoperator.io/v1 +kind: Istio +metadata: + name: default +spec: + namespace: istio-system + global: + defaultPodDisruptionBudget: + enabled: false # <-- disable default Pod Disruption Budget +``` + +`spec.global.defaultPodDisruptionBudget.enabled: false` disables the default Pod Disruption Budget for Istiod. In single-node clusters, a PDB can block operations such as node drains or pod evictions, as it prevents the number of available Istiod replicas from falling below the PDB's minimum desired count. Disabling it ensures smooth operations in this specific topology. diff --git a/docs/guidelines/example-documentation-following-guidelines.md b/docs/guidelines/example-documentation-following-guidelines.md index 527141e41..fbbea184b 100644 --- a/docs/guidelines/example-documentation-following-guidelines.md +++ b/docs/guidelines/example-documentation-following-guidelines.md @@ -1,3 +1,11 @@ +[Return to Project Root](../../README.md) + +# Table of Contents + +- [Example documentation where the guidelines are followed](#example-documentation-where-the-guidelines-are-followed) + - [Runme Test: Installing the operator from the helm repo and creating a Istio resource](#runme-test-installing-the-operator-from-the-helm-repo-and-creating-a-istio-resource) + - [Setting a Istio resource](#setting-a-istio-resource) + # Example documentation where the guidelines are followed This is an example doc where the guidelines are followed to achieve the best documentation possible. The doc is going to be used to test the automation workflow that is going to be used to run the tests over the documentation. @@ -59,20 +67,24 @@ EOF ``` > [!NOTE] -> These commented code blocks are validation steps, and they are added like this to be hidden in the final documentation, but they are going to be used in the automation workflow to validate the documentation. For more information, please check the [documentation](/docs/guidelines/guidelines.md#L146). - +``` +--> - +``` +--> - To check the status of the Istio resource, you can use the following command: ```bash { name=check-istio tag=example} @@ -86,20 +98,24 @@ kubectl create namespace sample kubectl label namespace sample istio-injection=enabled kubectl apply -n sample -f https://raw.githubusercontent.com/istio/istio/release-1.24/samples/bookinfo/platform/kube/bookinfo.yaml ``` - +``` +--> - Check the status of the sample application: ```bash { name=check-sample-app tag=example} kubectl get pods -n sample ``` - +``` +--> - Check the proxy version of the sample application: ```bash { name=check-proxy-version tag=example} diff --git a/docs/guidelines/guidelines.md b/docs/guidelines/guidelines.md index d137fadcd..9b90dd9e7 100644 --- a/docs/guidelines/guidelines.md +++ b/docs/guidelines/guidelines.md @@ -1,4 +1,22 @@ -[Return to Documentation Dir](../docs/README.md) # Path: docs/README.md +[Return to Project Root](../README.md) + +# Table of Contents + +- [Documentation Guidelines](#documentation-guidelines) + - [Introduction](#introduction) + - [Documentation Structure](#documentation-structure) + - [Structure of the entire folder and new folders](#structure-of-the-entire-folder-and-new-folders) + - [Table of Contents](#table-of-contents) + - [Headings](#headings) + - [Formatting](#formatting) + - [Links](#links) + - [Images](#images) + - [Nomemclature](#nomemclature) + - [Examples](#examples) + - [Automated Testing over the Documentation](#automated-testing-over-the-documentation) + - [What does update-docs-examples.sh do?](#what-does-update-docs-examplessh-do) + - [Adding a topic example to the automation workflow](#adding-a-topic-example-to-the-automation-workflow) + - [Debugging a specific docs example locally](#debugging-a-specific-docs-example-locally) # Documentation Guidelines @@ -50,20 +68,19 @@ Use code block for command or groups of command that has the same context. Use v Any documentation step need to be tested to ensure that the steps are correct and the user can follow them without any issue. To do this, we use `runme` (check the docs [here](https://docs.runme.dev/getting-started/cli)) to run the tests over the documentation. The workflow of the automation is the following: - The documentation files are in the `docs` folder. -- The `make update` target is going to run the script `tests/documentation_test/scripts/update-docs-examples.sh` that is going to generate the md modified files to run the test (For more information check next topic). -- The `make test.docs` target is going to run the script `tests/documentation_test/scripts/run-docs-examples.sh` that is going to run the tests over the documentation files. This script is going to run all the steps in the examples marked following the guidelines explained in the next section. +- The `make test.docs` target temporarily copies the documentation files to the `ARTIFACTS` directory (defaulting to a temporary folder) and uncomments all commented validation steps. This process is managed by the `tests/documentation_test/scripts/update-docs-examples.sh` script. Once the files are prepared, the `tests/documentation_test/scripts/run-docs-examples.sh` script executes the tests on the documentation files. It runs all example steps marked according to the guidelines described in the following sections. After the tests are completed, the temporary files are cleaned up. -### What does update-docs-examples.sh do? +### What does update-docs-examples.sh do The script `update-docs-examples.sh` is going to run the following steps: - Check all the md files in the `docs` folder and exclude from the list the files described inside the `EXCLUDE_FILES` variable and then copy all the files to the `tests/documentation_test/` folder that meets the criteria, check [Adding a topic example to the automation workflow](/docs/guidelines/guidelines.md#L62) section for more information about the criteria. -- Once the files are copied into the destination path it checka those files to uncomment the bash commented steps. This bash commented code block are going to be hiden in the original md files but we will uncomment this code block to be able to run validation steps that wait for conditions an avoid flakiness of the test. This means that most of the validation steps will be commented code blocks, more information [here](/docs/guidelines/guidelines.md#L146). +- Once the files are copied into the destination path it checks those files to uncomment the bash commented steps. This bash commented code block are going to be hiden in the original md files but we will uncomment this code block to be able to run validation steps that wait for conditions an avoid flakiness of the test. This means that most of the validation steps will be commented code blocks, more information [here](/docs/guidelines/guidelines.md#L146). ### Adding a topic example to the automation workflow To add a new topic to the automation workflow you need to: 1. In your documentation topic, each bash code block that you want to execute as part of the automation must include the following pattern: - `bash { name=example-name tag=example-tag }`: the fields used here are: - - `name`: the name of the example step, this is usefull to identify the step in the output of the test. The name should be short and descriptive. For example: `deploy-operator`, `wait-operator`, etc. + - `name`: the name of the example step, this is useful to identify the step in the output of the test. The name should be short and descriptive. For example: `deploy-operator`, `wait-operator`, etc. - `tag`: the tag of the example that is going to be used to run the test, is important to be a unique name. This tag should be unique and should not be used in other examples. The tag can be used to run only a specific example inside a file. For example: @@ -154,11 +171,11 @@ kubectl wait --for=condition=available --timeout=600s deployment/sail-operator - To avoid putting duplicated validation steps and help the users to easily get information, validate steps, etc. you can use prebuilt validation steps that are already created in the [prebuilts-func.sh](tests/documentation_tests/scripts/prebuilt-func.sh) script. For example, if you want to check if the istiod pod is ready you can use the `wait_istio_ready` function that is already created in the script. To use this function you need to add the following code block in your documentation: ```md ``` -To check the entire list of prebuilt functions please check the [prebuilts-func.sh](tests/documentation_tests/scripts/prebuilt-func.sh) script. +To check the entire list of prebuilt functions please check the [prebuilts-func.sh](tests/documentation_tests/scripts/prebuilt-func.sh) script. Note that `SCRIPT_DIR` is a variable that is already defined in the `run-docs-examples.sh` script, so you can use it directly in your documentation. > [!IMPORTANT] > Always include validation steps to avoid flakiness. They ensure resources are in expected conditions and the test fails clearly if they don’t. @@ -168,4 +185,213 @@ To check the entire list of prebuilt functions please check the [prebuilts-func. ```bash runme list --filename docs/common/runme-test.md ``` -This will output the entire list of commands but does not have filter for the tags. \ No newline at end of file +This will output the entire list of commands but does not have filter for the tags. + +### Debugging a specific docs example locally +To debug a specific example locally, you can use the the make target with the use of the env variable `FOCUS_DOC_TAGS`. For example, if you want to run the example with the tag `example-tag` you can run the following command: + +```bash +make test.docs FOCUS_DOC_TAGS=example-tag +``` + +This will run only the example with the tag `example-tag` and will not run any other example. This is useful to debug a specific example without running all the examples in the documentation. Take into account that the make target already creates their own kind cluster and installs the Sail Operator, so you don't need to do it manually. Also, the make target will clean up the cluster after the test is finished.[Return to Project Root](../README.md) + +# Table of Contents + +- [Documentation Guidelines](#documentation-guidelines) + - [Introduction](#introduction) + - [Documentation Structure](#documentation-structure) + - [Structure of the entire folder and new folders](#structure-of-the-entire-folder-and-new-folders) + - [Table of Contents](#table-of-contents) + - [Headings](#headings) + - [Formatting](#formatting) + - [Links](#links) + - [Images](#images) + - [Nomemclature](#nomemclature) + - [Examples](#examples) + - [Automated Testing over the Documentation](#automated-testing-over-the-documentation) + - [What does update-docs-examples.sh do?](#what-does-update-docs-examplessh-do) + - [Adding a topic example to the automation workflow](#adding-a-topic-example-to-the-automation-workflow) + - [Debugging a specific docs example locally](#debugging-a-specific-docs-example-locally) + +# Documentation Guidelines + +## Introduction +This guide aims to set basic guidelines for writing documentation for the project. The documentation should be clear, concise, and easy to understand. It should be written in a way that is accessible to both technical and non-technical users. + +## Documentation Structure +The documentation should be structured in a way that is easy to navigate. It should be divided into sections and subsections, with a table of contents at the beginning of the document. Each section should cover a specific topic, and each subsection should cover a specific aspect of that topic. + +### Structure of the entire folder and new folders +All the documentation lives unders the folder `docs` in the root of the project. The documentation is divided into the following sections: +- `guidelines`: Contains guidelines for writing documentation. +- `api-reference`: Contains the API reference documentation. It is generated from the code, so if you want to update it, you need to update the docstrings. +- `common`: Contains common documentation that is relevant to the entire project, for example: `create-and-configure-gateways`, `install-bookinfo-app`, etc. All the docs located here should be linked to from the main README.md file for easy access. +- `multicluster`: contains resources referenced in the README.md file for the multicluster setup. +- `README.md`: The main README.md file that contains the project main doc and links to the other documentation topics. + +Any new folder or file added to the documentation should be linked to from the main README.md and you should only create a new folder if the documentation is not relevant to the entire project or if it is a new section that does not fit into the existing structure. Also, new folders can contain resources that are referenced in the README.md to be able to run examples or to get more information about the project (for an example, see the `multicluster` folder). + +### Table of Contents +For long documents, it is recommended to include a table of contents at the beginning of the document. The table of contents should list all the sections and subsections in the document. + +### Headings +Use headings to break up the content into sections and subsections. Headings should be descriptive and should clearly indicate the topic of the section or subsection. + +### Formatting +Use formatting to make the text more readable. Use bullet points for lists, code blocks for code snippets, and inline code for short code snippets. Use bold or italic text to highlight important points. Any topic that is important to the reader should be highlighted in some way. + +### Links +Use links to reference other sections of the documentation, external resources, or related topics. Links should be descriptive and should clearly indicate the target of the link. Links should be used to provide additional information or context for the reader. Avoid the use of raw URLs in the documentation. + +### Images +Use images to illustrate concepts and provide visual guidance. Images should be accompanied by explanations and annotations to help the reader understand the content. Images should be used to enhance the text and provide additional context for the reader and they are going to be stored in a `images` folder. + +### Nomemclature +Use consistent terminology throughout the documentation. Use the same terms to refer to the same concepts and avoid using different terms to refer to the same concept. For this project, we are going to use the following terms: +- `Sail Operator` to refer to the Sail Operator project. +- `istio` to refer to the Istio service mesh upstream project. +- `Istio` to the resource created and managed by the Sail Operator. +- `IstioCNI` to refer to the Istio CNI resource managed by the Sail Operator. +- `istio-cni` to refer to the Istio CNI project. + +### Examples +Use examples to illustrate concepts and provide practical guidance. Examples should be clear, concise, and easy to follow. They should be accompanied by explanations and annotations to help the reader understand the code. Also, the examples provided can be used to run automated tests over the example steps. This is going to be explained in the next section. + +Use code block for command or groups of command that has the same context. Use validation steps to ensure that conditions are met and avoid flakiness of the test, in the next section we are going to explain how to add validation steps to the examples. Also, use the `ignore` tag to ignore code blocks that are not going to be run by the automation tool. For example, yaml files or any other code block that is not going to be run in the terminal. + +## Automated Testing over the Documentation +Any documentation step need to be tested to ensure that the steps are correct and the user can follow them without any issue. To do this, we use `runme` (check the docs [here](https://docs.runme.dev/getting-started/cli)) to run the tests over the documentation. The workflow of the automation is the following: + +- The documentation files are in the `docs` folder. +- The `make test.docs` target temporarily copies the documentation files to the `test/documentation_test` directory and uncomments all commented validation steps. This process is handled by the `tests/documentation_test/scripts/update-docs-examples.sh` script. Afterward, the `tests/documentation_test/scripts/run-docs-examples.sh` script executes the tests on the documentation files. It runs all example steps marked according to the guidelines outlined in the next section. Once the tests are complete, the temporary documentation files are removed. + +### What does update-docs-examples.sh do +The script `update-docs-examples.sh` is going to run the following steps: +- Check all the md files in the `docs` folder and exclude from the list the files described inside the `EXCLUDE_FILES` variable and then copy all the files to the `tests/documentation_test/` folder that meets the criteria, check [Adding a topic example to the automation workflow](/docs/guidelines/guidelines.md#L62) section for more information about the criteria. +- Once the files are copied into the destination path it checks those files to uncomment the bash commented steps. This bash commented code block are going to be hiden in the original md files but we will uncomment this code block to be able to run validation steps that wait for conditions an avoid flakiness of the test. This means that most of the validation steps will be commented code blocks, more information [here](/docs/guidelines/guidelines.md#L146). + + +### Adding a topic example to the automation workflow +To add a new topic to the automation workflow you need to: +1. In your documentation topic, each bash code block that you want to execute as part of the automation must include the following pattern: +- `bash { name=example-name tag=example-tag }`: the fields used here are: + - `name`: the name of the example step, this is useful to identify the step in the output of the test. The name should be short and descriptive. For example: `deploy-operator`, `wait-operator`, etc. + - `tag`: the tag of the example that is going to be used to run the test, is important to be a unique name. This tag should be unique and should not be used in other examples. The tag can be used to run only a specific example inside a file. +For example: + +````md +```bash { name=deploy-operator tag=example-tag} +helm repo add sail-operator https://istio-ecosystem.github.io/sail-operator +``` +```` + +- use `bash { ignore=true }` to prevent a code block from being executed by the `runme` tool. This is helpful when you include YAML examples or other content that shouldn’t be part of automation workflow. Using `ignore` attribute ensures only the intended code blocks are run. For example: + +````md +- Example of a Istio resource: +```yaml { ignore=true } +apiVersion: sailoperator.io/v1 +kind: Istio +metadata: + name: default +spec: + namespace: istio-system + updateStrategy: + type: RevisionBased + inactiveRevisionDeletionGracePeriodSeconds: 30 + version: v1.24.2 +``` +```` + +> [!Note] +> When using the `runme` tool, make sure that all code blocks intended for execution are marked with the bash language at the start of the block. These blocks should include only commands that are meant to be run in the terminal. Any steps that are not actual commands or are not essential for the goal of the example should be skipped using the `ignore=true` tag. For example: + +````md +- You should set version for Istio in the `Istio` resource and `IstioRevisionTag` resource should reference the name of the `Istio` resource: +```yaml { ignore=true } +apiVersion: sailoperator.io/v1 +kind: Istio +metadata: + name: default +spec: + namespace: istio-system + updateStrategy: + type: RevisionBased + inactiveRevisionDeletionGracePeriodSeconds: 30 + version: v1.24.2 +--- +apiVersion: sailoperator.io/v1 +kind: IstioRevisionTag +metadata: + name: default +spec: + targetRef: + kind: Istio + name: default +``` +- Create ns, `Istio` and `IstioRevisionTag` resources: +```bash { name=create-istio tag=example-tag} +kubectl create ns istio-system +cat < +``` + +To avoid putting duplicated validation steps and help the users to easily get information, validate steps, etc. you can use prebuilt validation steps that are already created in the [prebuilts-func.sh](../../tests/documentation_tests/scripts/prebuilt-func.sh) script. For example, if you want to check if the istiod pod is ready you can use the `wait_istio_ready` function that is already created in the script. To use this function you need to add the following code block in your documentation: +```md + +``` +To check the entire list of prebuilt functions please check the [prebuilts-func.sh](../../tests/documentation_tests/scripts/prebuilt-func.sh) script. + +> [!IMPORTANT] +> Always include validation steps to avoid flakiness. They ensure resources are in expected conditions and the test fails clearly if they don’t. + +> [!Note] +> If you want to check all the commands that inside a md file you can execute the following command: +```bash +runme list --filename docs/common/runme-test.md +``` +This will output the entire list of commands but does not have filter for the tags. + +### Debugging a specific docs example locally +To debug a specific example locally, you can use the the make target with the use of the env variable `FOCUS_DOC_TAGS`. For example, if you want to run the example with the tag `example-tag` you can run the following command: + +```bash +make test.docs FOCUS_DOC_TAGS=example-tag +``` + +This will run only the example with the tag `example-tag` and will not run any other example. This is useful to debug a specific example without running all the examples in the documentation. Take into account that the make target already creates their own kind cluster and installs the Sail Operator, so you don't need to do it manually. Also, the make target will clean up the cluster after the test is finished. \ No newline at end of file diff --git a/docs/update-strategy/update-strategy.md b/docs/update-strategy/update-strategy.md new file mode 100644 index 000000000..18bdc87e3 --- /dev/null +++ b/docs/update-strategy/update-strategy.md @@ -0,0 +1,598 @@ +[Return to Project Root](../../README.md) + +# Table of Contents + +- [Update Strategy](#update-strategy) + - [InPlace](#inplace) + - [Example using the InPlace strategy](#example-using-the-inplace-strategy) + - [Recommendations for InPlace Strategy](#recommendations-for-inplace-strategy) + - [RevisionBased](#revisionbased) + - [Example using the RevisionBased strategy](#example-using-the-revisionbased-strategy) + - [Example using the RevisionBased strategy and an IstioRevisionTag](#example-using-the-revisionbased-strategy-and-an-istiorevisiontag) + +# Update Strategy + +The Sail Operator supports two update strategies to update the version of the Istio control plane: `InPlace` and `RevisionBased`. The default strategy is `InPlace`. + +## InPlace +When the `InPlace` strategy is used, the existing Istio control plane is replaced with a new version. The workload sidecars immediately connect to the new control plane. The workloads therefore don't need to be moved from one control plane instance to another. + +### Example using the InPlace strategy + +Prerequisites: +* Sail Operator is installed. +* `istioctl` is [installed](../../docs/common/install-istioctl-tool.md). + +Steps: +1. Create the `istio-system` namespace. + + ```bash { name=create-istio-ns tag=inplace-update} + kubectl create namespace istio-system + ``` + +2. Create the `Istio` resource. + + ```bash { name=create-istio-resource tag=inplace-update} + cat < +3. Confirm the installation and version of the control plane. + + ```console + $ kubectl get istio -n istio-system + NAME REVISIONS READY IN USE ACTIVE REVISION STATUS VERSION AGE + default 1 1 0 default Healthy v1.25.3 23s + ``` + Note: `IN USE` field shows as 0, as `Istio` has just been installed and there are no workloads using it. + +4. Create namespace `bookinfo` and deploy bookinfo application. + + ```bash { name=deploy-bookinfo tag=inplace-update} + kubectl create namespace bookinfo + kubectl label namespace bookinfo istio-injection=enabled + kubectl apply -n bookinfo -f https://raw.githubusercontent.com/istio/istio/release-1.22/samples/bookinfo/platform/kube/bookinfo.yaml + ``` + Note: if the `Istio` resource name is other than `default`, you need to set the `istio.io/rev` label to the name of the `Istio` resource instead of adding the `istio-injection=enabled` label. + +5. Review the `Istio` resource after application deployment. + + ```console + $ kubectl get istio -n istio-system + NAME REVISIONS READY IN USE ACTIVE REVISION STATUS VERSION AGE + default 1 1 1 default Healthy v1.25.3 115s + ``` + Note: `IN USE` field shows as 1, as the namespace label and the injected proxies reference the IstioRevision. + +6. Perform the update of the control plane by changing the version in the Istio resource. + + ```bash + kubectl patch istio default -n istio-system --type='merge' -p '{"spec":{"version":"v1.26.0"}}' + ``` + + +7. Confirm the `Istio` resource version was updated. + + ```console + $ kubectl get istio -n istio-system + NAME REVISIONS READY IN USE ACTIVE REVISION STATUS VERSION AGE + default 1 1 1 default Healthy v1.26.0 4m50s + ``` + +8. Delete `bookinfo` pods to trigger sidecar injection with the new version. + + ```bash + kubectl rollout restart deployment -n bookinfo + ``` + + +9. Confirm that the new version is used in the sidecar. + + ```bash { name=print-istio-version tag=inplace-update} + istioctl proxy-status + ``` + The column `VERSION` should match the new control plane version. + + +### Recommendations for InPlace Strategy +During `InPlace` updates, the control plane pods are restarted, which may cause temporary service disruptions. To minimize downtime during updates, we recommend configuring the `istiod` deployment with high availability (HA). For more information, please refer to this [guide](../../docs/general/istiod-ha.md). + +## RevisionBased +When the `RevisionBased` strategy is used, a new Istio control plane instance is created for every change to the `Istio.spec.version` field. The old control plane remains in place until all workloads have been moved to the new control plane instance. This needs to be done by the user by updating the namespace label and restarting all the pods. The old control plane will be deleted after the grace period specified in the `Istio` resource field `spec.updateStrategy.inactiveRevisionDeletionGracePeriodSeconds`. + +### Example using the RevisionBased strategy + +Prerequisites: +* Sail Operator is installed. +* `istioctl` is [installed](../../docs/common/install-istioctl-tool.md). + +Steps: + +1. Create the `istio-system` namespace. + + ```bash { name=create-ns tag=revision-based-update} + kubectl create namespace istio-system + ``` + +2. Create the `Istio` resource. + + ```bash { name=create-istio tag=revision-based-update} + cat < + +3. Confirm the control plane is installed and is using the desired version. + + ```console + $ kubectl get istio -n istio-system + NAME REVISIONS READY IN USE ACTIVE REVISION STATUS VERSION AGE + default 1 1 0 default-v1-25-3 Healthy v1.25.3 52s + ``` + Note: `IN USE` field shows as 0, as the control plane has just been installed and there are no workloads using it. + +4. Get the `IstioRevision` name. + + ```console + $ kubectl get istiorevision -n istio-system + NAME TYPE READY STATUS IN USE VERSION AGE + default-v1-25-3 Local True Healthy False v1.25.3 3m4s + ``` + Note: `IstioRevision` name is in the format `-`. + + +5. Create `bookinfo` namespace and label it with the revision name. + + ```bash { name=create-bookinfo-ns tag=revision-based-update} + kubectl create namespace bookinfo + kubectl label namespace bookinfo istio.io/rev=default-v1-25-3 + ``` + +6. Deploy bookinfo application. + + ```bash { name=deploy-bookinfo tag=revision-based-update} + kubectl apply -n bookinfo -f https://raw.githubusercontent.com/istio/istio/release-1.22/samples/bookinfo/platform/kube/bookinfo.yaml + ``` + +7. Review the `Istio` resource after application deployment. + + ```console + $ kubectl get istio -n istio-system + NAME REVISIONS READY IN USE ACTIVE REVISION STATUS VERSION AGE + default 1 1 1 default-v1-25-3 Healthy v1.25.3 5m13s + ``` + Note: `IN USE` field shows as 1, after application being deployed. + +8. Confirm that the proxy version matches the control plane version. + + ```bash + istioctl proxy-status + ``` + The column `VERSION` should match the control plane version. + +9. Update the control plane to a new version. + + ```bash + kubectl patch istio default -n istio-system --type='merge' -p '{"spec":{"version":"v1.26.0"}}' + ``` + +10. Verify the `Istio` and `IstioRevision` resources. There will be a new revision created with the new version. + + ```console + $ kubectl get istio + NAME REVISIONS READY IN USE ACTIVE REVISION STATUS VERSION AGE + default 2 2 1 default-v1-26-0 Healthy v1.26.0 9m23s + + $ kubectl get istiorevision + NAME TYPE READY STATUS IN USE VERSION AGE + default-v1-25-3 Local True Healthy True v1.25.3 10m + default-v1-26-0 Local True Healthy False v1.26.0 66s + ``` + +11. Confirm there are two control plane pods running, one for each revision. + + ```console + $ kubectl get pods -n istio-system + NAME READY STATUS RESTARTS AGE + istiod-default-v1-25-3-c98fd9675-r7bfw 1/1 Running 0 10m + istiod-default-v1-26-0-7495cdc7bf-v8t4g 1/1 Running 0 113s + ``` + +12. Confirm the proxy sidecar version remains the same: + + ```bash + istioctl proxy-status + ``` + The column `VERSION` should still match the old control plane version. + +13. Change the label of the `bookinfo` namespace to use the new revision. + + ```bash { name=update-bookinfo-ns-revision tag=revision-based-update} + kubectl label namespace bookinfo istio.io/rev=default-v1-26-0 --overwrite + ``` + The existing workload sidecars will continue to run and will remain connected to the old control plane instance. They will not be replaced with a new version until the pods are deleted and recreated. + +14. Restart all Deplyments in the `bookinfo` namespace. + + ```bash + kubectl rollout restart deployment -n bookinfo + ``` + +15. Confirm the new version is used in the sidecars. + + ```bash + istioctl proxy-status + ``` + The column `VERSION` should match the updated control plane version. + +16. Confirm the deletion of the old control plane and IstioRevision. + + ```console + $ kubectl get pods -n istio-system + NAME READY STATUS RESTARTS AGE + istiod-default-v1-26-0-7495cdc7bf-v8t4g 1/1 Running 0 4m40s + + $ kubectl get istio + NAME REVISIONS READY IN USE ACTIVE REVISION STATUS VERSION AGE + default 1 1 1 default-v1-26-0 Healthy v1.26.0 5m + + $ kubectl get istiorevision + NAME TYPE READY STATUS IN USE VERSION AGE + default-v1-26-0 Local True Healthy True v1.26.0 5m31s + ``` + The old `IstioRevision` resource and the old control plane will be deleted when the grace period specified in the `Istio` resource field `spec.updateStrategy.inactiveRevisionDeletionGracePeriodSeconds` expires. + +### Example using the RevisionBased strategy and an IstioRevisionTag + +Prerequisites: +* Sail Operator is installed. +* `istioctl` is [installed](../../docs/common/install-istioctl-tool.md). + +Steps: + +1. Create the `istio-system` namespace. + + ```bash { name=create-ns tag=istiorevisiontag} + kubectl create namespace istio-system + ``` + +2. Create the `Istio` and `IstioRevisionTag` resources. + + ```bash { name=create-istio-and-revision-tag tag=istiorevisiontag} + cat < +3. Confirm the control plane is installed and is using the desired version. + + ```console + $ kubectl get istio + NAME REVISIONS READY IN USE ACTIVE REVISION STATUS VERSION AGE + default 1 1 1 default-v1-25-3 Healthy v1.25.3 52s + ``` + Note: `IN USE` field shows as 1, even though no workloads are using the control plane. This is because the `IstioRevisionTag` is referencing it. + +4. Inspect the `IstioRevisionTag`. + + ```console + $ kubectl get istiorevisiontags + NAME STATUS IN USE REVISION AGE + default NotReferencedByAnything False default-v1-25-3 52s + ``` + + Note: `IN USE` field shows as `False`, as the tag is not referenced by any workloads or namespaces. +5. Create `bookinfo` namespace and label it to mark it for injection. + + ```bash { name=create-bookinfo-ns tag=istiorevisiontag} + kubectl create namespace bookinfo + kubectl label namespace bookinfo istio-injection=enabled + ``` + +6. Deploy bookinfo application. + + ```bash { name=deploy-bookinfo tag=istiorevisiontag} + kubectl apply -n bookinfo -f https://raw.githubusercontent.com/istio/istio/release-1.23/samples/bookinfo/platform/kube/bookinfo.yaml + ``` + +7. Review the `IstioRevisionTag` resource after application deployment. + + ```console + $ kubectl get istiorevisiontag + NAME STATUS IN USE REVISION AGE + default Healthy True default-v1-25-3 2m46s + ``` + Note: `IN USE` field shows 'True', as the tag is now referenced by both active workloads and the bookinfo namespace. + +8. Confirm that the proxy version matches the control plane version. + + ```bash + istioctl proxy-status + ``` + The column `VERSION` should match the control plane version. + +9. Update the control plane to a new version. + + ```bash { name=update-istio-version tag=istiorevisiontag} + kubectl patch istio default -n istio-system --type='merge' -p '{"spec":{"version":"v1.26.0"}}' + ``` + +10. Verify the `Istio`, `IstioRevision` and `IstioRevisionTag` resources. There will be a new revision created with the new version. + + ```console + $ kubectl get istio + NAME REVISIONS READY IN USE ACTIVE REVISION STATUS VERSION AGE + default 2 2 1 default-v1-26-0 Healthy v1.26.0 9m23s + + $ kubectl get istiorevision + NAME TYPE READY STATUS IN USE VERSION AGE + default-v1-25-3 Local True Healthy True v1.25.3 10m + default-v1-26-0 Local True Healthy True v1.26.0 66s + + $ kubectl get istiorevisiontag + NAME STATUS IN USE REVISION AGE + default Healthy True default-v1-26-0 10m44s + ``` + Now, both our IstioRevisions and the IstioRevisionTag are considered in use. The old revision default-v1-25-3 because it is being used by proxies, the new revision default-v1-26-0 because it is referenced by the tag, and lastly the tag because it is referenced by the bookinfo namespace. + +11. Confirm there are two control plane pods running, one for each revision. + + ```console + $ kubectl get pods -n istio-system + NAME READY STATUS RESTARTS AGE + istiod-default-v1-25-3-c98fd9675-r7bfw 1/1 Running 0 10m + istiod-default-v1-26-0-7495cdc7bf-v8t4g 1/1 Running 0 113s + ``` + +12. Confirm the proxy sidecar version remains the same: + + ```bash { name=validation-istio-proxy-version tag=istiorevisiontag} + istioctl proxy-status + ``` + The column `VERSION` should still match the old control plane version. + +13. Restart all the Deployments in the `bookinfo` namespace. + + ```bash + kubectl rollout restart deployment -n bookinfo + ``` + +14. Confirm the new version is used in the sidecars. Note that it might take a few seconds for the restarts to complete. + + ```bash + istioctl proxy-status + ``` + The column `VERSION` should match the updated control plane version. + +16. Confirm the deletion of the old control plane and IstioRevision. + + ```console + $ kubectl get pods -n istio-system + NAME READY STATUS RESTARTS AGE + istiod-default-v1-26-0-7495cdc7bf-v8t4g 1/1 Running 0 4m40s + + $ kubectl get istio -n istio-system + NAME REVISIONS READY IN USE ACTIVE REVISION STATUS VERSION AGE + default 1 1 1 default-v1-26-0 Healthy v1.26.0 5m + + $ kubectl get istiorevision -n istio-system + NAME TYPE READY STATUS IN USE VERSION AGE + default-v1-26-0 Local True Healthy True v1.26.0 5m31s + ``` + The old `IstioRevision` resource and the old control plane will be deleted when the grace period specified in the `Istio` resource field `spec.updateStrategy.inactiveRevisionDeletionGracePeriodSeconds` expires. + diff --git a/go.mod b/go.mod index 76a0b110a..93f748863 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/onsi/gomega v1.37.0 github.com/prometheus/common v0.63.0 github.com/stretchr/testify v1.10.0 + go.uber.org/zap v1.27.0 golang.org/x/mod v0.24.0 golang.org/x/text v0.25.0 golang.org/x/tools v0.33.0 @@ -161,7 +162,6 @@ require ( go.opentelemetry.io/proto/otlp v1.6.0 // indirect go.uber.org/automaxprocs v1.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.0 // indirect golang.org/x/crypto v0.38.0 // indirect golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 // indirect golang.org/x/net v0.40.0 // indirect diff --git a/hack/operatorhub/publish-bundle.sh b/hack/operatorhub/publish-bundle.sh index e72e12858..afea2c0dd 100755 --- a/hack/operatorhub/publish-bundle.sh +++ b/hack/operatorhub/publish-bundle.sh @@ -1,5 +1,5 @@ #!/bin/bash -# shellcheck disable=SC1091 +# shellcheck disable=SC1091,SC2001 # Copyright Istio Authors # @@ -29,6 +29,8 @@ GIT_CONFIG_USER_EMAIL="${GIT_CONFIG_USER_EMAIL:-}" # The OPERATOR_NAME is defined in Makefile : "${OPERATOR_NAME:?"Missing OPERATOR_NAME variable"}" : "${OPERATOR_VERSION:?"Missing OPERATOR_VERSION variable"}" +: "${CHANNELS:?"Missing CHANNELS variable"}" +: "${PREVIOUS_VERSION:?"Missing PREVIOUS_VERSION variable"}" show_help() { echo "publish-bundle - raises PR to Operator Hub" @@ -99,6 +101,39 @@ BUNDLE_DIR="${CUR_DIR}"/../../bundle mkdir -p "${OPERATORS_DIR}" cp -a "${BUNDLE_DIR}"/. "${OPERATORS_DIR}" +# Generate release-config.yaml which is required to update FBC. FBC is only available in community-operators-prod atm +if [ "${OPERATOR_HUB}" = "community-operators-prod" ] +then + # when publishing a nightly build, we want to get previous build version automatically + if [[ ${OPERATOR_VERSION} == *"nightly"* ]] + then + # expecting there is only one channel in $CHANNELS when pushing nightly builds + LATEST_VERSION=$(yq '.entries[] | select(.schema == "olm.channel" and .name == '\""${CHANNELS}"\"').entries[-1].name' "${OPERATORS_DIR}../catalog-templates/basic.yaml") + # there is no entry in the given channel, probably a new channel and first version to be pushed there. Let's use previous nightly channel. + if [ -z "${LATEST_VERSION}" ] + then + PREVIOUS_MINOR=$(echo "${PREVIOUS_VERSION}" | cut -f1,2 -d'.') + LATEST_VERSION=$(yq '.entries[] | select(.schema == "olm.channel" and .name == '\""${PREVIOUS_MINOR}-nightly"\"').entries[-1].name' "${OPERATORS_DIR}../catalog-templates/basic.yaml") + if [ -z "${LATEST_VERSION}" ] + then + echo "Unable to find previous nightly version. Exiting." + exit 1 + fi + fi + else + LATEST_VERSION="${OPERATOR_NAME}.v${PREVIOUS_VERSION}" + fi + # yaml linter in community-operators-prod CI is expecting a space after a comma + CHANNELS_SANITIZED=$(echo "${CHANNELS}" | sed 's/, */, /g') + cat < "${OPERATORS_DIR}/release-config.yaml" +catalog_templates: + - template_name: basic.yaml + channels: [${CHANNELS_SANITIZED}] + replaces: ${LATEST_VERSION} + skipRange: '>=1.0.0 <${OPERATOR_VERSION}' +EOF +fi + if ! git config --global user.name; then skipInDryRun git config --global user.name "${GIT_CONFIG_USER_NAME}" fi @@ -110,7 +145,7 @@ fi TITLE="operator ${OPERATOR_NAME} (${OPERATOR_VERSION})" skipInDryRun git add . skipInDryRun git commit -s -m"${TITLE}" -skipInDryRun git push fork "${BRANCH}" +skipInDryRun git push -f fork "${BRANCH}" PAYLOAD="${TMP_DIR}/PAYLOAD" diff --git a/pkg/istiovalues/vendor_defaults.go b/pkg/istiovalues/vendor_defaults.go index e164832ff..c60e73e99 100644 --- a/pkg/istiovalues/vendor_defaults.go +++ b/pkg/istiovalues/vendor_defaults.go @@ -16,6 +16,8 @@ package istiovalues import ( _ "embed" + "fmt" + "strings" v1 "github.com/istio-ecosystem/sail-operator/api/v1" "github.com/istio-ecosystem/sail-operator/pkg/helm" @@ -41,10 +43,75 @@ func MustParseVendorDefaultsYAML(defaultsYAML []byte) map[string]map[string]any return parsedDefaults } -func ApplyVendorDefaults(version string, values *v1.Values) (*v1.Values, error) { +// OverrideVendorDefaults allows tests to override the vendor defaults +func OverrideVendorDefaults(defaults map[string]map[string]any) { + // Set the vendorDefaults to the provided defaults for testing purposes. + // This allows tests to override the defaults without modifying the original file. + vendorDefaults = defaults +} + +// ApplyIstioVendorDefaults applies vendor-specific default values to the provided +// Istio configuration (*v1.Values) for a given Istio version. +func ApplyIstioVendorDefaults(version string, values *v1.Values) (*v1.Values, error) { + // Convert the specific *v1.Values struct to a generic map[string]any + userHelmValues := helm.FromValues(values) + + mergedHelmValues, err := applyVendorDefaultsForResourceType(version, v1.IstioKind, userHelmValues) + if err != nil { + return nil, fmt.Errorf("failed to apply vendor defaults for istio: %w", err) + } + + finalValues, err := helm.ToValues(mergedHelmValues, &v1.Values{}) + if err != nil { + return nil, fmt.Errorf("failed to convert merged values back to *v1.Values: %w", err) + } + + return finalValues, nil +} + +// ApplyIstioCNIVendorDefaults applies vendor-specific default values to the provided +// Istio CNI configuration (*v1.CNIValues) for a given Istio version. +func ApplyIstioCNIVendorDefaults(version string, values *v1.CNIValues) (*v1.CNIValues, error) { + // Convert the specific *v1.CNIValues struct to a generic map[string]any + userHelmValues := helm.FromValues(values) + + mergedHelmValues, err := applyVendorDefaultsForResourceType(version, v1.IstioCNIKind, userHelmValues) + if err != nil { + return nil, fmt.Errorf("failed to apply vendor defaults for istiocni: %w", err) + } + + finalValues, err := helm.ToValues(mergedHelmValues, &v1.CNIValues{}) + if err != nil { + return nil, fmt.Errorf("failed to convert merged values back to *v1.CNIValues: %w", err) + } + + return finalValues, nil +} + +// applyVendorDefaultsForResourceType retrieves defaults for +// a given version and resourceType, current valid resources are: "istio" and "istiocni" +// It returns the merged map and an error if the defaults are not found or malformed. +// The userValuesMap is the 'base', and resource-specific defaults are 'overrides'. +func applyVendorDefaultsForResourceType(version string, resourceType string, userValuesMap map[string]any) (map[string]any, error) { if len(vendorDefaults) == 0 { - return values, nil + return userValuesMap, nil // No vendor defaults defined } - mergedValues := helm.Values(mergeOverwrite(vendorDefaults[version], helm.FromValues(values))) - return helm.ToValues(mergedValues, &v1.Values{}) + + versionSpecificDefaults, versionExists := vendorDefaults[version] + if !versionExists { + return userValuesMap, nil // No vendor defaults defined for this version + } + + resourceSpecificDefaults, resourceExists := versionSpecificDefaults[strings.ToLower(resourceType)] + if !resourceExists { + return userValuesMap, nil // No vendor defaults defined for this resource type in this version + } + + // Check if the resource-specific defaults are a map[string]any + resourceSpecificDefaultsMap, ok := resourceSpecificDefaults.(map[string]any) + if !ok { + return nil, fmt.Errorf("vendor defaults for resource '%s' (version '%s') are not a map[string]any", resourceType, version) + } + + return mergeOverwrite(resourceSpecificDefaultsMap, userValuesMap), nil } diff --git a/pkg/istiovalues/vendor_defaults.yaml b/pkg/istiovalues/vendor_defaults.yaml index b4c0482c2..49f50e6bc 100644 --- a/pkg/istiovalues/vendor_defaults.yaml +++ b/pkg/istiovalues/vendor_defaults.yaml @@ -1,10 +1,17 @@ # This file can be overwritten in vendor distributions of sail-operator to set # vendor-specific defaults. You can configure version-specific defaults for -# every field in `spec.values`, see the following example: +# every field in `spec.values` for both Istio and IstioCNI resource, +# see the following example: # -# v1.24.2: -# pilot: -# env: -# test: "true" +# v1.26.0: +# istio: +# pilot: +# env: +# test: "true" +# istiocni: +# cni: +# cniConfDir: custom/cni/conf/dir # # These defaults are type-checked at compile time. +# Note: After modifying this file, run `make test` to ensure that the +# generated defaults values from this files are valid. diff --git a/pkg/istiovalues/vendor_defaults_test.go b/pkg/istiovalues/vendor_defaults_test.go index c5daff383..8d79b7cb8 100644 --- a/pkg/istiovalues/vendor_defaults_test.go +++ b/pkg/istiovalues/vendor_defaults_test.go @@ -20,70 +20,223 @@ import ( "github.com/google/go-cmp/cmp" v1 "github.com/istio-ecosystem/sail-operator/api/v1" + "github.com/stretchr/testify/assert" ) func TestApplyVendorDefaults(t *testing.T) { testcases := []struct { - name string - vendorDefaults string - version string - preValues *v1.Values - postValues *v1.Values - err error + name string + vendorDefaults string + version string + istioPreValues *v1.Values + istoPostValues *v1.Values + istioCNIPreValues *v1.CNIValues + istioCniPostValues *v1.CNIValues + expectedIstioError bool + expectedIstioCniError bool + expectedErrSubstring string }{ { - name: "no user values", + name: "adding custom values for both Istio and IstioCNI", vendorDefaults: ` v1.24.2: - pilot: - env: - someEnvVar: "true" + istio: + pilot: + env: + someEnvVar: "true" + istiocni: + cni: + cniConfDir: example/path `, - version: "v1.24.2", - preValues: &v1.Values{}, - postValues: &v1.Values{ + version: "v1.24.2", + istioPreValues: &v1.Values{}, + istioCNIPreValues: &v1.CNIValues{}, + istoPostValues: &v1.Values{ Pilot: &v1.PilotConfig{ Env: map[string]string{ "someEnvVar": "true", }, }, }, - err: nil, + istioCniPostValues: &v1.CNIValues{ + Cni: &v1.CNIConfig{ + CniConfDir: StringPtr("example/path"), + }, + }, + expectedIstioError: false, + expectedIstioCniError: false, + expectedErrSubstring: "", // no error expected + }, + { + name: "adding custom values for Istio only", + vendorDefaults: ` +v1.24.2: + istio: + pilot: + env: + someEnvVar: "true" +`, + version: "v1.24.2", + istioPreValues: &v1.Values{}, + istioCNIPreValues: &v1.CNIValues{}, + istoPostValues: &v1.Values{ + Pilot: &v1.PilotConfig{ + Env: map[string]string{ + "someEnvVar": "true", + }, + }, + }, + istioCniPostValues: &v1.CNIValues{}, + expectedIstioError: false, + expectedIstioCniError: false, + expectedErrSubstring: "", // no error expected + }, + { + name: "adding custom values for IstioCNI only", + vendorDefaults: ` +v1.24.2: + istiocni: + cni: + cniConfDir: example/path +`, + version: "v1.24.2", + istioPreValues: &v1.Values{}, + istioCNIPreValues: &v1.CNIValues{}, + istoPostValues: &v1.Values{}, + istioCniPostValues: &v1.CNIValues{ + Cni: &v1.CNIConfig{ + CniConfDir: StringPtr("example/path"), + }, + }, + expectedIstioError: false, + expectedIstioCniError: false, + expectedErrSubstring: "", // no error expected + }, + { + name: "empty vendor defaults", + vendorDefaults: ` +`, // empty vendor defaults + version: "v1.24.2", + istioPreValues: &v1.Values{}, + istioCNIPreValues: &v1.CNIValues{}, + istoPostValues: &v1.Values{}, + istioCniPostValues: &v1.CNIValues{}, + expectedIstioError: false, + expectedIstioCniError: false, + expectedErrSubstring: "", // no error expected + }, + { + name: "non-existent version", + vendorDefaults: ` +v1.24.2: + istio: + pilot: + env: + someEnvVar: "true" +`, + version: "v1.25.0", // version not in vendor defaults + istioPreValues: &v1.Values{}, + istioCNIPreValues: &v1.CNIValues{}, + istoPostValues: &v1.Values{}, + istioCniPostValues: &v1.CNIValues{}, + expectedIstioError: false, + expectedIstioCniError: false, + expectedErrSubstring: "", // no error expected + }, + { + name: "invalid version format", + vendorDefaults: ` +v1.24.2: + istio: + pilot: + env: + someEnvVar: "true" +`, + version: "v1.24", // invalid version format + istioPreValues: &v1.Values{}, + istioCNIPreValues: &v1.CNIValues{}, + istoPostValues: &v1.Values{}, + istioCniPostValues: &v1.CNIValues{}, + expectedIstioError: false, + expectedIstioCniError: false, + expectedErrSubstring: "", // no error expected + }, + { + name: "malformed vendor defaults", + vendorDefaults: ` +v1.24.2: + istio: + pilot: "env" +`, + version: "v1.24.2", + istioPreValues: &v1.Values{}, + istioCNIPreValues: &v1.CNIValues{}, + istoPostValues: nil, // expect nil due to malformed defaults + istioCniPostValues: &v1.CNIValues{}, + expectedIstioError: true, + expectedIstioCniError: false, + expectedErrSubstring: "cannot unmarshal string into Go struct field Values.pilot", // expect a specific error for malformed defaults }, { name: "user values override vendor defaults", vendorDefaults: ` v1.24.2: - pilot: - env: - someEnvVar: "true" + istio: + pilot: + env: + someEnvVar: "true" `, version: "v1.24.2", - preValues: &v1.Values{ + istioPreValues: &v1.Values{ Pilot: &v1.PilotConfig{ Env: map[string]string{ - "someEnvVar": "false", + "someEnvVar": "false", // user value overrides vendor default }, }, }, - postValues: &v1.Values{ + istioCNIPreValues: &v1.CNIValues{}, + istoPostValues: &v1.Values{ Pilot: &v1.PilotConfig{ Env: map[string]string{ - "someEnvVar": "false", // user value overrides vendor default + "someEnvVar": "false", // user value takes precedence }, }, }, - err: nil, + istioCniPostValues: &v1.CNIValues{}, + expectedIstioError: false, + expectedIstioCniError: false, + expectedErrSubstring: "", // no error expected }, } for _, tc := range testcases { vendorDefaults = MustParseVendorDefaultsYAML([]byte(tc.vendorDefaults)) - result, err := ApplyVendorDefaults(tc.version, tc.preValues) - if err != tc.err { - t.Errorf("unexpected error: %v, expected %v", err, tc.err) + + // Test Istio values + istioResult, istioErr := ApplyIstioVendorDefaults(tc.version, tc.istioPreValues) + if tc.expectedIstioError { + if assert.Error(t, istioErr, "expected an error for Istio on %s but got none", tc.name) { + assert.ErrorContains(t, istioErr, tc.expectedErrSubstring, + "Istio default values on %s should unwrap JSON-unmarshal errors", tc.name) + } + } else { + assert.NoError(t, istioErr, "unexpected error for Istio on %s: %v", tc.name, istioErr) } - if diff := cmp.Diff(tc.postValues, result); diff != "" { - t.Errorf("unexpected merge result; diff (-expected, +actual):\n%v", diff) + if diff := cmp.Diff(tc.istoPostValues, istioResult); diff != "" { + t.Errorf("unexpected Istio merge result; diff (-expected, +actual):\n%v", diff) + } + + // Test IstioCNI values + cniResult, cniErr := ApplyIstioCNIVendorDefaults(tc.version, tc.istioCNIPreValues) + if tc.expectedIstioCniError { + if assert.Error(t, cniErr, "expected an error for IstioCNI on %s but got none", tc.name) { + assert.ErrorContains(t, cniErr, tc.expectedErrSubstring, + "IstioCNI default values on %s should unwrap JSON-unmarshal errors", tc.name) + } + } else { + assert.NoError(t, cniErr, "unexpected error for IstioCNI on %s: %v", tc.name, cniErr) + } + if diff := cmp.Diff(tc.istioCniPostValues, cniResult); diff != "" { + t.Errorf("unexpected IstioCNI merge result; diff (-expected, +actual):\n%v", diff) } } } @@ -99,9 +252,47 @@ func TestValidateVendorDefaultsFile(t *testing.T) { } for version := range vendorDefaults { - _, err := ApplyVendorDefaults(version, &v1.Values{}) + _, err := ApplyIstioVendorDefaults(version, &v1.Values{}) if err != nil { t.Errorf("failed to parse vendor_defaults.yaml at version %s: %v", version, err) } } } + +// OverrideVendorDefaults allows tests to override the vendor defaults +func TestOverrideVendorDefaults(t *testing.T) { + // Create a copy of the original vendor defaults + originalDefaults := vendorDefaults + + // Define new defaults to override + newDefaults := map[string]map[string]any{ + "v1.24.2": { + "istio": map[string]any{ + "pilot": map[string]any{ + "env": map[string]string{ + "newEnvVar": "true", + }, + }, + }, + "istiocni": map[string]any{ + "cni": map[string]any{ + "cniConfDir": "new/path", + }, + }, + }, + } + + // Override the vendor defaults + OverrideVendorDefaults(newDefaults) + + // Validate the override + assert.Equal(t, newDefaults, vendorDefaults, "Vendor defaults should be overridden") + + // Restore the original defaults + vendorDefaults = originalDefaults +} + +// StringPtr returns a pointer to a string literal. +func StringPtr(s string) *string { + return &s +} diff --git a/pkg/revision/values.go b/pkg/revision/values.go index f5ebcb9ed..759e84a59 100644 --- a/pkg/revision/values.go +++ b/pkg/revision/values.go @@ -37,7 +37,7 @@ func ComputeValues( userValues = istiovalues.ApplyDigests(version, userValues, config.Config) // apply vendor-specific default values - userValues, err := istiovalues.ApplyVendorDefaults(version, userValues) + userValues, err := istiovalues.ApplyIstioVendorDefaults(version, userValues) if err != nil { return nil, fmt.Errorf("failed to apply vendor defaults: %w", err) } diff --git a/pkg/test/environment.go b/pkg/test/environment.go index ea379025f..3680a5ee1 100644 --- a/pkg/test/environment.go +++ b/pkg/test/environment.go @@ -20,6 +20,7 @@ import ( "github.com/istio-ecosystem/sail-operator/pkg/scheme" "github.com/istio-ecosystem/sail-operator/pkg/test/project" + "go.uber.org/zap/zapcore" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" @@ -28,7 +29,7 @@ import ( ) func SetupEnv(logWriter io.Writer, installCRDs bool) (*envtest.Environment, client.Client, *rest.Config) { - logf.SetLogger(zap.New(zap.WriteTo(logWriter), zap.UseDevMode(true))) + logf.SetLogger(zap.New(zap.WriteTo(logWriter), zap.UseDevMode(true), zap.Level(zapcore.Level(-3)))) var crdDirectoryPaths []string if installCRDs { diff --git a/tests/documentation_tests/README-runme.md b/tests/documentation_tests/README-runme.md deleted file mode 100644 index 9e0be03d2..000000000 --- a/tests/documentation_tests/README-runme.md +++ /dev/null @@ -1,2564 +0,0 @@ -[Return to Project Root](../) -*Note*: To add new topics to this documentation, please follow the guidelines in the [guidelines](../docs/guidelines/guidelines.md) doc. - -# Table of Contents - -- [User Documentation](#user-documentation) -- [Concepts](#concepts) - - [Istio resource](#istio-resource) - - [IstioRevision resource](#istiorevision-resource) - - [IstioRevisionTag resource](#istiorevisiontag-resource) - - [IstioCNI resource](#istiocni-resource) - - [Updating the IstioCNI resource](#updating-the-istiocni-resource) - - [Resource Status](#resource-status) - - [InUse Detection](#inuse-detection) -- [API Reference documentation](#api-reference-documentation) -- [Getting Started](#getting-started) - - [Installation on OpenShift](#installation-on-openshift) - - [Installing through the web console](#installing-through-the-web-console) - - [Installing using the CLI](#installing-using-the-cli) - - [Installation from Source](#installation-from-source) -- [Migrating from Istio in-cluster Operator](#migrating-from-istio-in-cluster-operator) -- [Gateways](#gateways) -- [Update Strategy](#update-strategy) - - [InPlace](#inplace) - - [Example using the InPlace strategy](#example-using-the-inplace-strategy) - - [RevisionBased](#revisionbased) - - [Example using the RevisionBased strategy](#example-using-the-revisionbased-strategy) - - [Example using the RevisionBased strategy and an IstioRevisionTag](#example-using-the-revisionbased-strategy-and-an-istiorevisiontag) -- [Multiple meshes on a single cluster](#multiple-meshes-on-a-single-cluster) - - [Prerequisites](#prerequisites) - - [Installation Steps](#installation-steps) - - [Deploying the control planes](#deploying-the-control-planes) - - [Deploying the applications](#deploying-the-applications) - - [Validation](#validation) - - [Checking application to control plane mapping](#checking-application-to-control-plane-mapping) - - [Checking application connectivity](#checking-application-connectivity) -- [Multi-cluster](#multi-cluster) - - [Prerequisites](#prerequisites) - - [Common Setup](#common-setup) - - [Multi-Primary](#multi-primary---multi-network) - - [Primary-Remote](#primary-remote---multi-network) - - [External Control Plane](#external-control-plane) -- [Dual-stack Support](#dual-stack-support) - - [Prerequisites](#prerequisites-1) - - [Installation Steps](#installation-steps) - - [Validation](#validation) -- [Ambient mode](common/istio-ambient-mode.md) - - [Getting Started](common/istio-ambient-mode.md#getting-started) - - [Use Waypoint Proxies](common/istio-ambient-waypoint.md) -- [Addons](#addons) - - [Deploy Prometheus and Jaeger addons](#deploy-prometheus-and-jaeger-addons) - - [Deploy Kiali addon](#deploy-kiali-addon) - - [Deploy Gateway and Bookinfo](#deploy-gateway-and-bookinfo) - - [Generate traffic and visualize your mesh](#generate-traffic-and-visualize-your-mesh) -- [Observability Integrations](#observability-integrations) - - [Scraping metrics using the OpenShift monitoring stack](#scraping-metrics-using-the-openshift-monitoring-stack) - - [Configure Tracing with OpenShift distributed tracing](#configure-tracing-with-openshift-distributed-tracing) - - [Integrating with Kiali](#integrating-with-kiali) - - [Integrating Kiali with the OpenShift monitoring stack](#integrating-kiali-with-the-openshift-monitoring-stack) - - [Integrating Kiali with OpenShift distributed tracing](#integrating-kiali-with-openshift-distributed-tracing) -- [Uninstalling](#uninstalling) - - [Deleting Istio](#deleting-istio) - - [Deleting IstioCNI](#deleting-istiocni) - - [Deleting the Sail Operator](#deleting-the-sail-operator) - - [Deleting the istio-system and istio-cni Projects](#deleting-the-istio-system-and-istiocni-projects) - - [Decide whether you want to delete the CRDs as well](#decide-whether-you-want-to-delete-the-crds-as-well) - - -# User Documentation -Sail Operator manages the lifecycle of your Istio control planes. Instead of creating a new configuration schema, Sail Operator APIs are built around Istio's helm chart APIs. All installation and configuration options that are exposed by Istio's helm charts are available through the Sail Operator CRDs' `values` fields. - -Similar to using Istio's Helm charts, the final set of values used to render the charts is determined by a combination of user-provided values, default chart values, and values from selected profiles. -These profiles can include the user-defined profile, the platform profile, and the compatibility version profile. -To view the final set of values, inspect the ConfigMap named `values` (or `values-`) in the namespace where the control plane is installed. - -## Concepts - -### Istio resource -The `Istio` resource is used to manage your Istio control planes. It is a cluster-wide resource, as the Istio control plane operates in and requires access to the entire cluster. To select a namespace to run the control plane pods in, you can use the `spec.namespace` field. Note that this field is immutable, though: in order to move a control plane to another namespace, you have to remove the Istio resource and recreate it with a different `spec.namespace`. You can access all helm chart options through the `values` field in the `spec`: - -```yaml -apiVersion: sailoperator.io/v1 -kind: Istio -metadata: - name: default -spec: - namespace: istio-system - updateStrategy: - type: InPlace - values: - pilot: - resources: - requests: - cpu: 100m - memory: 1024Mi -``` - -Note: If you need a specific Istio version, you can explicitly set it using `spec.version`. If not specified, the Operator will install the latest supported version. - -Istio uses a ConfigMap for its global configuration, called the MeshConfig. All of its settings are available through `spec.meshConfig`. - -To support canary updates of the control plane, Sail Operator includes support for multiple Istio versions. You can select a version by setting the `version` field in the `spec` to the version you would like to install, prefixed with a `v`. You can then update to a new version just by changing this field. An `vX.Y-latest` alias can be used for the latest z/patch versions of each supported y/minor versions. As per the example above, `v1.26-latest` can be specified in the `version` field. By doing so, the operator will keep the istio version with the latest `z` version of the same `y` version. - -Sail Operator supports two different update strategies for your control planes: `InPlace` and `RevisionBased`. When using `InPlace`, the operator will immediately replace your existing control plane resources with the ones for the new version, whereas `RevisionBased` uses Istio's canary update mechanism by creating a second control plane to which you can migrate your workloads to complete the update. - -After creation of an `Istio` resource, the Sail Operator will generate a revision name for it based on the updateStrategy that was chosen, and create a corresponding [`IstioRevision`](#istiorevision-resource). - -### IstioRevision resource -The `IstioRevision` is the lowest-level API the Sail Operator provides, and it is usually not created by the user, but by the operator itself. It's schema closely resembles that of the `Istio` resource - but instead of representing the state of a control plane you want to be present in your cluster, it represents a *revision* of that control plane, which is an instance of Istio with a specific version and revision name, and its revision name can be used to add workloads or entire namespaces to the mesh, e.g. by using the `istio.io/rev=` label. It is also a cluster-wide resource. - -You can think of the relationship between the `Istio` and `IstioRevision` resource as similar to the one between Kubernetes' `ReplicaSet` and `Pod`: a `ReplicaSet` can be created by users and results in the automatic creation of `Pods`, which will trigger the instantiation of your containers. Similarly, users create an `Istio` resource which instructs the operator to create a matching `IstioRevision`, which then in turn triggers the creation of the Istio control plane. To do that, the Sail Operator will copy all of your relevant configuration from the `Istio` resource to the `IstioRevision` resource. - -### IstioRevisionTag resource -The `IstioRevisionTag` resource represents a *Stable Revision Tag*, which functions as an alias for Istio control plane revisions. With a stable tag `prod`, you can e.g. use the label `istio.io/rev=prod` to inject proxies into your workloads. When you perform an upgrade to a control plane with a new revision name, you can simply update your tag to point to the new revision, instead of having to re-label your workloads and namespaces. Also see the [Stable Revision Tags](https://istio.io/latest/docs/setup/upgrade/canary/#stable-revision-labels) section of Istio's [Canary Upgrades documentation](https://istio.io/latest/docs/setup/upgrade/canary/) for more details. - -In Istio, stable revision tags are usually created using `istioctl`, but if you're using the Sail Operator, you can use the `IstioRevisionTag` resource, which comes with an additional feature: instead of just being able to reference an `IstioRevision`, you can also reference an `Istio` resource. When you now update your control plane and the underlying `IstioRevision` changes, the Sail Operator will update your revision tag for you. You only need to restart your deployments to re-inject the new proxies. - -```yaml -apiVersion: sailoperator.io/v1 -kind: IstioRevisionTag -metadata: - name: default -spec: - targetRef: - kind: Istio # can be either Istio or IstioRevision - name: prod # the name of the Istio/IstioRevision resource -``` - -As you can see in the YAML above, `IstioRevisionTag` really only has one field in its spec: `targetRef`. With this field, you can reference an `Istio` or `IstioRevision` resource. So after deploying this, you will be able to use both the `istio.io/rev=default` and also `istio-injection=enabled` labels to inject proxies into your workloads. The `istio-injection` label can only be used for revisions and revision tags named `default`, like the `IstioRevisionTag` in the above example. - -### IstioCNI resource -The lifecycle of Istio's CNI plugin is managed separately when using Sail Operator. To install it, you can create an `IstioCNI` resource. The `IstioCNI` resource is a cluster-wide resource as it will install a `DaemonSet` that will be operating on all nodes of your cluster. You can select a version by setting the `spec.version` field, as you can see in the sample below: - -```yaml -apiVersion: sailoperator.io/v1 -kind: IstioCNI -metadata: - name: default -spec: - namespace: istio-cni - values: - cni: - cniConfDir: /etc/cni/net.d - excludeNamespaces: - - kube-system -``` - -Note: If you need a specific Istio version, you can explicitly set it using `spec.version`. If not specified, the Operator will install the latest supported version. - -#### Updating the IstioCNI resource -Updates for the `IstioCNI` resource are `Inplace` updates, this means that the `DaemonSet` will be updated with the new version of the CNI plugin once the resource is updated and the `istio-cni-node` pods are going to be replaced with the new version. -To update the CNI plugin, just change the `version` field to the version you want to install. Just like the `Istio` resource, it also has a `values` field that exposes all of the options provided in the `istio-cni` chart: - -1. Create the `IstioCNI` resource. - ```bash { name=install-cni tag=cni-update} - kubectl create ns istio-cni - cat < [!NOTE] -> The CNI plugin at version `1.x` is compatible with `Istio` at version `1.x-1`, `1.x` and `1.x+1`. - -### Resource Status -All of the Sail Operator API resources have a `status` subresource that contains information about their current state in the Kubernetes cluster. - -#### Conditions -All resources have a `Ready` condition which is set to `true` as soon as all child resource have been created and are deemed Ready by their respective controllers. To see additional conditions for each of the resources, check the [API reference documentation](https://github.com/istio-ecosystem/sail-operator/tree/main/docs/api-reference/sailoperator.io.md). - -#### InUse Detection -The Sail Operator uses InUse detection to determine whether an object is referenced. This is currently present on all resources apart from `IstioCNI`. On the `Istio` resource, it is a counter as it only aggregates the `InUse` conditions on its child `IstioRevisions`. - -|API |Type |Name|Description -|------------------|------------|----|------------------------------------------- -|Istio |Counter |Status.Revisions.InUse|Aggregates across all child `IstioRevisions`. -|IstioRevision |Condition |Status.Conditions[type="InUse']|Set to `true` if the `IstioRevision` is referenced by a namespace, workload or `IstioRevisionTag`. -|IstioRevisionTag |Condition |Status.Conditions[type="InUse']|Set to `true` if the `IstioRevisionTag` is referenced by a namespace or workload. - -## API Reference documentation -The Sail Operator API reference documentation can be found [here](https://github.com/istio-ecosystem/sail-operator/tree/main/docs/api-reference/sailoperator.io.md). - -## Getting Started - -### Installation on OpenShift - -#### Installing through the web console - -1. In the OpenShift Console, navigate to the OperatorHub by clicking **Operator** -> **Operator Hub** in the left side-pane. - -1. Search for "sail". - -1. Locate the Sail Operator, and click to select it. - -1. When the prompt that discusses the community operator appears, click **Continue**, then click **Install**. - -1. Use the default installation settings presented, and click **Install** to continue. - -1. Click **Operators** -> **Installed Operators** to verify that the Sail Operator -is installed. `Succeeded` should appear in the **Status** column. - -#### Installing using the CLI - -*Prerequisites* - -* You have access to the cluster as a user with the `cluster-admin` cluster role. - -*Steps* - -1. Create the `openshift-operators` namespace (if it does not already exist). - - ```bash - kubectl create namespace openshift-operators - ``` - -1. Create the `Subscription` object with the desired `spec.channel`. - - ```bash - kubectl apply -f - <.enabled: true/false`. Other functionality exposed through `spec.components` like the k8s overlays is not currently available. - -### Converter Script -This script is used to convert an Istio in-cluster operator configuration to a Sail Operator configuration. Upon execution, the script takes an input YAML file and istio version and generates a sail operator configuration file. - -#### Usage -To run the configuration-converter.sh script, you need to provide four arguments, only input file is required other arguments are optional: - -1. Input file path (): The path to your Istio operator configuration YAML file (required). -2. Output file path (): The path where the converted Sail configuration will be saved. If not provided, the script will save the output with -sail.yaml appended to the input file name. -3. Namespace (-n ): The Kubernetes namespace for the Istio deployment. Defaults to istio-system if not provided. -4. Version (-v ): The version of Istio to be used. If not provided, the `spec.version` field will be omitted from the output file and the operator will deploy the latest version when the YAML manifest is applied. - -```bash -./tools/configuration-converter.sh [/path/to/output.yaml] [-n namespace] [-v version] -``` - -##### Sample command only with input file: - -```bash -./tools/configuration-converter.sh /home/user/istio_config.yaml -``` - -##### Sample command with custom output, namespace, and version: - -```bash -./tools/configuration-converter.sh /home/user/input/istio_config.yaml /home/user/output/output.yaml -n custom-namespace -v v1.24.3 -``` - -> [!WARNING] -> This script is still under development. -> Please verify the resulting configuration carefully after conversion to ensure it meets your expectations and requirements. - -### CNI - -The CNI plugin's lifecycle is managed separately from the control plane. You will have to create a [IstioCNI resource](#istiocni-resource) to use CNI. - -## Gateways - -[Gateways in Istio](https://istio.io/latest/docs/concepts/traffic-management/#gateways) are used to manage inbound and outbound traffic for the mesh. The Sail Operator does not deploy or manage Gateways. You can deploy a gateway either through [gateway-api](https://istio.io/latest/docs/tasks/traffic-management/ingress/gateway-api/) or through [gateway injection](https://istio.io/latest/docs/setup/additional-setup/gateway/#deploying-a-gateway). As you are following the gateway installation instructions, skip the step to install Istio since this is handled by the Sail Operator. - -**Note:** The `IstioOperator` / `istioctl` example is separate from the Sail Operator. Setting `spec.components` or `spec.values.gateways` on your Sail Operator `Istio` resource **will not work**. - -For examples installing Gateways on OpenShift, see the [Gateways](common/create-and-configure-gateways.md) page. - -## Update Strategy - -The Sail Operator supports two update strategies to update the version of the Istio control plane: `InPlace` and `RevisionBased`. The default strategy is `InPlace`. - -### InPlace -When the `InPlace` strategy is used, the existing Istio control plane is replaced with a new version. The workload sidecars immediately connect to the new control plane. The workloads therefore don't need to be moved from one control plane instance to another. - -#### Example using the InPlace strategy - -Prerequisites: -* Sail Operator is installed. -* `istioctl` is [installed](common/install-istioctl-tool.md). - -Steps: -1. Create the `istio-system` namespace. - - ```bash { name=create-istio-ns tag=inplace-update} - kubectl create namespace istio-system - ``` - -2. Create the `Istio` resource. - - ```bash { name=create-istio-resource tag=inplace-update} - cat <-`. -```bash { name=validation-print-revision tag=revision-based-update} - kubectl get istiorevision -n istio-system -``` - -5. Create `bookinfo` namespace and label it with the revision name. - - ```bash { name=create-bookinfo-ns tag=revision-based-update} - kubectl create namespace bookinfo - kubectl label namespace bookinfo istio.io/rev=default-v1-25-3 - ``` - -6. Deploy bookinfo application. - - ```bash { name=deploy-bookinfo tag=revision-based-update} - kubectl apply -n bookinfo -f https://raw.githubusercontent.com/istio/istio/release-1.22/samples/bookinfo/platform/kube/bookinfo.yaml - ``` -```bash { name=validation-wait-bookinfo tag=revision-based-update} - . scripts/prebuilt-func.sh - with_retries wait_pods_ready_by_ns "bookinfo" - kubectl get pods -n bookinfo - istioctl proxy-status - with_retries pods_istio_version_match "bookinfo" "1.25.3" -``` -7. Review the `Istio` resource after application deployment. - - ```console - $ kubectl get istio -n istio-system - NAME REVISIONS READY IN USE ACTIVE REVISION STATUS VERSION AGE - default 1 1 1 default-v1-25-3 Healthy v1.25.3 5m13s - ``` - Note: `IN USE` field shows as 1, after application being deployed. -```bash { name=validation-istio-in-use tag=revision-based-update} - . scripts/prebuilt-func.sh - with_retries istio_active_revision_match "default-v1-25-3" -``` -8. Confirm that the proxy version matches the control plane version. - - ```bash - istioctl proxy-status - ``` - The column `VERSION` should match the control plane version. - -9. Update the control plane to a new version. - - ```bash - kubectl patch istio default -n istio-system --type='merge' -p '{"spec":{"version":"v1.26.0"}}' - ``` -```bash { name=validation-wait-istio-updated tag=revision-based-update} - kubectl patch istio default -n istio-system --type='merge' -p '{"spec":{"version":"v1.26.0"}}' - . scripts/prebuilt-func.sh - with_retries istiod_pods_count "2" - wait_istio_ready "istio-system" - print_istio_info -``` -10. Verify the `Istio` and `IstioRevision` resources. There will be a new revision created with the new version. - - ```console - $ kubectl get istio - NAME REVISIONS READY IN USE ACTIVE REVISION STATUS VERSION AGE - default 2 2 1 default-v1-26-0 Healthy v1.26.0 9m23s - - $ kubectl get istiorevision - NAME TYPE READY STATUS IN USE VERSION AGE - default-v1-25-3 Local True Healthy True v1.25.3 10m - default-v1-26-0 Local True Healthy False v1.26.0 66s - ``` -```bash { name=validation-istiorevision tag=revision-based-update} - . scripts/prebuilt-func.sh - kubectl get istio - kubectl get istiorevision -n istio-system - with_retries istio_active_revision_match "default-v1-26-0" - with_retries istio_revisions_ready_count "2" -``` -11. Confirm there are two control plane pods running, one for each revision. - - ```console - $ kubectl get pods -n istio-system - NAME READY STATUS RESTARTS AGE - istiod-default-v1-25-3-c98fd9675-r7bfw 1/1 Running 0 10m - istiod-default-v1-26-0-7495cdc7bf-v8t4g 1/1 Running 0 113s - ``` -```bash { name=validation-istiod-count tag=revision-based-update} - . scripts/prebuilt-func.sh - with_retries istiod_pods_count "2" -``` -12. Confirm the proxy sidecar version remains the same: - - ```bash - istioctl proxy-status - ``` - The column `VERSION` should still match the old control plane version. -```bash { name=validation-wait-bookinfo tag=revision-based-update} - . scripts/prebuilt-func.sh - istioctl proxy-status - with_retries pods_istio_version_match "bookinfo" "1.25.3" -``` -13. Change the label of the `bookinfo` namespace to use the new revision. - - ```bash { name=update-bookinfo-ns-revision tag=revision-based-update} - kubectl label namespace bookinfo istio.io/rev=default-v1-26-0 --overwrite - ``` - The existing workload sidecars will continue to run and will remain connected to the old control plane instance. They will not be replaced with a new version until the pods are deleted and recreated. - -14. Restart all Deplyments in the `bookinfo` namespace. - - ```bash - kubectl rollout restart deployment -n bookinfo - ``` -```bash { name=validation-wait-bookinfo tag=revision-based-update} - pod_names=$(kubectl get pods -n bookinfo -o name) - kubectl rollout restart deployment -n bookinfo - # Wait pod deletion - for pod in $pod_names; do - kubectl wait --for=delete $pod -n bookinfo --timeout=60s - done - . scripts/prebuilt-func.sh - with_retries wait_pods_ready_by_ns "bookinfo" - kubectl get pods -n bookinfo - istioctl proxy-status - with_retries pods_istio_version_match "bookinfo" "1.26.0" -``` -15. Confirm the new version is used in the sidecars. - - ```bash - istioctl proxy-status - ``` - The column `VERSION` should match the updated control plane version. - -16. Confirm the deletion of the old control plane and IstioRevision. - - ```console - $ kubectl get pods -n istio-system - NAME READY STATUS RESTARTS AGE - istiod-default-v1-26-0-7495cdc7bf-v8t4g 1/1 Running 0 4m40s - - $ kubectl get istio - NAME REVISIONS READY IN USE ACTIVE REVISION STATUS VERSION AGE - default 1 1 1 default-v1-26-0 Healthy v1.26.0 5m - - $ kubectl get istiorevision - NAME TYPE READY STATUS IN USE VERSION AGE - default-v1-26-0 Local True Healthy True v1.26.0 5m31s - ``` - The old `IstioRevision` resource and the old control plane will be deleted when the grace period specified in the `Istio` resource field `spec.updateStrategy.inactiveRevisionDeletionGracePeriodSeconds` expires. -```bash { name=validation-resources-deletion tag=revision-based-update} - . scripts/prebuilt-func.sh - echo "Confirm istiod pod is deleted" - with_retries istiod_pods_count "1" - echo "Confirm istiorevision is deleted" - with_retries istio_revisions_ready_count "1" - print_istio_info -``` -#### Example using the RevisionBased strategy and an IstioRevisionTag - -Prerequisites: -* Sail Operator is installed. -* `istioctl` is [installed](common/install-istioctl-tool.md). - -Steps: - -1. Create the `istio-system` namespace. - - ```bash { name=create-ns tag=istiorevisiontag} - kubectl create namespace istio-system - ``` - -2. Create the `Istio` and `IstioRevisionTag` resources. - - ```bash { name=create-istio-and-revision-tag tag=istiorevisiontag} - cat < - export CTX_CLUSTER2= - export ISTIO_VERSION=1.26.0 - ``` - -2. Create `istio-system` namespace on each cluster. - - ```bash - kubectl get ns istio-system --context "${CTX_CLUSTER1}" || kubectl create namespace istio-system --context "${CTX_CLUSTER1}" - kubectl get ns istio-system --context "${CTX_CLUSTER2}" || kubectl create namespace istio-system --context "${CTX_CLUSTER2}" - ``` - -4. Create a shared root certificate. - - If you have [established trust](https://istio.io/latest/docs/setup/install/multicluster/before-you-begin/#configure-trust) between your clusters already you can skip this and the following steps. - - ```bash - openssl genrsa -out root-key.pem 4096 - cat < root-ca.conf - [ req ] - encrypt_key = no - prompt = no - utf8 = yes - default_md = sha256 - default_bits = 4096 - req_extensions = req_ext - x509_extensions = req_ext - distinguished_name = req_dn - [ req_ext ] - subjectKeyIdentifier = hash - basicConstraints = critical, CA:true - keyUsage = critical, digitalSignature, nonRepudiation, keyEncipherment, keyCertSign - [ req_dn ] - O = Istio - CN = Root CA - EOF - - openssl req -sha256 -new -key root-key.pem \ - -config root-ca.conf \ - -out root-cert.csr - - openssl x509 -req -sha256 -days 3650 \ - -signkey root-key.pem \ - -extensions req_ext -extfile root-ca.conf \ - -in root-cert.csr \ - -out root-cert.pem - ``` -5. Create intermediate certificates. - - ```bash - for cluster in west east; do - mkdir $cluster - - openssl genrsa -out ${cluster}/ca-key.pem 4096 - cat < ${cluster}/intermediate.conf - [ req ] - encrypt_key = no - prompt = no - utf8 = yes - default_md = sha256 - default_bits = 4096 - req_extensions = req_ext - x509_extensions = req_ext - distinguished_name = req_dn - [ req_ext ] - subjectKeyIdentifier = hash - basicConstraints = critical, CA:true, pathlen:0 - keyUsage = critical, digitalSignature, nonRepudiation, keyEncipherment, keyCertSign - subjectAltName=@san - [ san ] - DNS.1 = istiod.istio-system.svc - [ req_dn ] - O = Istio - CN = Intermediate CA - L = $cluster - EOF - - openssl req -new -config ${cluster}/intermediate.conf \ - -key ${cluster}/ca-key.pem \ - -out ${cluster}/cluster-ca.csr - - openssl x509 -req -sha256 -days 3650 \ - -CA root-cert.pem \ - -CAkey root-key.pem -CAcreateserial \ - -extensions req_ext -extfile ${cluster}/intermediate.conf \ - -in ${cluster}/cluster-ca.csr \ - -out ${cluster}/ca-cert.pem - - cat ${cluster}/ca-cert.pem root-cert.pem \ - > ${cluster}/cert-chain.pem - cp root-cert.pem ${cluster} - done - ``` - -6. Push the intermediate CAs to each cluster. - ```bash - kubectl --context "${CTX_CLUSTER1}" label namespace istio-system topology.istio.io/network=network1 - kubectl get secret -n istio-system --context "${CTX_CLUSTER1}" cacerts || kubectl create secret generic cacerts -n istio-system --context "${CTX_CLUSTER1}" \ - --from-file=east/ca-cert.pem \ - --from-file=east/ca-key.pem \ - --from-file=east/root-cert.pem \ - --from-file=east/cert-chain.pem - kubectl --context "${CTX_CLUSTER2}" label namespace istio-system topology.istio.io/network=network2 - kubectl get secret -n istio-system --context "${CTX_CLUSTER2}" cacerts || kubectl create secret generic cacerts -n istio-system --context "${CTX_CLUSTER2}" \ - --from-file=west/ca-cert.pem \ - --from-file=west/ca-key.pem \ - --from-file=west/root-cert.pem \ - --from-file=west/cert-chain.pem - ``` - -### Multi-Primary - Multi-Network - -These instructions install a [multi-primary/multi-network](https://istio.io/latest/docs/setup/install/multicluster/multi-primary_multi-network/) Istio deployment using the Sail Operator and Sail CRDs. **Before you begin**, ensure you complete the [common setup](#common-setup). - -You can follow the steps below to install manually or you can run [this script](multicluster/setup-multi-primary.sh) which will setup a local environment for you with kind. Before running the setup script, you must install [kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) and [cloud-provider-kind](https://kind.sigs.k8s.io/docs/user/loadbalancer/#installing-cloud-provider-kind) then ensure the `cloud-provider-kind` binary is running in the background. - -These installation instructions are adapted from: https://istio.io/latest/docs/setup/install/multicluster/multi-primary_multi-network/. - -1. Create an `Istio` resource on `cluster1`. - - ```bash - kubectl apply --context "${CTX_CLUSTER1}" -f - < /dev/null || \ - { kubectl kustomize "github.com/kubernetes-sigs/gateway-api/config/crd?ref=v1.1.0" | kubectl apply -f - --context="${CTX_CLUSTER2}"; } - ``` - - Expose `helloworld` through the ingress gateway. - ```bash - kubectl apply -f "https://raw.githubusercontent.com/istio/istio/${ISTIO_VERSION}/samples/helloworld/gateway-api/helloworld-gateway.yaml" -n sample --context="${CTX_CLUSTER2}" - kubectl -n sample --context="${CTX_CLUSTER2}" wait --for=condition=programmed gtw helloworld-gateway - ``` - - Confirm you can access the `helloworld` application through the ingress gateway created in the Remote cluster. - ```bash - curl -s "http://$(kubectl -n sample --context="${CTX_CLUSTER2}" get gtw helloworld-gateway -o jsonpath='{.status.addresses[0].value}'):80/hello" - ``` - You should see a response from the `helloworld` application: - ```bash - Hello version: v1, instance: helloworld-v1-6d65866976-jb6qc - ``` - -15. Cleanup - - ```bash - kubectl delete istios default --context="${CTX_CLUSTER1}" - kubectl delete ns istio-system --context="${CTX_CLUSTER1}" - kubectl delete istios external-istiod --context="${CTX_CLUSTER1}" - kubectl delete ns external-istiod --context="${CTX_CLUSTER1}" - kubectl delete istios external-istiod --context="${CTX_CLUSTER2}" - kubectl delete ns external-istiod --context="${CTX_CLUSTER2}" - kubectl delete ns sample --context="${CTX_CLUSTER2}" - ``` - -## Dual-stack Support - -Kubernetes supports dual-stack networking as a stable feature starting from -[v1.23](https://kubernetes.io/docs/concepts/services-networking/dual-stack/), allowing clusters to handle both -IPv4 and IPv6 traffic. With many cloud providers also beginning to offer dual-stack Kubernetes clusters, it's easier -than ever to run services that function across both address types. Istio introduced dual-stack as an experimental -feature in version 1.17, and promoted it to [Alpha](https://istio.io/latest/news/releases/1.24.x/announcing-1.24/change-notes/) in -version 1.24. With Istio in dual-stack mode, services can communicate over both IPv4 and IPv6 endpoints, which helps -organizations transition to IPv6 while still maintaining compatibility with their existing IPv4 infrastructure. - -When Kubernetes is configured for dual-stack, it automatically assigns an IPv4 and an IPv6 address to each pod, -enabling them to communicate over both IP families. For services, however, you can control how they behave using -the `ipFamilyPolicy` setting. - -Service.Spec.ipFamilyPolicy can take the following values -- SingleStack: Only one IP family is configured for the service, which can be either IPv4 or IPv6. -- PreferDualStack: Both IPv4 and IPv6 cluster IPs are assigned to the Service when dual-stack is enabled. - However, if dual-stack is not enabled or supported, it falls back to singleStack behavior. -- RequireDualStack: The service will be created only if both IPv4 and IPv6 addresses can be assigned. - -This allows you to specify the type of service, providing flexibility in managing your network configuration. -For more details, you can refer to the Kubernetes [documentation](https://kubernetes.io/docs/concepts/services-networking/dual-stack/#services). - -### Prerequisites - -- Kubernetes 1.23 or later configured with dual-stack support. -- Sail Operator is installed. - -### Installation Steps - -You can use any existing Kind cluster that supports dual-stack networking or, alternatively, install one using the following command. - -```bash -kind create cluster --name istio-ds --config - < /dev/null || \ - { kubectl kustomize "github.com/kubernetes-sigs/gateway-api/config/crd?ref=v1.1.0" | kubectl apply -f -; } -``` - -Create bookinfo gateway. - -```bash -kubectl apply -n bookinfo -f https://raw.githubusercontent.com/istio/istio/master/samples/bookinfo/gateway-api/bookinfo-gateway.yaml -kubectl wait -n bookinfo --for=condition=programmed gtw bookinfo-gateway -``` - -### Generate traffic and visualize your mesh - -Send traffic to the productpage service. Note that this command will run until cancelled. - -```bash -export INGRESS_HOST=$(kubectl get gtw bookinfo-gateway -n bookinfo -o jsonpath='{.status.addresses[0].value}') -export INGRESS_PORT=$(kubectl get gtw bookinfo-gateway -n bookinfo -o jsonpath='{.spec.listeners[?(@.name=="http")].port}') -export GATEWAY_URL=$INGRESS_HOST:$INGRESS_PORT -watch curl http://${GATEWAY_URL}/productpage &> /dev/null -``` - -In a separate terminal, open Kiali to visualize your mesh. - -If using Openshift, open the Kiali route: - -```bash -echo https://$(kubectl get routes -n istio-system kiali -o jsonpath='{.spec.host}') -``` - -Otherwise, port forward to the kiali pod directly: - -```bash -kubectl port-forward -n istio-system svc/kiali 20001:20001 -``` - -You can view Kiali dashboard at: http://localhost:20001 - -## Observability Integrations - -### Scraping metrics using the OpenShift monitoring stack -The easiest way to get started with production-grade metrics collection is to use OpenShift's user-workload monitoring stack. The following steps assume that you installed Istio into the `istio-system` namespace. Note that these steps are not specific to the Sail Operator, but describe how to configure user-workload monitoring for Istio in general. - -*Prerequisites* -* User Workload monitoring is [enabled](https://docs.openshift.com/container-platform/latest/observability/monitoring/enabling-monitoring-for-user-defined-projects.html) - -*Steps* -1. Create a ServiceMonitor for istiod. - - ```yaml - apiVersion: monitoring.coreos.com/v1 - kind: ServiceMonitor - metadata: - name: istiod-monitor - namespace: istio-system - spec: - targetLabels: - - app - selector: - matchLabels: - istio: pilot - endpoints: - - port: http-monitoring - interval: 30s - ``` -1. Create a PodMonitor to scrape metrics from the istio-proxy containers. Note that *this resource has to be created in all namespaces where you are running sidecars*. - - ```yaml - apiVersion: monitoring.coreos.com/v1 - kind: PodMonitor - metadata: - name: istio-proxies-monitor - namespace: istio-system - spec: - selector: - matchExpressions: - - key: istio-prometheus-ignore - operator: DoesNotExist - podMetricsEndpoints: - - path: /stats/prometheus - interval: 30s - relabelings: - - action: keep - sourceLabels: ["__meta_kubernetes_pod_container_name"] - regex: "istio-proxy" - - action: keep - sourceLabels: ["__meta_kubernetes_pod_annotationpresent_prometheus_io_scrape"] - - action: replace - regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4}) - replacement: "[$2]:$1" - sourceLabels: ["__meta_kubernetes_pod_annotation_prometheus_io_port","__meta_kubernetes_pod_ip"] - targetLabel: "__address__" - - action: replace - regex: (\d+);((([0-9]+?)(\.|$)){4}) - replacement: "$2:$1" - sourceLabels: ["__meta_kubernetes_pod_annotation_prometheus_io_port","__meta_kubernetes_pod_ip"] - targetLabel: "__address__" - - action: labeldrop - regex: "__meta_kubernetes_pod_label_(.+)" - - sourceLabels: ["__meta_kubernetes_namespace"] - action: replace - targetLabel: namespace - - sourceLabels: ["__meta_kubernetes_pod_name"] - action: replace - targetLabel: pod_name - ``` - -Congratulations! You should now be able to see your control plane and data plane metrics in the OpenShift Console. Just go to Observe -> Metrics and try the query `istio_requests_total`. - -### Configure tracing with OpenShift distributed tracing -This section describes how to setup Istio with OpenShift Distributed Tracing to send distributed traces. - -*Prerequisites* -* A Tempo stack is installed and configured -* An instance of an OpenTelemetry collector is already configured in the istio-system namespace -* An Istio instance is created with the `openshift` profile -* An Istio CNI instance is created with the `openshift` profile - -*Steps* -1. Configure Istio to enable tracing and include the OpenTelemetry settings: - ```yaml - meshConfig: - enableTracing: true - extensionProviders: - - name: otel-tracing - opentelemetry: - port: 4317 - service: otel-collector.istio-system.svc.cluster.local - ``` -The *service* field is the OpenTelemetry collector service in the `istio-system` namespace. - -2. Create an Istio telemetry resource to active the OpenTelemetry tracer - ```yaml - apiVersion: telemetry.istio.io/v1 - kind: Telemetry - metadata: - name: otel-demo - namespace: istio-system - spec: - tracing: - - providers: - - name: otel-tracing - randomSamplingPercentage: 100 - ``` - -3. Validate the integration: Generate some traffic - -We can [Deploy Bookinfo](#deploy-gateway-and-bookinfo) and generate some traffic. - -4. Validate the integration: See the traces in the UI - -```bash -kubectl get routes -n tempo tempo-sample-query-frontend-tempo -``` - -If you [configure Kiali with OpenShift distributed tracing](#integrating-kiali-with-openshift-distributed-tracing) you can verify from there. - -### Integrating with Kiali -Integration with Kiali really depends on how you collect your metrics and traces. Note that Kiali is a separate project which for the purpose of this document we'll expect is installed using the Kiali operator. The steps here are not specific to Sail Operator, but describe how to configure Kiali for use with Istio in general. - -#### Integrating Kiali with the OpenShift monitoring stack -If you followed [Scraping metrics using the OpenShift monitoring stack](#scraping-metrics-using-the-openshift-monitoring-stack), you can set up Kiali to retrieve metrics from there. - -*Prerequisites* -* User Workload monitoring is [enabled](https://docs.openshift.com/container-platform/latest/observability/monitoring/enabling-monitoring-for-user-defined-projects.html) and [configured](#scraping-metrics-using-the-openshift-monitoring-stack) -* Kiali Operator is installed - -*Steps* -1. Create a ClusterRoleBinding for Kiali, so it can view metrics from user-workload monitoring - - ```yaml - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - name: kiali-monitoring-rbac - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-monitoring-view - subjects: - - kind: ServiceAccount - name: kiali-service-account - namespace: istio-system - ``` -1. Find out the revision name of your Istio instance. In our case it is `test`. - - ```console - $ kubectl get istiorevisions.sailoperator.io - NAME READY STATUS IN USE VERSION AGE - test True Healthy True v1.21.0 119m - ``` -1. Create a Kiali resource and point it to your Istio instance. Make sure to replace `test` with your revision name in the fields `config_map_name`, `istio_sidecar_injector_config_map_name`, `istiod_deployment_name` and `url_service_version`. - - ```yaml - apiVersion: kiali.io/v1alpha1 - kind: Kiali - metadata: - name: kiali-user-workload-monitoring - namespace: istio-system - spec: - external_services: - istio: - config_map_name: istio-test - istio_sidecar_injector_config_map_name: istio-sidecar-injector-test - istiod_deployment_name: istiod-test - url_service_version: 'http://istiod-test.istio-system:15014/version' - prometheus: - auth: - type: bearer - use_kiali_token: true - thanos_proxy: - enabled: true - url: https://thanos-querier.openshift-monitoring.svc.cluster.local:9091 - ``` -#### Integrating Kiali with OpenShift Distributed Tracing -This section describes how to setup Kiali with OpenShift Distributed Tracing to read the distributed traces. - -*Prerequisites* -* Istio tracing is [Configured with OpenShift distributed tracing](#configure-tracing-with-openshift-distributed-tracing) - -*Steps* -1. Setup Kiali to access traces from the Tempo frontend: - ```yaml - external_services: - grafana: - enabled: true - url: "http://grafana-istio-system.apps-crc.testing/" - tracing: - enabled: true - provider: tempo - use_grpc: false - in_cluster_url: http://tempo-sample-query-frontend.tempo:3200 - url: 'https://tempo-sample-query-frontend-tempo.apps-crc.testing' - tempo_config: - org_id: "1" - datasource_uid: "a8d2ef1c-d31c-4de5-a90b-e7bc5252cd00" - ``` - -Where: -* `external_services.grafana` section: Is just needed to see the "View in Tracing" link from the Traces tab -* `external_services.tracing.tempo_config`: Is just needed to see the "View in Tracing" link from the Traces tab and redirect to the proper Tempo datasource - -Now, we should be able to see traces from Kiali. For this, you can: -1. Select a Workload/Service/App -2. Click in the "Traces" tab - -## Uninstalling - -### Deleting Istio -1. In the OpenShift Container Platform web console, click **Operators** -> **Installed Operators**. -1. Click **Istio** in the **Provided APIs** column. -1. Click the Options menu, and select **Delete Istio**. -1. At the prompt to confirm the action, click **Delete**. - -### Deleting IstioCNI -1. In the OpenShift Container Platform web console, click **Operators** -> **Installed Operators**. -1. Click **IstioCNI** in the **Provided APIs** column. -1. Click the Options menu, and select **Delete IstioCNI**. -1. At the prompt to confirm the action, click **Delete**. - -### Deleting the Sail Operator -1. In the OpenShift Container Platform web console, click **Operators** -> **Installed Operators**. -1. Locate the Sail Operator. Click the Options menu, and select **Uninstall Operator**. -1. At the prompt to confirm the action, click **Uninstall**. - -### Deleting the istio-system and istio-cni Projects -1. In the OpenShift Container Platform web console, click **Home** -> **Projects**. -1. Locate the name of the project and click the Options menu. -1. Click **Delete Project**. -1. At the prompt to confirm the action, enter the name of the project. -1. Click **Delete**. - -### Decide whether you want to delete the CRDs as well -OLM leaves this [decision](https://olm.operatorframework.io/docs/tasks/uninstall-operator/#step-4-deciding-whether-or-not-to-delete-the-crds-and-apiservices) to the users. -If you want to delete the Istio CRDs, you can use the following command. -```bash -kubectl get crds -oname | grep istio.io | xargs kubectl delete -``` diff --git a/tests/documentation_tests/example-documentation-following-guidelines-runme.md b/tests/documentation_tests/example-documentation-following-guidelines-runme.md deleted file mode 100644 index e27680366..000000000 --- a/tests/documentation_tests/example-documentation-following-guidelines-runme.md +++ /dev/null @@ -1,107 +0,0 @@ -# Example documentation where the guidelines are followed -This is an example doc where the guidelines are followed to achieve the best documentation possible. The doc is going to be used to test the automation workflow that is going to be used to run the tests over the documentation. - -> [!NOTE] -> The docs shown here may not be updated to the latest version of the project, so please take into account that the examples may not work as expected. The goal of this doc is to show how to use the guidelines and how to add new examples to the automation workflow. - -## Runme Test: Installing the operator from the helm repo and creating a Istio resource - -To run this test, you need to have a Kubernetes cluster running and have `kubectl` and `helm` installed on your local machine. Also, `istioctl` should be installed and configured to work with your cluster. - -- Check the commands that are going to be executed: -```bash { ignore=true } -runme list --filename docs/guidelines/example-documentation-following-guidelines.md -``` - -- Run the commands: -```bash { ignore=true } -runme run --filename docs/guidelines/example-documentation-following-guidelines.md --all --skip-prompts -``` -This will run *all* the commands in the file. If you want to run only a specific command, you can use the `--tag` option to filter the commands by tag. For example, to run only the commands with the tag `example`, you can use the following command: -```bash { ignore=true } -runme run --filename docs/common/runme-test.md --tag example --skip-prompts -``` -For more information about tags and how to use them, you can check the [Runme documentation](https://docs.runme.dev/usage/run-tag). - -More information: -- [Cell configuration keys](https://docs.runme.dev/configuration/cell-level#cell-configuration-keys) -- [Running from cli](https://docs.runme.dev/getting-started/cli) - -### Setting a Istio resource - -The `Istio` resource is a custom resource that is used to configure Istio in your cluster. An example of an Istio resource is shown below: - -```yaml { ignore=true } -apiVersion: sailoperator.io/v1 -kind: Istio -metadata: - name: default -spec: - namespace: istio-system - updateStrategy: - type: RevisionBased - inactiveRevisionDeletionGracePeriodSeconds: 30 -``` -- To create the Istio resource, you can use the following command: -```bash { name=create-istio tag=example} -kubectl create ns istio-system -cat < [!NOTE] -> These commented code blocks are validation steps, and they are added like this to be hidden in the final documentation, but they are going to be used in the automation workflow to validate the documentation. For more information, please check the [documentation](/docs/guidelines/guidelines.md#L146). -```bash { name=validation-print-istio-resource tag=example} -kubectl get istio -o yaml -kubectl get deployment sail-operator -n sail-operator -o yaml -``` - -```bash { name=validation-wait-istiod tag=example} -for i in 1 2 3 4 5; do - pods=$(kubectl get pod -l app=istiod -n istio-system -o jsonpath='{.items[*].status.phase}') - echo "Waiting for istiod pod to be running... (current: $pods)" - echo "$pods" | grep -q Running && break - sleep 5 -done -``` - -- To check the status of the Istio resource, you can use the following command: -```bash { name=check-istio tag=example} -kubectl get pods -n istio-system -kubectl get istio -``` - -- Deploy sample application: -```bash { name=deploy-sample-app tag=example} -kubectl create namespace sample -kubectl label namespace sample istio-injection=enabled -kubectl apply -n sample -f https://raw.githubusercontent.com/istio/istio/release-1.24/samples/bookinfo/platform/kube/bookinfo.yaml -``` -```bash { name=validation-wait-sample-app tag=example} -for i in {1..5}; do kubectl wait --for=condition=available --timeout=600s deployment/productpage-v1 -n sample && break || sleep 5; done -``` - -- Check the status of the sample application: -```bash { name=check-sample-app tag=example} -kubectl get pods -n sample -``` -```bash { name=check-sidecar-exist tag=example} -if ! kubectl get pods -n sample -l app=productpage -o jsonpath='{range .items[*]}{@.metadata.name}{" "}{range .spec.containers[*]}{@.name}{" "}{end}{"\n"}{end}' | grep -q istio-proxy; then - echo "No Istio sidecar (istio-proxy) injected in productpage pod!" - exit 1 -fi -``` - -- Check the proxy version of the sample application: -```bash { name=check-proxy-version tag=example} -istioctl proxy-status -``` diff --git a/tests/documentation_tests/scripts/run-docs-examples.sh b/tests/documentation_tests/scripts/run-docs-examples.sh index 209acdefc..0f5f2fe58 100755 --- a/tests/documentation_tests/scripts/run-docs-examples.sh +++ b/tests/documentation_tests/scripts/run-docs-examples.sh @@ -16,9 +16,10 @@ set -eu -o pipefail -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC2155 +export SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT_DIR="$(cd "$SCRIPT_DIR/../../.." && pwd)" -TEST_DIR="$ROOT_DIR/tests/documentation_tests" +FOCUS_DOC_TAGS="${FOCUS_DOC_TAGS:-}" export KIND_CLUSTER_NAME="docs-automation" export IP_FAMILY="ipv4" @@ -35,9 +36,27 @@ export HUB="${KIND_REGISTRY}" # Workaround make inside make: ovewrite this variable so it is not recomputed in Makefile.core.mk export IMAGE="${HUB}/${IMAGE_BASE}:${TAG}" export ARTIFACTS="${ARTIFACTS:-$(mktemp -d)}" +export TEST_DIR="${ARTIFACTS}/docs.test" export KUBECONFIG="${KUBECONFIG:-"${ARTIFACTS}/config"}" export HELM_TEMPL_DEF_FLAGS="--include-crds --values chart/values.yaml" +# Check that TEST_DIR exist, if not create the directory +if [[ ! -d "$TEST_DIR" ]]; then + echo "Creating TEST_DIR directory: $TEST_DIR" + mkdir -p "$TEST_DIR" +else + echo "Using existing TEST_DIR directory: $TEST_DIR" +fi + +# Run the update-docs-examples.sh script to update the documentation files into the artifacts directory. +"${ROOT_DIR}/tests/documentation_tests/scripts/update-docs-examples.sh" + +# Check that .md files were copied to the artifacts directory. If there is no files, then exit with an error. +if ! find "$TEST_DIR" -maxdepth 1 -name "*.md"; then + echo "No .md files found in the artifacts directory: $TEST_DIR" + exit 1 +fi + # Validate that istioctl is installed if ! command -v istioctl &> /dev/null; then echo "istioctl could not be found. Please install it." @@ -58,11 +77,22 @@ for file in "$TEST_DIR"/*.md; do fi done -# Build list of file-tag pairs +# Build a list of file-tag pairs, isolating 'dual-stack' since it requires special treatment TAGS_LIST=() +dual_stack_tag="" + for file in "${FILES_TO_CHECK[@]}"; do TAGS=$(grep -oP 'tag=\K[^} ]+' "$file" | sort -u) for tag in $TAGS; do + if [[ -n "$FOCUS_DOC_TAGS" && "$tag" != "$FOCUS_DOC_TAGS" ]]; then + continue + fi + + if [[ "$tag" == "dual-stack" ]]; then + dual_stack_tag="$file -t $tag" + continue + fi + TAGS_LIST+=("$file -t $tag") done done @@ -71,22 +101,14 @@ echo "Tags list:" for tag in "${TAGS_LIST[@]}"; do echo "$tag" done +echo "$dual_stack_tag" -# Run each test in its own KIND cluster -for tag in "${TAGS_LIST[@]}"; do +# Run the tests on a separate cluster for all given tags +function run_tests() { ( - echo "Setting up cluster for: $tag" - - # Run the actual doc test - FILE=$(echo "$tag" | cut -d' ' -f1) - RUNME_TAG=$(echo "$tag" | cut -d' ' -f3-) + echo "Setting up cluster: $KIND_CLUSTER_NAME to run tests for tags: $*" + kind delete cluster --name "$KIND_CLUSTER_NAME" > /dev/null || true - # Set IP family only for the dual-stack tag - if [ "$RUNME_TAG" == "dual-stack" ]; then - echo "Setting up dual-stack cluster" - export IP_FAMILY="dual" - export KIND_CLUSTER_NAME="docs-automation-dual-stack" - fi # Source setup and build scripts to preserve trap and env source "${ROOT_DIR}/tests/e2e/setup/setup-kind.sh" @@ -98,19 +120,51 @@ for tag in "${TAGS_LIST[@]}"; do # TODO: check why KUBECONFIG is not properly set kind export kubeconfig --name="${KIND_CLUSTER_NAME}" - # Deploy operator + # Deploy the sail operator kubectl create ns sail-operator || echo "namespace sail-operator already exists" # shellcheck disable=SC2086 helm template chart chart ${HELM_TEMPL_DEF_FLAGS} --set image="${IMAGE}" --namespace sail-operator | kubectl apply --server-side=true -f - kubectl wait --for=condition=available --timeout=600s deployment/sail-operator -n sail-operator - echo "Running: runme run --filename $FILE -t $RUNME_TAG --skip-prompts" - runme run --filename "$FILE" -t "$RUNME_TAG" --skip-prompts - - # Unset IP_FAMILY and KIND_CLUSTER_NAME to avoid conflicts - if [ "$RUNME_TAG" == "dual-stack" ]; then - unset IP_FAMILY - unset KIND_CLUSTER_NAME - fi + # Record all namespaces before each test, to restore to this point + declare -A namespaces + for ns in $(kubectl get namespaces -o name); do + namespaces["$ns"]=1 + done + + for tag in "$@"; do + FILE=$(echo "$tag" | cut -d' ' -f1) + RUNME_TAG=$(echo "$tag" | cut -d' ' -f3-) + + echo "*** Testing '$RUNME_TAG' in file '$file' *** " + runme run --filename "$FILE" -t "$RUNME_TAG" --skip-prompts + echo "*** Testing concluded for '$RUNME_TAG' in file '$file' *** " + + # Save some time by avoiding cleanup for last tag, as the cluster will be deleted anyway. + [ "$tag" != "${!#}" ] || break + + # Clean up any of our cluster-wide custom resources + for crd in $(cat chart/crds/sailoperator.io_*.yaml | yq -rN 'select(.kind == "CustomResourceDefinition" and .spec.scope == "Cluster") | .metadata.name'); do + for cr in $(kubectl get "$crd" -o name); do + kubectl delete "$cr" + done + done + + # Clean up any new namespaces created after the cluster was deployed. + for ns in $(kubectl get namespaces -o name); do + if [ -z "${namespaces[$ns]-}" ]; then + kubectl delete "$ns" + kubectl wait --for=delete "$ns" --timeout=3m + fi + done + done ) -done +} + +# Run tests on a single cluster for all tags that we found +run_tests "${TAGS_LIST[@]}" + +# Run dual stack tests on it's own cluster, since it needs to be deployed with support for dual stack +if [[ -n "$dual_stack_tag" ]]; then + IP_FAMILY="dual" run_tests "$dual_stack_tag" +fi diff --git a/tests/documentation_tests/scripts/update-docs-examples.sh b/tests/documentation_tests/scripts/update-docs-examples.sh index fa5e70200..2429e43f8 100755 --- a/tests/documentation_tests/scripts/update-docs-examples.sh +++ b/tests/documentation_tests/scripts/update-docs-examples.sh @@ -17,7 +17,7 @@ set -eu -o pipefail # This script is used to update the example files in the documentation that are the resulting script to be executed by the runme tool. -# It will copy all the files from the docs/ respository to the test/documentation_tests/ directory. +# It will copy all the files from the docs/ respository to the TEST_DIR defined folder. # It will exclude this files: sailoperator.io.md (Add more here if is needed) # Beside that, after copy the files it will read each one and all the html commented bash code block and it will uncomment them. # This will allow us to hide from docs validation steps @@ -28,14 +28,17 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT_DIR="$(cd "$SCRIPT_DIR/../../.." && pwd)" # Set the documentation directory to the docs/ directory DOCS_DIR="$ROOT_DIR/docs" -# Set the test directory to the test/documentation_tests/ directory -TEST_DIR="$ROOT_DIR/tests/documentation_tests" -# Set the output directory to the test/documentation_tests/ directory -OUTPUT_DIR="$TEST_DIR" # Set the files to exclude from the copy EXCLUDE_FILES=( "sailoperator.io.md" "guidelines.md" ) +# Check if TEST_DIR is set, if not skip the script +if [[ -z "${TEST_DIR:-}" ]]; then + echo "TEST_DIR is not set. This script is not meant to be run directly. Please Check run-docs-examples.sh script." + exit 1 +fi + +echo "Using TEST_DIR directory to place the temporary docs files: $TEST_DIR" # Validate that no tag value is used in more than one file in the current documentation declare -A TAG_USAGE @@ -78,9 +81,6 @@ if [[ "$DUPLICATE_FOUND" -eq 1 ]]; then exit 1 fi -# Create the output directory if it does not exist -mkdir -p "$OUTPUT_DIR" - # Create first a list with all the files in the docs/ directory and subdirectories to be copied # Exclude the files in the EXCLUDE_FILES array and include only the files that contains at least one pattern in their code blocks like this: # bash { name= @@ -110,33 +110,26 @@ for file in "${FILES_TO_COPY[@]}"; do echo "$file" done -# Remove all the files in the output directory, but only the files in the output directory not the subdirectories -for file in "$OUTPUT_DIR"/*; do - if [ -f "$file" ]; then - echo "Removing file: $file" - rm "$file" - fi -done - - # Copy the files to the output directory and add the suffix -runme.md to each file for file in "${FILES_TO_COPY[@]}"; do # Get the base name of the file base_name=$(basename "$file") - echo "Copying file: $file to $OUTPUT_DIR/${base_name%.md}-runme.md" + echo "Copying file: $file to $TEST_DIR/${base_name%.md}-runme.md" # Copy the file to the output directory and add the suffix -runme.md to each file - cp "$file" "$OUTPUT_DIR/${base_name%.md}-runme.md" + cp "$file" "$TEST_DIR/${base_name%.md}-runme.md" done # Uncomment the html commented bash code block in the files # Read the files in the test/documentation_tests/ directory # The code block need to match this pattern: -# +# ``` +# --> # If any other pattern is found, it will be ignored. Add more patterns here if needed. -for file in "$OUTPUT_DIR"/*.md; do +for file in "$TEST_DIR"/*.md; do perl -0777 -pe 's//$1$2$3/gms' "$file" > "${file}.tmp" && mv "${file}.tmp" "$file" done diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 0b7b1a2c3..159a79e70 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -11,13 +11,13 @@ This end-to-end test suite utilizes Ginkgo, a testing framework known for its ex 1. [Adding a Test Suite](#adding-a-test-suite) 1. [Sub-Tests](#sub-tests) 1. [Best practices](#best-practices) -1. [Running Tests](#running-tests) - 1. [Pre-requisites](#Pre-requisites) - 1. [How to Run the test](#How-to-Run-the-test) - 1. [Running the test locally](#Running-the-test-locally) - 1. [Settings for end-to-end test execution](#Settings-for-end-to-end-test-execution) - 1. [Customizing the test run](#Customizing-the-test-run) - 1. [Get test definitions for the end-to-end test](#Get-test-definitions-for-the-end-to-end-test) +1. [Running Tests](#running-the-tests) + 1. [Pre-requisites](#pre-requisites) + 1. [How to Run the test](#how-to-run-the-test) + 1. [Running the test locally](#running-the-test-locally) + 1. [Settings for end-to-end test execution](#settings-for-end-to-end-test-execution) + 1. [Customizing the test run](#customizing-the-test-run) + 1. [Get test definitions for the end-to-end test](#get-test-definitions-for-the-end-to-end-test) 1. [Contributing](#contributing) ## Overview @@ -170,8 +170,30 @@ var _ = Describe("Operator", func() { * Use `Success` helper to print Success message in the test output. * Use `kubectl` and `helm` utils to make all the necessary operations in the test that are going to be done by a user. This means that the test should simulate the user behavior when using the operator. * Use `client` to interact with the Kubernetes API. This is useful when you need to interact with the Kubernetes API directly and not through `kubectl`. This needs to be used to make all the assertions if is possible. We use the client to make the assertions over `kubectl` because it is more reliable and faster, it will not need any complex parsing of the output. +* Use `cleaner` to clean up any resources your tests create after they finish. The cleaner records a "snapshot" of the cluster, and removes any resources that weren't recorded. The best way to use it is by adding call in `BeforeAll` and `AfterAll` blocks. For example: +```go +import "github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner" + +var _ = Describe("Testing with cleanup", Ordered, func() { + clr := cleaner.New(cl) + + BeforeAll(func(ctx SpecContext) { + clr.Record(ctx) + // Any additional set up goes here + }) + + // Tests go here + + AfterAll(func(ctx SpecContext) { + // Any finalizing logic goes here + clr.Cleanup(ctx) + }) +}) +``` + * You can use multiple cleaners, each with its own state. This is useful if the test does some global set up, e.g. sets up the operator, and then specific tests create further resources which you want cleaned. + * To clean resources without waiting, and waiting for them later, use `CleanupNoWait` followed by `WaitForDeletion`. This is particularly useful when working with more than one cluster. -## Running the test +## Running the tests The end-to-end test can be run in two different environments: OCP (OpenShift Container Platform) and KinD (Kubernetes in Docker). ### Pre-requisites @@ -206,7 +228,7 @@ To run a specific subset of tests, you can use the `GINKGO_FLAGS` environment va ``` $ GINKGO_FLAGS="-v --label-filter=smoke" make test.e2e.kind ``` -Note: `-v` is to add verbosity to the output. The `--label-filter` flag is used to filter the tests by label. You can use multiple labels separated by commas. Please take a look at the topic [Settings for end to end test execution](#Settings-for-end-to-end-test-execution) to see how you can set more customizations for the test run. +Note: `-v` is to add verbosity to the output. The `--label-filter` flag is used to filter the tests by label. You can use multiple labels separated by commas. Please take a look at the topic [Settings for end to end test execution](#settings-for-end-to-end-test-execution) to see how you can set more customizations for the test run. ### Running the test locally @@ -220,6 +242,12 @@ or $ make BUILD_WITH_CONTAINER=0 test.e2e.ocp ``` +Note: if you are running the test against a cluster that has a different architecture than the one you are running the test, you will need to set the `TARGET_ARCH` environment variable to the architecture of the cluster. For example, if you are running the test against an ARM64 cluster, you can use the following command: + +``` +TARGET_ARCH=arm64 make test.e2e.ocp +``` + ### Settings for end-to-end test execution The following environment variables define the behavior of the test run: @@ -234,6 +262,7 @@ The following environment variables define the behavior of the test run: * CONTROL_PLANE_NS=istio-system - The namespace where the control plane will be deployed. * DEPLOYMENT_NAME=sail-operator - The name of the operator deployment. * EXPECTED_REGISTRY=`^docker\.io|^gcr\.io` - Which image registry should the operand images come from. Useful for downstream tests. +* KEEP_ON_FAILURE - If set to true, when using a local KIND cluster, don't clean it up when the test fails. This allows to debug the failure. ### Customizing the test run @@ -252,6 +281,97 @@ Alternatively, set the following environment variables to change these file path `TCP_ECHO_*` are used in the `dual-stack` test suite, `SLEEP_YAML_PATH` and `HELLOWORLD_YAML_PATH` are used in both `Multicluster` and default test run. +### Using the e2e framework to test your cluster configuration +The e2e framework can be used to test your cluster configuration. The framework is designed to be flexible and extensible. It is easy to add new test suites and new tests. The idea is to be able to simulate what a real user scenario looks like when using the operator. + +To do this, we have a set of test cases under a test group called `smoke`. This test group will run a set of tests that are designed to test the basic functionality of the operator. The tests in this group are designed to be run against a cluster that is already configured and running. The tests will not modify the cluster configuration or the operator configuration. The tests will only check if the operator is working as expected. + +Pre-requisites: +* The operator is already installed and running in the cluster. + +To run the test group, you can use the following command: + +* Run the following command to run the smoke tests: +For running on kind: +``` +$ SKIP_BUILD=true SKIP_DEPLOY=true GINKGO_FLAGS="-v --label-filter=smoke" make test.e2e.kind +``` + +For running on OCP: +``` +$ SKIP_BUILD=true SKIP_DEPLOY=true GINKGO_FLAGS="-v --label-filter=smoke" make test.e2e.ocp +``` + +#### Running with specific configuration for the Istio and IstioCNI resource +There might be situations where you want to run tests with a specific configuration for the Istio and IstioCNI resource to match some cluster specific needs. For this, you can modify the `pkg/istiovalues/vendor_default.yaml` file to default `spec.values` for the Istio and IstioCNI resources. For more information and an example go to the [file](../../pkg/istiovalues/vendor_defaults.yaml) + +#### Running the testing framework against specific Istio versions +By default, the test framework will run the tests against all the latest patch minor version available for the operator. This is useful when you want to test the operator against all the latest patch minor versions available. The test framework will automatically detect the latest patch minor version available for the operator and run the tests against it by reading the versions located in the `pkg/istioversion/versions.yaml` file. + +To avoid this and run the tests against a specific Istio versions, you can create your own `versions.yaml` file and set the `VERSIONS_YAML_FILE` environment variable to point to it. The file should have the following structure: +```yaml +versions: + - name: v1.26-latest + ref: v1.26.0 + - name: v1.26.0 + version: 1.26.0 + repo: https://github.com/istio/istio + commit: 1.26.0 + charts: + - https://istio-release.storage.googleapis.com/charts/base-1.26.0.tgz + - https://istio-release.storage.googleapis.com/charts/istiod-1.26.0.tgz + - https://istio-release.storage.googleapis.com/charts/gateway-1.26.0.tgz + - https://istio-release.storage.googleapis.com/charts/cni-1.26.0.tgz + - https://istio-release.storage.googleapis.com/charts/ztunnel-1.26.0.tgz + - name: v1.25-latest + ref: v1.25.3 + - name: v1.25.3 + version: 1.25.3 + repo: https://github.com/istio/istio + commit: 1.25.3 + charts: + - https://istio-release.storage.googleapis.com/charts/base-1.25.3.tgz + - https://istio-release.storage.googleapis.com/charts/istiod-1.25.3.tgz + - https://istio-release.storage.googleapis.com/charts/gateway-1.25.3.tgz + - https://istio-release.storage.googleapis.com/charts/cni-1.25.3.tgz + - https://istio-release.storage.googleapis.com/charts/ztunnel-1.25.3.tgz +``` +*Important*: avoid adding in the custom file versions that are not available in the `pkg/istioversion/versions.yaml` file. The test framework will not be able to run the tests because the operator does not contains the charts for those versions. + +* To run the test framework against a specific Istio version, you can use the following command: +``` +$ VERSIONS_YAML_FILE=custom_versions.yaml SKIP_BUILD=true SKIP_DEPLOY=true GINKGO_FLAGS="-v --label-filter=smoke" make test.e2e.kind +``` +Note: The `custom_versions.yaml` file must be placed in the `pkg/istioversion` directory. The test framework uses this file to run tests against the specific Istio versions it defines. + +### Understanding the test output +By default, running the test using the make target will generate a report.xml file in the project's root directory. This file contains the test results in JUnit format, for example: +```xml + + + + +``` +As you can see, the test results are grouped by test suite. The `tests` attribute indicates the number of tests that were run, the `disabled` attribute indicates the number of tests that were skipped, and the `errors` and `failures` attributes indicate the number of tests that failed or had errors. The `time` attribute indicates the total time taken to run the tests. + +Also, in the terminal you will be able to see the test results in a human readable format. The test results will be printed to the console with the following format: +``` +Ran 82 of 82 Specs in 224.026 seconds +SUCCESS! -- 82 Passed | 0 Failed | 0 Pending | 0 Skipped +PASS + +Ginkgo ran 1 suite in 3m46.401610849s +Test Suite Passed +``` + +In case of failure, the test results will be printed to the console with the following format: +``` +Ran 82 of 82 Specs in 224.026 seconds +FAIL! -- 81 Passed | 1 Failed | 0 Pending | 0 Skipped +Ginkgo ran 1 suite in 3m46.401610849s +Test Suite Failed +``` + ### Get test definitions for the end-to-end test The end-to-end test suite is defined in the `tests/e2e/operator` directory. If you want to check the test definition without running the test, you can use the following make target: @@ -289,4 +409,4 @@ This can be used to show the actual coverage of the test suite. ## Contributing -Please refer to the [CONTRIBUTING.md](../../CONTRIBUTING.md) file for information about how to get involved. We welcome issues, questions, and pull requests. \ No newline at end of file +Please refer to the [CONTRIBUTING.md](../../CONTRIBUTING.md) file for information about how to get involved. We welcome issues, questions, and pull requests. diff --git a/tests/e2e/ambient/ambient_suite_test.go b/tests/e2e/ambient/ambient_suite_test.go index 2ecac88ee..fdc5b69f1 100644 --- a/tests/e2e/ambient/ambient_suite_test.go +++ b/tests/e2e/ambient/ambient_suite_test.go @@ -41,6 +41,7 @@ var ( skipDeploy = env.GetBool("SKIP_DEPLOY", false) expectedRegistry = env.Get("EXPECTED_REGISTRY", "^docker\\.io|^gcr\\.io") multicluster = env.GetBool("MULTICLUSTER", false) + keepOnFailure = env.GetBool("KEEP_ON_FAILURE", false) k kubectl.Kubectl ) diff --git a/tests/e2e/ambient/ambient_test.go b/tests/e2e/ambient/ambient_test.go index da4aa1f8b..3365d72a5 100644 --- a/tests/e2e/ambient/ambient_test.go +++ b/tests/e2e/ambient/ambient_test.go @@ -25,6 +25,7 @@ import ( "github.com/istio-ecosystem/sail-operator/pkg/istioversion" "github.com/istio-ecosystem/sail-operator/pkg/kube" . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" . "github.com/onsi/ginkgo/v2" @@ -44,8 +45,10 @@ var _ = Describe("Ambient configuration ", Label("smoke", "ambient"), Ordered, f SetDefaultEventuallyPollingInterval(time.Second) debugInfoLogged := false + clr := cleaner.New(cl) BeforeAll(func(ctx SpecContext) { + clr.Record(ctx) Expect(k.CreateNamespace(operatorNamespace)).To(Succeed(), "Namespace failed to be created") if skipDeploy { @@ -68,7 +71,9 @@ var _ = Describe("Ambient configuration ", Label("smoke", "ambient"), Ordered, f } Context(fmt.Sprintf("Istio version %s", version.Version), func() { - BeforeAll(func() { + clr := cleaner.New(cl) + BeforeAll(func(ctx SpecContext) { + clr.Record(ctx) Expect(k.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Istio namespace failed to be created") Expect(k.CreateNamespace(istioCniNamespace)).To(Succeed(), "IstioCNI namespace failed to be created") Expect(k.CreateNamespace(ztunnelNamespace)).To(Succeed(), "ZTunnel namespace failed to be created") @@ -275,13 +280,6 @@ spec: It("can access the httpbin service from the sleep pod", func(ctx SpecContext) { checkPodConnectivity(sleepPod.Items[0].Name, common.SleepNamespace, common.HttpbinNamespace) }) - - AfterAll(func(ctx SpecContext) { - By("Deleting the pods") - Expect(k.DeleteNamespace(common.HttpbinNamespace, common.SleepNamespace)). - To(Succeed(), "Failed to delete namespaces") - Success("Ambient validation pods deleted") - }) }) When("the Istio CR is deleted", func() { @@ -327,6 +325,14 @@ spec: Success("ztunnel namespace is empty") }) }) + + AfterAll(func(ctx SpecContext) { + if CurrentSpecReport().Failed() && keepOnFailure { + return + } + + clr.Cleanup(ctx) + }) }) } @@ -335,36 +341,22 @@ spec: common.LogDebugInfo(common.Ambient, k) debugInfoLogged = true } - - By("Cleaning up the Istio namespace") - Expect(k.DeleteNamespace(controlPlaneNamespace)).To(Succeed(), "Istio Namespace failed to be deleted") - - By("Cleaning up the IstioCNI namespace") - Expect(k.DeleteNamespace(istioCniNamespace)).To(Succeed(), "IstioCNI Namespace failed to be deleted") - - By("Cleaning up the ZTunnel namespace") - Expect(k.DeleteNamespace(ztunnelNamespace)).To(Succeed(), "ZTunnel Namespace failed to be deleted") }) }) - AfterAll(func() { - if CurrentSpecReport().Failed() && !debugInfoLogged { - common.LogDebugInfo(common.Ambient, k) - debugInfoLogged = true - } + AfterAll(func(ctx SpecContext) { + if CurrentSpecReport().Failed() { + if !debugInfoLogged { + common.LogDebugInfo(common.Ambient, k) + debugInfoLogged = true + } - if skipDeploy { - Success("Skipping operator undeploy because it was deployed externally") - return + if keepOnFailure { + return + } } - By("Deleting operator deployment") - Expect(common.UninstallOperator()). - To(Succeed(), "Operator failed to be deleted") - GinkgoWriter.Println("Operator uninstalled") - - Expect(k.DeleteNamespace(operatorNamespace)).To(Succeed(), "Namespace failed to be deleted") - Success("Namespace deleted") + clr.Cleanup(ctx) }) }) diff --git a/tests/e2e/common-operator-integ-suite.sh b/tests/e2e/common-operator-integ-suite.sh index 784ebd04c..501633243 100755 --- a/tests/e2e/common-operator-integ-suite.sh +++ b/tests/e2e/common-operator-integ-suite.sh @@ -146,6 +146,22 @@ export COMMAND OCP HUB IMAGE_BASE TAG NAMESPACE if [ "${SKIP_BUILD}" == "false" ]; then "${WD}/setup/build-and-push-operator.sh" + if [ "${OCP}" = "true" ]; then + # This is a workaround when pulling the image from internal registry + # To avoid errors of certificates meanwhile we are pulling the operator image from the internal registry + # We need to set image $HUB to a fixed known value after the push + # This value always will be equal to the svc url of the internal registry + HUB="image-registry.openshift-image-registry.svc:5000/istio-images" + echo "Using internal registry: ${HUB}" + + # Workaround for OCP helm operator installation issues: + # To avoid any cleanup issues, after we build and push the image we check if the namespace exists and delete it if it does. + # The test logic already handles the namespace creation and deletion during the test run. + if ${COMMAND} get ns "${NAMESPACE}" &>/dev/null; then + echo "Namespace ${NAMESPACE} already exists. Deleting it to avoid conflicts." + ${COMMAND} delete ns "${NAMESPACE}" + fi + fi # If OLM is enabled, deploy the operator using OLM # We are skipping the deploy via OLM test on OCP because the workaround to avoid the certificate issue is not working. # Jira ticket related to the limitation: https://issues.redhat.com/browse/OSSM-7993 @@ -182,14 +198,6 @@ if [ "${SKIP_BUILD}" == "false" ]; then fi fi -if [ "${OCP}" == "true" ]; then - # This is a workaround - # To avoid errors of certificates meanwhile we are pulling the operator image from the internal registry - # We need to set image $HUB to a fixed known value after the push - # This value always will be equal to the svc url of the internal registry - HUB="image-registry.openshift-image-registry.svc:5000/sail-operator" -fi - export SKIP_DEPLOY IP_FAMILY ISTIO_MANIFEST NAMESPACE CONTROL_PLANE_NS DEPLOYMENT_NAME MULTICLUSTER ARTIFACTS ISTIO_NAME COMMAND KUBECONFIG ISTIOCTL_PATH # shellcheck disable=SC2086 diff --git a/tests/e2e/controlplane/control_plane_suite_test.go b/tests/e2e/controlplane/control_plane_suite_test.go index 7bef72947..fefc6fe1b 100644 --- a/tests/e2e/controlplane/control_plane_suite_test.go +++ b/tests/e2e/controlplane/control_plane_suite_test.go @@ -22,6 +22,7 @@ import ( "github.com/istio-ecosystem/sail-operator/pkg/env" "github.com/istio-ecosystem/sail-operator/pkg/kube" . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner" k8sclient "github.com/istio-ecosystem/sail-operator/tests/e2e/util/client" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" @@ -46,9 +47,11 @@ var ( expectedRegistry = env.Get("EXPECTED_REGISTRY", "^docker\\.io|^gcr\\.io") sampleNamespace = env.Get("SAMPLE_NAMESPACE", "sample") multicluster = env.GetBool("MULTICLUSTER", false) + keepOnFailure = env.GetBool("KEEP_ON_FAILURE", false) ipFamily = env.Get("IP_FAMILY", "ipv4") - k kubectl.Kubectl + k kubectl.Kubectl + clr cleaner.Cleaner ) func TestControlPlane(t *testing.T) { @@ -68,9 +71,11 @@ func setup() { Expect(err).NotTo(HaveOccurred()) k = kubectl.New() + clr = cleaner.New(cl) } var _ = BeforeSuite(func(ctx SpecContext) { + clr.Record(ctx) Expect(k.CreateNamespace(namespace)).To(Succeed(), "Namespace failed to be created") if skipDeploy { @@ -85,17 +90,10 @@ var _ = BeforeSuite(func(ctx SpecContext) { Success("Operator is deployed in the namespace and Running") }) -var _ = AfterSuite(func(ctx SpecContext) { - if skipDeploy { - Success("Skipping operator undeploy because it was deployed externally") +var _ = ReportAfterSuite("Condiotnal cleanup", func(ctx SpecContext, r Report) { + if !r.SuiteSucceeded && keepOnFailure { return } - By("Deleting operator deployment") - Expect(common.UninstallOperator()). - To(Succeed(), "Operator failed to be deleted") - GinkgoWriter.Println("Operator uninstalled") - - Expect(k.DeleteNamespace(namespace)).To(Succeed(), "Namespace failed to be deleted") - Success("Namespace deleted") + clr.Cleanup(ctx) }) diff --git a/tests/e2e/controlplane/control_plane_test.go b/tests/e2e/controlplane/control_plane_test.go index ac69b90dd..96e158ec5 100644 --- a/tests/e2e/controlplane/control_plane_test.go +++ b/tests/e2e/controlplane/control_plane_test.go @@ -26,6 +26,7 @@ import ( "github.com/istio-ecosystem/sail-operator/pkg/istioversion" "github.com/istio-ecosystem/sail-operator/pkg/kube" . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" . "github.com/onsi/ginkgo/v2" @@ -98,7 +99,9 @@ metadata: Describe("given Istio version", func() { for _, version := range istioversion.GetLatestPatchVersions() { Context(version.Name, func() { - BeforeAll(func() { + clr := cleaner.New(cl) + BeforeAll(func(ctx SpecContext) { + clr.Record(ctx) Expect(k.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Istio namespace failed to be created") Expect(k.CreateNamespace(istioCniNamespace)).To(Succeed(), "IstioCNI namespace failed to be created") }) @@ -202,7 +205,7 @@ spec: }) When("sample pod is deployed", func() { - BeforeAll(func() { + BeforeAll(func(ctx SpecContext) { Expect(k.CreateNamespace(sampleNamespace)).To(Succeed(), "Sample namespace failed to be created") Expect(k.Label("namespace", sampleNamespace, "istio-injection", "enabled")).To(Succeed(), "Error labeling sample namespace") Expect(k.WithNamespace(sampleNamespace). @@ -238,12 +241,6 @@ spec: } Success("Istio sidecar version matches the expected Istio version") }) - - AfterAll(func(ctx SpecContext) { - By("Deleting sample") - Expect(k.DeleteNamespace(sampleNamespace)).To(Succeed(), "sample namespace failed to be deleted") - Success("sample deleted") - }) }) When("the Istio CR is deleted", func() { @@ -274,6 +271,14 @@ spec: Success("CNI namespace is empty") }) }) + + AfterAll(func(ctx SpecContext) { + if CurrentSpecReport().Failed() && keepOnFailure { + return + } + + clr.Cleanup(ctx) + }) }) } @@ -282,21 +287,15 @@ spec: common.LogDebugInfo(common.ControlPlane, k) debugInfoLogged = true } - - By("Cleaning up the Istio namespace") - Expect(k.DeleteNamespace(controlPlaneNamespace)).To(Succeed(), "Istio Namespace failed to be deleted") - - By("Cleaning up the IstioCNI namespace") - Expect(k.DeleteNamespace(istioCniNamespace)).To(Succeed(), "IstioCNI Namespace failed to be deleted") - - Success("Cleanup done") }) }) AfterAll(func() { - if CurrentSpecReport().Failed() && !debugInfoLogged { - common.LogDebugInfo(common.ControlPlane, k) - debugInfoLogged = true + if CurrentSpecReport().Failed() { + if !debugInfoLogged { + common.LogDebugInfo(common.ControlPlane, k) + debugInfoLogged = true + } } }) }) diff --git a/tests/e2e/controlplane/control_plane_update_test.go b/tests/e2e/controlplane/control_plane_update_test.go index 7dc42f709..3d09b9c87 100644 --- a/tests/e2e/controlplane/control_plane_update_test.go +++ b/tests/e2e/controlplane/control_plane_update_test.go @@ -26,6 +26,7 @@ import ( "github.com/istio-ecosystem/sail-operator/pkg/istioversion" "github.com/istio-ecosystem/sail-operator/pkg/kube" . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" . "github.com/onsi/ginkgo/v2" @@ -46,11 +47,14 @@ var _ = Describe("Control Plane updates", Label("control-plane", "slow"), Ordere } Context(istioversion.Base, func() { + clr := cleaner.New(cl) + BeforeAll(func(ctx SpecContext) { if len(istioversion.List) < 2 { Skip("Skipping update tests because there are not enough versions in versions.yaml") } + clr.Record(ctx) Expect(k.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Istio namespace failed to be created") Expect(k.CreateNamespace(istioCniNamespace)).To(Succeed(), "IstioCNI namespace failed to be created") @@ -304,29 +308,25 @@ spec: if CurrentSpecReport().Failed() { common.LogDebugInfo(common.ControlPlane, k) debugInfoLogged = true + if keepOnFailure { + return + } } - By("Cleaning up sample namespace") - Expect(k.DeleteNamespace(sampleNamespace)).To(Succeed(), "Sample Namespace failed to be deleted") - - By("Cleaning up the Istio namespace") - Expect(k.Delete("istio", istioName)).To(Succeed(), "Istio CR failed to be deleted") - Expect(k.DeleteNamespace(controlPlaneNamespace)).To(Succeed(), "Istio Namespace failed to be deleted") - - By("Cleaning up the IstioCNI namespace") - Expect(k.Delete("istiocni", istioCniName)).To(Succeed(), "IstioCNI CR failed to be deleted") - Expect(k.DeleteNamespace(istioCniNamespace)).To(Succeed(), "IstioCNI Namespace failed to be deleted") - - By("Deleting the IstioRevisionTag") - Expect(k.Delete("istiorevisiontag", "default")).To(Succeed(), "IstioRevisionTag failed to be deleted") - Success("Cleanup done") + clr.Cleanup(ctx) }) }) AfterAll(func() { - if CurrentSpecReport().Failed() && !debugInfoLogged { - common.LogDebugInfo(common.ControlPlane, k) - debugInfoLogged = true + if CurrentSpecReport().Failed() { + if !debugInfoLogged { + common.LogDebugInfo(common.ControlPlane, k) + debugInfoLogged = true + + if keepOnFailure { + return + } + } } }) }) diff --git a/tests/e2e/dualstack/dualstack_suite_test.go b/tests/e2e/dualstack/dualstack_suite_test.go index b7b245879..23ce5219e 100644 --- a/tests/e2e/dualstack/dualstack_suite_test.go +++ b/tests/e2e/dualstack/dualstack_suite_test.go @@ -40,6 +40,7 @@ var ( skipDeploy = env.GetBool("SKIP_DEPLOY", false) expectedRegistry = env.Get("EXPECTED_REGISTRY", "^docker\\.io|^gcr\\.io") multicluster = env.GetBool("MULTICLUSTER", false) + keepOnFailure = env.GetBool("KEEP_ON_FAILURE", false) ipFamily = env.Get("IP_FAMILY", "ipv4") k kubectl.Kubectl diff --git a/tests/e2e/dualstack/dualstack_test.go b/tests/e2e/dualstack/dualstack_test.go index 3ec9283f8..1b2124d8b 100644 --- a/tests/e2e/dualstack/dualstack_test.go +++ b/tests/e2e/dualstack/dualstack_test.go @@ -25,6 +25,7 @@ import ( "github.com/istio-ecosystem/sail-operator/pkg/istioversion" "github.com/istio-ecosystem/sail-operator/pkg/kube" . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" . "github.com/onsi/ginkgo/v2" @@ -47,8 +48,10 @@ var _ = Describe("DualStack configuration ", Label("dualstack"), Ordered, func() SetDefaultEventuallyPollingInterval(time.Second) debugInfoLogged := false + clr := cleaner.New(cl) BeforeAll(func(ctx SpecContext) { + clr.Record(ctx) Expect(k.CreateNamespace(namespace)).To(Succeed(), "Namespace failed to be created") if skipDeploy { @@ -71,7 +74,9 @@ var _ = Describe("DualStack configuration ", Label("dualstack"), Ordered, func() } Context(fmt.Sprintf("Istio version %s", version.Version), func() { - BeforeAll(func() { + clr := cleaner.New(cl) + BeforeAll(func(ctx SpecContext) { + clr.Record(ctx) Expect(k.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Istio namespace failed to be created") Expect(k.CreateNamespace(istioCniNamespace)).To(Succeed(), "IstioCNI namespace failed to be created") }) @@ -228,13 +233,6 @@ spec: It("can access the ipv6 only service from the sleep pod", func(ctx SpecContext) { checkPodConnectivity(sleepPod.Items[0].Name, SleepNamespace, IPv6Namespace) }) - - AfterAll(func(ctx SpecContext) { - By("Deleting the pods") - Expect(k.DeleteNamespace(DualStackNamespace, IPv4Namespace, IPv6Namespace, SleepNamespace)). - To(Succeed(), "Failed to delete namespaces") - Success("DualStack validation pods deleted") - }) }) When("the Istio CR is deleted", func() { @@ -265,6 +263,14 @@ spec: Success("CNI namespace is empty") }) }) + + AfterAll(func(ctx SpecContext) { + if CurrentSpecReport().Failed() && keepOnFailure { + return + } + + clr.Cleanup(ctx) + }) }) } @@ -273,35 +279,22 @@ spec: common.LogDebugInfo(common.DualStack, k) debugInfoLogged = true } - - By("Cleaning up the Istio namespace") - Expect(cl.Delete(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: controlPlaneNamespace}})).To(Succeed(), "Istio Namespace failed to be deleted") - - By("Cleaning up the IstioCNI namespace") - Expect(k.DeleteNamespace(istioCniNamespace)).To(Succeed(), "IstioCNI Namespace failed to be deleted") - - Success("Cleanup done") }) }) - AfterAll(func() { - if CurrentSpecReport().Failed() && !debugInfoLogged { - common.LogDebugInfo(common.DualStack, k) - debugInfoLogged = true - } + AfterAll(func(ctx SpecContext) { + if CurrentSpecReport().Failed() { + if !debugInfoLogged { + common.LogDebugInfo(common.DualStack, k) + debugInfoLogged = true + } - if skipDeploy { - Success("Skipping operator undeploy because it was deployed externally") - return + if keepOnFailure { + return + } } - By("Deleting operator deployment") - Expect(common.UninstallOperator()). - To(Succeed(), "Operator failed to be deleted") - GinkgoWriter.Println("Operator uninstalled") - - Expect(k.DeleteNamespace(namespace)).To(Succeed(), "Namespace failed to be deleted") - Success("Namespace deleted") + clr.Cleanup(ctx) }) }) diff --git a/tests/e2e/multicluster/multicluster_externalcontrolplane_test.go b/tests/e2e/multicluster/multicluster_externalcontrolplane_test.go index 65389ab59..7f1e463d7 100644 --- a/tests/e2e/multicluster/multicluster_externalcontrolplane_test.go +++ b/tests/e2e/multicluster/multicluster_externalcontrolplane_test.go @@ -18,18 +18,16 @@ package multicluster import ( "fmt" - "path/filepath" "time" v1 "github.com/istio-ecosystem/sail-operator/api/v1" "github.com/istio-ecosystem/sail-operator/pkg/istioversion" "github.com/istio-ecosystem/sail-operator/pkg/kube" - "github.com/istio-ecosystem/sail-operator/pkg/test/project" . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" "github.com/istio-ecosystem/sail-operator/pkg/version" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" - "github.com/istio-ecosystem/sail-operator/tests/e2e/util/helm" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/istioctl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -48,30 +46,6 @@ const ( var _ = Describe("Multicluster deployment models", Label("multicluster", "multicluster-external"), Ordered, func() { SetDefaultEventuallyTimeout(180 * time.Second) SetDefaultEventuallyPollingInterval(time.Second) - debugInfoLogged := false - - BeforeAll(func(ctx SpecContext) { - if !skipDeploy { - Expect(k1.CreateNamespace(namespace)).To(Succeed(), "Namespace failed to be created on Cluster #1") - Expect(k2.CreateNamespace(namespace)).To(Succeed(), "Namespace failed to be created on Cluster #2") - - Expect(helm.Install("sail-operator", filepath.Join(project.RootDir, "chart"), "--namespace "+namespace, "--set=image="+image, "--kubeconfig "+kubeconfig)). - To(Succeed(), "Operator failed to be deployed in Cluster #1") - - Expect(helm.Install("sail-operator", filepath.Join(project.RootDir, "chart"), "--namespace "+namespace, "--set=image="+image, "--kubeconfig "+kubeconfig2)). - To(Succeed(), "Operator failed to be deployed in Cluster #2") - - Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key(deploymentName, namespace), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Error getting Istio CRD") - Success("Operator is deployed in the Cluster #1 namespace and Running") - - Eventually(common.GetObject). - WithArguments(ctx, clRemote, kube.Key(deploymentName, namespace), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Error getting Istio CRD") - Success("Operator is deployed in the Cluster #2 namespace and Running") - } - }) Describe("External Control Plane Multi-Network configuration", func() { // Test the External Control Plane Multi-Network configuration for each supported Istio version @@ -83,6 +57,14 @@ var _ = Describe("Multicluster deployment models", Label("multicluster", "multic } Context(fmt.Sprintf("Istio version %s", v.Version), func() { + clr1 := cleaner.New(clPrimary, "cluster=primary") + clr2 := cleaner.New(clRemote, "cluster=remote") + + BeforeAll(func(ctx SpecContext) { + clr1.Record(ctx) + clr2.Record(ctx) + }) + When("default Istio is created in Cluster #1 to handle ingress to External Control Plane", func() { BeforeAll(func(ctx SpecContext) { Expect(k1.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Namespace failed to be created") @@ -242,16 +224,20 @@ metadata: Expect(apiURLCluster2).NotTo(BeEmpty(), "API URL is empty for the Cluster #2") Expect(err).NotTo(HaveOccurred()) - secret, err := istioctl.CreateRemoteSecret( - kubeconfig2, - externalControlPlaneNamespace, - "cluster2", - apiURLCluster2, - "--type=config", - "--service-account=istiod-"+externalIstioName, - "--create-service-account=false", - ) - Expect(err).NotTo(HaveOccurred()) + var secret string + Eventually(func() error { + secret, err = istioctl.CreateRemoteSecret( + kubeconfig2, + externalControlPlaneNamespace, + "cluster2", + apiURLCluster2, + "--type=config", + "--service-account=istiod-"+externalIstioName, + "--create-service-account=false", + ) + + return err + }).ShouldNot(HaveOccurred(), "Remote secret generation failed") Expect(k1.ApplyString(secret)).To(Succeed(), "Remote secret creation failed on Cluster #1") }) @@ -471,40 +457,17 @@ spec: if CurrentSpecReport().Failed() { common.LogDebugInfo(common.MultiCluster, k1, k2) debugInfoLogged = true + if keepOnFailure { + return + } } - // Delete namespaces to ensure clean up for new tests iteration - Expect(k1.DeleteNamespaceNoWait(controlPlaneNamespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #1") - Expect(k1.DeleteNamespaceNoWait(externalControlPlaneNamespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #1") - Expect(k1.DeleteNamespaceNoWait(istioCniNamespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #1") - Expect(k2.DeleteNamespaceNoWait(externalControlPlaneNamespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #2") - Expect(k2.DeleteNamespaceNoWait(sampleNamespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #2") - Expect(k2.DeleteNamespaceNoWait(istioCniNamespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #2") - - Expect(k1.WaitNamespaceDeleted(controlPlaneNamespace)).To(Succeed()) - Expect(k1.WaitNamespaceDeleted(externalControlPlaneNamespace)).To(Succeed()) - Expect(k2.WaitNamespaceDeleted(externalControlPlaneNamespace)).To(Succeed()) - Expect(k1.WaitNamespaceDeleted(istioCniNamespace)).To(Succeed()) - Expect(k2.WaitNamespaceDeleted(istioCniNamespace)).To(Succeed()) - Success("ControlPlane Namespaces are empty") - - Expect(k2.WaitNamespaceDeleted(sampleNamespace)).To(Succeed()) - Success("Sample app is deleted in Cluster #2") + c1Deleted := clr1.CleanupNoWait(ctx) + c2Deleted := clr2.CleanupNoWait(ctx) + clr1.WaitForDeletion(ctx, c1Deleted) + clr2.WaitForDeletion(ctx, c2Deleted) }) }) } }) - - AfterAll(func(ctx SpecContext) { - if CurrentSpecReport().Failed() && !debugInfoLogged { - common.LogDebugInfo(common.MultiCluster, k1, k2) - debugInfoLogged = true - } - - // Delete the Sail Operator from both clusters - Expect(k1.DeleteNamespaceNoWait(namespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #1") - Expect(k2.DeleteNamespaceNoWait(namespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #2") - Expect(k1.WaitNamespaceDeleted(namespace)).To(Succeed()) - Expect(k2.WaitNamespaceDeleted(namespace)).To(Succeed()) - }) }) diff --git a/tests/e2e/multicluster/multicluster_multiprimary_test.go b/tests/e2e/multicluster/multicluster_multiprimary_test.go index 0d6617515..ff00ac3ff 100644 --- a/tests/e2e/multicluster/multicluster_multiprimary_test.go +++ b/tests/e2e/multicluster/multicluster_multiprimary_test.go @@ -19,18 +19,16 @@ package multicluster import ( "context" "fmt" - "path/filepath" "time" v1 "github.com/istio-ecosystem/sail-operator/api/v1" "github.com/istio-ecosystem/sail-operator/pkg/istioversion" "github.com/istio-ecosystem/sail-operator/pkg/kube" - "github.com/istio-ecosystem/sail-operator/pkg/test/project" . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/certs" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" - "github.com/istio-ecosystem/sail-operator/tests/e2e/util/helm" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/istioctl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -43,36 +41,19 @@ import ( var _ = Describe("Multicluster deployment models", Label("multicluster", "multicluster-multiprimary"), Ordered, func() { SetDefaultEventuallyTimeout(180 * time.Second) SetDefaultEventuallyPollingInterval(time.Second) - debugInfoLogged := false - - BeforeAll(func(ctx SpecContext) { - if !skipDeploy { - // Deploy the Sail Operator on both clusters - Expect(k1.CreateNamespace(namespace)).To(Succeed(), "Namespace failed to be created on Cluster #1") - Expect(k2.CreateNamespace(namespace)).To(Succeed(), "Namespace failed to be created on Cluster #2") - - Expect(helm.Install("sail-operator", filepath.Join(project.RootDir, "chart"), "--namespace "+namespace, "--set=image="+image, "--kubeconfig "+kubeconfig)). - To(Succeed(), "Operator failed to be deployed in Cluster #1") - - Expect(helm.Install("sail-operator", filepath.Join(project.RootDir, "chart"), "--namespace "+namespace, "--set=image="+image, "--kubeconfig "+kubeconfig2)). - To(Succeed(), "Operator failed to be deployed in Cluster #2") - - Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key(deploymentName, namespace), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Error getting Istio CRD") - Success("Operator is deployed in the Cluster #1 namespace and Running") - - Eventually(common.GetObject). - WithArguments(ctx, clRemote, kube.Key(deploymentName, namespace), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Error getting Istio CRD") - Success("Operator is deployed in the Cluster #2 namespace and Running") - } - }) Describe("Multi-Primary Multi-Network configuration", func() { // Test the Multi-Primary Multi-Network configuration for each supported Istio version for _, version := range istioversion.GetLatestPatchVersions() { Context(fmt.Sprintf("Istio version %s", version.Version), func() { + clr1 := cleaner.New(clPrimary, "cluster=primary") + clr2 := cleaner.New(clRemote, "cluster=remote") + + BeforeAll(func(ctx SpecContext) { + clr1.Record(ctx) + clr2.Record(ctx) + }) + When("Istio and IstioCNI resources are created in both clusters", func() { BeforeAll(func(ctx SpecContext) { Expect(k1.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Istio namespace failed to be created") @@ -337,40 +318,17 @@ spec: if CurrentSpecReport().Failed() { common.LogDebugInfo(common.MultiCluster, k1, k2) debugInfoLogged = true + if keepOnFailure { + return + } } - // Delete namespaces to ensure clean up for new tests iteration - Expect(k1.DeleteNamespaceNoWait(istioCniNamespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #1") - Expect(k2.DeleteNamespaceNoWait(istioCniNamespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #2") - Expect(k1.DeleteNamespaceNoWait(controlPlaneNamespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #1") - Expect(k2.DeleteNamespaceNoWait(controlPlaneNamespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #2") - Expect(k1.DeleteNamespaceNoWait(sampleNamespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #1") - Expect(k2.DeleteNamespaceNoWait(sampleNamespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #2") - - Expect(k1.WaitNamespaceDeleted(istioCniNamespace)).To(Succeed()) - Expect(k2.WaitNamespaceDeleted(istioCniNamespace)).To(Succeed()) - Expect(k1.WaitNamespaceDeleted(controlPlaneNamespace)).To(Succeed()) - Expect(k2.WaitNamespaceDeleted(controlPlaneNamespace)).To(Succeed()) - Success("ControlPlane and CNI Namespaces are empty") - - Expect(k1.WaitNamespaceDeleted(sampleNamespace)).To(Succeed()) - Expect(k2.WaitNamespaceDeleted(sampleNamespace)).To(Succeed()) - Success("Sample app is deleted in both clusters") + c1Deleted := clr1.CleanupNoWait(ctx) + c2Deleted := clr2.CleanupNoWait(ctx) + clr1.WaitForDeletion(ctx, c1Deleted) + clr2.WaitForDeletion(ctx, c2Deleted) }) }) } }) - - AfterAll(func(ctx SpecContext) { - if CurrentSpecReport().Failed() && !debugInfoLogged { - common.LogDebugInfo(common.MultiCluster, k1, k2) - debugInfoLogged = true - } - - // Delete the Sail Operator from both clusters - Expect(k1.DeleteNamespaceNoWait(namespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #1") - Expect(k2.DeleteNamespaceNoWait(namespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #2") - Expect(k1.WaitNamespaceDeleted(namespace)).To(Succeed()) - Expect(k2.WaitNamespaceDeleted(namespace)).To(Succeed()) - }) }) diff --git a/tests/e2e/multicluster/multicluster_primaryremote_test.go b/tests/e2e/multicluster/multicluster_primaryremote_test.go index 9fc0722d0..773e332ea 100644 --- a/tests/e2e/multicluster/multicluster_primaryremote_test.go +++ b/tests/e2e/multicluster/multicluster_primaryremote_test.go @@ -27,6 +27,7 @@ import ( . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" "github.com/istio-ecosystem/sail-operator/pkg/version" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/certs" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/istioctl" @@ -41,31 +42,6 @@ import ( var _ = Describe("Multicluster deployment models", Label("multicluster", "multicluster-primaryremote"), Ordered, func() { SetDefaultEventuallyTimeout(180 * time.Second) SetDefaultEventuallyPollingInterval(time.Second) - debugInfoLogged := false - - BeforeAll(func(ctx SpecContext) { - if !skipDeploy { - // Deploy the Sail Operator on both clusters - Expect(k1.CreateNamespace(namespace)).To(Succeed(), "Namespace failed to be created on Primary Cluster") - Expect(k2.CreateNamespace(namespace)).To(Succeed(), "Namespace failed to be created on Remote Cluster") - - Expect(common.InstallOperatorViaHelm("--kubeconfig", kubeconfig)). - To(Succeed(), "Operator failed to be deployed in Primary Cluster") - - Expect(common.InstallOperatorViaHelm("--kubeconfig", kubeconfig2)). - To(Succeed(), "Operator failed to be deployed in Remote Cluster") - - Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key(deploymentName, namespace), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Error getting Istio CRD") - Success("Operator is deployed in the Primary namespace and Running") - - Eventually(common.GetObject). - WithArguments(ctx, clRemote, kube.Key(deploymentName, namespace), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Error getting Istio CRD") - Success("Operator is deployed in the Remote namespace and Running") - } - }) Describe("Primary-Remote - Multi-Network configuration", func() { // Test the Primary-Remote - Multi-Network configuration for each supported Istio version @@ -77,6 +53,14 @@ var _ = Describe("Multicluster deployment models", Label("multicluster", "multic } Context(fmt.Sprintf("Istio version %s", v.Version), func() { + clr1 := cleaner.New(clPrimary, "cluster=primary") + clr2 := cleaner.New(clRemote, "cluster=remote") + + BeforeAll(func(ctx SpecContext) { + clr1.Record(ctx) + clr2.Record(ctx) + }) + When("Istio and IstioCNI resources are created in both clusters", func() { BeforeAll(func(ctx SpecContext) { Expect(k1.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Istio namespace failed to be created") @@ -369,46 +353,17 @@ spec: if CurrentSpecReport().Failed() { common.LogDebugInfo(common.MultiCluster, k1, k2) debugInfoLogged = true + if keepOnFailure { + return + } } - // Delete namespaces to ensure clean up for new tests iteration - Expect(k1.DeleteNamespaceNoWait(istioCniNamespace)).To(Succeed(), "Namespace failed to be deleted on Primary Cluster") - Expect(k2.DeleteNamespaceNoWait(istioCniNamespace)).To(Succeed(), "Namespace failed to be deleted on Remote Cluster") - Expect(k1.DeleteNamespaceNoWait(controlPlaneNamespace)).To(Succeed(), "Namespace failed to be deleted on Primary Cluster") - Expect(k2.DeleteNamespaceNoWait(controlPlaneNamespace)).To(Succeed(), "Namespace failed to be deleted on Remote Cluster") - Expect(k1.DeleteNamespaceNoWait(sampleNamespace)).To(Succeed(), "Namespace failed to be deleted on Primary Cluster") - Expect(k2.DeleteNamespaceNoWait(sampleNamespace)).To(Succeed(), "Namespace failed to be deleted on Remote Cluster") - - Expect(k1.WaitNamespaceDeleted(istioCniNamespace)).To(Succeed()) - Expect(k2.WaitNamespaceDeleted(istioCniNamespace)).To(Succeed()) - Expect(k1.WaitNamespaceDeleted(controlPlaneNamespace)).To(Succeed()) - Expect(k2.WaitNamespaceDeleted(controlPlaneNamespace)).To(Succeed()) - Success("ControlPlane and CNI Namespaces were deleted") - - Expect(k1.WaitNamespaceDeleted(sampleNamespace)).To(Succeed()) - Expect(k2.WaitNamespaceDeleted(sampleNamespace)).To(Succeed()) - Success("Sample app is deleted in both clusters") - - // Delete the resources created by istioctl create-remote-secret - Expect(k2.Delete("ClusterRoleBinding", "istiod-clusterrole-istio-system")).To(Succeed()) - Expect(k2.Delete("ClusterRole", "istiod-clusterrole-istio-system")).To(Succeed()) - Expect(k2.Delete("ClusterRoleBinding", "istiod-gateway-controller-istio-system")).To(Succeed()) - Expect(k2.Delete("ClusterRole", "istiod-gateway-controller-istio-system")).To(Succeed()) + c1Deleted := clr1.CleanupNoWait(ctx) + c2Deleted := clr2.CleanupNoWait(ctx) + clr1.WaitForDeletion(ctx, c1Deleted) + clr2.WaitForDeletion(ctx, c2Deleted) }) }) } }) - - AfterAll(func(ctx SpecContext) { - if CurrentSpecReport().Failed() && !debugInfoLogged { - common.LogDebugInfo(common.MultiCluster, k1, k2) - debugInfoLogged = true - } - - // Delete the Sail Operator from both clusters - Expect(k1.DeleteNamespaceNoWait(namespace)).To(Succeed(), "Namespace failed to be deleted on Primary Cluster") - Expect(k2.DeleteNamespaceNoWait(namespace)).To(Succeed(), "Namespace failed to be deleted on Remote Cluster") - Expect(k1.WaitNamespaceDeleted(namespace)).To(Succeed()) - Expect(k2.WaitNamespaceDeleted(namespace)).To(Succeed()) - }) }) diff --git a/tests/e2e/multicluster/multicluster_suite_test.go b/tests/e2e/multicluster/multicluster_suite_test.go index 28d217fc9..5cf3c5540 100644 --- a/tests/e2e/multicluster/multicluster_suite_test.go +++ b/tests/e2e/multicluster/multicluster_suite_test.go @@ -23,11 +23,18 @@ import ( "testing" "github.com/istio-ecosystem/sail-operator/pkg/env" + "github.com/istio-ecosystem/sail-operator/pkg/kube" + . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/certs" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner" k8sclient "github.com/istio-ecosystem/sail-operator/tests/e2e/util/client" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" + . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -35,6 +42,7 @@ var ( clPrimary client.Client clRemote client.Client err error + debugInfoLogged bool namespace = env.Get("NAMESPACE", "sail-operator") deploymentName = env.Get("DEPLOYMENT_NAME", "sail-operator") controlPlaneNamespace = env.Get("CONTROL_PLANE_NS", "istio-system") @@ -42,9 +50,9 @@ var ( istioName = env.Get("ISTIO_NAME", "default") istioCniNamespace = env.Get("ISTIOCNI_NAMESPACE", "istio-cni") istioCniName = env.Get("ISTIOCNI_NAME", "default") - image = env.Get("IMAGE", "quay.io/maistra-dev/sail-operator:latest") skipDeploy = env.GetBool("SKIP_DEPLOY", false) multicluster = env.GetBool("MULTICLUSTER", false) + keepOnFailure = env.GetBool("KEEP_ON_FAILURE", false) kubeconfig = env.Get("KUBECONFIG", "") kubeconfig2 = env.Get("KUBECONFIG2", "") artifacts = env.Get("ARTIFACTS", "/tmp/artifacts") @@ -58,6 +66,9 @@ var ( k1 kubectl.Kubectl k2 kubectl.Kubectl + + clr1 cleaner.Cleaner + clr2 cleaner.Cleaner ) func TestMultiCluster(t *testing.T) { @@ -95,13 +106,60 @@ func setup(t *testing.T) { // Set base path baseRepoDir := filepath.Join(workDir, "../../..") - controlPlaneGatewayYAML = fmt.Sprintf("%s/docs/multicluster/controlplane-gateway.yaml", baseRepoDir) - eastGatewayYAML = fmt.Sprintf("%s/docs/multicluster/east-west-gateway-net1.yaml", baseRepoDir) - westGatewayYAML = fmt.Sprintf("%s/docs/multicluster/east-west-gateway-net2.yaml", baseRepoDir) - exposeServiceYAML = fmt.Sprintf("%s/docs/multicluster/expose-services.yaml", baseRepoDir) - exposeIstiodYAML = fmt.Sprintf("%s/docs/multicluster/expose-istiod.yaml", baseRepoDir) + controlPlaneGatewayYAML = fmt.Sprintf("%s/docs/deployment-models/resources/controlplane-gateway.yaml", baseRepoDir) + eastGatewayYAML = fmt.Sprintf("%s/docs/deployment-models/resources/east-west-gateway-net1.yaml", baseRepoDir) + westGatewayYAML = fmt.Sprintf("%s/docs/deployment-models/resources/east-west-gateway-net2.yaml", baseRepoDir) + exposeServiceYAML = fmt.Sprintf("%s/docs/deployment-models/resources/expose-services.yaml", baseRepoDir) + exposeIstiodYAML = fmt.Sprintf("%s/docs/deployment-models/resources/expose-istiod.yaml", baseRepoDir) // Initialize kubectl utilities, one for each cluster k1 = kubectl.New().WithKubeconfig(kubeconfig) k2 = kubectl.New().WithKubeconfig(kubeconfig2) + clr1 = cleaner.New(clPrimary, "cluster=primary") + clr2 = cleaner.New(clRemote, "cluster=remote") } + +var _ = BeforeSuite(func(ctx SpecContext) { + clr1.Record(ctx) + clr2.Record(ctx) + + if skipDeploy { + return + } + + Expect(k1.CreateNamespace(namespace)).To(Succeed(), "Namespace failed to be created on Primary Cluster") + Expect(k2.CreateNamespace(namespace)).To(Succeed(), "Namespace failed to be created on Remote Cluster") + + Expect(common.InstallOperatorViaHelm("--kubeconfig", kubeconfig)). + To(Succeed(), "Operator failed to be deployed in Primary Cluster") + + Expect(common.InstallOperatorViaHelm("--kubeconfig", kubeconfig2)). + To(Succeed(), "Operator failed to be deployed in Remote Cluster") + + Eventually(common.GetObject). + WithArguments(ctx, clPrimary, kube.Key(deploymentName, namespace), &appsv1.Deployment{}). + Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Error getting Istio CRD") + Success("Operator is deployed in the Primary namespace and Running") + + Eventually(common.GetObject). + WithArguments(ctx, clRemote, kube.Key(deploymentName, namespace), &appsv1.Deployment{}). + Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Error getting Istio CRD") + Success("Operator is deployed in the Remote namespace and Running") +}) + +var _ = ReportAfterSuite("Conditional cleanup", func(ctx SpecContext, r Report) { + if !r.SuiteSucceeded { + if !debugInfoLogged { + common.LogDebugInfo(common.MultiCluster, k1, k2) + } + + if keepOnFailure { + return + } + } + + c1Deleted := clr1.CleanupNoWait(ctx) + c2Deleted := clr2.CleanupNoWait(ctx) + clr1.WaitForDeletion(ctx, c1Deleted) + clr2.WaitForDeletion(ctx, c2Deleted) +}) diff --git a/tests/e2e/multicontrolplane/multi_control_plane_suite_test.go b/tests/e2e/multicontrolplane/multi_control_plane_suite_test.go index fb08bb324..7c0d90162 100644 --- a/tests/e2e/multicontrolplane/multi_control_plane_suite_test.go +++ b/tests/e2e/multicontrolplane/multi_control_plane_suite_test.go @@ -47,6 +47,7 @@ var ( appNamespace2a = env.Get("APP_NAMESPACE2A", "app2a") appNamespace2b = env.Get("APP_NAMESPACE2B", "app2b") multicluster = env.GetBool("MULTICLUSTER", false) + keepOnFailure = env.GetBool("KEEP_ON_FAILURE", false) ipFamily = env.Get("IP_FAMILY", "ipv4") k kubectl.Kubectl diff --git a/tests/e2e/multicontrolplane/multi_control_plane_test.go b/tests/e2e/multicontrolplane/multi_control_plane_test.go index 54eb438f5..ad788e775 100644 --- a/tests/e2e/multicontrolplane/multi_control_plane_test.go +++ b/tests/e2e/multicontrolplane/multi_control_plane_test.go @@ -24,6 +24,7 @@ import ( "github.com/istio-ecosystem/sail-operator/pkg/istioversion" "github.com/istio-ecosystem/sail-operator/pkg/kube" . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" . "github.com/onsi/ginkgo/v2" @@ -36,8 +37,10 @@ var _ = Describe("Multi control plane deployment model", Label("smoke", "multico SetDefaultEventuallyTimeout(180 * time.Second) SetDefaultEventuallyPollingInterval(time.Second) debugInfoLogged := false + clr := cleaner.New(cl) BeforeAll(func(ctx SpecContext) { + clr.Record(ctx) Expect(k.CreateNamespace(namespace)).To(Succeed(), "Namespace failed to be created") if skipDeploy { @@ -169,41 +172,18 @@ spec: }) }) - AfterAll(func() { - By("Cleaning up the application namespaces") - Expect(k.DeleteNamespace(appNamespace1, appNamespace2a, appNamespace2b)).To(Succeed()) + AfterAll(func(ctx SpecContext) { + if CurrentSpecReport().Failed() { + if !debugInfoLogged { + common.LogDebugInfo(common.MultiControlPlane, k) + debugInfoLogged = true + } - By("Cleaning up the Istio namespace") - Expect(k.DeleteNamespace(controlPlaneNamespace1, controlPlaneNamespace2)).To(Succeed(), "Istio Namespaces failed to be deleted") - - By("Cleaning up the IstioCNI namespace") - Expect(k.DeleteNamespace(istioCniNamespace)).To(Succeed(), "IstioCNI Namespace failed to be deleted") - - By("Deleting any left-over Istio and IstioRevision resources") - Expect(k.Delete("istio", istioName1)).To(Succeed(), "Failed to delete Istio") - Expect(k.Delete("istio", istioName2)).To(Succeed(), "Failed to delete Istio") - Expect(k.Delete("istiocni", istioCniName)).To(Succeed(), "Failed to delete IstioCNI") - Success("Istio Resources deleted") - Success("Cleanup done") - }) - - AfterAll(func() { - if CurrentSpecReport().Failed() && !debugInfoLogged { - common.LogDebugInfo(common.MultiControlPlane, k) - debugInfoLogged = true + if keepOnFailure { + return + } } - if skipDeploy { - Success("Skipping operator undeploy because it was deployed externally") - return - } - - By("Deleting operator deployment") - Expect(common.UninstallOperator()). - To(Succeed(), "Operator failed to be deleted") - GinkgoWriter.Println("Operator uninstalled") - - Expect(k.DeleteNamespace(namespace)).To(Succeed(), "Namespace failed to be deleted") - Success("Namespace deleted") + clr.Cleanup(ctx) }) }) diff --git a/tests/e2e/operator/operator_install_test.go b/tests/e2e/operator/operator_install_test.go index aea807705..16a40db9b 100644 --- a/tests/e2e/operator/operator_install_test.go +++ b/tests/e2e/operator/operator_install_test.go @@ -26,6 +26,7 @@ import ( "github.com/istio-ecosystem/sail-operator/pkg/kube" . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" . "github.com/onsi/ginkgo/v2" @@ -58,9 +59,11 @@ var sailCRDs = []string{ var _ = Describe("Operator", Label("smoke", "operator"), Ordered, func() { SetDefaultEventuallyTimeout(180 * time.Second) SetDefaultEventuallyPollingInterval(time.Second) + clr := cleaner.New(cl) Describe("installation", func() { - BeforeAll(func() { + BeforeAll(func(ctx SpecContext) { + clr.Record(ctx) Expect(k.CreateNamespace(namespace)).To(Succeed(), "Namespace failed to be created") if skipDeploy { @@ -185,9 +188,10 @@ spec: )) }) - AfterAll(func() { - Expect(k.DeleteNamespace(curlNamespace)).To(Succeed(), "failed to delete curl namespace") - exec.Command("kubectl", "delete", "--ignore-not-found", "clusterrolebinding", "metrics-reader-rolebinding").Run() + AfterAll(func(ctx SpecContext) { + if CurrentSpecReport().Failed() && keepOnFailure { + return + } if CurrentSpecReport().Failed() { common.LogDebugInfo(common.Operator, k) @@ -195,24 +199,15 @@ spec: }) }) - AfterAll(func() { + AfterAll(func(ctx SpecContext) { if CurrentSpecReport().Failed() { common.LogDebugInfo(common.Operator, k) + if keepOnFailure { + return + } } - if skipDeploy { - Success("Skipping operator undeploy because it was deployed externally") - return - } - - By("Uninstalling the operator") - Expect(common.UninstallOperator()). - To(Succeed(), "Operator failed to be deleted") - Success("Operator uninstalled") - - By("Deleting the CRDs") - Expect(k.DeleteCRDs(sailCRDs)).To(Succeed(), "CRDs failed to be deleted") - Success("CRDs deleted") + clr.Cleanup(ctx) }) }) diff --git a/tests/e2e/operator/operator_suite_test.go b/tests/e2e/operator/operator_suite_test.go index 2a66eb5e5..6da1f5fd0 100644 --- a/tests/e2e/operator/operator_suite_test.go +++ b/tests/e2e/operator/operator_suite_test.go @@ -35,6 +35,7 @@ var ( deploymentName = env.Get("DEPLOYMENT_NAME", "sail-operator") serviceAccountName = deploymentName multicluster = env.GetBool("MULTICLUSTER", false) + keepOnFailure = env.GetBool("KEEP_ON_FAILURE", false) curlNamespace = "curl-metrics" k kubectl.Kubectl diff --git a/tests/e2e/setup/build-and-push-operator.sh b/tests/e2e/setup/build-and-push-operator.sh index c74f0223e..f4527fa45 100755 --- a/tests/e2e/setup/build-and-push-operator.sh +++ b/tests/e2e/setup/build-and-push-operator.sh @@ -37,9 +37,13 @@ get_internal_registry() { fi URL=$(${COMMAND} get route default-route -n openshift-image-registry --template='{{ .spec.host }}') - export HUB="${URL}/${NAMESPACE}" + + # Create the istio-images namespace to store the operator image + # This will ensure that no matter the operator namespace is deleted, the image will still be available + export HUB="${URL}/istio-images" echo "Registry URL: ${HUB}" + ${COMMAND} create namespace istio-images || true ${COMMAND} create namespace "${NAMESPACE}" || true envsubst < "${WD}/config/role-bindings.yaml" | ${COMMAND} apply -f - @@ -56,18 +60,23 @@ build_and_push_operator_image() { echo "Building and pushing operator image: ${HUB}/${IMAGE_BASE}:${TAG}" DOCKER_BUILD_FLAGS="" - TARGET_ARCH=$(uname -m) + TARGET_ARCH=${TARGET_ARCH:-$(uname -m)} if [[ "$TARGET_ARCH" == "aarch64" || "$TARGET_ARCH" == "arm64" ]]; then TARGET_ARCH="arm64" DOCKER_BUILD_FLAGS="--platform=linux/${TARGET_ARCH}" elif [[ "$TARGET_ARCH" == "x86_64" || "$TARGET_ARCH" == "amd64" ]]; then TARGET_ARCH="amd64" + DOCKER_BUILD_FLAGS="--platform=linux/${TARGET_ARCH}" else echo "Unsupported architecture: ${TARGET_ARCH}" exit 1 fi + echo "Building for architecture: ${TARGET_ARCH}" + echo "Docker build flags: ${DOCKER_BUILD_FLAGS}" + echo "Using image base: ${HUB}/${IMAGE_BASE}:${TAG}" + BUILD_WITH_CONTAINER=0 \ DOCKER_BUILD_FLAGS=${DOCKER_BUILD_FLAGS} \ IMAGE=${HUB}/${IMAGE_BASE}:${TAG} \ @@ -80,4 +89,4 @@ if [ "${OCP}" == "true" ]; then get_internal_registry fi -build_and_push_operator_image +build_and_push_operator_image \ No newline at end of file diff --git a/tests/e2e/setup/config/role-bindings.yaml b/tests/e2e/setup/config/role-bindings.yaml index 1857c621f..d628f09ee 100644 --- a/tests/e2e/setup/config/role-bindings.yaml +++ b/tests/e2e/setup/config/role-bindings.yaml @@ -13,7 +13,7 @@ items: subjects: - kind: Group apiGroup: rbac.authorization.k8s.io - name: system:unauthenticated + name: system:authenticated - apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: @@ -24,6 +24,32 @@ items: kind: ClusterRole name: system:image-builder subjects: + - kind: Group + apiGroup: rbac.authorization.k8s.io + name: system:unauthenticated +- apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: image-puller + namespace: istio-images + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:image-puller + subjects: + - kind: Group + apiGroup: rbac.authorization.k8s.io + name: system:authenticated +- apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: image-pusher + namespace: istio-images + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:image-builder + subjects: - kind: Group apiGroup: rbac.authorization.k8s.io name: system:unauthenticated \ No newline at end of file diff --git a/tests/e2e/setup/setup-kind.sh b/tests/e2e/setup/setup-kind.sh index 8ff6ac665..a90eabccf 100755 --- a/tests/e2e/setup/setup-kind.sh +++ b/tests/e2e/setup/setup-kind.sh @@ -31,7 +31,6 @@ export IP_FAMILY="${IP_FAMILY:-ipv4}" export KIND_IP_FAMILY="${IP_FAMILY}" export ARTIFACTS="${ARTIFACTS:-$(mktemp -d)}" export MULTICLUSTER="${MULTICLUSTER:-false}" -export KIND_EXCLUDE_CLUSTERS="${KIND_EXCLUDE_CLUSTERS:-}" export KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-operator-integration-tests}" if [ "${MULTICLUSTER}" == "true" ]; then @@ -48,11 +47,7 @@ function setup_kind_registry() { docker network connect "kind" "${KIND_REGISTRY_NAME}" fi - for cluster in $(kind get clusters); do - if [[ "${KIND_EXCLUDE_CLUSTERS}" == *"${cluster}"* ]]; then - continue - fi - + for cluster in "$@"; do if [ "${MULTICLUSTER}" == "true" ]; then export KUBECONFIG="${KUBECONFIG_DIR}/${cluster}" else @@ -70,13 +65,13 @@ if [ "${MULTICLUSTER}" == "true" ]; then CLUSTER_TOPOLOGY_CONFIG_FILE="${SCRIPTPATH}/../setup/config/multicluster.json" load_cluster_topology "${CLUSTER_TOPOLOGY_CONFIG_FILE}" setup_kind_clusters "" "" - setup_kind_registry + setup_kind_registry "${CLUSTER_NAMES[@]}" export KUBECONFIG="${KUBECONFIGS[0]}" export KUBECONFIG2="${KUBECONFIGS[1]}" echo "Your KinD environment is ready, to use it: export KUBECONFIG=$(IFS=:; echo "${KUBECONFIGS[*]}")" else KUBECONFIG="${ARTIFACTS}/config" setup_kind_cluster "${KIND_CLUSTER_NAME}" "" "" "true" "true" - setup_kind_registry + setup_kind_registry "$KIND_CLUSTER_NAME" echo "Your KinD environment is ready, to use it: export KUBECONFIG=${ARTIFACTS}/config" fi diff --git a/tests/e2e/util/cleaner/cleaner.go b/tests/e2e/util/cleaner/cleaner.go new file mode 100644 index 000000000..7f52602e1 --- /dev/null +++ b/tests/e2e/util/cleaner/cleaner.go @@ -0,0 +1,301 @@ +//go:build e2e + +// Copyright Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR Condition OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cleaner + +import ( + "context" + "fmt" + "strings" + "time" + + . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// Cleaner records resources to keep, and cleans up any resources it didn't record. +type Cleaner struct { + cl client.Client + ctx []string + resources map[resource]bool + crds map[string]struct{} + crs map[string]map[resource]struct{} +} + +type resource struct { + kind string + name string + namespace string +} + +// New returns a Cleaner which can record resources to keep, and clean up any resources it didn't record. +// It needs an initialized client, and has optional (string) context which can be used to distinguish it's output. +func New(cl client.Client, ctx ...string) Cleaner { + return Cleaner{ + cl: cl, + ctx: ctx, + resources: make(map[resource]bool), + crds: make(map[string]struct{}), + crs: make(map[string]map[resource]struct{}), + } +} + +// Record will save the state of all resources it wants to keep in the cluster, so they won't be cleaned up. +func (c *Cleaner) Record(ctx context.Context) { + // Save all namespaces that exist so that we can skip them when cleaning + namespaceList := &corev1.NamespaceList{} + Expect(c.cl.List(ctx, namespaceList)).To(Succeed()) + for _, ns := range namespaceList.Items { + c.resources[c.resourceFromObj(&ns, ns.Name)] = true + } + + crList := &rbacv1.ClusterRoleList{} + Expect(c.cl.List(ctx, crList)).To(Succeed()) + for _, cr := range crList.Items { + c.resources[c.resourceFromObj(&cr, cr.Name)] = true + } + + crbList := &rbacv1.ClusterRoleBindingList{} + Expect(c.cl.List(ctx, crbList)).To(Succeed()) + for _, crb := range crbList.Items { + c.resources[c.resourceFromObj(&crb, crb.Name)] = true + } + + allCRDs := &apiextensionsv1.CustomResourceDefinitionList{} + Expect(c.cl.List(ctx, allCRDs)).To(Succeed()) + + // Save all existing custom resources so we can skip them when cleaning + for _, crd := range allCRDs.Items { + if !c.trackedCRD(&crd) { + continue + } + + c.crds[crd.Name] = struct{}{} + c.crs[crd.Name] = make(map[resource]struct{}) + customResources := &unstructured.UnstructuredList{} + customResources.SetGroupVersionKind(extractGVK(&crd)) + + Expect(c.cl.List(ctx, customResources)).To(Succeed()) + for _, cr := range customResources.Items { + c.crs[crd.Name][crKey(cr)] = struct{}{} + } + } +} + +func (c *Cleaner) resourceFromObj(obj client.Object, name string) resource { + kinds, _, _ := c.cl.Scheme().ObjectKinds(obj) + return resource{kind: kinds[0].Kind, name: name} +} + +func extractGVK(crd *apiextensionsv1.CustomResourceDefinition) schema.GroupVersionKind { + return schema.GroupVersionKind{ + Group: crd.Spec.Group, + Version: crd.Spec.Versions[0].Name, + Kind: crd.Spec.Names.Kind, + } +} + +func crKey(cr unstructured.Unstructured) resource { + return resource{name: cr.GetName(), namespace: cr.GetNamespace()} +} + +// CleanupNoWait will cleanup any resources not recorded, while not waiting for their deletion to complete. +// Use Cleaner.WaitForDeletion to wait for their deletion at a later point. +func (c *Cleaner) CleanupNoWait(ctx context.Context) (deleted []client.Object) { + return c.cleanup(ctx) +} + +// Cleanup will cleanup any resources not recorded, and wait for their deletion. +func (c *Cleaner) Cleanup(ctx context.Context) { + c.WaitForDeletion(ctx, c.cleanup(ctx)) +} + +type resObj struct { + key resource + obj client.Object +} + +func (c *Cleaner) cleanup(ctx context.Context) (deleted []client.Object) { + allCRDs := &apiextensionsv1.CustomResourceDefinitionList{} + Expect(c.cl.List(ctx, allCRDs)).To(Succeed()) + + // First, clean up all custom resources that didn't exist, to give the operator a shot at finalizing them + for _, crd := range allCRDs.Items { + if !c.trackedCRD(&crd) { + continue + } + + gvk := extractGVK(&crd) + customResources := &unstructured.UnstructuredList{} + customResources.SetGroupVersionKind(gvk) + + Expect(c.cl.List(ctx, customResources)).To(Succeed()) + for _, cr := range customResources.Items { + // Skip any recorded custom resource. + if mapHasKey(c.crs, crd.Name) && mapHasKey(c.crs[crd.Name], crKey(cr)) { + continue + } + + By(c.cleaningUpThe(&cr, gvk.Kind), func() { + Expect(c.delete(ctx, &cr)).To(Succeed()) + deleted = append(deleted, &cr) + }) + } + } + + // At this point we have to wait for cleanup to finish, since some CRs might get stuck "finalizing" if we later delete the operator namespace. + c.WaitForDeletion(ctx, deleted) + deleted = make([]client.Object, 0) + + // Clean up any resources we didn't record. + var resources []resObj + namespaceList := &corev1.NamespaceList{} + Expect(c.cl.List(ctx, namespaceList)).To(Succeed()) + for _, ns := range namespaceList.Items { + resources = append(resources, resObj{c.resourceFromObj(&ns, ns.Name), &ns}) + } + + crList := &rbacv1.ClusterRoleList{} + Expect(c.cl.List(ctx, crList)).To(Succeed()) + for _, cr := range crList.Items { + resources = append(resources, resObj{c.resourceFromObj(&cr, cr.Name), &cr}) + } + + crbList := &rbacv1.ClusterRoleBindingList{} + Expect(c.cl.List(ctx, crbList)).To(Succeed()) + for _, crb := range crbList.Items { + resources = append(resources, resObj{c.resourceFromObj(&crb, crb.Name), &crb}) + } + + // Clean up all resources that didn't exist before the tests + for _, res := range resources { + // Skip any resource we recorded previously. + // Also, skip any OLM managed resource as it'll be recreated upon deletion. + if mapHasKey(c.resources, res.key) || + mapHasKey(res.obj.GetLabels(), "olm.managed") { + continue + } + + By(c.cleaningUpThe(res.obj, res.key.kind), func() { + Expect(c.delete(ctx, res.obj)).To(Succeed()) + deleted = append(deleted, res.obj) + }) + } + + // Finally, remove any CRD that didn't exist at time of recording. + for _, crd := range allCRDs.Items { + // Skip any recorded CRD, and any CRD we don't track. + if mapHasKey(c.crds, crd.Name) || !c.trackedCRD(&crd) { + continue + } + + By(c.cleaningUpThe(&crd, "CRD"), func() { + Expect(c.delete(ctx, &crd)).To(Succeed()) + deleted = append(deleted, &crd) + }) + } + + return +} + +func mapHasKey[K comparable, V any](m map[K]V, k K) bool { + _, exists := m[k] + return exists +} + +func (c *Cleaner) trackedCRD(crd *apiextensionsv1.CustomResourceDefinition) bool { + return crd.Spec.Group == "sailoperator.io" || strings.HasSuffix(crd.Spec.Group, "istio.io") +} + +func (c *Cleaner) delete(ctx context.Context, obj client.Object) error { + if err := c.cl.Delete(ctx, obj); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + + return err + } + + return nil +} + +func (c *Cleaner) cleaningUpThe(obj client.Object, kind string) (s string) { + ctx := []string{} + ns := obj.GetNamespace() + if ns != "" { + ctx = append(ctx, fmt.Sprintf("namespace=%s", ns)) + } + + ctx = append(ctx, c.ctx...) + s = strings.Join(ctx, ", ") + if s != "" { + s = fmt.Sprintf(" on %s", s) + } + + return fmt.Sprintf("Cleaning up the %s %s%s", obj.GetName(), kind, s) +} + +// WaitForDeletion receives a slice of resources marked for deletion, and waits until they're all delete. +// It will fail the test suite if any resource hasn't been deleted in sufficient time. +func (c *Cleaner) WaitForDeletion(ctx context.Context, deleted []client.Object) { + if len(deleted) == 0 { + return + } + + s := strings.Join(c.ctx, ", ") + if s != "" { + s = fmt.Sprintf(" on %s", s) + } + + By(fmt.Sprintf("Waiting for resources to be deleted%s", s)) + for _, obj := range deleted { + Expect(c.waitForDeletion(ctx, obj)).To(Succeed()) + } + + Success(fmt.Sprintf("Finished cleaning up resources%s", s)) +} + +func (c *Cleaner) waitForDeletion(ctx context.Context, obj client.Object) error { + objKey := client.ObjectKeyFromObject(obj) + + err := wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, 1*time.Minute, true, func(ctx context.Context) (bool, error) { + gotObj := obj.DeepCopyObject().(client.Object) + + if err := c.cl.Get(ctx, objKey, gotObj); err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + + return false, err + } + + return false, nil + }) + if err != nil { + return err + } + + return nil +} diff --git a/tests/e2e/util/common/e2e_utils.go b/tests/e2e/util/common/e2e_utils.go index c8b8db26d..2690f6c81 100644 --- a/tests/e2e/util/common/e2e_utils.go +++ b/tests/e2e/util/common/e2e_utils.go @@ -33,6 +33,7 @@ import ( "github.com/istio-ecosystem/sail-operator/pkg/test/project" . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/helm" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/istioctl" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -217,6 +218,10 @@ func logIstioDebugInfo(k kubectl.Kubectl) { events, err := k.WithNamespace(controlPlaneNamespace).GetEvents() logDebugElement("=====Events in "+controlPlaneNamespace+"=====", events, err) + + // Running istioctl proxy-status to get the status of the proxies. + proxyStatus, err := istioctl.GetProxyStatus() + logDebugElement("=====Istioctl Proxy Status=====", proxyStatus, err) } func logCNIDebugInfo(k kubectl.Kubectl) { @@ -315,10 +320,6 @@ func InstallOperatorViaHelm(extraArgs ...string) error { return helm.Install("sail-operator", filepath.Join(project.RootDir, "chart"), args...) } -func UninstallOperator() error { - return helm.Uninstall("sail-operator", "--namespace", OperatorNamespace) -} - // GetSampleYAML returns the URL of the yaml file for the testing app. // args: // version: the version of the Istio to get the yaml file from. diff --git a/tests/e2e/util/helm/helm.go b/tests/e2e/util/helm/helm.go index 13e6c2f5c..d36868462 100644 --- a/tests/e2e/util/helm/helm.go +++ b/tests/e2e/util/helm/helm.go @@ -37,18 +37,3 @@ func Install(name string, chart string, args ...string) error { g.Success("Helm install executed successfully") return nil } - -// Uninstall runs helm uninstall in the given directory with params -// name: name of the helm release -// args: additional helm args, for example "--namespace sail-operator" -func Uninstall(name string, args ...string) error { - argsStr := strings.Join(args, " ") - command := fmt.Sprintf("helm uninstall %s %s", name, argsStr) - output, err := shell.ExecuteCommand(command) - if err != nil { - return fmt.Errorf("error running Helm uninstall: %w. Output: %s", err, output) - } - - g.Success("Helm uninstall executed successfully") - return nil -} diff --git a/tests/e2e/util/istioctl/istioctl.go b/tests/e2e/util/istioctl/istioctl.go index ccc46b5c5..c51027fdb 100644 --- a/tests/e2e/util/istioctl/istioctl.go +++ b/tests/e2e/util/istioctl/istioctl.go @@ -58,3 +58,13 @@ func CreateRemoteSecret(remoteKubeconfig, namespace, secretName, internalIP stri return yaml, err } + +// GetProxyStatus runs istioctl proxy-status command and return the output +func GetProxyStatus() (string, error) { + cmd := istioctl("proxy-status") + status, err := shell.ExecuteCommand(cmd) + if err != nil { + return "", fmt.Errorf("failed to get proxy status: %w", err) + } + return status, nil +} diff --git a/tests/e2e/util/kubectl/kubectl.go b/tests/e2e/util/kubectl/kubectl.go index 182fb8dff..a4e28d81e 100644 --- a/tests/e2e/util/kubectl/kubectl.go +++ b/tests/e2e/util/kubectl/kubectl.go @@ -17,7 +17,6 @@ package kubectl import ( "fmt" "os" - "strconv" "strings" "time" @@ -117,38 +116,6 @@ func (k Kubectl) CreateFromString(yamlString string) error { return nil } -// DeleteCRDs deletes the CRDs by given list of crds names -func (k Kubectl) DeleteCRDs(crds []string) error { - for _, crd := range crds { - cmd := k.build(" delete crd " + crd) - _, err := shell.ExecuteCommand(cmd) - if err != nil { - return fmt.Errorf("error deleting crd %s: %w", crd, err) - } - } - - return nil -} - -// DeleteNamespaceNoWait deletes a namespace and returns immediately (without waiting for the namespace to be removed). -func (k Kubectl) DeleteNamespaceNoWait(namespaces ...string) error { - return k.deleteNamespace(namespaces, false) -} - -// DeleteNamespace deletes a namespace and waits for it to be removed completely. -func (k Kubectl) DeleteNamespace(namespaces ...string) error { - return k.deleteNamespace(namespaces, true) -} - -func (k Kubectl) deleteNamespace(namespaces []string, wait bool) error { - cmd := k.build(" delete namespace " + strings.Join(namespaces, " ") + " --wait=" + strconv.FormatBool(wait)) - _, err := k.executeCommand(cmd) - if err != nil { - return fmt.Errorf("error deleting namespace: %w", err) - } - return nil -} - // ApplyString applies the given yaml string to the cluster func (k Kubectl) ApplyString(yamlString string) error { cmd := k.build(" apply --server-side -f -") @@ -199,13 +166,6 @@ func (k Kubectl) Delete(kind, name string) error { return nil } -// Wait waits for a specific condition on one or many resources -func (k Kubectl) Wait(waitFor, resource string, timeout time.Duration) error { - cmd := k.build(fmt.Sprintf("wait --for %s %s --timeout %s", waitFor, resource, timeout.String())) - _, err := k.executeCommand(cmd) - return err -} - // Patch patches a resource func (k Kubectl) Patch(kind, name, patchType, patch string) error { cmd := k.build(fmt.Sprintf(" patch %s %s --type=%s -p=%q", kind, name, patchType, patch)) @@ -331,11 +291,6 @@ func (k Kubectl) executeCommand(cmd string) (string, error) { return shell.ExecuteCommand(cmd) } -// WaitNamespaceDeleted waits for a namespace to be deleted -func (k Kubectl) WaitNamespaceDeleted(ns string) error { - return k.Wait("delete", "namespace/"+ns, 2*time.Minute) -} - func sinceFlag(since *time.Duration) string { if since == nil { return "" diff --git a/tools/update_deps.sh b/tools/update_deps.sh index 6a2152c82..0a62de430 100755 --- a/tools/update_deps.sh +++ b/tools/update_deps.sh @@ -90,5 +90,9 @@ RUNME_LATEST_VERSION=$(getLatestVersion runmedev/runme) RUNME_LATEST_VERSION=${RUNME_LATEST_VERSION#v} sed -i "s|RUNME_VERSION ?= .*|RUNME_VERSION ?= ${RUNME_LATEST_VERSION}|" "${ROOTDIR}/Makefile.core.mk" +# Update misspell +MISSPELL_LATEST_VERSION=$(getLatestVersion client9/misspell) +sed -i "s|MISSPELL_VERSION ?= .*|MISSPELL_VERSION ?= ${MISSPELL_LATEST_VERSION}|" "${ROOTDIR}/Makefile.core.mk" + # Regenerate files make update-istio gen