Skip to content

Commit 0f1461d

Browse files
andyatmiamiandyatmiami
committed
ci: scaffold e2e testing framework infrastructure
This commit establishes the foundational infrastructure for end-to-end testing of the workspaces components. This is an intermediate step that sets up the testing framework; no actual test execution is implemented yet and will be added in subsequent work. Changes include: - Add new `testing/` directory with Makefile and setup scripts: * `setup-kind.sh`: Automated Kind cluster creation and configuration * `setup-cert-manager.sh`: Cert-manager installation (v1.12.13 LTS) * `setup-istio.sh`: Istio service mesh installation * Makefile targets for cluster setup, component deployment, and e2e test execution (currently placeholder with TODO) - Add GitHub Actions workflow (`.github/workflows/ws-e2e-test.yml`): * Triggers on pushes to main branches and PRs affecting workspaces * Sets up Go environment and Kind cluster * Executes local-e2e target (placeholder for future test implementation) - Update workspace Makefiles (backend, controller, frontend): * Modify deploy targets to use `.output/` directories for kustomize operations, preventing modification of source manifests - Update .gitignore files: * Add `.output/` directories to prevent committing generated kustomize artifacts The framework enables automated deployment of all three components (controller, backend, frontend) to a Kind cluster with cert-manager and Istio pre-configured, providing the foundation for comprehensive e2e test scenarios in future commits. Signed-off-by: Andy Stoneberg <[email protected]>
1 parent 054ef4e commit 0f1461d

File tree

14 files changed

+388
-6
lines changed

14 files changed

+388
-6
lines changed

‎.github/workflows/ws-e2e-test.yml‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
name: Workspaces E2E Tests
2+
3+
permissions:
4+
contents: read
5+
6+
on:
7+
push:
8+
branches:
9+
- main
10+
- notebooks-v2
11+
- v*-branch
12+
pull_request:
13+
paths:
14+
- 'workspaces/**'
15+
- 'releasing/version/VERSION'
16+
17+
jobs:
18+
e2e-test:
19+
runs-on: ubuntu-latest
20+
env:
21+
GO_VERSION: '1.22'
22+
defaults:
23+
run:
24+
working-directory: testing
25+
steps:
26+
- name: Checkout code
27+
uses: actions/checkout@v4
28+
29+
- name: Set up Go
30+
uses: actions/setup-go@v5
31+
with:
32+
go-version: ${{ env.GO_VERSION }}
33+
cache: false
34+
35+
- name: Generate cache key from Makefile
36+
id: cache-key
37+
run: |
38+
VERSION_HASH=$(make dependency-hash)
39+
echo "cache_key=testing-bin-${{ runner.os }}-go${{ env.GO_VERSION }}-$VERSION_HASH" >> $GITHUB_OUTPUT
40+
41+
- name: Cache testing/bin directory
42+
uses: actions/cache@v4
43+
id: cache-testing-bin
44+
with:
45+
path: testing/bin
46+
key: ${{ steps.cache-key.outputs.cache_key }}
47+
48+
- name: Setup cluster
49+
run: make setup-cluster
50+
51+
- name: Run local-e2e tests
52+
run: make local-e2e
53+

‎testing/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# Binaries for programs and plugins
2+
bin/*

‎testing/Makefile‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
GIT_COMMIT := $(shell git rev-parse HEAD)
2+
GIT_TREE_STATE := $(shell test -n "`git status --porcelain`" && echo "-dirty" || echo "")
3+
4+
# Image URL to use all building/pushing image targets
5+
REGISTRY ?= ghcr.io/kubeflow/notebooks
6+
TAG ?= sha-$(GIT_COMMIT)$(GIT_TREE_STATE)
7+
8+
CONTROLLER_NAME ?= workspaces-controller
9+
CONTROLLER_IMG ?= $(REGISTRY)/$(CONTROLLER_NAME):$(TAG)
10+
11+
BACKEND_NAME ?= workspaces-backend
12+
BACKEND_IMG ?= $(REGISTRY)/$(BACKEND_NAME):$(TAG)
13+
14+
FRONTEND_NAME ?= workspaces-frontend
15+
FRONTEND_IMG ?= $(REGISTRY)/$(FRONTEND_NAME):$(TAG)
16+
17+
KIND_CLUSTER_NAME ?= local-e2e
18+
19+
# Setting SHELL to bash allows bash commands to be executed by recipes.
20+
# Options are set to exit when a recipe line exits non-zero or a piped command fails.
21+
SHELL = /usr/bin/env bash -o pipefail
22+
.SHELLFLAGS = -ec
23+
24+
25+
# Export KIND_EXPERIMENTAL_PROVIDER to honor it if set in user's environment
26+
# (e.g., KIND_EXPERIMENTAL_PROVIDER=podman for podman support)
27+
export KIND_EXPERIMENTAL_PROVIDER
28+
29+
##@ General
30+
31+
# The help target prints out all targets with their descriptions organized
32+
# beneath their categories. The categories are represented by '##@' and the
33+
# target descriptions by '##'. The awk command is responsible for reading the
34+
# entire set of makefiles included in this invocation, looking for lines of the
35+
# file as xyz: ## something, and then pretty-format the target and help. Then,
36+
# if there's a line with ##@ something, that gets pretty-printed as a category.
37+
# More info on the usage of ANSI control characters for terminal formatting:
38+
# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters
39+
# More info on the awk command:
40+
# http://linuxcommand.org/lc3_adv_awk.php
41+
42+
.PHONY: help
43+
help: ## Display this help.
44+
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
45+
46+
##@ Deployment
47+
48+
.PHONY: build-controller build-backend build-frontend build-all e2e
49+
50+
deploy-controller: kind check-kind-context ## Build and deploy the controller.
51+
cd ../workspaces/controller && $(MAKE) docker-build IMG=$(CONTROLLER_IMG)
52+
$(KIND) load docker-image $(CONTROLLER_IMG) --name $(KIND_CLUSTER_NAME)
53+
cd ../workspaces/controller && $(MAKE) deploy IMG=$(CONTROLLER_IMG)
54+
55+
deploy-backend: kind check-kind-context ## Build and deploy the backend.
56+
cd ../workspaces/backend && $(MAKE) docker-build IMG=$(BACKEND_IMG)
57+
$(KIND) load docker-image $(BACKEND_IMG) --name $(KIND_CLUSTER_NAME)
58+
cd ../workspaces/backend && $(MAKE) deploy IMG=$(BACKEND_IMG)
59+
60+
deploy-frontend: kind check-kind-context ## Build and deploy the frontend.
61+
cd ../workspaces/frontend && $(MAKE) docker-build IMG=$(FRONTEND_IMG)
62+
$(KIND) load docker-image $(FRONTEND_IMG) --name $(KIND_CLUSTER_NAME)
63+
cd ../workspaces/frontend && $(MAKE) deploy IMG=$(FRONTEND_IMG)
64+
65+
deploy-all: deploy-controller deploy-backend deploy-frontend ## Deploy all components.
66+
67+
local-e2e: deploy-all istioctl ## Run e2e tests.
68+
@echo "TODO: Run e2e tests..."
69+
70+
71+
##@ Dependencies
72+
73+
## Location to install dependencies to
74+
LOCALBIN ?= $(shell pwd)/bin
75+
$(LOCALBIN):
76+
mkdir -p $(LOCALBIN)
77+
78+
## Tool Binaries
79+
KIND ?= $(LOCALBIN)/kind
80+
KUBECTL ?= kubectl
81+
ISTIOCTL ?= $(LOCALBIN)/istioctl
82+
83+
## Tool Versions
84+
KIND_VERSION ?= v0.30.0
85+
ISTIOCTL_VERSION ?= 1.27.3
86+
87+
.PHONY: kind
88+
kind: $(KIND) ## Download kind locally if necessary.
89+
$(KIND): $(LOCALBIN)
90+
$(call go-install-tool,$(KIND),sigs.k8s.io/kind,$(KIND_VERSION))
91+
92+
.PHONY: check-kubectl
93+
check-kubectl: ## Verify that kubectl is available in PATH.
94+
@if ! command -v $(KUBECTL) >/dev/null 2>&1; then \
95+
echo "✗ ERROR: kubectl is not installed or not found in PATH"; \
96+
echo " Please install kubectl: https://kubernetes.io/docs/tasks/tools/#kubectl"; \
97+
exit 1; \
98+
fi
99+
@echo "✓ kubectl found: $$($(KUBECTL) version --client 2>/dev/null || echo 'version check failed')"
100+
101+
.PHONY: istioctl
102+
istioctl: $(ISTIOCTL) ## Download istioctl locally if necessary.
103+
$(ISTIOCTL): $(LOCALBIN)
104+
$(call go-install-tool,$(ISTIOCTL),istio.io/istio/istioctl/cmd/istioctl,$(ISTIOCTL_VERSION))
105+
106+
.PHONY: dependency-hash
107+
dependency-hash: ## Calculate hash of dependency versions for caching.
108+
@echo -e "KIND_VERSION=$(KIND_VERSION)\nISTIOCTL_VERSION=$(ISTIOCTL_VERSION)" | sha256sum | cut -d' ' -f1 | head -c 16
109+
110+
.PHONY: setup-cluster
111+
setup-cluster: check-kubectl kind istioctl ## Set up a complete kind cluster with cert-manager and Istio.
112+
@export PATH="$(LOCALBIN):$$PATH" && \
113+
bash scripts/setup-kind.sh && \
114+
bash scripts/setup-cert-manager.sh && \
115+
bash scripts/setup-istio.sh
116+
@echo "✓ Cluster setup complete"
117+
118+
.PHONY: check-kind-context
119+
check-kind-context: check-kubectl ## Verify that the current kubectl context is a kind cluster.
120+
@current_context=$$($(KUBECTL) config current-context 2>/dev/null) || { \
121+
echo "Error: Unable to get current kubectl context. Is kubectl configured?"; \
122+
exit 1; \
123+
}; \
124+
server_url=$$($(KUBECTL) config view --minify -o jsonpath='{.clusters[0].cluster.server}' 2>/dev/null) || { \
125+
echo "Error: Unable to get cluster server URL for context '$$current_context'"; \
126+
exit 1; \
127+
}; \
128+
context_check=0; \
129+
server_check=0; \
130+
if echo "$$current_context" | grep -qE '^kind-'; then \
131+
context_check=1; \
132+
fi; \
133+
if echo "$$server_url" | grep -qE '(127\.0\.0\.1|localhost)'; then \
134+
server_check=1; \
135+
fi; \
136+
if [ $$context_check -ne 1 ] || [ $$server_check -ne 1 ]; then \
137+
echo "✗ ERROR: Current context '$$current_context' does not appear to be a kind cluster!"; \
138+
if [ $$context_check -ne 1 ]; then \
139+
echo " ✗ Context name does not match kind-* pattern (got: $$current_context)"; \
140+
fi; \
141+
if [ $$server_check -ne 1 ]; then \
142+
echo " ✗ Server URL does not use localhost/127.0.0.1 (got: $$server_url)"; \
143+
fi; \
144+
exit 1; \
145+
fi
146+
147+
# go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist
148+
# $1 - target path with name of binary
149+
# $2 - package url which can be installed
150+
# $3 - specific version of package
151+
define go-install-tool
152+
@[ -f "$(1)-$(3)" ] || { \
153+
set -e; \
154+
package=$(2)@$(3) ;\
155+
echo "Downloading $${package}" ;\
156+
rm -f $(1) || true ;\
157+
GOBIN=$(LOCALBIN) go install $${package} ;\
158+
mv $(1) $(1)-$(3) ;\
159+
} ;\
160+
ln -sf $(1)-$(3) $(1)
161+
endef

‎testing/OWNERS‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
labels:
2+
- area/ci
3+
- area/v2
4+
approvers:
5+
- andyatmiami

‎testing/scripts/kind.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
apiVersion: kind.x-k8s.io/v1alpha4
2+
kind: Cluster
3+
# This is needed in order to support projected volumes with service account tokens.
4+
kubeadmConfigPatches:
5+
- |
6+
apiVersion: kubeadm.k8s.io/v1beta3
7+
kind: ClusterConfiguration
8+
metadata:
9+
name: config
10+
apiServer:
11+
extraArgs:
12+
"service-account-issuer": "kubernetes.default.svc"
13+
"service-account-signing-key-file": "/etc/kubernetes/pki/sa.key"
14+
nodes:
15+
- role: control-plane
16+
image: kindest/node:v1.33.1@sha256:050072256b9a903bd914c0b2866828150cb229cea0efe5892e2b644d5dd3b34f
17+
- role: worker
18+
image: kindest/node:v1.33.1@sha256:050072256b9a903bd914c0b2866828150cb229cea0efe5892e2b644d5dd3b34f

‎testing/scripts/setup-cert-manager.sh‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/usr/bin/env bash
2+
3+
# Setup script for cert-manager
4+
# This script checks if cert-manager is installed and installs it if needed
5+
# Uses the same version as the e2e tests: v1.12.13 (LTS version)
6+
7+
set -euo pipefail
8+
9+
# Use LTS version of cert-manager (matches e2e tests)
10+
CERT_MANAGER_VERSION="v1.12.13"
11+
CERT_MANAGER_URL="https://github.com/jetstack/cert-manager/releases/download/${CERT_MANAGER_VERSION}/cert-manager.yaml"
12+
13+
# Check if cert-manager is already installed
14+
if kubectl get crd certificates.cert-manager.io >/dev/null 2>&1; then
15+
echo "Cert-manager is already installed"
16+
exit 0
17+
fi
18+
19+
echo "Installing cert-manager ${CERT_MANAGER_VERSION}..."
20+
kubectl apply -f "${CERT_MANAGER_URL}"
21+
22+
echo "Waiting for cert-manager to be ready..."
23+
# Wait for cert-manager webhook to be ready (this is the critical component)
24+
kubectl wait --for=condition=ready pod \
25+
-l app.kubernetes.io/instance=cert-manager \
26+
-n cert-manager \
27+
--timeout=120s || {
28+
echo "Warning: cert-manager pods may not be fully ready, but continuing..."
29+
}
30+
31+
# Also wait for the CRDs to be established
32+
kubectl wait --for=condition=established crd/certificates.cert-manager.io --timeout=60s || true
33+
kubectl wait --for=condition=established crd/issuers.cert-manager.io --timeout=60s || true
34+
kubectl wait --for=condition=established crd/clusterissuers.cert-manager.io --timeout=60s || true
35+
36+
echo "Cert-manager installation complete"

‎testing/scripts/setup-istio.sh‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#!/usr/bin/env bash
2+
3+
# Setup script for Istio
4+
# This script checks if Istio is installed and installs it if needed
5+
# Uses istioctl to install the default profile
6+
7+
set -euo pipefail
8+
9+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
10+
TESTING_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
11+
LOCALBIN="${TESTING_DIR}/bin"
12+
13+
# Determine istioctl path - prefer LOCALBIN, fallback to PATH
14+
if [ -f "${LOCALBIN}/istioctl" ]; then
15+
ISTIOCTL="${LOCALBIN}/istioctl"
16+
elif command -v istioctl >/dev/null 2>&1; then
17+
ISTIOCTL="istioctl"
18+
else
19+
echo "ERROR: istioctl is not installed. Please install istioctl first:"
20+
echo " cd testing && make istioctl"
21+
echo " or visit: https://istio.io/latest/docs/setup/getting-started/#download"
22+
exit 1
23+
fi
24+
25+
# Check if Istio is already installed
26+
# Check for istio-system namespace or istio CRDs
27+
if kubectl get namespace istio-system >/dev/null 2>&1 && \
28+
kubectl get crd virtualservices.networking.istio.io >/dev/null 2>&1; then
29+
echo "Istio is already installed"
30+
exit 0
31+
fi
32+
33+
echo "Installing Istio with default profile..."
34+
"${ISTIOCTL}" install --set profile=default -y
35+
36+
echo "Waiting for Istio to be ready..."
37+
# Wait for istiod to be ready
38+
kubectl wait --for=condition=ready pod \
39+
-l app=istiod \
40+
-n istio-system \
41+
--timeout=120s || {
42+
echo "Warning: Istio pods may not be fully ready, but continuing..."
43+
}
44+
45+
# Wait for istio ingress gateway to be ready (if present)
46+
kubectl wait --for=condition=ready pod \
47+
-l app=istio-ingressgateway \
48+
-n istio-system \
49+
--timeout=120s || {
50+
echo "Warning: Istio ingress gateway may not be ready, but continuing..."
51+
}
52+
53+
echo "Istio installation complete"
54+

‎testing/scripts/setup-kind.sh‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/usr/bin/env bash
2+
3+
# Setup script for Kind cluster
4+
# This script checks if a Kind cluster exists and creates it if needed
5+
6+
set -euo pipefail
7+
8+
CLUSTER_NAME="local-e2e"
9+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
10+
KIND_CONFIG="${SCRIPT_DIR}/kind.yml"
11+
12+
# Check if kind command exists
13+
if ! command -v kind >/dev/null 2>&1; then
14+
echo "ERROR: kind is not installed. Please install kind first:"
15+
echo " brew install kind # macOS"
16+
echo " or visit: https://kind.sigs.k8s.io/docs/user/quick-start/#installation"
17+
exit 1
18+
fi
19+
20+
# Check if cluster exists
21+
if ! kind get clusters 2>/dev/null | grep -q "^${CLUSTER_NAME}$"; then
22+
echo "Creating Kind cluster '${CLUSTER_NAME}' with config from ${KIND_CONFIG}..."
23+
kind create cluster --name "${CLUSTER_NAME}" --config "${KIND_CONFIG}" --wait 60s
24+
echo "Kind cluster created successfully"
25+
else
26+
echo "Kind cluster '${CLUSTER_NAME}' already exists"
27+
fi
28+
29+
# Ensure kubectl context is set to the Kind cluster
30+
kubectl config use-context "kind-${CLUSTER_NAME}" || true
31+
32+
echo "Kind cluster setup complete"

‎workspaces/backend/.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,5 +13,8 @@ Dockerfile.cross
1313
# Output of the go coverage tool, specifically when used with LiteIDE
1414
*.out
1515

16+
# Intermediate output from kustomize
17+
**/kustomize/.output/
18+
1619
# Go workspace file
1720
go.work

‎workspaces/backend/Makefile‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -145,8 +145,12 @@ $(GOLANGCI_LINT): $(LOCALBIN)
145145

146146
.PHONY: deploy
147147
deploy: kustomize ## Deploy backend to the K8s cluster specified in ~/.kube/config.
148-
cd manifests/kustomize/overlays/istio && $(KUSTOMIZE) edit set image workspaces-backend=${IMG}
149-
$(KUBECTL) apply -k manifests/kustomize/overlays/istio
148+
@echo "Copying kustomize directory structure to .output..."
149+
@rm -rf manifests/kustomize/.output
150+
@mkdir -p manifests/kustomize/.output
151+
@cp -r manifests/kustomize/* manifests/kustomize/.output/
152+
@cd manifests/kustomize/.output/overlays/istio && $(KUSTOMIZE) edit set image workspaces-backend=${IMG}
153+
@$(KUBECTL) apply -k manifests/kustomize/.output/overlays/istio
150154

151155
.PHONY: undeploy
152156
undeploy: kustomize ## Undeploy backend from the K8s cluster specified in ~/.kube/config.

0 commit comments

Comments
 (0)