Skip to content

Commit

Permalink
refactor: rewrite acceptance tests using k8s e2e framework
Browse files Browse the repository at this point in the history
Signed-off-by: Mark Sagi-Kazar <[email protected]>
  • Loading branch information
sagikazarmark committed Jun 11, 2023
1 parent 535b600 commit fdbf50b
Show file tree
Hide file tree
Showing 19 changed files with 1,015 additions and 7 deletions.
2 changes: 2 additions & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
Dockerfile
/.devenv/
/.direnv/
/.github/
/bin/
build/
Expand Down
36 changes: 36 additions & 0 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,42 @@ jobs:
- name: Dependency Review
uses: actions/dependency-review-action@1360a344ccb0ab6e9475edef90ad2f46bf8003b1 # v3.0.6

e2e-test:
name: E2E test
runs-on: ubuntu-latest
needs: [artifacts]
strategy:
matrix:
k8s_version: ["v1.24.13", "v1.25.9", "v1.26.4", "v1.27.1" ]
# vault_version: ["1.10.11", "1.11.10", "1.12.6", "1.13.2"]

steps:
- name: Checkout repository
uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2

- name: Set up Nix
uses: cachix/install-nix-action@4b933aa7ebcc94a6174cf1364864e957b4910265 # v21
with:
extra_nix_config: |
access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
- name: Prepare Nix shell
run: nix develop --impure .#ci

- name: Download operator docker image
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
with:
name: "[container-image] Docker tarball"

- name: Test
run: nix develop --impure .#ci -c make test-e2e
env:
KIND_K8S_VERSION: ${{ matrix.k8s_version }}
LOAD_IMAGE_ARCHIVE: ${{ github.workspace }}/docker.tar
# VAULT_VERSION: ${{ matrix.vault_version }}
WEBHOOK_VERSION: ${{ needs.artifacts.outputs.container-image-tag }}
LOG_VERBOSE: "true"

acceptance-test:
name: Acceptance test
runs-on: ubuntu-latest
Expand Down
12 changes: 11 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

export PATH := $(abspath bin/):${PATH}

CONTAINER_IMAGE_REF = ghcr.io/bank-vaults/vault-secrets-webhook:dev

# Dependency versions
GOLANGCI_VERSION = 1.53.1
LICENSEI_VERSION = 0.8.0
Expand Down Expand Up @@ -52,7 +54,7 @@ artifacts: ## Build artifacts

.PHONY: container-image
container-image: ## Build container image
docker build .
docker build -t ${CONTAINER_IMAGE_REF} .

.PHONY: check
check: test lint ## Run checks (tests and linters)
Expand All @@ -65,6 +67,14 @@ test: ## Run tests
test-acceptance: ## Run acceptance tests
go test -race -v -timeout 900s -tags kubeall ./test

.PHONY: test-e2e
test-e2e: ## Run e2e tests
go test -race -v -timeout 900s -tags e2e ./e2e/

.PHONY: test-e2e-local
test-e2e-local: container-image ## Run e2e tests locally
LOAD_IMAGE=${CONTAINER_IMAGE_REF} WEBHOOK_VERSION=dev ${MAKE} test-e2e

.PHONY: lint
lint: ## Run linter
golangci-lint run ${LINT_ARGS}
Expand Down
3 changes: 0 additions & 3 deletions charts/vault-secrets-webhook/templates/_helpers.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,6 @@ Overrideable version for container image tags.
{{- define "vault-secrets-webhook.bank-vaults.version" -}}
{{- .Values.image.tag | default (printf "%s" .Chart.AppVersion) -}}
{{- end -}}
{{- define "vault-secrets-webhook.vault-env.version" -}}
{{- .Values.vaultEnv.tag | default (printf "%s" .Chart.AppVersion) -}}
{{- end -}}

{{/*
Create the name of the service account to use
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ spec:
value: "debug"
{{- end }}
- name: VAULT_ENV_IMAGE
value: "{{ .Values.vaultEnv.repository }}:{{ include "vault-secrets-webhook.vault-env.version" . }}"
value: "{{ .Values.vaultEnv.repository }}:{{ .Values.vaultEnv.tag }}"
{{- range $key, $value := .Values.env }}
- name: {{ $key }}
value: {{ $value | quote }}
Expand Down
4 changes: 2 additions & 2 deletions charts/vault-secrets-webhook/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,10 @@ webhookClientConfig:

vaultEnv:
repository: ghcr.io/bank-vaults/vault-env
# tag: ""
tag: "v1.20.1"

env:
VAULT_IMAGE: vault:1.6.2
VAULT_IMAGE: vault:1.13.2
# VAULT_CAPATH: /vault/tls
# # Used when the pod that should get secret injected does not
# # specify an imagePullSecret
Expand Down
12 changes: 12 additions & 0 deletions e2e/deploy/vault-secrets-webhook/values.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
replicaCount: 1

image:
pullPolicy: Never

configMapMutation: true
configmapFailurePolicy: "Fail"
podsFailurePolicy: "Fail"
secretsFailurePolicy: "Fail"

env:
VAULT_IMAGE: vault:1.13.2
46 changes: 46 additions & 0 deletions e2e/deploy/vault/rbac.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
kind: ServiceAccount
apiVersion: v1
metadata:
name: vault

---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: vault
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["*"]
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "update", "patch"]

---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: vault
roleRef:
kind: Role
name: vault
apiGroup: rbac.authorization.k8s.io
subjects:
- kind: ServiceAccount
name: vault

---
# This binding allows the deployed Vault instance to authenticate clients
# through Kubernetes ServiceAccounts (if configured so).
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: vault-auth-delegator
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:auth-delegator
subjects:
- kind: ServiceAccount
name: vault
namespace: default
172 changes: 172 additions & 0 deletions e2e/deploy/vault/vault.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
apiVersion: "vault.banzaicloud.com/v1alpha1"
kind: "Vault"
metadata:
name: "vault"
spec:
size: 1
image: vault:1.13.2

# Specify the ServiceAccount where the Vault Pod and the Bank-Vaults configurer/unsealer is running
serviceAccount: vault

# Specify the Service's type where the Vault Service is exposed
# Please note that some Ingress controllers like https://github.com/kubernetes/ingress-gce
# forces you to expose your Service on a NodePort
serviceType: ClusterIP

# Use local disk to store Vault file data, see config section.
volumes:
- name: vault-file
persistentVolumeClaim:
claimName: vault-file

volumeMounts:
- name: vault-file
mountPath: /vault/file

# Support for distributing the generated CA certificate Secret to other namespaces.
# Define a list of namespaces or use ["*"] for all namespaces.
caNamespaces:
- "vault-secrets-webhook"

# Describe where you would like to store the Vault unseal keys and root token.
unsealConfig:
options:
# The preFlightChecks flag enables unseal and root token storage tests
# This is true by default
preFlightChecks: true
# The storeRootToken flag enables storing of root token in chosen storage
# This is true by default
storeRootToken: true
kubernetes:
secretNamespace: default

# A YAML representation of a final vault config file.
# See https://www.vaultproject.io/docs/configuration/ for more information.
config:
storage:
file:
path: "${ .Env.VAULT_STORAGE_FILE }" # An example how Vault config environment interpolation can be used
listener:
tcp:
address: "0.0.0.0:8200"
# Uncommenting the following line and deleting tls_cert_file and tls_key_file disables TLS
# tls_disable: true
tls_cert_file: /vault/tls/server.crt
tls_key_file: /vault/tls/server.key
telemetry:
statsd_address: localhost:9125
ui: true

# See: https://banzaicloud.com/docs/bank-vaults/cli-tool/#example-external-vault-configuration
# The repository also contains a lot examples in the test/deploy and operator/deploy directories.
externalConfig:
policies:
- name: allow_secrets
rules: path "secret/*" {
capabilities = ["create", "read", "update", "delete", "list"]
}
- name: allow_pki
rules: path "pki/*" {
capabilities = ["create", "read", "update", "delete", "list"]
}

groups:
- name: admin1
policies:
- allow_secrets
metadata:
privileged: true
type: external
- name: admin2
policies:
- allow_secrets
metadata:
privileged: true
type: external

group-aliases:
- name: admin1
mountpath: token
group: admin1


auth:
- type: kubernetes
roles:
# Allow every pod in the default namespace to use the secret kv store
- name: default
bound_service_account_names: ["default", "vault-secrets-webhook", "vault"]
bound_service_account_namespaces: ["default", "vault-secrets-webhook"]
policies: ["allow_secrets", "allow_pki"]
ttl: 1h

secrets:
- path: secret
type: kv
description: General secrets.
options:
version: 2

- path: pki
type: pki
description: Vault PKI Backend
config:
default_lease_ttl: 168h
max_lease_ttl: 720h
configuration:
config:
- name: urls
issuing_certificates: https://vault.default:8200/v1/pki/ca
crl_distribution_points: https://vault.default:8200/v1/pki/crl
root/generate:
- name: internal
common_name: vault.default
roles:
- name: default
allowed_domains: localhost,pod,svc,default
allow_subdomains: true
generate_lease: true
ttl: 1m

# Allows writing some secrets to Vault (useful for development purposes).
# See https://www.vaultproject.io/docs/secrets/kv/index.html for more information.
startupSecrets:
- type: kv
path: secret/data/accounts/aws
data:
data:
AWS_ACCESS_KEY_ID: secretId
AWS_SECRET_ACCESS_KEY: s3cr3t
- type: kv
path: secret/data/dockerrepo
data:
data:
DOCKER_REPO_USER: dockerrepouser
DOCKER_REPO_PASSWORD: dockerrepopassword
- type: kv
path: secret/data/mysql
data:
data:
MYSQL_ROOT_PASSWORD: s3cr3t
MYSQL_PASSWORD: 3xtr3ms3cr3t

vaultEnvsConfig:
- name: VAULT_LOG_LEVEL
value: debug
- name: VAULT_STORAGE_FILE
value: "/vault/file"

---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: vault-file
spec:
# https://kubernetes.io/docs/concepts/storage/persistent-volumes/#class-1
# storageClassName: ""
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
2 changes: 2 additions & 0 deletions e2e/kind.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
Loading

0 comments on commit fdbf50b

Please sign in to comment.