diff --git a/Makefile b/Makefile index 9656c7e9d17..17e11e3aec0 100644 --- a/Makefile +++ b/Makefile @@ -512,6 +512,7 @@ guard-inotify: .PHONY: kind-integration-setup kind-integration-setup: guard-inotify .local/kind-kubeconfig + KUBECONFIG=$(CURDIR)/.local/kind-kubeconfig helmfile sync -f $(CURDIR)/hack/helmfile-federation-v0.yaml HELMFILE_ENV="kind" KUBECONFIG=$(CURDIR)/.local/kind-kubeconfig make kube-integration-setup .PHONY: kind-integration-test diff --git a/changelog.d/5-internal/v0-integration-setup b/changelog.d/5-internal/v0-integration-setup index a25f4d3a6c0..b38bc03fe24 100644 --- a/changelog.d/5-internal/v0-integration-setup +++ b/changelog.d/5-internal/v0-integration-setup @@ -1,3 +1,3 @@ Setup federation-v0 environment for use in integration tests: - add federation-v0 domain to test environment - - provision integration certificates with cert-manager + - provision integration certificates with cert-manager (#3849, #3898) diff --git a/hack/bin/integration-setup-federation.sh b/hack/bin/integration-setup-federation.sh index 95261e8cccc..b0abffc8184 100755 --- a/hack/bin/integration-setup-federation.sh +++ b/hack/bin/integration-setup-federation.sh @@ -44,7 +44,7 @@ export FEDERATION_DOMAIN_BASE_2="$NAMESPACE_2.svc.cluster.local" export FEDERATION_DOMAIN_2="federation-test-helper.$FEDERATION_DOMAIN_BASE_2" echo "Fetch federation-ca secret from cert-manager namespace" -FEDERATION_CA_CERTIFICATE=$(kubectl -n cert-manager get secrets federation-ca -o json -o jsonpath="{.data['tls\.crt']}") +FEDERATION_CA_CERTIFICATE=$(kubectl -n cert-manager get secrets federation-ca -o json -o jsonpath="{.data['tls\.crt']}" | base64 -d) export FEDERATION_CA_CERTIFICATE echo "Installing charts..." diff --git a/hack/helm_vars/common.yaml.gotmpl b/hack/helm_vars/common.yaml.gotmpl index 1e4b9b4d06d..4b296fb8fc4 100644 --- a/hack/helm_vars/common.yaml.gotmpl +++ b/hack/helm_vars/common.yaml.gotmpl @@ -4,7 +4,7 @@ federationDomainBase1: {{ requiredEnv "FEDERATION_DOMAIN_BASE_1" }} namespace2: {{ requiredEnv "NAMESPACE_2" }} federationDomain2: {{ requiredEnv "FEDERATION_DOMAIN_2" }} federationDomainBase2: {{ requiredEnv "FEDERATION_DOMAIN_BASE_2" }} -federationCACertificate: {{ requiredEnv "FEDERATION_CA_CERTIFICATE" }} +federationCACertificate: {{ requiredEnv "FEDERATION_CA_CERTIFICATE" | quote }} ingressChart: {{ requiredEnv "INGRESS_CHART" }} rabbitmqUsername: guest rabbitmqPassword: guest diff --git a/hack/helm_vars/nginx-ingress-services/values.yaml.gotmpl b/hack/helm_vars/nginx-ingress-services/values.yaml.gotmpl index 10ca09507ef..9cc214d779b 100644 --- a/hack/helm_vars/nginx-ingress-services/values.yaml.gotmpl +++ b/hack/helm_vars/nginx-ingress-services/values.yaml.gotmpl @@ -26,4 +26,4 @@ config: # certificateDomain: dynamically set by hack/helmfile.yaml secrets: - tlsClientCA: {{ .Values.federationCACertificate }} + tlsClientCA: {{ .Values.federationCACertificate | quote }} diff --git a/hack/helm_vars/wire-federation-v0/values.yaml.gotmpl b/hack/helm_vars/wire-federation-v0/values.yaml.gotmpl new file mode 100644 index 00000000000..c012a3b19f1 --- /dev/null +++ b/hack/helm_vars/wire-federation-v0/values.yaml.gotmpl @@ -0,0 +1,306 @@ +tags: + nginz: true + brig: true + galley: true + gundeck: true + cannon: true + cargohold: true + spar: true + federation: true # also see galley.config.enableFederation and brig.config.enableFederation + backoffice: true + proxy: false + webapp: false + team-settings: false + account-pages: false + legalhold: false + sftd: false + +cassandra-migrations: + cassandra: + host: cassandra-ephemeral + replicationFactor: 1 +elasticsearch-index: + elasticsearch: + host: elasticsearch-ephemeral + index: directory_test + cassandra: + host: cassandra-ephemeral + +brig: + replicaCount: 1 + resources: + requests: {} + limits: + memory: 512Mi + config: + externalUrls: + nginz: https://kube-staging-nginz-https.zinfra.io + teamCreatorWelcome: https://teams.wire.com/login + teamMemberWelcome: https://wire.com/download + cassandra: + host: cassandra-ephemeral + replicaCount: 1 + elasticsearch: + host: elasticsearch-ephemeral + index: directory_test + authSettings: + userTokenTimeout: 120 + sessionTokenTimeout: 20 + accessTokenTimeout: 30 + providerTokenTimeout: 60 + enableFederation: true # keep in sync with galley.config.enableFederation, cargohold.config.enableFederation and tags.federator! + optSettings: + setActivationTimeout: 10 + setVerificationTimeout: 10 + # keep this in sync with brigSettingsTeamInvitationTimeout in spar/templates/tests/configmap.yaml + setTeamInvitationTimeout: 10 + setExpiredUserCleanupTimeout: 1 + setUserMaxConnections: 16 + setCookieInsecure: true + setUserCookieRenewAge: 2 + setUserCookieLimit: 5 + setUserCookieThrottle: + stdDev: 5 + retryAfter: 5 + setLimitFailedLogins: + timeout: 5 # seconds. if you reach the limit, how long do you have to wait to try again. + retryLimit: 5 # how many times can you have a failed login in that timeframe. + setSuspendInactiveUsers: + suspendTimeout: 10 + setDefaultTemplateLocale: en + setDefaultUserLocale: en + setMaxConvAndTeamSize: 16 + setMaxTeamSize: 32 + setMaxConvSize: 16 + setFederationDomain: federation-test-helper.wire-federation-v0.svc.cluster.local + setFederationStrategy: allowAll + setFederationDomainConfigsUpdateFreq: 10 + set2FACodeGenerationDelaySecs: 5 + setNonceTtlSecs: 300 + setDpopMaxSkewSecs: 1 + setDpopTokenExpirationTimeSecs: 300 + setEnableMLS: true + setOAuthAuthCodeExpirationTimeSecs: 3 # 3 secs + setOAuthAccessTokenExpirationTimeSecs: 3 # 3 secs + setOAuthEnabled: true + setOAuthRefreshTokenExpirationTimeSecs: 14515200 # 24 weeks + setOAuthMaxActiveRefreshTokens: 10 + aws: + sesEndpoint: http://fake-aws-ses:4569 + sqsEndpoint: http://fake-aws-sqs:4568 + dynamoDBEndpoint: http://fake-aws-dynamodb:4567 + sesQueue: integration-brig-events + internalQueue: integration-brig-events-internal + prekeyTable: integration-brig-prekeys + emailSMS: + general: + emailSender: backend-integrationk8s@wire.com + smsSender: dummy + secrets: + # these secrets are only used during integration tests and should therefore be safe to include unencrypted in git. + # Normally these would live in a separately-encrypted secrets.yaml file and incorporated using the helm secrets plugin (wrapper around mozilla sops) + zAuth: + privateKeys: 7owt9MgvLd3D1nQ5s5Zm-5kOiUZcJ_iqASOYdzLUpjHRRbfyx7XJ6hzltU0S9_kvKsdYZmTK9wZNWKUraB4Z1Q== + publicKeys: 0UW38se1yeoc5bVNEvf5LyrHWGZkyvcGTVilK2geGdU= + turn: + secret: rPrUbws7PQZlfN2GG8Ggi7g5iOYPk7BiCoKHl3VoFZ + awsKeyId: dummykey + awsSecretKey: dummysecret + setTwilio: | + sid: "dummy" + token: "dummy" + setNexmo: |- + key: "dummy" + secret: "dummy" + smtpPassword: dummy-smtp-password + dpopSigKeyBundle: | + -----BEGIN PRIVATE KEY----- + MC4CAQAwBQYDK2VwBCIEIFANnxZLNE4p+GDzWzR3wm/v8x/0bxZYkCyke1aTRucX + -----END PRIVATE KEY----- + -----BEGIN PUBLIC KEY----- + MCowBQYDK2VwAyEACPvhIdimF20tOPjbb+fXJrwS2RKDp7686T90AZ0+Th8= + -----END PUBLIC KEY----- + oauthJwkKeyPair: | + { + "kty": "OKP", + "crv": "Ed25519", + "x": "mhP-NgFw3ifIXGZqJVB0kemt9L3BtD5P8q4Gah4Iklc", + "d": "R8-pV2-sPN7dykV8HFJ73S64F3kMHTNnJiSN8UdWk_o" + } + rabbitmq: + username: {{ .Values.rabbitmqUsername }} + password: {{ .Values.rabbitmqPassword }} + tests: + enableFederationTests: true +cannon: + replicaCount: 2 + resources: + requests: {} + limits: + memory: 512Mi + drainTimeout: 0 +cargohold: + replicaCount: 1 + resources: + requests: {} + limits: + memory: 512Mi + config: + aws: + s3Bucket: dummy-bucket + s3Endpoint: http://fake-aws-s3:9000 + enableFederation: true # keep in sync with brig.config.enableFederation, galley.config.enableFederation and tags.federator! + settings: + federationDomain: federation-test-helper.wire-federation-v0.svc.cluster.local + secrets: + awsKeyId: dummykey + awsSecretKey: dummysecret +galley: + replicaCount: 1 + config: + cassandra: + host: cassandra-ephemeral + replicaCount: 1 + enableFederation: true # keep in sync with brig.config.enableFederation, cargohold.config.enableFederation and tags.federator! + settings: + maxConvAndTeamSize: 16 + maxTeamSize: 32 + maxFanoutSize: 18 + maxConvSize: 16 + conversationCodeURI: https://kube-staging-nginz-https.zinfra.io/conversation-join/ + # See helmfile for the real value + federationDomain: federation-test-helper.wire-federation-v0.svc.cluster.local + featureFlags: + sso: disabled-by-default # this needs to be the default; tests can enable it when needed. + legalhold: whitelist-teams-and-implicit-consent + teamSearchVisibility: disabled-by-default + classifiedDomains: + status: enabled + config: + domains: ["example.com"] + journal: + endpoint: http://fake-aws-sqs:4568 + queueName: integration-team-events.fifo + secrets: + awsKeyId: dummykey + awsSecretKey: dummysecret + mlsPrivateKeys: + removal: + ed25519: | + -----BEGIN PRIVATE KEY----- + MC4CAQAwBQYDK2VwBCIEIAocCDXsKIAjb65gOUn5vEF0RIKnVJkKR4ebQzuZ709c + -----END PRIVATE KEY----- + rabbitmq: + username: {{ .Values.rabbitmqUsername }} + password: {{ .Values.rabbitmqPassword }} + +gundeck: + replicaCount: 1 + resources: + requests: {} + limits: + memory: 1024Mi + config: + cassandra: + host: cassandra-ephemeral + replicaCount: 1 + redis: + host: redis-ephemeral-master + connectionMode: master + aws: + account: "123456789012" + region: eu-west-1 + arnEnv: integration + queueName: integration-gundeck-events + sqsEndpoint: http://fake-aws-sqs:4568 + snsEndpoint: http://fake-aws-sns:4575 + bulkPush: true + setMaxConcurrentNativePushes: + hard: 30 + soft: 10 + secrets: + awsKeyId: dummykey + awsSecretKey: dummysecret +nginz: + replicaCount: 1 + nginx_conf: + env: staging + external_env_domain: zinfra.io + # NOTE: Web apps are disabled by default + allowlisted_origins: [] + randomport_allowlisted_origins: [] # default is empty by intention + rate_limit_reqs_per_user: "10r/s" + rate_limit_reqs_per_addr: "100r/s" + secrets: + basicAuth: "whatever" + zAuth: + # this must match the key in brig! + publicKeys: 0UW38se1yeoc5bVNEvf5LyrHWGZkyvcGTVilK2geGdU= + oAuth: + publicKeys: | + { + "kty": "OKP", + "crv": "Ed25519", + "x": "mhP-NgFw3ifIXGZqJVB0kemt9L3BtD5P8q4Gah4Iklc" + } +proxy: + replicaCount: 1 + secrets: + proxy_config: |- + secrets { + youtube = "..." + googlemaps = "..." + soundcloud = "..." + giphy = "..." + spotify = "Basic ..." + } +spar: + replicaCount: 1 + resources: + requests: {} + limits: + memory: 1024Mi + config: + tlsDisableCertValidation: true + cassandra: + host: cassandra-ephemeral + logLevel: Debug + domain: zinfra.io + appUri: http://spar:8080/ + ssoUri: http://spar:8080/sso + maxttlAuthreq: 5 + maxttlAuthresp: 7200 + maxScimTokens: 2 + contacts: + - type: ContactSupport + company: Example Company + email: email:backend+spar@wire.com + +federator: + replicaCount: 1 + resources: + requests: {} + config: + optSettings: + useSystemCAStore: false + remoteCAContents: {{ .Values.federationCACertificate | quote }} + tls: + useCertManager: true + useSharedFederatorSecret: true + +background-worker: + replicaCount: 1 + resources: + requests: {} + config: + backendNotificationPusher: + pushBackoffMinWait: 1000 # 1ms + pushBackoffMaxWait: 500000 # 0.5s + secrets: + rabbitmq: + username: {{ .Values.rabbitmqUsername }} + password: {{ .Values.rabbitmqPassword }} + +integration: + ingress: + class: "nginx-{{ .Release.Namespace }}" diff --git a/hack/helm_vars/wire-server/values.yaml.gotmpl b/hack/helm_vars/wire-server/values.yaml.gotmpl index 0d1ffba6b87..437159b7263 100644 --- a/hack/helm_vars/wire-server/values.yaml.gotmpl +++ b/hack/helm_vars/wire-server/values.yaml.gotmpl @@ -393,7 +393,7 @@ federator: resources: requests: {} imagePullPolicy: {{ .Values.imagePullPolicy }} - remoteCAContents: {{ .Values.federationCACertificate | b64dec | quote }} + remoteCAContents: {{ .Values.federationCACertificate | quote }} tls: useCertManager: true useSharedFederatorSecret: true diff --git a/hack/helmfile-federation-v0.yaml b/hack/helmfile-federation-v0.yaml new file mode 100644 index 00000000000..5400307d84b --- /dev/null +++ b/hack/helmfile-federation-v0.yaml @@ -0,0 +1,110 @@ +--- +helmDefaults: + wait: true + timeout: 600 + devel: true + createNamespace: true + +environments: + default: + values: + - federationCACertificate: {{ readFile "../services/nginz/integration-test/conf/nginz/integration-ca.pem" | quote }} + - rabbitmqUsername: guest + - rabbitmqPassword: guest +--- +repositories: + - name: jetstack + url: 'https://charts.jetstack.io' + + - name: bedag + url: 'https://bedag.github.io/helm-charts/' + + - name: wire + url: 'https://s3-eu-west-1.amazonaws.com/public.wire.com/charts-develop' + +releases: + - name: 'cert-manager' + namespace: cert-manager + chart: jetstack/cert-manager + set: + - name: installCRDs + value: true + + - name: 'federation-certs' + namespace: cert-manager + chart: bedag/raw + values: + - resources: + - apiVersion: v1 + kind: Secret + metadata: + name: federation-ca + namespace: cert-manager + data: + tls.crt: {{ readFile "../services/nginz/integration-test/conf/nginz/integration-ca.pem" | b64enc | quote }} + tls.key: {{ readFile "../services/nginz/integration-test/conf/nginz/integration-ca-key.pem" | b64enc | quote }} + - apiVersion: cert-manager.io/v1 + kind: ClusterIssuer + metadata: + name: federation + spec: + ca: + secretName: federation-ca + needs: + - 'cert-manager/cert-manager' + + - name: 'fake-aws' + namespace: wire-federation-v0 + chart: wire/fake-aws + version: 4.38.0-mandarin.14 + values: + - './helm_vars/fake-aws/values.yaml' + + - name: 'databases-ephemeral' + namespace: wire-federation-v0 + chart: 'wire/databases-ephemeral' + version: 4.38.0-mandarin.14 + + - name: 'rabbitmq' + namespace: wire-federation-v0 + chart: 'wire/rabbitmq' + version: 4.38.0-mandarin.14 + values: + - './helm_vars/rabbitmq/values.yaml.gotmpl' + + - name: 'ingress' + namespace: wire-federation-v0 + chart: 'wire/ingress-nginx-controller' + version: 4.38.0-mandarin.14 + values: + - './helm_vars/ingress-nginx-controller/values.yaml.gotmpl' + + - name: 'ingress-svc' + namespace: wire-federation-v0 + chart: 'wire/nginx-ingress-services' + version: 4.38.0-mandarin.14 + values: + - './helm_vars/nginx-ingress-services/values.yaml.gotmpl' + set: + # Federation domain is also the SRV record created by the + # federation-test-helper service. Maybe we can find a way to make these + # differ, so we don't make any silly assumptions in the code. + - name: config.dns.federator + value: wire-federation-v0.svc.cluster.local + - name: config.dns.certificateDomain + value: '*.wire-federation-v0.svc.cluster.local' + needs: + - 'ingress' + - 'cert-manager/cert-manager' + - 'cert-manager/federation-certs' + + - name: wire-server + namespace: wire-federation-v0 + chart: wire/wire-server + version: 4.38.0-mandarin.14 + values: + - './helm_vars/wire-federation-v0/values.yaml.gotmpl' + needs: + - 'cert-manager/cert-manager' + - 'cert-manager/federation-certs' + diff --git a/nix/overlay.nix b/nix/overlay.nix index 20aa6eb7995..a6390ab1d38 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -102,18 +102,5 @@ self: super: { inherit (super) stdenv fetchurl; }; - kind = staticBinary { - pname = "kind"; - version = "0.11.0"; - - darwinAmd64Url = "https://github.com/kubernetes-sigs/kind/releases/download/v0.11.1/kind-darwin-amd64"; - darwinAmd64Sha256 = "432bef555a70e9360b44661c759658265b9eaaf7f75f1beec4c4d1e6bbf97ce3"; - - linuxAmd64Url = "https://github.com/kubernetes-sigs/kind/releases/download/v0.11.1/kind-linux-amd64"; - linuxAmd64Sha256 = "949f81b3c30ca03a3d4effdecda04f100fa3edc07a28b19400f72ede7c5f0491"; - - inherit (super) stdenv fetchurl; - }; - rabbitmqadmin = super.callPackage ./pkgs/rabbitmqadmin { }; }