Federation-v0 setup in integration tests

changelog.d/5-internal/v0-integration-setup

@@ -0,0 +1,3 @@
+Setup federation-v0 environment for use in integration tests:
+ - add federation-v0 domain to test environment
+ - provision integration certificates with cert-manager

charts/federator/templates/tests/federator-integration.yaml

@@ -16,7 +16,7 @@ spec:
     # integration tests need access to the client certificate private key
     - name: "federator-secrets"
       secret:
-        secretName: "federator-secret"
+        secretName: {{ if .Values.tls.useCertManager }} "federator-certificate-secret" {{ else }} "federator-secret" {{ end }}
     # integration tests need access to the CA
     - name: "federator-ca"
       configMap:

charts/integration/templates/configmap.yaml

@@ -125,3 +125,42 @@ data:
       {{- if eq (include "useCassandraTLS" .Values.config) "true" }}
       tlsCa: /etc/wire/galley/cassandra/{{- (include "tlsSecretRef" .Values.config | fromYaml).key }}
       {{- end }}
+
+    federation-v0:
+      originDomain: federation-test-helper.wire-federation-v0.svc.cluster.local
+      brig:
+        host: brig.wire-federation-v0.svc.cluster.local
+        port: 8080
+      cannon:
+        host: cannon.wire-federation-v0.svc.cluster.local
+        port: 8080
+      cargohold:
+        host: cargohold.wire-federation-v0.svc.cluster.local
+        port: 8080
+      federatorInternal:
+        host: federator.wire-federation-v0.svc.cluster.local
+        port: 8080
+      federatorExternal:
+        host: federator.wire-federation-v0.svc.cluster.local
+        port: 8081
+      galley:
+        host: galley.wire-federation-v0.svc.cluster.local
+        port: 8080
+      gundeck:
+        host: gundeck.wire-federation-v0.svc.cluster.local
+        port: 8080
+      nginz:
+        host: nginz-integration-http.wire-federation-v0.svc.cluster.local
+        port: 8080
+      spar:
+        host: spar.wire-federation-v0.svc.cluster.local
+        port: 8080
+      proxy:
+        host: proxy.wire-federation-v0.svc.cluster.local
+        port: 8080
+      backgroundWorker:
+        host: backgroundWorker.wire-federation-v0.svc.cluster.local
+        port: 8080
+      stern:
+        host: stern.wire-federation-v0.svc.cluster.local
+        port: 8080

charts/integration/templates/ingress.yaml

@@ -17,7 +17,7 @@ metadata:
     nginx.ingress.kubernetes.io/backend-protocol: "HTTP"
     nginx.ingress.kubernetes.io/auth-tls-verify-client: "on"
     nginx.ingress.kubernetes.io/auth-tls-verify-depth: "{{ $.Values.tls.verify_depth }}"
-    nginx.ingress.kubernetes.io/auth-tls-secret: "{{ $.Release.Namespace }}/federator-ca-secret"
+    nginx.ingress.kubernetes.io/auth-tls-secret: "{{ or $.Values.tls.caNamespace $.Release.Namespace }}/federator-ca-secret"
     nginx.ingress.kubernetes.io/configuration-snippet: |
       proxy_set_header "X-SSL-Certificate" $ssl_client_escaped_cert;
 spec:

charts/nginx-ingress-services/templates/certificate_federator.yaml

@@ -31,5 +31,5 @@ spec:
     encoding: PKCS1
     rotationPolicy: Always
   dnsNames:
-    - {{ .Values.config.dns.federator }}
+    - "{{ or .Values.config.dns.certificateDomain .Values.config.dns.federator }}"
 {{- end -}}

charts/nginx-ingress-services/templates/ingress_federator.yaml

@@ -19,7 +19,7 @@ metadata:
     nginx.ingress.kubernetes.io/backend-protocol: "HTTP"
     nginx.ingress.kubernetes.io/auth-tls-verify-client: "on"
     nginx.ingress.kubernetes.io/auth-tls-verify-depth: "{{ .Values.tls.verify_depth }}"
-    nginx.ingress.kubernetes.io/auth-tls-secret: "{{ .Release.Namespace }}/federator-ca-secret"
+    nginx.ingress.kubernetes.io/auth-tls-secret: "{{ or $.Values.tls.caNamespace $.Release.Namespace }}/federator-ca-secret"
     nginx.ingress.kubernetes.io/configuration-snippet: |
       proxy_set_header "X-SSL-Certificate" $ssl_client_escaped_cert;
 spec:

charts/nginx-ingress-services/values.yaml

@@ -118,6 +118,8 @@ config:
 #     ^ fakeS3 is ignored if fakeS3.enabled == false
 #     federator: federator.<domain>
 #     ^ federator is ignored unless federator.enabled == true
+#     certificateDomain: federator.<domain>
+#     ^ domain to use in the CSR when using cert-manager
 #     teamSettings: teams.<domain>
 #     ^ teamSettings is ignored unless teamSettings.enabled == true
 #     accountPages: account.<domain>

deploy/dockerephemeral/coredns-config/db.example.com

@@ -17,4 +17,4 @@ _wire-server-federator._tcp.b  IN SRV 0 0 9443 localhost.
 _wire-server-federator._tcp.d1  IN SRV 0 0 10443 localhost.
 _wire-server-federator._tcp.d2  IN SRV 0 0 11443 localhost.
 _wire-server-federator._tcp.d3  IN SRV 0 0 12443 localhost.
-_wire-server-federator._tcp.v0  IN SRV 0 0 21443 localhost.
+_wire-server-federator._tcp.federation-v0  IN SRV 0 0 21443 localhost.

hack/bin/integration-setup-federation.sh

@@ -25,9 +25,6 @@ charts=(fake-aws databases-ephemeral redis-cluster rabbitmq wire-server ingress-
 mkdir -p ~/.parallel && touch ~/.parallel/will-cite
 printf '%s\n' "${charts[@]}" | parallel -P "${HELM_PARALLELISM}" "$DIR/update.sh" "$CHARTS_DIR/{}"
 
-# FUTUREWORK: use helm functions instead, see https://wearezeta.atlassian.net/browse/SQPIT-723
-echo "Generating self-signed certificates..."
-
 KUBERNETES_VERSION_MAJOR="$(kubectl version -o json | jq -r .serverVersion.major)"
 KUBERNETES_VERSION_MINOR="$(kubectl version -o json | jq -r .serverVersion.minor)"
 KUBERNETES_VERSION_MINOR="${KUBERNETES_VERSION_MINOR//[!0-9]/}" # some clusters report minor versions as a string like '27+'. Strip any non-digit characters.
@@ -39,14 +36,16 @@ else
 fi
 echo "kubeVersion: $KUBERNETES_VERSION and ingress controller=$INGRESS_CHART"
 export NAMESPACE_1="$NAMESPACE"
-export FEDERATION_DOMAIN_BASE="$NAMESPACE_1.svc.cluster.local"
-export FEDERATION_DOMAIN_1="federation-test-helper.$FEDERATION_DOMAIN_BASE"
-"$DIR/selfsigned-kubernetes.sh" namespace1
+export FEDERATION_DOMAIN_BASE_1="$NAMESPACE_1.svc.cluster.local"
+export FEDERATION_DOMAIN_1="federation-test-helper.$FEDERATION_DOMAIN_BASE_1"
 
 export NAMESPACE_2="$NAMESPACE-fed2"
-export FEDERATION_DOMAIN_BASE="$NAMESPACE_2.svc.cluster.local"
-export FEDERATION_DOMAIN_2="federation-test-helper.$FEDERATION_DOMAIN_BASE"
-"$DIR/selfsigned-kubernetes.sh" namespace2
+export FEDERATION_DOMAIN_BASE_2="$NAMESPACE_2.svc.cluster.local"
+export FEDERATION_DOMAIN_2="federation-test-helper.$FEDERATION_DOMAIN_BASE_2"
+
+echo "Fetch federation-ca secret from cert-manager namespace"
+FEDERATION_CA_CERTIFICATE="$(kubectl -n cert-manager get secrets federation-ca -o json | jq -r '.data."tls.crt"')"
+export FEDERATION_CA_CERTIFICATE
 
 echo "Installing charts..."
 

hack/helm_vars/common.yaml.gotmpl

@@ -1,7 +1,10 @@
 namespace1: {{ requiredEnv "NAMESPACE_1" }}
 federationDomain1: {{ requiredEnv "FEDERATION_DOMAIN_1" }}
+federationDomainBase1: {{ requiredEnv "FEDERATION_DOMAIN_BASE_1" }}
 namespace2: {{ requiredEnv "NAMESPACE_2" }}
 federationDomain2: {{ requiredEnv "FEDERATION_DOMAIN_2" }}
+federationDomainBase2: {{ requiredEnv "FEDERATION_DOMAIN_BASE_2" }}
+federationCACertificate: {{ requiredEnv "FEDERATION_CA_CERTIFICATE" }}
 ingressChart: {{ requiredEnv "INGRESS_CHART" }}
 rabbitmqUsername: guest
 rabbitmqPassword: guest

hack/helm_vars/nginx-ingress-services/values.yaml.gotmpl

@@ -6,7 +6,12 @@ federator:
   enabled: true
   integrationTestHelper: true
 tls:
-  useCertManager: false
+  useCertManager: true
+  issuer:
+    name: federation
+    kind: ClusterIssuer
+  createIssuer: false
+  caNamespace: wire-federation-v0
 
 config:
   ingressClass: "nginx-{{ .Release.Namespace }}"
@@ -18,6 +23,7 @@ config:
     teamSettings: "teams.{{ .Release.Namespace }}-integration.example.com"
     accountPages: "account.{{ .Release.Namespace }}-integration.example.com"
     # federator: dynamically set by hack/helmfile.yaml
+    # certificateDomain: dynamically set by hack/helmfile.yaml
 
-# secrets/tlsWildcardCert, secrets/tlsWildcardKey and secrets/tlsClientCA
-# are dynamically generated by hack/bin/selfsigned-kubernetes.sh
+secrets:
+  tlsClientCA: {{ .Values.federationCACertificate }}

hack/helm_vars/wire-server/values.yaml.gotmpl

@@ -393,6 +393,11 @@ federator:
   resources:
     requests: {}
   imagePullPolicy: {{ .Values.imagePullPolicy }}
+  remoteCAContents: {{ .Values.federationCACertificate | b64dec | quote }}
+  tls:
+    useCertManager: true
+    useSharedFederatorSecret: true
+
   config:
     optSettings:
       useSystemCAStore: false
@@ -441,6 +446,9 @@ integration:
     uploadXmlAwsAccessKeyId: {{ .Values.uploadXml.awsAccessKeyId }}
     uploadXmlAwsSecretAccessKey: {{ .Values.uploadXml.awsSecretAccessKey }}
   {{- end }}
+  tls:
+    caNamespace: wire-federation-v0
+
 backoffice:
   tests:
     {{- if .Values.uploadXml }}

hack/helmfile.yaml

@@ -118,13 +118,14 @@ releases:
     chart: '../.local/charts/nginx-ingress-services'
     values:
       - './helm_vars/nginx-ingress-services/values.yaml.gotmpl'
-      - './helm_vars/nginx-ingress-services/certificates-namespace1.yaml'
     set:
       # Federation domain is also the SRV record created by the
       # federation-test-helper service. Maybe we can find a way to make these
       # differ, so we don't make any silly assumptions in the code.
       - name: config.dns.federator
         value: '{{ .Values.federationDomain1 }}'
+      - name: config.dns.certificateDomain
+        value: '*.{{ .Values.federationDomainBase1 }}'
     needs:
       - 'ingress'
 
@@ -133,13 +134,14 @@ releases:
     chart: '../.local/charts/nginx-ingress-services'
     values:
       - './helm_vars/nginx-ingress-services/values.yaml.gotmpl'
-      - './helm_vars/nginx-ingress-services/certificates-namespace2.yaml'
     set:
       # Federation domain is also the SRV record created by the
       # federation-test-helper service. Maybe we can find a way to make these
       # differ, so we don't make any silly assumptions in the code.
       - name: config.dns.federator
         value: '{{ .Values.federationDomain2 }}'
+      - name: config.dns.certificateDomain
+        value: '*.{{ .Values.federationDomainBase2 }}'
     needs:
       - 'ingress'
 
@@ -153,7 +155,6 @@ releases:
     chart: '../.local/charts/wire-server'
     values:
       - './helm_vars/wire-server/values.yaml.gotmpl'
-      - './helm_vars/wire-server/certificates-namespace1.yaml'
     set:
       - name: brig.config.optSettings.setFederationDomain
         value: {{ .Values.federationDomain1 }}
@@ -169,7 +170,6 @@ releases:
     chart: '../.local/charts/wire-server'
     values:
       - './helm_vars/wire-server/values.yaml.gotmpl'
-      - './helm_vars/wire-server/certificates-namespace2.yaml'
     set:
       - name: brig.config.optSettings.setFederationDomain
         value: {{ .Values.federationDomain2 }}

integration/test/Test/Demo.hs

@@ -194,3 +194,16 @@ testUnrace = do
     True `shouldMatch` False
   -}
   retryT $ True `shouldMatch` True
+
+testFedV0Instance :: HasCallStack => App ()
+testFedV0Instance = do
+  res <- BrigP.getAPIVersion FedV0Domain >>= getJSON 200
+  res %. "domain" `shouldMatch` FedV0Domain
+
+testFedV0Federation :: HasCallStack => App ()
+testFedV0Federation = do
+  alice <- randomUser OwnDomain def
+  bob <- randomUser FedV0Domain def
+
+  bob' <- BrigP.getUser alice bob >>= getJSON 200
+  bob' %. "qualified_id" `shouldMatch` (bob %. "qualified_id")

integration/test/Testlib/App.hs

@@ -57,6 +57,11 @@ instance MakesValue Domain where
   make OwnDomain = asks (String . T.pack . (.domain1))
   make OtherDomain = asks (String . T.pack . (.domain2))
 
+data FedDomain = FedV0Domain
+
+instance MakesValue FedDomain where
+  make FedV0Domain = asks (String . T.pack . (.federationV0Domain))
+
 -- | Run an action, `recoverAll`ing with exponential backoff (min step 8ms, total timeout
 -- ~15s).  Search this package for examples how to use it.
 --

integration/test/Testlib/Env.hs

@@ -86,7 +86,8 @@ mkGlobalEnv cfgFile = do
   let sm =
         Map.fromList $
           [ (intConfig.backendOne.originDomain, intConfig.backendOne.beServiceMap),
-            (intConfig.backendTwo.originDomain, intConfig.backendTwo.beServiceMap)
+            (intConfig.backendTwo.originDomain, intConfig.backendTwo.beServiceMap),
+            (intConfig.federationV0.originDomain, intConfig.federationV0.beServiceMap)
           ]
             <> [(berDomain resource, resourceServiceMap resource) | resource <- resources]
   tempDir <- Codensity $ withSystemTempDirectory "test"
@@ -98,6 +99,7 @@ mkGlobalEnv cfgFile = do
       { gServiceMap = sm,
         gDomain1 = intConfig.backendOne.originDomain,
         gDomain2 = intConfig.backendTwo.originDomain,
+        gFederationV0Domain = intConfig.federationV0.originDomain,
         gDynamicDomains = (.domain) <$> Map.elems intConfig.dynamicBackends,
         gDefaultAPIVersion = 6,
         gManager = manager,
@@ -135,6 +137,7 @@ mkEnv ge = do
         { serviceMap = gServiceMap ge,
           domain1 = gDomain1 ge,
           domain2 = gDomain2 ge,
+          federationV0Domain = gFederationV0Domain ge,
           dynamicDomains = gDynamicDomains ge,
           defaultAPIVersion = gDefaultAPIVersion ge,
           manager = gManager ge,