Release_2021_10_29

.envrc

@@ -0,0 +1,5 @@
+env=$(nix-build --no-out-link "$PWD/direnv.nix")
+PATH_add "${env}/bin"
+
+# allow local .envrc overrides
+[[ -f .envrc.local ]] && source_env .envrc.local

.gitignore

@@ -33,7 +33,6 @@ TAGS
 .stack-docker-profile
 .metadata
 *.tix
-*.pem
 .DS_Store
 services/nginz/src
 services/.env
@@ -99,4 +98,4 @@ i.yaml
 b.yaml
 telepresence.log
 
-/.ghci
+/.ghci

CHANGELOG.md

@@ -1,5 +1,67 @@
 <!-- if you're not the release manager, do your edits to changelog under CHANGELOG.d/ -->
 
+# [2021-10-29]
+
+## Release notes
+
+* Upgrade SFT to 2.1.15 (#1849)
+* Upgrade team settings to Release: [v4.2.0](https://github.com/wireapp/wire-team-settings/releases/tag/v4.2.0) and image tag: 4.2.0-v0.28.28-1e2ef7 (#1856)
+* Upgrade Webapp to image tag: 20021-10-28-federation-m1 (#1856)
+
+## API changes
+
+* Remove `POST /list-conversations` endpoint. (#1840)
+* The member.self ID in conversation endpoints is qualified and available as
+  "qualified_id". The old unqualified "id" is still available. (#1866)
+
+## Features
+
+* Allow configuring nginz so it serve the deeplink for apps to discover the backend (#1889)
+* SFT: allow using TURN discovery using 'turnDiscoveryEnabled' (#1519)
+
+## Bug fixes and other updates
+
+* Fix an issue related to installing the SFT helm chart as a sub chart to the wire-server chart. (#1677)
+* SAML columns (Issuer, NameID) in CSV files with team members. (#1828)
+
+## Internal changes
+
+* Add a 'make flake-PATTERN' target to run a subset of tests multiple times to trigger a failure case in flaky tests (#1875)
+* Avoid a flaky test to fail related to phone updates and improve failure output. (#1874)
+* Brig: Delete deprecated `GET /i/users/connections-status` endpoint. (#1842)
+* Replace shell.nix with direnv + nixpkgs.buildEnv based setup (#1876)
+* Make connection DB functions work with Qualified IDs (#1819)
+* Fix more Swagger validation errors. (#1841)
+* Turn `Galley` into a polysemy monad stack. (#1881)
+* Internal CI tooling improvement: decrease integration setup time by using helmfile. (#1805)
+* Depend on hs-certificate master instead of our fork (#1822)
+* Add internal endpoint to insert or update a 1-1 conversation. This is to be used by brig when updating the status of a connection. (#1825)
+* Update helm to 3.6.3 in developer tooling (nix-shell) (#1862)
+* Improve the `Qualified` abstraction and make local/remote tagging safer (#1839)
+* Add some new Spar effects, completely isolating us from saml2-web-sso interface (#1827)
+* Convert legacy POST conversations/:cnv/members endpoint to Servant (#1838)
+* Simplify mock federator interface by removing unnecessary arguments. (#1870)
+* Replace the `Spar` newtype, instead using `Sem` directly. (#1833)
+
+## Federation changes
+
+* Remove remote guests as well as local ones when "Guests and services" is disabled in a group conversation, and propagate removal to remote members. (#1854)
+* Check connections when adding remote users to a local conversation and local users to remote conversations. (#1842)
+* Check connections when creating group and team conversations with remote members. (#1870)
+* Server certificates without the "serverAuth" extended usage flag are now rejected when connecting to a remote federator. (#1855)
+* Close GRPC client after making a request to a remote federator. (#1865)
+* Support deleting conversations with federated users (#1861)
+* Ensure that the conversation creator is included only once in notifications sent to remote users (#1879)
+* Allow connecting to remote users. One to one conversations are not created yet. (#1824)
+* Make federator's default log level Info (#1882)
+* The creator of a conversation now appears as a member when the conversation is fetched from a remote backend (#1842)
+* Include remote connections in the response to `POST /list-connections` (#1826)
+* When a user gets deleted, notify remotes about conversations and connections in chunks of 1000 (#1872, #1883)
+* Make federated requests to multiple backends in parallel. (#1860)
+* Make conversation ID of `RemoteConversation` unqualified and move it out of the metadata record. (#1839)
+* Make the conversation creator field in the `on-conversation-created` RPC unqualified. (#1858)
+* Update One2One conversation when connection status changes (#1850)
+
 # [2021-10-01]
 
 ## Release notes

Makefile

@@ -234,7 +234,7 @@ libzauth:
 .PHONY: hie.yaml
 hie.yaml: stack-dev.yaml
 	stack build implicit-hie
-	stack exec gen-hie | nix-shell --command 'yq "{cradle: {stack: {stackYaml: \"./stack-dev.yaml\", components: .cradle.stack}}}" > hie.yaml'
+	stack exec gen-hie | yq "{cradle: {stack: {stackYaml: \"./stack-dev.yaml\", components: .cradle.stack}}}" > hie.yaml
 
 .PHONY: stack-dev.yaml
 stack-dev.yaml:
@@ -311,7 +311,7 @@ release-chart-%:
 .PHONY: guard-tag
 guard-tag:
 	@if [ "${DOCKER_TAG}" = "${USER}" ]; then \
-	      echo "Environment variable DOCKER_TAG not set to non-default value. Re-run with DOCKER_TAG=<something>. Try using 'make latest-brig-tag' for latest develop docker image tag";\
+	      echo "Environment variable DOCKER_TAG not set to non-default value. Re-run with DOCKER_TAG=<something>. Try using 'make latest-tag' for latest develop docker image tag";\
 	    exit 1; \
 	fi
 
@@ -403,6 +403,7 @@ kind-reset: kind-delete kind-cluster
 .local/kind-kubeconfig:
 	mkdir -p $(CURDIR)/.local
 	kind get kubeconfig --name $(KIND_CLUSTER_NAME) > $(CURDIR)/.local/kind-kubeconfig
+	chmod 0600 $(CURDIR)/.local/kind-kubeconfig
 
 # This guard is a fail-early way to save needing to debug nginz container not
 # starting up in the second namespace of the kind cluster in some cases. Error
@@ -429,11 +430,11 @@ guard-inotify:
 
 .PHONY: kind-integration-setup
 kind-integration-setup: guard-inotify .local/kind-kubeconfig
-	ENABLE_KIND_VALUES="1" KUBECONFIG=$(CURDIR)/.local/kind-kubeconfig make kube-integration-setup
+	HELMFILE_ENV="kind" KUBECONFIG=$(CURDIR)/.local/kind-kubeconfig make kube-integration-setup
 
 .PHONY: kind-integration-test
 kind-integration-test: .local/kind-kubeconfig
-	ENABLE_KIND_VALUES="1" KUBECONFIG=$(CURDIR)/.local/kind-kubeconfig make kube-integration-test
+	HELMFILE_ENV="kind" KUBECONFIG=$(CURDIR)/.local/kind-kubeconfig make kube-integration-test
 
 kind-integration-e2e: .local/kind-kubeconfig
 	cd services/brig && KUBECONFIG=$(CURDIR)/.local/kind-kubeconfig ./federation-tests.sh $(NAMESPACE)

README.md

@@ -63,8 +63,8 @@ It also contains
 - **build**: Build scripts and Dockerfiles for some platforms
 - **deploy**: (Work-in-progress) - how to run wire-server in an ephemeral, in-memory demo mode
 - **doc**: Documentation
-- **hack**: scripts and configuration for kubernetes helm chart development/releases mainly used by CI
-- **charts**: kubernetes helm charts
+- **hack**: scripts and configuration for kuberentes helm chart development/releases mainly used by CI
+- **charts**: Kubernetes Helm charts. The charts are mirroed to S3 and can be used with `helm repo add wire https://s3-eu-west-1.amazonaws.com/public.wire.com/charts`. See the [Administrator's Guide](https://docs.wire.com) for more info.
 
 ## Architecture Overview
 

changelog.d/mk-changelog.sh

@@ -13,10 +13,14 @@ getPRNumber() {
 for d in "$DIR"/*; do
     if [[ ! -d "$d" ]]; then continue; fi
 
+    entries=("$d"/*[^~])
+
+    if [[ ${#entries[@]} -eq 0 ]]; then continue; fi
+
     echo -n "## "
     sed '$ a\' "$d/.title"
     echo ""
-    for f in "$d"/*[^~]; do
+    for f in "${entries[@]}"; do
         pr=$(getPRNumber $f)
         sed -r '
           # create a bullet point on the first line

charts/fake-aws-s3/values.yaml

@@ -7,6 +7,8 @@ minio:
     enabled: false
   environment:
     MINIO_BROWSER: "off"
+  defaultBucket:
+    name: dummy-bucket
   buckets:
     - name: dummy-bucket
       purge: true

charts/federator/values.yaml

@@ -23,7 +23,7 @@ resources:
     memory: "512Mi"
     cpu: "500m"
 config:
-  logLevel: Debug
+  logLevel: Info
   logFormat: JSON
   optSettings:
     # Defaults to using system CA store in the federator image for making

charts/nginz/templates/conf/_deeplink.html.tpl

@@ -0,0 +1,15 @@
+{{- define "nginz_deeplink.html" }}
+{{/* See https://docs.wire.com/how-to/associate/deeplink.html
+     (or search for "deeplink" on docs.wire.com)
+     for details on use of the deeplink*/}}
+<html>
+  <head></head>
+  <body>
+    {{- if hasKey .Values.nginx_conf "deeplink" }}
+    <a href="wire://access/?config={{ .Values.nginx_conf.deeplink.endpoints.backendURL }}/deeplink.json">Click here for access</a>
+    {{- else }}
+    No Deep Link.
+    {{- end }}
+  </body>
+</html>
+{{- end }}

charts/nginz/templates/conf/_deeplink.json.tpl

@@ -0,0 +1,24 @@
+{{- define "nginz_deeplink.json" }}
+{{- if hasKey .Values.nginx_conf "deeplink" }}
+{{- with .Values.nginx_conf.deeplink }}
+{{/* See https://docs.wire.com/how-to/associate/deeplink.html
+     (or search for "deeplink" on docs.wire.com)
+     for details on use of the deeplink*/}}
+{
+  "endpoints" : {
+      {{- with .endpoints }}
+      "backendURL" : {{ .backendURL | quote }},
+      "backendWSURL": {{ .backendWSURL | quote }},
+      "blackListURL": {{ .blackListURL | quote }},
+      "teamsURL": {{ .teamsURL | quote }},
+      "accountsURL": {{ .accountsURL | quote }},
+      "websiteURL": {{ .websiteURL | quote }}
+      {{- end }}
+   },
+   "title" : {{ .title | quote }}
+}
+{{- end }}
+{{- else }}
+{}
+{{- end }}
+{{- end }}

charts/nginz/templates/conf/_nginx.conf.tpl

@@ -344,6 +344,25 @@ http {
             image/png               png;
         }
     }
+
+    {{- if hasKey .Values.nginx_conf "deeplink" }}
+    location ~* ^/deeplink.(json|html)$ {
+        zauth off;
+        root /etc/wire/nginz/conf/;
+        types {
+            application/json  json;
+            text/html         html;
+        }
+        if ($request_method = 'OPTIONS') {
+                add_header 'Access-Control-Allow-Methods' "GET, OPTIONS";
+                add_header 'Access-Control-Allow-Headers' "$http_access_control_request_headers, DNT,X-Mx-ReqToken,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type";
+                add_header 'Content-Type' 'text/plain; charset=UTF-8';
+                add_header 'Content-Length' 0;
+                return 204;
+        }
+        more_set_headers 'Access-Control-Allow-Origin: $http_origin';
+    }
+    {{- end }}
   }
 }
 {{- end }}

charts/nginz/templates/configmap.yaml

@@ -6,6 +6,10 @@ data:
 {{- include "nginz_upstreams.txt" . | indent 4 }}
   zwagger-config.js: |2
 {{- include "nginz_zwagger-config.js" . | indent 4 }}
+  deeplink.json: |2
+{{- include "nginz_deeplink.json" . | indent 4 }}
+  deeplink.html: |2
+{{- include "nginz_deeplink.html" . | indent 4 }}
 {{ (.Files.Glob "conf/static/*").AsConfig | indent 2 }}
 kind: ConfigMap
 metadata:

charts/nginz/values.yaml

@@ -32,6 +32,15 @@ nginx_conf:
   worker_rlimit_nofile: 131072
   worker_connections: 65536
   swagger_root: /var/www/swagger
+  # deeplink:
+  #   endpoints:
+  #     backendURL: "https://prod-nginz-https.wire.com"
+  #     backendWSURL: "https://prod-nginz-ssl.wire.com"
+  #     blackListURL: "https://clientblacklist.wire.com/prod"
+  #     teamsURL: "https://teams.wire.com"
+  #     accountsURL: "https://accounts.wire.com"
+  #     websiteURL: "https://wire.com"
+  #   title: "Production"
   disabled_paths:
   - /conversations/last-events
   - ~* ^/conversations/([^/]*)/knock
@@ -304,9 +313,6 @@ nginx_conf:
       envs:
       - all
       doc: true
-    - path: ~* ^/list-conversations$
-      envs:
-      - all
     - path: ~* ^/teams$
       envs:
       - all

charts/sftd/Chart.yaml

@@ -11,4 +11,4 @@ version: 0.0.42
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
-appVersion: 2.0.127
+appVersion: 2.1.15

charts/sftd/README.md

@@ -111,6 +111,32 @@ able to reach the restund servers on their public IPs.
 More exotic setups _are_ possible but are currently *not* officially supported. Please
 contact us if you have different constraints.
 
+### No public IP on default interface
+
+Often on-prem or at certain cloud providers your nodes will not have directly routable public IP addresses
+but are deployed in 1:1 NAT.   This chart is able to auto-detect this scenario if your cloud providers adds
+an `ExternalIP` field to your kubernetes node objects.
+
+On on-prem you should set an `wire.com/external-ip` annotation on your kubernetes nodes so that sftd is aware
+of its external IP when it gets scheduled on a node.
+
+If you use our kubespray playbooks to bootstrap kubernetes, you simply have to
+set the `external_ip` field in your `group_vars`
+```yaml
+# inventory/group_vars/k8s-cluster
+node_annotations:
+  wire.com/external-ip: {{ external_ip }}
+```
+And the `external_ip` is set in the inventory per node:
+```
+node0 ansible_host=.... ip=...  external_ip=aaa.xxx.yyy.zzz
+```
+
+If you are hosting Kubernetes through other means you can annotate your nodes manually:
+```
+$ kubectl annotate node $HOSTNAME wire.com/external-ip=$EXTERNAL_IP
+```
+
 ## Rollout
 
 Kubernetes will shut down pods and start new ones when rolling out a release. Any calls
@@ -193,31 +219,6 @@ helm install wire-prod charts/wire-server --set 'nodeSelector.wire\.com/role=sft
 helm install wire-staging charts/wire-server --set 'nodeSelector.wire\.com/role=sftd-staging' ...other-flags
 ```
 
-## No public IP on default interface
-
-Often on-prem or at certain cloud providers your nodes will not have directly routable public IP addresses
-but are deployed in 1:1 NAT.   This chart is able to auto-detect this scenario if your cloud providers adds
-an `ExternalIP` field to your kubernetes node objects.
-
-On on-prem you should set an `wire.com/external-ip` annotation on your kubernetes nodes so that sftd is aware
-of its external IP when it gets scheduled on a node.
-
-If you use our kubespray playbooks to bootstrap kubernetes, you simply have to
-set the `external_ip` field in your `group_vars`
-```yaml
-# inventory/group_vars/k8s-cluster
-node_annotations:
-  wire.com/external-ip: {{ external_ip }}
-```
-And the `external_ip` is set in the inventory per node:
-```
-node0 ansible_host=.... ip=...  external_ip=aaa.xxx.yyy.zzz
-```
-
-If you are hosting Kubernetes through other means you can annotate your nodes manually:
-```
-$ kubectl annotate node $HOSTNAME wire.com/external-ip=$EXTERNAL_IP
-```
 
 ## Port conflicts and `hostNetwork`
 

charts/sftd/templates/configmap-join-call.yaml

@@ -14,7 +14,7 @@ data:
       location /healthz { return 204; }
 
       location ~ ^/sfts/([a-z0-9\-]+)/(.*) {
-        proxy_pass http://$1.sftd.${POD_NAMESPACE}.svc.cluster.local:8585/$2;
+        proxy_pass http://$1.{{ include "sftd.fullname" . }}.${POD_NAMESPACE}.svc.cluster.local:8585/$2;
       }
 
     }

charts/sftd/templates/statefulset.yaml

@@ -83,7 +83,12 @@ spec:
               else
                 ACCESS_ARGS="-A ${EXTERNAL_IP}"
               fi
-              exec sftd  -I "${POD_IP}" -M "${POD_IP}" ${ACCESS_ARGS} -u "https://{{ required "must specify host" .Values.host }}/sfts/${POD_NAME}"
+              exec sftd \
+                      -I "${POD_IP}" \
+                      -M "${POD_IP}" \
+                      ${ACCESS_ARGS} \
+                      {{ if .Values.turnDiscoveryEnabled }}-T{{ end }} \
+                      -u "https://{{ required "must specify host" .Values.host }}/sfts/${POD_NAME}"
           ports:
             - name: sft
               containerPort: 8585

charts/sftd/values.yaml

@@ -81,3 +81,7 @@ joinCall:
     # Overrides the image tag whose default is the chart appVersion.
     tag: "1.19.5"
 
+# Allow SFT instances to choose/consider using a TURN server for themselves as a proxy when
+# trying to establish a connection to clients
+# DOCS: https://docs.wire.com/understand/sft.html#prerequisites
+turnDiscoveryEnabled: false

charts/team-settings/values.yaml

@@ -9,7 +9,7 @@ resources:
     cpu: "1"
 image:
   repository: quay.io/wire/team-settings
-  tag: "4.0.0-v0.28.21-b92fca-2"
+  tag: "4.2.0-v0.28.28-1e2ef7"
 service:
   https:
     externalPort: 443

charts/webapp/values.yaml

@@ -9,7 +9,7 @@ resources:
     cpu: "1"
 image:
   repository: quay.io/wire/webapp
-  tag: 2021-09-06-staging.3-v0.28.24-e6e306b
+  tag: "2021-10-28-federation-M1"
 service:
   https:
     externalPort: 443