Release 2021-12-10

CHANGELOG.md

@@ -1,5 +1,55 @@
 <!-- if you're not the release manager, do your edits to changelog under CHANGELOG.d/ -->
 
+# [2021-12-10]
+
+## Release notes
+
+* If you have `selfDeletingMessages` configured in `galley.yaml`, add `lockStatus: unlocked`. (#1963)
+* Upgrade SFTD to 2.1.19. (#1983)
+
+## API changes
+
+* A new endpoint is added to Brig (`put /users/:uid/email`) that allows a team owner to initiate changing/setting a user email by (re-)sending an activation email. (#1948)
+* get team feature config for self deleting messages response includes lock status (#1963)
+* A new public Galley endpoint was added to dis-/enable the conversation guest link feature. The feature can only be configured through the public API if the lock status is unlocked in the server config. (#1964)
+* new internal endpoints for setting the lock status of self deleting messages (#1963)
+
+## Features
+
+* Team and server wide config for conversation guest link feature to configure feature status and lock status (#1964). If the feature is not configured on the server, the defaults will be:
+
+  ```txt
+    featureFlags:
+      ...
+      conversationGuestLinks:
+        defaults:
+          status: enabled
+          lockStatus: unlocked
+  ```
+* Lock status for the self deleting messages feature can be set internally by ibis and customer support (#1963)
+
+## Bug fixes and other updates
+
+* Correctly detect log level when rendering logs as structured JSON (#1959)
+
+## Documentation
+
+* Fix typo in swagger. (#1982)
+* Proposal for API versioning system. (#1958)
+* Update federation error documentation after changes to the federation API (#1956, #1975, #1978)
+
+## Internal changes
+
+* Suspend/unsuspend teams in backoffice/stern. (#1977)
+* Set request ID correctly in galley logs (#1967)
+* Improve cabal make targets: faster installation and better support for building and testing all packages (#1979)
+* sftd chart: add config key `additionalArgs` (#1972)
+
+## Federation changes
+
+* Add cargohold as a new federated component (#1973)
+
+
 # [2021-12-02]
 
 ## Release notes

Makefile

@@ -19,6 +19,9 @@ BUILDAH_PUSH          ?= 0
 KIND_CLUSTER_NAME     := wire-server
 BUILDAH_KIND_LOAD     ?= 1
 
+package ?= all
+EXE_SCHEMA := ./dist/$(package)-schema
+
 # This ensures that focused unit tests written in hspec fail. This is supposed
 # to help us avoid merging PRs with focused tests. This will not catch focused
 # integration tests as they are run in kubernetes where this Makefile doesn't
@@ -42,7 +45,7 @@ install: init
 ifeq ($(WIRE_BUILD_WITH_CABAL), 1)
 	cabal build all
 	./hack/bin/cabal-run-all-tests.sh
-	./hack/bin/cabal-install-all-artefacts.sh
+	./hack/bin/cabal-install-artefacts.sh all
 else
 	stack install --pedantic --test --bench --no-run-benchmarks --local-bin-path=dist
 endif
@@ -69,11 +72,19 @@ endif
 # Usage: make ci package=brig test=1
 .PHONY: ci
 ci: c
-ifeq ("$(pattern)", "")
-	make -C services/$(package) i
-else
-	make -C services/$(package) i-$(pattern)
-endif
+	./hack/bin/cabal-run-integration.sh $(package) $(pattern)
+
+# reset db using cabal
+.PHONY: db-reset-package
+db-reset-package: c
+	$(EXE_SCHEMA) --keyspace $(package)_test --replication-factor 1 --reset
+
+# migrate db using cabal
+# For using stack see the Makefile of the package, e.g. services/brig/Makefile
+# Usage: make db-migrate-package package=galley
+.PHONY: db-migrate-package
+db-migrate-package: c
+	$(EXE_SCHEMA) --keyspace $(package)_test --replication-factor 1
 
 # Build everything (Haskell services and nginz)
 .PHONY: services
@@ -230,12 +241,18 @@ run-docker-builder:
 	@echo "if this does not work, consider 'docker pull', 'docker tag', or 'make -C build-alpine builder'."
 	docker run --workdir /wire-server -it $(DOCKER_DEV_NETWORK) $(DOCKER_DEV_VOLUMES) --rm $(DOCKER_DEV_IMAGE) /bin/bash
 
-CASSANDRA_CONTAINER := $(shell docker ps | grep '/cassandra:' | perl -ne '/^(\S+)\s/ && print $$1')
 .PHONY: git-add-cassandra-schema
-git-add-cassandra-schema: db-reset
+git-add-cassandra-schema: db-reset git-add-cassandra-schema-impl
+
+CASSANDRA_CONTAINER := $(shell docker ps | grep '/cassandra:' | perl -ne '/^(\S+)\s/ && print $$1')
+.PHONY: git-add-cassandra-schema-impl
+git-add-cassandra-schema-impl:
 	( echo '-- automatically generated with `make git-add-cassandra-schema`' ; docker exec -i $(CASSANDRA_CONTAINER) /usr/bin/cqlsh -e "DESCRIBE schema;" ) > ./docs/reference/cassandra-schema.cql
 	git add ./docs/reference/cassandra-schema.cql
 
+.PHONY: git-add-cassandra-schema-cabal
+git-add-cassandra-schema-cabal: db-reset-cabal git-add-cassandra-schema-impl
+
 .PHONY: cqlsh
 cqlsh:
 	@echo "make sure you have ./deploy/dockerephemeral/run.sh running in another window!"
@@ -244,10 +261,17 @@ cqlsh:
 .PHONY: db-reset
 db-reset:
 	@echo "make sure you have ./deploy/dockerephemeral/run.sh running in another window!"
+ifeq ($(WIRE_BUILD_WITH_CABAL), 1)
+	make db-reset-package package=brig
+	make db-reset-package package=galley
+	make db-reset-package package=gundeck
+	make db-reset-package package=spar
+else
 	make -C services/brig db-reset
 	make -C services/galley db-reset
 	make -C services/gundeck db-reset
 	make -C services/spar db-reset
+endif
 
 #################################
 ## dependencies

charts/federator/templates/configmap.yaml

@@ -31,6 +31,10 @@ data:
       host: galley
       port: 8080
 
+    cargohold:
+      host: cargohold
+      port: 8080
+
     {{- with .Values.config }}
 
     logNetStrings: True # log using netstrings encoding:

charts/federator/templates/tests/configmap.yaml

@@ -16,6 +16,9 @@ data:
     galley:
       host: galley
       port: 8080
+    cargohold:
+      host: cargohold
+      port: 8080
     nginxIngress:
       host: federation-test-helper.{{ .Release.Namespace }}.svc.cluster.local
       port: 443

charts/sftd/Chart.yaml

@@ -11,4 +11,4 @@ version: 0.0.42
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
-appVersion: 2.1.15
+appVersion: 2.1.19

charts/sftd/templates/statefulset.yaml

@@ -87,6 +87,7 @@ spec:
                       -I "${POD_IP}" \
                       -M "${POD_IP}" \
                       ${ACCESS_ARGS} \
+                      {{ .Values.additionalArgs }} \
                       {{ if .Values.turnDiscoveryEnabled }}-T{{ end }} \
                       -u "https://{{ required "must specify host" .Values.host }}/sfts/${POD_NAME}"
           ports:

charts/sftd/values.yaml

@@ -85,3 +85,7 @@ joinCall:
 # trying to establish a connection to clients
 # DOCS: https://docs.wire.com/understand/sft.html#prerequisites
 turnDiscoveryEnabled: false
+
+# Additional arguments to be passed to `sftd`
+# Note: this might be removed in the future.
+additionalArgs: ""

dev-packages.nix

@@ -0,0 +1,190 @@
+{ pkgs ? import ./nix }:
+let
+  staticBinaryInTarball = { pname, version, linuxAmd64Url, linuxAmd64Sha256, darwinAmd64Url, darwinAmd64Sha256, binPath ? pname }:
+    pkgs.stdenv.mkDerivation {
+      inherit pname version;
+
+      src =
+        if pkgs.stdenv.isDarwin
+        then
+          pkgs.fetchurl
+            {
+              url = darwinAmd64Url;
+              sha256 = darwinAmd64Sha256;
+            }
+        else
+          pkgs.fetchurl {
+            url = linuxAmd64Url;
+            sha256 = linuxAmd64Sha256;
+          };
+
+      installPhase = ''
+        mkdir -p $out/bin
+        cp ${binPath} $out/bin
+      '';
+    };
+
+  staticBinary = { pname, version, linuxAmd64Url, linuxAmd64Sha256, darwinAmd64Url, darwinAmd64Sha256, binPath ? pname }:
+    pkgs.stdenv.mkDerivation {
+      inherit pname version;
+
+      src =
+        if pkgs.stdenv.isDarwin
+        then
+          pkgs.fetchurl
+            {
+              url = darwinAmd64Url;
+              sha256 = darwinAmd64Sha256;
+            }
+        else
+          pkgs.fetchurl {
+            url = linuxAmd64Url;
+            sha256 = linuxAmd64Sha256;
+          };
+      phases = [ "installPhase" "patchPhase" ];
+
+      installPhase = ''
+        mkdir -p $out/bin
+        cp $src $out/bin/${binPath}
+        chmod +x $out/bin/${binPath}
+      '';
+    };
+
+  pinned = {
+    stack = staticBinaryInTarball {
+      pname = "stack";
+      version = "2.3.1";
+
+      darwinAmd64Url = "https://github.com/commercialhaskell/stack/releases/download/v2.3.1/stack-2.3.1-osx-x86_64.tar.gz";
+      darwinAmd64Sha256 = "089nrb8mxf76a0r0hdccaxfvx1ly24b5zc0cy05gs4adybjygvkk";
+
+      linuxAmd64Url = "https://github.com/commercialhaskell/stack/releases/download/v2.3.1/stack-2.3.1-linux-x86_64-static.tar.gz";
+      linuxAmd64Sha256 = "0iqfqcd88rvlwgm2h8avs0rsi9f3pdxilvcacgrxskb1n8q8ibjb";
+    };
+
+    helm = staticBinaryInTarball {
+      pname = "helm";
+      version = "3.6.3";
+
+      darwinAmd64Url = "https://get.helm.sh/helm-v3.6.3-darwin-amd64.tar.gz";
+      darwinAmd64Sha256 = "0djjvgla8cw27h8s4y6jby19f74j58byb2vfv590cd03vlbzz8c4";
+
+      linuxAmd64Url = "https://get.helm.sh/helm-v3.6.3-linux-amd64.tar.gz";
+      linuxAmd64Sha256 = "0qp28fq137b07haz4vsdbc5biagh60dcs29jj70ksqi5k6201h87";
+    };
+
+    helmfile = staticBinary {
+      pname = "helmfile";
+      version = "0.141.0";
+
+      darwinAmd64Url = "https://github.com/roboll/helmfile/releases/download/v0.141.0/helmfile_darwin_amd64";
+      darwinAmd64Sha256 = "0szfd3vy6fzd5657079hz5vii86f9xkg3bdzp3g4knkcw5x1kpxy";
+
+      linuxAmd64Url = "https://github.com/roboll/helmfile/releases/download/v0.141.0/helmfile_linux_amd64";
+      linuxAmd64Sha256 = "0f5d9w3qjvwip4qn79hsigwp8nbjpj58p289hww503j43wjyxx8r";
+    };
+
+    kubectl = staticBinaryInTarball {
+      pname = "kubectl";
+      version = "1.19.8";
+
+      darwinAmd64Url = "https://dl.k8s.io/v1.19.8/kubernetes-client-darwin-amd64.tar.gz";
+      darwinAmd64Sha256 = "23b847bb8b545c748e9078e7660c654eef74d15ccab8696d294f3d6c619c788e";
+
+      linuxAmd64Url = "https://dl.k8s.io/v1.19.8/kubernetes-client-linux-amd64.tar.gz";
+      linuxAmd64Sha256 = "8388ff8b5c676bdbb8fe07ef7077de937b0bf60154f302df5f248f38f95122aa";
+
+      binPath = "client/bin/kubectl";
+    };
+
+    kind = staticBinary {
+      pname = "kind";
+      version = "0.11.0";
+
+      darwinAmd64Url = "https://github.com/kubernetes-sigs/kind/releases/download/v0.11.1/kind-darwin-amd64";
+      darwinAmd64Sha256 = "432bef555a70e9360b44661c759658265b9eaaf7f75f1beec4c4d1e6bbf97ce3";
+
+      linuxAmd64Url = "https://github.com/kubernetes-sigs/kind/releases/download/v0.11.1/kind-linux-amd64";
+      linuxAmd64Sha256 = "949f81b3c30ca03a3d4effdecda04f100fa3edc07a28b19400f72ede7c5f0491";
+    };
+  };
+
+  compile-deps = pkgs.buildEnv {
+    name = "wire-server-compile-deps";
+    paths = [
+      pkgs.bash
+      pkgs.coreutils
+      pkgs.gnused
+      pkgs.gnugrep
+      pkgs.pkgconfig
+      pkgs.gawk
+      pkgs.git
+
+      pkgs.haskell.compiler.ghc884
+      pkgs.protobuf
+
+      pkgs.cryptobox
+      pkgs.geoip
+      pkgs.icu.dev
+      pkgs.icu.out
+      pkgs.libsodium.dev
+      pkgs.libsodium.out
+      pkgs.libxml2.dev
+      pkgs.libxml2.out
+      pkgs.ncurses.dev
+      pkgs.ncurses.out
+      pkgs.openssl.dev
+      pkgs.openssl.out
+      pkgs.pcre.dev
+      pkgs.pcre.out
+      pkgs.snappy.dev
+      pkgs.snappy.out
+      pkgs.zlib.dev
+      pkgs.zlib.out
+      pkgs.lzma.dev
+      pkgs.lzma.out
+    ];
+  };
+
+  # This performs roughly the same setup as direnv's load_prefix function, but
+  # only when invoking cabal. This means that we can set LD_LIBRARY_PATH just
+  # for cabal, as setting it in direnv can interfere with programs in the host
+  # system, especially for non-NixOS users.
+  cabal-wrapper = pkgs.writeShellScriptBin "cabal" ''
+    export CPATH="${compile-deps}/include"
+    export LD_LIBRARY_PATH="${compile-deps}/lib"
+    export LIBRARY_PATH="${compile-deps}/lib"
+    export PKG_CONFIG_PATH="${compile-deps}/lib/pkgconfig"
+    export PATH="${compile-deps}/bin"
+    exec "${pkgs.cabal-install}/bin/cabal" "$@"
+  '';
+in
+[
+  pkgs.cfssl
+  pkgs.docker-compose
+  pkgs.gnumake
+  pkgs.haskell-language-server
+  pkgs.jq
+  pkgs.ormolu
+  pkgs.telepresence
+  pkgs.wget
+  pkgs.yq
+  pkgs.rsync
+  pkgs.netcat
+
+  # To actually run buildah on nixos, I had to follow this: https://gist.github.com/alexhrescale/474d55635154e6b2cd6362c3bb403faf
+  pkgs.buildah
+
+  pinned.stack
+  pinned.helm
+  pinned.helmfile
+  pinned.kubectl
+  pinned.kind
+
+  # For cabal-migration
+  pkgs.haskellPackages.cabal-plan
+
+  # We don't use pkgs.cabal-install here, as we invoke it with a wrapper
+  # which sets LD_LIBRARY_PATH and others correctly.
+  cabal-wrapper
+]