From 790431098a4186ac4e617e4c339158eeece450d3 Mon Sep 17 00:00:00 2001 From: Magnus Viernickel Date: Thu, 23 Nov 2023 16:56:36 +0100 Subject: [PATCH 01/10] [feat] bombon derivations --- nix/default.nix | 1 + nix/sources.json | 12 ++++++++++++ nix/wire-server.nix | 26 +++++++++++++++----------- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/nix/default.nix b/nix/default.nix index 6f75f4e105..b9580716e3 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -7,6 +7,7 @@ let # All wire-server specific packages (import ./overlay.nix) (import ./overlay-docs.nix) + (self: super: { lib = super.lib // (import sources.bombon).lib.${super.system}; }) ]; }; diff --git a/nix/sources.json b/nix/sources.json index 3f76c8b401..ff25e629a7 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -1,4 +1,16 @@ { + "bombon": { + "branch": "main", + "description": "Nix CycloneDX Software Bills of Materials (SBOMs)", + "homepage": "", + "owner": "nikstur", + "repo": "bombon", + "rev": "09dce0377beb87c24822f79501d6c76166105788", + "sha256": "1z80waaimga03m4b0nhc3djaca4y2bh0dq8mc1r8s59hqngc22ch", + "type": "tarball", + "url": "https://github.com/nikstur/bombon/archive/09dce0377beb87c24822f79501d6c76166105788.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, "nixpkgs": { "branch": "nixpkgs-unstable", "description": "Nix Packages collection", diff --git a/nix/wire-server.nix b/nix/wire-server.nix index 82a0211e42..35adf7e735 100644 --- a/nix/wire-server.nix +++ b/nix/wire-server.nix @@ -44,7 +44,7 @@ # with nixpkgs' dockerTools to make derivations for docker images that we need. pkgs: let - lib = pkgs.lib; + inherit (pkgs) lib; hlib = pkgs.haskell.lib; withCleanedPath = drv: hlib.overrideCabal drv (old: { @@ -459,9 +459,22 @@ let export LOCALE_ARCHIVE=${pkgs.glibcLocales}/lib/locale/locale-archive ''; }; + allLocalPackages = pkgs.symlinkJoin { + name = "all-local-packages"; + paths = map (e: (hPkgs localModsEnableAll).${e}) wireServerPackages; + }; + + allImages = pkgs.linkFarm "all-images" (images localModsEnableAll) ; + + allLocalPackagesBom = lib.buildBom allLocalPackages { + includeBuildtimeDependencies = true; + }; + allImagesBom = lib.buildBom allImages { + includeBuildtimeDependencies = true; + }; in { - inherit ciImage hoogleImage; + inherit ciImage hoogleImage allImages allImagesBom allLocalPackages allLocalPackagesBom; images = images localModsEnableAll; imagesUnoptimizedNoDocs = images localModsOnlyTests; @@ -522,13 +535,4 @@ in inherit brig-templates; haskellPackages = hPkgs localModsEnableAll; haskellPackagesUnoptimizedNoDocs = hPkgs localModsOnlyTests; - allLocalPackages = pkgs.symlinkJoin { - name = "all-local-packages"; - paths = map (e: (hPkgs localModsEnableAll).${e}) wireServerPackages; - }; - - allImages = pkgs.symlinkJoin { - name = "all-images"; - paths = builtins.attrValues (images localModsEnableAll); - }; } // attrsets.genAttrs wireServerPackages (e: (hPkgs localModsEnableAll).${e}) From 98954083710b7aaad2f627677af6554ac262ab79 Mon Sep 17 00:00:00 2001 From: Magnus Viernickel Date: Mon, 27 Nov 2023 14:07:04 +0100 Subject: [PATCH 02/10] [feat] add Makefile target to upload bombon to s3 --- Makefile | 4 ++++ hack/bin/bombon.hs | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100755 hack/bin/bombon.hs diff --git a/Makefile b/Makefile index db7ec781f6..d020031720 100644 --- a/Makefile +++ b/Makefile @@ -542,3 +542,7 @@ kind-restart-%: .local/kind-kubeconfig # make helm-template-wire-server helm-template-%: clean-charts charts-integration ./hack/bin/helm-template.sh $(*) + +.PHONY: upload-bombon +upload-bombon: + ./hack/bin/bombon -- -d ./nix diff --git a/hack/bin/bombon.hs b/hack/bin/bombon.hs new file mode 100755 index 0000000000..a1c6913b93 --- /dev/null +++ b/hack/bin/bombon.hs @@ -0,0 +1,21 @@ +#!/usr/bin/env -S nix -Lv run github:wireapp/ghc-flakr/ecb1f45f1549e06c92d71164e305ce501eb0e36e +{-# language BlockArguments, OverloadedStrings #-} + +import Turtle +import Data.Text qualified as T +import Data.Time.Clock.POSIX +import Data.Time + +main = do + nixDir <- options "Upload boms" do + optText "nixDir" 'd' "Where to find the nix code" + with (mktempdir "." "tmp") \tmpDir -> do + bomName <- ("wire-server-bom-" <>) . T.pack . show . nominalDiffTimeToSeconds <$> getPOSIXTime + let bomPath = T.pack tmpDir <> "/" <> bomName + proc + "nix" + ["build", "-f", nixDir, "wireServer.allLocalPackagesBom", "-o", bomPath] + mempty + printf ("Copying "%s%" to s3\n") bomName + proc "aws" ["s3", "cp", bomPath, "s3://wire-server-bom/" <> bomName] mempty + pure () From ed0ecfb2b284d4af121f564a3273ff5317e69e4a Mon Sep 17 00:00:00 2001 From: Magnus Viernickel Date: Mon, 27 Nov 2023 14:08:50 +0100 Subject: [PATCH 03/10] [chore] add a changelog.d entry --- changelog.d/5-internal/upload-bom | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/5-internal/upload-bom diff --git a/changelog.d/5-internal/upload-bom b/changelog.d/5-internal/upload-bom new file mode 100644 index 0000000000..af658e2002 --- /dev/null +++ b/changelog.d/5-internal/upload-bom @@ -0,0 +1 @@ +add a Makefile target to make it possible to upload a bom of all services to s3 on every CI run From a2b180b86788be47fa44cf8ddb696bce6bdce1d1 Mon Sep 17 00:00:00 2001 From: Magnus Viernickel Date: Mon, 27 Nov 2023 14:10:44 +0100 Subject: [PATCH 04/10] [fix] correct Makefile, make formatter happy --- Makefile | 2 +- hack/bin/bombon.hs | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index d020031720..e9d7db4ae7 100644 --- a/Makefile +++ b/Makefile @@ -545,4 +545,4 @@ helm-template-%: clean-charts charts-integration .PHONY: upload-bombon upload-bombon: - ./hack/bin/bombon -- -d ./nix + ./hack/bin/bombon.hs -- -d ./nix diff --git a/hack/bin/bombon.hs b/hack/bin/bombon.hs index a1c6913b93..6ca0829c66 100755 --- a/hack/bin/bombon.hs +++ b/hack/bin/bombon.hs @@ -1,10 +1,12 @@ #!/usr/bin/env -S nix -Lv run github:wireapp/ghc-flakr/ecb1f45f1549e06c92d71164e305ce501eb0e36e -{-# language BlockArguments, OverloadedStrings #-} +{-# LANGUAGE BlockArguments #-} +{-# LANGUAGE ImportQualifiedPost #-} +{-# LANGUAGE OverloadedStrings #-} -import Turtle import Data.Text qualified as T -import Data.Time.Clock.POSIX import Data.Time +import Data.Time.Clock.POSIX +import Turtle main = do nixDir <- options "Upload boms" do @@ -16,6 +18,6 @@ main = do "nix" ["build", "-f", nixDir, "wireServer.allLocalPackagesBom", "-o", bomPath] mempty - printf ("Copying "%s%" to s3\n") bomName + printf ("Copying " % s % " to s3\n") bomName proc "aws" ["s3", "cp", bomPath, "s3://wire-server-bom/" <> bomName] mempty pure () From a617320c4c1f8dcb036005d7cee86e05d05d9300 Mon Sep 17 00:00:00 2001 From: Magnus Viernickel Date: Mon, 27 Nov 2023 16:16:14 +0100 Subject: [PATCH 05/10] [feat] upload to release notes --- Makefile | 2 +- hack/bin/bombon.hs | 30 ++++++++++++++++++++++-------- nix/wire-server.nix | 1 + 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index e9d7db4ae7..f72034a490 100644 --- a/Makefile +++ b/Makefile @@ -545,4 +545,4 @@ helm-template-%: clean-charts charts-integration .PHONY: upload-bombon upload-bombon: - ./hack/bin/bombon.hs -- -d ./nix + ./hack/bin/bombon.hs -- "$@" diff --git a/hack/bin/bombon.hs b/hack/bin/bombon.hs index 6ca0829c66..cbc4eab857 100755 --- a/hack/bin/bombon.hs +++ b/hack/bin/bombon.hs @@ -9,15 +9,29 @@ import Data.Time.Clock.POSIX import Turtle main = do - nixDir <- options "Upload boms" do - optText "nixDir" 'd' "Where to find the nix code" + release <- options "Upload boms" do + optText "release" 't' "Which release tag to upload the artifacts to" + with (mktempdir "." "tmp") \tmpDir -> do - bomName <- ("wire-server-bom-" <>) . T.pack . show . nominalDiffTimeToSeconds <$> getPOSIXTime - let bomPath = T.pack tmpDir <> "/" <> bomName + cd tmpDir proc - "nix" - ["build", "-f", nixDir, "wireServer.allLocalPackagesBom", "-o", bomPath] + "wget" + ["https://github.com/wireapp/wire-server/archive/refs/tags/chart/" <> release <> ".zip"] mempty - printf ("Copying " % s % " to s3\n") bomName - proc "aws" ["s3", "cp", bomPath, "s3://wire-server-bom/" <> bomName] mempty + proc "unzip" [release] mempty + bomName <- ("wire-server-bom-" <>) . T.pack . show . nominalDiffTimeToSeconds <$> getPOSIXTime + let bomPath = "./" <> bomName <> ".json" + ExitSuccess <- + proc + "nix" + [ "build", + "-f", + "wire-server-chart-" <> release <> "/nix", + "wireServer.allLocalPackagesBom", + "-o", + bomPath + ] + mempty + printf ("uploading " % s % " to release" % s % "\n") bomName release + proc "gh" ["release", "upload", release, bomPath] mempty pure () diff --git a/nix/wire-server.nix b/nix/wire-server.nix index 35adf7e735..fa1a240c81 100644 --- a/nix/wire-server.nix +++ b/nix/wire-server.nix @@ -466,6 +466,7 @@ let allImages = pkgs.linkFarm "all-images" (images localModsEnableAll) ; + # BOM is an acronym for bill of materials allLocalPackagesBom = lib.buildBom allLocalPackages { includeBuildtimeDependencies = true; }; From a66cdf5c503700025c2999c8550d809ea7b6ec43 Mon Sep 17 00:00:00 2001 From: Magnus Viernickel Date: Tue, 28 Nov 2023 13:39:12 +0100 Subject: [PATCH 06/10] [fix] some fixes to make it work --- hack/bin/bombon.hs | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/hack/bin/bombon.hs b/hack/bin/bombon.hs index cbc4eab857..587091c90a 100755 --- a/hack/bin/bombon.hs +++ b/hack/bin/bombon.hs @@ -14,11 +14,10 @@ main = do with (mktempdir "." "tmp") \tmpDir -> do cd tmpDir - proc - "wget" - ["https://github.com/wireapp/wire-server/archive/refs/tags/chart/" <> release <> ".zip"] - mempty - proc "unzip" [release] mempty + let git l = proc "git" l mempty + git ["clone", "https://github.com/wireapp/wire-server", "."] + git ["checkout", release] + git ["submodule", "update", "--init", "--recursive"] bomName <- ("wire-server-bom-" <>) . T.pack . show . nominalDiffTimeToSeconds <$> getPOSIXTime let bomPath = "./" <> bomName <> ".json" ExitSuccess <- @@ -26,12 +25,21 @@ main = do "nix" [ "build", "-f", - "wire-server-chart-" <> release <> "/nix", + "nix", "wireServer.allLocalPackagesBom", "-o", bomPath ] mempty - printf ("uploading " % s % " to release" % s % "\n") bomName release - proc "gh" ["release", "upload", release, bomPath] mempty + printf ("uploading " % s % " to release " % s % "\n") bomName ("chart/" <> release) + proc + "gh" + [ "-R", + "wireapp/wire-server", + "release", + "upload", + "chart/" <> release, + bomPath + ] + mempty pure () From b08b3e8e53887d46904ba5ec6298a757d969f051 Mon Sep 17 00:00:00 2001 From: Magnus Viernickel Date: Tue, 28 Nov 2023 14:44:49 +0100 Subject: [PATCH 07/10] [fix] use revision instead of timestamp --- hack/bin/bombon.hs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/hack/bin/bombon.hs b/hack/bin/bombon.hs index 587091c90a..1c09195b49 100755 --- a/hack/bin/bombon.hs +++ b/hack/bin/bombon.hs @@ -4,8 +4,6 @@ {-# LANGUAGE OverloadedStrings #-} import Data.Text qualified as T -import Data.Time -import Data.Time.Clock.POSIX import Turtle main = do @@ -15,11 +13,10 @@ main = do with (mktempdir "." "tmp") \tmpDir -> do cd tmpDir let git l = proc "git" l mempty - git ["clone", "https://github.com/wireapp/wire-server", "."] + git ["clone", "https://github.com/mangoiv/wire-server", "."] git ["checkout", release] git ["submodule", "update", "--init", "--recursive"] - bomName <- ("wire-server-bom-" <>) . T.pack . show . nominalDiffTimeToSeconds <$> getPOSIXTime - let bomPath = "./" <> bomName <> ".json" + let bomName = "wire-server-bom-" <> release <> ".json" ExitSuccess <- proc "nix" @@ -28,18 +25,18 @@ main = do "nix", "wireServer.allLocalPackagesBom", "-o", - bomPath + bomName ] mempty printf ("uploading " % s % " to release " % s % "\n") bomName ("chart/" <> release) proc "gh" [ "-R", - "wireapp/wire-server", + "mangoiv/wire-server", "release", "upload", "chart/" <> release, - bomPath + bomName ] mempty pure () From 5f75037c9ef69b6c8388c3728e5df922ca78faef Mon Sep 17 00:00:00 2001 From: Magnus Viernickel Date: Mon, 4 Dec 2023 09:59:26 +0100 Subject: [PATCH 08/10] [chore] hi ci From 27fcf8825be9e5caa5d9b83ecc85395e2ebcc64a Mon Sep 17 00:00:00 2001 From: Magnus Viernickel Date: Wed, 6 Dec 2023 14:40:55 +0100 Subject: [PATCH 09/10] [chore] apply Akshay's suggestions --- hack/bin/bombon.hs | 41 ++++++++--------------------------------- nix/wire-server.nix | 7 ++----- 2 files changed, 10 insertions(+), 38 deletions(-) diff --git a/hack/bin/bombon.hs b/hack/bin/bombon.hs index 1c09195b49..2b10440079 100755 --- a/hack/bin/bombon.hs +++ b/hack/bin/bombon.hs @@ -7,36 +7,11 @@ import Data.Text qualified as T import Turtle main = do - release <- options "Upload boms" do - optText "release" 't' "Which release tag to upload the artifacts to" - - with (mktempdir "." "tmp") \tmpDir -> do - cd tmpDir - let git l = proc "git" l mempty - git ["clone", "https://github.com/mangoiv/wire-server", "."] - git ["checkout", release] - git ["submodule", "update", "--init", "--recursive"] - let bomName = "wire-server-bom-" <> release <> ".json" - ExitSuccess <- - proc - "nix" - [ "build", - "-f", - "nix", - "wireServer.allLocalPackagesBom", - "-o", - bomName - ] - mempty - printf ("uploading " % s % " to release " % s % "\n") bomName ("chart/" <> release) - proc - "gh" - [ "-R", - "mangoiv/wire-server", - "release", - "upload", - "chart/" <> release, - bomName - ] - mempty - pure () + (release, repo) <- options "Upload boms" do + (,) + <$> optText "release" 't' "Which release tag to upload the artifacts to" + <*> optText "repo" 'r' "Which repository to upload the artifacts to" + let bomName = "wire-server-bom-" <> release <> ".json" + ExitSuccess <- proc "nix" ["build", "-f", "nix", "wireServer.allLocalPackagesBom", "-o", bomName] mempty + printf ("uploading " % s % " to release " % s % "\n") bomName ("chart/" <> release) + proc "gh" ["-R", repo, "release", "upload", "chart/" <> release, bomName] mempty diff --git a/nix/wire-server.nix b/nix/wire-server.nix index fa1a240c81..a3db5c48a5 100644 --- a/nix/wire-server.nix +++ b/nix/wire-server.nix @@ -464,18 +464,15 @@ let paths = map (e: (hPkgs localModsEnableAll).${e}) wireServerPackages; }; - allImages = pkgs.linkFarm "all-images" (images localModsEnableAll) ; + allImages = pkgs.linkFarm "all-images" (images localModsEnableAll); # BOM is an acronym for bill of materials allLocalPackagesBom = lib.buildBom allLocalPackages { includeBuildtimeDependencies = true; }; - allImagesBom = lib.buildBom allImages { - includeBuildtimeDependencies = true; - }; in { - inherit ciImage hoogleImage allImages allImagesBom allLocalPackages allLocalPackagesBom; + inherit ciImage hoogleImage allImages allLocalPackages allLocalPackagesBom; images = images localModsEnableAll; imagesUnoptimizedNoDocs = images localModsOnlyTests; From a5f30f4ad1be8281660a3855cd626f4bbbc70985 Mon Sep 17 00:00:00 2001 From: Magnus Viernickel Date: Tue, 12 Dec 2023 11:44:14 +0100 Subject: [PATCH 10/10] [chore] hi ci