Merge pull request #281858 from nh2/fix-ceph-build-use-fmt_9

ceph: Fix build with GCC 13 by using fmt_8 -> fmt_9. Fixes #281027

Author: nh2

nixos/tests/ceph-multi-node.nix

@@ -185,6 +185,14 @@ let
     monA.succeed(
         "ceph osd pool create multi-node-test 32 32",
         "ceph osd pool ls | grep 'multi-node-test'",
+
+        # We need to enable an application on the pool, otherwise it will
+        # stay unhealthy in state POOL_APP_NOT_ENABLED.
+        # Creating a CephFS would do this automatically, but we haven't done that here.
+        # See: https://docs.ceph.com/en/reef/rados/operations/pools/#associating-a-pool-with-an-application
+        # We use the custom application name "nixos-test" for this.
+        "ceph osd pool application enable multi-node-test nixos-test",
+
         "ceph osd pool rename multi-node-test multi-node-other-test",
         "ceph osd pool ls | grep 'multi-node-other-test'",
     )

nixos/tests/ceph-single-node-bluestore.nix

@@ -145,6 +145,14 @@ let
     monA.succeed(
         "ceph osd pool create single-node-test 32 32",
         "ceph osd pool ls | grep 'single-node-test'",
+
+        # We need to enable an application on the pool, otherwise it will
+        # stay unhealthy in state POOL_APP_NOT_ENABLED.
+        # Creating a CephFS would do this automatically, but we haven't done that here.
+        # See: https://docs.ceph.com/en/reef/rados/operations/pools/#associating-a-pool-with-an-application
+        # We use the custom application name "nixos-test" for this.
+        "ceph osd pool application enable single-node-test nixos-test",
+
         "ceph osd pool rename single-node-test single-node-other-test",
         "ceph osd pool ls | grep 'single-node-other-test'",
     )

nixos/tests/ceph-single-node.nix

@@ -145,6 +145,14 @@ let
     monA.succeed(
         "ceph osd pool create single-node-test 32 32",
         "ceph osd pool ls | grep 'single-node-test'",
+
+        # We need to enable an application on the pool, otherwise it will
+        # stay unhealthy in state POOL_APP_NOT_ENABLED.
+        # Creating a CephFS would do this automatically, but we haven't done that here.
+        # See: https://docs.ceph.com/en/reef/rados/operations/pools/#associating-a-pool-with-an-application
+        # We use the custom application name "nixos-test" for this.
+        "ceph osd pool application enable single-node-test nixos-test",
+
         "ceph osd pool rename single-node-test single-node-other-test",
         "ceph osd pool ls | grep 'single-node-other-test'",
     )
@@ -182,19 +190,16 @@ let
     monA.wait_until_succeeds("ceph -s | grep 'mgr: ${cfg.monA.name}(active,'")
     monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
 
-    # This test has been commented out due to the upstream issue with pyo3
-    # that has broken this dashboard
-    # Reference: https://www.spinics.net/lists/ceph-users/msg77812.html
     # Enable the dashboard and recheck health
-    # monA.succeed(
-    #     "ceph mgr module enable dashboard",
-    #     "ceph config set mgr mgr/dashboard/ssl false",
-    #     # default is 8080 but it's better to be explicit
-    #     "ceph config set mgr mgr/dashboard/server_port 8080",
-    # )
-    # monA.wait_for_open_port(8080)
-    # monA.wait_until_succeeds("curl -q --fail http://localhost:8080")
-    # monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
+    monA.succeed(
+        "ceph mgr module enable dashboard",
+        "ceph config set mgr mgr/dashboard/ssl false",
+        # default is 8080 but it's better to be explicit
+        "ceph config set mgr mgr/dashboard/server_port 8080",
+    )
+    monA.wait_for_open_port(8080)
+    monA.wait_until_succeeds("curl -q --fail http://localhost:8080")
+    monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
   '';
 in {
   name = "basic-single-node-ceph-cluster";

pkgs/development/python-modules/cryptography/default.nix

@@ -4,6 +4,7 @@
 , callPackage
 , cargo
 , cffi
+, cryptography-vectors ? (callPackage ./vectors.nix { })
 , fetchPypi
 , hypothesis
 , iso8601
@@ -24,9 +25,6 @@
 , setuptoolsRustBuildHook
 }:
 
-let
-  cryptography-vectors = callPackage ./vectors.nix { };
-in
 buildPythonPackage rec {
   pname = "cryptography";
   version = "41.0.7"; # Also update the hash in vectors.nix

pkgs/tools/filesystems/ceph/default.nix

@@ -50,6 +50,9 @@
 , zlib
 , zstd
 
+# Dependencies of overridden Python dependencies, hopefully we can remove these soon.
+, rustPlatform
+
 # Optional Dependencies
 , curl ? null
 , expat ? null
@@ -166,7 +169,67 @@ let
 
   # Watch out for python <> boost compatibility
   python = python310.override {
-    packageOverrides = self: super: {
+    packageOverrides = self: super: let cryptographyOverrideVersion = "40.0.1"; in {
+      # Ceph does not support `cryptography` > 40 yet:
+      # * https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1899358602
+      # * Upstream issue: https://tracker.ceph.com/issues/63529
+      #   > Python Sub-Interpreter Model Used by ceph-mgr Incompatible With Python Modules Based on PyO3
+      #
+      # We pin the older `cryptography` 40 here;
+      # this also forces us to pin an older `pyopenssl` because the current one
+      # is not compatible with older `cryptography`, see:
+      #     https://github.com/pyca/pyopenssl/blob/d9752e44127ba36041b045417af8a0bf16ec4f1e/CHANGELOG.rst#2320-2023-05-30
+      cryptography = super.cryptography.overridePythonAttrs (old: rec {
+        version = cryptographyOverrideVersion;
+
+        src = fetchPypi {
+          inherit (old) pname;
+          version = cryptographyOverrideVersion;
+          hash = "sha256-KAPy+LHpX2FEGZJsfm9V2CivxhTKXtYVQ4d65mjMNHI=";
+        };
+
+        cargoDeps = rustPlatform.fetchCargoTarball {
+          inherit src;
+          sourceRoot = let cargoRoot = "src/rust"; in "${old.pname}-${cryptographyOverrideVersion}/${cargoRoot}";
+          name = "${old.pname}-${cryptographyOverrideVersion}";
+          hash = "sha256-gFfDTc2QWBWHBCycVH1dYlCsWQMVcRZfOBIau+njtDU=";
+        };
+
+        patches = (old.patches or []) ++ [
+          # Fix https://nvd.nist.gov/vuln/detail/CVE-2023-49083 which has no upstream backport.
+          # See https://github.com/pyca/cryptography/commit/f09c261ca10a31fe41b1262306db7f8f1da0e48a#diff-f5134bf8f3cf0a5cc8601df55e50697acc866c603a38caff98802bd8e17976c5R1893
+          ./python-cryptography-Cherry-pick-fix-for-CVE-2023-49083-on-cryptography-40.patch
+        ];
+
+        # Tests would require overriding `cryptography-vectors`, which is not currently
+        # possible/desired, see: https://github.com/NixOS/nixpkgs/pull/281858#pullrequestreview-1841421866
+        doCheck = false;
+      });
+
+      # This is the most recent version of `pyopenssl` that's still compatible with `cryptography` 40.
+      # See https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1899358602
+      pyopenssl = super.pyopenssl.overridePythonAttrs (old: rec {
+        version = "23.1.1";
+        src = fetchPypi {
+          pname = "pyOpenSSL";
+          inherit version;
+          hash = "sha256-hBSYub7GFiOxtsR+u8AjZ8B9YODhlfGXkIF/EMyNsLc=";
+        };
+      });
+
+      # Ceph does not support `kubernetes` >= 19, see:
+      #     https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1900324090
+      kubernetes = super.kubernetes.overridePythonAttrs (old: rec {
+        version = "18.20.0";
+        src = fetchFromGitHub {
+          owner = "kubernetes-client";
+          repo = "python";
+          rev = "v${version}";
+          sha256 = "1sawp62j7h0yksmg9jlv4ik9b9i1a1w9syywc9mv8x89wibf5ql1";
+          fetchSubmodules = true;
+        };
+      });
+
     };
   };
 

pkgs/tools/filesystems/ceph/python-cryptography-Cherry-pick-fix-for-CVE-2023-49083-on-cryptography-40.patch

@@ -0,0 +1,44 @@
+commit 08dbad8552ecca85d3e38072f94eb864b143d218
+Author: Niklas Hambüchen <[email protected]>
+Date:   Mon Jan 22 12:52:06 2024 +0000
+
+    Cherry-pick fix for CVE-2023-49083
+    
+    Cherry-Picked-From: f09c261ca10a31fe41b1262306db7f8f1da0e48a
+    Cherry-Picked-By: Niklas Hambüchen <[email protected]>
+
+diff --git a/src/cryptography/hazmat/backends/openssl/backend.py b/src/cryptography/hazmat/backends/openssl/backend.py
+index a3fe1bce4..58e7207c4 100644
+--- a/src/cryptography/hazmat/backends/openssl/backend.py
++++ b/src/cryptography/hazmat/backends/openssl/backend.py
+@@ -2383,9 +2383,12 @@ class Backend:
+                 _Reasons.UNSUPPORTED_SERIALIZATION,
+             )
+ 
++        certs: list[x509.Certificate] = []
++        if p7.d.sign == self._ffi.NULL:
++            return certs
++
+         sk_x509 = p7.d.sign.cert
+         num = self._lib.sk_X509_num(sk_x509)
+-        certs = []
+         for i in range(num):
+             x509 = self._lib.sk_X509_value(sk_x509, i)
+             self.openssl_assert(x509 != self._ffi.NULL)
+diff --git a/tests/hazmat/primitives/test_pkcs7.py b/tests/hazmat/primitives/test_pkcs7.py
+index 4e61c5ef5..d8170bfb9 100644
+--- a/tests/hazmat/primitives/test_pkcs7.py
++++ b/tests/hazmat/primitives/test_pkcs7.py
+@@ -89,6 +89,12 @@ class TestPKCS7Loading:
+                 mode="rb",
+             )
+ 
++    def test_load_pkcs7_empty_certificates(self, backend):
++        der = b"\x30\x0B\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x07\x02"
++
++        certificates = pkcs7.load_der_pkcs7_certificates(der)
++        assert certificates == []
++
+ 
+ # We have no public verification API and won't be adding one until we get
+ # some requirements from users so this function exists to give us basic

pkgs/top-level/all-packages.nix

@@ -6809,8 +6809,14 @@ with pkgs;
 
   libceph = ceph.lib;
   inherit (callPackages ../tools/filesystems/ceph {
-    lua = lua5_4;
-    fmt = fmt_8;
+    lua = lua5_4; # Ceph currently requires >= 5.3
+
+    # To see which `fmt` version Ceph upstream recommends, check its `src/fmt` submodule.
+    #
+    # Ceph does not currently build with `fmt_10`; see https://github.com/NixOS/nixpkgs/issues/281027#issuecomment-1899128557
+    # If we want to switch for that before upstream fixes it, use this patch:
+    # https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1899648638
+    fmt = fmt_9;
   })
     ceph
     ceph-client;