Merge pull request #782 from flyingcircusio/PL-131515-remove-old-ceph

[21.05] remove old ceph
flyingcircusio · Sep 19, 2023 · 670f625 · 670f625
2 parents 82e27dc + e1b75f4
commit 670f625
Show file tree

Hide file tree

Showing 38 changed files with 55 additions and 5,299 deletions.
diff --git a/nixos/lib/ceph-common.nix b/nixos/lib/ceph-common.nix
@@ -5,42 +5,20 @@ with lib;
 let
   # supported ceph release codenames, from newest to oldest
   # TODO: Once all ceph packages have a similar structure, releasePkgs can be
-  # generated from this ist
-  releaseOrder = [ "nautilus" "luminous" "jewel"];
+  # generated from this list (let's wait for pacific to see if the structure holds)
+  releaseOrder = [ "nautilus" ];
   cephReleaseType = types.enum (builtins.attrNames releasePkgs);
   defaultRelease = "nautilus";
 
   # ====== mapping of packages per ceph release =======
 
   releasePkgs = {
-    "jewel" = rec {
-      ceph = pkgs.ceph-jewel;
-      # temporary mapping until all actively used ceph releases are packaged in form of
-      # the new schema with subpackages
-      ceph-client = pkgs.ceph-jewel;
-      libceph = pkgs.ceph-jewel;
-      fcQemu = pkgs.fc.qemu-py2.override {
-        ceph = libceph;
-        qemu_ceph = qemu_ceph_versioned "jewel";
-      };
-      utilPhysical = pkgs.fc.util-physical.jewel.override {ceph = ceph-client;};
-    };
-    "luminous" = rec {
-      ceph = pkgs.ceph-luminous;
-      ceph-client = pkgs.ceph-luminous;
-      libceph = pkgs.ceph-luminous;
-      fcQemu = pkgs.fc.qemu-py2.override {
-        ceph = libceph;
-        qemu_ceph = qemu_ceph_versioned "luminous";
-      };
-      utilPhysical = pkgs.fc.util-physical.luminous.override {ceph = ceph-client;};
-    };
     "nautilus" = rec {
       ceph = pkgs.ceph-nautilus.ceph;
       ceph-client = pkgs.ceph-nautilus.ceph-client;
       # both the C lib and the python modules
       libceph = pkgs.ceph-nautilus.libceph;
-      fcQemu = pkgs.fc.qemu-py3.override {
+      fcQemu = pkgs.fc.qemu.override {
         inherit libceph ceph;
         qemu_ceph = qemu_ceph_versioned "nautilus";
       };

diff --git a/nixos/roles/ceph/mon.nix b/nixos/roles/ceph/mon.nix
@@ -19,15 +19,13 @@ let
 
   # default definitions for the mgr.* options:
   mgrEnabledModules = {
-    luminous = [ "balancer" "dashboard" "status" ];
     # always_on_modules are not listed here
     nautilus = [
       "telemetry"
       "iostat"
     ];
   };
   mgrDisabledModules = {
-    luminous = [];
     nautilus = [
       "restful"
     ];
@@ -180,19 +178,12 @@ in
         {
           commands = [
             "${fc-check-ceph-withVersion}/bin/check_ceph"
+            "${fc-check-ceph-withVersion}/bin/check_snapshot_restore_fill"
           ];
           groups = [ "sensuclient" ];
         }
       ];
 
-      flyingcircus.services.sensu-client.checks = {
-        ceph = {
-          notification = "Ceph cluster is unhealthy";
-          command = "sudo ${fc-check-ceph-withVersion}/bin/check_ceph -v -R 200 -A 300";
-          interval = 60;
-        };
-      };
-
       environment.systemPackages = [ fc-check-ceph-withVersion ];
 
       systemd.services.fc-ceph-load-vm-images = {
@@ -231,25 +222,6 @@ in
         };
       };
 
-    })
-    (lib.mkIf (role.enable && role.config == "") {
-      flyingcircus.services.ceph.extraSettingsSections = lib.recursiveUpdate
-      { mon = expandCamelCaseAttrs defaultMonSettings; }
-      (lib.recursiveUpdate
-        (expandCamelCaseSection (lib.foldr (attr: acc: acc // attr) { } (map perMonSettings (fclib.findServices "ceph_mon-mon"))))
-        (lib.recursiveUpdate
-          { mon = expandCamelCaseAttrs role.extraSettings; }
-          (lib.optionalAttrs (fclib.ceph.releaseAtLeast "luminous" role.cephRelease) {
-            mon = expandCamelCaseAttrs defaultMgrSettings;
-          })
-        )
-      );
-    })
-
-    (lib.mkIf (role.enable && role.config != "") {
-      environment.etc."ceph/ceph.conf".text = lib.mkAfter role.config;
-    })
-    (lib.mkIf (role.enable && fclib.ceph.releaseAtLeast "luminous" role.cephRelease ) {
       systemd.services.fc-ceph-mgr = rec {
         description = "Local Ceph MGR (via fc-ceph)";
         wantedBy = [ "multi-user.target" ];
@@ -275,23 +247,12 @@ in
           ${fc-ceph}/bin/fc-ceph mgr reactivate
         '';
 
-        preStart =
-          # dashboard only enabled for luminous, as the Nautilus release fails to build in the sandbox so far.
-          # If we ever manage to get it enabled, `ceph config set` needs to be used instead of `ceph config-key`
-          lib.optionalString (role.cephRelease == "luminous") ''
-            echo "ensure mgr dashboard binds to localhost only"
-            # make _all_ hosts bind the dashboard to localhost (v4) only (default port: 7000)
-            ${fclib.ceph.releasePkgs.${role.cephRelease}.ceph}/bin/ceph config-key set mgr/dashboard/server_addr 127.0.0.1
-          ''
-          # imperatively ensure mgr modules
-          + lib.concatStringsSep "\n" (
-              lib.forEach mgrEnabledModules.${role.cephRelease} (mod: "${fclib.ceph.releasePkgs.${role.cephRelease}.ceph}/bin/ceph mgr module enable ${mod} --force")
-            )
-          + "\n"
-          + lib.concatStringsSep "\n" (
-              lib.forEach mgrDisabledModules.${role.cephRelease} (mod: "${fclib.ceph.releasePkgs.${role.cephRelease}.ceph}/bin/ceph mgr module disable ${mod}")
-            )
-          ;
+        # imperatively ensure mgr modules
+        preStart = lib.concatStringsSep "\n" (
+          lib.forEach mgrEnabledModules.${role.cephRelease} (mod: "${fclib.ceph.releasePkgs.${role.cephRelease}.ceph}/bin/ceph mgr module enable ${mod} --force")
+          ++
+          lib.forEach mgrDisabledModules.${role.cephRelease} (mod: "${fclib.ceph.releasePkgs.${role.cephRelease}.ceph}/bin/ceph mgr module disable ${mod}")
+        );
 
         preStop = ''
           ${fc-ceph}/bin/fc-ceph mgr deactivate
@@ -302,16 +263,6 @@ in
           RemainAfterExit = true;
         };
       };
-    })
-    (lib.mkIf (role.enable && fclib.ceph.releaseAtLeast "nautilus" role.cephRelease ) {
-      flyingcircus.passwordlessSudoRules = [
-        {
-          commands = [
-            "${fc-check-ceph-withVersion}/bin/check_snapshot_restore_fill"
-          ];
-          groups = [ "sensuclient" ];
-        }
-      ];
 
       flyingcircus.services.sensu-client.checks = let
           # check config generated directly from our platform settings
@@ -330,7 +281,28 @@ in
             command = "sudo ${fc-check-ceph-withVersion}/bin/check_snapshot_restore_fill ${configtoml}";
             interval = 600;
           };
+          ceph = {
+            notification = "Ceph cluster is unhealthy";
+            command = "sudo ${fc-check-ceph-withVersion}/bin/check_ceph -v -R 200 -A 300";
+            interval = 60;
+          };
         };
+
+    })
+    (lib.mkIf (role.enable && role.config == "") {
+      flyingcircus.services.ceph.extraSettingsSections = lib.recursiveUpdate
+      { mon = expandCamelCaseAttrs defaultMonSettings; }
+      (lib.recursiveUpdate
+        (expandCamelCaseSection (lib.foldr (attr: acc: acc // attr) { } (map perMonSettings (fclib.findServices "ceph_mon-mon"))))
+        (lib.recursiveUpdate
+          { mon = expandCamelCaseAttrs role.extraSettings; }
+          { mon = expandCamelCaseAttrs defaultMgrSettings; }
+        )
+      );
+    })
+
+    (lib.mkIf (role.enable && role.config != "") {
+      environment.etc."ceph/ceph.conf".text = lib.mkAfter role.config;
     })
     (lib.mkIf (role.enable && role.primary) {
 

diff --git a/nixos/roles/ceph/rgw.nix b/nixos/roles/ceph/rgw.nix
@@ -24,15 +24,9 @@ let
     rgwEnableOpsLog = false;
     rgwMimeTypesFile = "${pkgs.mime-types}/etc/mime.types";
     debugRados = "1 5";
-  } // (if releaseAtLeast "nautilus" role.cephRelease
-    then {
-      rgwFrontends = "beast port=80";
-      debugRgw = "1 5";
-    } else {
-      rgwFrontends = "civetweb port=80";
-      debugCivetweb = "1 5";
-      debugRgw = "0 5";
-    });
+    rgwFrontends = "beast port=80";
+    debugRgw = "1 5";
+    };
 in
 {
   options = {

diff --git a/nixos/services/ceph/client.nix b/nixos/services/ceph/client.nix
@@ -40,7 +40,7 @@ let
     debugJournal = 4;
     debugThrottle = 4;
 
-    monCompactOnStart = true;     # Keep leveldb small
+    monCompactOnStart = true;     # Keep mondb small
     monHost = mons;
     monOsdDownOutInterval = 900;  # Allow 15 min for reboots to happen without backfilling.
     monOsdNearfullRatio = .9;
@@ -81,14 +81,6 @@ in
           Starting from Ceph Nautilus on, this is deprecated.
         '';
       };
-      # TODO: cleanup and simplify once Luminous is gone
-      extraConfig = lib.mkOption {
-        type = lib.types.lines;
-        default = "";
-        description = ''
-          Extra config in the [global] section.
-        '';
-      };
       extraSettings = lib.mkOption {
         # TODO: explicitly factoring out certain config options, like done in the
         # nixpkgs upstream ceph module, might allow for better type checking
@@ -104,6 +96,7 @@ in
       extraSettingsSections = lib.mkOption {
         # serves as interface for other Ceph roles and services, these can then add additional INI sections to ceph.conf
         # TODO: refactor type towards pkgs.formats.ini in conformance wit RFC 0042
+        # TODO: probably better make this a submodule type as well
         type = with lib.types; attrsOf (attrsOf (oneOf [ bool int str float ]));
         default = { };
         description = "Additional config sections of ceph.conf, for use by components and roles.";
@@ -145,16 +138,6 @@ in
           default =  fclib.ceph.releasePkgs.${cfg.client.cephRelease}.ceph-client;
         };
 
-        # legacy config for pre-Nautilus hosts (and migration to it), default value will
-        # already be served by structured settings instead
-        config = lib.mkOption {
-          type = lib.types.lines;
-          default = "";
-          description = ''
-            Contents of the Ceph config file for clients.
-            Starting from Ceph Nautilus on, this is deprecated.
-          '';
-        };
         extraSettings = lib.mkOption {
           type = with lib.types; attrsOf (oneOf [ str int float bool ]);
           default = {};   # defaults are provided in the config section with a lower priority
@@ -175,11 +158,6 @@ in
         assertion = (cfg.client.package.codename == cfg.client.cephRelease);
         message = "The ceph package set for this ceph client service must be of the same release series as defined in `cephRelease`";
       }
-      {
-        assertion = (cfg.extraSettings != {} || cfg.client.extraSettings != {}
-          -> cfg.config == "" && cfg.extraConfig == "" && cfg.client.config == "");
-          message = "Mixing the configuration styles (extra)Config and (extra)Settings is unsupported, please use either plaintext config or structured settings for ceph.";
-      }
     ];
 
     # config file to be read by fc-ceph
@@ -199,14 +177,7 @@ in
       "d /var/log/ceph 0755 root - - -"
     ];
 
-    services.udev.extraRules =
-      if fclib.ceph.releaseAtLeast "nautilus" cfg.client.cephRelease
-      then builtins.readFile "${cfg.client.package}/etc/udev/50-rbd.rules"
-      else
-      ''
-        KERNEL=="rbd[0-9]*", ENV{DEVTYPE}=="disk", PROGRAM="${cfg.client.package}/bin/ceph-rbdnamer %k", SYMLINK+="rbd/%c{1}/%c{2}"
-        KERNEL=="rbd[0-9]*", ENV{DEVTYPE}=="partition", PROGRAM="${cfg.client.package}/bin/ceph-rbdnamer %k", SYMLINK+="rbd/%c{1}/%c{2}-part%n"
-      '';
+    services.udev.extraRules = builtins.readFile "${cfg.client.package}/etc/udev/50-rbd.rules";
 
     flyingcircus.services.ceph.allMergedSettings = (
         lib.recursiveUpdate
@@ -218,25 +189,7 @@ in
             // { client = lib.recursiveUpdate (expandCamelCaseAttrs defaultClientSettings) (expandCamelCaseAttrs cfg.client.extraSettings); }
           )
       );
-    environment.etc."ceph/ceph.conf".text = let
-      mergedSettings = config.flyingcircus.services.ceph.allMergedSettings;
-      throwDeprecationWarning = lib.warnIf (fclib.ceph.releaseAtLeast "nautilus" cfg.client.cephRelease)
-        ("Configuring ceph via plaintext `config` and `extraConfig` is deprecated since "
-        + "the Nautilus role, please switch to `extraSettings`.");
-
-      globalConfig = (if (cfg.config != "")
-        # prefer old plaintext config if it has been customised, but possibly throw warning
-        then throwDeprecationWarning (cfg.config + "\n" + cfg.extraConfig)
-        else lib.generators.toINI { } { global = mergedSettings.global; })
-        + (if (cfg.extraConfig != "") then throwDeprecationWarning ("\n" + cfg.extraConfig) else "");
-      clientConfig = if (cfg.client.config != "")
-        then throwDeprecationWarning cfg.client.config
-        else lib.generators.toINI { } {client = mergedSettings.client;};
-      otherConfig = lib.generators.toINI { } (
-        lib.filterAttrs (k: _: ! builtins.elem k [ "global" "client"] ) mergedSettings
-        );
-    in
-      (globalConfig + "\n" + clientConfig + "\n" + otherConfig);
+    environment.etc."ceph/ceph.conf".text = lib.generators.toINI {} config.flyingcircus.services.ceph.allMergedSettings;
 
     environment.variables.CEPH_ARGS = fclib.mkPlatform "--id ${config.networking.hostName}";
 

diff --git a/pkgs/fc/ceph/src/fc/ceph/keys/__init__.py b/pkgs/fc/ceph/src/fc/ceph/keys/__init__.py
@@ -1,9 +1,8 @@
 import fc.ceph
 
-from . import jewel, luminous
+from . import nautilus
 
 
 class KeyManager(fc.ceph.VersionedSubsystem):
-    jewel = jewel.KeyManager
-    luminous = luminous.KeyManager
-    nautilus = luminous
+
+    nautilus = nautilus.KeyManager