Overhaul PostgreSQL packaging, native extensions, and NixOS support

nixos/doc/manual/release-notes/rl-1809.xml

@@ -18,6 +18,33 @@
   </para>
 
   <itemizedlist>
+   <listitem>
+     <para>
+       The PostgreSQL server infrastructure has been greatly improved and is
+       much easier to use. Extensions are always built against the correct
+       version of the server without overrides. Server versions are exposed as
+       "sets" of packages along with their compatible extensions, which are all
+       compiled and checked continuously. This means you can now use extensions
+       like plv8 with any PostgreSQL version without compiling anything.
+       Extensions expressions are easier to write, and new extensions are
+       available: <literal>pg_cron</literal>, <literal>postgres-hll</literal>,
+       <literal>postgres-topn</literal>, <literal>pg_journal</literal>,
+       <literal>pg_jobmon</literal>, <literal>pg_partman</literal>,
+       and <literal>citus</literal>. (Almost all existing extensions have seen
+       version updates, as well.)
+     </para>
+     <para>
+       NixOS 18.09 is the last release that will offer PostgreSQL 9.3 as a
+       server option. 9.3 will reach its 5-year End of Life in September 2018;
+       the NixOS 18.09 branch will see no updates at all to PostgreSQL 9.3. It
+       will only be available to keep existing servers running, so they may be
+       migrated to new versions.
+     </para>
+     <para>
+       PostgreSQL 10 is now the default option for new NixOS 18.09
+       installations.
+     </para>
+   </listitem>
    <listitem>
      <para>
        Support for wrapping binaries using <literal>firejail</literal> has been

nixos/modules/services/backup/bacula.nix

@@ -370,8 +370,8 @@ in {
       serviceConfig.ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
       preStart = ''
         if ! test -e "${libDir}/db-created"; then
-            ${pkgs.postgresql}/bin/createuser --no-superuser --no-createdb --no-createrole bacula
-            #${pkgs.postgresql}/bin/createdb --owner bacula bacula
+            ${config.services.postgresql.postgresqlPackage}/bin/createuser --no-superuser --no-createdb --no-createrole bacula
+            #${config.services.postgresql.postgresqlPackage}/bin/createdb --owner bacula bacula
 
             # populate DB
             ${pkgs.bacula}/etc/create_bacula_database postgresql

nixos/modules/services/backup/postgresql-backup.nix

@@ -3,46 +3,44 @@
 with lib;
 
 let
-
   cfg = config.services.postgresqlBackup;
 
-  postgresqlBackupService = db :
-    {
-      enable = true;
-
-      description = "Backup of database ${db}";
-
-      requires = [ "postgresql.service" ];
-
-      preStart = ''
-        mkdir -m 0700 -p ${cfg.location}
-        chown postgres ${cfg.location}
-      '';
-
-      script = ''
-        if [ -e ${cfg.location}/${db}.sql.gz ]; then
-          ${pkgs.coreutils}/bin/mv ${cfg.location}/${db}.sql.gz ${cfg.location}/${db}.prev.sql.gz
-        fi
-
-        ${config.services.postgresql.package}/bin/pg_dump ${cfg.pgdumpOptions} ${db} | \
-          ${pkgs.gzip}/bin/gzip -c > ${cfg.location}/${db}.sql.gz
-      '';
-
-      serviceConfig = {
-        Type = "oneshot";
-        PermissionsStartOnly = "true";
-        User = "postgres";
-      };
-
-      startAt = cfg.startAt;
+  postgresqlBackupService = db: {
+    description = "Backup of database ${db}";
+    enable = true;
+    requires = [ "postgresql.service" ];
+    startAt = cfg.startAt;
+
+    preStart = ''
+      mkdir -m 0700 -p ${cfg.location}
+      chown postgres ${cfg.location}
+    '';
+
+    script = ''
+      # Use the same PATH as the PostgreSQL instance, as it's configured
+      # with plugins and extensions. This is a bit of a hack; we can't easily
+      # set the nixos systemd options to do this, because the path is already
+      # constructed and is too long by the time the systemd.nix module examines
+      # it
+      export PATH="${config.systemd.services.postgresql.environment.PATH}"
+
+      if [ -e ${cfg.location}/${db}.sql.gz ]; then
+        mv ${cfg.location}/${db}.sql.gz ${cfg.location}/${db}.prev.sql.gz
+      fi
+
+      pg_dump ${cfg.pgdumpOptions} ${db} | ${pkgs.gzip}/bin/gzip -c > ${cfg.location}/${db}.sql.gz
+    '';
+
+    serviceConfig = {
+      Type = "oneshot";
+      PermissionsStartOnly = "true";
+      User = "postgres";
     };
+  };
 
 in {
-
   options = {
-
     services.postgresqlBackup = {
-
       enable = mkOption {
         default = false;
         description = ''
@@ -81,14 +79,14 @@ in {
         '';
       };
     };
-
   };
 
   config = mkIf config.services.postgresqlBackup.enable {
 
-    systemd.services = listToAttrs (map (db : {
-          name = "postgresqlBackup-${db}";
-          value = postgresqlBackupService db; } ) cfg.databases);
-  };
+    systemd.services = listToAttrs (map (db: {
+      name = "postgresqlBackup-${db}";
+      value = postgresqlBackupService db;
+    }) cfg.databases);
 
+  };
 }

nixos/modules/services/continuous-integration/hydra/default.nix

@@ -6,6 +6,8 @@ let
 
   cfg = config.services.hydra;
 
+  pg = config.services.postgresql;
+
   baseDir = "/var/lib/hydra";
 
   hydraConf = pkgs.writeScript "hydra.conf" cfg.extraConfig;
@@ -271,8 +273,8 @@ in
 
           ${optionalString haveLocalDB ''
             if ! [ -e ${baseDir}/.db-created ]; then
-              ${pkgs.sudo}/bin/sudo -u ${config.services.postgresql.superUser} ${config.services.postgresql.package}/bin/createuser hydra
-              ${pkgs.sudo}/bin/sudo -u ${config.services.postgresql.superUser} ${config.services.postgresql.package}/bin/createdb -O hydra hydra
+              ${pkgs.sudo}/bin/sudo -u ${pg.superUser} ${pg.postgresqlPackage}/bin/createuser hydra
+              ${pkgs.sudo}/bin/sudo -u ${pg.superUser} ${pg.postgresqlPackage}/bin/createdb -O hydra hydra
               touch ${baseDir}/.db-created
             fi
           ''}

nixos/modules/services/databases/postgresql.nix

@@ -6,23 +6,16 @@ let
 
   cfg = config.services.postgresql;
 
+  packageSetPlugins = cfg.plugins cfg.packages;
+  plugins = if cfg.extraPlugins != [] then cfg.extraPlugins else packageSetPlugins;
+
   # see description of extraPlugins
   postgresqlAndPlugins = pg:
-    if cfg.extraPlugins == [] then pg
-    else pkgs.buildEnv {
-      name = "postgresql-and-plugins-${(builtins.parseDrvName pg.name).version}";
-      paths = [ pg pg.lib ] ++ cfg.extraPlugins;
-      buildInputs = [ pkgs.makeWrapper ];
-      postBuild =
-        ''
-          mkdir -p $out/bin
-          rm $out/bin/{pg_config,postgres,pg_ctl}
-          cp --target-directory=$out/bin ${pg}/bin/{postgres,pg_config,pg_ctl}
-          wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib
-        '';
-    };
+    # See the note about withPackages in postgresql/packages.nix for more
+    let ps = import ../../../../pkgs/servers/sql/postgresql/packages.nix { inherit pkgs lib; };
+    in ps.withPackages pg plugins;
 
-  postgresql = postgresqlAndPlugins cfg.package;
+  postgresql = postgresqlAndPlugins cfg.postgresqlPackage;
 
   # The main PostgreSQL configuration file.
   configFile = pkgs.writeText "postgresql.conf"
@@ -35,6 +28,18 @@ let
       ${cfg.extraConfig}
     '';
 
+  # TODO: propose to move this to lib/types.nix
+  # since selector functions are used in at least 6 other options in NixOS.
+  selectorFunction = mkOptionType {
+    name = "selectorFunction";
+    description =
+      "function that takes an attribute set and returns a list " +
+      "containing a selection of the values of the input set";
+    check = select: isFunction select;
+    merge = _loc: defs:
+      as: concatMap (select: select as) (getValues defs);
+  };
+
 in
 
 {
@@ -54,10 +59,37 @@ in
       };
 
       package = mkOption {
-        type = types.package;
+        type = types.nullOr types.package;
         example = literalExample "pkgs.postgresql96";
+        default = null;
         description = ''
-          PostgreSQL package to use.
+          PostgreSQL package to use. Deprecated. Use services.postgresql.packages instead
+          to specify an entire package set (including compatible extensions) at once.
+        '';
+      };
+
+      packages = mkOption {
+        type = types.nullOr (types.attrsOf types.package);
+        default = null;
+        example = literalExample "pkgs.postgresql96Packages";
+        description = ''
+          The set of PostgreSQL packages to use, including the database
+          server and all available extensions.
+        '';
+      };
+
+      postgresqlPackage = mkOption {
+        type = types.package;
+        default =
+          if cfg.package != null
+          then cfg.package
+          else cfg.packages.postgresql;
+        readOnly = true;
+        visible = false;
+        description = ''
+          A deprecated, read-only and invisible option that refers to the
+          selected PostgreSQL package. This should be removed at the same time
+          that the deprecated config.services.postgresql.package option is removed.
         '';
       };
 
@@ -120,11 +152,31 @@ in
         default = [];
         example = literalExample "[ (pkgs.postgis.override { postgresql = pkgs.postgresql94; }) ]";
         description = ''
-          When this list contains elements a new store path is created.
+          Deprecated. When this list contains elements a new store path is created.
           PostgreSQL and the elements are symlinked into it. Then pg_config,
           postgres and pg_ctl are copied to make them use the new
           $out/lib directory as pkglibdir. This makes it possible to use postgis
           without patching the .sql files which reference $libdir/postgis-1.5.
+
+          This attribute is hard to use, and is deprecated in favor of the more general
+          services.postgresql.plugins attribute, which will correctly override the specified
+          packages for you.
+        '';
+      };
+
+      plugins = mkOption {
+        default = _: [];
+        type = selectorFunction;
+        example = literalExample "p: with p; [ postgis timescaledb ]";
+        description = ''
+          A function that selects plugins to include in the PostgreSQL server.
+          When this function returns a non-empty list, a new store path is created.
+          PostgreSQL and the elements are symlinked into it. Then pg_config,
+          postgres and pg_ctl are copied to make them use the new
+          $out/lib directory as pkglibdir.
+
+          The value provided as an argument to the function is the set of available
+          PostgreSQL plugins, specified by the services.postgresql.packages argument.
         '';
         # Note: the duplication of executables is about 4MB size.
         # So a nicer solution was patching postgresql to allow setting the
@@ -163,16 +215,46 @@ in
 
   config = mkIf config.services.postgresql.enable {
 
-    services.postgresql.package =
+    assertions =
+      [ # The user specified both 'package' and 'packages', which are mutually exclusive
+        { assertion = (cfg.package != null -> cfg.packages != null);
+          message = ''
+            The option services.postgresql.{package,packages} cannot both be set. Please use the 'packages' option
+            to specify the entire set of PostgreSQL packages.
+          '';
+        }
+
+        # The user specified the old extraPlugins attribute, but they *also* specified some 'plugins' function
+        # which returned a non-empty list. These are mutually exclusive.
+        { assertion = (cfg.extraPlugins != [] -> packageSetPlugins == []);
+          message = ''
+            The option services.postgresql.extraPlugins and .plugins cannot both be non-empty at the same time; please remove
+            uses of .extraPlugins.
+          '';
+        }
+      ];
+
+    warnings =
+      (optional (cfg.package != null)
+        ''The services.postgresql.package attribute is deprecated. Please use services.postgresql.packages (with an 's') instead.
+          See the NixOS manual ('nixos-help') for more information.'') ++
+      (optional (cfg.extraPlugins != [])
+        ''The services.postgresql.extraPlugins attribute is deprecated. Please use services.postgresql.plugins instead,
+          which will correctly override PostgreSQL attributes for you and will provide a list of compatible plugins for
+          the given server version.
+        '');
+
+    services.postgresql.packages =
       # Note: when changing the default, make it conditional on
       # ‘system.stateVersion’ to maintain compatibility with existing
       # systems!
-      mkDefault (if versionAtLeast config.system.stateVersion "17.09" then pkgs.postgresql96
-            else if versionAtLeast config.system.stateVersion "16.03" then pkgs.postgresql95
-            else pkgs.postgresql94);
+      mkDefault (if versionAtLeast config.system.stateVersion "18.09" then pkgs.postgresql10Packages
+            else if versionAtLeast config.system.stateVersion "17.09" then pkgs.postgresql96Packages
+            else if versionAtLeast config.system.stateVersion "16.03" then pkgs.postgresql95Packages
+            else pkgs.postgresql94Packages);
 
     services.postgresql.dataDir =
-      mkDefault (if versionAtLeast config.system.stateVersion "17.09" then "/var/lib/postgresql/${config.services.postgresql.package.psqlSchema}"
+      mkDefault (if versionAtLeast config.system.stateVersion "17.09" then "/var/lib/postgresql/${cfg.postgresqlPackage.psqlSchema}"
                  else "/var/db/postgresql");
 
     services.postgresql.authentication = mkAfter
@@ -188,6 +270,8 @@ in
         uid = config.ids.uids.postgres;
         group = "postgres";
         description = "PostgreSQL server user";
+        home = "${cfg.dataDir}";
+        useDefaultShell = true;
       };
 
     users.groups.postgres.gid = config.ids.gids.postgres;
@@ -245,6 +329,8 @@ in
             # Give Postgres a decent amount of time to clean up after
             # receiving systemd's SIGINT.
             TimeoutSec = 120;
+
+            Type = if versionAtLeast cfg.postgresqlPackage.psqlSchema "9.6" then "notify" else "simple";
           };
 
         # Wait for PostgreSQL to be ready to accept connections.
@@ -269,5 +355,5 @@ in
   };
 
   meta.doc = ./postgresql.xml;
-
+  meta.maintainers = with lib.maintainers; [ thoughtpolice ];
 }