From 869bd97a6ade7787051ec0691f95fcbec3022af8 Mon Sep 17 00:00:00 2001 From: danbst Date: Fri, 18 Jan 2019 23:18:55 +0200 Subject: [PATCH 1/4] postgresql: reorganize package and it's extensions Extracts some useful parts of https://github.com/NixOS/nixpkgs/pull/38698, in particular, it's vision that postgresql plugins should be namespaced. Original approach had several problems: - not gonna happen in forseeable future - did lots of deprecations - was all-in-one solution, which is hard to sell to nixpkgs - even that we have postgresqlPackages now, we can't do arbitrary overrides to postgresql and plugins. Several required functions were not exported Here I've fixed all of those problems: - deprecates nothing (though plugins were moved now into `aliases.nix`) - this doesn't touch NixOS at all, and doesn't break anything - hashes for plugins and PGs are not changed (I hope) - no extra fixes to pg itself - default PG is not changed - plugins and PGs are extensible Last one is the most interesting thing, because it introduces novel way to manage `XXX withPackages` problem. It is novel, but has got lots of inspiration from existing approaches: - python, so we have now `postgresql.pkgs.*`, `postgresql_11.pkgs.*` which all contain plugins compiled with correct PG. - python, so we have now `postgresql.withPackages` as well - in contrast to python, there are no `postgresql11Packages`, only `postgresql_11.pkgs` - https://github.com/NixOS/nixpkgs/pull/44196, so plugins are referenced starting at self-fixpoint. This allows override/add plugins with mere `//` in overlay. This works for both `postgresqlPackages` (overrides are applied to all postgresql_xx.pkgs) and `postgresql_xx.pkgs` (overrides are specific to this postgresql) sets - I've made it compatible with proposed mergeable overlays (https://github.com/NixOS/nixpkgs/pull/54266) however this PR doesn't depend on it - last, but not least, `postgresql/default.nix` is now an overlay! This replaces previous `callPackages` approach with a modern, extensible concept. --- pkgs/servers/sql/postgresql/default.nix | 116 ++++++++++++++++++++---- pkgs/top-level/aliases.nix | 16 ++++ pkgs/top-level/all-packages.nix | 6 +- 3 files changed, 119 insertions(+), 19 deletions(-) diff --git a/pkgs/servers/sql/postgresql/default.nix b/pkgs/servers/sql/postgresql/default.nix index 2b22c36e829fc..21d0940d7b2a2 100644 --- a/pkgs/servers/sql/postgresql/default.nix +++ b/pkgs/servers/sql/postgresql/default.nix @@ -1,14 +1,22 @@ -{ lib, stdenv, glibc, fetchurl, zlib, readline, libossp_uuid, openssl, libxml2, makeWrapper, tzdata, systemd, icu, pkgconfig }: - let - common = { version, sha256, psqlSchema }: + generic = + # dependencies + { stdenv, lib, fetchurl, makeWrapper + , glibc, zlib, readline, openssl, icu, systemd, libossp_uuid + , pkgconfig, libxml2, tzdata + + # for extensible postgreql.pkgs + , postgresqlPackages, newScope, this + + # source specification + , version, sha256, psqlSchema + }: let atLeast = lib.versionAtLeast version; - - # Build with ICU by default on versions that support it icuEnabled = atLeast "10"; - in stdenv.mkDerivation (rec { + + in stdenv.mkDerivation rec { name = "postgresql-${version}"; inherit version; @@ -96,51 +104,127 @@ let disallowedReferences = [ stdenv.cc ]; - passthru = { - inherit readline psqlSchema; + passthru = rec { + inherit readline psqlSchema version; + pkgs = lib.mapAttrs (name: value: + if builtins.isAttrs value && builtins.hasAttr "override" value + then value.override { postgresql = this; } + else value) + postgresqlPackages; + callPackage = newScope (pkgs // { postgresql = this; }); + withPackages = postgresqlPackages.withPackages this (builtins.removeAttrs pkgs [ + "withPackages" "__unfix__" "extend" + ]); }; meta = with lib; { homepage = https://www.postgresql.org; description = "A powerful, open source object-relational database system"; license = licenses.postgresql; - maintainers = with maintainers; [ ocharles thoughtpolice ]; + maintainers = with maintainers; [ ocharles thoughtpolice danbst ]; platforms = platforms.unix; knownVulnerabilities = optional (!atLeast "9.4") "PostgreSQL versions older than 9.4 are not maintained anymore!"; }; + }; + +in self: super: { + + postgresqlPackages = self.lib.makeExtensible (self_: + let this = self // self.postgresqlPackages // { + callPackage = self.newScope this; + }; + in { + withPackages = postgresql: pkgs: f: self.buildEnv { + name = "postgresql-and-plugins-${postgresql.version}"; + paths = f pkgs ++ [ + postgresql + postgresql.lib + postgresql.man # in case user installs this into environment + ]; + buildInputs = [ self.makeWrapper ]; + + # We include /bin to ensure the $out/bin directory is created, which is + # needed because we'll be removing the files from that directory in postBuild + # below. See #22653 + pathsToLink = ["/" "/bin"]; + + postBuild = '' + mkdir -p $out/bin + rm $out/bin/{pg_config,postgres,pg_ctl} + cp --target-directory=$out/bin ${postgresql}/bin/{postgres,pg_config,pg_ctl} + wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib + ''; + }; + + pg_repack = this.callPackage ./pg_repack { }; + + pg_similarity = this.callPackage ./pg_similarity { }; + + pgroonga = self.callPackage ./pgroonga { }; + + plv8 = this.callPackage ./plv8 { + v8 = this.callPackage ../../../development/libraries/v8/plv8_6_x.nix { + python = this.python2; + }; + }; + + jdbc = this.callPackage ./pgjwt { }; + + pgjwt = this.callPackage ./pgjwt { }; + + cstore_fdw = this.callPackage ./cstore_fdw { }; + + pg_hll = this.callPackage ./pg_hll { }; + + pg_cron = this.callPackage ./pg_cron { }; + + pg_topn = this.callPackage ./topn { }; + + pgtap = this.callPackage ./pgtap { }; + + psqlodbc = this.callPackage ./psqlodbc { }; + + timescaledb = this.callPackage ./timescaledb { }; + + tsearch_extras = this.callPackage ./tsearch_extras { }; }); -in { + postgresql = self.postgresql_9_6; - postgresql_9_4 = common { + postgresql_9_4 = self.callPackage generic { version = "9.4.20"; psqlSchema = "9.4"; sha256 = "0zzqjz5jrn624hzh04drpj6axh30a9k6bgawid6rwk45nbfxicgf"; + this = self.postgresql_9_4; }; - postgresql_9_5 = common { + postgresql_9_5 = self.callPackage generic { version = "9.5.15"; psqlSchema = "9.5"; sha256 = "0i2lylgmsmy2g1ixlvl112fryp7jmrd0i2brk8sxb7vzzpg3znnv"; + this = self.postgresql_9_5; }; - postgresql_9_6 = common { + postgresql_9_6 = self.callPackage generic { version = "9.6.11"; psqlSchema = "9.6"; sha256 = "0c55akrkzqd6p6a8hr0338wk246hl76r9j16p4zn3s51d7f0l99q"; + this = self.postgresql_9_6; }; - postgresql_10 = common { + postgresql_10 = self.callPackage generic { version = "10.6"; psqlSchema = "10.0"; sha256 = "0jv26y3f10svrjxzsgqxg956c86b664azyk2wppzpa5x11pjga38"; + this = self.postgresql_10; }; - postgresql_11 = common { + postgresql_11 = self.callPackage generic { version = "11.1"; psqlSchema = "11.1"; sha256 = "026v0sicsh7avzi45waf8shcbhivyxmi7qgn9fd1x0vl520mx0ch"; + this = self.postgresql_11; }; -} +} \ No newline at end of file diff --git a/pkgs/top-level/aliases.nix b/pkgs/top-level/aliases.nix index e13e9adfa1657..72cdf885465ff 100644 --- a/pkgs/top-level/aliases.nix +++ b/pkgs/top-level/aliases.nix @@ -240,6 +240,22 @@ mapAliases ({ postgresql95 = postgresql_9_5; postgresql96 = postgresql_9_6; postgresql100 = throw "deprecated 2018-10-21: use postgresql_10 instead"; + # postgresql plugins + pgjwt = postgresqlPackages.pgjwt; + pg_repack = postgresqlPackages.pg_repack; + pgroonga = postgresqlPackages.pgroonga; + pg_similarity = postgresqlPackages.pg_similarity; + pgtap = postgresqlPackages.pgtap; + plv8 = postgresqlPackages.plv8; + timescaledb = postgresqlPackages.timescaledb; + tsearch_extras = postgresqlPackages.tsearch-extras; + postgresql_jdbc = postgresqlPackages.jdbc; + cstore_fdw = postgresqlPackages.cstore_fdw; + pg_hll = postgresqlPackages.pg_hll; + pg_cron = postgresqlPackages.pg_cron; + pg_topn = postgresqlPackages.pg_topn; + psqlodbc = postgresqlPackages.psqlodbc; + # end procps-ng = procps; # added 2018-06-08 prometheus-statsd-bridge = prometheus-statsd-exporter; # added 2017-08-27 pulseaudioLight = pulseaudio; # added 2018-04-25 diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index b732d930167c3..bbcec9df1c701 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -13933,9 +13933,9 @@ in timescaledb-parallel-copy = callPackage ../development/tools/database/timescaledb-parallel-copy { }; - postgresql = postgresql_9_6; - - inherit (callPackages ../servers/sql/postgresql { }) + inherit (import ../servers/sql/postgresql pkgs super) + postgresql + postgresqlPackages postgresql_9_4 postgresql_9_5 postgresql_9_6 From a88af1ce40cdb083efe6de16f42aa223d6d55171 Mon Sep 17 00:00:00 2001 From: danbst Date: Mon, 21 Jan 2019 00:20:52 +0200 Subject: [PATCH 2/4] postgresql: redo extensible packages via overlays Previous approach turned out to be awful. It was impossible to perform deep override. This time I didn't invent bycicles and used overlays for subpackages. Big change is that now overriding `postgresqlPackages` doesn't override all other package sets. But `postgresqlPackages` are now also available as an overlay! So you can get that, reorganize whatever you want and then attach to some postgresql. --- pkgs/servers/sql/postgresql/default.nix | 126 +++++++++-------------- pkgs/servers/sql/postgresql/packages.nix | 30 ++++++ pkgs/top-level/aliases.nix | 4 +- pkgs/top-level/all-packages.nix | 7 +- 4 files changed, 82 insertions(+), 85 deletions(-) create mode 100644 pkgs/servers/sql/postgresql/packages.nix diff --git a/pkgs/servers/sql/postgresql/default.nix b/pkgs/servers/sql/postgresql/default.nix index 21d0940d7b2a2..0624998e5deba 100644 --- a/pkgs/servers/sql/postgresql/default.nix +++ b/pkgs/servers/sql/postgresql/default.nix @@ -6,8 +6,8 @@ let , glibc, zlib, readline, openssl, icu, systemd, libossp_uuid , pkgconfig, libxml2, tzdata - # for extensible postgreql.pkgs - , postgresqlPackages, newScope, this + # for postgreql.pkgs + , this, self, newScope, buildEnv # source specification , version, sha256, psqlSchema @@ -104,17 +104,20 @@ let disallowedReferences = [ stdenv.cc ]; - passthru = rec { + passthru = { inherit readline psqlSchema version; - pkgs = lib.mapAttrs (name: value: - if builtins.isAttrs value && builtins.hasAttr "override" value - then value.override { postgresql = this; } - else value) - postgresqlPackages; - callPackage = newScope (pkgs // { postgresql = this; }); - withPackages = postgresqlPackages.withPackages this (builtins.removeAttrs pkgs [ - "withPackages" "__unfix__" "extend" - ]); + + pkgs = let + scope = { postgresql = this; }; + newSelf = self // scope; + newSuper = { callPackage = newScope (scope // this.pkgs); }; + in import ./packages.nix newSelf newSuper; + + withPackages = postgresqlWithPackages { + inherit makeWrapper buildEnv; + postgresql = this; + } + this.pkgs; }; meta = with lib; { @@ -128,103 +131,68 @@ let }; }; -in self: super: { - - postgresqlPackages = self.lib.makeExtensible (self_: - let this = self // self.postgresqlPackages // { - callPackage = self.newScope this; - }; - in { - withPackages = postgresql: pkgs: f: self.buildEnv { - name = "postgresql-and-plugins-${postgresql.version}"; - paths = f pkgs ++ [ - postgresql - postgresql.lib - postgresql.man # in case user installs this into environment - ]; - buildInputs = [ self.makeWrapper ]; - - # We include /bin to ensure the $out/bin directory is created, which is - # needed because we'll be removing the files from that directory in postBuild - # below. See #22653 - pathsToLink = ["/" "/bin"]; - - postBuild = '' - mkdir -p $out/bin - rm $out/bin/{pg_config,postgres,pg_ctl} - cp --target-directory=$out/bin ${postgresql}/bin/{postgres,pg_config,pg_ctl} - wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib - ''; - }; - - pg_repack = this.callPackage ./pg_repack { }; - - pg_similarity = this.callPackage ./pg_similarity { }; - - pgroonga = self.callPackage ./pgroonga { }; - - plv8 = this.callPackage ./plv8 { - v8 = this.callPackage ../../../development/libraries/v8/plv8_6_x.nix { - python = this.python2; - }; - }; - - jdbc = this.callPackage ./pgjwt { }; - - pgjwt = this.callPackage ./pgjwt { }; - - cstore_fdw = this.callPackage ./cstore_fdw { }; - - pg_hll = this.callPackage ./pg_hll { }; - - pg_cron = this.callPackage ./pg_cron { }; - - pg_topn = this.callPackage ./topn { }; - - pgtap = this.callPackage ./pgtap { }; - - psqlodbc = this.callPackage ./psqlodbc { }; - - timescaledb = this.callPackage ./timescaledb { }; - - tsearch_extras = this.callPackage ./tsearch_extras { }; - }); + postgresqlWithPackages = { postgresql, makeWrapper, buildEnv }: pkgs: f: buildEnv { + name = "postgresql-and-plugins-${postgresql.version}"; + paths = f pkgs ++ [ + postgresql + postgresql.lib + postgresql.man # in case user installs this into environment + ]; + buildInputs = [ makeWrapper ]; + + # We include /bin to ensure the $out/bin directory is created, which is + # needed because we'll be removing the files from that directory in postBuild + # below. See #22653 + pathsToLink = ["/" "/bin"]; + + postBuild = '' + mkdir -p $out/bin + rm $out/bin/{pg_config,postgres,pg_ctl} + cp --target-directory=$out/bin ${postgresql}/bin/{postgres,pg_config,pg_ctl} + wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib + ''; + }; - postgresql = self.postgresql_9_6; +in self: super: { - postgresql_9_4 = self.callPackage generic { + postgresql_9_4 = super.callPackage generic { version = "9.4.20"; psqlSchema = "9.4"; sha256 = "0zzqjz5jrn624hzh04drpj6axh30a9k6bgawid6rwk45nbfxicgf"; this = self.postgresql_9_4; + inherit self; }; - postgresql_9_5 = self.callPackage generic { + postgresql_9_5 = super.callPackage generic { version = "9.5.15"; psqlSchema = "9.5"; sha256 = "0i2lylgmsmy2g1ixlvl112fryp7jmrd0i2brk8sxb7vzzpg3znnv"; this = self.postgresql_9_5; + inherit self; }; - postgresql_9_6 = self.callPackage generic { + postgresql_9_6 = super.callPackage generic { version = "9.6.11"; psqlSchema = "9.6"; sha256 = "0c55akrkzqd6p6a8hr0338wk246hl76r9j16p4zn3s51d7f0l99q"; this = self.postgresql_9_6; + inherit self; }; - postgresql_10 = self.callPackage generic { + postgresql_10 = super.callPackage generic { version = "10.6"; psqlSchema = "10.0"; sha256 = "0jv26y3f10svrjxzsgqxg956c86b664azyk2wppzpa5x11pjga38"; this = self.postgresql_10; + inherit self; }; - postgresql_11 = self.callPackage generic { + postgresql_11 = super.callPackage generic { version = "11.1"; psqlSchema = "11.1"; sha256 = "026v0sicsh7avzi45waf8shcbhivyxmi7qgn9fd1x0vl520mx0ch"; this = self.postgresql_11; + inherit self; }; } \ No newline at end of file diff --git a/pkgs/servers/sql/postgresql/packages.nix b/pkgs/servers/sql/postgresql/packages.nix new file mode 100644 index 0000000000000..d6e86005d27f8 --- /dev/null +++ b/pkgs/servers/sql/postgresql/packages.nix @@ -0,0 +1,30 @@ +self: super: { + + pg_repack = super.callPackage ./pg_repack { }; + + pg_similarity = super.callPackage ./pg_similarity { }; + + pgroonga = super.callPackage ./pgroonga { }; + + plv8 = super.callPackage ./plv8 { + v8 = super.callPackage ../../../development/libraries/v8/plv8_6_x.nix { + python = self.python2; + }; + }; + + pgjwt = super.callPackage ./pgjwt { }; + + cstore_fdw = super.callPackage ./cstore_fdw { }; + + pg_hll = super.callPackage ./pg_hll { }; + + pg_cron = super.callPackage ./pg_cron { }; + + pg_topn = super.callPackage ./topn { }; + + pgtap = super.callPackage ./pgtap { }; + + timescaledb = super.callPackage ./timescaledb { }; + + tsearch_extras = super.callPackage ./tsearch_extras { }; +} \ No newline at end of file diff --git a/pkgs/top-level/aliases.nix b/pkgs/top-level/aliases.nix index 72cdf885465ff..fcabdf7c69a8b 100644 --- a/pkgs/top-level/aliases.nix +++ b/pkgs/top-level/aliases.nix @@ -248,13 +248,11 @@ mapAliases ({ pgtap = postgresqlPackages.pgtap; plv8 = postgresqlPackages.plv8; timescaledb = postgresqlPackages.timescaledb; - tsearch_extras = postgresqlPackages.tsearch-extras; - postgresql_jdbc = postgresqlPackages.jdbc; + tsearch_extras = postgresqlPackages.tsearch_extras; cstore_fdw = postgresqlPackages.cstore_fdw; pg_hll = postgresqlPackages.pg_hll; pg_cron = postgresqlPackages.pg_cron; pg_topn = postgresqlPackages.pg_topn; - psqlodbc = postgresqlPackages.psqlodbc; # end procps-ng = procps; # added 2018-06-08 prometheus-statsd-bridge = prometheus-statsd-exporter; # added 2017-08-27 diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index bbcec9df1c701..eb190732a8a48 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -13934,13 +13934,14 @@ in timescaledb-parallel-copy = callPackage ../development/tools/database/timescaledb-parallel-copy { }; inherit (import ../servers/sql/postgresql pkgs super) - postgresql - postgresqlPackages postgresql_9_4 postgresql_9_5 postgresql_9_6 postgresql_10 - postgresql_11; + postgresql_11 + ; + postgresql = postgresql_9_6.override { this = postgresql; }; + postgresqlPackages = recurseIntoAttrs postgresql.pkgs; postgresql_jdbc = callPackage ../development/java-modules/postgresql_jdbc { }; From 8a9e8f49550effb83103481584a0f90052dd121c Mon Sep 17 00:00:00 2001 From: danbst Date: Mon, 21 Jan 2019 00:47:12 +0200 Subject: [PATCH 3/4] fix postgresql test --- nixos/tests/postgresql.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nixos/tests/postgresql.nix b/nixos/tests/postgresql.nix index 1d434b62a5cb5..c28c13043adde 100644 --- a/nixos/tests/postgresql.nix +++ b/nixos/tests/postgresql.nix @@ -7,7 +7,7 @@ with import ../lib/testing.nix { inherit system pkgs; }; with pkgs.lib; let - postgresql-versions = pkgs.callPackages ../../pkgs/servers/sql/postgresql { }; + postgresql-versions = import ../../pkgs/servers/sql/postgresql pkgs pkgs; test-sql = pkgs.writeText "postgresql-test" '' CREATE EXTENSION pgcrypto; -- just to check if lib loading works CREATE TABLE sth ( @@ -29,8 +29,8 @@ let machine = {...}: { - services.postgresql.package=postgresql-package; services.postgresql.enable = true; + services.postgresql.package = postgresql-package; services.postgresqlBackup.enable = true; services.postgresqlBackup.databases = [ "postgres" ]; From 9e8f256c12ef007c56d9649405f01875f582efbc Mon Sep 17 00:00:00 2001 From: danbst Date: Mon, 21 Jan 2019 01:06:38 +0200 Subject: [PATCH 4/4] rebase to master --- pkgs/servers/sql/postgresql/packages.nix | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pkgs/servers/sql/postgresql/packages.nix b/pkgs/servers/sql/postgresql/packages.nix index d6e86005d27f8..ba3da25bc16be 100644 --- a/pkgs/servers/sql/postgresql/packages.nix +++ b/pkgs/servers/sql/postgresql/packages.nix @@ -1,30 +1,30 @@ self: super: { - pg_repack = super.callPackage ./pg_repack { }; + pg_repack = super.callPackage ./ext/pg_repack.nix { }; - pg_similarity = super.callPackage ./pg_similarity { }; + pg_similarity = super.callPackage ./ext/pg_similarity.nix { }; - pgroonga = super.callPackage ./pgroonga { }; + pgroonga = super.callPackage ./ext/pgroonga.nix { }; - plv8 = super.callPackage ./plv8 { + plv8 = super.callPackage ./ext/plv8.nix { v8 = super.callPackage ../../../development/libraries/v8/plv8_6_x.nix { python = self.python2; }; }; - pgjwt = super.callPackage ./pgjwt { }; + pgjwt = super.callPackage ./ext/pgjwt.nix { }; - cstore_fdw = super.callPackage ./cstore_fdw { }; + cstore_fdw = super.callPackage ./ext/cstore_fdw.nix { }; - pg_hll = super.callPackage ./pg_hll { }; + pg_hll = super.callPackage ./ext/pg_hll.nix { }; - pg_cron = super.callPackage ./pg_cron { }; + pg_cron = super.callPackage ./ext/pg_cron.nix { }; - pg_topn = super.callPackage ./topn { }; + pg_topn = super.callPackage ./ext/pg_topn.nix { }; - pgtap = super.callPackage ./pgtap { }; + pgtap = super.callPackage ./ext/pgtap.nix { }; - timescaledb = super.callPackage ./timescaledb { }; + timescaledb = super.callPackage ./ext/timescaledb.nix { }; - tsearch_extras = super.callPackage ./tsearch_extras { }; + tsearch_extras = super.callPackage ./ext/tsearch_extras.nix { }; } \ No newline at end of file