From 677c607d63f4615372f5a9bca2e3c24ad7eb7fbb Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:17:47 -0500 Subject: [PATCH 01/61] nixpkgs: deprecate the badly named 'postgresql100' attribute See #38616. This is a minor issue, and for now we keep around the old attribute name with a warning to tell people how to switch. Signed-off-by: Austin Seipp --- nixos/tests/postgis.nix | 2 +- pkgs/servers/sql/postgresql/default.nix | 6 +++++- pkgs/top-level/all-packages.nix | 9 ++++++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/nixos/tests/postgis.nix b/nixos/tests/postgis.nix index f8b63c5b6a27b..0ff880e7aa527 100644 --- a/nixos/tests/postgis.nix +++ b/nixos/tests/postgis.nix @@ -9,7 +9,7 @@ import ./make-test.nix ({ pkgs, ...} : { { pkgs, ... }: { - services.postgresql = let mypg = pkgs.postgresql100; in { + services.postgresql = let mypg = pkgs.postgresql10; in { enable = true; package = mypg; extraPlugins = [ (pkgs.postgis.override { postgresql = mypg; }) ]; diff --git a/pkgs/servers/sql/postgresql/default.nix b/pkgs/servers/sql/postgresql/default.nix index bccd54ca4ba89..ecbaa1cc1b08c 100644 --- a/pkgs/servers/sql/postgresql/default.nix +++ b/pkgs/servers/sql/postgresql/default.nix @@ -123,7 +123,11 @@ in { sha256 = "09l4zqs74fqnazdsyln9x657mq3wsbgng9wpvq71yh26cv2sq5c6"; }; - postgresql100 = common { + # NOTE: starting with PostgreSQL 10, the versioning scheme changed from + # .. to .. Thus there is no longer a + # 3rd component, and we should name attributes following only the major + # number. + postgresql10 = common { version = "10.5"; psqlSchema = "10.0"; sha256 = "04a07jkvc5s6zgh6jr78149kcjmsxclizsqabjw44ld4j5n633kc"; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 182eb55557e97..92f56dab7891e 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -13357,7 +13357,14 @@ with pkgs; postgresql94 postgresql95 postgresql96 - postgresql100; + postgresql10; + + # TODO FIXME: this name is a misnomer (see #38616), but might be in use in + # the wild. leave it for now, but remove this after 18.09 is released; it was + # already in 18.03 and it's too late for that, so we should keep it around + # for at least one more release as the 10.x series sees more updates. + postgresql100 = builtins.trace "The 'postgresql100' attribute was badly misnamed, and is deprecated; use 'postgresql10' instead" + postgresql10; postgresql_jdbc = callPackage ../servers/sql/postgresql/jdbc { }; From ac197bee74ca6f1017c43f30675b3bf211318a08 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:17:49 -0500 Subject: [PATCH 02/61] nixpkgs: move postgresql patches into a common directory This simply makes it easier to browse the PostgreSQL package directory. More to come. Signed-off-by: Austin Seipp --- pkgs/servers/sql/postgresql/default.nix | 8 ++++---- .../{ => patches}/disable-resolve_symlinks-94.patch | 0 .../{ => patches}/disable-resolve_symlinks.patch | 0 .../postgresql/{ => patches}/hardcode-pgxs-path-96.patch | 0 .../sql/postgresql/{ => patches}/hardcode-pgxs-path.patch | 0 .../sql/postgresql/{ => patches}/less-is-more-96.patch | 0 .../sql/postgresql/{ => patches}/less-is-more.patch | 0 .../{ => patches}/specify_pkglibdir_at_runtime.patch | 0 8 files changed, 4 insertions(+), 4 deletions(-) rename pkgs/servers/sql/postgresql/{ => patches}/disable-resolve_symlinks-94.patch (100%) rename pkgs/servers/sql/postgresql/{ => patches}/disable-resolve_symlinks.patch (100%) rename pkgs/servers/sql/postgresql/{ => patches}/hardcode-pgxs-path-96.patch (100%) rename pkgs/servers/sql/postgresql/{ => patches}/hardcode-pgxs-path.patch (100%) rename pkgs/servers/sql/postgresql/{ => patches}/less-is-more-96.patch (100%) rename pkgs/servers/sql/postgresql/{ => patches}/less-is-more.patch (100%) rename pkgs/servers/sql/postgresql/{ => patches}/specify_pkglibdir_at_runtime.patch (100%) diff --git a/pkgs/servers/sql/postgresql/default.nix b/pkgs/servers/sql/postgresql/default.nix index ecbaa1cc1b08c..07473145f6920 100644 --- a/pkgs/servers/sql/postgresql/default.nix +++ b/pkgs/servers/sql/postgresql/default.nix @@ -37,10 +37,10 @@ let ]; patches = - [ (if atLeast "9.4" then ./disable-resolve_symlinks-94.patch else ./disable-resolve_symlinks.patch) - (if atLeast "9.6" then ./less-is-more-96.patch else ./less-is-more.patch) - (if atLeast "9.6" then ./hardcode-pgxs-path-96.patch else ./hardcode-pgxs-path.patch) - ./specify_pkglibdir_at_runtime.patch + [ (if atLeast "9.4" then ./patches/disable-resolve_symlinks-94.patch else ./patches/disable-resolve_symlinks.patch) + (if atLeast "9.6" then ./patches/less-is-more-96.patch else ./patches/less-is-more.patch) + (if atLeast "9.6" then ./patches/hardcode-pgxs-path-96.patch else ./patches/hardcode-pgxs-path.patch) + ./patches/specify_pkglibdir_at_runtime.patch ]; installTargets = [ "install-world" ]; diff --git a/pkgs/servers/sql/postgresql/disable-resolve_symlinks-94.patch b/pkgs/servers/sql/postgresql/patches/disable-resolve_symlinks-94.patch similarity index 100% rename from pkgs/servers/sql/postgresql/disable-resolve_symlinks-94.patch rename to pkgs/servers/sql/postgresql/patches/disable-resolve_symlinks-94.patch diff --git a/pkgs/servers/sql/postgresql/disable-resolve_symlinks.patch b/pkgs/servers/sql/postgresql/patches/disable-resolve_symlinks.patch similarity index 100% rename from pkgs/servers/sql/postgresql/disable-resolve_symlinks.patch rename to pkgs/servers/sql/postgresql/patches/disable-resolve_symlinks.patch diff --git a/pkgs/servers/sql/postgresql/hardcode-pgxs-path-96.patch b/pkgs/servers/sql/postgresql/patches/hardcode-pgxs-path-96.patch similarity index 100% rename from pkgs/servers/sql/postgresql/hardcode-pgxs-path-96.patch rename to pkgs/servers/sql/postgresql/patches/hardcode-pgxs-path-96.patch diff --git a/pkgs/servers/sql/postgresql/hardcode-pgxs-path.patch b/pkgs/servers/sql/postgresql/patches/hardcode-pgxs-path.patch similarity index 100% rename from pkgs/servers/sql/postgresql/hardcode-pgxs-path.patch rename to pkgs/servers/sql/postgresql/patches/hardcode-pgxs-path.patch diff --git a/pkgs/servers/sql/postgresql/less-is-more-96.patch b/pkgs/servers/sql/postgresql/patches/less-is-more-96.patch similarity index 100% rename from pkgs/servers/sql/postgresql/less-is-more-96.patch rename to pkgs/servers/sql/postgresql/patches/less-is-more-96.patch diff --git a/pkgs/servers/sql/postgresql/less-is-more.patch b/pkgs/servers/sql/postgresql/patches/less-is-more.patch similarity index 100% rename from pkgs/servers/sql/postgresql/less-is-more.patch rename to pkgs/servers/sql/postgresql/patches/less-is-more.patch diff --git a/pkgs/servers/sql/postgresql/specify_pkglibdir_at_runtime.patch b/pkgs/servers/sql/postgresql/patches/specify_pkglibdir_at_runtime.patch similarity index 100% rename from pkgs/servers/sql/postgresql/specify_pkglibdir_at_runtime.patch rename to pkgs/servers/sql/postgresql/patches/specify_pkglibdir_at_runtime.patch From 6d00a38ec970357742aee9ee6c2ef78a42b91636 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:17:52 -0500 Subject: [PATCH 03/61] nixpkgs: reorganize the postgresql extensions This is a backwards compatible change; it mostly puts all the extensions for postgresql in a common directory to keep them isolated. It also moves a few things that /were not/ extensions out into other parts of the filesystem namespace; namely the postgresql_jdbc and psqlodbc libraries were moved under development/java-modules and development/libraries, respectively. Because these libraries use the libpq postgresql client drivers, they're less sensitive to underlying version changes anyway (since the protocol is relatively stable). No attributes were renamed or harmed in the creation of this patch. Signed-off-by: Austin Seipp --- .../java-modules/postgresql_jdbc}/default.nix | 0 .../libraries}/psqlodbc/default.nix | 0 .../default.nix => ext/cstore_fdw.nix} | 0 .../{pg_cron/default.nix => ext/pg_cron.nix} | 0 .../{pg_hll/default.nix => ext/pg_hll.nix} | 0 .../default.nix => ext/pg_repack.nix} | 0 .../default.nix => ext/pg_similarity.nix} | 0 .../{topn/default.nix => ext/pg_topn.nix} | 0 .../{pgjwt/default.nix => ext/pgjwt.nix} | 0 .../default.nix => ext/pgroonga.nix} | 0 .../{pgtap/default.nix => ext/pgtap.nix} | 0 .../{plv8/default.nix => ext/plv8.nix} | 0 .../default.nix => ext/timescaledb.nix} | 0 .../default.nix => ext/tsearch_extras.nix} | 0 pkgs/top-level/all-packages.nix | 47 ++++++++++--------- 15 files changed, 25 insertions(+), 22 deletions(-) rename pkgs/{servers/sql/postgresql/jdbc => development/java-modules/postgresql_jdbc}/default.nix (100%) rename pkgs/{servers/sql/postgresql => development/libraries}/psqlodbc/default.nix (100%) rename pkgs/servers/sql/postgresql/{cstore_fdw/default.nix => ext/cstore_fdw.nix} (100%) rename pkgs/servers/sql/postgresql/{pg_cron/default.nix => ext/pg_cron.nix} (100%) rename pkgs/servers/sql/postgresql/{pg_hll/default.nix => ext/pg_hll.nix} (100%) rename pkgs/servers/sql/postgresql/{pg_repack/default.nix => ext/pg_repack.nix} (100%) rename pkgs/servers/sql/postgresql/{pg_similarity/default.nix => ext/pg_similarity.nix} (100%) rename pkgs/servers/sql/postgresql/{topn/default.nix => ext/pg_topn.nix} (100%) rename pkgs/servers/sql/postgresql/{pgjwt/default.nix => ext/pgjwt.nix} (100%) rename pkgs/servers/sql/postgresql/{pgroonga/default.nix => ext/pgroonga.nix} (100%) rename pkgs/servers/sql/postgresql/{pgtap/default.nix => ext/pgtap.nix} (100%) rename pkgs/servers/sql/postgresql/{plv8/default.nix => ext/plv8.nix} (100%) rename pkgs/servers/sql/postgresql/{timescaledb/default.nix => ext/timescaledb.nix} (100%) rename pkgs/servers/sql/postgresql/{tsearch_extras/default.nix => ext/tsearch_extras.nix} (100%) diff --git a/pkgs/servers/sql/postgresql/jdbc/default.nix b/pkgs/development/java-modules/postgresql_jdbc/default.nix similarity index 100% rename from pkgs/servers/sql/postgresql/jdbc/default.nix rename to pkgs/development/java-modules/postgresql_jdbc/default.nix diff --git a/pkgs/servers/sql/postgresql/psqlodbc/default.nix b/pkgs/development/libraries/psqlodbc/default.nix similarity index 100% rename from pkgs/servers/sql/postgresql/psqlodbc/default.nix rename to pkgs/development/libraries/psqlodbc/default.nix diff --git a/pkgs/servers/sql/postgresql/cstore_fdw/default.nix b/pkgs/servers/sql/postgresql/ext/cstore_fdw.nix similarity index 100% rename from pkgs/servers/sql/postgresql/cstore_fdw/default.nix rename to pkgs/servers/sql/postgresql/ext/cstore_fdw.nix diff --git a/pkgs/servers/sql/postgresql/pg_cron/default.nix b/pkgs/servers/sql/postgresql/ext/pg_cron.nix similarity index 100% rename from pkgs/servers/sql/postgresql/pg_cron/default.nix rename to pkgs/servers/sql/postgresql/ext/pg_cron.nix diff --git a/pkgs/servers/sql/postgresql/pg_hll/default.nix b/pkgs/servers/sql/postgresql/ext/pg_hll.nix similarity index 100% rename from pkgs/servers/sql/postgresql/pg_hll/default.nix rename to pkgs/servers/sql/postgresql/ext/pg_hll.nix diff --git a/pkgs/servers/sql/postgresql/pg_repack/default.nix b/pkgs/servers/sql/postgresql/ext/pg_repack.nix similarity index 100% rename from pkgs/servers/sql/postgresql/pg_repack/default.nix rename to pkgs/servers/sql/postgresql/ext/pg_repack.nix diff --git a/pkgs/servers/sql/postgresql/pg_similarity/default.nix b/pkgs/servers/sql/postgresql/ext/pg_similarity.nix similarity index 100% rename from pkgs/servers/sql/postgresql/pg_similarity/default.nix rename to pkgs/servers/sql/postgresql/ext/pg_similarity.nix diff --git a/pkgs/servers/sql/postgresql/topn/default.nix b/pkgs/servers/sql/postgresql/ext/pg_topn.nix similarity index 100% rename from pkgs/servers/sql/postgresql/topn/default.nix rename to pkgs/servers/sql/postgresql/ext/pg_topn.nix diff --git a/pkgs/servers/sql/postgresql/pgjwt/default.nix b/pkgs/servers/sql/postgresql/ext/pgjwt.nix similarity index 100% rename from pkgs/servers/sql/postgresql/pgjwt/default.nix rename to pkgs/servers/sql/postgresql/ext/pgjwt.nix diff --git a/pkgs/servers/sql/postgresql/pgroonga/default.nix b/pkgs/servers/sql/postgresql/ext/pgroonga.nix similarity index 100% rename from pkgs/servers/sql/postgresql/pgroonga/default.nix rename to pkgs/servers/sql/postgresql/ext/pgroonga.nix diff --git a/pkgs/servers/sql/postgresql/pgtap/default.nix b/pkgs/servers/sql/postgresql/ext/pgtap.nix similarity index 100% rename from pkgs/servers/sql/postgresql/pgtap/default.nix rename to pkgs/servers/sql/postgresql/ext/pgtap.nix diff --git a/pkgs/servers/sql/postgresql/plv8/default.nix b/pkgs/servers/sql/postgresql/ext/plv8.nix similarity index 100% rename from pkgs/servers/sql/postgresql/plv8/default.nix rename to pkgs/servers/sql/postgresql/ext/plv8.nix diff --git a/pkgs/servers/sql/postgresql/timescaledb/default.nix b/pkgs/servers/sql/postgresql/ext/timescaledb.nix similarity index 100% rename from pkgs/servers/sql/postgresql/timescaledb/default.nix rename to pkgs/servers/sql/postgresql/ext/timescaledb.nix diff --git a/pkgs/servers/sql/postgresql/tsearch_extras/default.nix b/pkgs/servers/sql/postgresql/ext/tsearch_extras.nix similarity index 100% rename from pkgs/servers/sql/postgresql/tsearch_extras/default.nix rename to pkgs/servers/sql/postgresql/ext/tsearch_extras.nix diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 92f56dab7891e..80ba0a8f8759c 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -3034,17 +3034,34 @@ with pkgs; pgf_graphics = callPackage ../tools/graphics/pgf { }; - pgjwt = callPackage ../servers/sql/postgresql/pgjwt {}; - cstore_fdw = callPackage ../servers/sql/postgresql/cstore_fdw {}; + # postgresql extensions - pg_hll = callPackage ../servers/sql/postgresql/pg_hll {}; + cstore_fdw = callPackage ../servers/sql/postgresql/ext/cstore_fdw.nix {}; - pg_cron = callPackage ../servers/sql/postgresql/pg_cron {}; + pg_cron = callPackage ../servers/sql/postgresql/ext/pg_cron.nix {}; - pgtap = callPackage ../servers/sql/postgresql/pgtap {}; + pg_hll = callPackage ../servers/sql/postgresql/ext/pg_hll.nix {}; - pg_topn = callPackage ../servers/sql/postgresql/topn {}; + pgjwt = callPackage ../servers/sql/postgresql/ext/pgjwt.nix {}; + + pg_repack = callPackage ../servers/sql/postgresql/ext/pg_repack.nix {}; + + pgroonga = callPackage ../servers/sql/postgresql/ext/pgroonga.nix {}; + + pg_similarity = callPackage ../servers/sql/postgresql/ext/pg_similarity.nix {}; + + pgtap = callPackage ../servers/sql/postgresql/ext/pgtap.nix {}; + + pg_topn = callPackage ../servers/sql/postgresql/ext/pg_topn.nix {}; + + plv8 = callPackage ../servers/sql/postgresql/ext/plv8.nix { + v8 = v8_6_x; + }; + + timescaledb = callPackage ../servers/sql/postgresql/ext/timescaledb.nix {}; + + tsearch_extras = callPackage ../servers/sql/postgresql/ext/tsearch_extras.nix { }; pigz = callPackage ../tools/compression/pigz { }; @@ -11501,18 +11518,8 @@ with pkgs; pdf2xml = callPackage ../development/libraries/pdf2xml {} ; - pg_repack = callPackage ../servers/sql/postgresql/pg_repack {}; - - pg_similarity = callPackage ../servers/sql/postgresql/pg_similarity {}; - pg_tmp = callPackage ../development/tools/database/pg_tmp { }; - pgroonga = callPackage ../servers/sql/postgresql/pgroonga {}; - - plv8 = callPackage ../servers/sql/postgresql/plv8 { - v8 = v8_6_x; - }; - phonon = callPackage ../development/libraries/phonon {}; phonon-backend-gstreamer = callPackage ../development/libraries/phonon/backends/gstreamer.nix {}; @@ -13366,7 +13373,7 @@ with pkgs; postgresql100 = builtins.trace "The 'postgresql100' attribute was badly misnamed, and is deprecated; use 'postgresql10' instead" postgresql10; - postgresql_jdbc = callPackage ../servers/sql/postgresql/jdbc { }; + postgresql_jdbc = callPackage ../development/java-modules/postgresql_jdbc { }; inherit (callPackage ../servers/monitoring/prometheus { buildGoPackage = buildGo110Package; @@ -13405,7 +13412,7 @@ with pkgs; prometheus-varnish-exporter = callPackage ../servers/monitoring/prometheus/varnish-exporter.nix { }; prometheus-jmx-httpserver = callPackage ../servers/monitoring/prometheus/jmx-httpserver.nix { }; - psqlodbc = callPackage ../servers/sql/postgresql/psqlodbc { }; + psqlodbc = callPackage ../development/libraries/psqlodbc { }; pure-ftpd = callPackage ../servers/ftp/pure-ftpd { }; @@ -18872,8 +18879,6 @@ with pkgs; fftw = fftwSinglePrec; }; - timescaledb = callPackage ../servers/sql/postgresql/timescaledb {}; - timewarrior = callPackage ../applications/misc/timewarrior { }; timidity = callPackage ../tools/misc/timidity { }; @@ -18957,8 +18962,6 @@ with pkgs; trojita = libsForQt5.callPackage ../applications/networking/mailreaders/trojita { }; - tsearch_extras = callPackage ../servers/sql/postgresql/tsearch_extras { }; - tudu = callPackage ../applications/office/tudu { }; tuxguitar = callPackage ../applications/editors/music/tuxguitar { }; From abf7e3c583a78e040e61184846d98b42fcbca0bf Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:17:54 -0500 Subject: [PATCH 04/61] nixpkgs: tidy up pgjwt package This makes pgjwt take a dummy 'postgresql' argument, which it does not *need* in the buildInputs (it is purely a SQL extension with no C code). However, this argument will be necessary for an upcoming change that will parameterize the extensions over a particular PostgreSQL version. It also does some tiny cleanup, setting a null build phase. Signed-off-by: Austin Seipp --- pkgs/servers/sql/postgresql/ext/pgjwt.nix | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/pkgs/servers/sql/postgresql/ext/pgjwt.nix b/pkgs/servers/sql/postgresql/ext/pgjwt.nix index 3e01d9bfe2ca0..ab7ba8943a7bf 100644 --- a/pkgs/servers/sql/postgresql/ext/pgjwt.nix +++ b/pkgs/servers/sql/postgresql/ext/pgjwt.nix @@ -1,19 +1,23 @@ -{ stdenv, fetchFromGitHub }: +{ stdenv, fetchFromGitHub, postgresql }: + stdenv.mkDerivation rec { - name = "pgjwt-${version}"; + name = "pgjwt-${version}"; version = "unstable-2017-04-24"; + src = fetchFromGitHub { - owner = "michelp"; - repo = "pgjwt"; - rev = "546a2911027b716586e241be7fd4c6f1785237cd"; + owner = "michelp"; + repo = "pgjwt"; + rev = "546a2911027b716586e241be7fd4c6f1785237cd"; sha256 = "1riz0xvwb6y02j0fljbr9hcbqb2jqs4njlivmavy9ysbcrrv1vrf"; }; - dontBuild = true; + + buildPhase = ":"; installPhase = '' mkdir -p $out/bin # current postgresql extension mechanism in nixos requires bin directory mkdir -p $out/share/extension cp pg*sql *.control $out/share/extension ''; + meta = with stdenv.lib; { description = "PostgreSQL implementation of JSON Web Tokens"; longDescription = '' From 3cbd0b267cc85a80f50ec2eeba52d3c2772e8fee Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:17:56 -0500 Subject: [PATCH 05/61] nixpkgs: implement new postgresqlPackages infrastructure This adds five new top-level attrsets: pkgs.postgresql93Packages pkgs.postgresql94Packages pkgs.postgresql95Packages pkgs.postgresql96Packages pkgs.postgresql10Packages Each of these contains a set of PostgreSQL extensions, built against the particular specified postgresql package. This makes it far easier for specific PostgreSQL extensions to be built and used in NixOS modules, and provides an easier way to test extension builds against multiple versions. For example, we can now use either: pkgs.postgresql96Packages.timescaledb or pkgs.postgresql10Packages.timescaledb when building and specifying extensions. By default, the prior top-level attribute names such as 'timescaledb' HAVE NOT been removed. They will be deprecated in a future commit to encourage people to move to explicitly-versioned attrsets. Signed-off-by: Austin Seipp --- pkgs/servers/sql/postgresql/packages.nix | 65 ++++++++++++++++++ pkgs/top-level/all-packages.nix | 85 +++++++++++++----------- 2 files changed, 111 insertions(+), 39 deletions(-) create mode 100644 pkgs/servers/sql/postgresql/packages.nix diff --git a/pkgs/servers/sql/postgresql/packages.nix b/pkgs/servers/sql/postgresql/packages.nix new file mode 100644 index 0000000000000..99a2ecf8f2ecc --- /dev/null +++ b/pkgs/servers/sql/postgresql/packages.nix @@ -0,0 +1,65 @@ +{ pkgs }: + +let + postgresqlPackages = pkgs.callPackages ./default.nix {}; + + makePackageSet = postgresql: + let + callPackage = p: args: pkgs.callPackage p (args // { inherit postgresql; }); + in + { + # Convenience function for end-users to easily build packages against a specific + # version. + inherit callPackage; + + # Convenience function for top-level attributes that were previously available, + # but are deprecated and renamed. TODO FIXME: remove this after 18.09 is released. + # see all-packages.nix (in the postgresql section) for more + callPackageWithDeprecation = name: p: args: + builtins.trace '' + The PostgreSQL-related attribute '${name}' is deprecated. Please use the + 'postgresqlPackages.${name}' attribute instead (to use the default version of + PostgreSQL in Nixpkgs), or use 'postgresqlPackages.${name}' for a given + version . + '' (callPackage p args); + + # Convenience attribute that exports the postgres derivation used for builds. We + # export these from all-packages.nix to consolidate everything here. + inherit postgresql; + + # PostgreSQL extensions follow from here. + + cstore_fdw = callPackage ./ext/cstore_fdw.nix {}; + + pg_cron = callPackage ./ext/pg_cron.nix {}; + + pg_hll = callPackage ./ext/pg_hll.nix {}; + + pgjwt = callPackage ./ext/pgjwt.nix {}; + + pg_repack = callPackage ./ext/pg_repack.nix {}; + + pgroonga = callPackage ./ext/pgroonga.nix {}; + + pg_similarity = callPackage ./ext/pg_similarity.nix {}; + + pgtap = callPackage ./ext/pgtap.nix {}; + + pg_topn = callPackage ./ext/pg_topn.nix {}; + + plv8 = callPackage ./ext/plv8.nix { + v8 = pkgs.v8_6_x; + }; + + timescaledb = callPackage ./ext/timescaledb.nix {}; + + tsearch_extras = callPackage ./ext/tsearch_extras.nix { }; + }; +in +with postgresqlPackages; { + postgresql93Packages = makePackageSet postgresql93; + postgresql94Packages = makePackageSet postgresql94; + postgresql95Packages = makePackageSet postgresql95; + postgresql96Packages = makePackageSet postgresql96; + postgresql10Packages = makePackageSet postgresql10; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 80ba0a8f8759c..ca624ead5465b 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -3034,35 +3034,6 @@ with pkgs; pgf_graphics = callPackage ../tools/graphics/pgf { }; - - # postgresql extensions - - cstore_fdw = callPackage ../servers/sql/postgresql/ext/cstore_fdw.nix {}; - - pg_cron = callPackage ../servers/sql/postgresql/ext/pg_cron.nix {}; - - pg_hll = callPackage ../servers/sql/postgresql/ext/pg_hll.nix {}; - - pgjwt = callPackage ../servers/sql/postgresql/ext/pgjwt.nix {}; - - pg_repack = callPackage ../servers/sql/postgresql/ext/pg_repack.nix {}; - - pgroonga = callPackage ../servers/sql/postgresql/ext/pgroonga.nix {}; - - pg_similarity = callPackage ../servers/sql/postgresql/ext/pg_similarity.nix {}; - - pgtap = callPackage ../servers/sql/postgresql/ext/pgtap.nix {}; - - pg_topn = callPackage ../servers/sql/postgresql/ext/pg_topn.nix {}; - - plv8 = callPackage ../servers/sql/postgresql/ext/plv8.nix { - v8 = v8_6_x; - }; - - timescaledb = callPackage ../servers/sql/postgresql/ext/timescaledb.nix {}; - - tsearch_extras = callPackage ../servers/sql/postgresql/ext/tsearch_extras.nix { }; - pigz = callPackage ../tools/compression/pigz { }; pixz = callPackage ../tools/compression/pixz { }; @@ -13357,24 +13328,62 @@ with pkgs; libmemcached = null; # Detection is broken upstream }; - postgresql = postgresql96; + ## ----- POSTGRESQL ------------------------------------------------------------------- + + # all postgresql packages, including extensions + inherit (import ../servers/sql/postgresql/packages.nix { inherit pkgs; }) + postgresql93Packages + postgresql94Packages + postgresql95Packages + postgresql96Packages + postgresql10Packages; + + # the default postgres version + package set + postgresqlPackages = postgresql96Packages; + postgresql = postgresqlPackages.postgresql; - inherit (callPackages ../servers/sql/postgresql { }) - postgresql93 - postgresql94 - postgresql95 - postgresql96 - postgresql10; + # named attributes for non-default versions of postgresql by itself. + postgresql93 = postgresql93Packages.postgresql; + postgresql94 = postgresql94Packages.postgresql; + postgresql95 = postgresql95Packages.postgresql; + postgresql96 = postgresql96Packages.postgresql; + postgresql10 = postgresql10Packages.postgresql; # TODO FIXME: this name is a misnomer (see #38616), but might be in use in # the wild. leave it for now, but remove this after 18.09 is released; it was # already in 18.03 and it's too late for that, so we should keep it around # for at least one more release as the 10.x series sees more updates. postgresql100 = builtins.trace "The 'postgresql100' attribute was badly misnamed, and is deprecated; use 'postgresql10' instead" - postgresql10; + postgresql10Packages.postgresql; + + ## -- miscellaneous postgresql-related packages. postgresql_jdbc = callPackage ../development/java-modules/postgresql_jdbc { }; + psqlodbc = callPackage ../development/libraries/psqlodbc { }; + + ## -- deprecated top-level names for extensions. TODO FIXME: remove after 18.09 + + pgjwt = postgresqlPackages.callPackageWithDeprecation "pgjwt" ../servers/sql/postgresql/ext/pgjwt.nix {}; + + pg_repack = postgresqlPackages.callPackageWithDeprecation "pg_repack" ../servers/sql/postgresql/ext/pg_repack.nix {}; + + pgroonga = postgresqlPackages.callPackageWithDeprecation "pgroonga" ../servers/sql/postgresql/ext/pgroonga.nix {}; + + pg_similarity = postgresqlPackages.callPackageWithDeprecation "pg_similarity" ../servers/sql/postgresql/ext/pg_similarity.nix {}; + + pgtap = postgresqlPackages.callPackageWithDeprecation "pgtap" ../servers/sql/postgresql/ext/pgtap.nix {}; + + plv8 = postgresqlPackages.callPackageWithDeprecation "plv8" ../servers/sql/postgresql/ext/plv8.nix { + v8 = v8_6_x; + }; + + timescaledb = postgresqlPackages.callPackageWithDeprecation "timescaledb" ../servers/sql/postgresql/ext/timescaledb.nix {}; + + tsearch_extras = postgresqlPackages.callPackageWithDeprecation "tsearch_extras" ../servers/sql/postgresql/ext/tsearch_extras.nix { }; + + ## ----- END: POSTGRESQL -------------------------------------------------------------- + inherit (callPackage ../servers/monitoring/prometheus { buildGoPackage = buildGo110Package; }) @@ -13412,8 +13421,6 @@ with pkgs; prometheus-varnish-exporter = callPackage ../servers/monitoring/prometheus/varnish-exporter.nix { }; prometheus-jmx-httpserver = callPackage ../servers/monitoring/prometheus/jmx-httpserver.nix { }; - psqlodbc = callPackage ../development/libraries/psqlodbc { }; - pure-ftpd = callPackage ../servers/ftp/pure-ftpd { }; pyIRCt = callPackage ../servers/xmpp/pyIRCt {}; From 1cee11f10acb0a0cd4860e2474a27886dda7aecf Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:17:59 -0500 Subject: [PATCH 06/61] nixpkgs: migrate postgis to new postgresqlPackages infrastructure Signed-off-by: Austin Seipp --- .../default.nix => servers/sql/postgresql/ext/postgis.nix} | 0 pkgs/servers/sql/postgresql/packages.nix | 2 ++ pkgs/top-level/all-packages.nix | 4 ++-- 3 files changed, 4 insertions(+), 2 deletions(-) rename pkgs/{development/libraries/postgis/default.nix => servers/sql/postgresql/ext/postgis.nix} (100%) diff --git a/pkgs/development/libraries/postgis/default.nix b/pkgs/servers/sql/postgresql/ext/postgis.nix similarity index 100% rename from pkgs/development/libraries/postgis/default.nix rename to pkgs/servers/sql/postgresql/ext/postgis.nix diff --git a/pkgs/servers/sql/postgresql/packages.nix b/pkgs/servers/sql/postgresql/packages.nix index 99a2ecf8f2ecc..6c40fb1dcf939 100644 --- a/pkgs/servers/sql/postgresql/packages.nix +++ b/pkgs/servers/sql/postgresql/packages.nix @@ -31,6 +31,8 @@ let cstore_fdw = callPackage ./ext/cstore_fdw.nix {}; + postgis = callPackage ./ext/postgis.nix {}; + pg_cron = callPackage ./ext/pg_cron.nix {}; pg_hll = callPackage ./ext/pg_hll.nix {}; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index ca624ead5465b..13ad8e6ce0abe 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -11566,8 +11566,6 @@ with pkgs; buildPythonApplication click future six; }; - postgis = callPackage ../development/libraries/postgis { }; - protobuf = protobuf3_4; protobuf3_5 = callPackage ../development/libraries/protobuf/3.5.nix { }; @@ -13364,6 +13362,8 @@ with pkgs; ## -- deprecated top-level names for extensions. TODO FIXME: remove after 18.09 + postgis = postgresqlPackages.callPackageWithDeprecation "postgis" ../servers/sql/postgresql/ext/postgis.nix {}; + pgjwt = postgresqlPackages.callPackageWithDeprecation "pgjwt" ../servers/sql/postgresql/ext/pgjwt.nix {}; pg_repack = postgresqlPackages.callPackageWithDeprecation "pg_repack" ../servers/sql/postgresql/ext/pg_repack.nix {}; From 1864fc3b947da158fd54a1ea5d32291f30f98d14 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:18:04 -0500 Subject: [PATCH 07/61] postgresqlPackages: add myself as a maintainer Signed-off-by: Austin Seipp --- pkgs/servers/sql/postgresql/default.nix | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pkgs/servers/sql/postgresql/default.nix b/pkgs/servers/sql/postgresql/default.nix index 07473145f6920..38f3bf083646e 100644 --- a/pkgs/servers/sql/postgresql/default.nix +++ b/pkgs/servers/sql/postgresql/default.nix @@ -89,11 +89,11 @@ let }; meta = with lib; { - homepage = https://www.postgresql.org; description = "A powerful, open source object-relational database system"; - license = licenses.postgresql; - maintainers = [ maintainers.ocharles ]; - platforms = platforms.unix; + homepage = https://www.postgresql.org; + license = licenses.postgresql; + maintainers = with maintainers; [ ocharles thoughtpolice ]; + platforms = platforms.unix; }; }); @@ -132,5 +132,4 @@ in { psqlSchema = "10.0"; sha256 = "04a07jkvc5s6zgh6jr78149kcjmsxclizsqabjw44ld4j5n633kc"; }; - } From 4eaf18bdfb3bfb9a1b8c551a1a33b8dee0adfea9 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:18:07 -0500 Subject: [PATCH 08/61] postgresqlPackages: add versionCheck passthru to extension builds for checking PostgreSQL compatibility Not every PostgreSQL extension builds with every version we have available; for instance, timescaledb is only available for PostgreSQL 9.6 and later. To do this, extensions may use a 'passthru.versionCheck' attribute, which should return 'true' if the extension can be built and used against the given postgresql version. This avoids needless build failures and helps indicate to the user that an extension is not available. Unfortunately the PostgreSQL expression did not previously offer a 'version' attribute to check against, so we must add it (as expected). This implies a rebuild of the PostgreSQL derivations, unfortunately. Signed-off-by: Austin Seipp --- pkgs/servers/sql/postgresql/default.nix | 1 + pkgs/servers/sql/postgresql/packages.nix | 7 +++++-- pkgs/top-level/all-packages.nix | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pkgs/servers/sql/postgresql/default.nix b/pkgs/servers/sql/postgresql/default.nix index 38f3bf083646e..47022a9a34703 100644 --- a/pkgs/servers/sql/postgresql/default.nix +++ b/pkgs/servers/sql/postgresql/default.nix @@ -5,6 +5,7 @@ let common = { version, sha256, psqlSchema }: let atLeast = lib.versionAtLeast version; in stdenv.mkDerivation (rec { name = "postgresql-${version}"; + inherit version; src = fetchurl { url = "mirror://postgresql/source/v${version}/${name}.tar.bz2"; diff --git a/pkgs/servers/sql/postgresql/packages.nix b/pkgs/servers/sql/postgresql/packages.nix index 6c40fb1dcf939..08774e5c918d8 100644 --- a/pkgs/servers/sql/postgresql/packages.nix +++ b/pkgs/servers/sql/postgresql/packages.nix @@ -1,13 +1,16 @@ -{ pkgs }: +{ pkgs, lib }: let postgresqlPackages = pkgs.callPackages ./default.nix {}; + # Filter out any versions which fail a version check. + filterPackages = lib.filterAttrs (_: drv: drv.versionCheck or true); + makePackageSet = postgresql: let callPackage = p: args: pkgs.callPackage p (args // { inherit postgresql; }); in - { + filterPackages { # Convenience function for end-users to easily build packages against a specific # version. inherit callPackage; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 13ad8e6ce0abe..9f65c33f2a344 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -13329,7 +13329,7 @@ with pkgs; ## ----- POSTGRESQL ------------------------------------------------------------------- # all postgresql packages, including extensions - inherit (import ../servers/sql/postgresql/packages.nix { inherit pkgs; }) + inherit (import ../servers/sql/postgresql/packages.nix { inherit pkgs lib; }) postgresql93Packages postgresql94Packages postgresql95Packages From 83e0444a541570e711bede53a6098c4e77eb8e89 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:18:09 -0500 Subject: [PATCH 09/61] postgresqlPackages.timescaledb: only works on 9.6+ Signed-off-by: Austin Seipp --- pkgs/servers/sql/postgresql/ext/timescaledb.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkgs/servers/sql/postgresql/ext/timescaledb.nix b/pkgs/servers/sql/postgresql/ext/timescaledb.nix index fab515035ce69..80df12dee93b4 100644 --- a/pkgs/servers/sql/postgresql/ext/timescaledb.nix +++ b/pkgs/servers/sql/postgresql/ext/timescaledb.nix @@ -41,6 +41,10 @@ stdenv.mkDerivation rec { mkdir -p $out/bin ''; + passthru = { + versionCheck = builtins.compareVersions postgresql.version "9.6" >= 0; + }; + meta = with stdenv.lib; { description = "Scales PostgreSQL for time-series data via automatic partitioning across time and space"; homepage = https://www.timescale.com/; From 96babf1b1e01ba832262e0f345518acaa362de80 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:18:12 -0500 Subject: [PATCH 10/61] postgresqlPackages.pg_cron: only works on 9.5+ Signed-off-by: Austin Seipp --- pkgs/servers/sql/postgresql/ext/pg_cron.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkgs/servers/sql/postgresql/ext/pg_cron.nix b/pkgs/servers/sql/postgresql/ext/pg_cron.nix index c5a7a40546ef2..d7eb352742565 100644 --- a/pkgs/servers/sql/postgresql/ext/pg_cron.nix +++ b/pkgs/servers/sql/postgresql/ext/pg_cron.nix @@ -21,6 +21,10 @@ stdenv.mkDerivation rec { cp *.control $out/share/extension ''; + passthru = { + versionCheck = builtins.compareVersions postgresql.version "9.5" >= 0; + }; + meta = with stdenv.lib; { description = "Run Cron jobs through PostgreSQL"; homepage = https://www.citusdata.com/; From 4c257d7364ff7ae1e1922dfa5f5b39d9b2312fc8 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:18:14 -0500 Subject: [PATCH 11/61] postgresqlPackages.pg_topn: only works on 9.4+ Signed-off-by: Austin Seipp --- pkgs/servers/sql/postgresql/ext/pg_topn.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkgs/servers/sql/postgresql/ext/pg_topn.nix b/pkgs/servers/sql/postgresql/ext/pg_topn.nix index 6886c80cf50d1..1624b912c78a1 100644 --- a/pkgs/servers/sql/postgresql/ext/pg_topn.nix +++ b/pkgs/servers/sql/postgresql/ext/pg_topn.nix @@ -22,6 +22,10 @@ stdenv.mkDerivation rec { cp *.control $out/share/extension ''; + passthru = { + versionCheck = builtins.compareVersions postgresql.version "9.4" >= 0; + }; + meta = with stdenv.lib; { description = "Efficient querying of 'top values' for PostgreSQL"; homepage = https://www.citusdata.com/; From ebcffbcb52843a8ce8052259e57c737969fa9a13 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:18:16 -0500 Subject: [PATCH 12/61] postgresqlPackages.pg_jobmon: init at 1.3.3 Signed-off-by: Austin Seipp --- pkgs/servers/sql/postgresql/ext/pg_jobmon.nix | 31 +++++++++++++++++++ pkgs/servers/sql/postgresql/packages.nix | 2 ++ 2 files changed, 33 insertions(+) create mode 100644 pkgs/servers/sql/postgresql/ext/pg_jobmon.nix diff --git a/pkgs/servers/sql/postgresql/ext/pg_jobmon.nix b/pkgs/servers/sql/postgresql/ext/pg_jobmon.nix new file mode 100644 index 0000000000000..9f60cf3ff5b39 --- /dev/null +++ b/pkgs/servers/sql/postgresql/ext/pg_jobmon.nix @@ -0,0 +1,31 @@ +{ stdenv, fetchFromGitHub, postgresql }: + +stdenv.mkDerivation rec { + name = "pg_jobmon-${version}"; + version = "1.3.3"; + + buildInputs = [ postgresql ]; + + src = fetchFromGitHub { + owner = "omniti-labs"; + repo = "pg_jobmon"; + rev = "refs/tags/v${version}"; + sha256 = "09izh6j1rpkllmy2kjhd9pwzld6lryp7825129k5jbbvnavxv6g8"; + }; + + installPhase = '' + mkdir -p $out/share/extension + + cp -v updates/*.sql $out/share/extension + cp -v sql/*.sql $out/share/extension + cp -v *.control $out/share/extension + ''; + + meta = with stdenv.lib; { + description = "Log and monitor PostgreSQL jobs"; + homepage = https://github.com/omniti-labs/pg_jobmon; + maintainers = with maintainers; [ thoughtpolice ]; + platforms = platforms.linux; + license = licenses.postgresql; + }; +} diff --git a/pkgs/servers/sql/postgresql/packages.nix b/pkgs/servers/sql/postgresql/packages.nix index 08774e5c918d8..0c65d38dc47c7 100644 --- a/pkgs/servers/sql/postgresql/packages.nix +++ b/pkgs/servers/sql/postgresql/packages.nix @@ -42,6 +42,8 @@ let pgjwt = callPackage ./ext/pgjwt.nix {}; + pg_jobmon = callPackage ./ext/pg_jobmon.nix {}; + pg_repack = callPackage ./ext/pg_repack.nix {}; pgroonga = callPackage ./ext/pgroonga.nix {}; From f9c2f8da3d331c63509ad19ffe9ed6db036bc652 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:18:18 -0500 Subject: [PATCH 13/61] postgresqlPackages.pg_partman: init at 3.1.2 Signed-off-by: Austin Seipp --- .../servers/sql/postgresql/ext/pg_partman.nix | 36 +++++++++++++++++++ pkgs/servers/sql/postgresql/packages.nix | 2 ++ 2 files changed, 38 insertions(+) create mode 100644 pkgs/servers/sql/postgresql/ext/pg_partman.nix diff --git a/pkgs/servers/sql/postgresql/ext/pg_partman.nix b/pkgs/servers/sql/postgresql/ext/pg_partman.nix new file mode 100644 index 0000000000000..e1fc1db403d76 --- /dev/null +++ b/pkgs/servers/sql/postgresql/ext/pg_partman.nix @@ -0,0 +1,36 @@ +{ stdenv, fetchFromGitHub, postgresql }: + +stdenv.mkDerivation rec { + name = "pg_partman-${version}"; + version = "3.1.2"; + + buildInputs = [ postgresql ]; + + src = fetchFromGitHub { + owner = "keithf4"; + repo = "pg_partman"; + rev = "refs/tags/v${version}"; + sha256 = "170kv6a7dwfkgpnjafbg3wk0j6zgcp0h79q7a31drhxw0hfab4y5"; + }; + + installPhase = '' + mkdir -p $out/{lib,share/extension} + + cp -v src/*.so $out/lib + cp -v updates/*.sql $out/share/extension + cp -v sql/*.sql $out/share/extension + cp -v *.control $out/share/extension + ''; + + passthru = { + versionCheck = builtins.compareVersions postgresql.version "9.4" >= 0; + }; + + meta = with stdenv.lib; { + description = "Partition management extension for PostgreSQL"; + homepage = https://github.com/keithf4/pg_partman; + maintainers = with maintainers; [ thoughtpolice ]; + platforms = platforms.linux; + license = licenses.postgresql; + }; +} diff --git a/pkgs/servers/sql/postgresql/packages.nix b/pkgs/servers/sql/postgresql/packages.nix index 0c65d38dc47c7..44a9cf005d401 100644 --- a/pkgs/servers/sql/postgresql/packages.nix +++ b/pkgs/servers/sql/postgresql/packages.nix @@ -44,6 +44,8 @@ let pg_jobmon = callPackage ./ext/pg_jobmon.nix {}; + pg_partman = callPackage ./ext/pg_partman.nix {}; + pg_repack = callPackage ./ext/pg_repack.nix {}; pgroonga = callPackage ./ext/pgroonga.nix {}; From e5ad1dc508bbdec578175c8596e1ed4b018c5726 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:18:20 -0500 Subject: [PATCH 14/61] nixos/postgresql: add myself as the module maintainer Signed-off-by: Austin Seipp --- nixos/modules/services/databases/postgresql.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/modules/services/databases/postgresql.nix b/nixos/modules/services/databases/postgresql.nix index f59fb1c817726..6958c52689d5d 100644 --- a/nixos/modules/services/databases/postgresql.nix +++ b/nixos/modules/services/databases/postgresql.nix @@ -269,5 +269,5 @@ in }; meta.doc = ./postgresql.xml; - + meta.maintainers = with lib.maintainers; [ thoughtpolice ]; } From 2842c505f8d6795a9c2bbec6c8dfc9e1eb23269c Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:18:22 -0500 Subject: [PATCH 15/61] nixos/postgresql: kill ugly hack, now that postgresql has 'version' attrs Signed-off-by: Austin Seipp --- nixos/modules/services/databases/postgresql.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/modules/services/databases/postgresql.nix b/nixos/modules/services/databases/postgresql.nix index 6958c52689d5d..121ccd15afc9c 100644 --- a/nixos/modules/services/databases/postgresql.nix +++ b/nixos/modules/services/databases/postgresql.nix @@ -10,7 +10,7 @@ let postgresqlAndPlugins = pg: if cfg.extraPlugins == [] then pg else pkgs.buildEnv { - name = "postgresql-and-plugins-${(builtins.parseDrvName pg.name).version}"; + name = "postgresql-and-plugins-${pg.version}"; paths = [ pg pg.lib ] ++ cfg.extraPlugins; buildInputs = [ pkgs.makeWrapper ]; postBuild = From 16624c613f5f4604507e19529f077560251c6bc3 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:18:25 -0500 Subject: [PATCH 16/61] nixos/postgresql: add new nixos module API Signed-off-by: Austin Seipp --- .../modules/services/databases/postgresql.nix | 88 ++++++++++++++++--- 1 file changed, 77 insertions(+), 11 deletions(-) diff --git a/nixos/modules/services/databases/postgresql.nix b/nixos/modules/services/databases/postgresql.nix index 121ccd15afc9c..77e5a0c3ddc5b 100644 --- a/nixos/modules/services/databases/postgresql.nix +++ b/nixos/modules/services/databases/postgresql.nix @@ -6,12 +6,15 @@ let cfg = config.services.postgresql; + packageSetPlugins = cfg.plugins cfg.packages; + plugins = if cfg.extraPlugins != [] then cfg.extraPlugins else packageSetPlugins; + # see description of extraPlugins postgresqlAndPlugins = pg: - if cfg.extraPlugins == [] then pg + if plugins == [] then pg else pkgs.buildEnv { name = "postgresql-and-plugins-${pg.version}"; - paths = [ pg pg.lib ] ++ cfg.extraPlugins; + paths = [ pg pg.lib ] ++ plugins; buildInputs = [ pkgs.makeWrapper ]; postBuild = '' @@ -22,7 +25,10 @@ let ''; }; - postgresql = postgresqlAndPlugins cfg.package; + postgresqlPackage = if (cfg.package != null) then cfg.package else + cfg.packages.postgresql; + + postgresql = postgresqlAndPlugins postgresqlPackage; # The main PostgreSQL configuration file. configFile = pkgs.writeText "postgresql.conf" @@ -54,10 +60,22 @@ in }; package = mkOption { - type = types.package; + type = types.nullOr types.package; example = literalExample "pkgs.postgresql96"; + default = null; description = '' - PostgreSQL package to use. + PostgreSQL package to use. Deprecated. Use services.postgresql.packages instead + to specify an entire package set (including compatible extensions) at once. + ''; + }; + + packages = mkOption { + type = types.nullOr (types.attrsOf types.package); + default = null; + example = literalExample "pkgs.postgresql96Packages"; + description = '' + The set of PostgreSQL packages to use, including the database + server and all available extensions. ''; }; @@ -120,11 +138,30 @@ in default = []; example = literalExample "[ (pkgs.postgis.override { postgresql = pkgs.postgresql94; }) ]"; description = '' - When this list contains elements a new store path is created. + Deprecated. When this list contains elements a new store path is created. PostgreSQL and the elements are symlinked into it. Then pg_config, postgres and pg_ctl are copied to make them use the new $out/lib directory as pkglibdir. This makes it possible to use postgis without patching the .sql files which reference $libdir/postgis-1.5. + + This attribute is hard to use, and is deprecated in favor of the more general + services.postgresql.plugins attribute, which will correctly override the specified + packages for you. + ''; + }; + + plugins = mkOption { + default = _: []; + example = literalExample "p: with p; [ postgis timescaledb ]"; + description = '' + A function that selects plugins to include in the PostgreSQL server. + When this function returns a non-empty list, a new store path is created. + PostgreSQL and the elements are symlinked into it. Then pg_config, + postgres and pg_ctl are copied to make them use the new + $out/lib directory as pkglibdir. + + The value provided as an argument to the function is the set of available + PostgreSQL plugins, specified by the services.postgresql.packages argument. ''; # Note: the duplication of executables is about 4MB size. # So a nicer solution was patching postgresql to allow setting the @@ -163,16 +200,45 @@ in config = mkIf config.services.postgresql.enable { - services.postgresql.package = + assertions = + [ # The user specified both 'package' and 'packages', which are mutually exclusive + { assertion = (cfg.package != null -> cfg.packages != null); + message = '' + The option services.postgresql.{package,packages} cannot both be set. Please use the 'packages' option + to specify the entire set of PostgreSQL packages. + ''; + } + + # The user specified the old extraPlugins attribute, but they *also* specified some 'plugins' function + # which returned a non-empty list. These are mutually exclusive. + { assertion = (cfg.extraPlugins != [] -> packageSetPlugins == []); + message = '' + The option services.postgresql.extraPlugins and .plugins cannot both be non-empty at the same time; please remove + uses of .extraPlugins. + ''; + } + ]; + + warnings = + (optional (cfg.package != null) + ''The services.postgresql.package attribute is deprecated. Please use services.postgresql.packages (with an 's') instead. + See the NixOS manual ('nixos-help') for more information.'') ++ + (optional (cfg.extraPlugins != []) + ''The services.postgresql.extraPlugins attribute is deprecated. Please use services.postgresql.plugins instead, + which will correctly override PostgreSQL attributes for you and will provide a list of compatible plugins for + the given server version. + ''); + + services.postgresql.packages = # Note: when changing the default, make it conditional on # ‘system.stateVersion’ to maintain compatibility with existing # systems! - mkDefault (if versionAtLeast config.system.stateVersion "17.09" then pkgs.postgresql96 - else if versionAtLeast config.system.stateVersion "16.03" then pkgs.postgresql95 - else pkgs.postgresql94); + mkDefault (if versionAtLeast config.system.stateVersion "17.09" then pkgs.postgresql96Packages + else if versionAtLeast config.system.stateVersion "16.03" then pkgs.postgresql95Packages + else pkgs.postgresql94Packages); services.postgresql.dataDir = - mkDefault (if versionAtLeast config.system.stateVersion "17.09" then "/var/lib/postgresql/${config.services.postgresql.package.psqlSchema}" + mkDefault (if versionAtLeast config.system.stateVersion "17.09" then "/var/lib/postgresql/${postgresqlPackage.psqlSchema}" else "/var/db/postgresql"); services.postgresql.authentication = mkAfter From 31e15dad357c9dce90d3b423ad38705e68d61b8c Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:18:27 -0500 Subject: [PATCH 17/61] nixos/tests: migrate postgis test to new postgres API Signed-off-by: Austin Seipp --- nixos/tests/postgis.nix | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nixos/tests/postgis.nix b/nixos/tests/postgis.nix index 0ff880e7aa527..a7c93baa52ca9 100644 --- a/nixos/tests/postgis.nix +++ b/nixos/tests/postgis.nix @@ -1,7 +1,7 @@ import ./make-test.nix ({ pkgs, ...} : { name = "postgis"; meta = with pkgs.stdenv.lib.maintainers; { - maintainers = [ lsix ]; + maintainers = [ thoughtpolice lsix ]; }; nodes = { @@ -9,10 +9,10 @@ import ./make-test.nix ({ pkgs, ...} : { { pkgs, ... }: { - services.postgresql = let mypg = pkgs.postgresql10; in { - enable = true; - package = mypg; - extraPlugins = [ (pkgs.postgis.override { postgresql = mypg; }) ]; + services.postgresql = { + enable = true; + packages = pkgs.postgresqlPackages; + plugins = p: with p; [ postgis ]; }; }; }; From 87ccdbe548b8fb719f184243cb68d6a9d1bdb169 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:18:29 -0500 Subject: [PATCH 18/61] nixpkgs/nixos: switch to PostgreSQL 10 by default Signed-off-by: Austin Seipp --- nixos/modules/services/databases/postgresql.nix | 3 ++- pkgs/top-level/all-packages.nix | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/nixos/modules/services/databases/postgresql.nix b/nixos/modules/services/databases/postgresql.nix index 77e5a0c3ddc5b..07ff6989057e1 100644 --- a/nixos/modules/services/databases/postgresql.nix +++ b/nixos/modules/services/databases/postgresql.nix @@ -233,7 +233,8 @@ in # Note: when changing the default, make it conditional on # ‘system.stateVersion’ to maintain compatibility with existing # systems! - mkDefault (if versionAtLeast config.system.stateVersion "17.09" then pkgs.postgresql96Packages + mkDefault (if versionAtLeast config.system.stateVersion "18.09" then pkgs.postgresql10Packages + else if versionAtLeast config.system.stateVersion "17.09" then pkgs.postgresql96Packages else if versionAtLeast config.system.stateVersion "16.03" then pkgs.postgresql95Packages else pkgs.postgresql94Packages); diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 9f65c33f2a344..1a5b730ffbc8b 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -13337,7 +13337,7 @@ with pkgs; postgresql10Packages; # the default postgres version + package set - postgresqlPackages = postgresql96Packages; + postgresqlPackages = postgresql10Packages; postgresql = postgresqlPackages.postgresql; # named attributes for non-default versions of postgresql by itself. From ef1f8be3c1033c7afa164225c468c77093b9bdf7 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:18:31 -0500 Subject: [PATCH 19/61] nixos/postgres: update manual Signed-off-by: Austin Seipp --- .../modules/services/databases/postgresql.xml | 110 ++++++++++++++---- 1 file changed, 90 insertions(+), 20 deletions(-) diff --git a/nixos/modules/services/databases/postgresql.xml b/nixos/modules/services/databases/postgresql.xml index 1aaf339632454..fa13c87ddf0c1 100644 --- a/nixos/modules/services/databases/postgresql.xml +++ b/nixos/modules/services/databases/postgresql.xml @@ -6,16 +6,60 @@ PostgreSQL - - - Source: modules/services/databases/postgresql.nix Upstream documentation: - +Maintainer: Austin Seipp + +Default version: 10.x + +PostgreSQL is an advanced, free, feature-rich relational database. NixOS +provides multiple versions of PostgreSQL, as well as a set of extensions that +provide powerful features on top of the default installation. + +
+Introduction to the PostgreSQL packaging infrastructure + +NixOS provides multiple versions of PostgreSQL, available as +sets of packages, which you use to configure the +PostgreSQL system services and specify extensions. Extensions are available +only for the correct PostgreSQL versions that they are compatible with, and +they are automatically built against the proper PostgreSQL server for +you. + +The available set of PostgreSQL packages are available under the name +pkgs.postgresqlXXPackages, where XX +contains the major version number, without any bugfix releases. For example, +the following are all available: + + +pkgs.postgresql93Packages +pkgs.postgresql94Packages +pkgs.postgresql95Packages +pkgs.postgresql96Packages +pkgs.postgresql10Packages + +# points to default version specified by Nixpkgs +pkgs.postgresqlPackages + + + +These names are each attribute sets containing A) the +.postgresql attribute, pointing to the derivation for the +PostgreSQL server, and an assorted set of other attributes which specify +extensions that can be installed into the server. Because extensions are tied +to the version of PostgreSQL they're built against, they must all be bundled +together. For example, the following attributes yield the derivations for +PostgreSQL 9.6, and the PostGIS extension built against PostgreSQL 9.6: + + +pkgs.postgresql96Packages.postgres +pkgs.postgresql96Packages.postgis + + -PostgreSQL is an advanced, free relational database. +
Configuring @@ -27,25 +71,23 @@ = pkgs.postgresql94; -Note that you are required to specify the desired version of -PostgreSQL (e.g. pkgs.postgresql94). Since -upgrading your PostgreSQL version requires a database dump and reload -(see below), NixOS cannot provide a default value for - such as the most recent -release of PostgreSQL. +Note that you are required to specify the desired version of PostgreSQL (e.g. +pkgs.postgresql96Packages). Since upgrading your PostgreSQL +version requires a database dump and reload (see below), NixOS cannot provide a +default value for such as the +most recent release of PostgreSQL. - +postgres=> + + By default, PostgreSQL stores its databases in /var/db/postgresql. You can override this using @@ -59,6 +101,35 @@ alice=>
+
Enabling and installing extensions + +In order to install extensions, use the + option. This option should contain +a function which takes an attribute set, and from that attribute set, returns a +list of the extensions the user wishes to enable. This extension set is the +same one specified by . + +For example, to enable the PostGIS and TimescaleDB extension, you can +say: + + +services.postgresql.plugins = p: with p; + [ postgis + timescaledb + ]; + + + +This example imports the set of PostgreSQL attributes into scope and +selects the postgis and timescaledb +attributes. Note that not all of these attributes are available for every +version of PostgreSQL, e.g. TimescaleDB does not work with PostgreSQL 9.5. +Should you specify v9.5 (using ), +the above example would fail with an evaluation error. + +FIXME: talk about CREATE EXTENSION. + +
Upgrading @@ -66,12 +137,11 @@ alice=>
-
Options - A complete list of options for the PostgreSQL module may be found here. +A complete list of options for the PostgreSQL module may be found here.
- From 44fc8a90b4105059ceac337ef58bc54dd227b600 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:18:33 -0500 Subject: [PATCH 20/61] nixpkgs: actually deprecate old postgresql extension names Signed-off-by: Austin Seipp --- pkgs/servers/sql/postgresql/packages.nix | 11 --------- pkgs/top-level/aliases.nix | 13 +++++++++++ pkgs/top-level/all-packages.nix | 29 ------------------------ 3 files changed, 13 insertions(+), 40 deletions(-) diff --git a/pkgs/servers/sql/postgresql/packages.nix b/pkgs/servers/sql/postgresql/packages.nix index 44a9cf005d401..b14342396b0eb 100644 --- a/pkgs/servers/sql/postgresql/packages.nix +++ b/pkgs/servers/sql/postgresql/packages.nix @@ -15,17 +15,6 @@ let # version. inherit callPackage; - # Convenience function for top-level attributes that were previously available, - # but are deprecated and renamed. TODO FIXME: remove this after 18.09 is released. - # see all-packages.nix (in the postgresql section) for more - callPackageWithDeprecation = name: p: args: - builtins.trace '' - The PostgreSQL-related attribute '${name}' is deprecated. Please use the - 'postgresqlPackages.${name}' attribute instead (to use the default version of - PostgreSQL in Nixpkgs), or use 'postgresqlPackages.${name}' for a given - version . - '' (callPackage p args); - # Convenience attribute that exports the postgres derivation used for builds. We # export these from all-packages.nix to consolidate everything here. inherit postgresql; diff --git a/pkgs/top-level/aliases.nix b/pkgs/top-level/aliases.nix index fed02f40134d2..9f0b886f5d07c 100644 --- a/pkgs/top-level/aliases.nix +++ b/pkgs/top-level/aliases.nix @@ -214,6 +214,19 @@ mapAliases ({ pltScheme = racket; # just to be sure poppler_qt5 = libsForQt5.poppler; # added 2015-12-19 procps-ng = procps; # added 2018-06-08 + + # added 2018-04-11, remove after 18.09 release + postgresql100 = throw "deprecated: 'postgresql100' was badly misnamed, use 'postgresql10' instead"; + postgis = throw "deprecated: use 'postgresqlPackages.postgis' instead"; + pgjwt = throw "deprecated: use 'postgresqlPackages.pgjwt' instead"; + pg_repack = throw "deprecated: use 'postgresqlPackages.pg_repack' instead"; + pgroonga = throw "deprecated: use 'postgresqlPackages.pgroonga' instead"; + pg_similarity = throw "deprecated: use 'postgresqlPackages.pg_similarity' instead"; + pgtap = throw "deprecated: use 'postgresqlPackages.pgtap' instead"; + plv8 = throw "deprecated: use 'postgresqlPackages.plv8' instead"; + timescaledb = throw "deprecated: use 'postgresqlPackages.timescaledb' instead"; + tsearch_extras = throw "deprecated: use 'postgresqlPackages.tsearch_extras' instead"; + prometheus-statsd-bridge = prometheus-statsd-exporter; # added 2017-08-27 pulseaudioLight = pulseaudio; # added 2018-04-25 qca-qt5 = libsForQt5.qca-qt5; # added 2015-12-19 diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 1a5b730ffbc8b..bcb90e6940ae0 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -13347,41 +13347,12 @@ with pkgs; postgresql96 = postgresql96Packages.postgresql; postgresql10 = postgresql10Packages.postgresql; - # TODO FIXME: this name is a misnomer (see #38616), but might be in use in - # the wild. leave it for now, but remove this after 18.09 is released; it was - # already in 18.03 and it's too late for that, so we should keep it around - # for at least one more release as the 10.x series sees more updates. - postgresql100 = builtins.trace "The 'postgresql100' attribute was badly misnamed, and is deprecated; use 'postgresql10' instead" - postgresql10Packages.postgresql; - ## -- miscellaneous postgresql-related packages. postgresql_jdbc = callPackage ../development/java-modules/postgresql_jdbc { }; psqlodbc = callPackage ../development/libraries/psqlodbc { }; - ## -- deprecated top-level names for extensions. TODO FIXME: remove after 18.09 - - postgis = postgresqlPackages.callPackageWithDeprecation "postgis" ../servers/sql/postgresql/ext/postgis.nix {}; - - pgjwt = postgresqlPackages.callPackageWithDeprecation "pgjwt" ../servers/sql/postgresql/ext/pgjwt.nix {}; - - pg_repack = postgresqlPackages.callPackageWithDeprecation "pg_repack" ../servers/sql/postgresql/ext/pg_repack.nix {}; - - pgroonga = postgresqlPackages.callPackageWithDeprecation "pgroonga" ../servers/sql/postgresql/ext/pgroonga.nix {}; - - pg_similarity = postgresqlPackages.callPackageWithDeprecation "pg_similarity" ../servers/sql/postgresql/ext/pg_similarity.nix {}; - - pgtap = postgresqlPackages.callPackageWithDeprecation "pgtap" ../servers/sql/postgresql/ext/pgtap.nix {}; - - plv8 = postgresqlPackages.callPackageWithDeprecation "plv8" ../servers/sql/postgresql/ext/plv8.nix { - v8 = v8_6_x; - }; - - timescaledb = postgresqlPackages.callPackageWithDeprecation "timescaledb" ../servers/sql/postgresql/ext/timescaledb.nix {}; - - tsearch_extras = postgresqlPackages.callPackageWithDeprecation "tsearch_extras" ../servers/sql/postgresql/ext/tsearch_extras.nix { }; - ## ----- END: POSTGRESQL -------------------------------------------------------------- inherit (callPackage ../servers/monitoring/prometheus { From 2f32398f5495a01224df2d3bb8a21ba1fc476cc3 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:18:36 -0500 Subject: [PATCH 21/61] nixos/tests: migrate to new postgresql api Signed-off-by: Austin Seipp --- nixos/tests/pgjwt.nix | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/nixos/tests/pgjwt.nix b/nixos/tests/pgjwt.nix index a2d81288c8122..953d8594787ca 100644 --- a/nixos/tests/pgjwt.nix +++ b/nixos/tests/pgjwt.nix @@ -1,10 +1,12 @@ import ./make-test.nix ({ pkgs, lib, ...}: let + pgsql = pkgs.postgresqlPackages; + test = with pkgs; runCommand "patch-test" { - nativeBuildInputs = [ pgjwt ]; + nativeBuildInputs = [ pgsql.pgjwt ]; } '' - sed -e '12 i CREATE EXTENSION pgcrypto;\nCREATE EXTENSION pgtap;\nSET search_path TO tap,public;' ${pgjwt.src}/test.sql > $out; + sed -e '12 i CREATE EXTENSION pgcrypto;\nCREATE EXTENSION pgtap;\nSET search_path TO tap,public;' ${pgsql.pgjwt.src}/test.sql > $out; ''; in with pkgs; { @@ -18,7 +20,8 @@ with pkgs; { { services.postgresql = { enable = true; - extraPlugins = [ pgjwt pgtap ]; + packages = pgsql; + plugins = p: [ p.pgjwt p.pgtap ]; }; }; }; From f20b30670c09610490a077313d92c514285261ed Mon Sep 17 00:00:00 2001 From: Bas van Dijk Date: Mon, 6 Aug 2018 10:18:38 -0500 Subject: [PATCH 22/61] postgresql: always create $out/bin (issue #22653). This is needed because some PostgreSQL plugins don't have a bin directory. If only these plugins are listed in cfg.extraPlugins buildEnv will turn $out/bin into a symbolic link to ${pg}/bin. Lateron we try to rm $out/bin/{pg_config,postgres,pg_ctl} which will then fail because $out/bin will be read-only. Signed-off-by: Austin Seipp --- nixos/modules/services/databases/postgresql.nix | 4 ++++ pkgs/servers/sql/postgresql/ext/pgjwt.nix | 1 - pkgs/servers/sql/postgresql/ext/timescaledb.nix | 7 ------- pkgs/servers/sql/postgresql/ext/tsearch_extras.nix | 1 - 4 files changed, 4 insertions(+), 9 deletions(-) diff --git a/nixos/modules/services/databases/postgresql.nix b/nixos/modules/services/databases/postgresql.nix index 07ff6989057e1..3d549f33c6ff7 100644 --- a/nixos/modules/services/databases/postgresql.nix +++ b/nixos/modules/services/databases/postgresql.nix @@ -15,6 +15,10 @@ let else pkgs.buildEnv { name = "postgresql-and-plugins-${pg.version}"; paths = [ pg pg.lib ] ++ plugins; + # We include /bin to ensure the $out/bin directory is created, which is + # needed because we'll be removing the files from that directory in postBuild + # below. See #22653 + pathsToLink = ["/" "/bin"]; buildInputs = [ pkgs.makeWrapper ]; postBuild = '' diff --git a/pkgs/servers/sql/postgresql/ext/pgjwt.nix b/pkgs/servers/sql/postgresql/ext/pgjwt.nix index ab7ba8943a7bf..3185e2265744e 100644 --- a/pkgs/servers/sql/postgresql/ext/pgjwt.nix +++ b/pkgs/servers/sql/postgresql/ext/pgjwt.nix @@ -13,7 +13,6 @@ stdenv.mkDerivation rec { buildPhase = ":"; installPhase = '' - mkdir -p $out/bin # current postgresql extension mechanism in nixos requires bin directory mkdir -p $out/share/extension cp pg*sql *.control $out/share/extension ''; diff --git a/pkgs/servers/sql/postgresql/ext/timescaledb.nix b/pkgs/servers/sql/postgresql/ext/timescaledb.nix index 80df12dee93b4..fb11472f123f6 100644 --- a/pkgs/servers/sql/postgresql/ext/timescaledb.nix +++ b/pkgs/servers/sql/postgresql/ext/timescaledb.nix @@ -34,13 +34,6 @@ stdenv.mkDerivation rec { done ''; - postInstall = '' - # work around an annoying bug, by creating $out/bin, so buildEnv doesn't freak out later - # see https://github.com/NixOS/nixpkgs/issues/22653 - - mkdir -p $out/bin - ''; - passthru = { versionCheck = builtins.compareVersions postgresql.version "9.6" >= 0; }; diff --git a/pkgs/servers/sql/postgresql/ext/tsearch_extras.nix b/pkgs/servers/sql/postgresql/ext/tsearch_extras.nix index 5140ae1a228f1..ee28cc0ae6600 100644 --- a/pkgs/servers/sql/postgresql/ext/tsearch_extras.nix +++ b/pkgs/servers/sql/postgresql/ext/tsearch_extras.nix @@ -15,7 +15,6 @@ stdenv.mkDerivation rec { buildInputs = [ postgresql ]; installPhase = '' - mkdir -p $out/bin install -D tsearch_extras.so -t $out/lib/ install -D ./{tsearch_extras--1.0.sql,tsearch_extras.control} -t $out/share/extension ''; From b671e687b58bbebe5d2a04bcdd883890711ce82c Mon Sep 17 00:00:00 2001 From: Bas van Dijk Date: Mon, 6 Aug 2018 10:18:40 -0500 Subject: [PATCH 23/61] postgresqlPackages.pg_journal: init at 0.2.0 Signed-off-by: Austin Seipp --- .../servers/sql/postgresql/ext/pg_journal.nix | 41 +++++++++++++++++++ pkgs/servers/sql/postgresql/packages.nix | 2 + 2 files changed, 43 insertions(+) create mode 100644 pkgs/servers/sql/postgresql/ext/pg_journal.nix diff --git a/pkgs/servers/sql/postgresql/ext/pg_journal.nix b/pkgs/servers/sql/postgresql/ext/pg_journal.nix new file mode 100644 index 0000000000000..3a5637e468d75 --- /dev/null +++ b/pkgs/servers/sql/postgresql/ext/pg_journal.nix @@ -0,0 +1,41 @@ +{ stdenv, fetchFromGitHub, fetchpatch, pkgconfig, postgresql, systemd, openssl }: + +# # To enable on NixOS: +# config.services.postgresql = { +# extraPlugins = [ pkgs.pg_journal ]; +# extraConfig = "shared_preload_libraries = 'pg_journal'"; +# } + +stdenv.mkDerivation rec { + name = "pg_journal-${version}"; + version = "0.2.0"; + + buildInputs = [ postgresql pkgconfig systemd.dev openssl.dev ]; + + src = fetchFromGitHub { + owner = "intgr"; + repo = "pg_journal"; + rev = "e3847a1ff34c0aa8e7d16646bf921f69433b39ae"; + sha256 = "0kgqw75hnm7dx9haiv26gwa8h73c39276pqkmnryn3fg469hl8wb"; + }; + + patches = [ + # See: https://github.com/intgr/pg_journal/issues/1 + (fetchpatch { + url = https://github.com/gosuai/pg_journal/commit/a8e534ee1089c35bc073113358166ce2fd9e2c99.patch; + sha256 = "1bgc1jzhkvwc86gvf5s8qfzlr9xiqdrdbi9zgppd1zb0i4ndmqcp"; + }) + ]; + + installPhase = '' + mkdir -p $out/lib + cp pg_journal.so $out/lib + ''; + + meta = with stdenv.lib; { + description = "Log PostgreSQL messages to systemd journal"; + homepage = https://github.com/intgr/pg_journal; + maintainers = with maintainers; [ basvandijk ]; + license = licenses.postgresql; + }; +} diff --git a/pkgs/servers/sql/postgresql/packages.nix b/pkgs/servers/sql/postgresql/packages.nix index b14342396b0eb..005bde3e29406 100644 --- a/pkgs/servers/sql/postgresql/packages.nix +++ b/pkgs/servers/sql/postgresql/packages.nix @@ -33,6 +33,8 @@ let pg_jobmon = callPackage ./ext/pg_jobmon.nix {}; + pg_journal = callPackage ./ext/pg_journal.nix {}; + pg_partman = callPackage ./ext/pg_partman.nix {}; pg_repack = callPackage ./ext/pg_repack.nix {}; From 8bdf30e1b998cb1077bb5c67ed53d39d7fc97e5e Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:18:42 -0500 Subject: [PATCH 24/61] postgresqlPackages.tsearch_extras: 0.3 -> 0.4 Signed-off-by: Austin Seipp --- pkgs/servers/sql/postgresql/ext/tsearch_extras.nix | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkgs/servers/sql/postgresql/ext/tsearch_extras.nix b/pkgs/servers/sql/postgresql/ext/tsearch_extras.nix index ee28cc0ae6600..8b9e9322b4796 100644 --- a/pkgs/servers/sql/postgresql/ext/tsearch_extras.nix +++ b/pkgs/servers/sql/postgresql/ext/tsearch_extras.nix @@ -2,13 +2,13 @@ stdenv.mkDerivation rec { name = "tsearch-extras-${version}"; - version = "0.3"; + version = "0.4"; src = fetchFromGitHub { - owner = "zulip"; - repo = "tsearch_extras"; - rev = version; - sha256 = "0i3i99lw80jwd4xflgdqabxmn1dnm1gm7dzf1mqv2drllxcy3yix"; + owner = "zulip"; + repo = "tsearch_extras"; + rev = "84e78f00931c4ef261d98197d6b5d94fc141f742"; # no release tag? + sha256 = "18j0saqblg3jhrz38splk173xjwdf32c67ymm18m8n5y94h8d2ba"; }; nativenativeBuildInputs = [ pkgconfig ]; From 50834369c1075249bc78c1236fd8c94ba215bb89 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:18:46 -0500 Subject: [PATCH 25/61] postgresqlPackages.pg_cron: 1.0.2 -> 1.1.2 Signed-off-by: Austin Seipp --- pkgs/servers/sql/postgresql/ext/pg_cron.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/servers/sql/postgresql/ext/pg_cron.nix b/pkgs/servers/sql/postgresql/ext/pg_cron.nix index d7eb352742565..98bcc32b53db6 100644 --- a/pkgs/servers/sql/postgresql/ext/pg_cron.nix +++ b/pkgs/servers/sql/postgresql/ext/pg_cron.nix @@ -2,7 +2,7 @@ stdenv.mkDerivation rec { name = "pg_cron-${version}"; - version = "1.0.2"; + version = "1.1.2"; buildInputs = [ postgresql ]; @@ -10,7 +10,7 @@ stdenv.mkDerivation rec { owner = "citusdata"; repo = "pg_cron"; rev = "refs/tags/v${version}"; - sha256 = "0z743bbal9j0pvqskznfj0zvjsqvdl7p90d4fdrl0sc0crc3nvyx"; + sha256 = "0n74dx1wkg9qxvjhnx03028465ap3p97v2kzqww833dws1wqk5m1"; }; installPhase = '' From 7571fb47e7d04f8012083351d5e4af26a7e9323c Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:18:49 -0500 Subject: [PATCH 26/61] postgresqlPackages.pg_partman: 3.1.2 -> 3.2.1 Signed-off-by: Austin Seipp --- pkgs/servers/sql/postgresql/ext/pg_partman.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/servers/sql/postgresql/ext/pg_partman.nix b/pkgs/servers/sql/postgresql/ext/pg_partman.nix index e1fc1db403d76..486c626247404 100644 --- a/pkgs/servers/sql/postgresql/ext/pg_partman.nix +++ b/pkgs/servers/sql/postgresql/ext/pg_partman.nix @@ -2,7 +2,7 @@ stdenv.mkDerivation rec { name = "pg_partman-${version}"; - version = "3.1.2"; + version = "3.2.1"; buildInputs = [ postgresql ]; @@ -10,7 +10,7 @@ stdenv.mkDerivation rec { owner = "keithf4"; repo = "pg_partman"; rev = "refs/tags/v${version}"; - sha256 = "170kv6a7dwfkgpnjafbg3wk0j6zgcp0h79q7a31drhxw0hfab4y5"; + sha256 = "069m2y20pkpk7lw7d2spaa1zpkb47dmm76rxi1d4qvq3vgfd739m"; }; installPhase = '' From 6f6dcc07db05eec6cefea025f878ef32cbd2d1a1 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:18:51 -0500 Subject: [PATCH 27/61] postgresqlPackages.pg_topn 2.0.2 -> 2.1.0 Signed-off-by: Austin Seipp --- pkgs/servers/sql/postgresql/ext/pg_topn.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/servers/sql/postgresql/ext/pg_topn.nix b/pkgs/servers/sql/postgresql/ext/pg_topn.nix index 1624b912c78a1..89c3867cd0e5d 100644 --- a/pkgs/servers/sql/postgresql/ext/pg_topn.nix +++ b/pkgs/servers/sql/postgresql/ext/pg_topn.nix @@ -2,7 +2,7 @@ stdenv.mkDerivation rec { name = "pg_topn-${version}"; - version = "2.0.2"; + version = "2.1.0"; nativeBuildInputs = [ protobufc ]; buildInputs = [ postgresql ]; From 1e83cf317443cbc381af72aa09959266bc2d207b Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:18:53 -0500 Subject: [PATCH 28/61] postgresql: enable systemd service notification on 9.6+ This results in a marginal closure size increase of ~137mb -> ~141mb. This only affects the 'out' output, not the 'lib' output, so libpq-dependent expressions should not bloat as a result of this. The type of the systemd service is changed to 'notify' on 9.6+, which will help identify when Postgres is starting/shutting down. Signed-off-by: Austin Seipp --- nixos/modules/services/databases/postgresql.nix | 2 ++ pkgs/servers/sql/postgresql/default.nix | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/nixos/modules/services/databases/postgresql.nix b/nixos/modules/services/databases/postgresql.nix index 3d549f33c6ff7..c2f46d945df05 100644 --- a/nixos/modules/services/databases/postgresql.nix +++ b/nixos/modules/services/databases/postgresql.nix @@ -316,6 +316,8 @@ in # Give Postgres a decent amount of time to clean up after # receiving systemd's SIGINT. TimeoutSec = 120; + + Type = if versionAtLeast postgresqlPackage.psqlSchema "9.6" then "notify" else "simple"; }; # Wait for PostgreSQL to be ready to accept connections. diff --git a/pkgs/servers/sql/postgresql/default.nix b/pkgs/servers/sql/postgresql/default.nix index 47022a9a34703..9b03136a75e8b 100644 --- a/pkgs/servers/sql/postgresql/default.nix +++ b/pkgs/servers/sql/postgresql/default.nix @@ -1,4 +1,6 @@ -{ lib, stdenv, glibc, fetchurl, zlib, readline, libossp_uuid, openssl, libxml2, makeWrapper, tzdata }: +{ stdenv, lib, fetchurl, makeWrapper +, glibc, zlib, readline, libossp_uuid, openssl, libxml2, tzdata, systemd +}: let @@ -17,7 +19,8 @@ let buildInputs = [ zlib readline openssl libxml2 makeWrapper ] - ++ lib.optionals (!stdenv.isDarwin) [ libossp_uuid ]; + ++ lib.optionals (!stdenv.isDarwin) [ libossp_uuid ] + ++ lib.optionals (atLeast "9.6" && !stdenv.isDarwin) [ systemd ]; enableParallelBuilding = true; @@ -34,6 +37,7 @@ let "--sysconfdir=/etc" "--libdir=$(lib)/lib" "--with-system-tzdata=${tzdata}/share/zoneinfo" + (lib.optionalString (atLeast "9.6" && !stdenv.isDarwin) "--with-systemd") (if stdenv.isDarwin then "--with-uuid=e2fs" else "--with-ossp-uuid") ]; From af3d0e533e6bb2dda48959561e3d36bebeb03d79 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:18:55 -0500 Subject: [PATCH 29/61] postgresqlPackages: simplify versionCheck comparisons Extensions need to compare against the version of PostgreSQL they're building against, so providing a partially applied function is easier and less verbose for these cases. Signed-off-by: Austin Seipp --- pkgs/servers/sql/postgresql/default.nix | 1 + pkgs/servers/sql/postgresql/ext/pg_cron.nix | 2 +- pkgs/servers/sql/postgresql/ext/pg_partman.nix | 2 +- pkgs/servers/sql/postgresql/ext/pg_topn.nix | 2 +- pkgs/servers/sql/postgresql/ext/timescaledb.nix | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pkgs/servers/sql/postgresql/default.nix b/pkgs/servers/sql/postgresql/default.nix index 9b03136a75e8b..a2892bfdac9e3 100644 --- a/pkgs/servers/sql/postgresql/default.nix +++ b/pkgs/servers/sql/postgresql/default.nix @@ -91,6 +91,7 @@ let passthru = { inherit readline psqlSchema; + compareVersion = builtins.compareVersions version; }; meta = with lib; { diff --git a/pkgs/servers/sql/postgresql/ext/pg_cron.nix b/pkgs/servers/sql/postgresql/ext/pg_cron.nix index 98bcc32b53db6..25118f7c758ac 100644 --- a/pkgs/servers/sql/postgresql/ext/pg_cron.nix +++ b/pkgs/servers/sql/postgresql/ext/pg_cron.nix @@ -22,7 +22,7 @@ stdenv.mkDerivation rec { ''; passthru = { - versionCheck = builtins.compareVersions postgresql.version "9.5" >= 0; + versionCheck = postgresql.compareVersion "9.5" >= 0; }; meta = with stdenv.lib; { diff --git a/pkgs/servers/sql/postgresql/ext/pg_partman.nix b/pkgs/servers/sql/postgresql/ext/pg_partman.nix index 486c626247404..2b951e6abb6e3 100644 --- a/pkgs/servers/sql/postgresql/ext/pg_partman.nix +++ b/pkgs/servers/sql/postgresql/ext/pg_partman.nix @@ -23,7 +23,7 @@ stdenv.mkDerivation rec { ''; passthru = { - versionCheck = builtins.compareVersions postgresql.version "9.4" >= 0; + versionCheck = postgresql.compareVersion "9.4" >= 0; }; meta = with stdenv.lib; { diff --git a/pkgs/servers/sql/postgresql/ext/pg_topn.nix b/pkgs/servers/sql/postgresql/ext/pg_topn.nix index 89c3867cd0e5d..00b712975a310 100644 --- a/pkgs/servers/sql/postgresql/ext/pg_topn.nix +++ b/pkgs/servers/sql/postgresql/ext/pg_topn.nix @@ -23,7 +23,7 @@ stdenv.mkDerivation rec { ''; passthru = { - versionCheck = builtins.compareVersions postgresql.version "9.4" >= 0; + versionCheck = postgresql.compareVersion "9.4" >= 0; }; meta = with stdenv.lib; { diff --git a/pkgs/servers/sql/postgresql/ext/timescaledb.nix b/pkgs/servers/sql/postgresql/ext/timescaledb.nix index fb11472f123f6..6cb9160625f0b 100644 --- a/pkgs/servers/sql/postgresql/ext/timescaledb.nix +++ b/pkgs/servers/sql/postgresql/ext/timescaledb.nix @@ -35,7 +35,7 @@ stdenv.mkDerivation rec { ''; passthru = { - versionCheck = builtins.compareVersions postgresql.version "9.6" >= 0; + versionCheck = postgresql.compareVersion "9.6" >= 0; }; meta = with stdenv.lib; { From 7c8281c3c7d60aa6e2e7b4ef903e3cabf46b535a Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:18:57 -0500 Subject: [PATCH 30/61] postgresqlPackages.pg_repack: (re)indent to 2 spaces Signed-off-by: Austin Seipp --- pkgs/servers/sql/postgresql/ext/pg_repack.nix | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/pkgs/servers/sql/postgresql/ext/pg_repack.nix b/pkgs/servers/sql/postgresql/ext/pg_repack.nix index 1b7cd08b082b7..69fed13a6db6c 100644 --- a/pkgs/servers/sql/postgresql/ext/pg_repack.nix +++ b/pkgs/servers/sql/postgresql/ext/pg_repack.nix @@ -1,35 +1,35 @@ { stdenv, fetchFromGitHub, postgresql, openssl, zlib, readline }: stdenv.mkDerivation rec { - name = "pg_repack-${version}"; - version = "1.4.3"; + name = "pg_repack-${version}"; + version = "1.4.3"; - buildInputs = [ postgresql openssl zlib readline ]; + buildInputs = [ postgresql openssl zlib readline ]; - src = fetchFromGitHub { - owner = "reorg"; - repo = "pg_repack"; - rev = "refs/tags/ver_${version}"; - sha256 = "1mmd22nfaxjwnbl3i95f3ivmjvfqwdflgaczlg3129dbpwg265xr"; - }; + src = fetchFromGitHub { + owner = "reorg"; + repo = "pg_repack"; + rev = "refs/tags/ver_${version}"; + sha256 = "1mmd22nfaxjwnbl3i95f3ivmjvfqwdflgaczlg3129dbpwg265xr"; + }; - installPhase = '' - install -D bin/pg_repack -t $out/bin/ - install -D lib/pg_repack.so -t $out/lib/ - install -D lib/{pg_repack--${version}.sql,pg_repack.control} -t $out/share/extension - ''; + installPhase = '' + install -D bin/pg_repack -t $out/bin/ + install -D lib/pg_repack.so -t $out/lib/ + install -D lib/{pg_repack--${version}.sql,pg_repack.control} -t $out/share/extension + ''; - meta = with stdenv.lib; { - description = "Reorganize tables in PostgreSQL databases with minimal locks"; - longDescription = '' - pg_repack is a PostgreSQL extension which lets you remove bloat from tables and indexes, and optionally restore - the physical order of clustered indexes. Unlike CLUSTER and VACUUM FULL it works online, without holding an - exclusive lock on the processed tables during processing. pg_repack is efficient to boot, - with performance comparable to using CLUSTER directly. - ''; - license = licenses.bsd3; - maintainers = with maintainers; [ danbst ]; - inherit (postgresql.meta) platforms; - inherit (src.meta) homepage; - }; + meta = with stdenv.lib; { + description = "Reorganize tables in PostgreSQL databases with minimal locks"; + longDescription = '' + pg_repack is a PostgreSQL extension which lets you remove bloat from tables and indexes, and optionally restore + the physical order of clustered indexes. Unlike CLUSTER and VACUUM FULL it works online, without holding an + exclusive lock on the processed tables during processing. pg_repack is efficient to boot, + with performance comparable to using CLUSTER directly. + ''; + license = licenses.bsd3; + maintainers = with maintainers; [ danbst ]; + inherit (postgresql.meta) platforms; + inherit (src.meta) homepage; + }; } From fafa88f0dae305e87fe8853aded5990a1d5d11fe Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:18:59 -0500 Subject: [PATCH 31/61] nixos/manual: add 18.09 release notes for PostgreSQL Signed-off-by: Austin Seipp --- nixos/doc/manual/release-notes/rl-1809.xml | 24 ++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/nixos/doc/manual/release-notes/rl-1809.xml b/nixos/doc/manual/release-notes/rl-1809.xml index 9ec465d8955eb..6006ba9760be0 100644 --- a/nixos/doc/manual/release-notes/rl-1809.xml +++ b/nixos/doc/manual/release-notes/rl-1809.xml @@ -18,6 +18,30 @@ + + + The PostgreSQL server infrastructure has been greatly improved and is + much easier to use. Extensions are always built against the correct + version of the server without overrides. Server versions are exposed as + "sets" of packages along with their compatible extensions, which are all + compiled and checked continuously. This means you can now use extensions + like plv8 with any PostgreSQL version without compiling anything. + Extensions expressions are easier to write, and new extensions are + available: cstore_fdw, pg_cron, + postgres-hll, and postgres-topn. + + + NixOS 18.09 is the last release that will offer PostgreSQL 9.3 as a + server option. 9.3 will reach its 5-year End of Life in September 2018; + the NixOS 18.09 branch will see no updates at all to PostgreSQL 9.3. It + will only be available to keep existing servers running, so they may be + migrated to new versions. + + + PostgreSQL 10 is now the default option for new NixOS 18.09 + installations. + + Support for wrapping binaries using firejail has been From f30de35e7801de27bbd14aabd7213c35c5528ac9 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:19:02 -0500 Subject: [PATCH 32/61] postgresql: add postgresql 11.0 beta2 package This also bars all extensions which currently do not build. Signed-off-by: Austin Seipp --- pkgs/servers/sql/postgresql/default.nix | 6 ++++++ pkgs/servers/sql/postgresql/ext/cstore_fdw.nix | 4 ++++ pkgs/servers/sql/postgresql/ext/pg_hll.nix | 4 ++++ pkgs/servers/sql/postgresql/ext/pg_partman.nix | 2 +- pkgs/servers/sql/postgresql/ext/pg_repack.nix | 4 ++++ pkgs/servers/sql/postgresql/ext/pgroonga.nix | 5 ++++- pkgs/servers/sql/postgresql/ext/plv8.nix | 5 ++++- pkgs/servers/sql/postgresql/ext/timescaledb.nix | 2 +- pkgs/servers/sql/postgresql/packages.nix | 1 + pkgs/top-level/all-packages.nix | 4 +++- 10 files changed, 32 insertions(+), 5 deletions(-) diff --git a/pkgs/servers/sql/postgresql/default.nix b/pkgs/servers/sql/postgresql/default.nix index a2892bfdac9e3..f36f5d2982a06 100644 --- a/pkgs/servers/sql/postgresql/default.nix +++ b/pkgs/servers/sql/postgresql/default.nix @@ -138,4 +138,10 @@ in { psqlSchema = "10.0"; sha256 = "04a07jkvc5s6zgh6jr78149kcjmsxclizsqabjw44ld4j5n633kc"; }; + + postgresql11 = common { + version = "11beta2"; + psqlSchema = "11.0"; + sha256 = "0qxlfh1a7bhhamrbs3msk71pny7jxx0c0fs26zlmp7jjn138zqii"; + }; } diff --git a/pkgs/servers/sql/postgresql/ext/cstore_fdw.nix b/pkgs/servers/sql/postgresql/ext/cstore_fdw.nix index b6b9f3a5650f5..28243960848fc 100644 --- a/pkgs/servers/sql/postgresql/ext/cstore_fdw.nix +++ b/pkgs/servers/sql/postgresql/ext/cstore_fdw.nix @@ -22,6 +22,10 @@ stdenv.mkDerivation rec { cp *.control $out/share/extension ''; + passthru = { + versionCheck = postgresql.compareVersion "11" < 0; + }; + meta = with stdenv.lib; { description = "Columnar storage for PostgreSQL"; homepage = https://www.citusdata.com/; diff --git a/pkgs/servers/sql/postgresql/ext/pg_hll.nix b/pkgs/servers/sql/postgresql/ext/pg_hll.nix index 6c453f6a40c32..5f9cea00f9728 100644 --- a/pkgs/servers/sql/postgresql/ext/pg_hll.nix +++ b/pkgs/servers/sql/postgresql/ext/pg_hll.nix @@ -21,6 +21,10 @@ stdenv.mkDerivation rec { cp *.control $out/share/extension ''; + passthru = { + versionCheck = postgresql.compareVersion "11" < 0; + }; + meta = with stdenv.lib; { description = "HyperLogLog for PostgreSQL"; homepage = https://www.citusdata.com/; diff --git a/pkgs/servers/sql/postgresql/ext/pg_partman.nix b/pkgs/servers/sql/postgresql/ext/pg_partman.nix index 2b951e6abb6e3..29734c3da6964 100644 --- a/pkgs/servers/sql/postgresql/ext/pg_partman.nix +++ b/pkgs/servers/sql/postgresql/ext/pg_partman.nix @@ -23,7 +23,7 @@ stdenv.mkDerivation rec { ''; passthru = { - versionCheck = postgresql.compareVersion "9.4" >= 0; + versionCheck = postgresql.compareVersion "9.4" >= 0 && postgresql.compareVersion "11" < 0; }; meta = with stdenv.lib; { diff --git a/pkgs/servers/sql/postgresql/ext/pg_repack.nix b/pkgs/servers/sql/postgresql/ext/pg_repack.nix index 69fed13a6db6c..47ac3b2ac4244 100644 --- a/pkgs/servers/sql/postgresql/ext/pg_repack.nix +++ b/pkgs/servers/sql/postgresql/ext/pg_repack.nix @@ -19,6 +19,10 @@ stdenv.mkDerivation rec { install -D lib/{pg_repack--${version}.sql,pg_repack.control} -t $out/share/extension ''; + passthru = { + versionCheck = postgresql.compareVersion "11" < 0; + }; + meta = with stdenv.lib; { description = "Reorganize tables in PostgreSQL databases with minimal locks"; longDescription = '' diff --git a/pkgs/servers/sql/postgresql/ext/pgroonga.nix b/pkgs/servers/sql/postgresql/ext/pgroonga.nix index f4c7bfb1b85f1..21f3b03c83b81 100644 --- a/pkgs/servers/sql/postgresql/ext/pgroonga.nix +++ b/pkgs/servers/sql/postgresql/ext/pgroonga.nix @@ -15,11 +15,14 @@ stdenv.mkDerivation rec { makeFlags = [ "HAVE_MSGPACK=1" ]; installPhase = '' - mkdir -p $out/bin install -D pgroonga.so -t $out/lib/ install -D ./{pgroonga-*.sql,pgroonga.control} -t $out/share/extension ''; + passthru = { + versionCheck = postgresql.compareVersion "11" < 0; + }; + meta = with stdenv.lib; { description = "A PostgreSQL extension to use Groonga as the index"; longDescription = "PGroonga is a PostgreSQL extension to use Groonga as the index. PostgreSQL supports full text search against languages that use only alphabet and digit. It means that PostgreSQL doesn't support full text search against Japanese, Chinese and so on. You can use super fast full text search feature against all languages by installing PGroonga into your PostgreSQL."; diff --git a/pkgs/servers/sql/postgresql/ext/plv8.nix b/pkgs/servers/sql/postgresql/ext/plv8.nix index 009f65b9d81c6..e37ada352a50a 100644 --- a/pkgs/servers/sql/postgresql/ext/plv8.nix +++ b/pkgs/servers/sql/postgresql/ext/plv8.nix @@ -19,11 +19,14 @@ stdenv.mkDerivation rec { ''; installPhase = '' - mkdir -p $out/bin install -D plv8.so -t $out/lib install -D {plls,plcoffee,plv8}{--${version}.sql,.control} -t $out/share/extension ''; + passthru = { + versionCheck = postgresql.compareVersion "11" < 0; + }; + meta = with stdenv.lib; { description = "PL/v8 - A Procedural Language in JavaScript powered by V8"; homepage = https://pgxn.org/dist/plv8/; diff --git a/pkgs/servers/sql/postgresql/ext/timescaledb.nix b/pkgs/servers/sql/postgresql/ext/timescaledb.nix index 6cb9160625f0b..93c68dce7486d 100644 --- a/pkgs/servers/sql/postgresql/ext/timescaledb.nix +++ b/pkgs/servers/sql/postgresql/ext/timescaledb.nix @@ -35,7 +35,7 @@ stdenv.mkDerivation rec { ''; passthru = { - versionCheck = postgresql.compareVersion "9.6" >= 0; + versionCheck = postgresql.compareVersion "9.6" >= 0 && postgresql.compareVersion "11" < 0; }; meta = with stdenv.lib; { diff --git a/pkgs/servers/sql/postgresql/packages.nix b/pkgs/servers/sql/postgresql/packages.nix index 005bde3e29406..9da69628d820e 100644 --- a/pkgs/servers/sql/postgresql/packages.nix +++ b/pkgs/servers/sql/postgresql/packages.nix @@ -62,4 +62,5 @@ with postgresqlPackages; { postgresql95Packages = makePackageSet postgresql95; postgresql96Packages = makePackageSet postgresql96; postgresql10Packages = makePackageSet postgresql10; + postgresql11Packages = makePackageSet postgresql11; } diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index bcb90e6940ae0..2b5222cae64e8 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -13334,7 +13334,8 @@ with pkgs; postgresql94Packages postgresql95Packages postgresql96Packages - postgresql10Packages; + postgresql10Packages + postgresql11Packages; # the default postgres version + package set postgresqlPackages = postgresql10Packages; @@ -13346,6 +13347,7 @@ with pkgs; postgresql95 = postgresql95Packages.postgresql; postgresql96 = postgresql96Packages.postgresql; postgresql10 = postgresql10Packages.postgresql; + postgresql11 = postgresql11Packages.postgresql; ## -- miscellaneous postgresql-related packages. From dcd94557fc697d6cc9234a247bddfe43f91ae674 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:19:04 -0500 Subject: [PATCH 33/61] postgresql11: preliminary JIT support This adds preliminary support for JIT'ing SQL queries via LLVM to PostgreSQL, enabled on versions 11+, on Linux. The default stdenv of PostgreSQL itself and all exposed sub-packages are overridden to use a particular major version of llvmPackages.stdenv so that 'clang' is chosen as the default compiler. Currently, we use llvmPackages_6. To do this, we expose another component from postgresqlXXPackages -- a new .stdenv attribute, which exposes which stdenv PostgreSQL was built with. PostgreSQL JIT support works by compiling all of the source code for the system into LLVM bitcode, and shipping this as part of the binary distribution (under the .lib output, in our case, $lib/lib/bitcode/). At runtime, Postgres uses the LLVM API in order to load this bitcode and JIT queries directly against the source code. This allows inlining database code into queries, in particular inlining operator and expression definitions for custom types, in any extensions. This feature is enabled via the '--with-llvm' flag during configurePhase. This feature is integrated with 'PGXS', the Makefile infrastructure for writing and distributing Postgres extensions. If an extension is being compiled against a version of PostgreSQL that has LLVM JIT support, then the extension code is *also* compiled to bitcode and distributed transparently in the binary output, alongside the extension shared object. In order to ensure consistent bitcode is emitted, when Postgres is compiled with LLVM support, it *must* be compiled with Clang (hence the stdenv choice), and it also *must* embed the copy/path of Clang into the resulting binaries, so PGXS can use the exact same compiler later on during 3rd party extension builds. This design decision is likely because on systems such as Debian, multiple versions of Clang can exist, so any compiled Postgres code/extensions must use a consistent compiler for all future builds. However, this decision has extremely negative consequences for Nix-based packages, because it inserts LLVM and Clang into the closure of the PostgreSQL derivations as a hard runtime-dependency. This bloats the closure size by over a gigabyte (~140MB -> 1.4GB), which is fairly unwieldly and unlikely to be permissible by default. Currently this bloat only applies to the .out (binary) outputs. But this is made worse by the fact that the .lib output is also bloated by having a hard runtime dependency on llvmPackages.llvm.lib. This is because postgresql.lib now ships libllvmjit.so which talks to libLLVM.so, but postgresql.lib also contains client libraries like libpq.so. This effectively bloats every libpq client expression as well by about 200MB. Finally, because of the way PGXS's default installation logic works, it wants to install binary artifacts into the postgresql lib/ directory, which obviously isn't possible in the Nix store as it's read-only. Hence, we create environments composed of all extensions outputs and patch postgresql to load that. But this means the installPhase for every extension is currently a custom hand-written script, and *that* means every extension must now contain logic to install LLVM .bc files on top of .sql and .so files. Instead, we should probably patch PGXS to install to a proper external directory so its install logic can take over and we can remove custom installPhase scripts for most extensions. Postgres' LLVM support logic in its configure script and PGXS code will need to be patched to remove hard-coded references to clang-wrapper, since we always control the exact version of clang used and can remove it as a run-time dependency. Finally, bitcode should probably be moved to separate .bitcode derivations for all server versions and extension outputs, so that libpq clients aren't bloated by indirect dependencies on libLLVM.so (by way of libllvmjit.so). Oh, and PostGIS fails to build with JIT support/clang as the compiler, for some reason. As a result of these significant complications, this support is disabled by default, and should only be considered supported for vanilla PostgreSQL with no third-party extensions. Darwin may also be supported in the future; it may even build, but can't be tested. Signed-off-by: Austin Seipp --- pkgs/servers/sql/postgresql/default.nix | 31 +++++++++++++++++---- pkgs/servers/sql/postgresql/ext/postgis.nix | 4 +++ pkgs/servers/sql/postgresql/packages.nix | 6 ++-- 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/pkgs/servers/sql/postgresql/default.nix b/pkgs/servers/sql/postgresql/default.nix index f36f5d2982a06..bd6f3c7344213 100644 --- a/pkgs/servers/sql/postgresql/default.nix +++ b/pkgs/servers/sql/postgresql/default.nix @@ -1,11 +1,27 @@ { stdenv, lib, fetchurl, makeWrapper , glibc, zlib, readline, libossp_uuid, openssl, libxml2, tzdata, systemd -}: + +# Gate JIT support right now behind a flag; it increases closure size +# dramatically due to the PostgreSQL build system requiring a hard dependency +# on clang-wrapper (~140MB -> 1.4GB). This must be worked around before it can +# be enabled by default by making clang-wrapper a build-time only dependency. +, llvmPackages, enableJitSupport ? false +}@deps: let common = { version, sha256, psqlSchema }: - let atLeast = lib.versionAtLeast version; in stdenv.mkDerivation (rec { + let + atLeast = lib.versionAtLeast version; + + # JIT is only supported on Linux, for now. (Darwin may build, but must be + # tested). + jitEnabled = atLeast "11" && enableJitSupport && deps.stdenv.isLinux; + + # Note: use deps.stdenv, not just 'stdenv', otherwise infinite recursion + # will occur due to lexical scoping rules. + stdenv = if jitEnabled then llvmPackages.stdenv else deps.stdenv; + in stdenv.mkDerivation (rec { name = "postgresql-${version}"; inherit version; @@ -20,7 +36,8 @@ let buildInputs = [ zlib readline openssl libxml2 makeWrapper ] ++ lib.optionals (!stdenv.isDarwin) [ libossp_uuid ] - ++ lib.optionals (atLeast "9.6" && !stdenv.isDarwin) [ systemd ]; + ++ lib.optionals (atLeast "9.6" && !stdenv.isDarwin) [ systemd ] + ++ lib.optionals jitEnabled (with llvmPackages; [ clang llvm ]); enableParallelBuilding = true; @@ -39,6 +56,7 @@ let "--with-system-tzdata=${tzdata}/share/zoneinfo" (lib.optionalString (atLeast "9.6" && !stdenv.isDarwin) "--with-systemd") (if stdenv.isDarwin then "--with-uuid=e2fs" else "--with-ossp-uuid") + (lib.optionalString jitEnabled "--with-llvm") ]; patches = @@ -87,11 +105,14 @@ let doInstallCheck = false; # needs a running daemon? - disallowedReferences = [ stdenv.cc ]; + disallowedReferences = lib.optionals (!jitEnabled) [ stdenv.cc ]; passthru = { - inherit readline psqlSchema; + # Note: we export 'stdenv', because the chosen stdenv *might* be a llvmPackages-based + # one, and we want to propagate that to all extensions. + inherit readline psqlSchema stdenv; compareVersion = builtins.compareVersions version; + hasJitSupport = jitEnabled; }; meta = with lib; { diff --git a/pkgs/servers/sql/postgresql/ext/postgis.nix b/pkgs/servers/sql/postgresql/ext/postgis.nix index a577d318523db..2700461f595bd 100644 --- a/pkgs/servers/sql/postgresql/ext/postgis.nix +++ b/pkgs/servers/sql/postgresql/ext/postgis.nix @@ -95,6 +95,10 @@ in stdenv.mkDerivation rec { "raster/scripts/python/Makefile"; ''; + passthru = { + versionCheck = postgresql.hasJitSupport == false; + }; + meta = with stdenv.lib; { description = "Geographic Objects for PostgreSQL"; homepage = http://postgis.refractions.net; diff --git a/pkgs/servers/sql/postgresql/packages.nix b/pkgs/servers/sql/postgresql/packages.nix index 9da69628d820e..94c1ad8c34f90 100644 --- a/pkgs/servers/sql/postgresql/packages.nix +++ b/pkgs/servers/sql/postgresql/packages.nix @@ -1,14 +1,16 @@ { pkgs, lib }: let - postgresqlPackages = pkgs.callPackages ./default.nix {}; + llvmPackages = pkgs.llvmPackages_6; + postgresqlPackages = pkgs.callPackages ./default.nix { inherit llvmPackages; }; # Filter out any versions which fail a version check. filterPackages = lib.filterAttrs (_: drv: drv.versionCheck or true); makePackageSet = postgresql: let - callPackage = p: args: pkgs.callPackage p (args // { inherit postgresql; }); + stdenv = postgresql.stdenv; # Use the stdenv for the particular version of Postgres + callPackage = p: args: pkgs.callPackage p (args // { inherit postgresql stdenv; }); in filterPackages { # Convenience function for end-users to easily build packages against a specific From 8caf52331f6a3d8c1ddaf2aa7fd0530c6c748e0f Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:19:07 -0500 Subject: [PATCH 34/61] postgresql11: remove hard clang dependency for JIT support When JIT support was enabled, PGXS embedded a copy of clang-wrapper's path into its $out, resulting in a large increase in closure size due to dragging the whole compiler along. A similar problem existed previously by way of binutils 'ld', which was worked around. Perform that same workaround for clang-wrapper, too. With JIT support enabled, this reduces the closure size of postgresql11.out from ~1.4GB to ~410MB. Signed-off-by: Austin Seipp --- pkgs/servers/sql/postgresql/default.nix | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pkgs/servers/sql/postgresql/default.nix b/pkgs/servers/sql/postgresql/default.nix index bd6f3c7344213..dba3e6f8c83d2 100644 --- a/pkgs/servers/sql/postgresql/default.nix +++ b/pkgs/servers/sql/postgresql/default.nix @@ -83,9 +83,6 @@ let moveToOutput "lib/*.a" "$out" moveToOutput "lib/libecpg*" "$out" - # Prevent a retained dependency on gcc-wrapper. - substituteInPlace "$out/lib/pgxs/src/Makefile.global" --replace ${stdenv.cc}/bin/ld ld - if [ -z "''${dontDisableStatic:-}" ]; then # Remove static libraries in case dynamic are available. for i in $out/lib/*.a; do @@ -95,6 +92,12 @@ let fi done fi + + # Prevent a retained dependency on gcc-wrapper. + substituteInPlace "$out/lib/pgxs/src/Makefile.global" --replace ${stdenv.cc}/bin/ld ld + '' + lib.optionalString jitEnabled '' + # In the case of JIT support, prevent a retained dependency on clang-wrapper, too + substituteInPlace "$out/lib/pgxs/src/Makefile.global" --replace ${stdenv.cc}/bin/clang clang ''; postFixup = lib.optionalString (!stdenv.isDarwin && stdenv.hostPlatform.libc == "glibc") @@ -105,7 +108,7 @@ let doInstallCheck = false; # needs a running daemon? - disallowedReferences = lib.optionals (!jitEnabled) [ stdenv.cc ]; + disallowedReferences = [ stdenv.cc ]; passthru = { # Note: we export 'stdenv', because the chosen stdenv *might* be a llvmPackages-based From 91841ffa74f616f65af5ea5fb421fa534c26acaa Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:19:09 -0500 Subject: [PATCH 35/61] postgresql11: keep bitcode/llvmjit.so in $out, not $lib With JIT support enabled, the Postgres build system keeps a copy of bitcode for the server, and a new llvmjit.so extension, in the $lib output. However, this bloats the closure of any libpq-dependent libraries by requiring a hard dependency on libLLVM.so, as well as the actual bitcode files (which aren't useful at all to clients). Moving both of these into $out does not hurt binary output size (since it already was dependent on $lib anyway), but does help all clients. Signed-off-by: Austin Seipp --- pkgs/servers/sql/postgresql/default.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkgs/servers/sql/postgresql/default.nix b/pkgs/servers/sql/postgresql/default.nix index dba3e6f8c83d2..2e08d5fa3f186 100644 --- a/pkgs/servers/sql/postgresql/default.nix +++ b/pkgs/servers/sql/postgresql/default.nix @@ -98,6 +98,11 @@ let '' + lib.optionalString jitEnabled '' # In the case of JIT support, prevent a retained dependency on clang-wrapper, too substituteInPlace "$out/lib/pgxs/src/Makefile.global" --replace ${stdenv.cc}/bin/clang clang + + # Move the bitcode and libllvmjit.so library out of $lib; otherwise, every client that + # depends on libpq.so will also have libLLVM.so in its closure too, bloating it + moveToOutput "lib/bitcode" "$out" + moveToOutput "lib/llvmjit*" "$out" ''; postFixup = lib.optionalString (!stdenv.isDarwin && stdenv.hostPlatform.libc == "glibc") From dce94b7a3bcc2dd2184e7b094e50d1aed3dcb5e3 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:19:11 -0500 Subject: [PATCH 36/61] postgresqlPackages: add new withPackages combinator withPackages is like a local version of the services.postgresql.plugins option: it allows you to build an environment containing PostgreSQL and all the extensions you desire, by way of attrsets. For example, the following command will drop you in a Nix shell containing all the binaries from PostgreSQL 10, including the TimescaleDB extension as a loadable library: $ nix-shell -p \ 'postgresql10Packages.withPackages (p: with p; [ timescaledb ])' This is primarily useful for running demonstrations or local experiments, but may also be useful for performing database upgrades (as you may want pg_dumpall, etc from _newer_ PostgreSQL versions, to run against your current server). Signed-off-by: Austin Seipp --- .../modules/services/databases/postgresql.nix | 20 +--- pkgs/servers/sql/postgresql/packages.nix | 98 +++++++++++++------ 2 files changed, 73 insertions(+), 45 deletions(-) diff --git a/nixos/modules/services/databases/postgresql.nix b/nixos/modules/services/databases/postgresql.nix index c2f46d945df05..a0e4b71c1d903 100644 --- a/nixos/modules/services/databases/postgresql.nix +++ b/nixos/modules/services/databases/postgresql.nix @@ -11,23 +11,9 @@ let # see description of extraPlugins postgresqlAndPlugins = pg: - if plugins == [] then pg - else pkgs.buildEnv { - name = "postgresql-and-plugins-${pg.version}"; - paths = [ pg pg.lib ] ++ plugins; - # We include /bin to ensure the $out/bin directory is created, which is - # needed because we'll be removing the files from that directory in postBuild - # below. See #22653 - pathsToLink = ["/" "/bin"]; - buildInputs = [ pkgs.makeWrapper ]; - postBuild = - '' - mkdir -p $out/bin - rm $out/bin/{pg_config,postgres,pg_ctl} - cp --target-directory=$out/bin ${pg}/bin/{postgres,pg_config,pg_ctl} - wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib - ''; - }; + # See the note about withPackages in postgresql/packages.nix for more + let ps = import ../../../../pkgs/servers/sql/postgresql/packages.nix { inherit pkgs lib; }; + in ps.withPackages pg plugins; postgresqlPackage = if (cfg.package != null) then cfg.package else cfg.packages.postgresql; diff --git a/pkgs/servers/sql/postgresql/packages.nix b/pkgs/servers/sql/postgresql/packages.nix index 94c1ad8c34f90..eea938accc2ba 100644 --- a/pkgs/servers/sql/postgresql/packages.nix +++ b/pkgs/servers/sql/postgresql/packages.nix @@ -7,56 +7,91 @@ let # Filter out any versions which fail a version check. filterPackages = lib.filterAttrs (_: drv: drv.versionCheck or true); + # Build an environment containing PostgreSQL, its libraries, and any paths + # containing binary-compatible installed plugins. + withPackages = postgresql: plugins: + if plugins == [] + then postgresql + else pkgs.buildEnv { + name = "postgresql-and-plugins-${postgresql.version}"; + paths = [ postgresql postgresql.lib ] ++ plugins; + + # We include /bin to ensure the $out/bin directory is created, which is + # needed because we'll be removing the files from that directory in postBuild + # below. See #22653 + pathsToLink = ["/" "/bin"]; + + buildInputs = [ pkgs.makeWrapper ]; + postBuild = '' + mkdir -p $out/bin + rm $out/bin/{pg_config,postgres,pg_ctl} + cp --target-directory=$out/bin ${postgresql}/bin/{postgres,pg_config,pg_ctl} + wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib + ''; + }; + + # This is the same as above, but a bit more spicy: it takes a postgres + # package set and a continuation, which determines which extensions to use in + # the resulting environment. This is the analog of the NixOS module's method + # for selecting compatible plugins -- incompatible plugins will not exist in + # the attrset, resulting in an evaluation error. + withPackageSet = ps: k: withPackages ps.postgresql (k ps); + makePackageSet = postgresql: let stdenv = postgresql.stdenv; # Use the stdenv for the particular version of Postgres callPackage = p: args: pkgs.callPackage p (args // { inherit postgresql stdenv; }); - in - filterPackages { - # Convenience function for end-users to easily build packages against a specific - # version. - inherit callPackage; - # Convenience attribute that exports the postgres derivation used for builds. We - # export these from all-packages.nix to consolidate everything here. - inherit postgresql; + self = filterPackages { + # Convenience function for end-users to easily build packages against a specific + # version. + inherit callPackage; - # PostgreSQL extensions follow from here. + # Convenience function for getting a full PostgreSQL derivation with extensions + # for the given version. + withPackages = withPackageSet self; - cstore_fdw = callPackage ./ext/cstore_fdw.nix {}; + # Convenience attribute that exports the postgres derivation used for builds. We + # export these from all-packages.nix to consolidate everything here. + inherit postgresql; - postgis = callPackage ./ext/postgis.nix {}; + # PostgreSQL extensions follow from here. - pg_cron = callPackage ./ext/pg_cron.nix {}; + cstore_fdw = callPackage ./ext/cstore_fdw.nix {}; - pg_hll = callPackage ./ext/pg_hll.nix {}; + pg_cron = callPackage ./ext/pg_cron.nix {}; - pgjwt = callPackage ./ext/pgjwt.nix {}; + pg_hll = callPackage ./ext/pg_hll.nix {}; - pg_jobmon = callPackage ./ext/pg_jobmon.nix {}; + pgjwt = callPackage ./ext/pgjwt.nix {}; - pg_journal = callPackage ./ext/pg_journal.nix {}; + pg_jobmon = callPackage ./ext/pg_jobmon.nix {}; - pg_partman = callPackage ./ext/pg_partman.nix {}; + pg_journal = callPackage ./ext/pg_journal.nix {}; - pg_repack = callPackage ./ext/pg_repack.nix {}; + pg_partman = callPackage ./ext/pg_partman.nix {}; - pgroonga = callPackage ./ext/pgroonga.nix {}; + pg_repack = callPackage ./ext/pg_repack.nix {}; - pg_similarity = callPackage ./ext/pg_similarity.nix {}; + pgroonga = callPackage ./ext/pgroonga.nix {}; - pgtap = callPackage ./ext/pgtap.nix {}; + pg_similarity = callPackage ./ext/pg_similarity.nix {}; - pg_topn = callPackage ./ext/pg_topn.nix {}; + pgtap = callPackage ./ext/pgtap.nix {}; - plv8 = callPackage ./ext/plv8.nix { - v8 = pkgs.v8_6_x; - }; + pg_topn = callPackage ./ext/pg_topn.nix {}; - timescaledb = callPackage ./ext/timescaledb.nix {}; + plv8 = callPackage ./ext/plv8.nix { + v8 = pkgs.v8_6_x; + }; - tsearch_extras = callPackage ./ext/tsearch_extras.nix { }; - }; + postgis = callPackage ./ext/postgis.nix {}; + + timescaledb = callPackage ./ext/timescaledb.nix {}; + + tsearch_extras = callPackage ./ext/tsearch_extras.nix { }; + }; + in self; in with postgresqlPackages; { postgresql93Packages = makePackageSet postgresql93; @@ -65,4 +100,11 @@ with postgresqlPackages; { postgresql96Packages = makePackageSet postgresql96; postgresql10Packages = makePackageSet postgresql10; postgresql11Packages = makePackageSet postgresql11; + + # HEADS UP: do NOT export this from the top level of all-packages.nix! + # it is only used by the NixOS module for PostgreSQL in order to reduce + # some of the duplicated logic needed for withPackages. this should + # _only_ be used by postgresql.nix, and nothing more. Do not taunt + # Happy Fun Ball. + inherit withPackages; } From 9b06c7af150755f6c4efee289b4a91b26a8a9c70 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:19:14 -0500 Subject: [PATCH 37/61] postgresqlPackages.pg_similarity: remove needless dir Signed-off-by: Austin Seipp --- pkgs/servers/sql/postgresql/ext/pg_similarity.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/pkgs/servers/sql/postgresql/ext/pg_similarity.nix b/pkgs/servers/sql/postgresql/ext/pg_similarity.nix index 32945c9fa62b3..aa204eec5a6d3 100644 --- a/pkgs/servers/sql/postgresql/ext/pg_similarity.nix +++ b/pkgs/servers/sql/postgresql/ext/pg_similarity.nix @@ -13,7 +13,6 @@ stdenv.mkDerivation { buildInputs = [ postgresql gcc ]; buildPhase = "USE_PGXS=1 make"; installPhase = '' - mkdir -p $out/bin # for buildEnv to setup proper symlinks install -D pg_similarity.so -t $out/lib/ install -D ./{pg_similarity--unpackaged--1.0.sql,pg_similarity--1.0.sql,pg_similarity.control} -t $out/share/extension ''; From 49dbf9f28ec32af8cbe0413f5c612b3bc4538372 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:19:16 -0500 Subject: [PATCH 38/61] postgresqlPackages: make PGXS install phase Nix-compatible By fixing the (non-overrideable) PGXS installation directory variables in the Postgres expression, any further extensions can be installed using traditional pg_config-based PGXS Makefile build systems. This that overriding 'installPhase' is no longer necessary; an ordinary PREFIX=... will work, just like most Makefile build systems. Aside from being cleaner, less-error prone, and easier to write, this is also another piece necessary for useable JIT support. Signed-off-by: Austin Seipp --- pkgs/servers/sql/postgresql/default.nix | 5 ++++ .../servers/sql/postgresql/ext/cstore_fdw.nix | 8 +---- pkgs/servers/sql/postgresql/ext/pg_cron.nix | 8 +---- pkgs/servers/sql/postgresql/ext/pg_hll.nix | 8 +---- pkgs/servers/sql/postgresql/ext/pg_jobmon.nix | 8 +---- .../servers/sql/postgresql/ext/pg_journal.nix | 5 +--- .../servers/sql/postgresql/ext/pg_partman.nix | 9 +----- pkgs/servers/sql/postgresql/ext/pg_repack.nix | 6 +--- .../sql/postgresql/ext/pg_similarity.nix | 12 ++++---- pkgs/servers/sql/postgresql/ext/pg_topn.nix | 8 +---- pkgs/servers/sql/postgresql/ext/pgjwt.nix | 7 ++--- pkgs/servers/sql/postgresql/ext/pgroonga.nix | 7 +---- pkgs/servers/sql/postgresql/ext/pgtap.nix | 4 +-- pkgs/servers/sql/postgresql/ext/plv8.nix | 10 +++---- .../sql/postgresql/ext/tsearch_extras.nix | 6 +--- .../sql/postgresql/patches/pgxs-nix.patch | 29 +++++++++++++++++++ 16 files changed, 56 insertions(+), 84 deletions(-) create mode 100644 pkgs/servers/sql/postgresql/patches/pgxs-nix.patch diff --git a/pkgs/servers/sql/postgresql/default.nix b/pkgs/servers/sql/postgresql/default.nix index 2e08d5fa3f186..3b7a5774541fe 100644 --- a/pkgs/servers/sql/postgresql/default.nix +++ b/pkgs/servers/sql/postgresql/default.nix @@ -93,6 +93,11 @@ let done fi + # Fix the PGXS installation paths, since they can't write into the Nix store + patch -d "$out/lib/pgxs/src" < ${./patches/pgxs-nix.patch} + substituteInPlace "$out/lib/pgxs/src/Makefile.global" \ + --subst-var-by NIX_POSTGRES_INCLUDE "$out/include" + # Prevent a retained dependency on gcc-wrapper. substituteInPlace "$out/lib/pgxs/src/Makefile.global" --replace ${stdenv.cc}/bin/ld ld '' + lib.optionalString jitEnabled '' diff --git a/pkgs/servers/sql/postgresql/ext/cstore_fdw.nix b/pkgs/servers/sql/postgresql/ext/cstore_fdw.nix index 28243960848fc..d6b0b3da74306 100644 --- a/pkgs/servers/sql/postgresql/ext/cstore_fdw.nix +++ b/pkgs/servers/sql/postgresql/ext/cstore_fdw.nix @@ -14,13 +14,7 @@ stdenv.mkDerivation rec { sha256 = "1cpkpbv4c82l961anzwp74r1jc8f0n5z5cvwy4lyrqg5jr501nd4"; }; - installPhase = '' - mkdir -p $out/{lib,share/extension} - - cp *.so $out/lib - cp *.sql $out/share/extension - cp *.control $out/share/extension - ''; + makeFlags = [ "PREFIX=$(out)" ]; passthru = { versionCheck = postgresql.compareVersion "11" < 0; diff --git a/pkgs/servers/sql/postgresql/ext/pg_cron.nix b/pkgs/servers/sql/postgresql/ext/pg_cron.nix index 25118f7c758ac..ce5e636084112 100644 --- a/pkgs/servers/sql/postgresql/ext/pg_cron.nix +++ b/pkgs/servers/sql/postgresql/ext/pg_cron.nix @@ -13,13 +13,7 @@ stdenv.mkDerivation rec { sha256 = "0n74dx1wkg9qxvjhnx03028465ap3p97v2kzqww833dws1wqk5m1"; }; - installPhase = '' - mkdir -p $out/{lib,share/extension} - - cp *.so $out/lib - cp *.sql $out/share/extension - cp *.control $out/share/extension - ''; + makeFlags = [ "PREFIX=$(out)" ]; passthru = { versionCheck = postgresql.compareVersion "9.5" >= 0; diff --git a/pkgs/servers/sql/postgresql/ext/pg_hll.nix b/pkgs/servers/sql/postgresql/ext/pg_hll.nix index 5f9cea00f9728..7e9816d6136bd 100644 --- a/pkgs/servers/sql/postgresql/ext/pg_hll.nix +++ b/pkgs/servers/sql/postgresql/ext/pg_hll.nix @@ -13,13 +13,7 @@ stdenv.mkDerivation rec { sha256 = "044x9v9kjhxb0idqb9f5i7c3yygxxsqliswl4kspqy9f9qcblckl"; }; - installPhase = '' - mkdir -p $out/{lib,share/extension} - - cp *.so $out/lib - cp *.sql $out/share/extension - cp *.control $out/share/extension - ''; + makeFlags = [ "PREFIX=$(out)" ]; passthru = { versionCheck = postgresql.compareVersion "11" < 0; diff --git a/pkgs/servers/sql/postgresql/ext/pg_jobmon.nix b/pkgs/servers/sql/postgresql/ext/pg_jobmon.nix index 9f60cf3ff5b39..b92618a2efa97 100644 --- a/pkgs/servers/sql/postgresql/ext/pg_jobmon.nix +++ b/pkgs/servers/sql/postgresql/ext/pg_jobmon.nix @@ -13,13 +13,7 @@ stdenv.mkDerivation rec { sha256 = "09izh6j1rpkllmy2kjhd9pwzld6lryp7825129k5jbbvnavxv6g8"; }; - installPhase = '' - mkdir -p $out/share/extension - - cp -v updates/*.sql $out/share/extension - cp -v sql/*.sql $out/share/extension - cp -v *.control $out/share/extension - ''; + makeFlags = [ "PREFIX=$(out)" ]; meta = with stdenv.lib; { description = "Log and monitor PostgreSQL jobs"; diff --git a/pkgs/servers/sql/postgresql/ext/pg_journal.nix b/pkgs/servers/sql/postgresql/ext/pg_journal.nix index 3a5637e468d75..956fcd3617f99 100644 --- a/pkgs/servers/sql/postgresql/ext/pg_journal.nix +++ b/pkgs/servers/sql/postgresql/ext/pg_journal.nix @@ -27,10 +27,7 @@ stdenv.mkDerivation rec { }) ]; - installPhase = '' - mkdir -p $out/lib - cp pg_journal.so $out/lib - ''; + makeFlags = [ "PREFIX=$(out)" ]; meta = with stdenv.lib; { description = "Log PostgreSQL messages to systemd journal"; diff --git a/pkgs/servers/sql/postgresql/ext/pg_partman.nix b/pkgs/servers/sql/postgresql/ext/pg_partman.nix index 29734c3da6964..abf7d76824e8f 100644 --- a/pkgs/servers/sql/postgresql/ext/pg_partman.nix +++ b/pkgs/servers/sql/postgresql/ext/pg_partman.nix @@ -13,14 +13,7 @@ stdenv.mkDerivation rec { sha256 = "069m2y20pkpk7lw7d2spaa1zpkb47dmm76rxi1d4qvq3vgfd739m"; }; - installPhase = '' - mkdir -p $out/{lib,share/extension} - - cp -v src/*.so $out/lib - cp -v updates/*.sql $out/share/extension - cp -v sql/*.sql $out/share/extension - cp -v *.control $out/share/extension - ''; + makeFlags = [ "PREFIX=$(out)" ]; passthru = { versionCheck = postgresql.compareVersion "9.4" >= 0 && postgresql.compareVersion "11" < 0; diff --git a/pkgs/servers/sql/postgresql/ext/pg_repack.nix b/pkgs/servers/sql/postgresql/ext/pg_repack.nix index 47ac3b2ac4244..d74716ae50134 100644 --- a/pkgs/servers/sql/postgresql/ext/pg_repack.nix +++ b/pkgs/servers/sql/postgresql/ext/pg_repack.nix @@ -13,11 +13,7 @@ stdenv.mkDerivation rec { sha256 = "1mmd22nfaxjwnbl3i95f3ivmjvfqwdflgaczlg3129dbpwg265xr"; }; - installPhase = '' - install -D bin/pg_repack -t $out/bin/ - install -D lib/pg_repack.so -t $out/lib/ - install -D lib/{pg_repack--${version}.sql,pg_repack.control} -t $out/share/extension - ''; + makeFlags = [ "PREFIX=$(out)" ]; passthru = { versionCheck = postgresql.compareVersion "11" < 0; diff --git a/pkgs/servers/sql/postgresql/ext/pg_similarity.nix b/pkgs/servers/sql/postgresql/ext/pg_similarity.nix index aa204eec5a6d3..806a8ab81cdd3 100644 --- a/pkgs/servers/sql/postgresql/ext/pg_similarity.nix +++ b/pkgs/servers/sql/postgresql/ext/pg_similarity.nix @@ -1,4 +1,4 @@ -{ stdenv, lib, fetchFromGitHub, gcc, postgresql }: +{ stdenv, lib, fetchFromGitHub, postgresql }: stdenv.mkDerivation { @@ -10,12 +10,10 @@ stdenv.mkDerivation { sha256 = "1z4v4r2yccdr8kz3935fnk1bc5vj0qj0apscldyap4wxlyi89xim"; }; - buildInputs = [ postgresql gcc ]; - buildPhase = "USE_PGXS=1 make"; - installPhase = '' - install -D pg_similarity.so -t $out/lib/ - install -D ./{pg_similarity--unpackaged--1.0.sql,pg_similarity--1.0.sql,pg_similarity.control} -t $out/share/extension - ''; + buildInputs = [ postgresql ]; + makeFlags = [ "PREFIX=$(out)" ]; + + USE_PGXS=1; # needed for 'make' to work meta = { description = '' diff --git a/pkgs/servers/sql/postgresql/ext/pg_topn.nix b/pkgs/servers/sql/postgresql/ext/pg_topn.nix index 00b712975a310..1f89f65b3efd3 100644 --- a/pkgs/servers/sql/postgresql/ext/pg_topn.nix +++ b/pkgs/servers/sql/postgresql/ext/pg_topn.nix @@ -14,13 +14,7 @@ stdenv.mkDerivation rec { sha256 = "00hc3hgnqv9xaalizbcvprb7s55sydj2qgk3rhgrdlwg2g025h62"; }; - installPhase = '' - mkdir -p $out/{lib,share/extension} - - cp *.so $out/lib - cp *.sql $out/share/extension - cp *.control $out/share/extension - ''; + makeFlags = [ "PREFIX=$(out)" ]; passthru = { versionCheck = postgresql.compareVersion "9.4" >= 0; diff --git a/pkgs/servers/sql/postgresql/ext/pgjwt.nix b/pkgs/servers/sql/postgresql/ext/pgjwt.nix index 3185e2265744e..503a09b0cd58e 100644 --- a/pkgs/servers/sql/postgresql/ext/pgjwt.nix +++ b/pkgs/servers/sql/postgresql/ext/pgjwt.nix @@ -11,11 +11,8 @@ stdenv.mkDerivation rec { sha256 = "1riz0xvwb6y02j0fljbr9hcbqb2jqs4njlivmavy9ysbcrrv1vrf"; }; - buildPhase = ":"; - installPhase = '' - mkdir -p $out/share/extension - cp pg*sql *.control $out/share/extension - ''; + buildInputs = [ postgresql ]; + makeFlags = [ "PREFIX=$(out)" ]; meta = with stdenv.lib; { description = "PostgreSQL implementation of JSON Web Tokens"; diff --git a/pkgs/servers/sql/postgresql/ext/pgroonga.nix b/pkgs/servers/sql/postgresql/ext/pgroonga.nix index 21f3b03c83b81..1b48e33aada78 100644 --- a/pkgs/servers/sql/postgresql/ext/pgroonga.nix +++ b/pkgs/servers/sql/postgresql/ext/pgroonga.nix @@ -12,12 +12,7 @@ stdenv.mkDerivation rec { nativeBuildInputs = [ pkgconfig ]; buildInputs = [ postgresql msgpack groonga ]; - makeFlags = [ "HAVE_MSGPACK=1" ]; - - installPhase = '' - install -D pgroonga.so -t $out/lib/ - install -D ./{pgroonga-*.sql,pgroonga.control} -t $out/share/extension - ''; + makeFlags = [ "HAVE_MSGPACK=1" "PREFIX=$(out)" ]; passthru = { versionCheck = postgresql.compareVersion "11" < 0; diff --git a/pkgs/servers/sql/postgresql/ext/pgtap.nix b/pkgs/servers/sql/postgresql/ext/pgtap.nix index c6eb4013c5ce4..fa493448b8c93 100644 --- a/pkgs/servers/sql/postgresql/ext/pgtap.nix +++ b/pkgs/servers/sql/postgresql/ext/pgtap.nix @@ -13,9 +13,7 @@ stdenv.mkDerivation rec { nativeBuildInputs = [ postgresql perl perlPackages.TAPParserSourceHandlerpgTAP which ]; - installPhase = '' - install -D {sql/pgtap--${version}.sql,pgtap.control} -t $out/share/extension - ''; + makeFlags = [ "PREFIX=$(out)" ]; meta = with stdenv.lib; { description = "pgTAP is a unit testing framework for PostgreSQL"; diff --git a/pkgs/servers/sql/postgresql/ext/plv8.nix b/pkgs/servers/sql/postgresql/ext/plv8.nix index e37ada352a50a..c090ebcc8777e 100644 --- a/pkgs/servers/sql/postgresql/ext/plv8.nix +++ b/pkgs/servers/sql/postgresql/ext/plv8.nix @@ -14,13 +14,11 @@ stdenv.mkDerivation rec { sha256 = "1sfpxz0zcbinn6822j12lkwgrw9kfacrs83ic968rm489rl9w241"; }; - preConfigure = '' - substituteInPlace Makefile --replace '-lv8_libplatform' '-lv8_libplatform -lv8_libbase' - ''; + makeFlags = [ "PREFIX=$(out)" ]; - installPhase = '' - install -D plv8.so -t $out/lib - install -D {plls,plcoffee,plv8}{--${version}.sql,.control} -t $out/share/extension + preConfigure = '' + substituteInPlace Makefile \ + --replace '-lv8_libplatform' '-lv8_libplatform -lv8_libbase' ''; passthru = { diff --git a/pkgs/servers/sql/postgresql/ext/tsearch_extras.nix b/pkgs/servers/sql/postgresql/ext/tsearch_extras.nix index 8b9e9322b4796..42814e87f6060 100644 --- a/pkgs/servers/sql/postgresql/ext/tsearch_extras.nix +++ b/pkgs/servers/sql/postgresql/ext/tsearch_extras.nix @@ -13,11 +13,7 @@ stdenv.mkDerivation rec { nativenativeBuildInputs = [ pkgconfig ]; buildInputs = [ postgresql ]; - - installPhase = '' - install -D tsearch_extras.so -t $out/lib/ - install -D ./{tsearch_extras--1.0.sql,tsearch_extras.control} -t $out/share/extension - ''; + makeFlags = [ "PREFIX=$(out)" ]; meta = with stdenv.lib; { description = "Provides a few PostgreSQL functions for a lower-level data full text search"; diff --git a/pkgs/servers/sql/postgresql/patches/pgxs-nix.patch b/pkgs/servers/sql/postgresql/patches/pgxs-nix.patch new file mode 100644 index 0000000000000..478dc0a4686f8 --- /dev/null +++ b/pkgs/servers/sql/postgresql/patches/pgxs-nix.patch @@ -0,0 +1,29 @@ +--- Makefile.global 2018-08-06 02:21:37.865065005 -0500 ++++ Makefile.global.new 2018-08-06 02:30:17.063261754 -0500 +@@ -153,16 +153,16 @@ + PG_CONFIG = pg_config + endif + +-bindir := $(shell $(PG_CONFIG) --bindir) +-datadir := $(shell $(PG_CONFIG) --sharedir) +-sysconfdir := $(shell $(PG_CONFIG) --sysconfdir) +-libdir := $(shell $(PG_CONFIG) --libdir) +-pkglibdir := $(shell $(PG_CONFIG) --pkglibdir) +-includedir := $(shell $(PG_CONFIG) --includedir) +-pkgincludedir := $(shell $(PG_CONFIG) --pkgincludedir) +-mandir := $(shell $(PG_CONFIG) --mandir) +-docdir := $(shell $(PG_CONFIG) --docdir) +-localedir := $(shell $(PG_CONFIG) --localedir) ++bindir := $(PREFIX)/bin ++datadir := $(PREFIX)/share ++sysconfdir := $(PREFIX)/etc ++libdir := $(PREFIX)/lib ++pkglibdir := $(PREFIX)/lib ++includedir := $(PREFIX)/include ++pkgincludedir := @NIX_POSTGRES_INCLUDE@ ++mandir := $(PREFIX)/share/man ++docdir := $(PREFIX)/share/doc ++localedir := $(PREFIX)/share/locale + + endif # PGXS + From 54ec98dc58802f03e0d27e2fb84a1d4ba82a7d06 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:19:19 -0500 Subject: [PATCH 39/61] nixos/manual: finish section on PostgreSQL server upgrades Signed-off-by: Austin Seipp --- .../modules/services/databases/postgresql.xml | 80 ++++++++++++++++++- 1 file changed, 78 insertions(+), 2 deletions(-) diff --git a/nixos/modules/services/databases/postgresql.xml b/nixos/modules/services/databases/postgresql.xml index fa13c87ddf0c1..a0dbc163edcda 100644 --- a/nixos/modules/services/databases/postgresql.xml +++ b/nixos/modules/services/databases/postgresql.xml @@ -127,13 +127,83 @@ version of PostgreSQL, e.g. TimescaleDB does not work with PostgreSQL 9.5. Should you specify v9.5 (using ), the above example would fail with an evaluation error. -FIXME: talk about CREATE EXTENSION. +Once this has been done and nixos-rebuild has been +run, you may run CREATE EXTENSION> in order to +install the extension for the given database you've selected: + +postgres=> CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE; + + + +Any functions, types, etc that are provided by the extension should +now be available.
Upgrading -FIXME: document dump/upgrade/load cycle. +This section will be brief. Be sure to read the +official +PostgreSQL Upgrading documentation for more detailed information +about the steps below. This section will use the pg_dumpall +method. + +First, you must dump all data from the server using +pg_dumpall, but this must be executed by the +new version of the PostgreSQL tools. This can be done +easily using the existing Postgres infrastructure, by using nix-shell +in order to install the tools: + + +a@link> nix-shell -p 'postgresql10Packages.withPackages (_: [])' +[nix-shell:~]$ sudo -u postgres pg_dumpall > dumpfile + + + +At this point, the file dumpfile contains a dump of +the entire server, and can be backed up or transported elsewhere. Also note +that the above command is analagous to the NixOS module configuration: you +can enter a shell with extensions available, by selecting them from the +attribute set provided to the lambda. The above example uses no extensions. + + +Next, stop the PostgreSQL service using systemctl stop: + + +[nix-shell:~]$ sudo systemctl stop postgres + + + +At this point, you may move the data directory out of the way, so that +the database will be re-initialized in a clean manner once the new +server version is installed. By default, the NixOS module for +PostgreSQL installs each database by default into its own directory +under /var/lib/postgresql with a unique name based +on the version number, so installing a new version will not overwrite +data from the old one. However, you may wish to do this anyway at this +point, and you must do it if you changed the data +directory yourself: + + +[nix-shell:~]$ sudo mv /var/lib/postgresql/* ... + + + +Finally, upgrade your configuration.nix to point the + option to point to the new +version of PostgreSQL; for example, postgresql10Packages. +Then start the new service using nixos-rebuild switch. Note +that you do not need to keep a backup or resupply copies of any configuration +files, since they're managed by Nix. However, you might want to make some tweaks +for the new version of the server, as well. + +At this point, you can simply use psql in order to load +the old data into the new server, including roles, databases, etc: + + +[nix-shell:~]$ sudo -u postgres psql -d postgres -f dumpfile + +
@@ -144,4 +214,10 @@ linkend="opt-services.postgresql.enable">here. +
Authoring new extensions + +FIXME: document how to write an extension + +
+ From 4119c9fde040aa0a938974950ad65318bfa4bf16 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:19:21 -0500 Subject: [PATCH 40/61] postgresqlPackages: init citus 7.5.0 Signed-off-by: Austin Seipp --- pkgs/servers/sql/postgresql/ext/citus.nix | 33 +++++++++++++++++++++++ pkgs/servers/sql/postgresql/packages.nix | 2 ++ 2 files changed, 35 insertions(+) create mode 100644 pkgs/servers/sql/postgresql/ext/citus.nix diff --git a/pkgs/servers/sql/postgresql/ext/citus.nix b/pkgs/servers/sql/postgresql/ext/citus.nix new file mode 100644 index 0000000000000..f963ea63e316e --- /dev/null +++ b/pkgs/servers/sql/postgresql/ext/citus.nix @@ -0,0 +1,33 @@ +{ stdenv, fetchFromGitHub, postgresql, curl }: + +stdenv.mkDerivation rec { + name = "citus-${version}"; + version = "7.5.0"; + + src = fetchFromGitHub { + owner = "citusdata"; + repo = "citus"; + rev = "refs/tags/v${version}"; + sha256 = "1ipzla0whfr4zfzz4csjrzw1w9chlpgxxjlxblbsb9v9y9wd7i72"; + }; + + buildInputs = [ postgresql curl ]; + makeFlags = [ "PREFIX=$(out)" ]; + + patchPhase = '' + substituteInPlace ./Makefile \ + --replace '$(DESTDIR)$(includedir_server)' "$out/include/server" + ''; + + passthru = { + versionCheck = postgresql.compareVersion "9.6" >= 0 && postgresql.compareVersion "11" < 0; + }; + + meta = with stdenv.lib; { + description = "Transparent, distributed sharding and replication for PostgreSQL"; + homepage = https://www.citusdata.com/; + maintainers = with maintainers; [ thoughtpolice ]; + platforms = platforms.linux; + license = licenses.asl20; + }; +} diff --git a/pkgs/servers/sql/postgresql/packages.nix b/pkgs/servers/sql/postgresql/packages.nix index eea938accc2ba..ccd48f7516834 100644 --- a/pkgs/servers/sql/postgresql/packages.nix +++ b/pkgs/servers/sql/postgresql/packages.nix @@ -57,6 +57,8 @@ let # PostgreSQL extensions follow from here. + citus = callPackage ./ext/citus.nix {}; + cstore_fdw = callPackage ./ext/cstore_fdw.nix {}; pg_cron = callPackage ./ext/pg_cron.nix {}; From 429f350e5bacea4e1eb9ec1dc027679d10196458 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:19:24 -0500 Subject: [PATCH 41/61] postgresqlPackages.postgis: fix gdal against the same postgresql the extension is built against GDAL's Postgres support had a weird cyclic dependency: GDAL depends on Postgres (for possible spatial information querying for GDAL clients), and PostGIS depends on GDAL. This sounds fine, except it has a dirty follow up: GDAL is *only* built against the default version of PostgreSQL. This means if you try to use a non-default version of Postgres with PostGIS, it pulls in GDAL, which pulls in some other version of Postgres. This results in weird situations like PostgreSQL 9.4 + PostGIS depending on PostgreSQL 9.6 in their closure. In a previous experiment I tried making this optional, but there's a better way: instead of making it optional, we simply let the default 'gdal' expression depend on 'postgresql' (whatever version the default might be), and then we simply override the version of GDAL used by postgis to be the *same* postgres that postgis is built against. This effectively results in a private copy of gdal that only is used by the postgis extension builds. Signed-off-by: Austin Seipp --- pkgs/servers/sql/postgresql/packages.nix | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkgs/servers/sql/postgresql/packages.nix b/pkgs/servers/sql/postgresql/packages.nix index ccd48f7516834..75da52a0eeb56 100644 --- a/pkgs/servers/sql/postgresql/packages.nix +++ b/pkgs/servers/sql/postgresql/packages.nix @@ -87,7 +87,14 @@ let v8 = pkgs.v8_6_x; }; - postgis = callPackage ./ext/postgis.nix {}; + postgis = callPackage ./ext/postgis.nix { + ## NOTE: set postgresql to the version we use, so we don't end up in + ## the odd case where 'gdal' pulls in a different postgresql version and + ## causes various versions to depend on each other. e.g. if 9.6 is the + ## default and this is not overridden, and you choose 10, then psql 10's + ## closure will depend on psql 9.6 + gdal = pkgs.gdal.override { inherit postgresql; }; + }; timescaledb = callPackage ./ext/timescaledb.nix {}; From a022a958038b8edce50343b7a66fecdd69039176 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 10:59:42 -0500 Subject: [PATCH 42/61] postgresqlPackages: expose a set containing all postgresqlPackages sets, for testing Occasionally, it may be useful to enumerate _all_ supported package sets, for example, in the NixOS tests. This refactoring supports that while keeping the level of redundancy the same, for the most part. Signed-off-by: Austin Seipp --- pkgs/servers/sql/postgresql/packages.nix | 37 +++++++++++++++--------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/pkgs/servers/sql/postgresql/packages.nix b/pkgs/servers/sql/postgresql/packages.nix index 75da52a0eeb56..431840c69c5e8 100644 --- a/pkgs/servers/sql/postgresql/packages.nix +++ b/pkgs/servers/sql/postgresql/packages.nix @@ -101,19 +101,28 @@ let tsearch_extras = callPackage ./ext/tsearch_extras.nix { }; }; in self; + + allPostgresqlPackages = with postgresqlPackages; { + postgresql93Packages = makePackageSet postgresql93; + postgresql94Packages = makePackageSet postgresql94; + postgresql95Packages = makePackageSet postgresql95; + postgresql96Packages = makePackageSet postgresql96; + postgresql10Packages = makePackageSet postgresql10; + postgresql11Packages = makePackageSet postgresql11; + }; in -with postgresqlPackages; { - postgresql93Packages = makePackageSet postgresql93; - postgresql94Packages = makePackageSet postgresql94; - postgresql95Packages = makePackageSet postgresql95; - postgresql96Packages = makePackageSet postgresql96; - postgresql10Packages = makePackageSet postgresql10; - postgresql11Packages = makePackageSet postgresql11; - - # HEADS UP: do NOT export this from the top level of all-packages.nix! - # it is only used by the NixOS module for PostgreSQL in order to reduce - # some of the duplicated logic needed for withPackages. this should - # _only_ be used by postgresql.nix, and nothing more. Do not taunt - # Happy Fun Ball. +{ + # HEADS UP: do NOT export this from the top level of all-packages.nix! it is + # only used by the NixOS module for PostgreSQL in order to reduce some of the + # duplicated logic needed for withPackages. this should _only_ be used by + # postgresql.nix, and nothing more. Do not taunt Happy Fun Ball. inherit withPackages; -} + + # This expression is useful primarily for NixOS tests: they run the basic + # tests with every version of PostgreSQL available, so they use this in order + # to enumerate them. It should also not be exported from all-packages.nix. + inherit allPostgresqlPackages; + + # Finally, include all of the package sets directly as well, so they can + # be exposed in all-packages.nix +} // allPostgresqlPackages From 364c06bc3f6ff273459e25f6e3d49cd9465faeb4 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 11:55:12 -0500 Subject: [PATCH 43/61] nixos/postgresql-backup: migrate to new module API Signed-off-by: Austin Seipp --- .../services/backup/postgresql-backup.nix | 74 +++++++++---------- 1 file changed, 36 insertions(+), 38 deletions(-) diff --git a/nixos/modules/services/backup/postgresql-backup.nix b/nixos/modules/services/backup/postgresql-backup.nix index 2ec78ce6f2cfa..039c079463187 100644 --- a/nixos/modules/services/backup/postgresql-backup.nix +++ b/nixos/modules/services/backup/postgresql-backup.nix @@ -3,46 +3,44 @@ with lib; let - cfg = config.services.postgresqlBackup; - postgresqlBackupService = db : - { - enable = true; - - description = "Backup of database ${db}"; - - requires = [ "postgresql.service" ]; - - preStart = '' - mkdir -m 0700 -p ${cfg.location} - chown postgres ${cfg.location} - ''; - - script = '' - if [ -e ${cfg.location}/${db}.sql.gz ]; then - ${pkgs.coreutils}/bin/mv ${cfg.location}/${db}.sql.gz ${cfg.location}/${db}.prev.sql.gz - fi - - ${config.services.postgresql.package}/bin/pg_dump ${cfg.pgdumpOptions} ${db} | \ - ${pkgs.gzip}/bin/gzip -c > ${cfg.location}/${db}.sql.gz - ''; - - serviceConfig = { - Type = "oneshot"; - PermissionsStartOnly = "true"; - User = "postgres"; - }; - - startAt = cfg.startAt; + postgresqlBackupService = db: { + description = "Backup of database ${db}"; + enable = true; + requires = [ "postgresql.service" ]; + startAt = cfg.startAt; + + preStart = '' + mkdir -m 0700 -p ${cfg.location} + chown postgres ${cfg.location} + ''; + + script = '' + # Use the same PATH as the PostgreSQL instance, as it's configured + # with plugins and extensions. This is a bit of a hack; we can't easily + # set the nixos systemd options to do this, because the path is already + # constructed and is too long by the time the systemd.nix module examines + # it + export PATH="${config.systemd.services.postgresql.environment.PATH}" + + if [ -e ${cfg.location}/${db}.sql.gz ]; then + mv ${cfg.location}/${db}.sql.gz ${cfg.location}/${db}.prev.sql.gz + fi + + pg_dump ${cfg.pgdumpOptions} ${db} | ${pkgs.gzip}/bin/gzip -c > ${cfg.location}/${db}.sql.gz + ''; + + serviceConfig = { + Type = "oneshot"; + PermissionsStartOnly = "true"; + User = "postgres"; }; + }; in { - options = { - services.postgresqlBackup = { - enable = mkOption { default = false; description = '' @@ -81,14 +79,14 @@ in { ''; }; }; - }; config = mkIf config.services.postgresqlBackup.enable { - systemd.services = listToAttrs (map (db : { - name = "postgresqlBackup-${db}"; - value = postgresqlBackupService db; } ) cfg.databases); - }; + systemd.services = listToAttrs (map (db: { + name = "postgresqlBackup-${db}"; + value = postgresqlBackupService db; + }) cfg.databases); + }; } From 73866e535a6f5d7991158f29e5445b8f72f9ec8e Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 11:55:27 -0500 Subject: [PATCH 44/61] nixos/tests: touch up PostgreSQL tests, migrate to new module API Signed-off-by: Austin Seipp --- nixos/tests/postgresql.nix | 43 ++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/nixos/tests/postgresql.nix b/nixos/tests/postgresql.nix index f1f09277f342a..46dcf75cd3619 100644 --- a/nixos/tests/postgresql.nix +++ b/nixos/tests/postgresql.nix @@ -1,9 +1,24 @@ -{ system ? builtins.currentSystem }: +{ system ? builtins.currentSystem +}: + with import ../lib/testing.nix { inherit system; }; with pkgs.lib; + let - postgresql-versions = pkgs.callPackages ../../pkgs/servers/sql/postgresql { }; - test-sql = pkgs.writeText "postgresql-test" '' + + # An attrset containing every version of PostgreSQL, shipped by Nixpkgs. The + # tests are run once for each version. + postgresql-versions = + # This is an existing attrset containing every supported version... + let allPackages = (pkgs.callPackage ../../pkgs/servers/sql/postgresql/packages.nix { }).allPostgresqlPackages; + # ... now swizzle the names of the attrset in order to be more user-friendly. This is a bit of a hack; + # ideally we would use postgresql.version, but that normally results in something like 'postgresql-10.4' + # which is an attribute name that can't be evaluated easily by 'nix-build' + in mapAttrs' (name: value: { name = "${builtins.substring 0 12 name}"; inherit value; }) allPackages; + + # Sample SQL script to use. Note: this should work on _every_ available, supported + # version of PostgreSQL shipped by Nixpkgs. + test-sql = pkgs.writeText "test.sql" '' CREATE EXTENSION pgcrypto; -- just to check if lib loading works CREATE TABLE sth ( id int @@ -16,16 +31,19 @@ let CREATE TABLE xmltest ( doc xml ); INSERT INTO xmltest (doc) VALUES ('ok'); -- check if libxml2 enabled ''; - make-postgresql-test = postgresql-name: postgresql-package: makeTest { - name = postgresql-name; + + # Actual test + make-test = name: packages: makeTest { + inherit name; + meta = with pkgs.stdenv.lib.maintainers; { - maintainers = [ zagy ]; + maintainers = [ thoughtpolice zagy ]; }; machine = {...}: { - services.postgresql.package=postgresql-package; services.postgresql.enable = true; + services.postgresql.packages = packages; services.postgresqlBackup.enable = true; services.postgresqlBackup.databases = [ "postgres" ]; @@ -38,11 +56,14 @@ let } $machine->start; - $machine->waitForUnit("postgresql"); + # postgresql should be available just after unit start + $machine->waitForUnit("postgresql"); $machine->succeed("cat ${test-sql} | sudo -u postgres psql"); $machine->shutdown; # make sure that postgresql survive restart (bug #1735) sleep(2); + + # run some basic queries against the schema $machine->start; $machine->waitForUnit("postgresql"); $machine->fail(check_count("SELECT * FROM sth;", 3)); @@ -55,7 +76,7 @@ let $machine->succeed("zcat /var/backup/postgresql/postgres.sql.gz | grep 'ok'"); $machine->shutdown; ''; - }; -in - mapAttrs' (p-name: p-package: {name=p-name; value=make-postgresql-test p-name p-package;}) postgresql-versions + + results = mapAttrs' (name: pkg: { inherit name; value = make-test name pkg; }) postgresql-versions; +in results From c0a7848f01f84a16eb41e3ac4b3bf8c75c81d185 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 11:59:03 -0500 Subject: [PATCH 45/61] nixos/tests: wibble for postgis tests [NFC] Signed-off-by: Austin Seipp --- nixos/tests/postgis.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/nixos/tests/postgis.nix b/nixos/tests/postgis.nix index a7c93baa52ca9..79dfcaf427ea1 100644 --- a/nixos/tests/postgis.nix +++ b/nixos/tests/postgis.nix @@ -11,7 +11,6 @@ import ./make-test.nix ({ pkgs, ...} : { { services.postgresql = { enable = true; - packages = pkgs.postgresqlPackages; plugins = p: with p; [ postgis ]; }; }; From 2fc694e6e9ac0cf96075b20c77363d66ce8ebfe2 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 12:07:59 -0500 Subject: [PATCH 46/61] nixos/tests: touch up pgjwt tests Signed-off-by: Austin Seipp --- nixos/tests/pgjwt.nix | 32 +++++++++++--------------------- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/nixos/tests/pgjwt.nix b/nixos/tests/pgjwt.nix index 953d8594787ca..9f4cce806b422 100644 --- a/nixos/tests/pgjwt.nix +++ b/nixos/tests/pgjwt.nix @@ -1,40 +1,30 @@ import ./make-test.nix ({ pkgs, lib, ...}: + let - pgsql = pkgs.postgresqlPackages; + pgjwt = pkgs.postgresqlPackages.pgjwt; + pg_prove = "${pkgs.perlPackages.TAPParserSourceHandlerpgTAP}/bin/pg_prove"; - test = with pkgs; runCommand "patch-test" { - nativeBuildInputs = [ pgsql.pgjwt ]; - } - '' - sed -e '12 i CREATE EXTENSION pgcrypto;\nCREATE EXTENSION pgtap;\nSET search_path TO tap,public;' ${pgsql.pgjwt.src}/test.sql > $out; + test = pkgs.runCommand "test.sql" {} '' + sed -e '12 i CREATE EXTENSION pgcrypto;\nCREATE EXTENSION pgtap;\nSET search_path TO tap,public;' ${pgjwt.src}/test.sql > $out; ''; in -with pkgs; { +{ name = "pgjwt"; - meta = with lib.maintainers; { - maintainers = [ spinus willibutz ]; - }; + meta.maintainers = with lib.maintainers; [ thoughtpolice spinus willibutz ]; nodes = { - master = { ... }: - { + master = { ... }: { services.postgresql = { enable = true; - packages = pgsql; - plugins = p: [ p.pgjwt p.pgtap ]; + plugins = p: with p; [ pgjwt pgtap ]; }; }; }; - testScript = { nodes, ... }: - let - sqlSU = "${nodes.master.config.services.postgresql.superUser}"; - pgProve = "${pkgs.perlPackages.TAPParserSourceHandlerpgTAP}"; - in - '' + testScript = { nodes, ... }: '' startAll; $master->waitForUnit("postgresql"); $master->copyFileFromHost("${test}","/tmp/test.sql"); - $master->succeed("${pkgs.sudo}/bin/sudo -u ${sqlSU} PGOPTIONS=--search_path=tap,public ${pgProve}/bin/pg_prove -d postgres -v -f /tmp/test.sql"); + $master->succeed("${pkgs.sudo}/bin/sudo -u postgres PGOPTIONS=--search_path=tap,public ${pg_prove} -v -d postgres -f /tmp/test.sql"); ''; }) From 69dd764f701c8f4c0aee0c60c6e3bb3fb630a3c6 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 12:10:10 -0500 Subject: [PATCH 47/61] nixos/tests: migrate trac to new postgresql API This test doesn't work, but it suppresses a needless warning, anyway. Signed-off-by: Austin Seipp --- nixos/tests/trac.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/nixos/tests/trac.nix b/nixos/tests/trac.nix index 4599885acde69..fc440c96d3b23 100644 --- a/nixos/tests/trac.nix +++ b/nixos/tests/trac.nix @@ -17,7 +17,6 @@ import ./make-test.nix ({ pkgs, ... }: { postgresql = { pkgs, ... }: { services.postgresql.enable = true; - services.postgresql.package = pkgs.postgresql; services.postgresql.enableTCPIP = true; services.postgresql.authentication = '' # Generated file; do not edit! From 01610cfdd31b748159f7c2aa9ca6b93e88d983bb Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 12:28:38 -0500 Subject: [PATCH 48/61] nixos/virtualisation: fix example Signed-off-by: Austin Seipp --- nixos/modules/virtualisation/containers.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/nixos/modules/virtualisation/containers.nix b/nixos/modules/virtualisation/containers.nix index b91165ce3b827..fd04951d918c4 100644 --- a/nixos/modules/virtualisation/containers.nix +++ b/nixos/modules/virtualisation/containers.nix @@ -605,8 +605,6 @@ in { config = { config, pkgs, ... }: { services.postgresql.enable = true; - services.postgresql.package = pkgs.postgresql96; - system.stateVersion = "17.03"; }; }; From 224ca9fb755abd9ef609324520f4313cf71f3f2b Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 12:30:17 -0500 Subject: [PATCH 49/61] nixos/gitea: update to new PostgreSQL interface Signed-off-by: Austin Seipp --- nixos/modules/services/misc/gitea.nix | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nixos/modules/services/misc/gitea.nix b/nixos/modules/services/misc/gitea.nix index a222325579fec..6fb85d0ccb10c 100644 --- a/nixos/modules/services/misc/gitea.nix +++ b/nixos/modules/services/misc/gitea.nix @@ -5,7 +5,10 @@ with lib; let cfg = config.services.gitea; gitea = cfg.package; + pg = config.services.postgresql; + pgbin = if (pg.package != null) then pg.package else pg.packages.postgresql; + usePostgresql = cfg.database.type == "postgres"; configFile = pkgs.writeText "app.ini" '' APP_NAME = ${cfg.appName} @@ -308,9 +311,9 @@ in echo "CREATE ROLE ${cfg.database.user} WITH ENCRYPTED PASSWORD '$(head -n1 ${cfg.database.passwordFile})' NOCREATEDB NOCREATEROLE LOGIN" | - ${pkgs.sudo}/bin/sudo -u ${pg.superUser} ${pg.package}/bin/psql + ${pkgs.sudo}/bin/sudo -u ${pg.superUser} ${pgbin}/bin/psql ${pkgs.sudo}/bin/sudo -u ${pg.superUser} \ - ${pg.package}/bin/createdb \ + ${pgbin}/bin/createdb \ --owner=${cfg.database.user} \ --encoding=UTF8 \ --lc-collate=C \ From 9e398a11a0d729f844b8904ddebe107b53b2326c Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 12:49:16 -0500 Subject: [PATCH 50/61] nixos/gitlab: use new postgres module API Signed-off-by: Austin Seipp --- nixos/modules/services/misc/gitlab.nix | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/nixos/modules/services/misc/gitlab.nix b/nixos/modules/services/misc/gitlab.nix index 5bf66354f487e..d591aaf8d1855 100644 --- a/nixos/modules/services/misc/gitlab.nix +++ b/nixos/modules/services/misc/gitlab.nix @@ -14,6 +14,10 @@ let pathUrlQuote = url: replaceStrings ["/"] ["%2F"] url; pgSuperUser = config.services.postgresql.superUser; + pgBinaryDir = if (config.services.postgresql.package != null) + then config.services.postgresql.package + else config.services.postgresql.packages.postgresql; + databaseYml = '' production: adapter: postgresql @@ -162,7 +166,7 @@ let makeWrapper ${cfg.packages.gitlab.rubyEnv}/bin/rake $out/bin/gitlab-rake \ ${concatStrings (mapAttrsToList (name: value: "--set ${name} '${value}' ") gitlabEnv)} \ --set GITLAB_CONFIG_PATH '${cfg.statePath}/config' \ - --set PATH '${lib.makeBinPath [ pkgs.nodejs pkgs.gzip pkgs.git pkgs.gnutar config.services.postgresql.package ]}:$PATH' \ + --set PATH '${lib.makeBinPath [ pkgs.nodejs pkgs.gzip pkgs.git pkgs.gnutar pgBinaryDir ]}:$PATH' \ --set RAKEOPT '-f ${cfg.packages.gitlab}/share/gitlab/Rakefile' \ --run 'cd ${cfg.packages.gitlab}/share/gitlab' ''; @@ -465,7 +469,7 @@ in { partOf = [ "gitlab.service" ]; environment = gitlabEnv; path = with pkgs; [ - config.services.postgresql.package + pgBinaryDir gitAndTools.git ruby openssh @@ -541,7 +545,7 @@ in { wantedBy = [ "multi-user.target" ]; environment = gitlabEnv; path = with pkgs; [ - config.services.postgresql.package + pgBinaryDir gitAndTools.git openssh nodejs @@ -609,7 +613,7 @@ in { if [ "${cfg.databaseHost}" = "127.0.0.1" ]; then if ! test -e "${cfg.statePath}/db-created"; then ${pkgs.sudo}/bin/sudo -u ${pgSuperUser} psql postgres -c "CREATE ROLE ${cfg.databaseUsername} WITH LOGIN NOCREATEDB NOCREATEROLE ENCRYPTED PASSWORD '${cfg.databasePassword}'" - ${pkgs.sudo}/bin/sudo -u ${pgSuperUser} ${config.services.postgresql.package}/bin/createdb --owner ${cfg.databaseUsername} ${cfg.databaseName} + ${pkgs.sudo}/bin/sudo -u ${pgSuperUser} ${pgBinaryDir}/bin/createdb --owner ${cfg.databaseUsername} ${cfg.databaseName} touch "${cfg.statePath}/db-created" fi From 075f1b7d83bf30da1c14867aa78d61e758a542e8 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 12:49:25 -0500 Subject: [PATCH 51/61] nixos/matrix-synapse: use new postgres module API Signed-off-by: Austin Seipp --- nixos/modules/services/misc/matrix-synapse.nix | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/nixos/modules/services/misc/matrix-synapse.nix b/nixos/modules/services/misc/matrix-synapse.nix index 18e13f6ac0300..d4fbb901d6e90 100644 --- a/nixos/modules/services/misc/matrix-synapse.nix +++ b/nixos/modules/services/misc/matrix-synapse.nix @@ -5,6 +5,11 @@ with lib; let cfg = config.services.matrix-synapse; pg = config.services.postgresql; + + pgBinaryDir = if (config.services.postgresql.package != null) + then config.services.postgresql.package + else config.services.postgresql.packages.postgresql; + usePostgresql = cfg.database_type == "psycopg2"; logConfigFile = pkgs.writeText "log_config.yaml" cfg.logConfig; mkResource = r: ''{names: ${builtins.toJSON r.names}, compress: ${boolToString r.compress}}''; @@ -663,14 +668,14 @@ in { '' + optionalString (usePostgresql && cfg.create_local_database) '' if ! test -e "${cfg.dataDir}/db-created"; then ${pkgs.sudo}/bin/sudo -u ${pg.superUser} \ - ${pg.package}/bin/createuser \ + ${pgBinaryDir}/bin/createuser \ --login \ --no-createdb \ --no-createrole \ --encrypted \ ${cfg.database_user} ${pkgs.sudo}/bin/sudo -u ${pg.superUser} \ - ${pg.package}/bin/createdb \ + ${pgBinaryDir}/bin/createdb \ --owner=${cfg.database_user} \ --encoding=UTF8 \ --lc-collate=C \ From 0b3d62278105c816c5974edc285f0c53a57cb3dc Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 13:27:42 -0500 Subject: [PATCH 52/61] postgresqlPackages.postgis: enable PostgreSQL 11 JIT support This was a bug that was fixed upstream; the applied patch is slightly different but with the same result. Should be removed when a new version of PostGIS is out. Signed-off-by: Austin Seipp --- pkgs/servers/sql/postgresql/ext/postgis.nix | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/pkgs/servers/sql/postgresql/ext/postgis.nix b/pkgs/servers/sql/postgresql/ext/postgis.nix index 2700461f595bd..10bce15815bee 100644 --- a/pkgs/servers/sql/postgresql/ext/postgis.nix +++ b/pkgs/servers/sql/postgresql/ext/postgis.nix @@ -71,20 +71,28 @@ in stdenv.mkDerivation rec { ''; buildInputs = [ libxml2 postgresql geos proj perl gdal json_c pkgconfig ]; + dontDisableStatic = true; sql_comments = "postgis_comments.sql"; - sql_srcs = ["postgis.sql" "spatial_ref_sys.sql"]; # postgis config directory assumes /include /lib from the same root for json-c library NIX_LDFLAGS = "-L${stdenv.lib.getLib json_c}/lib"; - dontDisableStatic = true; preConfigure = '' sed -i 's@/usr/bin/file@${file}/bin/file@' configure configureFlags="--datadir=$out/share --datarootdir=$out/share --bindir=$out/bin --with-gdalconfig=${gdal}/bin/gdal-config --with-jsondir=${json_c.dev}" + makeFlags="PERL=${perl}/bin/perl datadir=$out/share pkglibdir=$out/lib bindir=$out/bin" + + # fix the build with clang/JIT support + # see https://trac.osgeo.org/postgis/ticket/4060 -- can be removed later + substituteInPlace ./postgis/Makefile.in \ + --replace 'PG_CPPFLAGS +=' 'PG_CPPFLAGS += -I../liblwgeom' + substituteInPlace ./raster/rt_pg/Makefile.in \ + --replace 'LIBPGCOMMON_CFLAGS="-I../../libpgcommon"' 'LIBPGCOMMON_CFLAGS=-I ../../liblwgeom -I../../libpgcommon' ''; + postConfigure = '' sed -i "s|@mkdir -p \$(DESTDIR)\$(PGSQL_BINDIR)||g ; s|\$(DESTDIR)\$(PGSQL_BINDIR)|$prefix/bin|g @@ -95,10 +103,6 @@ in stdenv.mkDerivation rec { "raster/scripts/python/Makefile"; ''; - passthru = { - versionCheck = postgresql.hasJitSupport == false; - }; - meta = with stdenv.lib; { description = "Geographic Objects for PostgreSQL"; homepage = http://postgis.refractions.net; From 0b7377a873c6040c76ddd721785ca79b26c089b4 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 14:41:15 -0500 Subject: [PATCH 53/61] postgresqlPackages: remove outdated comments Signed-off-by: Austin Seipp --- .../servers/sql/postgresql/ext/pg_journal.nix | 6 ---- pkgs/servers/sql/postgresql/ext/postgis.nix | 30 ------------------- 2 files changed, 36 deletions(-) diff --git a/pkgs/servers/sql/postgresql/ext/pg_journal.nix b/pkgs/servers/sql/postgresql/ext/pg_journal.nix index 956fcd3617f99..1754aec27ab90 100644 --- a/pkgs/servers/sql/postgresql/ext/pg_journal.nix +++ b/pkgs/servers/sql/postgresql/ext/pg_journal.nix @@ -1,11 +1,5 @@ { stdenv, fetchFromGitHub, fetchpatch, pkgconfig, postgresql, systemd, openssl }: -# # To enable on NixOS: -# config.services.postgresql = { -# extraPlugins = [ pkgs.pg_journal ]; -# extraConfig = "shared_preload_libraries = 'pg_journal'"; -# } - stdenv.mkDerivation rec { name = "pg_journal-${version}"; version = "0.2.0"; diff --git a/pkgs/servers/sql/postgresql/ext/postgis.nix b/pkgs/servers/sql/postgresql/ext/postgis.nix index 10bce15815bee..e2fc89ff0fae6 100644 --- a/pkgs/servers/sql/postgresql/ext/postgis.nix +++ b/pkgs/servers/sql/postgresql/ext/postgis.nix @@ -11,36 +11,6 @@ , file }: - /* - - ### NixOS - usage: - ================== - - services.postgresql.extraPlugins = [ (pkgs.postgis.override { postgresql = pkgs.postgresql95; }) ]; - - - ### important Postgis implementation details: - ============================================= - - Postgis provides a shared library implementing many operations. They are - exposed to the Postgres SQL interpreter by special SQL queries eg: - - CREATE FUNCTION [...] - AS '[..]liblwgeom', 'lwhistogram2d_in' LANGUAGE 'C' IMMUTABLE STRICT; -- WITH (isstrict); - - where liblwgeom is the shared library. - Postgis < 1.5 used absolute paths, in NixOS $libdir is always used. - - Thus if you want to use postgresql dumps which were created by non NixOS - systems you have to adopt the library path. - - - ### TODO: - ========= - the bin commands to have gtk gui: - */ - - let version = "2.4.4"; sha256 = "1hm8migjb53cymp4qvg1h20yqllmy9f7x0awv5450391i6syyqq6"; From c2e3535cb530f43f448692466b269f8659b8382b Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Mon, 6 Aug 2018 14:43:51 -0500 Subject: [PATCH 54/61] nixos/manual: add more postgresql modules to 18.09 release notes Signed-off-by: Austin Seipp --- nixos/doc/manual/release-notes/rl-1809.xml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nixos/doc/manual/release-notes/rl-1809.xml b/nixos/doc/manual/release-notes/rl-1809.xml index 6006ba9760be0..b8db03ca8ad11 100644 --- a/nixos/doc/manual/release-notes/rl-1809.xml +++ b/nixos/doc/manual/release-notes/rl-1809.xml @@ -27,8 +27,11 @@ compiled and checked continuously. This means you can now use extensions like plv8 with any PostgreSQL version without compiling anything. Extensions expressions are easier to write, and new extensions are - available: cstore_fdw, pg_cron, - postgres-hll, and postgres-topn. + available: pg_cron, postgres-hll, + postgres-topn, pg_journal, + pg_jobmon, pg_partman, + and citus. (Almost all existing extensions have seen + version updates, as well.) NixOS 18.09 is the last release that will offer PostgreSQL 9.3 as a From c3919f5d716cb0e2bb6f0652cc0c14af12cf5b0d Mon Sep 17 00:00:00 2001 From: Bas van Dijk Date: Tue, 7 Aug 2018 03:27:26 -0500 Subject: [PATCH 55/61] nixos/postgresql: give the plugins option the selectorFunction type This allows to merge configurations that both define plugins as in: mkMerge [ { services.postgresql.plugins = p: [ p.pg_journal ]; } { services.postgresql.plugins = p: [ p.postgis ]; } ] selectorFunction should be moved to lib/types.nix since options like "plugins" occur in at least 6 other places in NixOS. Signed-off-by: Austin Seipp --- nixos/modules/services/databases/postgresql.nix | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/nixos/modules/services/databases/postgresql.nix b/nixos/modules/services/databases/postgresql.nix index a0e4b71c1d903..a4d24096f1b03 100644 --- a/nixos/modules/services/databases/postgresql.nix +++ b/nixos/modules/services/databases/postgresql.nix @@ -31,6 +31,18 @@ let ${cfg.extraConfig} ''; + # TODO: propose to move this to lib/types.nix + # since selector functions are used in at least 6 other options in NixOS. + selectorFunction = mkOptionType { + name = "selectorFunction"; + description = + "function that takes an attribute set and returns a list " + + "containing a selection of the values of the input set"; + check = select: isFunction select; + merge = _loc: defs: + as: concatMap (select: select as) (getValues defs); + }; + in { @@ -142,6 +154,7 @@ in plugins = mkOption { default = _: []; + type = selectorFunction; example = literalExample "p: with p; [ postgis timescaledb ]"; description = '' A function that selects plugins to include in the PostgreSQL server. From da504b673918f44921907aa5ef318a1eb292f7dd Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Tue, 7 Aug 2018 14:34:56 -0500 Subject: [PATCH 56/61] nixos/postgresql: give the postgres user a homedir and shell For certain SQL operations such as COPY (...) [TO|FROM] PROGRAM '...' WITH (FORMAT CSV) the designated PostgreSQL server user must be able to execute the specified program and pipe the data into it; however, Postgres executes these commands *under a shell* in order to support features such as traditional piping, for commands like split or gzip. With no homedir and no shell assigned, Postgres fails to execute the shell, resulting in a hang that then later results in the query failing. Depending on the program and query executed, this may happen at seemingly random times due to buffering/piping issues. While it isn't traditional to assign a shell to the database user (at least on NixOS), doing so restores this functionality, which is extremely useful for e.g. bulk ETL pipelines that copy to/from external data files. Signed-off-by: Austin Seipp --- nixos/modules/services/databases/postgresql.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nixos/modules/services/databases/postgresql.nix b/nixos/modules/services/databases/postgresql.nix index a4d24096f1b03..59196f1811a3e 100644 --- a/nixos/modules/services/databases/postgresql.nix +++ b/nixos/modules/services/databases/postgresql.nix @@ -258,6 +258,8 @@ in uid = config.ids.uids.postgres; group = "postgres"; description = "PostgreSQL server user"; + home = "${cfg.dataDir}"; + useDefaultShell = true; }; users.groups.postgres.gid = config.ids.gids.postgres; From bcf48585c7522473e3268ff7dee0654cf8c37d55 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Tue, 7 Aug 2018 17:17:18 -0500 Subject: [PATCH 57/61] timescaledb-parallel-copy: init at 2018-05-14 Signed-off-by: Austin Seipp --- .../timescaledb-parallel-copy/default.nix | 26 +++++++++++++++++++ .../timescaledb-parallel-copy/deps.nix | 21 +++++++++++++++ pkgs/top-level/all-packages.nix | 2 ++ 3 files changed, 49 insertions(+) create mode 100644 pkgs/development/tools/database/timescaledb-parallel-copy/default.nix create mode 100644 pkgs/development/tools/database/timescaledb-parallel-copy/deps.nix diff --git a/pkgs/development/tools/database/timescaledb-parallel-copy/default.nix b/pkgs/development/tools/database/timescaledb-parallel-copy/default.nix new file mode 100644 index 0000000000000..d667e49e7cbeb --- /dev/null +++ b/pkgs/development/tools/database/timescaledb-parallel-copy/default.nix @@ -0,0 +1,26 @@ +{ stdenv, buildGoPackage, fetchFromGitHub }: + +buildGoPackage rec { + name = "timescaledb-parallel-copy-${version}"; + version = "2018-05-14"; + + owner = "timescale"; + repo = "timescaledb-parallel-copy"; + + goPackagePath = with src; "github.com/${owner}/${repo}"; + goDeps = ./deps.nix; + + src = fetchFromGitHub { + inherit owner repo; + rev = "20d3e8f8219329f2f4b0a5aa985f280dd04d10bb"; + sha256 = "0waaccw991cnxaxjdxh9ksb94kiiyx1r7gif6pkd5k58js0kfvdn"; + }; + + meta = with stdenv.lib; { + description = "Bulk, parallel insert of CSV records into PostgreSQL"; + homepage = http://github.com/timescale/timescaledb-parallel-copy; + license = licenses.asl20; + platforms = platforms.linux; + maintainers = with maintainers; [ thoughtpolice ]; + }; +} diff --git a/pkgs/development/tools/database/timescaledb-parallel-copy/deps.nix b/pkgs/development/tools/database/timescaledb-parallel-copy/deps.nix new file mode 100644 index 0000000000000..c672611e09c10 --- /dev/null +++ b/pkgs/development/tools/database/timescaledb-parallel-copy/deps.nix @@ -0,0 +1,21 @@ +# This file was generated by https://github.com/kamilchm/go2nix v1.2.1 +[ + { + goPackagePath = "github.com/jmoiron/sqlx"; + fetch = { + type = "git"; + url = "https://github.com/jmoiron/sqlx"; + rev = "0dae4fefe7c0e190f7b5a78dac28a1c82cc8d849"; + sha256 = "0r8fyj70n0v84byvagw8w8rzz532s94mjr72b9sx018j0b6xglmy"; + }; + } + { + goPackagePath = "github.com/lib/pq"; + fetch = { + type = "git"; + url = "https://github.com/lib/pq"; + rev = "90697d60dd844d5ef6ff15135d0203f65d2f53b8"; + sha256 = "0hb4bfsk8g5473yzbf3lzrb373xicakjznkf0v085xgimz991i9r"; + }; + } +] diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 2b5222cae64e8..d51cf5013cbb8 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -13355,6 +13355,8 @@ with pkgs; psqlodbc = callPackage ../development/libraries/psqlodbc { }; + timescaledb-parallel-copy = callPackage ../development/tools/database/timescaledb-parallel-copy { }; + ## ----- END: POSTGRESQL -------------------------------------------------------------- inherit (callPackage ../servers/monitoring/prometheus { From 8a0deebc63a053fa3991ebe02c1edc64cca5e377 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Tue, 7 Aug 2018 18:50:07 -0500 Subject: [PATCH 58/61] postgresqlPackages: add .man to all withPackage/NixOS configurations Otherwise, features like 'man psql' don't work, which is weird for a machine that has PostgreSQL running as a service. Signed-off-by: Austin Seipp --- pkgs/servers/sql/postgresql/packages.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkgs/servers/sql/postgresql/packages.nix b/pkgs/servers/sql/postgresql/packages.nix index 431840c69c5e8..40718ee64f747 100644 --- a/pkgs/servers/sql/postgresql/packages.nix +++ b/pkgs/servers/sql/postgresql/packages.nix @@ -14,7 +14,9 @@ let then postgresql else pkgs.buildEnv { name = "postgresql-and-plugins-${postgresql.version}"; - paths = [ postgresql postgresql.lib ] ++ plugins; + # Make sure we include .man so that manual pages work for the user in their + # environment + paths = [ postgresql postgresql.lib postgresql.man ] ++ plugins; # We include /bin to ensure the $out/bin directory is created, which is # needed because we'll be removing the files from that directory in postBuild From 0ca1defc2e58cfbda2208c7c40ad0905867107f6 Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Tue, 14 Aug 2018 20:48:58 -0500 Subject: [PATCH 59/61] postgresqlPackages: enable LLVM JIT by default on PostgreSQL 11+ This turns on JIT support by default for PostgreSQL 11-compatible packages, now that most of the problems are fixed: 1) Closure sizes don't include Clang, only LLVM, 2) libpq clients don't suffer bloat from libLLVM.so dependencies, 3) PGXS support is improved, so hand-written installPhases are mostly not needed, making it much more viable to ship bitcode for 3rd party extensions. As a side note, this change also should not impact the nixos-small channel builds very much (I hope), because those already have LLVM 6.0 in their closure, by way of QEMU/mesa packages. Signed-off-by: Austin Seipp --- pkgs/servers/sql/postgresql/default.nix | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pkgs/servers/sql/postgresql/default.nix b/pkgs/servers/sql/postgresql/default.nix index 3b7a5774541fe..09876fd21e5ef 100644 --- a/pkgs/servers/sql/postgresql/default.nix +++ b/pkgs/servers/sql/postgresql/default.nix @@ -1,11 +1,10 @@ { stdenv, lib, fetchurl, makeWrapper , glibc, zlib, readline, libossp_uuid, openssl, libxml2, tzdata, systemd -# Gate JIT support right now behind a flag; it increases closure size -# dramatically due to the PostgreSQL build system requiring a hard dependency -# on clang-wrapper (~140MB -> 1.4GB). This must be worked around before it can -# be enabled by default by making clang-wrapper a build-time only dependency. -, llvmPackages, enableJitSupport ? false +# For now, keep JIT support behind a conditional flag, even though it is on by +# default -- so we can turn all features (and impacts to closure size, +# dependencies, etc) off easily, if we need to. +, llvmPackages, enableJitSupport ? true }@deps: let From 9a88df84997d0c6ba57132156ac5f22496535c1d Mon Sep 17 00:00:00 2001 From: Bas van Dijk Date: Sun, 9 Sep 2018 22:52:33 +0200 Subject: [PATCH 60/61] postgresql: add doc section id module-services-postgres-new-extensions --- nixos/modules/services/databases/postgresql.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/modules/services/databases/postgresql.xml b/nixos/modules/services/databases/postgresql.xml index a0dbc163edcda..9e021691ddaca 100644 --- a/nixos/modules/services/databases/postgresql.xml +++ b/nixos/modules/services/databases/postgresql.xml @@ -214,7 +214,7 @@ linkend="opt-services.postgresql.enable">here. -
Authoring new extensions +
Authoring new extensions FIXME: document how to write an extension From a1d18b73a3388f6a2f0275adb8bf337f1aa2336d Mon Sep 17 00:00:00 2001 From: Bas van Dijk Date: Mon, 10 Sep 2018 23:46:42 +0200 Subject: [PATCH 61/61] postgresql: introduce the deprecated, readOnly and invisible postgresqlPackage option This option is used in modules that need to refer to the selected PostgreSQL package. Without this option these modules need to duplicate some conditional logic which is error-prone. This option should be removed when the deprecated config.services.postgresql.package is removed. --- nixos/modules/services/backup/bacula.nix | 4 ++-- .../continuous-integration/hydra/default.nix | 6 +++-- .../modules/services/databases/postgresql.nix | 24 ++++++++++++++----- nixos/modules/services/misc/gammu-smsd.nix | 2 +- nixos/modules/services/misc/gitea.nix | 5 ++-- nixos/modules/services/misc/gitlab.nix | 18 ++++++-------- .../modules/services/misc/matrix-synapse.nix | 8 ++----- .../services/monitoring/zabbix-server.nix | 12 ++++++---- .../modules/services/web-apps/mattermost.nix | 11 +++++---- .../services/web-apps/restya-board.nix | 13 +++++----- nixos/modules/services/web-apps/tt-rss.nix | 8 ++++--- .../web-servers/apache-httpd/owncloud.nix | 2 +- nixos/modules/services/web-servers/hydron.nix | 12 +++++----- nixos/modules/services/web-servers/meguca.nix | 8 +++---- 14 files changed, 72 insertions(+), 61 deletions(-) diff --git a/nixos/modules/services/backup/bacula.nix b/nixos/modules/services/backup/bacula.nix index a0565ca26204b..a4a1f6532ff21 100644 --- a/nixos/modules/services/backup/bacula.nix +++ b/nixos/modules/services/backup/bacula.nix @@ -370,8 +370,8 @@ in { serviceConfig.ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; preStart = '' if ! test -e "${libDir}/db-created"; then - ${pkgs.postgresql}/bin/createuser --no-superuser --no-createdb --no-createrole bacula - #${pkgs.postgresql}/bin/createdb --owner bacula bacula + ${config.services.postgresql.postgresqlPackage}/bin/createuser --no-superuser --no-createdb --no-createrole bacula + #${config.services.postgresql.postgresqlPackage}/bin/createdb --owner bacula bacula # populate DB ${pkgs.bacula}/etc/create_bacula_database postgresql diff --git a/nixos/modules/services/continuous-integration/hydra/default.nix b/nixos/modules/services/continuous-integration/hydra/default.nix index c7fe4eeeab996..4fb4a54e9230f 100644 --- a/nixos/modules/services/continuous-integration/hydra/default.nix +++ b/nixos/modules/services/continuous-integration/hydra/default.nix @@ -6,6 +6,8 @@ let cfg = config.services.hydra; + pg = config.services.postgresql; + baseDir = "/var/lib/hydra"; hydraConf = pkgs.writeScript "hydra.conf" cfg.extraConfig; @@ -271,8 +273,8 @@ in ${optionalString haveLocalDB '' if ! [ -e ${baseDir}/.db-created ]; then - ${pkgs.sudo}/bin/sudo -u ${config.services.postgresql.superUser} ${config.services.postgresql.package}/bin/createuser hydra - ${pkgs.sudo}/bin/sudo -u ${config.services.postgresql.superUser} ${config.services.postgresql.package}/bin/createdb -O hydra hydra + ${pkgs.sudo}/bin/sudo -u ${pg.superUser} ${pg.postgresqlPackage}/bin/createuser hydra + ${pkgs.sudo}/bin/sudo -u ${pg.superUser} ${pg.postgresqlPackage}/bin/createdb -O hydra hydra touch ${baseDir}/.db-created fi ''} diff --git a/nixos/modules/services/databases/postgresql.nix b/nixos/modules/services/databases/postgresql.nix index 59196f1811a3e..2566ba7c7626f 100644 --- a/nixos/modules/services/databases/postgresql.nix +++ b/nixos/modules/services/databases/postgresql.nix @@ -15,10 +15,7 @@ let let ps = import ../../../../pkgs/servers/sql/postgresql/packages.nix { inherit pkgs lib; }; in ps.withPackages pg plugins; - postgresqlPackage = if (cfg.package != null) then cfg.package else - cfg.packages.postgresql; - - postgresql = postgresqlAndPlugins postgresqlPackage; + postgresql = postgresqlAndPlugins cfg.postgresqlPackage; # The main PostgreSQL configuration file. configFile = pkgs.writeText "postgresql.conf" @@ -81,6 +78,21 @@ in ''; }; + postgresqlPackage = mkOption { + type = types.package; + default = + if cfg.package != null + then cfg.package + else cfg.packages.postgresql; + readOnly = true; + visible = false; + description = '' + A deprecated, read-only and invisible option that refers to the + selected PostgreSQL package. This should be removed at the same time + that the deprecated config.services.postgresql.package option is removed. + ''; + }; + port = mkOption { type = types.int; default = 5432; @@ -242,7 +254,7 @@ in else pkgs.postgresql94Packages); services.postgresql.dataDir = - mkDefault (if versionAtLeast config.system.stateVersion "17.09" then "/var/lib/postgresql/${postgresqlPackage.psqlSchema}" + mkDefault (if versionAtLeast config.system.stateVersion "17.09" then "/var/lib/postgresql/${cfg.postgresqlPackage.psqlSchema}" else "/var/db/postgresql"); services.postgresql.authentication = mkAfter @@ -318,7 +330,7 @@ in # receiving systemd's SIGINT. TimeoutSec = 120; - Type = if versionAtLeast postgresqlPackage.psqlSchema "9.6" then "notify" else "simple"; + Type = if versionAtLeast cfg.postgresqlPackage.psqlSchema "9.6" then "notify" else "simple"; }; # Wait for PostgreSQL to be ready to accept connections. diff --git a/nixos/modules/services/misc/gammu-smsd.nix b/nixos/modules/services/misc/gammu-smsd.nix index 3057d7fd1a095..72c3ba43be363 100644 --- a/nixos/modules/services/misc/gammu-smsd.nix +++ b/nixos/modules/services/misc/gammu-smsd.nix @@ -232,7 +232,7 @@ in { '' + (let execPsql = extraArgs: concatStringsSep " " [ (optionalString (sql.password != null) "PGPASSWORD=${sql.password}") - "${config.services.postgresql.package}/bin/psql" + "${config.services.postgresql.postgresqlPackage}/bin/psql" (optionalString (sql.host != null) "-h ${sql.host}") (optionalString (sql.user != null) "-U ${sql.user}") "$extraArgs" diff --git a/nixos/modules/services/misc/gitea.nix b/nixos/modules/services/misc/gitea.nix index 6fb85d0ccb10c..61981cde64cf5 100644 --- a/nixos/modules/services/misc/gitea.nix +++ b/nixos/modules/services/misc/gitea.nix @@ -7,7 +7,6 @@ let gitea = cfg.package; pg = config.services.postgresql; - pgbin = if (pg.package != null) then pg.package else pg.packages.postgresql; usePostgresql = cfg.database.type == "postgres"; configFile = pkgs.writeText "app.ini" '' @@ -311,9 +310,9 @@ in echo "CREATE ROLE ${cfg.database.user} WITH ENCRYPTED PASSWORD '$(head -n1 ${cfg.database.passwordFile})' NOCREATEDB NOCREATEROLE LOGIN" | - ${pkgs.sudo}/bin/sudo -u ${pg.superUser} ${pgbin}/bin/psql + ${pkgs.sudo}/bin/sudo -u ${pg.superUser} ${pg.postgresqlPackage}/bin/psql ${pkgs.sudo}/bin/sudo -u ${pg.superUser} \ - ${pgbin}/bin/createdb \ + ${pg.postgresqlPackage}/bin/createdb \ --owner=${cfg.database.user} \ --encoding=UTF8 \ --lc-collate=C \ diff --git a/nixos/modules/services/misc/gitlab.nix b/nixos/modules/services/misc/gitlab.nix index d591aaf8d1855..b4bbd22822f46 100644 --- a/nixos/modules/services/misc/gitlab.nix +++ b/nixos/modules/services/misc/gitlab.nix @@ -12,11 +12,7 @@ let gitlabSocket = "${cfg.statePath}/tmp/sockets/gitlab.socket"; gitalySocket = "${cfg.statePath}/tmp/sockets/gitaly.socket"; pathUrlQuote = url: replaceStrings ["/"] ["%2F"] url; - pgSuperUser = config.services.postgresql.superUser; - - pgBinaryDir = if (config.services.postgresql.package != null) - then config.services.postgresql.package - else config.services.postgresql.packages.postgresql; + pg = config.services.postgresql; databaseYml = '' production: @@ -166,7 +162,7 @@ let makeWrapper ${cfg.packages.gitlab.rubyEnv}/bin/rake $out/bin/gitlab-rake \ ${concatStrings (mapAttrsToList (name: value: "--set ${name} '${value}' ") gitlabEnv)} \ --set GITLAB_CONFIG_PATH '${cfg.statePath}/config' \ - --set PATH '${lib.makeBinPath [ pkgs.nodejs pkgs.gzip pkgs.git pkgs.gnutar pgBinaryDir ]}:$PATH' \ + --set PATH '${lib.makeBinPath [ pkgs.nodejs pkgs.gzip pkgs.git pkgs.gnutar pg.postgresqlPackage ]}:$PATH' \ --set RAKEOPT '-f ${cfg.packages.gitlab}/share/gitlab/Rakefile' \ --run 'cd ${cfg.packages.gitlab}/share/gitlab' ''; @@ -469,7 +465,7 @@ in { partOf = [ "gitlab.service" ]; environment = gitlabEnv; path = with pkgs; [ - pgBinaryDir + pg.postgresqlPackage gitAndTools.git ruby openssh @@ -545,7 +541,7 @@ in { wantedBy = [ "multi-user.target" ]; environment = gitlabEnv; path = with pkgs; [ - pgBinaryDir + pg.postgresqlPackage gitAndTools.git openssh nodejs @@ -612,13 +608,13 @@ in { if [ "${cfg.databaseHost}" = "127.0.0.1" ]; then if ! test -e "${cfg.statePath}/db-created"; then - ${pkgs.sudo}/bin/sudo -u ${pgSuperUser} psql postgres -c "CREATE ROLE ${cfg.databaseUsername} WITH LOGIN NOCREATEDB NOCREATEROLE ENCRYPTED PASSWORD '${cfg.databasePassword}'" - ${pkgs.sudo}/bin/sudo -u ${pgSuperUser} ${pgBinaryDir}/bin/createdb --owner ${cfg.databaseUsername} ${cfg.databaseName} + ${pkgs.sudo}/bin/sudo -u ${pg.superUser} psql postgres -c "CREATE ROLE ${cfg.databaseUsername} WITH LOGIN NOCREATEDB NOCREATEROLE ENCRYPTED PASSWORD '${cfg.databasePassword}'" + ${pkgs.sudo}/bin/sudo -u ${pg.superUser} createdb --owner ${cfg.databaseUsername} ${cfg.databaseName} touch "${cfg.statePath}/db-created" fi # enable required pg_trgm extension for gitlab - ${pkgs.sudo}/bin/sudo -u ${pgSuperUser} psql ${cfg.databaseName} -c "CREATE EXTENSION IF NOT EXISTS pg_trgm" + ${pkgs.sudo}/bin/sudo -u ${pg.superUser} psql ${cfg.databaseName} -c "CREATE EXTENSION IF NOT EXISTS pg_trgm" fi # Always do the db migrations just to be sure the database is up-to-date diff --git a/nixos/modules/services/misc/matrix-synapse.nix b/nixos/modules/services/misc/matrix-synapse.nix index d4fbb901d6e90..373814e113772 100644 --- a/nixos/modules/services/misc/matrix-synapse.nix +++ b/nixos/modules/services/misc/matrix-synapse.nix @@ -6,10 +6,6 @@ let cfg = config.services.matrix-synapse; pg = config.services.postgresql; - pgBinaryDir = if (config.services.postgresql.package != null) - then config.services.postgresql.package - else config.services.postgresql.packages.postgresql; - usePostgresql = cfg.database_type == "psycopg2"; logConfigFile = pkgs.writeText "log_config.yaml" cfg.logConfig; mkResource = r: ''{names: ${builtins.toJSON r.names}, compress: ${boolToString r.compress}}''; @@ -668,14 +664,14 @@ in { '' + optionalString (usePostgresql && cfg.create_local_database) '' if ! test -e "${cfg.dataDir}/db-created"; then ${pkgs.sudo}/bin/sudo -u ${pg.superUser} \ - ${pgBinaryDir}/bin/createuser \ + ${pg.postgresqlPackage}/bin/createuser \ --login \ --no-createdb \ --no-createrole \ --encrypted \ ${cfg.database_user} ${pkgs.sudo}/bin/sudo -u ${pg.superUser} \ - ${pgBinaryDir}/bin/createdb \ + ${pg.postgresqlPackage}/bin/createdb \ --owner=${cfg.database_user} \ --encoding=UTF8 \ --lc-collate=C \ diff --git a/nixos/modules/services/monitoring/zabbix-server.nix b/nixos/modules/services/monitoring/zabbix-server.nix index 5f9fc12832fc0..bd27e35d3c6be 100644 --- a/nixos/modules/services/monitoring/zabbix-server.nix +++ b/nixos/modules/services/monitoring/zabbix-server.nix @@ -7,6 +7,8 @@ let cfg = config.services.zabbixServer; + pg = config.services.postgresql; + stateDir = "/var/run/zabbix"; logDir = "/var/log/zabbix"; @@ -103,11 +105,11 @@ in chown zabbix ${stateDir} ${logDir} ${libDir} if ! test -e "${libDir}/db-created"; then - ${pkgs.su}/bin/su -s "$SHELL" ${config.services.postgresql.superUser} -c '${pkgs.postgresql}/bin/createuser --no-superuser --no-createdb --no-createrole zabbix' || true - ${pkgs.su}/bin/su -s "$SHELL" ${config.services.postgresql.superUser} -c '${pkgs.postgresql}/bin/createdb --owner zabbix zabbix' || true - cat ${pkgs.zabbix.server}/share/zabbix/db/schema/postgresql.sql | ${pkgs.su}/bin/su -s "$SHELL" zabbix -c '${pkgs.postgresql}/bin/psql zabbix' - cat ${pkgs.zabbix.server}/share/zabbix/db/data/images_pgsql.sql | ${pkgs.su}/bin/su -s "$SHELL" zabbix -c '${pkgs.postgresql}/bin/psql zabbix' - cat ${pkgs.zabbix.server}/share/zabbix/db/data/data.sql | ${pkgs.su}/bin/su -s "$SHELL" zabbix -c '${pkgs.postgresql}/bin/psql zabbix' + ${pkgs.su}/bin/su -s "$SHELL" ${pg.superUser} -c '${pg.postgresqlPackage}/bin/createuser --no-superuser --no-createdb --no-createrole zabbix' || true + ${pkgs.su}/bin/su -s "$SHELL" ${pg.superUser} -c '${pg.postgresqlPackage}/bin/createdb --owner zabbix zabbix' || true + cat ${pkgs.zabbix.server}/share/zabbix/db/schema/postgresql.sql | ${pkgs.su}/bin/su -s "$SHELL" zabbix -c '${pg.postgresqlPackage}/bin/psql zabbix' + cat ${pkgs.zabbix.server}/share/zabbix/db/data/images_pgsql.sql | ${pkgs.su}/bin/su -s "$SHELL" zabbix -c '${pg.postgresqlPackage}/bin/psql zabbix' + cat ${pkgs.zabbix.server}/share/zabbix/db/data/data.sql | ${pkgs.su}/bin/su -s "$SHELL" zabbix -c '${pg.postgresqlPackage}/bin/psql zabbix' touch "${libDir}/db-created" fi ''; diff --git a/nixos/modules/services/web-apps/mattermost.nix b/nixos/modules/services/web-apps/mattermost.nix index 8c7fc4056adc9..50a0dac7e7119 100644 --- a/nixos/modules/services/web-apps/mattermost.nix +++ b/nixos/modules/services/web-apps/mattermost.nix @@ -6,6 +6,8 @@ let cfg = config.services.mattermost; + pg = config.services.postgresql; + defaultConfig = builtins.fromJSON (readFile "${pkgs.mattermost}/config/config.json"); mattermostConf = foldl recursiveUpdate defaultConfig @@ -184,11 +186,11 @@ in fi '' + lib.optionalString cfg.localDatabaseCreate '' if ! test -e "${cfg.statePath}/.db-created"; then - ${pkgs.sudo}/bin/sudo -u ${config.services.postgresql.superUser} \ - ${config.services.postgresql.package}/bin/psql postgres -c \ + ${pkgs.sudo}/bin/sudo -u ${pg.superUser} \ + ${pg.postgresqlPackage}/bin/psql postgres -c \ "CREATE ROLE ${cfg.localDatabaseUser} WITH LOGIN NOCREATEDB NOCREATEROLE ENCRYPTED PASSWORD '${cfg.localDatabasePassword}'" - ${pkgs.sudo}/bin/sudo -u ${config.services.postgresql.superUser} \ - ${config.services.postgresql.package}/bin/createdb \ + ${pkgs.sudo}/bin/sudo -u ${pg.superUser} \ + ${pg.postgresqlPackage}/bin/createdb \ --owner ${cfg.localDatabaseUser} ${cfg.localDatabaseName} touch ${cfg.statePath}/.db-created fi @@ -227,4 +229,3 @@ in }) ]; } - diff --git a/nixos/modules/services/web-apps/restya-board.nix b/nixos/modules/services/web-apps/restya-board.nix index bc6689bdb2712..4c493253ba6c3 100644 --- a/nixos/modules/services/web-apps/restya-board.nix +++ b/nixos/modules/services/web-apps/restya-board.nix @@ -10,6 +10,8 @@ with lib; let cfg = config.services.restya-board; + pg = config.services.postgresql; + runDir = "/run/restya-board"; poolName = "restya-board"; @@ -312,16 +314,16 @@ in ${optionalString (isNull cfg.database.host) '' if ! [ -e "${cfg.dataDir}/.db-initialized" ]; then - ${pkgs.sudo}/bin/sudo -u ${config.services.postgresql.superUser} \ - ${config.services.postgresql.package}/bin/psql -U ${config.services.postgresql.superUser} \ + ${pkgs.sudo}/bin/sudo -u ${pg.superUser} \ + ${pg.postgresqlPackage}/bin/psql -U ${pg.superUser} \ -c "CREATE USER ${cfg.database.user} WITH ENCRYPTED PASSWORD 'restya'" - ${pkgs.sudo}/bin/sudo -u ${config.services.postgresql.superUser} \ - ${config.services.postgresql.package}/bin/psql -U ${config.services.postgresql.superUser} \ + ${pkgs.sudo}/bin/sudo -u ${pg.superUser} \ + ${pg.postgresqlPackage}/bin/psql -U ${pg.superUser} \ -c "CREATE DATABASE ${cfg.database.name} OWNER ${cfg.database.user} ENCODING 'UTF8' TEMPLATE template0" ${pkgs.sudo}/bin/sudo -u ${cfg.user} \ - ${config.services.postgresql.package}/bin/psql -U ${cfg.database.user} \ + ${pg.postgresqlPackage}/bin/psql -U ${cfg.database.user} \ -d ${cfg.database.name} -f "${runDir}/sql/restyaboard_with_empty_data.sql" touch "${cfg.dataDir}/.db-initialized" @@ -381,4 +383,3 @@ in }; } - diff --git a/nixos/modules/services/web-apps/tt-rss.nix b/nixos/modules/services/web-apps/tt-rss.nix index 2b171aa1b2b29..f37dc9af26db1 100644 --- a/nixos/modules/services/web-apps/tt-rss.nix +++ b/nixos/modules/services/web-apps/tt-rss.nix @@ -4,6 +4,8 @@ with lib; let cfg = config.services.tt-rss; + pg = config.services.postgresql; + configVersion = 26; cacheDir = "cache"; @@ -528,7 +530,7 @@ let callSql = e: if cfg.database.type == "pgsql" then '' ${optionalString (cfg.database.password != null) "PGPASSWORD=${cfg.database.password}"} \ - ${pkgs.sudo}/bin/sudo -u ${cfg.user} ${config.services.postgresql.package}/bin/psql \ + ${pkgs.sudo}/bin/sudo -u ${cfg.user} ${pg.postgresqlPackage}/bin/psql \ -U ${cfg.database.user} \ ${optionalString (cfg.database.host != null) "-h ${cfg.database.host} --port ${toString dbPort}"} \ -c '${e}' \ @@ -565,8 +567,8 @@ let + (optionalString (cfg.database.type == "pgsql") '' ${optionalString (cfg.database.host == null && cfg.database.password == null) '' if ! [ -e ${cfg.root}/.db-created ]; then - ${pkgs.sudo}/bin/sudo -u ${config.services.postgresql.superUser} ${config.services.postgresql.package}/bin/createuser ${cfg.database.user} - ${pkgs.sudo}/bin/sudo -u ${config.services.postgresql.superUser} ${config.services.postgresql.package}/bin/createdb -O ${cfg.database.user} ${cfg.database.name} + ${pkgs.sudo}/bin/sudo -u ${pg.superUser} ${pg.postgresqlPackage}/bin/createuser ${cfg.database.user} + ${pkgs.sudo}/bin/sudo -u ${pg.superUser} ${pg.postgresqlPackage}/bin/createdb -O ${cfg.database.user} ${cfg.database.name} touch ${cfg.root}/.db-created fi ''} diff --git a/nixos/modules/services/web-servers/apache-httpd/owncloud.nix b/nixos/modules/services/web-servers/apache-httpd/owncloud.nix index 6345a9a569355..b81a084892e12 100644 --- a/nixos/modules/services/web-servers/apache-httpd/owncloud.nix +++ b/nixos/modules/services/web-servers/apache-httpd/owncloud.nix @@ -332,7 +332,7 @@ let 'logtimezone' => '${tz}', ''; - postgresql = serverInfo.fullConfig.services.postgresql.package; + postgresql = serverInfo.fullConfig.services.postgresql.postgresqlPackage; setupDb = pkgs.writeScript "setup-owncloud-db" '' #!${pkgs.runtimeShell} diff --git a/nixos/modules/services/web-servers/hydron.nix b/nixos/modules/services/web-servers/hydron.nix index ed63230bc7842..c1c3a41a2eea0 100644 --- a/nixos/modules/services/web-servers/hydron.nix +++ b/nixos/modules/services/web-servers/hydron.nix @@ -2,7 +2,7 @@ let cfg = config.services.hydron; - postgres = config.services.postgresql; + pg = config.services.postgresql; in with lib; { options.services.hydron = { enable = mkEnableOption "hydron"; @@ -105,11 +105,11 @@ in with lib; { chown -R hydron:hydron ${escapeShellArg cfg.dataDir} # Ensure the database is correct or create it - ${pkgs.sudo}/bin/sudo -u ${postgres.superUser} ${postgres.package}/bin/createuser \ + ${pkgs.sudo}/bin/sudo -u ${pg.superUser} ${pg.postgresqlPackage}/bin/createuser \ -SDR hydron || true - ${pkgs.sudo}/bin/sudo -u ${postgres.superUser} ${postgres.package}/bin/createdb \ + ${pkgs.sudo}/bin/sudo -u ${pg.superUser} ${pg.postgresqlPackage}/bin/createdb \ -T template0 -E UTF8 -O hydron hydron || true - ${pkgs.sudo}/bin/sudo -u hydron ${postgres.package}/bin/psql \ + ${pkgs.sudo}/bin/sudo -u hydron ${pg.postgresqlPackage}/bin/psql \ -c "ALTER ROLE hydron WITH PASSWORD '$(cat ${escapeShellArg cfg.passwordFile})';" || true ''; @@ -139,7 +139,7 @@ in with lib; { description = "Automatically import paths into hydron and possibly fetch tags"; after = [ "network.target" "hydron.service" ]; wantedBy = [ "timers.target" ]; - + timerConfig = { Persistent = true; OnCalendar = cfg.interval; @@ -148,7 +148,7 @@ in with lib; { users = { groups.hydron.gid = config.ids.gids.hydron; - + users.hydron = { description = "hydron server service user"; home = cfg.dataDir; diff --git a/nixos/modules/services/web-servers/meguca.nix b/nixos/modules/services/web-servers/meguca.nix index 11aebcb91d88a..6db66edf66a9a 100644 --- a/nixos/modules/services/web-servers/meguca.nix +++ b/nixos/modules/services/web-servers/meguca.nix @@ -2,7 +2,7 @@ let cfg = config.services.meguca; - postgres = config.services.postgresql; + pg = config.services.postgresql; in with lib; { options.services.meguca = { enable = mkEnableOption "meguca"; @@ -108,11 +108,11 @@ in with lib; { chown -R meguca:meguca ${escapeShellArg cfg.dataDir} # Ensure the database is correct or create it - ${pkgs.sudo}/bin/sudo -u ${postgres.superUser} ${postgres.package}/bin/createuser \ + ${pkgs.sudo}/bin/sudo -u ${pg.superUser} ${pg.postgresqlPackage}/bin/createuser \ -SDR meguca || true - ${pkgs.sudo}/bin/sudo -u ${postgres.superUser} ${postgres.package}/bin/createdb \ + ${pkgs.sudo}/bin/sudo -u ${pg.superUser} ${pg.postgresqlPackage}/bin/createdb \ -T template0 -E UTF8 -O meguca meguca || true - ${pkgs.sudo}/bin/sudo -u meguca ${postgres.package}/bin/psql \ + ${pkgs.sudo}/bin/sudo -u meguca ${pg.postgresqlPackage}/bin/psql \ -c "ALTER ROLE meguca WITH PASSWORD '$(cat ${escapeShellArg cfg.passwordFile})';" || true '';