postgresqlPackages: replace custom installPhase with buildPostgresqlExtension helper

pkgs/servers/sql/postgresql/buildPostgresqlExtension.nix

@@ -0,0 +1,129 @@
+# PostgreSQL's build system for extensions (PGXS) makes the following assumptions:
+# - All extensions will be installed in the same prefix as PostgreSQL itself.
+# - pg_config is able to return the correct paths for bindir/libdir/datadir etc.
+#
+# Both of those assumptions break with nix. Since each extension is a separate
+# derivation, we need to put all its files into a different folder. At the same
+# time, pg_config only points to the PostgreSQL derivation's paths.
+#
+# When building extensions, the paths provided by pg_config are used for two
+# purposes:
+# - To find postgres libs and headers and reference those paths via -L and -I flags.
+# - To determine the correct install directory.
+#
+# The PGXS Makefiles also support an environment variable DESTDIR, which is added as
+# a prefix to all install locations. This is primarily used for temporary installs
+# while running the test suite. Since pg_config returns absolute paths to /nix/store
+# for us, using DESTDIR will result in install locations of the form:
+#   $DESTIDR/nix/store/<postgresql-output>/...
+#
+# In multiple iterations, the following approaches have been tried to work around all
+# of this:
+# 1. For a long time, all extensions in nixpkgs just overwrote the installPhase
+#    and moved the respective files to the correct location manually. This approach
+#    is not maintainable, because whenever upstream adds a new file, we'd have to
+#    make sure the file is correctly installed as well. Additionally, it makes adding
+#    a new extension harder than it should be.
+#
+# 2. A wrapper around pg_config could just replace the returned paths with paths to
+#    $out of currently building derivation, i.e. the extension. This works for install-
+#    ation, but breaks for any of the libs and headers the extension needs from postgres
+#    itself.
+#
+# 3. A variation of 2., but make the pg_config wrapper only return the changed paths
+#    during the installPahse. During configure and build, it would return the regular
+#    paths to the PostgreSQL derivation. This works better, but not for every case.
+#    Some extensions try to be smarter and search for the "postgres" binary to deduce
+#    the necessary paths from that. Those would still need special handling.
+#
+# 4. Use the fact that DESTDIR is prepended to every installation directory - and only
+#    there, to run a replacement of all Makefiles in postgres' lib/pgxs/ folder and
+#    all Makefiles in the extension's source. "$DESTDIR/$bindir" can be replaced with
+#    "$out/bin" etc. - thus mapping the installPhase directly into the right output.
+#    This works beautifully - for the majority of cases. But it doesn't work for
+#    some extensions that use CMake. And it doesn't work for some extensions that use
+#    custom variables instead of the default "bindir" and friends.
+#
+# 5. Just set DESTDIR to the extensions's output and then clean up afterward. This will
+#    result in paths like this:
+#      /nix/store/<extension-output>/nix/store/<postgresql-output>/...
+#    Directly after the installPhase, we'll move the files in the right folder.
+#    This seems to work consistently across all extensions we have in nixpkgs right now.
+#    Of course, it would break down for any extension that doesn't support DESTDIR -
+#    but that just means PGXS is not used either, so that's OK.
+#
+# This last approach is the one we're taking in this file. To make sure the removal of the
+# nested nix/store happens immediately after the installPhase, before any other postInstall
+# hooks run, this needs to be run in an override of `mkDerivation` and not in a setup hook.
+
+{
+  lib,
+  stdenv,
+  postgresql,
+}:
+
+args:
+
+let
+  buildPostgresqlExtension = finalAttrs: prevAttrs: {
+    buildInputs = [ postgresql ] ++ prevAttrs.buildInputs or [ ];
+
+    installFlags = [
+      "DESTDIR=${placeholder "out"}"
+    ] ++ prevAttrs.installFlags or [ ];
+
+    postInstall =
+      ''
+        # DESTDIR + pg_config install the files into
+        # /nix/store/<extension>/nix/store/<postgresql>/...
+        # We'll now remove the /nix/store/<postgresql> part:
+        if [[ -d "$out${postgresql}" ]]; then
+            cp -alt "$out" "$out${postgresql}"/*
+            rm -r "$out${postgresql}"
+        fi
+
+        if [[ -d "$out${postgresql.dev}" ]]; then
+            mkdir -p "''${dev:-$out}"
+            cp -alt "''${dev:-$out}" "$out${postgresql.dev}"/*
+            rm -r "$out${postgresql.dev}"
+        fi
+
+        if [[ -d "$out${postgresql.lib}" ]]; then
+            mkdir -p "''${lib:-$out}"
+            cp -alt "''${lib:-$out}" "$out${postgresql.lib}"/*
+            rm -r "$out${postgresql.lib}"
+        fi
+
+        if [[ -d "$out${postgresql.doc}" ]]; then
+            mkdir -p "''${doc:-$out}"
+            cp -alt "''${doc:-$out}" "$out${postgresql.doc}"/*
+            rm -r "$out${postgresql.doc}"
+        fi
+
+        if [[ -d "$out${postgresql.man}" ]]; then
+            mkdir -p "''${man:-$out}"
+            cp -alt "''${man:-$out}" "$out${postgresql.man}"/*
+            rm -r "$out${postgresql.man}"
+        fi
+
+        # In some cases (postgis) parts of the install script
+        # actually work "OK", before we add DESTDIR, so some
+        # files end up in
+        # /nix/store/<extension>/nix/store/<extension>/...
+        if [[ -d "$out$out" ]]; then
+            cp -alt "$out" "$out$out"/*
+            rm -r "$out$out"
+        fi
+
+        if [[ -d "$out/nix/store" ]]; then
+            if ! rmdir "$out/nix/store" "$out/nix"; then
+              find "$out/nix"
+              nixErrorLog 'Found left-overs in $out/nix/store, make sure to move them into $out properly.'
+              exit 1
+            fi
+        fi
+      ''
+      + prevAttrs.postInstall or "";
+  };
+in
+stdenv.mkDerivation (lib.extends buildPostgresqlExtension (lib.toFunction args))

pkgs/servers/sql/postgresql/ext/age.nix

@@ -1,4 +1,4 @@
-{ lib, stdenv, bison, fetchFromGitHub, flex, perl, postgresql }:
+{ lib, stdenv, bison, fetchFromGitHub, flex, perl, postgresql, buildPostgresqlExtension }:
 
 let
   hashes = {
@@ -11,7 +11,7 @@ let
     "12" = "sha256-JFNk17ESsIt20dwXrfBkQ5E6DbZzN/Q9eS6+WjCXGd4=";
   };
 in
-stdenv.mkDerivation rec {
+buildPostgresqlExtension rec {
   pname = "age";
   version = "1.5.0-rc0";
 
@@ -22,20 +22,12 @@ stdenv.mkDerivation rec {
     hash = hashes.${lib.versions.major postgresql.version} or (throw "Source for Age is not available for ${postgresql.version}");
   };
 
-  buildInputs = [ postgresql ];
-
   makeFlags = [
     "BISON=${bison}/bin/bison"
     "FLEX=${flex}/bin/flex"
     "PERL=${perl}/bin/perl"
   ];
 
-  installPhase = ''
-    install -D -t $out/lib *${postgresql.dlSuffix}
-    install -D -t $out/share/postgresql/extension *.sql
-    install -D -t $out/share/postgresql/extension *.control
-  '';
-
   passthru.tests = stdenv.mkDerivation {
     inherit version src;
 

pkgs/servers/sql/postgresql/ext/anonymizer.nix

@@ -1,23 +1,16 @@
-{ lib, stdenv, pg-dump-anon, postgresql, runtimeShell, jitSupport, llvm }:
+{ lib, stdenv, pg-dump-anon, postgresql, runtimeShell, jitSupport, llvm, buildPostgresqlExtension }:
 
-stdenv.mkDerivation (finalAttrs: {
+buildPostgresqlExtension (finalAttrs: {
   pname = "postgresql_anonymizer";
 
   inherit (pg-dump-anon) version src passthru;
 
-  buildInputs = [ postgresql ];
   nativeBuildInputs = [ postgresql ] ++ lib.optional jitSupport llvm;
 
   strictDeps = true;
 
-  makeFlags = [
-    "BINDIR=${placeholder "out"}/bin"
-    "datadir=${placeholder "out"}/share/postgresql"
-    "pkglibdir=${placeholder "out"}/lib"
-    "DESTDIR="
-  ];
-
-  postInstall = ''
+  # Needs to be after postInstall, where removeNestedNixStore runs
+  preFixup = ''
     cat >$out/bin/pg_dump_anon.sh <<'EOF'
     #!${runtimeShell}
     echo "This script is deprecated by upstream. To use the new script,"

pkgs/servers/sql/postgresql/ext/apache_datasketches.nix

@@ -1,4 +1,4 @@
-{ stdenv, lib, fetchFromGitHub, postgresql, boost182, nixosTests }:
+{ stdenv, lib, fetchFromGitHub, postgresql, boost182, nixosTests, buildPostgresqlExtension }:
 
 let
   version = "1.7.0";
@@ -20,47 +20,22 @@ let
   };
 in
 
-stdenv.mkDerivation {
+buildPostgresqlExtension {
   pname = "apache_datasketches";
   inherit version;
 
   srcs = [ main_src cpp_src ];
 
   sourceRoot = main_src.name;
 
-  buildInputs = [ postgresql boost182 ];
+  buildInputs = [ boost182 ];
 
   patchPhase = ''
     runHook prePatch
     cp -r ../${cpp_src.name} .
     runHook postPatch
   '';
 
-  installPhase = ''
-    runHook preInstall
-    install -D -m 644 ./datasketches${postgresql.dlSuffix} -t $out/lib/
-    cat \
-      sql/datasketches_cpc_sketch.sql \
-      sql/datasketches_kll_float_sketch.sql \
-      sql/datasketches_kll_double_sketch.sql \
-      sql/datasketches_theta_sketch.sql \
-      sql/datasketches_frequent_strings_sketch.sql \
-      sql/datasketches_hll_sketch.sql \
-      sql/datasketches_aod_sketch.sql \
-      sql/datasketches_req_float_sketch.sql \
-      sql/datasketches_quantiles_double_sketch.sql \
-      > sql/datasketches--${version}.sql
-    install -D -m 644 ./datasketches.control -t $out/share/postgresql/extension
-    install -D -m 644 \
-      ./sql/datasketches--${version}.sql \
-      ./sql/datasketches--1.3.0--1.4.0.sql \
-      ./sql/datasketches--1.4.0--1.5.0.sql \
-      ./sql/datasketches--1.5.0--1.6.0.sql \
-      ./sql/datasketches--1.6.0--1.7.0.sql \
-      -t $out/share/postgresql/extension
-    runHook postInstall
-  '';
-
   passthru.tests.apache_datasketches = nixosTests.apache_datasketches;
 
   meta = {

pkgs/servers/sql/postgresql/ext/citus.nix

@@ -4,9 +4,10 @@
 , fetchFromGitHub
 , lz4
 , postgresql
+, buildPostgresqlExtension
 }:
 
-stdenv.mkDerivation rec {
+buildPostgresqlExtension rec {
   pname = "citus";
   version = "12.1.2";
 
@@ -20,23 +21,8 @@ stdenv.mkDerivation rec {
   buildInputs = [
     curl
     lz4
-    postgresql
   ];
 
-  installPhase = ''
-    runHook preInstall
-
-    install -D -t $out/lib src/backend/columnar/citus_columnar${postgresql.dlSuffix}
-    install -D -t $out/share/postgresql/extension src/backend/columnar/build/sql/*.sql
-    install -D -t $out/share/postgresql/extension src/backend/columnar/*.control
-
-    install -D -t $out/lib src/backend/distributed/citus${postgresql.dlSuffix}
-    install -D -t $out/share/postgresql/extension src/backend/distributed/build/sql/*.sql
-    install -D -t $out/share/postgresql/extension src/backend/distributed/*.control
-
-    runHook postInstall
-  '';
-
   meta = with lib; {
     # "Our soft policy for Postgres version compatibility is to support Citus'
     # latest release with Postgres' 3 latest releases."

pkgs/servers/sql/postgresql/ext/cstore_fdw.nix

@@ -1,11 +1,10 @@
-{ lib, stdenv, fetchFromGitHub, postgresql, protobufc }:
+{ lib, stdenv, fetchFromGitHub, postgresql, protobufc, buildPostgresqlExtension }:
 
-stdenv.mkDerivation rec {
+buildPostgresqlExtension rec {
   pname = "cstore_fdw";
   version = "unstable-2022-03-08";
 
   nativeBuildInputs = [ protobufc ];
-  buildInputs = [ postgresql ];
 
   src = fetchFromGitHub {
     owner  = "citusdata";
@@ -14,14 +13,6 @@ stdenv.mkDerivation rec {
     sha256 = "sha256-02wcCqs8A5ZOZX080fgcNJTQrYQctnlwnA8+YPaRTZc=";
   };
 
-  installPhase = ''
-    mkdir -p $out/{lib,share/postgresql/extension}
-
-    cp *.so      $out/lib
-    cp *.sql     $out/share/postgresql/extension
-    cp *.control $out/share/postgresql/extension
-  '';
-
   meta = with lib; {
     broken      = versionAtLeast postgresql.version "14";
     description = "Columnar storage for PostgreSQL";

pkgs/servers/sql/postgresql/ext/h3-pg.nix

@@ -5,9 +5,10 @@
 , h3_4
 , postgresql
 , postgresqlTestExtension
+, buildPostgresqlExtension
 }:
 
-stdenv.mkDerivation (finalAttrs: {
+buildPostgresqlExtension (finalAttrs: {
   pname = "h3-pg";
   version = "4.1.3";
 
@@ -32,16 +33,8 @@ stdenv.mkDerivation (finalAttrs: {
 
   buildInputs = [
     h3_4
-    postgresql
   ];
 
-  installPhase = ''
-    install -D -t $out/lib h3/h3.so
-    install -D -t $out/share/postgresql/extension h3/h3-*.sql h3/h3.control
-    install -D -t $out/lib h3_postgis/h3_postgis.so
-    install -D -t $out/share/postgresql/extension h3_postgis/h3_postgis-*.sql h3_postgis/h3_postgis.control
-  '';
-
   passthru.tests.extension = postgresqlTestExtension {
     inherit (finalAttrs) finalPackage;
     withPackages = [ "postgis" ];

pkgs/servers/sql/postgresql/ext/hypopg.nix

@@ -1,6 +1,6 @@
-{ lib, stdenv, fetchFromGitHub, postgresql, gitUpdater }:
+{ lib, stdenv, fetchFromGitHub, postgresql, gitUpdater, buildPostgresqlExtension }:
 
-stdenv.mkDerivation rec {
+buildPostgresqlExtension rec {
   pname = "hypopg";
   version = "1.4.1";
 
@@ -11,14 +11,6 @@ stdenv.mkDerivation rec {
     hash = "sha256-88uKPSnITRZ2VkelI56jZ9GWazG/Rn39QlyHKJKSKMM=";
   };
 
-  buildInputs = [ postgresql ];
-
-  installPhase = ''
-    install -D -t $out/lib *${postgresql.dlSuffix}
-    install -D -t $out/share/postgresql/extension *.control
-    install -D -t $out/share/postgresql/extension *.sql
-  '';
-
   passthru = {
     updateScript = gitUpdater {
       ignoredVersions = "beta";

pkgs/servers/sql/postgresql/ext/jsonb_deep_sum.nix

@@ -1,6 +1,6 @@
-{ lib, stdenv, fetchFromGitHub, postgresql }:
+{ lib, stdenv, fetchFromGitHub, postgresql, buildPostgresqlExtension }:
 
-stdenv.mkDerivation rec {
+buildPostgresqlExtension rec {
   pname = "jsonb_deep_sum";
   version = "unstable-2021-12-24";
 
@@ -11,16 +11,6 @@ stdenv.mkDerivation rec {
     sha256 = "sha256-W1wNILAwTAjFPezq+grdRMA59KEnMZDz69n9xQUqdc0=";
   };
 
-  buildInputs = [ postgresql ];
-
-  installPhase = ''
-    mkdir -p $out/{lib,share/postgresql/extension}
-
-    cp *${postgresql.dlSuffix} $out/lib
-    cp *.sql     $out/share/postgresql/extension
-    cp *.control $out/share/postgresql/extension
-  '';
-
   meta = with lib; {
     description = "PostgreSQL extension to easily add jsonb numeric";
     homepage = "https://github.com/furstenheim/jsonb_deep_sum";