Skip to content

Commit

Permalink
fix dynamic builds with liburing
Browse files Browse the repository at this point in the history
The original implementation of this was really weird, so I restructed it
a lot while debugging, and am just gonna leave the restructured version.

Root cause of the segfault seems to be that upstream nixpkgs liburing
derivation is generating both static and dynamic libraries, causing
rocksdb to statically link liburing in a dynamic build, pulling in some
allocator stuff at the same time. I created a PR[1] to fix this upstream,
but it probably won't be available on nixos-unstable for quite a while,
so we can also patch it locally.

[1]: NixOS/nixpkgs#314945
  • Loading branch information
Benjamin Lee committed May 26, 2024
1 parent 001138a commit 251aaa1
Show file tree
Hide file tree
Showing 2 changed files with 27 additions and 21 deletions.
18 changes: 18 additions & 0 deletions flake.nix
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,24 @@
(builtins.fromJSON (builtins.readFile ./flake.lock))
.nodes.rocksdb.original.ref;
});
# TODO: remove once https://github.com/NixOS/nixpkgs/pull/314945 is available
liburing = pkgs.liburing.overrideAttrs (old: {
# the configure script doesn't support these, and unconditionally
# builds both static and dynamic libraries.
configureFlags = pkgs.lib.subtractLists
[ "--enable-static" "--disable-shared" ]
old.configureFlags;

postInstall = old.postInstall + ''
# we remove the extra outputs
#
# we need to do this to prevent rocksdb from trying to link the
# static library in a dynamic stdenv
rm $out/lib/liburing*${
if pkgs.stdenv.hostPlatform.isStatic then ".so*" else ".a"
}
'';
});
});

scopeHost = mkScope pkgsHost;
Expand Down
30 changes: 9 additions & 21 deletions nix/pkgs/main/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,7 @@
, rocksdb
, rust
, rust-jemalloc-sys
, snappy
, stdenv
, pkgsStatic

# Options (keep sorted)
, default_features ? true
Expand Down Expand Up @@ -48,6 +46,8 @@ features'' = lib.subtractLists disable_features' features';

featureEnabled = feature : builtins.elem feature features'';

enableLiburing = featureEnabled "io_uring" && stdenv.isLinux;

# This derivation will set the JEMALLOC_OVERRIDE variable, causing the
# tikv-jemalloc-sys crate to use the nixpkgs jemalloc instead of building it's
# own. In order for this to work, we need to set flags on the build that match
Expand All @@ -61,15 +61,9 @@ rust-jemalloc-sys' = (rust-jemalloc-sys.override {
# tikv-jemalloc-sys/profiling feature
lib.optional (featureEnabled "jemalloc_prof") "--enable-prof";
});
liburing' = pkgsStatic.liburing.overrideAttrs {
configureFlags = []; # liburing's configure file is handwritten so the default assumptions don't apply
isStatic = true;
};

buildDepsOnlyEnv =
let
uring = featureEnabled "io_uring" && stdenv.isLinux;
extraDeps = lib.optionals uring [ liburing'.dev liburing'.out];
rocksdb' = (rocksdb.override {
jemalloc = rust-jemalloc-sys';
# rocksdb fails to build with prefixed jemalloc, which is required on
Expand All @@ -83,7 +77,8 @@ buildDepsOnlyEnv =
# TODO: static rocksdb fails to build on darwin
# build log at <https://girlboss.ceo/~strawberry/pb/JjGH>
meta.broken = stdenv.hostPlatform.isStatic && stdenv.isDarwin;
propagatedBuildInputs = old.propagatedBuildInputs ++ extraDeps;
# TODO: switch to enableUring option once https://github.com/NixOS/nixpkgs/pull/314945 is available
buildInputs = old.buildInputs ++ lib.optional enableLiburing liburing;
});
in
{
Expand All @@ -104,18 +99,11 @@ buildDepsOnlyEnv =
buildPackageEnv = {
CONDUWUIT_VERSION_EXTRA = inputs.self.shortRev or inputs.self.dirtyShortRev or "";
} // buildDepsOnlyEnv // {
CARGO_BUILD_RUSTFLAGS =
let
uring = featureEnabled "io_uring";
valid = (stdenv.hostPlatform.isAarch64 || stdenv.hostPlatform.isx86_64)
&& stdenv.hostPlatform.isStatic
&& !stdenv.isDarwin
&& !stdenv.cc.bintools.isLLVM;
in
buildDepsOnlyEnv.CARGO_BUILD_RUSTFLAGS
+ lib.optionalString (uring && valid) " -L${lib.getLib liburing'}/lib/ -luring"
+ " -L${lib.getLib snappy}/lib/ -lsnappy";
};
# Only needed in static stdenv because these are transitive dependencies of rocksdb
CARGO_BUILD_RUSTFLAGS = buildDepsOnlyEnv.CARGO_BUILD_RUSTFLAGS
+ lib.optionalString (enableLiburing && stdenv.hostPlatform.isStatic)
" -L${lib.getLib liburing}/lib -luring";
};



Expand Down

0 comments on commit 251aaa1

Please sign in to comment.