From 970942f45836172fda410a638853382952189eb9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 12 Feb 2025 21:50:20 +0100 Subject: [PATCH 01/28] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index ed1d60050..3953e8ad5 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.26.2 +2.26.3 From 86ccad698eb1c0679fc2be8ac59149211371358e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 20 May 2024 08:36:58 -0400 Subject: [PATCH 02/28] Expose a bunch of things in the Legacy SSH Store for Hydra (cherry picked from commit 5eade4825221d3284fc6555cb20de2c7aa171d72) --- src/libstore/legacy-ssh-store.cc | 99 +++++++++++++++++++++++++++----- src/libstore/legacy-ssh-store.hh | 55 ++++++++++++++++++ 2 files changed, 140 insertions(+), 14 deletions(-) diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index eac360a4f..3f62794ef 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -69,7 +69,7 @@ ref LegacySSHStore::openConnection() command.push_back("--store"); command.push_back(remoteStore.get()); } - conn->sshConn = master.startCommand(std::move(command)); + conn->sshConn = master.startCommand(std::move(command), std::list{extraSshArgs}); conn->to = FdSink(conn->sshConn->in.get()); conn->from = FdSource(conn->sshConn->out.get()); @@ -100,19 +100,31 @@ std::string LegacySSHStore::getUri() return *uriSchemes().begin() + "://" + host; } +std::map LegacySSHStore::queryPathInfosUncached( + const StorePathSet & paths) +{ + auto conn(connections->get()); + + /* No longer support missing NAR hash */ + assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4); + + debug("querying remote host '%s' for info on '%s'", host, concatStringsSep(", ", printStorePathSet(paths))); + + auto infos = conn->queryPathInfos(*this, paths); + + for (const auto & [_, info] : infos) { + if (info.narHash == Hash::dummy) + throw Error("NAR hash is now mandatory"); + } + + return infos; +} void LegacySSHStore::queryPathInfoUncached(const StorePath & path, Callback> callback) noexcept { try { - auto conn(connections->get()); - - /* No longer support missing NAR hash */ - assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4); - - debug("querying remote host '%s' for info on '%s'", host, printStorePath(path)); - - auto infos = conn->queryPathInfos(*this, {path}); + auto infos = queryPathInfosUncached({path}); switch (infos.size()) { case 0: @@ -120,9 +132,6 @@ void LegacySSHStore::queryPathInfoUncached(const StorePath & path, case 1: { auto & [path2, info] = *infos.begin(); - if (info.narHash == Hash::dummy) - throw Error("NAR hash is now mandatory"); - assert(path == path2); return callback(std::make_shared( std::move(path), @@ -193,13 +202,19 @@ void LegacySSHStore::addToStore(const ValidPathInfo & info, Source & source, void LegacySSHStore::narFromPath(const StorePath & path, Sink & sink) { - auto conn(connections->get()); - conn->narFromPath(*this, path, [&](auto & source) { + narFromPath(path, [&](auto & source) { copyNAR(source, sink); }); } +void LegacySSHStore::narFromPath(const StorePath & path, std::function fun) +{ + auto conn(connections->get()); + conn->narFromPath(*this, path, fun); +} + + static ServeProto::BuildOptions buildSettings() { return { @@ -223,6 +238,19 @@ BuildResult LegacySSHStore::buildDerivation(const StorePath & drvPath, const Bas return conn->getBuildDerivationResponse(*this); } +std::function LegacySSHStore::buildDerivationAsync( + const StorePath & drvPath, const BasicDerivation & drv, + const ServeProto::BuildOptions & options) +{ + // Until we have C++23 std::move_only_function + auto conn = std::make_shared::Handle>(connections->get()); + (*conn)->putBuildDerivationRequest(*this, drvPath, drv, options); + + return [this,conn]() -> BuildResult { + return (*conn)->getBuildDerivationResponse(*this); + }; +} + void LegacySSHStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode, std::shared_ptr evalStore) { @@ -294,6 +322,32 @@ StorePathSet LegacySSHStore::queryValidPaths(const StorePathSet & paths, } +StorePathSet LegacySSHStore::queryValidPaths(const StorePathSet & paths, + bool lock, SubstituteFlag maybeSubstitute) +{ + auto conn(connections->get()); + return conn->queryValidPaths(*this, + lock, paths, maybeSubstitute); +} + + +void LegacySSHStore::addMultipleToStoreLegacy(Store & srcStore, const StorePathSet & paths) +{ + auto conn(connections->get()); + conn->to << ServeProto::Command::ImportPaths; + try { + srcStore.exportPaths(paths, conn->to); + } catch (...) { + conn->good = false; + throw; + } + conn->to.flush(); + + if (readInt(conn->from) != 1) + throw Error("remote machine failed to import closure"); +} + + void LegacySSHStore::connect() { auto conn(connections->get()); @@ -307,6 +361,23 @@ unsigned int LegacySSHStore::getProtocol() } +pid_t LegacySSHStore::getConnectionPid() +{ + auto conn(connections->get()); + return conn->sshConn->sshPid; +} + + +LegacySSHStore::ConnectionStats LegacySSHStore::getConnectionStats() +{ + auto conn(connections->get()); + return { + .bytesReceived = conn->from.read, + .bytesSent = conn->to.written, + }; +} + + /** * The legacy ssh protocol doesn't support checking for trusted-user. * Try using ssh-ng:// instead if you want to know. diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index b541455b4..2444a7a66 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -6,6 +6,7 @@ #include "ssh.hh" #include "callback.hh" #include "pool.hh" +#include "serve-protocol.hh" namespace nix { @@ -24,6 +25,11 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig const Setting maxConnections{this, 1, "max-connections", "Maximum number of concurrent SSH connections."}; + /** + * Hack for hydra + */ + Strings extraSshArgs = {}; + const std::string name() override { return "SSH Store"; } static std::set uriSchemes() { return {"ssh"}; } @@ -60,11 +66,24 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor void queryPathInfoUncached(const StorePath & path, Callback> callback) noexcept override; + std::map queryPathInfosUncached( + const StorePathSet & paths); + void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) override; void narFromPath(const StorePath & path, Sink & sink) override; + /** + * Hands over the connection temporarily as source to the given + * function. The function must not consume beyond the NAR; it can + * not just blindly try to always read more bytes until it is + * cut-off. + * + * This is exposed for sake of Hydra. + */ + void narFromPath(const StorePath & path, std::function fun); + std::optional queryPathFromHashPart(const std::string & hashPart) override { unsupported("queryPathFromHashPart"); } @@ -93,6 +112,16 @@ public: BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; + /** + * Note, the returned function must only be called once, or we'll + * try to read from the connection twice. + * + * @todo Use C++23 `std::move_only_function`. + */ + std::function buildDerivationAsync( + const StorePath & drvPath, const BasicDerivation & drv, + const ServeProto::BuildOptions & options); + void buildPaths(const std::vector & drvPaths, BuildMode buildMode, std::shared_ptr evalStore) override; void ensurePath(const StorePath & path) override @@ -119,10 +148,36 @@ public: StorePathSet queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute = NoSubstitute) override; + /** + * Custom variation that atomically creates temp locks on the remote + * side. + * + * This exists to prevent a race where the remote host + * garbage-collects paths that are already there. Optionally, ask + * the remote host to substitute missing paths. + */ + StorePathSet queryValidPaths(const StorePathSet & paths, + bool lock, + SubstituteFlag maybeSubstitute = NoSubstitute); + + /** + * Just exists because this is exactly what Hydra was doing, and we + * don't yet want an algorithmic change. + */ + void addMultipleToStoreLegacy(Store & srcStore, const StorePathSet & paths); + void connect() override; unsigned int getProtocol() override; + struct ConnectionStats { + size_t bytesReceived, bytesSent; + }; + + ConnectionStats getConnectionStats(); + + pid_t getConnectionPid(); + /** * The legacy ssh protocol doesn't support checking for trusted-user. * Try using ssh-ng:// instead if you want to know. From 7112f8294c162db536b15f9d527033c9d641e057 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 23 May 2024 11:53:17 -0400 Subject: [PATCH 03/28] Add `SSHMaster::Connection::trySetBufferSize` It is unused in Nix currently, but will be used in Hydra. This reflects what Hydra does in https://github.com/NixOS/hydra/pull/1387. We may probably to use it more widely for better SSH store performance, but this needs to be subject to more testing before we do that. (cherry picked from commit 0d25cc65417647c454e3095650b87bc88351b384) --- src/libstore/ssh.cc | 15 +++++++++++++++ src/libstore/ssh.hh | 12 ++++++++++++ 2 files changed, 27 insertions(+) diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 116a480ba..f47cfbbec 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -240,4 +240,19 @@ Path SSHMaster::startMaster() #endif +void SSHMaster::Connection::trySetBufferSize(size_t size) +{ +#ifdef F_SETPIPE_SZ + /* This `fcntl` method of doing this takes a positive `int`. Check + and convert accordingly. + + The function overall still takes `size_t` because this is more + portable for a platform-agnostic interface. */ + assert(size <= INT_MAX); + int pipesize = size; + fcntl(in.get(), F_SETPIPE_SZ, pipesize); + fcntl(out.get(), F_SETPIPE_SZ, pipesize); +#endif +} + } diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh index 85be704ec..eb05df011 100644 --- a/src/libstore/ssh.hh +++ b/src/libstore/ssh.hh @@ -54,6 +54,18 @@ public: Pid sshPid; #endif AutoCloseFD out, in; + + /** + * Try to set the buffer size in both directions to the + * designated amount, if possible. If not possible, does + * nothing. + * + * Current implementation is to use `fcntl` with `F_SETPIPE_SZ`, + * which is Linux-only. For this implementation, `size` must + * convertable to an `int`. In other words, it must be within + * `[0, INT_MAX]`. + */ + void trySetBufferSize(size_t size); }; /** From fa7f0d6d07bdbedd06904d52bd111e58cb3d64c9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 16 Feb 2025 20:01:03 -0500 Subject: [PATCH 04/28] Allow setting `ssh://` pipe size Exposed for Hydra. We could make it fancier but with (a) new store settings (b) switch to `ssh-ng://` both in the works, it doesn't seem worth it. (cherry picked from commit 94a7c34b2f8285650e3130e5dc6ff5333eaa6dc8) --- src/libstore/legacy-ssh-store.cc | 3 +++ src/libstore/legacy-ssh-store.hh | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 3f62794ef..3849f088d 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -70,6 +70,9 @@ ref LegacySSHStore::openConnection() command.push_back(remoteStore.get()); } conn->sshConn = master.startCommand(std::move(command), std::list{extraSshArgs}); + if (connPipeSize) { + conn->sshConn->trySetBufferSize(*connPipeSize); + } conn->to = FdSink(conn->sshConn->in.get()); conn->from = FdSource(conn->sshConn->out.get()); diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index 2444a7a66..92aa4ae56 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -30,6 +30,11 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig */ Strings extraSshArgs = {}; + /** + * Exposed for hydra + */ + std::optional connPipeSize; + const std::string name() override { return "SSH Store"; } static std::set uriSchemes() { return {"ssh"}; } From 7d168db83cde11e3cf9872f7275fd6664f594740 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Feb 2025 11:36:47 +0100 Subject: [PATCH 05/28] getDefaultNixPath(): Don't add symlinks if the target doesn't exist (cherry picked from commit 8ac49ea5de0b763175af5b266dd258c544192036) --- src/libexpr/eval-settings.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index 4cbcb39b9..ade0abf9a 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -57,7 +57,7 @@ Strings EvalSettings::getDefaultNixPath() { Strings res; auto add = [&](const Path & p, const std::string & s = std::string()) { - if (pathAccessible(p)) { + if (std::filesystem::exists(p)) { if (s.empty()) { res.push_back(p); } else { From 640ce50da12e81c078142d344c89a9b3494a3ccb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Feb 2025 11:50:54 +0100 Subject: [PATCH 06/28] resolveLookupPathPath(): Fix caching of negative lookups This avoids spamming in case the missing search path entry does not exist (#12480). (cherry picked from commit df08e1e204d04924bc546ed3ebb2fabf936aa5be) --- src/libexpr/eval.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index dee764429..8aef85dc5 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -3070,8 +3070,11 @@ std::optional EvalState::resolveLookupPathPath(const LookupPath::Pat auto i = lookupPathResolved.find(value); if (i != lookupPathResolved.end()) return i->second; - auto finish = [&](SourcePath res) { - debug("resolved search path element '%s' to '%s'", value, res); + auto finish = [&](std::optional res) { + if (res) + debug("resolved search path element '%s' to '%s'", value, *res); + else + debug("failed to resolve search path element '%s'", value); lookupPathResolved.emplace(value, res); return res; }; @@ -3123,8 +3126,7 @@ std::optional EvalState::resolveLookupPathPath(const LookupPath::Pat } } - debug("failed to resolve search path element '%s'", value); - return std::nullopt; + return finish(std::nullopt); } From 80db87bd4c0ec214be8cc1705e6b5f11212605a1 Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman <145775305+xokdvium@users.noreply.github.com> Date: Tue, 18 Feb 2025 01:57:33 +0300 Subject: [PATCH 07/28] Move code related to NIX_MAN_DIR from libstore to nix-cli This is a prerequisite to properly fixing man-pages once and for all [1]. Note that this patch leaves manpages for legacy commands in a borked state, pending the movement of manpages from nix-manual to nix-cli [2]. [1]: https://www.github.com/NixOS/nix/issues/12382 [2]: https://www.github.com/NixOS/nix/issues/12382#issuecomment-2663782043 (cherry picked from commit b1a38b3efe214b093910776f4a92cad0fc125a3e) --- src/libmain/shared.cc | 14 --------- src/libmain/shared.hh | 5 ---- src/libstore/globals.cc | 1 - src/libstore/globals.hh | 5 ---- src/nix-build/nix-build.cc | 1 + src/nix-channel/nix-channel.cc | 1 + .../nix-collect-garbage.cc | 1 + src/nix-copy-closure/nix-copy-closure.cc | 1 + src/nix-env/nix-env.cc | 1 + src/nix-instantiate/nix-instantiate.cc | 1 + src/nix-store/nix-store.cc | 1 + src/nix/hash.cc | 1 + src/nix/man-pages.cc | 29 +++++++++++++++++++ src/nix/man-pages.hh | 28 ++++++++++++++++++ src/nix/meson.build | 12 ++++++++ src/nix/prefetch.cc | 1 + src/nix/unix/daemon.cc | 1 + 17 files changed, 79 insertions(+), 25 deletions(-) create mode 100644 src/nix/man-pages.cc create mode 100644 src/nix/man-pages.hh diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 50f90bfb3..30e76c349 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -315,20 +315,6 @@ void printVersion(const std::string & programName) throw Exit(); } - -void showManPage(const std::string & name) -{ - restoreProcessContext(); - setEnv("MANPATH", settings.nixManDir.c_str()); - execlp("man", "man", name.c_str(), nullptr); - if (errno == ENOENT) { - // Not SysError because we don't want to suffix the errno, aka No such file or directory. - throw Error("The '%1%' command was not found, but it is needed for '%2%' and some other '%3%' commands' help text. Perhaps you could install the '%1%' command?", "man", name.c_str(), "nix-*"); - } - throw SysError("command 'man %1%' failed", name.c_str()); -} - - int handleExceptions(const std::string & programName, std::function fun) { ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 712b404d3..a6a18ceb0 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -70,11 +70,6 @@ struct LegacyArgs : public MixCommonArgs, public RootArgs }; -/** - * Show the manual page for the specified program. - */ -void showManPage(const std::string & name); - /** * The constructor of this class starts a pager if standard output is a * terminal and $PAGER is set. Standard output is redirected to the diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index b64e73c26..e908fc81f 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -65,7 +65,6 @@ Settings::Settings() , nixStateDir(canonPath(getEnvNonEmpty("NIX_STATE_DIR").value_or(NIX_STATE_DIR))) , nixConfDir(canonPath(getEnvNonEmpty("NIX_CONF_DIR").value_or(NIX_CONF_DIR))) , nixUserConfFiles(getUserConfigFiles()) - , nixManDir(canonPath(NIX_MAN_DIR)) , nixDaemonSocketFile(canonPath(getEnvNonEmpty("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH))) { #ifndef _WIN32 diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index ff3df46ba..6b9a87ce3 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -84,11 +84,6 @@ public: */ std::vector nixUserConfFiles; - /** - * The directory where the man pages are stored. - */ - Path nixManDir; - /** * File name of the socket the daemon listens to. */ diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index de01e1afc..5410f0cab 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -27,6 +27,7 @@ #include "users.hh" #include "network-proxy.hh" #include "compatibility-settings.hh" +#include "man-pages.hh" using namespace nix; using namespace std::string_literals; diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 56d1d7abb..ee61db994 100644 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -8,6 +8,7 @@ #include "users.hh" #include "tarball.hh" #include "self-exe.hh" +#include "man-pages.hh" #include #include diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index 20d5161df..a060a01fd 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -7,6 +7,7 @@ #include "shared.hh" #include "globals.hh" #include "legacy.hh" +#include "man-pages.hh" #include #include diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index b64af758f..15bff0a0a 100644 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -2,6 +2,7 @@ #include "realisation.hh" #include "store-api.hh" #include "legacy.hh" +#include "man-pages.hh" using namespace nix; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index c99c1088e..aa1edb4c8 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -17,6 +17,7 @@ #include "legacy.hh" #include "eval-settings.hh" // for defexpr #include "terminal.hh" +#include "man-pages.hh" #include #include diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 09d354832..0cf926369 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -12,6 +12,7 @@ #include "local-fs-store.hh" #include "common-eval-args.hh" #include "legacy.hh" +#include "man-pages.hh" #include #include diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 99bb2c726..3fb69a29d 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -12,6 +12,7 @@ #include "legacy.hh" #include "posix-source-accessor.hh" #include "path-with-outputs.hh" +#include "man-pages.hh" #ifndef _WIN32 // TODO implement on Windows or provide allowed-to-noop interface # include "local-store.hh" diff --git a/src/nix/hash.cc b/src/nix/hash.cc index eac421d12..91bba47f4 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -8,6 +8,7 @@ #include "git.hh" #include "posix-source-accessor.hh" #include "misc-store-flags.hh" +#include "man-pages.hh" using namespace nix; diff --git a/src/nix/man-pages.cc b/src/nix/man-pages.cc new file mode 100644 index 000000000..a98a771cc --- /dev/null +++ b/src/nix/man-pages.cc @@ -0,0 +1,29 @@ +#include "man-pages.hh" +#include "file-system.hh" +#include "current-process.hh" +#include "environment-variables.hh" + +namespace nix { + +std::filesystem::path getNixManDir() +{ + return canonPath(NIX_MAN_DIR); +} + +void showManPage(const std::string & name) +{ + restoreProcessContext(); + setEnv("MANPATH", getNixManDir().c_str()); + execlp("man", "man", name.c_str(), nullptr); + if (errno == ENOENT) { + // Not SysError because we don't want to suffix the errno, aka No such file or directory. + throw Error( + "The '%1%' command was not found, but it is needed for '%2%' and some other '%3%' commands' help text. Perhaps you could install the '%1%' command?", + "man", + name.c_str(), + "nix-*"); + } + throw SysError("command 'man %1%' failed", name.c_str()); +} + +} diff --git a/src/nix/man-pages.hh b/src/nix/man-pages.hh new file mode 100644 index 000000000..9ba035af8 --- /dev/null +++ b/src/nix/man-pages.hh @@ -0,0 +1,28 @@ +#pragma once +///@file + +#include +#include + +namespace nix { + +/** + * @brief Get path to the nix manual dir. + * + * Nix relies on the man pages being available at a NIX_MAN_DIR for + * displaying help messaged for legacy cli. + * + * NIX_MAN_DIR is a compile-time parameter, so man pages are unlikely to work + * for cases when the nix executable is installed out-of-store or as a static binary. + * + */ +std::filesystem::path getNixManDir(); + +/** + * Show the manual page for the specified program. + * + * @param name Name of the man item. + */ +void showManPage(const std::string & name); + +} diff --git a/src/nix/meson.build b/src/nix/meson.build index 2698cc873..e8d740803 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -90,6 +90,7 @@ nix_sources = [config_h] + files( 'ls.cc', 'main.cc', 'make-content-addressed.cc', + 'man-pages.cc', 'nar.cc', 'optimise-store.cc', 'path-from-hash-part.cc', @@ -182,6 +183,16 @@ if host_machine.system() != 'windows' ] endif +fs = import('fs') +prefix = get_option('prefix') + +mandir = get_option('mandir') +mandir = fs.is_absolute(mandir) ? mandir : prefix / mandir + +cpp_args= [ + '-DNIX_MAN_DIR="@0@"'.format(mandir) +] + include_dirs = [include_directories('.')] this_exe = executable( @@ -189,6 +200,7 @@ this_exe = executable( sources, dependencies : deps_private_subproject + deps_private + deps_other, include_directories : include_dirs, + cpp_args : cpp_args, link_args: linker_export_flags, install : true, ) diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index db7d9e4ef..84c0224e2 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -12,6 +12,7 @@ #include "posix-source-accessor.hh" #include "misc-store-flags.hh" #include "terminal.hh" +#include "man-pages.hh" #include diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 746963a01..b4c7c10ed 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -15,6 +15,7 @@ #include "finally.hh" #include "legacy.hh" #include "daemon.hh" +#include "man-pages.hh" #include #include From a004c84e85fa03137afeb08d8f93a6c0df7ed6bf Mon Sep 17 00:00:00 2001 From: Sergei Zimmerman <145775305+xokdvium@users.noreply.github.com> Date: Tue, 18 Feb 2025 13:01:39 +0300 Subject: [PATCH 08/28] Don't override default man search paths By appending a colon to MANPATH NIX_MAN_DIR gets prepended to the final MANPATH before default search paths. This makes man still consider default search paths, but prefers NIX_MAN_DIR (if it exists). It still makes sense to point NIX_MAN_DIR to a correct location by moving man pages build from nix-manual.man to nix-cli.man, but this should fix most common use-cases where nix is installed globally. (cherry picked from commit 95f16a3275a3d23afe4f311cb793d7a5d47222e1) --- src/nix/man-pages.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/man-pages.cc b/src/nix/man-pages.cc index a98a771cc..e9e89bb62 100644 --- a/src/nix/man-pages.cc +++ b/src/nix/man-pages.cc @@ -13,7 +13,7 @@ std::filesystem::path getNixManDir() void showManPage(const std::string & name) { restoreProcessContext(); - setEnv("MANPATH", getNixManDir().c_str()); + setEnv("MANPATH", (getNixManDir().string() + ":").c_str()); execlp("man", "man", name.c_str(), nullptr); if (errno == ENOENT) { // Not SysError because we don't want to suffix the errno, aka No such file or directory. From f02a7b880e32015dd165975a4c6c87beab2ee077 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 18 Feb 2025 11:56:19 +0100 Subject: [PATCH 09/28] startDaemon(): Detect if the daemon crashes before creating the socket This avoids timeouts like those seen in https://github.com/NixOS/nix/actions/runs/13376958708/job/37358120348?pr=6962. (cherry picked from commit 11c42cb2e1b5bb44719e40d9c17750fb8a99d750) --- tests/functional/common/functions.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/functional/common/functions.sh b/tests/functional/common/functions.sh index bf3dd2ca8..1b2ec8fe0 100644 --- a/tests/functional/common/functions.sh +++ b/tests/functional/common/functions.sh @@ -67,7 +67,7 @@ startDaemon() { die "startDaemon: not supported when testing on NixOS. Is it really needed? If so add conditionals; e.g. if ! isTestOnNixOS; then ..." fi - # Don’t start the daemon twice, as this would just make it loop indefinitely + # Don't start the daemon twice, as this would just make it loop indefinitely. if [[ "${_NIX_TEST_DAEMON_PID-}" != '' ]]; then return fi @@ -76,15 +76,19 @@ startDaemon() { PATH=$DAEMON_PATH nix --extra-experimental-features 'nix-command' daemon & _NIX_TEST_DAEMON_PID=$! export _NIX_TEST_DAEMON_PID - for ((i = 0; i < 300; i++)); do + for ((i = 0; i < 60; i++)); do if [[ -S $NIX_DAEMON_SOCKET_PATH ]]; then DAEMON_STARTED=1 break; fi + if ! kill -0 "$_NIX_TEST_DAEMON_PID"; then + echo "daemon died unexpectedly" >&2 + exit 1 + fi sleep 0.1 done if [[ -z ${DAEMON_STARTED+x} ]]; then - fail "Didn’t manage to start the daemon" + fail "Didn't manage to start the daemon" fi trap "killDaemon" EXIT # Save for if daemon is killed @@ -97,7 +101,7 @@ killDaemon() { die "killDaemon: not supported when testing on NixOS. Is it really needed? If so add conditionals; e.g. if ! isTestOnNixOS; then ..." fi - # Don’t fail trying to stop a non-existant daemon twice + # Don't fail trying to stop a non-existant daemon twice. if [[ "${_NIX_TEST_DAEMON_PID-}" == '' ]]; then return fi @@ -219,7 +223,7 @@ assertStderr() { needLocalStore() { if [[ "$NIX_REMOTE" == "daemon" ]]; then - skipTest "Can’t run through the daemon ($1)" + skipTest "Can't run through the daemon ($1)" fi } From cc3ad9bd3af7da510a1c41eabf6f761c713143fd Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 17 Feb 2025 17:17:37 +0100 Subject: [PATCH 10/28] Support libgit2 1.9.0 For when the overlay is used with nixos-unstable. 1.9.0 has our patches. (cherry picked from commit b0bbb1252a8ae8d925e2cb45d1c778b9c00587e2) --- packaging/dependencies.nix | 51 ++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index afbc31fc6..b23c9cbcd 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -138,7 +138,8 @@ let enableParallelBuilding = true; }; in -scope: { +scope: +{ inherit stdenv; aws-sdk-cpp = @@ -174,6 +175,31 @@ scope: { installPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.installPhase; }); + inherit resolvePath filesetToSource; + + mkMesonDerivation = mkPackageBuilder [ + miscGoodPractice + localSourceLayer + mesonLayer + ]; + mkMesonExecutable = mkPackageBuilder [ + miscGoodPractice + bsdNoLinkAsNeeded + localSourceLayer + mesonLayer + mesonBuildLayer + ]; + mkMesonLibrary = mkPackageBuilder [ + miscGoodPractice + bsdNoLinkAsNeeded + localSourceLayer + mesonLayer + mesonBuildLayer + mesonLibraryLayer + ]; +} +# libgit2: Nixpkgs 24.11 has < 1.9.0 +// lib.optionalAttrs (!lib.versionAtLeast pkgs.libgit2.version "1.9.0") { libgit2 = pkgs.libgit2.overrideAttrs (attrs: { cmakeFlags = attrs.cmakeFlags or [ ] ++ [ "-DUSE_SSH=exec" ]; nativeBuildInputs = @@ -203,27 +229,4 @@ scope: { ./patches/libgit2-packbuilder-callback-interruptible.patch ]; }); - - inherit resolvePath filesetToSource; - - mkMesonDerivation = mkPackageBuilder [ - miscGoodPractice - localSourceLayer - mesonLayer - ]; - mkMesonExecutable = mkPackageBuilder [ - miscGoodPractice - bsdNoLinkAsNeeded - localSourceLayer - mesonLayer - mesonBuildLayer - ]; - mkMesonLibrary = mkPackageBuilder [ - miscGoodPractice - bsdNoLinkAsNeeded - localSourceLayer - mesonLayer - mesonBuildLayer - mesonLibraryLayer - ]; } From 21998464b7986f50bc20aa4a8ca3ab416b10d536 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 18 Feb 2025 11:22:00 +0100 Subject: [PATCH 11/28] Restore detailed Nix CLI version ... as intended. Requirements: - don't build fresh libraries for each git commit - have git commit in the CLI Bug: - echo ${version} went into the wrong file => use the fact that it's a symlink, not just for reading but also for writing. (cherry picked from commit bba4e6b061f53cbc77d47408468f9bc0f534281b) --- src/libstore/globals.cc | 2 +- src/libstore/globals.hh | 10 +++++++++- src/nix/main.cc | 3 +++ src/nix/meson.build | 3 +++ src/nix/package.nix | 6 +++--- 5 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index e908fc81f..d7c000dfa 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -242,7 +242,7 @@ Path Settings::getDefaultSSLCertFile() return ""; } -const std::string nixVersion = PACKAGE_VERSION; +std::string nixVersion = PACKAGE_VERSION; NLOHMANN_JSON_SERIALIZE_ENUM(SandboxMode, { {SandboxMode::smEnabled, true}, diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 6b9a87ce3..1682d572c 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -1248,7 +1248,15 @@ void loadConfFile(AbstractConfig & config); // Used by the Settings constructor std::vector getUserConfigFiles(); -extern const std::string nixVersion; +/** + * The version of Nix itself. + * + * This is not `const`, so that the Nix CLI can provide a more detailed version + * number including the git revision, without having to "re-compile" the entire + * set of Nix libraries to include that version, even when those libraries are + * not affected by the change. + */ +extern std::string nixVersion; /** * @param loadConfig Whether to load configuration from `nix.conf`, `NIX_CONFIG`, etc. May be disabled for unit tests. diff --git a/src/nix/main.cc b/src/nix/main.cc index b0e26e093..3db17ef69 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -557,6 +557,9 @@ void mainWrapped(int argc, char * * argv) int main(int argc, char * * argv) { + // The CLI has a more detailed version than the libraries; see nixVersion. + nix::nixVersion = NIX_CLI_VERSION; + // Increase the default stack size for the evaluator and for // libstdc++'s std::regex. nix::setStackSize(64 * 1024 * 1024); diff --git a/src/nix/meson.build b/src/nix/meson.build index e8d740803..1ad3d5b5a 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -35,6 +35,9 @@ subdir('nix-meson-build-support/windows-version') configdata = configuration_data() +# The CLI has a more detailed version string than the libraries; see `nixVersion` +configdata.set_quoted('NIX_CLI_VERSION', meson.project_version()) + fs = import('fs') bindir = get_option('bindir') diff --git a/src/nix/package.nix b/src/nix/package.nix index 6e59adc38..bb90be1ef 100644 --- a/src/nix/package.nix +++ b/src/nix/package.nix @@ -92,11 +92,11 @@ mkMesonExecutable (finalAttrs: { ]; preConfigure = - # "Inline" .version so it's not a symlink, and includes the suffix. - # Do the meson utils, without modification. + # Update the repo-global .version file. + # Symlink ./.version points there, but by default only workDir is writable. '' chmod u+w ./.version - echo ${version} > ../../../.version + echo ${version} > ./.version ''; mesonFlags = [ From b175e5bb6dcd945316cbab531a0d97574ad3f0b1 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 18 Feb 2025 11:41:35 +0100 Subject: [PATCH 12/28] Write just ./.version on all components This way it's easier to get right. See previous commit. (cherry picked from commit 3556f6bf4cd6aa7ffea760c03b8e91ddbe3fcde8) --- packaging/dependencies.nix | 15 +++++++++++++++ src/libcmd/package.nix | 8 -------- src/libexpr-c/package.nix | 8 -------- src/libexpr-test-support/package.nix | 8 -------- src/libexpr-tests/package.nix | 8 -------- src/libexpr/package.nix | 8 -------- src/libfetchers-tests/package.nix | 8 -------- src/libfetchers/package.nix | 8 -------- src/libflake-c/package.nix | 8 -------- src/libflake-tests/package.nix | 8 -------- src/libflake/package.nix | 8 -------- src/libmain-c/package.nix | 8 -------- src/libmain/package.nix | 8 -------- src/libstore-c/package.nix | 8 -------- src/libstore-test-support/package.nix | 8 -------- src/libstore-tests/package.nix | 8 -------- src/libstore/package.nix | 8 -------- src/libutil-c/package.nix | 8 -------- src/libutil-test-support/package.nix | 8 -------- src/libutil-tests/package.nix | 8 -------- src/libutil/package.nix | 11 ----------- src/nix/package.nix | 8 -------- tests/functional/package.nix | 8 +------- 23 files changed, 16 insertions(+), 178 deletions(-) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index b23c9cbcd..20992555c 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -42,6 +42,18 @@ let mkPackageBuilder = exts: userFn: stdenv.mkDerivation (lib.extends (lib.composeManyExtensions exts) userFn); + setVersionLayer = finalAttrs: prevAttrs: { + preConfigure = + prevAttrs.prevAttrs or "" + + + # Update the repo-global .version file. + # Symlink ./.version points there, but by default only workDir is writable. + '' + chmod u+w ./.version + echo ${finalAttrs.version} > ./.version + ''; + }; + localSourceLayer = finalAttrs: prevAttrs: let @@ -180,12 +192,14 @@ scope: mkMesonDerivation = mkPackageBuilder [ miscGoodPractice localSourceLayer + setVersionLayer mesonLayer ]; mkMesonExecutable = mkPackageBuilder [ miscGoodPractice bsdNoLinkAsNeeded localSourceLayer + setVersionLayer mesonLayer mesonBuildLayer ]; @@ -194,6 +208,7 @@ scope: bsdNoLinkAsNeeded localSourceLayer mesonLayer + setVersionLayer mesonBuildLayer mesonLibraryLayer ]; diff --git a/src/libcmd/package.nix b/src/libcmd/package.nix index d155d9f1e..d459d1c20 100644 --- a/src/libcmd/package.nix +++ b/src/libcmd/package.nix @@ -64,14 +64,6 @@ mkMesonLibrary (finalAttrs: { nlohmann_json ]; - preConfigure = - # "Inline" .version so it's not a symlink, and includes the suffix. - # Do the meson utils, without modification. - '' - chmod u+w ./.version - echo ${version} > ../../.version - ''; - mesonFlags = [ (lib.mesonEnable "markdown" enableMarkdown) (lib.mesonOption "readline-flavor" readlineFlavor) diff --git a/src/libexpr-c/package.nix b/src/libexpr-c/package.nix index ad1ea371c..694fbc1fe 100644 --- a/src/libexpr-c/package.nix +++ b/src/libexpr-c/package.nix @@ -36,14 +36,6 @@ mkMesonLibrary (finalAttrs: { nix-expr ]; - preConfigure = - # "Inline" .version so it's not a symlink, and includes the suffix. - # Do the meson utils, without modification. - '' - chmod u+w ./.version - echo ${version} > ../../.version - ''; - mesonFlags = [ ]; diff --git a/src/libexpr-test-support/package.nix b/src/libexpr-test-support/package.nix index 5628d606a..44b0ff386 100644 --- a/src/libexpr-test-support/package.nix +++ b/src/libexpr-test-support/package.nix @@ -40,14 +40,6 @@ mkMesonLibrary (finalAttrs: { rapidcheck ]; - preConfigure = - # "Inline" .version so it's not a symlink, and includes the suffix. - # Do the meson utils, without modification. - '' - chmod u+w ./.version - echo ${version} > ../../.version - ''; - mesonFlags = [ ]; diff --git a/src/libexpr-tests/package.nix b/src/libexpr-tests/package.nix index bb5acb7c8..51d52e935 100644 --- a/src/libexpr-tests/package.nix +++ b/src/libexpr-tests/package.nix @@ -46,14 +46,6 @@ mkMesonExecutable (finalAttrs: { gtest ]; - preConfigure = - # "Inline" .version so it's not a symlink, and includes the suffix. - # Do the meson utils, without modification. - '' - chmod u+w ./.version - echo ${version} > ../../.version - ''; - mesonFlags = [ ]; diff --git a/src/libexpr/package.nix b/src/libexpr/package.nix index afd01c384..533dae9f2 100644 --- a/src/libexpr/package.nix +++ b/src/libexpr/package.nix @@ -77,14 +77,6 @@ mkMesonLibrary (finalAttrs: { nlohmann_json ] ++ lib.optional enableGC boehmgc; - preConfigure = - # "Inline" .version so it's not a symlink, and includes the suffix. - # Do the meson utils, without modification. - '' - chmod u+w ./.version - echo ${version} > ../../.version - ''; - mesonFlags = [ (lib.mesonEnable "gc" enableGC) ]; diff --git a/src/libfetchers-tests/package.nix b/src/libfetchers-tests/package.nix index f2680e9b3..1e379fc5a 100644 --- a/src/libfetchers-tests/package.nix +++ b/src/libfetchers-tests/package.nix @@ -44,14 +44,6 @@ mkMesonExecutable (finalAttrs: { gtest ]; - preConfigure = - # "Inline" .version so it's not a symlink, and includes the suffix. - # Do the meson utils, without modification. - '' - chmod u+w ./.version - echo ${version} > ../../.version - ''; - mesonFlags = [ ]; diff --git a/src/libfetchers/package.nix b/src/libfetchers/package.nix index b0aecd049..3f52e9878 100644 --- a/src/libfetchers/package.nix +++ b/src/libfetchers/package.nix @@ -41,14 +41,6 @@ mkMesonLibrary (finalAttrs: { nlohmann_json ]; - preConfigure = - # "Inline" .version so it's not a symlink, and includes the suffix. - # Do the meson utils, without modification. - '' - chmod u+w ./.version - echo ${version} > ../../.version - ''; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libflake-c/package.nix b/src/libflake-c/package.nix index f0615a427..114950852 100644 --- a/src/libflake-c/package.nix +++ b/src/libflake-c/package.nix @@ -38,14 +38,6 @@ mkMesonLibrary (finalAttrs: { nix-flake ]; - preConfigure = - # "Inline" .version so it's not a symlink, and includes the suffix. - # Do the meson utils, without modification. - '' - chmod u+w ./.version - echo ${version} > ../../.version - ''; - mesonFlags = [ ]; diff --git a/src/libflake-tests/package.nix b/src/libflake-tests/package.nix index f9d9b0bc0..714f3791a 100644 --- a/src/libflake-tests/package.nix +++ b/src/libflake-tests/package.nix @@ -46,14 +46,6 @@ mkMesonExecutable (finalAttrs: { gtest ]; - preConfigure = - # "Inline" .version so it's not a symlink, and includes the suffix. - # Do the meson utils, without modification. - '' - chmod u+w ./.version - echo ${version} > ../../.version - ''; - mesonFlags = [ ]; diff --git a/src/libflake/package.nix b/src/libflake/package.nix index ebd38e140..5240ce5e3 100644 --- a/src/libflake/package.nix +++ b/src/libflake/package.nix @@ -40,14 +40,6 @@ mkMesonLibrary (finalAttrs: { nlohmann_json ]; - preConfigure = - # "Inline" .version so it's not a symlink, and includes the suffix. - # Do the meson utils, without modification. - '' - chmod u+w ./.version - echo ${version} > ../../.version - ''; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libmain-c/package.nix b/src/libmain-c/package.nix index cf710e03b..f019a917d 100644 --- a/src/libmain-c/package.nix +++ b/src/libmain-c/package.nix @@ -40,14 +40,6 @@ mkMesonLibrary (finalAttrs: { nix-main ]; - preConfigure = - # "Inline" .version so it's not a symlink, and includes the suffix. - # Do the meson utils, without modification. - '' - chmod u+w ./.version - echo ${version} > ../../.version - ''; - mesonFlags = [ ]; diff --git a/src/libmain/package.nix b/src/libmain/package.nix index 046b505df..c03697c48 100644 --- a/src/libmain/package.nix +++ b/src/libmain/package.nix @@ -37,14 +37,6 @@ mkMesonLibrary (finalAttrs: { openssl ]; - preConfigure = - # "Inline" .version so it's not a symlink, and includes the suffix. - # Do the meson utils, without modification. - '' - chmod u+w ./.version - echo ${version} > ../../.version - ''; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libstore-c/package.nix b/src/libstore-c/package.nix index 89abeaab8..fde17c78e 100644 --- a/src/libstore-c/package.nix +++ b/src/libstore-c/package.nix @@ -36,14 +36,6 @@ mkMesonLibrary (finalAttrs: { nix-store ]; - preConfigure = - # "Inline" .version so it's not a symlink, and includes the suffix. - # Do the meson utils, without modification. - '' - chmod u+w ./.version - echo ${version} > ../../.version - ''; - mesonFlags = [ ]; diff --git a/src/libstore-test-support/package.nix b/src/libstore-test-support/package.nix index 7cc29795c..ccac25ee1 100644 --- a/src/libstore-test-support/package.nix +++ b/src/libstore-test-support/package.nix @@ -40,14 +40,6 @@ mkMesonLibrary (finalAttrs: { rapidcheck ]; - preConfigure = - # "Inline" .version so it's not a symlink, and includes the suffix. - # Do the meson utils, without modification. - '' - chmod u+w ./.version - echo ${version} > ../../.version - ''; - mesonFlags = [ ]; diff --git a/src/libstore-tests/package.nix b/src/libstore-tests/package.nix index 670386c4a..b39ee7fa7 100644 --- a/src/libstore-tests/package.nix +++ b/src/libstore-tests/package.nix @@ -52,14 +52,6 @@ mkMesonExecutable (finalAttrs: { nix-store-test-support ]; - preConfigure = - # "Inline" .version so it's not a symlink, and includes the suffix. - # Do the meson utils, without modification. - '' - chmod u+w ./.version - echo ${version} > ../../.version - ''; - mesonFlags = [ ]; diff --git a/src/libstore/package.nix b/src/libstore/package.nix index c982b44f0..31867d331 100644 --- a/src/libstore/package.nix +++ b/src/libstore/package.nix @@ -69,14 +69,6 @@ mkMesonLibrary (finalAttrs: { nlohmann_json ]; - preConfigure = - # "Inline" .version so it's not a symlink, and includes the suffix. - # Do the meson utils, without modification. - '' - chmod u+w ./.version - echo ${version} > ../../.version - ''; - mesonFlags = [ (lib.mesonEnable "seccomp-sandboxing" stdenv.hostPlatform.isLinux) diff --git a/src/libutil-c/package.nix b/src/libutil-c/package.nix index 72f57d6f9..f26f57775 100644 --- a/src/libutil-c/package.nix +++ b/src/libutil-c/package.nix @@ -34,14 +34,6 @@ mkMesonLibrary (finalAttrs: { nix-util ]; - preConfigure = - # "Inline" .version so it's not a symlink, and includes the suffix. - # Do the meson utils, without modification. - '' - chmod u+w ./.version - echo ${version} > ../../.version - ''; - mesonFlags = [ ]; diff --git a/src/libutil-test-support/package.nix b/src/libutil-test-support/package.nix index 33cd5217d..fafd47c86 100644 --- a/src/libutil-test-support/package.nix +++ b/src/libutil-test-support/package.nix @@ -38,14 +38,6 @@ mkMesonLibrary (finalAttrs: { rapidcheck ]; - preConfigure = - # "Inline" .version so it's not a symlink, and includes the suffix. - # Do the meson utils, without modification. - '' - chmod u+w ./.version - echo ${version} > ../../.version - ''; - mesonFlags = [ ]; diff --git a/src/libutil-tests/package.nix b/src/libutil-tests/package.nix index d89c54453..c06de6894 100644 --- a/src/libutil-tests/package.nix +++ b/src/libutil-tests/package.nix @@ -45,14 +45,6 @@ mkMesonExecutable (finalAttrs: { gtest ]; - preConfigure = - # "Inline" .version so it's not a symlink, and includes the suffix. - # Do the meson utils, without modification. - '' - chmod u+w ./.version - echo ${version} > ../../.version - ''; - mesonFlags = [ ]; diff --git a/src/libutil/package.nix b/src/libutil/package.nix index 586119a6e..47dcb54a2 100644 --- a/src/libutil/package.nix +++ b/src/libutil/package.nix @@ -52,17 +52,6 @@ mkMesonLibrary (finalAttrs: { nlohmann_json ]; - preConfigure = - # "Inline" .version so it's not a symlink, and includes the suffix. - # Do the meson utils, without modification. - # - # TODO: change release process to add `pre` in `.version`, remove it - # before tagging, and restore after. - '' - chmod u+w ./.version - echo ${version} > ../../.version - ''; - mesonFlags = [ (lib.mesonEnable "cpuid" stdenv.hostPlatform.isx86_64) ]; diff --git a/src/nix/package.nix b/src/nix/package.nix index bb90be1ef..40a280437 100644 --- a/src/nix/package.nix +++ b/src/nix/package.nix @@ -91,14 +91,6 @@ mkMesonExecutable (finalAttrs: { nix-cmd ]; - preConfigure = - # Update the repo-global .version file. - # Symlink ./.version points there, but by default only workDir is writable. - '' - chmod u+w ./.version - echo ${version} > ./.version - ''; - mesonFlags = [ ]; diff --git a/tests/functional/package.nix b/tests/functional/package.nix index 74c034196..64ffa540a 100644 --- a/tests/functional/package.nix +++ b/tests/functional/package.nix @@ -75,16 +75,10 @@ mkMesonDerivation ( ]; preConfigure = - # "Inline" .version so it's not a symlink, and includes the suffix. - # Do the meson utils, without modification. - '' - chmod u+w ./.version - echo ${version} > ../../../.version - '' # TEMP hack for Meson before make is gone, where # `src/nix-functional-tests` is during the transition a symlink and # not the actual directory directory. - + '' + '' cd $(readlink -e $PWD) echo $PWD | grep tests/functional ''; From fd062585acde7178d9df9bc3121691eafe3dffa6 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 18 Feb 2025 15:55:19 +0100 Subject: [PATCH 13/28] tests: Fix installTests (cherry picked from commit 2b5365bcf73bd7584af79b1c5afc84935a2df536) --- tests/functional/misc.sh | 2 +- tests/functional/package.nix | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/functional/misc.sh b/tests/functional/misc.sh index 7d63756b7..cb4d4139f 100755 --- a/tests/functional/misc.sh +++ b/tests/functional/misc.sh @@ -11,7 +11,7 @@ source common.sh #nix-hash --help | grepQuiet base32 # Can we ask for the version number? -nix-env --version | grep "$version" +nix-env --version | grep -F "${_NIX_TEST_CLIENT_VERSION:-$version}" nix_env=$(type -P nix-env) (PATH=""; ! $nix_env --help 2>&1 ) | grepQuiet -F "The 'man' command was not found, but it is needed for 'nix-env' and some other 'nix-*' commands' help text. Perhaps you could install the 'man' command?" diff --git a/tests/functional/package.nix b/tests/functional/package.nix index 64ffa540a..a84ad1791 100644 --- a/tests/functional/package.nix +++ b/tests/functional/package.nix @@ -99,6 +99,8 @@ mkMesonDerivation ( } // lib.optionalAttrs (test-daemon != null) { + # TODO rename to _NIX_TEST_DAEMON_PACKAGE NIX_DAEMON_PACKAGE = test-daemon; + _NIX_TEST_CLIENT_VERSION = nix-cli.version; } ) From 540e8cb90809863e32a4fe588e49be388f4a67e4 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 18 Feb 2025 12:19:48 +0100 Subject: [PATCH 14/28] packaging: Move layers from dependencies to components This makes it easier to implement batch overriding for the components. (cherry picked from commit cca01407a7e661e589de165d9a873210ce91353f) --- flake.nix | 2 + packaging/components.nix | 160 ++++++++++++++++++++++++++++++++++++- packaging/dependencies.nix | 146 --------------------------------- 3 files changed, 161 insertions(+), 147 deletions(-) diff --git a/flake.nix b/flake.nix index eafb65353..7158f1ac8 100644 --- a/flake.nix +++ b/flake.nix @@ -165,6 +165,8 @@ f = import ./packaging/components.nix { inherit (final) lib; inherit officialRelease; + inherit stdenv; + pkgs = final; src = self; }; }; diff --git a/packaging/components.nix b/packaging/components.nix index 07bb209cd..5c03408dd 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -1,13 +1,22 @@ { lib, + pkgs, src, + stdenv, officialRelease, }: scope: let - inherit (scope) callPackage; + inherit (scope) + callPackage + ; + inherit (pkgs.buildPackages) + meson + ninja + pkg-config + ; baseVersion = lib.fileContents ../.version; @@ -20,6 +29,129 @@ let }_${src.shortRev or "dirty"}"; fineVersion = baseVersion + fineVersionSuffix; + + root = ../.; + + # Nixpkgs implements this by returning a subpath into the fetched Nix sources. + resolvePath = p: p; + + # Indirection for Nixpkgs to override when package.nix files are vendored + filesetToSource = lib.fileset.toSource; + + /** + Given a set of layers, create a mkDerivation-like function + */ + mkPackageBuilder = + exts: userFn: stdenv.mkDerivation (lib.extends (lib.composeManyExtensions exts) userFn); + + setVersionLayer = finalAttrs: prevAttrs: { + preConfigure = + prevAttrs.prevAttrs or "" + + + # Update the repo-global .version file. + # Symlink ./.version points there, but by default only workDir is writable. + '' + chmod u+w ./.version + echo ${finalAttrs.version} > ./.version + ''; + }; + + localSourceLayer = + finalAttrs: prevAttrs: + let + workDirPath = + # Ideally we'd pick finalAttrs.workDir, but for now `mkDerivation` has + # the requirement that everything except passthru and meta must be + # serialized by mkDerivation, which doesn't work for this. + prevAttrs.workDir; + + workDirSubpath = lib.path.removePrefix root workDirPath; + sources = + assert prevAttrs.fileset._type == "fileset"; + prevAttrs.fileset; + src = lib.fileset.toSource { + fileset = sources; + inherit root; + }; + + in + { + sourceRoot = "${src.name}/" + workDirSubpath; + inherit src; + + # Clear what `derivation` can't/shouldn't serialize; see prevAttrs.workDir. + fileset = null; + workDir = null; + }; + + mesonLayer = finalAttrs: prevAttrs: { + # NOTE: + # As of https://github.com/NixOS/nixpkgs/blob/8baf8241cea0c7b30e0b8ae73474cb3de83c1a30/pkgs/by-name/me/meson/setup-hook.sh#L26, + # `mesonBuildType` defaults to `plain` if not specified. We want our Nix-built binaries to be optimized by default. + # More on build types here: https://mesonbuild.com/Builtin-options.html#details-for-buildtype. + mesonBuildType = "release"; + # NOTE: + # Users who are debugging Nix builds are expected to set the environment variable `mesonBuildType`, per the + # guidance in https://github.com/NixOS/nix/blob/8a3fc27f1b63a08ac983ee46435a56cf49ebaf4a/doc/manual/source/development/debugging.md?plain=1#L10. + # For this reason, we don't want to refer to `finalAttrs.mesonBuildType` here, but rather use the environment variable. + preConfigure = + prevAttrs.preConfigure or "" + + + lib.optionalString + ( + !stdenv.hostPlatform.isWindows + # build failure + && !stdenv.hostPlatform.isStatic + # LTO breaks exception handling on x86-64-darwin. + && stdenv.system != "x86_64-darwin" + ) + '' + case "$mesonBuildType" in + release|minsize) appendToVar mesonFlags "-Db_lto=true" ;; + *) appendToVar mesonFlags "-Db_lto=false" ;; + esac + ''; + nativeBuildInputs = [ + meson + ninja + ] ++ prevAttrs.nativeBuildInputs or [ ]; + mesonCheckFlags = prevAttrs.mesonCheckFlags or [ ] ++ [ + "--print-errorlogs" + ]; + }; + + mesonBuildLayer = finalAttrs: prevAttrs: { + nativeBuildInputs = prevAttrs.nativeBuildInputs or [ ] ++ [ + pkg-config + ]; + separateDebugInfo = !stdenv.hostPlatform.isStatic; + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + env = + prevAttrs.env or { } + // lib.optionalAttrs ( + stdenv.isLinux + && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") + && !(stdenv.hostPlatform.useLLVM or false) + ) { LDFLAGS = "-fuse-ld=gold"; }; + }; + + mesonLibraryLayer = finalAttrs: prevAttrs: { + outputs = prevAttrs.outputs or [ "out" ] ++ [ "dev" ]; + }; + + # Work around weird `--as-needed` linker behavior with BSD, see + # https://github.com/mesonbuild/meson/issues/3593 + bsdNoLinkAsNeeded = + finalAttrs: prevAttrs: + lib.optionalAttrs stdenv.hostPlatform.isBSD { + mesonFlags = [ (lib.mesonBool "b_asneeded" false) ] ++ prevAttrs.mesonFlags or [ ]; + }; + + miscGoodPractice = finalAttrs: prevAttrs: { + strictDeps = prevAttrs.strictDeps or true; + enableParallelBuilding = true; + }; + in # This becomes the pkgs.nixComponents attribute set @@ -27,6 +159,32 @@ in version = baseVersion + versionSuffix; inherit versionSuffix; + inherit resolvePath filesetToSource; + + mkMesonDerivation = mkPackageBuilder [ + miscGoodPractice + localSourceLayer + setVersionLayer + mesonLayer + ]; + mkMesonExecutable = mkPackageBuilder [ + miscGoodPractice + bsdNoLinkAsNeeded + localSourceLayer + setVersionLayer + mesonLayer + mesonBuildLayer + ]; + mkMesonLibrary = mkPackageBuilder [ + miscGoodPractice + bsdNoLinkAsNeeded + localSourceLayer + mesonLayer + setVersionLayer + mesonBuildLayer + mesonLibraryLayer + ]; + nix-util = callPackage ../src/libutil/package.nix { }; nix-util-c = callPackage ../src/libutil-c/package.nix { }; nix-util-test-support = callPackage ../src/libutil-test-support/package.nix { }; diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 20992555c..2060672f7 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -17,8 +17,6 @@ in let inherit (pkgs) lib; - root = ../.; - stdenv = if prevStdenv.isDarwin && prevStdenv.isx86_64 then darwinStdenv else prevStdenv; # Fix the following error with the default x86_64-darwin SDK: @@ -30,125 +28,6 @@ let # all the way back to 10.6. darwinStdenv = pkgs.overrideSDK prevStdenv { darwinMinVersion = "10.13"; }; - # Nixpkgs implements this by returning a subpath into the fetched Nix sources. - resolvePath = p: p; - - # Indirection for Nixpkgs to override when package.nix files are vendored - filesetToSource = lib.fileset.toSource; - - /** - Given a set of layers, create a mkDerivation-like function - */ - mkPackageBuilder = - exts: userFn: stdenv.mkDerivation (lib.extends (lib.composeManyExtensions exts) userFn); - - setVersionLayer = finalAttrs: prevAttrs: { - preConfigure = - prevAttrs.prevAttrs or "" - + - # Update the repo-global .version file. - # Symlink ./.version points there, but by default only workDir is writable. - '' - chmod u+w ./.version - echo ${finalAttrs.version} > ./.version - ''; - }; - - localSourceLayer = - finalAttrs: prevAttrs: - let - workDirPath = - # Ideally we'd pick finalAttrs.workDir, but for now `mkDerivation` has - # the requirement that everything except passthru and meta must be - # serialized by mkDerivation, which doesn't work for this. - prevAttrs.workDir; - - workDirSubpath = lib.path.removePrefix root workDirPath; - sources = - assert prevAttrs.fileset._type == "fileset"; - prevAttrs.fileset; - src = lib.fileset.toSource { - fileset = sources; - inherit root; - }; - - in - { - sourceRoot = "${src.name}/" + workDirSubpath; - inherit src; - - # Clear what `derivation` can't/shouldn't serialize; see prevAttrs.workDir. - fileset = null; - workDir = null; - }; - - mesonLayer = finalAttrs: prevAttrs: { - # NOTE: - # As of https://github.com/NixOS/nixpkgs/blob/8baf8241cea0c7b30e0b8ae73474cb3de83c1a30/pkgs/by-name/me/meson/setup-hook.sh#L26, - # `mesonBuildType` defaults to `plain` if not specified. We want our Nix-built binaries to be optimized by default. - # More on build types here: https://mesonbuild.com/Builtin-options.html#details-for-buildtype. - mesonBuildType = "release"; - # NOTE: - # Users who are debugging Nix builds are expected to set the environment variable `mesonBuildType`, per the - # guidance in https://github.com/NixOS/nix/blob/8a3fc27f1b63a08ac983ee46435a56cf49ebaf4a/doc/manual/source/development/debugging.md?plain=1#L10. - # For this reason, we don't want to refer to `finalAttrs.mesonBuildType` here, but rather use the environment variable. - preConfigure = - prevAttrs.preConfigure or "" - + - lib.optionalString - ( - !stdenv.hostPlatform.isWindows - # build failure - && !stdenv.hostPlatform.isStatic - # LTO breaks exception handling on x86-64-darwin. - && stdenv.system != "x86_64-darwin" - ) - '' - case "$mesonBuildType" in - release|minsize) appendToVar mesonFlags "-Db_lto=true" ;; - *) appendToVar mesonFlags "-Db_lto=false" ;; - esac - ''; - nativeBuildInputs = [ - pkgs.buildPackages.meson - pkgs.buildPackages.ninja - ] ++ prevAttrs.nativeBuildInputs or [ ]; - mesonCheckFlags = prevAttrs.mesonCheckFlags or [ ] ++ [ - "--print-errorlogs" - ]; - }; - - mesonBuildLayer = finalAttrs: prevAttrs: { - nativeBuildInputs = prevAttrs.nativeBuildInputs or [ ] ++ [ - pkgs.buildPackages.pkg-config - ]; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - env = - prevAttrs.env or { } - // lib.optionalAttrs ( - stdenv.isLinux - && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") - && !(stdenv.hostPlatform.useLLVM or false) - ) { LDFLAGS = "-fuse-ld=gold"; }; - }; - - mesonLibraryLayer = finalAttrs: prevAttrs: { - outputs = prevAttrs.outputs or [ "out" ] ++ [ "dev" ]; - }; - - # Work around weird `--as-needed` linker behavior with BSD, see - # https://github.com/mesonbuild/meson/issues/3593 - bsdNoLinkAsNeeded = - finalAttrs: prevAttrs: - lib.optionalAttrs stdenv.hostPlatform.isBSD { - mesonFlags = [ (lib.mesonBool "b_asneeded" false) ] ++ prevAttrs.mesonFlags or [ ]; - }; - - miscGoodPractice = finalAttrs: prevAttrs: { - strictDeps = prevAttrs.strictDeps or true; - enableParallelBuilding = true; - }; in scope: { @@ -187,31 +66,6 @@ scope: installPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.installPhase; }); - inherit resolvePath filesetToSource; - - mkMesonDerivation = mkPackageBuilder [ - miscGoodPractice - localSourceLayer - setVersionLayer - mesonLayer - ]; - mkMesonExecutable = mkPackageBuilder [ - miscGoodPractice - bsdNoLinkAsNeeded - localSourceLayer - setVersionLayer - mesonLayer - mesonBuildLayer - ]; - mkMesonLibrary = mkPackageBuilder [ - miscGoodPractice - bsdNoLinkAsNeeded - localSourceLayer - mesonLayer - setVersionLayer - mesonBuildLayer - mesonLibraryLayer - ]; } # libgit2: Nixpkgs 24.11 has < 1.9.0 // lib.optionalAttrs (!lib.versionAtLeast pkgs.libgit2.version "1.9.0") { From 44fb6479860f831a0d34540d3b4bae335cb39a59 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 18 Feb 2025 13:35:12 +0100 Subject: [PATCH 15/28] packaging: Add overrideAllMesonComponents (cherry picked from commit f31d86284f1027edf173d92967b609de67e1bb2e) --- packaging/components.nix | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/packaging/components.nix b/packaging/components.nix index 5c03408dd..546d5829d 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -161,11 +161,28 @@ in inherit resolvePath filesetToSource; + /** + A user-provided extension function to apply to each component derivation. + */ + mesonComponentOverrides = finalAttrs: prevAttrs: { }; + + /** + Apply an extension function (i.e. overlay-shaped) to all component derivations. + */ + overrideAllMesonComponents = + f: + scope.overrideScope ( + finalScope: prevScope: { + mesonComponentOverrides = lib.composeExtensions scope.mesonComponentOverrides f; + } + ); + mkMesonDerivation = mkPackageBuilder [ miscGoodPractice localSourceLayer setVersionLayer mesonLayer + scope.mesonComponentOverrides ]; mkMesonExecutable = mkPackageBuilder [ miscGoodPractice @@ -174,6 +191,7 @@ in setVersionLayer mesonLayer mesonBuildLayer + scope.mesonComponentOverrides ]; mkMesonLibrary = mkPackageBuilder [ miscGoodPractice @@ -183,6 +201,7 @@ in setVersionLayer mesonBuildLayer mesonLibraryLayer + scope.mesonComponentOverrides ]; nix-util = callPackage ../src/libutil/package.nix { }; @@ -224,5 +243,18 @@ in nix-perl-bindings = callPackage ../src/perl/package.nix { }; - nix-everything = callPackage ../packaging/everything.nix { }; + nix-everything = callPackage ../packaging/everything.nix { } // { + # Note: no `passthru.overrideAllMesonComponents` + # This would propagate into `nix.overrideAttrs f`, but then discard + # `f` when `.overrideAllMesonComponents` is used. + # Both "methods" should be views on the same fixpoint overriding mechanism + # for that to work. For now, we intentionally don't support the broken + # two-fixpoint solution. + /** + Apply an extension function (i.e. overlay-shaped) to all component derivations, and return the nix package. + */ + overrideAllMesonComponents = f: (scope.overrideAllMesonComponents f).nix-everything; + + scope = scope; + }; } From cc3fb612496a08c35fd8daf31101e7c2279ca032 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 18 Feb 2025 15:31:55 +0100 Subject: [PATCH 16/28] packaging: Add source overriding "methods" (cherry picked from commit 48fb6fdde955afd1078ea7bb7f0e8c73e0185f8f) --- flake.nix | 21 +++++++ packaging/components.nix | 123 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 136 insertions(+), 8 deletions(-) diff --git a/flake.nix b/flake.nix index 7158f1ac8..a92fd74fc 100644 --- a/flake.nix +++ b/flake.nix @@ -237,6 +237,27 @@ LANG=C.UTF-8 ${pkgs.changelog-d}/bin/changelog-d ${./doc/manual/rl-next} >$out ''; repl-completion = nixpkgsFor.${system}.native.callPackage ./tests/repl-completion.nix { }; + + /** + Checks for our packaging expressions. + This shouldn't build anything significant; just check that things + (including derivations) are _set up_ correctly. + */ + packaging-overriding = + let + pkgs = nixpkgsFor.${system}.native; + nix = self.packages.${system}.nix; + in + assert (nix.appendPatches [ pkgs.emptyFile ]).libs.nix-util.src.patches == [ pkgs.emptyFile ]; + # If this fails, something might be wrong with how we've wired the scope, + # or something could be broken in Nixpkgs. + pkgs.testers.testEqualContents { + assertion = "trivial patch does not change source contents"; + expected = "${./.}"; + actual = + # Same for all components; nix-util is an arbitrary pick + (nix.appendPatches [ pkgs.emptyFile ]).libs.nix-util.src; + }; } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) { dockerImage = self.hydraJobs.dockerImage.${system}; diff --git a/packaging/components.nix b/packaging/components.nix index 546d5829d..de02f052b 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -32,9 +32,6 @@ let root = ../.; - # Nixpkgs implements this by returning a subpath into the fetched Nix sources. - resolvePath = p: p; - # Indirection for Nixpkgs to override when package.nix files are vendored filesetToSource = lib.fileset.toSource; @@ -84,6 +81,31 @@ let workDir = null; }; + resolveRelPath = p: lib.path.removePrefix root p; + + makeFetchedSourceLayer = + finalScope: finalAttrs: prevAttrs: + let + workDirPath = + # Ideally we'd pick finalAttrs.workDir, but for now `mkDerivation` has + # the requirement that everything except passthru and meta must be + # serialized by mkDerivation, which doesn't work for this. + prevAttrs.workDir; + + workDirSubpath = resolveRelPath workDirPath; + # sources = assert prevAttrs.fileset._type == "fileset"; prevAttrs.fileset; + # src = lib.fileset.toSource { fileset = sources; inherit root; }; + + in + { + sourceRoot = "${finalScope.patchedSrc.name}/" + workDirSubpath; + src = finalScope.patchedSrc; + + # Clear what `derivation` can't/shouldn't serialize; see prevAttrs.workDir. + fileset = null; + workDir = null; + }; + mesonLayer = finalAttrs: prevAttrs: { # NOTE: # As of https://github.com/NixOS/nixpkgs/blob/8baf8241cea0c7b30e0b8ae73474cb3de83c1a30/pkgs/by-name/me/meson/setup-hook.sh#L26, @@ -152,6 +174,17 @@ let enableParallelBuilding = true; }; + /** + Append patches to the source layer. + */ + appendPatches = + scope: patches: + scope.overrideScope ( + finalScope: prevScope: { + patches = prevScope.patches ++ patches; + } + ); + in # This becomes the pkgs.nixComponents attribute set @@ -159,13 +192,24 @@ in version = baseVersion + versionSuffix; inherit versionSuffix; - inherit resolvePath filesetToSource; + inherit filesetToSource; /** A user-provided extension function to apply to each component derivation. */ mesonComponentOverrides = finalAttrs: prevAttrs: { }; + /** + An overridable derivation layer for handling the sources. + */ + sourceLayer = localSourceLayer; + + /** + Resolve a path value to either itself or a path in the `src`, depending + whether `overrideSource` was called. + */ + resolvePath = p: p; + /** Apply an extension function (i.e. overlay-shaped) to all component derivations. */ @@ -177,9 +221,57 @@ in } ); + /** + Provide an alternate source. This allows the expressions to be vendored without copying the sources, + but it does make the build non-granular; all components will use a complete source. + + Packaging expressions will be ignored. + */ + overrideSource = + src: + scope.overrideScope ( + finalScope: prevScope: { + sourceLayer = makeFetchedSourceLayer finalScope; + /** + Unpatched source for the build of Nix. Packaging expressions will be ignored. + */ + src = src; + /** + Patches for the whole Nix source. Changes to packaging expressions will be ignored. + */ + patches = [ ]; + /** + Fetched and patched source to be used in component derivations. + */ + patchedSrc = + if finalScope.patches == [ ] then + src + else + pkgs.buildPackages.srcOnly ( + pkgs.buildPackages.stdenvNoCC.mkDerivation { + name = "${finalScope.src.name or "nix-source"}-patched"; + inherit (finalScope) src patches; + } + ); + resolvePath = p: finalScope.patchedSrc + "/${resolveRelPath p}"; + appendPatches = appendPatches finalScope; + } + ); + + /** + Append patches to be applied to the whole Nix source. + This affects all components. + + Changes to the packaging expressions will be ignored. + */ + appendPatches = + patches: + # switch to "fetched" source first, so that patches apply to the whole tree. + (scope.overrideSource "${./..}").appendPatches patches; + mkMesonDerivation = mkPackageBuilder [ miscGoodPractice - localSourceLayer + scope.sourceLayer setVersionLayer mesonLayer scope.mesonComponentOverrides @@ -187,7 +279,7 @@ in mkMesonExecutable = mkPackageBuilder [ miscGoodPractice bsdNoLinkAsNeeded - localSourceLayer + scope.sourceLayer setVersionLayer mesonLayer mesonBuildLayer @@ -196,7 +288,7 @@ in mkMesonLibrary = mkPackageBuilder [ miscGoodPractice bsdNoLinkAsNeeded - localSourceLayer + scope.sourceLayer mesonLayer setVersionLayer mesonBuildLayer @@ -255,6 +347,21 @@ in */ overrideAllMesonComponents = f: (scope.overrideAllMesonComponents f).nix-everything; - scope = scope; + /** + Append patches to be applied to the whole Nix source. + This affects all components. + + Changes to the packaging expressions will be ignored. + */ + appendPatches = ps: (scope.appendPatches ps).nix-everything; + + /** + Provide an alternate source. This allows the expressions to be vendored without copying the sources, + but it does make the build non-granular; all components will use a complete source. + + Packaging expressions will be ignored. + */ + overrideSource = src: (scope.overrideSource src).nix-everything; + }; } From a4641be4e92737fe213c166705949f570f0cc64c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 18 Feb 2025 17:13:57 +0100 Subject: [PATCH 17/28] test: Ignore packaging-overriding check on darwin for now (cherry picked from commit 03efba30dacc79e64f4107206b13231473bf2670) --- flake.nix | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/flake.nix b/flake.nix index a92fd74fc..f5c7780d5 100644 --- a/flake.nix +++ b/flake.nix @@ -249,15 +249,18 @@ nix = self.packages.${system}.nix; in assert (nix.appendPatches [ pkgs.emptyFile ]).libs.nix-util.src.patches == [ pkgs.emptyFile ]; - # If this fails, something might be wrong with how we've wired the scope, - # or something could be broken in Nixpkgs. - pkgs.testers.testEqualContents { - assertion = "trivial patch does not change source contents"; - expected = "${./.}"; - actual = - # Same for all components; nix-util is an arbitrary pick - (nix.appendPatches [ pkgs.emptyFile ]).libs.nix-util.src; - }; + if pkgs.stdenv.buildPlatform.isDarwin then + lib.warn "packaging-overriding check currently disabled because of a permissions issue on macOS" pkgs.emptyFile + else + # If this fails, something might be wrong with how we've wired the scope, + # or something could be broken in Nixpkgs. + pkgs.testers.testEqualContents { + assertion = "trivial patch does not change source contents"; + expected = "${./.}"; + actual = + # Same for all components; nix-util is an arbitrary pick + (nix.appendPatches [ pkgs.emptyFile ]).libs.nix-util.src; + }; } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) { dockerImage = self.hydraJobs.dockerImage.${system}; From bfb6f37b374eb71747419afb12de372fe02e51a7 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 18 Feb 2025 19:01:58 +0100 Subject: [PATCH 18/28] packaging: Add patch count to version (cherry picked from commit 0dbe28ad9d5f82d11bc5626310822a404f07eb60) --- packaging/components.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packaging/components.nix b/packaging/components.nix index de02f052b..c26b4b9a8 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -100,6 +100,11 @@ let { sourceRoot = "${finalScope.patchedSrc.name}/" + workDirSubpath; src = finalScope.patchedSrc; + version = + let + n = lib.count (p: p != null) finalScope.patches; + in + if n == 0 then finalAttrs.version else finalAttrs.version + "+${toString n}"; # Clear what `derivation` can't/shouldn't serialize; see prevAttrs.workDir. fileset = null; From dade40573e63528070a510d09a41980993e3724c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 18 Feb 2025 19:06:36 +0100 Subject: [PATCH 19/28] packaging: Make patch count lazier This makes `nix.version` quicker to evaluate, which should speed up package listing operations. If you want an accurate count, use `lib.optionals` in your override instead of `null` values. (cherry picked from commit d47e3c95762881e35e894ca1ba1f77c00f8b7ba3) --- packaging/components.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/components.nix b/packaging/components.nix index c26b4b9a8..bec4dc865 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -102,7 +102,7 @@ let src = finalScope.patchedSrc; version = let - n = lib.count (p: p != null) finalScope.patches; + n = lib.length finalScope.patches; in if n == 0 then finalAttrs.version else finalAttrs.version + "+${toString n}"; From 244735270a4a1d5f06edd569012cdb1dd222ec4a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 19 Feb 2025 00:10:05 +0100 Subject: [PATCH 20/28] packaging: Remove dead code ... from nixpkgs, my bad. (cherry picked from commit f0bdb652161f142999134dd7756e41a3942f57b6) --- packaging/components.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/packaging/components.nix b/packaging/components.nix index bec4dc865..b1ef38302 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -93,8 +93,6 @@ let prevAttrs.workDir; workDirSubpath = resolveRelPath workDirPath; - # sources = assert prevAttrs.fileset._type == "fileset"; prevAttrs.fileset; - # src = lib.fileset.toSource { fileset = sources; inherit root; }; in { From 8028579060d5ddb05ab1e998827341f82438ee18 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 19 Feb 2025 00:36:29 +0100 Subject: [PATCH 21/28] packaging: Restore libgit2 USE_SSH=exec ... when nixpkgs is nixos-unstable or the overlay is used. (cherry picked from commit 5488e29d2f0b77c3106fb295a9464ba2dd326d9a) --- packaging/dependencies.nix | 70 ++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 2060672f7..535b3ff37 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -29,8 +29,7 @@ let darwinStdenv = pkgs.overrideSDK prevStdenv { darwinMinVersion = "10.13"; }; in -scope: -{ +scope: { inherit stdenv; aws-sdk-cpp = @@ -66,36 +65,39 @@ scope: installPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.installPhase; }); -} -# libgit2: Nixpkgs 24.11 has < 1.9.0 -// lib.optionalAttrs (!lib.versionAtLeast pkgs.libgit2.version "1.9.0") { - libgit2 = pkgs.libgit2.overrideAttrs (attrs: { - cmakeFlags = attrs.cmakeFlags or [ ] ++ [ "-DUSE_SSH=exec" ]; - nativeBuildInputs = - attrs.nativeBuildInputs or [ ] - # gitMinimal does not build on Windows. See packbuilder patch. - ++ lib.optionals (!stdenv.hostPlatform.isWindows) [ - # Needed for `git apply`; see `prePatch` - pkgs.buildPackages.gitMinimal - ]; - # Only `git apply` can handle git binary patches - prePatch = - attrs.prePatch or "" - + lib.optionalString (!stdenv.hostPlatform.isWindows) '' - patch() { - git apply - } - ''; - patches = - attrs.patches or [ ] - ++ [ - ./patches/libgit2-mempack-thin-packfile.patch - ] - # gitMinimal does not build on Windows, but fortunately this patch only - # impacts interruptibility - ++ lib.optionals (!stdenv.hostPlatform.isWindows) [ - # binary patch; see `prePatch` - ./patches/libgit2-packbuilder-callback-interruptible.patch - ]; - }); + libgit2 = pkgs.libgit2.overrideAttrs ( + attrs: + { + cmakeFlags = attrs.cmakeFlags or [ ] ++ [ "-DUSE_SSH=exec" ]; + } + # libgit2: Nixpkgs 24.11 has < 1.9.0, which needs our patches + // lib.optionalAttrs (!lib.versionAtLeast pkgs.libgit2.version "1.9.0") { + nativeBuildInputs = + attrs.nativeBuildInputs or [ ] + # gitMinimal does not build on Windows. See packbuilder patch. + ++ lib.optionals (!stdenv.hostPlatform.isWindows) [ + # Needed for `git apply`; see `prePatch` + pkgs.buildPackages.gitMinimal + ]; + # Only `git apply` can handle git binary patches + prePatch = + attrs.prePatch or "" + + lib.optionalString (!stdenv.hostPlatform.isWindows) '' + patch() { + git apply + } + ''; + patches = + attrs.patches or [ ] + ++ [ + ./patches/libgit2-mempack-thin-packfile.patch + ] + # gitMinimal does not build on Windows, but fortunately this patch only + # impacts interruptibility + ++ lib.optionals (!stdenv.hostPlatform.isWindows) [ + # binary patch; see `prePatch` + ./patches/libgit2-packbuilder-callback-interruptible.patch + ]; + } + ); } From cd149b56c76f886bc0b08d1d6da3f4e5f631d591 Mon Sep 17 00:00:00 2001 From: MaxHearnden Date: Sun, 9 Feb 2025 20:53:58 +0000 Subject: [PATCH 22/28] Set FD_CLOEXEC on sockets created by curl Curl creates sockets without setting FD_CLOEXEC/SOCK_CLOEXEC, this can cause connections to remain open forever when using commands like `nix shell` This change sets the FD_CLOEXEC flag using a CURLOPT_SOCKOPTFUNCTION callback. (cherry picked from commit 12d25272764bf2f9f828d5d129ec26622baf75eb) --- doc/manual/rl-next/curl-cloexec.md | 10 ++++++++++ src/libstore/filetransfer.cc | 12 ++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 doc/manual/rl-next/curl-cloexec.md diff --git a/doc/manual/rl-next/curl-cloexec.md b/doc/manual/rl-next/curl-cloexec.md new file mode 100644 index 000000000..2fcdfb0d1 --- /dev/null +++ b/doc/manual/rl-next/curl-cloexec.md @@ -0,0 +1,10 @@ +--- +synopsis: Set FD_CLOEXEC on sockets created by curl +issues: [] +prs: [12439] +--- + + +Curl creates sockets without setting FD_CLOEXEC/SOCK_CLOEXEC, this can cause connections to remain open forever when using commands like `nix shell` + +This change sets the FD_CLOEXEC flag using a CURLOPT_SOCKOPTFUNCTION callback. diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 8439cc39c..932e1d756 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -300,6 +300,14 @@ struct curlFileTransfer : public FileTransfer return ((TransferItem *) userp)->readCallback(buffer, size, nitems); } + #if !defined(_WIN32) && LIBCURL_VERSION_NUM >= 0x071000 + static int cloexec_callback(void *, curl_socket_t curlfd, curlsocktype purpose) { + unix::closeOnExec(curlfd); + vomit("cloexec set for fd %i", curlfd); + return CURL_SOCKOPT_OK; + } + #endif + void init() { if (!req) req = curl_easy_init(); @@ -359,6 +367,10 @@ struct curlFileTransfer : public FileTransfer curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0); } + #if !defined(_WIN32) && LIBCURL_VERSION_NUM >= 0x071000 + curl_easy_setopt(req, CURLOPT_SOCKOPTFUNCTION, cloexec_callback); + #endif + curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransferSettings.connectTimeout.get()); curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L); From 65583ca79b9945ef588c03886209243b1d6cc1cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Sat, 1 Feb 2025 16:49:31 +0100 Subject: [PATCH 23/28] Only try to chmod /nix/var/nix/profiles/per-user when necessary Co-authored-by: Eelco Dolstra (cherry picked from commit dcbf4dcc09805ea3d1f22a7f8a55f313473338ed) --- src/libstore/local-store.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index f708bd1b0..9a7a941b6 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -136,7 +136,12 @@ LocalStore::LocalStore( for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { createDirs(perUserDir); if (!readOnly) { - if (chmod(perUserDir.c_str(), 0755) == -1) + auto st = lstat(perUserDir); + + // Skip chmod call if the directory already has the correct permissions (0755). + // This is to avoid failing when the executing user lacks permissions to change the directory's permissions + // even if it would be no-op. + if ((st.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO)) != 0755 && chmod(perUserDir.c_str(), 0755) == -1) throw SysError("could not set permissions on '%s' to 755", perUserDir); } } From cd1935468d7e6a38c9dbb7212c87a5122afc47f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20M=C3=B6ller?= Date: Fri, 21 Feb 2025 11:49:00 +0100 Subject: [PATCH 24/28] Fix perl store bindings When #9863 converted the `Nix::Store` free functions into member functions, the implicit `this` argument was not accounted for when iterating over the variable number of arguments in some functions. (cherry picked from commit 5cf9e18167b86f39864e39e5fe129e5f6c1a15e0) --- src/perl/lib/Nix/Store.xs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/perl/lib/Nix/Store.xs b/src/perl/lib/Nix/Store.xs index 172c3500d..cfc3ac034 100644 --- a/src/perl/lib/Nix/Store.xs +++ b/src/perl/lib/Nix/Store.xs @@ -194,7 +194,7 @@ StoreWrapper::computeFSClosure(int flipDirection, int includeOutputs, ...) PPCODE: try { StorePathSet paths; - for (int n = 2; n < items; ++n) + for (int n = 3; n < items; ++n) THIS->store->computeFSClosure(THIS->store->parseStorePath(SvPV_nolen(ST(n))), paths, flipDirection, includeOutputs); for (auto & i : paths) XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(i).c_str(), 0))); @@ -208,7 +208,7 @@ StoreWrapper::topoSortPaths(...) PPCODE: try { StorePathSet paths; - for (int n = 0; n < items; ++n) paths.insert(THIS->store->parseStorePath(SvPV_nolen(ST(n)))); + for (int n = 1; n < items; ++n) paths.insert(THIS->store->parseStorePath(SvPV_nolen(ST(n)))); auto sorted = THIS->store->topoSortPaths(paths); for (auto & i : sorted) XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(i).c_str(), 0))); @@ -234,7 +234,7 @@ StoreWrapper::exportPaths(int fd, ...) PPCODE: try { StorePathSet paths; - for (int n = 1; n < items; ++n) paths.insert(THIS->store->parseStorePath(SvPV_nolen(ST(n)))); + for (int n = 2; n < items; ++n) paths.insert(THIS->store->parseStorePath(SvPV_nolen(ST(n)))); FdSink sink(fd); THIS->store->exportPaths(paths, sink); } catch (Error & e) { From c69d5af1053ed36b3d20c4b2cd84c843ed6f49d2 Mon Sep 17 00:00:00 2001 From: Ivan Trubach Date: Tue, 18 Feb 2025 22:09:05 +0300 Subject: [PATCH 25/28] libstore: fix expected bytes in progress bar (cherry picked from commit eb73bfcf73bae4d6e4d37a4882231cd9cb7fbddd) --- src/libstore/store-api.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 236622eae..fc3fbcc0f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -230,18 +230,22 @@ void Store::addMultipleToStore( { std::atomic nrDone{0}; std::atomic nrFailed{0}; - std::atomic bytesExpected{0}; std::atomic nrRunning{0}; using PathWithInfo = std::pair>; + uint64_t bytesExpected = 0; + std::map infosMap; StorePathSet storePathsToAdd; for (auto & thingToAdd : pathsToCopy) { + bytesExpected += thingToAdd.first.narSize; infosMap.insert_or_assign(thingToAdd.first.path, &thingToAdd); storePathsToAdd.insert(thingToAdd.first.path); } + act.setExpected(actCopyPath, bytesExpected); + auto showProgress = [&, nrTotal = pathsToCopy.size()]() { act.progress(nrDone, nrTotal, nrRunning, nrFailed); }; @@ -259,9 +263,6 @@ void Store::addMultipleToStore( return StorePathSet(); } - bytesExpected += info.narSize; - act.setExpected(actCopyPath, bytesExpected); - return info.references; }, From d670380bd9f63d83655a0bde71b285103735b072 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Feb 2025 15:30:30 +0100 Subject: [PATCH 26/28] nix flake archive: Skip relative path inputs Fixes #12438. (cherry picked from commit b4dfeafed5e2b0d8d6fd90bef4d3bed24caa4734) --- src/nix/flake.cc | 4 +++- tests/functional/flakes/relative-paths.sh | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 4d5cad1a8..87eaafd15 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1088,12 +1088,14 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun nlohmann::json jsonObj2 = json ? json::object() : nlohmann::json(nullptr); for (auto & [inputName, input] : node.inputs) { if (auto inputNode = std::get_if<0>(&input)) { + if ((*inputNode)->lockedRef.input.isRelative()) + continue; auto storePath = dryRun ? (*inputNode)->lockedRef.input.computeStorePath(*store) : (*inputNode)->lockedRef.input.fetchToStore(store).first; if (json) { - auto& jsonObj3 = jsonObj2[inputName]; + auto & jsonObj3 = jsonObj2[inputName]; jsonObj3["path"] = store->printStorePath(storePath); sources.insert(std::move(storePath)); jsonObj3["inputs"] = traverse(**inputNode); diff --git a/tests/functional/flakes/relative-paths.sh b/tests/functional/flakes/relative-paths.sh index 9b93da9c1..ac4b07eb2 100644 --- a/tests/functional/flakes/relative-paths.sh +++ b/tests/functional/flakes/relative-paths.sh @@ -76,6 +76,9 @@ if ! isTestOnNixOS; then fi (! grep narHash "$subflake2/flake.lock") +# Test `nix flake archive` with relative path flakes. +nix flake archive --json "$rootFlake" + # Test circular relative path flakes. FIXME: doesn't work at the moment. if false; then From 9e87a583142e0dccb04588445d7a807392385903 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 24 Feb 2025 16:44:12 +0100 Subject: [PATCH 27/28] packaging: Use correct stdenv for x86_64-darwin (cherry picked from commit 0772c2e3abc269f5e3aa8dd1fa055fba523d60ee) --- flake.nix | 1 - packaging/components.nix | 10 +++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index f5c7780d5..0c0ddfa47 100644 --- a/flake.nix +++ b/flake.nix @@ -165,7 +165,6 @@ f = import ./packaging/components.nix { inherit (final) lib; inherit officialRelease; - inherit stdenv; pkgs = final; src = self; }; diff --git a/packaging/components.nix b/packaging/components.nix index b1ef38302..9da864887 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -2,7 +2,6 @@ lib, pkgs, src, - stdenv, officialRelease, }: @@ -12,6 +11,15 @@ let inherit (scope) callPackage ; + inherit + (scope.callPackage ( + { stdenv }: + { + inherit stdenv; + } + ) { }) + stdenv + ; inherit (pkgs.buildPackages) meson ninja From 605b2371f96c020516ee3e9596ff6df3db0f0be5 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Mon, 24 Feb 2025 21:30:35 +0000 Subject: [PATCH 28/28] windows: fix compilation after recent changes (backport #12495) (#12561) * windows: fix compilation after recent changes Specifically last few week's merges involving legacy SSH options and dynamic derivations. (cherry picked from commit e0617d25453760e2f5817ece317914eee9330768) # Conflicts: # src/libstore/build/derivation-creation-and-realisation-goal.hh * Remove unneeded * Remove unneeded --------- Co-authored-by: Brian McKenna Co-authored-by: Eelco Dolstra --- src/libstore/legacy-ssh-store.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 3849f088d..480f41059 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -367,7 +367,12 @@ unsigned int LegacySSHStore::getProtocol() pid_t LegacySSHStore::getConnectionPid() { auto conn(connections->get()); +#ifndef _WIN32 return conn->sshConn->sshPid; +#else + // TODO: Implement + return 0; +#endif }