From b2be6fed8600ee48c05cc9643c101d5eab4a5727 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 23 Apr 2024 16:00:52 +0200 Subject: [PATCH 01/68] Improve support for subflakes Subflakes are flakes in the same tree, accessed in flake inputs via relative paths (e.g. `inputs.foo.url = "path:./subdir"`). Previously these didn't work very well because they would be separately copied to the store, which is inefficient and makes references to parent directories tricky or impossible. Furthermore, they had their own NAR hash in the lock file, which is superfluous since the parent is already locked. Now subflakes are accessed via the accessor of the calling flake. This avoids the unnecessary copy and makes it possible for subflakes to depend on flakes in a parent directory (so long as they're in the same tree). Lock file nodes for relative flake inputs now have a new `parent` field: { "locked": { "path": "./subdir", "type": "path" }, "original": { "path": "./subdir", "type": "path" }, "parent": [ "foo", "bar" ] } which denotes that `./subdir` is to be interpreted relative to the directory of the `bar` input of the `foo` input of the root flake. Extracted from the lazy-trees branch. --- src/libexpr/flake/call-flake.nix | 7 + src/libexpr/flake/flake.cc | 160 +++++++++++++++------- src/libexpr/flake/flakeref.cc | 68 +++++---- src/libexpr/flake/flakeref.hh | 6 +- src/libexpr/flake/lockfile.cc | 9 +- src/libexpr/flake/lockfile.hh | 12 +- src/libexpr/primops/fetchTree.cc | 5 +- src/libfetchers/fetchers.cc | 6 + src/libfetchers/fetchers.hh | 14 +- src/libfetchers/path.cc | 25 +--- src/nix/flake.md | 25 +++- tests/functional/flakes/follow-paths.sh | 25 +++- tests/functional/flakes/relative-paths.sh | 89 ++++++++++++ tests/functional/local.mk | 1 + 14 files changed, 332 insertions(+), 120 deletions(-) create mode 100644 tests/functional/flakes/relative-paths.sh diff --git a/src/libexpr/flake/call-flake.nix b/src/libexpr/flake/call-flake.nix index a411564df..43ecb7f15 100644 --- a/src/libexpr/flake/call-flake.nix +++ b/src/libexpr/flake/call-flake.nix @@ -38,10 +38,17 @@ let (key: node: let + parentNode = allNodes.${getInputByPath lockFile.root node.parent}; + sourceInfo = if overrides ? ${key} then overrides.${key}.sourceInfo + else if node.locked.type == "path" && builtins.substring 0 1 node.locked.path != "/" + then + parentNode.sourceInfo // { + outPath = parentNode.sourceInfo.outPath + ("/" + node.locked.path); + } else # FIXME: remove obsolete node.info. fetchTree (node.info or {} // removeAttrs node.locked ["dir"]); diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 3af9ef14e..d4cabe68f 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -93,12 +93,17 @@ static void expectType(EvalState & state, ValueType type, } static std::map parseFlakeInputs( - EvalState & state, Value * value, const PosIdx pos, - const std::optional & baseDir, InputPath lockRootPath); + EvalState & state, + Value * value, + const PosIdx pos, + InputPath lockRootPath); -static FlakeInput parseFlakeInput(EvalState & state, - const std::string & inputName, Value * value, const PosIdx pos, - const std::optional & baseDir, InputPath lockRootPath) +static FlakeInput parseFlakeInput( + EvalState & state, + const std::string & inputName, + Value * value, + const PosIdx pos, + InputPath lockRootPath) { expectType(state, nAttrs, *value, pos); @@ -122,7 +127,7 @@ static FlakeInput parseFlakeInput(EvalState & state, expectType(state, nBool, *attr.value, attr.pos); input.isFlake = attr.value->boolean(); } else if (attr.name == sInputs) { - input.overrides = parseFlakeInputs(state, attr.value, attr.pos, baseDir, lockRootPath); + input.overrides = parseFlakeInputs(state, attr.value, attr.pos, lockRootPath); } else if (attr.name == sFollows) { expectType(state, nString, *attr.value, attr.pos); auto follows(parseInputPath(attr.value->c_str())); @@ -173,7 +178,7 @@ static FlakeInput parseFlakeInput(EvalState & state, if (!attrs.empty()) throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, state.positions[pos]); if (url) - input.ref = parseFlakeRef(*url, baseDir, true, input.isFlake); + input.ref = parseFlakeRef(*url, {}, true, input.isFlake, true); } if (!input.follows && !input.ref) @@ -183,8 +188,10 @@ static FlakeInput parseFlakeInput(EvalState & state, } static std::map parseFlakeInputs( - EvalState & state, Value * value, const PosIdx pos, - const std::optional & baseDir, InputPath lockRootPath) + EvalState & state, + Value * value, + const PosIdx pos, + InputPath lockRootPath) { std::map inputs; @@ -196,7 +203,6 @@ static std::map parseFlakeInputs( state.symbols[inputAttr.name], inputAttr.value, inputAttr.pos, - baseDir, lockRootPath)); } @@ -232,7 +238,7 @@ static Flake readFlake( auto sInputs = state.symbols.create("inputs"); if (auto inputs = vInfo.attrs()->get(sInputs)) - flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, flakePath.parent().path.abs(), lockRootPath); // FIXME + flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, lockRootPath); auto sOutputs = state.symbols.create("outputs"); @@ -366,13 +372,29 @@ LockedFlake lockFlake( debug("old lock file: %s", oldLockFile); - std::map overrides; + struct Override + { + FlakeInput input; + SourcePath sourcePath; + std::optional parentPath; // FIXME: rename to inputPathPrefix? + }; + + std::map overrides; std::set explicitCliOverrides; std::set overridesUsed, updatesUsed; std::map, SourcePath> nodePaths; for (auto & i : lockFlags.inputOverrides) { - overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second }); + overrides.emplace( + i.first, + Override { + .input = FlakeInput { .ref = i.second }, + /* Note: any relative overrides + (e.g. `--override-input B/C "path:./foo/bar"`) + are interpreted relative to the top-level + flake. */ + .sourcePath = flake.path, + }); explicitCliOverrides.insert(i.first); } @@ -386,7 +408,7 @@ LockedFlake lockFlake( const InputPath & inputPathPrefix, std::shared_ptr oldNode, const InputPath & lockRootPath, - const Path & parentPath, + const SourcePath & sourcePath, bool trustLock)> computeLocks; @@ -402,7 +424,8 @@ LockedFlake lockFlake( copied. */ std::shared_ptr oldNode, const InputPath & lockRootPath, - const Path & parentPath, + /* The source path of this node's flake. */ + const SourcePath & sourcePath, bool trustLock) { debug("computing lock file node '%s'", printInputPath(inputPathPrefix)); @@ -414,7 +437,12 @@ LockedFlake lockFlake( auto inputPath(inputPathPrefix); inputPath.push_back(id); inputPath.push_back(idOverride); - overrides.insert_or_assign(inputPath, inputOverride); + overrides.emplace(inputPath, + Override { + .input = inputOverride, + .sourcePath = sourcePath, + .parentPath = inputPathPrefix // FIXME: should this be inputPath? + }); } } @@ -446,13 +474,18 @@ LockedFlake lockFlake( auto i = overrides.find(inputPath); bool hasOverride = i != overrides.end(); bool hasCliOverride = explicitCliOverrides.contains(inputPath); - if (hasOverride) { + if (hasOverride) overridesUsed.insert(inputPath); - // Respect the “flakeness” of the input even if we - // override it - i->second.isFlake = input2.isFlake; - } - auto & input = hasOverride ? i->second : input2; + auto input = hasOverride ? i->second.input : input2; + + /* Resolve relative 'path:' inputs relative to + the source path of the overrider. */ + auto overridenSourcePath = hasOverride ? i->second.sourcePath : sourcePath; + + /* Respect the "flakeness" of the input even if we + override it. */ + if (hasOverride) + input.isFlake = input2.isFlake; /* Resolve 'follows' later (since it may refer to an input path we haven't processed yet. */ @@ -468,6 +501,33 @@ LockedFlake lockFlake( assert(input.ref); + auto overridenParentPath = + input.ref->input.isRelative() + ? std::optional(hasOverride ? i->second.parentPath : inputPathPrefix) + : std::nullopt; + + auto resolveRelativePath = [&]() -> std::optional + { + if (auto relativePath = input.ref->input.isRelative()) { + return SourcePath { + overridenSourcePath.accessor, + CanonPath(*relativePath, overridenSourcePath.path.parent().value()) + }; + } else + return std::nullopt; + }; + + /* Get the input flake, resolve 'path:./...' + flakerefs relative to the parent flake. */ + auto getInputFlake = [&]() + { + if (auto resolvedPath = resolveRelativePath()) { + return readFlake(state, *input.ref, *input.ref, *input.ref, *resolvedPath, inputPath); + } else { + return getFlake(state, *input.ref, useRegistries, flakeCache, inputPath); + } + }; + /* Do we have an entry in the existing lock file? And the input is not in updateInputs? */ std::shared_ptr oldLock; @@ -481,6 +541,7 @@ LockedFlake lockFlake( if (oldLock && oldLock->originalRef == *input.ref + && oldLock->parentPath == overridenParentPath && !hasCliOverride) { debug("keeping existing input '%s'", inputPathS); @@ -489,7 +550,10 @@ LockedFlake lockFlake( didn't change and there is no override from a higher level flake. */ auto childNode = make_ref( - oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake); + oldLock->lockedRef, + oldLock->originalRef, + oldLock->isFlake, + oldLock->parentPath); node->inputs.insert_or_assign(id, childNode); @@ -541,11 +605,15 @@ LockedFlake lockFlake( } if (mustRefetch) { - auto inputFlake = getFlake(state, oldLock->lockedRef, false, flakeCache, inputPath); + auto inputFlake = getInputFlake(); nodePaths.emplace(childNode, inputFlake.path.parent()); - computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, lockRootPath, parentPath, false); + computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, lockRootPath, inputFlake.path, false); } else { - computeLocks(fakeInputs, childNode, inputPath, oldLock, lockRootPath, parentPath, true); + // FIXME: sourcePath is wrong here, we + // should pass a lambda that lazily + // fetches the parent flake if needed + // (i.e. getInputFlake()). + computeLocks(fakeInputs, childNode, inputPath, oldLock, lockRootPath, sourcePath, true); } } else { @@ -553,7 +621,9 @@ LockedFlake lockFlake( this input. */ debug("creating new input '%s'", inputPathS); - if (!lockFlags.allowUnlocked && !input.ref->input.isLocked()) + if (!lockFlags.allowUnlocked + && !input.ref->input.isLocked() + && !input.ref->input.isRelative()) throw Error("cannot update unlocked flake input '%s' in pure mode", inputPathS); /* Note: in case of an --override-input, we use @@ -566,17 +636,9 @@ LockedFlake lockFlake( auto ref = (input2.ref && explicitCliOverrides.contains(inputPath)) ? *input2.ref : *input.ref; if (input.isFlake) { - Path localPath = parentPath; - FlakeRef localRef = *input.ref; + auto inputFlake = getInputFlake(); - // If this input is a path, recurse it down. - // This allows us to resolve path inputs relative to the current flake. - if (localRef.input.getType() == "path") - localPath = absPath(*input.ref->input.getSourcePath(), parentPath); - - auto inputFlake = getFlake(state, localRef, useRegistries, flakeCache, inputPath); - - auto childNode = make_ref(inputFlake.lockedRef, ref); + auto childNode = make_ref(inputFlake.lockedRef, ref, true, overridenParentPath); node->inputs.insert_or_assign(id, childNode); @@ -598,17 +660,26 @@ LockedFlake lockFlake( ? std::dynamic_pointer_cast(oldLock) : readLockFile(inputFlake.lockFilePath()).root.get_ptr(), oldLock ? lockRootPath : inputPath, - localPath, + inputFlake.path, false); } else { - auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree( - state, *input.ref, useRegistries, flakeCache); + auto [path, lockedRef] = [&]() -> std::tuple + { + // Handle non-flake 'path:./...' inputs. + if (auto resolvedPath = resolveRelativePath()) { + return {*resolvedPath, *input.ref}; + } else { + auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree( + state, *input.ref, useRegistries, flakeCache); + return {state.rootPath(state.store->toRealPath(storePath)), lockedRef}; + } + }(); - auto childNode = make_ref(lockedRef, ref, false); + auto childNode = make_ref(lockedRef, ref, false, overridenParentPath); - nodePaths.emplace(childNode, state.rootPath(state.store->toRealPath(storePath))); + nodePaths.emplace(childNode, path); node->inputs.insert_or_assign(id, childNode); } @@ -621,9 +692,6 @@ LockedFlake lockFlake( } }; - // Bring in the current ref for relative path resolution if we have it - auto parentPath = flake.path.parent().path.abs(); - nodePaths.emplace(newLockFile.root, flake.path.parent()); computeLocks( @@ -632,7 +700,7 @@ LockedFlake lockFlake( {}, lockFlags.recreateLockFile ? nullptr : oldLockFile.root.get_ptr(), {}, - parentPath, + flake.path, false); for (auto & i : lockFlags.inputOverrides) diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 6e4aad64d..bac237a2a 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -51,9 +51,10 @@ FlakeRef parseFlakeRef( const std::string & url, const std::optional & baseDir, bool allowMissing, - bool isFlake) + bool isFlake, + bool allowRelative) { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing, isFlake); + auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing, isFlake, allowRelative); if (fragment != "") throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url); return flakeRef; @@ -69,11 +70,25 @@ std::optional maybeParseFlakeRef( } } +static std::pair fromParsedURL( + ParsedURL && parsedURL, + bool isFlake) +{ + auto dir = getOr(parsedURL.query, "dir", ""); + parsedURL.query.erase("dir"); + + std::string fragment; + std::swap(fragment, parsedURL.fragment); + + return std::make_pair(FlakeRef(fetchers::Input::fromURL(parsedURL, isFlake), dir), fragment); +} + std::pair parsePathFlakeRefWithFragment( const std::string & url, const std::optional & baseDir, bool allowMissing, - bool isFlake) + bool isFlake, + bool allowRelative) { std::string path = url; std::string fragment = ""; @@ -90,7 +105,7 @@ std::pair parsePathFlakeRefWithFragment( fragment = percentDecode(url.substr(fragmentStart+1)); } if (pathEnd != std::string::npos && fragmentStart != std::string::npos) { - query = decodeQuery(url.substr(pathEnd+1, fragmentStart-pathEnd-1)); + query = decodeQuery(url.substr(pathEnd + 1, fragmentStart - pathEnd - 1)); } if (baseDir) { @@ -154,6 +169,7 @@ std::pair parsePathFlakeRefWithFragment( .authority = "", .path = flakeRoot, .query = query, + .fragment = fragment, }; if (subdir != "") { @@ -165,9 +181,7 @@ std::pair parsePathFlakeRefWithFragment( if (pathExists(flakeRoot + "/.git/shallow")) parsedURL.query.insert_or_assign("shallow", "1"); - return std::make_pair( - FlakeRef(fetchers::Input::fromURL(parsedURL), getOr(parsedURL.query, "dir", "")), - fragment); + return fromParsedURL(std::move(parsedURL), isFlake); } subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir); @@ -176,25 +190,30 @@ std::pair parsePathFlakeRefWithFragment( } } else { - if (!hasPrefix(path, "/")) + if (!allowRelative && !hasPrefix(path, "/")) throw BadURL("flake reference '%s' is not an absolute path", url); - path = canonPath(path + "/" + getOr(query, "dir", "")); } fetchers::Attrs attrs; attrs.insert_or_assign("type", "path"); attrs.insert_or_assign("path", path); - return std::make_pair(FlakeRef(fetchers::Input::fromAttrs(std::move(attrs)), ""), fragment); -}; - + return fromParsedURL({ + .url = path, + .base = path, + .scheme = "path", + .authority = "", + .path = path, + .query = query, + .fragment = fragment + }, isFlake); +} /* Check if 'url' is a flake ID. This is an abbreviated syntax for 'flake:?ref=&rev='. */ std::optional> parseFlakeIdRef( const std::string & url, - bool isFlake -) + bool isFlake) { std::smatch match; @@ -223,32 +242,21 @@ std::optional> parseFlakeIdRef( std::optional> parseURLFlakeRef( const std::string & url, const std::optional & baseDir, - bool isFlake -) + bool isFlake) { - ParsedURL parsedURL; try { - parsedURL = parseURL(url); + return fromParsedURL(parseURL(url), isFlake); } catch (BadURL &) { return std::nullopt; } - - std::string fragment; - std::swap(fragment, parsedURL.fragment); - - auto input = fetchers::Input::fromURL(parsedURL, isFlake); - input.parent = baseDir; - - return std::make_pair( - FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")), - fragment); } std::pair parseFlakeRefWithFragment( const std::string & url, const std::optional & baseDir, bool allowMissing, - bool isFlake) + bool isFlake, + bool allowRelative) { using namespace fetchers; @@ -259,7 +267,7 @@ std::pair parseFlakeRefWithFragment( } else if (auto res = parseURLFlakeRef(url, baseDir, isFlake)) { return *res; } else { - return parsePathFlakeRefWithFragment(url, baseDir, allowMissing, isFlake); + return parsePathFlakeRefWithFragment(url, baseDir, allowMissing, isFlake, allowRelative); } } diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index 04c812ed0..ea6c4e4d7 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -75,7 +75,8 @@ FlakeRef parseFlakeRef( const std::string & url, const std::optional & baseDir = {}, bool allowMissing = false, - bool isFlake = true); + bool isFlake = true, + bool allowRelative = false); /** * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) @@ -90,7 +91,8 @@ std::pair parseFlakeRefWithFragment( const std::string & url, const std::optional & baseDir = {}, bool allowMissing = false, - bool isFlake = true); + bool isFlake = true, + bool allowRelative = false); /** * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index d252214dd..2884ca262 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -36,8 +36,9 @@ LockedNode::LockedNode(const nlohmann::json & json) : lockedRef(getFlakeRef(json, "locked", "info")) // FIXME: remove "info" , originalRef(getFlakeRef(json, "original", nullptr)) , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) + , parentPath(json.find("parent") != json.end() ? (std::optional) json["parent"] : std::nullopt) { - if (!lockedRef.input.isLocked()) + if (!lockedRef.input.isLocked() && !lockedRef.input.isRelative()) throw Error("lock file contains unlocked input '%s'", fetchers::attrsToJSON(lockedRef.input.toAttrs())); } @@ -184,6 +185,8 @@ std::pair LockFile::toJSON() const n["locked"] = fetchers::attrsToJSON(lockedNode->lockedRef.toAttrs()); if (!lockedNode->isFlake) n["flake"] = false; + if (lockedNode->parentPath) + n["parent"] = *lockedNode->parentPath; } nodes[key] = std::move(n); @@ -230,7 +233,9 @@ std::optional LockFile::isUnlocked() const for (auto & i : nodes) { if (i == ref(root)) continue; auto node = i.dynamic_pointer_cast(); - if (node && !node->lockedRef.input.isLocked()) + if (node + && !node->lockedRef.input.isLocked() + && !node->lockedRef.input.isRelative()) return node->lockedRef; } diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index 7e62e6d09..aad805baf 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -38,11 +38,19 @@ struct LockedNode : Node FlakeRef lockedRef, originalRef; bool isFlake = true; + /* The node relative to which relative source paths + (e.g. 'path:../foo') are interpreted. */ + std::optional parentPath; + LockedNode( const FlakeRef & lockedRef, const FlakeRef & originalRef, - bool isFlake = true) - : lockedRef(lockedRef), originalRef(originalRef), isFlake(isFlake) + bool isFlake = true, + std::optional parentPath = {}) + : lockedRef(lockedRef) + , originalRef(originalRef) + , isFlake(isFlake) + , parentPath(parentPath) { } LockedNode(const nlohmann::json & json); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index e27f30512..1c5cd4ec2 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -31,9 +31,8 @@ void emitTreeAttrs( // FIXME: support arbitrary input attributes. - auto narHash = input.getNarHash(); - assert(narHash); - attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true)); + if (auto narHash = input.getNarHash()) + attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true)); if (input.getType() == "git") attrs.alloc("submodules").mkBool( diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 73923907c..73cb4fea3 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -141,6 +141,12 @@ bool Input::isLocked() const return scheme && scheme->isLocked(*this); } +std::optional Input::isRelative() const +{ + assert(scheme); + return scheme->isRelative(*this); +} + Attrs Input::toAttrs() const { return attrs; diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 551be9a1f..66cac7064 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -31,11 +31,6 @@ struct Input std::shared_ptr scheme; // note: can be null Attrs attrs; - /** - * path of the parent of this input, used for relative path resolution - */ - std::optional parent; - public: /** * Create an `Input` from a URL. @@ -73,6 +68,12 @@ public: */ bool isLocked() const; + /** + * Only for relative path flakes, i.e. 'path:./foo', returns the + * relative path, i.e. './foo'. + */ + std::optional isRelative() const; + bool operator ==(const Input & other) const; bool contains(const Input & other) const; @@ -220,6 +221,9 @@ struct InputScheme * if there is a mismatch. */ virtual void checkLocks(const Input & specified, const Input & final) const; + + virtual std::optional isRelative(const Input & input) const + { return std::nullopt; } }; void registerInputScheme(std::shared_ptr && fetcher); diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 68958d559..62500bbcb 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -116,31 +116,14 @@ struct PathInputScheme : InputScheme std::pair, Input> getAccessor(ref store, const Input & _input) const override { Input input(_input); - std::string absPath; auto path = getStrAttr(input.attrs, "path"); - if (path[0] != '/') { - if (!input.parent) - throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string()); + auto absPath = getAbsPath(input); - auto parent = canonPath(*input.parent); - - // the path isn't relative, prefix it - absPath = nix::absPath(path, parent); - - // for security, ensure that if the parent is a store path, it's inside it - if (store->isInStore(parent)) { - auto storePath = store->printStorePath(store->toStorePath(parent).first); - if (!isDirOrInDir(absPath, storePath)) - throw BadStorePath("relative path '%s' points outside of its parent's store path '%s'", path, storePath); - } - } else - absPath = path; - - Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s'", absPath)); + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s' to the store", absPath)); // FIXME: check whether access to 'path' is allowed. - auto storePath = store->maybeParseStorePath(absPath); + auto storePath = store->maybeParseStorePath(absPath.abs()); if (storePath) store->addTempRoot(*storePath); @@ -149,7 +132,7 @@ struct PathInputScheme : InputScheme if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) { // FIXME: try to substitute storePath. auto src = sinkToSource([&](Sink & sink) { - mtime = dumpPathAndGetMtime(absPath, sink, defaultPathFilter); + mtime = dumpPathAndGetMtime(absPath.abs(), sink, defaultPathFilter); }); storePath = store->addToStoreFromDump(*src, "source"); } diff --git a/src/nix/flake.md b/src/nix/flake.md index 661dd2f73..e60abe5fc 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -195,18 +195,29 @@ Currently the `type` attribute can be one of the following: If the flake at *path* is not inside a git repository, the `path:` prefix is implied and can be omitted. - *path* generally must be an absolute path. However, on the command - line, it can be a relative path (e.g. `.` or `./foo`) which is - interpreted as relative to the current directory. In this case, it - must start with `.` to avoid ambiguity with registry lookups - (e.g. `nixpkgs` is a registry lookup; `./nixpkgs` is a relative - path). + If *path* is a relative path (i.e. if it does not start with `/`), + it is interpreted as follows: + + - If *path* is a command line argument, it is interpreted relative + to the current directory. + + - If *path* is used in a `flake.nix`, it is interpreted relative to + the directory containing that `flake.nix`. However, the resolved + path must be in the same tree. For instance, a `flake.nix` in the + root of a tree can use `path:./foo` to access the flake in + subdirectory `foo`, but `path:../bar` is illegal. + + Note that if you omit `path:`, relative paths must start with `.` to + avoid ambiguity with registry lookups (e.g. `nixpkgs` is a registry + lookup; `./nixpkgs` is a relative path). For example, these are valid path flake references: * `path:/home/user/sub/dir` * `/home/user/sub/dir` (if `dir/flake.nix` is *not* in a git repository) - * `./sub/dir` (when used on the command line and `dir/flake.nix` is *not* in a git repository) + * `path:sub/dir` + * `./sub/dir` + * `path:../parent` * `git`: Git repositories. The location of the repository is specified by the attribute `url`. diff --git a/tests/functional/flakes/follow-paths.sh b/tests/functional/flakes/follow-paths.sh index 1afd91bd2..35f52f3ba 100644 --- a/tests/functional/flakes/follow-paths.sh +++ b/tests/functional/flakes/follow-paths.sh @@ -115,7 +115,7 @@ nix flake lock $flakeFollowsA [[ $(jq -c .nodes.B.inputs.foobar $flakeFollowsA/flake.lock) = '"foobar"' ]] jq -r -c '.nodes | keys | .[]' $flakeFollowsA/flake.lock | grep "^foobar$" -# Ensure a relative path is not allowed to go outside the store path +# Check that path: inputs cannot escape from their root. cat > $flakeFollowsA/flake.nix <&1 | grep 'points outside' +expect 1 nix flake lock $flakeFollowsA 2>&1 | grep '/flakeB.*is forbidden in pure evaluation mode' +expect 1 nix flake lock --impure $flakeFollowsA 2>&1 | grep '/flakeB.*does not exist' + +# Test relative non-flake inputs. +cat > $flakeFollowsA/flake.nix < $flakeFollowsA/foo.nix + +git -C $flakeFollowsA add flake.nix foo.nix + +nix flake lock $flakeFollowsA + +[[ $(nix eval --json $flakeFollowsA#e) = 123 ]] # Non-existant follows should print a warning. cat >$flakeFollowsA/flake.nix < $rootFlake/flake.nix < $subflake0/flake.nix < $subflake1/flake.nix < $subflake2/flake.nix < $rootFlake/flake.nix < Date: Fri, 17 May 2024 16:38:01 +0200 Subject: [PATCH 02/68] call-flake.nix: Fix relative path resolution `parentNode.sourceInfo.outPath` does not include the subdir of the parent flake, while `parentNode.outPath` does. So we need to use the latter. --- src/libexpr/flake/call-flake.nix | 2 +- tests/functional/flakes/relative-paths.sh | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libexpr/flake/call-flake.nix b/src/libexpr/flake/call-flake.nix index 43ecb7f15..38dc74ba1 100644 --- a/src/libexpr/flake/call-flake.nix +++ b/src/libexpr/flake/call-flake.nix @@ -47,7 +47,7 @@ let else if node.locked.type == "path" && builtins.substring 0 1 node.locked.path != "/" then parentNode.sourceInfo // { - outPath = parentNode.sourceInfo.outPath + ("/" + node.locked.path); + outPath = parentNode.outPath + ("/" + node.locked.path); } else # FIXME: remove obsolete node.info. diff --git a/tests/functional/flakes/relative-paths.sh b/tests/functional/flakes/relative-paths.sh index cecda44d4..38987d6af 100644 --- a/tests/functional/flakes/relative-paths.sh +++ b/tests/functional/flakes/relative-paths.sh @@ -63,6 +63,10 @@ git -C $rootFlake add flake.nix sub2/flake.nix [[ $(nix eval $subflake2#y) = 15 ]] +# Make sure that this still works after commiting the lock file. +git -C $rootFlake add sub2/flake.lock +[[ $(nix eval $subflake2#y) = 15 ]] + # Make sure there are no content locks for relative path flakes. (! grep "$TEST_ROOT" $subflake2/flake.lock) (! grep "$NIX_STORE_DIR" $subflake2/flake.lock) From 3180671cabeb6a6010057770731e12761ed5666c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 May 2024 19:49:40 +0200 Subject: [PATCH 03/68] Allow the 'url' flake input attribute to be a path literal https://github.com/NixOS/nix/pull/10089#issuecomment-1978133326 --- src/libexpr/flake/flake.cc | 34 +++++++++++++++++------ tests/functional/flakes/relative-paths.sh | 4 +-- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index d4cabe68f..1401f5dda 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -96,14 +96,16 @@ static std::map parseFlakeInputs( EvalState & state, Value * value, const PosIdx pos, - InputPath lockRootPath); + InputPath lockRootPath, + const SourcePath & flakeDir); static FlakeInput parseFlakeInput( EvalState & state, const std::string & inputName, Value * value, const PosIdx pos, - InputPath lockRootPath) + InputPath lockRootPath, + const SourcePath & flakeDir) { expectType(state, nAttrs, *value, pos); @@ -120,14 +122,25 @@ static FlakeInput parseFlakeInput( for (auto & attr : *value->attrs()) { try { if (attr.name == sUrl) { - expectType(state, nString, *attr.value, attr.pos); - url = attr.value->string_view(); + forceTrivialValue(state, *attr.value, pos); + if (attr.value->type() == nString) + url = attr.value->string_view(); + else if (attr.value->type() == nPath) { + auto path = attr.value->path(); + if (path.accessor != flakeDir.accessor) + throw Error("input path '%s' at %s must be in the same source tree as %s", + path, state.positions[attr.pos], flakeDir); + url = "path:" + flakeDir.path.makeRelative(path.path); + } + else + throw Error("expected a string or a path but got %s at %s", + showType(attr.value->type()), state.positions[attr.pos]); attrs.emplace("url", *url); } else if (attr.name == sFlake) { expectType(state, nBool, *attr.value, attr.pos); input.isFlake = attr.value->boolean(); } else if (attr.name == sInputs) { - input.overrides = parseFlakeInputs(state, attr.value, attr.pos, lockRootPath); + input.overrides = parseFlakeInputs(state, attr.value, attr.pos, lockRootPath, flakeDir); } else if (attr.name == sFollows) { expectType(state, nString, *attr.value, attr.pos); auto follows(parseInputPath(attr.value->c_str())); @@ -191,7 +204,8 @@ static std::map parseFlakeInputs( EvalState & state, Value * value, const PosIdx pos, - InputPath lockRootPath) + InputPath lockRootPath, + const SourcePath & flakeDir) { std::map inputs; @@ -203,7 +217,8 @@ static std::map parseFlakeInputs( state.symbols[inputAttr.name], inputAttr.value, inputAttr.pos, - lockRootPath)); + lockRootPath, + flakeDir)); } return inputs; @@ -217,7 +232,8 @@ static Flake readFlake( const SourcePath & rootDir, const InputPath & lockRootPath) { - auto flakePath = rootDir / CanonPath(resolvedRef.subdir) / "flake.nix"; + auto flakeDir = rootDir / CanonPath(resolvedRef.subdir); + auto flakePath = flakeDir / "flake.nix"; // NOTE evalFile forces vInfo to be an attrset because mustBeTrivial is true. Value vInfo; @@ -238,7 +254,7 @@ static Flake readFlake( auto sInputs = state.symbols.create("inputs"); if (auto inputs = vInfo.attrs()->get(sInputs)) - flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, lockRootPath); + flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, lockRootPath, flakeDir); auto sOutputs = state.symbols.create("outputs"); diff --git a/tests/functional/flakes/relative-paths.sh b/tests/functional/flakes/relative-paths.sh index 38987d6af..3e4c97cc4 100644 --- a/tests/functional/flakes/relative-paths.sh +++ b/tests/functional/flakes/relative-paths.sh @@ -12,7 +12,7 @@ mkdir -p $rootFlake $subflake0 $subflake1 $subflake2 cat > $rootFlake/flake.nix < $subflake2/flake.nix < Date: Mon, 16 Sep 2024 14:11:08 +0200 Subject: [PATCH 04/68] shellcheck --- tests/functional/flakes/relative-paths.sh | 54 ++++++++++++----------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/tests/functional/flakes/relative-paths.sh b/tests/functional/flakes/relative-paths.sh index 3e4c97cc4..e72339b7f 100644 --- a/tests/functional/flakes/relative-paths.sh +++ b/tests/functional/flakes/relative-paths.sh @@ -1,16 +1,18 @@ +#!/usr/bin/env bash + source ./common.sh requireGit -rootFlake=$TEST_ROOT/flake1 -subflake0=$rootFlake/sub0 -subflake1=$rootFlake/sub1 -subflake2=$rootFlake/sub2 +rootFlake="$TEST_ROOT/flake1" +subflake0="$rootFlake/sub0" +subflake1="$rootFlake/sub1" +subflake2="$rootFlake/sub2" -rm -rf $rootFlake -mkdir -p $rootFlake $subflake0 $subflake1 $subflake2 +rm -rf "$rootFlake" +mkdir -p "$rootFlake" "$subflake0" "$subflake1" "$subflake2" -cat > $rootFlake/flake.nix < "$rootFlake/flake.nix" < $rootFlake/flake.nix < $subflake0/flake.nix < "$subflake0/flake.nix" < $subflake0/flake.nix < $subflake1/flake.nix < "$subflake1/flake.nix" < $subflake1/flake.nix < $subflake2/flake.nix < "$subflake2/flake.nix" < $subflake2/flake.nix < $rootFlake/flake.nix < "$rootFlake/flake.nix" < $rootFlake/flake.nix < Date: Mon, 16 Sep 2024 14:52:23 +0200 Subject: [PATCH 05/68] parentPath -> parentInputPath --- src/libflake/flake/flake.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index 632968f50..461ac0a90 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -404,7 +404,7 @@ LockedFlake lockFlake( { FlakeInput input; SourcePath sourcePath; - std::optional parentPath; // FIXME: rename to inputPathPrefix? + std::optional parentInputPath; // FIXME: rename to inputPathPrefix? }; std::map overrides; @@ -469,7 +469,7 @@ LockedFlake lockFlake( Override { .input = inputOverride, .sourcePath = sourcePath, - .parentPath = inputPathPrefix // FIXME: should this be inputPath? + .parentInputPath = inputPathPrefix // FIXME: should this be inputPath? }); } } @@ -531,7 +531,7 @@ LockedFlake lockFlake( auto overridenParentPath = input.ref->input.isRelative() - ? std::optional(hasOverride ? i->second.parentPath : inputPathPrefix) + ? std::optional(hasOverride ? i->second.parentInputPath : inputPathPrefix) : std::nullopt; auto resolveRelativePath = [&]() -> std::optional From f2063255a40d534035b706235752a5c0a09c6d15 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 25 Sep 2024 16:29:43 +0200 Subject: [PATCH 06/68] tests/functional/flakes/relative-paths.sh: Fix build failure in hydraJobs.tests.functional_user --- tests/functional/flakes/relative-paths.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/functional/flakes/relative-paths.sh b/tests/functional/flakes/relative-paths.sh index e72339b7f..9b93da9c1 100644 --- a/tests/functional/flakes/relative-paths.sh +++ b/tests/functional/flakes/relative-paths.sh @@ -71,7 +71,9 @@ git -C "$rootFlake" add sub2/flake.lock # Make sure there are no content locks for relative path flakes. (! grep "$TEST_ROOT" "$subflake2/flake.lock") -(! grep "$NIX_STORE_DIR" "$subflake2/flake.lock") +if ! isTestOnNixOS; then + (! grep "$NIX_STORE_DIR" "$subflake2/flake.lock") +fi (! grep narHash "$subflake2/flake.lock") # Test circular relative path flakes. FIXME: doesn't work at the moment. From 00b99b8bc0aa7f5f0a1a04b9704c2120e5eca6db Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 22 Nov 2024 16:23:34 +0100 Subject: [PATCH 07/68] Remove FIXME --- src/libflake/flake/flake.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index e49333f25..8e381c0d2 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -470,7 +470,7 @@ LockedFlake lockFlake( Override { .input = inputOverride, .sourcePath = sourcePath, - .parentInputPath = inputPathPrefix // FIXME: should this be inputPath? + .parentInputPath = inputPathPrefix }); } } From 985b2f9df30e62a21c32d7ffaae7aebfe4c56d3a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Nov 2024 15:23:56 +0100 Subject: [PATCH 08/68] Remove FIXME --- src/libflake/flake/flake.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index 8e381c0d2..9d5760ed1 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -638,10 +638,6 @@ LockedFlake lockFlake( nodePaths.emplace(childNode, inputFlake.path.parent()); computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, lockRootPath, inputFlake.path, false); } else { - // FIXME: sourcePath is wrong here, we - // should pass a lambda that lazily - // fetches the parent flake if needed - // (i.e. getInputFlake()). computeLocks(fakeInputs, childNode, inputPath, oldLock, lockRootPath, sourcePath, true); } From 9223d64ac62385d3b75cfbf5bac491fc14d033ad Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 23 Dec 2024 16:03:13 +0100 Subject: [PATCH 09/68] Remove dead code --- src/libflake/flake/flakeref.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index 41e991c20..3a43a51f8 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -195,10 +195,6 @@ std::pair parsePathFlakeRefWithFragment( throw BadURL("flake reference '%s' is not an absolute path", url); } - fetchers::Attrs attrs; - attrs.insert_or_assign("type", "path"); - attrs.insert_or_assign("path", path); - return fromParsedURL(fetchSettings, { .url = path, .base = path, From 01c96f9fd587912a50e746309580f75e95c56a97 Mon Sep 17 00:00:00 2001 From: Mutsuha Asada Date: Thu, 2 Jan 2025 13:26:13 +0900 Subject: [PATCH 10/68] libmain: fix ignoring empty lines in the print-build-logs option --- src/libmain/progress-bar.cc | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index fa0b73ebe..961850b58 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -287,23 +287,21 @@ public: else if (type == resBuildLogLine || type == resPostBuildLogLine) { auto lastLine = chomp(getS(fields, 0)); - if (!lastLine.empty()) { - auto i = state->its.find(act); - assert(i != state->its.end()); - ActInfo info = *i->second; - if (printBuildLogs) { - auto suffix = "> "; - if (type == resPostBuildLogLine) { - suffix = " (post)> "; - } - log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + suffix + ANSI_NORMAL + lastLine); - } else { - state->activities.erase(i->second); - info.lastLine = lastLine; - state->activities.emplace_back(info); - i->second = std::prev(state->activities.end()); - update(*state); + auto i = state->its.find(act); + assert(i != state->its.end()); + ActInfo info = *i->second; + if (printBuildLogs) { + auto suffix = "> "; + if (type == resPostBuildLogLine) { + suffix = " (post)> "; } + log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + suffix + ANSI_NORMAL + lastLine); + } else { + state->activities.erase(i->second); + info.lastLine = lastLine; + state->activities.emplace_back(info); + i->second = std::prev(state->activities.end()); + update(*state); } } From 75cda2da7fbd6283f88cf0fd0b934e1047ca1408 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Jan 2025 13:40:18 +0100 Subject: [PATCH 11/68] Document path values in inputs Co-authored-by: Robert Hensing --- src/nix/flake.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/nix/flake.md b/src/nix/flake.md index a55bba4fc..5bb18328d 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -212,6 +212,13 @@ Currently the `type` attribute can be one of the following: root of a tree can use `path:./foo` to access the flake in subdirectory `foo`, but `path:../bar` is illegal. +Path inputs can be specified with path values in `flake.nix`. Path values are a syntax for `path` inputs, and they are converted by +1. resolving them into relative paths, relative to the base directory of `flake.nix` +2. escaping URL characters (refer to IETF RFC?) +3. prepending `path:` + +Note that the allowed syntax for path values in flake `inputs` may be more restrictive than general Nix, so you may need to use `path:` if your path contains certain special characters. See [Path literals](@docroot@/language/syntax#path-literal) + Note that if you omit `path:`, relative paths must start with `.` to avoid ambiguity with registry lookups (e.g. `nixpkgs` is a registry lookup; `./nixpkgs` is a relative path). From e8c7dd9971aaac0ac14e08f299a41ef0a93afef7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Jan 2025 13:43:56 +0100 Subject: [PATCH 12/68] Rename allowRelative -> preserveRelativePaths --- src/libflake/flake/flakeref.cc | 12 ++++++------ src/libflake/flake/flakeref.hh | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index 3a43a51f8..06198b10c 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -49,9 +49,9 @@ FlakeRef parseFlakeRef( const std::optional & baseDir, bool allowMissing, bool isFlake, - bool allowRelative) + bool preserveRelativePaths) { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(fetchSettings, url, baseDir, allowMissing, isFlake, allowRelative); + auto [flakeRef, fragment] = parseFlakeRefWithFragment(fetchSettings, url, baseDir, allowMissing, isFlake, preserveRelativePaths); if (fragment != "") throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url); return flakeRef; @@ -89,7 +89,7 @@ std::pair parsePathFlakeRefWithFragment( const std::optional & baseDir, bool allowMissing, bool isFlake, - bool allowRelative) + bool preserveRelativePaths) { std::string path = url; std::string fragment = ""; @@ -191,7 +191,7 @@ std::pair parsePathFlakeRefWithFragment( } } else { - if (!allowRelative && !hasPrefix(path, "/")) + if (!preserveRelativePaths && !hasPrefix(path, "/")) throw BadURL("flake reference '%s' is not an absolute path", url); } @@ -258,7 +258,7 @@ std::pair parseFlakeRefWithFragment( const std::optional & baseDir, bool allowMissing, bool isFlake, - bool allowRelative) + bool preserveRelativePaths) { using namespace fetchers; @@ -267,7 +267,7 @@ std::pair parseFlakeRefWithFragment( } else if (auto res = parseURLFlakeRef(fetchSettings, url, baseDir, isFlake)) { return *res; } else { - return parsePathFlakeRefWithFragment(fetchSettings, url, baseDir, allowMissing, isFlake, allowRelative); + return parsePathFlakeRefWithFragment(fetchSettings, url, baseDir, allowMissing, isFlake, preserveRelativePaths); } } diff --git a/src/libflake/flake/flakeref.hh b/src/libflake/flake/flakeref.hh index 32094d381..c9cf7952d 100644 --- a/src/libflake/flake/flakeref.hh +++ b/src/libflake/flake/flakeref.hh @@ -85,7 +85,7 @@ FlakeRef parseFlakeRef( const std::optional & baseDir = {}, bool allowMissing = false, bool isFlake = true, - bool allowRelative = false); + bool preserveRelativePaths = false); /** * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) @@ -104,7 +104,7 @@ std::pair parseFlakeRefWithFragment( const std::optional & baseDir = {}, bool allowMissing = false, bool isFlake = true, - bool allowRelative = false); + bool preserveRelativePaths = false); /** * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) From 0792152627a94dbb2fbbbd8f0f05ff36a480dabf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Jan 2025 13:54:19 +0100 Subject: [PATCH 13/68] Rename Override -> OverrideTarget --- src/libflake/flake/flake.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index e9fa28f06..4f0b922ab 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -408,14 +408,14 @@ LockedFlake lockFlake( debug("old lock file: %s", oldLockFile); - struct Override + struct OverrideTarget { FlakeInput input; SourcePath sourcePath; std::optional parentInputPath; // FIXME: rename to inputPathPrefix? }; - std::map overrides; + std::map overrides; std::set explicitCliOverrides; std::set overridesUsed, updatesUsed; std::map, SourcePath> nodePaths; @@ -423,7 +423,7 @@ LockedFlake lockFlake( for (auto & i : lockFlags.inputOverrides) { overrides.emplace( i.first, - Override { + OverrideTarget { .input = FlakeInput { .ref = i.second }, /* Note: any relative overrides (e.g. `--override-input B/C "path:./foo/bar"`) @@ -474,7 +474,7 @@ LockedFlake lockFlake( inputPath.push_back(id); inputPath.push_back(idOverride); overrides.emplace(inputPath, - Override { + OverrideTarget { .input = inputOverride, .sourcePath = sourcePath, .parentInputPath = inputPathPrefix From ef2739b7c9071149fb9333865bd3cc2ab60f2178 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Jan 2025 14:01:49 +0100 Subject: [PATCH 14/68] Example of referencing parent directories --- src/nix/flake.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nix/flake.md b/src/nix/flake.md index 5bb18328d..35ac3ea9b 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -210,7 +210,9 @@ Currently the `type` attribute can be one of the following: the directory containing that `flake.nix`. However, the resolved path must be in the same tree. For instance, a `flake.nix` in the root of a tree can use `path:./foo` to access the flake in - subdirectory `foo`, but `path:../bar` is illegal. + subdirectory `foo`, but `path:../bar` is illegal. On the other + hand, a flake in the `/foo` directory of a tree can use + `path:../bar` to refer to the flake in `/bar`. Path inputs can be specified with path values in `flake.nix`. Path values are a syntax for `path` inputs, and they are converted by 1. resolving them into relative paths, relative to the base directory of `flake.nix` From d329b2632a91b05e1151decbb2b4c4011ebf5aa1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Jan 2025 17:04:06 +0100 Subject: [PATCH 15/68] Fix manual --- src/nix/flake.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/flake.md b/src/nix/flake.md index 35ac3ea9b..e79a8ee0b 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -219,7 +219,7 @@ Path inputs can be specified with path values in `flake.nix`. Path values are a 2. escaping URL characters (refer to IETF RFC?) 3. prepending `path:` -Note that the allowed syntax for path values in flake `inputs` may be more restrictive than general Nix, so you may need to use `path:` if your path contains certain special characters. See [Path literals](@docroot@/language/syntax#path-literal) +Note that the allowed syntax for path values in flake `inputs` may be more restrictive than general Nix, so you may need to use `path:` if your path contains certain special characters. See [Path literals](@docroot@/language/syntax.md#path-literal) Note that if you omit `path:`, relative paths must start with `.` to avoid ambiguity with registry lookups (e.g. `nixpkgs` is a registry From ed4f2c32040afcc7f1e5adb89b67b3f28e384f40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 7 Jan 2025 21:47:55 +0100 Subject: [PATCH 16/68] scripts/install-darwin-multi-user: workaround dscl failing sometimes --- scripts/install-darwin-multi-user.sh | 31 +++++++++++++++++++++------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh index 89c66b8f4..c74999512 100644 --- a/scripts/install-darwin-multi-user.sh +++ b/scripts/install-darwin-multi-user.sh @@ -145,13 +145,28 @@ poly_user_id_get() { dsclattr "/Users/$1" "UniqueID" } +dscl_create() { + # workaround a bug in dscl where it sometimes fails with eNotYetImplemented: + # https://github.com/NixOS/nix/issues/12140 + while ! _sudo "$1" /usr/bin/dscl . -create "$2" "$3" "$4" 2> "$SCRATCH/dscl.err"; do + local err=$? + if [[ $err -eq 140 ]] && grep -q "-14988 (eNotYetImplemented)" "$SCRATCH/dscl.err"; then + echo "dscl failed with eNotYetImplemented, retrying..." + sleep 1 + continue + fi + cat "$SCRATCH/dscl.err" + return $err + done +} + poly_user_hidden_get() { dsclattr "/Users/$1" "IsHidden" } poly_user_hidden_set() { - _sudo "in order to make $1 a hidden user" \ - /usr/bin/dscl . -create "/Users/$1" "IsHidden" "1" + dscl_create "in order to make $1 a hidden user" \ + "/Users/$1" "IsHidden" "1" } poly_user_home_get() { @@ -161,8 +176,8 @@ poly_user_home_get() { poly_user_home_set() { # This can trigger a permission prompt now: # "Terminal" would like to administer your computer. Administration can include modifying passwords, networking, and system settings. - _sudo "in order to give $1 a safe home directory" \ - /usr/bin/dscl . -create "/Users/$1" "NFSHomeDirectory" "$2" + dscl_create "in order to give $1 a safe home directory" \ + "/Users/$1" "NFSHomeDirectory" "$2" } poly_user_note_get() { @@ -170,8 +185,8 @@ poly_user_note_get() { } poly_user_note_set() { - _sudo "in order to give $username a useful note" \ - /usr/bin/dscl . -create "/Users/$1" "RealName" "$2" + dscl_create "in order to give $1 a useful note" \ + "/Users/$1" "RealName" "$2" } poly_user_shell_get() { @@ -179,8 +194,8 @@ poly_user_shell_get() { } poly_user_shell_set() { - _sudo "in order to give $1 a safe shell" \ - /usr/bin/dscl . -create "/Users/$1" "UserShell" "$2" + dscl_create "in order to give $1 a safe shell" \ + "/Users/$1" "UserShell" "$2" } poly_user_in_group_check() { From fccfdbea5719f66b01870d483195f66146553b19 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 9 Jan 2025 20:44:25 +0100 Subject: [PATCH 17/68] nix upgrade-nix: Give a better error message if the profile is using 'nix profile' --- src/nix/upgrade-nix.cc | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 77fb798a7..28174c4fb 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -15,7 +15,7 @@ using namespace nix; struct CmdUpgradeNix : MixDryRun, StoreCommand { - Path profileDir; + std::filesystem::path profileDir; CmdUpgradeNix() { @@ -64,7 +64,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand if (profileDir == "") profileDir = getProfileDir(store); - printInfo("upgrading Nix in profile '%s'", profileDir); + printInfo("upgrading Nix in profile %s", profileDir); auto storePath = getLatestNix(store); @@ -93,7 +93,9 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand { Activity act(*logger, lvlInfo, actUnknown, - fmt("installing '%s' into profile '%s'...", store->printStorePath(storePath), profileDir)); + fmt("installing '%s' into profile %s...", store->printStorePath(storePath), profileDir)); + + // FIXME: don't call an external process. runProgram(getNixBin("nix-env").string(), false, {"--profile", profileDir, "-i", store->printStorePath(storePath), "--no-sandbox"}); } @@ -102,31 +104,33 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand } /* Return the profile in which Nix is installed. */ - Path getProfileDir(ref store) + std::filesystem::path getProfileDir(ref store) { auto whereOpt = ExecutablePath::load().findName(OS_STR("nix-env")); if (!whereOpt) throw Error("couldn't figure out how Nix is installed, so I can't upgrade it"); const auto & where = whereOpt->parent_path(); - printInfo("found Nix in '%s'", where); + printInfo("found Nix in %s", where); if (hasPrefix(where.string(), "/run/current-system")) throw Error("Nix on NixOS must be upgraded via 'nixos-rebuild'"); - Path profileDir = where.parent_path().string(); + auto profileDir = where.parent_path(); // Resolve profile to /nix/var/nix/profiles/ link. - while (canonPath(profileDir).find("/profiles/") == std::string::npos && std::filesystem::is_symlink(profileDir)) + while (canonPath(profileDir.string()).find("/profiles/") == std::string::npos && std::filesystem::is_symlink(profileDir)) profileDir = readLink(profileDir); - printInfo("found profile '%s'", profileDir); + printInfo("found profile %s", profileDir); - Path userEnv = canonPath(profileDir, true); + Path userEnv = canonPath(profileDir.string(), true); - if (where.filename() != "bin" || - !hasSuffix(userEnv, "user-environment")) - throw Error("directory %s does not appear to be part of a Nix profile", where); + if (std::filesystem::exists(profileDir / "manifest.json")) + throw Error("directory %s is managed by 'nix profile' and currently cannot be upgraded by 'nix upgrade-nix'", profileDir); + + if (!std::filesystem::exists(profileDir / "manifest.nix")) + throw Error("directory %s does not appear to be part of a Nix profile", profileDir); if (!store->isValidPath(store->parseStorePath(userEnv))) throw Error("directory '%s' is not in the Nix store", userEnv); From 96bd9bad2f434fe3489b039ac665679878460b0c Mon Sep 17 00:00:00 2001 From: Bryan Lai Date: Thu, 26 Dec 2024 20:50:36 +0800 Subject: [PATCH 18/68] fetchers/git: make path absolute for local repo --- src/libfetchers/git.cc | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index d894550c0..c4dfc27a2 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -426,7 +426,16 @@ struct GitInputScheme : InputScheme auto url = parseURL(getStrAttr(input.attrs, "url")); bool isBareRepository = url.scheme == "file" && !pathExists(url.path + "/.git"); repoInfo.isLocal = url.scheme == "file" && !forceHttp && !isBareRepository; - repoInfo.url = repoInfo.isLocal ? url.path : url.to_string(); + // + // FIXME: here we turn a possibly relative path into an absolute path. + // This allows relative git flake inputs to be resolved against the + // **current working directory** (as in POSIX), which tends to work out + // ok in the context of flakes, but is the wrong behavior, + // as it should resolve against the flake.nix base directory instead. + // + // See: https://discourse.nixos.org/t/57783 and #9708 + // + repoInfo.url = repoInfo.isLocal ? std::filesystem::absolute(url.path).string() : url.to_string(); // If this is a local directory and no ref or revision is // given, then allow the use of an unclean working tree. From 9d088fa50242c5fe2ad94bffb61a4742f232ff0b Mon Sep 17 00:00:00 2001 From: Bryan Lai Date: Fri, 27 Dec 2024 17:35:44 +0800 Subject: [PATCH 19/68] tests/flakes: check git+file:./${submodule} protocol Relative, local git repo used to work (for submodules), but it fails after 3e0129ce3b9eb094d4a3cc8023884f372f1d7ff6. This commit adds a test to prevent such failure in the future. --- tests/functional/flakes/flakes.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 6c466a0c7..acbd32a91 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -106,6 +106,14 @@ nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir#default" nix build -o "$TEST_ROOT/result" "$flake1Dir?ref=HEAD#default" nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir?ref=HEAD#default" +# Check that relative paths are allowed for git flakes. +# This may change in the future once git submodule support is refined. +# See: https://discourse.nixos.org/t/57783 and #9708. +( + cd "$flake1Dir/.." + nix build -o "$TEST_ROOT/result" "git+file:./$(basename "$flake1Dir")" +) + # Check that store symlinks inside a flake are not interpreted as flakes. nix build -o "$flake1Dir/result" "git+file://$flake1Dir" nix path-info "$flake1Dir/result" From 37ac18d1d9faba24ce09653723342465bcac3e0b Mon Sep 17 00:00:00 2001 From: Bryan Lai Date: Wed, 1 Jan 2025 14:53:04 +0800 Subject: [PATCH 20/68] tests/flake-in-submodule: git+file:./* input --- tests/functional/flakes/flake-in-submodule.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/functional/flakes/flake-in-submodule.sh b/tests/functional/flakes/flake-in-submodule.sh index 643d729ff..f98c19aa8 100755 --- a/tests/functional/flakes/flake-in-submodule.sh +++ b/tests/functional/flakes/flake-in-submodule.sh @@ -76,3 +76,21 @@ git -C "$rootRepo" commit -m "Add flake.nix" storePath=$(nix flake metadata --json "$rootRepo?submodules=1" | jq -r .path) [[ -e "$storePath/submodule" ]] + +# The root repo may use the submodule repo as an input +# through the relative path. This may change in the future; +# see: https://discourse.nixos.org/t/57783 and #9708. +cat > "$rootRepo"/flake.nix < Date: Fri, 10 Jan 2025 09:57:54 +0100 Subject: [PATCH 21/68] Clarify cd call in tests/functional/flakes/flakes.sh --- tests/functional/flakes/flakes.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index acbd32a91..995ca03b0 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -110,6 +110,7 @@ nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir?ref=HEAD#default" # This may change in the future once git submodule support is refined. # See: https://discourse.nixos.org/t/57783 and #9708. ( + # This `cd` should not be required and is indicative of aforementioned bug. cd "$flake1Dir/.." nix build -o "$TEST_ROOT/result" "git+file:./$(basename "$flake1Dir")" ) From 22adffec347d551919eb53a3ec7a57039915346b Mon Sep 17 00:00:00 2001 From: Tristan Ross Date: Thu, 9 Jan 2025 12:50:05 -0800 Subject: [PATCH 22/68] nix flake: clarify error message when file is an unknown type --- src/libstore/nar-accessor.cc | 6 +++++- src/libutil/fs-sink.cc | 10 ++++++---- src/libutil/git.cc | 14 +++++++++++--- src/libutil/posix-source-accessor.cc | 12 ++++++++++-- src/libutil/source-accessor.cc | 20 ++++++++++++++++++++ src/libutil/source-accessor.hh | 8 ++++++-- src/nix/flake.cc | 2 +- 7 files changed, 59 insertions(+), 13 deletions(-) diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 9a541bb77..c4e0b137b 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -291,7 +291,11 @@ json listNar(ref accessor, const CanonPath & path, bool recurse) obj["type"] = "symlink"; obj["target"] = accessor->readLink(path); break; - case SourceAccessor::Type::tMisc: + case SourceAccessor::Type::tBlock: + case SourceAccessor::Type::tChar: + case SourceAccessor::Type::tSocket: + case SourceAccessor::Type::tFifo: + case SourceAccessor::Type::tUnknown: assert(false); // cannot happen for NARs } return obj; diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index 72e5c731f..dd979f83f 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -49,11 +49,13 @@ void copyRecursive( break; } - case SourceAccessor::tMisc: - throw Error("file '%1%' has an unsupported type", from); - + case SourceAccessor::tChar: + case SourceAccessor::tBlock: + case SourceAccessor::tSocket: + case SourceAccessor::tFifo: + case SourceAccessor::tUnknown: default: - unreachable(); + throw Error("file '%1%' has an unsupported type of %2%", from, stat.typeString()); } } diff --git a/src/libutil/git.cc b/src/libutil/git.cc index af91fa643..3303dbc32 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -200,7 +200,11 @@ std::optional convertMode(SourceAccessor::Type type) case SourceAccessor::tSymlink: return Mode::Symlink; case SourceAccessor::tRegular: return Mode::Regular; case SourceAccessor::tDirectory: return Mode::Directory; - case SourceAccessor::tMisc: return std::nullopt; + case SourceAccessor::tChar: + case SourceAccessor::tBlock: + case SourceAccessor::tSocket: + case SourceAccessor::tFifo: return std::nullopt; + case SourceAccessor::tUnknown: default: unreachable(); } } @@ -314,9 +318,13 @@ Mode dump( return Mode::Symlink; } - case SourceAccessor::tMisc: + case SourceAccessor::tChar: + case SourceAccessor::tBlock: + case SourceAccessor::tSocket: + case SourceAccessor::tFifo: + case SourceAccessor::tUnknown: default: - throw Error("file '%1%' has an unsupported type", path); + throw Error("file '%1%' has an unsupported type of %2%", path, st.typeString()); } } diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 50b436893..8ee986d3f 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -122,7 +122,11 @@ std::optional PosixSourceAccessor::maybeLstat(const CanonP S_ISREG(st->st_mode) ? tRegular : S_ISDIR(st->st_mode) ? tDirectory : S_ISLNK(st->st_mode) ? tSymlink : - tMisc, + S_ISCHR(st->st_mode) ? tChar : + S_ISBLK(st->st_mode) ? tBlock : + S_ISSOCK(st->st_mode) ? tSocket : + S_ISFIFO(st->st_mode) ? tFifo : + tUnknown, .fileSize = S_ISREG(st->st_mode) ? std::optional(st->st_size) : std::nullopt, .isExecutable = S_ISREG(st->st_mode) && st->st_mode & S_IXUSR, }; @@ -156,7 +160,11 @@ SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & case std::filesystem::file_type::regular: return Type::tRegular; break; case std::filesystem::file_type::symlink: return Type::tSymlink; break; case std::filesystem::file_type::directory: return Type::tDirectory; break; - default: return tMisc; + case std::filesystem::file_type::character: return Type::tChar; break; + case std::filesystem::file_type::block: return Type::tBlock; break; + case std::filesystem::file_type::fifo: return Type::tFifo; break; + case std::filesystem::file_type::socket: return Type::tSocket; break; + default: return tUnknown; } #pragma GCC diagnostic pop }(); diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index d3e304f74..aa8dcf9fd 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -5,6 +5,26 @@ namespace nix { static std::atomic nextNumber{0}; +bool SourceAccessor::Stat::isNotNARSerialisable() +{ + return this->type != tRegular && this->type != tSymlink && this->type != tDirectory; +} + +std::string SourceAccessor::Stat::typeString() { + switch (this->type) { + case tRegular: return "regular"; + case tSymlink: return "symlink"; + case tDirectory: return "directory"; + case tChar: return "character device"; + case tBlock: return "block device"; + case tSocket: return "socket"; + case tFifo: return "fifo"; + case tUnknown: + default: return "unknown"; + } + return "unknown"; +} + SourceAccessor::SourceAccessor() : number(++nextNumber) , displayPrefix{"«unknown»"} diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index b16960d4a..42af8256a 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -88,12 +88,13 @@ struct SourceAccessor : std::enable_shared_from_this Unlike `DT_UNKNOWN`, this must not be used for deferring the lookup of types. */ - tMisc + tChar, tBlock, tSocket, tFifo, + tUnknown }; struct Stat { - Type type = tMisc; + Type type = tUnknown; /** * For regular files only: the size of the file. Not all @@ -112,6 +113,9 @@ struct SourceAccessor : std::enable_shared_from_this * file in the NAR. Only returned by NAR accessors. */ std::optional narOffset; + + bool isNotNARSerialisable(); + std::string typeString(); }; Stat lstat(const CanonPath & path); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 7189689cc..3da194bcc 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -941,7 +941,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand createSymlink(target, os_string_to_string(PathViewNG { to2 })); } else - throw Error("file '%s' has unsupported type", from2); + throw Error("path '%s' needs to be a symlink, file, or directory but instead is a %s", from2, st.typeString()); changedFiles.push_back(to2); notice("wrote: %s", to2); } From e1613932991b719854a0b23ff0ab0b42fc9fc747 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 10 Jan 2025 16:27:40 +0100 Subject: [PATCH 23/68] Add setting 'allow-dirty-locks' This allows writing lock files with dirty inputs, so long as they have a NAR hash. (Currently they always have a NAR hash, but with lazy trees that may not always be the case.) Generally dirty locks are bad for reproducibility (we can detect if the dirty input has changed, but we have no way to fetch it except substitution). Hence we don't allow them by default. Fixes #11181. --- src/libcmd/installables.cc | 2 +- src/libexpr/primops/fetchTree.cc | 2 +- src/libfetchers/fetch-settings.hh | 16 ++++++++++++++++ src/libfetchers/fetchers.cc | 7 +++++++ src/libfetchers/fetchers.hh | 9 +++++++++ src/libflake/flake/flake.cc | 16 +++++++++++----- src/libflake/flake/flake.hh | 4 +++- src/libflake/flake/lockfile.cc | 9 +++++---- src/libflake/flake/lockfile.hh | 2 +- src/libflake/flake/settings.hh | 2 +- src/nix/flake.cc | 4 ++-- tests/functional/flakes/unlocked-override.sh | 11 ++++++++++- 12 files changed, 67 insertions(+), 17 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 250cd1413..ab3ab3104 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -450,7 +450,7 @@ ref openEvalCache( std::shared_ptr lockedFlake) { auto fingerprint = evalSettings.useEvalCache && evalSettings.pureEval - ? lockedFlake->getFingerprint(state.store) + ? lockedFlake->getFingerprint(state.store, state.fetchSettings) : std::nullopt; auto rootLoader = [&state, lockedFlake]() { diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 47d1be1cc..fe42b88f1 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -182,7 +182,7 @@ static void fetchTree( if (!state.settings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes)) input = lookupInRegistries(state.store, input).first; - if (state.settings.pureEval && !input.isLocked()) { + if (state.settings.pureEval && !input.isConsideredLocked(state.fetchSettings)) { auto fetcher = "fetchTree"; if (params.isFetchGit) fetcher = "fetchGit"; diff --git a/src/libfetchers/fetch-settings.hh b/src/libfetchers/fetch-settings.hh index f7cb34a02..2ad8aa327 100644 --- a/src/libfetchers/fetch-settings.hh +++ b/src/libfetchers/fetch-settings.hh @@ -70,6 +70,22 @@ struct Settings : public Config Setting warnDirty{this, true, "warn-dirty", "Whether to warn about dirty Git/Mercurial trees."}; + Setting allowDirtyLocks{ + this, + false, + "allow-dirty-locks", + R"( + Whether to allow dirty inputs (such as dirty Git workdirs) + to be locked via their NAR hash. This is generally bad + practice since Nix has no way to obtain such inputs if they + are subsequently modified. Therefore lock files with dirty + locks should generally only be used for local testing, and + should not be pushed to other users. + )", + {}, + true, + Xp::Flakes}; + Setting trustTarballsFromGitForges{ this, true, "trust-tarballs-from-git-forges", R"( diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 87bf0dd71..35976f6a3 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -4,6 +4,7 @@ #include "fetch-to-store.hh" #include "json-utils.hh" #include "store-path-accessor.hh" +#include "fetch-settings.hh" #include @@ -154,6 +155,12 @@ bool Input::isLocked() const return scheme && scheme->isLocked(*this); } +bool Input::isConsideredLocked( + const Settings & settings) const +{ + return isLocked() || (settings.allowDirtyLocks && getNarHash()); +} + bool Input::isFinal() const { return maybeGetBoolAttr(attrs, "__final").value_or(false); diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 841a44041..8cabd5103 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -95,6 +95,15 @@ public: */ bool isLocked() const; + /** + * Return whether the input is either locked, or, if + * `allow-dirty-locks` is enabled, it has a NAR hash. In the + * latter case, we can verify the input but we may not be able to + * fetch it from anywhere. + */ + bool isConsideredLocked( + const Settings & settings) const; + /** * Return whether this is a "final" input, meaning that fetching * it will not add, remove or change any attributes. (See diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index 29090b900..8c0c36eb3 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -345,6 +345,7 @@ Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup } static LockFile readLockFile( + const Settings & settings, const fetchers::Settings & fetchSettings, const SourcePath & lockFilePath) { @@ -380,6 +381,7 @@ LockedFlake lockFlake( } auto oldLockFile = readLockFile( + settings, state.fetchSettings, lockFlags.referenceLockFilePath.value_or( flake.lockFilePath())); @@ -616,7 +618,7 @@ LockedFlake lockFlake( inputFlake.inputs, childNode, inputPath, oldLock ? std::dynamic_pointer_cast(oldLock) - : readLockFile(state.fetchSettings, inputFlake.lockFilePath()).root.get_ptr(), + : readLockFile(settings, state.fetchSettings, inputFlake.lockFilePath()).root.get_ptr(), oldLock ? lockRootPath : inputPath, localPath, false); @@ -678,9 +680,11 @@ LockedFlake lockFlake( if (lockFlags.writeLockFile) { if (sourcePath || lockFlags.outputLockFilePath) { - if (auto unlockedInput = newLockFile.isUnlocked()) { + if (auto unlockedInput = newLockFile.isUnlocked(state.fetchSettings)) { if (lockFlags.failOnUnlocked) - throw Error("cannot write lock file of flake '%s' because it has an unlocked input ('%s').\n", topRef, *unlockedInput); + throw Error( + "Will not write lock file of flake '%s' because it has an unlocked input ('%s'). " + "Use '--allow-dirty-locks' to allow this anyway.", topRef, *unlockedInput); if (state.fetchSettings.warnDirty) warn("will not write lock file of flake '%s' because it has an unlocked input ('%s')", topRef, *unlockedInput); } else { @@ -979,9 +983,11 @@ static RegisterPrimOp r4({ } -std::optional LockedFlake::getFingerprint(ref store) const +std::optional LockedFlake::getFingerprint( + ref store, + const fetchers::Settings & fetchSettings) const { - if (lockFile.isUnlocked()) return std::nullopt; + if (lockFile.isUnlocked(fetchSettings)) return std::nullopt; auto fingerprint = flake.lockedRef.input.getFingerprint(store); if (!fingerprint) return std::nullopt; diff --git a/src/libflake/flake/flake.hh b/src/libflake/flake/flake.hh index 0dfd9440d..f85671a41 100644 --- a/src/libflake/flake/flake.hh +++ b/src/libflake/flake/flake.hh @@ -129,7 +129,9 @@ struct LockedFlake */ std::map, SourcePath> nodePaths; - std::optional getFingerprint(ref store) const; + std::optional getFingerprint( + ref store, + const fetchers::Settings & fetchSettings) const; }; struct LockFlags diff --git a/src/libflake/flake/lockfile.cc b/src/libflake/flake/lockfile.cc index 668ed165f..336054f5b 100644 --- a/src/libflake/flake/lockfile.cc +++ b/src/libflake/flake/lockfile.cc @@ -10,6 +10,7 @@ #include #include "strings.hh" +#include "flake/settings.hh" namespace nix::flake { @@ -43,8 +44,8 @@ LockedNode::LockedNode( , originalRef(getFlakeRef(fetchSettings, json, "original", nullptr)) , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) { - if (!lockedRef.input.isLocked()) - throw Error("lock file contains unlocked input '%s'", + if (!lockedRef.input.isConsideredLocked(fetchSettings)) + throw Error("Lock file contains unlocked input '%s'. Use '--allow-dirty-locks' to accept this lock file.", fetchers::attrsToJSON(lockedRef.input.toAttrs())); // For backward compatibility, lock file entries are implicitly final. @@ -228,7 +229,7 @@ std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile) return stream; } -std::optional LockFile::isUnlocked() const +std::optional LockFile::isUnlocked(const fetchers::Settings & fetchSettings) const { std::set> nodes; @@ -247,7 +248,7 @@ std::optional LockFile::isUnlocked() const for (auto & i : nodes) { if (i == ref(root)) continue; auto node = i.dynamic_pointer_cast(); - if (node && (!node->lockedRef.input.isLocked() || !node->lockedRef.input.isFinal())) + if (node && (!node->lockedRef.input.isConsideredLocked(fetchSettings) || !node->lockedRef.input.isFinal())) return node->lockedRef; } diff --git a/src/libflake/flake/lockfile.hh b/src/libflake/flake/lockfile.hh index a2711a516..bcc3c460c 100644 --- a/src/libflake/flake/lockfile.hh +++ b/src/libflake/flake/lockfile.hh @@ -71,7 +71,7 @@ struct LockFile * Check whether this lock file has any unlocked or non-final * inputs. If so, return one. */ - std::optional isUnlocked() const; + std::optional isUnlocked(const fetchers::Settings & fetchSettings) const; bool operator ==(const LockFile & other) const; diff --git a/src/libflake/flake/settings.hh b/src/libflake/flake/settings.hh index fee247a7d..991eaca1f 100644 --- a/src/libflake/flake/settings.hh +++ b/src/libflake/flake/settings.hh @@ -29,7 +29,7 @@ struct Settings : public Config this, false, "accept-flake-config", - "Whether to accept nix configuration from a flake without prompting.", + "Whether to accept Nix configuration settings from a flake without prompting.", {}, true, Xp::Flakes}; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 7189689cc..d92f644d9 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -238,7 +238,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["lastModified"] = *lastModified; j["path"] = storePath; j["locks"] = lockedFlake.lockFile.toJSON().first; - if (auto fingerprint = lockedFlake.getFingerprint(store)) + if (auto fingerprint = lockedFlake.getFingerprint(store, fetchSettings)) j["fingerprint"] = fingerprint->to_string(HashFormat::Base16, false); logger->cout("%s", j.dump()); } else { @@ -272,7 +272,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON logger->cout( ANSI_BOLD "Last modified:" ANSI_NORMAL " %s", std::put_time(std::localtime(&*lastModified), "%F %T")); - if (auto fingerprint = lockedFlake.getFingerprint(store)) + if (auto fingerprint = lockedFlake.getFingerprint(store, fetchSettings)) logger->cout( ANSI_BOLD "Fingerprint:" ANSI_NORMAL " %s", fingerprint->to_string(HashFormat::Base16, false)); diff --git a/tests/functional/flakes/unlocked-override.sh b/tests/functional/flakes/unlocked-override.sh index ebad332d0..dcb427a8f 100755 --- a/tests/functional/flakes/unlocked-override.sh +++ b/tests/functional/flakes/unlocked-override.sh @@ -31,5 +31,14 @@ echo 456 > "$flake1Dir"/x.nix [[ $(nix eval --json "$flake2Dir#x" --override-input flake1 "$TEST_ROOT/flake1") = 456 ]] +# Dirty overrides require --allow-dirty-locks. expectStderr 1 nix flake lock "$flake2Dir" --override-input flake1 "$TEST_ROOT/flake1" | - grepQuiet "cannot write lock file.*because it has an unlocked input" + grepQuiet "Will not write lock file.*because it has an unlocked input" + +nix flake lock "$flake2Dir" --override-input flake1 "$TEST_ROOT/flake1" --allow-dirty-locks + +# Using a lock file with a dirty lock requires --allow-dirty-locks as well. +expectStderr 1 nix eval "$flake2Dir#x" | + grepQuiet "Lock file contains unlocked input" + +[[ $(nix eval "$flake2Dir#x" --allow-dirty-locks) = 456 ]] From 47cf93ba80b8be3701589a25ba5d8e8382b3962f Mon Sep 17 00:00:00 2001 From: Tristan Ross Date: Fri, 10 Jan 2025 18:08:27 -0800 Subject: [PATCH 24/68] Add LLVM to Flake --- flake.nix | 5 +++++ src/libcmd/package.nix | 2 +- src/libexpr/package.nix | 2 +- src/libfetchers/package.nix | 2 +- src/libflake/package.nix | 2 +- src/libmain/package.nix | 2 +- src/libstore/package.nix | 2 +- src/libutil/package.nix | 2 +- src/nix/package.nix | 2 +- 9 files changed, 13 insertions(+), 8 deletions(-) diff --git a/flake.nix b/flake.nix index 83bdc5ecc..8edc2266f 100644 --- a/flake.nix +++ b/flake.nix @@ -107,6 +107,7 @@ in { inherit stdenvs native; static = native.pkgsStatic; + llvm = native.pkgsLLVM; cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); }); @@ -282,6 +283,7 @@ # These attributes go right into `packages.`. "${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName}; "${pkgName}-static" = nixpkgsFor.${system}.static.nixComponents.${pkgName}; + "${pkgName}-llvm" = nixpkgsFor.${system}.llvm.nixComponents.${pkgName}; } // lib.optionalAttrs supportsCross (flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: { # These attributes go right into `packages.`. @@ -321,6 +323,9 @@ prefixAttrs "static" (forAllStdenvs (stdenvName: makeShell { pkgs = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".pkgsStatic; })) // + prefixAttrs "llvm" (forAllStdenvs (stdenvName: makeShell { + pkgs = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".pkgsLLVM; + })) // prefixAttrs "cross" (forAllCrossSystems (crossSystem: makeShell { pkgs = nixpkgsFor.${system}.cross.${crossSystem}; })) diff --git a/src/libcmd/package.nix b/src/libcmd/package.nix index 53e54d2f8..9244a780a 100644 --- a/src/libcmd/package.nix +++ b/src/libcmd/package.nix @@ -76,7 +76,7 @@ mkMesonLibrary (finalAttrs: { (lib.mesonOption "readline-flavor" readlineFlavor) ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM or false)) { LDFLAGS = "-fuse-ld=gold"; }; diff --git a/src/libexpr/package.nix b/src/libexpr/package.nix index 5171d70fd..9ef56b28c 100644 --- a/src/libexpr/package.nix +++ b/src/libexpr/package.nix @@ -96,7 +96,7 @@ mkMesonLibrary (finalAttrs: { # https://github.com/NixOS/nixpkgs/issues/86131. BOOST_INCLUDEDIR = "${lib.getDev boost}/include"; BOOST_LIBRARYDIR = "${lib.getLib boost}/lib"; - } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM or false)) { LDFLAGS = "-fuse-ld=gold"; }; diff --git a/src/libfetchers/package.nix b/src/libfetchers/package.nix index 7dad00025..86d505fbf 100644 --- a/src/libfetchers/package.nix +++ b/src/libfetchers/package.nix @@ -49,7 +49,7 @@ mkMesonLibrary (finalAttrs: { echo ${version} > ../../.version ''; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM or false)) { LDFLAGS = "-fuse-ld=gold"; }; diff --git a/src/libflake/package.nix b/src/libflake/package.nix index 92445739f..29c9fdd80 100644 --- a/src/libflake/package.nix +++ b/src/libflake/package.nix @@ -48,7 +48,7 @@ mkMesonLibrary (finalAttrs: { echo ${version} > ../../.version ''; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM or false)) { LDFLAGS = "-fuse-ld=gold"; }; diff --git a/src/libmain/package.nix b/src/libmain/package.nix index 7d9d99b61..ede1f005b 100644 --- a/src/libmain/package.nix +++ b/src/libmain/package.nix @@ -45,7 +45,7 @@ mkMesonLibrary (finalAttrs: { echo ${version} > ../../.version ''; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM or false)) { LDFLAGS = "-fuse-ld=gold"; }; diff --git a/src/libstore/package.nix b/src/libstore/package.nix index 47a203f83..195efbb8a 100644 --- a/src/libstore/package.nix +++ b/src/libstore/package.nix @@ -87,7 +87,7 @@ mkMesonLibrary (finalAttrs: { # https://github.com/NixOS/nixpkgs/issues/86131. BOOST_INCLUDEDIR = "${lib.getDev boost}/include"; BOOST_LIBRARYDIR = "${lib.getLib boost}/lib"; - } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM or false)) { LDFLAGS = "-fuse-ld=gold"; }; diff --git a/src/libutil/package.nix b/src/libutil/package.nix index 0a9f65782..11f7249ee 100644 --- a/src/libutil/package.nix +++ b/src/libutil/package.nix @@ -72,7 +72,7 @@ mkMesonLibrary (finalAttrs: { # https://github.com/NixOS/nixpkgs/issues/86131. BOOST_INCLUDEDIR = "${lib.getDev boost}/include"; BOOST_LIBRARYDIR = "${lib.getLib boost}/lib"; - } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM or false)) { LDFLAGS = "-fuse-ld=gold"; }; diff --git a/src/nix/package.nix b/src/nix/package.nix index 9bc139c3b..67bf79f3f 100644 --- a/src/nix/package.nix +++ b/src/nix/package.nix @@ -99,7 +99,7 @@ mkMesonExecutable (finalAttrs: { mesonFlags = [ ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM)) { LDFLAGS = "-fuse-ld=gold"; }; From afac093b347dbc8a745d9fbee92257bd1623426f Mon Sep 17 00:00:00 2001 From: Dominique Martinet Date: Sun, 12 Jan 2025 15:01:49 +0900 Subject: [PATCH 25/68] libutil: thread-pool: ensure threads finished on error This fixes segfaults with nix copy when there was an error processing addMultipleToStore. Running with ASAN/TSAN pointed at an use-after-free with threads from the pool accessing the graph declared in processGraph after the function was exiting and destructing the variables. It turns out that if there is an error before pool.process() is called, for example while we are still enqueuing tasks, then pool.process() isn't called and threads are still left to run. By creating the pool last we ensure that it is stopped first before running other destructors even if an exception happens early. [ lix porting note: nix does not name threads so the patch has been adapted to not pass thread name ] Link: https://git.lix.systems/lix-project/lix/issues/618 Link: https://gerrit.lix.systems/c/lix/+/2355 --- src/libstore/store-api.cc | 8 ++------ src/libutil/thread-pool.hh | 5 ++++- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 78cc3b917..3b5167730 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -246,9 +246,7 @@ void Store::addMultipleToStore( act.progress(nrDone, pathsToCopy.size(), nrRunning, nrFailed); }; - ThreadPool pool; - - processGraph(pool, + processGraph( storePathsToAdd, [&](const StorePath & path) { @@ -1028,12 +1026,10 @@ std::map copyPaths( } auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute); - ThreadPool pool; - try { // Copy the realisation closure processGraph( - pool, Realisation::closure(srcStore, toplevelRealisations), + Realisation::closure(srcStore, toplevelRealisations), [&](const Realisation & current) -> std::set { std::set children; for (const auto & [drvOutput, _] : current.dependentRealisations) { diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh index 02765badc..dc056481a 100644 --- a/src/libutil/thread-pool.hh +++ b/src/libutil/thread-pool.hh @@ -83,7 +83,6 @@ private: */ template void processGraph( - ThreadPool & pool, const std::set & nodes, std::function(const T &)> getEdges, std::function processNode) @@ -97,6 +96,10 @@ void processGraph( std::function worker; + /* Create pool last to ensure threads are stopped before other destructors + * run */ + ThreadPool pool; + worker = [&](const T & node) { { From 29a1a21ce4448bc00c63d26a0c4238031c612cba Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 12 Jan 2025 13:50:55 +0100 Subject: [PATCH 26/68] Reject merge conflicts They're usually found by other checks, but docs would remain susceptible. --- maintainers/flake-module.nix | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index 6b311c62e..fcf370b71 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -10,6 +10,27 @@ # https://flake.parts/options/git-hooks-nix#options pre-commit.settings = { hooks = { + # Conflicts are usually found by other checks, but not those in docs, + # and potentially other places. + check-merge-conflicts.enable = true; + # built-in check-merge-conflicts seems ineffective against those produced by mergify backports + check-merge-conflicts-2 = { + enable = true; + entry = "${pkgs.writeScript "check-merge-conflicts" '' + #!${pkgs.runtimeShell} + conflicts=false + for file in "$@"; do + if grep --with-filename --line-number -E '^>>>>>>> ' -- "$file"; then + conflicts=true + fi + done + if $conflicts; then + echo "ERROR: found merge/patch conflicts in files" + exit 1 + fi + touch $out + ''}"; + }; clang-format = { enable = true; # https://github.com/cachix/git-hooks.nix/pull/532 From 4fac767b5295542da040d02807dc4b3a175c0337 Mon Sep 17 00:00:00 2001 From: Andrew Poelstra Date: Sun, 12 Jan 2025 18:36:32 +0000 Subject: [PATCH 27/68] gc: replace ordered sets with unordered sets for in-memory caches During garbage collection we cache several things -- a set of known-dead paths, a set of known-alive paths, and a map of paths to their derivers. Currently they use STL maps and sets, which are ordered structures that typically are backed by binary trees. Since we are putting pseudorandom paths into these and looking them up by exact key, we don't need the ordering, and we're paying a nontrivial cost per insertion. The existing maps require O(n log n) memory and have O(log n) insertion and lookup time. We could instead use unordered maps, which are typically backed by hashmaps. These require O(n) memory and have O(1) insertion and lookup time. On my system this appears to result in a dramatic speedup -- prior to this patch I was able to delete 400k paths out of 9.5 million over the course of 34.5 hours. After this patch the same result took 89 minutes. This result should NOT be taken at face value because the two runs aren't really comparable; in particular the first started when I had 9.5 million store paths and the seconcd started with 7.8 million, so we are deleting a different set of paths starting from a much cleaner filesystem. But I do think it's indicative. Related: https://github.com/NixOS/nix/issues/9581 --- src/libstore/gc.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 45dfe4ad8..ac354f3fa 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -455,7 +455,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) bool gcKeepOutputs = settings.gcKeepOutputs; bool gcKeepDerivations = settings.gcKeepDerivations; - StorePathSet roots, dead, alive; + std::unordered_set roots, dead, alive; struct Shared { @@ -661,7 +661,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } }; - std::map referrersCache; + std::unordered_map referrersCache; /* Helper function that visits all paths reachable from `start` via the referrers edges and optionally derivers and derivation From fd053fdcad7bbd0dcff7a2daefd8011235653f06 Mon Sep 17 00:00:00 2001 From: Siddarth Kumar Date: Mon, 13 Jan 2025 14:12:41 +0530 Subject: [PATCH 28/68] scripts/install-multi-user: fix typo --- scripts/install-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index b65e08682..f051ccc46 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -562,7 +562,7 @@ create_build_user_for_core() { if [ "$actual_uid" != "$uid" ]; then failure < Date: Fri, 15 Nov 2024 10:51:22 +1100 Subject: [PATCH 29/68] windows: create files if they don't exist, and with write permission --- src/libutil/fs-sink.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index dd979f83f..fadba5972 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -112,7 +112,7 @@ void RestoreSink::createRegularFile(const CanonPath & path, std::function Date: Tue, 14 Jan 2025 14:51:49 +0100 Subject: [PATCH 30/68] Add release note --- doc/manual/rl-next/relative-path-flakes.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/manual/rl-next/relative-path-flakes.md diff --git a/doc/manual/rl-next/relative-path-flakes.md b/doc/manual/rl-next/relative-path-flakes.md new file mode 100644 index 000000000..5a4412316 --- /dev/null +++ b/doc/manual/rl-next/relative-path-flakes.md @@ -0,0 +1,12 @@ +--- +synopsis: "Support for relative path inputs" +prs: [10089] +--- + +Flakes can now refer to other flakes in the same repository using relative paths, e.g. +```nix +inputs.foo.url = "path:./foo"; +``` +uses the flake in the `foo` subdirectory of the referring flake. + +This feature required a change to the lock file format. Previous Nix versions will not be able to use lock files that have locks for relative path inputs in them. From ff8e2fe84e4950cd20dbc513cf6eaa1d40d6bf65 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 14 Jan 2025 17:30:13 +0100 Subject: [PATCH 31/68] Fix relative 'path:' flakerefs in the CLI And handle relative 'git+file:' flakerefs while we're at it (these crashed with an assertion failure). Fixes #12248. --- src/libflake/flake/flakeref.cc | 11 ++++++++--- tests/functional/flakes/flakes.sh | 3 +++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index cdf0a40e8..15d1191b3 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -102,8 +102,8 @@ std::pair parsePathFlakeRefWithFragment( if (baseDir) { /* Check if 'url' is a path (either absolute or relative - to 'baseDir'). If so, search upward to the root of the - repo (i.e. the directory containing .git). */ + to 'baseDir'). If so, search upward to the root of the + repo (i.e. the directory containing .git). */ path = absPath(path, baseDir); @@ -232,7 +232,12 @@ std::optional> parseURLFlakeRef( ) { try { - return fromParsedURL(fetchSettings, parseURL(url), isFlake); + auto parsed = parseURL(url); + if (baseDir + && (parsed.scheme == "path" || parsed.scheme == "git+file") + && !hasPrefix(parsed.path, "/")) + parsed.path = absPath(parsed.path, *baseDir); + return fromParsedURL(fetchSettings, std::move(parsed), isFlake); } catch (BadURL &) { return std::nullopt; } diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 6c466a0c7..342c075a8 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -97,6 +97,9 @@ nix build -o "$TEST_ROOT/result" flake1 nix build -o "$TEST_ROOT/result" "$flake1Dir" nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir" +(cd "$flake1Dir" && nix build -o "$TEST_ROOT/result" ".") +(cd "$flake1Dir" && nix build -o "$TEST_ROOT/result" "path:.") +(cd "$flake1Dir" && nix build -o "$TEST_ROOT/result" "git+file:.") # Test explicit packages.default. nix build -o "$TEST_ROOT/result" "$flake1Dir#default" From ff9d886f3cb28db13bfb88d1dc4d6fe3dc83270f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 14 Jan 2025 17:42:26 +0100 Subject: [PATCH 32/68] Use isAbsolute() --- src/libexpr/eval.cc | 2 +- src/libfetchers/path.cc | 2 +- src/libfetchers/registry.cc | 2 +- src/libflake/flake/flakeref.cc | 4 ++-- src/libutil/file-system.cc | 7 +------ src/libutil/file-system.hh | 5 +++++ src/libutil/source-accessor.cc | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 21dd5a294..345c09e7e 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -406,7 +406,7 @@ void EvalState::checkURI(const std::string & uri) /* If the URI is a path, then check it against allowedPaths as well. */ - if (hasPrefix(uri, "/")) { + if (isAbsolute(uri)) { if (auto rootFS2 = rootFS.dynamic_pointer_cast()) rootFS2->checkAccess(CanonPath(uri)); return; diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index f628e042c..340e7f4fa 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -97,7 +97,7 @@ struct PathInputScheme : InputScheme std::optional isRelative(const Input & input) const { auto path = getStrAttr(input.attrs, "path"); - if (hasPrefix(path, "/")) + if (isAbsolute(path)) return std::nullopt; else return path; diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 171afcea7..c18e12d23 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -153,7 +153,7 @@ static std::shared_ptr getGlobalRegistry(const Settings & settings, re return std::make_shared(settings, Registry::Global); // empty registry } - if (!hasPrefix(path, "/")) { + if (!isAbsolute(path)) { auto storePath = downloadFile(store, path, "flake-registry.json").storePath; if (auto store2 = store.dynamic_pointer_cast()) store2->addPermRoot(storePath, getCacheDir() + "/flake-registry.json"); diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index 15d1191b3..b81eb1292 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -178,7 +178,7 @@ std::pair parsePathFlakeRefWithFragment( } } else { - if (!hasPrefix(path, "/")) + if (!isAbsolute(path)) throw BadURL("flake reference '%s' is not an absolute path", url); path = canonPath(path + "/" + getOr(query, "dir", "")); } @@ -235,7 +235,7 @@ std::optional> parseURLFlakeRef( auto parsed = parseURL(url); if (baseDir && (parsed.scheme == "path" || parsed.scheme == "git+file") - && !hasPrefix(parsed.path, "/")) + && !isAbsolute(parsed.path)) parsed.path = absPath(parsed.path, *baseDir); return fromParsedURL(fetchSettings, std::move(parsed), isFlake); } catch (BadURL &) { diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 923220fd0..a86f1ba4f 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -31,12 +31,7 @@ namespace nix { namespace fs { using namespace std::filesystem; } -/** - * Treat the string as possibly an absolute path, by inspecting the - * start of it. Return whether it was probably intended to be - * absolute. - */ -static bool isAbsolute(PathView path) +bool isAbsolute(PathView path) { return fs::path { path }.is_absolute(); } diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index 7fdaba811..204907339 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -42,6 +42,11 @@ namespace nix { struct Sink; struct Source; +/** + * Return whether the path denotes an absolute path. + */ +bool isAbsolute(PathView path); + /** * @return An absolutized path, resolving paths relative to the * specified directory, or the current directory otherwise. The path diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index aa8dcf9fd..78f038cf3 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -115,7 +115,7 @@ CanonPath SourceAccessor::resolveSymlinks( throw Error("infinite symlink recursion in path '%s'", showPath(path)); auto target = readLink(res); res.pop(); - if (hasPrefix(target, "/")) + if (isAbsolute(target)) res = CanonPath::root; todo.splice(todo.begin(), tokenizeString>(target, "/")); } From 6a874c2865636348d192f436d22cd5a10f26a29a Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Wed, 15 Jan 2025 08:59:14 -0600 Subject: [PATCH 33/68] sequoia-nixbld-user-migration: nail down PATH Fixes a user report of trouble with toybox grep and avoids potential of same basic issue with other utils. --- scripts/sequoia-nixbld-user-migration.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/sequoia-nixbld-user-migration.sh b/scripts/sequoia-nixbld-user-migration.sh index 88e801706..58b5fea64 100755 --- a/scripts/sequoia-nixbld-user-migration.sh +++ b/scripts/sequoia-nixbld-user-migration.sh @@ -2,6 +2,9 @@ set -eo pipefail +# stock path to avoid unexpected command versions +PATH="$(/usr/bin/getconf PATH)" + ((NEW_NIX_FIRST_BUILD_UID=351)) ((TEMP_NIX_FIRST_BUILD_UID=31000)) From 3d078cd508cf9fa3b623cffc2d26a6942b44b720 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 15 Jan 2025 20:00:23 +0100 Subject: [PATCH 34/68] Add RossComputerGuy as `-llvm` maintainer --- flake.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/flake.nix b/flake.nix index 8edc2266f..9d224310b 100644 --- a/flake.nix +++ b/flake.nix @@ -283,6 +283,10 @@ # These attributes go right into `packages.`. "${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName}; "${pkgName}-static" = nixpkgsFor.${system}.static.nixComponents.${pkgName}; + /** + Nix and dependencies built with LLVM and Clang, maintainers: + @RossComputerGuy, Nix team + */ "${pkgName}-llvm" = nixpkgsFor.${system}.llvm.nixComponents.${pkgName}; } // lib.optionalAttrs supportsCross (flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: { From cab347b4ebb2f8fe2e06a84a29c1c3027b9edc85 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 15 Jan 2025 19:54:02 +0100 Subject: [PATCH 35/68] refactor: Move ld=gold rule to mesonBuildLayer --- flake.nix | 4 ---- packaging/dependencies.nix | 6 ++++++ src/libcmd/package.nix | 4 ---- src/libexpr-c/package.nix | 4 ---- src/libexpr-test-support/package.nix | 4 ---- src/libexpr-tests/package.nix | 4 ---- src/libexpr/package.nix | 2 -- src/libfetchers-tests/package.nix | 4 ---- src/libfetchers/package.nix | 4 ---- src/libflake-c/package.nix | 4 ---- src/libflake-tests/package.nix | 4 ---- src/libflake/package.nix | 4 ---- src/libmain-c/package.nix | 4 ---- src/libmain/package.nix | 4 ---- src/libstore-c/package.nix | 4 ---- src/libstore-test-support/package.nix | 4 ---- src/libstore-tests/package.nix | 4 ---- src/libstore/package.nix | 2 -- src/libutil-c/package.nix | 4 ---- src/libutil-test-support/package.nix | 4 ---- src/libutil-tests/package.nix | 4 ---- src/libutil/package.nix | 2 -- src/nix/package.nix | 4 ---- 23 files changed, 6 insertions(+), 82 deletions(-) diff --git a/flake.nix b/flake.nix index 9d224310b..8edc2266f 100644 --- a/flake.nix +++ b/flake.nix @@ -283,10 +283,6 @@ # These attributes go right into `packages.`. "${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName}; "${pkgName}-static" = nixpkgsFor.${system}.static.nixComponents.${pkgName}; - /** - Nix and dependencies built with LLVM and Clang, maintainers: - @RossComputerGuy, Nix team - */ "${pkgName}-llvm" = nixpkgsFor.${system}.llvm.nixComponents.${pkgName}; } // lib.optionalAttrs supportsCross (flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: { diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 4bc7495e7..08d179b82 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -97,6 +97,12 @@ let ]; separateDebugInfo = !stdenv.hostPlatform.isStatic; hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + env = prevAttrs.env or {} + // lib.optionalAttrs + (stdenv.isLinux + && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") + && !(stdenv.hostPlatform.useLLVM or false)) + { LDFLAGS = "-fuse-ld=gold"; }; }; mesonLibraryLayer = finalAttrs: prevAttrs: diff --git a/src/libcmd/package.nix b/src/libcmd/package.nix index 9244a780a..5cafb4dc1 100644 --- a/src/libcmd/package.nix +++ b/src/libcmd/package.nix @@ -76,10 +76,6 @@ mkMesonLibrary (finalAttrs: { (lib.mesonOption "readline-flavor" readlineFlavor) ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM or false)) { - LDFLAGS = "-fuse-ld=gold"; - }; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libexpr-c/package.nix b/src/libexpr-c/package.nix index 727b3a811..22726ac36 100644 --- a/src/libexpr-c/package.nix +++ b/src/libexpr-c/package.nix @@ -47,10 +47,6 @@ mkMesonLibrary (finalAttrs: { mesonFlags = [ ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { - LDFLAGS = "-fuse-ld=gold"; - }; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libexpr-test-support/package.nix b/src/libexpr-test-support/package.nix index 4842f5f17..9726fa57a 100644 --- a/src/libexpr-test-support/package.nix +++ b/src/libexpr-test-support/package.nix @@ -51,10 +51,6 @@ mkMesonLibrary (finalAttrs: { mesonFlags = [ ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { - LDFLAGS = "-fuse-ld=gold"; - }; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libexpr-tests/package.nix b/src/libexpr-tests/package.nix index 70e497b7e..a4a3bb0e7 100644 --- a/src/libexpr-tests/package.nix +++ b/src/libexpr-tests/package.nix @@ -56,10 +56,6 @@ mkMesonExecutable (finalAttrs: { mesonFlags = [ ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { - LDFLAGS = "-fuse-ld=gold"; - }; - passthru = { tests = { run = runCommand "${finalAttrs.pname}-run" { diff --git a/src/libexpr/package.nix b/src/libexpr/package.nix index 9ef56b28c..3d5b78e35 100644 --- a/src/libexpr/package.nix +++ b/src/libexpr/package.nix @@ -96,8 +96,6 @@ mkMesonLibrary (finalAttrs: { # https://github.com/NixOS/nixpkgs/issues/86131. BOOST_INCLUDEDIR = "${lib.getDev boost}/include"; BOOST_LIBRARYDIR = "${lib.getLib boost}/lib"; - } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM or false)) { - LDFLAGS = "-fuse-ld=gold"; }; meta = { diff --git a/src/libfetchers-tests/package.nix b/src/libfetchers-tests/package.nix index e18d644ed..5336672a2 100644 --- a/src/libfetchers-tests/package.nix +++ b/src/libfetchers-tests/package.nix @@ -54,10 +54,6 @@ mkMesonExecutable (finalAttrs: { mesonFlags = [ ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { - LDFLAGS = "-fuse-ld=gold"; - }; - passthru = { tests = { run = runCommand "${finalAttrs.pname}-run" { diff --git a/src/libfetchers/package.nix b/src/libfetchers/package.nix index 86d505fbf..79dd81cc9 100644 --- a/src/libfetchers/package.nix +++ b/src/libfetchers/package.nix @@ -49,10 +49,6 @@ mkMesonLibrary (finalAttrs: { echo ${version} > ../../.version ''; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM or false)) { - LDFLAGS = "-fuse-ld=gold"; - }; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libflake-c/package.nix b/src/libflake-c/package.nix index 7425d6140..9d339dd0a 100644 --- a/src/libflake-c/package.nix +++ b/src/libflake-c/package.nix @@ -49,10 +49,6 @@ mkMesonLibrary (finalAttrs: { mesonFlags = [ ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { - LDFLAGS = "-fuse-ld=gold"; - }; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libflake-tests/package.nix b/src/libflake-tests/package.nix index f1abbb32d..51b68ad58 100644 --- a/src/libflake-tests/package.nix +++ b/src/libflake-tests/package.nix @@ -56,10 +56,6 @@ mkMesonExecutable (finalAttrs: { mesonFlags = [ ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { - LDFLAGS = "-fuse-ld=gold"; - }; - passthru = { tests = { run = runCommand "${finalAttrs.pname}-run" { diff --git a/src/libflake/package.nix b/src/libflake/package.nix index 29c9fdd80..868832b75 100644 --- a/src/libflake/package.nix +++ b/src/libflake/package.nix @@ -48,10 +48,6 @@ mkMesonLibrary (finalAttrs: { echo ${version} > ../../.version ''; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM or false)) { - LDFLAGS = "-fuse-ld=gold"; - }; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libmain-c/package.nix b/src/libmain-c/package.nix index d65792921..ba47d5cb9 100644 --- a/src/libmain-c/package.nix +++ b/src/libmain-c/package.nix @@ -51,10 +51,6 @@ mkMesonLibrary (finalAttrs: { mesonFlags = [ ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { - LDFLAGS = "-fuse-ld=gold"; - }; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libmain/package.nix b/src/libmain/package.nix index ede1f005b..856aa660f 100644 --- a/src/libmain/package.nix +++ b/src/libmain/package.nix @@ -45,10 +45,6 @@ mkMesonLibrary (finalAttrs: { echo ${version} > ../../.version ''; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM or false)) { - LDFLAGS = "-fuse-ld=gold"; - }; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libstore-c/package.nix b/src/libstore-c/package.nix index 351d4510a..e9b548473 100644 --- a/src/libstore-c/package.nix +++ b/src/libstore-c/package.nix @@ -47,10 +47,6 @@ mkMesonLibrary (finalAttrs: { mesonFlags = [ ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { - LDFLAGS = "-fuse-ld=gold"; - }; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libstore-test-support/package.nix b/src/libstore-test-support/package.nix index 62f783c4f..705297b4c 100644 --- a/src/libstore-test-support/package.nix +++ b/src/libstore-test-support/package.nix @@ -51,10 +51,6 @@ mkMesonLibrary (finalAttrs: { mesonFlags = [ ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { - LDFLAGS = "-fuse-ld=gold"; - }; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libstore-tests/package.nix b/src/libstore-tests/package.nix index 4937b5329..3acf4e25c 100644 --- a/src/libstore-tests/package.nix +++ b/src/libstore-tests/package.nix @@ -62,10 +62,6 @@ mkMesonExecutable (finalAttrs: { mesonFlags = [ ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { - LDFLAGS = "-fuse-ld=gold"; - }; - passthru = { tests = { run = let diff --git a/src/libstore/package.nix b/src/libstore/package.nix index 195efbb8a..4fbaea4ac 100644 --- a/src/libstore/package.nix +++ b/src/libstore/package.nix @@ -87,8 +87,6 @@ mkMesonLibrary (finalAttrs: { # https://github.com/NixOS/nixpkgs/issues/86131. BOOST_INCLUDEDIR = "${lib.getDev boost}/include"; BOOST_LIBRARYDIR = "${lib.getLib boost}/lib"; - } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM or false)) { - LDFLAGS = "-fuse-ld=gold"; }; meta = { diff --git a/src/libutil-c/package.nix b/src/libutil-c/package.nix index 4caf96804..294593b52 100644 --- a/src/libutil-c/package.nix +++ b/src/libutil-c/package.nix @@ -45,10 +45,6 @@ mkMesonLibrary (finalAttrs: { mesonFlags = [ ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { - LDFLAGS = "-fuse-ld=gold"; - }; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libutil-test-support/package.nix b/src/libutil-test-support/package.nix index 19b5d6b77..8185a701f 100644 --- a/src/libutil-test-support/package.nix +++ b/src/libutil-test-support/package.nix @@ -49,10 +49,6 @@ mkMesonLibrary (finalAttrs: { mesonFlags = [ ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { - LDFLAGS = "-fuse-ld=gold"; - }; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libutil-tests/package.nix b/src/libutil-tests/package.nix index f06704e26..28769e115 100644 --- a/src/libutil-tests/package.nix +++ b/src/libutil-tests/package.nix @@ -55,10 +55,6 @@ mkMesonExecutable (finalAttrs: { mesonFlags = [ ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { - LDFLAGS = "-fuse-ld=gold"; - }; - passthru = { tests = { run = runCommand "${finalAttrs.pname}-run" { diff --git a/src/libutil/package.nix b/src/libutil/package.nix index 11f7249ee..679872a75 100644 --- a/src/libutil/package.nix +++ b/src/libutil/package.nix @@ -72,8 +72,6 @@ mkMesonLibrary (finalAttrs: { # https://github.com/NixOS/nixpkgs/issues/86131. BOOST_INCLUDEDIR = "${lib.getDev boost}/include"; BOOST_LIBRARYDIR = "${lib.getLib boost}/lib"; - } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM or false)) { - LDFLAGS = "-fuse-ld=gold"; }; meta = { diff --git a/src/nix/package.nix b/src/nix/package.nix index 67bf79f3f..ac0181529 100644 --- a/src/nix/package.nix +++ b/src/nix/package.nix @@ -99,10 +99,6 @@ mkMesonExecutable (finalAttrs: { mesonFlags = [ ]; - env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux") && !(stdenv.hostPlatform.useLLVM)) { - LDFLAGS = "-fuse-ld=gold"; - }; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; From 7ad02b62e04e8845243f2c80d9891375b465d0af Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 15 Jan 2025 19:58:36 +0100 Subject: [PATCH 36/68] refactor: Remove redundant parameters --- src/libexpr-c/package.nix | 1 - src/libexpr-test-support/package.nix | 1 - src/libfetchers/package.nix | 1 - src/libflake-c/package.nix | 1 - src/libflake/package.nix | 1 - src/libmain-c/package.nix | 1 - src/libmain/package.nix | 1 - src/libstore-c/package.nix | 1 - src/libstore-test-support/package.nix | 1 - src/libutil-c/package.nix | 1 - src/libutil-test-support/package.nix | 1 - src/nix/package.nix | 1 - 12 files changed, 12 deletions(-) diff --git a/src/libexpr-c/package.nix b/src/libexpr-c/package.nix index 22726ac36..5047f3e2e 100644 --- a/src/libexpr-c/package.nix +++ b/src/libexpr-c/package.nix @@ -1,5 +1,4 @@ { lib -, stdenv , mkMesonLibrary , nix-store-c diff --git a/src/libexpr-test-support/package.nix b/src/libexpr-test-support/package.nix index 9726fa57a..48118fa0c 100644 --- a/src/libexpr-test-support/package.nix +++ b/src/libexpr-test-support/package.nix @@ -1,5 +1,4 @@ { lib -, stdenv , mkMesonLibrary , nix-store-test-support diff --git a/src/libfetchers/package.nix b/src/libfetchers/package.nix index 79dd81cc9..d4ca18555 100644 --- a/src/libfetchers/package.nix +++ b/src/libfetchers/package.nix @@ -1,5 +1,4 @@ { lib -, stdenv , mkMesonLibrary , nix-util diff --git a/src/libflake-c/package.nix b/src/libflake-c/package.nix index 9d339dd0a..dcd6c4966 100644 --- a/src/libflake-c/package.nix +++ b/src/libflake-c/package.nix @@ -1,5 +1,4 @@ { lib -, stdenv , mkMesonLibrary , nix-store-c diff --git a/src/libflake/package.nix b/src/libflake/package.nix index 868832b75..3fc96a20e 100644 --- a/src/libflake/package.nix +++ b/src/libflake/package.nix @@ -1,5 +1,4 @@ { lib -, stdenv , mkMesonLibrary , nix-util diff --git a/src/libmain-c/package.nix b/src/libmain-c/package.nix index ba47d5cb9..b96901bb4 100644 --- a/src/libmain-c/package.nix +++ b/src/libmain-c/package.nix @@ -1,5 +1,4 @@ { lib -, stdenv , mkMesonLibrary , nix-util-c diff --git a/src/libmain/package.nix b/src/libmain/package.nix index 856aa660f..9a5b9e8c2 100644 --- a/src/libmain/package.nix +++ b/src/libmain/package.nix @@ -1,5 +1,4 @@ { lib -, stdenv , mkMesonLibrary , openssl diff --git a/src/libstore-c/package.nix b/src/libstore-c/package.nix index e9b548473..c2413c389 100644 --- a/src/libstore-c/package.nix +++ b/src/libstore-c/package.nix @@ -1,5 +1,4 @@ { lib -, stdenv , mkMesonLibrary , nix-util-c diff --git a/src/libstore-test-support/package.nix b/src/libstore-test-support/package.nix index 705297b4c..5d3f41b3e 100644 --- a/src/libstore-test-support/package.nix +++ b/src/libstore-test-support/package.nix @@ -1,5 +1,4 @@ { lib -, stdenv , mkMesonLibrary , nix-util-test-support diff --git a/src/libutil-c/package.nix b/src/libutil-c/package.nix index 294593b52..f80e0b7f0 100644 --- a/src/libutil-c/package.nix +++ b/src/libutil-c/package.nix @@ -1,5 +1,4 @@ { lib -, stdenv , mkMesonLibrary , nix-util diff --git a/src/libutil-test-support/package.nix b/src/libutil-test-support/package.nix index 8185a701f..a8a239717 100644 --- a/src/libutil-test-support/package.nix +++ b/src/libutil-test-support/package.nix @@ -1,5 +1,4 @@ { lib -, stdenv , mkMesonLibrary , nix-util diff --git a/src/nix/package.nix b/src/nix/package.nix index ac0181529..171621af9 100644 --- a/src/nix/package.nix +++ b/src/nix/package.nix @@ -1,5 +1,4 @@ { lib -, stdenv , mkMesonExecutable , nix-store From 521667eb8908843925dc57e195f7702d66d0ac16 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Jan 2025 11:19:20 +0100 Subject: [PATCH 37/68] Fix follow-paths test Since ff8e2fe84e4950cd20dbc513cf6eaa1d40d6bf65, 'path:' URLs on the CLI are interpreted as relative to the current directory of the user, not the path of the flake we're overriding. --- tests/functional/flakes/follow-paths.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/flakes/follow-paths.sh b/tests/functional/flakes/follow-paths.sh index 966f69cf7..a71d4c6d7 100755 --- a/tests/functional/flakes/follow-paths.sh +++ b/tests/functional/flakes/follow-paths.sh @@ -356,6 +356,6 @@ json=$(nix flake metadata "$flakeFollowsCustomUrlA" --json) rm "$flakeFollowsCustomUrlA"/flake.lock # if override-input is specified, lock "original" entry should contain original url -json=$(nix flake metadata "$flakeFollowsCustomUrlA" --override-input B/C "path:./flakeB/flakeD" --json) +json=$(nix flake metadata "$flakeFollowsCustomUrlA" --override-input B/C "$flakeFollowsCustomUrlD" --json) echo "$json" | jq .locks.nodes.C.original [[ $(echo "$json" | jq -r .locks.nodes.C.original.path) = './flakeC' ]] From 5d03ef9caf136de06b79298b4a5f8c80a4abadbf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Jan 2025 11:26:14 +0100 Subject: [PATCH 38/68] PathInputSchema::getAbsPath(): Return std::filesystem::path --- src/libfetchers/path.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 9691cc9c5..351e21a1b 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -91,7 +91,7 @@ struct PathInputScheme : InputScheme std::string_view contents, std::optional commitMsg) const override { - writeFile((CanonPath(getAbsPath(input)) / path).abs(), contents); + writeFile(getAbsPath(input) / path.rel(), contents); } std::optional isRelative(const Input & input) const override @@ -108,12 +108,12 @@ struct PathInputScheme : InputScheme return (bool) input.getNarHash(); } - CanonPath getAbsPath(const Input & input) const + std::filesystem::path getAbsPath(const Input & input) const { auto path = getStrAttr(input.attrs, "path"); - if (path[0] == '/') - return CanonPath(path); + if (isAbsolute(path)) + return canonPath(path); throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string()); } @@ -128,7 +128,7 @@ struct PathInputScheme : InputScheme Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s' to the store", absPath)); // FIXME: check whether access to 'path' is allowed. - auto storePath = store->maybeParseStorePath(absPath.abs()); + auto storePath = store->maybeParseStorePath(absPath.string()); if (storePath) store->addTempRoot(*storePath); @@ -137,7 +137,7 @@ struct PathInputScheme : InputScheme if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) { // FIXME: try to substitute storePath. auto src = sinkToSource([&](Sink & sink) { - mtime = dumpPathAndGetMtime(absPath.abs(), sink, defaultPathFilter); + mtime = dumpPathAndGetMtime(absPath.string(), sink, defaultPathFilter); }); storePath = store->addToStoreFromDump(*src, "source"); } @@ -159,7 +159,7 @@ struct PathInputScheme : InputScheme store object and the subpath. */ auto path = getAbsPath(input); try { - auto [storePath, subPath] = store->toStorePath(path.abs()); + auto [storePath, subPath] = store->toStorePath(path.string()); auto info = store->queryPathInfo(storePath); return fmt("path:%s:%s", info->narHash.to_string(HashFormat::Base16, false), subPath); } catch (Error &) { From 8b1fb92a0ce7fd8c0ad7955de1a59bb42866339e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Jan 2025 11:31:22 +0100 Subject: [PATCH 39/68] flakes.md: Fix indentation that broke the list --- src/nix/flake.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/nix/flake.md b/src/nix/flake.md index e79a8ee0b..2072dd3dd 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -214,12 +214,12 @@ Currently the `type` attribute can be one of the following: hand, a flake in the `/foo` directory of a tree can use `path:../bar` to refer to the flake in `/bar`. -Path inputs can be specified with path values in `flake.nix`. Path values are a syntax for `path` inputs, and they are converted by -1. resolving them into relative paths, relative to the base directory of `flake.nix` -2. escaping URL characters (refer to IETF RFC?) -3. prepending `path:` + Path inputs can be specified with path values in `flake.nix`. Path values are a syntax for `path` inputs, and they are converted by + 1. resolving them into relative paths, relative to the base directory of `flake.nix` + 2. escaping URL characters (refer to IETF RFC?) + 3. prepending `path:` -Note that the allowed syntax for path values in flake `inputs` may be more restrictive than general Nix, so you may need to use `path:` if your path contains certain special characters. See [Path literals](@docroot@/language/syntax.md#path-literal) + Note that the allowed syntax for path values in flake `inputs` may be more restrictive than general Nix, so you may need to use `path:` if your path contains certain special characters. See [Path literals](@docroot@/language/syntax.md#path-literal) Note that if you omit `path:`, relative paths must start with `.` to avoid ambiguity with registry lookups (e.g. `nixpkgs` is a registry From db46d40b12cc57de47c468a73262f1c20b0374c8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Jan 2025 13:15:20 +0100 Subject: [PATCH 40/68] Update release note --- doc/manual/rl-next/relative-path-flakes.md | 2 +- src/nix/flake.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/rl-next/relative-path-flakes.md b/doc/manual/rl-next/relative-path-flakes.md index 5a4412316..3616f3467 100644 --- a/doc/manual/rl-next/relative-path-flakes.md +++ b/doc/manual/rl-next/relative-path-flakes.md @@ -7,6 +7,6 @@ Flakes can now refer to other flakes in the same repository using relative paths ```nix inputs.foo.url = "path:./foo"; ``` -uses the flake in the `foo` subdirectory of the referring flake. +uses the flake in the `foo` subdirectory of the referring flake. For more information, see the documentation on [the `path` flake input type](@docroot@/command-ref/new-cli/nix3-flake.md#path-fetcher). This feature required a change to the lock file format. Previous Nix versions will not be able to use lock files that have locks for relative path inputs in them. diff --git a/src/nix/flake.md b/src/nix/flake.md index 2072dd3dd..364302b61 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -187,7 +187,7 @@ Currently the `type` attribute can be one of the following: * `nixpkgs/nixos-unstable/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293` * `sub/dir` (if a flake named `sub` is in the registry) -* `path`: arbitrary local directories. The required attribute `path` +* `path`: arbitrary local directories. The required attribute `path` specifies the path of the flake. The URL form is ``` From 2ca0c62a8d577e9b4bc1e6d313239e0cb9452ca2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Jan 2025 17:06:59 +0100 Subject: [PATCH 41/68] Remove some unnecessary quotes around std::filesystem::path --- src/libutil/file-system.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index a86f1ba4f..6fe93b63a 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -643,7 +643,7 @@ void setWriteTime( // doesn't support access time just modification time. // // System clock vs File clock issues also make that annoying. - warn("Changing file times is not yet implemented on Windows, path is '%s'", path); + warn("Changing file times is not yet implemented on Windows, path is %s", path); #elif HAVE_UTIMENSAT && HAVE_DECL_AT_SYMLINK_NOFOLLOW struct timespec times[2] = { { @@ -656,7 +656,7 @@ void setWriteTime( }, }; if (utimensat(AT_FDCWD, path.c_str(), times, AT_SYMLINK_NOFOLLOW) == -1) - throw SysError("changing modification time of '%s' (using `utimensat`)", path); + throw SysError("changing modification time of %s (using `utimensat`)", path); #else struct timeval times[2] = { { @@ -670,7 +670,7 @@ void setWriteTime( }; #if HAVE_LUTIMES if (lutimes(path.c_str(), times) == -1) - throw SysError("changing modification time of '%s'", path); + throw SysError("changing modification time of %s", path); #else bool isSymlink = optIsSymlink ? *optIsSymlink @@ -678,9 +678,9 @@ void setWriteTime( if (!isSymlink) { if (utimes(path.c_str(), times) == -1) - throw SysError("changing modification time of '%s' (not a symlink)", path); + throw SysError("changing modification time of %s (not a symlink)", path); } else { - throw Error("Cannot modification time of symlink '%s'", path); + throw Error("Cannot modification time of symlink %s", path); } #endif #endif @@ -709,7 +709,7 @@ void copyFile(const fs::path & from, const fs::path & to, bool andDelete) copyFile(entry, to / entry.path().filename(), andDelete); } } else { - throw Error("file '%s' has an unsupported type", from); + throw Error("file %s has an unsupported type", from); } setWriteTime(to, lstat(from.string().c_str())); @@ -736,7 +736,7 @@ void moveFile(const Path & oldName, const Path & newName) auto tempCopyTarget = temp / "copy-target"; if (e.code().value() == EXDEV) { fs::remove(newPath); - warn("Can’t rename %s as %s, copying instead", oldName, newName); + warn("can’t rename %s as %s, copying instead", oldName, newName); copyFile(oldPath, tempCopyTarget, true); std::filesystem::rename( os_string_to_string(PathViewNG { tempCopyTarget }), From 12e14956e22a42c26715ca084bb6b17d35628bb9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Jan 2025 17:22:54 +0100 Subject: [PATCH 42/68] Warn against the use of relative 'git+file:' flake inputs --- src/libfetchers/git.cc | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index c4dfc27a2..3f196da54 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -435,7 +435,16 @@ struct GitInputScheme : InputScheme // // See: https://discourse.nixos.org/t/57783 and #9708 // - repoInfo.url = repoInfo.isLocal ? std::filesystem::absolute(url.path).string() : url.to_string(); + if (repoInfo.isLocal) { + if (!isAbsolute(url.path)) { + warn( + "Fetching Git repository '%s', which uses a path relative to the current directory. " + "This is not supported and will stop working in a future release.", + url); + } + repoInfo.url = std::filesystem::absolute(url.path).string(); + } else + repoInfo.url = url.to_string(); // If this is a local directory and no ref or revision is // given, then allow the use of an unclean working tree. From 3716ded8df6de1f38527b6b3fab72427f92c73f3 Mon Sep 17 00:00:00 2001 From: Andy Hamon Date: Thu, 16 Jan 2025 00:24:47 -0800 Subject: [PATCH 43/68] nix-env: add a --priority flag to --install nix-env can read priorities from a derivations meta attributes, but this only works when installing a nix expression. nix-env can also install bare store paths, however meta attributes are not readable in that case. This means that a store path can not be installed with a specific priority. Some cases where it is advantageous to install a store path: a remote host following a `nix copy`, or any time you want to save some evaluation time and happen to already know the store path. This PR addresses this shortcoming by adding a --priority flag to nix-env --install. --- .../source/command-ref/nix-env/install.md | 6 +++- src/nix-env/nix-env.cc | 35 ++++++++++++------- tests/functional/user-envs-test-case.sh | 10 +++++- 3 files changed, 37 insertions(+), 14 deletions(-) diff --git a/doc/manual/source/command-ref/nix-env/install.md b/doc/manual/source/command-ref/nix-env/install.md index 748dd1e7a..aa5c2fbba 100644 --- a/doc/manual/source/command-ref/nix-env/install.md +++ b/doc/manual/source/command-ref/nix-env/install.md @@ -11,6 +11,7 @@ [`--from-profile` *path*] [`--preserve-installed` | `-P`] [`--remove-all` | `-r`] + [`--priority` *priority*] # Description @@ -61,6 +62,10 @@ The arguments *args* map to store paths in a number of possible ways: The derivations returned by those function calls are installed. This allows derivations to be specified in an unambiguous way, which is necessary if there are multiple derivations with the same name. +- If `--priority` *priority* is given, the priority of the derivations being installed is set to *priority*. + This can be used to override the priority of the derivations being installed. + This is useful if *args* are [store paths], which don't have any priority information. + - If *args* are [store derivations](@docroot@/glossary.md#gloss-store-derivation), then these are [realised], and the resulting output paths are installed. - If *args* are [store paths] that are not store derivations, then these are [realised] and installed. @@ -235,4 +240,3 @@ channel: ```console $ nix-env --file https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz --install --attr firefox ``` - diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index e9eb52708..fa6d6afb3 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -501,9 +501,17 @@ static bool keep(PackageInfo & drv) return drv.queryMetaBool("keep", false); } +static void setMetaFlag(EvalState & state, PackageInfo & drv, + const std::string & name, const std::string & value) +{ + auto v = state.allocValue(); + v->mkString(value); + drv.setMeta(name, v); +} + static void installDerivations(Globals & globals, - const Strings & args, const Path & profile) + const Strings & args, const Path & profile, std::optional priority) { debug("installing derivations"); @@ -527,6 +535,11 @@ static void installDerivations(Globals & globals, newNames.insert(DrvName(i.queryName()).name); } + if (priority) { + for (auto & drv : newElems) { + setMetaFlag(*globals.state, drv, "priority", std::to_string((priority.value()))); + } + } while (true) { auto lockToken = optimisticLockProfile(profile); @@ -564,6 +577,7 @@ static void installDerivations(Globals & globals, static void opInstall(Globals & globals, Strings opFlags, Strings opArgs) { + std::optional priority; for (Strings::iterator i = opFlags.begin(); i != opFlags.end(); ) { auto arg = *i++; if (parseInstallSourceOptions(globals, i, opFlags, arg)) ; @@ -571,10 +585,15 @@ static void opInstall(Globals & globals, Strings opFlags, Strings opArgs) globals.preserveInstalled = true; else if (arg == "--remove-all" || arg == "-r") globals.removeAll = true; + else if (arg == "--priority") { + if (i == opFlags.end()) + throw UsageError("'%1%' requires an argument", arg); + priority = std::stoi(*i++); + } else throw UsageError("unknown flag '%1%'", arg); } - installDerivations(globals, opArgs, globals.profile); + installDerivations(globals, opArgs, globals.profile, priority); } @@ -689,15 +708,6 @@ static void opUpgrade(Globals & globals, Strings opFlags, Strings opArgs) } -static void setMetaFlag(EvalState & state, PackageInfo & drv, - const std::string & name, const std::string & value) -{ - auto v = state.allocValue(); - v->mkString(value); - drv.setMeta(name, v); -} - - static void opSetFlag(Globals & globals, Strings opFlags, Strings opArgs) { if (opFlags.size() > 0) @@ -1507,7 +1517,8 @@ static int main_nix_env(int argc, char * * argv) opFlags.push_back(*arg); /* FIXME: hacky */ if (*arg == "--from-profile" || - (op == opQuery && (*arg == "--attr" || *arg == "-A"))) + (op == opQuery && (*arg == "--attr" || *arg == "-A")) || + (op == opInstall && (*arg == "--priority"))) opFlags.push_back(getArg(*arg, arg, end)); } else diff --git a/tests/functional/user-envs-test-case.sh b/tests/functional/user-envs-test-case.sh index 117c6c7a4..3483a4600 100644 --- a/tests/functional/user-envs-test-case.sh +++ b/tests/functional/user-envs-test-case.sh @@ -173,13 +173,21 @@ nix-env -q '*' | grepQuiet bar-0.1.1 # Test priorities: foo-0.1 has a lower priority than foo-1.0, so it # should be possible to install both without a collision. Also test -# ‘--set-flag priority’ to manually override the declared priorities. +# '-i --priority' and '--set-flag priority' to manually override the +# declared priorities. nix-env -e '*' nix-env -i foo-0.1 foo-1.0 [ "$($profiles/test/bin/foo)" = "foo-1.0" ] nix-env --set-flag priority 1 foo-0.1 [ "$($profiles/test/bin/foo)" = "foo-0.1" ] +# Priorities can be overridden with the --priority flag +nix-env -e '*' +nix-env -i foo-1.0 +[ "$($profiles/test/bin/foo)" = "foo-1.0" ] +nix-env -i --priority 1 foo-0.1 +[ "$($profiles/test/bin/foo)" = "foo-0.1" ] + # Test nix-env --set. nix-env --set $outPath10 [ "$(nix-store -q --resolve $profiles/test)" = $outPath10 ] From 6ea339ce8a5f0e925f601990c9fb0c700be29187 Mon Sep 17 00:00:00 2001 From: Andy Hamon Date: Thu, 16 Jan 2025 11:53:43 -0800 Subject: [PATCH 44/68] Update src/nix-env/nix-env.cc Co-authored-by: Eelco Dolstra --- src/nix-env/nix-env.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index fa6d6afb3..c99c1088e 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -588,7 +588,9 @@ static void opInstall(Globals & globals, Strings opFlags, Strings opArgs) else if (arg == "--priority") { if (i == opFlags.end()) throw UsageError("'%1%' requires an argument", arg); - priority = std::stoi(*i++); + priority = string2Int(*i++); + if (!priority) + throw UsageError("'--priority' requires an integer argument"); } else throw UsageError("unknown flag '%1%'", arg); } From 5807b5cda0198ec96688039eab42ec0e6a86ed27 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Jan 2025 23:44:18 +0100 Subject: [PATCH 45/68] Trivial changes from lazy-trees Rename allowLookup -> useRegistries, rename lockRootPath -> followsPrefix, drop an unnecessary getFlake() variant. --- src/libflake/flake/flake.cc | 53 ++++++++++++++++++++----------------- src/libflake/flake/flake.hh | 2 +- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index 4e70b4089..cac39f8ab 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -43,7 +43,7 @@ static std::optional lookupInFlakeCache( static std::tuple fetchOrSubstituteTree( EvalState & state, const FlakeRef & originalRef, - bool allowLookup, + bool useRegistries, FlakeCache & flakeCache) { auto fetched = lookupInFlakeCache(flakeCache, originalRef); @@ -54,7 +54,7 @@ static std::tuple fetchOrSubstituteTree( auto [storePath, lockedRef] = originalRef.fetchTree(state.store); fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .storePath = storePath}); } else { - if (allowLookup) { + if (useRegistries) { resolvedRef = originalRef.resolve( state.store, [](fetchers::Registry::RegistryType type) { @@ -105,7 +105,7 @@ static std::map parseFlakeInputs( EvalState & state, Value * value, const PosIdx pos, - InputPath lockRootPath, + const InputPath & lockRootPath, const SourcePath & flakeDir); static FlakeInput parseFlakeInput( @@ -113,7 +113,7 @@ static FlakeInput parseFlakeInput( std::string_view inputName, Value * value, const PosIdx pos, - InputPath lockRootPath, + const InputPath & lockRootPath, const SourcePath & flakeDir) { expectType(state, nAttrs, *value, pos); @@ -220,7 +220,7 @@ static std::map parseFlakeInputs( EvalState & state, Value * value, const PosIdx pos, - InputPath lockRootPath, + const InputPath & lockRootPath, const SourcePath & flakeDir) { std::map inputs; @@ -345,25 +345,20 @@ static Flake readFlake( static Flake getFlake( EvalState & state, const FlakeRef & originalRef, - bool allowLookup, + bool useRegistries, FlakeCache & flakeCache, - InputPath lockRootPath) + const InputPath & lockRootPath) { auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree( - state, originalRef, allowLookup, flakeCache); + state, originalRef, useRegistries, flakeCache); return readFlake(state, originalRef, resolvedRef, lockedRef, state.rootPath(state.store->toRealPath(storePath)), lockRootPath); } -Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup, FlakeCache & flakeCache) -{ - return getFlake(state, originalRef, allowLookup, flakeCache, {}); -} - -Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup) +Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool useRegistries) { FlakeCache flakeCache; - return getFlake(state, originalRef, allowLookup, flakeCache); + return getFlake(state, originalRef, useRegistries, flakeCache, {}); } static LockFile readLockFile( @@ -390,7 +385,7 @@ LockedFlake lockFlake( auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries); - auto flake = getFlake(state, topRef, useRegistries, flakeCache); + auto flake = getFlake(state, topRef, useRegistries, flakeCache, {}); if (lockFlags.applyNixConfig) { flake.config.apply(settings); @@ -445,7 +440,7 @@ LockedFlake lockFlake( ref node, const InputPath & inputPathPrefix, std::shared_ptr oldNode, - const InputPath & lockRootPath, + const InputPath & followsPrefix, const SourcePath & sourcePath, bool trustLock)> computeLocks; @@ -461,7 +456,11 @@ LockedFlake lockFlake( /* The old node, if any, from which locks can be copied. */ std::shared_ptr oldNode, - const InputPath & lockRootPath, + /* The prefix relative to which 'follows' should be + interpreted. When a node is initially locked, it's + relative to the node's flake; when it's already locked, + it's relative to the root of the lock file. */ + const InputPath & followsPrefix, /* The source path of this node's flake. */ const SourcePath & sourcePath, bool trustLock) @@ -633,7 +632,7 @@ LockedFlake lockFlake( break; } } - auto absoluteFollows(lockRootPath); + auto absoluteFollows(followsPrefix); absoluteFollows.insert(absoluteFollows.end(), follows->begin(), follows->end()); fakeInputs.emplace(i.first, FlakeInput { .follows = absoluteFollows, @@ -645,9 +644,10 @@ LockedFlake lockFlake( if (mustRefetch) { auto inputFlake = getInputFlake(); nodePaths.emplace(childNode, inputFlake.path.parent()); - computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, lockRootPath, inputFlake.path, false); + computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, followsPrefix, + inputFlake.path, false); } else { - computeLocks(fakeInputs, childNode, inputPath, oldLock, lockRootPath, sourcePath, true); + computeLocks(fakeInputs, childNode, inputPath, oldLock, followsPrefix, sourcePath, true); } } else { @@ -672,7 +672,11 @@ LockedFlake lockFlake( if (input.isFlake) { auto inputFlake = getInputFlake(); - auto childNode = make_ref(inputFlake.lockedRef, ref, true, overridenParentPath); + auto childNode = make_ref( + inputFlake.lockedRef, + ref, + true, + overridenParentPath); node->inputs.insert_or_assign(id, childNode); @@ -693,7 +697,7 @@ LockedFlake lockFlake( oldLock ? std::dynamic_pointer_cast(oldLock) : readLockFile(settings, state.fetchSettings, inputFlake.lockFilePath()).root.get_ptr(), - oldLock ? lockRootPath : inputPath, + oldLock ? followsPrefix : inputPath, inputFlake.path, false); } @@ -817,8 +821,7 @@ LockedFlake lockFlake( repo, so we should re-read it. FIXME: we could also just clear the 'rev' field... */ auto prevLockedRef = flake.lockedRef; - FlakeCache dummyCache; - flake = getFlake(state, topRef, useRegistries, dummyCache); + flake = getFlake(state, topRef, useRegistries); if (lockFlags.commitLockFile && flake.lockedRef.input.getRev() && diff --git a/src/libflake/flake/flake.hh b/src/libflake/flake/flake.hh index f85671a41..9ab661fce 100644 --- a/src/libflake/flake/flake.hh +++ b/src/libflake/flake/flake.hh @@ -110,7 +110,7 @@ struct Flake } }; -Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool allowLookup); +Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool useRegistries); /** * Fingerprint of a locked flake; used as a cache key. From f0271090b24035ff5f19899f9971519854738e3b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Jan 2025 23:47:57 +0100 Subject: [PATCH 46/68] Drop unused 'settings' argument --- src/libflake/flake/flake.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index cac39f8ab..873a26d5c 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -362,7 +362,6 @@ Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool useRegistri } static LockFile readLockFile( - const Settings & settings, const fetchers::Settings & fetchSettings, const SourcePath & lockFilePath) { @@ -398,7 +397,6 @@ LockedFlake lockFlake( } auto oldLockFile = readLockFile( - settings, state.fetchSettings, lockFlags.referenceLockFilePath.value_or( flake.lockFilePath())); @@ -696,7 +694,7 @@ LockedFlake lockFlake( inputFlake.inputs, childNode, inputPath, oldLock ? std::dynamic_pointer_cast(oldLock) - : readLockFile(settings, state.fetchSettings, inputFlake.lockFilePath()).root.get_ptr(), + : readLockFile(state.fetchSettings, inputFlake.lockFilePath()).root.get_ptr(), oldLock ? followsPrefix : inputPath, inputFlake.path, false); From 3197c19a31f5ce42b099d257b6ba22fb25e707c4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 Jan 2025 12:33:23 +0100 Subject: [PATCH 47/68] Add link to tracking issue --- src/libfetchers/git.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 3f196da54..5a9679199 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -439,7 +439,8 @@ struct GitInputScheme : InputScheme if (!isAbsolute(url.path)) { warn( "Fetching Git repository '%s', which uses a path relative to the current directory. " - "This is not supported and will stop working in a future release.", + "This is not supported and will stop working in a future release. " + "See https://github.com/NixOS/nix/issues/12281 for details.", url); } repoInfo.url = std::filesystem::absolute(url.path).string(); From 9003343b5345622399571d8e232b7f94ec3aec28 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 Jan 2025 13:11:07 +0100 Subject: [PATCH 48/68] Git fetcher: Replace RepoInfo::url by a std::variant Previously the 'url' field was either a path or a URL, depending on 'isLocal'. This replaces both fields by a std::variant, which is more type-safe. --- src/libfetchers/git.cc | 122 +++++++++++++++++++++++++---------------- 1 file changed, 74 insertions(+), 48 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 5a9679199..bca4b2d38 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -297,7 +297,7 @@ struct GitInputScheme : InputScheme Strings args = {"clone"}; - args.push_back(repoInfo.url); + args.push_back(repoInfo.locationToArg()); if (auto ref = input.getRef()) { args.push_back("--branch"); @@ -311,11 +311,14 @@ struct GitInputScheme : InputScheme runProgram("git", true, args, {}, true); } + // FIXME: return std::filesystem::path std::optional getSourcePath(const Input & input) const override { auto repoInfo = getRepoInfo(input); - if (repoInfo.isLocal) return repoInfo.url; - return std::nullopt; + if (auto path = repoInfo.getPath()) + return *path; + else + return std::nullopt; } void putFile( @@ -325,14 +328,15 @@ struct GitInputScheme : InputScheme std::optional commitMsg) const override { auto repoInfo = getRepoInfo(input); - if (!repoInfo.isLocal) + auto repoPath = repoInfo.getPath(); + if (!repoPath) throw Error("cannot commit '%s' to Git repository '%s' because it's not a working tree", path, input.to_string()); - writeFile((CanonPath(repoInfo.url) / path).abs(), contents); + writeFile(*repoPath / path.rel(), contents); auto result = runProgram(RunOptions { .program = "git", - .args = {"-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "check-ignore", "--quiet", std::string(path.rel())}, + .args = {"-C", *repoPath, "--git-dir", repoInfo.gitDir, "check-ignore", "--quiet", std::string(path.rel())}, }); auto exitCode = #ifndef WIN32 // TODO abstract over exit status handling on Windows @@ -345,7 +349,7 @@ struct GitInputScheme : InputScheme if (exitCode != 0) { // The path is not `.gitignore`d, we can add the file. runProgram("git", true, - { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) }); + { "-C", *repoPath, "--git-dir", repoInfo.gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) }); if (commitMsg) { @@ -353,7 +357,7 @@ struct GitInputScheme : InputScheme logger->pause(); Finally restoreLogger([]() { logger->resume(); }); runProgram("git", true, - { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-F", "-" }, + { "-C", *repoPath, "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-F", "-" }, *commitMsg); } } @@ -361,24 +365,41 @@ struct GitInputScheme : InputScheme struct RepoInfo { - /* Whether this is a local, non-bare repository. */ - bool isLocal = false; + /* Either the path of the repo (for local, non-bare repos), or + the URL (which is never a `file` URL). */ + std::variant location; /* Working directory info: the complete list of files, and whether the working directory is dirty compared to HEAD. */ GitRepo::WorkdirInfo workdirInfo; - /* URL of the repo, or its path if isLocal. Never a `file` URL. */ - std::string url; + std::string locationToArg() const + { + return std::visit( + overloaded { + [&](const std::filesystem::path & path) + { return path.string(); }, + [&](const ParsedURL & url) + { return url.to_string(); } + }, location); + } + + std::optional getPath() const + { + if (auto path = std::get_if(&location)) + return *path; + else + return std::nullopt; + } void warnDirty(const Settings & settings) const { if (workdirInfo.isDirty) { if (!settings.allowDirty) - throw Error("Git tree '%s' is dirty", url); + throw Error("Git tree '%s' is dirty", locationToArg()); if (settings.warnDirty) - warn("Git tree '%s' is dirty", url); + warn("Git tree '%s' is dirty", locationToArg()); } } @@ -425,7 +446,6 @@ struct GitInputScheme : InputScheme static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing auto url = parseURL(getStrAttr(input.attrs, "url")); bool isBareRepository = url.scheme == "file" && !pathExists(url.path + "/.git"); - repoInfo.isLocal = url.scheme == "file" && !forceHttp && !isBareRepository; // // FIXME: here we turn a possibly relative path into an absolute path. // This allows relative git flake inputs to be resolved against the @@ -435,7 +455,7 @@ struct GitInputScheme : InputScheme // // See: https://discourse.nixos.org/t/57783 and #9708 // - if (repoInfo.isLocal) { + if (url.scheme == "file" && !forceHttp && !isBareRepository) { if (!isAbsolute(url.path)) { warn( "Fetching Git repository '%s', which uses a path relative to the current directory. " @@ -443,14 +463,14 @@ struct GitInputScheme : InputScheme "See https://github.com/NixOS/nix/issues/12281 for details.", url); } - repoInfo.url = std::filesystem::absolute(url.path).string(); + repoInfo.location = std::filesystem::absolute(url.path); } else - repoInfo.url = url.to_string(); + repoInfo.location = url; // If this is a local directory and no ref or revision is // given, then allow the use of an unclean working tree. - if (!input.getRef() && !input.getRev() && repoInfo.isLocal) - repoInfo.workdirInfo = GitRepo::getCachedWorkdirInfo(repoInfo.url); + if (auto repoPath = repoInfo.getPath(); !input.getRef() && !input.getRev() && repoPath) + repoInfo.workdirInfo = GitRepo::getCachedWorkdirInfo(*repoPath); return repoInfo; } @@ -480,7 +500,7 @@ struct GitInputScheme : InputScheme if (auto revCountAttrs = cache->lookup(key)) return getIntAttr(*revCountAttrs, "revCount"); - Activity act(*logger, lvlChatty, actUnknown, fmt("getting Git revision count of '%s'", repoInfo.url)); + Activity act(*logger, lvlChatty, actUnknown, fmt("getting Git revision count of '%s'", repoInfo.locationToArg())); auto revCount = GitRepo::openRepo(repoDir)->getRevCount(rev); @@ -491,11 +511,15 @@ struct GitInputScheme : InputScheme std::string getDefaultRef(const RepoInfo & repoInfo) const { - auto head = repoInfo.isLocal - ? GitRepo::openRepo(repoInfo.url)->getWorkdirRef() - : readHeadCached(repoInfo.url); + auto head = std::visit( + overloaded { + [&](const std::filesystem::path & path) + { return GitRepo::openRepo(path)->getWorkdirRef(); }, + [&](const ParsedURL & url) + { return readHeadCached(url.to_string()); } + }, repoInfo.location); if (!head) { - warn("could not read HEAD ref from repo at '%s', using 'master'", repoInfo.url); + warn("could not read HEAD ref from repo at '%s', using 'master'", repoInfo.locationToArg()); return "master"; } return *head; @@ -540,12 +564,13 @@ struct GitInputScheme : InputScheme Path repoDir; - if (repoInfo.isLocal) { - repoDir = repoInfo.url; + if (auto repoPath = repoInfo.getPath()) { + repoDir = *repoPath; if (!input.getRev()) input.attrs.insert_or_assign("rev", GitRepo::openRepo(repoDir)->resolveRef(ref).gitRev()); } else { - Path cacheDir = getCachePath(repoInfo.url, getShallowAttr(input)); + auto repoUrl = std::get(repoInfo.location); + Path cacheDir = getCachePath(repoUrl.to_string(), getShallowAttr(input)); repoDir = cacheDir; repoInfo.gitDir = "."; @@ -555,7 +580,7 @@ struct GitInputScheme : InputScheme auto repo = GitRepo::openRepo(cacheDir, true, true); // We need to set the origin so resolving submodule URLs works - repo->setRemote("origin", repoInfo.url); + repo->setRemote("origin", repoUrl.to_string()); Path localRefFile = ref.compare(0, 5, "refs/") == 0 @@ -594,11 +619,11 @@ struct GitInputScheme : InputScheme ? ref : "refs/heads/" + ref; - repo->fetch(repoInfo.url, fmt("%s:%s", fetchRef, fetchRef), getShallowAttr(input)); + repo->fetch(repoUrl.to_string(), fmt("%s:%s", fetchRef, fetchRef), getShallowAttr(input)); } catch (Error & e) { if (!pathExists(localRefFile)) throw; logError(e.info()); - warn("could not update local clone of Git repository '%s'; continuing with the most recent version", repoInfo.url); + warn("could not update local clone of Git repository '%s'; continuing with the most recent version", repoInfo.locationToArg()); } try { @@ -607,8 +632,8 @@ struct GitInputScheme : InputScheme } catch (Error & e) { warn("could not update mtime for file '%s': %s", localRefFile, e.info().msg); } - if (!originalRef && !storeCachedHead(repoInfo.url, ref)) - warn("could not update cached head '%s' for '%s'", ref, repoInfo.url); + if (!originalRef && !storeCachedHead(repoUrl.to_string(), ref)) + warn("could not update cached head '%s' for '%s'", ref, repoInfo.locationToArg()); } if (auto rev = input.getRev()) { @@ -620,8 +645,7 @@ struct GitInputScheme : InputScheme "allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".", rev->gitRev(), ref, - repoInfo.url - ); + repoInfo.locationToArg()); } else input.attrs.insert_or_assign("rev", repo->resolveRef(ref).gitRev()); @@ -633,7 +657,7 @@ struct GitInputScheme : InputScheme auto isShallow = repo->isShallow(); if (isShallow && !getShallowAttr(input)) - throw Error("'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified", repoInfo.url); + throw Error("'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified", repoInfo.locationToArg()); // FIXME: check whether rev is an ancestor of ref? @@ -648,7 +672,7 @@ struct GitInputScheme : InputScheme infoAttrs.insert_or_assign("revCount", getRevCount(repoInfo, repoDir, rev)); - printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.url); + printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg()); verifyCommit(input, repo); @@ -702,21 +726,23 @@ struct GitInputScheme : InputScheme RepoInfo & repoInfo, Input && input) const { + auto repoPath = repoInfo.getPath().value(); + if (getSubmodulesAttr(input)) /* Create mountpoints for the submodules. */ for (auto & submodule : repoInfo.workdirInfo.submodules) repoInfo.workdirInfo.files.insert(submodule.path); - auto repo = GitRepo::openRepo(repoInfo.url, false, false); + auto repo = GitRepo::openRepo(repoPath, false, false); auto exportIgnore = getExportIgnoreAttr(input); ref accessor = repo->getAccessor(repoInfo.workdirInfo, exportIgnore, - makeNotAllowedError(repoInfo.url)); + makeNotAllowedError(repoInfo.locationToArg())); - accessor->setPathDisplay(repoInfo.url); + accessor->setPathDisplay(repoInfo.locationToArg()); /* If the repo has submodules, return a mounted input accessor consisting of the accessor for the top-level repo and the @@ -725,10 +751,10 @@ struct GitInputScheme : InputScheme std::map> mounts; for (auto & submodule : repoInfo.workdirInfo.submodules) { - auto submodulePath = CanonPath(repoInfo.url) / submodule.path; + auto submodulePath = repoPath / submodule.path.rel(); fetchers::Attrs attrs; attrs.insert_or_assign("type", "git"); - attrs.insert_or_assign("url", submodulePath.abs()); + attrs.insert_or_assign("url", submodulePath.string()); attrs.insert_or_assign("exportIgnore", Explicit{ exportIgnore }); attrs.insert_or_assign("submodules", Explicit{ true }); // TODO: fall back to getAccessorFromCommit-like fetch when submodules aren't checked out @@ -752,7 +778,7 @@ struct GitInputScheme : InputScheme } if (!repoInfo.workdirInfo.isDirty) { - auto repo = GitRepo::openRepo(repoInfo.url); + auto repo = GitRepo::openRepo(repoPath); if (auto ref = repo->getWorkdirRef()) input.attrs.insert_or_assign("ref", *ref); @@ -762,7 +788,7 @@ struct GitInputScheme : InputScheme input.attrs.insert_or_assign("rev", rev.gitRev()); input.attrs.insert_or_assign("revCount", - rev == nullRev ? 0 : getRevCount(repoInfo, repoInfo.url, rev)); + rev == nullRev ? 0 : getRevCount(repoInfo, repoPath, rev)); verifyCommit(input, repo); } else { @@ -781,7 +807,7 @@ struct GitInputScheme : InputScheme input.attrs.insert_or_assign( "lastModified", repoInfo.workdirInfo.headRev - ? getLastModified(repoInfo, repoInfo.url, *repoInfo.workdirInfo.headRev) + ? getLastModified(repoInfo, repoPath, *repoInfo.workdirInfo.headRev) : 0); return {accessor, std::move(input)}; @@ -804,7 +830,7 @@ struct GitInputScheme : InputScheme } auto [accessor, final] = - input.getRef() || input.getRev() || !repoInfo.isLocal + input.getRef() || input.getRev() || !repoInfo.getPath() ? getAccessorFromCommit(store, repoInfo, std::move(input)) : getAccessorFromWorkdir(store, repoInfo, std::move(input)); @@ -822,14 +848,14 @@ struct GitInputScheme : InputScheme return makeFingerprint(*rev); else { auto repoInfo = getRepoInfo(input); - if (repoInfo.isLocal && repoInfo.workdirInfo.headRev && repoInfo.workdirInfo.submodules.empty()) { + if (auto repoPath = repoInfo.getPath(); repoPath && repoInfo.workdirInfo.headRev && repoInfo.workdirInfo.submodules.empty()) { /* Calculate a fingerprint that takes into account the deleted and modified/added files. */ HashSink hashSink{HashAlgorithm::SHA512}; for (auto & file : repoInfo.workdirInfo.dirtyFiles) { writeString("modified:", hashSink); writeString(file.abs(), hashSink); - dumpPath(repoInfo.url + "/" + file.abs(), hashSink); + dumpPath(*repoPath / file.rel(), hashSink); } for (auto & file : repoInfo.workdirInfo.deletedFiles) { writeString("deleted:", hashSink); From f5548c17ed61bf6e6392ece9c83eb76c9f31cf66 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 Jan 2025 13:17:58 +0100 Subject: [PATCH 49/68] getSourcePath(): Return std::filesystem::path --- src/libfetchers/fetchers.cc | 4 ++-- src/libfetchers/fetchers.hh | 4 ++-- src/libfetchers/git.cc | 9 ++------- src/libfetchers/mercurial.cc | 2 +- src/libfetchers/path.cc | 4 ++-- src/libflake/flake/flake.cc | 2 +- src/nix/develop.cc | 2 +- 7 files changed, 11 insertions(+), 16 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 1cfc48d52..9459db087 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -358,7 +358,7 @@ void Input::clone(const Path & destDir) const scheme->clone(*this, destDir); } -std::optional Input::getSourcePath() const +std::optional Input::getSourcePath() const { assert(scheme); return scheme->getSourcePath(*this); @@ -461,7 +461,7 @@ Input InputScheme::applyOverrides( return input; } -std::optional InputScheme::getSourcePath(const Input & input) const +std::optional InputScheme::getSourcePath(const Input & input) const { return {}; } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 87c7b08c0..644c267c1 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -164,7 +164,7 @@ public: void clone(const Path & destDir) const; - std::optional getSourcePath() const; + std::optional getSourcePath() const; /** * Write a file to this input, for input types that support @@ -247,7 +247,7 @@ struct InputScheme virtual void clone(const Input & input, const Path & destDir) const; - virtual std::optional getSourcePath(const Input & input) const; + virtual std::optional getSourcePath(const Input & input) const; virtual void putFile( const Input & input, diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index bca4b2d38..4523f49ca 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -311,14 +311,9 @@ struct GitInputScheme : InputScheme runProgram("git", true, args, {}, true); } - // FIXME: return std::filesystem::path - std::optional getSourcePath(const Input & input) const override + std::optional getSourcePath(const Input & input) const override { - auto repoInfo = getRepoInfo(input); - if (auto path = repoInfo.getPath()) - return *path; - else - return std::nullopt; + return getRepoInfo(input).getPath(); } void putFile( diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index c2fd8139c..61cbca202 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -126,7 +126,7 @@ struct MercurialInputScheme : InputScheme return res; } - std::optional getSourcePath(const Input & input) const override + std::optional getSourcePath(const Input & input) const override { auto url = parseURL(getStrAttr(input.attrs, "url")); if (url.scheme == "file" && !input.getRef() && !input.getRev()) diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 351e21a1b..9d1cce0f3 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -80,9 +80,9 @@ struct PathInputScheme : InputScheme }; } - std::optional getSourcePath(const Input & input) const override + std::optional getSourcePath(const Input & input) const override { - return getStrAttr(input.attrs, "path"); + return getAbsPath(input); } void putFile( diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index 873a26d5c..c2145ab39 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -781,7 +781,7 @@ LockedFlake lockFlake( writeFile(*lockFlags.outputLockFilePath, newLockFileS); } else { auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; - auto outputLockFilePath = *sourcePath + "/" + relPath; + auto outputLockFilePath = *sourcePath / relPath; bool lockFileExists = pathExists(outputLockFilePath); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 1736add9a..deee89aa1 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -696,7 +696,7 @@ struct CmdDevelop : Common, MixEnvironment auto sourcePath = installableFlake->getLockedFlake()->flake.resolvedRef.input.getSourcePath(); if (sourcePath) { if (chdir(sourcePath->c_str()) == -1) { - throw SysError("chdir to '%s' failed", *sourcePath); + throw SysError("chdir to %s failed", *sourcePath); } } } From c59aa3ec8723cc0afe2f7dda4755a3951710c851 Mon Sep 17 00:00:00 2001 From: Ilja Kotirinta Date: Mon, 20 Jan 2025 14:03:29 +0200 Subject: [PATCH 50/68] Remove character not needed for a command --- doc/manual/source/installation/uninstall.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/source/installation/uninstall.md b/doc/manual/source/installation/uninstall.md index 47689a16e..8d45da6bb 100644 --- a/doc/manual/source/installation/uninstall.md +++ b/doc/manual/source/installation/uninstall.md @@ -160,6 +160,6 @@ which you may remove. To remove a [single-user installation](./installing-binary.md#single-user-installation) of Nix, run: ```console -$ rm -rf /nix ~/.nix-channels ~/.nix-defexpr ~/.nix-profile +rm -rf /nix ~/.nix-channels ~/.nix-defexpr ~/.nix-profile ``` You might also want to manually remove references to Nix from your `~/.profile`. From a8c69cc907044565f9e43371663cc8315602b28a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 20 Jan 2025 13:22:50 +0100 Subject: [PATCH 51/68] processGraph(): Don't throw ThreadPoolShutDown if there is an exception Fixes $ nix copy --derivation --to /tmp/nix /nix/store/... error: cannot enqueue a work item while the thread pool is shutting down The ThreadPoolShutDown exception was hiding the reason for the thread pool shut down, e.g. error: cannot add path '/nix/store/03sl46khd8gmjpsad7223m32ma965vy9-fix-static.patch' because it lacks a signature by a trusted key --- src/libutil/thread-pool.hh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh index dc056481a..4adc48657 100644 --- a/src/libutil/thread-pool.hh +++ b/src/libutil/thread-pool.hh @@ -150,8 +150,16 @@ void processGraph( } }; - for (auto & node : nodes) - pool.enqueue(std::bind(worker, std::ref(node))); + for (auto & node : nodes) { + try { + pool.enqueue(std::bind(worker, std::ref(node))); + } catch (ThreadPoolShutDown &) { + /* Stop if the thread pool is shutting down. It means a + previous work item threw an exception, so process() + below will rethrow it. */ + break; + } + } pool.process(); From cc838e81818fa6d78fb2e2686eb83dacac07d7d6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 20 Jan 2025 14:23:02 +0100 Subject: [PATCH 52/68] addMultipleToStore(): Move pathsToCopy This allows RemoteStore::addMultipleToStore() to free the Source objects early (and in particular the associated sinkToSource() buffers). This should fix #7359. For example, memory consumption of nix copy --derivation --to ssh-ng://localhost?remote-store=/tmp/nix --derivation --no-check-sigs \ /nix/store/4p9xmfgnvclqpii8pxqcwcvl9bxqy2xf-nixos-system-...drv went from 353 MB to 74 MB. --- src/libstore/remote-store.cc | 7 +++++-- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.cc | 4 ++-- src/libstore/store-api.hh | 2 +- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 69bbc64fc..87c58a690 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -534,14 +534,16 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, void RemoteStore::addMultipleToStore( - PathsSource & pathsToCopy, + PathsSource && pathsToCopy, Activity & act, RepairFlag repair, CheckSigsFlag checkSigs) { auto source = sinkToSource([&](Sink & sink) { sink << pathsToCopy.size(); - for (auto & [pathInfo, pathSource] : pathsToCopy) { + std::reverse(pathsToCopy.begin(), pathsToCopy.end()); + while (!pathsToCopy.empty()) { + auto & [pathInfo, pathSource] = pathsToCopy.back(); WorkerProto::Serialise::write(*this, WorkerProto::WriteConn { .to = sink, @@ -549,6 +551,7 @@ void RemoteStore::addMultipleToStore( }, pathInfo); pathSource->drainInto(sink); + pathsToCopy.pop_back(); } }); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 4e1896268..ea6cd471e 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -102,7 +102,7 @@ public: CheckSigsFlag checkSigs) override; void addMultipleToStore( - PathsSource & pathsToCopy, + PathsSource && pathsToCopy, Activity & act, RepairFlag repair, CheckSigsFlag checkSigs) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 3b5167730..6cd8e47f0 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -223,7 +223,7 @@ StorePath Store::addToStore( } void Store::addMultipleToStore( - PathsSource & pathsToCopy, + PathsSource && pathsToCopy, Activity & act, RepairFlag repair, CheckSigsFlag checkSigs) @@ -1138,7 +1138,7 @@ std::map copyPaths( pathsToCopy.push_back(std::pair{infoForDst, std::move(source)}); } - dstStore.addMultipleToStore(pathsToCopy, act, repair, checkSigs); + dstStore.addMultipleToStore(std::move(pathsToCopy), act, repair, checkSigs); return pathsMap; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index f45012061..474dffcb5 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -425,7 +425,7 @@ public: CheckSigsFlag checkSigs = CheckSigs); virtual void addMultipleToStore( - PathsSource & pathsToCopy, + PathsSource && pathsToCopy, Activity & act, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); From c656725a1521e9bb6f7a1e80e00459ac05a181e4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 20 Jan 2025 14:52:23 +0100 Subject: [PATCH 53/68] mingw: Check for S_ISSOCK --- src/libutil/posix-source-accessor.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 8ee986d3f..70ad6474f 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -124,7 +124,9 @@ std::optional PosixSourceAccessor::maybeLstat(const CanonP S_ISLNK(st->st_mode) ? tSymlink : S_ISCHR(st->st_mode) ? tChar : S_ISBLK(st->st_mode) ? tBlock : +#ifdef S_ISSOCK S_ISSOCK(st->st_mode) ? tSocket : +#endif S_ISFIFO(st->st_mode) ? tFifo : tUnknown, .fileSize = S_ISREG(st->st_mode) ? std::optional(st->st_size) : std::nullopt, From d8636843b1d20ec8067281efb8082dfcd3797db5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 20 Jan 2025 14:52:50 +0100 Subject: [PATCH 54/68] mingw: Don't do LTO This breaks the build with "symbol wrong type (4 vs 3)". https://stackoverflow.com/questions/28267100/dll-linking-failed-with-lto-using-mingw-w64 --- packaging/dependencies.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 08d179b82..a3d7a73e4 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -75,7 +75,7 @@ let # Users who are debugging Nix builds are expected to set the environment variable `mesonBuildType`, per the # guidance in https://github.com/NixOS/nix/blob/8a3fc27f1b63a08ac983ee46435a56cf49ebaf4a/doc/manual/source/development/debugging.md?plain=1#L10. # For this reason, we don't want to refer to `finalAttrs.mesonBuildType` here, but rather use the environment variable. - preConfigure = prevAttrs.preConfigure or "" + '' + preConfigure = prevAttrs.preConfigure or "" + lib.optionalString (!stdenv.hostPlatform.isWindows) '' case "$mesonBuildType" in release|minsize) appendToVar mesonFlags "-Db_lto=true" ;; *) appendToVar mesonFlags "-Db_lto=false" ;; From 2669e4ac4f8b1deda501587fb8d4d41b5dc9064c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 20 Jan 2025 14:57:56 +0100 Subject: [PATCH 55/68] Add comment Co-authored-by: Robert Hensing --- src/libstore/remote-store.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 87c58a690..6781e4743 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -541,6 +541,7 @@ void RemoteStore::addMultipleToStore( { auto source = sinkToSource([&](Sink & sink) { sink << pathsToCopy.size(); + // Reverse, so we can release memory at the original start std::reverse(pathsToCopy.begin(), pathsToCopy.end()); while (!pathsToCopy.empty()) { auto & [pathInfo, pathSource] = pathsToCopy.back(); From a78f55ef978b59419366590871966d6aa35a5959 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 20 Jan 2025 15:36:19 +0100 Subject: [PATCH 56/68] GitInputScheme: Fix mingw build Mingw apparently is less smart about coercing std::filesystem::path into strings. --- src/libfetchers/git.cc | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 4523f49ca..b411e112f 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -331,7 +331,7 @@ struct GitInputScheme : InputScheme auto result = runProgram(RunOptions { .program = "git", - .args = {"-C", *repoPath, "--git-dir", repoInfo.gitDir, "check-ignore", "--quiet", std::string(path.rel())}, + .args = {"-C", repoPath->string(), "--git-dir", repoInfo.gitDir, "check-ignore", "--quiet", std::string(path.rel())}, }); auto exitCode = #ifndef WIN32 // TODO abstract over exit status handling on Windows @@ -344,7 +344,7 @@ struct GitInputScheme : InputScheme if (exitCode != 0) { // The path is not `.gitignore`d, we can add the file. runProgram("git", true, - { "-C", *repoPath, "--git-dir", repoInfo.gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) }); + { "-C", repoPath->string(), "--git-dir", repoInfo.gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) }); if (commitMsg) { @@ -352,7 +352,7 @@ struct GitInputScheme : InputScheme logger->pause(); Finally restoreLogger([]() { logger->resume(); }); runProgram("git", true, - { "-C", *repoPath, "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-F", "-" }, + { "-C", repoPath->string(), "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-F", "-" }, *commitMsg); } } @@ -470,7 +470,7 @@ struct GitInputScheme : InputScheme return repoInfo; } - uint64_t getLastModified(const RepoInfo & repoInfo, const std::string & repoDir, const Hash & rev) const + uint64_t getLastModified(const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const { Cache::Key key{"gitLastModified", {{"rev", rev.gitRev()}}}; @@ -486,7 +486,7 @@ struct GitInputScheme : InputScheme return lastModified; } - uint64_t getRevCount(const RepoInfo & repoInfo, const std::string & repoDir, const Hash & rev) const + uint64_t getRevCount(const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const { Cache::Key key{"gitRevCount", {{"rev", rev.gitRev()}}}; @@ -557,7 +557,7 @@ struct GitInputScheme : InputScheme auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo); input.attrs.insert_or_assign("ref", ref); - Path repoDir; + std::filesystem::path repoDir; if (auto repoPath = repoInfo.getPath()) { repoDir = *repoPath; @@ -565,22 +565,22 @@ struct GitInputScheme : InputScheme input.attrs.insert_or_assign("rev", GitRepo::openRepo(repoDir)->resolveRef(ref).gitRev()); } else { auto repoUrl = std::get(repoInfo.location); - Path cacheDir = getCachePath(repoUrl.to_string(), getShallowAttr(input)); + std::filesystem::path cacheDir = getCachePath(repoUrl.to_string(), getShallowAttr(input)); repoDir = cacheDir; repoInfo.gitDir = "."; - createDirs(dirOf(cacheDir)); - PathLocks cacheDirLock({cacheDir}); + std::filesystem::create_directories(cacheDir.parent_path()); + PathLocks cacheDirLock({cacheDir.string()}); auto repo = GitRepo::openRepo(cacheDir, true, true); // We need to set the origin so resolving submodule URLs works repo->setRemote("origin", repoUrl.to_string()); - Path localRefFile = + auto localRefFile = ref.compare(0, 5, "refs/") == 0 - ? cacheDir + "/" + ref - : cacheDir + "/refs/heads/" + ref; + ? cacheDir / ref + : cacheDir / "refs/heads" / ref; bool doFetch; time_t now = time(0); @@ -596,7 +596,7 @@ struct GitInputScheme : InputScheme /* If the local ref is older than ‘tarball-ttl’ seconds, do a git fetch to update the local ref to the remote ref. */ struct stat st; - doFetch = stat(localRefFile.c_str(), &st) != 0 || + doFetch = stat(localRefFile.string().c_str(), &st) != 0 || !isCacheFileWithinTtl(now, st); } } @@ -616,7 +616,7 @@ struct GitInputScheme : InputScheme repo->fetch(repoUrl.to_string(), fmt("%s:%s", fetchRef, fetchRef), getShallowAttr(input)); } catch (Error & e) { - if (!pathExists(localRefFile)) throw; + if (!std::filesystem::exists(localRefFile)) throw; logError(e.info()); warn("could not update local clone of Git repository '%s'; continuing with the most recent version", repoInfo.locationToArg()); } @@ -850,7 +850,7 @@ struct GitInputScheme : InputScheme for (auto & file : repoInfo.workdirInfo.dirtyFiles) { writeString("modified:", hashSink); writeString(file.abs(), hashSink); - dumpPath(*repoPath / file.rel(), hashSink); + dumpPath((*repoPath / file.rel()).string(), hashSink); } for (auto & file : repoInfo.workdirInfo.deletedFiles) { writeString("deleted:", hashSink); From 15073e86a88cd03935591dde4448354836a6b632 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 20 Jan 2025 15:47:10 +0100 Subject: [PATCH 57/68] tests: Wait for network *online* and multi-user targets This should help prevent some test stalls. By default, multi-user.target does not imply that the network is fully up. --- tests/nixos/git-submodules.nix | 4 +++- tests/nixos/nix-copy-closure.nix | 5 ++++- tests/nixos/nix-copy.nix | 5 ++++- tests/nixos/remote-builds-ssh-ng.nix | 5 ++++- tests/nixos/remote-builds.nix | 3 ++- 5 files changed, 17 insertions(+), 5 deletions(-) diff --git a/tests/nixos/git-submodules.nix b/tests/nixos/git-submodules.nix index 570b1822b..b18300a91 100644 --- a/tests/nixos/git-submodules.nix +++ b/tests/nixos/git-submodules.nix @@ -39,11 +39,13 @@ client.succeed("chmod 600 /root/.ssh/id_ed25519") # Install the SSH key on the builders. - client.wait_for_unit("network.target") + client.wait_for_unit("network-online.target") remote.succeed("mkdir -p -m 700 /root/.ssh") remote.copy_from_host("key.pub", "/root/.ssh/authorized_keys") remote.wait_for_unit("sshd") + remote.wait_for_unit("multi-user.target") + remote.wait_for_unit("network-online.target") client.succeed(f"ssh -o StrictHostKeyChecking=no {remote.name} 'echo hello world'") remote.succeed(""" diff --git a/tests/nixos/nix-copy-closure.nix b/tests/nixos/nix-copy-closure.nix index b9daa0a1f..44324e989 100644 --- a/tests/nixos/nix-copy-closure.nix +++ b/tests/nixos/nix-copy-closure.nix @@ -48,7 +48,10 @@ in { server.succeed("mkdir -m 700 /root/.ssh") server.copy_from_host("key.pub", "/root/.ssh/authorized_keys") server.wait_for_unit("sshd") - client.wait_for_unit("network.target") + server.wait_for_unit("multi-user.target") + server.wait_for_unit("network-online.target") + + client.wait_for_unit("network-online.target") client.succeed(f"ssh -o StrictHostKeyChecking=no {server.name} 'echo hello world'") # Copy the closure of package A from the client to the server. diff --git a/tests/nixos/nix-copy.nix b/tests/nixos/nix-copy.nix index 8691d0138..a6a04b52c 100644 --- a/tests/nixos/nix-copy.nix +++ b/tests/nixos/nix-copy.nix @@ -56,7 +56,10 @@ in { start_all() server.wait_for_unit("sshd") - client.wait_for_unit("network.target") + server.wait_for_unit("multi-user.target") + server.wait_for_unit("network-online.target") + + client.wait_for_unit("network-online.target") client.wait_for_unit("getty@tty1.service") # Either the prompt: ]# # or an OCR misreading of it: 1# diff --git a/tests/nixos/remote-builds-ssh-ng.nix b/tests/nixos/remote-builds-ssh-ng.nix index 926ec00fe..3562d2d2f 100644 --- a/tests/nixos/remote-builds-ssh-ng.nix +++ b/tests/nixos/remote-builds-ssh-ng.nix @@ -89,10 +89,13 @@ in client.succeed("chmod 600 /root/.ssh/id_ed25519") # Install the SSH key on the builder. - client.wait_for_unit("network.target") + client.wait_for_unit("network-online.target") builder.succeed("mkdir -p -m 700 /root/.ssh") builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys") builder.wait_for_unit("sshd") + builder.wait_for_unit("multi-user.target") + builder.wait_for_unit("network-online.target") + client.succeed(f"ssh -o StrictHostKeyChecking=no {builder.name} 'echo hello world'") # Perform a build diff --git a/tests/nixos/remote-builds.nix b/tests/nixos/remote-builds.nix index 84e5176b7..4fca4b938 100644 --- a/tests/nixos/remote-builds.nix +++ b/tests/nixos/remote-builds.nix @@ -112,11 +112,12 @@ in client.succeed("chmod 600 /root/.ssh/id_ed25519") # Install the SSH key on the builders. - client.wait_for_unit("network.target") + client.wait_for_unit("network-online.target") for builder in [builder1, builder2]: builder.succeed("mkdir -p -m 700 /root/.ssh") builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys") builder.wait_for_unit("sshd") + builder.wait_for_unit("network-online.target") # Make sure the builder can handle our login correctly builder.wait_for_unit("multi-user.target") # Make sure there's no funny business on the client either From 89e5047e12a9da201f35f0dc4852082b0d188392 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 20 Jan 2025 15:50:04 +0100 Subject: [PATCH 58/68] Fix libflake build on mingw --- src/libflake/flake/flake.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index c2145ab39..d4708e1f1 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -783,7 +783,7 @@ LockedFlake lockFlake( auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; auto outputLockFilePath = *sourcePath / relPath; - bool lockFileExists = pathExists(outputLockFilePath); + bool lockFileExists = std::filesystem::exists(outputLockFilePath); auto s = chomp(diff); if (lockFileExists) { From 0c85477f8e4f77307c9e63ac9d80973370dd46b9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 20 Jan 2025 14:48:06 +0100 Subject: [PATCH 59/68] maint: Remove perl bindings from static build for now --- packaging/everything.nix | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packaging/everything.nix b/packaging/everything.nix index 0b04d2c6d..33d1bec7f 100644 --- a/packaging/everything.nix +++ b/packaging/everything.nix @@ -61,6 +61,8 @@ let nix-store-c nix-util nix-util-c + ] ++ lib.optionals (!stdenv.hostPlatform.isStatic) [ + # Currently fails in static build nix-perl-bindings ]; installPhase = '' @@ -131,6 +133,8 @@ in # (checkInputs must be empty paths??) (runCommand "check-pkg-config" { checked = dev.tests.pkg-config; } "mkdir $out") ] ++ + lib.optionals (!stdenv.hostPlatform.isStatic) ( + # Perl currently fails in static build (if stdenv.buildPlatform.canExecute stdenv.hostPlatform then [ # TODO: add perl.tests @@ -138,7 +142,7 @@ in ] else [ nix-perl-bindings - ]); + ])); installCheckInputs = [ nix-functional-tests ]; From 2cb494f56119e202c31f59bf6f7fd8b7ad0a47c0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 20 Jan 2025 16:06:55 +0100 Subject: [PATCH 60/68] nix upgrade-nix: Fix build on mingw --- src/nix/upgrade-nix.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 28174c4fb..1e8032af6 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -97,7 +97,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand // FIXME: don't call an external process. runProgram(getNixBin("nix-env").string(), false, - {"--profile", profileDir, "-i", store->printStorePath(storePath), "--no-sandbox"}); + {"--profile", profileDir.string(), "-i", store->printStorePath(storePath), "--no-sandbox"}); } printInfo(ANSI_GREEN "upgrade to version %s done" ANSI_NORMAL, version); @@ -120,7 +120,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand // Resolve profile to /nix/var/nix/profiles/ link. while (canonPath(profileDir.string()).find("/profiles/") == std::string::npos && std::filesystem::is_symlink(profileDir)) - profileDir = readLink(profileDir); + profileDir = readLink(profileDir.string()); printInfo("found profile %s", profileDir); From 809f157a49dc0851b3199f972b0ee3a70dc4262f Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 20 Jan 2025 16:11:48 +0100 Subject: [PATCH 61/68] maint: Disable LTO for static due to build failure --- packaging/dependencies.nix | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index a3d7a73e4..22d4cf9e3 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -75,7 +75,11 @@ let # Users who are debugging Nix builds are expected to set the environment variable `mesonBuildType`, per the # guidance in https://github.com/NixOS/nix/blob/8a3fc27f1b63a08ac983ee46435a56cf49ebaf4a/doc/manual/source/development/debugging.md?plain=1#L10. # For this reason, we don't want to refer to `finalAttrs.mesonBuildType` here, but rather use the environment variable. - preConfigure = prevAttrs.preConfigure or "" + lib.optionalString (!stdenv.hostPlatform.isWindows) '' + preConfigure = prevAttrs.preConfigure or "" + lib.optionalString ( + !stdenv.hostPlatform.isWindows + # build failure + && !stdenv.hostPlatform.isStatic + ) '' case "$mesonBuildType" in release|minsize) appendToVar mesonFlags "-Db_lto=true" ;; *) appendToVar mesonFlags "-Db_lto=false" ;; From 8e05ddfd84caec7e1e0461490b2e4bc4d3ec25f5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 20 Jan 2025 16:33:52 +0100 Subject: [PATCH 62/68] Use fs::symlink_exists --- src/libflake/flake/flake.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index d4708e1f1..06260c67a 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -783,7 +783,7 @@ LockedFlake lockFlake( auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; auto outputLockFilePath = *sourcePath / relPath; - bool lockFileExists = std::filesystem::exists(outputLockFilePath); + bool lockFileExists = fs::symlink_exists(outputLockFilePath); auto s = chomp(diff); if (lockFileExists) { From fcc501b73025bb1be796661bbb2545c7697553aa Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 20 Jan 2025 17:06:20 +0100 Subject: [PATCH 63/68] mergify: Add automatic backport label This label will be useful for constructing queries to find backportable PRs. Specifically, those should omit both automatic backports and "backports reviewed" PRs. --- .mergify.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.mergify.yml b/.mergify.yml index 5d2bf8520..5b03feca0 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -27,6 +27,7 @@ pull_request_rules: branches: - 2.18-maintenance labels: + - automatic backport - merge-queue - name: backport patches to 2.19 @@ -37,6 +38,7 @@ pull_request_rules: branches: - 2.19-maintenance labels: + - automatic backport - merge-queue - name: backport patches to 2.20 @@ -47,6 +49,7 @@ pull_request_rules: branches: - 2.20-maintenance labels: + - automatic backport - merge-queue - name: backport patches to 2.21 @@ -57,6 +60,7 @@ pull_request_rules: branches: - 2.21-maintenance labels: + - automatic backport - merge-queue - name: backport patches to 2.22 @@ -67,6 +71,7 @@ pull_request_rules: branches: - 2.22-maintenance labels: + - automatic backport - merge-queue - name: backport patches to 2.23 @@ -77,6 +82,7 @@ pull_request_rules: branches: - 2.23-maintenance labels: + - automatic backport - merge-queue - name: backport patches to 2.24 @@ -87,6 +93,7 @@ pull_request_rules: branches: - "2.24-maintenance" labels: + - automatic backport - merge-queue - name: backport patches to 2.25 @@ -97,4 +104,5 @@ pull_request_rules: branches: - "2.25-maintenance" labels: + - automatic backport - merge-queue From 0fe92067fbdc4c40663189a39412866dddb18faa Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 20 Jan 2025 11:18:27 -0500 Subject: [PATCH 64/68] Fixes for `nix-everything` wrapper 1. Fix this eval error: https://hydra.nixos.org/jobset/nix/master#tabs-errors The dev package output (actually a separate derivation) needs to skip this for cross just as the main package output does. 2. Deduplicate libs attrset and list. 3. Move `nix-functional-tests` to `checkInputs`. With the Meson build system, we no longer need a `check` vs `install-check` distinction, so it is simpler to just keeep everything in one place. --- packaging/everything.nix | 73 ++++++++++++++++------------------------ src/perl/meson.build | 3 ++ 2 files changed, 32 insertions(+), 44 deletions(-) diff --git a/packaging/everything.nix b/packaging/everything.nix index 33d1bec7f..5761cc40b 100644 --- a/packaging/everything.nix +++ b/packaging/everything.nix @@ -42,29 +42,34 @@ }: let + libs = { + inherit + nix-util + nix-util-c + nix-store + nix-store-c + nix-fetchers + nix-expr + nix-expr-c + nix-flake + nix-flake-c + nix-main + nix-main-c + ; + } // lib.optionalAttrs (!stdenv.hostPlatform.isStatic && stdenv.buildPlatform.canExecute stdenv.hostPlatform) { + # Currently fails in static build + inherit + nix-perl-bindings + ; + }; + dev = stdenv.mkDerivation (finalAttrs: { name = "nix-${nix-cli.version}-dev"; pname = "nix"; version = nix-cli.version; dontUnpack = true; dontBuild = true; - libs = map lib.getDev [ - nix-cmd - nix-expr - nix-expr-c - nix-fetchers - nix-flake - nix-flake-c - nix-main - nix-main-c - nix-store - nix-store-c - nix-util - nix-util-c - ] ++ lib.optionals (!stdenv.hostPlatform.isStatic) [ - # Currently fails in static build - nix-perl-bindings - ]; + libs = map lib.getDev (lib.attrValues libs); installPhase = '' mkdir -p $out/nix-support echo $libs >> $out/nix-support/propagated-build-inputs @@ -129,22 +134,16 @@ in nix-fetchers-tests.tests.run nix-flake-tests.tests.run + # Make sure the functional tests have passed + nix-functional-tests + # dev bundle is ok # (checkInputs must be empty paths??) (runCommand "check-pkg-config" { checked = dev.tests.pkg-config; } "mkdir $out") - ] ++ - lib.optionals (!stdenv.hostPlatform.isStatic) ( + ] ++ lib.optionals (!stdenv.hostPlatform.isStatic && stdenv.buildPlatform.canExecute stdenv.hostPlatform) [ # Perl currently fails in static build - (if stdenv.buildPlatform.canExecute stdenv.hostPlatform - then [ - # TODO: add perl.tests - nix-perl-bindings - ] - else [ - nix-perl-bindings - ])); - installCheckInputs = [ - nix-functional-tests + # TODO: Split out tests into a separate derivation? + nix-perl-bindings ]; passthru = prevAttrs.passthru // { inherit (nix-cli) version; @@ -166,21 +165,7 @@ in disallowedReferences = nix.all; ``` */ - libs = { - inherit - nix-util - nix-util-c - nix-store - nix-store-c - nix-fetchers - nix-expr - nix-expr-c - nix-flake - nix-flake-c - nix-main - nix-main-c - ; - }; + inherit libs; tests = prevAttrs.passthru.tests or {} // { # TODO: create a proper fixpoint and: diff --git a/src/perl/meson.build b/src/perl/meson.build index 52d85fd60..7b3716c17 100644 --- a/src/perl/meson.build +++ b/src/perl/meson.build @@ -82,6 +82,9 @@ nix_store_dep = dependency('nix-store') # pkgconfig available, are not in a standard location, # and are installed into a version folder. Use the # Perl binary to give hints about perl include dir. +# +# Note that until we have a better solution for this, cross-compiling +# the perl bindings does not appear to be possible. #------------------------------------------------- perl_archname = run_command( perl, '-e', 'use Config; print $Config{archname};', check: true).stdout() From 6f0bdd9ae70c9bda2d9e2e9c641c9f193429fcd3 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 20 Jan 2025 18:21:07 +0100 Subject: [PATCH 65/68] tests: Wait for network *online* targets More prevalent than I thought in 15073e86a See also https://github.com/NixOS/nix/actions/runs/12872412321/job/35887830320?pr=12310 which is a failed github-flakes test without "Network is Online" --- tests/nixos/git-submodules.nix | 1 + tests/nixos/github-flakes.nix | 2 ++ tests/nixos/nix-docker.nix | 1 + tests/nixos/nss-preload.nix | 2 ++ tests/nixos/s3-binary-cache-store.nix | 3 +++ tests/nixos/sourcehut-flakes.nix | 2 ++ 6 files changed, 11 insertions(+) diff --git a/tests/nixos/git-submodules.nix b/tests/nixos/git-submodules.nix index b18300a91..a82ddf418 100644 --- a/tests/nixos/git-submodules.nix +++ b/tests/nixos/git-submodules.nix @@ -46,6 +46,7 @@ remote.wait_for_unit("sshd") remote.wait_for_unit("multi-user.target") remote.wait_for_unit("network-online.target") + client.wait_for_unit("network-online.target") client.succeed(f"ssh -o StrictHostKeyChecking=no {remote.name} 'echo hello world'") remote.succeed(""" diff --git a/tests/nixos/github-flakes.nix b/tests/nixos/github-flakes.nix index 8e646f6dd..69d1df410 100644 --- a/tests/nixos/github-flakes.nix +++ b/tests/nixos/github-flakes.nix @@ -161,7 +161,9 @@ in github.succeed("cat /var/log/httpd/*.log >&2") github.wait_for_unit("httpd.service") + github.wait_for_unit("network-online.target") + client.wait_for_unit("network-online.target") client.succeed("curl -v https://github.com/ >&2") out = client.succeed("nix registry list") print(out) diff --git a/tests/nixos/nix-docker.nix b/tests/nixos/nix-docker.nix index dfd508988..00b04482c 100644 --- a/tests/nixos/nix-docker.nix +++ b/tests/nixos/nix-docker.nix @@ -37,6 +37,7 @@ in { testScript = { nodes }: '' cache.wait_for_unit("harmonia.service") + cache.wait_for_unit("network-online.target") machine.succeed("mkdir -p /etc/containers") machine.succeed("""echo '{"default":[{"type":"insecureAcceptAnything"}]}' > /etc/containers/policy.json""") diff --git a/tests/nixos/nss-preload.nix b/tests/nixos/nss-preload.nix index 610769c8d..b7e704f39 100644 --- a/tests/nixos/nss-preload.nix +++ b/tests/nixos/nss-preload.nix @@ -102,6 +102,7 @@ in }; testScript = { nodes, ... }: '' + http_dns.wait_for_unit("network-online.target") http_dns.wait_for_unit("nginx") http_dns.wait_for_open_port(80) http_dns.wait_for_unit("unbound") @@ -109,6 +110,7 @@ in client.start() client.wait_for_unit('multi-user.target') + client.wait_for_unit('network-online.target') with subtest("can fetch data from a remote server outside sandbox"): client.succeed("nix --version >&2") diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix index 83b85c032..f8659b830 100644 --- a/tests/nixos/s3-binary-cache-store.nix +++ b/tests/nixos/s3-binary-cache-store.nix @@ -52,12 +52,15 @@ in { # Create a binary cache. server.wait_for_unit("minio") + server.wait_for_unit("network-online.target") server.succeed("mc config host add minio http://localhost:9000 ${accessKey} ${secretKey} --api s3v4") server.succeed("mc mb minio/my-cache") server.succeed("${env} nix copy --to '${storeUrl}' ${pkgA}") + client.wait_for_unit("network-online.target") + # Test fetchurl on s3:// URLs while we're at it. client.succeed("${env} nix eval --impure --expr 'builtins.fetchurl { name = \"foo\"; url = \"s3://my-cache/nix-cache-info?endpoint=http://server:9000®ion=eu-west-1\"; }'") diff --git a/tests/nixos/sourcehut-flakes.nix b/tests/nixos/sourcehut-flakes.nix index 04f3590e1..2f469457a 100644 --- a/tests/nixos/sourcehut-flakes.nix +++ b/tests/nixos/sourcehut-flakes.nix @@ -122,6 +122,8 @@ in start_all() sourcehut.wait_for_unit("httpd.service") + sourcehut.wait_for_unit("network-online.target") + client.wait_for_unit("network-online.target") client.succeed("curl -v https://git.sr.ht/ >&2") client.succeed("nix registry list | grep nixpkgs") From 6b987206ce51839a41cd0540236e5f5ecdc6940e Mon Sep 17 00:00:00 2001 From: Mike Kusold Date: Mon, 20 Jan 2025 11:04:53 -0700 Subject: [PATCH 66/68] Add unit of measurement for download-buffer-size I started getting these warnings `warning: download buffer is full; consider increasing the 'download-buffer-size' setting` but the documentation does not make it obvious what unit of measurement it accepts. --- src/libstore/filetransfer.hh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index d836ab2c4..e9a166af7 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -50,8 +50,9 @@ struct FileTransferSettings : Config Setting downloadBufferSize{this, 64 * 1024 * 1024, "download-buffer-size", R"( - The size of Nix's internal download buffer during `curl` transfers. If data is + The size of Nix's internal download buffer in bytes during `curl` transfers. If data is not processed quickly enough to exceed the size of this buffer, downloads may stall. + The default is 67108864 (64MB). )"}; }; From f0c209fb14cb511370ea2b7787d9a8dd90070162 Mon Sep 17 00:00:00 2001 From: Mike Kusold Date: Mon, 20 Jan 2025 11:32:47 -0700 Subject: [PATCH 67/68] Use standardize units for megabytes Co-authored-by: Eelco Dolstra --- src/libstore/filetransfer.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index e9a166af7..43a384d71 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -52,7 +52,7 @@ struct FileTransferSettings : Config R"( The size of Nix's internal download buffer in bytes during `curl` transfers. If data is not processed quickly enough to exceed the size of this buffer, downloads may stall. - The default is 67108864 (64MB). + The default is 67108864 (64 MiB). )"}; }; From a82c63f5d8bdc916dc01a77c177f58d766d38680 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 21 Jan 2025 09:54:45 +0100 Subject: [PATCH 68/68] Fix nix-everything build https://hydra.nixos.org/build/286306734 --- packaging/everything.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/packaging/everything.nix b/packaging/everything.nix index 5761cc40b..7ca878d8d 100644 --- a/packaging/everything.nix +++ b/packaging/everything.nix @@ -55,6 +55,7 @@ let nix-flake-c nix-main nix-main-c + nix-cmd ; } // lib.optionalAttrs (!stdenv.hostPlatform.isStatic && stdenv.buildPlatform.canExecute stdenv.hostPlatform) { # Currently fails in static build