From b2be6fed8600ee48c05cc9643c101d5eab4a5727 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 23 Apr 2024 16:00:52 +0200 Subject: [PATCH 01/19] Improve support for subflakes Subflakes are flakes in the same tree, accessed in flake inputs via relative paths (e.g. `inputs.foo.url = "path:./subdir"`). Previously these didn't work very well because they would be separately copied to the store, which is inefficient and makes references to parent directories tricky or impossible. Furthermore, they had their own NAR hash in the lock file, which is superfluous since the parent is already locked. Now subflakes are accessed via the accessor of the calling flake. This avoids the unnecessary copy and makes it possible for subflakes to depend on flakes in a parent directory (so long as they're in the same tree). Lock file nodes for relative flake inputs now have a new `parent` field: { "locked": { "path": "./subdir", "type": "path" }, "original": { "path": "./subdir", "type": "path" }, "parent": [ "foo", "bar" ] } which denotes that `./subdir` is to be interpreted relative to the directory of the `bar` input of the `foo` input of the root flake. Extracted from the lazy-trees branch. --- src/libexpr/flake/call-flake.nix | 7 + src/libexpr/flake/flake.cc | 160 +++++++++++++++------- src/libexpr/flake/flakeref.cc | 68 +++++---- src/libexpr/flake/flakeref.hh | 6 +- src/libexpr/flake/lockfile.cc | 9 +- src/libexpr/flake/lockfile.hh | 12 +- src/libexpr/primops/fetchTree.cc | 5 +- src/libfetchers/fetchers.cc | 6 + src/libfetchers/fetchers.hh | 14 +- src/libfetchers/path.cc | 25 +--- src/nix/flake.md | 25 +++- tests/functional/flakes/follow-paths.sh | 25 +++- tests/functional/flakes/relative-paths.sh | 89 ++++++++++++ tests/functional/local.mk | 1 + 14 files changed, 332 insertions(+), 120 deletions(-) create mode 100644 tests/functional/flakes/relative-paths.sh diff --git a/src/libexpr/flake/call-flake.nix b/src/libexpr/flake/call-flake.nix index a411564df..43ecb7f15 100644 --- a/src/libexpr/flake/call-flake.nix +++ b/src/libexpr/flake/call-flake.nix @@ -38,10 +38,17 @@ let (key: node: let + parentNode = allNodes.${getInputByPath lockFile.root node.parent}; + sourceInfo = if overrides ? ${key} then overrides.${key}.sourceInfo + else if node.locked.type == "path" && builtins.substring 0 1 node.locked.path != "/" + then + parentNode.sourceInfo // { + outPath = parentNode.sourceInfo.outPath + ("/" + node.locked.path); + } else # FIXME: remove obsolete node.info. fetchTree (node.info or {} // removeAttrs node.locked ["dir"]); diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 3af9ef14e..d4cabe68f 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -93,12 +93,17 @@ static void expectType(EvalState & state, ValueType type, } static std::map parseFlakeInputs( - EvalState & state, Value * value, const PosIdx pos, - const std::optional & baseDir, InputPath lockRootPath); + EvalState & state, + Value * value, + const PosIdx pos, + InputPath lockRootPath); -static FlakeInput parseFlakeInput(EvalState & state, - const std::string & inputName, Value * value, const PosIdx pos, - const std::optional & baseDir, InputPath lockRootPath) +static FlakeInput parseFlakeInput( + EvalState & state, + const std::string & inputName, + Value * value, + const PosIdx pos, + InputPath lockRootPath) { expectType(state, nAttrs, *value, pos); @@ -122,7 +127,7 @@ static FlakeInput parseFlakeInput(EvalState & state, expectType(state, nBool, *attr.value, attr.pos); input.isFlake = attr.value->boolean(); } else if (attr.name == sInputs) { - input.overrides = parseFlakeInputs(state, attr.value, attr.pos, baseDir, lockRootPath); + input.overrides = parseFlakeInputs(state, attr.value, attr.pos, lockRootPath); } else if (attr.name == sFollows) { expectType(state, nString, *attr.value, attr.pos); auto follows(parseInputPath(attr.value->c_str())); @@ -173,7 +178,7 @@ static FlakeInput parseFlakeInput(EvalState & state, if (!attrs.empty()) throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, state.positions[pos]); if (url) - input.ref = parseFlakeRef(*url, baseDir, true, input.isFlake); + input.ref = parseFlakeRef(*url, {}, true, input.isFlake, true); } if (!input.follows && !input.ref) @@ -183,8 +188,10 @@ static FlakeInput parseFlakeInput(EvalState & state, } static std::map parseFlakeInputs( - EvalState & state, Value * value, const PosIdx pos, - const std::optional & baseDir, InputPath lockRootPath) + EvalState & state, + Value * value, + const PosIdx pos, + InputPath lockRootPath) { std::map inputs; @@ -196,7 +203,6 @@ static std::map parseFlakeInputs( state.symbols[inputAttr.name], inputAttr.value, inputAttr.pos, - baseDir, lockRootPath)); } @@ -232,7 +238,7 @@ static Flake readFlake( auto sInputs = state.symbols.create("inputs"); if (auto inputs = vInfo.attrs()->get(sInputs)) - flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, flakePath.parent().path.abs(), lockRootPath); // FIXME + flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, lockRootPath); auto sOutputs = state.symbols.create("outputs"); @@ -366,13 +372,29 @@ LockedFlake lockFlake( debug("old lock file: %s", oldLockFile); - std::map overrides; + struct Override + { + FlakeInput input; + SourcePath sourcePath; + std::optional parentPath; // FIXME: rename to inputPathPrefix? + }; + + std::map overrides; std::set explicitCliOverrides; std::set overridesUsed, updatesUsed; std::map, SourcePath> nodePaths; for (auto & i : lockFlags.inputOverrides) { - overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second }); + overrides.emplace( + i.first, + Override { + .input = FlakeInput { .ref = i.second }, + /* Note: any relative overrides + (e.g. `--override-input B/C "path:./foo/bar"`) + are interpreted relative to the top-level + flake. */ + .sourcePath = flake.path, + }); explicitCliOverrides.insert(i.first); } @@ -386,7 +408,7 @@ LockedFlake lockFlake( const InputPath & inputPathPrefix, std::shared_ptr oldNode, const InputPath & lockRootPath, - const Path & parentPath, + const SourcePath & sourcePath, bool trustLock)> computeLocks; @@ -402,7 +424,8 @@ LockedFlake lockFlake( copied. */ std::shared_ptr oldNode, const InputPath & lockRootPath, - const Path & parentPath, + /* The source path of this node's flake. */ + const SourcePath & sourcePath, bool trustLock) { debug("computing lock file node '%s'", printInputPath(inputPathPrefix)); @@ -414,7 +437,12 @@ LockedFlake lockFlake( auto inputPath(inputPathPrefix); inputPath.push_back(id); inputPath.push_back(idOverride); - overrides.insert_or_assign(inputPath, inputOverride); + overrides.emplace(inputPath, + Override { + .input = inputOverride, + .sourcePath = sourcePath, + .parentPath = inputPathPrefix // FIXME: should this be inputPath? + }); } } @@ -446,13 +474,18 @@ LockedFlake lockFlake( auto i = overrides.find(inputPath); bool hasOverride = i != overrides.end(); bool hasCliOverride = explicitCliOverrides.contains(inputPath); - if (hasOverride) { + if (hasOverride) overridesUsed.insert(inputPath); - // Respect the “flakeness” of the input even if we - // override it - i->second.isFlake = input2.isFlake; - } - auto & input = hasOverride ? i->second : input2; + auto input = hasOverride ? i->second.input : input2; + + /* Resolve relative 'path:' inputs relative to + the source path of the overrider. */ + auto overridenSourcePath = hasOverride ? i->second.sourcePath : sourcePath; + + /* Respect the "flakeness" of the input even if we + override it. */ + if (hasOverride) + input.isFlake = input2.isFlake; /* Resolve 'follows' later (since it may refer to an input path we haven't processed yet. */ @@ -468,6 +501,33 @@ LockedFlake lockFlake( assert(input.ref); + auto overridenParentPath = + input.ref->input.isRelative() + ? std::optional(hasOverride ? i->second.parentPath : inputPathPrefix) + : std::nullopt; + + auto resolveRelativePath = [&]() -> std::optional + { + if (auto relativePath = input.ref->input.isRelative()) { + return SourcePath { + overridenSourcePath.accessor, + CanonPath(*relativePath, overridenSourcePath.path.parent().value()) + }; + } else + return std::nullopt; + }; + + /* Get the input flake, resolve 'path:./...' + flakerefs relative to the parent flake. */ + auto getInputFlake = [&]() + { + if (auto resolvedPath = resolveRelativePath()) { + return readFlake(state, *input.ref, *input.ref, *input.ref, *resolvedPath, inputPath); + } else { + return getFlake(state, *input.ref, useRegistries, flakeCache, inputPath); + } + }; + /* Do we have an entry in the existing lock file? And the input is not in updateInputs? */ std::shared_ptr oldLock; @@ -481,6 +541,7 @@ LockedFlake lockFlake( if (oldLock && oldLock->originalRef == *input.ref + && oldLock->parentPath == overridenParentPath && !hasCliOverride) { debug("keeping existing input '%s'", inputPathS); @@ -489,7 +550,10 @@ LockedFlake lockFlake( didn't change and there is no override from a higher level flake. */ auto childNode = make_ref( - oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake); + oldLock->lockedRef, + oldLock->originalRef, + oldLock->isFlake, + oldLock->parentPath); node->inputs.insert_or_assign(id, childNode); @@ -541,11 +605,15 @@ LockedFlake lockFlake( } if (mustRefetch) { - auto inputFlake = getFlake(state, oldLock->lockedRef, false, flakeCache, inputPath); + auto inputFlake = getInputFlake(); nodePaths.emplace(childNode, inputFlake.path.parent()); - computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, lockRootPath, parentPath, false); + computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, lockRootPath, inputFlake.path, false); } else { - computeLocks(fakeInputs, childNode, inputPath, oldLock, lockRootPath, parentPath, true); + // FIXME: sourcePath is wrong here, we + // should pass a lambda that lazily + // fetches the parent flake if needed + // (i.e. getInputFlake()). + computeLocks(fakeInputs, childNode, inputPath, oldLock, lockRootPath, sourcePath, true); } } else { @@ -553,7 +621,9 @@ LockedFlake lockFlake( this input. */ debug("creating new input '%s'", inputPathS); - if (!lockFlags.allowUnlocked && !input.ref->input.isLocked()) + if (!lockFlags.allowUnlocked + && !input.ref->input.isLocked() + && !input.ref->input.isRelative()) throw Error("cannot update unlocked flake input '%s' in pure mode", inputPathS); /* Note: in case of an --override-input, we use @@ -566,17 +636,9 @@ LockedFlake lockFlake( auto ref = (input2.ref && explicitCliOverrides.contains(inputPath)) ? *input2.ref : *input.ref; if (input.isFlake) { - Path localPath = parentPath; - FlakeRef localRef = *input.ref; + auto inputFlake = getInputFlake(); - // If this input is a path, recurse it down. - // This allows us to resolve path inputs relative to the current flake. - if (localRef.input.getType() == "path") - localPath = absPath(*input.ref->input.getSourcePath(), parentPath); - - auto inputFlake = getFlake(state, localRef, useRegistries, flakeCache, inputPath); - - auto childNode = make_ref(inputFlake.lockedRef, ref); + auto childNode = make_ref(inputFlake.lockedRef, ref, true, overridenParentPath); node->inputs.insert_or_assign(id, childNode); @@ -598,17 +660,26 @@ LockedFlake lockFlake( ? std::dynamic_pointer_cast(oldLock) : readLockFile(inputFlake.lockFilePath()).root.get_ptr(), oldLock ? lockRootPath : inputPath, - localPath, + inputFlake.path, false); } else { - auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree( - state, *input.ref, useRegistries, flakeCache); + auto [path, lockedRef] = [&]() -> std::tuple + { + // Handle non-flake 'path:./...' inputs. + if (auto resolvedPath = resolveRelativePath()) { + return {*resolvedPath, *input.ref}; + } else { + auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree( + state, *input.ref, useRegistries, flakeCache); + return {state.rootPath(state.store->toRealPath(storePath)), lockedRef}; + } + }(); - auto childNode = make_ref(lockedRef, ref, false); + auto childNode = make_ref(lockedRef, ref, false, overridenParentPath); - nodePaths.emplace(childNode, state.rootPath(state.store->toRealPath(storePath))); + nodePaths.emplace(childNode, path); node->inputs.insert_or_assign(id, childNode); } @@ -621,9 +692,6 @@ LockedFlake lockFlake( } }; - // Bring in the current ref for relative path resolution if we have it - auto parentPath = flake.path.parent().path.abs(); - nodePaths.emplace(newLockFile.root, flake.path.parent()); computeLocks( @@ -632,7 +700,7 @@ LockedFlake lockFlake( {}, lockFlags.recreateLockFile ? nullptr : oldLockFile.root.get_ptr(), {}, - parentPath, + flake.path, false); for (auto & i : lockFlags.inputOverrides) diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 6e4aad64d..bac237a2a 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -51,9 +51,10 @@ FlakeRef parseFlakeRef( const std::string & url, const std::optional & baseDir, bool allowMissing, - bool isFlake) + bool isFlake, + bool allowRelative) { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing, isFlake); + auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing, isFlake, allowRelative); if (fragment != "") throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url); return flakeRef; @@ -69,11 +70,25 @@ std::optional maybeParseFlakeRef( } } +static std::pair fromParsedURL( + ParsedURL && parsedURL, + bool isFlake) +{ + auto dir = getOr(parsedURL.query, "dir", ""); + parsedURL.query.erase("dir"); + + std::string fragment; + std::swap(fragment, parsedURL.fragment); + + return std::make_pair(FlakeRef(fetchers::Input::fromURL(parsedURL, isFlake), dir), fragment); +} + std::pair parsePathFlakeRefWithFragment( const std::string & url, const std::optional & baseDir, bool allowMissing, - bool isFlake) + bool isFlake, + bool allowRelative) { std::string path = url; std::string fragment = ""; @@ -90,7 +105,7 @@ std::pair parsePathFlakeRefWithFragment( fragment = percentDecode(url.substr(fragmentStart+1)); } if (pathEnd != std::string::npos && fragmentStart != std::string::npos) { - query = decodeQuery(url.substr(pathEnd+1, fragmentStart-pathEnd-1)); + query = decodeQuery(url.substr(pathEnd + 1, fragmentStart - pathEnd - 1)); } if (baseDir) { @@ -154,6 +169,7 @@ std::pair parsePathFlakeRefWithFragment( .authority = "", .path = flakeRoot, .query = query, + .fragment = fragment, }; if (subdir != "") { @@ -165,9 +181,7 @@ std::pair parsePathFlakeRefWithFragment( if (pathExists(flakeRoot + "/.git/shallow")) parsedURL.query.insert_or_assign("shallow", "1"); - return std::make_pair( - FlakeRef(fetchers::Input::fromURL(parsedURL), getOr(parsedURL.query, "dir", "")), - fragment); + return fromParsedURL(std::move(parsedURL), isFlake); } subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir); @@ -176,25 +190,30 @@ std::pair parsePathFlakeRefWithFragment( } } else { - if (!hasPrefix(path, "/")) + if (!allowRelative && !hasPrefix(path, "/")) throw BadURL("flake reference '%s' is not an absolute path", url); - path = canonPath(path + "/" + getOr(query, "dir", "")); } fetchers::Attrs attrs; attrs.insert_or_assign("type", "path"); attrs.insert_or_assign("path", path); - return std::make_pair(FlakeRef(fetchers::Input::fromAttrs(std::move(attrs)), ""), fragment); -}; - + return fromParsedURL({ + .url = path, + .base = path, + .scheme = "path", + .authority = "", + .path = path, + .query = query, + .fragment = fragment + }, isFlake); +} /* Check if 'url' is a flake ID. This is an abbreviated syntax for 'flake:?ref=&rev='. */ std::optional> parseFlakeIdRef( const std::string & url, - bool isFlake -) + bool isFlake) { std::smatch match; @@ -223,32 +242,21 @@ std::optional> parseFlakeIdRef( std::optional> parseURLFlakeRef( const std::string & url, const std::optional & baseDir, - bool isFlake -) + bool isFlake) { - ParsedURL parsedURL; try { - parsedURL = parseURL(url); + return fromParsedURL(parseURL(url), isFlake); } catch (BadURL &) { return std::nullopt; } - - std::string fragment; - std::swap(fragment, parsedURL.fragment); - - auto input = fetchers::Input::fromURL(parsedURL, isFlake); - input.parent = baseDir; - - return std::make_pair( - FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")), - fragment); } std::pair parseFlakeRefWithFragment( const std::string & url, const std::optional & baseDir, bool allowMissing, - bool isFlake) + bool isFlake, + bool allowRelative) { using namespace fetchers; @@ -259,7 +267,7 @@ std::pair parseFlakeRefWithFragment( } else if (auto res = parseURLFlakeRef(url, baseDir, isFlake)) { return *res; } else { - return parsePathFlakeRefWithFragment(url, baseDir, allowMissing, isFlake); + return parsePathFlakeRefWithFragment(url, baseDir, allowMissing, isFlake, allowRelative); } } diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index 04c812ed0..ea6c4e4d7 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -75,7 +75,8 @@ FlakeRef parseFlakeRef( const std::string & url, const std::optional & baseDir = {}, bool allowMissing = false, - bool isFlake = true); + bool isFlake = true, + bool allowRelative = false); /** * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) @@ -90,7 +91,8 @@ std::pair parseFlakeRefWithFragment( const std::string & url, const std::optional & baseDir = {}, bool allowMissing = false, - bool isFlake = true); + bool isFlake = true, + bool allowRelative = false); /** * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index d252214dd..2884ca262 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -36,8 +36,9 @@ LockedNode::LockedNode(const nlohmann::json & json) : lockedRef(getFlakeRef(json, "locked", "info")) // FIXME: remove "info" , originalRef(getFlakeRef(json, "original", nullptr)) , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) + , parentPath(json.find("parent") != json.end() ? (std::optional) json["parent"] : std::nullopt) { - if (!lockedRef.input.isLocked()) + if (!lockedRef.input.isLocked() && !lockedRef.input.isRelative()) throw Error("lock file contains unlocked input '%s'", fetchers::attrsToJSON(lockedRef.input.toAttrs())); } @@ -184,6 +185,8 @@ std::pair LockFile::toJSON() const n["locked"] = fetchers::attrsToJSON(lockedNode->lockedRef.toAttrs()); if (!lockedNode->isFlake) n["flake"] = false; + if (lockedNode->parentPath) + n["parent"] = *lockedNode->parentPath; } nodes[key] = std::move(n); @@ -230,7 +233,9 @@ std::optional LockFile::isUnlocked() const for (auto & i : nodes) { if (i == ref(root)) continue; auto node = i.dynamic_pointer_cast(); - if (node && !node->lockedRef.input.isLocked()) + if (node + && !node->lockedRef.input.isLocked() + && !node->lockedRef.input.isRelative()) return node->lockedRef; } diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index 7e62e6d09..aad805baf 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -38,11 +38,19 @@ struct LockedNode : Node FlakeRef lockedRef, originalRef; bool isFlake = true; + /* The node relative to which relative source paths + (e.g. 'path:../foo') are interpreted. */ + std::optional parentPath; + LockedNode( const FlakeRef & lockedRef, const FlakeRef & originalRef, - bool isFlake = true) - : lockedRef(lockedRef), originalRef(originalRef), isFlake(isFlake) + bool isFlake = true, + std::optional parentPath = {}) + : lockedRef(lockedRef) + , originalRef(originalRef) + , isFlake(isFlake) + , parentPath(parentPath) { } LockedNode(const nlohmann::json & json); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index e27f30512..1c5cd4ec2 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -31,9 +31,8 @@ void emitTreeAttrs( // FIXME: support arbitrary input attributes. - auto narHash = input.getNarHash(); - assert(narHash); - attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true)); + if (auto narHash = input.getNarHash()) + attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true)); if (input.getType() == "git") attrs.alloc("submodules").mkBool( diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 73923907c..73cb4fea3 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -141,6 +141,12 @@ bool Input::isLocked() const return scheme && scheme->isLocked(*this); } +std::optional Input::isRelative() const +{ + assert(scheme); + return scheme->isRelative(*this); +} + Attrs Input::toAttrs() const { return attrs; diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 551be9a1f..66cac7064 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -31,11 +31,6 @@ struct Input std::shared_ptr scheme; // note: can be null Attrs attrs; - /** - * path of the parent of this input, used for relative path resolution - */ - std::optional parent; - public: /** * Create an `Input` from a URL. @@ -73,6 +68,12 @@ public: */ bool isLocked() const; + /** + * Only for relative path flakes, i.e. 'path:./foo', returns the + * relative path, i.e. './foo'. + */ + std::optional isRelative() const; + bool operator ==(const Input & other) const; bool contains(const Input & other) const; @@ -220,6 +221,9 @@ struct InputScheme * if there is a mismatch. */ virtual void checkLocks(const Input & specified, const Input & final) const; + + virtual std::optional isRelative(const Input & input) const + { return std::nullopt; } }; void registerInputScheme(std::shared_ptr && fetcher); diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 68958d559..62500bbcb 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -116,31 +116,14 @@ struct PathInputScheme : InputScheme std::pair, Input> getAccessor(ref store, const Input & _input) const override { Input input(_input); - std::string absPath; auto path = getStrAttr(input.attrs, "path"); - if (path[0] != '/') { - if (!input.parent) - throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string()); + auto absPath = getAbsPath(input); - auto parent = canonPath(*input.parent); - - // the path isn't relative, prefix it - absPath = nix::absPath(path, parent); - - // for security, ensure that if the parent is a store path, it's inside it - if (store->isInStore(parent)) { - auto storePath = store->printStorePath(store->toStorePath(parent).first); - if (!isDirOrInDir(absPath, storePath)) - throw BadStorePath("relative path '%s' points outside of its parent's store path '%s'", path, storePath); - } - } else - absPath = path; - - Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s'", absPath)); + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s' to the store", absPath)); // FIXME: check whether access to 'path' is allowed. - auto storePath = store->maybeParseStorePath(absPath); + auto storePath = store->maybeParseStorePath(absPath.abs()); if (storePath) store->addTempRoot(*storePath); @@ -149,7 +132,7 @@ struct PathInputScheme : InputScheme if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) { // FIXME: try to substitute storePath. auto src = sinkToSource([&](Sink & sink) { - mtime = dumpPathAndGetMtime(absPath, sink, defaultPathFilter); + mtime = dumpPathAndGetMtime(absPath.abs(), sink, defaultPathFilter); }); storePath = store->addToStoreFromDump(*src, "source"); } diff --git a/src/nix/flake.md b/src/nix/flake.md index 661dd2f73..e60abe5fc 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -195,18 +195,29 @@ Currently the `type` attribute can be one of the following: If the flake at *path* is not inside a git repository, the `path:` prefix is implied and can be omitted. - *path* generally must be an absolute path. However, on the command - line, it can be a relative path (e.g. `.` or `./foo`) which is - interpreted as relative to the current directory. In this case, it - must start with `.` to avoid ambiguity with registry lookups - (e.g. `nixpkgs` is a registry lookup; `./nixpkgs` is a relative - path). + If *path* is a relative path (i.e. if it does not start with `/`), + it is interpreted as follows: + + - If *path* is a command line argument, it is interpreted relative + to the current directory. + + - If *path* is used in a `flake.nix`, it is interpreted relative to + the directory containing that `flake.nix`. However, the resolved + path must be in the same tree. For instance, a `flake.nix` in the + root of a tree can use `path:./foo` to access the flake in + subdirectory `foo`, but `path:../bar` is illegal. + + Note that if you omit `path:`, relative paths must start with `.` to + avoid ambiguity with registry lookups (e.g. `nixpkgs` is a registry + lookup; `./nixpkgs` is a relative path). For example, these are valid path flake references: * `path:/home/user/sub/dir` * `/home/user/sub/dir` (if `dir/flake.nix` is *not* in a git repository) - * `./sub/dir` (when used on the command line and `dir/flake.nix` is *not* in a git repository) + * `path:sub/dir` + * `./sub/dir` + * `path:../parent` * `git`: Git repositories. The location of the repository is specified by the attribute `url`. diff --git a/tests/functional/flakes/follow-paths.sh b/tests/functional/flakes/follow-paths.sh index 1afd91bd2..35f52f3ba 100644 --- a/tests/functional/flakes/follow-paths.sh +++ b/tests/functional/flakes/follow-paths.sh @@ -115,7 +115,7 @@ nix flake lock $flakeFollowsA [[ $(jq -c .nodes.B.inputs.foobar $flakeFollowsA/flake.lock) = '"foobar"' ]] jq -r -c '.nodes | keys | .[]' $flakeFollowsA/flake.lock | grep "^foobar$" -# Ensure a relative path is not allowed to go outside the store path +# Check that path: inputs cannot escape from their root. cat > $flakeFollowsA/flake.nix <&1 | grep 'points outside' +expect 1 nix flake lock $flakeFollowsA 2>&1 | grep '/flakeB.*is forbidden in pure evaluation mode' +expect 1 nix flake lock --impure $flakeFollowsA 2>&1 | grep '/flakeB.*does not exist' + +# Test relative non-flake inputs. +cat > $flakeFollowsA/flake.nix < $flakeFollowsA/foo.nix + +git -C $flakeFollowsA add flake.nix foo.nix + +nix flake lock $flakeFollowsA + +[[ $(nix eval --json $flakeFollowsA#e) = 123 ]] # Non-existant follows should print a warning. cat >$flakeFollowsA/flake.nix < $rootFlake/flake.nix < $subflake0/flake.nix < $subflake1/flake.nix < $subflake2/flake.nix < $rootFlake/flake.nix < Date: Fri, 17 May 2024 16:38:01 +0200 Subject: [PATCH 02/19] call-flake.nix: Fix relative path resolution `parentNode.sourceInfo.outPath` does not include the subdir of the parent flake, while `parentNode.outPath` does. So we need to use the latter. --- src/libexpr/flake/call-flake.nix | 2 +- tests/functional/flakes/relative-paths.sh | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libexpr/flake/call-flake.nix b/src/libexpr/flake/call-flake.nix index 43ecb7f15..38dc74ba1 100644 --- a/src/libexpr/flake/call-flake.nix +++ b/src/libexpr/flake/call-flake.nix @@ -47,7 +47,7 @@ let else if node.locked.type == "path" && builtins.substring 0 1 node.locked.path != "/" then parentNode.sourceInfo // { - outPath = parentNode.sourceInfo.outPath + ("/" + node.locked.path); + outPath = parentNode.outPath + ("/" + node.locked.path); } else # FIXME: remove obsolete node.info. diff --git a/tests/functional/flakes/relative-paths.sh b/tests/functional/flakes/relative-paths.sh index cecda44d4..38987d6af 100644 --- a/tests/functional/flakes/relative-paths.sh +++ b/tests/functional/flakes/relative-paths.sh @@ -63,6 +63,10 @@ git -C $rootFlake add flake.nix sub2/flake.nix [[ $(nix eval $subflake2#y) = 15 ]] +# Make sure that this still works after commiting the lock file. +git -C $rootFlake add sub2/flake.lock +[[ $(nix eval $subflake2#y) = 15 ]] + # Make sure there are no content locks for relative path flakes. (! grep "$TEST_ROOT" $subflake2/flake.lock) (! grep "$NIX_STORE_DIR" $subflake2/flake.lock) From 3180671cabeb6a6010057770731e12761ed5666c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 May 2024 19:49:40 +0200 Subject: [PATCH 03/19] Allow the 'url' flake input attribute to be a path literal https://github.com/NixOS/nix/pull/10089#issuecomment-1978133326 --- src/libexpr/flake/flake.cc | 34 +++++++++++++++++------ tests/functional/flakes/relative-paths.sh | 4 +-- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index d4cabe68f..1401f5dda 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -96,14 +96,16 @@ static std::map parseFlakeInputs( EvalState & state, Value * value, const PosIdx pos, - InputPath lockRootPath); + InputPath lockRootPath, + const SourcePath & flakeDir); static FlakeInput parseFlakeInput( EvalState & state, const std::string & inputName, Value * value, const PosIdx pos, - InputPath lockRootPath) + InputPath lockRootPath, + const SourcePath & flakeDir) { expectType(state, nAttrs, *value, pos); @@ -120,14 +122,25 @@ static FlakeInput parseFlakeInput( for (auto & attr : *value->attrs()) { try { if (attr.name == sUrl) { - expectType(state, nString, *attr.value, attr.pos); - url = attr.value->string_view(); + forceTrivialValue(state, *attr.value, pos); + if (attr.value->type() == nString) + url = attr.value->string_view(); + else if (attr.value->type() == nPath) { + auto path = attr.value->path(); + if (path.accessor != flakeDir.accessor) + throw Error("input path '%s' at %s must be in the same source tree as %s", + path, state.positions[attr.pos], flakeDir); + url = "path:" + flakeDir.path.makeRelative(path.path); + } + else + throw Error("expected a string or a path but got %s at %s", + showType(attr.value->type()), state.positions[attr.pos]); attrs.emplace("url", *url); } else if (attr.name == sFlake) { expectType(state, nBool, *attr.value, attr.pos); input.isFlake = attr.value->boolean(); } else if (attr.name == sInputs) { - input.overrides = parseFlakeInputs(state, attr.value, attr.pos, lockRootPath); + input.overrides = parseFlakeInputs(state, attr.value, attr.pos, lockRootPath, flakeDir); } else if (attr.name == sFollows) { expectType(state, nString, *attr.value, attr.pos); auto follows(parseInputPath(attr.value->c_str())); @@ -191,7 +204,8 @@ static std::map parseFlakeInputs( EvalState & state, Value * value, const PosIdx pos, - InputPath lockRootPath) + InputPath lockRootPath, + const SourcePath & flakeDir) { std::map inputs; @@ -203,7 +217,8 @@ static std::map parseFlakeInputs( state.symbols[inputAttr.name], inputAttr.value, inputAttr.pos, - lockRootPath)); + lockRootPath, + flakeDir)); } return inputs; @@ -217,7 +232,8 @@ static Flake readFlake( const SourcePath & rootDir, const InputPath & lockRootPath) { - auto flakePath = rootDir / CanonPath(resolvedRef.subdir) / "flake.nix"; + auto flakeDir = rootDir / CanonPath(resolvedRef.subdir); + auto flakePath = flakeDir / "flake.nix"; // NOTE evalFile forces vInfo to be an attrset because mustBeTrivial is true. Value vInfo; @@ -238,7 +254,7 @@ static Flake readFlake( auto sInputs = state.symbols.create("inputs"); if (auto inputs = vInfo.attrs()->get(sInputs)) - flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, lockRootPath); + flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, lockRootPath, flakeDir); auto sOutputs = state.symbols.create("outputs"); diff --git a/tests/functional/flakes/relative-paths.sh b/tests/functional/flakes/relative-paths.sh index 38987d6af..3e4c97cc4 100644 --- a/tests/functional/flakes/relative-paths.sh +++ b/tests/functional/flakes/relative-paths.sh @@ -12,7 +12,7 @@ mkdir -p $rootFlake $subflake0 $subflake1 $subflake2 cat > $rootFlake/flake.nix < $subflake2/flake.nix < Date: Mon, 16 Sep 2024 14:11:08 +0200 Subject: [PATCH 04/19] shellcheck --- tests/functional/flakes/relative-paths.sh | 54 ++++++++++++----------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/tests/functional/flakes/relative-paths.sh b/tests/functional/flakes/relative-paths.sh index 3e4c97cc4..e72339b7f 100644 --- a/tests/functional/flakes/relative-paths.sh +++ b/tests/functional/flakes/relative-paths.sh @@ -1,16 +1,18 @@ +#!/usr/bin/env bash + source ./common.sh requireGit -rootFlake=$TEST_ROOT/flake1 -subflake0=$rootFlake/sub0 -subflake1=$rootFlake/sub1 -subflake2=$rootFlake/sub2 +rootFlake="$TEST_ROOT/flake1" +subflake0="$rootFlake/sub0" +subflake1="$rootFlake/sub1" +subflake2="$rootFlake/sub2" -rm -rf $rootFlake -mkdir -p $rootFlake $subflake0 $subflake1 $subflake2 +rm -rf "$rootFlake" +mkdir -p "$rootFlake" "$subflake0" "$subflake1" "$subflake2" -cat > $rootFlake/flake.nix < "$rootFlake/flake.nix" < $rootFlake/flake.nix < $subflake0/flake.nix < "$subflake0/flake.nix" < $subflake0/flake.nix < $subflake1/flake.nix < "$subflake1/flake.nix" < $subflake1/flake.nix < $subflake2/flake.nix < "$subflake2/flake.nix" < $subflake2/flake.nix < $rootFlake/flake.nix < "$rootFlake/flake.nix" < $rootFlake/flake.nix < Date: Mon, 16 Sep 2024 14:52:23 +0200 Subject: [PATCH 05/19] parentPath -> parentInputPath --- src/libflake/flake/flake.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index 632968f50..461ac0a90 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -404,7 +404,7 @@ LockedFlake lockFlake( { FlakeInput input; SourcePath sourcePath; - std::optional parentPath; // FIXME: rename to inputPathPrefix? + std::optional parentInputPath; // FIXME: rename to inputPathPrefix? }; std::map overrides; @@ -469,7 +469,7 @@ LockedFlake lockFlake( Override { .input = inputOverride, .sourcePath = sourcePath, - .parentPath = inputPathPrefix // FIXME: should this be inputPath? + .parentInputPath = inputPathPrefix // FIXME: should this be inputPath? }); } } @@ -531,7 +531,7 @@ LockedFlake lockFlake( auto overridenParentPath = input.ref->input.isRelative() - ? std::optional(hasOverride ? i->second.parentPath : inputPathPrefix) + ? std::optional(hasOverride ? i->second.parentInputPath : inputPathPrefix) : std::nullopt; auto resolveRelativePath = [&]() -> std::optional From f2063255a40d534035b706235752a5c0a09c6d15 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 25 Sep 2024 16:29:43 +0200 Subject: [PATCH 06/19] tests/functional/flakes/relative-paths.sh: Fix build failure in hydraJobs.tests.functional_user --- tests/functional/flakes/relative-paths.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/functional/flakes/relative-paths.sh b/tests/functional/flakes/relative-paths.sh index e72339b7f..9b93da9c1 100644 --- a/tests/functional/flakes/relative-paths.sh +++ b/tests/functional/flakes/relative-paths.sh @@ -71,7 +71,9 @@ git -C "$rootFlake" add sub2/flake.lock # Make sure there are no content locks for relative path flakes. (! grep "$TEST_ROOT" "$subflake2/flake.lock") -(! grep "$NIX_STORE_DIR" "$subflake2/flake.lock") +if ! isTestOnNixOS; then + (! grep "$NIX_STORE_DIR" "$subflake2/flake.lock") +fi (! grep narHash "$subflake2/flake.lock") # Test circular relative path flakes. FIXME: doesn't work at the moment. From 00b99b8bc0aa7f5f0a1a04b9704c2120e5eca6db Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 22 Nov 2024 16:23:34 +0100 Subject: [PATCH 07/19] Remove FIXME --- src/libflake/flake/flake.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index e49333f25..8e381c0d2 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -470,7 +470,7 @@ LockedFlake lockFlake( Override { .input = inputOverride, .sourcePath = sourcePath, - .parentInputPath = inputPathPrefix // FIXME: should this be inputPath? + .parentInputPath = inputPathPrefix }); } } From 985b2f9df30e62a21c32d7ffaae7aebfe4c56d3a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Nov 2024 15:23:56 +0100 Subject: [PATCH 08/19] Remove FIXME --- src/libflake/flake/flake.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index 8e381c0d2..9d5760ed1 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -638,10 +638,6 @@ LockedFlake lockFlake( nodePaths.emplace(childNode, inputFlake.path.parent()); computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, lockRootPath, inputFlake.path, false); } else { - // FIXME: sourcePath is wrong here, we - // should pass a lambda that lazily - // fetches the parent flake if needed - // (i.e. getInputFlake()). computeLocks(fakeInputs, childNode, inputPath, oldLock, lockRootPath, sourcePath, true); } From 9223d64ac62385d3b75cfbf5bac491fc14d033ad Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 23 Dec 2024 16:03:13 +0100 Subject: [PATCH 09/19] Remove dead code --- src/libflake/flake/flakeref.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index 41e991c20..3a43a51f8 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -195,10 +195,6 @@ std::pair parsePathFlakeRefWithFragment( throw BadURL("flake reference '%s' is not an absolute path", url); } - fetchers::Attrs attrs; - attrs.insert_or_assign("type", "path"); - attrs.insert_or_assign("path", path); - return fromParsedURL(fetchSettings, { .url = path, .base = path, From 75cda2da7fbd6283f88cf0fd0b934e1047ca1408 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Jan 2025 13:40:18 +0100 Subject: [PATCH 10/19] Document path values in inputs Co-authored-by: Robert Hensing --- src/nix/flake.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/nix/flake.md b/src/nix/flake.md index a55bba4fc..5bb18328d 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -212,6 +212,13 @@ Currently the `type` attribute can be one of the following: root of a tree can use `path:./foo` to access the flake in subdirectory `foo`, but `path:../bar` is illegal. +Path inputs can be specified with path values in `flake.nix`. Path values are a syntax for `path` inputs, and they are converted by +1. resolving them into relative paths, relative to the base directory of `flake.nix` +2. escaping URL characters (refer to IETF RFC?) +3. prepending `path:` + +Note that the allowed syntax for path values in flake `inputs` may be more restrictive than general Nix, so you may need to use `path:` if your path contains certain special characters. See [Path literals](@docroot@/language/syntax#path-literal) + Note that if you omit `path:`, relative paths must start with `.` to avoid ambiguity with registry lookups (e.g. `nixpkgs` is a registry lookup; `./nixpkgs` is a relative path). From e8c7dd9971aaac0ac14e08f299a41ef0a93afef7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Jan 2025 13:43:56 +0100 Subject: [PATCH 11/19] Rename allowRelative -> preserveRelativePaths --- src/libflake/flake/flakeref.cc | 12 ++++++------ src/libflake/flake/flakeref.hh | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index 3a43a51f8..06198b10c 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -49,9 +49,9 @@ FlakeRef parseFlakeRef( const std::optional & baseDir, bool allowMissing, bool isFlake, - bool allowRelative) + bool preserveRelativePaths) { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(fetchSettings, url, baseDir, allowMissing, isFlake, allowRelative); + auto [flakeRef, fragment] = parseFlakeRefWithFragment(fetchSettings, url, baseDir, allowMissing, isFlake, preserveRelativePaths); if (fragment != "") throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url); return flakeRef; @@ -89,7 +89,7 @@ std::pair parsePathFlakeRefWithFragment( const std::optional & baseDir, bool allowMissing, bool isFlake, - bool allowRelative) + bool preserveRelativePaths) { std::string path = url; std::string fragment = ""; @@ -191,7 +191,7 @@ std::pair parsePathFlakeRefWithFragment( } } else { - if (!allowRelative && !hasPrefix(path, "/")) + if (!preserveRelativePaths && !hasPrefix(path, "/")) throw BadURL("flake reference '%s' is not an absolute path", url); } @@ -258,7 +258,7 @@ std::pair parseFlakeRefWithFragment( const std::optional & baseDir, bool allowMissing, bool isFlake, - bool allowRelative) + bool preserveRelativePaths) { using namespace fetchers; @@ -267,7 +267,7 @@ std::pair parseFlakeRefWithFragment( } else if (auto res = parseURLFlakeRef(fetchSettings, url, baseDir, isFlake)) { return *res; } else { - return parsePathFlakeRefWithFragment(fetchSettings, url, baseDir, allowMissing, isFlake, allowRelative); + return parsePathFlakeRefWithFragment(fetchSettings, url, baseDir, allowMissing, isFlake, preserveRelativePaths); } } diff --git a/src/libflake/flake/flakeref.hh b/src/libflake/flake/flakeref.hh index 32094d381..c9cf7952d 100644 --- a/src/libflake/flake/flakeref.hh +++ b/src/libflake/flake/flakeref.hh @@ -85,7 +85,7 @@ FlakeRef parseFlakeRef( const std::optional & baseDir = {}, bool allowMissing = false, bool isFlake = true, - bool allowRelative = false); + bool preserveRelativePaths = false); /** * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) @@ -104,7 +104,7 @@ std::pair parseFlakeRefWithFragment( const std::optional & baseDir = {}, bool allowMissing = false, bool isFlake = true, - bool allowRelative = false); + bool preserveRelativePaths = false); /** * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) From 0792152627a94dbb2fbbbd8f0f05ff36a480dabf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Jan 2025 13:54:19 +0100 Subject: [PATCH 12/19] Rename Override -> OverrideTarget --- src/libflake/flake/flake.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index e9fa28f06..4f0b922ab 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -408,14 +408,14 @@ LockedFlake lockFlake( debug("old lock file: %s", oldLockFile); - struct Override + struct OverrideTarget { FlakeInput input; SourcePath sourcePath; std::optional parentInputPath; // FIXME: rename to inputPathPrefix? }; - std::map overrides; + std::map overrides; std::set explicitCliOverrides; std::set overridesUsed, updatesUsed; std::map, SourcePath> nodePaths; @@ -423,7 +423,7 @@ LockedFlake lockFlake( for (auto & i : lockFlags.inputOverrides) { overrides.emplace( i.first, - Override { + OverrideTarget { .input = FlakeInput { .ref = i.second }, /* Note: any relative overrides (e.g. `--override-input B/C "path:./foo/bar"`) @@ -474,7 +474,7 @@ LockedFlake lockFlake( inputPath.push_back(id); inputPath.push_back(idOverride); overrides.emplace(inputPath, - Override { + OverrideTarget { .input = inputOverride, .sourcePath = sourcePath, .parentInputPath = inputPathPrefix From ef2739b7c9071149fb9333865bd3cc2ab60f2178 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Jan 2025 14:01:49 +0100 Subject: [PATCH 13/19] Example of referencing parent directories --- src/nix/flake.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nix/flake.md b/src/nix/flake.md index 5bb18328d..35ac3ea9b 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -210,7 +210,9 @@ Currently the `type` attribute can be one of the following: the directory containing that `flake.nix`. However, the resolved path must be in the same tree. For instance, a `flake.nix` in the root of a tree can use `path:./foo` to access the flake in - subdirectory `foo`, but `path:../bar` is illegal. + subdirectory `foo`, but `path:../bar` is illegal. On the other + hand, a flake in the `/foo` directory of a tree can use + `path:../bar` to refer to the flake in `/bar`. Path inputs can be specified with path values in `flake.nix`. Path values are a syntax for `path` inputs, and they are converted by 1. resolving them into relative paths, relative to the base directory of `flake.nix` From d329b2632a91b05e1151decbb2b4c4011ebf5aa1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Jan 2025 17:04:06 +0100 Subject: [PATCH 14/19] Fix manual --- src/nix/flake.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/flake.md b/src/nix/flake.md index 35ac3ea9b..e79a8ee0b 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -219,7 +219,7 @@ Path inputs can be specified with path values in `flake.nix`. Path values are a 2. escaping URL characters (refer to IETF RFC?) 3. prepending `path:` -Note that the allowed syntax for path values in flake `inputs` may be more restrictive than general Nix, so you may need to use `path:` if your path contains certain special characters. See [Path literals](@docroot@/language/syntax#path-literal) +Note that the allowed syntax for path values in flake `inputs` may be more restrictive than general Nix, so you may need to use `path:` if your path contains certain special characters. See [Path literals](@docroot@/language/syntax.md#path-literal) Note that if you omit `path:`, relative paths must start with `.` to avoid ambiguity with registry lookups (e.g. `nixpkgs` is a registry From 6cc5b48a291fc9f805d0957ad73fc770ef426b39 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 14 Jan 2025 14:51:49 +0100 Subject: [PATCH 15/19] Add release note --- doc/manual/rl-next/relative-path-flakes.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/manual/rl-next/relative-path-flakes.md diff --git a/doc/manual/rl-next/relative-path-flakes.md b/doc/manual/rl-next/relative-path-flakes.md new file mode 100644 index 000000000..5a4412316 --- /dev/null +++ b/doc/manual/rl-next/relative-path-flakes.md @@ -0,0 +1,12 @@ +--- +synopsis: "Support for relative path inputs" +prs: [10089] +--- + +Flakes can now refer to other flakes in the same repository using relative paths, e.g. +```nix +inputs.foo.url = "path:./foo"; +``` +uses the flake in the `foo` subdirectory of the referring flake. + +This feature required a change to the lock file format. Previous Nix versions will not be able to use lock files that have locks for relative path inputs in them. From 521667eb8908843925dc57e195f7702d66d0ac16 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Jan 2025 11:19:20 +0100 Subject: [PATCH 16/19] Fix follow-paths test Since ff8e2fe84e4950cd20dbc513cf6eaa1d40d6bf65, 'path:' URLs on the CLI are interpreted as relative to the current directory of the user, not the path of the flake we're overriding. --- tests/functional/flakes/follow-paths.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/flakes/follow-paths.sh b/tests/functional/flakes/follow-paths.sh index 966f69cf7..a71d4c6d7 100755 --- a/tests/functional/flakes/follow-paths.sh +++ b/tests/functional/flakes/follow-paths.sh @@ -356,6 +356,6 @@ json=$(nix flake metadata "$flakeFollowsCustomUrlA" --json) rm "$flakeFollowsCustomUrlA"/flake.lock # if override-input is specified, lock "original" entry should contain original url -json=$(nix flake metadata "$flakeFollowsCustomUrlA" --override-input B/C "path:./flakeB/flakeD" --json) +json=$(nix flake metadata "$flakeFollowsCustomUrlA" --override-input B/C "$flakeFollowsCustomUrlD" --json) echo "$json" | jq .locks.nodes.C.original [[ $(echo "$json" | jq -r .locks.nodes.C.original.path) = './flakeC' ]] From 5d03ef9caf136de06b79298b4a5f8c80a4abadbf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Jan 2025 11:26:14 +0100 Subject: [PATCH 17/19] PathInputSchema::getAbsPath(): Return std::filesystem::path --- src/libfetchers/path.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 9691cc9c5..351e21a1b 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -91,7 +91,7 @@ struct PathInputScheme : InputScheme std::string_view contents, std::optional commitMsg) const override { - writeFile((CanonPath(getAbsPath(input)) / path).abs(), contents); + writeFile(getAbsPath(input) / path.rel(), contents); } std::optional isRelative(const Input & input) const override @@ -108,12 +108,12 @@ struct PathInputScheme : InputScheme return (bool) input.getNarHash(); } - CanonPath getAbsPath(const Input & input) const + std::filesystem::path getAbsPath(const Input & input) const { auto path = getStrAttr(input.attrs, "path"); - if (path[0] == '/') - return CanonPath(path); + if (isAbsolute(path)) + return canonPath(path); throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string()); } @@ -128,7 +128,7 @@ struct PathInputScheme : InputScheme Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s' to the store", absPath)); // FIXME: check whether access to 'path' is allowed. - auto storePath = store->maybeParseStorePath(absPath.abs()); + auto storePath = store->maybeParseStorePath(absPath.string()); if (storePath) store->addTempRoot(*storePath); @@ -137,7 +137,7 @@ struct PathInputScheme : InputScheme if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) { // FIXME: try to substitute storePath. auto src = sinkToSource([&](Sink & sink) { - mtime = dumpPathAndGetMtime(absPath.abs(), sink, defaultPathFilter); + mtime = dumpPathAndGetMtime(absPath.string(), sink, defaultPathFilter); }); storePath = store->addToStoreFromDump(*src, "source"); } @@ -159,7 +159,7 @@ struct PathInputScheme : InputScheme store object and the subpath. */ auto path = getAbsPath(input); try { - auto [storePath, subPath] = store->toStorePath(path.abs()); + auto [storePath, subPath] = store->toStorePath(path.string()); auto info = store->queryPathInfo(storePath); return fmt("path:%s:%s", info->narHash.to_string(HashFormat::Base16, false), subPath); } catch (Error &) { From 8b1fb92a0ce7fd8c0ad7955de1a59bb42866339e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Jan 2025 11:31:22 +0100 Subject: [PATCH 18/19] flakes.md: Fix indentation that broke the list --- src/nix/flake.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/nix/flake.md b/src/nix/flake.md index e79a8ee0b..2072dd3dd 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -214,12 +214,12 @@ Currently the `type` attribute can be one of the following: hand, a flake in the `/foo` directory of a tree can use `path:../bar` to refer to the flake in `/bar`. -Path inputs can be specified with path values in `flake.nix`. Path values are a syntax for `path` inputs, and they are converted by -1. resolving them into relative paths, relative to the base directory of `flake.nix` -2. escaping URL characters (refer to IETF RFC?) -3. prepending `path:` + Path inputs can be specified with path values in `flake.nix`. Path values are a syntax for `path` inputs, and they are converted by + 1. resolving them into relative paths, relative to the base directory of `flake.nix` + 2. escaping URL characters (refer to IETF RFC?) + 3. prepending `path:` -Note that the allowed syntax for path values in flake `inputs` may be more restrictive than general Nix, so you may need to use `path:` if your path contains certain special characters. See [Path literals](@docroot@/language/syntax.md#path-literal) + Note that the allowed syntax for path values in flake `inputs` may be more restrictive than general Nix, so you may need to use `path:` if your path contains certain special characters. See [Path literals](@docroot@/language/syntax.md#path-literal) Note that if you omit `path:`, relative paths must start with `.` to avoid ambiguity with registry lookups (e.g. `nixpkgs` is a registry From db46d40b12cc57de47c468a73262f1c20b0374c8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Jan 2025 13:15:20 +0100 Subject: [PATCH 19/19] Update release note --- doc/manual/rl-next/relative-path-flakes.md | 2 +- src/nix/flake.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/rl-next/relative-path-flakes.md b/doc/manual/rl-next/relative-path-flakes.md index 5a4412316..3616f3467 100644 --- a/doc/manual/rl-next/relative-path-flakes.md +++ b/doc/manual/rl-next/relative-path-flakes.md @@ -7,6 +7,6 @@ Flakes can now refer to other flakes in the same repository using relative paths ```nix inputs.foo.url = "path:./foo"; ``` -uses the flake in the `foo` subdirectory of the referring flake. +uses the flake in the `foo` subdirectory of the referring flake. For more information, see the documentation on [the `path` flake input type](@docroot@/command-ref/new-cli/nix3-flake.md#path-fetcher). This feature required a change to the lock file format. Previous Nix versions will not be able to use lock files that have locks for relative path inputs in them. diff --git a/src/nix/flake.md b/src/nix/flake.md index 2072dd3dd..364302b61 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -187,7 +187,7 @@ Currently the `type` attribute can be one of the following: * `nixpkgs/nixos-unstable/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293` * `sub/dir` (if a flake named `sub` is in the registry) -* `path`: arbitrary local directories. The required attribute `path` +* `path`: arbitrary local directories. The required attribute `path` specifies the path of the flake. The URL form is ```