diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 87a14b4bc..13d911518 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,7 +91,7 @@ jobs: flake_regressions: if: github.event_name == 'merge_group' needs: build_x86_64-linux - runs-on: blacksmith-32vcpu-ubuntu-2204 + runs-on: namespace-profile-x86-32cpu-64gb steps: - name: Checkout nix uses: actions/checkout@v4 @@ -109,7 +109,30 @@ jobs: with: determinate: true - uses: DeterminateSystems/flakehub-cache-action@main - - run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH MAX_FLAKES=50 flake-regressions/eval-all.sh + - run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH PARALLEL="-P 50%" flake-regressions/eval-all.sh + + flake_regressions_lazy: + if: github.event_name == 'merge_group' + needs: build_x86_64-linux + runs-on: namespace-profile-x86-32cpu-64gb + steps: + - name: Checkout nix + uses: actions/checkout@v4 + - name: Checkout flake-regressions + uses: actions/checkout@v4 + with: + repository: DeterminateSystems/flake-regressions + path: flake-regressions + - name: Checkout flake-regressions-data + uses: actions/checkout@v4 + with: + repository: DeterminateSystems/flake-regressions-data + path: flake-regressions/tests + - uses: DeterminateSystems/nix-installer-action@main + with: + determinate: true + - uses: DeterminateSystems/flakehub-cache-action@main + - run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH PARALLEL="-P 50%" NIX_CONFIG="lazy-trees = true" flake-regressions/eval-all.sh manual: if: github.event_name != 'merge_group' diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc index d9ac3a29e..4eb4993b1 100644 --- a/src/libcmd/installable-value.cc +++ b/src/libcmd/installable-value.cc @@ -57,7 +57,8 @@ std::optional InstallableValue::trySinglePathToDerivedPaths else if (v.type() == nString) { return {{ .path = DerivedPath::fromSingle( - state->coerceToSingleDerivedPath(pos, v, errorCtx)), + state->devirtualize( + state->coerceToSingleDerivedPath(pos, v, errorCtx))), .info = make_ref(), }}; } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f095b393b..6505de7bc 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -20,6 +20,7 @@ #include "nix/util/url.hh" #include "nix/fetchers/fetch-to-store.hh" #include "nix/fetchers/tarball.hh" +#include "nix/fetchers/input-cache.hh" #include "parser-tab.hh" @@ -266,12 +267,9 @@ EvalState::EvalState( /nix/store while using a chroot store. */ auto accessor = getFSSourceAccessor(); - auto realStoreDir = dirOf(store->toRealPath(StorePath::dummy)); - if (settings.pureEval || store->storeDir != realStoreDir) { - accessor = settings.pureEval - ? storeFS.cast() - : makeUnionSourceAccessor({accessor, storeFS}); - } + accessor = settings.pureEval + ? storeFS.cast() + : makeUnionSourceAccessor({accessor, storeFS}); /* Apply access control if needed. */ if (settings.restrictEval || settings.pureEval) @@ -293,6 +291,7 @@ EvalState::EvalState( )} , store(store) , buildStore(buildStore ? buildStore : store) + , inputCache(fetchers::InputCache::create()) , debugRepl(nullptr) , debugStop(false) , trylevel(0) @@ -949,7 +948,16 @@ void EvalState::mkPos(Value & v, PosIdx p) auto origin = positions.originOf(p); if (auto path = std::get_if(&origin)) { auto attrs = buildBindings(3); - attrs.alloc(sFile).mkString(path->path.abs()); + if (path->accessor == rootFS && store->isInStore(path->path.abs())) + // FIXME: only do this for virtual store paths? + attrs.alloc(sFile).mkString(path->path.abs(), + { + NixStringContextElem::Opaque{ + .path = store->toStorePath(path->path.abs()).first + } + }); + else + attrs.alloc(sFile).mkString(path->path.abs()); makePositionThunks(*this, p, attrs.alloc(sLine), attrs.alloc(sColumn)); v.mkAttrs(attrs); } else @@ -1135,6 +1143,7 @@ void EvalState::resetFileCache() { fileEvalCache.clear(); fileParseCache.clear(); + inputCache->clear(); } @@ -2317,6 +2326,9 @@ BackedStringView EvalState::coerceToString( } if (v.type() == nPath) { + // FIXME: instead of copying the path to the store, we could + // return a virtual store path that lazily copies the path to + // the store in devirtualize(). return !canonicalizePath && !copyToStore ? // FIXME: hack to preserve path literals that end in a @@ -2406,7 +2418,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat *store, path.resolveSymlinks(SymlinkResolution::Ancestors), settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, - path.baseName(), + computeBaseName(path), ContentAddressMethod::Raw::NixArchive, nullptr, repair); @@ -2461,7 +2473,7 @@ StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringCon auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); if (auto storePath = store->maybeParseStorePath(path)) return *storePath; - error("path '%1%' is not in the Nix store", path).withTrace(pos, errorCtx).debugThrow(); + error("cannot coerce '%s' to a store path because it is not a subpath of the Nix store", path).withTrace(pos, errorCtx).debugThrow(); } diff --git a/src/libexpr/include/nix/expr/eval-settings.hh b/src/libexpr/include/nix/expr/eval-settings.hh index fb482568a..6e5bbca20 100644 --- a/src/libexpr/include/nix/expr/eval-settings.hh +++ b/src/libexpr/include/nix/expr/eval-settings.hh @@ -247,6 +247,11 @@ struct EvalSettings : Config This option can be enabled by setting `NIX_ABORT_ON_WARN=1` in the environment. )"}; + + Setting lazyTrees{this, false, "lazy-trees", + R"( + If set to true, flakes and trees fetched by [`builtins.fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) are only copied to the Nix store when they're used as a dependency of a derivation. This avoids copying (potentially large) source trees unnecessarily. + )"}; }; /** diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index 9623c2a9c..d82baddb1 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -33,7 +33,11 @@ namespace nix { constexpr size_t maxPrimOpArity = 8; class Store; -namespace fetchers { struct Settings; } +namespace fetchers { +struct Settings; +struct InputCache; +struct Input; +} struct EvalSettings; class EvalState; class StorePath; @@ -301,6 +305,8 @@ public: RootValue vImportedDrvToDerivation = nullptr; + ref inputCache; + /** * Debugger */ @@ -445,6 +451,15 @@ public: void checkURI(const std::string & uri); + /** + * Mount an input on the Nix store. + */ + StorePath mountInput( + fetchers::Input & input, + const fetchers::Input & originalInput, + ref accessor, + bool requireLockable); + /** * Parse a Nix expression from the specified file. */ @@ -554,6 +569,18 @@ public: std::optional tryAttrsToString(const PosIdx pos, Value & v, NixStringContext & context, bool coerceMore = false, bool copyToStore = true); + StorePath devirtualize( + const StorePath & path, + StringMap * rewrites = nullptr); + + SingleDerivedPath devirtualize( + const SingleDerivedPath & path, + StringMap * rewrites = nullptr); + + std::string devirtualize( + std::string_view s, + const NixStringContext & context); + /** * String coercion. * @@ -569,6 +596,19 @@ public: StorePath copyPathToStore(NixStringContext & context, const SourcePath & path); + + /** + * Compute the base name for a `SourcePath`. For non-store paths, + * this is just `SourcePath::baseName()`. But for store paths, for + * backwards compatibility, it needs to be `-source`, + * i.e. as if the path were copied to the Nix store. This results + * in a "double-copied" store path like + * `/nix/store/--source`. We don't need to + * materialize /nix/store/-source though. Still, this + * requires reading/hashing the path twice. + */ + std::string computeBaseName(const SourcePath & path); + /** * Path coercion. * diff --git a/src/libexpr/include/nix/expr/print-ambiguous.hh b/src/libexpr/include/nix/expr/print-ambiguous.hh index 09a849c49..1dafd5d56 100644 --- a/src/libexpr/include/nix/expr/print-ambiguous.hh +++ b/src/libexpr/include/nix/expr/print-ambiguous.hh @@ -15,10 +15,10 @@ namespace nix { * See: https://github.com/NixOS/nix/issues/9730 */ void printAmbiguous( - Value &v, - const SymbolTable &symbols, - std::ostream &str, - std::set *seen, + EvalState & state, + Value & v, + std::ostream & str, + std::set * seen, int depth); } diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index c5107de3a..da1408e9b 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -1,5 +1,7 @@ #include "nix/store/store-api.hh" #include "nix/expr/eval.hh" +#include "nix/util/mounted-source-accessor.hh" +#include "nix/fetchers/fetch-to-store.hh" namespace nix { @@ -18,4 +20,81 @@ SourcePath EvalState::storePath(const StorePath & path) return {rootFS, CanonPath{store->printStorePath(path)}}; } +StorePath EvalState::devirtualize(const StorePath & path, StringMap * rewrites) +{ + if (auto mount = storeFS->getMount(CanonPath(store->printStorePath(path)))) { + auto storePath = fetchToStore( + *store, SourcePath{ref(mount)}, settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, path.name()); + assert(storePath.name() == path.name()); + if (rewrites) + rewrites->emplace(path.hashPart(), storePath.hashPart()); + return storePath; + } else + return path; +} + +SingleDerivedPath EvalState::devirtualize(const SingleDerivedPath & path, StringMap * rewrites) +{ + if (auto o = std::get_if(&path.raw())) + return SingleDerivedPath::Opaque{devirtualize(o->path, rewrites)}; + else + return path; +} + +std::string EvalState::devirtualize(std::string_view s, const NixStringContext & context) +{ + StringMap rewrites; + + for (auto & c : context) + if (auto o = std::get_if(&c.raw)) + devirtualize(o->path, &rewrites); + + return rewriteStrings(std::string(s), rewrites); +} + +std::string EvalState::computeBaseName(const SourcePath & path) +{ + if (path.accessor == rootFS) { + if (auto storePath = store->maybeParseStorePath(path.path.abs())) { + warn( + "Performing inefficient double copy of path '%s' to the store. " + "This can typically be avoided by rewriting an attribute like `src = ./.` " + "to `src = builtins.path { path = ./.; name = \"source\"; }`.", + path); + return std::string(fetchToStore(*store, path, FetchMode::DryRun, storePath->name()).to_string()); + } + } + return std::string(path.baseName()); +} + +StorePath EvalState::mountInput( + fetchers::Input & input, const fetchers::Input & originalInput, ref accessor, bool requireLockable) +{ + auto storePath = settings.lazyTrees ? StorePath::random(input.getName()) + : fetchToStore(*store, accessor, FetchMode::Copy, input.getName()); + + allowPath(storePath); // FIXME: should just whitelist the entire virtual store + + storeFS->mount(CanonPath(store->printStorePath(storePath)), accessor); + + if (requireLockable && !input.isLocked() && !input.getNarHash()) { + auto narHash = accessor->hashPath(CanonPath::root); + input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); + } + + // FIXME: what to do with the NAR hash in lazy mode? + if (!settings.lazyTrees && originalInput.getNarHash()) { + auto expected = originalInput.computeStorePath(*store); + if (storePath != expected) + throw Error( + (unsigned int) 102, + "NAR hash mismatch in input '%s', expected '%s' but got '%s'", + originalInput.to_string(), + store->printStorePath(storePath), + store->printStorePath(expected)); + } + + return storePath; +} + } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 47f048aef..7243f09ce 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -14,6 +14,7 @@ #include "nix/expr/value-to-xml.hh" #include "nix/expr/primops.hh" #include "nix/fetchers/fetch-to-store.hh" +#include "nix/util/mounted-source-accessor.hh" #include #include @@ -75,7 +76,10 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS ensureValid(b.drvPath->getBaseStorePath()); }, [&](const NixStringContextElem::Opaque & o) { - ensureValid(o.path); + // We consider virtual store paths valid here. They'll + // be devirtualized if needed elsewhere. + if (!storeFS->getMount(CanonPath(store->printStorePath(o.path)))) + ensureValid(o.path); if (maybePathsOut) maybePathsOut->emplace(o.path); }, @@ -1408,6 +1412,8 @@ static void derivationStrictInternal( /* Everything in the context of the strings in the derivation attributes should be added as dependencies of the resulting derivation. */ + StringMap rewrites; + for (auto & c : context) { std::visit(overloaded { /* Since this allows the builder to gain access to every @@ -1430,11 +1436,13 @@ static void derivationStrictInternal( drv.inputDrvs.ensureSlot(*b.drvPath).value.insert(b.output); }, [&](const NixStringContextElem::Opaque & o) { - drv.inputSrcs.insert(o.path); + drv.inputSrcs.insert(state.devirtualize(o.path, &rewrites)); }, }, c.raw); } + drv.applyRewrites(rewrites); + /* Do we have all required attributes? */ if (drv.builder == "") state.error("required attribute 'builder' missing") @@ -2500,6 +2508,7 @@ static void addPath( {})); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { + // FIXME: make this lazy? auto dstPath = fetchToStore( *state.store, path.resolveSymlinks(), @@ -2530,7 +2539,7 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * arg "while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"); state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource"); - addPath(state, pos, path.baseName(), path, args[0], ContentAddressMethod::Raw::NixArchive, std::nullopt, v, context); + addPath(state, pos, state.computeBaseName(path), path, args[0], ContentAddressMethod::Raw::NixArchive, std::nullopt, v, context); } static RegisterPrimOp primop_filterSource({ diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 9e135c31a..c82fb82c5 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -10,6 +10,7 @@ #include "nix/util/url.hh" #include "nix/expr/value-to-json.hh" #include "nix/fetchers/fetch-to-store.hh" +#include "nix/fetchers/input-cache.hh" #include "nix/util/mounted-source-accessor.hh" #include @@ -201,13 +202,11 @@ static void fetchTree( throw Error("input '%s' is not allowed to use the '__final' attribute", input.to_string()); } - auto [storePath, accessor, input2] = input.fetchToStore(state.store); + auto cachedInput = state.inputCache->getAccessor(state.store, input, fetchers::UseRegistries::No); - state.allowPath(storePath); + auto storePath = state.mountInput(cachedInput.lockedInput, input, cachedInput.accessor, true); - state.storeFS->mount(CanonPath(state.store->printStorePath(storePath)), accessor); - - emitTreeAttrs(state, storePath, input2, v, params.emptyRevFallback, false); + emitTreeAttrs(state, storePath, cachedInput.lockedInput, v, params.emptyRevFallback, false); } static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, Value & v) diff --git a/src/libexpr/print-ambiguous.cc b/src/libexpr/print-ambiguous.cc index 0646783c2..e5bfe3ccd 100644 --- a/src/libexpr/print-ambiguous.cc +++ b/src/libexpr/print-ambiguous.cc @@ -7,10 +7,10 @@ namespace nix { // See: https://github.com/NixOS/nix/issues/9730 void printAmbiguous( - Value &v, - const SymbolTable &symbols, - std::ostream &str, - std::set *seen, + EvalState & state, + Value & v, + std::ostream & str, + std::set * seen, int depth) { checkInterrupt(); @@ -26,9 +26,13 @@ void printAmbiguous( case nBool: printLiteralBool(str, v.boolean()); break; - case nString: - printLiteralString(str, v.string_view()); + case nString: { + NixStringContext context; + copyContext(v, context); + // FIXME: make devirtualization configurable? + printLiteralString(str, state.devirtualize(v.string_view(), context)); break; + } case nPath: str << v.path().to_string(); // !!! escaping? break; @@ -40,9 +44,9 @@ void printAmbiguous( str << "«repeated»"; else { str << "{ "; - for (auto & i : v.attrs()->lexicographicOrder(symbols)) { - str << symbols[i->name] << " = "; - printAmbiguous(*i->value, symbols, str, seen, depth - 1); + for (auto & i : v.attrs()->lexicographicOrder(state.symbols)) { + str << state.symbols[i->name] << " = "; + printAmbiguous(state, *i->value, str, seen, depth - 1); str << "; "; } str << "}"; @@ -56,7 +60,7 @@ void printAmbiguous( str << "[ "; for (auto v2 : v.listItems()) { if (v2) - printAmbiguous(*v2, symbols, str, seen, depth - 1); + printAmbiguous(state, *v2, str, seen, depth - 1); else str << "(nullptr)"; str << " "; diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 51652db1f..6230fa585 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -31,7 +31,9 @@ json printValueAsJSON(EvalState & state, bool strict, case nString: copyContext(v, context); - out = v.c_str(); + // FIXME: only use the context from `v`. + // FIXME: make devirtualization configurable? + out = state.devirtualize(v.c_str(), context); break; case nPath: diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 9693f1773..33301933c 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -228,6 +228,9 @@ void Input::checkLocks(Input specified, Input & result) if (auto prevNarHash = specified.getNarHash()) specified.attrs.insert_or_assign("narHash", prevNarHash->to_string(HashFormat::SRI, true)); + if (auto narHash = result.getNarHash()) + result.attrs.insert_or_assign("narHash", narHash->to_string(HashFormat::SRI, true)); + for (auto & field : specified.attrs) { auto field2 = result.attrs.find(field.first); if (field2 != result.attrs.end() && field.second != field2->second) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index f24c07fc2..bb82f751f 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -332,6 +332,13 @@ struct GitArchiveInputScheme : InputScheme false, "«" + input.to_string() + "»"); + if (!input.settings->trustTarballsFromGitForges) + // FIXME: computing the NAR hash here is wasteful if + // copyInputToStore() is just going to hash/copy it as + // well. + input.attrs.insert_or_assign("narHash", + accessor->hashPath(CanonPath::root).to_string(HashFormat::SRI, true)); + return {accessor, input}; } diff --git a/src/libfetchers/include/nix/fetchers/input-cache.hh b/src/libfetchers/include/nix/fetchers/input-cache.hh new file mode 100644 index 000000000..9b1c5a310 --- /dev/null +++ b/src/libfetchers/include/nix/fetchers/input-cache.hh @@ -0,0 +1,35 @@ +#include "fetchers.hh" + +namespace nix::fetchers { + +enum class UseRegistries : int; + +struct InputCache +{ + struct CachedResult + { + ref accessor; + Input resolvedInput; + Input lockedInput; + }; + + CachedResult getAccessor(ref store, const Input & originalInput, UseRegistries useRegistries); + + struct CachedInput + { + Input lockedInput; + ref accessor; + }; + + virtual std::optional lookup(const Input & originalInput) const = 0; + + virtual void upsert(Input key, CachedInput cachedInput) = 0; + + virtual void clear() = 0; + + static ref create(); + + virtual ~InputCache() = default; +}; + +} diff --git a/src/libfetchers/include/nix/fetchers/meson.build b/src/libfetchers/include/nix/fetchers/meson.build index 3a752d9cb..e6ddedd97 100644 --- a/src/libfetchers/include/nix/fetchers/meson.build +++ b/src/libfetchers/include/nix/fetchers/meson.build @@ -9,6 +9,7 @@ headers = files( 'filtering-source-accessor.hh', 'git-lfs-fetch.hh', 'git-utils.hh', + 'input-cache.hh', 'registry.hh', 'store-path-accessor.hh', 'tarball.hh', diff --git a/src/libfetchers/input-cache.cc b/src/libfetchers/input-cache.cc new file mode 100644 index 000000000..1f43e951a --- /dev/null +++ b/src/libfetchers/input-cache.cc @@ -0,0 +1,76 @@ +#include "nix/fetchers/input-cache.hh" +#include "nix/fetchers/registry.hh" +#include "nix/util/sync.hh" +#include "nix/util/source-path.hh" + +namespace nix::fetchers { + +InputCache::CachedResult +InputCache::getAccessor(ref store, const Input & originalInput, UseRegistries useRegistries) +{ + auto fetched = lookup(originalInput); + Input resolvedInput = originalInput; + + if (!fetched) { + if (originalInput.isDirect()) { + auto [accessor, lockedInput] = originalInput.getAccessor(store); + fetched.emplace(CachedInput{.lockedInput = lockedInput, .accessor = accessor}); + } else { + if (useRegistries != fetchers::UseRegistries::No) { + auto [res, extraAttrs] = lookupInRegistries(store, originalInput, useRegistries); + resolvedInput = std::move(res); + fetched = lookup(resolvedInput); + if (!fetched) { + auto [accessor, lockedInput] = resolvedInput.getAccessor(store); + fetched.emplace(CachedInput{.lockedInput = lockedInput, .accessor = accessor}); + } + upsert(resolvedInput, *fetched); + } else { + throw Error( + "'%s' is an indirect flake reference, but registry lookups are not allowed", + originalInput.to_string()); + } + } + upsert(originalInput, *fetched); + } + + debug("got tree '%s' from '%s'", fetched->accessor, fetched->lockedInput.to_string()); + + return {fetched->accessor, resolvedInput, fetched->lockedInput}; +} + +struct InputCacheImpl : InputCache +{ + Sync> cache_; + + std::optional lookup(const Input & originalInput) const override + { + auto cache(cache_.readLock()); + auto i = cache->find(originalInput); + if (i == cache->end()) + return std::nullopt; + debug( + "mapping '%s' to previously seen input '%s' -> '%s", + originalInput.to_string(), + i->first.to_string(), + i->second.lockedInput.to_string()); + return i->second; + } + + void upsert(Input key, CachedInput cachedInput) override + { + cache_.lock()->insert_or_assign(std::move(key), std::move(cachedInput)); + } + + void clear() override + { + cache_.lock()->clear(); + } +}; + +ref InputCache::create() +{ + return make_ref(); +} + +} diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build index 6e7129f4c..321146ca4 100644 --- a/src/libfetchers/meson.build +++ b/src/libfetchers/meson.build @@ -44,6 +44,7 @@ sources = files( 'git.cc', 'github.cc', 'indirect.cc', + 'input-cache.cc', 'mercurial.cc', 'path.cc', 'registry.cc', diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 31ef25c6e..f503541ce 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -15,89 +15,17 @@ #include "nix/fetchers/fetch-to-store.hh" #include "nix/util/memory-source-accessor.hh" #include "nix/util/mounted-source-accessor.hh" +#include "nix/fetchers/input-cache.hh" #include namespace nix { using namespace flake; +using namespace fetchers; namespace flake { -struct FetchedFlake -{ - FlakeRef lockedRef; - ref accessor; -}; - -typedef std::map FlakeCache; - -static std::optional lookupInFlakeCache( - const FlakeCache & flakeCache, - const FlakeRef & flakeRef) -{ - auto i = flakeCache.find(flakeRef); - if (i == flakeCache.end()) return std::nullopt; - debug("mapping '%s' to previously seen input '%s' -> '%s", - flakeRef, i->first, i->second.lockedRef); - return i->second; -} - -static std::tuple, FlakeRef, FlakeRef> fetchOrSubstituteTree( - EvalState & state, - const FlakeRef & originalRef, - fetchers::UseRegistries useRegistries, - FlakeCache & flakeCache) -{ - auto fetched = lookupInFlakeCache(flakeCache, originalRef); - FlakeRef resolvedRef = originalRef; - - if (!fetched) { - if (originalRef.input.isDirect()) { - auto [accessor, lockedRef] = originalRef.lazyFetch(state.store); - fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .accessor = accessor}); - } else { - if (useRegistries != fetchers::UseRegistries::No) { - resolvedRef = originalRef.resolve(state.store, useRegistries); - fetched = lookupInFlakeCache(flakeCache, originalRef); - if (!fetched) { - auto [accessor, lockedRef] = resolvedRef.lazyFetch(state.store); - fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .accessor = accessor}); - } - flakeCache.insert_or_assign(resolvedRef, *fetched); - } - else { - throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", originalRef); - } - } - flakeCache.insert_or_assign(originalRef, *fetched); - } - - debug("got tree '%s' from '%s'", fetched->accessor, fetched->lockedRef); - - return {fetched->accessor, resolvedRef, fetched->lockedRef}; -} - -static StorePath copyInputToStore( - EvalState & state, - fetchers::Input & input, - const fetchers::Input & originalInput, - ref accessor) -{ - auto storePath = fetchToStore(*state.store, accessor, FetchMode::Copy, input.getName()); - - state.allowPath(storePath); // FIXME: should just whitelist the entire virtual store - - state.storeFS->mount(CanonPath(state.store->printStorePath(storePath)), accessor); - - auto narHash = state.store->queryPathInfo(storePath)->narHash; - input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); - - assert(!originalInput.getNarHash() || storePath == originalInput.computeStorePath(*state.store)); - - return storePath; -} - static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos) { if (value.isThunk() && value.isTrivial()) @@ -123,7 +51,7 @@ static std::pair, fetchers::Attrs> parseFlakeInput static void parseFlakeInputAttr( EvalState & state, - const Attr & attr, + const nix::Attr & attr, fetchers::Attrs & attrs) { // Allow selecting a subset of enum values @@ -394,15 +322,17 @@ static Flake getFlake( EvalState & state, const FlakeRef & originalRef, fetchers::UseRegistries useRegistries, - FlakeCache & flakeCache, - const InputAttrPath & lockRootAttrPath) + const InputAttrPath & lockRootAttrPath, + bool requireLockable) { // Fetch a lazy tree first. - auto [accessor, resolvedRef, lockedRef] = fetchOrSubstituteTree( - state, originalRef, useRegistries, flakeCache); + auto cachedInput = state.inputCache->getAccessor(state.store, originalRef.input, useRegistries); + + auto resolvedRef = FlakeRef(std::move(cachedInput.resolvedInput), originalRef.subdir); + auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), originalRef.subdir); // Parse/eval flake.nix to get at the input.self attributes. - auto flake = readFlake(state, originalRef, resolvedRef, lockedRef, {accessor}, lockRootAttrPath); + auto flake = readFlake(state, originalRef, resolvedRef, lockedRef, {cachedInput.accessor}, lockRootAttrPath); // Re-fetch the tree if necessary. auto newLockedRef = applySelfAttrs(lockedRef, flake); @@ -411,23 +341,21 @@ static Flake getFlake( debug("refetching input '%s' due to self attribute", newLockedRef); // FIXME: need to remove attrs that are invalidated by the changed input attrs, such as 'narHash'. newLockedRef.input.attrs.erase("narHash"); - auto [accessor2, resolvedRef2, lockedRef2] = fetchOrSubstituteTree( - state, newLockedRef, fetchers::UseRegistries::No, flakeCache); - accessor = accessor2; - lockedRef = lockedRef2; + auto cachedInput2 = state.inputCache->getAccessor(state.store, newLockedRef.input, fetchers::UseRegistries::No); + cachedInput.accessor = cachedInput2.accessor; + lockedRef = FlakeRef(std::move(cachedInput2.lockedInput), newLockedRef.subdir); } - // Copy the tree to the store. - auto storePath = copyInputToStore(state, lockedRef.input, originalRef.input, accessor); - // Re-parse flake.nix from the store. - return readFlake(state, originalRef, resolvedRef, lockedRef, state.storePath(storePath), lockRootAttrPath); + return readFlake( + state, originalRef, resolvedRef, lockedRef, + state.storePath(state.mountInput(lockedRef.input, originalRef.input, cachedInput.accessor, requireLockable)), + lockRootAttrPath); } -Flake getFlake(EvalState & state, const FlakeRef & originalRef, fetchers::UseRegistries useRegistries) +Flake getFlake(EvalState & state, const FlakeRef & originalRef, fetchers::UseRegistries useRegistries, bool requireLockable) { - FlakeCache flakeCache; - return getFlake(state, originalRef, useRegistries, flakeCache, {}); + return getFlake(state, originalRef, useRegistries, {}, requireLockable); } static LockFile readLockFile( @@ -447,8 +375,6 @@ LockedFlake lockFlake( const FlakeRef & topRef, const LockFlags & lockFlags) { - FlakeCache flakeCache; - auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries); auto useRegistriesTop = useRegistries ? fetchers::UseRegistries::All : fetchers::UseRegistries::No; auto useRegistriesInputs = useRegistries ? fetchers::UseRegistries::Limited : fetchers::UseRegistries::No; @@ -457,8 +383,8 @@ LockedFlake lockFlake( state, topRef, useRegistriesTop, - flakeCache, - {}); + {}, + lockFlags.requireLockable); if (lockFlags.applyNixConfig) { flake.config.apply(settings); @@ -637,8 +563,8 @@ LockedFlake lockFlake( state, ref, useRegistriesInputs, - flakeCache, - inputAttrPath); + inputAttrPath, + true); } }; @@ -811,15 +737,17 @@ LockedFlake lockFlake( if (auto resolvedPath = resolveRelativePath()) { return {*resolvedPath, *input.ref}; } else { - auto [accessor, resolvedRef, lockedRef] = fetchOrSubstituteTree( - state, *input.ref, useRegistriesInputs, flakeCache); + auto cachedInput = state.inputCache->getAccessor(state.store, input.ref->input, useRegistriesTop); + + auto resolvedRef = FlakeRef(std::move(cachedInput.resolvedInput), input.ref->subdir); + auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), input.ref->subdir); warnRegistry(resolvedRef); - // FIXME: allow input to be lazy. - auto storePath = copyInputToStore(state, lockedRef.input, input.ref->input, accessor); - - return {state.storePath(storePath), lockedRef}; + return { + state.storePath(state.mountInput(lockedRef.input, input.ref->input, cachedInput.accessor, true)), + lockedRef + }; } }(); @@ -932,7 +860,8 @@ LockedFlake lockFlake( flake = getFlake( state, topRef, - useRegistriesTop); + useRegistriesTop, + lockFlags.requireLockable); if (lockFlags.commitLockFile && flake.lockedRef.input.getRev() && diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index ef0bb349b..fdac4397f 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -115,7 +115,11 @@ struct Flake } }; -Flake getFlake(EvalState & state, const FlakeRef & flakeRef, fetchers::UseRegistries useRegistries); +Flake getFlake( + EvalState & state, + const FlakeRef & flakeRef, + fetchers::UseRegistries useRegistries, + bool requireLockable = true); /** * Fingerprint of a locked flake; used as a cache key. @@ -213,6 +217,11 @@ struct LockFlags * for those inputs will be ignored. */ std::set inputUpdates; + + /** + * Whether to require a locked input. + */ + bool requireLockable = true; }; LockedFlake lockFlake( diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index d3bccd7af..ae1a53e79 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -214,8 +214,12 @@ StorePath Store::addToStore( auto sink = sourceToSink([&](Source & source) { LengthSource lengthSource(source); storePath = addToStoreFromDump(lengthSource, name, fsm, method, hashAlgo, references, repair); - if (settings.warnLargePathThreshold && lengthSource.total >= settings.warnLargePathThreshold) - warn("copied large path '%s' to the store (%s)", path, renderSize(lengthSource.total)); + if (settings.warnLargePathThreshold && lengthSource.total >= settings.warnLargePathThreshold) { + static bool failOnLargePath = getEnv("_NIX_TEST_FAIL_ON_LARGE_PATH").value_or("") == "1"; + if (failOnLargePath) + throw Error("won't copy large path '%s' to the store (%d)", path, renderSize(lengthSource.total)); + warn("copied large path '%s' to the store (%d)", path, renderSize(lengthSource.total)); + } }); dumpPath(path, *sink, fsm, filter); sink->finish(); diff --git a/src/libutil/include/nix/util/mounted-source-accessor.hh b/src/libutil/include/nix/util/mounted-source-accessor.hh index 4e75edfaf..2e8d45dd6 100644 --- a/src/libutil/include/nix/util/mounted-source-accessor.hh +++ b/src/libutil/include/nix/util/mounted-source-accessor.hh @@ -7,6 +7,12 @@ namespace nix { struct MountedSourceAccessor : SourceAccessor { virtual void mount(CanonPath mountPoint, ref accessor) = 0; + + /** + * Return the accessor mounted on `mountPoint`, or `nullptr` if + * there is no such mount point. + */ + virtual std::shared_ptr getMount(CanonPath mountPoint) = 0; }; ref makeMountedSourceAccessor(std::map> mounts); diff --git a/src/libutil/mounted-source-accessor.cc b/src/libutil/mounted-source-accessor.cc index 89063b10f..28e799e4c 100644 --- a/src/libutil/mounted-source-accessor.cc +++ b/src/libutil/mounted-source-accessor.cc @@ -81,6 +81,15 @@ struct MountedSourceAccessorImpl : MountedSourceAccessor // FIXME: thread-safety mounts.insert_or_assign(std::move(mountPoint), accessor); } + + std::shared_ptr getMount(CanonPath mountPoint) override + { + auto i = mounts.find(mountPoint); + if (i != mounts.end()) + return i->second; + else + return nullptr; + } }; ref makeMountedSourceAccessor(std::map> mounts) diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index e149b6aeb..c49f2885d 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -110,7 +110,7 @@ bool createUserEnv(EvalState & state, PackageInfos & elems, environment. */ auto manifestFile = ({ std::ostringstream str; - printAmbiguous(manifest, state.symbols, str, nullptr, std::numeric_limits::max()); + printAmbiguous(state, manifest, str, nullptr, std::numeric_limits::max()); StringSource source { toView(str) }; state.store->addToStoreFromDump( source, "env-manifest.nix", FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, references); diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index c1b6cc66a..4ae82b2bf 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -52,7 +52,10 @@ void processExpr(EvalState & state, const Strings & attrPaths, else state.autoCallFunction(autoArgs, v, vRes); if (output == okRaw) - std::cout << *state.coerceToString(noPos, vRes, context, "while generating the nix-instantiate output"); + std::cout << + state.devirtualize( + *state.coerceToString(noPos, vRes, context, "while generating the nix-instantiate output"), + context); // We intentionally don't output a newline here. The default PS1 for Bash in NixOS starts with a newline // and other interactive shells like Zsh are smart enough to print a missing newline before the prompt. else if (output == okXML) @@ -63,7 +66,7 @@ void processExpr(EvalState & state, const Strings & attrPaths, } else { if (strict) state.forceValueDeep(vRes); std::set seen; - printAmbiguous(vRes, state.symbols, std::cout, &seen, std::numeric_limits::max()); + printAmbiguous(state, vRes, std::cout, &seen, std::numeric_limits::max()); std::cout << std::endl; } } else { diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 24a87f140..d03d09916 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -114,7 +114,11 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption else if (raw) { logger->stop(); - writeFull(getStandardOutput(), *state->coerceToString(noPos, *v, context, "while generating the eval command output")); + writeFull( + getStandardOutput(), + state->devirtualize( + *state->coerceToString(noPos, *v, context, "while generating the eval command output"), + context)); } else if (json) { diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 6df025e35..4782cbb29 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -134,6 +134,7 @@ public: lockFlags.recreateLockFile = updateAll; lockFlags.writeLockFile = true; lockFlags.applyNixConfig = true; + lockFlags.requireLockable = false; lockFlake(); } @@ -166,6 +167,7 @@ struct CmdFlakeLock : FlakeCommand lockFlags.writeLockFile = true; lockFlags.failOnUnlocked = true; lockFlags.applyNixConfig = true; + lockFlags.requireLockable = false; lockFlake(); } @@ -212,11 +214,17 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON void run(nix::ref store) override { + lockFlags.requireLockable = false; auto lockedFlake = lockFlake(); auto & flake = lockedFlake.flake; - // Currently, all flakes are in the Nix store via the rootFS accessor. - auto storePath = store->printStorePath(store->toStorePath(flake.path.path.abs()).first); + /* Hack to show the store path if available. */ + std::optional storePath; + if (store->isInStore(flake.path.path.abs())) { + auto path = store->toStorePath(flake.path.path.abs()).first; + if (store->isValidPath(path)) + storePath = path; + } if (json) { nlohmann::json j; @@ -238,7 +246,8 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["revCount"] = *revCount; if (auto lastModified = flake.lockedRef.input.getLastModified()) j["lastModified"] = *lastModified; - j["path"] = storePath; + if (storePath) + j["path"] = store->printStorePath(*storePath); j["locks"] = lockedFlake.lockFile.toJSON().first; if (auto fingerprint = lockedFlake.getFingerprint(store, fetchSettings)) j["fingerprint"] = fingerprint->to_string(HashFormat::Base16, false); @@ -255,9 +264,10 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON logger->cout( ANSI_BOLD "Description:" ANSI_NORMAL " %s", *flake.description); - logger->cout( - ANSI_BOLD "Path:" ANSI_NORMAL " %s", - storePath); + if (storePath) + logger->cout( + ANSI_BOLD "Path:" ANSI_NORMAL " %s", + store->printStorePath(*storePath)); if (auto rev = flake.lockedRef.input.getRev()) logger->cout( ANSI_BOLD "Revision:" ANSI_NORMAL " %s", @@ -1079,7 +1089,10 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun StorePathSet sources; - auto storePath = store->toStorePath(flake.flake.path.path.abs()).first; + auto storePath = + dryRun + ? flake.flake.lockedRef.input.computeStorePath(*store) + : std::get(flake.flake.lockedRef.input.fetchToStore(store)); sources.insert(storePath); @@ -1095,7 +1108,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun storePath = dryRun ? (*inputNode)->lockedRef.input.computeStorePath(*store) - : std::get<0>((*inputNode)->lockedRef.input.fetchToStore(store)); + : std::get((*inputNode)->lockedRef.input.fetchToStore(store)); sources.insert(*storePath); } if (json) { diff --git a/tests/functional/fetchGit.sh b/tests/functional/fetchGit.sh index 5e5e8e61f..6fc8ca8b0 100755 --- a/tests/functional/fetchGit.sh +++ b/tests/functional/fetchGit.sh @@ -292,7 +292,7 @@ path11=$(nix eval --impure --raw --expr "(builtins.fetchGit ./.).outPath") empty="$TEST_ROOT/empty" git init "$empty" -emptyAttrs='{ lastModified = 0; lastModifiedDate = "19700101000000"; narHash = "sha256-pQpattmS9VmO3ZIQUFn66az8GSmB4IvYhTTCFn6SUmo="; rev = "0000000000000000000000000000000000000000"; revCount = 0; shortRev = "0000000"; submodules = false; }' +emptyAttrs='{ lastModified = 0; lastModifiedDate = "19700101000000"; rev = "0000000000000000000000000000000000000000"; revCount = 0; shortRev = "0000000"; submodules = false; }' [[ $(nix eval --impure --expr "builtins.removeAttrs (builtins.fetchGit $empty) [\"outPath\"]") = $emptyAttrs ]] diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 72fe79838..7ec438d74 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -77,6 +77,7 @@ hash1=$(echo "$json" | jq -r .revision) echo foo > "$flake1Dir/foo" git -C "$flake1Dir" add $flake1Dir/foo [[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "$hash1-dirty" ]] +[[ $(_NIX_TEST_FAIL_ON_LARGE_PATH=1 nix flake metadata flake1 --json --refresh --warn-large-path-threshold 1 --lazy-trees | jq -r .dirtyRevision) == "$hash1-dirty" ]] [[ "$(nix flake metadata flake1 --json | jq -r .fingerprint)" != null ]] echo -n '# foo' >> "$flake1Dir/flake.nix" diff --git a/tests/functional/flakes/unlocked-override.sh b/tests/functional/flakes/unlocked-override.sh index 512aca401..bd73929dc 100755 --- a/tests/functional/flakes/unlocked-override.sh +++ b/tests/functional/flakes/unlocked-override.sh @@ -36,6 +36,7 @@ expectStderr 1 nix flake lock "$flake2Dir" --override-input flake1 "$TEST_ROOT/f grepQuiet "Will not write lock file.*because it has an unlocked input" nix flake lock "$flake2Dir" --override-input flake1 "$TEST_ROOT/flake1" --allow-dirty-locks +_NIX_TEST_FAIL_ON_LARGE_PATH=1 nix flake lock "$flake2Dir" --override-input flake1 "$TEST_ROOT/flake1" --allow-dirty-locks --warn-large-path-threshold 1 --lazy-trees # Using a lock file with a dirty lock does not require --allow-dirty-locks, but should print a warning. expectStderr 0 nix eval "$flake2Dir#x" | diff --git a/tests/functional/lang/eval-fail-hashfile-missing.err.exp b/tests/functional/lang/eval-fail-hashfile-missing.err.exp index 0d3747a6d..901dea2b5 100644 --- a/tests/functional/lang/eval-fail-hashfile-missing.err.exp +++ b/tests/functional/lang/eval-fail-hashfile-missing.err.exp @@ -10,4 +10,4 @@ error: … while calling the 'hashFile' builtin - error: opening file '/pwd/lang/this-file-is-definitely-not-there-7392097': No such file or directory + error: path '/pwd/lang/this-file-is-definitely-not-there-7392097' does not exist diff --git a/tests/nixos/github-flakes.nix b/tests/nixos/github-flakes.nix index e36725827..eb7266b8c 100644 --- a/tests/nixos/github-flakes.nix +++ b/tests/nixos/github-flakes.nix @@ -204,7 +204,7 @@ in cat_log() # ... otherwise it should use the API - out = client.succeed("nix flake metadata private-flake --json --access-tokens github.com=ghp_000000000000000000000000000000000000 --tarball-ttl 0") + out = client.succeed("nix flake metadata private-flake --json --access-tokens github.com=ghp_000000000000000000000000000000000000 --tarball-ttl 0 --no-trust-tarballs-from-git-forges") print(out) info = json.loads(out) assert info["revision"] == "${private-flake-rev}", f"revision mismatch: {info['revision']} != ${private-flake-rev}" @@ -223,6 +223,10 @@ in hash = client.succeed(f"nix eval --no-trust-tarballs-from-git-forges --raw --expr '(fetchTree {info['url']}).narHash'") assert hash == info['locked']['narHash'] + # Fetching with an incorrect NAR hash should fail. + out = client.fail(f"nix eval --no-trust-tarballs-from-git-forges --raw --expr '(fetchTree \"github:fancy-enterprise/private-flake/{info['revision']}?narHash=sha256-HsrRFZYg69qaVe/wDyWBYLeS6ca7ACEJg2Z%2BGpEFw4A%3D\").narHash' 2>&1") + assert "NAR hash mismatch in input" in out, "NAR hash check did not fail with the expected error" + # Fetching without a narHash should succeed if trust-github is set and fail otherwise. client.succeed(f"nix eval --raw --expr 'builtins.fetchTree github:github:fancy-enterprise/private-flake/{info['revision']}'") out = client.fail(f"nix eval --no-trust-tarballs-from-git-forges --raw --expr 'builtins.fetchTree github:github:fancy-enterprise/private-flake/{info['revision']}' 2>&1")