wroclaw-git-backups/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-06-25 06:31:14 +02:00
Code Activity

Merge pull request #27 from DeterminateSystems/lazy-trees-v2

Browse source 
Lazy trees v2
This commit is contained in:
Eelco Dolstra 2025-05-07 15:03:25 +00:00 • committed by GitHub
commit df93fa8604
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
31 changed files with 438 additions and 158 deletions
Download patch file Download diff file Expand all files Collapse all files

27
.github/workflows/ci.yml vendored
View file

@ -91,7 +91,7 @@ jobs:
flake_regressions:
if: github.event_name == 'merge_group'
needs: build_x86_64-linux
runs-on: blacksmith-32vcpu-ubuntu-2204
runs-on: namespace-profile-x86-32cpu-64gb
steps:
- name: Checkout nix
uses: actions/checkout@v4
@ -109,7 +109,30 @@ jobs:
with:
determinate: true
- uses: DeterminateSystems/flakehub-cache-action@main
- run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH MAX_FLAKES=50 flake-regressions/eval-all.sh
- run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH PARALLEL="-P 50%" flake-regressions/eval-all.sh
flake_regressions_lazy:
if: github.event_name == 'merge_group'
needs: build_x86_64-linux
runs-on: namespace-profile-x86-32cpu-64gb
steps:
- name: Checkout nix
uses: actions/checkout@v4
- name: Checkout flake-regressions
uses: actions/checkout@v4
with:
repository: DeterminateSystems/flake-regressions
path: flake-regressions
- name: Checkout flake-regressions-data
uses: actions/checkout@v4
with:
repository: DeterminateSystems/flake-regressions-data
path: flake-regressions/tests
- uses: DeterminateSystems/nix-installer-action@main
with:
determinate: true
- uses: DeterminateSystems/flakehub-cache-action@main
- run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH PARALLEL="-P 50%" NIX_CONFIG="lazy-trees = true" flake-regressions/eval-all.sh
manual:
if: github.event_name != 'merge_group'

3
src/libcmd/installable-value.cc
View file

@ -57,7 +57,8 @@ std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths
else if (v.type() == nString) {
return {{
.path = DerivedPath::fromSingle(
state->coerceToSingleDerivedPath(pos, v, errorCtx)),
state->devirtualize(
state->coerceToSingleDerivedPath(pos, v, errorCtx))),
.info = make_ref<ExtraPathInfo>(),
}};
}

30
src/libexpr/eval.cc
View file

@ -20,6 +20,7 @@
#include "nix/util/url.hh"
#include "nix/fetchers/fetch-to-store.hh"
#include "nix/fetchers/tarball.hh"
#include "nix/fetchers/input-cache.hh"
#include "parser-tab.hh"
@ -266,12 +267,9 @@ EvalState::EvalState(
/nix/store while using a chroot store. */
auto accessor = getFSSourceAccessor();
auto realStoreDir = dirOf(store->toRealPath(StorePath::dummy));
if (settings.pureEval || store->storeDir != realStoreDir) {
accessor = settings.pureEval
? storeFS.cast<SourceAccessor>()
: makeUnionSourceAccessor({accessor, storeFS});
}
accessor = settings.pureEval
? storeFS.cast<SourceAccessor>()
: makeUnionSourceAccessor({accessor, storeFS});
/* Apply access control if needed. */
if (settings.restrictEval || settings.pureEval)
@ -293,6 +291,7 @@ EvalState::EvalState(
)}
, store(store)
, buildStore(buildStore ? buildStore : store)
, inputCache(fetchers::InputCache::create())
, debugRepl(nullptr)
, debugStop(false)
, trylevel(0)
@ -949,7 +948,16 @@ void EvalState::mkPos(Value & v, PosIdx p)
auto origin = positions.originOf(p);
if (auto path = std::get_if<SourcePath>(&origin)) {
auto attrs = buildBindings(3);
attrs.alloc(sFile).mkString(path->path.abs());
if (path->accessor == rootFS && store->isInStore(path->path.abs()))
// FIXME: only do this for virtual store paths?
attrs.alloc(sFile).mkString(path->path.abs(),
{
NixStringContextElem::Opaque{
.path = store->toStorePath(path->path.abs()).first
}
});
else
attrs.alloc(sFile).mkString(path->path.abs());
makePositionThunks(*this, p, attrs.alloc(sLine), attrs.alloc(sColumn));
v.mkAttrs(attrs);
} else
@ -1135,6 +1143,7 @@ void EvalState::resetFileCache()
{
fileEvalCache.clear();
fileParseCache.clear();
inputCache->clear();
}
@ -2317,6 +2326,9 @@ BackedStringView EvalState::coerceToString(
}
if (v.type() == nPath) {
// FIXME: instead of copying the path to the store, we could
// return a virtual store path that lazily copies the path to
// the store in devirtualize().
return
!canonicalizePath && !copyToStore
? // FIXME: hack to preserve path literals that end in a
@ -2406,7 +2418,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
*store,
path.resolveSymlinks(SymlinkResolution::Ancestors),
settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,
path.baseName(),
computeBaseName(path),
ContentAddressMethod::Raw::NixArchive,
nullptr,
repair);
@ -2461,7 +2473,7 @@ StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringCon
auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned();
if (auto storePath = store->maybeParseStorePath(path))
return *storePath;
error<EvalError>("path '%1%' is not in the Nix store", path).withTrace(pos, errorCtx).debugThrow();
error<EvalError>("cannot coerce '%s' to a store path because it is not a subpath of the Nix store", path).withTrace(pos, errorCtx).debugThrow();
}

5
src/libexpr/include/nix/expr/eval-settings.hh
View file

@ -247,6 +247,11 @@ struct EvalSettings : Config
This option can be enabled by setting `NIX_ABORT_ON_WARN=1` in the environment.
)"};
Setting<bool> lazyTrees{this, false, "lazy-trees",
R"(
If set to true, flakes and trees fetched by [`builtins.fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) are only copied to the Nix store when they're used as a dependency of a derivation. This avoids copying (potentially large) source trees unnecessarily.
)"};
};
/**

42
src/libexpr/include/nix/expr/eval.hh
View file

@ -33,7 +33,11 @@ namespace nix {
constexpr size_t maxPrimOpArity = 8;
class Store;
namespace fetchers { struct Settings; }
namespace fetchers {
struct Settings;
struct InputCache;
struct Input;
}
struct EvalSettings;
class EvalState;
class StorePath;
@ -301,6 +305,8 @@ public:
RootValue vImportedDrvToDerivation = nullptr;
ref<fetchers::InputCache> inputCache;
/**
* Debugger
*/
@ -445,6 +451,15 @@ public:
void checkURI(const std::string & uri);
/**
* Mount an input on the Nix store.
*/
StorePath mountInput(
fetchers::Input & input,
const fetchers::Input & originalInput,
ref<SourceAccessor> accessor,
bool requireLockable);
/**
* Parse a Nix expression from the specified file.
*/
@ -554,6 +569,18 @@ public:
std::optional<std::string> tryAttrsToString(const PosIdx pos, Value & v,
NixStringContext & context, bool coerceMore = false, bool copyToStore = true);
StorePath devirtualize(
const StorePath & path,
StringMap * rewrites = nullptr);
SingleDerivedPath devirtualize(
const SingleDerivedPath & path,
StringMap * rewrites = nullptr);
std::string devirtualize(
std::string_view s,
const NixStringContext & context);
/**
* String coercion.
*
@ -569,6 +596,19 @@ public:
StorePath copyPathToStore(NixStringContext & context, const SourcePath & path);
/**
* Compute the base name for a `SourcePath`. For non-store paths,
* this is just `SourcePath::baseName()`. But for store paths, for
* backwards compatibility, it needs to be `<hash>-source`,
* i.e. as if the path were copied to the Nix store. This results
* in a "double-copied" store path like
* `/nix/store/<hash1>-<hash2>-source`. We don't need to
* materialize /nix/store/<hash2>-source though. Still, this
* requires reading/hashing the path twice.
*/
std::string computeBaseName(const SourcePath & path);
/**
* Path coercion.
*

8
src/libexpr/include/nix/expr/print-ambiguous.hh
View file

@ -15,10 +15,10 @@ namespace nix {
* See: https://github.com/NixOS/nix/issues/9730
*/
void printAmbiguous(
Value &v,
const SymbolTable &symbols,
std::ostream &str,
std::set<const void *> *seen,
EvalState & state,
Value & v,
std::ostream & str,
std::set<const void *> * seen,
int depth);
}

79
src/libexpr/paths.cc
View file

@ -1,5 +1,7 @@
#include "nix/store/store-api.hh"
#include "nix/expr/eval.hh"
#include "nix/util/mounted-source-accessor.hh"
#include "nix/fetchers/fetch-to-store.hh"
namespace nix {
@ -18,4 +20,81 @@ SourcePath EvalState::storePath(const StorePath & path)
return {rootFS, CanonPath{store->printStorePath(path)}};
}
StorePath EvalState::devirtualize(const StorePath & path, StringMap * rewrites)
{
if (auto mount = storeFS->getMount(CanonPath(store->printStorePath(path)))) {
auto storePath = fetchToStore(
*store, SourcePath{ref(mount)}, settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, path.name());
assert(storePath.name() == path.name());
if (rewrites)
rewrites->emplace(path.hashPart(), storePath.hashPart());
return storePath;
} else
return path;
}
SingleDerivedPath EvalState::devirtualize(const SingleDerivedPath & path, StringMap * rewrites)
{
if (auto o = std::get_if<SingleDerivedPath::Opaque>(&path.raw()))
return SingleDerivedPath::Opaque{devirtualize(o->path, rewrites)};
else
return path;
}
std::string EvalState::devirtualize(std::string_view s, const NixStringContext & context)
{
StringMap rewrites;
for (auto & c : context)
if (auto o = std::get_if<NixStringContextElem::Opaque>(&c.raw))
devirtualize(o->path, &rewrites);
return rewriteStrings(std::string(s), rewrites);
}
std::string EvalState::computeBaseName(const SourcePath & path)
{
if (path.accessor == rootFS) {
if (auto storePath = store->maybeParseStorePath(path.path.abs())) {
warn(
"Performing inefficient double copy of path '%s' to the store. "
"This can typically be avoided by rewriting an attribute like `src = ./.` "
"to `src = builtins.path { path = ./.; name = \"source\"; }`.",
path);
return std::string(fetchToStore(*store, path, FetchMode::DryRun, storePath->name()).to_string());
}
}
return std::string(path.baseName());
}
StorePath EvalState::mountInput(
fetchers::Input & input, const fetchers::Input & originalInput, ref<SourceAccessor> accessor, bool requireLockable)
{
auto storePath = settings.lazyTrees ? StorePath::random(input.getName())
: fetchToStore(*store, accessor, FetchMode::Copy, input.getName());
allowPath(storePath); // FIXME: should just whitelist the entire virtual store
storeFS->mount(CanonPath(store->printStorePath(storePath)), accessor);
if (requireLockable && !input.isLocked() && !input.getNarHash()) {
auto narHash = accessor->hashPath(CanonPath::root);
input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
}
// FIXME: what to do with the NAR hash in lazy mode?
if (!settings.lazyTrees && originalInput.getNarHash()) {
auto expected = originalInput.computeStorePath(*store);
if (storePath != expected)
throw Error(
(unsigned int) 102,
"NAR hash mismatch in input '%s', expected '%s' but got '%s'",
originalInput.to_string(),
store->printStorePath(storePath),
store->printStorePath(expected));
}
return storePath;
}
}

15
src/libexpr/primops.cc
View file

@ -14,6 +14,7 @@
#include "nix/expr/value-to-xml.hh"
#include "nix/expr/primops.hh"
#include "nix/fetchers/fetch-to-store.hh"
#include "nix/util/mounted-source-accessor.hh"
#include <boost/container/small_vector.hpp>
#include <nlohmann/json.hpp>
@ -75,7 +76,10 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS
ensureValid(b.drvPath->getBaseStorePath());
},
[&](const NixStringContextElem::Opaque & o) {
ensureValid(o.path);
// We consider virtual store paths valid here. They'll
// be devirtualized if needed elsewhere.
if (!storeFS->getMount(CanonPath(store->printStorePath(o.path))))
ensureValid(o.path);
if (maybePathsOut)
maybePathsOut->emplace(o.path);
},
@ -1408,6 +1412,8 @@ static void derivationStrictInternal(
/* Everything in the context of the strings in the derivation
attributes should be added as dependencies of the resulting
derivation. */
StringMap rewrites;
for (auto & c : context) {
std::visit(overloaded {
/* Since this allows the builder to gain access to every
@ -1430,11 +1436,13 @@ static void derivationStrictInternal(
drv.inputDrvs.ensureSlot(*b.drvPath).value.insert(b.output);
},
[&](const NixStringContextElem::Opaque & o) {
drv.inputSrcs.insert(o.path);
drv.inputSrcs.insert(state.devirtualize(o.path, &rewrites));
},
}, c.raw);
}
drv.applyRewrites(rewrites);
/* Do we have all required attributes? */
if (drv.builder == "")
state.error<EvalError>("required attribute 'builder' missing")
@ -2500,6 +2508,7 @@ static void addPath(
{}));
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
// FIXME: make this lazy?
auto dstPath = fetchToStore(
*state.store,
path.resolveSymlinks(),
@ -2530,7 +2539,7 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * arg
"while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'");
state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource");
addPath(state, pos, path.baseName(), path, args[0], ContentAddressMethod::Raw::NixArchive, std::nullopt, v, context);
addPath(state, pos, state.computeBaseName(path), path, args[0], ContentAddressMethod::Raw::NixArchive, std::nullopt, v, context);
}
static RegisterPrimOp primop_filterSource({

9
src/libexpr/primops/fetchTree.cc
View file

@ -10,6 +10,7 @@
#include "nix/util/url.hh"
#include "nix/expr/value-to-json.hh"
#include "nix/fetchers/fetch-to-store.hh"
#include "nix/fetchers/input-cache.hh"
#include "nix/util/mounted-source-accessor.hh"
#include <nlohmann/json.hpp>
@ -201,13 +202,11 @@ static void fetchTree(
throw Error("input '%s' is not allowed to use the '__final' attribute", input.to_string());
}
auto [storePath, accessor, input2] = input.fetchToStore(state.store);
auto cachedInput = state.inputCache->getAccessor(state.store, input, fetchers::UseRegistries::No);
state.allowPath(storePath);
auto storePath = state.mountInput(cachedInput.lockedInput, input, cachedInput.accessor, true);
state.storeFS->mount(CanonPath(state.store->printStorePath(storePath)), accessor);
emitTreeAttrs(state, storePath, input2, v, params.emptyRevFallback, false);
emitTreeAttrs(state, storePath, cachedInput.lockedInput, v, params.emptyRevFallback, false);
}
static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, Value & v)

24
src/libexpr/print-ambiguous.cc
View file

@ -7,10 +7,10 @@ namespace nix {
// See: https://github.com/NixOS/nix/issues/9730
void printAmbiguous(
Value &v,
const SymbolTable &symbols,
std::ostream &str,
std::set<const void *> *seen,
EvalState & state,
Value & v,
std::ostream & str,
std::set<const void *> * seen,
int depth)
{
checkInterrupt();
@ -26,9 +26,13 @@ void printAmbiguous(
case nBool:
printLiteralBool(str, v.boolean());
break;
case nString:
printLiteralString(str, v.string_view());
case nString: {
NixStringContext context;
copyContext(v, context);
// FIXME: make devirtualization configurable?
printLiteralString(str, state.devirtualize(v.string_view(), context));
break;
}
case nPath:
str << v.path().to_string(); // !!! escaping?
break;
@ -40,9 +44,9 @@ void printAmbiguous(
str << "«repeated»";
else {
str << "{ ";
for (auto & i : v.attrs()->lexicographicOrder(symbols)) {
str << symbols[i->name] << " = ";
printAmbiguous(*i->value, symbols, str, seen, depth - 1);
for (auto & i : v.attrs()->lexicographicOrder(state.symbols)) {
str << state.symbols[i->name] << " = ";
printAmbiguous(state, *i->value, str, seen, depth - 1);
str << "; ";
}
str << "}";
@ -56,7 +60,7 @@ void printAmbiguous(
str << "[ ";
for (auto v2 : v.listItems()) {
if (v2)
printAmbiguous(*v2, symbols, str, seen, depth - 1);
printAmbiguous(state, *v2, str, seen, depth - 1);
else
str << "(nullptr)";
str << " ";

4
src/libexpr/value-to-json.cc
View file

@ -31,7 +31,9 @@ json printValueAsJSON(EvalState & state, bool strict,
case nString:
copyContext(v, context);
out = v.c_str();
// FIXME: only use the context from `v`.
// FIXME: make devirtualization configurable?
out = state.devirtualize(v.c_str(), context);
break;
case nPath:

3
src/libfetchers/fetchers.cc
View file

@ -228,6 +228,9 @@ void Input::checkLocks(Input specified, Input & result)
if (auto prevNarHash = specified.getNarHash())
specified.attrs.insert_or_assign("narHash", prevNarHash->to_string(HashFormat::SRI, true));
if (auto narHash = result.getNarHash())
result.attrs.insert_or_assign("narHash", narHash->to_string(HashFormat::SRI, true));
for (auto & field : specified.attrs) {
auto field2 = result.attrs.find(field.first);
if (field2 != result.attrs.end() && field.second != field2->second)

7
src/libfetchers/github.cc
View file

@ -332,6 +332,13 @@ struct GitArchiveInputScheme : InputScheme
false,
"«" + input.to_string() + "»");
if (!input.settings->trustTarballsFromGitForges)
// FIXME: computing the NAR hash here is wasteful if
// copyInputToStore() is just going to hash/copy it as
// well.
input.attrs.insert_or_assign("narHash",
accessor->hashPath(CanonPath::root).to_string(HashFormat::SRI, true));
return {accessor, input};
}

35
src/libfetchers/include/nix/fetchers/input-cache.hh Normal file
View file

@ -0,0 +1,35 @@
#include "fetchers.hh"
namespace nix::fetchers {
enum class UseRegistries : int;
struct InputCache
{
struct CachedResult
{
ref<SourceAccessor> accessor;
Input resolvedInput;
Input lockedInput;
};
CachedResult getAccessor(ref<Store> store, const Input & originalInput, UseRegistries useRegistries);
struct CachedInput
{
Input lockedInput;
ref<SourceAccessor> accessor;
};
virtual std::optional<CachedInput> lookup(const Input & originalInput) const = 0;
virtual void upsert(Input key, CachedInput cachedInput) = 0;
virtual void clear() = 0;
static ref<InputCache> create();
virtual ~InputCache() = default;
};
}

1
src/libfetchers/include/nix/fetchers/meson.build
View file

@ -9,6 +9,7 @@ headers = files(
'filtering-source-accessor.hh',
'git-lfs-fetch.hh',
'git-utils.hh',
'input-cache.hh',
'registry.hh',
'store-path-accessor.hh',
'tarball.hh',

76
src/libfetchers/input-cache.cc Normal file
View file

@ -0,0 +1,76 @@
#include "nix/fetchers/input-cache.hh"
#include "nix/fetchers/registry.hh"
#include "nix/util/sync.hh"
#include "nix/util/source-path.hh"
namespace nix::fetchers {
InputCache::CachedResult
InputCache::getAccessor(ref<Store> store, const Input & originalInput, UseRegistries useRegistries)
{
auto fetched = lookup(originalInput);
Input resolvedInput = originalInput;
if (!fetched) {
if (originalInput.isDirect()) {
auto [accessor, lockedInput] = originalInput.getAccessor(store);
fetched.emplace(CachedInput{.lockedInput = lockedInput, .accessor = accessor});
} else {
if (useRegistries != fetchers::UseRegistries::No) {
auto [res, extraAttrs] = lookupInRegistries(store, originalInput, useRegistries);
resolvedInput = std::move(res);
fetched = lookup(resolvedInput);
if (!fetched) {
auto [accessor, lockedInput] = resolvedInput.getAccessor(store);
fetched.emplace(CachedInput{.lockedInput = lockedInput, .accessor = accessor});
}
upsert(resolvedInput, *fetched);
} else {
throw Error(
"'%s' is an indirect flake reference, but registry lookups are not allowed",
originalInput.to_string());
}
}
upsert(originalInput, *fetched);
}
debug("got tree '%s' from '%s'", fetched->accessor, fetched->lockedInput.to_string());
return {fetched->accessor, resolvedInput, fetched->lockedInput};
}
struct InputCacheImpl : InputCache
{
Sync<std::map<Input, CachedInput>> cache_;
std::optional<CachedInput> lookup(const Input & originalInput) const override
{
auto cache(cache_.readLock());
auto i = cache->find(originalInput);
if (i == cache->end())
return std::nullopt;
debug(
"mapping '%s' to previously seen input '%s' -> '%s",
originalInput.to_string(),
i->first.to_string(),
i->second.lockedInput.to_string());
return i->second;
}
void upsert(Input key, CachedInput cachedInput) override
{
cache_.lock()->insert_or_assign(std::move(key), std::move(cachedInput));
}
void clear() override
{
cache_.lock()->clear();
}
};
ref<InputCache> InputCache::create()
{
return make_ref<InputCacheImpl>();
}
}

1
src/libfetchers/meson.build
View file

@ -44,6 +44,7 @@ sources = files(
'git.cc',
'github.cc',
'indirect.cc',
'input-cache.cc',
'mercurial.cc',
'path.cc',
'registry.cc',

137
src/libflake/flake.cc
View file

@ -15,89 +15,17 @@
#include "nix/fetchers/fetch-to-store.hh"
#include "nix/util/memory-source-accessor.hh"
#include "nix/util/mounted-source-accessor.hh"
#include "nix/fetchers/input-cache.hh"
#include <nlohmann/json.hpp>
namespace nix {
using namespace flake;
using namespace fetchers;
namespace flake {
struct FetchedFlake
{
FlakeRef lockedRef;
ref<SourceAccessor> accessor;
};
typedef std::map<FlakeRef, FetchedFlake> FlakeCache;
static std::optional<FetchedFlake> lookupInFlakeCache(
const FlakeCache & flakeCache,
const FlakeRef & flakeRef)
{
auto i = flakeCache.find(flakeRef);
if (i == flakeCache.end()) return std::nullopt;
debug("mapping '%s' to previously seen input '%s' -> '%s",
flakeRef, i->first, i->second.lockedRef);
return i->second;
}
static std::tuple<ref<SourceAccessor>, FlakeRef, FlakeRef> fetchOrSubstituteTree(
EvalState & state,
const FlakeRef & originalRef,
fetchers::UseRegistries useRegistries,
FlakeCache & flakeCache)
{
auto fetched = lookupInFlakeCache(flakeCache, originalRef);
FlakeRef resolvedRef = originalRef;
if (!fetched) {
if (originalRef.input.isDirect()) {
auto [accessor, lockedRef] = originalRef.lazyFetch(state.store);
fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .accessor = accessor});
} else {
if (useRegistries != fetchers::UseRegistries::No) {
resolvedRef = originalRef.resolve(state.store, useRegistries);
fetched = lookupInFlakeCache(flakeCache, originalRef);
if (!fetched) {
auto [accessor, lockedRef] = resolvedRef.lazyFetch(state.store);
fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .accessor = accessor});
}
flakeCache.insert_or_assign(resolvedRef, *fetched);
}
else {
throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", originalRef);
}
}
flakeCache.insert_or_assign(originalRef, *fetched);
}
debug("got tree '%s' from '%s'", fetched->accessor, fetched->lockedRef);
return {fetched->accessor, resolvedRef, fetched->lockedRef};
}
static StorePath copyInputToStore(
EvalState & state,
fetchers::Input & input,
const fetchers::Input & originalInput,
ref<SourceAccessor> accessor)
{
auto storePath = fetchToStore(*state.store, accessor, FetchMode::Copy, input.getName());
state.allowPath(storePath); // FIXME: should just whitelist the entire virtual store
state.storeFS->mount(CanonPath(state.store->printStorePath(storePath)), accessor);
auto narHash = state.store->queryPathInfo(storePath)->narHash;
input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
assert(!originalInput.getNarHash() || storePath == originalInput.computeStorePath(*state.store));
return storePath;
}
static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos)
{
if (value.isThunk() && value.isTrivial())
@ -123,7 +51,7 @@ static std::pair<std::map<FlakeId, FlakeInput>, fetchers::Attrs> parseFlakeInput
static void parseFlakeInputAttr(
EvalState & state,
const Attr & attr,
const nix::Attr & attr,
fetchers::Attrs & attrs)
{
// Allow selecting a subset of enum values
@ -394,15 +322,17 @@ static Flake getFlake(
EvalState & state,
const FlakeRef & originalRef,
fetchers::UseRegistries useRegistries,
FlakeCache & flakeCache,
const InputAttrPath & lockRootAttrPath)
const InputAttrPath & lockRootAttrPath,
bool requireLockable)
{
// Fetch a lazy tree first.
auto [accessor, resolvedRef, lockedRef] = fetchOrSubstituteTree(
state, originalRef, useRegistries, flakeCache);
auto cachedInput = state.inputCache->getAccessor(state.store, originalRef.input, useRegistries);
auto resolvedRef = FlakeRef(std::move(cachedInput.resolvedInput), originalRef.subdir);
auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), originalRef.subdir);
// Parse/eval flake.nix to get at the input.self attributes.
auto flake = readFlake(state, originalRef, resolvedRef, lockedRef, {accessor}, lockRootAttrPath);
auto flake = readFlake(state, originalRef, resolvedRef, lockedRef, {cachedInput.accessor}, lockRootAttrPath);
// Re-fetch the tree if necessary.
auto newLockedRef = applySelfAttrs(lockedRef, flake);
@ -411,23 +341,21 @@ static Flake getFlake(
debug("refetching input '%s' due to self attribute", newLockedRef);
// FIXME: need to remove attrs that are invalidated by the changed input attrs, such as 'narHash'.
newLockedRef.input.attrs.erase("narHash");
auto [accessor2, resolvedRef2, lockedRef2] = fetchOrSubstituteTree(
state, newLockedRef, fetchers::UseRegistries::No, flakeCache);
accessor = accessor2;
lockedRef = lockedRef2;
auto cachedInput2 = state.inputCache->getAccessor(state.store, newLockedRef.input, fetchers::UseRegistries::No);
cachedInput.accessor = cachedInput2.accessor;
lockedRef = FlakeRef(std::move(cachedInput2.lockedInput), newLockedRef.subdir);
}
// Copy the tree to the store.
auto storePath = copyInputToStore(state, lockedRef.input, originalRef.input, accessor);
// Re-parse flake.nix from the store.
return readFlake(state, originalRef, resolvedRef, lockedRef, state.storePath(storePath), lockRootAttrPath);
return readFlake(
state, originalRef, resolvedRef, lockedRef,
state.storePath(state.mountInput(lockedRef.input, originalRef.input, cachedInput.accessor, requireLockable)),
lockRootAttrPath);
}
Flake getFlake(EvalState & state, const FlakeRef & originalRef, fetchers::UseRegistries useRegistries)
Flake getFlake(EvalState & state, const FlakeRef & originalRef, fetchers::UseRegistries useRegistries, bool requireLockable)
{
FlakeCache flakeCache;
return getFlake(state, originalRef, useRegistries, flakeCache, {});
return getFlake(state, originalRef, useRegistries, {}, requireLockable);
}
static LockFile readLockFile(
@ -447,8 +375,6 @@ LockedFlake lockFlake(
const FlakeRef & topRef,
const LockFlags & lockFlags)
{
FlakeCache flakeCache;
auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries);
auto useRegistriesTop = useRegistries ? fetchers::UseRegistries::All : fetchers::UseRegistries::No;
auto useRegistriesInputs = useRegistries ? fetchers::UseRegistries::Limited : fetchers::UseRegistries::No;
@ -457,8 +383,8 @@ LockedFlake lockFlake(
state,
topRef,
useRegistriesTop,
flakeCache,
{});
{},
lockFlags.requireLockable);
if (lockFlags.applyNixConfig) {
flake.config.apply(settings);
@ -637,8 +563,8 @@ LockedFlake lockFlake(
state,
ref,
useRegistriesInputs,
flakeCache,
inputAttrPath);
inputAttrPath,
true);
}
};
@ -811,15 +737,17 @@ LockedFlake lockFlake(
if (auto resolvedPath = resolveRelativePath()) {
return {*resolvedPath, *input.ref};
} else {
auto [accessor, resolvedRef, lockedRef] = fetchOrSubstituteTree(
state, *input.ref, useRegistriesInputs, flakeCache);
auto cachedInput = state.inputCache->getAccessor(state.store, input.ref->input, useRegistriesTop);
auto resolvedRef = FlakeRef(std::move(cachedInput.resolvedInput), input.ref->subdir);
auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), input.ref->subdir);
warnRegistry(resolvedRef);
// FIXME: allow input to be lazy.
auto storePath = copyInputToStore(state, lockedRef.input, input.ref->input, accessor);
return {state.storePath(storePath), lockedRef};
return {
state.storePath(state.mountInput(lockedRef.input, input.ref->input, cachedInput.accessor, true)),
lockedRef
};
}
}();
@ -932,7 +860,8 @@ LockedFlake lockFlake(
flake = getFlake(
state,
topRef,
useRegistriesTop);
useRegistriesTop,
lockFlags.requireLockable);
if (lockFlags.commitLockFile &&
flake.lockedRef.input.getRev() &&

11
src/libflake/include/nix/flake/flake.hh
View file

@ -115,7 +115,11 @@ struct Flake
}
};
Flake getFlake(EvalState & state, const FlakeRef & flakeRef, fetchers::UseRegistries useRegistries);
Flake getFlake(
EvalState & state,
const FlakeRef & flakeRef,
fetchers::UseRegistries useRegistries,
bool requireLockable = true);
/**
* Fingerprint of a locked flake; used as a cache key.
@ -213,6 +217,11 @@ struct LockFlags
* for those inputs will be ignored.
*/
std::set<InputAttrPath> inputUpdates;
/**
* Whether to require a locked input.
*/
bool requireLockable = true;
};
LockedFlake lockFlake(

8
src/libstore/store-api.cc
View file

@ -214,8 +214,12 @@ StorePath Store::addToStore(
auto sink = sourceToSink([&](Source & source) {
LengthSource lengthSource(source);
storePath = addToStoreFromDump(lengthSource, name, fsm, method, hashAlgo, references, repair);
if (settings.warnLargePathThreshold && lengthSource.total >= settings.warnLargePathThreshold)
warn("copied large path '%s' to the store (%s)", path, renderSize(lengthSource.total));
if (settings.warnLargePathThreshold && lengthSource.total >= settings.warnLargePathThreshold) {
static bool failOnLargePath = getEnv("_NIX_TEST_FAIL_ON_LARGE_PATH").value_or("") == "1";
if (failOnLargePath)
throw Error("won't copy large path '%s' to the store (%d)", path, renderSize(lengthSource.total));
warn("copied large path '%s' to the store (%d)", path, renderSize(lengthSource.total));
}
});
dumpPath(path, *sink, fsm, filter);
sink->finish();

6
src/libutil/include/nix/util/mounted-source-accessor.hh
View file

@ -7,6 +7,12 @@ namespace nix {
struct MountedSourceAccessor : SourceAccessor
{
virtual void mount(CanonPath mountPoint, ref<SourceAccessor> accessor) = 0;
/**
* Return the accessor mounted on `mountPoint`, or `nullptr` if
* there is no such mount point.
*/
virtual std::shared_ptr<SourceAccessor> getMount(CanonPath mountPoint) = 0;
};
ref<MountedSourceAccessor> makeMountedSourceAccessor(std::map<CanonPath, ref<SourceAccessor>> mounts);

9
src/libutil/mounted-source-accessor.cc
View file

@ -81,6 +81,15 @@ struct MountedSourceAccessorImpl : MountedSourceAccessor
// FIXME: thread-safety
mounts.insert_or_assign(std::move(mountPoint), accessor);
}
std::shared_ptr<SourceAccessor> getMount(CanonPath mountPoint) override
{
auto i = mounts.find(mountPoint);
if (i != mounts.end())
return i->second;
else
return nullptr;
}
};
ref<MountedSourceAccessor> makeMountedSourceAccessor(std::map<CanonPath, ref<SourceAccessor>> mounts)

2
src/nix-env/user-env.cc
View file

@ -110,7 +110,7 @@ bool createUserEnv(EvalState & state, PackageInfos & elems,
environment. */
auto manifestFile = ({
std::ostringstream str;
printAmbiguous(manifest, state.symbols, str, nullptr, std::numeric_limits<int>::max());
printAmbiguous(state, manifest, str, nullptr, std::numeric_limits<int>::max());
StringSource source { toView(str) };
state.store->addToStoreFromDump(
source, "env-manifest.nix", FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, references);

7
src/nix-instantiate/nix-instantiate.cc
View file

@ -52,7 +52,10 @@ void processExpr(EvalState & state, const Strings & attrPaths,
else
state.autoCallFunction(autoArgs, v, vRes);
if (output == okRaw)
std::cout << *state.coerceToString(noPos, vRes, context, "while generating the nix-instantiate output");
std::cout <<
state.devirtualize(
*state.coerceToString(noPos, vRes, context, "while generating the nix-instantiate output"),
context);
// We intentionally don't output a newline here. The default PS1 for Bash in NixOS starts with a newline
// and other interactive shells like Zsh are smart enough to print a missing newline before the prompt.
else if (output == okXML)
@ -63,7 +66,7 @@ void processExpr(EvalState & state, const Strings & attrPaths,
} else {
if (strict) state.forceValueDeep(vRes);
std::set<const void *> seen;
printAmbiguous(vRes, state.symbols, std::cout, &seen, std::numeric_limits<int>::max());
printAmbiguous(state, vRes, std::cout, &seen, std::numeric_limits<int>::max());
std::cout << std::endl;
}
} else {

6
src/nix/eval.cc
View file

@ -114,7 +114,11 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption
else if (raw) {
logger->stop();
writeFull(getStandardOutput(), *state->coerceToString(noPos, *v, context, "while generating the eval command output"));
writeFull(
getStandardOutput(),
state->devirtualize(
*state->coerceToString(noPos, *v, context, "while generating the eval command output"),
context));
}
else if (json) {

29
src/nix/flake.cc
View file

@ -134,6 +134,7 @@ public:
lockFlags.recreateLockFile = updateAll;
lockFlags.writeLockFile = true;
lockFlags.applyNixConfig = true;
lockFlags.requireLockable = false;
lockFlake();
}
@ -166,6 +167,7 @@ struct CmdFlakeLock : FlakeCommand
lockFlags.writeLockFile = true;
lockFlags.failOnUnlocked = true;
lockFlags.applyNixConfig = true;
lockFlags.requireLockable = false;
lockFlake();
}
@ -212,11 +214,17 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON
void run(nix::ref<nix::Store> store) override
{
lockFlags.requireLockable = false;
auto lockedFlake = lockFlake();
auto & flake = lockedFlake.flake;
// Currently, all flakes are in the Nix store via the rootFS accessor.
auto storePath = store->printStorePath(store->toStorePath(flake.path.path.abs()).first);
/* Hack to show the store path if available. */
std::optional<StorePath> storePath;
if (store->isInStore(flake.path.path.abs())) {
auto path = store->toStorePath(flake.path.path.abs()).first;
if (store->isValidPath(path))
storePath = path;
}
if (json) {
nlohmann::json j;
@ -238,7 +246,8 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON
j["revCount"] = *revCount;
if (auto lastModified = flake.lockedRef.input.getLastModified())
j["lastModified"] = *lastModified;
j["path"] = storePath;
if (storePath)
j["path"] = store->printStorePath(*storePath);
j["locks"] = lockedFlake.lockFile.toJSON().first;
if (auto fingerprint = lockedFlake.getFingerprint(store, fetchSettings))
j["fingerprint"] = fingerprint->to_string(HashFormat::Base16, false);
@ -255,9 +264,10 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON
logger->cout(
ANSI_BOLD "Description:" ANSI_NORMAL " %s",
*flake.description);
logger->cout(
ANSI_BOLD "Path:" ANSI_NORMAL " %s",
storePath);
if (storePath)
logger->cout(
ANSI_BOLD "Path:" ANSI_NORMAL " %s",
store->printStorePath(*storePath));
if (auto rev = flake.lockedRef.input.getRev())
logger->cout(
ANSI_BOLD "Revision:" ANSI_NORMAL " %s",
@ -1079,7 +1089,10 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun
StorePathSet sources;
auto storePath = store->toStorePath(flake.flake.path.path.abs()).first;
auto storePath =
dryRun
? flake.flake.lockedRef.input.computeStorePath(*store)
: std::get<StorePath>(flake.flake.lockedRef.input.fetchToStore(store));
sources.insert(storePath);
@ -1095,7 +1108,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun
storePath =
dryRun
? (*inputNode)->lockedRef.input.computeStorePath(*store)
: std::get<0>((*inputNode)->lockedRef.input.fetchToStore(store));
: std::get<StorePath>((*inputNode)->lockedRef.input.fetchToStore(store));
sources.insert(*storePath);
}
if (json) {

2
tests/functional/fetchGit.sh
View file

@ -292,7 +292,7 @@ path11=$(nix eval --impure --raw --expr "(builtins.fetchGit ./.).outPath")
empty="$TEST_ROOT/empty"
git init "$empty"
emptyAttrs='{ lastModified = 0; lastModifiedDate = "19700101000000"; narHash = "sha256-pQpattmS9VmO3ZIQUFn66az8GSmB4IvYhTTCFn6SUmo="; rev = "0000000000000000000000000000000000000000"; revCount = 0; shortRev = "0000000"; submodules = false; }'
emptyAttrs='{ lastModified = 0; lastModifiedDate = "19700101000000"; rev = "0000000000000000000000000000000000000000"; revCount = 0; shortRev = "0000000"; submodules = false; }'
[[ $(nix eval --impure --expr "builtins.removeAttrs (builtins.fetchGit $empty) [\"outPath\"]") = $emptyAttrs ]]

1
tests/functional/flakes/flakes.sh
View file

@ -77,6 +77,7 @@ hash1=$(echo "$json" | jq -r .revision)
echo foo > "$flake1Dir/foo"
git -C "$flake1Dir" add $flake1Dir/foo
[[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "$hash1-dirty" ]]
[[ $(_NIX_TEST_FAIL_ON_LARGE_PATH=1 nix flake metadata flake1 --json --refresh --warn-large-path-threshold 1 --lazy-trees | jq -r .dirtyRevision) == "$hash1-dirty" ]]
[[ "$(nix flake metadata flake1 --json | jq -r .fingerprint)" != null ]]
echo -n '# foo' >> "$flake1Dir/flake.nix"

1
tests/functional/flakes/unlocked-override.sh
View file

@ -36,6 +36,7 @@ expectStderr 1 nix flake lock "$flake2Dir" --override-input flake1 "$TEST_ROOT/f
grepQuiet "Will not write lock file.*because it has an unlocked input"
nix flake lock "$flake2Dir" --override-input flake1 "$TEST_ROOT/flake1" --allow-dirty-locks
_NIX_TEST_FAIL_ON_LARGE_PATH=1 nix flake lock "$flake2Dir" --override-input flake1 "$TEST_ROOT/flake1" --allow-dirty-locks --warn-large-path-threshold 1 --lazy-trees
# Using a lock file with a dirty lock does not require --allow-dirty-locks, but should print a warning.
expectStderr 0 nix eval "$flake2Dir#x" |

2
tests/functional/lang/eval-fail-hashfile-missing.err.exp
View file

@ -10,4 +10,4 @@ error:
… while calling the 'hashFile' builtin
error: opening file '/pwd/lang/this-file-is-definitely-not-there-7392097': No such file or directory
error: path '/pwd/lang/this-file-is-definitely-not-there-7392097' does not exist

6
tests/nixos/github-flakes.nix
View file

@ -204,7 +204,7 @@ in
cat_log()
# ... otherwise it should use the API
out = client.succeed("nix flake metadata private-flake --json --access-tokens github.com=ghp_000000000000000000000000000000000000 --tarball-ttl 0")
out = client.succeed("nix flake metadata private-flake --json --access-tokens github.com=ghp_000000000000000000000000000000000000 --tarball-ttl 0 --no-trust-tarballs-from-git-forges")
print(out)
info = json.loads(out)
assert info["revision"] == "${private-flake-rev}", f"revision mismatch: {info['revision']} != ${private-flake-rev}"
@ -223,6 +223,10 @@ in
hash = client.succeed(f"nix eval --no-trust-tarballs-from-git-forges --raw --expr '(fetchTree {info['url']}).narHash'")
assert hash == info['locked']['narHash']
# Fetching with an incorrect NAR hash should fail.
out = client.fail(f"nix eval --no-trust-tarballs-from-git-forges --raw --expr '(fetchTree \"github:fancy-enterprise/private-flake/{info['revision']}?narHash=sha256-HsrRFZYg69qaVe/wDyWBYLeS6ca7ACEJg2Z%2BGpEFw4A%3D\").narHash' 2>&1")
assert "NAR hash mismatch in input" in out, "NAR hash check did not fail with the expected error"
# Fetching without a narHash should succeed if trust-github is set and fail otherwise.
client.succeed(f"nix eval --raw --expr 'builtins.fetchTree github:github:fancy-enterprise/private-flake/{info['revision']}'")
out = client.fail(f"nix eval --no-trust-tarballs-from-git-forges --raw --expr 'builtins.fetchTree github:github:fancy-enterprise/private-flake/{info['revision']}' 2>&1")