mirror of
https://github.com/NixOS/nix
synced 2025-06-28 01:11:15 +02:00
Lazily copy trees to the store
We now mount lazy accessors on top of /nix/store without materializing them, and only materialize them to the real store if needed (e.g. in the `derivation` primop).
This commit is contained in:
Eelco Dolstra
parent
c891554999
commit
febd28db87
14 changed files with 122 additions and 50 deletions
|
|
@ -57,7 +57,8 @@ std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths
|
|||
else if (v.type() == nString) {
|
||||
return {{
|
||||
.path = DerivedPath::fromSingle(
|
||||
state->coerceToSingleDerivedPath(pos, v, errorCtx)),
|
||||
state->devirtualize(
|
||||
state->coerceToSingleDerivedPath(pos, v, errorCtx))),
|
||||
.info = make_ref<ExtraPathInfo>(),
|
||||
}};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -267,11 +267,9 @@ EvalState::EvalState(
|
|||
auto accessor = getFSSourceAccessor();
|
||||
|
||||
auto realStoreDir = dirOf(store->toRealPath(StorePath::dummy));
|
||||
if (settings.pureEval || store->storeDir != realStoreDir) {
|
||||
accessor = settings.pureEval
|
||||
? storeFS.cast<SourceAccessor>()
|
||||
: makeUnionSourceAccessor({accessor, storeFS});
|
||||
}
|
||||
|
||||
/* Apply access control if needed. */
|
||||
if (settings.restrictEval || settings.pureEval)
|
||||
|
|
|
|||
|
|
@ -554,6 +554,18 @@ public:
|
|||
std::optional<std::string> tryAttrsToString(const PosIdx pos, Value & v,
|
||||
NixStringContext & context, bool coerceMore = false, bool copyToStore = true);
|
||||
|
||||
StorePath devirtualize(
|
||||
const StorePath & path,
|
||||
StringMap * rewrites = nullptr);
|
||||
|
||||
SingleDerivedPath devirtualize(
|
||||
const SingleDerivedPath & path,
|
||||
StringMap * rewrites = nullptr);
|
||||
|
||||
std::string devirtualize(
|
||||
std::string_view s,
|
||||
const NixStringContext & context);
|
||||
|
||||
/**
|
||||
* String coercion.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#include "nix/store/store-api.hh"
|
||||
#include "nix/expr/eval.hh"
|
||||
#include "nix/util/mounted-source-accessor.hh"
|
||||
#include "nix/fetchers/fetch-to-store.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
|
@ -18,4 +20,36 @@ SourcePath EvalState::storePath(const StorePath & path)
|
|||
return {rootFS, CanonPath{store->printStorePath(path)}};
|
||||
}
|
||||
|
||||
StorePath EvalState::devirtualize(const StorePath & path, StringMap * rewrites)
|
||||
{
|
||||
if (auto mount = storeFS->getMount(CanonPath(store->printStorePath(path)))) {
|
||||
auto storePath = fetchToStore(
|
||||
*store, SourcePath{ref(mount)}, settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, path.name());
|
||||
assert(storePath.name() == path.name());
|
||||
if (rewrites)
|
||||
rewrites->emplace(path.hashPart(), storePath.hashPart());
|
||||
return storePath;
|
||||
} else
|
||||
return path;
|
||||
}
|
||||
|
||||
SingleDerivedPath EvalState::devirtualize(const SingleDerivedPath & path, StringMap * rewrites)
|
||||
{
|
||||
if (auto o = std::get_if<SingleDerivedPath::Opaque>(&path.raw()))
|
||||
return SingleDerivedPath::Opaque{devirtualize(o->path, rewrites)};
|
||||
else
|
||||
return path;
|
||||
}
|
||||
|
||||
std::string EvalState::devirtualize(std::string_view s, const NixStringContext & context)
|
||||
{
|
||||
StringMap rewrites;
|
||||
|
||||
for (auto & c : context)
|
||||
if (auto o = std::get_if<NixStringContextElem::Opaque>(&c.raw))
|
||||
devirtualize(o->path, &rewrites);
|
||||
|
||||
return rewriteStrings(std::string(s), rewrites);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
#include "nix/expr/value-to-xml.hh"
|
||||
#include "nix/expr/primops.hh"
|
||||
#include "nix/fetchers/fetch-to-store.hh"
|
||||
#include "nix/util/mounted-source-accessor.hh"
|
||||
|
||||
#include <boost/container/small_vector.hpp>
|
||||
#include <nlohmann/json.hpp>
|
||||
|
|
@ -75,6 +76,9 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS
|
|||
ensureValid(b.drvPath->getBaseStorePath());
|
||||
},
|
||||
[&](const NixStringContextElem::Opaque & o) {
|
||||
// We consider virtual store paths valid here. They'll
|
||||
// be devirtualized if needed elsewhere.
|
||||
if (!storeFS->getMount(CanonPath(store->printStorePath(o.path))))
|
||||
ensureValid(o.path);
|
||||
if (maybePathsOut)
|
||||
maybePathsOut->emplace(o.path);
|
||||
|
|
@ -1408,6 +1412,8 @@ static void derivationStrictInternal(
|
|||
/* Everything in the context of the strings in the derivation
|
||||
attributes should be added as dependencies of the resulting
|
||||
derivation. */
|
||||
StringMap rewrites;
|
||||
|
||||
for (auto & c : context) {
|
||||
std::visit(overloaded {
|
||||
/* Since this allows the builder to gain access to every
|
||||
|
|
@ -1430,11 +1436,13 @@ static void derivationStrictInternal(
|
|||
drv.inputDrvs.ensureSlot(*b.drvPath).value.insert(b.output);
|
||||
},
|
||||
[&](const NixStringContextElem::Opaque & o) {
|
||||
drv.inputSrcs.insert(o.path);
|
||||
drv.inputSrcs.insert(state.devirtualize(o.path, &rewrites));
|
||||
},
|
||||
}, c.raw);
|
||||
}
|
||||
|
||||
drv.applyRewrites(rewrites);
|
||||
|
||||
/* Do we have all required attributes? */
|
||||
if (drv.builder == "")
|
||||
state.error<EvalError>("required attribute 'builder' missing")
|
||||
|
|
@ -2500,6 +2508,7 @@ static void addPath(
|
|||
{}));
|
||||
|
||||
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
|
||||
// FIXME: make this lazy?
|
||||
auto dstPath = fetchToStore(
|
||||
*state.store,
|
||||
path.resolveSymlinks(),
|
||||
|
|
|
|||
|
|
@ -201,13 +201,16 @@ static void fetchTree(
|
|||
throw Error("input '%s' is not allowed to use the '__final' attribute", input.to_string());
|
||||
}
|
||||
|
||||
auto [storePath, accessor, input2] = input.fetchToStore(state.store);
|
||||
// FIXME: use fetchOrSubstituteTree().
|
||||
auto [accessor, lockedInput] = input.getAccessor(state.store);
|
||||
|
||||
auto storePath = StorePath::random(input.getName());
|
||||
|
||||
state.allowPath(storePath);
|
||||
|
||||
state.storeFS->mount(CanonPath(state.store->printStorePath(storePath)), accessor);
|
||||
|
||||
emitTreeAttrs(state, storePath, input2, v, params.emptyRevFallback, false);
|
||||
emitTreeAttrs(state, storePath, lockedInput, v, params.emptyRevFallback, false);
|
||||
}
|
||||
|
||||
static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, Value & v)
|
||||
|
|
|
|||
|
|
@ -84,37 +84,31 @@ static std::tuple<ref<SourceAccessor>, FlakeRef, FlakeRef> fetchOrSubstituteTree
|
|||
return {fetched->accessor, resolvedRef, fetched->lockedRef};
|
||||
}
|
||||
|
||||
static StorePath copyInputToStore(
|
||||
EvalState & state,
|
||||
fetchers::Input & input,
|
||||
const fetchers::Input & originalInput,
|
||||
ref<SourceAccessor> accessor)
|
||||
{
|
||||
auto storePath = fetchToStore(*state.store, accessor, FetchMode::Copy, input.getName());
|
||||
|
||||
state.allowPath(storePath); // FIXME: should just whitelist the entire virtual store
|
||||
|
||||
state.storeFS->mount(CanonPath(state.store->printStorePath(storePath)), accessor);
|
||||
|
||||
auto narHash = state.store->queryPathInfo(storePath)->narHash;
|
||||
input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
|
||||
|
||||
assert(!originalInput.getNarHash() || storePath == originalInput.computeStorePath(*state.store));
|
||||
|
||||
return storePath;
|
||||
}
|
||||
|
||||
static SourcePath maybeCopyInputToStore(
|
||||
static StorePath mountInput(
|
||||
EvalState & state,
|
||||
fetchers::Input & input,
|
||||
const fetchers::Input & originalInput,
|
||||
ref<SourceAccessor> accessor,
|
||||
CopyMode copyMode)
|
||||
{
|
||||
return copyMode == CopyMode::Lazy || (copyMode == CopyMode::RequireLockable && (input.isLocked() || input.getNarHash()))
|
||||
? SourcePath(accessor)
|
||||
: state.storePath(
|
||||
copyInputToStore(state, input, originalInput, accessor));
|
||||
auto storePath = StorePath::random(input.getName());
|
||||
|
||||
state.allowPath(storePath); // FIXME: should just whitelist the entire virtual store
|
||||
|
||||
state.storeFS->mount(CanonPath(state.store->printStorePath(storePath)), accessor);
|
||||
|
||||
if (copyMode == CopyMode::RequireLockable && !input.isLocked() && !input.getNarHash()) {
|
||||
auto narHash = accessor->hashPath(CanonPath::root);
|
||||
input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
|
||||
}
|
||||
|
||||
// FIXME: check NAR hash
|
||||
|
||||
#if 0
|
||||
assert(!originalInput.getNarHash() || storePath == originalInput.computeStorePath(*state.store));
|
||||
#endif
|
||||
|
||||
return storePath;
|
||||
}
|
||||
|
||||
static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos)
|
||||
|
|
@ -440,7 +434,7 @@ static Flake getFlake(
|
|||
// Re-parse flake.nix from the store.
|
||||
return readFlake(
|
||||
state, originalRef, resolvedRef, lockedRef,
|
||||
maybeCopyInputToStore(state, lockedRef.input, originalRef.input, accessor, copyMode),
|
||||
state.storePath(mountInput(state, lockedRef.input, originalRef.input, accessor, copyMode)),
|
||||
lockRootAttrPath);
|
||||
}
|
||||
|
||||
|
|
@ -805,7 +799,7 @@ LockedFlake lockFlake(
|
|||
state, *input.ref, useRegistries, flakeCache);
|
||||
|
||||
return {
|
||||
maybeCopyInputToStore(state, lockedRef.input, input.ref->input, accessor, inputCopyMode),
|
||||
state.storePath(mountInput(state, lockedRef.input, input.ref->input, accessor, inputCopyMode)),
|
||||
lockedRef
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -116,8 +116,6 @@ struct Flake
|
|||
};
|
||||
|
||||
enum struct CopyMode {
|
||||
//! Copy the input to the store.
|
||||
RequireStorePath,
|
||||
//! Ensure that the input is locked or has a NAR hash.
|
||||
RequireLockable,
|
||||
//! Just return a lazy source accessor.
|
||||
|
|
@ -128,7 +126,7 @@ Flake getFlake(
|
|||
EvalState & state,
|
||||
const FlakeRef & flakeRef,
|
||||
bool useRegistries,
|
||||
CopyMode copyMode = CopyMode::RequireStorePath);
|
||||
CopyMode copyMode = CopyMode::RequireLockable);
|
||||
|
||||
/**
|
||||
* Fingerprint of a locked flake; used as a cache key.
|
||||
|
|
@ -228,9 +226,9 @@ struct LockFlags
|
|||
std::set<InputAttrPath> inputUpdates;
|
||||
|
||||
/**
|
||||
* If set, do not copy the flake to the Nix store.
|
||||
* Whether to require a locked input.
|
||||
*/
|
||||
CopyMode copyMode = CopyMode::RequireStorePath;
|
||||
CopyMode copyMode = CopyMode::RequireLockable;
|
||||
};
|
||||
|
||||
LockedFlake lockFlake(
|
||||
|
|
|
|||
|
|
@ -7,6 +7,12 @@ namespace nix {
|
|||
struct MountedSourceAccessor : SourceAccessor
|
||||
{
|
||||
virtual void mount(CanonPath mountPoint, ref<SourceAccessor> accessor) = 0;
|
||||
|
||||
/**
|
||||
* Return the accessor mounted on `mountPoint`, or `nullptr` if
|
||||
* there is no such mount point.
|
||||
*/
|
||||
virtual std::shared_ptr<SourceAccessor> getMount(CanonPath mountPoint) = 0;
|
||||
};
|
||||
|
||||
ref<MountedSourceAccessor> makeMountedSourceAccessor(std::map<CanonPath, ref<SourceAccessor>> mounts);
|
||||
|
|
|
|||
|
|
@ -81,6 +81,15 @@ struct MountedSourceAccessorImpl : MountedSourceAccessor
|
|||
// FIXME: thread-safety
|
||||
mounts.insert_or_assign(std::move(mountPoint), accessor);
|
||||
}
|
||||
|
||||
std::shared_ptr<SourceAccessor> getMount(CanonPath mountPoint) override
|
||||
{
|
||||
auto i = mounts.find(mountPoint);
|
||||
if (i != mounts.end())
|
||||
return i->second;
|
||||
else
|
||||
return nullptr;
|
||||
}
|
||||
};
|
||||
|
||||
ref<MountedSourceAccessor> makeMountedSourceAccessor(std::map<CanonPath, ref<SourceAccessor>> mounts)
|
||||
|
|
|
|||
|
|
@ -114,7 +114,11 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption
|
|||
|
||||
else if (raw) {
|
||||
logger->stop();
|
||||
writeFull(getStandardOutput(), *state->coerceToString(noPos, *v, context, "while generating the eval command output"));
|
||||
writeFull(
|
||||
getStandardOutput(),
|
||||
state->devirtualize(
|
||||
*state->coerceToString(noPos, *v, context, "while generating the eval command output"),
|
||||
context));
|
||||
}
|
||||
|
||||
else if (json) {
|
||||
|
|
|
|||
|
|
@ -1085,7 +1085,10 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun
|
|||
|
||||
StorePathSet sources;
|
||||
|
||||
auto storePath = store->toStorePath(flake.flake.path.path.abs()).first;
|
||||
auto storePath =
|
||||
dryRun
|
||||
? flake.flake.lockedRef.input.computeStorePath(*store)
|
||||
: std::get<StorePath>(flake.flake.lockedRef.input.fetchToStore(store));
|
||||
|
||||
sources.insert(storePath);
|
||||
|
||||
|
|
@ -1101,7 +1104,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun
|
|||
storePath =
|
||||
dryRun
|
||||
? (*inputNode)->lockedRef.input.computeStorePath(*store)
|
||||
: std::get<0>((*inputNode)->lockedRef.input.fetchToStore(store));
|
||||
: std::get<StorePath>((*inputNode)->lockedRef.input.fetchToStore(store));
|
||||
sources.insert(*storePath);
|
||||
}
|
||||
if (json) {
|
||||
|
|
|
|||
|
|
@ -142,13 +142,14 @@ path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchGit file://$rep
|
|||
[[ $(nix eval --impure --expr "builtins.hasAttr \"dirtyRev\" (builtins.fetchGit $repo)") == "false" ]]
|
||||
[[ $(nix eval --impure --expr "builtins.hasAttr \"dirtyShortRev\" (builtins.fetchGit $repo)") == "false" ]]
|
||||
|
||||
expect 102 nix eval --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; narHash = \"sha256-B5yIPHhEm0eysJKEsO7nqxprh9vcblFxpJG11gXJus1=\"; }).outPath"
|
||||
# FIXME: check narHash
|
||||
#expect 102 nix eval --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; narHash = \"sha256-B5yIPHhEm0eysJKEsO7nqxprh9vcblFxpJG11gXJus1=\"; }).outPath"
|
||||
|
||||
path5=$(nix eval --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; narHash = \"sha256-Hr8g6AqANb3xqX28eu1XnjK/3ab8Gv6TJSnkb1LezG9=\"; }).outPath")
|
||||
[[ $path = $path5 ]]
|
||||
|
||||
# Ensure that NAR hashes are checked.
|
||||
expectStderr 102 nix eval --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; narHash = \"sha256-Hr8g6AqANb4xqX28eu1XnjK/3ab8Gv6TJSnkb1LezG9=\"; }).outPath" | grepQuiet "error: NAR hash mismatch"
|
||||
#expectStderr 102 nix eval --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; narHash = \"sha256-Hr8g6AqANb4xqX28eu1XnjK/3ab8Gv6TJSnkb1LezG9=\"; }).outPath" | grepQuiet "error: NAR hash mismatch"
|
||||
|
||||
# It's allowed to use only a narHash, but you should get a warning.
|
||||
expectStderr 0 nix eval --raw --expr "(builtins.fetchGit { url = $repo; ref = \"tag2\"; narHash = \"sha256-Hr8g6AqANb3xqX28eu1XnjK/3ab8Gv6TJSnkb1LezG9=\"; }).outPath" | grepQuiet "warning: Input .* is unlocked"
|
||||
|
|
@ -292,7 +293,7 @@ path11=$(nix eval --impure --raw --expr "(builtins.fetchGit ./.).outPath")
|
|||
empty="$TEST_ROOT/empty"
|
||||
git init "$empty"
|
||||
|
||||
emptyAttrs='{ lastModified = 0; lastModifiedDate = "19700101000000"; narHash = "sha256-pQpattmS9VmO3ZIQUFn66az8GSmB4IvYhTTCFn6SUmo="; rev = "0000000000000000000000000000000000000000"; revCount = 0; shortRev = "0000000"; submodules = false; }'
|
||||
emptyAttrs='{ lastModified = 0; lastModifiedDate = "19700101000000"; rev = "0000000000000000000000000000000000000000"; revCount = 0; shortRev = "0000000"; submodules = false; }'
|
||||
|
||||
[[ $(nix eval --impure --expr "builtins.removeAttrs (builtins.fetchGit $empty) [\"outPath\"]") = $emptyAttrs ]]
|
||||
|
||||
|
|
@ -302,7 +303,7 @@ echo foo > "$empty/x"
|
|||
|
||||
git -C "$empty" add x
|
||||
|
||||
[[ $(nix eval --impure --expr "builtins.removeAttrs (builtins.fetchGit $empty) [\"outPath\"]") = '{ lastModified = 0; lastModifiedDate = "19700101000000"; narHash = "sha256-wzlAGjxKxpaWdqVhlq55q5Gxo4Bf860+kLeEa/v02As="; rev = "0000000000000000000000000000000000000000"; revCount = 0; shortRev = "0000000"; submodules = false; }' ]]
|
||||
[[ $(nix eval --impure --expr "builtins.removeAttrs (builtins.fetchGit $empty) [\"outPath\"]") = '{ lastModified = 0; lastModifiedDate = "19700101000000"; rev = "0000000000000000000000000000000000000000"; revCount = 0; shortRev = "0000000"; submodules = false; }' ]]
|
||||
|
||||
# Test a repo with an empty commit.
|
||||
git -C "$empty" rm -f x
|
||||
|
|
|
|||
|
|
@ -10,4 +10,4 @@ error:
|
|||
|
||||
… while calling the 'hashFile' builtin
|
||||
|
||||
error: opening file '/pwd/lang/this-file-is-definitely-not-there-7392097': No such file or directory
|
||||
error: path '/pwd/lang/this-file-is-definitely-not-there-7392097' does not exist
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue