wroclaw-git-backups/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-06-30 19:57:59 +02:00
Code Activity

Merge branch 'master' into debug-merge

Browse source
This commit is contained in:
Ben Burdette 2021-11-25 08:53:59 -07:00
commit 64c4ba8f66
348 changed files with 20809 additions and 9834 deletions
Download patch file Download diff file Expand all files Collapse all files

31
src/build-remote/build-remote.cc
View file

@ -18,6 +18,7 @@
#include "derivations.hh"
#include "local-store.hh"
#include "legacy.hh"
#include "experimental-features.hh"
using namespace nix;
using std::cin;
@ -130,11 +131,14 @@ static int main_build_remote(int argc, char * * argv)
for (auto & m : machines) {
debug("considering building on remote machine '%s'", m.storeUri);
if (m.enabled && std::find(m.systemTypes.begin(),
m.systemTypes.end(),
neededSystem) != m.systemTypes.end() &&
if (m.enabled
&& (neededSystem == "builtin"
|| std::find(m.systemTypes.begin(),
m.systemTypes.end(),
neededSystem) != m.systemTypes.end()) &&
m.allSupported(requiredFeatures) &&
m.mandatoryMet(requiredFeatures)) {
m.mandatoryMet(requiredFeatures))
{
rightType = true;
AutoCloseFD free;
uint64_t load = 0;
@ -270,14 +274,23 @@ connected:
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri));
copyPaths(store, ref<Store>(sshStore), store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute);
copyPaths(*store, *sshStore, store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute);
}
uploadLock = -1;
auto drv = store->readDerivation(*drvPath);
auto outputHashes = staticOutputHashes(*store, drv);
drv.inputSrcs = store->parseStorePathSet(inputs);
// Hijack the inputs paths of the derivation to include all the paths
// that come from the `inputDrvs` set.
// We don’t do that for the derivations whose `inputDrvs` is empty
// because
// 1. It’s not needed
// 2. Changing the `inputSrcs` set changes the associated output ids,
// which break CA derivations
if (!drv.inputDrvs.empty())
drv.inputSrcs = store->parseStorePathSet(inputs);
auto result = sshStore->buildDerivation(*drvPath, drv);
@ -286,7 +299,7 @@ connected:
std::set<Realisation> missingRealisations;
StorePathSet missingPaths;
if (settings.isExperimentalFeatureEnabled("ca-derivations") && !derivationHasKnownOutputPaths(drv.type())) {
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && !derivationHasKnownOutputPaths(drv.type())) {
for (auto & outputName : wantedOutputs) {
auto thisOutputHash = outputHashes.at(outputName);
auto thisOutputId = DrvOutput{ thisOutputHash, outputName };
@ -312,13 +325,13 @@ connected:
if (auto localStore = store.dynamic_pointer_cast<LocalStore>())
for (auto & path : missingPaths)
localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */
copyPaths(ref<Store>(sshStore), store, missingPaths, NoRepair, NoCheckSigs, NoSubstitute);
copyPaths(*sshStore, *store, missingPaths, NoRepair, NoCheckSigs, NoSubstitute);
}
// XXX: Should be done as part of `copyPaths`
for (auto & realisation : missingRealisations) {
// Should hold, because if the feature isn't enabled the set
// of missing realisations should be empty
settings.requireExperimentalFeature("ca-derivations");
settings.requireExperimentalFeature(Xp::CaDerivations);
store->registerDrvOutput(realisation);
}

69
src/libcmd/command.cc
View file

@ -95,8 +95,21 @@ EvalCommand::~EvalCommand()
evalState->printStats();
}
ref<Store> EvalCommand::getEvalStore()
{
if (!evalStore)
evalStore = evalStoreUrl ? openStore(*evalStoreUrl) : getStore();
return ref<Store>(evalStore);
}
RealisedPathsCommand::RealisedPathsCommand(bool recursive)
ref<EvalState> EvalCommand::getEvalState()
{
if (!evalState)
evalState = std::make_shared<EvalState>(searchPath, getEvalStore(), getStore());
return ref<EvalState>(evalState);
}
BuiltPathsCommand::BuiltPathsCommand(bool recursive)
: recursive(recursive)
{
if (recursive)
@ -123,44 +136,53 @@ RealisedPathsCommand::RealisedPathsCommand(bool recursive)
});
}
void RealisedPathsCommand::run(ref<Store> store)
void BuiltPathsCommand::run(ref<Store> store)
{
std::vector<RealisedPath> paths;
BuiltPaths paths;
if (all) {
if (installables.size())
throw UsageError("'--all' does not expect arguments");
// XXX: Only uses opaque paths, ignores all the realisations
for (auto & p : store->queryAllValidPaths())
paths.push_back(p);
paths.push_back(BuiltPath::Opaque{p});
} else {
auto pathSet = toRealisedPaths(store, realiseMode, operateOn, installables);
paths = toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables);
if (recursive) {
auto roots = std::move(pathSet);
pathSet = {};
RealisedPath::closure(*store, roots, pathSet);
// XXX: This only computes the store path closure, ignoring
// intermediate realisations
StorePathSet pathsRoots, pathsClosure;
for (auto & root : paths) {
auto rootFromThis = root.outPaths();
pathsRoots.insert(rootFromThis.begin(), rootFromThis.end());
}
store->computeFSClosure(pathsRoots, pathsClosure);
for (auto & path : pathsClosure)
paths.push_back(BuiltPath::Opaque{path});
}
for (auto & path : pathSet)
paths.push_back(path);
}
run(store, std::move(paths));
}
StorePathsCommand::StorePathsCommand(bool recursive)
: RealisedPathsCommand(recursive)
: BuiltPathsCommand(recursive)
{
}
void StorePathsCommand::run(ref<Store> store, std::vector<RealisedPath> paths)
void StorePathsCommand::run(ref<Store> store, BuiltPaths && paths)
{
StorePaths storePaths;
for (auto & p : paths)
storePaths.push_back(p.path());
StorePathSet storePaths;
for (auto & builtPath : paths)
for (auto & p : builtPath.outPaths())
storePaths.insert(p);
run(store, std::move(storePaths));
auto sorted = store->topoSortPaths(storePaths);
std::reverse(sorted.begin(), sorted.end());
run(store, std::move(sorted));
}
void StorePathCommand::run(ref<Store> store, std::vector<StorePath> storePaths)
void StorePathCommand::run(ref<Store> store, std::vector<StorePath> && storePaths)
{
if (storePaths.size() != 1)
throw UsageError("this command requires exactly one store path");
@ -204,7 +226,7 @@ void MixProfile::updateProfile(const StorePath & storePath)
profile2, storePath));
}
void MixProfile::updateProfile(const DerivedPathsWithHints & buildables)
void MixProfile::updateProfile(const BuiltPaths & buildables)
{
if (!profile) return;
@ -212,22 +234,19 @@ void MixProfile::updateProfile(const DerivedPathsWithHints & buildables)
for (auto & buildable : buildables) {
std::visit(overloaded {
[&](DerivedPathWithHints::Opaque bo) {
[&](const BuiltPath::Opaque & bo) {
result.push_back(bo.path);
},
[&](DerivedPathWithHints::Built bfd) {
[&](const BuiltPath::Built & bfd) {
for (auto & output : bfd.outputs) {
/* Output path should be known because we just tried to
build it. */
assert(output.second);
result.push_back(*output.second);
result.push_back(output.second);
}
},
}, buildable.raw());
}
if (result.size() != 1)
throw Error("'--profile' requires that the arguments produce a single store path, but there are %d", result.size());
throw UsageError("'--profile' requires that the arguments produce a single store path, but there are %d", result.size());
updateProfile(result[0]);
}

76
src/libcmd/command.hh
View file

@ -47,13 +47,18 @@ struct EvalCommand : virtual StoreCommand, MixEvalArgs
{
bool startReplOnEvalErrors = false;
ref<EvalState> getEvalState();
EvalCommand();
std::shared_ptr<EvalState> evalState;
~EvalCommand();
ref<Store> getEvalStore();
ref<EvalState> getEvalState();
private:
std::shared_ptr<Store> evalStore;
std::shared_ptr<EvalState> evalState;
};
struct MixFlakeOptions : virtual Args, EvalCommand
@ -105,6 +110,8 @@ enum class Realise {
exists. */
Derivation,
/* Evaluate in dry-run mode. Postcondition: nothing. */
// FIXME: currently unused, but could be revived if we can
// evaluate derivations in-memory.
Nothing
};
@ -147,7 +154,7 @@ private:
};
/* A command that operates on zero or more store paths. */
struct RealisedPathsCommand : public InstallablesCommand
struct BuiltPathsCommand : public InstallablesCommand
{
private:
@ -160,26 +167,26 @@ protected:
public:
RealisedPathsCommand(bool recursive = false);
BuiltPathsCommand(bool recursive = false);
using StoreCommand::run;
virtual void run(ref<Store> store, std::vector<RealisedPath> paths) = 0;
virtual void run(ref<Store> store, BuiltPaths && paths) = 0;
void run(ref<Store> store) override;
bool useDefaultInstallables() override { return !all; }
};
struct StorePathsCommand : public RealisedPathsCommand
struct StorePathsCommand : public BuiltPathsCommand
{
StorePathsCommand(bool recursive = false);
using RealisedPathsCommand::run;
using BuiltPathsCommand::run;
virtual void run(ref<Store> store, std::vector<StorePath> storePaths) = 0;
virtual void run(ref<Store> store, std::vector<StorePath> && storePaths) = 0;
void run(ref<Store> store, std::vector<RealisedPath> paths) override;
void run(ref<Store> store, BuiltPaths && paths) override;
};
/* A command that operates on exactly one store path. */
@ -189,7 +196,7 @@ struct StorePathCommand : public StorePathsCommand
virtual void run(ref<Store> store, const StorePath & storePath) = 0;
void run(ref<Store> store, std::vector<StorePath> storePaths) override;
void run(ref<Store> store, std::vector<StorePath> && storePaths) override;
};
/* A helper class for registering commands globally. */
@ -220,26 +227,37 @@ static RegisterCommand registerCommand2(std::vector<std::string> && name)
return RegisterCommand(std::move(name), [](){ return make_ref<T>(); });
}
DerivedPathsWithHints build(ref<Store> store, Realise mode,
std::vector<std::shared_ptr<Installable>> installables, BuildMode bMode = bmNormal);
BuiltPaths build(
ref<Store> evalStore,
ref<Store> store, Realise mode,
const std::vector<std::shared_ptr<Installable>> & installables,
BuildMode bMode = bmNormal);
std::set<StorePath> toStorePaths(ref<Store> store,
Realise mode, OperateOn operateOn,
std::vector<std::shared_ptr<Installable>> installables);
StorePath toStorePath(ref<Store> store,
Realise mode, OperateOn operateOn,
std::shared_ptr<Installable> installable);
std::set<StorePath> toDerivations(ref<Store> store,
std::vector<std::shared_ptr<Installable>> installables,
bool useDeriver = false);
std::set<RealisedPath> toRealisedPaths(
std::set<StorePath> toStorePaths(
ref<Store> evalStore,
ref<Store> store,
Realise mode,
OperateOn operateOn,
std::vector<std::shared_ptr<Installable>> installables);
const std::vector<std::shared_ptr<Installable>> & installables);
StorePath toStorePath(
ref<Store> evalStore,
ref<Store> store,
Realise mode,
OperateOn operateOn,
std::shared_ptr<Installable> installable);
std::set<StorePath> toDerivations(
ref<Store> store,
const std::vector<std::shared_ptr<Installable>> & installables,
bool useDeriver = false);
BuiltPaths toBuiltPaths(
ref<Store> evalStore,
ref<Store> store,
Realise mode,
OperateOn operateOn,
const std::vector<std::shared_ptr<Installable>> & installables);
/* Helper function to generate args that invoke $EDITOR on
filename:lineno. */
@ -256,7 +274,7 @@ struct MixProfile : virtual StoreCommand
/* If 'profile' is set, make it point at the store path produced
by 'buildables'. */
void updateProfile(const DerivedPathsWithHints & buildables);
void updateProfile(const BuiltPaths & buildables);
};
struct MixDefaultProfile : MixProfile

311
src/libcmd/installables.cc
View file

@ -58,9 +58,13 @@ MixFlakeOptions::MixFlakeOptions()
addFlag({
.longName = "no-registries",
.description = "Don't allow lookups in the flake registries.",
.description =
"Don't allow lookups in the flake registries. This option is deprecated; use `--no-use-registries`.",
.category = category,
.handler = {&lockFlags.useRegistries, false}
.handler = {[&]() {
lockFlags.useRegistries = false;
warn("'--no-registries' is deprecated; use '--no-use-registries'");
}}
});
addFlag({
@ -171,14 +175,50 @@ Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes()
void SourceExprCommand::completeInstallable(std::string_view prefix)
{
if (file) return; // FIXME
if (file) {
evalSettings.pureEval = false;
auto state = getEvalState();
Expr *e = state->parseExprFromFile(
resolveExprPath(state->checkSourcePath(lookupFileArg(*state, *file)))
);
completeFlakeRefWithFragment(
getEvalState(),
lockFlags,
getDefaultFlakeAttrPathPrefixes(),
getDefaultFlakeAttrPaths(),
prefix);
Value root;
state->eval(e, root);
auto autoArgs = getAutoArgs(*state);
std::string prefix_ = std::string(prefix);
auto sep = prefix_.rfind('.');
std::string searchWord;
if (sep != std::string::npos) {
searchWord = prefix_.substr(sep, std::string::npos);
prefix_ = prefix_.substr(0, sep);
} else {
searchWord = prefix_;
prefix_ = "";
}
Value &v1(*findAlongAttrPath(*state, prefix_, *autoArgs, root).first);
state->forceValue(v1);
Value v2;
state->autoCallFunction(*autoArgs, v1, v2);
if (v2.type() == nAttrs) {
for (auto & i : *v2.attrs) {
std::string name = i.name;
if (name.find(searchWord) == 0) {
completions->add(i.name);
}
}
}
} else {
completeFlakeRefWithFragment(
getEvalState(),
lockFlags,
getDefaultFlakeAttrPathPrefixes(),
getDefaultFlakeAttrPaths(),
prefix);
}
}
void completeFlakeRefWithFragment(
@ -249,7 +289,6 @@ void completeFlakeRefWithFragment(
completeFlakeRef(evalState->store, prefix);
}
void completeFlakeRef(ref<Store> store, std::string_view prefix)
{
if (prefix == "")
@ -273,9 +312,9 @@ void completeFlakeRef(ref<Store> store, std::string_view prefix)
}
}
DerivedPathWithHints Installable::toDerivedPathWithHints()
DerivedPath Installable::toDerivedPath()
{
auto buildables = toDerivedPathsWithHints();
auto buildables = toDerivedPaths();
if (buildables.size() != 1)
throw Error("installable '%s' evaluates to %d derivations, where only one is expected", what(), buildables.size());
return std::move(buildables[0]);
@ -309,22 +348,19 @@ struct InstallableStorePath : Installable
std::string what() override { return store->printStorePath(storePath); }
DerivedPathsWithHints toDerivedPathsWithHints() override
DerivedPaths toDerivedPaths() override
{
if (storePath.isDerivation()) {
std::map<std::string, std::optional<StorePath>> outputs;
auto drv = store->readDerivation(storePath);
for (auto & [name, output] : drv.outputsAndOptPaths(*store))
outputs.emplace(name, output.second);
return {
DerivedPathWithHints::Built {
DerivedPath::Built {
.drvPath = storePath,
.outputs = std::move(outputs)
.outputs = drv.outputNames(),
}
};
} else {
return {
DerivedPathWithHints::Opaque {
DerivedPath::Opaque {
.path = storePath,
}
};
@ -337,22 +373,24 @@ struct InstallableStorePath : Installable
}
};
DerivedPathsWithHints InstallableValue::toDerivedPathsWithHints()
DerivedPaths InstallableValue::toDerivedPaths()
{
DerivedPathsWithHints res;
DerivedPaths res;
std::map<StorePath, std::map<std::string, std::optional<StorePath>>> drvsToOutputs;
std::map<StorePath, std::set<std::string>> drvsToOutputs;
RealisedPath::Set drvsToCopy;
// Group by derivation, helps with .all in particular
for (auto & drv : toDerivations()) {
auto outputName = drv.outputName;
if (outputName == "")
throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(drv.drvPath));
drvsToOutputs[drv.drvPath].insert_or_assign(outputName, drv.outPath);
drvsToOutputs[drv.drvPath].insert(outputName);
drvsToCopy.insert(drv.drvPath);
}
for (auto & i : drvsToOutputs)
res.push_back(DerivedPathWithHints::Built { i.first, i.second });
res.push_back(DerivedPath::Built { i.first, i.second });
return res;
}
@ -564,10 +602,10 @@ InstallableFlake::getCursors(EvalState & state)
std::shared_ptr<flake::LockedFlake> InstallableFlake::getLockedFlake() const
{
flake::LockFlags lockFlagsApplyConfig = lockFlags;
lockFlagsApplyConfig.applyNixConfig = true;
if (!_lockedFlake) {
_lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlags));
_lockedFlake->flake.config.apply();
// FIXME: send new config to the daemon.
_lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlagsApplyConfig));
}
return _lockedFlake;
}
@ -616,6 +654,17 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
for (auto & s : ss) {
std::exception_ptr ex;
if (s.find('/') != std::string::npos) {
try {
result.push_back(std::make_shared<InstallableStorePath>(store, store->followLinksToStorePath(s)));
continue;
} catch (BadStorePath &) {
} catch (...) {
if (!ex)
ex = std::current_exception();
}
}
try {
auto [flakeRef, fragment] = parseFlakeRefWithFragment(s, absPath("."));
result.push_back(std::make_shared<InstallableFlake>(
@ -630,25 +679,7 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
ex = std::current_exception();
}
if (s.find('/') != std::string::npos) {
try {
result.push_back(std::make_shared<InstallableStorePath>(store, store->followLinksToStorePath(s)));
continue;
} catch (BadStorePath &) {
} catch (...) {
if (!ex)
ex = std::current_exception();
}
}
std::rethrow_exception(ex);
/*
throw Error(
pathExists(s)
? "path '%s' is not a flake or a store path"
: "don't know how to handle argument '%s'", s);
*/
}
}
@ -663,107 +694,121 @@ std::shared_ptr<Installable> SourceExprCommand::parseInstallable(
return installables.front();
}
DerivedPathsWithHints build(ref<Store> store, Realise mode,
std::vector<std::shared_ptr<Installable>> installables, BuildMode bMode)
BuiltPaths getBuiltPaths(ref<Store> evalStore, ref<Store> store, const DerivedPaths & hopefullyBuiltPaths)
{
if (mode == Realise::Nothing)
settings.readOnlyMode = true;
DerivedPathsWithHints buildables;
std::vector<DerivedPath> pathsToBuild;
for (auto & i : installables) {
for (auto & b : i->toDerivedPathsWithHints()) {
std::visit(overloaded {
[&](DerivedPathWithHints::Opaque bo) {
pathsToBuild.push_back(bo);
BuiltPaths res;
for (const auto & b : hopefullyBuiltPaths)
std::visit(
overloaded{
[&](const DerivedPath::Opaque & bo) {
res.push_back(BuiltPath::Opaque{bo.path});
},
[&](DerivedPathWithHints::Built bfd) {
StringSet outputNames;
for (auto & output : bfd.outputs)
outputNames.insert(output.first);
pathsToBuild.push_back(
DerivedPath::Built{bfd.drvPath, outputNames});
},
}, b.raw());
buildables.push_back(std::move(b));
}
}
if (mode == Realise::Nothing)
printMissing(store, pathsToBuild, lvlError);
else if (mode == Realise::Outputs)
store->buildPaths(pathsToBuild, bMode);
return buildables;
}
std::set<RealisedPath> toRealisedPaths(
ref<Store> store,
Realise mode,
OperateOn operateOn,
std::vector<std::shared_ptr<Installable>> installables)
{
std::set<RealisedPath> res;
if (operateOn == OperateOn::Output) {
for (auto & b : build(store, mode, installables))
std::visit(overloaded {
[&](DerivedPathWithHints::Opaque bo) {
res.insert(bo.path);
},
[&](DerivedPathWithHints::Built bfd) {
auto drv = store->readDerivation(bfd.drvPath);
auto outputHashes = staticOutputHashes(*store, drv);
[&](const DerivedPath::Built & bfd) {
OutputPathMap outputs;
auto drv = evalStore->readDerivation(bfd.drvPath);
auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
auto drvOutputs = drv.outputsAndOptPaths(*store);
for (auto & output : bfd.outputs) {
if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
if (!outputHashes.count(output.first))
throw Error(
"the derivation '%s' doesn't have an output named '%s'",
store->printStorePath(bfd.drvPath),
output.first);
auto outputId = DrvOutput{outputHashes.at(output.first), output.first};
auto realisation = store->queryRealisation(outputId);
if (!outputHashes.count(output))
throw Error(
"the derivation '%s' doesn't have an output named '%s'",
store->printStorePath(bfd.drvPath), output);
if (settings.isExperimentalFeatureEnabled(
Xp::CaDerivations)) {
auto outputId =
DrvOutput{outputHashes.at(output), output};
auto realisation =
store->queryRealisation(outputId);
if (!realisation)
throw Error("cannot operate on an output of unbuilt content-addresed derivation '%s'", outputId.to_string());
res.insert(RealisedPath{*realisation});
}
else {
// If ca-derivations isn't enabled, behave as if
// all the paths are opaque to keep the default
// behavior
assert(output.second);
res.insert(*output.second);
throw Error(
"cannot operate on an output of unbuilt "
"content-addressed derivation '%s'",
outputId.to_string());
outputs.insert_or_assign(
output, realisation->outPath);
} else {
// If ca-derivations isn't enabled, assume that
// the output path is statically known.
assert(drvOutputs.count(output));
assert(drvOutputs.at(output).second);
outputs.insert_or_assign(
output, *drvOutputs.at(output).second);
}
}
res.push_back(BuiltPath::Built{bfd.drvPath, outputs});
},
}, b.raw());
} else {
if (mode == Realise::Nothing)
settings.readOnlyMode = true;
auto drvPaths = toDerivations(store, installables, true);
res.insert(drvPaths.begin(), drvPaths.end());
}
},
b.raw());
return res;
}
StorePathSet toStorePaths(ref<Store> store,
BuiltPaths build(
ref<Store> evalStore,
ref<Store> store,
Realise mode,
const std::vector<std::shared_ptr<Installable>> & installables,
BuildMode bMode)
{
if (mode == Realise::Nothing)
settings.readOnlyMode = true;
std::vector<DerivedPath> pathsToBuild;
for (auto & i : installables) {
auto b = i->toDerivedPaths();
pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end());
}
if (mode == Realise::Nothing || mode == Realise::Derivation)
printMissing(store, pathsToBuild, lvlError);
else if (mode == Realise::Outputs)
store->buildPaths(pathsToBuild, bMode, evalStore);
return getBuiltPaths(evalStore, store, pathsToBuild);
}
BuiltPaths toBuiltPaths(
ref<Store> evalStore,
ref<Store> store,
Realise mode,
OperateOn operateOn,
const std::vector<std::shared_ptr<Installable>> & installables)
{
if (operateOn == OperateOn::Output)
return build(evalStore, store, mode, installables);
else {
if (mode == Realise::Nothing)
settings.readOnlyMode = true;
BuiltPaths res;
for (auto & drvPath : toDerivations(store, installables, true))
res.push_back(BuiltPath::Opaque{drvPath});
return res;
}
}
StorePathSet toStorePaths(
ref<Store> evalStore,
ref<Store> store,
Realise mode, OperateOn operateOn,
std::vector<std::shared_ptr<Installable>> installables)
const std::vector<std::shared_ptr<Installable>> & installables)
{
StorePathSet outPaths;
for (auto & path : toRealisedPaths(store, mode, operateOn, installables))
outPaths.insert(path.path());
for (auto & path : toBuiltPaths(evalStore, store, mode, operateOn, installables)) {
auto thisOutPaths = path.outPaths();
outPaths.insert(thisOutPaths.begin(), thisOutPaths.end());
}
return outPaths;
}
StorePath toStorePath(ref<Store> store,
StorePath toStorePath(
ref<Store> evalStore,
ref<Store> store,
Realise mode, OperateOn operateOn,
std::shared_ptr<Installable> installable)
{
auto paths = toStorePaths(store, mode, operateOn, {installable});
auto paths = toStorePaths(evalStore, store, mode, operateOn, {installable});
if (paths.size() != 1)
throw Error("argument '%s' should evaluate to one store path", installable->what());
@ -771,15 +816,17 @@ StorePath toStorePath(ref<Store> store,
return *paths.begin();
}
StorePathSet toDerivations(ref<Store> store,
std::vector<std::shared_ptr<Installable>> installables, bool useDeriver)
StorePathSet toDerivations(
ref<Store> store,
const std::vector<std::shared_ptr<Installable>> & installables,
bool useDeriver)
{
StorePathSet drvPaths;
for (auto & i : installables)
for (auto & b : i->toDerivedPathsWithHints())
for (const auto & i : installables)
for (const auto & b : i->toDerivedPaths())
std::visit(overloaded {
[&](DerivedPathWithHints::Opaque bo) {
[&](const DerivedPath::Opaque & bo) {
if (!useDeriver)
throw Error("argument '%s' did not evaluate to a derivation", i->what());
auto derivers = store->queryValidDerivers(bo.path);
@ -788,7 +835,7 @@ StorePathSet toDerivations(ref<Store> store,
// FIXME: use all derivers?
drvPaths.insert(*derivers.begin());
},
[&](DerivedPathWithHints::Built bfd) {
[&](const DerivedPath::Built & bfd) {
drvPaths.insert(bfd.drvPath);
},
}, b.raw());

14
src/libcmd/installables.hh
View file

@ -23,17 +23,23 @@ struct App
// FIXME: add args, sandbox settings, metadata, ...
};
struct UnresolvedApp
{
App unresolved;
App resolve(ref<Store> evalStore, ref<Store> store);
};
struct Installable
{
virtual ~Installable() { }
virtual std::string what() = 0;
virtual DerivedPathsWithHints toDerivedPathsWithHints() = 0;
virtual DerivedPaths toDerivedPaths() = 0;
DerivedPathWithHints toDerivedPathWithHints();
DerivedPath toDerivedPath();
App toApp(EvalState & state);
UnresolvedApp toApp(EvalState & state);
virtual std::pair<Value *, Pos> toValue(EvalState & state)
{
@ -74,7 +80,7 @@ struct InstallableValue : Installable
virtual std::vector<DerivationInfo> toDerivations() = 0;
DerivedPathsWithHints toDerivedPathsWithHints() override;
DerivedPaths toDerivedPaths() override;
};
struct InstallableFlake : InstallableValue

5
src/libcmd/local.mk
View file

@ -8,8 +8,9 @@ libcmd_SOURCES := $(wildcard $(d)/*.cc)
libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers -I src/nix
libcmd_LDFLAGS = $(EDITLINE_LIBS) -llowdown
# libcmd_LDFLAGS = $(EDITLINE_LIBS) -llowdown
libcmd_LDFLAGS += -llowdown -pthread
libcmd_LIBS = libstore libutil libexpr libmain libfetchers libnix
$(eval $(call install-file-in, $(d)/nix-cmd.pc, $(prefix)/lib/pkgconfig, 0644))
$(eval $(call install-file-in, $(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644))

8
src/libcmd/markdown.cc
View file

@ -12,7 +12,7 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
struct lowdown_opts opts {
.type = LOWDOWN_TERM,
.maxdepth = 20,
.cols = std::min(getWindowSize().second, (unsigned short) 80),
.cols = std::max(getWindowSize().second, (unsigned short) 80),
.hmargin = 0,
.vmargin = 0,
.feat = LOWDOWN_COMMONMARK | LOWDOWN_FENCED | LOWDOWN_DEFLIST | LOWDOWN_TABLES,
@ -25,7 +25,7 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
Finally freeDoc([&]() { lowdown_doc_free(doc); });
size_t maxn = 0;
auto node = lowdown_doc_parse(doc, &maxn, markdown.data(), markdown.size());
auto node = lowdown_doc_parse(doc, &maxn, markdown.data(), markdown.size(), nullptr);
if (!node)
throw Error("cannot parse Markdown document");
Finally freeNode([&]() { lowdown_node_free(node); });
@ -40,11 +40,11 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
throw Error("cannot allocate Markdown output buffer");
Finally freeBuffer([&]() { lowdown_buf_free(buf); });
int rndr_res = lowdown_term_rndr(buf, nullptr, renderer, node);
int rndr_res = lowdown_term_rndr(buf, renderer, node);
if (!rndr_res)
throw Error("allocation error while rendering Markdown");
return std::string(buf->data, buf->size);
return filterANSIEscapes(std::string(buf->data, buf->size), !shouldANSI());
}
}

108
src/libcmd/repl.cc
View file

@ -68,6 +68,7 @@ struct NixRepl
StorePath getDerivationPath(Value & v);
bool processLine(string line);
void loadFile(const Path & path);
void loadFlake(const std::string & flakeRef);
void initEnv();
void reloadFiles();
void addAttrsToScope(Value & attrs);
@ -104,6 +105,25 @@ NixRepl::~NixRepl()
write_history(historyFile.c_str());
}
string runNix(Path program, const Strings & args,
const std::optional<std::string> & input = {})
{
auto subprocessEnv = getEnv();
subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue();
auto res = runProgram(RunOptions {
.program = settings.nixBinDir+ "/" + program,
.args = args,
.environment = subprocessEnv,
.input = input,
});
if (!statusOk(res.first))
throw ExecError(res.first, fmt("program '%1%' %2%", program, statusToString(res.first)));
return res.second;
}
static NixRepl * curRepl; // ugly
static char * completionCallback(char * s, int *match) {
@ -180,15 +200,14 @@ namespace {
void NixRepl::mainLoop(const std::vector<std::string> & files)
{
string error = ANSI_RED "error:" ANSI_NORMAL " ";
std::cout << "Welcome to Nix version " << nixVersion << ". Type :? for help." << std::endl << std::endl;
notice("Welcome to Nix " + nixVersion + ". Type :? for help.\n");
if (!files.empty()) {
for (auto & i : files)
loadedFiles.push_back(i);
reloadFiles();
if (!loadedFiles.empty()) std::cout << std::endl;
}
reloadFiles();
if (!loadedFiles.empty()) notice("");
// Allow nix-repl specific settings in .inputrc
rl_readline_name = "nix-repl";
@ -378,6 +397,8 @@ bool NixRepl::processLine(string line)
{
if (line == "") return true;
_isInterrupted = false;
string command, arg;
if (line[0] == ':') {
@ -397,9 +418,10 @@ bool NixRepl::processLine(string line)
<< " <x> = <expr> Bind expression to variable\n"
<< " :a <expr> Add attributes from resulting set to scope\n"
<< " :b <expr> Build derivation\n"
<< " :e <expr> Open the derivation in $EDITOR\n"
<< " :e <expr> Open package or function in $EDITOR\n"
<< " :i <expr> Build derivation, then install result into current profile\n"
<< " :l <path> Load Nix expression and add it to scope\n"
<< " :lf <ref> Load Nix flake and add it to scope\n"
<< " :p <expr> Evaluate and print expression recursively\n"
<< " :q Exit nix-repl\n"
<< " :r Reload all files\n"
@ -420,6 +442,10 @@ bool NixRepl::processLine(string line)
loadFile(arg);
}
else if (command == ":lf" || command == ":load-flake") {
loadFlake(arg);
}
else if (command == ":r" || command == ":reload") {
state->resetFileCache();
reloadFiles();
@ -439,14 +465,17 @@ bool NixRepl::processLine(string line)
pos = v.lambda.fun->pos;
} else {
// assume it's a derivation
pos = findDerivationFilename(*state, v, arg);
pos = findPackageFilename(*state, v, arg);
}
// Open in EDITOR
auto args = editorFor(pos);
auto editor = args.front();
args.pop_front();
runProgram(editor, true, args);
// runProgram redirects stdout to a StringSink,
// using runProgram2 to allow editors to display their UI
runProgram2(RunOptions { .program = editor, .searchPath = true, .args = args });
// Reload right after exiting the editor
state->resetFileCache();
@ -456,16 +485,17 @@ bool NixRepl::processLine(string line)
else if (command == ":t") {
Value v;
evalString(arg, v);
std::cout << showType(v) << std::endl;
logger->cout(showType(v));
}
} else if (command == ":u") {
else if (command == ":u") {
Value v, f, result;
evalString(arg, v);
evalString("drv: (import <nixpkgs> {}).runCommand \"shell\" { buildInputs = [ drv ]; } \"\"", f);
state->callFunction(f, v, result, Pos());
StorePath drvPath = getDerivationPath(result);
runProgram(settings.nixBinDir + "/nix-shell", true, {state->store->printStorePath(drvPath)});
runNix("nix-shell", {state->store->printStorePath(drvPath)});
}
else if (command == ":b" || command == ":i" || command == ":s") {
@ -475,21 +505,15 @@ bool NixRepl::processLine(string line)
Path drvPathRaw = state->store->printStorePath(drvPath);
if (command == ":b") {
/* We could do the build in this process using buildPaths(),
but doing it in a child makes it easier to recover from
problems / SIGINT. */
try {
runProgram(settings.nixBinDir + "/nix", true, {"build", "--no-link", drvPathRaw});
auto drv = state->store->readDerivation(drvPath);
std::cout << std::endl << "this derivation produced the following outputs:" << std::endl;
for (auto & i : drv.outputsAndOptPaths(*state->store))
std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(*i.second.second));
} catch (ExecError &) {
}
state->store->buildPaths({DerivedPath::Built{drvPath}});
auto drv = state->store->readDerivation(drvPath);
logger->cout("\nThis derivation produced the following outputs:");
for (auto & [outputName, outputPath] : state->store->queryDerivationOutputMap(drvPath))
logger->cout(" %s -> %s", outputName, state->store->printStorePath(outputPath));
} else if (command == ":i") {
runProgram(settings.nixBinDir + "/nix-env", true, {"-i", drvPathRaw});
runNix("nix-env", {"-i", drvPathRaw});
} else {
runProgram(settings.nixBinDir + "/nix-shell", true, {drvPathRaw});
runNix("nix-shell", {drvPathRaw});
}
}
@ -518,9 +542,9 @@ bool NixRepl::processLine(string line)
+ concatStringsSep(" ", args) + "\n\n";
}
markdown += trim(stripIndentation(doc->doc));
markdown += stripIndentation(doc->doc);
std::cout << renderMarkdownToTerminal(markdown);
logger->cout(trim(renderMarkdownToTerminal(markdown)));
} else
throw Error("value does not have documentation");
}
@ -561,6 +585,25 @@ void NixRepl::loadFile(const Path & path)
addAttrsToScope(v2);
}
void NixRepl::loadFlake(const std::string & flakeRefS)
{
auto flakeRef = parseFlakeRef(flakeRefS, absPath("."), true);
if (evalSettings.pureEval && !flakeRef.input.isImmutable())
throw Error("cannot use ':load-flake' on mutable flake reference '%s' (use --impure to override)", flakeRefS);
Value v;
flake::callFlake(*state,
flake::lockFlake(*state, flakeRef,
flake::LockFlags {
.updateLockFile = false,
.useRegistries = !evalSettings.pureEval,
.allowMutable = !evalSettings.pureEval,
}),
v);
addAttrsToScope(v);
}
void NixRepl::initEnv()
{
@ -584,9 +627,9 @@ void NixRepl::reloadFiles()
bool first = true;
for (auto & i : old) {
if (!first) std::cout << std::endl;
if (!first) notice("");
first = false;
std::cout << format("Loading '%1%'...") % i << std::endl;
notice("Loading '%1%'...", i);
loadFile(i);
}
}
@ -596,8 +639,8 @@ void NixRepl::addAttrsToScope(Value & attrs)
{
state->forceAttrs(attrs);
for (auto & i : *attrs.attrs)
addVarToScope(i.name, i.value);
std::cout << format("Added %1% variables.") % attrs.attrs->size() << std::endl;
addVarToScope(i.name, *i.value);
notice("Added %1% variables.", attrs.attrs->size());
}
@ -605,8 +648,9 @@ void NixRepl::addVarToScope(const Symbol & name, Value * v)
{
if (displ >= envSize)
throw Error("environment full; cannot add more variables");
staticEnv->vars[name] = displ;
env->values[displ++] = v;
staticEnv->vars.emplace_back(name, displ);
staticEnv->sort();
env->values[displ++] = &v;
varNames.insert((string) name);
}
@ -665,7 +709,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
break;
case nString:
str << ANSI_YELLOW;
str << ANSI_WARNING;
printStringValue(str, v.string.s);
str << ANSI_NORMAL;
break;

8
src/libexpr/attr-path.cc
View file

@ -19,7 +19,7 @@ static Strings parseAttrPath(std::string_view s)
++i;
while (1) {
if (i == s.end())
throw Error("missing closing quote in selection path '%1%'", s);
throw ParseError("missing closing quote in selection path '%1%'", s);
if (*i == '"') break;
cur.push_back(*i++);
}
@ -100,7 +100,7 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
}
Pos findDerivationFilename(EvalState & state, Value & v, std::string what)
Pos findPackageFilename(EvalState & state, Value & v, std::string what)
{
Value * v2;
try {
@ -116,14 +116,14 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what)
auto colon = pos.rfind(':');
if (colon == std::string::npos)
throw Error("cannot parse meta.position attribute '%s'", pos);
throw ParseError("cannot parse meta.position attribute '%s'", pos);
std::string filename(pos, 0, colon);
unsigned int lineno;
try {
lineno = std::stoi(std::string(pos, colon + 1));
} catch (std::invalid_argument & e) {
throw Error("cannot parse line number '%s'", pos);
throw ParseError("cannot parse line number '%s'", pos);
}
Symbol file = state.symbols.create(filename);

2
src/libexpr/attr-path.hh
View file

@ -14,7 +14,7 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
Bindings & autoArgs, Value & vIn);
/* Heuristic to find the filename and lineno or a nix value. */
Pos findDerivationFilename(EvalState & state, Value & v, std::string what);
Pos findPackageFilename(EvalState & state, Value & v, std::string what);
std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s);

8
src/libexpr/attr-set.hh
View file

@ -17,8 +17,8 @@ struct Attr
{
Symbol name;
Value * value;
Pos * pos;
Attr(Symbol name, Value * value, Pos * pos = &noPos)
ptr<Pos> pos;
Attr(Symbol name, Value * value, ptr<Pos> pos = ptr(&noPos))
: name(name), value(value), pos(pos) { };
Attr() : pos(&noPos) { };
bool operator < (const Attr & a) const
@ -35,13 +35,13 @@ class Bindings
{
public:
typedef uint32_t size_t;
Pos *pos;
ptr<Pos> pos;
private:
size_t size_, capacity_;
Attr attrs[0];
Bindings(size_t capacity) : size_(0), capacity_(capacity) { }
Bindings(size_t capacity) : pos(&noPos), size_(0), capacity_(capacity) { }
Bindings(const Bindings & bindings) = delete;
public:

8
src/libexpr/common-eval-args.cc
View file

@ -61,6 +61,14 @@ MixEvalArgs::MixEvalArgs()
fetchers::overrideRegistry(from.input, to.input, extraAttrs);
}}
});
addFlag({
.longName = "eval-store",
.description = "The Nix store to use for evaluations.",
.category = category,
.labels = {"store-url"},
.handler = {&evalStoreUrl},
});
}
Bindings * MixEvalArgs::getAutoArgs(EvalState & state)

3
src/libexpr/common-eval-args.hh
View file

@ -16,8 +16,9 @@ struct MixEvalArgs : virtual Args
Strings searchPath;
private:
std::optional<std::string> evalStoreUrl;
private:
std::map<std::string, std::string> autoArgs;
};

502
src/libexpr/eval.cc
View file

@ -65,7 +65,11 @@ static char * dupStringWithLen(const char * s, size_t size)
RootValue allocRootValue(Value * v)
{
#if HAVE_BOEHMGC
return std::allocate_shared<Value *>(traceable_allocator<Value *>(), v);
#else
return std::make_shared<Value *>(v);
#endif
}
@ -234,22 +238,34 @@ static void * oomHandler(size_t requested)
}
class BoehmGCStackAllocator : public StackAllocator {
boost::coroutines2::protected_fixedsize_stack stack {
// We allocate 8 MB, the default max stack size on NixOS.
// A smaller stack might be quicker to allocate but reduces the stack
// depth available for source filter expressions etc.
std::max(boost::context::stack_traits::default_size(), static_cast<std::size_t>(8 * 1024 * 1024))
boost::coroutines2::protected_fixedsize_stack stack {
// We allocate 8 MB, the default max stack size on NixOS.
// A smaller stack might be quicker to allocate but reduces the stack
// depth available for source filter expressions etc.
std::max(boost::context::stack_traits::default_size(), static_cast<std::size_t>(8 * 1024 * 1024))
};
// This is specific to boost::coroutines2::protected_fixedsize_stack.
// The stack protection page is included in sctx.size, so we have to
// subtract one page size from the stack size.
std::size_t pfss_usable_stack_size(boost::context::stack_context &sctx) {
return sctx.size - boost::context::stack_traits::page_size();
}
public:
boost::context::stack_context allocate() override {
auto sctx = stack.allocate();
GC_add_roots(static_cast<char *>(sctx.sp) - sctx.size, sctx.sp);
// Stacks generally start at a high address and grow to lower addresses.
// Architectures that do the opposite are rare; in fact so rare that
// boost_routine does not implement it.
// So we subtract the stack size.
GC_add_roots(static_cast<char *>(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp);
return sctx;
}
void deallocate(boost::context::stack_context sctx) override {
GC_remove_roots(static_cast<char *>(sctx.sp) - sctx.size, sctx.sp);
GC_remove_roots(static_cast<char *>(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp);
stack.deallocate(sctx);
}
@ -363,7 +379,10 @@ static Strings parseNixPath(const string & s)
}
EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
EvalState::EvalState(
const Strings & _searchPath,
ref<Store> store,
std::shared_ptr<Store> buildStore)
: sWith(symbols.create("<with>"))
, sOutPath(symbols.create("outPath"))
, sDrvPath(symbols.create("drvPath"))
@ -396,6 +415,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
, sEpsilon(symbols.create(""))
, repair(NoRepair)
, store(store)
, buildStore(buildStore ? buildStore : store)
, regexCache(makeRegexCache())
, baseEnv(allocEnv(128))
, staticBaseEnv(new StaticEnv(false, 0))
@ -426,12 +446,12 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
StorePathSet closure;
store->computeFSClosure(store->toStorePath(r.second).first, closure);
for (auto & path : closure)
allowedPaths->insert(store->printStorePath(path));
allowPath(path);
} catch (InvalidPath &) {
allowedPaths->insert(r.second);
allowPath(r.second);
}
} else
allowedPaths->insert(r.second);
allowPath(r.second);
}
}
@ -446,6 +466,35 @@ EvalState::~EvalState()
}
void EvalState::requireExperimentalFeatureOnEvaluation(
const ExperimentalFeature & feature,
const std::string_view fName,
const Pos & pos)
{
if (!settings.isExperimentalFeatureEnabled(feature)) {
throw EvalError({
.msg = hintfmt(
"Cannot call '%2%' because experimental Nix feature '%1%' is disabled. You can enable it via '--extra-experimental-features %1%'.",
feature,
fName
),
.errPos = pos
});
}
}
void EvalState::allowPath(const Path & path)
{
if (allowedPaths)
allowedPaths->insert(path);
}
void EvalState::allowPath(const StorePath & storePath)
{
if (allowedPaths)
allowedPaths->insert(store->toRealPath(storePath));
}
Path EvalState::checkSourcePath(const Path & path_)
{
if (!allowedPaths) return path_;
@ -472,7 +521,7 @@ Path EvalState::checkSourcePath(const Path & path_)
}
if (!found)
throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", abspath);
throw RestrictedPathError("access to absolute path '%1%' is forbidden in restricted mode", abspath);
/* Resolve symlinks. */
debug(format("checking access to '%s'") % abspath);
@ -485,7 +534,7 @@ Path EvalState::checkSourcePath(const Path & path_)
}
}
throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", path);
throw RestrictedPathError("access to canonical path '%1%' is forbidden in restricted mode", path);
}
@ -535,14 +584,20 @@ Value * EvalState::addConstant(const string & name, Value & v)
{
Value * v2 = allocValue();
*v2 = v;
staticBaseEnv->vars[symbols.create(name)] = baseEnvDispl;
baseEnv.values[baseEnvDispl++] = v2;
string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name;
baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v2));
addConstant(name, v2);
return v2;
}
void EvalState::addConstant(const string & name, Value * v)
{
staticBaseEnv.vars.emplace_back(symbols.create(name), baseEnvDispl);
baseEnv.values[baseEnvDispl++] = v;
string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name;
baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v));
}
Value * EvalState::addPrimOp(const string & name,
size_t arity, PrimOpFun primOp)
{
@ -561,7 +616,7 @@ Value * EvalState::addPrimOp(const string & name,
Value * v = allocValue();
v->mkPrimOp(new PrimOp { .fun = primOp, .arity = arity, .name = sym });
staticBaseEnv->vars[symbols.create(name)] = baseEnvDispl;
staticBaseEnv->vars.emplace_back(symbols.create(name), baseEnvDispl);
baseEnv.values[baseEnvDispl++] = v;
baseEnv.values[0]->attrs->push_back(Attr(sym, v));
return v;
@ -587,7 +642,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
Value * v = allocValue();
v->mkPrimOp(new PrimOp(std::move(primOp)));
staticBaseEnv->vars[envName] = baseEnvDispl;
staticBaseEnv->vars.emplace_back(envName, baseEnvDispl);
baseEnv.values[baseEnvDispl++] = v;
baseEnv.values[0]->attrs->push_back(Attr(primOp.name, v));
return v;
@ -861,7 +916,7 @@ void mkPath(Value & v, const char * s)
inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
{
for (size_t l = var.level; l; --l, env = env->up) ;
for (auto l = var.level; l; --l, env = env->up) ;
if (!var.fromWith) return env->values[var.displ];
@ -875,7 +930,7 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
}
Bindings::iterator j = env->values[0]->attrs->find(var.name);
if (j != env->values[0]->attrs->end()) {
if (countCalls && j->pos) attrSelects[*j->pos]++;
if (countCalls) attrSelects[*j->pos]++;
return j->value;
}
if (!env->prevWith) {
@ -886,18 +941,10 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
}
std::atomic<uint64_t> nrValuesFreed{0};
void finalizeValue(void * obj, void * data)
{
nrValuesFreed++;
}
Value * EvalState::allocValue()
{
nrValues++;
auto v = (Value *) allocBytes(sizeof(Value));
//GC_register_finalizer_no_order(v, finalizeValue, nullptr, nullptr, nullptr);
return v;
}
@ -949,9 +996,9 @@ void EvalState::mkThunk_(Value & v, Expr * expr)
}
void EvalState::mkPos(Value & v, Pos * pos)
void EvalState::mkPos(Value & v, ptr<Pos> pos)
{
if (pos && pos->file.set()) {
if (pos->file.set()) {
mkAttrs(v, 3);
mkString(*allocAttr(v, sFile), pos->file);
mkInt(*allocAttr(v, sLine), pos->line);
@ -974,39 +1021,37 @@ Value * Expr::maybeThunk(EvalState & state, Env & env)
}
unsigned long nrAvoided = 0;
Value * ExprVar::maybeThunk(EvalState & state, Env & env)
{
Value * v = state.lookupVar(&env, *this, true);
/* The value might not be initialised in the environment yet.
In that case, ignore it. */
if (v) { nrAvoided++; return v; }
if (v) { state.nrAvoided++; return v; }
return Expr::maybeThunk(state, env);
}
Value * ExprString::maybeThunk(EvalState & state, Env & env)
{
nrAvoided++;
state.nrAvoided++;
return &v;
}
Value * ExprInt::maybeThunk(EvalState & state, Env & env)
{
nrAvoided++;
state.nrAvoided++;
return &v;
}
Value * ExprFloat::maybeThunk(EvalState & state, Env & env)
{
nrAvoided++;
state.nrAvoided++;
return &v;
}
Value * ExprPath::maybeThunk(EvalState & state, Env & env)
{
nrAvoided++;
state.nrAvoided++;
return &v;
}
@ -1021,38 +1066,23 @@ void EvalState::evalFile(const Path & path_, Value & v, bool mustBeTrivial)
return;
}
Path path2 = resolveExprPath(path);
if ((i = fileEvalCache.find(path2)) != fileEvalCache.end()) {
Path resolvedPath = resolveExprPath(path);
if ((i = fileEvalCache.find(resolvedPath)) != fileEvalCache.end()) {
v = i->second;
return;
}
printTalkative("evaluating file '%1%'", path2);
printTalkative("evaluating file '%1%'", resolvedPath);
Expr * e = nullptr;
auto j = fileParseCache.find(path2);
auto j = fileParseCache.find(resolvedPath);
if (j != fileParseCache.end())
e = j->second;
if (!e)
e = parseExprFromFile(checkSourcePath(path2));
e = parseExprFromFile(checkSourcePath(resolvedPath));
fileParseCache[path2] = e;
try {
// Enforce that 'flake.nix' is a direct attrset, not a
// computation.
if (mustBeTrivial &&
!(dynamic_cast<ExprAttrs *>(e)))
throw Error("file '%s' must be an attribute set", path);
eval(e, v);
} catch (Error & e) {
addErrorTrace(e, "while evaluating the file '%1%':", path2);
throw;
}
fileEvalCache[path2] = v;
if (path != path2) fileEvalCache[path] = v;
cacheFile(path, resolvedPath, e, v, mustBeTrivial);
}
@ -1063,6 +1093,32 @@ void EvalState::resetFileCache()
}
void EvalState::cacheFile(
const Path & path,
const Path & resolvedPath,
Expr * e,
Value & v,
bool mustBeTrivial)
{
fileParseCache[resolvedPath] = e;
try {
// Enforce that 'flake.nix' is a direct attrset, not a
// computation.
if (mustBeTrivial &&
!(dynamic_cast<ExprAttrs *>(e)))
throw EvalError("file '%s' must be an attribute set", path);
eval(e, v);
} catch (Error & e) {
addErrorTrace(e, "while evaluating the file '%1%':", resolvedPath);
throw;
}
fileEvalCache[resolvedPath] = v;
if (path != resolvedPath) fileEvalCache[path] = v;
}
void EvalState::eval(Expr * e, Value & v)
{
e->eval(*this, baseEnv, v);
@ -1144,7 +1200,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
/* The recursive attributes are evaluated in the new
environment, while the inherited attributes are evaluated
in the original environment. */
size_t displ = 0;
Displacement displ = 0;
for (auto & i : attrs) {
Value * vAttr;
if (hasOverrides && !i.second.inherited) {
@ -1153,7 +1209,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
} else
vAttr = i.second.e->maybeThunk(state, i.second.inherited ? env : env2);
env2.values[displ++] = vAttr;
v.attrs->push_back(Attr(i.first, vAttr, &i.second.pos));
v.attrs->push_back(Attr(i.first, vAttr, ptr(&i.second.pos)));
}
/* If the rec contains an attribute called `__overrides', then
@ -1185,7 +1241,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
else
for (auto & i : attrs)
v.attrs->push_back(Attr(i.first, i.second.e->maybeThunk(state, env), &i.second.pos));
v.attrs->push_back(Attr(i.first, i.second.e->maybeThunk(state, env), ptr(&i.second.pos)));
/* Dynamic attrs apply *after* rec and __overrides. */
for (auto & i : dynamicAttrs) {
@ -1203,11 +1259,11 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
i.valueExpr->setName(nameSym);
/* Keep sorted order so find can catch duplicates */
v.attrs->push_back(Attr(nameSym, i.valueExpr->maybeThunk(state, *dynamicEnv), &i.pos));
v.attrs->push_back(Attr(nameSym, i.valueExpr->maybeThunk(state, *dynamicEnv), ptr(&i.pos)));
v.attrs->sort(); // FIXME: inefficient
}
v.attrs->pos = &pos;
v.attrs->pos = ptr(&pos);
}
@ -1221,7 +1277,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
/* The recursive attributes are evaluated in the new environment,
while the inherited attributes are evaluated in the original
environment. */
size_t displ = 0;
Displacement displ = 0;
for (auto & i : attrs->attrs)
env2.values[displ++] = i.second.e->maybeThunk(state, i.second.inherited ? env : env2);
@ -1262,12 +1318,10 @@ static string showAttrPath(EvalState & state, Env & env, const AttrPath & attrPa
}
unsigned long nrLookups = 0;
void ExprSelect::eval(EvalState & state, Env & env, Value & v)
{
Value vTmp;
Pos * pos2 = 0;
ptr<Pos> pos2(&noPos);
Value * vAttrs = &vTmp;
e->eval(state, env, vTmp);
@ -1275,7 +1329,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
try {
for (auto & i : attrPath) {
nrLookups++;
state.nrLookups++;
Bindings::iterator j;
Symbol name = getName(i, state, env);
if (def) {
@ -1293,13 +1347,13 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
}
vAttrs = j->value;
pos2 = j->pos;
if (state.countCalls && pos2) state.attrSelects[*pos2]++;
if (state.countCalls) state.attrSelects[*pos2]++;
}
state.forceValue(*vAttrs, ( pos2 != NULL ? *pos2 : this->pos ) );
state.forceValue(*vAttrs, (*pos2 != noPos ? *pos2 : this->pos ) );
} catch (Error & e) {
if (pos2 && pos2->file != state.sDerivationNix)
if (*pos2 != noPos && pos2->file != state.sDerivationNix)
addErrorTrace(e, *pos2, "while evaluating the attribute '%1%'",
showAttrPath(state, env, attrPath));
throw;
@ -1340,160 +1394,183 @@ void ExprLambda::eval(EvalState & state, Env & env, Value & v)
}
void ExprApp::eval(EvalState & state, Env & env, Value & v)
{
/* FIXME: vFun prevents GCC from doing tail call optimisation. */
Value vFun;
e1->eval(state, env, vFun);
state.callFunction(vFun, *(e2->maybeThunk(state, env)), v, pos);
}
void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos)
{
/* Figure out the number of arguments still needed. */
size_t argsDone = 0;
Value * primOp = &fun;
while (primOp->isPrimOpApp()) {
argsDone++;
primOp = primOp->primOpApp.left;
}
assert(primOp->isPrimOp());
auto arity = primOp->primOp->arity;
auto argsLeft = arity - argsDone;
if (argsLeft == 1) {
/* We have all the arguments, so call the primop. */
/* Put all the arguments in an array. */
Value * vArgs[arity];
auto n = arity - 1;
vArgs[n--] = &arg;
for (Value * arg = &fun; arg->isPrimOpApp(); arg = arg->primOpApp.left)
vArgs[n--] = arg->primOpApp.right;
/* And call the primop. */
nrPrimOpCalls++;
if (countCalls) primOpCalls[primOp->primOp->name]++;
primOp->primOp->fun(*this, pos, vArgs, v);
} else {
Value * fun2 = allocValue();
*fun2 = fun;
v.mkPrimOpApp(fun2, &arg);
}
}
void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & pos)
void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const Pos & pos)
{
auto trace = evalSettings.traceFunctionCalls ? std::make_unique<FunctionCallTrace>(pos) : nullptr;
forceValue(fun, pos);
if (fun.isPrimOp() || fun.isPrimOpApp()) {
callPrimOp(fun, arg, v, pos);
return;
}
Value vCur(fun);
if (fun.type() == nAttrs) {
auto found = fun.attrs->find(sFunctor);
if (found != fun.attrs->end()) {
/* fun may be allocated on the stack of the calling function,
* but for functors we may keep a reference, so heap-allocate
* a copy and use that instead.
*/
auto & fun2 = *allocValue();
fun2 = fun;
/* !!! Should we use the attr pos here? */
Value v2;
callFunction(*found->value, fun2, v2, pos);
return callFunction(v2, arg, v, pos);
}
}
auto makeAppChain = [&]()
{
vRes = vCur;
for (size_t i = 0; i < nrArgs; ++i) {
auto fun2 = allocValue();
*fun2 = vRes;
vRes.mkPrimOpApp(fun2, args[i]);
}
};
if (!fun.isLambda()) {
throwTypeError(
pos,
"attempt to call something which is not a function but %1%",
showType(fun).c_str(),
fakeEnv(1), 0);
}
Attr * functor;
ExprLambda & lambda(*fun.lambda.fun);
while (nrArgs > 0) {
auto size =
(lambda.arg.empty() ? 0 : 1) +
(lambda.matchAttrs ? lambda.formals->formals.size() : 0);
Env & env2(allocEnv(size));
env2.up = fun.lambda.env;
if (vCur.isLambda()) {
size_t displ = 0;
ExprLambda & lambda(*vCur.lambda.fun);
if (!lambda.matchAttrs){
env2.values[displ++] = &arg;
}
else {
forceAttrs(arg, pos);
auto size =
(lambda.arg.empty() ? 0 : 1) +
(lambda.hasFormals() ? lambda.formals->formals.size() : 0);
Env & env2(allocEnv(size));
env2.up = vCur.lambda.env;
if (!lambda.arg.empty())
env2.values[displ++] = &arg;
Displacement displ = 0;
/* For each formal argument, get the actual argument. If
there is no matching actual argument but the formal
argument has a default, use the default. */
size_t attrsUsed = 0;
for (auto & i : lambda.formals->formals) {
Bindings::iterator j = arg.attrs->find(i.name);
if (j == arg.attrs->end()) {
if (!i.def)
throwTypeError(
pos,
"%1% called without required argument '%2%'",
lambda,
i.name,
*fun.lambda.env, &lambda);
env2.values[displ++] = i.def->maybeThunk(*this, env2);
if (!lambda.hasFormals())
env2.values[displ++] = args[0];
else {
forceAttrs(*args[0], pos);
if (!lambda.arg.empty())
env2.values[displ++] = args[0];
/* For each formal argument, get the actual argument. If
there is no matching actual argument but the formal
argument has a default, use the default. */
size_t attrsUsed = 0;
for (auto & i : lambda.formals->formals) {
auto j = args[0]->attrs->get(i.name);
if (!j) {
if (!i.def) throwTypeError(pos, "%1% called without required argument '%2%'",
lambda, i.name, *fun.lambda.env, &lambda);
env2.values[displ++] = i.def->maybeThunk(*this, env2);
} else {
attrsUsed++;
env2.values[displ++] = j->value;
}
}
/* Check that each actual argument is listed as a formal
argument (unless the attribute match specifies a `...'). */
if (!lambda.formals->ellipsis && attrsUsed != args[0]->attrs->size()) {
/* Nope, so show the first unexpected argument to the
user. */
for (auto & i : *args[0]->attrs)
if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end())
throwTypeError(pos, "%1% called with unexpected argument '%2%'", lambda, i.name);
abort(); // can't happen
}
}
nrFunctionCalls++;
if (countCalls) incrFunctionCall(&lambda);
/* Evaluate the body. */
try {
lambda.body->eval(*this, env2, vCur);
} catch (Error & e) {
if (loggerSettings.showTrace.get()) {
addErrorTrace(e, lambda.pos, "while evaluating %s",
(lambda.name.set()
? "'" + (string) lambda.name + "'"
: "anonymous lambda"));
addErrorTrace(e, pos, "from call site%s", "");
}
throw;
}
nrArgs--;
args += 1;
}
else if (vCur.isPrimOp()) {
size_t argsLeft = vCur.primOp->arity;
if (nrArgs < argsLeft) {
/* We don't have enough arguments, so create a tPrimOpApp chain. */
makeAppChain();
return;
} else {
attrsUsed++;
env2.values[displ++] = j->value;
/* We have all the arguments, so call the primop. */
nrPrimOpCalls++;
if (countCalls) primOpCalls[vCur.primOp->name]++;
vCur.primOp->fun(*this, pos, args, vCur);
nrArgs -= argsLeft;
args += argsLeft;
}
}
else if (vCur.isPrimOpApp()) {
/* Figure out the number of arguments still needed. */
size_t argsDone = 0;
Value * primOp = &vCur;
while (primOp->isPrimOpApp()) {
argsDone++;
primOp = primOp->primOpApp.left;
}
assert(primOp->isPrimOp());
auto arity = primOp->primOp->arity;
auto argsLeft = arity - argsDone;
/* Check that each actual argument is listed as a formal
argument (unless the attribute match specifies a `...'). */
if (!lambda.formals->ellipsis && attrsUsed != arg.attrs->size()) {
/* Nope, so show the first unexpected argument to the
user. */
for (auto & i : *arg.attrs)
if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end())
throwTypeError(pos,
"%1% called with unexpected argument '%2%'",
lambda,
i.name,
*fun.lambda.env, &lambda);
abort(); // can't happen
if (nrArgs < argsLeft) {
/* We still don't have enough arguments, so extend the tPrimOpApp chain. */
makeAppChain();
return;
} else {
/* We have all the arguments, so call the primop with
the previous and new arguments. */
Value * vArgs[arity];
auto n = argsDone;
for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left)
vArgs[--n] = arg->primOpApp.right;
for (size_t i = 0; i < argsLeft; ++i)
vArgs[argsDone + i] = args[i];
nrPrimOpCalls++;
if (countCalls) primOpCalls[primOp->primOp->name]++;
primOp->primOp->fun(*this, pos, vArgs, vCur);
nrArgs -= argsLeft;
args += argsLeft;
}
}
else if (vCur.type() == nAttrs && (functor = vCur.attrs->get(sFunctor))) {
/* 'vCur' may be allocated on the stack of the calling
function, but for functors we may keep a reference, so
heap-allocate a copy and use that instead. */
Value * args2[] = {allocValue(), args[0]};
*args2[0] = vCur;
/* !!! Should we use the attr pos here? */
callFunction(*functor->value, 2, args2, vCur, pos);
nrArgs--;
args++;
}
else
throwTypeError(pos, "attempt to call something which is not a function but %1%", vCur);
}
nrFunctionCalls++;
if (countCalls) incrFunctionCall(&lambda);
vRes = vCur;
}
/* Evaluate the body. This is conditional on showTrace, because
catching exceptions makes this function not tail-recursive. */
if (loggerSettings.showTrace.get())
try {
lambda.body->eval(*this, env2, v);
} catch (Error & e) {
addErrorTrace(e, lambda.pos, "while evaluating %s",
(lambda.name.set()
? "'" + (string) lambda.name + "'"
: "anonymous lambda"));
addErrorTrace(e, pos, "from call site%s", "");
throw;
}
else
fun.lambda.fun->body->eval(*this, env2, v);
void ExprCall::eval(EvalState & state, Env & env, Value & v)
{
Value vFun;
fun->eval(state, env, vFun);
Value * vArgs[args.size()];
for (size_t i = 0; i < args.size(); ++i)
vArgs[i] = args[i]->maybeThunk(state, env);
state.callFunction(vFun, args.size(), vArgs, v, pos);
}
@ -1519,7 +1596,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
}
}
if (!fun.isLambda() || !fun.lambda.fun->matchAttrs) {
if (!fun.isLambda() || !fun.lambda.fun->hasFormals()) {
res = fun;
return;
}
@ -1722,7 +1799,6 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
and none of the strings are allowed to have contexts. */
if (first) {
firstType = vTmp.type();
first = false;
}
if (firstType == nInt) {
@ -1744,7 +1820,12 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
} else
throwEvalError(pos, "cannot add %1% to a float", showType(vTmp), env, this);
} else
s << state.coerceToString(pos, vTmp, context, false, firstType == nString);
/* skip canonization of first path, which would only be not
canonized in the first place if it's coming from a ./${foo} type
path */
s << state.coerceToString(pos, vTmp, context, false, firstType == nString, !first);
first = false;
}
if (firstType == nInt)
@ -1763,7 +1844,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
void ExprPos::eval(EvalState & state, Env & env, Value & v)
{
state.mkPos(v, &pos);
state.mkPos(v, ptr(&pos));
}
@ -1935,7 +2016,7 @@ std::optional<string> EvalState::tryAttrsToString(const Pos & pos, Value & v,
}
string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
bool coerceMore, bool copyToStore)
bool coerceMore, bool copyToStore, bool canonicalizePath)
{
forceValue(v, pos);
@ -1947,7 +2028,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
}
if (v.type() == nPath) {
Path path(canonPath(v.path));
Path path(canonicalizePath ? canonPath(v.path) : v.path);
return copyToStore ? copyPathToStore(context, path) : path;
}
@ -2010,6 +2091,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path)
? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first
: store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair);
dstPath = store->printStorePath(p);
allowPath(p);
srcToStore.insert_or_assign(path, std::move(p));
printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath);
}

59
src/libexpr/eval.hh
View file

@ -5,6 +5,7 @@
#include "nixexpr.hh"
#include "symbol-table.hh"
#include "config.hh"
#include "experimental-features.hh"
#include <map>
#include <optional>
@ -98,8 +99,14 @@ public:
Value vEmptySet;
/* Store used to materialise .drv files. */
const ref<Store> store;
/* Store used to build stuff. */
const ref<Store> buildStore;
RootValue vCallFlake = nullptr;
RootValue vImportedDrvToDerivation = nullptr;
private:
SrcToStore srcToStore;
@ -132,13 +139,31 @@ private:
public:
EvalState(const Strings & _searchPath, ref<Store> store);
EvalState(
const Strings & _searchPath,
ref<Store> store,
std::shared_ptr<Store> buildStore = nullptr);
~EvalState();
void requireExperimentalFeatureOnEvaluation(
const ExperimentalFeature &,
const std::string_view fName,
const Pos & pos
);
void addToSearchPath(const string & s);
SearchPath getSearchPath() { return searchPath; }
/* Allow access to a path. */
void allowPath(const Path & path);
/* Allow access to a store path. Note that this gets remapped to
the real store path if `store` is a chroot store. */
void allowPath(const StorePath & storePath);
/* Check whether access to a path is allowed and throw an error if
not. Otherwise return the canonicalised path. */
Path checkSourcePath(const Path & path);
void checkURI(const std::string & uri);
@ -167,6 +192,14 @@ public:
trivial (i.e. doesn't require arbitrary computation). */
void evalFile(const Path & path, Value & v, bool mustBeTrivial = false);
/* Like `cacheFile`, but with an already parsed expression. */
void cacheFile(
const Path & path,
const Path & resolvedPath,
Expr * e,
Value & v,
bool mustBeTrivial = false);
void resetFileCache();
/* Look up a file in the search path. */
@ -221,7 +254,8 @@ public:
booleans and lists to a string. If `copyToStore' is set,
referenced paths are copied to the Nix store as a side effect. */
string coerceToString(const Pos & pos, Value & v, PathSet & context,
bool coerceMore = false, bool copyToStore = true);
bool coerceMore = false, bool copyToStore = true,
bool canonicalizePath = true);
string copyPathToStore(PathSet & context, const Path & path);
@ -247,6 +281,8 @@ private:
Value * addConstant(const string & name, Value & v);
void addConstant(const string & name, Value * v);
Value * addPrimOp(const string & name,
size_t arity, PrimOpFun primOp);
@ -286,8 +322,14 @@ public:
bool isFunctor(Value & fun);
void callFunction(Value & fun, Value & arg, Value & v, const Pos & pos);
void callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos);
// FIXME: use std::span
void callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const Pos & pos);
void callFunction(Value & fun, Value & arg, Value & vRes, const Pos & pos)
{
Value * args[] = {&arg};
callFunction(fun, 1, args, vRes, pos);
}
/* Automatically call a function for which each argument has a
default value or has a binding in the `args' map. */
@ -305,7 +347,7 @@ public:
void mkList(Value & v, size_t length);
void mkAttrs(Value & v, size_t capacity);
void mkThunk_(Value & v, Expr * expr);
void mkPos(Value & v, Pos * pos);
void mkPos(Value & v, ptr<Pos> pos);
void concatLists(Value & v, size_t nrLists, Value * * lists, const Pos & pos);
@ -320,8 +362,10 @@ private:
unsigned long nrValuesInEnvs = 0;
unsigned long nrValues = 0;
unsigned long nrListElems = 0;
unsigned long nrLookups = 0;
unsigned long nrAttrsets = 0;
unsigned long nrAttrsInAttrsets = 0;
unsigned long nrAvoided = 0;
unsigned long nrOpUpdates = 0;
unsigned long nrOpUpdateValuesCopied = 0;
unsigned long nrListConcats = 0;
@ -343,6 +387,11 @@ private:
friend struct ExprOpUpdate;
friend struct ExprOpConcatLists;
friend struct ExprVar;
friend struct ExprString;
friend struct ExprInt;
friend struct ExprFloat;
friend struct ExprPath;
friend struct ExprSelect;
friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v);
friend void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v);

29
src/libexpr/flake/config.cc
View file

@ -1,4 +1,5 @@
#include "flake.hh"
#include "globals.hh"
#include <nlohmann/json.hpp>
@ -22,12 +23,14 @@ static TrustedList readTrustedList()
static void writeTrustedList(const TrustedList & trustedList)
{
writeFile(trustedListPath(), nlohmann::json(trustedList).dump());
auto path = trustedListPath();
createDirs(dirOf(path));
writeFile(path, nlohmann::json(trustedList).dump());
}
void ConfigFile::apply()
{
std::set<std::string> whitelist{"bash-prompt", "bash-prompt-suffix"};
std::set<std::string> whitelist{"bash-prompt", "bash-prompt-suffix", "flake-registry"};
for (auto & [name, value] : settings) {
@ -50,21 +53,19 @@ void ConfigFile::apply()
auto trustedList = readTrustedList();
bool trusted = false;
if (auto saved = get(get(trustedList, name).value_or(std::map<std::string, bool>()), valueS)) {
if (nix::settings.acceptFlakeConfig){
trusted = true;
} else if (auto saved = get(get(trustedList, name).value_or(std::map<std::string, bool>()), valueS)) {
trusted = *saved;
warn("Using saved setting for '%s = %s' from ~/.local/share/nix/trusted-settings.json.", name,valueS);
} else {
// FIXME: filter ANSI escapes, newlines, \r, etc.
if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) != 'y') {
if (std::tolower(logger->ask("do you want to permanently mark this value as untrusted (y/N)?").value_or('n')) == 'y') {
trustedList[name][valueS] = false;
writeTrustedList(trustedList);
}
} else {
if (std::tolower(logger->ask("do you want to permanently mark this value as trusted (y/N)?").value_or('n')) == 'y') {
trustedList[name][valueS] = trusted = true;
writeTrustedList(trustedList);
}
if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) == 'y') {
trusted = true;
}
if (std::tolower(logger->ask(fmt("do you want to permanently mark this value as %s (y/N)?", trusted ? "trusted": "untrusted" )).value_or('n')) == 'y') {
trustedList[name][valueS] = trusted;
writeTrustedList(trustedList);
}
}

181
src/libexpr/flake/flake.cc
View file

@ -1,4 +1,5 @@
#include "flake.hh"
#include "eval.hh"
#include "lockfile.hh"
#include "primops.hh"
#include "eval-inline.hh"
@ -63,8 +64,7 @@ static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree(
debug("got tree '%s' from '%s'",
state.store->printStorePath(tree.storePath), lockedRef);
if (state.allowedPaths)
state.allowedPaths->insert(tree.actualPath);
state.allowPath(tree.storePath);
assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store));
@ -88,10 +88,12 @@ static void expectType(EvalState & state, ValueType type,
}
static std::map<FlakeId, FlakeInput> parseFlakeInputs(
EvalState & state, Value * value, const Pos & pos);
EvalState & state, Value * value, const Pos & pos,
const std::optional<Path> & baseDir);
static FlakeInput parseFlakeInput(EvalState & state,
const std::string & inputName, Value * value, const Pos & pos)
const std::string & inputName, Value * value, const Pos & pos,
const std::optional<Path> & baseDir)
{
expectType(state, nAttrs, *value, pos);
@ -115,7 +117,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
expectType(state, nBool, *attr.value, *attr.pos);
input.isFlake = attr.value->boolean;
} else if (attr.name == sInputs) {
input.overrides = parseFlakeInputs(state, attr.value, *attr.pos);
input.overrides = parseFlakeInputs(state, attr.value, *attr.pos, baseDir);
} else if (attr.name == sFollows) {
expectType(state, nString, *attr.value, *attr.pos);
input.follows = parseInputPath(attr.value->string.s);
@ -153,7 +155,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
if (!attrs.empty())
throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, pos);
if (url)
input.ref = parseFlakeRef(*url, {}, true);
input.ref = parseFlakeRef(*url, baseDir, true);
}
if (!input.follows && !input.ref)
@ -163,7 +165,8 @@ static FlakeInput parseFlakeInput(EvalState & state,
}
static std::map<FlakeId, FlakeInput> parseFlakeInputs(
EvalState & state, Value * value, const Pos & pos)
EvalState & state, Value * value, const Pos & pos,
const std::optional<Path> & baseDir)
{
std::map<FlakeId, FlakeInput> inputs;
@ -174,7 +177,8 @@ static std::map<FlakeId, FlakeInput> parseFlakeInputs(
parseFlakeInput(state,
inputAttr.name,
inputAttr.value,
*inputAttr.pos));
*inputAttr.pos,
baseDir));
}
return inputs;
@ -190,7 +194,8 @@ static Flake getFlake(
state, originalRef, allowLookup, flakeCache);
// Guard against symlink attacks.
auto flakeFile = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir + "/flake.nix");
auto flakeDir = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir);
auto flakeFile = canonPath(flakeDir + "/flake.nix");
if (!isInDir(flakeFile, sourceInfo.actualPath))
throw Error("'flake.nix' file of flake '%s' escapes from '%s'",
lockedRef, state.store->printStorePath(sourceInfo.storePath));
@ -218,14 +223,14 @@ static Flake getFlake(
auto sInputs = state.symbols.create("inputs");
if (auto inputs = vInfo.attrs->get(sInputs))
flake.inputs = parseFlakeInputs(state, inputs->value, *inputs->pos);
flake.inputs = parseFlakeInputs(state, inputs->value, *inputs->pos, flakeDir);
auto sOutputs = state.symbols.create("outputs");
if (auto outputs = vInfo.attrs->get(sOutputs)) {
expectType(state, nFunction, *outputs->value, *outputs->pos);
if (outputs->value->isLambda() && outputs->value->lambda.fun->matchAttrs) {
if (outputs->value->isLambda() && outputs->value->lambda.fun->hasFormals()) {
for (auto & formal : outputs->value->lambda.fun->formals->formals) {
if (formal.name != state.sSelf)
flake.inputs.emplace(formal.name, FlakeInput {
@ -292,11 +297,18 @@ LockedFlake lockFlake(
const FlakeRef & topRef,
const LockFlags & lockFlags)
{
settings.requireExperimentalFeature("flakes");
settings.requireExperimentalFeature(Xp::Flakes);
FlakeCache flakeCache;
auto flake = getFlake(state, topRef, lockFlags.useRegistries, flakeCache);
auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries);
auto flake = getFlake(state, topRef, useRegistries, flakeCache);
if (lockFlags.applyNixConfig) {
flake.config.apply();
state.store->setOptions();
}
try {
@ -317,25 +329,38 @@ LockedFlake lockFlake(
std::vector<FlakeRef> parents;
struct LockParent {
/* The path to this parent. */
InputPath path;
/* Whether we are currently inside a top-level lockfile
(inputs absolute) or subordinate lockfile (inputs
relative). */
bool absolute;
};
std::function<void(
const FlakeInputs & flakeInputs,
std::shared_ptr<Node> node,
const InputPath & inputPathPrefix,
std::shared_ptr<const Node> oldNode)>
std::shared_ptr<const Node> oldNode,
const LockParent & parent,
const Path & parentPath)>
computeLocks;
computeLocks = [&](
const FlakeInputs & flakeInputs,
std::shared_ptr<Node> node,
const InputPath & inputPathPrefix,
std::shared_ptr<const Node> oldNode)
std::shared_ptr<const Node> oldNode,
const LockParent & parent,
const Path & parentPath)
{
debug("computing lock file node '%s'", printInputPath(inputPathPrefix));
/* Get the overrides (i.e. attributes of the form
'inputs.nixops.inputs.nixpkgs.url = ...'). */
// FIXME: check this
for (auto & [id, input] : flake.inputs) {
for (auto & [id, input] : flakeInputs) {
for (auto & [idOverride, inputOverride] : input.overrides) {
auto inputPath(inputPathPrefix);
inputPath.push_back(id);
@ -359,22 +384,31 @@ LockedFlake lockFlake(
ancestors? */
auto i = overrides.find(inputPath);
bool hasOverride = i != overrides.end();
if (hasOverride) overridesUsed.insert(inputPath);
if (hasOverride) {
overridesUsed.insert(inputPath);
// Respect the “flakeness” of the input even if we
// override it
i->second.isFlake = input2.isFlake;
}
auto & input = hasOverride ? i->second : input2;
/* Resolve 'follows' later (since it may refer to an input
path we haven't processed yet. */
if (input.follows) {
InputPath target;
if (hasOverride || input.absolute)
/* 'follows' from an override is relative to the
root of the graph. */
if (parent.absolute && !hasOverride) {
target = *input.follows;
else {
/* Otherwise, it's relative to the current flake. */
target = inputPathPrefix;
} else {
if (hasOverride) {
target = inputPathPrefix;
target.pop_back();
} else
target = parent.path;
for (auto & i : *input.follows) target.push_back(i);
}
debug("input '%s' follows '%s'", inputPathS, printInputPath(target));
node->inputs.insert_or_assign(id, target);
continue;
@ -412,22 +446,18 @@ LockedFlake lockFlake(
update it. */
auto lb = lockFlags.inputUpdates.lower_bound(inputPath);
auto hasChildUpdate =
auto mustRefetch =
lb != lockFlags.inputUpdates.end()
&& lb->size() > inputPath.size()
&& std::equal(inputPath.begin(), inputPath.end(), lb->begin());
if (hasChildUpdate) {
auto inputFlake = getFlake(
state, oldLock->lockedRef, false, flakeCache);
computeLocks(inputFlake.inputs, childNode, inputPath, oldLock);
} else {
FlakeInputs fakeInputs;
if (!mustRefetch) {
/* No need to fetch this flake, we can be
lazy. However there may be new overrides on the
inputs of this flake, so we need to check
those. */
FlakeInputs fakeInputs;
for (auto & i : oldLock->inputs) {
if (auto lockedNode = std::get_if<0>(&i.second)) {
fakeInputs.emplace(i.first, FlakeInput {
@ -435,16 +465,28 @@ LockedFlake lockFlake(
.isFlake = (*lockedNode)->isFlake,
});
} else if (auto follows = std::get_if<1>(&i.second)) {
auto o = input.overrides.find(i.first);
// If the override disappeared, we have to refetch the flake,
// since some of the inputs may not be present in the lockfile.
if (o == input.overrides.end()) {
mustRefetch = true;
// There's no point populating the rest of the fake inputs,
// since we'll refetch the flake anyways.
break;
}
fakeInputs.emplace(i.first, FlakeInput {
.follows = *follows,
.absolute = true
});
}
}
computeLocks(fakeInputs, childNode, inputPath, oldLock);
}
computeLocks(
mustRefetch
? getFlake(state, oldLock->lockedRef, false, flakeCache).inputs
: fakeInputs,
childNode, inputPath, oldLock, parent, parentPath);
} else {
/* We need to create a new lock file entry. So fetch
this input. */
@ -454,7 +496,15 @@ LockedFlake lockFlake(
throw Error("cannot update flake input '%s' in pure mode", inputPathS);
if (input.isFlake) {
auto inputFlake = getFlake(state, *input.ref, lockFlags.useRegistries, flakeCache);
Path localPath = parentPath;
FlakeRef localRef = *input.ref;
// If this input is a path, recurse it down.
// This allows us to resolve path inputs relative to the current flake.
if (localRef.input.getType() == "path")
localPath = absPath(*input.ref->input.getSourcePath(), parentPath);
auto inputFlake = getFlake(state, localRef, useRegistries, flakeCache);
/* Note: in case of an --override-input, we use
the *original* ref (input2.ref) for the
@ -475,6 +525,13 @@ LockedFlake lockFlake(
parents.push_back(*input.ref);
Finally cleanup([&]() { parents.pop_back(); });
// Follows paths from existing inputs in the top-level lockfile are absolute,
// whereas paths in subordinate lockfiles are relative to those lockfiles.
LockParent newParent {
.path = inputPath,
.absolute = oldLock ? true : false
};
/* Recursively process the inputs of this
flake. Also, unless we already have this flake
in the top-level lock file, use this flake's
@ -484,12 +541,13 @@ LockedFlake lockFlake(
oldLock
? std::dynamic_pointer_cast<const Node>(oldLock)
: LockFile::read(
inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root);
inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root,
newParent, localPath);
}
else {
auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree(
state, *input.ref, lockFlags.useRegistries, flakeCache);
state, *input.ref, useRegistries, flakeCache);
node->inputs.insert_or_assign(id,
std::make_shared<LockedNode>(lockedRef, *input.ref, false));
}
@ -502,9 +560,17 @@ LockedFlake lockFlake(
}
};
LockParent parent {
.path = {},
.absolute = true
};
// Bring in the current ref for relative path resolution if we have it
auto parentPath = canonPath(flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir);
computeLocks(
flake.inputs, newLockFile.root, {},
lockFlags.recreateLockFile ? nullptr : oldLockFile.root);
lockFlags.recreateLockFile ? nullptr : oldLockFile.root, parent, parentPath);
for (auto & i : lockFlags.inputOverrides)
if (!overridesUsed.count(i.first))
@ -554,8 +620,8 @@ LockedFlake lockFlake(
topRef.input.markChangedFile(
(topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock",
lockFlags.commitLockFile
? std::optional<std::string>(fmt("%s: %s\n\nFlake input changes:\n\n%s",
relPath, lockFileExists ? "Update" : "Add", diff))
? std::optional<std::string>(fmt("%s: %s\n\nFlake lock file changes:\n\n%s",
relPath, lockFileExists ? "Update" : "Add", filterANSIEscapes(diff, true)))
: std::nullopt);
/* Rewriting the lockfile changed the top-level
@ -563,7 +629,7 @@ LockedFlake lockFlake(
also just clear the 'rev' field... */
auto prevLockedRef = flake.lockedRef;
FlakeCache dummyCache;
flake = getFlake(state, topRef, lockFlags.useRegistries, dummyCache);
flake = getFlake(state, topRef, useRegistries, dummyCache);
if (lockFlags.commitLockFile &&
flake.lockedRef.input.getRev() &&
@ -580,8 +646,10 @@ LockedFlake lockFlake(
}
} else
throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef);
} else
} else {
warn("not writing modified lock file of flake '%s':\n%s", topRef, chomp(diff));
flake.forceDirty = true;
}
}
return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) };
@ -604,26 +672,32 @@ void callFlake(EvalState & state,
mkString(*vLocks, lockedFlake.lockFile.to_string());
emitTreeAttrs(state, *lockedFlake.flake.sourceInfo, lockedFlake.flake.lockedRef.input, *vRootSrc);
emitTreeAttrs(
state,
*lockedFlake.flake.sourceInfo,
lockedFlake.flake.lockedRef.input,
*vRootSrc,
false,
lockedFlake.flake.forceDirty);
mkString(*vRootSubdir, lockedFlake.flake.lockedRef.subdir);
static RootValue vCallFlake = nullptr;
if (!vCallFlake) {
vCallFlake = allocRootValue(state.allocValue());
if (!state.vCallFlake) {
state.vCallFlake = allocRootValue(state.allocValue());
state.eval(state.parseExprFromString(
#include "call-flake.nix.gen.hh"
, "/"), **vCallFlake);
, "/"), **state.vCallFlake);
}
state.callFunction(**vCallFlake, *vLocks, *vTmp1, noPos);
state.callFunction(**state.vCallFlake, *vLocks, *vTmp1, noPos);
state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos);
state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos);
}
static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
state.requireExperimentalFeatureOnEvaluation(Xp::Flakes, "builtins.getFlake", pos);
auto flakeRefS = state.forceStringNoCtx(*args[0], pos);
auto flakeRef = parseFlakeRef(flakeRefS, {}, true);
if (evalSettings.pureEval && !flakeRef.input.isImmutable())
@ -633,13 +707,13 @@ static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Va
lockFlake(state, flakeRef,
LockFlags {
.updateLockFile = false,
.useRegistries = !evalSettings.pureEval,
.useRegistries = !evalSettings.pureEval && settings.useRegistries,
.allowMutable = !evalSettings.pureEval,
}),
v);
}
static RegisterPrimOp r2("__getFlake", 1, prim_getFlake, "flakes");
static RegisterPrimOp r2("__getFlake", 1, prim_getFlake);
}
@ -649,8 +723,9 @@ Fingerprint LockedFlake::getFingerprint() const
// and we haven't changed it, then it's sufficient to use
// flake.sourceInfo.storePath for the fingerprint.
return hashString(htSHA256,
fmt("%s;%d;%d;%s",
fmt("%s;%s;%d;%d;%s",
flake.sourceInfo->storePath.to_string(),
flake.lockedRef.subdir,
flake.lockedRef.input.getRevCount().value_or(0),
flake.lockedRef.input.getLastModified().value_or(0),
lockFile));

18
src/libexpr/flake/flake.hh
View file

@ -43,7 +43,6 @@ struct FlakeInput
std::optional<FlakeRef> ref;
bool isFlake = true; // true = process flake to get outputs, false = (fetched) static source path
std::optional<InputPath> follows;
bool absolute = false; // whether 'follows' is relative to the flake root
FlakeInputs overrides;
};
@ -59,9 +58,10 @@ struct ConfigFile
/* The contents of a flake.nix file. */
struct Flake
{
FlakeRef originalRef; // the original flake specification (by the user)
FlakeRef resolvedRef; // registry references and caching resolved to the specific underlying flake
FlakeRef lockedRef; // the specific local store result of invoking the fetcher
FlakeRef originalRef; // the original flake specification (by the user)
FlakeRef resolvedRef; // registry references and caching resolved to the specific underlying flake
FlakeRef lockedRef; // the specific local store result of invoking the fetcher
bool forceDirty = false; // pretend that 'lockedRef' is dirty
std::optional<std::string> description;
std::shared_ptr<const fetchers::Tree> sourceInfo;
FlakeInputs inputs;
@ -102,7 +102,11 @@ struct LockFlags
/* Whether to use the registries to lookup indirect flake
references like 'nixpkgs'. */
bool useRegistries = true;
std::optional<bool> useRegistries = std::nullopt;
/* Whether to apply flake's nixConfig attribute to the configuration */
bool applyNixConfig = false;
/* Whether mutable flake references (i.e. those without a Git
revision or similar) without a corresponding lock are
@ -137,6 +141,8 @@ void emitTreeAttrs(
EvalState & state,
const fetchers::Tree & tree,
const fetchers::Input & input,
Value & v, bool emptyRevFallback = false);
Value & v,
bool emptyRevFallback = false,
bool forceDirty = false);
}

6
src/libexpr/flake/flakeref.cc
View file

@ -172,8 +172,12 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
auto parsedURL = parseURL(url);
std::string fragment;
std::swap(fragment, parsedURL.fragment);
auto input = Input::fromURL(parsedURL);
input.parent = baseDir;
return std::make_pair(
FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
FlakeRef(std::move(input), get(parsedURL.query, "dir").value_or("")),
fragment);
}
}

21
src/libexpr/flake/lockfile.cc
View file

@ -2,6 +2,8 @@
#include "store-api.hh"
#include "url-parts.hh"
#include <iomanip>
#include <nlohmann/json.hpp>
namespace nix::flake {
@ -268,10 +270,20 @@ std::map<InputPath, Node::Edge> LockFile::getAllInputs() const
return res;
}
static std::string describe(const FlakeRef & flakeRef)
{
auto s = fmt("'%s'", flakeRef.to_string());
if (auto lastModified = flakeRef.input.getLastModified())
s += fmt(" (%s)", std::put_time(std::gmtime(&*lastModified), "%Y-%m-%d"));
return s;
}
std::ostream & operator <<(std::ostream & stream, const Node::Edge & edge)
{
if (auto node = std::get_if<0>(&edge))
stream << "'" << (*node)->lockedRef << "'";
stream << describe((*node)->lockedRef);
else if (auto follows = std::get_if<1>(&edge))
stream << fmt("follows '%s'", printInputPath(*follows));
return stream;
@ -299,14 +311,15 @@ std::string LockFile::diff(const LockFile & oldLocks, const LockFile & newLocks)
while (i != oldFlat.end() || j != newFlat.end()) {
if (j != newFlat.end() && (i == oldFlat.end() || i->first > j->first)) {
res += fmt("* Added '%s': %s\n", printInputPath(j->first), j->second);
res += fmt("• " ANSI_GREEN "Added input '%s':" ANSI_NORMAL "\n %s\n",
printInputPath(j->first), j->second);
++j;
} else if (i != oldFlat.end() && (j == newFlat.end() || i->first < j->first)) {
res += fmt("* Removed '%s'\n", printInputPath(i->first));
res += fmt("• " ANSI_RED "Removed input '%s'" ANSI_NORMAL "\n", printInputPath(i->first));
++i;
} else {
if (!equals(i->second, j->second)) {
res += fmt("* Updated '%s': %s -> %s\n",
res += fmt("• " ANSI_BOLD "Updated input '%s':" ANSI_NORMAL "\n %s\n → %s\n",
printInputPath(i->first),
i->second,
j->second);

101
src/libexpr/lexer.l
View file

@ -9,6 +9,9 @@
%s DEFAULT
%x STRING
%x IND_STRING
%x INPATH
%x INPATH_SLASH
%x PATH_START
%{
@ -25,6 +28,8 @@ using namespace nix;
namespace nix {
// backup to recover from yyless(0)
YYLTYPE prev_yylloc;
static void initLoc(YYLTYPE * loc)
{
@ -35,14 +40,18 @@ static void initLoc(YYLTYPE * loc)
static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
{
prev_yylloc = *loc;
loc->first_line = loc->last_line;
loc->first_column = loc->last_column;
while (len--) {
for (size_t i = 0; i < len; i++) {
switch (*s++) {
case '\r':
if (*s == '\n') /* cr/lf */
if (*s == '\n') { /* cr/lf */
i++;
s++;
}
/* fall through */
case '\n':
++loc->last_line;
@ -55,6 +64,7 @@ static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
}
// FIXME: optimize
static Expr * unescapeStr(SymbolTable & symbols, const char * s, size_t length)
{
string t;
@ -95,9 +105,12 @@ ANY .|\n
ID [a-zA-Z\_][a-zA-Z0-9\_\'\-]*
INT [0-9]+
FLOAT (([1-9][0-9]*\.[0-9]*)|(0?\.[0-9]+))([Ee][+-]?[0-9]+)?
PATH [a-zA-Z0-9\.\_\-\+]*(\/[a-zA-Z0-9\.\_\-\+]+)+\/?
HPATH \~(\/[a-zA-Z0-9\.\_\-\+]+)+\/?
SPATH \<[a-zA-Z0-9\.\_\-\+]+(\/[a-zA-Z0-9\.\_\-\+]+)*\>
PATH_CHAR [a-zA-Z0-9\.\_\-\+]
PATH {PATH_CHAR}*(\/{PATH_CHAR}+)+\/?
PATH_SEG {PATH_CHAR}*\/
HPATH \~(\/{PATH_CHAR}+)+\/?
HPATH_START \~\/
SPATH \<{PATH_CHAR}+(\/{PATH_CHAR}+)*\>
URI [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+
@ -198,17 +211,75 @@ or { return OR_KW; }
return IND_STR;
}
{PATH_SEG}\$\{ |
{HPATH_START}\$\{ {
PUSH_STATE(PATH_START);
yyless(0);
*yylloc = prev_yylloc;
}
<PATH_START>{PATH_SEG} {
POP_STATE();
PUSH_STATE(INPATH_SLASH);
yylval->path = strdup(yytext);
return PATH;
}
<PATH_START>{HPATH_START} {
POP_STATE();
PUSH_STATE(INPATH_SLASH);
yylval->path = strdup(yytext);
return HPATH;
}
{PATH} {
if (yytext[yyleng-1] == '/')
PUSH_STATE(INPATH_SLASH);
else
PUSH_STATE(INPATH);
yylval->path = strdup(yytext);
return PATH;
}
{HPATH} {
if (yytext[yyleng-1] == '/')
PUSH_STATE(INPATH_SLASH);
else
PUSH_STATE(INPATH);
yylval->path = strdup(yytext);
return HPATH;
}
<INPATH,INPATH_SLASH>\$\{ {
POP_STATE();
PUSH_STATE(INPATH);
PUSH_STATE(DEFAULT);
return DOLLAR_CURLY;
}
<INPATH,INPATH_SLASH>{PATH}|{PATH_SEG}|{PATH_CHAR}+ {
POP_STATE();
if (yytext[yyleng-1] == '/')
PUSH_STATE(INPATH_SLASH);
else
PUSH_STATE(INPATH);
yylval->e = new ExprString(data->symbols.create(string(yytext)));
return STR;
}
<INPATH>{ANY} |
<INPATH><<EOF>> {
/* if we encounter a non-path character we inform the parser that the path has
ended with a PATH_END token and re-parse this character in the default
context (it may be ')', ';', or something of that sort) */
POP_STATE();
yyless(0);
*yylloc = prev_yylloc;
return PATH_END;
}
<INPATH_SLASH>{ANY} |
<INPATH_SLASH><<EOF>> {
throw ParseError("path has a trailing slash");
}
{PATH} { if (yytext[yyleng-1] == '/')
throw ParseError("path '%s' has a trailing slash", yytext);
yylval->path = strdup(yytext);
return PATH;
}
{HPATH} { if (yytext[yyleng-1] == '/')
throw ParseError("path '%s' has a trailing slash", yytext);
yylval->path = strdup(yytext);
return HPATH;
}
{SPATH} { yylval->path = strdup(yytext); return SPATH; }
{URI} { yylval->uri = strdup(yytext); return URI; }

6
src/libexpr/local.mk
View file

@ -15,8 +15,8 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib
libexpr_LIBS = libutil libstore libfetchers
libexpr_LDFLAGS = -lboost_context
ifneq ($(OS), FreeBSD)
libexpr_LDFLAGS += -lboost_context -pthread
ifdef HOST_LINUX
libexpr_LDFLAGS += -ldl
endif
@ -35,7 +35,7 @@ $(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l
clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
$(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644))
$(eval $(call install-file-in, $(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644))
$(foreach i, $(wildcard src/libexpr/flake/*.hh), \
$(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644)))

66
src/libexpr/nixexpr.cc
View file

@ -123,7 +123,7 @@ void ExprList::show(std::ostream & str) const
void ExprLambda::show(std::ostream & str) const
{
str << "(";
if (matchAttrs) {
if (hasFormals()) {
str << "{ ";
bool first = true;
for (auto & i : formals->formals) {
@ -142,6 +142,16 @@ void ExprLambda::show(std::ostream & str) const
str << ": " << *body << ")";
}
void ExprCall::show(std::ostream & str) const
{
str << '(' << *fun;
for (auto e : args) {
str << ' ';
str << *e;
}
str << ')';
}
void ExprLet::show(std::ostream & str) const
{
str << "(let ";
@ -273,13 +283,13 @@ void ExprVar::bindVars(const std::shared_ptr<const StaticEnv> &env)
/* Check whether the variable appears in the environment. If so,
set its level and displacement. */
const StaticEnv * curEnv;
unsigned int level;
Level level;
int withLevel = -1;
for (curEnv = env.get(), level = 0; curEnv; curEnv = curEnv->up, level++) {
if (curEnv->isWith) {
if (withLevel == -1) withLevel = level;
} else {
StaticEnv::Vars::const_iterator i = curEnv->vars.find(name);
auto i = curEnv->find(name);
if (i != curEnv->vars.end()) {
fromWith = false;
this->level = level;
@ -332,12 +342,13 @@ void ExprAttrs::bindVars(const std::shared_ptr<const StaticEnv> &env)
staticenv = env;
if (recursive) {
auto newEnv = std::shared_ptr<StaticEnv>(new StaticEnv(false, env.get()));
auto newEnv = std::shared_ptr<StaticEnv>(new StaticEnv(false, env.get(), recursive ? attrs.size() : 0));
unsigned int displ = 0;
for (auto & i : attrs) {
newEnv->vars[i.first] = i.second.displ = displ++;
}
Displacement displ = 0;
for (auto & i : attrs)
newEnv.vars.emplace_back(i.first, i.second.displ = displ++);
// No need to sort newEnv since attrs is in sorted order.
for (auto & i : attrs)
i.second.e->bindVars(i.second.inherited ? env : newEnv);
@ -372,15 +383,21 @@ void ExprLambda::bindVars(const std::shared_ptr<const StaticEnv> &env)
if (debuggerHook)
staticenv = env;
auto newEnv = std::shared_ptr<StaticEnv>(new StaticEnv(false, env.get()));
auto newEnv = std::shared_ptr<StaticEnv>(
new StaticEnv(
false, env.get(),
(hasFormals() ? formals->formals.size() : 0) +
(arg.empty() ? 0 : 1)));
unsigned int displ = 0;
Displacement displ = 0;
if (!arg.empty()) newEnv->vars[arg] = displ++;
if (!arg.empty()) newEnv.vars.emplace_back(arg, displ++);
if (matchAttrs) {
if (hasFormals()) {
for (auto & i : formals->formals)
newEnv->vars[i.name] = displ++;
newEnv.vars.emplace_back(i.name, displ++);
newEnv.sort();
for (auto & i : formals->formals)
if (i.def) i.def->bindVars(newEnv);
@ -389,16 +406,28 @@ void ExprLambda::bindVars(const std::shared_ptr<const StaticEnv> &env)
body->bindVars(newEnv);
}
void ExprLet::bindVars(const std::shared_ptr<const StaticEnv> &env)
void ExprCall::bindVars(const StaticEnv & env)
{
if (debuggerHook)
staticenv = env;
auto newEnv = std::shared_ptr<StaticEnv>(new StaticEnv(false, env.get()));
fun->bindVars(env);
for (auto e : args)
e->bindVars(env);
}
unsigned int displ = 0;
void ExprLet::bindVars(const StaticEnv & env)
{
if (debuggerHook)
staticenv = env;
auto newEnv = std::shared_ptr<StaticEnv>(new StaticEnv(false, env.get(), attrs->attrs.size()));
Displacement displ = 0;
for (auto & i : attrs->attrs)
newEnv->vars[i.first] = i.second.displ = displ++;
newEnv.vars.emplace_back(i.first, i.second.displ = displ++);
// No need to sort newEnv since attrs->attrs is in sorted order.
for (auto & i : attrs->attrs)
i.second.e->bindVars(i.second.inherited ? env : newEnv);
@ -415,7 +444,7 @@ void ExprWith::bindVars(const std::shared_ptr<const StaticEnv> &env)
level so that `lookupVar' can look up variables in the previous
`with' if this one doesn't contain the desired attribute. */
const StaticEnv * curEnv;
unsigned int level;
Level level;
prevWith = 0;
for (curEnv = env.get(), level = 1; curEnv; curEnv = curEnv->up, level++)
if (curEnv->isWith) {
@ -503,5 +532,4 @@ size_t SymbolTable::totalSize() const
return n;
}
}

52
src/libexpr/nixexpr.hh
View file

@ -4,8 +4,6 @@
#include "symbol-table.hh"
#include "error.hh"
#include <map>
namespace nix {
@ -138,6 +136,9 @@ struct ExprPath : Expr
Value * maybeThunk(EvalState & state, Env & env);
};
typedef uint32_t Level;
typedef uint32_t Displacement;
struct ExprVar : Expr
{
Pos pos;
@ -153,8 +154,8 @@ struct ExprVar : Expr
value is obtained by getting the attribute named `name' from
the set stored in the environment that is `level' levels up
from the current one.*/
unsigned int level;
unsigned int displ;
Level level;
Displacement displ;
ExprVar(const Symbol & name) : name(name) { };
ExprVar(const Pos & pos, const Symbol & name) : pos(pos), name(name) { };
@ -188,7 +189,7 @@ struct ExprAttrs : Expr
bool inherited;
Expr * e;
Pos pos;
unsigned int displ; // displacement
Displacement displ; // displacement
AttrDef(Expr * e, const Pos & pos, bool inherited=false)
: inherited(inherited), e(e), pos(pos) { };
AttrDef() { };
@ -236,11 +237,10 @@ struct ExprLambda : Expr
Pos pos;
Symbol name;
Symbol arg;
bool matchAttrs;
Formals * formals;
Expr * body;
ExprLambda(const Pos & pos, const Symbol & arg, bool matchAttrs, Formals * formals, Expr * body)
: pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body)
ExprLambda(const Pos & pos, const Symbol & arg, Formals * formals, Expr * body)
: pos(pos), arg(arg), formals(formals), body(body)
{
if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
throw ParseError({
@ -250,6 +250,18 @@ struct ExprLambda : Expr
};
void setName(Symbol & name);
string showNamePos() const;
inline bool hasFormals() const { return formals != nullptr; }
COMMON_METHODS
};
struct ExprCall : Expr
{
Expr * fun;
std::vector<Expr *> args;
Pos pos;
ExprCall(const Pos & pos, Expr * fun, std::vector<Expr *> && args)
: fun(fun), args(args), pos(pos)
{ }
COMMON_METHODS
};
@ -311,7 +323,6 @@ struct ExprOpNot : Expr
void eval(EvalState & state, Env & env, Value & v); \
};
MakeBinOp(ExprApp, "")
MakeBinOp(ExprOpEq, "==")
MakeBinOp(ExprOpNEq, "!=")
MakeBinOp(ExprOpAnd, "&&")
@ -345,9 +356,28 @@ struct StaticEnv
{
bool isWith;
const StaticEnv * up;
typedef std::map<Symbol, unsigned int> Vars;
// Note: these must be in sorted order.
typedef std::vector<std::pair<Symbol, Displacement>> Vars;
Vars vars;
StaticEnv(bool isWith, const StaticEnv * up) : isWith(isWith), up(up) { };
StaticEnv(bool isWith, const StaticEnv * up, size_t expectedSize = 0) : isWith(isWith), up(up) {
vars.reserve(expectedSize);
};
void sort()
{
std::sort(vars.begin(), vars.end(),
[](const Vars::value_type & a, const Vars::value_type & b) { return a.first < b.first; });
}
Vars::const_iterator find(const Symbol & name) const
{
Vars::value_type key(name, 0);
auto i = std::lower_bound(vars.begin(), vars.end(), key);
if (i != vars.end() && i->first == name) return i;
return vars.end();
}
};

86
src/libexpr/parser.y
View file

@ -33,11 +33,9 @@ namespace nix {
Symbol file;
FileOrigin origin;
std::optional<ErrorInfo> error;
Symbol sLetBody;
ParseData(EvalState & state)
: state(state)
, symbols(state.symbols)
, sLetBody(symbols.create("<let-body>"))
{ };
};
@ -126,14 +124,14 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath,
auto j2 = jAttrs->attrs.find(ad.first);
if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error.
dupAttr(ad.first, j2->second.pos, ad.second.pos);
jAttrs->attrs[ad.first] = ad.second;
jAttrs->attrs.emplace(ad.first, ad.second);
}
} else {
dupAttr(attrPath, pos, j->second.pos);
}
} else {
// This attr path is not defined. Let's create it.
attrs->attrs[i->symbol] = ExprAttrs::AttrDef(e, pos);
attrs->attrs.emplace(i->symbol, ExprAttrs::AttrDef(e, pos));
e->setName(i->symbol);
}
} else {
@ -283,20 +281,20 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
}
%type <e> start expr expr_function expr_if expr_op
%type <e> expr_app expr_select expr_simple
%type <e> expr_select expr_simple expr_app
%type <list> expr_list
%type <attrs> binds
%type <formals> formals
%type <formal> formal
%type <attrNames> attrs attrpath
%type <string_parts> string_parts_interpolated ind_string_parts
%type <e> string_parts string_attr
%type <e> path_start string_parts string_attr
%type <id> attr
%token <id> ID ATTRPATH
%token <e> STR IND_STR
%token <n> INT
%token <nf> FLOAT
%token <path> PATH HPATH SPATH
%token <path> PATH HPATH SPATH PATH_END
%token <uri> URI
%token IF THEN ELSE ASSERT WITH LET IN REC INHERIT EQ NEQ AND OR IMPL OR_KW
%token DOLLAR_CURLY /* == ${ */
@ -324,13 +322,13 @@ expr: expr_function;
expr_function
: ID ':' expr_function
{ $$ = new ExprLambda(CUR_POS, data->symbols.create($1), false, 0, $3); }
{ $$ = new ExprLambda(CUR_POS, data->symbols.create($1), 0, $3); }
| '{' formals '}' ':' expr_function
{ $$ = new ExprLambda(CUR_POS, data->symbols.create(""), true, $2, $5); }
{ $$ = new ExprLambda(CUR_POS, data->symbols.create(""), $2, $5); }
| '{' formals '}' '@' ID ':' expr_function
{ $$ = new ExprLambda(CUR_POS, data->symbols.create($5), true, $2, $7); }
{ $$ = new ExprLambda(CUR_POS, data->symbols.create($5), $2, $7); }
| ID '@' '{' formals '}' ':' expr_function
{ $$ = new ExprLambda(CUR_POS, data->symbols.create($1), true, $4, $7); }
{ $$ = new ExprLambda(CUR_POS, data->symbols.create($1), $4, $7); }
| ASSERT expr ';' expr_function
{ $$ = new ExprAssert(CUR_POS, $2, $4); }
| WITH expr ';' expr_function
@ -353,13 +351,13 @@ expr_if
expr_op
: '!' expr_op %prec NOT { $$ = new ExprOpNot($2); }
| '-' expr_op %prec NEGATE { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__sub")), new ExprInt(0)), $2); }
| '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {new ExprInt(0), $2}); }
| expr_op EQ expr_op { $$ = new ExprOpEq($1, $3); }
| expr_op NEQ expr_op { $$ = new ExprOpNEq($1, $3); }
| expr_op '<' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $1), $3); }
| expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $3), $1)); }
| expr_op '>' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $3), $1); }
| expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $1), $3)); }
| expr_op '<' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$1, $3}); }
| expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$3, $1})); }
| expr_op '>' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$3, $1}); }
| expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$1, $3})); }
| expr_op AND expr_op { $$ = new ExprOpAnd(CUR_POS, $1, $3); }
| expr_op OR expr_op { $$ = new ExprOpOr(CUR_POS, $1, $3); }
| expr_op IMPL expr_op { $$ = new ExprOpImpl(CUR_POS, $1, $3); }
@ -367,17 +365,22 @@ expr_op
| expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, *$3); }
| expr_op '+' expr_op
{ $$ = new ExprConcatStrings(CUR_POS, false, new vector<Expr *>({$1, $3})); }
| expr_op '-' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__sub")), $1), $3); }
| expr_op '*' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__mul")), $1), $3); }
| expr_op '/' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__div")), $1), $3); }
| expr_op '-' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {$1, $3}); }
| expr_op '*' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__mul")), {$1, $3}); }
| expr_op '/' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__div")), {$1, $3}); }
| expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(CUR_POS, $1, $3); }
| expr_app
;
expr_app
: expr_app expr_select
{ $$ = new ExprApp(CUR_POS, $1, $2); }
| expr_select { $$ = $1; }
: expr_app expr_select {
if (auto e2 = dynamic_cast<ExprCall *>($1)) {
e2->args.push_back($2);
$$ = $1;
} else
$$ = new ExprCall(CUR_POS, $1, {$2});
}
| expr_select
;
expr_select
@ -388,7 +391,7 @@ expr_select
| /* Backwards compatibility: because Nixpkgs has a rarely used
function named ‘or’, allow stuff like ‘map or [...]’. */
expr_simple OR_KW
{ $$ = new ExprApp(CUR_POS, $1, new ExprVar(CUR_POS, data->symbols.create("or"))); }
{ $$ = new ExprCall(CUR_POS, $1, {new ExprVar(CUR_POS, data->symbols.create("or"))}); }
| expr_simple { $$ = $1; }
;
@ -405,17 +408,20 @@ expr_simple
| IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE {
$$ = stripIndentation(CUR_POS, data->symbols, *$2);
}
| PATH { $$ = new ExprPath(absPath($1, data->basePath)); }
| HPATH { $$ = new ExprPath(getHome() + string{$1 + 1}); }
| path_start PATH_END { $$ = $1; }
| path_start string_parts_interpolated PATH_END {
$2->insert($2->begin(), $1);
$$ = new ExprConcatStrings(CUR_POS, false, $2);
}
| SPATH {
string path($1 + 1, strlen($1) - 2);
$$ = new ExprApp(CUR_POS,
new ExprApp(new ExprVar(data->symbols.create("__findFile")),
new ExprVar(data->symbols.create("__nixPath"))),
new ExprString(data->symbols.create(path)));
$$ = new ExprCall(CUR_POS,
new ExprVar(data->symbols.create("__findFile")),
{new ExprVar(data->symbols.create("__nixPath")),
new ExprString(data->symbols.create(path))});
}
| URI {
static bool noURLLiterals = settings.isExperimentalFeatureEnabled("no-url-literals");
static bool noURLLiterals = settings.isExperimentalFeatureEnabled(Xp::NoUrlLiterals);
if (noURLLiterals)
throw ParseError({
.msg = hintfmt("URL literals are disabled"),
@ -452,6 +458,20 @@ string_parts_interpolated
}
;
path_start
: PATH {
Path path(absPath($1, data->basePath));
/* add back in the trailing '/' to the first segment */
if ($1[strlen($1)-1] == '/' && strlen($1) > 1)
path += "/";
$$ = new ExprPath(path);
}
| HPATH {
Path path(getHome() + string($1 + 1));
$$ = new ExprPath(path);
}
;
ind_string_parts
: ind_string_parts IND_STR { $$ = $1; $1->push_back($2); }
| ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->push_back($3); }
@ -466,7 +486,7 @@ binds
if ($$->attrs.find(i.symbol) != $$->attrs.end())
dupAttr(i.symbol, makeCurPos(@3, data), $$->attrs[i.symbol].pos);
Pos pos = makeCurPos(@3, data);
$$->attrs[i.symbol] = ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true);
$$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true));
}
}
| binds INHERIT '(' expr ')' attrs ';'
@ -475,7 +495,7 @@ binds
for (auto & i : *$6) {
if ($$->attrs.find(i.symbol) != $$->attrs.end())
dupAttr(i.symbol, makeCurPos(@6, data), $$->attrs[i.symbol].pos);
$$->attrs[i.symbol] = ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data));
$$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data)));
}
}
| { $$ = new ExprAttrs(makeCurPos(@0, data)); }
@ -735,7 +755,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
res = { true, path };
else {
logWarning({
.msg = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second)
.msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", elem.second)
});
res = { false, "" };
}

336
src/libexpr/primops.cc
View file

@ -21,6 +21,8 @@
#include <regex>
#include <dlfcn.h>
#include <cmath>
namespace nix {
@ -50,16 +52,13 @@ void EvalState::realiseContext(const PathSet & context)
if (drvs.empty()) return;
if (!evalSettings.enableImportFromDerivation)
throw EvalError("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false",
throw Error(
"cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled",
store->printStorePath(drvs.begin()->drvPath));
/* For performance, prefetch all substitute info. */
StorePathSet willBuild, willSubstitute, unknown;
uint64_t downloadSize, narSize;
/* Build/substitute the context. */
std::vector<DerivedPath> buildReqs;
for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d });
store->queryMissing(buildReqs, willBuild, willSubstitute, unknown, downloadSize, narSize);
store->buildPaths(buildReqs);
/* Add the output of this derivations to the allowed
@ -71,7 +70,7 @@ void EvalState::realiseContext(const PathSet & context)
if (outputPaths.count(outputName) == 0)
throw Error("derivation '%s' does not have an output named '%s'",
store->printStorePath(drvPath), outputName);
allowedPaths->insert(store->printStorePath(outputPaths.at(outputName)));
allowPath(outputPaths.at(outputName));
}
}
}
@ -122,7 +121,7 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS
});
} catch (Error & e) {
e.addTrace(pos, "while importing '%s'", path);
throw e;
throw;
}
Path realPath = state.checkSourcePath(state.toRealPath(path, context));
@ -158,16 +157,15 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS
}
w.attrs->sort();
static RootValue fun;
if (!fun) {
fun = allocRootValue(state.allocValue());
if (!state.vImportedDrvToDerivation) {
state.vImportedDrvToDerivation = allocRootValue(state.allocValue());
state.eval(state.parseExprFromString(
#include "imported-drv-to-derivation.nix.gen.hh"
, "/"), **fun);
, "/"), **state.vImportedDrvToDerivation);
}
state.forceFunction(**fun, pos);
mkApp(v, **fun, w);
state.forceFunction(**state.vImportedDrvToDerivation, pos);
mkApp(v, **state.vImportedDrvToDerivation, w);
state.forceAttrs(v, pos);
}
@ -186,14 +184,17 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS
Env * env = &state.allocEnv(vScope->attrs->size());
env->up = &state.baseEnv;
auto staticEnv = std::shared_ptr<StaticEnv>(new StaticEnv(false, state.staticBaseEnv.get()));
auto staticEnv = std::shared_ptr<StaticEnv>(new StaticEnv(false, state.staticBaseEnv.get(), vScope->attrs->size()));
unsigned int displ = 0;
for (auto & attr : *vScope->attrs) {
staticEnv->vars[attr.name] = displ;
staticEnv.vars.emplace_back(attr.name, displ);
env->values[displ++] = attr.value;
}
// No need to call staticEnv.sort(), because
// args[0]->attrs is already sorted.
printTalkative("evaluating file '%1%'", realPath);
Expr * e = state.parseExprFromFile(resolveExprPath(realPath), staticEnv);
@ -412,7 +413,7 @@ static RegisterPrimOp primop_isNull({
Return `true` if *e* evaluates to `null`, and `false` otherwise.
> **Warning**
>
>
> This function is *deprecated*; just write `e == null` instead.
)",
.fun = prim_isNull,
@ -516,7 +517,11 @@ static RegisterPrimOp primop_isPath({
struct CompareValues
{
bool operator () (const Value * v1, const Value * v2) const
EvalState & state;
CompareValues(EvalState & state) : state(state) { };
bool operator () (Value * v1, Value * v2) const
{
if (v1->type() == nFloat && v2->type() == nInt)
return v1->fpoint < v2->integer;
@ -533,6 +538,17 @@ struct CompareValues
return strcmp(v1->string.s, v2->string.s) < 0;
case nPath:
return strcmp(v1->path, v2->path) < 0;
case nList:
// Lexicographic comparison
for (size_t i = 0;; i++) {
if (i == v2->listSize()) {
return false;
} else if (i == v1->listSize()) {
return true;
} else if (!state.eqValues(*v1->listElems()[i], *v2->listElems()[i])) {
return (*this)(v1->listElems()[i], v2->listElems()[i]);
}
}
default:
throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2));
}
@ -620,7 +636,8 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
ValueList res;
// `doneKeys' doesn't need to be a GC root, because its values are
// reachable from res.
set<Value *, CompareValues> doneKeys;
auto cmp = CompareValues(state);
set<Value *, decltype(cmp)> doneKeys(cmp);
while (!workSet.empty()) {
Value * e = *(workSet.begin());
workSet.pop_front();
@ -714,6 +731,44 @@ static RegisterPrimOp primop_addErrorContext(RegisterPrimOp::Info {
.fun = prim_addErrorContext,
});
static void prim_ceil(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
auto value = state.forceFloat(*args[0], args[0]->determinePos(pos));
mkInt(v, ceil(value));
}
static RegisterPrimOp primop_ceil({
.name = "__ceil",
.args = {"double"},
.doc = R"(
Converts an IEEE-754 double-precision floating-point number (*double*) to
the next higher integer.
If the datatype is neither an integer nor a "float", an evaluation error will be
thrown.
)",
.fun = prim_ceil,
});
static void prim_floor(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
auto value = state.forceFloat(*args[0], args[0]->determinePos(pos));
mkInt(v, floor(value));
}
static RegisterPrimOp primop_floor({
.name = "__floor",
.args = {"double"},
.doc = R"(
Converts an IEEE-754 double-precision floating-point number (*double*) to
the next lower integer.
If the datatype is neither an integer nor a "float", an evaluation error will be
thrown.
)",
.fun = prim_floor,
});
/* Try evaluating the argument. Success => {success=true; value=something;},
* else => {success=false; value=false;} */
static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Value & v)
@ -949,8 +1004,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
}
if (i->name == state.sContentAddressed) {
settings.requireExperimentalFeature("ca-derivations");
contentAddressed = state.forceBool(*i->value, pos);
if (contentAddressed)
settings.requireExperimentalFeature(Xp::CaDerivations);
}
/* The `args' attribute is special: it supplies the
@ -972,7 +1028,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
if (i->name == state.sStructuredAttrs) continue;
auto placeholder(jsonObject->placeholder(key));
printValueAsJSON(state, true, *i->value, placeholder, context);
printValueAsJSON(state, true, *i->value, pos, placeholder, context);
if (i->name == state.sBuilder)
drv.builder = state.forceString(*i->value, context, posDrvName);
@ -1134,7 +1190,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
// hash per output.
auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true);
std::visit(overloaded {
[&](Hash h) {
[&](Hash & h) {
for (auto & i : outputs) {
auto outPath = state.store->makeOutputPath(i, h, drvName);
drv.env[i] = state.store->printStorePath(outPath);
@ -1146,11 +1202,11 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
});
}
},
[&](CaOutputHashes) {
[&](CaOutputHashes &) {
// Shouldn't happen as the toplevel derivation is not CA.
assert(false);
},
[&](DeferredHash _) {
[&](DeferredHash &) {
for (auto & i : outputs) {
drv.outputs.insert_or_assign(i,
DerivationOutput {
@ -1453,15 +1509,20 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va
string type = state.forceStringNoCtx(*args[0], pos);
std::optional<HashType> ht = parseHashType(type);
if (!ht)
throw Error({
.msg = hintfmt("unknown hash type '%1%'", type),
.errPos = pos
});
throw Error({
.msg = hintfmt("unknown hash type '%1%'", type),
.errPos = pos
});
PathSet context; // discarded
Path p = state.coerceToPath(pos, *args[1], context);
PathSet context;
Path path = state.coerceToPath(pos, *args[1], context);
try {
state.realiseContext(context);
} catch (InvalidPathError & e) {
throw EvalError("cannot read '%s' since path '%s' is not valid, at %s", path, e.path, pos);
}
mkString(v, hashFile(*ht, state.checkSourcePath(p)).to_string(Base16, false), context);
mkString(v, hashFile(*ht, state.checkSourcePath(state.toRealPath(path, context))).to_string(Base16, false));
}
static RegisterPrimOp primop_hashFile({
@ -1538,7 +1599,7 @@ static void prim_toXML(EvalState & state, const Pos & pos, Value * * args, Value
{
std::ostringstream out;
PathSet context;
printValueAsXML(state, true, false, *args[0], out, context);
printValueAsXML(state, true, false, *args[0], out, context, pos);
mkString(v, out.str(), context);
}
@ -1646,7 +1707,7 @@ static void prim_toJSON(EvalState & state, const Pos & pos, Value * * args, Valu
{
std::ostringstream out;
PathSet context;
printValueAsJSON(state, true, *args[0], out, context);
printValueAsJSON(state, true, *args[0], pos, out, context);
mkString(v, out.str(), context);
}
@ -1672,7 +1733,7 @@ static void prim_fromJSON(EvalState & state, const Pos & pos, Value * * args, Va
parseJSON(state, s, v);
} catch (JSONParseError &e) {
e.addTrace(pos, "while decoding a JSON string");
throw e;
throw;
}
}
@ -1802,50 +1863,79 @@ static RegisterPrimOp primop_toFile({
.fun = prim_toFile,
});
static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_,
Value * filterFun, FileIngestionMethod method, const std::optional<Hash> expectedHash, Value & v)
static void addPath(
EvalState & state,
const Pos & pos,
const string & name,
Path path,
Value * filterFun,
FileIngestionMethod method,
const std::optional<Hash> expectedHash,
Value & v,
const PathSet & context)
{
const auto path = evalSettings.pureEval && expectedHash ?
path_ :
state.checkSourcePath(path_);
PathFilter filter = filterFun ? ([&](const Path & path) {
auto st = lstat(path);
try {
// FIXME: handle CA derivation outputs (where path needs to
// be rewritten to the actual output).
state.realiseContext(context);
/* Call the filter function. The first argument is the path,
the second is a string indicating the type of the file. */
Value arg1;
mkString(arg1, path);
StorePathSet refs;
Value fun2;
state.callFunction(*filterFun, arg1, fun2, noPos);
if (state.store->isInStore(path)) {
auto [storePath, subPath] = state.store->toStorePath(path);
// FIXME: we should scanForReferences on the path before adding it
refs = state.store->queryPathInfo(storePath)->references;
path = state.store->toRealPath(storePath) + subPath;
}
Value arg2;
mkString(arg2,
S_ISREG(st.st_mode) ? "regular" :
S_ISDIR(st.st_mode) ? "directory" :
S_ISLNK(st.st_mode) ? "symlink" :
"unknown" /* not supported, will fail! */);
path = evalSettings.pureEval && expectedHash
? path
: state.checkSourcePath(path);
Value res;
state.callFunction(fun2, arg2, res, noPos);
PathFilter filter = filterFun ? ([&](const Path & path) {
auto st = lstat(path);
return state.forceBool(res, pos);
}) : defaultPathFilter;
/* Call the filter function. The first argument is the path,
the second is a string indicating the type of the file. */
Value arg1;
mkString(arg1, path);
std::optional<StorePath> expectedStorePath;
if (expectedHash)
expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name);
Path dstPath;
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
dstPath = state.store->printStorePath(settings.readOnlyMode
? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first
: state.store->addToStore(name, path, method, htSHA256, filter, state.repair));
if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath))
throw Error("store path mismatch in (possibly filtered) path added from '%s'", path);
} else
dstPath = state.store->printStorePath(*expectedStorePath);
Value arg2;
mkString(arg2,
S_ISREG(st.st_mode) ? "regular" :
S_ISDIR(st.st_mode) ? "directory" :
S_ISLNK(st.st_mode) ? "symlink" :
"unknown" /* not supported, will fail! */);
mkString(v, dstPath, {dstPath});
Value * args []{&arg1, &arg2};
Value res;
state.callFunction(*filterFun, 2, args, res, pos);
return state.forceBool(res, pos);
}) : defaultPathFilter;
std::optional<StorePath> expectedStorePath;
if (expectedHash)
expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name);
Path dstPath;
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
dstPath = state.store->printStorePath(settings.readOnlyMode
? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first
: state.store->addToStore(name, path, method, htSHA256, filter, state.repair, refs));
if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath))
throw Error("store path mismatch in (possibly filtered) path added from '%s'", path);
} else
dstPath = state.store->printStorePath(*expectedStorePath);
mkString(v, dstPath, {dstPath});
state.allowPath(dstPath);
} catch (Error & e) {
e.addTrace(pos, "while adding path '%s'", path);
throw;
}
}
@ -1853,11 +1943,6 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
{
PathSet context;
Path path = state.coerceToPath(pos, *args[1], context);
if (!context.empty())
throw EvalError({
.msg = hintfmt("string '%1%' cannot refer to other paths", path),
.errPos = pos
});
state.forceValue(*args[0], pos);
if (args[0]->type() != nFunction)
@ -1868,13 +1953,26 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
.errPos = pos
});
addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v);
addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v, context);
}
static RegisterPrimOp primop_filterSource({
.name = "__filterSource",
.args = {"e1", "e2"},
.doc = R"(
> **Warning**
>
> `filterSource` should not be used to filter store paths. Since
> `filterSource` uses the name of the input directory while naming
> the output directory, doing so will produce a directory name in
> the form of `<hash2>-<hash>-<name>`, where `<hash>-<name>` is
> the name of the input directory. Since `<hash>` depends on the
> unfiltered directory, the name of the output directory will
> indirectly depend on files that are filtered out by the
> function. This will trigger a rebuild even when a filtered out
> file is changed. Use `builtins.path` instead, which allows
> specifying the name of the output directory.
This function allows you to copy sources into the Nix store while
filtering certain files. For instance, suppose that you want to use
the directory `source-dir` as an input to a Nix expression, e.g.
@ -1921,18 +2019,13 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
Value * filterFun = nullptr;
auto method = FileIngestionMethod::Recursive;
std::optional<Hash> expectedHash;
PathSet context;
for (auto & attr : *args[0]->attrs) {
const string & n(attr.name);
if (n == "path") {
PathSet context;
if (n == "path")
path = state.coerceToPath(*attr.pos, *attr.value, context);
if (!context.empty())
throw EvalError({
.msg = hintfmt("string '%1%' cannot refer to other paths", path),
.errPos = *attr.pos
});
} else if (attr.name == state.sName)
else if (attr.name == state.sName)
name = state.forceStringNoCtx(*attr.value, *attr.pos);
else if (n == "filter") {
state.forceValue(*attr.value, pos);
@ -1955,7 +2048,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
if (name.empty())
name = baseNameOf(path);
addPath(state, pos, name, path, filterFun, method, expectedHash, v);
addPath(state, pos, name, path, filterFun, method, expectedHash, v, context);
}
static RegisterPrimOp primop_path({
@ -2069,7 +2162,7 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v)
pos
);
// !!! add to stack trace?
if (state.countCalls && i->pos) state.attrSelects[*i->pos]++;
if (state.countCalls && *i->pos != noPos) state.attrSelects[*i->pos]++;
state.forceValue(*i->value, pos);
v = *i->value;
}
@ -2320,7 +2413,7 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
.errPos = pos
});
if (!args[0]->lambda.fun->matchAttrs) {
if (!args[0]->lambda.fun->hasFormals()) {
state.mkAttrs(v, 0);
return;
}
@ -2329,7 +2422,7 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
for (auto & i : args[0]->lambda.fun->formals->formals) {
// !!! should optimise booleans (allocate only once)
Value * value = state.allocValue();
v.attrs->push_back(Attr(i.name, value, &i.pos));
v.attrs->push_back(Attr(i.name, value, ptr(&i.pos)));
mkBool(*value, i.def);
}
v.attrs->sort();
@ -2475,7 +2568,7 @@ static RegisterPrimOp primop_tail({
the argument isn’t a list or is an empty list.
> **Warning**
>
>
> This function should generally be avoided since it's inefficient:
> unlike Haskell's `tail`, it takes O(n) time, so recursing over a
> list by repeatedly calling `tail` takes O(n^2) time.
@ -2617,10 +2710,9 @@ static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args,
Value * vCur = args[1];
for (unsigned int n = 0; n < args[2]->listSize(); ++n) {
Value vTmp;
state.callFunction(*args[0], *vCur, vTmp, pos);
Value * vs []{vCur, args[2]->listElems()[n]};
vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue();
state.callFunction(vTmp, *args[2]->listElems()[n], *vCur, pos);
state.callFunction(*args[0], 2, vs, *vCur, pos);
}
state.forceValue(v, pos);
} else {
@ -2634,9 +2726,9 @@ static RegisterPrimOp primop_foldlStrict({
.args = {"op", "nul", "list"},
.doc = R"(
Reduce a list by applying a binary operator, from left to right,
e.g. `foldl’ op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2)
e.g. `foldl' op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2)
...`. The operator is applied strictly, i.e., its arguments are
evaluated first. For example, `foldl’ (x: y: x + y) 0 [1 2 3]`
evaluated first. For example, `foldl' (x: y: x + y) 0 [1 2 3]`
evaluates to 6.
)",
.fun = prim_foldlStrict,
@ -2741,17 +2833,16 @@ static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value
v.listElems()[n] = args[1]->listElems()[n];
}
auto comparator = [&](Value * a, Value * b) {
/* Optimization: if the comparator is lessThan, bypass
callFunction. */
if (args[0]->isPrimOp() && args[0]->primOp->fun == prim_lessThan)
return CompareValues()(a, b);
return CompareValues(state)(a, b);
Value vTmp1, vTmp2;
state.callFunction(*args[0], *a, vTmp1, pos);
state.callFunction(vTmp1, *b, vTmp2, pos);
return state.forceBool(vTmp2, pos);
Value * vs[] = {a, b};
Value vBool;
state.callFunction(*args[0], 2, vs, vBool, pos);
return state.forceBool(vBool, pos);
};
/* FIXME: std::sort can segfault if the comparator is not a strict
@ -2857,7 +2948,7 @@ static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, V
state.forceList(lists[n], lists[n].determinePos(args[0]->determinePos(pos)));
} catch (TypeError &e) {
e.addTrace(pos, hintfmt("while invoking '%s'", "concatMap"));
throw e;
throw;
}
len += lists[n].listSize();
}
@ -3028,7 +3119,7 @@ static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Va
{
state.forceValue(*args[0], pos);
state.forceValue(*args[1], pos);
CompareValues comp;
CompareValues comp{state};
mkBool(v, comp(args[0], args[1]));
}
@ -3069,7 +3160,7 @@ static RegisterPrimOp primop_toString({
- A path (e.g., `toString /foo/bar` yields `"/foo/bar"`.
- A set containing `{ __toString = self: ...; }`.
- A set containing `{ __toString = self: ...; }` or `{ outPath = ...; }`.
- An integer.
@ -3154,7 +3245,7 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args,
PathSet context; // discarded
string s = state.forceString(*args[1], context, pos);
mkString(v, hashString(*ht, s).to_string(Base16, false), context);
mkString(v, hashString(*ht, s).to_string(Base16, false));
}
static RegisterPrimOp primop_hashString({
@ -3561,15 +3652,13 @@ static RegisterPrimOp primop_splitVersion({
RegisterPrimOp::PrimOps * RegisterPrimOp::primOps;
RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun,
std::optional<std::string> requiredFeature)
RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun)
{
if (!primOps) primOps = new PrimOps;
primOps->push_back({
.name = name,
.args = {},
.arity = arity,
.requiredFeature = std::move(requiredFeature),
.fun = fun
});
}
@ -3605,9 +3694,7 @@ void EvalState::createBaseEnv()
if (!evalSettings.pureEval) {
mkInt(v, time(0));
addConstant("__currentTime", v);
}
if (!evalSettings.pureEval) {
mkString(v, settings.thisSystem.get());
addConstant("__currentSystem", v);
}
@ -3622,7 +3709,7 @@ void EvalState::createBaseEnv()
language feature gets added. It's not necessary to increase it
when primops get added, because you can just use `builtins ?
primOp' to check. */
mkInt(v, 5);
mkInt(v, 6);
addConstant("__langVersion", v);
// Miscellaneous
@ -3645,26 +3732,31 @@ void EvalState::createBaseEnv()
if (RegisterPrimOp::primOps)
for (auto & primOp : *RegisterPrimOp::primOps)
if (!primOp.requiredFeature || settings.isExperimentalFeatureEnabled(*primOp.requiredFeature))
addPrimOp({
.fun = primOp.fun,
.arity = std::max(primOp.args.size(), primOp.arity),
.name = symbols.create(primOp.name),
.args = std::move(primOp.args),
.doc = primOp.doc,
});
addPrimOp({
.fun = primOp.fun,
.arity = std::max(primOp.args.size(), primOp.arity),
.name = symbols.create(primOp.name),
.args = std::move(primOp.args),
.doc = primOp.doc,
});
/* Add a wrapper around the derivation primop that computes the
`drvPath' and `outPath' attributes lazily. */
sDerivationNix = symbols.create("//builtin/derivation.nix");
eval(parse(
#include "primops/derivation.nix.gen.hh"
, foFile, sDerivationNix, "/", staticBaseEnv), v);
addConstant("derivation", v);
auto vDerivation = allocValue();
addConstant("derivation", vDerivation);
/* Now that we've added all primops, sort the `builtins' set,
because attribute lookups expect it to be sorted. */
baseEnv.values[0]->attrs->sort();
staticBaseEnv.sort();
/* Note: we have to initialize the 'derivation' constant *after*
building baseEnv/staticBaseEnv because it uses 'builtins'. */
eval(parse(
#include "primops/derivation.nix.gen.hh"
, foFile, sDerivationNix, "/", staticBaseEnv), *vDerivation);
}

4
src/libexpr/primops.hh
View file

@ -15,7 +15,6 @@ struct RegisterPrimOp
std::vector<std::string> args;
size_t arity = 0;
const char * doc;
std::optional<std::string> requiredFeature;
PrimOpFun fun;
};
@ -28,8 +27,7 @@ struct RegisterPrimOp
RegisterPrimOp(
std::string name,
size_t arity,
PrimOpFun fun,
std::optional<std::string> requiredFeature = {});
PrimOpFun fun);
RegisterPrimOp(Info && info);
};

6
src/libexpr/primops/fetchMercurial.cc
View file

@ -15,7 +15,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
std::string name = "source";
PathSet context;
state.forceValue(*args[0]);
state.forceValue(*args[0], pos);
if (args[0]->type() == nAttrs) {
@ -62,6 +62,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
fetchers::Attrs attrs;
attrs.insert_or_assign("type", "hg");
attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
attrs.insert_or_assign("name", name);
if (ref) attrs.insert_or_assign("ref", *ref);
if (rev) attrs.insert_or_assign("rev", rev->gitRev());
auto input = fetchers::Input::fromAttrs(std::move(attrs));
@ -83,8 +84,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
v.attrs->sort();
if (state.allowedPaths)
state.allowedPaths->insert(tree.actualPath);
state.allowPath(tree.storePath);
}
static RegisterPrimOp r_fetchMercurial("fetchMercurial", 1, prim_fetchMercurial);

166
src/libexpr/primops/fetchTree.cc
View file

@ -7,6 +7,7 @@
#include <ctime>
#include <iomanip>
#include <regex>
namespace nix {
@ -15,7 +16,8 @@ void emitTreeAttrs(
const fetchers::Tree & tree,
const fetchers::Input & input,
Value & v,
bool emptyRevFallback)
bool emptyRevFallback,
bool forceDirty)
{
assert(input.isImmutable());
@ -32,24 +34,28 @@ void emitTreeAttrs(
mkString(*state.allocAttr(v, state.symbols.create("narHash")),
narHash->to_string(SRI, true));
if (auto rev = input.getRev()) {
mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev());
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev());
} else if (emptyRevFallback) {
// Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev
auto emptyHash = Hash(htSHA1);
mkString(*state.allocAttr(v, state.symbols.create("rev")), emptyHash.gitRev());
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitShortRev());
}
if (input.getType() == "git")
mkBool(*state.allocAttr(v, state.symbols.create("submodules")),
fetchers::maybeGetBoolAttr(input.attrs, "submodules").value_or(false));
if (auto revCount = input.getRevCount())
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
else if (emptyRevFallback)
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), 0);
if (!forceDirty) {
if (auto rev = input.getRev()) {
mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev());
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev());
} else if (emptyRevFallback) {
// Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev
auto emptyHash = Hash(htSHA1);
mkString(*state.allocAttr(v, state.symbols.create("rev")), emptyHash.gitRev());
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitShortRev());
}
if (auto revCount = input.getRevCount())
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
else if (emptyRevFallback)
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), 0);
}
if (auto lastModified = input.getLastModified()) {
mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *lastModified);
@ -60,47 +66,74 @@ void emitTreeAttrs(
v.attrs->sort();
}
std::string fixURI(std::string uri, EvalState &state)
std::string fixURI(std::string uri, EvalState & state, const std::string & defaultScheme = "file")
{
state.checkURI(uri);
return uri.find("://") != std::string::npos ? uri : "file://" + uri;
return uri.find("://") != std::string::npos ? uri : defaultScheme + "://" + uri;
}
void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v)
std::string fixURIForGit(std::string uri, EvalState & state)
{
string n(name);
attrs.emplace(name, n == "url" ? fixURI(v, state) : v);
/* Detects scp-style uris (e.g. git@github.com:NixOS/nix) and fixes
* them by removing the `:` and assuming a scheme of `ssh://`
* */
static std::regex scp_uri("([^/]*)@(.*):(.*)");
if (uri[0] != '/' && std::regex_match(uri, scp_uri))
return fixURI(std::regex_replace(uri, scp_uri, "$1@$2/$3"), state, "ssh");
else
return fixURI(uri, state);
}
struct FetchTreeParams {
bool emptyRevFallback = false;
bool allowNameArgument = false;
};
static void fetchTree(
EvalState &state,
const Pos &pos,
Value **args,
Value &v,
const std::optional<std::string> type,
bool emptyRevFallback = false
EvalState & state,
const Pos & pos,
Value * * args,
Value & v,
std::optional<std::string> type,
const FetchTreeParams & params = FetchTreeParams{}
) {
fetchers::Input input;
PathSet context;
state.forceValue(*args[0]);
state.forceValue(*args[0], pos);
if (args[0]->type() == nAttrs) {
state.forceAttrs(*args[0], pos);
fetchers::Attrs attrs;
if (auto aType = args[0]->attrs->get(state.sType)) {
if (type)
throw Error({
.msg = hintfmt("unexpected attribute 'type'"),
.errPos = pos
});
type = state.forceStringNoCtx(*aType->value, *aType->pos);
} else if (!type)
throw Error({
.msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
.errPos = pos
});
attrs.emplace("type", type.value());
for (auto & attr : *args[0]->attrs) {
state.forceValue(*attr.value);
if (attr.value->type() == nPath || attr.value->type() == nString)
addURI(
state,
attrs,
attr.name,
state.coerceToString(*attr.pos, *attr.value, context, false, false)
);
else if (attr.value->type() == nString)
addURI(state, attrs, attr.name, attr.value->string.s);
if (attr.name == state.sType) continue;
state.forceValue(*attr.value, *attr.pos);
if (attr.value->type() == nPath || attr.value->type() == nString) {
auto s = state.coerceToString(*attr.pos, *attr.value, context, false, false);
attrs.emplace(attr.name,
attr.name == "url"
? type == "git"
? fixURIForGit(s, state)
: fixURI(s, state)
: s);
}
else if (attr.value->type() == nBool)
attrs.emplace(attr.name, Explicit<bool>{attr.value->boolean});
else if (attr.value->type() == nInt)
@ -110,26 +143,24 @@ static void fetchTree(
attr.name, showType(*attr.value));
}
if (type)
attrs.emplace("type", type.value());
if (!attrs.count("type"))
throw Error({
.msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
.errPos = pos
});
if (!params.allowNameArgument)
if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
throw Error({
.msg = hintfmt("attribute 'name' isn’t supported in call to 'fetchTree'"),
.errPos = pos
});
input = fetchers::Input::fromAttrs(std::move(attrs));
} else {
auto url = fixURI(state.coerceToString(pos, *args[0], context, false, false), state);
auto url = state.coerceToString(pos, *args[0], context, false, false);
if (type == "git") {
fetchers::Attrs attrs;
attrs.emplace("type", "git");
attrs.emplace("url", url);
attrs.emplace("url", fixURIForGit(url, state));
input = fetchers::Input::fromAttrs(std::move(attrs));
} else {
input = fetchers::Input::fromURL(url);
input = fetchers::Input::fromURL(fixURI(url, state));
}
}
@ -141,16 +172,15 @@ static void fetchTree(
auto [tree, input2] = input.fetch(state.store);
if (state.allowedPaths)
state.allowedPaths->insert(tree.actualPath);
state.allowPath(tree.storePath);
emitTreeAttrs(state, tree, input2, v, emptyRevFallback);
emitTreeAttrs(state, tree, input2, v, params.emptyRevFallback, false);
}
static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
settings.requireExperimentalFeature("flakes");
fetchTree(state, pos, args, v, std::nullopt);
settings.requireExperimentalFeature(Xp::Flakes);
fetchTree(state, pos, args, v, std::nullopt, FetchTreeParams { .allowNameArgument = false });
}
// FIXME: document
@ -162,7 +192,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
std::optional<std::string> url;
std::optional<Hash> expectedHash;
state.forceValue(*args[0]);
state.forceValue(*args[0], pos);
if (args[0]->type() == nAttrs) {
@ -206,20 +236,18 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath
: fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
auto path = state.store->toRealPath(storePath);
if (expectedHash) {
auto hash = unpack
? state.store->queryPathInfo(storePath)->narHash
: hashFile(htSHA256, path);
: hashFile(htSHA256, state.store->toRealPath(storePath));
if (hash != *expectedHash)
throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s",
*url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true));
}
if (state.allowedPaths)
state.allowedPaths->insert(path);
state.allowPath(storePath);
auto path = state.store->printStorePath(storePath);
mkString(v, path, PathSet({path}));
}
@ -262,13 +290,13 @@ static RegisterPrimOp primop_fetchTarball({
stdenv.mkDerivation { … }
```
The fetched tarball is cached for a certain amount of time (1 hour
by default) in `~/.cache/nix/tarballs/`. You can change the cache
timeout either on the command line with `--option tarball-ttl number
of seconds` or in the Nix configuration file with this option: `
number of seconds to cache `.
The fetched tarball is cached for a certain amount of time (1
hour by default) in `~/.cache/nix/tarballs/`. You can change the
cache timeout either on the command line with `--tarball-ttl`
*number-of-seconds* or in the Nix configuration file by adding
the line `tarball-ttl = ` *number-of-seconds*.
Note that when obtaining the hash with ` nix-prefetch-url ` the
Note that when obtaining the hash with `nix-prefetch-url` the
option `--unpack` is required.
This function can also verify the contents against a hash. In that
@ -292,7 +320,7 @@ static RegisterPrimOp primop_fetchTarball({
static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v)
{
fetchTree(state, pos, args, v, "git", true);
fetchTree(state, pos, args, v, "git", FetchTreeParams { .emptyRevFallback = true, .allowNameArgument = true });
}
static RegisterPrimOp primop_fetchGit({
@ -368,7 +396,7 @@ static RegisterPrimOp primop_fetchGit({
```
> **Note**
>
>
> It is nice to always specify the branch which a revision
> belongs to. Without the branch being specified, the fetcher
> might fail if the default branch changes. Additionally, it can
@ -405,12 +433,12 @@ static RegisterPrimOp primop_fetchGit({
```
> **Note**
>
>
> Nix will refetch the branch in accordance with
> the option `tarball-ttl`.
> **Note**
>
>
> This behavior is disabled in *Pure evaluation mode*.
)",
.fun = prim_fetchGit,

24
src/libexpr/value-to-json.cc
View file

@ -10,11 +10,11 @@
namespace nix {
void printValueAsJSON(EvalState & state, bool strict,
Value & v, JSONPlaceholder & out, PathSet & context)
Value & v, const Pos & pos, JSONPlaceholder & out, PathSet & context)
{
checkInterrupt();
if (strict) state.forceValue(v);
if (strict) state.forceValue(v, pos);
switch (v.type()) {
@ -40,7 +40,7 @@ void printValueAsJSON(EvalState & state, bool strict,
break;
case nAttrs: {
auto maybeString = state.tryAttrsToString(noPos, v, context, false, false);
auto maybeString = state.tryAttrsToString(pos, v, context, false, false);
if (maybeString) {
out.write(*maybeString);
break;
@ -54,10 +54,10 @@ void printValueAsJSON(EvalState & state, bool strict,
for (auto & j : names) {
Attr & a(*v.attrs->find(state.symbols.create(j)));
auto placeholder(obj.placeholder(j));
printValueAsJSON(state, strict, *a.value, placeholder, context);
printValueAsJSON(state, strict, *a.value, *a.pos, placeholder, context);
}
} else
printValueAsJSON(state, strict, *i->value, out, context);
printValueAsJSON(state, strict, *i->value, *i->pos, out, context);
break;
}
@ -65,7 +65,7 @@ void printValueAsJSON(EvalState & state, bool strict,
auto list(out.list());
for (unsigned int n = 0; n < v.listSize(); ++n) {
auto placeholder(list.placeholder());
printValueAsJSON(state, strict, *v.listElems()[n], placeholder, context);
printValueAsJSON(state, strict, *v.listElems()[n], pos, placeholder, context);
}
break;
}
@ -79,18 +79,20 @@ void printValueAsJSON(EvalState & state, bool strict,
break;
case nThunk:
throw TypeError("cannot convert %1% to JSON", showType(v));
case nFunction:
throw TypeError("cannot convert %1% to JSON", showType(v));
auto e = TypeError({
.msg = hintfmt("cannot convert %1% to JSON", showType(v)),
.errPos = v.determinePos(pos)
});
throw e.addTrace(pos, hintfmt("message for the trace"));
}
}
void printValueAsJSON(EvalState & state, bool strict,
Value & v, std::ostream & str, PathSet & context)
Value & v, const Pos & pos, std::ostream & str, PathSet & context)
{
JSONPlaceholder out(str);
printValueAsJSON(state, strict, v, out, context);
printValueAsJSON(state, strict, v, pos, out, context);
}
void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict,

4
src/libexpr/value-to-json.hh
View file

@ -11,9 +11,9 @@ namespace nix {
class JSONPlaceholder;
void printValueAsJSON(EvalState & state, bool strict,
Value & v, JSONPlaceholder & out, PathSet & context);
Value & v, const Pos & pos, JSONPlaceholder & out, PathSet & context);
void printValueAsJSON(EvalState & state, bool strict,
Value & v, std::ostream & str, PathSet & context);
Value & v, const Pos & pos, std::ostream & str, PathSet & context);
}

29
src/libexpr/value-to-xml.cc
View file

@ -18,7 +18,8 @@ static XMLAttrs singletonAttrs(const string & name, const string & value)
static void printValueAsXML(EvalState & state, bool strict, bool location,
Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen);
Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen,
const Pos & pos);
static void posToXML(XMLAttrs & xmlAttrs, const Pos & pos)
@ -42,21 +43,22 @@ static void showAttrs(EvalState & state, bool strict, bool location,
XMLAttrs xmlAttrs;
xmlAttrs["name"] = i;
if (location && a.pos != &noPos) posToXML(xmlAttrs, *a.pos);
if (location && a.pos != ptr(&noPos)) posToXML(xmlAttrs, *a.pos);
XMLOpenElement _(doc, "attr", xmlAttrs);
printValueAsXML(state, strict, location,
*a.value, doc, context, drvsSeen);
*a.value, doc, context, drvsSeen, *a.pos);
}
}
static void printValueAsXML(EvalState & state, bool strict, bool location,
Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen)
Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen,
const Pos & pos)
{
checkInterrupt();
if (strict) state.forceValue(v);
if (strict) state.forceValue(v, pos);
switch (v.type()) {
@ -91,14 +93,14 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
Path drvPath;
a = v.attrs->find(state.sDrvPath);
if (a != v.attrs->end()) {
if (strict) state.forceValue(*a->value);
if (strict) state.forceValue(*a->value, *a->pos);
if (a->value->type() == nString)
xmlAttrs["drvPath"] = drvPath = a->value->string.s;
}
a = v.attrs->find(state.sOutPath);
if (a != v.attrs->end()) {
if (strict) state.forceValue(*a->value);
if (strict) state.forceValue(*a->value, *a->pos);
if (a->value->type() == nString)
xmlAttrs["outPath"] = a->value->string.s;
}
@ -121,7 +123,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
case nList: {
XMLOpenElement _(doc, "list");
for (unsigned int n = 0; n < v.listSize(); ++n)
printValueAsXML(state, strict, location, *v.listElems()[n], doc, context, drvsSeen);
printValueAsXML(state, strict, location, *v.listElems()[n], doc, context, drvsSeen, pos);
break;
}
@ -135,7 +137,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
if (location) posToXML(xmlAttrs, v.lambda.fun->pos);
XMLOpenElement _(doc, "function", xmlAttrs);
if (v.lambda.fun->matchAttrs) {
if (v.lambda.fun->hasFormals()) {
XMLAttrs attrs;
if (!v.lambda.fun->arg.empty()) attrs["name"] = v.lambda.fun->arg;
if (v.lambda.fun->formals->ellipsis) attrs["ellipsis"] = "1";
@ -149,7 +151,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
}
case nExternal:
v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen);
v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen, pos);
break;
case nFloat:
@ -163,19 +165,20 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
void ExternalValueBase::printValueAsXML(EvalState & state, bool strict,
bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) const
bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen,
const Pos & pos) const
{
doc.writeEmptyElement("unevaluated");
}
void printValueAsXML(EvalState & state, bool strict, bool location,
Value & v, std::ostream & out, PathSet & context)
Value & v, std::ostream & out, PathSet & context, const Pos & pos)
{
XMLWriter doc(true, out);
XMLOpenElement root(doc, "expr");
PathSet drvsSeen;
printValueAsXML(state, strict, location, v, doc, context, drvsSeen);
printValueAsXML(state, strict, location, v, doc, context, drvsSeen, pos);
}

4
src/libexpr/value-to-xml.hh
View file

@ -9,6 +9,6 @@
namespace nix {
void printValueAsXML(EvalState & state, bool strict, bool location,
Value & v, std::ostream & out, PathSet & context);
Value & v, std::ostream & out, PathSet & context, const Pos & pos);
}

3
src/libexpr/value.hh
View file

@ -94,7 +94,8 @@ class ExternalValueBase
/* Print the value as XML. Defaults to unevaluated */
virtual void printValueAsXML(EvalState & state, bool strict, bool location,
XMLWriter & doc, PathSet & context, PathSet & drvsSeen) const;
XMLWriter & doc, PathSet & context, PathSet & drvsSeen,
const Pos & pos) const;
virtual ~ExternalValueBase()
{

2
src/libfetchers/attrs.hh
View file

@ -6,6 +6,8 @@
#include <nlohmann/json_fwd.hpp>
#include <optional>
namespace nix::fetchers {
typedef std::variant<std::string, uint64_t, Explicit<bool>> Attr;

7
src/libfetchers/fetchers.cc
View file

@ -200,12 +200,17 @@ void Input::markChangedFile(
return scheme->markChangedFile(*this, file, commitMsg);
}
std::string Input::getName() const
{
return maybeGetStrAttr(attrs, "name").value_or("source");
}
StorePath Input::computeStorePath(Store & store) const
{
auto narHash = getNarHash();
if (!narHash)
throw Error("cannot compute store path for mutable input '%s'", to_string());
return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source");
return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, getName());
}
std::string Input::getType() const

5
src/libfetchers/fetchers.hh
View file

@ -38,6 +38,9 @@ struct Input
bool immutable = false;
bool direct = true;
/* path of the parent of this input, used for relative path resolution */
std::optional<Path> parent;
public:
static Input fromURL(const std::string & url);
@ -81,6 +84,8 @@ public:
std::string_view file,
std::optional<std::string> commitMsg) const;
std::string getName() const;
StorePath computeStorePath(Store & store) const;
// Convenience functions for common attributes.

52
src/libfetchers/git.cc
View file

@ -4,6 +4,7 @@
#include "tarfile.hh"
#include "store-api.hh"
#include "url-parts.hh"
#include "pathlocks.hh"
#include <sys/time.h>
#include <sys/wait.h>
@ -12,6 +13,12 @@ using namespace std::string_literals;
namespace nix::fetchers {
// Explicit initial branch of our bare repo to suppress warnings from new version of git.
// The value itself does not matter, since we always fetch a specific revision or branch.
// It is set with `-c init.defaultBranch=` instead of `--initial-branch=` to stay compatible with
// old version of git, which will ignore unrecognized `-c` options.
const std::string gitInitialBranch = "__nix_dummy_branch";
static std::string readHead(const Path & path)
{
return chomp(runProgram("git", true, { "-C", path, "rev-parse", "--abbrev-ref", "HEAD" }));
@ -44,7 +51,7 @@ struct GitInputScheme : InputScheme
for (auto &[name, value] : url.query) {
if (name == "rev" || name == "ref")
attrs.emplace(name, value);
else if (name == "shallow")
else if (name == "shallow" || name == "submodules")
attrs.emplace(name, Explicit<bool> { value == "1" });
else
url2.query.emplace(name, value);
@ -60,7 +67,7 @@ struct GitInputScheme : InputScheme
if (maybeGetStrAttr(attrs, "type") != "git") return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs")
if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name")
throw Error("unsupported Git input attribute '%s'", name);
parseURL(getStrAttr(attrs, "url"));
@ -167,10 +174,10 @@ struct GitInputScheme : InputScheme
std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
{
auto name = "source";
Input input(_input);
std::string name = input.getName();
bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false);
@ -270,7 +277,7 @@ struct GitInputScheme : InputScheme
return files.count(file);
};
auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
auto storePath = store->addToStore(input.getName(), actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
// FIXME: maybe we should use the timestamp of the last
// modified dirty file?
@ -317,9 +324,11 @@ struct GitInputScheme : InputScheme
Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false);
repoDir = cacheDir;
createDirs(dirOf(cacheDir));
PathLocks cacheDirLock({cacheDir + ".lock"});
if (!pathExists(cacheDir)) {
createDirs(dirOf(cacheDir));
runProgram("git", true, { "init", "--bare", repoDir });
runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", "--bare", repoDir });
}
Path localRefFile =
@ -386,6 +395,8 @@ struct GitInputScheme : InputScheme
if (!input.getRev())
input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev());
// cache dir lock is removed at scope end; we will only use read-only operations on specific revisions in the remainder
}
bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true";
@ -406,17 +417,14 @@ struct GitInputScheme : InputScheme
AutoDelete delTmpDir(tmpDir, true);
PathFilter filter = defaultPathFilter;
RunOptions checkCommitOpts(
"git",
{ "-C", repoDir, "cat-file", "commit", input.getRev()->gitRev() }
);
checkCommitOpts.searchPath = true;
checkCommitOpts.mergeStderrToStdout = true;
auto result = runProgram(checkCommitOpts);
auto result = runProgram(RunOptions {
.program = "git",
.args = { "-C", repoDir, "cat-file", "commit", input.getRev()->gitRev() },
.mergeStderrToStdout = true
});
if (WEXITSTATUS(result.first) == 128
&& result.second.find("bad file") != std::string::npos
) {
&& result.second.find("bad file") != std::string::npos)
{
throw Error(
"Cannot find Git revision '%s' in ref '%s' of repository '%s'! "
"Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the "
@ -432,7 +440,7 @@ struct GitInputScheme : InputScheme
Path tmpGitDir = createTempDir();
AutoDelete delTmpGitDir(tmpGitDir, true);
runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir });
runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", tmpDir, "--separate-git-dir", tmpGitDir });
// TODO: repoDir might lack the ref (it only checks if rev
// exists, see FIXME above) so use a big hammer and fetch
// everything to ensure we get the rev.
@ -448,9 +456,11 @@ struct GitInputScheme : InputScheme
// FIXME: should pipe this, or find some better way to extract a
// revision.
auto source = sinkToSource([&](Sink & sink) {
RunOptions gitOptions("git", { "-C", repoDir, "archive", input.getRev()->gitRev() });
gitOptions.standardOut = &sink;
runProgram2(gitOptions);
runProgram2({
.program = "git",
.args = { "-C", repoDir, "archive", input.getRev()->gitRev() },
.standardOut = &sink
});
});
unpackTarfile(*source, tmpDir);

12
src/libfetchers/github.cc
View file

@ -207,7 +207,7 @@ struct GitArchiveInputScheme : InputScheme
auto url = getDownloadUrl(input);
auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, url.headers);
auto [tree, lastModified] = downloadTarball(store, url.url, input.getName(), true, url.headers);
input.attrs.insert_or_assign("lastModified", uint64_t(lastModified));
@ -273,9 +273,9 @@ struct GitHubInputScheme : GitArchiveInputScheme
void clone(const Input & input, const Path & destDir) override
{
auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
Input::fromURL(fmt("git+https://%s/%s/%s.git",
host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
.applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
.applyOverrides(input.getRef(), input.getRev())
.clone(destDir);
}
};
@ -300,7 +300,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
if ("PAT" == token.substr(0, fldsplit))
return std::make_pair("Private-token", token.substr(fldsplit+1));
warn("Unrecognized GitLab token type %s", token.substr(0, fldsplit));
return std::nullopt;
return std::make_pair(token.substr(0,fldsplit), token.substr(fldsplit+1));
}
Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
@ -341,9 +341,9 @@ struct GitLabInputScheme : GitArchiveInputScheme
{
auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
// FIXME: get username somewhere
Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
Input::fromURL(fmt("git+https://%s/%s/%s.git",
host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
.applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
.applyOverrides(input.getRef(), input.getRev())
.clone(destDir);
}
};

2
src/libfetchers/local.mk
View file

@ -8,4 +8,6 @@ libfetchers_SOURCES := $(wildcard $(d)/*.cc)
libfetchers_CXXFLAGS += -I src/libutil -I src/libstore
libfetchers_LDFLAGS += -pthread
libfetchers_LIBS = libutil libstore

50
src/libfetchers/mercurial.cc
View file

@ -11,34 +11,32 @@ using namespace std::string_literals;
namespace nix::fetchers {
namespace {
static RunOptions hgOptions(const Strings & args)
{
auto env = getEnv();
// Set HGPLAIN: this means we get consistent output from hg and avoids leakage from a user or system .hgrc.
env["HGPLAIN"] = "";
RunOptions hgOptions(const Strings & args) {
RunOptions opts("hg", args);
opts.searchPath = true;
auto env = getEnv();
// Set HGPLAIN: this means we get consistent output from hg and avoids leakage from a user or system .hgrc.
env["HGPLAIN"] = "";
opts.environment = env;
return opts;
return {
.program = "hg",
.searchPath = true,
.args = args,
.environment = env
};
}
// runProgram wrapper that uses hgOptions instead of stock RunOptions.
string runHg(const Strings & args, const std::optional<std::string> & input = {})
static string runHg(const Strings & args, const std::optional<std::string> & input = {})
{
RunOptions opts = hgOptions(args);
opts.input = input;
RunOptions opts = hgOptions(args);
opts.input = input;
auto res = runProgram(opts);
auto res = runProgram(std::move(opts));
if (!statusOk(res.first))
throw ExecError(res.first, fmt("hg %1%", statusToString(res.first)));
return res.second;
}
if (!statusOk(res.first))
throw ExecError(res.first, fmt("hg %1%", statusToString(res.first)));
return res.second;
}
struct MercurialInputScheme : InputScheme
@ -74,7 +72,7 @@ struct MercurialInputScheme : InputScheme
if (maybeGetStrAttr(attrs, "type") != "hg") return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash")
if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash" && name != "name")
throw Error("unsupported Mercurial input attribute '%s'", name);
parseURL(getStrAttr(attrs, "url"));
@ -147,10 +145,10 @@ struct MercurialInputScheme : InputScheme
std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
{
auto name = "source";
Input input(_input);
auto name = input.getName();
auto [isLocal, actualUrl_] = getActualUrl(input);
auto actualUrl = actualUrl_; // work around clang bug
@ -193,7 +191,7 @@ struct MercurialInputScheme : InputScheme
return files.count(file);
};
auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
auto storePath = store->addToStore(input.getName(), actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
return {
Tree(store->toRealPath(storePath), std::move(storePath)),
@ -253,9 +251,7 @@ struct MercurialInputScheme : InputScheme
have to pull again. */
if (!(input.getRev()
&& pathExists(cacheDir)
&& runProgram(
hgOptions({ "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" })
.killStderr(true)).second == "1"))
&& runProgram(hgOptions({ "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" })).second == "1"))
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl));

26
src/libfetchers/path.cc
View file

@ -82,18 +82,38 @@ struct PathInputScheme : InputScheme
std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
{
std::string absPath;
auto path = getStrAttr(input.attrs, "path");
// FIXME: check whether access to 'path' is allowed.
if (path[0] != '/') {
if (!input.parent)
throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string());
auto storePath = store->maybeParseStorePath(path);
auto parent = canonPath(*input.parent);
// the path isn't relative, prefix it
absPath = nix::absPath(path, parent);
// for security, ensure that if the parent is a store path, it's inside it
if (store->isInStore(parent)) {
auto storePath = store->printStorePath(store->toStorePath(parent).first);
if (!isInDir(absPath, storePath))
throw BadStorePath("relative path '%s' points outside of its parent's store path '%s'", path, storePath);
}
} else
absPath = path;
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s'", absPath));
// FIXME: check whether access to 'path' is allowed.
auto storePath = store->maybeParseStorePath(absPath);
if (storePath)
store->addTempRoot(*storePath);
if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath))
// FIXME: try to substitute storePath.
storePath = store->addToStore("source", path);
storePath = store->addToStore("source", absPath);
return {
Tree(store->toRealPath(*storePath), std::move(*storePath)),

7
src/libfetchers/registry.cc
View file

@ -124,6 +124,13 @@ std::shared_ptr<Registry> getUserRegistry()
return userRegistry;
}
std::shared_ptr<Registry> getCustomRegistry(const Path & p)
{
static auto customRegistry =
Registry::read(p, Registry::Custom);
return customRegistry;
}
static std::shared_ptr<Registry> flagRegistry =
std::make_shared<Registry>(Registry::Flag);

3
src/libfetchers/registry.hh
View file

@ -14,6 +14,7 @@ struct Registry
User = 1,
System = 2,
Global = 3,
Custom = 4,
};
RegistryType type;
@ -48,6 +49,8 @@ typedef std::vector<std::shared_ptr<Registry>> Registries;
std::shared_ptr<Registry> getUserRegistry();
std::shared_ptr<Registry> getCustomRegistry(const Path & p);
Path getUserRegistryPath();
Registries getRegistries(ref<Store> store);

7
src/libfetchers/tarball.cc
View file

@ -178,7 +178,8 @@ struct TarballInputScheme : InputScheme
&& !hasSuffix(url.path, ".tar")
&& !hasSuffix(url.path, ".tar.gz")
&& !hasSuffix(url.path, ".tar.xz")
&& !hasSuffix(url.path, ".tar.bz2"))
&& !hasSuffix(url.path, ".tar.bz2")
&& !hasSuffix(url.path, ".tar.zst"))
return {};
Input input;
@ -195,7 +196,7 @@ struct TarballInputScheme : InputScheme
if (maybeGetStrAttr(attrs, "type") != "tarball") return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash")
if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash" && name != "name")
throw Error("unsupported tarball input attribute '%s'", name);
Input input;
@ -225,7 +226,7 @@ struct TarballInputScheme : InputScheme
std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
{
auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first;
auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), input.getName(), false).first;
return {std::move(tree), input};
}
};

4
src/libmain/local.mk
View file

@ -8,10 +8,10 @@ libmain_SOURCES := $(wildcard $(d)/*.cc)
libmain_CXXFLAGS += -I src/libutil -I src/libstore
libmain_LDFLAGS = $(OPENSSL_LIBS)
libmain_LDFLAGS += $(OPENSSL_LIBS)
libmain_LIBS = libstore libutil
libmain_ALLOW_UNDEFINED = 1
$(eval $(call install-file-in, $(d)/nix-main.pc, $(prefix)/lib/pkgconfig, 0644))
$(eval $(call install-file-in, $(d)/nix-main.pc, $(libdir)/pkgconfig, 0644))

18
src/libmain/progress-bar.cc
View file

@ -103,17 +103,19 @@ public:
~ProgressBar()
{
stop();
updateThread.join();
}
void stop() override
{
auto state(state_.lock());
if (!state->active) return;
state->active = false;
writeToStderr("\r\e[K");
updateCV.notify_one();
quitCV.notify_one();
{
auto state(state_.lock());
if (!state->active) return;
state->active = false;
writeToStderr("\r\e[K");
updateCV.notify_one();
quitCV.notify_one();
}
updateThread.join();
}
bool isVerbose() override {
@ -484,7 +486,7 @@ Logger * makeProgressBar(bool printBuildLogs)
{
return new ProgressBar(
printBuildLogs,
isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb"
shouldANSI()
);
}

32
src/libmain/shared.cc
View file

@ -15,6 +15,9 @@
#include <sys/stat.h>
#include <unistd.h>
#include <signal.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netdb.h>
#include <openssl/crypto.h>
@ -110,6 +113,31 @@ static void opensslLockCallback(int mode, int type, const char * file, int line)
}
#endif
static std::once_flag dns_resolve_flag;
static void preloadNSS() {
/* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of
one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already
been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to
load its lookup libraries in the parent before any child gets a chance to. */
std::call_once(dns_resolve_flag, []() {
struct addrinfo *res = NULL;
/* nss will only force the "local" (not through nscd) dns resolution if its on the LOCALDOMAIN.
We need the resolution to be done locally, as nscd socket will not be accessible in the
sandbox. */
char * previous_env = getenv("LOCALDOMAIN");
setenv("LOCALDOMAIN", "invalid", 1);
if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) == 0) {
if (res) freeaddrinfo(res);
}
if (previous_env) {
setenv("LOCALDOMAIN", previous_env, 1);
} else {
unsetenv("LOCALDOMAIN");
}
});
}
static void sigHandler(int signo) { }
@ -176,6 +204,8 @@ void initNix()
if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/"))
unsetenv("TMPDIR");
#endif
preloadNSS();
}
@ -238,7 +268,7 @@ LegacyArgs::LegacyArgs(const std::string & programName,
addFlag({
.longName = "no-gc-warning",
.description = "Disable warnings about not using `--add-root`.",
.handler = {&gcWarning, true},
.handler = {&gcWarning, false},
});
addFlag({

65
src/libstore/binary-cache-store.cc
View file

@ -52,9 +52,9 @@ void BinaryCacheStore::init()
throw Error("binary cache '%s' is for Nix stores with prefix '%s', not '%s'",
getUri(), value, storeDir);
} else if (name == "WantMassQuery") {
wantMassQuery.setDefault(value == "1" ? "true" : "false");
wantMassQuery.setDefault(value == "1");
} else if (name == "Priority") {
priority.setDefault(fmt("%d", std::stoi(value)));
priority.setDefault(std::stoi(value));
}
}
}
@ -111,15 +111,15 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo");
std::string hashPart(narInfo->path.hashPart());
{
auto state_(state.lock());
state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = std::shared_ptr<NarInfo>(narInfo) });
state_->pathInfoCache.upsert(
std::string(narInfo->path.to_string()),
PathInfoCacheValue { .value = std::shared_ptr<NarInfo>(narInfo) });
}
if (diskCache)
diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
diskCache->upsertNarInfo(getUri(), std::string(narInfo->path.hashPart()), std::shared_ptr<NarInfo>(narInfo));
}
AutoCloseFD openFile(const Path & path)
@ -130,17 +130,6 @@ AutoCloseFD openFile(const Path & path)
return fd;
}
struct FileSource : FdSource
{
AutoCloseFD fd2;
FileSource(const Path & path)
: fd2(openFile(path))
{
fd = fd2.get();
}
};
ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs,
std::function<ValidPathInfo(HashResult)> mkInfo)
@ -160,7 +149,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
{
FdSink fileSink(fdTemp.get());
TeeSink teeSinkCompressed { fileSink, fileHashSink };
auto compressionSink = makeCompressionSink(compression, teeSinkCompressed);
auto compressionSink = makeCompressionSink(compression, teeSinkCompressed, parallelCompression, compressionLevel);
TeeSink teeSinkUncompressed { *compressionSink, narHashSink };
TeeSource teeSource { narSource, teeSinkUncompressed };
narAccessor = makeNarAccessor(teeSource);
@ -319,16 +308,17 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
}
StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references)
{
if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256)
unsupported("addToStoreFromDump");
return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info {
makeFixedOutputPath(method, nar.first, name),
makeFixedOutputPath(method, nar.first, name, references),
nar.first,
};
info.narSize = nar.second;
info.references = references;
return info;
})->path;
}
@ -396,7 +386,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
}
StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair, const StorePathSet & references)
{
/* FIXME: Make BinaryCacheStore::addToStoreCommon support
non-recursive+sha256 so we can just use the default
@ -415,10 +405,11 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
});
return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info {
makeFixedOutputPath(method, h, name),
makeFixedOutputPath(method, h, name, references),
nar.first,
};
info.narSize = nar.second;
info.references = references;
info.ca = FixedOutputHash {
.method = method,
.hash = h,
@ -448,20 +439,34 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
})->path;
}
std::optional<const Realisation> BinaryCacheStore::queryRealisation(const DrvOutput & id)
void BinaryCacheStore::queryRealisationUncached(const DrvOutput & id,
Callback<std::shared_ptr<const Realisation>> callback) noexcept
{
auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi";
auto rawOutputInfo = getFile(outputInfoFilePath);
if (rawOutputInfo) {
return {Realisation::fromJSON(
nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath)};
} else {
return std::nullopt;
}
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
Callback<std::shared_ptr<std::string>> newCallback = {
[=](std::future<std::shared_ptr<std::string>> fut) {
try {
auto data = fut.get();
if (!data) return (*callbackPtr)(nullptr);
auto realisation = Realisation::fromJSON(
nlohmann::json::parse(*data), outputInfoFilePath);
return (*callbackPtr)(std::make_shared<const Realisation>(realisation));
} catch (...) {
callbackPtr->rethrow();
}
}
};
getFile(outputInfoFilePath, std::move(newCallback));
}
void BinaryCacheStore::registerDrvOutput(const Realisation& info) {
if (diskCache)
diskCache->upsertRealisation(getUri(), info);
auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi";
upsertFile(filePath, info.toJSON().dump(), "application/json");
}

17
src/libstore/binary-cache-store.hh
View file

@ -15,13 +15,17 @@ struct BinaryCacheStoreConfig : virtual StoreConfig
{
using StoreConfig::StoreConfig;
const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"};
const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', 'gzip', 'zstd', or 'none')"};
const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"};
const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key", "path to secret key used to sign the binary cache"};
const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache", "path to a local cache of NARs"};
const Setting<bool> parallelCompression{(StoreConfig*) this, false, "parallel-compression",
"enable multi-threading compression, available for xz only currently"};
"enable multi-threading compression for NARs, available for xz and zstd only currently"};
const Setting<int> compressionLevel{(StoreConfig*) this, -1, "compression-level",
"specify 'preset level' of compression to be used with NARs: "
"meaning and accepted range of values depends on compression method selected, "
"other than -1 which we reserve to indicate Nix defaults should be used"};
};
class BinaryCacheStore : public virtual BinaryCacheStoreConfig, public virtual Store
@ -34,7 +38,7 @@ private:
protected:
// The prefix under which realisation infos will be stored
const std::string realisationsPrefix = "/realisations";
const std::string realisationsPrefix = "realisations";
BinaryCacheStore(const Params & params);
@ -93,18 +97,19 @@ public:
RepairFlag repair, CheckSigsFlag checkSigs) override;
StorePath addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override;
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references ) override;
StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo,
PathFilter & filter, RepairFlag repair) override;
PathFilter & filter, RepairFlag repair, const StorePathSet & references) override;
StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override;
void registerDrvOutput(const Realisation & info) override;
std::optional<const Realisation> queryRealisation(const DrvOutput &) override;
void queryRealisationUncached(const DrvOutput &,
Callback<std::shared_ptr<const Realisation>> callback) noexcept override;
void narFromPath(const StorePath & path, Sink & sink) override;

215
src/libstore/build/derivation-goal.cc
View file

@ -143,7 +143,6 @@ void DerivationGoal::work()
(this->*state)();
}
void DerivationGoal::addWantedOutputs(const StringSet & outputs)
{
/* If we already want all outputs, there is nothing to do. */
@ -166,7 +165,7 @@ void DerivationGoal::getDerivation()
/* The first thing to do is to make sure that the derivation
exists. If it doesn't, it may be created through a
substitute. */
if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) {
if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) {
loadDerivation();
return;
}
@ -189,12 +188,12 @@ void DerivationGoal::loadDerivation()
/* `drvPath' should already be a root, but let's be on the safe
side: if the user forgot to make it a root, we wouldn't want
things being garbage collected while we're busy. */
worker.store.addTempRoot(drvPath);
worker.evalStore.addTempRoot(drvPath);
assert(worker.store.isValidPath(drvPath));
assert(worker.evalStore.isValidPath(drvPath));
/* Get the derivation. */
drv = std::make_unique<Derivation>(worker.store.derivationFromPath(drvPath));
drv = std::make_unique<Derivation>(worker.evalStore.derivationFromPath(drvPath));
haveDerivation();
}
@ -205,7 +204,7 @@ void DerivationGoal::haveDerivation()
trace("have derivation");
if (drv->type() == DerivationType::CAFloating)
settings.requireExperimentalFeature("ca-derivations");
settings.requireExperimentalFeature(Xp::CaDerivations);
retrySubstitution = false;
@ -213,8 +212,8 @@ void DerivationGoal::haveDerivation()
if (i.second.second)
worker.store.addTempRoot(*i.second.second);
auto outputHashes = staticOutputHashes(worker.store, *drv);
for (auto &[outputName, outputHash] : outputHashes)
auto outputHashes = staticOutputHashes(worker.evalStore, *drv);
for (auto & [outputName, outputHash] : outputHashes)
initialOutputs.insert({
outputName,
InitialOutput{
@ -338,6 +337,15 @@ void DerivationGoal::gaveUpOnSubstitution()
for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs)
addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal));
/* Copy the input sources from the eval store to the build
store. */
if (&worker.evalStore != &worker.store) {
RealisedPath::Set inputSrcs;
for (auto & i : drv->inputSrcs)
inputSrcs.insert(i);
copyClosure(worker.evalStore, worker.store, inputSrcs);
}
for (auto & i : drv->inputSrcs) {
if (worker.store.isValidPath(i)) continue;
if (!settings.useSubstitutes)
@ -445,7 +453,7 @@ void DerivationGoal::inputsRealised()
if (useDerivation) {
auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
if (settings.isExperimentalFeatureEnabled("ca-derivations") &&
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) &&
((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type()))
|| fullDrv.type() == DerivationType::DeferredInputAddressed)) {
/* We are be able to resolve this derivation based on the
@ -479,8 +487,8 @@ void DerivationGoal::inputsRealised()
/* Add the relevant output closures of the input derivation
`i' as input paths. Only add the closures of output paths
that are specified as inputs. */
assert(worker.store.isValidPath(drvPath));
auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath);
assert(worker.evalStore.isValidPath(drvPath));
auto outputs = worker.evalStore.queryPartialDerivationOutputMap(depDrvPath);
for (auto & j : wantedDepOutputs) {
if (outputs.count(j) > 0) {
auto optRealizedInput = outputs.at(j);
@ -545,7 +553,7 @@ void DerivationGoal::tryToBuild()
PathSet lockFiles;
/* FIXME: Should lock something like the drv itself so we don't build same
CA drv concurrently */
if (dynamic_cast<LocalStore *>(&worker.store))
if (dynamic_cast<LocalStore *>(&worker.store)) {
/* If we aren't a local store, we might need to use the local store as
a build remote, but that would cause a deadlock. */
/* FIXME: Make it so we can use ourselves as a build remote even if we
@ -553,9 +561,15 @@ void DerivationGoal::tryToBuild()
/* FIXME: find some way to lock for scheduling for the other stores so
a forking daemon with --store still won't farm out redundant builds.
*/
for (auto & i : drv->outputsAndOptPaths(worker.store))
for (auto & i : drv->outputsAndOptPaths(worker.store)) {
if (i.second.second)
lockFiles.insert(worker.store.Store::toRealPath(*i.second.second));
else
lockFiles.insert(
worker.store.Store::toRealPath(drvPath) + "." + i.first
);
}
}
if (!outputLocks.lockPaths(lockFiles, "", false)) {
if (!actLock)
@ -602,7 +616,9 @@ void DerivationGoal::tryToBuild()
/* Don't do a remote build if the derivation has the attribute
`preferLocalBuild' set. Also, check and repair modes are only
supported for local builds. */
bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
bool buildLocally =
(buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store))
&& settings.maxBuildJobs.get() != 0;
if (!buildLocally) {
switch (tryBuildHook()) {
@ -739,6 +755,64 @@ void DerivationGoal::cleanupPostOutputsRegisteredModeNonCheck()
{
}
void runPostBuildHook(
Store & store,
Logger & logger,
const StorePath & drvPath,
StorePathSet outputPaths
)
{
auto hook = settings.postBuildHook;
if (hook == "")
return;
Activity act(logger, lvlInfo, actPostBuildHook,
fmt("running post-build-hook '%s'", settings.postBuildHook),
Logger::Fields{store.printStorePath(drvPath)});
PushActivity pact(act.id);
std::map<std::string, std::string> hookEnvironment = getEnv();
hookEnvironment.emplace("DRV_PATH", store.printStorePath(drvPath));
hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", store.printStorePathSet(outputPaths))));
hookEnvironment.emplace("NIX_CONFIG", globalConfig.toKeyValue());
struct LogSink : Sink {
Activity & act;
std::string currentLine;
LogSink(Activity & act) : act(act) { }
void operator() (std::string_view data) override {
for (auto c : data) {
if (c == '\n') {
flushLine();
} else {
currentLine += c;
}
}
}
void flushLine() {
act.result(resPostBuildLogLine, currentLine);
currentLine.clear();
}
~LogSink() {
if (currentLine != "") {
currentLine += '\n';
flushLine();
}
}
};
LogSink sink(act);
runProgram2({
.program = settings.postBuildHook,
.environment = hookEnvironment,
.standardOut = &sink,
.mergeStderrToStdout = true,
});
}
void DerivationGoal::buildDone()
{
@ -804,57 +878,15 @@ void DerivationGoal::buildDone()
being valid. */
registerOutputs();
if (settings.postBuildHook != "") {
Activity act(*logger, lvlInfo, actPostBuildHook,
fmt("running post-build-hook '%s'", settings.postBuildHook),
Logger::Fields{worker.store.printStorePath(drvPath)});
PushActivity pact(act.id);
StorePathSet outputPaths;
for (auto i : drv->outputs) {
outputPaths.insert(finalOutputs.at(i.first));
}
std::map<std::string, std::string> hookEnvironment = getEnv();
hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath));
hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths))));
RunOptions opts(settings.postBuildHook, {});
opts.environment = hookEnvironment;
struct LogSink : Sink {
Activity & act;
std::string currentLine;
LogSink(Activity & act) : act(act) { }
void operator() (std::string_view data) override {
for (auto c : data) {
if (c == '\n') {
flushLine();
} else {
currentLine += c;
}
}
}
void flushLine() {
act.result(resPostBuildLogLine, currentLine);
currentLine.clear();
}
~LogSink() {
if (currentLine != "") {
currentLine += '\n';
flushLine();
}
}
};
LogSink sink(act);
opts.standardOut = &sink;
opts.mergeStderrToStdout = true;
runProgram2(opts);
}
StorePathSet outputPaths;
for (auto & [_, path] : finalOutputs)
outputPaths.insert(path);
runPostBuildHook(
worker.store,
*logger,
drvPath,
outputPaths
);
if (buildMode == bmCheck) {
cleanupPostOutputsRegisteredModeCheck();
@ -910,6 +942,8 @@ void DerivationGoal::resolvedFinished() {
auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv);
StorePathSet outputPaths;
// `wantedOutputs` might be empty, which means “all the outputs”
auto realWantedOutputs = wantedOutputs;
if (realWantedOutputs.empty())
@ -927,8 +961,10 @@ void DerivationGoal::resolvedFinished() {
auto newRealisation = *realisation;
newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput};
newRealisation.signatures.clear();
newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath);
signRealisation(newRealisation);
worker.store.registerDrvOutput(newRealisation);
outputPaths.insert(realisation->outPath);
} else {
// If we don't have a realisation, then it must mean that something
// failed when building the resolved drv
@ -936,6 +972,13 @@ void DerivationGoal::resolvedFinished() {
}
}
runPostBuildHook(
worker.store,
*logger,
drvPath,
outputPaths
);
// This is potentially a bit fishy in terms of error reporting. Not sure
// how to do it in a cleaner way
amDone(nrFailed == 0 ? ecSuccess : ecFailed, ex);
@ -968,7 +1011,7 @@ HookReply DerivationGoal::tryBuildHook()
return readLine(worker.hook->fromHook.readSide.get());
} catch (Error & e) {
e.addTrace({}, "while reading the response from the build hook");
throw e;
throw;
}
}();
if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true))
@ -1014,7 +1057,7 @@ HookReply DerivationGoal::tryBuildHook()
machineName = readLine(hook->fromHook.readSide.get());
} catch (Error & e) {
e.addTrace({}, "while reading the machine name from the build hook");
throw e;
throw;
}
/* Tell the hook all the inputs that have to be copied to the
@ -1048,42 +1091,6 @@ HookReply DerivationGoal::tryBuildHook()
}
StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
{
StorePathSet paths;
for (auto & storePath : storePaths) {
if (!inputPaths.count(storePath))
throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath));
worker.store.computeFSClosure({storePath}, paths);
}
/* If there are derivations in the graph, then include their
outputs as well. This is useful if you want to do things
like passing all build-time dependencies of some path to a
derivation that builds a NixOS DVD image. */
auto paths2 = paths;
for (auto & j : paths2) {
if (j.isDerivation()) {
Derivation drv = worker.store.derivationFromPath(j);
for (auto & k : drv.outputsAndOptPaths(worker.store)) {
if (!k.second.second)
/* FIXME: I am confused why we are calling
`computeFSClosure` on the output path, rather than
derivation itself. That doesn't seem right to me, so I
won't try to implemented this for CA derivations. */
throw UnimplementedError("exportReferences on CA derivations is not yet implemented");
worker.store.computeFSClosure(*k.second.second, paths);
}
}
}
return paths;
}
void DerivationGoal::registerOutputs()
{
/* When using a build hook, the build hook can register the output
@ -1268,7 +1275,7 @@ void DerivationGoal::checkPathValidity()
: PathStatus::Corrupt,
};
}
if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
auto drvOutput = DrvOutput{initialOutputs.at(i.first).outputHash, i.first};
if (auto real = worker.store.queryRealisation(drvOutput)) {
info.known = {

71
src/libstore/build/drv-output-substitution-goal.cc
View file

@ -1,6 +1,8 @@
#include "drv-output-substitution-goal.hh"
#include "finally.hh"
#include "worker.hh"
#include "substitution-goal.hh"
#include "callback.hh"
namespace nix {
@ -17,6 +19,13 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput& id, Worker
void DrvOutputSubstitutionGoal::init()
{
trace("init");
/* If the derivation already exists, we’re done */
if (worker.store.queryRealisation(id)) {
amDone(ecSuccess);
return;
}
subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
tryNext();
}
@ -43,14 +52,62 @@ void DrvOutputSubstitutionGoal::tryNext()
return;
}
auto sub = subs.front();
sub = subs.front();
subs.pop_front();
// FIXME: Make async
outputInfo = sub->queryRealisation(id);
// outputInfo = sub->queryRealisation(id);
outPipe.create();
promise = decltype(promise)();
sub->queryRealisation(
id, { [&](std::future<std::shared_ptr<const Realisation>> res) {
try {
Finally updateStats([this]() { outPipe.writeSide.close(); });
promise.set_value(res.get());
} catch (...) {
promise.set_exception(std::current_exception());
}
} });
worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false);
state = &DrvOutputSubstitutionGoal::realisationFetched;
}
void DrvOutputSubstitutionGoal::realisationFetched()
{
worker.childTerminated(this);
try {
outputInfo = promise.get_future().get();
} catch (std::exception & e) {
printError(e.what());
substituterFailed = true;
}
if (!outputInfo) {
tryNext();
return;
return tryNext();
}
for (const auto & [depId, depPath] : outputInfo->dependentRealisations) {
if (depId != id) {
if (auto localOutputInfo = worker.store.queryRealisation(depId);
localOutputInfo && localOutputInfo->outPath != depPath) {
warn(
"substituter '%s' has an incompatible realisation for '%s', ignoring.\n"
"Local: %s\n"
"Remote: %s",
sub->getUri(),
depId.to_string(),
worker.store.printStorePath(localOutputInfo->outPath),
worker.store.printStorePath(depPath)
);
tryNext();
return;
}
addWaitee(worker.makeDrvOutputSubstitutionGoal(depId));
}
}
addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath));
@ -92,4 +149,10 @@ void DrvOutputSubstitutionGoal::work()
(this->*state)();
}
void DrvOutputSubstitutionGoal::handleEOF(int fd)
{
if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
}
}

14
src/libstore/build/drv-output-substitution-goal.hh
View file

@ -3,6 +3,8 @@
#include "store-api.hh"
#include "goal.hh"
#include "realisation.hh"
#include <thread>
#include <future>
namespace nix {
@ -20,11 +22,18 @@ private:
// The realisation corresponding to the given output id.
// Will be filled once we can get it.
std::optional<Realisation> outputInfo;
std::shared_ptr<const Realisation> outputInfo;
/* The remaining substituters. */
std::list<ref<Store>> subs;
/* The current substituter. */
std::shared_ptr<Store> sub;
Pipe outPipe;
std::thread thr;
std::promise<std::shared_ptr<const Realisation>> promise;
/* Whether a substituter failed. */
bool substituterFailed = false;
@ -36,6 +45,7 @@ public:
void init();
void tryNext();
void realisationFetched();
void outPathValid();
void finished();
@ -44,7 +54,7 @@ public:
string key() override;
void work() override;
void handleEOF(int fd) override;
};
}

19
src/libstore/build/entry-points.cc
View file

@ -1,4 +1,3 @@
#include "machines.hh"
#include "worker.hh"
#include "substitution-goal.hh"
#include "derivation-goal.hh"
@ -6,17 +5,17 @@
namespace nix {
void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMode)
void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMode, std::shared_ptr<Store> evalStore)
{
Worker worker(*this);
Worker worker(*this, evalStore ? *evalStore : *this);
Goals goals;
for (auto & br : reqs) {
for (const auto & br : reqs) {
std::visit(overloaded {
[&](DerivedPath::Built bfd) {
[&](const DerivedPath::Built & bfd) {
goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode));
},
[&](DerivedPath::Opaque bo) {
[&](const DerivedPath::Opaque & bo) {
goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair));
},
}, br.raw());
@ -51,7 +50,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode)
{
Worker worker(*this);
Worker worker(*this, *this);
auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode);
BuildResult result;
@ -74,7 +73,7 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat
outputId,
Realisation{ outputId, *staticOutput.second}
);
if (settings.isExperimentalFeatureEnabled("ca-derivations") && !derivationHasKnownOutputPaths(drv.type())) {
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && !derivationHasKnownOutputPaths(drv.type())) {
auto realisation = this->queryRealisation(outputId);
if (realisation)
result.builtOutputs.insert_or_assign(
@ -93,7 +92,7 @@ void Store::ensurePath(const StorePath & path)
/* If the path is already valid, we're done. */
if (isValidPath(path)) return;
Worker worker(*this);
Worker worker(*this, *this);
GoalPtr goal = worker.makePathSubstitutionGoal(path);
Goals goals = {goal};
@ -111,7 +110,7 @@ void Store::ensurePath(const StorePath & path)
void LocalStore::repairPath(const StorePath & path)
{
Worker worker(*this);
Worker worker(*this, *this);
GoalPtr goal = worker.makePathSubstitutionGoal(path, Repair);
Goals goals = {goal};

13
src/libstore/build/goal.cc
View file

@ -13,11 +13,9 @@ bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
void addToWeakGoals(WeakGoals & goals, GoalPtr p)
{
// FIXME: necessary?
// FIXME: O(n)
for (auto & i : goals)
if (i.lock() == p) return;
goals.push_back(p);
if (goals.find(p) != goals.end())
return;
goals.insert(p);
}
@ -46,10 +44,7 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
/* If we failed and keepGoing is not set, we remove all
remaining waitees. */
for (auto & goal : waitees) {
WeakGoals waiters2;
for (auto & j : goal->waiters)
if (j.lock() != shared_from_this()) waiters2.push_back(j);
goal->waiters = waiters2;
goal->waiters.extract(shared_from_this());
}
waitees.clear();

2
src/libstore/build/goal.hh
View file

@ -19,7 +19,7 @@ struct CompareGoalPtrs {
/* Set of goals. */
typedef set<GoalPtr, CompareGoalPtrs> Goals;
typedef list<WeakGoalPtr> WeakGoals;
typedef set<WeakGoalPtr, std::owner_less<WeakGoalPtr>> WeakGoals;
/* A map of paths to goals (and the other way around). */
typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;

327
src/libstore/build/local-derivation-goal.cc
View file

@ -17,16 +17,14 @@
#include <regex>
#include <queue>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <netdb.h>
#include <fcntl.h>
#include <termios.h>
#include <unistd.h>
#include <sys/mman.h>
#include <sys/utsname.h>
#include <sys/resource.h>
#include <sys/socket.h>
#if HAVE_STATVFS
#include <sys/statvfs.h>
@ -34,7 +32,6 @@
/* Includes required for chroot support. */
#if __linux__
#include <sys/socket.h>
#include <sys/ioctl.h>
#include <net/if.h>
#include <netinet/ip.h>
@ -70,12 +67,14 @@ void handleDiffHook(
auto diffHook = settings.diffHook;
if (diffHook != "" && settings.runDiffHook) {
try {
RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir});
diffHookOptions.searchPath = true;
diffHookOptions.uid = uid;
diffHookOptions.gid = gid;
diffHookOptions.chdir = "/";
auto diffRes = runProgram(diffHookOptions);
auto diffRes = runProgram(RunOptions {
.program = diffHook,
.searchPath = true,
.args = {tryA, tryB, drvPath, tmpDir},
.uid = uid,
.gid = gid,
.chdir = "/"
});
if (!statusOk(diffRes.first))
throw ExecError(diffRes.first,
"diff-hook program '%1%' %2%",
@ -153,6 +152,7 @@ void LocalDerivationGoal::killChild()
void LocalDerivationGoal::tryLocalBuild() {
unsigned int curBuilds = worker.getNrLocalBuilds();
if (curBuilds >= settings.maxBuildJobs) {
state = &DerivationGoal::tryToBuild;
worker.waitForBuildSlot(shared_from_this());
outputLocks.unlock();
return;
@ -291,7 +291,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull()
auto & localStore = getLocalStore();
uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
struct statvfs st;
if (statvfs(localStore.realStoreDir.c_str(), &st) == 0 &&
if (statvfs(localStore.realStoreDir.get().c_str(), &st) == 0 &&
(uint64_t) st.f_bavail * st.f_bsize < required)
diskFull = true;
if (statvfs(tmpDir.c_str(), &st) == 0 &&
@ -342,24 +342,7 @@ int childEntry(void * arg)
return 1;
}
static std::once_flag dns_resolve_flag;
static void preloadNSS() {
/* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of
one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already
been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to
load its lookup libraries in the parent before any child gets a chance to. */
std::call_once(dns_resolve_flag, []() {
struct addrinfo *res = NULL;
if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) {
if (res) freeaddrinfo(res);
}
});
}
#if __linux__
static void linkOrCopy(const Path & from, const Path & to)
{
if (link(from.c_str(), to.c_str()) == -1) {
@ -375,6 +358,7 @@ static void linkOrCopy(const Path & from, const Path & to)
copyPath(from, to);
}
}
#endif
void LocalDerivationGoal::startBuilder()
@ -388,9 +372,6 @@ void LocalDerivationGoal::startBuilder()
settings.thisSystem,
concatStringsSep<StringSet>(", ", worker.store.systemFeatures));
if (drv->isBuiltin())
preloadNSS();
#if __APPLE__
additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or("");
#endif
@ -416,7 +397,7 @@ void LocalDerivationGoal::startBuilder()
}
auto & localStore = getLocalStore();
if (localStore.storeDir != localStore.realStoreDir) {
if (localStore.storeDir != localStore.realStoreDir.get()) {
#if __linux__
useChroot = true;
#else
@ -517,7 +498,7 @@ void LocalDerivationGoal::startBuilder()
/* Write closure info to <fileName>. */
writeFile(tmpDir + "/" + fileName,
worker.store.makeValidityRegistration(
exportReferences({storePath}), false, false));
worker.store.exportReferences({storePath}, inputPaths), false, false));
}
}
@ -581,7 +562,9 @@ void LocalDerivationGoal::startBuilder()
throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps",
worker.store.printStorePath(drvPath), i);
dirsInChroot[i] = i;
/* Allow files in __impureHostDeps to be missing; e.g.
macOS 11+ has no /usr/lib/libSystem*.dylib */
dirsInChroot[i] = {i, true};
}
#if __linux__
@ -729,6 +712,7 @@ void LocalDerivationGoal::startBuilder()
if (!builderOut.readSide)
throw SysError("opening pseudoterminal master");
// FIXME: not thread-safe, use ptsname_r
std::string slaveName(ptsname(builderOut.readSide.get()));
if (buildUser) {
@ -772,7 +756,6 @@ void LocalDerivationGoal::startBuilder()
result.startTime = time(0);
/* Fork a child to build the package. */
ProcessOptions options;
#if __linux__
if (useChroot) {
@ -815,8 +798,6 @@ void LocalDerivationGoal::startBuilder()
userNamespaceSync.create();
options.allowVfork = false;
Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces";
static bool userNamespacesEnabled =
pathExists(maxUserNamespaces)
@ -874,7 +855,7 @@ void LocalDerivationGoal::startBuilder()
writeFull(builderOut.writeSide.get(),
fmt("%d %d\n", usingUserNamespace, child));
_exit(0);
}, options);
});
int res = helper.wait();
if (res != 0 && settings.sandboxFallback) {
@ -937,11 +918,12 @@ void LocalDerivationGoal::startBuilder()
} else
#endif
{
#if __linux__
fallback:
options.allowVfork = !buildUser && !drv->isBuiltin();
#endif
pid = startProcess([&]() {
runChild();
}, options);
});
}
/* parent */
@ -956,9 +938,12 @@ void LocalDerivationGoal::startBuilder()
try {
return readLine(builderOut.readSide.get());
} catch (Error & e) {
e.addTrace({}, "while waiting for the build environment to initialize (previous messages: %s)",
auto status = pid.wait();
e.addTrace({}, "while waiting for the build environment for '%s' to initialize (%s, previous messages: %s)",
worker.store.printStorePath(drvPath),
statusToString(status),
concatStringsSep("|", msgs));
throw e;
throw;
}
}();
if (string(msg, 0, 1) == "\2") break;
@ -1081,123 +1066,38 @@ void LocalDerivationGoal::initEnv()
}
static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
void LocalDerivationGoal::writeStructuredAttrs()
{
auto structuredAttrs = parsedDrv->getStructuredAttrs();
if (!structuredAttrs) return;
if (auto structAttrsJson = parsedDrv->prepareStructuredAttrs(worker.store, inputPaths)) {
auto json = structAttrsJson.value();
nlohmann::json rewritten;
for (auto & [i, v] : json["outputs"].get<nlohmann::json::object_t>()) {
/* The placeholder must have a rewrite, so we use it to cover both the
cases where we know or don't know the output path ahead of time. */
rewritten[i] = rewriteStrings(v, inputRewrites);
}
auto json = *structuredAttrs;
json["outputs"] = rewritten;
/* Add an "outputs" object containing the output paths. */
nlohmann::json outputs;
for (auto & i : drv->outputs) {
/* The placeholder must have a rewrite, so we use it to cover both the
cases where we know or don't know the output path ahead of time. */
outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites);
auto jsonSh = writeStructuredAttrsShell(json);
writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
chownToBuilder(tmpDir + "/.attrs.sh");
env["NIX_ATTRS_SH_FILE"] = tmpDir + "/.attrs.sh";
writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites));
chownToBuilder(tmpDir + "/.attrs.json");
env["NIX_ATTRS_JSON_FILE"] = tmpDir + "/.attrs.json";
}
json["outputs"] = outputs;
/* Handle exportReferencesGraph. */
auto e = json.find("exportReferencesGraph");
if (e != json.end() && e->is_object()) {
for (auto i = e->begin(); i != e->end(); ++i) {
std::ostringstream str;
{
JSONPlaceholder jsonRoot(str, true);
StorePathSet storePaths;
for (auto & p : *i)
storePaths.insert(worker.store.parseStorePath(p.get<std::string>()));
worker.store.pathInfoToJSON(jsonRoot,
exportReferences(storePaths), false, true);
}
json[i.key()] = nlohmann::json::parse(str.str()); // urgh
}
}
writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites));
chownToBuilder(tmpDir + "/.attrs.json");
/* As a convenience to bash scripts, write a shell file that
maps all attributes that are representable in bash -
namely, strings, integers, nulls, Booleans, and arrays and
objects consisting entirely of those values. (So nested
arrays or objects are not supported.) */
auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> {
if (value.is_string())
return shellEscape(value);
if (value.is_number()) {
auto f = value.get<float>();
if (std::ceil(f) == f)
return std::to_string(value.get<int>());
}
if (value.is_null())
return std::string("''");
if (value.is_boolean())
return value.get<bool>() ? std::string("1") : std::string("");
return {};
};
std::string jsonSh;
for (auto i = json.begin(); i != json.end(); ++i) {
if (!std::regex_match(i.key(), shVarName)) continue;
auto & value = i.value();
auto s = handleSimpleType(value);
if (s)
jsonSh += fmt("declare %s=%s\n", i.key(), *s);
else if (value.is_array()) {
std::string s2;
bool good = true;
for (auto i = value.begin(); i != value.end(); ++i) {
auto s3 = handleSimpleType(i.value());
if (!s3) { good = false; break; }
s2 += *s3; s2 += ' ';
}
if (good)
jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2);
}
else if (value.is_object()) {
std::string s2;
bool good = true;
for (auto i = value.begin(); i != value.end(); ++i) {
auto s3 = handleSimpleType(i.value());
if (!s3) { good = false; break; }
s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3);
}
if (good)
jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2);
}
}
writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
chownToBuilder(tmpDir + "/.attrs.sh");
}
static StorePath pathPartOfReq(const DerivedPath & req)
{
return std::visit(overloaded {
[&](DerivedPath::Opaque bo) {
[&](const DerivedPath::Opaque & bo) {
return bo.path;
},
[&](DerivedPath::Built bfd) {
[&](const DerivedPath::Built & bfd) {
return bfd.drvPath;
},
}, req.raw());
@ -1282,7 +1182,8 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair,
const StorePathSet & references = StorePathSet()) override
{ throw Error("addToStore"); }
void addToStore(const ValidPathInfo & info, Source & narSource,
@ -1301,9 +1202,10 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
}
StorePath addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair,
const StorePathSet & references = StorePathSet()) override
{
auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair);
auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair, references);
goal.addDependency(path);
return path;
}
@ -1327,16 +1229,24 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
// corresponds to an allowed derivation
{ throw Error("registerDrvOutput"); }
std::optional<const Realisation> queryRealisation(const DrvOutput & id) override
void queryRealisationUncached(const DrvOutput & id,
Callback<std::shared_ptr<const Realisation>> callback) noexcept override
// XXX: This should probably be allowed if the realisation corresponds to
// an allowed derivation
{ throw Error("queryRealisation"); }
void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode) override
{
if (!goal.isAllowed(id))
callback(nullptr);
next->queryRealisation(id, std::move(callback));
}
void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override
{
assert(!evalStore);
if (buildMode != bmNormal) throw Error("unsupported build mode");
StorePathSet newPaths;
std::set<Realisation> newRealisations;
for (auto & req : paths) {
if (!goal.isAllowed(req))
@ -1349,16 +1259,28 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
auto p = std::get_if<DerivedPath::Built>(&path);
if (!p) continue;
auto & bfd = *p;
auto drv = readDerivation(bfd.drvPath);
auto drvHashes = staticOutputHashes(*this, drv);
auto outputs = next->queryDerivationOutputMap(bfd.drvPath);
for (auto & [outputName, outputPath] : outputs)
if (wantOutput(outputName, bfd.outputs))
if (wantOutput(outputName, bfd.outputs)) {
newPaths.insert(outputPath);
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
auto thisRealisation = next->queryRealisation(
DrvOutput{drvHashes.at(outputName), outputName}
);
assert(thisRealisation);
newRealisations.insert(*thisRealisation);
}
}
}
StorePathSet closure;
next->computeFSClosure(newPaths, closure);
for (auto & path : closure)
goal.addDependency(path);
for (auto & real : Realisation::closure(*next, newRealisations))
goal.addedDrvOutputs.insert(real.id);
}
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
@ -1404,7 +1326,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
void LocalDerivationGoal::startDaemon()
{
settings.requireExperimentalFeature("recursive-nix");
settings.requireExperimentalFeature(Xp::RecursiveNix);
Store::Params params;
params["path-info-cache-size"] = "0";
@ -1437,7 +1359,7 @@ void LocalDerivationGoal::startDaemon()
AutoCloseFD remote = accept(daemonSocket.get(),
(struct sockaddr *) &remoteAddr, &remoteAddrLen);
if (!remote) {
if (errno == EINTR) continue;
if (errno == EINTR || errno == EAGAIN) continue;
if (errno == EINVAL) break;
throw SysError("accepting connection");
}
@ -1734,7 +1656,7 @@ void LocalDerivationGoal::runChild()
/* N.B. it is realistic that these paths might not exist. It
happens when testing Nix building fixed-output derivations
within a pure derivation. */
for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts", "/var/run/nscd/socket" })
for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts" })
if (pathExists(path))
ss.push_back(path);
}
@ -1916,7 +1838,7 @@ void LocalDerivationGoal::runChild()
/* Fill in the arguments. */
Strings args;
const char *builder = "invalid";
std::string builder = "invalid";
if (drv->isBuiltin()) {
;
@ -2042,13 +1964,13 @@ void LocalDerivationGoal::runChild()
}
args.push_back(drv->builder);
} else {
builder = drv->builder.c_str();
builder = drv->builder;
args.push_back(std::string(baseNameOf(drv->builder)));
}
}
#else
else {
builder = drv->builder.c_str();
builder = drv->builder;
args.push_back(std::string(baseNameOf(drv->builder)));
}
#endif
@ -2075,7 +1997,7 @@ void LocalDerivationGoal::runChild()
else if (drv->builder == "builtin:unpack-channel")
builtinUnpackChannel(drv2);
else
throw Error("unsupported builtin function '%1%'", string(drv->builder, 8));
throw Error("unsupported builtin builder '%1%'", string(drv->builder, 8));
_exit(0);
} catch (std::exception & e) {
writeFull(STDERR_FILENO, e.what() + std::string("\n"));
@ -2104,9 +2026,9 @@ void LocalDerivationGoal::runChild()
posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL);
}
posix_spawn(NULL, builder, NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
posix_spawn(NULL, builder.c_str(), NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
#else
execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
execve(builder.c_str(), stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
#endif
throw SysError("executing '%1%'", drv->builder);
@ -2221,8 +2143,7 @@ void LocalDerivationGoal::registerOutputs()
/* Pass blank Sink as we are not ready to hash data at this stage. */
NullSink blank;
auto references = worker.store.parseStorePathSet(
scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths)));
auto references = scanForReferences(blank, actualPath, referenceablePaths);
outputReferencesIfUnregistered.insert_or_assign(
outputName,
@ -2236,8 +2157,8 @@ void LocalDerivationGoal::registerOutputs()
/* Since we'll use the already installed versions of these, we
can treat them as leaves and ignore any references they
have. */
[&](AlreadyRegistered _) { return StringSet {}; },
[&](PerhapsNeedToRegister refs) {
[&](const AlreadyRegistered &) { return StringSet {}; },
[&](const PerhapsNeedToRegister & refs) {
StringSet referencedOutputs;
/* FIXME build inverted map up front so no quadratic waste here */
for (auto & r : refs.refs)
@ -2273,11 +2194,11 @@ void LocalDerivationGoal::registerOutputs()
};
std::optional<StorePathSet> referencesOpt = std::visit(overloaded {
[&](AlreadyRegistered skippedFinalPath) -> std::optional<StorePathSet> {
[&](const AlreadyRegistered & skippedFinalPath) -> std::optional<StorePathSet> {
finish(skippedFinalPath.path);
return std::nullopt;
},
[&](PerhapsNeedToRegister r) -> std::optional<StorePathSet> {
[&](const PerhapsNeedToRegister & r) -> std::optional<StorePathSet> {
return r.refs;
},
}, outputReferencesIfUnregistered.at(outputName));
@ -2289,7 +2210,7 @@ void LocalDerivationGoal::registerOutputs()
auto rewriteOutput = [&]() {
/* Apply hash rewriting if necessary. */
if (!outputRewrites.empty()) {
warn("rewriting hashes in '%1%'; cross fingers", actualPath);
debug("rewriting hashes in '%1%'; cross fingers", actualPath);
/* FIXME: this is in-memory. */
StringSink sink;
@ -2298,10 +2219,6 @@ void LocalDerivationGoal::registerOutputs()
sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites));
StringSource source(*sink.s);
restorePath(actualPath, source);
/* FIXME: set proper permissions in restorePath() so
we don't have to do another traversal. */
canonicalisePathMetaData(actualPath, -1, inodesSeen);
}
};
@ -2355,32 +2272,19 @@ void LocalDerivationGoal::registerOutputs()
}
auto got = caSink.finish().first;
auto refs = rewriteRefs();
HashModuloSink narSink { htSHA256, oldHashPart };
dumpPath(actualPath, narSink);
auto narHashAndSize = narSink.finish();
ValidPathInfo newInfo0 {
worker.store.makeFixedOutputPath(
auto finalPath = worker.store.makeFixedOutputPath(
outputHash.method,
got,
outputPathName(drv->name, outputName),
refs.second,
refs.first),
narHashAndSize.first,
};
newInfo0.narSize = narHashAndSize.second;
newInfo0.ca = FixedOutputHash {
.method = outputHash.method,
.hash = got,
};
newInfo0.references = refs.second;
if (refs.first)
newInfo0.references.insert(newInfo0.path);
if (scratchPath != newInfo0.path) {
refs.first);
if (scratchPath != finalPath) {
// Also rewrite the output path
auto source = sinkToSource([&](Sink & nextSink) {
StringSink sink;
dumpPath(actualPath, sink);
RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink);
RewritingSink rsink2(oldHashPart, std::string(finalPath.hashPart()), nextSink);
rsink2(*sink.s);
rsink2.flush();
});
@ -2390,12 +2294,27 @@ void LocalDerivationGoal::registerOutputs()
movePath(tmpPath, actualPath);
}
HashResult narHashAndSize = hashPath(htSHA256, actualPath);
ValidPathInfo newInfo0 {
finalPath,
narHashAndSize.first,
};
newInfo0.narSize = narHashAndSize.second;
newInfo0.ca = FixedOutputHash {
.method = outputHash.method,
.hash = got,
};
newInfo0.references = refs.second;
if (refs.first)
newInfo0.references.insert(newInfo0.path);
assert(newInfo0.ca);
return newInfo0;
};
ValidPathInfo newInfo = std::visit(overloaded {
[&](DerivationOutputInputAddressed output) {
[&](const DerivationOutputInputAddressed & output) {
/* input-addressed case */
auto requiredFinalPath = output.path;
/* Preemptively add rewrite rule for final hash, as that is
@ -2414,14 +2333,14 @@ void LocalDerivationGoal::registerOutputs()
newInfo0.references.insert(newInfo0.path);
return newInfo0;
},
[&](DerivationOutputCAFixed dof) {
[&](const DerivationOutputCAFixed & dof) {
auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating {
.method = dof.hash.method,
.hashType = dof.hash.hash.type,
});
/* Check wanted hash */
Hash & wanted = dof.hash.hash;
const Hash & wanted = dof.hash.hash;
assert(newInfo0.ca);
auto got = getContentAddressHash(*newInfo0.ca);
if (wanted != got) {
@ -2450,6 +2369,10 @@ void LocalDerivationGoal::registerOutputs()
},
}, output.output);
/* FIXME: set proper permissions in restorePath() so
we don't have to do another traversal. */
canonicalisePathMetaData(actualPath, -1, inodesSeen);
/* Calculate where we'll move the output files. In the checking case we
will leave leave them where they are, for now, rather than move to
their usual "final destination" */
@ -2459,6 +2382,7 @@ void LocalDerivationGoal::registerOutputs()
floating CA derivations and hash-mismatching fixed-output
derivations. */
PathLocks dynamicOutputLock;
dynamicOutputLock.setDeletion(true);
auto optFixedPath = output.path(worker.store, drv->name, outputName);
if (!optFixedPath ||
worker.store.printStorePath(*optFixedPath) != finalDestPath)
@ -2482,6 +2406,7 @@ void LocalDerivationGoal::registerOutputs()
assert(newInfo.ca);
} else {
auto destPath = worker.store.toRealPath(finalDestPath);
deletePath(destPath);
movePath(actualPath, destPath);
actualPath = destPath;
}
@ -2551,7 +2476,13 @@ void LocalDerivationGoal::registerOutputs()
infos.emplace(outputName, std::move(newInfo));
}
if (buildMode == bmCheck) return;
if (buildMode == bmCheck) {
// In case of FOD mismatches on `--check` an error must be thrown as this is also
// a source for non-determinism.
if (delayedException)
std::rethrow_exception(delayedException);
return;
}
/* Apply output checks. */
checkOutputs(infos);
@ -2636,7 +2567,7 @@ void LocalDerivationGoal::registerOutputs()
that for floating CA derivations, which otherwise couldn't be cached,
but it's fine to do in all cases. */
if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
for (auto& [outputName, newInfo] : infos) {
auto thisRealisation = Realisation{
.id = DrvOutput{initialOutputs.at(outputName).outputHash,

8
src/libstore/build/local-derivation-goal.hh
View file

@ -108,6 +108,9 @@ struct LocalDerivationGoal : public DerivationGoal
/* Paths that were added via recursive Nix calls. */
StorePathSet addedPaths;
/* Realisations that were added via recursive Nix calls. */
std::set<DrvOutput> addedDrvOutputs;
/* Recursive Nix calls are only allowed to build or realize paths
in the original input closure or added via a recursive Nix call
(so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
@ -116,6 +119,11 @@ struct LocalDerivationGoal : public DerivationGoal
{
return inputPaths.count(path) || addedPaths.count(path);
}
bool isAllowed(const DrvOutput & id)
{
return addedDrvOutputs.count(id);
}
bool isAllowed(const DerivedPath & req);
friend struct RestrictedStore;

2
src/libstore/build/substitution-goal.cc
View file

@ -204,7 +204,7 @@ void PathSubstitutionGoal::tryToRun()
Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()});
PushActivity pact(act.id);
copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()),
copyStorePath(*sub, worker.store,
subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
promise.set_value();

5
src/libstore/build/worker.cc
View file

@ -9,11 +9,12 @@
namespace nix {
Worker::Worker(Store & store)
Worker::Worker(Store & store, Store & evalStore)
: act(*logger, actRealise)
, actDerivations(*logger, actBuilds)
, actSubstitutions(*logger, actCopyPaths)
, store(store)
, evalStore(evalStore)
{
/* Debugging: prevent recursive workers. */
nrLocalBuilds = 0;
@ -238,7 +239,7 @@ void Worker::run(const Goals & _topGoals)
}
}
/* Call queryMissing() efficiently query substitutes. */
/* Call queryMissing() to efficiently query substitutes. */
StorePathSet willBuild, willSubstitute, unknown;
uint64_t downloadSize, narSize;
store.queryMissing(topPaths, willBuild, willSubstitute, unknown, downloadSize, narSize);

3
src/libstore/build/worker.hh
View file

@ -110,6 +110,7 @@ public:
bool checkMismatch;
Store & store;
Store & evalStore;
std::unique_ptr<HookInstance> hook;
@ -131,7 +132,7 @@ public:
it answers with "decline-permanently", we don't try again. */
bool tryBuildHook = true;
Worker(Store & store);
Worker(Store & store, Store & evalStore);
~Worker();
/* Make a goal (with caching). */

11
src/libstore/ca-specific-schema.sql
View file

@ -3,10 +3,19 @@
-- is enabled
create table if not exists Realisations (
id integer primary key autoincrement not null,
drvPath text not null,
outputName text not null, -- symbolic output id, usually "out"
outputPath integer not null,
signatures text, -- space-separated list
primary key (drvPath, outputName),
foreign key (outputPath) references ValidPaths(id) on delete cascade
);
create index if not exists IndexRealisations on Realisations(drvPath, outputName);
create table if not exists RealisationsRefs (
referrer integer not null,
realisationReference integer,
foreign key (referrer) references Realisations(id) on delete cascade,
foreign key (realisationReference) references Realisations(id) on delete restrict
);

22
src/libstore/content-address.cc
View file

@ -31,10 +31,10 @@ std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
std::string renderContentAddress(ContentAddress ca)
{
return std::visit(overloaded {
[](TextHash th) {
[](TextHash & th) {
return "text:" + th.hash.to_string(Base32, true);
},
[](FixedOutputHash fsh) {
[](FixedOutputHash & fsh) {
return makeFixedOutputCA(fsh.method, fsh.hash);
}
}, ca);
@ -43,10 +43,10 @@ std::string renderContentAddress(ContentAddress ca)
std::string renderContentAddressMethod(ContentAddressMethod cam)
{
return std::visit(overloaded {
[](TextHashMethod &th) {
[](TextHashMethod & th) {
return std::string{"text:"} + printHashType(htSHA256);
},
[](FixedOutputHashMethod &fshm) {
[](FixedOutputHashMethod & fshm) {
return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType);
}
}, cam);
@ -104,12 +104,12 @@ ContentAddress parseContentAddress(std::string_view rawCa) {
return std::visit(
overloaded {
[&](TextHashMethod thm) {
[&](TextHashMethod & thm) {
return ContentAddress(TextHash {
.hash = Hash::parseNonSRIUnprefixed(rest, htSHA256)
});
},
[&](FixedOutputHashMethod fohMethod) {
[&](FixedOutputHashMethod & fohMethod) {
return ContentAddress(FixedOutputHash {
.method = fohMethod.fileIngestionMethod,
.hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)),
@ -120,8 +120,10 @@ ContentAddress parseContentAddress(std::string_view rawCa) {
ContentAddressMethod parseContentAddressMethod(std::string_view caMethod)
{
std::string_view asPrefix {std::string{caMethod} + ":"};
return parseContentAddressMethodPrefix(asPrefix);
std::string asPrefix = std::string{caMethod} + ":";
// parseContentAddressMethodPrefix takes its argument by reference
std::string_view asPrefixView = asPrefix;
return parseContentAddressMethodPrefix(asPrefixView);
}
std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt)
@ -137,10 +139,10 @@ std::string renderContentAddress(std::optional<ContentAddress> ca)
Hash getContentAddressHash(const ContentAddress & ca)
{
return std::visit(overloaded {
[](TextHash th) {
[](const TextHash & th) {
return th.hash;
},
[](FixedOutputHash fsh) {
[](const FixedOutputHash & fsh) {
return fsh.hash;
}
}, ca);

82
src/libstore/daemon.cc
View file

@ -227,8 +227,15 @@ struct ClientSettings
try {
if (name == "ssh-auth-sock") // obsolete
;
else if (name == settings.experimentalFeatures.name) {
// We don’t want to forward the experimental features to
// the daemon, as that could cause some pretty weird stuff
if (parseFeatures(tokenizeString<StringSet>(value)) != settings.experimentalFeatures.get())
debug("Ignoring the client-specified experimental features");
}
else if (trusted
|| name == settings.buildTimeout.name
|| name == settings.buildRepeat.name
|| name == "connect-timeout"
|| (name == "builders" && value == ""))
settings.set(name, value);
@ -243,27 +250,10 @@ struct ClientSettings
}
};
static void writeValidPathInfo(
ref<Store> store,
unsigned int clientVersion,
Sink & to,
std::shared_ptr<const ValidPathInfo> info)
{
to << (info->deriver ? store->printStorePath(*info->deriver) : "")
<< info->narHash.to_string(Base16, false);
worker_proto::write(*store, to, info->references);
to << info->registrationTime << info->narSize;
if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
to << info->ultimate
<< info->sigs
<< renderContentAddress(info->ca);
}
}
static std::vector<DerivedPath> readDerivedPaths(Store & store, unsigned int clientVersion, Source & from)
{
std::vector<DerivedPath> reqs;
if (GET_PROTOCOL_MINOR(clientVersion) >= 29) {
if (GET_PROTOCOL_MINOR(clientVersion) >= 30) {
reqs = worker_proto::read(store, from, Phantom<std::vector<DerivedPath>> {});
} else {
for (auto & s : readStrings<Strings>(from))
@ -406,25 +396,21 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
FramedSource source(from);
// TODO this is essentially RemoteStore::addCAToStore. Move it up to Store.
return std::visit(overloaded {
[&](TextHashMethod &_) {
[&](TextHashMethod &) {
// We could stream this by changing Store
std::string contents = source.drain();
auto path = store->addTextToStore(name, contents, refs, repair);
return store->queryPathInfo(path);
},
[&](FixedOutputHashMethod &fohm) {
if (!refs.empty())
throw UnimplementedError("cannot yet have refs with flat or nar-hashed data");
auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair);
[&](FixedOutputHashMethod & fohm) {
auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair, refs);
return store->queryPathInfo(path);
},
}, contentAddressMethod);
}();
logger->stopWork();
to << store->printStorePath(pathInfo->path);
writeValidPathInfo(store, clientVersion, to, pathInfo);
pathInfo->write(to, *store, GET_PROTOCOL_MINOR(clientVersion));
} else {
HashType hashAlgo;
std::string baseName;
@ -471,6 +457,21 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
break;
}
case wopAddMultipleToStore: {
bool repair, dontCheckSigs;
from >> repair >> dontCheckSigs;
if (!trusted && dontCheckSigs)
dontCheckSigs = false;
logger->startWork();
FramedSource source(from);
store->addMultipleToStore(source,
RepairFlag{repair},
dontCheckSigs ? NoCheckSigs : CheckSigs);
logger->stopWork();
break;
}
case wopAddTextToStore: {
string suffix = readString(from);
string s = readString(from);
@ -622,9 +623,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
break;
}
// Obsolete.
case wopSyncWithGC: {
logger->startWork();
store->syncWithGC();
logger->stopWork();
to << 1;
break;
@ -770,7 +771,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (info) {
if (GET_PROTOCOL_MINOR(clientVersion) >= 17)
to << 1;
writeValidPathInfo(store, clientVersion, to, info);
info->write(to, *store, GET_PROTOCOL_MINOR(clientVersion), false);
} else {
assert(GET_PROTOCOL_MINOR(clientVersion) >= 17);
to << 0;
@ -885,10 +886,15 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopRegisterDrvOutput: {
logger->startWork();
auto outputId = DrvOutput::parse(readString(from));
auto outputPath = StorePath(readString(from));
store->registerDrvOutput(Realisation{
.id = outputId, .outPath = outputPath});
if (GET_PROTOCOL_MINOR(clientVersion) < 31) {
auto outputId = DrvOutput::parse(readString(from));
auto outputPath = StorePath(readString(from));
store->registerDrvOutput(Realisation{
.id = outputId, .outPath = outputPath});
} else {
auto realisation = worker_proto::read(*store, from, Phantom<Realisation>());
store->registerDrvOutput(realisation);
}
logger->stopWork();
break;
}
@ -898,9 +904,15 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
auto outputId = DrvOutput::parse(readString(from));
auto info = store->queryRealisation(outputId);
logger->stopWork();
std::set<StorePath> outPaths;
if (info) outPaths.insert(info->outPath);
worker_proto::write(*store, to, outPaths);
if (GET_PROTOCOL_MINOR(clientVersion) < 31) {
std::set<StorePath> outPaths;
if (info) outPaths.insert(info->outPath);
worker_proto::write(*store, to, outPaths);
} else {
std::set<Realisation> realisations;
if (info) realisations.insert(*info);
worker_proto::write(*store, to, realisations);
}
break;
}

48
src/libstore/derivations.cc
View file

@ -10,18 +10,18 @@ namespace nix {
std::optional<StorePath> DerivationOutput::path(const Store & store, std::string_view drvName, std::string_view outputName) const
{
return std::visit(overloaded {
[](DerivationOutputInputAddressed doi) -> std::optional<StorePath> {
[](const DerivationOutputInputAddressed & doi) -> std::optional<StorePath> {
return { doi.path };
},
[&](DerivationOutputCAFixed dof) -> std::optional<StorePath> {
[&](const DerivationOutputCAFixed & dof) -> std::optional<StorePath> {
return {
dof.path(store, drvName, outputName)
};
},
[](DerivationOutputCAFloating dof) -> std::optional<StorePath> {
[](const DerivationOutputCAFloating & dof) -> std::optional<StorePath> {
return std::nullopt;
},
[](DerivationOutputDeferred) -> std::optional<StorePath> {
[](const DerivationOutputDeferred &) -> std::optional<StorePath> {
return std::nullopt;
},
}, output);
@ -187,7 +187,7 @@ static DerivationOutput parseDerivationOutput(const Store & store,
},
};
} else {
settings.requireExperimentalFeature("ca-derivations");
settings.requireExperimentalFeature(Xp::CaDerivations);
assert(pathS == "");
return DerivationOutput {
.output = DerivationOutputCAFloating {
@ -332,22 +332,22 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
if (first) first = false; else s += ',';
s += '('; printUnquotedString(s, i.first);
std::visit(overloaded {
[&](DerivationOutputInputAddressed doi) {
[&](const DerivationOutputInputAddressed & doi) {
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(doi.path));
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
},
[&](DerivationOutputCAFixed dof) {
[&](const DerivationOutputCAFixed & dof) {
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first)));
s += ','; printUnquotedString(s, dof.hash.printMethodAlgo());
s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false));
},
[&](DerivationOutputCAFloating dof) {
[&](const DerivationOutputCAFloating & dof) {
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
s += ','; printUnquotedString(s, "");
},
[&](DerivationOutputDeferred) {
[&](const DerivationOutputDeferred &) {
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
@ -420,13 +420,13 @@ DerivationType BasicDerivation::type() const
std::optional<HashType> floatingHashType;
for (auto & i : outputs) {
std::visit(overloaded {
[&](DerivationOutputInputAddressed _) {
[&](const DerivationOutputInputAddressed &) {
inputAddressedOutputs.insert(i.first);
},
[&](DerivationOutputCAFixed _) {
[&](const DerivationOutputCAFixed &) {
fixedCAOutputs.insert(i.first);
},
[&](DerivationOutputCAFloating dof) {
[&](const DerivationOutputCAFloating & dof) {
floatingCAOutputs.insert(i.first);
if (!floatingHashType) {
floatingHashType = dof.hashType;
@ -435,7 +435,7 @@ DerivationType BasicDerivation::type() const
throw Error("All floating outputs must use the same hash type");
}
},
[&](DerivationOutputDeferred _) {
[&](const DerivationOutputDeferred &) {
deferredIAOutputs.insert(i.first);
},
}, i.second.output);
@ -538,15 +538,15 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
const auto & res = pathDerivationModulo(store, i.first);
std::visit(overloaded {
// Regular non-CA derivation, replace derivation
[&](Hash drvHash) {
[&](const Hash & drvHash) {
inputs2.insert_or_assign(drvHash.to_string(Base16, false), i.second);
},
[&](DeferredHash deferredHash) {
[&](const DeferredHash & deferredHash) {
isDeferred = true;
inputs2.insert_or_assign(deferredHash.hash.to_string(Base16, false), i.second);
},
// CA derivation's output hashes
[&](CaOutputHashes outputHashes) {
[&](const CaOutputHashes & outputHashes) {
std::set<std::string> justOut = { "out" };
for (auto & output : i.second) {
/* Put each one in with a single "out" output.. */
@ -568,21 +568,21 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
}
std::map<std::string, Hash> staticOutputHashes(Store& store, const Derivation& drv)
std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv)
{
std::map<std::string, Hash> res;
std::visit(overloaded {
[&](Hash drvHash) {
[&](const Hash & drvHash) {
for (auto & outputName : drv.outputNames()) {
res.insert({outputName, drvHash});
}
},
[&](DeferredHash deferredHash) {
[&](const DeferredHash & deferredHash) {
for (auto & outputName : drv.outputNames()) {
res.insert({outputName, deferredHash.hash});
}
},
[&](CaOutputHashes outputHashes) {
[&](const CaOutputHashes & outputHashes) {
res = outputHashes;
},
}, hashDerivationModulo(store, drv, true));
@ -666,22 +666,22 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
for (auto & i : drv.outputs) {
out << i.first;
std::visit(overloaded {
[&](DerivationOutputInputAddressed doi) {
[&](const DerivationOutputInputAddressed & doi) {
out << store.printStorePath(doi.path)
<< ""
<< "";
},
[&](DerivationOutputCAFixed dof) {
[&](const DerivationOutputCAFixed & dof) {
out << store.printStorePath(dof.path(store, drv.name, i.first))
<< dof.hash.printMethodAlgo()
<< dof.hash.hash.to_string(Base16, false);
},
[&](DerivationOutputCAFloating dof) {
[&](const DerivationOutputCAFloating & dof) {
out << ""
<< (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
<< "";
},
[&](DerivationOutputDeferred) {
[&](const DerivationOutputDeferred &) {
out << ""
<< ""
<< "";

4
src/libstore/derivations.hh
View file

@ -138,8 +138,8 @@ struct Derivation : BasicDerivation
/* Return the underlying basic derivation but with these changes:
1. Input drvs are emptied, but the outputs of them that were used are
added directly to input sources.
1. Input drvs are emptied, but the outputs of them that were used are
added directly to input sources.
2. Input placeholders are replaced with realized input store paths. */
std::optional<BasicDerivation> tryResolve(Store & store);

51
src/libstore/derived-path.cc
View file

@ -11,18 +11,33 @@ nlohmann::json DerivedPath::Opaque::toJSON(ref<Store> store) const {
return res;
}
nlohmann::json DerivedPathWithHints::Built::toJSON(ref<Store> store) const {
nlohmann::json BuiltPath::Built::toJSON(ref<Store> store) const {
nlohmann::json res;
res["drvPath"] = store->printStorePath(drvPath);
for (const auto& [output, path] : outputs) {
res["outputs"][output] = path ? store->printStorePath(*path) : "";
res["outputs"][output] = store->printStorePath(path);
}
return res;
}
nlohmann::json derivedPathsWithHintsToJSON(const DerivedPathsWithHints & buildables, ref<Store> store) {
StorePathSet BuiltPath::outPaths() const
{
return std::visit(
overloaded{
[](const BuiltPath::Opaque & p) { return StorePathSet{p.path}; },
[](const BuiltPath::Built & b) {
StorePathSet res;
for (auto & [_, path] : b.outputs)
res.insert(path);
return res;
},
}, raw()
);
}
nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref<Store> store) {
auto res = nlohmann::json::array();
for (const DerivedPathWithHints & buildable : buildables) {
for (const BuiltPath & buildable : buildables) {
std::visit([&res, store](const auto & buildable) {
res.push_back(buildable.toJSON(store));
}, buildable.raw());
@ -62,7 +77,7 @@ DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_vi
auto outputsS = s.substr(n + 1);
std::set<string> outputs;
if (outputsS != "*")
outputs = tokenizeString<std::set<string>>(outputsS);
outputs = tokenizeString<std::set<string>>(outputsS, ",");
return {drvPath, outputs};
}
@ -74,4 +89,30 @@ DerivedPath DerivedPath::parse(const Store & store, std::string_view s)
: (DerivedPath) DerivedPath::Built::parse(store, s);
}
RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const
{
RealisedPath::Set res;
std::visit(
overloaded{
[&](const BuiltPath::Opaque & p) { res.insert(p.path); },
[&](const BuiltPath::Built & p) {
auto drvHashes =
staticOutputHashes(store, store.readDerivation(p.drvPath));
for (auto& [outputName, outputPath] : p.outputs) {
if (settings.isExperimentalFeatureEnabled(
Xp::CaDerivations)) {
auto thisRealisation = store.queryRealisation(
DrvOutput{drvHashes.at(outputName), outputName});
assert(thisRealisation); // We’ve built it, so we must h
// ve the realisation
res.insert(*thisRealisation);
} else {
res.insert(outputPath);
}
}
},
},
raw());
return res;
}
}

42
src/libstore/derived-path.hh
View file

@ -2,6 +2,7 @@
#include "util.hh"
#include "path.hh"
#include "realisation.hh"
#include <optional>
@ -79,51 +80,44 @@ struct DerivedPath : _DerivedPathRaw {
/**
* A built derived path with hints in the form of optional concrete output paths.
*
* See 'DerivedPathWithHints' for more an explanation.
* See 'BuiltPath' for more an explanation.
*/
struct DerivedPathWithHintsBuilt {
struct BuiltPathBuilt {
StorePath drvPath;
std::map<std::string, std::optional<StorePath>> outputs;
std::map<std::string, StorePath> outputs;
nlohmann::json toJSON(ref<Store> store) const;
static DerivedPathWithHintsBuilt parse(const Store & store, std::string_view);
static BuiltPathBuilt parse(const Store & store, std::string_view);
};
using _DerivedPathWithHintsRaw = std::variant<
using _BuiltPathRaw = std::variant<
DerivedPath::Opaque,
DerivedPathWithHintsBuilt
BuiltPathBuilt
>;
/**
* A derived path with hints in the form of optional concrete output paths in the built case.
*
* This type is currently just used by the CLI. The paths are filled in
* during evaluation for derivations that know what paths they will
* produce in advanced, i.e. input-addressed or fixed-output content
* addressed derivations.
*
* That isn't very good, because it puts floating content-addressed
* derivations "at a disadvantage". It would be better to never rely on
* the output path of unbuilt derivations, and exclusively use the
* realizations types to work with built derivations' concrete output
* paths.
* A built path. Similar to a `DerivedPath`, but enriched with the corresponding
* output path(s).
*/
// FIXME Stop using and delete this, or if that is not possible move out of libstore to libcmd.
struct DerivedPathWithHints : _DerivedPathWithHintsRaw {
using Raw = _DerivedPathWithHintsRaw;
struct BuiltPath : _BuiltPathRaw {
using Raw = _BuiltPathRaw;
using Raw::Raw;
using Opaque = DerivedPathOpaque;
using Built = DerivedPathWithHintsBuilt;
using Built = BuiltPathBuilt;
inline const Raw & raw() const {
return static_cast<const Raw &>(*this);
}
StorePathSet outPaths() const;
RealisedPath::Set toRealisedPaths(Store & store) const;
};
typedef std::vector<DerivedPathWithHints> DerivedPathsWithHints;
typedef std::vector<DerivedPath> DerivedPaths;
typedef std::vector<BuiltPath> BuiltPaths;
nlohmann::json derivedPathsWithHintsToJSON(const DerivedPathsWithHints & buildables, ref<Store> store);
nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref<Store> store);
}

10
src/libstore/dummy-store.cc
View file

@ -43,11 +43,6 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store
RepairFlag repair, CheckSigsFlag checkSigs) override
{ unsupported("addToStore"); }
StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo,
PathFilter & filter, RepairFlag repair) override
{ unsupported("addToStore"); }
StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override
{ unsupported("addTextToStore"); }
@ -55,8 +50,9 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store
void narFromPath(const StorePath & path, Sink & sink) override
{ unsupported("narFromPath"); }
std::optional<const Realisation> queryRealisation(const DrvOutput&) override
{ unsupported("queryRealisation"); }
void queryRealisationUncached(const DrvOutput &,
Callback<std::shared_ptr<const Realisation>> callback) noexcept override
{ callback(nullptr); }
};
static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regDummyStore;

27
src/libstore/filetransfer.cc
View file

@ -7,7 +7,7 @@
#include "finally.hh"
#include "callback.hh"
#ifdef ENABLE_S3
#if ENABLE_S3
#include <aws/core/client/ClientConfiguration.h>
#endif
@ -544,6 +544,14 @@ struct curlFileTransfer : public FileTransfer
stopWorkerThread();
});
#ifdef __linux__
/* Cause this thread to not share any FS attributes with the main thread,
because this causes setns() in restoreMountNamespace() to fail.
Ideally, this would happen in the std::thread() constructor. */
if (unshare(CLONE_FS) != 0)
throw SysError("unsharing filesystem state in download thread");
#endif
std::map<CURL *, std::shared_ptr<TransferItem>> items;
bool quit = false;
@ -665,7 +673,7 @@ struct curlFileTransfer : public FileTransfer
writeFull(wakeupPipe.writeSide.get(), " ");
}
#ifdef ENABLE_S3
#if ENABLE_S3
std::tuple<std::string, std::string, Store::Params> parseS3Uri(std::string uri)
{
auto [path, params] = splitUriAndParams(uri);
@ -688,7 +696,7 @@ struct curlFileTransfer : public FileTransfer
if (hasPrefix(request.uri, "s3://")) {
// FIXME: do this on a worker thread
try {
#ifdef ENABLE_S3
#if ENABLE_S3
auto [bucketName, key, params] = parseS3Uri(request.uri);
std::string profile = get(params, "profile").value_or("");
@ -716,15 +724,24 @@ struct curlFileTransfer : public FileTransfer
}
};
ref<curlFileTransfer> makeCurlFileTransfer()
{
return make_ref<curlFileTransfer>();
}
ref<FileTransfer> getFileTransfer()
{
static ref<FileTransfer> fileTransfer = makeFileTransfer();
static ref<curlFileTransfer> fileTransfer = makeCurlFileTransfer();
if (fileTransfer->state_.lock()->quit)
fileTransfer = makeCurlFileTransfer();
return fileTransfer;
}
ref<FileTransfer> makeFileTransfer()
{
return make_ref<curlFileTransfer>();
return makeCurlFileTransfer();
}
std::future<FileTransferResult> FileTransfer::enqueueFileTransfer(const FileTransferRequest & request)

793
src/libstore/gc.cc
View file

@ -10,48 +10,22 @@
#include <regex>
#include <random>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/statvfs.h>
#include <climits>
#include <errno.h>
#include <fcntl.h>
#include <poll.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/statvfs.h>
#include <sys/types.h>
#include <sys/un.h>
#include <unistd.h>
#include <climits>
namespace nix {
static string gcLockName = "gc.lock";
static string gcRootsDir = "gcroots";
/* Acquire the global GC lock. This is used to prevent new Nix
processes from starting after the temporary root files have been
read. To be precise: when they try to create a new temporary root
file, they will block until the garbage collector has finished /
yielded the GC lock. */
AutoCloseFD LocalStore::openGCLock(LockType lockType)
{
Path fnGCLock = (format("%1%/%2%")
% stateDir % gcLockName).str();
debug(format("acquiring global GC lock '%1%'") % fnGCLock);
AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
if (!fdGCLock)
throw SysError("opening global GC lock '%1%'", fnGCLock);
if (!lockFile(fdGCLock.get(), lockType, false)) {
printInfo("waiting for the big garbage collector lock...");
lockFile(fdGCLock.get(), lockType, true);
}
/* !!! Restrict read permission on the GC root. Otherwise any
process that can open the file for reading can DoS the
collector. */
return fdGCLock;
}
static std::string gcSocketPath = "/gc-socket/socket";
static std::string gcRootsDir = "gcroots";
static void makeSymlink(const Path & link, const Path & target)
@ -71,12 +45,6 @@ static void makeSymlink(const Path & link, const Path & target)
}
void LocalStore::syncWithGC()
{
AutoCloseFD fdGCLock = openGCLock(ltRead);
}
void LocalStore::addIndirectRoot(const Path & path)
{
string hash = hashString(htSHA1, path).to_string(Base32, false);
@ -95,6 +63,12 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot
"creating a garbage collector root (%1%) in the Nix store is forbidden "
"(are you running nix-build inside the store?)", gcRoot);
/* Register this root with the garbage collector, if it's
running. This should be superfluous since the caller should
have registered this root yet, but let's be on the safe
side. */
addTempRoot(storePath);
/* Don't clobber the link if it already exists and doesn't
point to the Nix store. */
if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot))))
@ -102,11 +76,6 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot
makeSymlink(gcRoot, printStorePath(storePath));
addIndirectRoot(gcRoot);
/* Grab the global GC root, causing us to block while a GC is in
progress. This prevents the set of permanent roots from
increasing while a GC is in progress. */
syncWithGC();
return gcRoot;
}
@ -119,8 +88,6 @@ void LocalStore::addTempRoot(const StorePath & path)
if (!state->fdTempRoots) {
while (1) {
AutoCloseFD fdGCLock = openGCLock(ltRead);
if (pathExists(fnTempRoots))
/* It *must* be stale, since there can be no two
processes with the same pid. */
@ -128,10 +95,8 @@ void LocalStore::addTempRoot(const StorePath & path)
state->fdTempRoots = openLockFile(fnTempRoots, true);
fdGCLock = -1;
debug(format("acquiring read lock on '%1%'") % fnTempRoots);
lockFile(state->fdTempRoots.get(), ltRead, true);
debug("acquiring write lock on '%s'", fnTempRoots);
lockFile(state->fdTempRoots.get(), ltWrite, true);
/* Check whether the garbage collector didn't get in our
way. */
@ -147,24 +112,55 @@ void LocalStore::addTempRoot(const StorePath & path)
}
/* Upgrade the lock to a write lock. This will cause us to block
if the garbage collector is holding our lock. */
debug(format("acquiring write lock on '%1%'") % fnTempRoots);
lockFile(state->fdTempRoots.get(), ltWrite, true);
if (!state->fdGCLock)
state->fdGCLock = openGCLock();
restart:
FdLock gcLock(state->fdGCLock.get(), ltRead, false, "");
if (!gcLock.acquired) {
/* We couldn't get a shared global GC lock, so the garbage
collector is running. So we have to connect to the garbage
collector and inform it about our root. */
if (!state->fdRootsSocket) {
auto socketPath = stateDir.get() + gcSocketPath;
debug("connecting to '%s'", socketPath);
state->fdRootsSocket = createUnixDomainSocket();
nix::connect(state->fdRootsSocket.get(), socketPath);
}
try {
debug("sending GC root '%s'", printStorePath(path));
writeFull(state->fdRootsSocket.get(), printStorePath(path) + "\n", false);
char c;
readFull(state->fdRootsSocket.get(), &c, 1);
assert(c == '1');
debug("got ack for GC root '%s'", printStorePath(path));
} catch (SysError & e) {
/* The garbage collector may have exited, so we need to
restart. */
if (e.errNo == EPIPE) {
debug("GC socket disconnected");
state->fdRootsSocket.close();
goto restart;
}
} catch (EndOfFile & e) {
debug("GC socket disconnected");
state->fdRootsSocket.close();
goto restart;
}
}
/* Append the store path to the temporary roots file. */
string s = printStorePath(path) + '\0';
writeFull(state->fdTempRoots.get(), s);
/* Downgrade to a read lock. */
debug(format("downgrading to read lock on '%1%'") % fnTempRoots);
lockFile(state->fdTempRoots.get(), ltRead, true);
}
static std::string censored = "{censored}";
void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
void LocalStore::findTempRoots(Roots & tempRoots, bool censor)
{
/* Read the `temproots' directory for per-process temporary root
files. */
@ -179,35 +175,25 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
pid_t pid = std::stoi(i.name);
debug(format("reading temporary root file '%1%'") % path);
FDPtr fd(new AutoCloseFD(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666)));
if (!*fd) {
AutoCloseFD fd(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666));
if (!fd) {
/* It's okay if the file has disappeared. */
if (errno == ENOENT) continue;
throw SysError("opening temporary roots file '%1%'", path);
}
/* This should work, but doesn't, for some reason. */
//FDPtr fd(new AutoCloseFD(openLockFile(path, false)));
//if (*fd == -1) continue;
/* Try to acquire a write lock without blocking. This can
only succeed if the owning process has died. In that case
we don't care about its temporary roots. */
if (lockFile(fd->get(), ltWrite, false)) {
if (lockFile(fd.get(), ltWrite, false)) {
printInfo("removing stale temporary roots file '%1%'", path);
unlink(path.c_str());
writeFull(fd->get(), "d");
writeFull(fd.get(), "d");
continue;
}
/* Acquire a read lock. This will prevent the owning process
from upgrading to a write lock, therefore it will block in
addTempRoot(). */
debug(format("waiting for read lock on '%1%'") % path);
lockFile(fd->get(), ltRead, true);
/* Read the entire file. */
string contents = readFile(fd->get());
string contents = readFile(fd.get());
/* Extract the roots. */
string::size_type pos = 0, end;
@ -218,8 +204,6 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
tempRoots[parseStorePath(root)].emplace(censor ? censored : fmt("{temp:%d}", pid));
pos = end + 1;
}
fds.push_back(fd); /* keep open */
}
}
@ -304,8 +288,7 @@ Roots LocalStore::findRoots(bool censor)
Roots roots;
findRootsNoTemp(roots, censor);
FDs fds;
findTempRoots(fds, roots, censor);
findTempRoots(roots, censor);
return roots;
}
@ -341,6 +324,7 @@ static string quoteRegexChars(const string & raw)
return std::regex_replace(raw, specialRegex, R"(\$&)");
}
#if __linux__
static void readFileRoots(const char * path, UncheckedRoots & roots)
{
try {
@ -350,6 +334,7 @@ static void readFileRoots(const char * path, UncheckedRoots & roots)
throw;
}
}
#endif
void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
{
@ -431,7 +416,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
}
#endif
#if defined(__linux__)
#if __linux__
readFileRoots("/proc/sys/kernel/modprobe", unchecked);
readFileRoots("/proc/sys/kernel/fbsplash", unchecked);
readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked);
@ -455,265 +440,139 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
struct GCLimitReached { };
struct LocalStore::GCState
{
const GCOptions & options;
GCResults & results;
StorePathSet roots;
StorePathSet tempRoots;
StorePathSet dead;
StorePathSet alive;
bool gcKeepOutputs;
bool gcKeepDerivations;
uint64_t bytesInvalidated;
bool moveToTrash = true;
bool shouldDelete;
GCState(const GCOptions & options, GCResults & results)
: options(options), results(results), bytesInvalidated(0) { }
};
bool LocalStore::isActiveTempFile(const GCState & state,
const Path & path, const string & suffix)
{
return hasSuffix(path, suffix)
&& state.tempRoots.count(parseStorePath(string(path, 0, path.size() - suffix.size())));
}
void LocalStore::deleteGarbage(GCState & state, const Path & path)
{
uint64_t bytesFreed;
deletePath(path, bytesFreed);
state.results.bytesFreed += bytesFreed;
}
void LocalStore::deletePathRecursive(GCState & state, const Path & path)
{
checkInterrupt();
uint64_t size = 0;
auto storePath = maybeParseStorePath(path);
if (storePath && isValidPath(*storePath)) {
StorePathSet referrers;
queryReferrers(*storePath, referrers);
for (auto & i : referrers)
if (printStorePath(i) != path) deletePathRecursive(state, printStorePath(i));
size = queryPathInfo(*storePath)->narSize;
invalidatePathChecked(*storePath);
}
Path realPath = realStoreDir + "/" + std::string(baseNameOf(path));
struct stat st;
if (lstat(realPath.c_str(), &st)) {
if (errno == ENOENT) return;
throw SysError("getting status of %1%", realPath);
}
printInfo(format("deleting '%1%'") % path);
state.results.paths.insert(path);
/* If the path is not a regular file or symlink, move it to the
trash directory. The move is to ensure that later (when we're
not holding the global GC lock) we can delete the path without
being afraid that the path has become alive again. Otherwise
delete it right away. */
if (state.moveToTrash && S_ISDIR(st.st_mode)) {
// Estimate the amount freed using the narSize field. FIXME:
// if the path was not valid, need to determine the actual
// size.
try {
if (chmod(realPath.c_str(), st.st_mode | S_IWUSR) == -1)
throw SysError("making '%1%' writable", realPath);
Path tmp = trashDir + "/" + std::string(baseNameOf(path));
if (rename(realPath.c_str(), tmp.c_str()))
throw SysError("unable to rename '%1%' to '%2%'", realPath, tmp);
state.bytesInvalidated += size;
} catch (SysError & e) {
if (e.errNo == ENOSPC) {
printInfo(format("note: can't create move '%1%': %2%") % realPath % e.msg());
deleteGarbage(state, realPath);
}
}
} else
deleteGarbage(state, realPath);
if (state.results.bytesFreed + state.bytesInvalidated > state.options.maxFreed) {
printInfo(format("deleted or invalidated more than %1% bytes; stopping") % state.options.maxFreed);
throw GCLimitReached();
}
}
bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const StorePath & path)
{
if (visited.count(path)) return false;
if (state.alive.count(path)) return true;
if (state.dead.count(path)) return false;
if (state.roots.count(path)) {
debug("cannot delete '%1%' because it's a root", printStorePath(path));
state.alive.insert(path);
return true;
}
visited.insert(path);
if (!isValidPath(path)) return false;
StorePathSet incoming;
/* Don't delete this path if any of its referrers are alive. */
queryReferrers(path, incoming);
/* If keep-derivations is set and this is a derivation, then
don't delete the derivation if any of the outputs are alive. */
if (state.gcKeepDerivations && path.isDerivation()) {
for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(path))
if (maybeOutPath &&
isValidPath(*maybeOutPath) &&
queryPathInfo(*maybeOutPath)->deriver == path
)
incoming.insert(*maybeOutPath);
}
/* If keep-outputs is set, then don't delete this path if there
are derivers of this path that are not garbage. */
if (state.gcKeepOutputs) {
auto derivers = queryValidDerivers(path);
for (auto & i : derivers)
incoming.insert(i);
}
for (auto & i : incoming)
if (i != path)
if (canReachRoot(state, visited, i)) {
state.alive.insert(path);
return true;
}
return false;
}
void LocalStore::tryToDelete(GCState & state, const Path & path)
{
checkInterrupt();
auto realPath = realStoreDir + "/" + std::string(baseNameOf(path));
if (realPath == linksDir || realPath == trashDir) return;
//Activity act(*logger, lvlDebug, format("considering whether to delete '%1%'") % path);
auto storePath = maybeParseStorePath(path);
if (!storePath || !isValidPath(*storePath)) {
/* A lock file belonging to a path that we're building right
now isn't garbage. */
if (isActiveTempFile(state, path, ".lock")) return;
/* Don't delete .chroot directories for derivations that are
currently being built. */
if (isActiveTempFile(state, path, ".chroot")) return;
/* Don't delete .check directories for derivations that are
currently being built, because we may need to run
diff-hook. */
if (isActiveTempFile(state, path, ".check")) return;
}
StorePathSet visited;
if (storePath && canReachRoot(state, visited, *storePath)) {
debug("cannot delete '%s' because it's still reachable", path);
} else {
/* No path we visited was a root, so everything is garbage.
But we only delete ‘path’ and its referrers here so that
‘nix-store --delete’ doesn't have the unexpected effect of
recursing into derivations and outputs. */
for (auto & i : visited)
state.dead.insert(i);
if (state.shouldDelete)
deletePathRecursive(state, path);
}
}
/* Unlink all files in /nix/store/.links that have a link count of 1,
which indicates that there are no other links and so they can be
safely deleted. FIXME: race condition with optimisePath(): we
might see a link count of 1 just before optimisePath() increases
the link count. */
void LocalStore::removeUnusedLinks(const GCState & state)
{
AutoCloseDir dir(opendir(linksDir.c_str()));
if (!dir) throw SysError("opening directory '%1%'", linksDir);
int64_t actualSize = 0, unsharedSize = 0;
struct dirent * dirent;
while (errno = 0, dirent = readdir(dir.get())) {
checkInterrupt();
string name = dirent->d_name;
if (name == "." || name == "..") continue;
Path path = linksDir + "/" + name;
auto st = lstat(path);
if (st.st_nlink != 1) {
actualSize += st.st_size;
unsharedSize += (st.st_nlink - 1) * st.st_size;
continue;
}
printMsg(lvlTalkative, format("deleting unused link '%1%'") % path);
if (unlink(path.c_str()) == -1)
throw SysError("deleting '%1%'", path);
state.results.bytesFreed += st.st_size;
}
struct stat st;
if (stat(linksDir.c_str(), &st) == -1)
throw SysError("statting '%1%'", linksDir);
int64_t overhead = st.st_blocks * 512ULL;
printInfo("note: currently hard linking saves %.2f MiB",
((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
}
void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
{
GCState state(options, results);
state.gcKeepOutputs = settings.gcKeepOutputs;
state.gcKeepDerivations = settings.gcKeepDerivations;
bool shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific;
bool gcKeepOutputs = settings.gcKeepOutputs;
bool gcKeepDerivations = settings.gcKeepDerivations;
StorePathSet roots, dead, alive;
struct Shared
{
// The temp roots only store the hash part to make it easier to
// ignore suffixes like '.lock', '.chroot' and '.check'.
std::unordered_set<std::string> tempRoots;
// Hash part of the store path currently being deleted, if
// any.
std::optional<std::string> pending;
};
Sync<Shared> _shared;
std::condition_variable wakeup;
/* Using `--ignore-liveness' with `--delete' can have unintended
consequences if `keep-outputs' or `keep-derivations' are true
(the garbage collector will recurse into deleting the outputs
or derivers, respectively). So disable them. */
if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) {
state.gcKeepOutputs = false;
state.gcKeepDerivations = false;
gcKeepOutputs = false;
gcKeepDerivations = false;
}
state.shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific;
if (state.shouldDelete)
if (shouldDelete)
deletePath(reservedPath);
/* Acquire the global GC root. This prevents
a) New roots from being added.
b) Processes from creating new temporary root files. */
AutoCloseFD fdGCLock = openGCLock(ltWrite);
/* Acquire the global GC root. Note: we don't use fdGCLock
here because then in auto-gc mode, another thread could
downgrade our exclusive lock. */
auto fdGCLock = openGCLock();
FdLock gcLock(fdGCLock.get(), ltWrite, true, "waiting for the big garbage collector lock...");
/* Start the server for receiving new roots. */
auto socketPath = stateDir.get() + gcSocketPath;
createDirs(dirOf(socketPath));
auto fdServer = createUnixDomainSocket(socketPath, 0666);
if (fcntl(fdServer.get(), F_SETFL, fcntl(fdServer.get(), F_GETFL) | O_NONBLOCK) == -1)
throw SysError("making socket '%1%' non-blocking", socketPath);
Pipe shutdownPipe;
shutdownPipe.create();
std::thread serverThread([&]() {
Sync<std::map<int, std::thread>> connections;
Finally cleanup([&]() {
debug("GC roots server shutting down");
while (true) {
auto item = remove_begin(*connections.lock());
if (!item) break;
auto & [fd, thread] = *item;
shutdown(fd, SHUT_RDWR);
thread.join();
}
});
while (true) {
std::vector<struct pollfd> fds;
fds.push_back({.fd = shutdownPipe.readSide.get(), .events = POLLIN});
fds.push_back({.fd = fdServer.get(), .events = POLLIN});
auto count = poll(fds.data(), fds.size(), -1);
assert(count != -1);
if (fds[0].revents)
/* Parent is asking us to quit. */
break;
if (fds[1].revents) {
/* Accept a new connection. */
assert(fds[1].revents & POLLIN);
AutoCloseFD fdClient = accept(fdServer.get(), nullptr, nullptr);
if (!fdClient) continue;
/* Process the connection in a separate thread. */
auto fdClient_ = fdClient.get();
std::thread clientThread([&, fdClient = std::move(fdClient)]() {
Finally cleanup([&]() {
auto conn(connections.lock());
auto i = conn->find(fdClient.get());
if (i != conn->end()) {
i->second.detach();
conn->erase(i);
}
});
while (true) {
try {
auto path = readLine(fdClient.get());
auto storePath = maybeParseStorePath(path);
if (storePath) {
debug("got new GC root '%s'", path);
auto hashPart = std::string(storePath->hashPart());
auto shared(_shared.lock());
shared->tempRoots.insert(hashPart);
/* If this path is currently being
deleted, then we have to wait until
deletion is finished to ensure that
the client doesn't start
re-creating it before we're
done. FIXME: ideally we would use a
FD for this so we don't block the
poll loop. */
while (shared->pending == hashPart) {
debug("synchronising with deletion of path '%s'", path);
shared.wait(wakeup);
}
} else
printError("received garbage instead of a root from client");
writeFull(fdClient.get(), "1", false);
} catch (Error &) { break; }
}
});
connections.lock()->insert({fdClient_, std::move(clientThread)});
}
}
});
Finally stopServer([&]() {
writeFull(shutdownPipe.writeSide.get(), "x", false);
wakeup.notify_all();
if (serverThread.joinable()) serverThread.join();
});
/* Find the roots. Since we've grabbed the GC lock, the set of
permanent roots cannot increase now. */
@ -722,124 +581,256 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
if (!options.ignoreLiveness)
findRootsNoTemp(rootMap, true);
for (auto & i : rootMap) state.roots.insert(i.first);
for (auto & i : rootMap) roots.insert(i.first);
/* Read the temporary roots. This acquires read locks on all
per-process temporary root files. So after this point no paths
can be added to the set of temporary roots. */
FDs fds;
/* Read the temporary roots created before we acquired the global
GC root. Any new roots will be sent to our socket. */
Roots tempRoots;
findTempRoots(fds, tempRoots, true);
findTempRoots(tempRoots, true);
for (auto & root : tempRoots) {
state.tempRoots.insert(root.first);
state.roots.insert(root.first);
_shared.lock()->tempRoots.insert(std::string(root.first.hashPart()));
roots.insert(root.first);
}
/* After this point the set of roots or temporary roots cannot
increase, since we hold locks on everything. So everything
that is not reachable from `roots' is garbage. */
/* Helper function that deletes a path from the store and throws
GCLimitReached if we've deleted enough garbage. */
auto deleteFromStore = [&](std::string_view baseName)
{
Path path = storeDir + "/" + std::string(baseName);
Path realPath = realStoreDir + "/" + std::string(baseName);
if (state.shouldDelete) {
if (pathExists(trashDir)) deleteGarbage(state, trashDir);
try {
createDirs(trashDir);
} catch (SysError & e) {
if (e.errNo == ENOSPC) {
printInfo("note: can't create trash directory: %s", e.msg());
state.moveToTrash = false;
printInfo("deleting '%1%'", path);
results.paths.insert(path);
uint64_t bytesFreed;
deletePath(realPath, bytesFreed);
results.bytesFreed += bytesFreed;
if (results.bytesFreed > options.maxFreed) {
printInfo("deleted more than %d bytes; stopping", options.maxFreed);
throw GCLimitReached();
}
};
std::map<StorePath, StorePathSet> referrersCache;
/* Helper function that visits all paths reachable from `start`
via the referrers edges and optionally derivers and derivation
output edges. If none of those paths are roots, then all
visited paths are garbage and are deleted. */
auto deleteReferrersClosure = [&](const StorePath & start) {
StorePathSet visited;
std::queue<StorePath> todo;
/* Wake up any GC client waiting for deletion of the paths in
'visited' to finish. */
Finally releasePending([&]() {
auto shared(_shared.lock());
shared->pending.reset();
wakeup.notify_all();
});
auto enqueue = [&](const StorePath & path) {
if (visited.insert(path).second)
todo.push(path);
};
enqueue(start);
while (auto path = pop(todo)) {
checkInterrupt();
/* Bail out if we've previously discovered that this path
is alive. */
if (alive.count(*path)) {
alive.insert(start);
return;
}
/* If we've previously deleted this path, we don't have to
handle it again. */
if (dead.count(*path)) continue;
auto markAlive = [&]()
{
alive.insert(*path);
alive.insert(start);
try {
StorePathSet closure;
computeFSClosure(*path, closure);
for (auto & p : closure)
alive.insert(p);
} catch (InvalidPath &) { }
};
/* If this is a root, bail out. */
if (roots.count(*path)) {
debug("cannot delete '%s' because it's a root", printStorePath(*path));
return markAlive();
}
if (options.action == GCOptions::gcDeleteSpecific
&& !options.pathsToDelete.count(*path))
return;
{
auto hashPart = std::string(path->hashPart());
auto shared(_shared.lock());
if (shared->tempRoots.count(hashPart)) {
debug("cannot delete '%s' because it's a temporary root", printStorePath(*path));
return markAlive();
}
shared->pending = hashPart;
}
if (isValidPath(*path)) {
/* Visit the referrers of this path. */
auto i = referrersCache.find(*path);
if (i == referrersCache.end()) {
StorePathSet referrers;
queryReferrers(*path, referrers);
referrersCache.emplace(*path, std::move(referrers));
i = referrersCache.find(*path);
}
for (auto & p : i->second)
enqueue(p);
/* If keep-derivations is set and this is a
derivation, then visit the derivation outputs. */
if (gcKeepDerivations && path->isDerivation()) {
for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(*path))
if (maybeOutPath &&
isValidPath(*maybeOutPath) &&
queryPathInfo(*maybeOutPath)->deriver == *path)
enqueue(*maybeOutPath);
}
/* If keep-outputs is set, then visit the derivers. */
if (gcKeepOutputs) {
auto derivers = queryValidDerivers(*path);
for (auto & i : derivers)
enqueue(i);
}
}
}
}
/* Now either delete all garbage paths, or just the specified
for (auto & path : topoSortPaths(visited)) {
if (!dead.insert(path).second) continue;
if (shouldDelete) {
invalidatePathChecked(path);
deleteFromStore(path.to_string());
referrersCache.erase(path);
}
}
};
/* Synchronisation point for testing, see tests/gc-concurrent.sh. */
if (auto p = getEnv("_NIX_TEST_GC_SYNC"))
readFile(*p);
/* Either delete all garbage paths, or just the specified
paths (for gcDeleteSpecific). */
if (options.action == GCOptions::gcDeleteSpecific) {
for (auto & i : options.pathsToDelete) {
tryToDelete(state, printStorePath(i));
if (state.dead.find(i) == state.dead.end())
deleteReferrersClosure(i);
if (!dead.count(i))
throw Error(
"cannot delete path '%1%' since it is still alive. "
"To find out why use: "
"Cannot delete path '%1%' since it is still alive. "
"To find out why, use: "
"nix-store --query --roots",
printStorePath(i));
}
} else if (options.maxFreed > 0) {
if (state.shouldDelete)
if (shouldDelete)
printInfo("deleting garbage...");
else
printInfo("determining live/dead paths...");
try {
AutoCloseDir dir(opendir(realStoreDir.c_str()));
AutoCloseDir dir(opendir(realStoreDir.get().c_str()));
if (!dir) throw SysError("opening directory '%1%'", realStoreDir);
/* Read the store and immediately delete all paths that
aren't valid. When using --max-freed etc., deleting
invalid paths is preferred over deleting unreachable
paths, since unreachable paths could become reachable
again. We don't use readDirectory() here so that GCing
can start faster. */
/* Read the store and delete all paths that are invalid or
unreachable. We don't use readDirectory() here so that
GCing can start faster. */
auto linksName = baseNameOf(linksDir);
Paths entries;
struct dirent * dirent;
while (errno = 0, dirent = readdir(dir.get())) {
checkInterrupt();
string name = dirent->d_name;
if (name == "." || name == "..") continue;
Path path = storeDir + "/" + name;
auto storePath = maybeParseStorePath(path);
if (storePath && isValidPath(*storePath))
entries.push_back(path);
if (name == "." || name == ".." || name == linksName) continue;
if (auto storePath = maybeParseStorePath(storeDir + "/" + name))
deleteReferrersClosure(*storePath);
else
tryToDelete(state, path);
deleteFromStore(name);
}
dir.reset();
/* Now delete the unreachable valid paths. Randomise the
order in which we delete entries to make the collector
less biased towards deleting paths that come
alphabetically first (e.g. /nix/store/000...). This
matters when using --max-freed etc. */
vector<Path> entries_(entries.begin(), entries.end());
std::mt19937 gen(1);
std::shuffle(entries_.begin(), entries_.end(), gen);
for (auto & i : entries_)
tryToDelete(state, i);
} catch (GCLimitReached & e) {
}
}
if (state.options.action == GCOptions::gcReturnLive) {
for (auto & i : state.alive)
state.results.paths.insert(printStorePath(i));
if (options.action == GCOptions::gcReturnLive) {
for (auto & i : alive)
results.paths.insert(printStorePath(i));
return;
}
if (state.options.action == GCOptions::gcReturnDead) {
for (auto & i : state.dead)
state.results.paths.insert(printStorePath(i));
if (options.action == GCOptions::gcReturnDead) {
for (auto & i : dead)
results.paths.insert(printStorePath(i));
return;
}
/* Allow other processes to add to the store from here on. */
fdGCLock = -1;
fds.clear();
/* Delete the trash directory. */
printInfo(format("deleting '%1%'") % trashDir);
deleteGarbage(state, trashDir);
/* Clean up the links directory. */
/* Unlink all files in /nix/store/.links that have a link count of 1,
which indicates that there are no other links and so they can be
safely deleted. FIXME: race condition with optimisePath(): we
might see a link count of 1 just before optimisePath() increases
the link count. */
if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) {
printInfo("deleting unused links...");
removeUnusedLinks(state);
AutoCloseDir dir(opendir(linksDir.c_str()));
if (!dir) throw SysError("opening directory '%1%'", linksDir);
int64_t actualSize = 0, unsharedSize = 0;
struct dirent * dirent;
while (errno = 0, dirent = readdir(dir.get())) {
checkInterrupt();
string name = dirent->d_name;
if (name == "." || name == "..") continue;
Path path = linksDir + "/" + name;
auto st = lstat(path);
if (st.st_nlink != 1) {
actualSize += st.st_size;
unsharedSize += (st.st_nlink - 1) * st.st_size;
continue;
}
printMsg(lvlTalkative, format("deleting unused link '%1%'") % path);
if (unlink(path.c_str()) == -1)
throw SysError("deleting '%1%'", path);
results.bytesFreed += st.st_size;
}
struct stat st;
if (stat(linksDir.c_str(), &st) == -1)
throw SysError("statting '%1%'", linksDir);
int64_t overhead = st.st_blocks * 512ULL;
printInfo("note: currently hard linking saves %.2f MiB",
((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
}
/* While we're at it, vacuum the database. */
@ -856,7 +847,7 @@ void LocalStore::autoGC(bool sync)
return std::stoll(readFile(*fakeFreeSpaceFile));
struct statvfs st;
if (statvfs(realStoreDir.c_str(), &st))
if (statvfs(realStoreDir.get().c_str(), &st))
throw SysError("getting filesystem info about '%s'", realStoreDir);
return (uint64_t) st.f_bavail * st.f_frsize;

18
src/libstore/globals.cc
View file

@ -148,7 +148,8 @@ StringSet Settings::getDefaultExtraPlatforms()
// machines. Note that we can’t force processes from executing
// x86_64 in aarch64 environments or vice versa since they can
// always exec with their own binary preferences.
if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) {
if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist") ||
pathExists("/System/Library/LaunchDaemons/com.apple.oahd.plist")) {
if (std::string{SYSTEM} == "x86_64-darwin")
extraPlatforms.insert("aarch64-darwin");
else if (std::string{SYSTEM} == "aarch64-darwin")
@ -159,21 +160,16 @@ StringSet Settings::getDefaultExtraPlatforms()
return extraPlatforms;
}
bool Settings::isExperimentalFeatureEnabled(const std::string & name)
bool Settings::isExperimentalFeatureEnabled(const ExperimentalFeature & feature)
{
auto & f = experimentalFeatures.get();
return std::find(f.begin(), f.end(), name) != f.end();
return std::find(f.begin(), f.end(), feature) != f.end();
}
MissingExperimentalFeature::MissingExperimentalFeature(std::string feature)
: Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", feature)
, missingFeature(feature)
{}
void Settings::requireExperimentalFeature(const std::string & name)
void Settings::requireExperimentalFeature(const ExperimentalFeature & feature)
{
if (!isExperimentalFeatureEnabled(name))
throw MissingExperimentalFeature(name);
if (!isExperimentalFeatureEnabled(feature))
throw MissingExperimentalFeature(feature);
}
bool Settings::isWSL1()

28
src/libstore/globals.hh
View file

@ -3,6 +3,7 @@
#include "types.hh"
#include "config.hh"
#include "util.hh"
#include "experimental-features.hh"
#include <map>
#include <limits>
@ -45,15 +46,6 @@ struct PluginFilesSetting : public BaseSetting<Paths>
void set(const std::string & str, bool append = false) override;
};
class MissingExperimentalFeature: public Error
{
public:
std::string missingFeature;
MissingExperimentalFeature(std::string feature);
virtual const char* sname() const override { return "MissingExperimentalFeature"; }
};
class Settings : public Config {
unsigned int getDefaultCores();
@ -617,8 +609,10 @@ public:
Strings{"https://cache.nixos.org/"},
"substituters",
R"(
A list of URLs of substituters, separated by whitespace. The default
is `https://cache.nixos.org`.
A list of URLs of substituters, separated by whitespace. Substituters
are tried based on their Priority value, which each substituter can set
independently. Lower value means higher priority.
The default is `https://cache.nixos.org`, with a Priority of 40.
)",
{"binary-caches"}};
@ -923,12 +917,12 @@ public:
value.
)"};
Setting<Strings> experimentalFeatures{this, {}, "experimental-features",
Setting<std::set<ExperimentalFeature>> experimentalFeatures{this, {}, "experimental-features",
"Experimental Nix features to enable."};
bool isExperimentalFeatureEnabled(const std::string & name);
bool isExperimentalFeatureEnabled(const ExperimentalFeature &);
void requireExperimentalFeature(const std::string & name);
void requireExperimentalFeature(const ExperimentalFeature &);
Setting<bool> allowDirty{this, true, "allow-dirty",
"Whether to allow dirty Git/Mercurial trees."};
@ -954,6 +948,12 @@ public:
resolves to a different location from that of the build machine. You
can enable this setting if you are sure you're not going to do that.
)"};
Setting<bool> useRegistries{this, true, "use-registries",
"Whether to use flake registries to resolve flake references."};
Setting<bool> acceptFlakeConfig{this, false, "accept-flake-config",
"Whether to accept nix configuration from a flake without prompting."};
};

4
src/libstore/http-binary-cache-store.cc
View file

@ -57,8 +57,8 @@ public:
{
// FIXME: do this lazily?
if (auto cacheInfo = diskCache->cacheExists(cacheUri)) {
wantMassQuery.setDefault(cacheInfo->wantMassQuery ? "true" : "false");
priority.setDefault(fmt("%d", cacheInfo->priority));
wantMassQuery.setDefault(cacheInfo->wantMassQuery);
priority.setDefault(cacheInfo->priority);
} else {
try {
BinaryCacheStore::init();

35
src/libstore/legacy-ssh-store.cc
View file

@ -82,9 +82,20 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION;
conn->to.flush();
unsigned int magic = readInt(conn->from);
if (magic != SERVE_MAGIC_2)
throw Error("protocol mismatch with 'nix-store --serve' on '%s'", host);
StringSink saved;
try {
TeeSource tee(conn->from, saved);
unsigned int magic = readInt(tee);
if (magic != SERVE_MAGIC_2)
throw Error("'nix-store --serve' protocol mismatch from '%s'", host);
} catch (SerialisationError & e) {
/* In case the other side is waiting for our input,
close it. */
conn->sshConn->in.close();
auto msg = conn->from.drain();
throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'",
host, chomp(*saved.s + msg));
}
conn->remoteVersion = readInt(conn->from);
if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200)
throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host);
@ -216,7 +227,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo,
PathFilter & filter, RepairFlag repair) override
PathFilter & filter, RepairFlag repair, const StorePathSet & references) override
{ unsupported("addToStore"); }
StorePath addTextToStore(const string & name, const string & s,
@ -237,6 +248,10 @@ private:
conn.to
<< settings.buildRepeat
<< settings.enforceDeterminism;
if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 7) {
conn.to << ((int) settings.keepFailed);
}
}
public:
@ -267,8 +282,11 @@ public:
return status;
}
void buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode) override
void buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override
{
if (evalStore && evalStore.get() != this)
throw Error("building on an SSH store is incompatible with '--eval-store'");
auto conn(connections->get());
conn->to << cmdBuildPaths;
@ -276,10 +294,10 @@ public:
for (auto & p : drvPaths) {
auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p);
std::visit(overloaded {
[&](StorePathWithOutputs s) {
[&](const StorePathWithOutputs & s) {
ss.push_back(s.to_string(*this));
},
[&](StorePath drvPath) {
[&](const StorePath & drvPath) {
throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath));
},
}, sOrDrvPath);
@ -349,7 +367,8 @@ public:
return conn->remoteVersion;
}
std::optional<const Realisation> queryRealisation(const DrvOutput&) override
void queryRealisationUncached(const DrvOutput &,
Callback<std::shared_ptr<const Realisation>> callback) noexcept override
// TODO: Implement
{ unsupported("queryRealisation"); }
};

2
src/libstore/local-binary-cache-store.cc
View file

@ -93,7 +93,7 @@ protected:
void LocalBinaryCacheStore::init()
{
createDirs(binaryCacheDir + "/nar");
createDirs(binaryCacheDir + realisationsPrefix);
createDirs(binaryCacheDir + "/" + realisationsPrefix);
if (writeDebugInfo)
createDirs(binaryCacheDir + "/debuginfo");
BinaryCacheStore::init();

5
src/libstore/local-fs-store.hh
View file

@ -18,6 +18,9 @@ struct LocalFSStoreConfig : virtual StoreConfig
const PathSetting logDir{(StoreConfig*) this, false,
rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir,
"log", "directory where Nix will store state"};
const PathSetting realStoreDir{(StoreConfig*) this, false,
rootDir != "" ? rootDir + "/nix/store" : storeDir, "real",
"physical path to the Nix store"};
};
class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store
@ -34,7 +37,7 @@ public:
/* Register a permanent GC root. */
Path addPermRoot(const StorePath & storePath, const Path & gcRoot);
virtual Path getRealStoreDir() { return storeDir; }
virtual Path getRealStoreDir() { return realStoreDir; }
Path toRealPath(const Path & storePath) override
{

350
src/libstore/local-store.cc
View file

@ -53,12 +53,15 @@ struct LocalStore::State::Stmts {
SQLiteStmt InvalidatePath;
SQLiteStmt AddDerivationOutput;
SQLiteStmt RegisterRealisedOutput;
SQLiteStmt UpdateRealisedOutput;
SQLiteStmt QueryValidDerivers;
SQLiteStmt QueryDerivationOutputs;
SQLiteStmt QueryRealisedOutput;
SQLiteStmt QueryAllRealisedOutputs;
SQLiteStmt QueryPathFromHashPart;
SQLiteStmt QueryValidPaths;
SQLiteStmt QueryRealisationReferences;
SQLiteStmt AddRealisationReference;
};
int getSchema(Path schemaPath)
@ -76,7 +79,7 @@ int getSchema(Path schemaPath)
void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd)
{
const int nixCASchemaVersion = 1;
const int nixCASchemaVersion = 2;
int curCASchema = getSchema(schemaPath);
if (curCASchema != nixCASchemaVersion) {
if (curCASchema > nixCASchemaVersion) {
@ -94,7 +97,39 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd)
#include "ca-specific-schema.sql.gen.hh"
;
db.exec(schema);
curCASchema = nixCASchemaVersion;
}
if (curCASchema < 2) {
SQLiteTxn txn(db);
// Ugly little sql dance to add a new `id` column and make it the primary key
db.exec(R"(
create table Realisations2 (
id integer primary key autoincrement not null,
drvPath text not null,
outputName text not null, -- symbolic output id, usually "out"
outputPath integer not null,
signatures text, -- space-separated list
foreign key (outputPath) references ValidPaths(id) on delete cascade
);
insert into Realisations2 (drvPath, outputName, outputPath, signatures)
select drvPath, outputName, outputPath, signatures from Realisations;
drop table Realisations;
alter table Realisations2 rename to Realisations;
)");
db.exec(R"(
create index if not exists IndexRealisations on Realisations(drvPath, outputName);
create table if not exists RealisationsRefs (
referrer integer not null,
realisationReference integer,
foreign key (referrer) references Realisations(id) on delete cascade,
foreign key (realisationReference) references Realisations(id) on delete restrict
);
)");
txn.commit();
}
writeFile(schemaPath, fmt("%d", nixCASchemaVersion));
lockFile(lockFd.get(), ltRead, true);
}
@ -106,14 +141,10 @@ LocalStore::LocalStore(const Params & params)
, LocalStoreConfig(params)
, Store(params)
, LocalFSStore(params)
, realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real",
"physical path to the Nix store"}
, realStoreDir(realStoreDir_)
, dbDir(stateDir + "/db")
, linksDir(realStoreDir + "/.links")
, reservedPath(dbDir + "/reserved")
, schemaPath(dbDir + "/schema")
, trashDir(realStoreDir + "/trash")
, tempRootsDir(stateDir + "/temproots")
, fnTempRoots(fmt("%s/%d", tempRootsDir, getpid()))
, locksHeld(tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS").value_or("")))
@ -153,13 +184,13 @@ LocalStore::LocalStore(const Params & params)
printError("warning: the group '%1%' specified in 'build-users-group' does not exist", settings.buildUsersGroup);
else {
struct stat st;
if (stat(realStoreDir.c_str(), &st))
if (stat(realStoreDir.get().c_str(), &st))
throw SysError("getting attributes of path '%1%'", realStoreDir);
if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) {
if (chown(realStoreDir.c_str(), 0, gr->gr_gid) == -1)
if (chown(realStoreDir.get().c_str(), 0, gr->gr_gid) == -1)
throw SysError("changing ownership of path '%1%'", realStoreDir);
if (chmod(realStoreDir.c_str(), perm) == -1)
if (chmod(realStoreDir.get().c_str(), perm) == -1)
throw SysError("changing permissions on path '%1%'", realStoreDir);
}
}
@ -277,7 +308,7 @@ LocalStore::LocalStore(const Params & params)
else openDB(*state, false);
if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
migrateCASchema(state->db, dbDir + "/ca-schema", globalLock);
}
@ -307,16 +338,25 @@ LocalStore::LocalStore(const Params & params)
state->stmts->QueryPathFromHashPart.create(state->db,
"select path from ValidPaths where path >= ? limit 1;");
state->stmts->QueryValidPaths.create(state->db, "select path from ValidPaths");
if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
state->stmts->RegisterRealisedOutput.create(state->db,
R"(
insert or replace into Realisations (drvPath, outputName, outputPath, signatures)
insert into Realisations (drvPath, outputName, outputPath, signatures)
values (?, ?, (select id from ValidPaths where path = ?), ?)
;
)");
state->stmts->UpdateRealisedOutput.create(state->db,
R"(
update Realisations
set signatures = ?
where
drvPath = ? and
outputName = ?
;
)");
state->stmts->QueryRealisedOutput.create(state->db,
R"(
select Output.path, Realisations.signatures from Realisations
select Realisations.id, Output.path, Realisations.signatures from Realisations
inner join ValidPaths as Output on Output.id = Realisations.outputPath
where drvPath = ? and outputName = ?
;
@ -328,10 +368,33 @@ LocalStore::LocalStore(const Params & params)
where drvPath = ?
;
)");
state->stmts->QueryRealisationReferences.create(state->db,
R"(
select drvPath, outputName from Realisations
join RealisationsRefs on realisationReference = Realisations.id
where referrer = ?;
)");
state->stmts->AddRealisationReference.create(state->db,
R"(
insert or replace into RealisationsRefs (referrer, realisationReference)
values (
(select id from Realisations where drvPath = ? and outputName = ?),
(select id from Realisations where drvPath = ? and outputName = ?));
)");
}
}
AutoCloseFD LocalStore::openGCLock()
{
Path fnGCLock = stateDir + "/gc.lock";
auto fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
if (!fdGCLock)
throw SysError("opening global GC lock '%1%'", fnGCLock);
return fdGCLock;
}
LocalStore::~LocalStore()
{
std::shared_future<void> future;
@ -437,14 +500,11 @@ void LocalStore::makeStoreWritable()
if (getuid() != 0) return;
/* Check if /nix/store is on a read-only mount. */
struct statvfs stat;
if (statvfs(realStoreDir.c_str(), &stat) != 0)
if (statvfs(realStoreDir.get().c_str(), &stat) != 0)
throw SysError("getting info about the Nix store mount point");
if (stat.f_flag & ST_RDONLY) {
if (unshare(CLONE_NEWNS) == -1)
throw SysError("setting up a private mount namespace");
if (mount(0, realStoreDir.c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1)
if (mount(0, realStoreDir.get().c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1)
throw SysError("remounting %1% writable", realStoreDir);
}
#endif
@ -627,7 +687,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
std::optional<Hash> h;
for (auto & i : drv.outputs) {
std::visit(overloaded {
[&](DerivationOutputInputAddressed doia) {
[&](const DerivationOutputInputAddressed & doia) {
if (!h) {
// somewhat expensive so we do lazily
auto temp = hashDerivationModulo(*this, drv, true);
@ -639,14 +699,14 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed));
envHasRightPath(doia.path, i.first);
},
[&](DerivationOutputCAFixed dof) {
[&](const DerivationOutputCAFixed & dof) {
StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName);
envHasRightPath(path, i.first);
},
[&](DerivationOutputCAFloating _) {
[&](const DerivationOutputCAFloating &) {
/* Nothing to check */
},
[&](DerivationOutputDeferred) {
[&](const DerivationOutputDeferred &) {
},
}, i.second.output);
}
@ -654,7 +714,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs)
{
settings.requireExperimentalFeature("ca-derivations");
settings.requireExperimentalFeature(Xp::CaDerivations);
if (checkSigs == NoCheckSigs || !realisationIsUntrusted(info))
registerDrvOutput(info);
else
@ -663,15 +723,55 @@ void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag check
void LocalStore::registerDrvOutput(const Realisation & info)
{
settings.requireExperimentalFeature("ca-derivations");
auto state(_state.lock());
settings.requireExperimentalFeature(Xp::CaDerivations);
retrySQLite<void>([&]() {
state->stmts->RegisterRealisedOutput.use()
(info.id.strHash())
(info.id.outputName)
(printStorePath(info.outPath))
(concatStringsSep(" ", info.signatures))
.exec();
auto state(_state.lock());
if (auto oldR = queryRealisation_(*state, info.id)) {
if (info.isCompatibleWith(*oldR)) {
auto combinedSignatures = oldR->signatures;
combinedSignatures.insert(info.signatures.begin(),
info.signatures.end());
state->stmts->UpdateRealisedOutput.use()
(concatStringsSep(" ", combinedSignatures))
(info.id.strHash())
(info.id.outputName)
.exec();
} else {
throw Error("Trying to register a realisation of '%s', but we already "
"have another one locally.\n"
"Local: %s\n"
"Remote: %s",
info.id.to_string(),
printStorePath(oldR->outPath),
printStorePath(info.outPath)
);
}
} else {
state->stmts->RegisterRealisedOutput.use()
(info.id.strHash())
(info.id.outputName)
(printStorePath(info.outPath))
(concatStringsSep(" ", info.signatures))
.exec();
}
for (auto & [outputId, depPath] : info.dependentRealisations) {
auto localRealisation = queryRealisationCore_(*state, outputId);
if (!localRealisation)
throw Error("unable to register the derivation '%s' as it "
"depends on the non existent '%s'",
info.id.to_string(), outputId.to_string());
if (localRealisation->second.outPath != depPath)
throw Error("unable to register the derivation '%s' as it "
"depends on a realisation of '%s' that doesn’t"
"match what we have locally",
info.id.to_string(), outputId.to_string());
state->stmts->AddRealisationReference.use()
(info.id.strHash())
(info.id.outputName)
(outputId.strHash())
(outputId.outputName)
.exec();
}
});
}
@ -731,7 +831,7 @@ uint64_t LocalStore::addValidPath(State & state,
{
auto state_(Store::state.lock());
state_->pathInfoCache.upsert(std::string(info.path.hashPart()),
state_->pathInfoCache.upsert(std::string(info.path.to_string()),
PathInfoCacheValue{ .value = std::make_shared<const ValidPathInfo>(info) });
}
@ -909,7 +1009,7 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath & path_)
return outputs;
});
if (!settings.isExperimentalFeatureEnabled("ca-derivations"))
if (!settings.isExperimentalFeatureEnabled(Xp::CaDerivations))
return outputs;
auto drv = readInvalidDerivation(path);
@ -977,14 +1077,19 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths)
}
// FIXME: move this, it's not specific to LocalStore.
void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos)
{
if (!settings.useSubstitutes) return;
for (auto & sub : getDefaultSubstituters()) {
for (auto & path : paths) {
if (infos.count(path.first))
// Choose first succeeding substituter.
continue;
auto subPath(path.first);
// recompute store path so that we can use a different store root
// Recompute store path so that we can use a different store root.
if (path.second) {
subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second);
if (sub->storeDir == storeDir)
@ -1099,7 +1204,7 @@ void LocalStore::invalidatePath(State & state, const StorePath & path)
{
auto state_(Store::state.lock());
state_->pathInfoCache.erase(std::string(path.hashPart()));
state_->pathInfoCache.erase(std::string(path.to_string()));
}
}
@ -1145,24 +1250,15 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
deletePath(realPath);
// text hashing has long been allowed to have non-self-references because it is used for drv files.
bool refersToSelf = info.references.count(info.path) > 0;
if (info.ca.has_value() && !info.references.empty() && !(std::holds_alternative<TextHash>(*info.ca) && !refersToSelf))
settings.requireExperimentalFeature("ca-references");
/* While restoring the path from the NAR, compute the hash
of the NAR. */
std::unique_ptr<AbstractHashSink> hashSink;
if (!info.ca.has_value() || !info.references.count(info.path))
hashSink = std::make_unique<HashSink>(htSHA256);
else
hashSink = std::make_unique<HashModuloSink>(htSHA256, std::string(info.path.hashPart()));
HashSink hashSink(htSHA256);
TeeSource wrapperSource { source, *hashSink };
TeeSource wrapperSource { source, hashSink };
restorePath(realPath, wrapperSource);
auto hashResult = hashSink->finish();
auto hashResult = hashSink.finish();
if (hashResult.first != info.narHash)
throw Error("hash mismatch importing path '%s';\n specified: %s\n got: %s",
@ -1172,6 +1268,31 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
throw Error("size mismatch importing path '%s';\n specified: %s\n got: %s",
printStorePath(info.path), info.narSize, hashResult.second);
if (info.ca) {
if (auto foHash = std::get_if<FixedOutputHash>(&*info.ca)) {
auto actualFoHash = hashCAPath(
foHash->method,
foHash->hash.type,
info.path
);
if (foHash->hash != actualFoHash.hash) {
throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s",
printStorePath(info.path),
foHash->hash.to_string(Base32, true),
actualFoHash.hash.to_string(Base32, true));
}
}
if (auto textHash = std::get_if<TextHash>(&*info.ca)) {
auto actualTextHash = hashString(htSHA256, readFile(realPath));
if (textHash->hash != actualTextHash) {
throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s",
printStorePath(info.path),
textHash->hash.to_string(Base32, true),
actualTextHash.to_string(Base32, true));
}
}
}
autoGC();
canonicalisePathMetaData(realPath, -1);
@ -1187,7 +1308,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references)
{
/* For computing the store path. */
auto hashSink = std::make_unique<HashSink>(hashAlgo);
@ -1243,7 +1364,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
auto [hash, size] = hashSink->finish();
auto dstPath = makeFixedOutputPath(method, hash, name);
auto dstPath = makeFixedOutputPath(method, hash, name, references);
addTempRoot(dstPath);
@ -1290,6 +1411,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
ValidPathInfo info { dstPath, narHash.first };
info.narSize = narHash.second;
info.references = references;
info.ca = FixedOutputHash { .method = method, .hash = hash };
registerValidPath(info);
}
@ -1390,7 +1512,8 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
/* Acquire the global GC lock to get a consistent snapshot of
existing and valid paths. */
AutoCloseFD fdGCLock = openGCLock(ltWrite);
auto fdGCLock = openGCLock();
FdLock gcLock(fdGCLock.get(), ltRead, true, "waiting for the big garbage collector lock...");
StringSet store;
for (auto & i : readDirectory(realStoreDir)) store.insert(i.name);
@ -1401,8 +1524,6 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
StorePathSet validPaths;
PathSet done;
fdGCLock = -1;
for (auto & i : queryAllValidPaths())
verifyPath(printStorePath(i), store, done, validPaths, repair, errors);
@ -1440,14 +1561,10 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
/* Check the content hash (optionally - slow). */
printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i));
std::unique_ptr<AbstractHashSink> hashSink;
if (!info->ca || !info->references.count(info->path))
hashSink = std::make_unique<HashSink>(info->narHash.type);
else
hashSink = std::make_unique<HashModuloSink>(info->narHash.type, std::string(info->path.hashPart()));
auto hashSink = HashSink(info->narHash.type);
dumpPath(Store::toRealPath(i), *hashSink);
auto current = hashSink->finish();
dumpPath(Store::toRealPath(i), hashSink);
auto current = hashSink.finish();
if (info->narHash != nullHash && info->narHash != current.first) {
printError("path '%s' was modified! expected hash '%s', got '%s'",
@ -1665,19 +1782,108 @@ void LocalStore::createUser(const std::string & userName, uid_t userId)
}
}
std::optional<const Realisation> LocalStore::queryRealisation(
const DrvOutput& id) {
typedef std::optional<const Realisation> Ret;
return retrySQLite<Ret>([&]() -> Ret {
auto state(_state.lock());
auto use(state->stmts->QueryRealisedOutput.use()(id.strHash())(
id.outputName));
if (!use.next())
return std::nullopt;
auto outputPath = parseStorePath(use.getStr(0));
auto signatures = tokenizeString<StringSet>(use.getStr(1));
return Ret{Realisation{
.id = id, .outPath = outputPath, .signatures = signatures}};
});
std::optional<std::pair<int64_t, Realisation>> LocalStore::queryRealisationCore_(
LocalStore::State & state,
const DrvOutput & id)
{
auto useQueryRealisedOutput(
state.stmts->QueryRealisedOutput.use()
(id.strHash())
(id.outputName));
if (!useQueryRealisedOutput.next())
return std::nullopt;
auto realisationDbId = useQueryRealisedOutput.getInt(0);
auto outputPath = parseStorePath(useQueryRealisedOutput.getStr(1));
auto signatures =
tokenizeString<StringSet>(useQueryRealisedOutput.getStr(2));
return {{
realisationDbId,
Realisation{
.id = id,
.outPath = outputPath,
.signatures = signatures,
}
}};
}
std::optional<const Realisation> LocalStore::queryRealisation_(
LocalStore::State & state,
const DrvOutput & id)
{
auto maybeCore = queryRealisationCore_(state, id);
if (!maybeCore)
return std::nullopt;
auto [realisationDbId, res] = *maybeCore;
std::map<DrvOutput, StorePath> dependentRealisations;
auto useRealisationRefs(
state.stmts->QueryRealisationReferences.use()
(realisationDbId));
while (useRealisationRefs.next()) {
auto depId = DrvOutput {
Hash::parseAnyPrefixed(useRealisationRefs.getStr(0)),
useRealisationRefs.getStr(1),
};
auto dependentRealisation = queryRealisationCore_(state, depId);
assert(dependentRealisation); // Enforced by the db schema
auto outputPath = dependentRealisation->second.outPath;
dependentRealisations.insert({depId, outputPath});
}
res.dependentRealisations = dependentRealisations;
return { res };
}
void LocalStore::queryRealisationUncached(const DrvOutput & id,
Callback<std::shared_ptr<const Realisation>> callback) noexcept
{
try {
auto maybeRealisation
= retrySQLite<std::optional<const Realisation>>([&]() {
auto state(_state.lock());
return queryRealisation_(*state, id);
});
if (maybeRealisation)
callback(
std::make_shared<const Realisation>(maybeRealisation.value()));
else
callback(nullptr);
} catch (...) {
callback.rethrow();
}
}
FixedOutputHash LocalStore::hashCAPath(
const FileIngestionMethod & method, const HashType & hashType,
const StorePath & path)
{
return hashCAPath(method, hashType, Store::toRealPath(path), path.hashPart());
}
FixedOutputHash LocalStore::hashCAPath(
const FileIngestionMethod & method,
const HashType & hashType,
const Path & path,
const std::string_view pathHash
)
{
HashModuloSink caSink ( hashType, std::string(pathHash) );
switch (method) {
case FileIngestionMethod::Recursive:
dumpPath(path, caSink);
break;
case FileIngestionMethod::Flat:
readFile(path, caSink);
break;
}
auto hash = caSink.finish().first;
return FixedOutputHash{
.method = method,
.hash = hash,
};
}
} // namespace nix

56
src/libstore/local-store.hh
View file

@ -58,9 +58,15 @@ private:
struct Stmts;
std::unique_ptr<Stmts> stmts;
/* The global GC lock */
AutoCloseFD fdGCLock;
/* The file to which we write our temporary roots. */
AutoCloseFD fdTempRoots;
/* Connection to the garbage collector. */
AutoCloseFD fdRootsSocket;
/* The last time we checked whether to do an auto-GC, or an
auto-GC finished. */
std::chrono::time_point<std::chrono::steady_clock> lastGCCheck;
@ -83,14 +89,10 @@ private:
public:
PathSetting realStoreDir_;
const Path realStoreDir;
const Path dbDir;
const Path linksDir;
const Path reservedPath;
const Path schemaPath;
const Path trashDir;
const Path tempRootsDir;
const Path fnTempRoots;
@ -143,7 +145,7 @@ public:
RepairFlag repair, CheckSigsFlag checkSigs) override;
StorePath addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override;
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override;
StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override;
@ -152,14 +154,11 @@ public:
void addIndirectRoot(const Path & path) override;
void syncWithGC() override;
private:
typedef std::shared_ptr<AutoCloseFD> FDPtr;
typedef list<FDPtr> FDs;
void findTempRoots(Roots & roots, bool censor);
void findTempRoots(FDs & fds, Roots & roots, bool censor);
AutoCloseFD openGCLock();
public:
@ -206,7 +205,10 @@ public:
void registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) override;
void cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output);
std::optional<const Realisation> queryRealisation(const DrvOutput&) override;
std::optional<const Realisation> queryRealisation_(State & state, const DrvOutput & id);
std::optional<std::pair<int64_t, Realisation>> queryRealisationCore_(State & state, const DrvOutput & id);
void queryRealisationUncached(const DrvOutput&,
Callback<std::shared_ptr<const Realisation>> callback) noexcept override;
private:
@ -237,29 +239,12 @@ private:
PathSet queryValidPathsOld();
ValidPathInfo queryPathInfoOld(const Path & path);
struct GCState;
void deleteGarbage(GCState & state, const Path & path);
void tryToDelete(GCState & state, const Path & path);
bool canReachRoot(GCState & state, StorePathSet & visited, const StorePath & path);
void deletePathRecursive(GCState & state, const Path & path);
bool isActiveTempFile(const GCState & state,
const Path & path, const string & suffix);
AutoCloseFD openGCLock(LockType lockType);
void findRoots(const Path & path, unsigned char type, Roots & roots);
void findRootsNoTemp(Roots & roots, bool censor);
void findRuntimeRoots(Roots & roots, bool censor);
void removeUnusedLinks(const GCState & state);
Path createTempDirInStore();
void checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv);
@ -279,10 +264,21 @@ private:
void signPathInfo(ValidPathInfo & info);
void signRealisation(Realisation &);
Path getRealStoreDir() override { return realStoreDir; }
void createUser(const std::string & userName, uid_t userId) override;
// XXX: Make a generic `Store` method
FixedOutputHash hashCAPath(
const FileIngestionMethod & method,
const HashType & hashType,
const StorePath & path);
FixedOutputHash hashCAPath(
const FileIngestionMethod & method,
const HashType & hashType,
const Path & path,
const std::string_view pathHash
);
friend struct LocalDerivationGoal;
friend struct PathSubstitutionGoal;
friend struct SubstitutionGoal;

10
src/libstore/local.mk
View file

@ -8,12 +8,12 @@ libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc)
libstore_LIBS = libutil
libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread
ifneq ($(OS), FreeBSD)
libstore_LDFLAGS += $(SQLITE3_LIBS) $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread
ifdef HOST_LINUX
libstore_LDFLAGS += -ldl
endif
ifeq ($(OS), Darwin)
ifdef HOST_DARWIN
libstore_FILES = sandbox-defaults.sb sandbox-minimal.sb sandbox-network.sb
endif
@ -23,7 +23,7 @@ ifeq ($(ENABLE_S3), 1)
libstore_LDFLAGS += -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core
endif
ifeq ($(OS), SunOS)
ifdef HOST_SOLARIS
libstore_LDFLAGS += -lsocket
endif
@ -60,7 +60,7 @@ $(d)/build.cc:
clean-files += $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh
$(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644))
$(eval $(call install-file-in, $(d)/nix-store.pc, $(libdir)/pkgconfig, 0644))
$(foreach i, $(wildcard src/libstore/builtins/*.hh), \
$(eval $(call install-file-in, $(i), $(includedir)/nix/builtins, 0644)))

115
src/libstore/machines.cc
View file

@ -16,13 +16,18 @@ Machine::Machine(decltype(storeUri) storeUri,
decltype(mandatoryFeatures) mandatoryFeatures,
decltype(sshPublicHostKey) sshPublicHostKey) :
storeUri(
// Backwards compatibility: if the URI is a hostname,
// prepend ssh://.
// Backwards compatibility: if the URI is schemeless, is not a path,
// and is not one of the special store connection words, prepend
// ssh://.
storeUri.find("://") != std::string::npos
|| hasPrefix(storeUri, "local")
|| hasPrefix(storeUri, "remote")
|| hasPrefix(storeUri, "auto")
|| hasPrefix(storeUri, "/")
|| storeUri.find("/") != std::string::npos
|| storeUri == "auto"
|| storeUri == "daemon"
|| storeUri == "local"
|| hasPrefix(storeUri, "auto?")
|| hasPrefix(storeUri, "daemon?")
|| hasPrefix(storeUri, "local?")
|| hasPrefix(storeUri, "?")
? storeUri
: "ssh://" + storeUri),
systemTypes(systemTypes),
@ -34,7 +39,8 @@ Machine::Machine(decltype(storeUri) storeUri,
sshPublicHostKey(sshPublicHostKey)
{}
bool Machine::allSupported(const std::set<string> & features) const {
bool Machine::allSupported(const std::set<string> & features) const
{
return std::all_of(features.begin(), features.end(),
[&](const string & feature) {
return supportedFeatures.count(feature) ||
@ -42,14 +48,16 @@ bool Machine::allSupported(const std::set<string> & features) const {
});
}
bool Machine::mandatoryMet(const std::set<string> & features) const {
bool Machine::mandatoryMet(const std::set<string> & features) const
{
return std::all_of(mandatoryFeatures.begin(), mandatoryFeatures.end(),
[&](const string & feature) {
return features.count(feature);
});
}
ref<Store> Machine::openStore() const {
ref<Store> Machine::openStore() const
{
Store::Params storeParams;
if (hasPrefix(storeUri, "ssh://")) {
storeParams["max-connections"] = "1";
@ -78,53 +86,86 @@ ref<Store> Machine::openStore() const {
return nix::openStore(storeUri, storeParams);
}
void parseMachines(const std::string & s, Machines & machines)
static std::vector<std::string> expandBuilderLines(const std::string & builders)
{
for (auto line : tokenizeString<std::vector<string>>(s, "\n;")) {
std::vector<std::string> result;
for (auto line : tokenizeString<std::vector<string>>(builders, "\n;")) {
trim(line);
line.erase(std::find(line.begin(), line.end(), '#'), line.end());
if (line.empty()) continue;
if (line[0] == '@') {
auto file = trim(std::string(line, 1));
const std::string path = trim(std::string(line, 1));
std::string text;
try {
parseMachines(readFile(file), machines);
text = readFile(path);
} catch (const SysError & e) {
if (e.errNo != ENOENT)
throw;
debug("cannot find machines file '%s'", file);
debug("cannot find machines file '%s'", path);
}
const auto lines = expandBuilderLines(text);
result.insert(end(result), begin(lines), end(lines));
continue;
}
auto tokens = tokenizeString<std::vector<string>>(line);
auto sz = tokens.size();
if (sz < 1)
throw FormatError("bad machine specification '%s'", line);
auto isSet = [&](size_t n) {
return tokens.size() > n && tokens[n] != "" && tokens[n] != "-";
};
machines.emplace_back(tokens[0],
isSet(1) ? tokenizeString<std::vector<string>>(tokens[1], ",") : std::vector<string>{settings.thisSystem},
isSet(2) ? tokens[2] : "",
isSet(3) ? std::stoull(tokens[3]) : 1LL,
isSet(4) ? std::stoull(tokens[4]) : 1LL,
isSet(5) ? tokenizeString<std::set<string>>(tokens[5], ",") : std::set<string>{},
isSet(6) ? tokenizeString<std::set<string>>(tokens[6], ",") : std::set<string>{},
isSet(7) ? tokens[7] : "");
result.emplace_back(line);
}
return result;
}
static Machine parseBuilderLine(const std::string & line)
{
const auto tokens = tokenizeString<std::vector<string>>(line);
auto isSet = [&](size_t fieldIndex) {
return tokens.size() > fieldIndex && tokens[fieldIndex] != "" && tokens[fieldIndex] != "-";
};
auto parseUnsignedIntField = [&](size_t fieldIndex) {
const auto result = string2Int<unsigned int>(tokens[fieldIndex]);
if (!result) {
throw FormatError("bad machine specification: failed to convert column #%lu in a row: '%s' to 'unsigned int'", fieldIndex, line);
}
return result.value();
};
auto ensureBase64 = [&](size_t fieldIndex) {
const auto & str = tokens[fieldIndex];
try {
base64Decode(str);
} catch (const Error & e) {
throw FormatError("bad machine specification: a column #%lu in a row: '%s' is not valid base64 string: %s", fieldIndex, line, e.what());
}
return str;
};
if (!isSet(0))
throw FormatError("bad machine specification: store URL was not found at the first column of a row: '%s'", line);
return {
tokens[0],
isSet(1) ? tokenizeString<std::vector<string>>(tokens[1], ",") : std::vector<string>{settings.thisSystem},
isSet(2) ? tokens[2] : "",
isSet(3) ? parseUnsignedIntField(3) : 1U,
isSet(4) ? parseUnsignedIntField(4) : 1U,
isSet(5) ? tokenizeString<std::set<string>>(tokens[5], ",") : std::set<string>{},
isSet(6) ? tokenizeString<std::set<string>>(tokens[6], ",") : std::set<string>{},
isSet(7) ? ensureBase64(7) : ""
};
}
static Machines parseBuilderLines(const std::vector<std::string>& builders) {
Machines result;
std::transform(builders.begin(), builders.end(), std::back_inserter(result), parseBuilderLine);
return result;
}
Machines getMachines()
{
static auto machines = [&]() {
Machines machines;
parseMachines(settings.builders, machines);
return machines;
}();
return machines;
const auto builderLines = expandBuilderLines(settings.builders);
return parseBuilderLines(builderLines);
}
}

181
src/libstore/misc.cc
View file

@ -6,98 +6,73 @@
#include "thread-pool.hh"
#include "topo-sort.hh"
#include "callback.hh"
#include "closure.hh"
namespace nix {
void Store::computeFSClosure(const StorePathSet & startPaths,
StorePathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers)
{
struct State
{
size_t pending;
StorePathSet & paths;
std::exception_ptr exc;
};
std::function<std::set<StorePath>(const StorePath & path, std::future<ref<const ValidPathInfo>> &)> queryDeps;
if (flipDirection)
queryDeps = [&](const StorePath& path,
std::future<ref<const ValidPathInfo>> & fut) {
StorePathSet res;
StorePathSet referrers;
queryReferrers(path, referrers);
for (auto& ref : referrers)
if (ref != path)
res.insert(ref);
Sync<State> state_(State{0, paths_, 0});
if (includeOutputs)
for (auto& i : queryValidDerivers(path))
res.insert(i);
std::function<void(const StorePath &)> enqueue;
if (includeDerivers && path.isDerivation())
for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path))
if (maybeOutPath && isValidPath(*maybeOutPath))
res.insert(*maybeOutPath);
return res;
};
else
queryDeps = [&](const StorePath& path,
std::future<ref<const ValidPathInfo>> & fut) {
StorePathSet res;
auto info = fut.get();
for (auto& ref : info->references)
if (ref != path)
res.insert(ref);
std::condition_variable done;
if (includeOutputs && path.isDerivation())
for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path))
if (maybeOutPath && isValidPath(*maybeOutPath))
res.insert(*maybeOutPath);
enqueue = [&](const StorePath & path) -> void {
{
auto state(state_.lock());
if (state->exc) return;
if (!state->paths.insert(path).second) return;
state->pending++;
}
if (includeDerivers && info->deriver && isValidPath(*info->deriver))
res.insert(*info->deriver);
return res;
};
queryPathInfo(path, {[&](std::future<ref<const ValidPathInfo>> fut) {
// FIXME: calls to isValidPath() should be async
try {
auto info = fut.get();
if (flipDirection) {
StorePathSet referrers;
queryReferrers(path, referrers);
for (auto & ref : referrers)
if (ref != path)
enqueue(ref);
if (includeOutputs)
for (auto & i : queryValidDerivers(path))
enqueue(i);
if (includeDerivers && path.isDerivation())
for (auto & i : queryDerivationOutputs(path))
if (isValidPath(i) && queryPathInfo(i)->deriver == path)
enqueue(i);
} else {
for (auto & ref : info->references)
if (ref != path)
enqueue(ref);
if (includeOutputs && path.isDerivation())
for (auto & i : queryDerivationOutputs(path))
if (isValidPath(i)) enqueue(i);
if (includeDerivers && info->deriver && isValidPath(*info->deriver))
enqueue(*info->deriver);
}
{
auto state(state_.lock());
assert(state->pending);
if (!--state->pending) done.notify_one();
}
} catch (...) {
auto state(state_.lock());
if (!state->exc) state->exc = std::current_exception();
assert(state->pending);
if (!--state->pending) done.notify_one();
};
}});
};
for (auto & startPath : startPaths)
enqueue(startPath);
{
auto state(state_.lock());
while (state->pending) state.wait(done);
if (state->exc) std::rethrow_exception(state->exc);
}
computeClosure<StorePath>(
startPaths, paths_,
[&](const StorePath& path,
std::function<void(std::promise<std::set<StorePath>>&)>
processEdges) {
std::promise<std::set<StorePath>> promise;
std::function<void(std::future<ref<const ValidPathInfo>>)>
getDependencies =
[&](std::future<ref<const ValidPathInfo>> fut) {
try {
promise.set_value(queryDeps(path, fut));
} catch (...) {
promise.set_exception(std::current_exception());
}
};
queryPathInfo(path, getDependencies);
processEdges(promise);
});
}
void Store::computeFSClosure(const StorePath & startPath,
StorePathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers)
{
@ -191,7 +166,7 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
}
std::visit(overloaded {
[&](DerivedPath::Built bfd) {
[&](const DerivedPath::Built & bfd) {
if (!isValidPath(bfd.drvPath)) {
// FIXME: we could try to substitute the derivation.
auto state(state_.lock());
@ -224,7 +199,7 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
mustBuildDrv(bfd.drvPath, *drv);
},
[&](DerivedPath::Opaque bo) {
[&](const DerivedPath::Opaque & bo) {
if (isValidPath(bo.path)) return;
@ -264,12 +239,11 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths)
{
return topoSort(paths,
{[&](const StorePath & path) {
StorePathSet references;
try {
references = queryPathInfo(path)->references;
return queryPathInfo(path)->references;
} catch (InvalidPath &) {
return StorePathSet();
}
return references;
}},
{[&](const StorePath & path, const StorePath & parent) {
return BuildError(
@ -279,5 +253,44 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths)
}});
}
std::map<DrvOutput, StorePath> drvOutputReferences(
const std::set<Realisation> & inputRealisations,
const StorePathSet & pathReferences)
{
std::map<DrvOutput, StorePath> res;
for (const auto & input : inputRealisations) {
if (pathReferences.count(input.outPath)) {
res.insert({input.id, input.outPath});
}
}
return res;
}
std::map<DrvOutput, StorePath> drvOutputReferences(
Store & store,
const Derivation & drv,
const StorePath & outputPath)
{
std::set<Realisation> inputRealisations;
for (const auto& [inputDrv, outputNames] : drv.inputDrvs) {
auto outputHashes =
staticOutputHashes(store, store.readDerivation(inputDrv));
for (const auto& outputName : outputNames) {
auto thisRealisation = store.queryRealisation(
DrvOutput{outputHashes.at(outputName), outputName});
if (!thisRealisation)
throw Error(
"output '%s' of derivation '%s' isn’t built", outputName,
store.printStorePath(inputDrv));
inputRealisations.insert(*thisRealisation);
}
}
auto info = store.queryPathInfo(outputPath);
return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references);
}
}

99
src/libstore/nar-info-disk-cache.cc
View file

@ -4,6 +4,7 @@
#include "globals.hh"
#include <sqlite3.h>
#include <nlohmann/json.hpp>
namespace nix {
@ -38,6 +39,15 @@ create table if not exists NARs (
foreign key (cache) references BinaryCaches(id) on delete cascade
);
create table if not exists Realisations (
cache integer not null,
outputId text not null,
content blob, -- Json serialisation of the realisation, or null if the realisation is absent
timestamp integer not null,
primary key (cache, outputId),
foreign key (cache) references BinaryCaches(id) on delete cascade
);
create table if not exists LastPurge (
dummy text primary key,
value integer
@ -63,7 +73,9 @@ public:
struct State
{
SQLite db;
SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, queryNAR, purgeCache;
SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR,
queryNAR, insertRealisation, insertMissingRealisation,
queryRealisation, purgeCache;
std::map<std::string, Cache> caches;
};
@ -98,6 +110,26 @@ public:
state->queryNAR.create(state->db,
"select present, namePart, url, compression, fileHash, fileSize, narHash, narSize, refs, deriver, sigs, ca from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))");
state->insertRealisation.create(state->db,
R"(
insert or replace into Realisations(cache, outputId, content, timestamp)
values (?, ?, ?, ?)
)");
state->insertMissingRealisation.create(state->db,
R"(
insert or replace into Realisations(cache, outputId, timestamp)
values (?, ?, ?)
)");
state->queryRealisation.create(state->db,
R"(
select content from Realisations
where cache = ? and outputId = ? and
((content is null and timestamp > ?) or
(content is not null and timestamp > ?))
)");
/* Periodically purge expired entries from the database. */
retrySQLite<void>([&]() {
auto now = time(0);
@ -212,6 +244,38 @@ public:
});
}
std::pair<Outcome, std::shared_ptr<Realisation>> lookupRealisation(
const std::string & uri, const DrvOutput & id) override
{
return retrySQLite<std::pair<Outcome, std::shared_ptr<Realisation>>>(
[&]() -> std::pair<Outcome, std::shared_ptr<Realisation>> {
auto state(_state.lock());
auto & cache(getCache(*state, uri));
auto now = time(0);
auto queryRealisation(state->queryRealisation.use()
(cache.id)
(id.to_string())
(now - settings.ttlNegativeNarInfoCache)
(now - settings.ttlPositiveNarInfoCache));
if (!queryRealisation.next())
return {oUnknown, 0};
if (queryRealisation.isNull(0))
return {oInvalid, 0};
auto realisation =
std::make_shared<Realisation>(Realisation::fromJSON(
nlohmann::json::parse(queryRealisation.getStr(0)),
"Local disk cache"));
return {oValid, realisation};
});
}
void upsertNarInfo(
const std::string & uri, const std::string & hashPart,
std::shared_ptr<const ValidPathInfo> info) override
@ -251,6 +315,39 @@ public:
}
});
}
void upsertRealisation(
const std::string & uri,
const Realisation & realisation) override
{
retrySQLite<void>([&]() {
auto state(_state.lock());
auto & cache(getCache(*state, uri));
state->insertRealisation.use()
(cache.id)
(realisation.id.to_string())
(realisation.toJSON().dump())
(time(0)).exec();
});
}
virtual void upsertAbsentRealisation(
const std::string & uri,
const DrvOutput & id) override
{
retrySQLite<void>([&]() {
auto state(_state.lock());
auto & cache(getCache(*state, uri));
state->insertMissingRealisation.use()
(cache.id)
(id.to_string())
(time(0)).exec();
});
}
};
ref<NarInfoDiskCache> getNarInfoDiskCache()

10
src/libstore/nar-info-disk-cache.hh
View file

@ -2,6 +2,7 @@
#include "ref.hh"
#include "nar-info.hh"
#include "realisation.hh"
namespace nix {
@ -29,6 +30,15 @@ public:
virtual void upsertNarInfo(
const std::string & uri, const std::string & hashPart,
std::shared_ptr<const ValidPathInfo> info) = 0;
virtual void upsertRealisation(
const std::string & uri,
const Realisation & realisation) = 0;
virtual void upsertAbsentRealisation(
const std::string & uri,
const DrvOutput & id) = 0;
virtual std::pair<Outcome, std::shared_ptr<Realisation>> lookupRealisation(
const std::string & uri, const DrvOutput & id) = 0;
};
/* Return a singleton cache object that can be used concurrently by

2
src/libstore/optimise-store.cc
View file

@ -198,7 +198,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
/* Make the containing directory writable, but only if it's not
the store itself (we don't want or need to mess with its
permissions). */
bool mustToggle = dirOf(path) != realStoreDir;
bool mustToggle = dirOf(path) != realStoreDir.get();
if (mustToggle) makeWritable(dirOf(path));
/* When we're done, make the directory read-only again and reset

107
src/libstore/parsed-derivations.cc
View file

@ -1,6 +1,8 @@
#include "parsed-derivations.hh"
#include <nlohmann/json.hpp>
#include <regex>
#include "json.hh"
namespace nix {
@ -91,6 +93,8 @@ StringSet ParsedDerivation::getRequiredSystemFeatures() const
StringSet res;
for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings()))
res.insert(i);
if (!derivationHasKnownOutputPaths(drv.type()))
res.insert("ca-derivations");
return res;
}
@ -121,4 +125,107 @@ bool ParsedDerivation::substitutesAllowed() const
return getBoolAttr("allowSubstitutes", true);
}
static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
std::optional<nlohmann::json> ParsedDerivation::prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths)
{
auto structuredAttrs = getStructuredAttrs();
if (!structuredAttrs) return std::nullopt;
auto json = *structuredAttrs;
/* Add an "outputs" object containing the output paths. */
nlohmann::json outputs;
for (auto & i : drv.outputs)
outputs[i.first] = hashPlaceholder(i.first);
json["outputs"] = outputs;
/* Handle exportReferencesGraph. */
auto e = json.find("exportReferencesGraph");
if (e != json.end() && e->is_object()) {
for (auto i = e->begin(); i != e->end(); ++i) {
std::ostringstream str;
{
JSONPlaceholder jsonRoot(str, true);
StorePathSet storePaths;
for (auto & p : *i)
storePaths.insert(store.parseStorePath(p.get<std::string>()));
store.pathInfoToJSON(jsonRoot,
store.exportReferences(storePaths, inputPaths), false, true);
}
json[i.key()] = nlohmann::json::parse(str.str()); // urgh
}
}
return json;
}
/* As a convenience to bash scripts, write a shell file that
maps all attributes that are representable in bash -
namely, strings, integers, nulls, Booleans, and arrays and
objects consisting entirely of those values. (So nested
arrays or objects are not supported.) */
std::string writeStructuredAttrsShell(const nlohmann::json & json)
{
auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> {
if (value.is_string())
return shellEscape(value);
if (value.is_number()) {
auto f = value.get<float>();
if (std::ceil(f) == f)
return std::to_string(value.get<int>());
}
if (value.is_null())
return std::string("''");
if (value.is_boolean())
return value.get<bool>() ? std::string("1") : std::string("");
return {};
};
std::string jsonSh;
for (auto & [key, value] : json.items()) {
if (!std::regex_match(key, shVarName)) continue;
auto s = handleSimpleType(value);
if (s)
jsonSh += fmt("declare %s=%s\n", key, *s);
else if (value.is_array()) {
std::string s2;
bool good = true;
for (auto & value2 : value) {
auto s3 = handleSimpleType(value2);
if (!s3) { good = false; break; }
s2 += *s3; s2 += ' ';
}
if (good)
jsonSh += fmt("declare -a %s=(%s)\n", key, s2);
}
else if (value.is_object()) {
std::string s2;
bool good = true;
for (auto & [key2, value2] : value.items()) {
auto s3 = handleSimpleType(value2);
if (!s3) { good = false; break; }
s2 += fmt("[%s]=%s ", shellEscape(key2), *s3);
}
if (good)
jsonSh += fmt("declare -A %s=(%s)\n", key, s2);
}
}
return jsonSh;
}
}

4
src/libstore/parsed-derivations.hh
View file

@ -36,6 +36,10 @@ public:
bool willBuildLocally(Store & localStore) const;
bool substitutesAllowed() const;
std::optional<nlohmann::json> prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths);
};
std::string writeStructuredAttrsShell(const nlohmann::json & json);
}

46
src/libstore/path-info.cc Normal file
View file

@ -0,0 +1,46 @@
#include "path-info.hh"
#include "worker-protocol.hh"
namespace nix {
ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format)
{
return read(source, store, format, store.parseStorePath(readString(source)));
}
ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format, StorePath && path)
{
auto deriver = readString(source);
auto narHash = Hash::parseAny(readString(source), htSHA256);
ValidPathInfo info(path, narHash);
if (deriver != "") info.deriver = store.parseStorePath(deriver);
info.references = worker_proto::read(store, source, Phantom<StorePathSet> {});
source >> info.registrationTime >> info.narSize;
if (format >= 16) {
source >> info.ultimate;
info.sigs = readStrings<StringSet>(source);
info.ca = parseContentAddressOpt(readString(source));
}
return info;
}
void ValidPathInfo::write(
Sink & sink,
const Store & store,
unsigned int format,
bool includePath) const
{
if (includePath)
sink << store.printStorePath(path);
sink << (deriver ? store.printStorePath(*deriver) : "")
<< narHash.to_string(Base16, false);
worker_proto::write(store, sink, references);
sink << registrationTime << narSize;
if (format >= 16) {
sink << ultimate
<< sigs
<< renderContentAddress(ca);
}
}
}

5
src/libstore/path-info.hh
View file

@ -105,6 +105,11 @@ struct ValidPathInfo
ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { };
virtual ~ValidPathInfo() { }
static ValidPathInfo read(Source & source, const Store & store, unsigned int format);
static ValidPathInfo read(Source & source, const Store & store, unsigned int format, StorePath && path);
void write(Sink & sink, const Store & store, unsigned int format, bool includePath = true) const;
};
typedef std::map<StorePath, ValidPathInfo> ValidPathInfos;

4
src/libstore/path-with-outputs.cc
View file

@ -31,14 +31,14 @@ std::vector<DerivedPath> toDerivedPaths(const std::vector<StorePathWithOutputs>
std::variant<StorePathWithOutputs, StorePath> StorePathWithOutputs::tryFromDerivedPath(const DerivedPath & p)
{
return std::visit(overloaded {
[&](DerivedPath::Opaque bo) -> std::variant<StorePathWithOutputs, StorePath> {
[&](const DerivedPath::Opaque & bo) -> std::variant<StorePathWithOutputs, StorePath> {
if (bo.path.isDerivation()) {
// drv path gets interpreted as "build", not "get drv file itself"
return bo.path;
}
return StorePathWithOutputs { bo.path };
},
[&](DerivedPath::Built bfd) -> std::variant<StorePathWithOutputs, StorePath> {
[&](const DerivedPath::Built & bfd) -> std::variant<StorePathWithOutputs, StorePath> {
return StorePathWithOutputs { bfd.drvPath, bfd.outputs };
},
}, p.raw());

13
src/libstore/pathlocks.cc
View file

@ -176,4 +176,17 @@ void PathLocks::setDeletion(bool deletePaths)
}
FdLock::FdLock(int fd, LockType lockType, bool wait, std::string_view waitMsg)
: fd(fd)
{
if (wait) {
if (!lockFile(fd, lockType, false)) {
printInfo("%s", waitMsg);
acquired = lockFile(fd, lockType, true);
}
} else
acquired = lockFile(fd, lockType, false);
}
}

14
src/libstore/pathlocks.hh
View file

@ -35,4 +35,18 @@ public:
void setDeletion(bool deletePaths);
};
struct FdLock
{
int fd;
bool acquired = false;
FdLock(int fd, LockType lockType, bool wait, std::string_view waitMsg);
~FdLock()
{
if (acquired)
lockFile(fd, ltNone, false);
}
};
}

42
src/libstore/profiles.cc
View file

@ -126,9 +126,9 @@ void deleteGeneration(const Path & profile, GenerationNumber gen)
static void deleteGeneration2(const Path & profile, GenerationNumber gen, bool dryRun)
{
if (dryRun)
printInfo(format("would remove generation %1%") % gen);
notice("would remove profile version %1%", gen);
else {
printInfo(format("removing generation %1%") % gen);
notice("removing profile version %1%", gen);
deleteGeneration(profile, gen);
}
}
@ -142,7 +142,7 @@ void deleteGenerations(const Path & profile, const std::set<GenerationNumber> &
auto [gens, curGen] = findGenerations(profile);
if (gensToDelete.count(*curGen))
throw Error("cannot delete current generation of profile %1%'", profile);
throw Error("cannot delete current version of profile %1%'", profile);
for (auto & i : gens) {
if (!gensToDelete.count(i.number)) continue;
@ -211,12 +211,15 @@ void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun)
void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, bool dryRun)
{
if (timeSpec.empty() || timeSpec[timeSpec.size() - 1] != 'd')
throw UsageError("invalid number of days specifier '%1%', expected something like '14d'", timeSpec);
time_t curTime = time(0);
string strDays = string(timeSpec, 0, timeSpec.size() - 1);
auto days = string2Int<int>(strDays);
if (!days || *days < 1)
throw Error("invalid number of days specifier '%1%'", timeSpec);
throw UsageError("invalid number of days specifier '%1%'", timeSpec);
time_t oldTime = curTime - *days * 24 * 3600;
@ -233,6 +236,37 @@ void switchLink(Path link, Path target)
}
void switchGeneration(
const Path & profile,
std::optional<GenerationNumber> dstGen,
bool dryRun)
{
PathLocks lock;
lockProfile(lock, profile);
auto [gens, curGen] = findGenerations(profile);
std::optional<Generation> dst;
for (auto & i : gens)
if ((!dstGen && i.number < curGen) ||
(dstGen && i.number == *dstGen))
dst = i;
if (!dst) {
if (dstGen)
throw Error("profile version %1% does not exist", *dstGen);
else
throw Error("no profile version older than the current (%1%) exists", curGen.value_or(0));
}
notice("switching profile from version %d to %d", curGen.value_or(0), dst->number);
if (dryRun) return;
switchLink(profile, dst->path);
}
void lockProfile(PathLocks & lock, const Path & profile)
{
lock.lockPaths({profile}, (format("waiting for lock on profile '%1%'") % profile).str());

9
src/libstore/profiles.hh
View file

@ -11,7 +11,7 @@ namespace nix {
class StorePath;
typedef unsigned int GenerationNumber;
typedef uint64_t GenerationNumber;
struct Generation
{
@ -46,6 +46,13 @@ void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, b
void switchLink(Path link, Path target);
/* Roll back a profile to the specified generation, or to the most
recent one older than the current. */
void switchGeneration(
const Path & profile,
std::optional<GenerationNumber> dstGen,
bool dryRun);
/* Ensure exclusive access to a profile. Any command that modifies
the profile first acquires this lock. */
void lockProfile(PathLocks & lock, const Path & profile);

66
src/libstore/realisation.cc
View file

@ -1,5 +1,6 @@
#include "realisation.hh"
#include "store-api.hh"
#include "closure.hh"
#include <nlohmann/json.hpp>
namespace nix {
@ -21,11 +22,52 @@ std::string DrvOutput::to_string() const {
return strHash() + "!" + outputName;
}
std::set<Realisation> Realisation::closure(Store & store, const std::set<Realisation> & startOutputs)
{
std::set<Realisation> res;
Realisation::closure(store, startOutputs, res);
return res;
}
void Realisation::closure(Store & store, const std::set<Realisation> & startOutputs, std::set<Realisation> & res)
{
auto getDeps = [&](const Realisation& current) -> std::set<Realisation> {
std::set<Realisation> res;
for (auto& [currentDep, _] : current.dependentRealisations) {
if (auto currentRealisation = store.queryRealisation(currentDep))
res.insert(*currentRealisation);
else
throw Error(
"Unrealised derivation '%s'", currentDep.to_string());
}
return res;
};
computeClosure<Realisation>(
startOutputs, res,
[&](const Realisation& current,
std::function<void(std::promise<std::set<Realisation>>&)>
processEdges) {
std::promise<std::set<Realisation>> promise;
try {
auto res = getDeps(current);
promise.set_value(res);
} catch (...) {
promise.set_exception(std::current_exception());
}
return processEdges(promise);
});
}
nlohmann::json Realisation::toJSON() const {
auto jsonDependentRealisations = nlohmann::json::object();
for (auto & [depId, depOutPath] : dependentRealisations)
jsonDependentRealisations.emplace(depId.to_string(), depOutPath.to_string());
return nlohmann::json{
{"id", id.to_string()},
{"outPath", outPath.to_string()},
{"signatures", signatures},
{"dependentRealisations", jsonDependentRealisations},
};
}
@ -51,10 +93,16 @@ Realisation Realisation::fromJSON(
if (auto signaturesIterator = json.find("signatures"); signaturesIterator != json.end())
signatures.insert(signaturesIterator->begin(), signaturesIterator->end());
std::map <DrvOutput, StorePath> dependentRealisations;
if (auto jsonDependencies = json.find("dependentRealisations"); jsonDependencies != json.end())
for (auto & [jsonDepId, jsonDepOutPath] : jsonDependencies->get<std::map<std::string, std::string>>())
dependentRealisations.insert({DrvOutput::parse(jsonDepId), StorePath(jsonDepOutPath)});
return Realisation{
.id = DrvOutput::parse(getField("id")),
.outPath = StorePath(getField("outPath")),
.signatures = signatures,
.dependentRealisations = dependentRealisations,
};
}
@ -92,6 +140,24 @@ StorePath RealisedPath::path() const {
return std::visit([](auto && arg) { return arg.getPath(); }, raw);
}
bool Realisation::isCompatibleWith(const Realisation & other) const
{
assert (id == other.id);
if (outPath == other.outPath) {
if (dependentRealisations.empty() != other.dependentRealisations.empty()) {
warn(
"Encountered a realisation for '%s' with an empty set of "
"dependencies. This is likely an artifact from an older Nix. "
"I’ll try to fix the realisation if I can",
id.to_string());
return true;
} else if (dependentRealisations == other.dependentRealisations) {
return true;
}
}
return false;
}
void RealisedPath::closure(
Store& store,
const RealisedPath::Set& startPaths,

13
src/libstore/realisation.hh
View file

@ -28,6 +28,14 @@ struct Realisation {
StringSet signatures;
/**
* The realisations that are required for the current one to be valid.
*
* When importing this realisation, the store will first check that all its
* dependencies exist, and map to the correct output path
*/
std::map<DrvOutput, StorePath> dependentRealisations;
nlohmann::json toJSON() const;
static Realisation fromJSON(const nlohmann::json& json, const std::string& whence);
@ -36,6 +44,11 @@ struct Realisation {
bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const;
size_t checkSignatures(const PublicKeys & publicKeys) const;
static std::set<Realisation> closure(Store &, const std::set<Realisation> &);
static void closure(Store &, const std::set<Realisation> &, std::set<Realisation> & res);
bool isCompatibleWith(const Realisation & other) const;
StorePath getPath() const { return outPath; }
GENERATE_CMP(Realisation, me->id, me->outPath);

97
src/libstore/references.cc
View file

@ -5,27 +5,29 @@
#include <map>
#include <cstdlib>
#include <mutex>
namespace nix {
static unsigned int refLength = 32; /* characters */
static size_t refLength = 32; /* characters */
static void search(const unsigned char * s, size_t len,
StringSet & hashes, StringSet & seen)
static void search(
std::string_view s,
StringSet & hashes,
StringSet & seen)
{
static bool initialised = false;
static std::once_flag initialised;
static bool isBase32[256];
if (!initialised) {
std::call_once(initialised, [](){
for (unsigned int i = 0; i < 256; ++i) isBase32[i] = false;
for (unsigned int i = 0; i < base32Chars.size(); ++i)
isBase32[(unsigned char) base32Chars[i]] = true;
initialised = true;
}
});
for (size_t i = 0; i + refLength <= len; ) {
for (size_t i = 0; i + refLength <= s.size(); ) {
int j;
bool match = true;
for (j = refLength - 1; j >= 0; --j)
@ -35,7 +37,7 @@ static void search(const unsigned char * s, size_t len,
break;
}
if (!match) continue;
string ref((const char *) s + i, refLength);
std::string ref(s.substr(i, refLength));
if (hashes.erase(ref)) {
debug(format("found reference to '%1%' at offset '%2%'")
% ref % i);
@ -46,69 +48,60 @@ static void search(const unsigned char * s, size_t len,
}
struct RefScanSink : Sink
void RefScanSink::operator () (std::string_view data)
{
StringSet hashes;
StringSet seen;
/* It's possible that a reference spans the previous and current
fragment, so search in the concatenation of the tail of the
previous fragment and the start of the current fragment. */
auto s = tail;
auto tailLen = std::min(data.size(), refLength);
s.append(data.data(), tailLen);
search(s, hashes, seen);
string tail;
search(data, hashes, seen);
RefScanSink() { }
void operator () (std::string_view data) override
{
/* It's possible that a reference spans the previous and current
fragment, so search in the concatenation of the tail of the
previous fragment and the start of the current fragment. */
string s = tail + std::string(data, 0, refLength);
search((const unsigned char *) s.data(), s.size(), hashes, seen);
search((const unsigned char *) data.data(), data.size(), hashes, seen);
size_t tailLen = data.size() <= refLength ? data.size() : refLength;
tail = std::string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen));
tail.append({data.data() + data.size() - tailLen, tailLen});
}
};
auto rest = refLength - tailLen;
if (rest < tail.size())
tail = tail.substr(tail.size() - rest);
tail.append(data.data() + data.size() - tailLen, tailLen);
}
std::pair<PathSet, HashResult> scanForReferences(const string & path,
const PathSet & refs)
std::pair<StorePathSet, HashResult> scanForReferences(
const string & path,
const StorePathSet & refs)
{
HashSink hashSink { htSHA256 };
auto found = scanForReferences(hashSink, path, refs);
auto hash = hashSink.finish();
return std::pair<PathSet, HashResult>(found, hash);
return std::pair<StorePathSet, HashResult>(found, hash);
}
PathSet scanForReferences(Sink & toTee,
const string & path, const PathSet & refs)
StorePathSet scanForReferences(
Sink & toTee,
const Path & path,
const StorePathSet & refs)
{
RefScanSink refsSink;
TeeSink sink { refsSink, toTee };
std::map<string, Path> backMap;
StringSet hashes;
std::map<std::string, StorePath> backMap;
for (auto & i : refs) {
auto baseName = std::string(baseNameOf(i));
string::size_type pos = baseName.find('-');
if (pos == string::npos)
throw Error("bad reference '%1%'", i);
string s = string(baseName, 0, pos);
assert(s.size() == refLength);
assert(backMap.find(s) == backMap.end());
// parseHash(htSHA256, s);
refsSink.hashes.insert(s);
backMap[s] = i;
std::string hashPart(i.hashPart());
auto inserted = backMap.emplace(hashPart, i).second;
assert(inserted);
hashes.insert(hashPart);
}
/* Look for the hashes in the NAR dump of the path. */
RefScanSink refsSink(std::move(hashes));
TeeSink sink { refsSink, toTee };
dumpPath(path, sink);
/* Map the hashes found back to their store paths. */
PathSet found;
for (auto & i : refsSink.seen) {
std::map<string, Path>::iterator j;
if ((j = backMap.find(i)) == backMap.end()) abort();
StorePathSet found;
for (auto & i : refsSink.getResult()) {
auto j = backMap.find(i);
assert(j != backMap.end());
found.insert(j->second);
}

24
src/libstore/references.hh
View file

@ -1,13 +1,31 @@
#pragma once
#include "types.hh"
#include "hash.hh"
#include "path.hh"
namespace nix {
std::pair<PathSet, HashResult> scanForReferences(const Path & path, const PathSet & refs);
std::pair<StorePathSet, HashResult> scanForReferences(const Path & path, const StorePathSet & refs);
PathSet scanForReferences(Sink & toTee, const Path & path, const PathSet & refs);
StorePathSet scanForReferences(Sink & toTee, const Path & path, const StorePathSet & refs);
class RefScanSink : public Sink
{
StringSet hashes;
StringSet seen;
std::string tail;
public:
RefScanSink(StringSet && hashes) : hashes(hashes)
{ }
StringSet & getResult()
{ return seen; }
void operator () (std::string_view data) override;
};
struct RewritingSink : Sink
{

139
src/libstore/remote-store.cc
View file

@ -162,8 +162,19 @@ void RemoteStore::initConnection(Connection & conn)
try {
conn.to << WORKER_MAGIC_1;
conn.to.flush();
unsigned int magic = readInt(conn.from);
if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch");
StringSink saved;
try {
TeeSource tee(conn.from, saved);
unsigned int magic = readInt(tee);
if (magic != WORKER_MAGIC_2)
throw Error("protocol mismatch");
} catch (SerialisationError & e) {
/* In case the other side is waiting for our input, close
it. */
conn.closeWrite();
auto msg = conn.from.drain();
throw Error("protocol mismatch, got '%s'", chomp(*saved.s + msg));
}
conn.from >> conn.daemonVersion;
if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION))
@ -222,6 +233,7 @@ void RemoteStore::setOptions(Connection & conn)
overrides.erase(settings.buildCores.name);
overrides.erase(settings.useSubstitutes.name);
overrides.erase(loggerSettings.showTrace.name);
overrides.erase(settings.experimentalFeatures.name);
conn.to << overrides.size();
for (auto & i : overrides)
conn.to << i.first << i.second.value;
@ -278,6 +290,10 @@ ConnectionHandle RemoteStore::getConnection()
return ConnectionHandle(connections->get());
}
void RemoteStore::setOptions()
{
setOptions(*(getConnection().handle));
}
bool RemoteStore::isValidPathUncached(const StorePath & path)
{
@ -386,23 +402,6 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
}
ref<const ValidPathInfo> RemoteStore::readValidPathInfo(ConnectionHandle & conn, const StorePath & path)
{
auto deriver = readString(conn->from);
auto narHash = Hash::parseAny(readString(conn->from), htSHA256);
auto info = make_ref<ValidPathInfo>(path, narHash);
if (deriver != "") info->deriver = parseStorePath(deriver);
info->references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
conn->from >> info->registrationTime >> info->narSize;
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
conn->from >> info->ultimate;
info->sigs = readStrings<StringSet>(conn->from);
info->ca = parseContentAddressOpt(readString(conn->from));
}
return info;
}
void RemoteStore::queryPathInfoUncached(const StorePath & path,
Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
{
@ -423,7 +422,8 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path,
bool valid; conn->from >> valid;
if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path));
}
info = readValidPathInfo(conn, path);
info = std::make_shared<ValidPathInfo>(
ValidPathInfo::read(conn->from, *this, GET_PROTOCOL_MINOR(conn->daemonVersion), StorePath{path}));
}
callback(std::move(info));
} catch (...) { callback.rethrow(); }
@ -525,20 +525,20 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
});
}
auto path = parseStorePath(readString(conn->from));
return readValidPathInfo(conn, path);
return make_ref<ValidPathInfo>(
ValidPathInfo::read(conn->from, *this, GET_PROTOCOL_MINOR(conn->daemonVersion)));
}
else {
if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25");
std::visit(overloaded {
[&](TextHashMethod thm) -> void {
[&](const TextHashMethod & thm) -> void {
std::string s = dump.drain();
conn->to << wopAddTextToStore << name << s;
worker_proto::write(*this, conn->to, references);
conn.processStderr();
},
[&](FixedOutputHashMethod fohm) -> void {
[&](const FixedOutputHashMethod & fohm) -> void {
conn->to
<< wopAddToStore
<< name
@ -582,9 +582,8 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method, HashType hashType, RepairFlag repair)
FileIngestionMethod method, HashType hashType, RepairFlag repair, const StorePathSet & references)
{
StorePathSet references;
return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path;
}
@ -642,6 +641,25 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
}
void RemoteStore::addMultipleToStore(
Source & source,
RepairFlag repair,
CheckSigsFlag checkSigs)
{
if (GET_PROTOCOL_MINOR(getConnection()->daemonVersion) >= 32) {
auto conn(getConnection());
conn->to
<< wopAddMultipleToStore
<< repair
<< !checkSigs;
conn.withFramedSink([&](Sink & sink) {
source.drainInto(sink);
});
} else
Store::addMultipleToStore(source, repair, checkSigs);
}
StorePath RemoteStore::addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair)
{
@ -653,36 +671,57 @@ void RemoteStore::registerDrvOutput(const Realisation & info)
{
auto conn(getConnection());
conn->to << wopRegisterDrvOutput;
conn->to << info.id.to_string();
conn->to << std::string(info.outPath.to_string());
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) {
conn->to << info.id.to_string();
conn->to << std::string(info.outPath.to_string());
} else {
worker_proto::write(*this, conn->to, info);
}
conn.processStderr();
}
std::optional<const Realisation> RemoteStore::queryRealisation(const DrvOutput & id)
void RemoteStore::queryRealisationUncached(const DrvOutput & id,
Callback<std::shared_ptr<const Realisation>> callback) noexcept
{
auto conn(getConnection());
conn->to << wopQueryRealisation;
conn->to << id.to_string();
conn.processStderr();
auto outPaths = worker_proto::read(*this, conn->from, Phantom<std::set<StorePath>>{});
if (outPaths.empty())
return std::nullopt;
return {Realisation{.id = id, .outPath = *outPaths.begin()}};
auto real = [&]() -> std::shared_ptr<const Realisation> {
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) {
auto outPaths = worker_proto::read(
*this, conn->from, Phantom<std::set<StorePath>> {});
if (outPaths.empty())
return nullptr;
return std::make_shared<const Realisation>(Realisation { .id = id, .outPath = *outPaths.begin() });
} else {
auto realisations = worker_proto::read(
*this, conn->from, Phantom<std::set<Realisation>> {});
if (realisations.empty())
return nullptr;
return std::make_shared<const Realisation>(*realisations.begin());
}
}();
try {
callback(std::shared_ptr<const Realisation>(real));
} catch (...) { return callback.rethrow(); }
}
static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector<DerivedPath> & reqs)
{
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) {
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 30) {
worker_proto::write(store, conn->to, reqs);
} else {
Strings ss;
for (auto & p : reqs) {
auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p);
std::visit(overloaded {
[&](StorePathWithOutputs s) {
[&](const StorePathWithOutputs & s) {
ss.push_back(s.to_string(store));
},
[&](StorePath drvPath) {
[&](const StorePath & drvPath) {
throw Error("trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file",
store.printStorePath(drvPath),
GET_PROTOCOL_MAJOR(conn->daemonVersion),
@ -694,8 +733,18 @@ static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, cons
}
}
void RemoteStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode)
void RemoteStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore)
{
if (evalStore && evalStore.get() != this) {
/* The remote doesn't have a way to access evalStore, so copy
the .drvs. */
RealisedPath::Set drvPaths2;
for (auto & i : drvPaths)
if (auto p = std::get_if<DerivedPath::Built>(&i))
drvPaths2.insert(p->drvPath);
copyClosure(*evalStore, *this, drvPaths2);
}
auto conn(getConnection());
conn->to << wopBuildPaths;
assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13);
@ -761,15 +810,6 @@ void RemoteStore::addIndirectRoot(const Path & path)
}
void RemoteStore::syncWithGC()
{
auto conn(getConnection());
conn->to << wopSyncWithGC;
conn.processStderr();
readInt(conn->from);
}
Roots RemoteStore::findRoots(bool censor)
{
auto conn(getConnection());
@ -990,14 +1030,14 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
return nullptr;
}
void ConnectionHandle::withFramedSink(std::function<void(Sink &sink)> fun)
void ConnectionHandle::withFramedSink(std::function<void(Sink & sink)> fun)
{
(*this)->to.flush();
std::exception_ptr ex;
/* Handle log messages / exceptions from the remote on a
separate thread. */
/* Handle log messages / exceptions from the remote on a separate
thread. */
std::thread stderrThread([&]()
{
try {
@ -1030,7 +1070,6 @@ void ConnectionHandle::withFramedSink(std::function<void(Sink &sink)> fun)
stderrThread.join();
if (ex)
std::rethrow_exception(ex);
}
}

21
src/libstore/remote-store.hh
View file

@ -73,19 +73,25 @@ public:
/* Add a content-addressable store path. Does not support references. `dump` will be drained. */
StorePath addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override;
void addToStore(const ValidPathInfo & info, Source & nar,
RepairFlag repair, CheckSigsFlag checkSigs) override;
void addMultipleToStore(
Source & source,
RepairFlag repair,
CheckSigsFlag checkSigs) override;
StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override;
void registerDrvOutput(const Realisation & info) override;
std::optional<const Realisation> queryRealisation(const DrvOutput &) override;
void queryRealisationUncached(const DrvOutput &,
Callback<std::shared_ptr<const Realisation>> callback) noexcept override;
void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode) override;
void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override;
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode) override;
@ -96,8 +102,6 @@ public:
void addIndirectRoot(const Path & path) override;
void syncWithGC() override;
Roots findRoots(bool censor) override;
void collectGarbage(const GCOptions & options, GCResults & results) override;
@ -120,7 +124,6 @@ public:
struct Connection
{
AutoCloseFD fd;
FdSink to;
FdSource from;
unsigned int daemonVersion;
@ -128,6 +131,8 @@ public:
virtual ~Connection();
virtual void closeWrite() = 0;
std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true);
};
@ -143,6 +148,8 @@ protected:
virtual void setOptions(Connection & conn);
void setOptions() override;
ConnectionHandle getConnection();
friend struct ConnectionHandle;
@ -151,8 +158,6 @@ protected:
virtual void narFromPath(const StorePath & path, Sink & sink) override;
ref<const ValidPathInfo> readValidPathInfo(ConnectionHandle & conn, const StorePath & path);
private:
std::atomic_bool failed{false};

Some files were not shown because too many files have changed in this diff Show more