wroclaw-git-backups/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-06-28 13:41:15 +02:00
Code Activity

Merge branch 'path-info' into ca-drv-exotic

Browse source
This commit is contained in:
John Ericson 2022-03-10 16:20:01 +00:00
commit 938650700f
427 changed files with 22834 additions and 36076 deletions
Download patch file Download diff file Expand all files Collapse all files

131
src/libstore/binary-cache-store.cc
View file

@ -31,7 +31,7 @@ BinaryCacheStore::BinaryCacheStore(const Params & params)
StringSink sink;
sink << narVersionMagic1;
narMagic = *sink.s;
narMagic = sink.s;
}
void BinaryCacheStore::init()
@ -68,7 +68,7 @@ void BinaryCacheStore::upsertFile(const std::string & path,
}
void BinaryCacheStore::getFile(const std::string & path,
Callback<std::shared_ptr<std::string>> callback) noexcept
Callback<std::optional<std::string>> callback) noexcept
{
try {
callback(getFile(path));
@ -77,9 +77,9 @@ void BinaryCacheStore::getFile(const std::string & path,
void BinaryCacheStore::getFile(const std::string & path, Sink & sink)
{
std::promise<std::shared_ptr<std::string>> promise;
std::promise<std::optional<std::string>> promise;
getFile(path,
{[&](std::future<std::shared_ptr<std::string>> result) {
{[&](std::future<std::optional<std::string>> result) {
try {
promise.set_value(result.get());
} catch (...) {
@ -89,15 +89,15 @@ void BinaryCacheStore::getFile(const std::string & path, Sink & sink)
sink(*promise.get_future().get());
}
std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
std::optional<std::string> BinaryCacheStore::getFile(const std::string & path)
{
StringSink sink;
try {
getFile(path, sink);
} catch (NoSuchBinaryCacheFile &) {
return nullptr;
return std::nullopt;
}
return sink.s;
return std::move(sink.s);
}
std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath)
@ -111,15 +111,15 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo");
std::string hashPart(narInfo->path.hashPart());
{
auto state_(state.lock());
state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = std::shared_ptr<NarInfo>(narInfo) });
state_->pathInfoCache.upsert(
std::string(narInfo->path.to_string()),
PathInfoCacheValue { .value = std::shared_ptr<NarInfo>(narInfo) });
}
if (diskCache)
diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
diskCache->upsertNarInfo(getUri(), std::string(narInfo->path.hashPart()), std::shared_ptr<NarInfo>(narInfo));
}
AutoCloseFD openFile(const Path & path)
@ -149,7 +149,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
{
FdSink fileSink(fdTemp.get());
TeeSink teeSinkCompressed { fileSink, fileHashSink };
auto compressionSink = makeCompressionSink(compression, teeSinkCompressed);
auto compressionSink = makeCompressionSink(compression, teeSinkCompressed, parallelCompression, compressionLevel);
TeeSink teeSinkUncompressed { *compressionSink, narHashSink };
TeeSource teeSource { narSource, teeSinkUncompressed };
narAccessor = makeNarAccessor(teeSource);
@ -306,8 +306,8 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
}});
}
StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view name,
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references)
{
if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256)
unsupported("addToStoreFromDump");
@ -315,13 +315,16 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & nam
ValidPathInfo info {
*this,
{
.name = name,
.name = std::string { name },
.info = FixedOutputInfo {
{
.method = method,
.hash = nar.first,
},
{},
{
.references = references,
.hasSelfReference = false,
},
},
},
nar.first,
@ -375,11 +378,11 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
getFile(narInfoFile,
{[=](std::future<std::shared_ptr<std::string>> fut) {
{[=](std::future<std::optional<std::string>> fut) {
try {
auto data = fut.get();
if (!data) return (*callbackPtr)(nullptr);
if (!data) return (*callbackPtr)({});
stats.narInfoRead++;
@ -393,8 +396,14 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
}});
}
StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
StorePath BinaryCacheStore::addToStore(
std::string_view name,
const Path & srcPath,
FileIngestionMethod method,
HashType hashAlgo,
PathFilter & filter,
RepairFlag repair,
const StorePathSet & references)
{
/* FIXME: Make BinaryCacheStore::addToStoreCommon support
non-recursive+sha256 so we can just use the default
@ -415,13 +424,16 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
ValidPathInfo info {
*this,
{
.name = name,
.name = std::string { name },
.info = FixedOutputInfo {
{
.method = method,
.hash = h,
},
{},
{
.references = references,
.hasSelfReference = false,
},
},
},
nar.first,
@ -431,8 +443,11 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
})->path;
}
StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair)
StorePath BinaryCacheStore::addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references,
RepairFlag repair)
{
auto textHash = hashString(htSHA256, s);
auto path = makeTextPath(name, TextInfo { textHash, references });
@ -442,12 +457,12 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
StringSink sink;
dumpString(s, sink);
auto source = StringSource { *sink.s };
StringSource source(sink.s);
return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info {
*this,
{
.name = name,
.name = std::string { name },
.info = TextInfo {
{ .hash = textHash },
references,
@ -456,45 +471,33 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
nar.first,
};
info.narSize = nar.second;
info.ca = TextHash { textHash };
return info;
})->path;
}
std::optional<const Realisation> BinaryCacheStore::queryRealisation(const DrvOutput & id)
void BinaryCacheStore::queryRealisationUncached(const DrvOutput & id,
Callback<std::shared_ptr<const Realisation>> callback) noexcept
{
if (diskCache) {
auto [cacheOutcome, maybeCachedRealisation] =
diskCache->lookupRealisation(getUri(), id);
switch (cacheOutcome) {
case NarInfoDiskCache::oValid:
debug("Returning a cached realisation for %s", id.to_string());
return *maybeCachedRealisation;
case NarInfoDiskCache::oInvalid:
debug("Returning a cached missing realisation for %s", id.to_string());
return {};
case NarInfoDiskCache::oUnknown:
break;
}
}
auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi";
auto rawOutputInfo = getFile(outputInfoFilePath);
if (rawOutputInfo) {
auto realisation = Realisation::fromJSON(
nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath);
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
if (diskCache)
diskCache->upsertRealisation(
getUri(), realisation);
Callback<std::optional<std::string>> newCallback = {
[=](std::future<std::optional<std::string>> fut) {
try {
auto data = fut.get();
if (!data) return (*callbackPtr)({});
return {realisation};
} else {
if (diskCache)
diskCache->upsertAbsentRealisation(getUri(), id);
return std::nullopt;
}
auto realisation = Realisation::fromJSON(
nlohmann::json::parse(*data), outputInfoFilePath);
return (*callbackPtr)(std::make_shared<const Realisation>(realisation));
} catch (...) {
callbackPtr->rethrow();
}
}
};
getFile(outputInfoFilePath, std::move(newCallback));
}
void BinaryCacheStore::registerDrvOutput(const Realisation& info) {
@ -523,7 +526,7 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe
writeNarInfo(narInfo);
}
std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & path)
std::optional<std::string> BinaryCacheStore::getBuildLog(const StorePath & path)
{
auto drvPath = path;
@ -531,10 +534,10 @@ std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & pat
try {
auto info = queryPathInfo(path);
// FIXME: add a "Log" field to .narinfo
if (!info->deriver) return nullptr;
if (!info->deriver) return std::nullopt;
drvPath = *info->deriver;
} catch (InvalidPath &) {
return nullptr;
return std::nullopt;
}
}
@ -545,4 +548,14 @@ std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & pat
return getFile(logPath);
}
void BinaryCacheStore::addBuildLog(const StorePath & drvPath, std::string_view log)
{
assert(drvPath.isDerivation());
upsertFile(
"log/" + std::string(drvPath.to_string()),
(std::string) log, // FIXME: don't copy
"text/plain; charset=utf-8");
}
}

45
src/libstore/binary-cache-store.hh
View file

@ -15,13 +15,17 @@ struct BinaryCacheStoreConfig : virtual StoreConfig
{
using StoreConfig::StoreConfig;
const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"};
const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', 'gzip', 'zstd', or 'none')"};
const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"};
const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key", "path to secret key used to sign the binary cache"};
const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache", "path to a local cache of NARs"};
const Setting<bool> parallelCompression{(StoreConfig*) this, false, "parallel-compression",
"enable multi-threading compression, available for xz only currently"};
"enable multi-threading compression for NARs, available for xz and zstd only currently"};
const Setting<int> compressionLevel{(StoreConfig*) this, -1, "compression-level",
"specify 'preset level' of compression to be used with NARs: "
"meaning and accepted range of values depends on compression method selected, "
"other than -1 which we reserve to indicate Nix defaults should be used"};
};
class BinaryCacheStore : public virtual BinaryCacheStoreConfig, public virtual Store
@ -47,6 +51,7 @@ public:
const std::string & mimeType) = 0;
void upsertFile(const std::string & path,
// FIXME: use std::string_view
std::string && data,
const std::string & mimeType);
@ -58,10 +63,11 @@ public:
/* Fetch the specified file and call the specified callback with
the result. A subclass may implement this asynchronously. */
virtual void getFile(const std::string & path,
Callback<std::shared_ptr<std::string>> callback) noexcept;
virtual void getFile(
const std::string & path,
Callback<std::optional<std::string>> callback) noexcept;
std::shared_ptr<std::string> getFile(const std::string & path);
std::optional<std::string> getFile(const std::string & path);
public:
@ -92,19 +98,28 @@ public:
void addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs) override;
StorePath addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override;
StorePath addToStoreFromDump(Source & dump, std::string_view name,
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override;
StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo,
PathFilter & filter, RepairFlag repair) override;
StorePath addToStore(
std::string_view name,
const Path & srcPath,
FileIngestionMethod method,
HashType hashAlgo,
PathFilter & filter,
RepairFlag repair,
const StorePathSet & references) override;
StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override;
StorePath addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references,
RepairFlag repair) override;
void registerDrvOutput(const Realisation & info) override;
std::optional<const Realisation> queryRealisation(const DrvOutput &) override;
void queryRealisationUncached(const DrvOutput &,
Callback<std::shared_ptr<const Realisation>> callback) noexcept override;
void narFromPath(const StorePath & path, Sink & sink) override;
@ -112,7 +127,9 @@ public:
void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
std::shared_ptr<std::string> getBuildLog(const StorePath & path) override;
std::optional<std::string> getBuildLog(const StorePath & path) override;
void addBuildLog(const StorePath & drvPath, std::string_view log) override;
};

89
src/libstore/build-result.hh Normal file
View file

@ -0,0 +1,89 @@
#pragma once
#include "realisation.hh"
#include <string>
#include <chrono>
namespace nix {
struct BuildResult
{
/* Note: don't remove status codes, and only add new status codes
at the end of the list, to prevent client/server
incompatibilities in the nix-store --serve protocol. */
enum Status {
Built = 0,
Substituted,
AlreadyValid,
PermanentFailure,
InputRejected,
OutputRejected,
TransientFailure, // possibly transient
CachedFailure, // no longer used
TimedOut,
MiscFailure,
DependencyFailed,
LogLimitExceeded,
NotDeterministic,
ResolvesToAlreadyValid,
NoSubstituters,
} status = MiscFailure;
std::string errorMsg;
std::string toString() const {
auto strStatus = [&]() {
switch (status) {
case Built: return "Built";
case Substituted: return "Substituted";
case AlreadyValid: return "AlreadyValid";
case PermanentFailure: return "PermanentFailure";
case InputRejected: return "InputRejected";
case OutputRejected: return "OutputRejected";
case TransientFailure: return "TransientFailure";
case CachedFailure: return "CachedFailure";
case TimedOut: return "TimedOut";
case MiscFailure: return "MiscFailure";
case DependencyFailed: return "DependencyFailed";
case LogLimitExceeded: return "LogLimitExceeded";
case NotDeterministic: return "NotDeterministic";
case ResolvesToAlreadyValid: return "ResolvesToAlreadyValid";
default: return "Unknown";
};
}();
return strStatus + ((errorMsg == "") ? "" : " : " + errorMsg);
}
/* How many times this build was performed. */
unsigned int timesBuilt = 0;
/* If timesBuilt > 1, whether some builds did not produce the same
result. (Note that 'isNonDeterministic = false' does not mean
the build is deterministic, just that we don't have evidence of
non-determinism.) */
bool isNonDeterministic = false;
/* The derivation we built or the store path we substituted. */
DerivedPath path;
/* For derivations, a mapping from the names of the wanted outputs
to actual paths. */
DrvOutputs builtOutputs;
/* The start/stop times of the build (or one of the rounds, if it
was repeated). */
time_t startTime = 0, stopTime = 0;
bool success()
{
return status == Built || status == Substituted || status == AlreadyValid || status == ResolvesToAlreadyValid;
}
void rethrow()
{
throw Error("%s", errorMsg);
}
};
}

256
src/libstore/build/derivation-goal.cc
View file

@ -17,6 +17,7 @@
#include <regex>
#include <queue>
#include <fstream>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/un.h>
@ -65,7 +66,7 @@ namespace nix {
DerivationGoal::DerivationGoal(const StorePath & drvPath,
const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
: Goal(worker)
: Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs })
, useDerivation(true)
, drvPath(drvPath)
, wantedOutputs(wantedOutputs)
@ -84,7 +85,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath,
DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
: Goal(worker)
: Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs })
, useDerivation(false)
, drvPath(drvPath)
, wantedOutputs(wantedOutputs)
@ -115,7 +116,7 @@ DerivationGoal::~DerivationGoal()
}
string DerivationGoal::key()
std::string DerivationGoal::key()
{
/* Ensure that derivations get built in order of their name,
i.e. a derivation named "aardvark" always comes before
@ -134,7 +135,7 @@ void DerivationGoal::killChild()
void DerivationGoal::timedOut(Error && ex)
{
killChild();
done(BuildResult::TimedOut, ex);
done(BuildResult::TimedOut, {}, ex);
}
@ -181,7 +182,7 @@ void DerivationGoal::loadDerivation()
trace("loading derivation");
if (nrFailed != 0) {
done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)));
done(BuildResult::MiscFailure, {}, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)));
return;
}
@ -193,7 +194,7 @@ void DerivationGoal::loadDerivation()
assert(worker.evalStore.isValidPath(drvPath));
/* Get the derivation. */
drv = std::make_unique<Derivation>(worker.evalStore.derivationFromPath(drvPath));
drv = std::make_unique<Derivation>(worker.evalStore.readDerivation(drvPath));
haveDerivation();
}
@ -204,7 +205,7 @@ void DerivationGoal::haveDerivation()
trace("have derivation");
if (drv->type() == DerivationType::CAFloating)
settings.requireExperimentalFeature("ca-derivations");
settings.requireExperimentalFeature(Xp::CaDerivations);
retrySubstitution = false;
@ -214,28 +215,20 @@ void DerivationGoal::haveDerivation()
auto outputHashes = staticOutputHashes(worker.evalStore, *drv);
for (auto & [outputName, outputHash] : outputHashes)
initialOutputs.insert({
initialOutputs.insert({
outputName,
InitialOutput{
InitialOutput {
.wanted = true, // Will be refined later
.outputHash = outputHash
}
});
});
/* Check what outputs paths are not already valid. */
checkPathValidity();
bool allValid = true;
for (auto & [_, status] : initialOutputs) {
if (!status.wanted) continue;
if (!status.known || !status.known->isValid()) {
allValid = false;
break;
}
}
auto [allValid, validOutputs] = checkPathValidity();
/* If they are all valid, then we're done. */
if (allValid && buildMode == bmNormal) {
done(BuildResult::AlreadyValid);
done(BuildResult::AlreadyValid, std::move(validOutputs));
return;
}
@ -276,8 +269,8 @@ void DerivationGoal::outputsSubstitutionTried()
trace("all outputs substituted (maybe)");
if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
done(BuildResult::TransientFailure,
fmt("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
done(BuildResult::TransientFailure, {},
Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
worker.store.printStorePath(drvPath)));
return;
}
@ -300,23 +293,17 @@ void DerivationGoal::outputsSubstitutionTried()
return;
}
checkPathValidity();
size_t nrInvalid = 0;
for (auto & [_, status] : initialOutputs) {
if (!status.wanted) continue;
if (!status.known || !status.known->isValid())
nrInvalid++;
}
auto [allValid, validOutputs] = checkPathValidity();
if (buildMode == bmNormal && nrInvalid == 0) {
done(BuildResult::Substituted);
if (buildMode == bmNormal && allValid) {
done(BuildResult::Substituted, std::move(validOutputs));
return;
}
if (buildMode == bmRepair && nrInvalid == 0) {
if (buildMode == bmRepair && allValid) {
repairClosure();
return;
}
if (buildMode == bmCheck && nrInvalid > 0)
if (buildMode == bmCheck && !allValid)
throw Error("some outputs of '%s' are not valid, so checking is not possible",
worker.store.printStorePath(drvPath));
@ -408,7 +395,7 @@ void DerivationGoal::repairClosure()
}
if (waitees.empty()) {
done(BuildResult::AlreadyValid);
done(BuildResult::AlreadyValid, assertPathValidity());
return;
}
@ -422,7 +409,7 @@ void DerivationGoal::closureRepaired()
if (nrFailed > 0)
throw Error("some paths in the output closure of derivation '%s' could not be repaired",
worker.store.printStorePath(drvPath));
done(BuildResult::AlreadyValid);
done(BuildResult::AlreadyValid, assertPathValidity());
}
@ -433,7 +420,7 @@ void DerivationGoal::inputsRealised()
if (nrFailed != 0) {
if (!useDerivation)
throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath));
done(BuildResult::DependencyFailed, Error(
done(BuildResult::DependencyFailed, {}, Error(
"%s dependencies of derivation '%s' failed to build",
nrFailed, worker.store.printStorePath(drvPath)));
return;
@ -453,7 +440,7 @@ void DerivationGoal::inputsRealised()
if (useDerivation) {
auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
if (settings.isExperimentalFeatureEnabled("ca-derivations") &&
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) &&
((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type()))
|| fullDrv.type() == DerivationType::DeferredInputAddressed)) {
/* We are be able to resolve this derivation based on the
@ -464,7 +451,6 @@ void DerivationGoal::inputsRealised()
Derivation drvResolved { *std::move(attempt) };
auto pathResolved = writeDerivation(worker.store, drvResolved);
resolvedDrv = drvResolved;
auto msg = fmt("Resolved derivation: '%s' -> '%s'",
worker.store.printStorePath(drvPath),
@ -475,9 +461,9 @@ void DerivationGoal::inputsRealised()
worker.store.printStorePath(pathResolved),
});
auto resolvedGoal = worker.makeDerivationGoal(
resolvedDrvGoal = worker.makeDerivationGoal(
pathResolved, wantedOutputs, buildMode);
addWaitee(resolvedGoal);
addWaitee(resolvedDrvGoal);
state = &DerivationGoal::resolvedFinished;
return;
@ -523,10 +509,11 @@ void DerivationGoal::inputsRealised()
state = &DerivationGoal::tryToBuild;
worker.wakeUp(shared_from_this());
result = BuildResult();
buildResult = BuildResult { .path = buildResult.path };
}
void DerivationGoal::started() {
void DerivationGoal::started()
{
auto msg = fmt(
buildMode == bmRepair ? "repairing outputs of '%s'" :
buildMode == bmCheck ? "checking outputs of '%s'" :
@ -588,19 +575,12 @@ void DerivationGoal::tryToBuild()
omitted, but that would be less efficient.) Note that since we
now hold the locks on the output paths, no other process can
build this derivation, so no further checks are necessary. */
checkPathValidity();
bool allValid = true;
for (auto & [_, status] : initialOutputs) {
if (!status.wanted) continue;
if (!status.known || !status.known->isValid()) {
allValid = false;
break;
}
}
auto [allValid, validOutputs] = checkPathValidity();
if (buildMode != bmCheck && allValid) {
debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath));
outputLocks.setDeletion(true);
done(BuildResult::AlreadyValid);
done(BuildResult::AlreadyValid, std::move(validOutputs));
return;
}
@ -616,7 +596,9 @@ void DerivationGoal::tryToBuild()
/* Don't do a remote build if the derivation has the attribute
`preferLocalBuild' set. Also, check and repair modes are only
supported for local builds. */
bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
bool buildLocally =
(buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store))
&& settings.maxBuildJobs.get() != 0;
if (!buildLocally) {
switch (tryBuildHook()) {
@ -624,7 +606,7 @@ void DerivationGoal::tryToBuild()
/* Yes, it has started doing so. Wait until we get
EOF from the hook. */
actLock.reset();
result.startTime = time(0); // inexact
buildResult.startTime = time(0); // inexact
state = &DerivationGoal::buildDone;
started();
return;
@ -653,7 +635,7 @@ void DerivationGoal::tryLocalBuild() {
throw Error(
"unable to build with a primary store that isn't a local store; "
"either pass a different '--store' or enable remote builds."
"\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
"\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html");
}
@ -828,8 +810,8 @@ void DerivationGoal::buildDone()
debug("builder process for '%s' finished", worker.store.printStorePath(drvPath));
result.timesBuilt++;
result.stopTime = time(0);
buildResult.timesBuilt++;
buildResult.stopTime = time(0);
/* So the child is gone now. */
worker.childTerminated(this);
@ -874,11 +856,11 @@ void DerivationGoal::buildDone()
/* Compute the FS closure of the outputs and register them as
being valid. */
registerOutputs();
auto builtOutputs = registerOutputs();
StorePathSet outputPaths;
for (auto & [_, path] : finalOutputs)
outputPaths.insert(path);
for (auto & [_, output] : buildResult.builtOutputs)
outputPaths.insert(output.outPath);
runPostBuildHook(
worker.store,
*logger,
@ -888,7 +870,7 @@ void DerivationGoal::buildDone()
if (buildMode == bmCheck) {
cleanupPostOutputsRegisteredModeCheck();
done(BuildResult::Built);
done(BuildResult::Built, std::move(builtOutputs));
return;
}
@ -909,6 +891,8 @@ void DerivationGoal::buildDone()
outputLocks.setDeletion(true);
outputLocks.unlock();
done(BuildResult::Built, std::move(builtOutputs));
} catch (BuildError & e) {
outputLocks.unlock();
@ -928,30 +912,32 @@ void DerivationGoal::buildDone()
BuildResult::PermanentFailure;
}
done(st, e);
done(st, {}, e);
return;
}
done(BuildResult::Built);
}
void DerivationGoal::resolvedFinished() {
assert(resolvedDrv);
void DerivationGoal::resolvedFinished()
{
assert(resolvedDrvGoal);
auto resolvedDrv = *resolvedDrvGoal->drv;
auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv);
auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv);
StorePathSet outputPaths;
// `wantedOutputs` might be empty, which means “all the outputs”
auto realWantedOutputs = wantedOutputs;
if (realWantedOutputs.empty())
realWantedOutputs = resolvedDrv->outputNames();
realWantedOutputs = resolvedDrv.outputNames();
DrvOutputs builtOutputs;
for (auto & wantedOutput : realWantedOutputs) {
assert(initialOutputs.count(wantedOutput) != 0);
assert(resolvedHashes.count(wantedOutput) != 0);
auto realisation = worker.store.queryRealisation(
DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput}
DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput}
);
// We've just built it, but maybe the build failed, in which case the
// realisation won't be there
@ -963,10 +949,11 @@ void DerivationGoal::resolvedFinished() {
signRealisation(newRealisation);
worker.store.registerDrvOutput(newRealisation);
outputPaths.insert(realisation->outPath);
builtOutputs.emplace(realisation->id, *realisation);
} else {
// If we don't have a realisation, then it must mean that something
// failed when building the resolved drv
assert(!result.success());
assert(!buildResult.success());
}
}
@ -977,9 +964,17 @@ void DerivationGoal::resolvedFinished() {
outputPaths
);
// This is potentially a bit fishy in terms of error reporting. Not sure
// how to do it in a cleaner way
amDone(nrFailed == 0 ? ecSuccess : ecFailed, ex);
auto status = [&]() {
auto & resolvedResult = resolvedDrvGoal->buildResult;
switch (resolvedResult.status) {
case BuildResult::AlreadyValid:
return BuildResult::ResolvesToAlreadyValid;
default:
return resolvedResult.status;
}
}();
done(status, std::move(builtOutputs));
}
HookReply DerivationGoal::tryBuildHook()
@ -1002,7 +997,7 @@ HookReply DerivationGoal::tryBuildHook()
/* Read the first line of input, which should be a word indicating
whether the hook wishes to perform the build. */
string reply;
std::string reply;
while (true) {
auto s = [&]() {
try {
@ -1014,8 +1009,8 @@ HookReply DerivationGoal::tryBuildHook()
}();
if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true))
;
else if (string(s, 0, 2) == "# ") {
reply = string(s, 2);
else if (s.substr(0, 2) == "# ") {
reply = s.substr(2);
break;
}
else {
@ -1080,7 +1075,7 @@ HookReply DerivationGoal::tryBuildHook()
/* Create the log file and pipe. */
Path logFile = openLogFile();
set<int> fds;
std::set<int> fds;
fds.insert(hook->fromHook.readSide.get());
fds.insert(hook->builderOut.readSide.get());
worker.childStarted(shared_from_this(), fds, false, false);
@ -1089,7 +1084,7 @@ HookReply DerivationGoal::tryBuildHook()
}
void DerivationGoal::registerOutputs()
DrvOutputs DerivationGoal::registerOutputs()
{
/* When using a build hook, the build hook can register the output
as valid (by doing `nix-store --import'). If so we don't have
@ -1098,21 +1093,7 @@ void DerivationGoal::registerOutputs()
We can only early return when the outputs are known a priori. For
floating content-addressed derivations this isn't the case.
*/
for (auto & [outputName, optOutputPath] : worker.store.queryPartialDerivationOutputMap(drvPath)) {
if (!wantOutput(outputName, wantedOutputs))
continue;
if (!optOutputPath)
throw BuildError(
"output '%s' from derivation '%s' does not have a known output path",
outputName, worker.store.printStorePath(drvPath));
auto & outputPath = *optOutputPath;
if (!worker.store.isValidPath(outputPath))
throw BuildError(
"output '%s' from derivation '%s' is supposed to be at '%s' but that path is not valid",
outputName, worker.store.printStorePath(drvPath), worker.store.printStorePath(outputPath));
finalOutputs.insert_or_assign(outputName, outputPath);
}
return assertPathValidity();
}
Path DerivationGoal::openLogFile()
@ -1129,10 +1110,10 @@ Path DerivationGoal::openLogFile()
logDir = localStore->logDir;
else
logDir = settings.nixLogDir;
Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, string(baseName, 0, 2));
Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, baseName.substr(0, 2));
createDirs(dir);
Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2),
Path logFileName = fmt("%s/%s%s", dir, baseName.substr(2),
settings.compressLog ? ".bz2" : "");
fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666);
@ -1164,16 +1145,17 @@ bool DerivationGoal::isReadDesc(int fd)
return fd == hook->builderOut.readSide.get();
}
void DerivationGoal::handleChildOutput(int fd, const string & data)
void DerivationGoal::handleChildOutput(int fd, std::string_view data)
{
if (isReadDesc(fd))
// local & `ssh://`-builds are dealt with here.
auto isWrittenToLog = isReadDesc(fd);
if (isWrittenToLog)
{
logSize += data.size();
if (settings.maxLogSize && logSize > settings.maxLogSize) {
killChild();
done(
BuildResult::LogLimitExceeded,
BuildResult::LogLimitExceeded, {},
Error("%s killed after writing more than %d bytes of log output",
getName(), settings.maxLogSize));
return;
@ -1196,7 +1178,16 @@ void DerivationGoal::handleChildOutput(int fd, const string & data)
if (hook && fd == hook->fromHook.readSide.get()) {
for (auto c : data)
if (c == '\n') {
handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true);
auto json = parseJSONMessage(currentHookLine);
if (json) {
auto s = handleJSONLogMessage(*json, worker.act, hook->activities, true);
// ensure that logs from a builder using `ssh-ng://` as protocol
// are also available to `nix log`.
if (s && !isWrittenToLog && logSink && (*json)["type"] == resBuildLogLine) {
auto f = (*json)["fields"];
(*logSink)((f.size() > 0 ? f.at(0).get<std::string>() : "") + "\n");
}
}
currentHookLine.clear();
} else
currentHookLine += c;
@ -1253,10 +1244,12 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap()
}
void DerivationGoal::checkPathValidity()
std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
{
bool checkHash = buildMode == bmRepair;
auto wantedOutputsLeft = wantedOutputs;
DrvOutputs validOutputs;
for (auto & i : queryPartialDerivationOutputMap()) {
InitialOutput & info = initialOutputs.at(i.first);
info.wanted = wantOutput(i.first, wantedOutputs);
@ -1273,26 +1266,28 @@ void DerivationGoal::checkPathValidity()
: PathStatus::Corrupt,
};
}
if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
auto drvOutput = DrvOutput{initialOutputs.at(i.first).outputHash, i.first};
auto drvOutput = DrvOutput{initialOutputs.at(i.first).outputHash, i.first};
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
if (auto real = worker.store.queryRealisation(drvOutput)) {
info.known = {
.path = real->outPath,
.status = PathStatus::Valid,
};
} else if (info.known && info.known->status == PathStatus::Valid) {
// We know the output because it' a static output of the
} else if (info.known && info.known->isValid()) {
// We know the output because it's a static output of the
// derivation, and the output path is valid, but we don't have
// its realisation stored (probably because it has been built
// without the `ca-derivations` experimental flag)
// without the `ca-derivations` experimental flag).
worker.store.registerDrvOutput(
Realisation{
Realisation {
drvOutput,
info.known->path,
}
);
}
}
if (info.wanted && info.known && info.known->isValid())
validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path });
}
// If we requested all the outputs via the empty set, we are always fine.
// If we requested specific elements, the loop above removes all the valid
@ -1301,24 +1296,50 @@ void DerivationGoal::checkPathValidity()
throw Error("derivation '%s' does not have wanted outputs %s",
worker.store.printStorePath(drvPath),
concatStringsSep(", ", quoteStrings(wantedOutputsLeft)));
bool allValid = true;
for (auto & [_, status] : initialOutputs) {
if (!status.wanted) continue;
if (!status.known || !status.known->isValid()) {
allValid = false;
break;
}
}
return { allValid, validOutputs };
}
void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex)
DrvOutputs DerivationGoal::assertPathValidity()
{
result.status = status;
auto [allValid, validOutputs] = checkPathValidity();
if (!allValid)
throw Error("some outputs are unexpectedly invalid");
return validOutputs;
}
void DerivationGoal::done(
BuildResult::Status status,
DrvOutputs builtOutputs,
std::optional<Error> ex)
{
buildResult.status = status;
if (ex)
result.errorMsg = ex->what();
amDone(result.success() ? ecSuccess : ecFailed, ex);
if (result.status == BuildResult::TimedOut)
// FIXME: strip: "error: "
buildResult.errorMsg = ex->what();
amDone(buildResult.success() ? ecSuccess : ecFailed, ex);
if (buildResult.status == BuildResult::TimedOut)
worker.timedOut = true;
if (result.status == BuildResult::PermanentFailure)
if (buildResult.status == BuildResult::PermanentFailure)
worker.permanentFailure = true;
mcExpectedBuilds.reset();
mcRunningBuilds.reset();
if (result.success()) {
if (buildResult.success()) {
assert(!builtOutputs.empty());
buildResult.builtOutputs = std::move(builtOutputs);
if (status == BuildResult::Built)
worker.doneBuilds++;
} else {
@ -1327,6 +1348,13 @@ void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex)
}
worker.updateProgress();
auto traceBuiltOutputsFile = getEnv("_NIX_TRACE_BUILT_OUTPUTS").value_or("");
if (traceBuiltOutputsFile != "") {
std::fstream fs;
fs.open(traceBuiltOutputsFile, std::fstream::out);
fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl;
}
}

38
src/libstore/build/derivation-goal.hh
View file

@ -50,8 +50,8 @@ struct DerivationGoal : public Goal
/* The path of the derivation. */
StorePath drvPath;
/* The path of the corresponding resolved derivation */
std::optional<BasicDerivation> resolvedDrv;
/* The goal for the corresponding resolved derivation */
std::shared_ptr<DerivationGoal> resolvedDrvGoal;
/* The specific outputs that we need to build. Empty means all of
them. */
@ -104,20 +104,8 @@ struct DerivationGoal : public Goal
typedef void (DerivationGoal::*GoalState)();
GoalState state;
/* The final output paths of the build.
- For input-addressed derivations, always the precomputed paths
- For content-addressed derivations, calcuated from whatever the hash
ends up being. (Note that fixed outputs derivations that produce the
"wrong" output still install that data under its true content-address.)
*/
OutputPathMap finalOutputs;
BuildMode buildMode;
BuildResult result;
/* The current round, if we're building multiple times. */
size_t curRound = 1;
@ -145,15 +133,13 @@ struct DerivationGoal : public Goal
void timedOut(Error && ex) override;
string key() override;
std::string key() override;
void work() override;
/* Add wanted outputs to an already existing derivation goal. */
void addWantedOutputs(const StringSet & outputs);
BuildResult getResult() { return result; }
/* The states. */
void getDerivation();
void loadDerivation();
@ -175,7 +161,7 @@ struct DerivationGoal : public Goal
/* Check that the derivation outputs all exist and register them
as valid. */
virtual void registerOutputs();
virtual DrvOutputs registerOutputs();
/* Open a log file and a pipe to it. */
Path openLogFile();
@ -200,7 +186,7 @@ struct DerivationGoal : public Goal
virtual bool isReadDesc(int fd);
/* Callback used by the worker to write to the log. */
void handleChildOutput(int fd, const string & data) override;
void handleChildOutput(int fd, std::string_view data) override;
void handleEOF(int fd) override;
void flushLine();
@ -210,8 +196,17 @@ struct DerivationGoal : public Goal
std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap();
OutputPathMap queryDerivationOutputMap();
/* Return the set of (in)valid paths. */
void checkPathValidity();
/* Update 'initialOutputs' to determine the current status of the
outputs of the derivation. Also returns a Boolean denoting
whether all outputs are valid and non-corrupt, and a
'DrvOutputs' structure containing the valid and wanted
outputs. */
std::pair<bool, DrvOutputs> checkPathValidity();
/* Aborts if any output is not valid or corrupt, and otherwise
returns a 'DrvOutputs' structure containing the wanted
outputs. */
DrvOutputs assertPathValidity();
/* Forcibly kill the child process, if any. */
virtual void killChild();
@ -222,6 +217,7 @@ struct DerivationGoal : public Goal
void done(
BuildResult::Status status,
DrvOutputs builtOutputs = {},
std::optional<Error> ex = {});
StorePathSet exportReferences(const StorePathSet & storePaths);

58
src/libstore/build/drv-output-substitution-goal.cc
View file

@ -1,11 +1,17 @@
#include "drv-output-substitution-goal.hh"
#include "finally.hh"
#include "worker.hh"
#include "substitution-goal.hh"
#include "callback.hh"
namespace nix {
DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
: Goal(worker)
DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(
const DrvOutput & id,
Worker & worker,
RepairFlag repair,
std::optional<ContentAddress> ca)
: Goal(worker, DerivedPath::Opaque { StorePath::dummy })
, id(id)
{
state = &DrvOutputSubstitutionGoal::init;
@ -30,7 +36,7 @@ void DrvOutputSubstitutionGoal::init()
void DrvOutputSubstitutionGoal::tryNext()
{
trace("Trying next substituter");
trace("trying next substituter");
if (subs.size() == 0) {
/* None left. Terminate this goal and let someone else deal
@ -50,14 +56,42 @@ void DrvOutputSubstitutionGoal::tryNext()
return;
}
auto sub = subs.front();
sub = subs.front();
subs.pop_front();
// FIXME: Make async
outputInfo = sub->queryRealisation(id);
// outputInfo = sub->queryRealisation(id);
outPipe.create();
promise = decltype(promise)();
sub->queryRealisation(
id, { [&](std::future<std::shared_ptr<const Realisation>> res) {
try {
Finally updateStats([this]() { outPipe.writeSide.close(); });
promise.set_value(res.get());
} catch (...) {
promise.set_exception(std::current_exception());
}
} });
worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false);
state = &DrvOutputSubstitutionGoal::realisationFetched;
}
void DrvOutputSubstitutionGoal::realisationFetched()
{
worker.childTerminated(this);
try {
outputInfo = promise.get_future().get();
} catch (std::exception & e) {
printError(e.what());
substituterFailed = true;
}
if (!outputInfo) {
tryNext();
return;
return tryNext();
}
for (const auto & [depId, depPath] : outputInfo->dependentRealisations) {
@ -89,7 +123,7 @@ void DrvOutputSubstitutionGoal::tryNext()
void DrvOutputSubstitutionGoal::outPathValid()
{
assert(outputInfo);
trace("Output path substituted");
trace("output path substituted");
if (nrFailed > 0) {
debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
@ -107,7 +141,7 @@ void DrvOutputSubstitutionGoal::finished()
amDone(ecSuccess);
}
string DrvOutputSubstitutionGoal::key()
std::string DrvOutputSubstitutionGoal::key()
{
/* "a$" ensures substitution goals happen before derivation
goals. */
@ -119,4 +153,10 @@ void DrvOutputSubstitutionGoal::work()
(this->*state)();
}
void DrvOutputSubstitutionGoal::handleEOF(int fd)
{
if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
}
}

16
src/libstore/build/drv-output-substitution-goal.hh
View file

@ -3,6 +3,8 @@
#include "store-api.hh"
#include "goal.hh"
#include "realisation.hh"
#include <thread>
#include <future>
namespace nix {
@ -20,11 +22,18 @@ private:
// The realisation corresponding to the given output id.
// Will be filled once we can get it.
std::optional<Realisation> outputInfo;
std::shared_ptr<const Realisation> outputInfo;
/* The remaining substituters. */
std::list<ref<Store>> subs;
/* The current substituter. */
std::shared_ptr<Store> sub;
Pipe outPipe;
std::thread thr;
std::promise<std::shared_ptr<const Realisation>> promise;
/* Whether a substituter failed. */
bool substituterFailed = false;
@ -36,15 +45,16 @@ public:
void init();
void tryNext();
void realisationFetched();
void outPathValid();
void finished();
void timedOut(Error && ex) override { abort(); };
string key() override;
std::string key() override;
void work() override;
void handleEOF(int fd) override;
};
}

65
src/libstore/build/entry-points.cc
View file

@ -1,4 +1,3 @@
#include "machines.hh"
#include "worker.hh"
#include "substitution-goal.hh"
#include "derivation-goal.hh"
@ -48,43 +47,51 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
}
}
std::vector<BuildResult> Store::buildPathsWithResults(
const std::vector<DerivedPath> & reqs,
BuildMode buildMode,
std::shared_ptr<Store> evalStore)
{
Worker worker(*this, evalStore ? *evalStore : *this);
Goals goals;
for (const auto & br : reqs) {
std::visit(overloaded {
[&](const DerivedPath::Built & bfd) {
goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode));
},
[&](const DerivedPath::Opaque & bo) {
goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair));
},
}, br.raw());
}
worker.run(goals);
std::vector<BuildResult> results;
for (auto & i : goals)
results.push_back(i->buildResult);
return results;
}
BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode)
{
Worker worker(*this, *this);
auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode);
BuildResult result;
try {
worker.run(Goals{goal});
result = goal->getResult();
return goal->buildResult;
} catch (Error & e) {
result.status = BuildResult::MiscFailure;
result.errorMsg = e.msg();
}
// XXX: Should use `goal->queryPartialDerivationOutputMap()` once it's
// extended to return the full realisation for each output
auto staticDrvOutputs = drv.outputsAndOptPaths(*this);
auto outputHashes = staticOutputHashes(*this, drv);
for (auto & [outputName, staticOutput] : staticDrvOutputs) {
auto outputId = DrvOutput{outputHashes.at(outputName), outputName};
if (staticOutput.second)
result.builtOutputs.insert_or_assign(
outputId,
Realisation{ outputId, *staticOutput.second}
);
if (settings.isExperimentalFeatureEnabled("ca-derivations") && !derivationHasKnownOutputPaths(drv.type())) {
auto realisation = this->queryRealisation(outputId);
if (realisation)
result.builtOutputs.insert_or_assign(
outputId,
*realisation
);
}
}
return result;
return BuildResult {
.status = BuildResult::MiscFailure,
.errorMsg = e.msg(),
.path = DerivedPath::Built { .drvPath = drvPath },
};
};
}

4
src/libstore/build/goal.cc
View file

@ -5,8 +5,8 @@ namespace nix {
bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
string s1 = a->key();
string s2 = b->key();
std::string s1 = a->key();
std::string s2 = b->key();
return s1 < s2;
}

20
src/libstore/build/goal.hh
View file

@ -2,6 +2,7 @@
#include "types.hh"
#include "store-api.hh"
#include "build-result.hh"
namespace nix {
@ -18,8 +19,8 @@ struct CompareGoalPtrs {
};
/* Set of goals. */
typedef set<GoalPtr, CompareGoalPtrs> Goals;
typedef set<WeakGoalPtr, std::owner_less<WeakGoalPtr>> WeakGoals;
typedef std::set<GoalPtr, CompareGoalPtrs> Goals;
typedef std::set<WeakGoalPtr, std::owner_less<WeakGoalPtr>> WeakGoals;
/* A map of paths to goals (and the other way around). */
typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
@ -50,15 +51,20 @@ struct Goal : public std::enable_shared_from_this<Goal>
unsigned int nrIncompleteClosure;
/* Name of this goal for debugging purposes. */
string name;
std::string name;
/* Whether the goal is finished. */
ExitCode exitCode;
/* Build result. */
BuildResult buildResult;
/* Exception containing an error message, if any. */
std::optional<Error> ex;
Goal(Worker & worker) : worker(worker)
Goal(Worker & worker, DerivedPath path)
: worker(worker)
, buildResult { .path = std::move(path) }
{
nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
exitCode = ecBusy;
@ -75,7 +81,7 @@ struct Goal : public std::enable_shared_from_this<Goal>
virtual void waiteeDone(GoalPtr waitee, ExitCode result);
virtual void handleChildOutput(int fd, const string & data)
virtual void handleChildOutput(int fd, std::string_view data)
{
abort();
}
@ -87,7 +93,7 @@ struct Goal : public std::enable_shared_from_this<Goal>
void trace(const FormatOrString & fs);
string getName()
std::string getName()
{
return name;
}
@ -97,7 +103,7 @@ struct Goal : public std::enable_shared_from_this<Goal>
by the worker (important!), etc. */
virtual void timedOut(Error && ex) = 0;
virtual string key() = 0;
virtual std::string key() = 0;
void amDone(ExitCode result, std::optional<Error> ex = {});

220
src/libstore/build/local-derivation-goal.cc
View file

@ -1,4 +1,5 @@
#include "local-derivation-goal.hh"
#include "gc-store.hh"
#include "hook-instance.hh"
#include "worker.hh"
#include "builtins.hh"
@ -193,7 +194,7 @@ void LocalDerivationGoal::tryLocalBuild() {
outputLocks.unlock();
buildUser.reset();
worker.permanentFailure = true;
done(BuildResult::InputRejected, e);
done(BuildResult::InputRejected, {}, e);
return;
}
@ -260,6 +261,7 @@ void LocalDerivationGoal::cleanupHookFinally()
void LocalDerivationGoal::cleanupPreChildKill()
{
sandboxMountNamespace = -1;
sandboxUserNamespace = -1;
}
@ -342,7 +344,7 @@ int childEntry(void * arg)
return 1;
}
#if __linux__
static void linkOrCopy(const Path & from, const Path & to)
{
if (link(from.c_str(), to.c_str()) == -1) {
@ -358,6 +360,7 @@ static void linkOrCopy(const Path & from, const Path & to)
copyPath(from, to);
}
}
#endif
void LocalDerivationGoal::startBuilder()
@ -479,12 +482,12 @@ void LocalDerivationGoal::startBuilder()
temporary build directory. The text files have the format used
by `nix-store --register-validity'. However, the deriver
fields are left empty. */
string s = get(drv->env, "exportReferencesGraph").value_or("");
auto s = get(drv->env, "exportReferencesGraph").value_or("");
Strings ss = tokenizeString<Strings>(s);
if (ss.size() % 2 != 0)
throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s);
for (Strings::iterator i = ss.begin(); i != ss.end(); ) {
string fileName = *i++;
auto fileName = *i++;
static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*");
if (!std::regex_match(fileName, regex))
throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName);
@ -515,10 +518,10 @@ void LocalDerivationGoal::startBuilder()
i.pop_back();
}
size_t p = i.find('=');
if (p == string::npos)
if (p == std::string::npos)
dirsInChroot[i] = {i, optional};
else
dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional};
dirsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional};
}
dirsInChroot[tmpDirInSandbox] = tmpDir;
@ -669,9 +672,10 @@ void LocalDerivationGoal::startBuilder()
auto state = stBegin;
auto lines = runProgram(settings.preBuildHook, false, args);
auto lastPos = std::string::size_type{0};
for (auto nlPos = lines.find('\n'); nlPos != string::npos;
nlPos = lines.find('\n', lastPos)) {
auto line = std::string{lines, lastPos, nlPos - lastPos};
for (auto nlPos = lines.find('\n'); nlPos != std::string::npos;
nlPos = lines.find('\n', lastPos))
{
auto line = lines.substr(lastPos, nlPos - lastPos);
lastPos = nlPos + 1;
if (state == stBegin) {
if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") {
@ -684,10 +688,10 @@ void LocalDerivationGoal::startBuilder()
state = stBegin;
} else {
auto p = line.find('=');
if (p == string::npos)
if (p == std::string::npos)
dirsInChroot[line] = line;
else
dirsInChroot[string(line, 0, p)] = string(line, p + 1);
dirsInChroot[line.substr(0, p)] = line.substr(p + 1);
}
}
}
@ -752,7 +756,7 @@ void LocalDerivationGoal::startBuilder()
if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term))
throw SysError("putting pseudoterminal into raw mode");
result.startTime = time(0);
buildResult.startTime = time(0);
/* Fork a child to build the package. */
@ -905,19 +909,27 @@ void LocalDerivationGoal::startBuilder()
"nobody:x:65534:65534:Nobody:/:/noshell\n",
sandboxUid(), sandboxGid(), settings.sandboxBuildDir));
/* Save the mount namespace of the child. We have to do this
/* Save the mount- and user namespace of the child. We have to do this
*before* the child does a chroot. */
sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY);
if (sandboxMountNamespace.get() == -1)
throw SysError("getting sandbox mount namespace");
if (usingUserNamespace) {
sandboxUserNamespace = open(fmt("/proc/%d/ns/user", (pid_t) pid).c_str(), O_RDONLY);
if (sandboxUserNamespace.get() == -1)
throw SysError("getting sandbox user namespace");
}
/* Signal the builder that we've updated its user namespace. */
writeFull(userNamespaceSync.writeSide.get(), "1");
} else
#endif
{
#if __linux__
fallback:
#endif
pid = startProcess([&]() {
runChild();
});
@ -931,7 +943,7 @@ void LocalDerivationGoal::startBuilder()
/* Check if setting up the build environment failed. */
std::vector<std::string> msgs;
while (true) {
string msg = [&]() {
std::string msg = [&]() {
try {
return readLine(builderOut.readSide.get());
} catch (Error & e) {
@ -943,12 +955,12 @@ void LocalDerivationGoal::startBuilder()
throw;
}
}();
if (string(msg, 0, 1) == "\2") break;
if (string(msg, 0, 1) == "\1") {
if (msg.substr(0, 1) == "\2") break;
if (msg.substr(0, 1) == "\1") {
FdSource source(builderOut.readSide.get());
auto ex = readError(source);
ex.addTrace({}, "while setting up the build environment");
throw;
throw ex;
}
debug("sandbox setup: " + msg);
msgs.push_back(std::move(msg));
@ -980,7 +992,7 @@ void LocalDerivationGoal::initTmpDir() {
env[i.first] = i.second;
} else {
auto hash = hashString(htSHA256, i.first);
string fn = ".attr-" + hash.to_string(Base32, false);
std::string fn = ".attr-" + hash.to_string(Base32, false);
Path p = tmpDir + "/" + fn;
writeFile(p, rewriteStrings(i.second, inputRewrites));
chownToBuilder(p);
@ -1071,7 +1083,7 @@ void LocalDerivationGoal::writeStructuredAttrs()
for (auto & [i, v] : json["outputs"].get<nlohmann::json::object_t>()) {
/* The placeholder must have a rewrite, so we use it to cover both the
cases where we know or don't know the output path ahead of time. */
rewritten[i] = rewriteStrings(v, inputRewrites);
rewritten[i] = rewriteStrings((std::string) v, inputRewrites);
}
json["outputs"] = rewritten;
@ -1116,7 +1128,7 @@ struct RestrictedStoreConfig : virtual LocalFSStoreConfig
/* A wrapper around LocalStore that only allows building/querying of
paths that are in the input closures of the build or were added via
recursive Nix calls. */
struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore
struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore, public virtual GcStore
{
ref<LocalStore> next;
@ -1177,9 +1189,14 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
{ throw Error("queryPathFromHashPart"); }
StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override
StorePath addToStore(
std::string_view name,
const Path & srcPath,
FileIngestionMethod method,
HashType hashAlgo,
PathFilter & filter,
RepairFlag repair,
const StorePathSet & references) override
{ throw Error("addToStore"); }
void addToStore(const ValidPathInfo & info, Source & narSource,
@ -1189,18 +1206,26 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
goal.addDependency(info.path);
}
StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair = NoRepair) override
StorePath addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references,
RepairFlag repair = NoRepair) override
{
auto path = next->addTextToStore(name, s, references, repair);
goal.addDependency(path);
return path;
}
StorePath addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override
StorePath addToStoreFromDump(
Source & dump,
std::string_view name,
FileIngestionMethod method,
HashType hashAlgo,
RepairFlag repair,
const StorePathSet & references) override
{
auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair);
auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair, references);
goal.addDependency(path);
return path;
}
@ -1224,16 +1249,27 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
// corresponds to an allowed derivation
{ throw Error("registerDrvOutput"); }
std::optional<const Realisation> queryRealisation(const DrvOutput & id) override
void queryRealisationUncached(const DrvOutput & id,
Callback<std::shared_ptr<const Realisation>> callback) noexcept override
// XXX: This should probably be allowed if the realisation corresponds to
// an allowed derivation
{
if (!goal.isAllowed(id))
throw InvalidPath("cannot query an unknown output id '%s' in recursive Nix", id.to_string());
return next->queryRealisation(id);
callback(nullptr);
next->queryRealisation(id, std::move(callback));
}
void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override
{
for (auto & result : buildPathsWithResults(paths, buildMode, evalStore))
if (!result.success())
result.rethrow();
}
std::vector<BuildResult> buildPathsWithResults(
const std::vector<DerivedPath> & paths,
BuildMode buildMode = bmNormal,
std::shared_ptr<Store> evalStore = nullptr) override
{
assert(!evalStore);
@ -1247,26 +1283,13 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
throw InvalidPath("cannot build '%s' in recursive Nix because path is unknown", req.to_string(*next));
}
next->buildPaths(paths, buildMode);
auto results = next->buildPathsWithResults(paths, buildMode);
for (auto & path : paths) {
auto p = std::get_if<DerivedPath::Built>(&path);
if (!p) continue;
auto & bfd = *p;
auto drv = readDerivation(bfd.drvPath);
auto drvHashes = staticOutputHashes(*this, drv);
auto outputs = next->queryDerivationOutputMap(bfd.drvPath);
for (auto & [outputName, outputPath] : outputs)
if (wantOutput(outputName, bfd.outputs)) {
newPaths.insert(outputPath);
if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
auto thisRealisation = next->queryRealisation(
DrvOutput{drvHashes.at(outputName), outputName}
);
assert(thisRealisation);
newRealisations.insert(*thisRealisation);
}
}
for (auto & result : results) {
for (auto & [outputName, output] : result.builtOutputs) {
newPaths.insert(output.outPath);
newRealisations.insert(output);
}
}
StorePathSet closure;
@ -1275,6 +1298,8 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
goal.addDependency(path);
for (auto & real : Realisation::closure(*next, newRealisations))
goal.addedDrvOutputs.insert(real.id);
return results;
}
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
@ -1320,7 +1345,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
void LocalDerivationGoal::startDaemon()
{
settings.requireExperimentalFeature("recursive-nix");
settings.requireExperimentalFeature(Xp::RecursiveNix);
Store::Params params;
params["path-info-cache-size"] = "0";
@ -1353,7 +1378,7 @@ void LocalDerivationGoal::startDaemon()
AutoCloseFD remote = accept(daemonSocket.get(),
(struct sockaddr *) &remoteAddr, &remoteAddrLen);
if (!remote) {
if (errno == EINTR) continue;
if (errno == EINTR || errno == EAGAIN) continue;
if (errno == EINVAL) break;
throw SysError("accepting connection");
}
@ -1432,6 +1457,9 @@ void LocalDerivationGoal::addDependency(const StorePath & path)
child process.*/
Pid child(startProcess([&]() {
if (usingUserNamespace && (setns(sandboxUserNamespace.get(), 0) == -1))
throw SysError("entering sandbox user namespace");
if (setns(sandboxMountNamespace.get(), 0) == -1)
throw SysError("entering sandbox mount namespace");
@ -1773,11 +1801,14 @@ void LocalDerivationGoal::runChild()
i686-linux build on an x86_64-linux machine. */
struct utsname utsbuf;
uname(&utsbuf);
if (drv->platform == "i686-linux" &&
(settings.thisSystem == "x86_64-linux" ||
(!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) {
if ((drv->platform == "i686-linux"
&& (settings.thisSystem == "x86_64-linux"
|| (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64"))))
|| drv->platform == "armv7l-linux"
|| drv->platform == "armv6l-linux")
{
if (personality(PER_LINUX32) == -1)
throw SysError("cannot set i686-linux personality");
throw SysError("cannot set 32-bit personality");
}
/* Impersonate a Linux 2.6 machine to get some determinism in
@ -1902,7 +1933,7 @@ void LocalDerivationGoal::runChild()
"can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin",
i.first, i.second.source);
string path = i.first;
std::string path = i.first;
struct stat st;
if (lstat(path.c_str(), &st)) {
if (i.second.optional && errno == ENOENT)
@ -1954,7 +1985,7 @@ void LocalDerivationGoal::runChild()
args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/");
if (allowLocalNetworking) {
args.push_back("-D");
args.push_back(string("_ALLOW_LOCAL_NETWORKING=1"));
args.push_back(std::string("_ALLOW_LOCAL_NETWORKING=1"));
}
args.push_back(drv->builder);
} else {
@ -1973,7 +2004,7 @@ void LocalDerivationGoal::runChild()
args.push_back(rewriteStrings(i, inputRewrites));
/* Indicate that we managed to set up the build environment. */
writeFull(STDERR_FILENO, string("\2\n"));
writeFull(STDERR_FILENO, std::string("\2\n"));
/* Execute the program. This should not return. */
if (drv->isBuiltin()) {
@ -1991,7 +2022,7 @@ void LocalDerivationGoal::runChild()
else if (drv->builder == "builtin:unpack-channel")
builtinUnpackChannel(drv2);
else
throw Error("unsupported builtin function '%1%'", string(drv->builder, 8));
throw Error("unsupported builtin builder '%1%'", drv->builder.substr(8));
_exit(0);
} catch (std::exception & e) {
writeFull(STDERR_FILENO, e.what() + std::string("\n"));
@ -2037,7 +2068,7 @@ void LocalDerivationGoal::runChild()
}
void LocalDerivationGoal::registerOutputs()
DrvOutputs LocalDerivationGoal::registerOutputs()
{
/* When using a build hook, the build hook can register the output
as valid (by doing `nix-store --import'). If so we don't have
@ -2046,10 +2077,8 @@ void LocalDerivationGoal::registerOutputs()
We can only early return when the outputs are known a priori. For
floating content-addressed derivations this isn't the case.
*/
if (hook) {
DerivationGoal::registerOutputs();
return;
}
if (hook)
return DerivationGoal::registerOutputs();
std::map<std::string, ValidPathInfo> infos;
@ -2172,6 +2201,8 @@ void LocalDerivationGoal::registerOutputs()
std::reverse(sortedOutputNames.begin(), sortedOutputNames.end());
OutputPathMap finalOutputs;
for (auto & outputName : sortedOutputNames) {
auto output = drv->outputs.at(outputName);
auto & scratchPath = scratchOutputs.at(outputName);
@ -2210,8 +2241,8 @@ void LocalDerivationGoal::registerOutputs()
StringSink sink;
dumpPath(actualPath, sink);
deletePath(actualPath);
sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites));
StringSource source(*sink.s);
sink.s = rewriteStrings(sink.s, outputRewrites);
StringSource source(sink.s);
restorePath(actualPath, source);
}
};
@ -2290,7 +2321,7 @@ void LocalDerivationGoal::registerOutputs()
StringSink sink;
dumpPath(actualPath, sink);
RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink);
rsink2(*sink.s);
rsink2(sink.s);
rsink2.flush();
});
Path tmpPath = actualPath + ".tmp";
@ -2308,6 +2339,7 @@ void LocalDerivationGoal::registerOutputs()
};
ValidPathInfo newInfo = std::visit(overloaded {
[&](const DerivationOutputInputAddressed & output) {
/* input-addressed case */
auto requiredFinalPath = output.path;
@ -2324,6 +2356,7 @@ void LocalDerivationGoal::registerOutputs()
static_cast<PathReferences<StorePath> &>(newInfo0) = rewriteRefs();
return newInfo0;
},
[&](const DerivationOutputCAFixed & dof) {
auto wanted = getContentAddressHash(dof.ca);
@ -2352,18 +2385,17 @@ void LocalDerivationGoal::registerOutputs()
return newInfo0;
},
[&](DerivationOutputCAFloating dof) {
[&](DerivationOutputCAFloating & dof) {
return newInfoFromCA(dof);
},
[&](DerivationOutputDeferred) {
[&](DerivationOutputDeferred) -> ValidPathInfo {
// No derivation should reach that point without having been
// rewritten first
assert(false);
// Ugly, but the compiler insists on having this return a value
// of type `ValidPathInfo` despite the `assert(false)`, so
// let's provide it
return *(ValidPathInfo*)0;
},
}, output.output);
/* FIXME: set proper permissions in restorePath() so
@ -2454,7 +2486,7 @@ void LocalDerivationGoal::registerOutputs()
}
if (curRound == nrRounds) {
localStore.optimisePath(actualPath); // FIXME: combine with scanForReferences()
localStore.optimisePath(actualPath, NoRepair); // FIXME: combine with scanForReferences()
worker.markContentsGood(newInfo.path);
}
@ -2474,11 +2506,12 @@ void LocalDerivationGoal::registerOutputs()
}
if (buildMode == bmCheck) {
// In case of FOD mismatches on `--check` an error must be thrown as this is also
// a source for non-determinism.
/* In case of fixed-output derivations, if there are
mismatches on `--check` an error must be thrown as this is
also a source for non-determinism. */
if (delayedException)
std::rethrow_exception(delayedException);
return;
return assertPathValidity();
}
/* Apply output checks. */
@ -2490,7 +2523,7 @@ void LocalDerivationGoal::registerOutputs()
assert(prevInfos.size() == infos.size());
for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j)
if (!(*i == *j)) {
result.isNonDeterministic = true;
buildResult.isNonDeterministic = true;
Path prev = worker.store.printStorePath(i->second.path) + checkSuffix;
bool prevExists = keepPreviousRound && pathExists(prev);
hintformat hint = prevExists
@ -2528,7 +2561,7 @@ void LocalDerivationGoal::registerOutputs()
if (curRound < nrRounds) {
prevInfos = std::move(infos);
return;
return {};
}
/* Remove the .check directories if we're done. FIXME: keep them
@ -2563,17 +2596,24 @@ void LocalDerivationGoal::registerOutputs()
means it's safe to link the derivation to the output hash. We must do
that for floating CA derivations, which otherwise couldn't be cached,
but it's fine to do in all cases. */
DrvOutputs builtOutputs;
if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
for (auto& [outputName, newInfo] : infos) {
auto thisRealisation = Realisation{
.id = DrvOutput{initialOutputs.at(outputName).outputHash,
outputName},
.outPath = newInfo.path};
for (auto & [outputName, newInfo] : infos) {
auto thisRealisation = Realisation {
.id = DrvOutput {
initialOutputs.at(outputName).outputHash,
outputName
},
.outPath = newInfo.path
};
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
signRealisation(thisRealisation);
worker.store.registerDrvOutput(thisRealisation);
}
builtOutputs.emplace(thisRealisation.id, thisRealisation);
}
return builtOutputs;
}
void LocalDerivationGoal::signRealisation(Realisation & realisation)
@ -2582,7 +2622,7 @@ void LocalDerivationGoal::signRealisation(Realisation & realisation)
}
void LocalDerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo> & outputs)
{
std::map<Path, const ValidPathInfo &> outputsByPath;
for (auto & output : outputs)
@ -2654,8 +2694,8 @@ void LocalDerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & out
for (auto & i : *value) {
if (worker.store.isStorePath(i))
spec.insert(worker.store.parseStorePath(i));
else if (finalOutputs.count(i))
spec.insert(finalOutputs.at(i));
else if (outputs.count(i))
spec.insert(outputs.at(i).path);
else throw BuildError("derivation contains an illegal reference specifier '%s'", i);
}
@ -2678,7 +2718,7 @@ void LocalDerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & out
}
if (!badPaths.empty()) {
string badPathsStr;
std::string badPathsStr;
for (auto & i : badPaths) {
badPathsStr += "\n ";
badPathsStr += worker.store.printStorePath(i);

9
src/libstore/build/local-derivation-goal.hh
View file

@ -27,9 +27,10 @@ struct LocalDerivationGoal : public DerivationGoal
/* Pipe for synchronising updates to the builder namespaces. */
Pipe userNamespaceSync;
/* The mount namespace of the builder, used to add additional
/* The mount namespace and user namespace of the builder, used to add additional
paths to the sandbox as a result of recursive Nix calls. */
AutoCloseFD sandboxMountNamespace;
AutoCloseFD sandboxUserNamespace;
/* On Linux, whether we're doing the build in its own user
namespace. */
@ -57,11 +58,11 @@ struct LocalDerivationGoal : public DerivationGoal
typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
DirsInChroot dirsInChroot;
typedef map<string, string> Environment;
typedef map<std::string, std::string> Environment;
Environment env;
#if __APPLE__
typedef string SandboxProfile;
typedef std::string SandboxProfile;
SandboxProfile additionalSandboxProfile;
#endif
@ -168,7 +169,7 @@ struct LocalDerivationGoal : public DerivationGoal
/* Check that the derivation outputs all exist and register them
as valid. */
void registerOutputs() override;
DrvOutputs registerOutputs() override;
void signRealisation(Realisation &) override;

25
src/libstore/build/substitution-goal.cc
View file

@ -6,7 +6,7 @@
namespace nix {
PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
: Goal(worker)
: Goal(worker, DerivedPath::Opaque { storePath })
, storePath(storePath)
, repair(repair)
, ca(ca)
@ -24,6 +24,13 @@ PathSubstitutionGoal::~PathSubstitutionGoal()
}
void PathSubstitutionGoal::done(ExitCode result, BuildResult::Status status)
{
buildResult.status = status;
amDone(result);
}
void PathSubstitutionGoal::work()
{
(this->*state)();
@ -38,7 +45,7 @@ void PathSubstitutionGoal::init()
/* If the path already exists we're done. */
if (!repair && worker.store.isValidPath(storePath)) {
amDone(ecSuccess);
done(ecSuccess, BuildResult::AlreadyValid);
return;
}
@ -65,7 +72,7 @@ void PathSubstitutionGoal::tryNext()
/* Hack: don't indicate failure if there were no substituters.
In that case the calling derivation should just do a
build. */
amDone(substituterFailed ? ecFailed : ecNoSubstituters);
done(substituterFailed ? ecFailed : ecNoSubstituters, BuildResult::NoSubstituters);
if (substituterFailed) {
worker.failedSubstitutions++;
@ -141,8 +148,8 @@ void PathSubstitutionGoal::tryNext()
only after we've downloaded the path. */
if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info))
{
warn("substituter '%s' does not have a valid signature for path '%s'",
sub->getUri(), worker.store.printStorePath(storePath));
warn("the substitute for '%s' from '%s' is not signed by any of the keys in 'trusted-public-keys'",
worker.store.printStorePath(storePath), sub->getUri());
tryNext();
return;
}
@ -165,7 +172,9 @@ void PathSubstitutionGoal::referencesValid()
if (nrFailed > 0) {
debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath));
amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
done(
nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed,
BuildResult::DependencyFailed);
return;
}
@ -269,11 +278,11 @@ void PathSubstitutionGoal::finished()
worker.updateProgress();
amDone(ecSuccess);
done(ecSuccess, BuildResult::Substituted);
}
void PathSubstitutionGoal::handleChildOutput(int fd, const string & data)
void PathSubstitutionGoal::handleChildOutput(int fd, std::string_view data)
{
}

6
src/libstore/build/substitution-goal.hh
View file

@ -53,13 +53,15 @@ struct PathSubstitutionGoal : public Goal
/* Content address for recomputing store path */
std::optional<ContentAddress> ca;
void done(ExitCode result, BuildResult::Status status);
public:
PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
~PathSubstitutionGoal();
void timedOut(Error && ex) override { abort(); };
string key() override
std::string key() override
{
/* "a$" ensures substitution goals happen before derivation
goals. */
@ -77,7 +79,7 @@ public:
void finished();
/* Callback used by the worker to write to the log. */
void handleChildOutput(int fd, const string & data) override;
void handleChildOutput(int fd, std::string_view data) override;
void handleEOF(int fd) override;
void cleanup() override;

10
src/libstore/build/worker.cc
View file

@ -161,7 +161,7 @@ unsigned Worker::getNrLocalBuilds()
}
void Worker::childStarted(GoalPtr goal, const set<int> & fds,
void Worker::childStarted(GoalPtr goal, const std::set<int> & fds,
bool inBuildSlot, bool respectTimeouts)
{
Child child;
@ -281,11 +281,11 @@ void Worker::run(const Goals & _topGoals)
if (getMachines().empty())
throw Error("unable to start any build; either increase '--max-jobs' "
"or enable remote builds."
"\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
"\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html");
else
throw Error("unable to start any build; remote machines may not have "
"all required system features."
"\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
"\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html");
}
assert(!awake.empty());
@ -377,7 +377,7 @@ void Worker::waitForInput()
GoalPtr goal = j->goal.lock();
assert(goal);
set<int> fds2(j->fds);
std::set<int> fds2(j->fds);
std::vector<unsigned char> buffer(4096);
for (auto & k : fds2) {
if (pollStatus.at(fdToPollStatus.at(k)).revents) {
@ -394,7 +394,7 @@ void Worker::waitForInput()
} else {
printMsg(lvlVomit, "%1%: read %2% bytes",
goal->getName(), rd);
string data((char *) buffer.data(), rd);
std::string data((char *) buffer.data(), rd);
j->lastOutput = after;
goal->handleChildOutput(k, data);
}

4
src/libstore/build/worker.hh
View file

@ -38,7 +38,7 @@ struct Child
{
WeakGoalPtr goal;
Goal * goal2; // ugly hackery
set<int> fds;
std::set<int> fds;
bool respectTimeouts;
bool inBuildSlot;
steady_time_point lastOutput; /* time we last got output on stdout/stderr */
@ -167,7 +167,7 @@ public:
/* Registers a running child process. `inBuildSlot' means that
the process counts towards the jobs limit. */
void childStarted(GoalPtr goal, const set<int> & fds,
void childStarted(GoalPtr goal, const std::set<int> & fds,
bool inBuildSlot, bool respectTimeouts);
/* Unregisters a running child process. `wakeSleepers' should be

4
src/libstore/builtins/buildenv.cc
View file

@ -123,7 +123,7 @@ void buildProfile(const Path & out, Packages && pkgs)
createLinks(state, pkgDir, out, priority);
try {
for (const auto & p : tokenizeString<std::vector<string>>(
for (const auto & p : tokenizeString<std::vector<std::string>>(
readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n"))
if (!done.count(p))
postponed.insert(p);
@ -161,7 +161,7 @@ void buildProfile(const Path & out, Packages && pkgs)
void builtinBuildenv(const BasicDerivation & drv)
{
auto getAttr = [&](const string & name) {
auto getAttr = [&](const std::string & name) {
auto i = drv.env.find(name);
if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
return i->second;

2
src/libstore/builtins/fetchurl.cc
View file

@ -16,7 +16,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
writeFile(settings.netrcFile, netrcData, 0600);
}
auto getAttr = [&](const string & name) {
auto getAttr = [&](const std::string & name) {
auto i = drv.env.find(name);
if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
return i->second;

2
src/libstore/builtins/unpack-channel.cc
View file

@ -5,7 +5,7 @@ namespace nix {
void builtinUnpackChannel(const BasicDerivation & drv)
{
auto getAttr = [&](const string & name) {
auto getAttr = [&](const std::string & name) {
auto i = drv.env.find(name);
if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
return i->second;

5
src/libstore/ca-specific-schema.sql
View file

@ -19,3 +19,8 @@ create table if not exists RealisationsRefs (
foreign key (referrer) references Realisations(id) on delete cascade,
foreign key (realisationReference) references Realisations(id) on delete restrict
);
-- used by QueryRealisationReferences
create index if not exists IndexRealisationsRefs on RealisationsRefs(referrer);
-- used by cascade deletion when ValidPaths is deleted
create index if not exists IndexRealisationsRefsOnOutputPath on Realisations(outputPath);

6
src/libstore/content-address.cc
View file

@ -144,8 +144,10 @@ ContentAddress parseContentAddress(std::string_view rawCa) {
std::pair<ContentAddressMethod, HashType> parseContentAddressMethod(std::string_view caMethod)
{
std::string_view asPrefix {std::string{caMethod} + ":"};
return parseContentAddressMethodPrefix(asPrefix);
std::string asPrefix = std::string{caMethod} + ":";
// parseContentAddressMethodPrefix takes its argument by reference
std::string_view asPrefixView = asPrefix;
return parseContentAddressMethodPrefix(asPrefixView);
}
std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt)

145
src/libstore/daemon.cc
View file

@ -1,10 +1,11 @@
#include "daemon.hh"
#include "monitor-fd.hh"
#include "worker-protocol.hh"
#include "build-result.hh"
#include "store-api.hh"
#include "gc-store.hh"
#include "path-with-outputs.hh"
#include "finally.hh"
#include "affinity.hh"
#include "archive.hh"
#include "derivations.hh"
#include "args.hh"
@ -70,7 +71,7 @@ struct TunnelLogger : public Logger
StringSink buf;
buf << STDERR_NEXT << (fs.s + "\n");
enqueueMsg(*buf.s);
enqueueMsg(buf.s);
}
void logEI(const ErrorInfo & ei) override
@ -82,7 +83,7 @@ struct TunnelLogger : public Logger
StringSink buf;
buf << STDERR_NEXT << oss.str();
enqueueMsg(*buf.s);
enqueueMsg(buf.s);
}
/* startWork() means that we're starting an operation for which we
@ -130,7 +131,7 @@ struct TunnelLogger : public Logger
StringSink buf;
buf << STDERR_START_ACTIVITY << act << lvl << type << s << fields << parent;
enqueueMsg(*buf.s);
enqueueMsg(buf.s);
}
void stopActivity(ActivityId act) override
@ -138,7 +139,7 @@ struct TunnelLogger : public Logger
if (GET_PROTOCOL_MINOR(clientVersion) < 20) return;
StringSink buf;
buf << STDERR_STOP_ACTIVITY << act;
enqueueMsg(*buf.s);
enqueueMsg(buf.s);
}
void result(ActivityId act, ResultType type, const Fields & fields) override
@ -146,7 +147,7 @@ struct TunnelLogger : public Logger
if (GET_PROTOCOL_MINOR(clientVersion) < 20) return;
StringSink buf;
buf << STDERR_RESULT << act << type << fields;
enqueueMsg(*buf.s);
enqueueMsg(buf.s);
}
};
@ -230,11 +231,12 @@ struct ClientSettings
else if (name == settings.experimentalFeatures.name) {
// We don’t want to forward the experimental features to
// the daemon, as that could cause some pretty weird stuff
if (tokenizeString<Strings>(value) != settings.experimentalFeatures.get())
if (parseFeatures(tokenizeString<StringSet>(value)) != settings.experimentalFeatures.get())
debug("Ignoring the client-specified experimental features");
}
else if (trusted
|| name == settings.buildTimeout.name
|| name == settings.buildRepeat.name
|| name == "connect-timeout"
|| (name == "builders" && value == ""))
settings.set(name, value);
@ -406,9 +408,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
return store->queryPathInfo(path);
},
[&](FileIngestionMethod & fim) {
if (!refs.empty())
throw UnimplementedError("cannot yet have refs with flat or nar-hashed data");
auto path = store->addToStoreFromDump(source, name, fim, hashType, repair);
auto path = store->addToStoreFromDump(source, name, fim, hashType, repair, refs);
return store->queryPathInfo(path);
},
}, contentAddressMethod);
@ -436,25 +436,30 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
hashAlgo = parseHashType(hashAlgoRaw);
}
StringSink saved;
TeeSource savedNARSource(from, saved);
RetrieveRegularNARSink savedRegular { saved };
if (method == FileIngestionMethod::Recursive) {
/* Get the entire NAR dump from the client and save it to
a string so that we can pass it to
addToStoreFromDump(). */
ParseSink sink; /* null sink; just parse the NAR */
parseDump(sink, savedNARSource);
} else
parseDump(savedRegular, from);
auto dumpSource = sinkToSource([&](Sink & saved) {
if (method == FileIngestionMethod::Recursive) {
/* We parse the NAR dump through into `saved` unmodified,
so why all this extra work? We still parse the NAR so
that we aren't sending arbitrary data to `saved`
unwittingly`, and we know when the NAR ends so we don't
consume the rest of `from` and can't parse another
command. (We don't trust `addToStoreFromDump` to not
eagerly consume the entire stream it's given, past the
length of the Nar. */
TeeSource savedNARSource(from, saved);
ParseSink sink; /* null sink; just parse the NAR */
parseDump(sink, savedNARSource);
} else {
/* Incrementally parse the NAR file, stripping the
metadata, and streaming the sole file we expect into
`saved`. */
RetrieveRegularNARSink savedRegular { saved };
parseDump(savedRegular, from);
if (!savedRegular.regular) throw Error("regular file expected");
}
});
logger->startWork();
if (!savedRegular.regular) throw Error("regular file expected");
// FIXME: try to stream directly from `from`.
StringSource dumpSource { *saved.s };
auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo);
auto path = store->addToStoreFromDump(*dumpSource, baseName, method, hashAlgo);
logger->stopWork();
to << store->printStorePath(path);
@ -469,17 +474,19 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
dontCheckSigs = false;
logger->startWork();
FramedSource source(from);
store->addMultipleToStore(source,
RepairFlag{repair},
dontCheckSigs ? NoCheckSigs : CheckSigs);
{
FramedSource source(from);
store->addMultipleToStore(source,
RepairFlag{repair},
dontCheckSigs ? NoCheckSigs : CheckSigs);
}
logger->stopWork();
break;
}
case wopAddTextToStore: {
string suffix = readString(from);
string s = readString(from);
std::string suffix = readString(from);
std::string s = readString(from);
auto refs = worker_proto::read(*store, from, Phantom<StorePathSet> {});
logger->startWork();
auto path = store->addTextToStore(suffix, s, refs, NoRepair);
@ -529,6 +536,25 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
break;
}
case wopBuildPathsWithResults: {
auto drvs = readDerivedPaths(*store, clientVersion, from);
BuildMode mode = bmNormal;
mode = (BuildMode) readInt(from);
/* Repairing is not atomic, so disallowed for "untrusted"
clients. */
if (mode == bmRepair && !trusted)
throw Error("repairing is not allowed because you are not in 'trusted-users'");
logger->startWork();
auto results = store->buildPathsWithResults(drvs, mode);
logger->stopWork();
worker_proto::write(*store, to, results);
break;
}
case wopBuildDerivation: {
auto drvPath = store->parseStorePath(readString(from));
BasicDerivation drv;
@ -621,16 +647,19 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopAddIndirectRoot: {
Path path = absPath(readString(from));
logger->startWork();
store->addIndirectRoot(path);
auto & gcStore = requireGcStore(*store);
gcStore.addIndirectRoot(path);
logger->stopWork();
to << 1;
break;
}
// Obsolete.
case wopSyncWithGC: {
logger->startWork();
store->syncWithGC();
logger->stopWork();
to << 1;
break;
@ -638,7 +667,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopFindRoots: {
logger->startWork();
Roots roots = store->findRoots(!trusted);
auto & gcStore = requireGcStore(*store);
Roots roots = gcStore.findRoots(!trusted);
logger->stopWork();
size_t size = 0;
@ -669,7 +699,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork();
if (options.ignoreLiveness)
throw Error("you are not allowed to ignore liveness");
store->collectGarbage(options, results);
auto & gcStore = requireGcStore(*store);
gcStore.collectGarbage(options, results);
logger->stopWork();
to << results.paths << results.bytesFreed << 0 /* obsolete */;
@ -697,8 +728,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (GET_PROTOCOL_MINOR(clientVersion) >= 12) {
unsigned int n = readInt(from);
for (unsigned int i = 0; i < n; i++) {
string name = readString(from);
string value = readString(from);
auto name = readString(from);
auto value = readString(from);
clientSettings.overrides.emplace(name, value);
}
}
@ -853,14 +884,14 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
else {
std::unique_ptr<Source> source;
StringSink saved;
if (GET_PROTOCOL_MINOR(clientVersion) >= 21)
source = std::make_unique<TunnelSource>(from, to);
else {
StringSink saved;
TeeSource tee { from, saved };
ParseSink ether;
parseDump(ether, tee);
source = std::make_unique<StringSource>(std::move(*saved.s));
source = std::make_unique<StringSource>(saved.s);
}
logger->startWork();
@ -921,6 +952,22 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
break;
}
case wopAddBuildLog: {
StorePath path{readString(from)};
logger->startWork();
if (!trusted)
throw Error("you are not privileged to add logs");
{
FramedSource source(from);
StringSink sink;
source.drainInto(sink);
store->addBuildLog(path, sink.s);
}
logger->stopWork();
to << 1;
break;
}
default:
throw Error("invalid operation %1%", op);
}
@ -956,15 +1003,19 @@ void processConnection(
Finally finally([&]() {
_isInterrupted = false;
prevLogger->log(lvlDebug, fmt("%d operations", opCount));
printMsgUsing(prevLogger, lvlDebug, "%d operations", opCount);
});
if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) {
auto affinity = readInt(from);
setAffinityTo(affinity);
// Obsolete CPU affinity.
readInt(from);
}
readInt(from); // obsolete reserveSpace
if (GET_PROTOCOL_MINOR(clientVersion) >= 11)
readInt(from); // obsolete reserveSpace
if (GET_PROTOCOL_MINOR(clientVersion) >= 33)
to << nixVersion;
/* Send startup error messages to the client. */
tunnelLogger->startWork();
@ -989,6 +1040,8 @@ void processConnection(
break;
}
printMsgUsing(prevLogger, lvlDebug, "received daemon op %d", op);
opCount++;
try {

41
src/libstore/derivations.cc
View file

@ -5,6 +5,7 @@
#include "split.hh"
#include "worker-protocol.hh"
#include "fs-accessor.hh"
#include <boost/container/small_vector.hpp>
namespace nix {
@ -83,7 +84,7 @@ bool derivationIsImpure(DerivationType dt) {
bool BasicDerivation::isBuiltin() const
{
return string(builder, 0, 8) == "builtin:";
return builder.substr(0, 8) == "builtin:";
}
@ -105,19 +106,19 @@ StorePath writeDerivation(Store & store,
/* Read string `s' from stream `str'. */
static void expect(std::istream & str, const string & s)
static void expect(std::istream & str, std::string_view s)
{
char s2[s.size()];
str.read(s2, s.size());
if (string(s2, s.size()) != s)
if (std::string(s2, s.size()) != s)
throw FormatError("expected string '%1%'", s);
}
/* Read a C-style string from stream `str'. */
static string parseString(std::istream & str)
static std::string parseString(std::istream & str)
{
string res;
std::string res;
expect(str, "\"");
int c;
while ((c = str.get()) != '"')
@ -185,7 +186,7 @@ static DerivationOutput parseDerivationOutput(const Store & store,
},
};
} else {
settings.requireExperimentalFeature("ca-derivations");
settings.requireExperimentalFeature(Xp::CaDerivations);
assert(pathS == "");
return DerivationOutput {
.output = DerivationOutputCAFloating {
@ -257,8 +258,8 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi
/* Parse the environment variables. */
expect(str, ",[");
while (!endOfList(str)) {
expect(str, "("); string name = parseString(str);
expect(str, ","); string value = parseString(str);
expect(str, "("); auto name = parseString(str);
expect(str, ","); auto value = parseString(str);
expect(str, ")");
drv.env[name] = value;
}
@ -268,9 +269,11 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi
}
static void printString(string & res, std::string_view s)
static void printString(std::string & res, std::string_view s)
{
char buf[s.size() * 2 + 2];
boost::container::small_vector<char, 64 * 1024> buffer;
buffer.reserve(s.size() * 2 + 2);
char * buf = buffer.data();
char * p = buf;
*p++ = '"';
for (auto c : s)
@ -284,7 +287,7 @@ static void printString(string & res, std::string_view s)
}
static void printUnquotedString(string & res, std::string_view s)
static void printUnquotedString(std::string & res, std::string_view s)
{
res += '"';
res.append(s);
@ -293,7 +296,7 @@ static void printUnquotedString(string & res, std::string_view s)
template<class ForwardIterator>
static void printStrings(string & res, ForwardIterator i, ForwardIterator j)
static void printStrings(std::string & res, ForwardIterator i, ForwardIterator j)
{
res += '[';
bool first = true;
@ -306,7 +309,7 @@ static void printStrings(string & res, ForwardIterator i, ForwardIterator j)
template<class ForwardIterator>
static void printUnquotedStrings(string & res, ForwardIterator i, ForwardIterator j)
static void printUnquotedStrings(std::string & res, ForwardIterator i, ForwardIterator j)
{
res += '[';
bool first = true;
@ -318,10 +321,10 @@ static void printUnquotedStrings(string & res, ForwardIterator i, ForwardIterato
}
string Derivation::unparse(const Store & store, bool maskOutputs,
std::string Derivation::unparse(const Store & store, bool maskOutputs,
std::map<std::string, StringSet> * actualInputs) const
{
string s;
std::string s;
s.reserve(65536);
s += "Derive([";
@ -396,7 +399,7 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
// FIXME: remove
bool isDerivation(const string & fileName)
bool isDerivation(const std::string & fileName)
{
return hasSuffix(fileName, drvExtension);
}
@ -588,7 +591,7 @@ std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation &
}
bool wantOutput(const string & output, const std::set<string> & wanted)
bool wantOutput(const std::string & output, const std::set<std::string> & wanted)
{
return wanted.empty() || wanted.find(output) != wanted.end();
}
@ -694,10 +697,10 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
}
std::string hashPlaceholder(const std::string & outputName)
std::string hashPlaceholder(const std::string_view outputName)
{
// FIXME: memoize?
return "/" + hashString(htSHA256, "nix-output:" + outputName).to_string(Base32, false);
return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(Base32, false);
}
std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName)

14
src/libstore/derivations.hh
View file

@ -59,21 +59,19 @@ struct DerivationOutput
std::optional<StorePath> path(const Store & store, std::string_view drvName, std::string_view outputName) const;
};
typedef std::map<string, DerivationOutput> DerivationOutputs;
typedef std::map<std::string, DerivationOutput> DerivationOutputs;
/* These are analogues to the previous DerivationOutputs data type, but they
also contains, for each output, the (optional) store path in which it would
be written. To calculate values of these types, see the corresponding
functions in BasicDerivation */
typedef std::map<string, std::pair<DerivationOutput, std::optional<StorePath>>>
typedef std::map<std::string, std::pair<DerivationOutput, std::optional<StorePath>>>
DerivationOutputsAndOptPaths;
/* For inputs that are sub-derivations, we specify exactly which
output IDs we are interested in. */
typedef std::map<StorePath, StringSet> DerivationInputs;
typedef std::map<string, string> StringPairs;
enum struct DerivationType : uint8_t {
InputAddressed,
DeferredInputAddressed,
@ -103,7 +101,7 @@ struct BasicDerivation
{
DerivationOutputs outputs; /* keyed on symbolic IDs */
StorePathSet inputSrcs; /* inputs that are sources */
string platform;
std::string platform;
Path builder;
Strings args;
StringPairs env;
@ -164,7 +162,7 @@ StorePath writeDerivation(Store & store,
Derivation parseDerivation(const Store & store, std::string && s, std::string_view name);
// FIXME: remove
bool isDerivation(const string & fileName);
bool isDerivation(const std::string & fileName);
/* Calculate the name that will be used for the store path for this
output.
@ -222,7 +220,7 @@ typedef std::map<StorePath, DrvHashModulo> DrvHashes;
// FIXME: global, though at least thread-safe.
extern Sync<DrvHashes> drvHashes;
bool wantOutput(const string & output, const std::set<string> & wanted);
bool wantOutput(const std::string & output, const std::set<std::string> & wanted);
struct Source;
struct Sink;
@ -236,7 +234,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
It is used as a placeholder to allow derivations to refer to their
own outputs without needing to use the hash of a derivation in
itself, making the hash near-impossible to calculate. */
std::string hashPlaceholder(const std::string & outputName);
std::string hashPlaceholder(const std::string_view outputName);
/* This creates an opaque and almost certainly unique string
deterministically from a derivation path and output name.

6
src/libstore/derived-path.cc
View file

@ -75,9 +75,9 @@ DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_vi
assert(n != s.npos);
auto drvPath = store.parseStorePath(s.substr(0, n));
auto outputsS = s.substr(n + 1);
std::set<string> outputs;
std::set<std::string> outputs;
if (outputsS != "*")
outputs = tokenizeString<std::set<string>>(outputsS, ",");
outputs = tokenizeString<std::set<std::string>>(outputsS, ",");
return {drvPath, outputs};
}
@ -100,7 +100,7 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const
staticOutputHashes(store, store.readDerivation(p.drvPath));
for (auto& [outputName, outputPath] : p.outputs) {
if (settings.isExperimentalFeatureEnabled(
"ca-derivations")) {
Xp::CaDerivations)) {
auto thisRealisation = store.queryRealisation(
DrvOutput{drvHashes.at(outputName), outputName});
assert(thisRealisation); // We’ve built it, so we must h

14
src/libstore/dummy-store.cc
View file

@ -21,7 +21,7 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store
, Store(params)
{ }
string getUri() override
std::string getUri() override
{
return *uriSchemes().begin();
}
@ -43,15 +43,19 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store
RepairFlag repair, CheckSigsFlag checkSigs) override
{ unsupported("addToStore"); }
StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override
StorePath addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references,
RepairFlag repair) override
{ unsupported("addTextToStore"); }
void narFromPath(const StorePath & path, Sink & sink) override
{ unsupported("narFromPath"); }
std::optional<const Realisation> queryRealisation(const DrvOutput&) override
{ unsupported("queryRealisation"); }
void queryRealisationUncached(const DrvOutput &,
Callback<std::shared_ptr<const Realisation>> callback) noexcept override
{ callback(nullptr); }
};
static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regDummyStore;

6
src/libstore/export-import.cc
View file

@ -75,20 +75,20 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
auto references = worker_proto::read(*this, source, Phantom<StorePathSet> {});
auto deriver = readString(source);
auto narHash = hashString(htSHA256, *saved.s);
auto narHash = hashString(htSHA256, saved.s);
ValidPathInfo info { path, narHash };
if (deriver != "")
info.deriver = parseStorePath(deriver);
info.setReferencesPossiblyToSelf(std::move(references));
info.narSize = saved.s->size();
info.narSize = saved.s.size();
// Ignore optional legacy signature.
if (readInt(source) == 1)
readString(source);
// Can't use underlying source, which would have been exhausted
auto source = StringSource { *saved.s };
auto source = StringSource(saved.s);
addToStore(info, source, NoRepair, checkSigs);
res.push_back(info.path);

65
src/libstore/filetransfer.cc
View file

@ -33,12 +33,12 @@ FileTransferSettings fileTransferSettings;
static GlobalConfig::Register rFileTransferSettings(&fileTransferSettings);
std::string resolveUri(const std::string & uri)
std::string resolveUri(std::string_view uri)
{
if (uri.compare(0, 8, "channel:") == 0)
return "https://nixos.org/channels/" + std::string(uri, 8) + "/nixexprs.tar.xz";
return "https://nixos.org/channels/" + std::string(uri.substr(8)) + "/nixexprs.tar.xz";
else
return uri;
return std::string(uri);
}
struct curlFileTransfer : public FileTransfer
@ -106,7 +106,7 @@ struct curlFileTransfer : public FileTransfer
this->request.dataCallback(data);
}
} else
this->result.data->append(data);
this->result.data.append(data);
})
{
if (!request.expectedETag.empty())
@ -128,7 +128,7 @@ struct curlFileTransfer : public FileTransfer
if (requestHeaders) curl_slist_free_all(requestHeaders);
try {
if (!done)
fail(FileTransferError(Interrupted, nullptr, "download of '%s' was interrupted", request.uri));
fail(FileTransferError(Interrupted, {}, "download of '%s' was interrupted", request.uri));
} catch (...) {
ignoreException();
}
@ -195,17 +195,17 @@ struct curlFileTransfer : public FileTransfer
std::smatch match;
if (std::regex_match(line, match, statusLine)) {
result.etag = "";
result.data = std::make_shared<std::string>();
result.data.clear();
result.bodySize = 0;
statusMsg = trim(match[1]);
statusMsg = trim(match.str(1));
acceptRanges = false;
encoding = "";
} else {
auto i = line.find(':');
if (i != string::npos) {
string name = toLower(trim(string(line, 0, i)));
if (i != std::string::npos) {
std::string name = toLower(trim(line.substr(0, i)));
if (name == "etag") {
result.etag = trim(string(line, i + 1));
result.etag = trim(line.substr(i + 1));
/* Hack to work around a GitHub bug: it sends
ETags, but ignores If-None-Match. So if we get
the expected ETag on a 200 response, then shut
@ -218,8 +218,8 @@ struct curlFileTransfer : public FileTransfer
return 0;
}
} else if (name == "content-encoding")
encoding = trim(string(line, i + 1));
else if (name == "accept-ranges" && toLower(trim(std::string(line, i + 1))) == "bytes")
encoding = trim(line.substr(i + 1));
else if (name == "accept-ranges" && toLower(trim(line.substr(i + 1))) == "bytes")
acceptRanges = true;
}
}
@ -340,7 +340,7 @@ struct curlFileTransfer : public FileTransfer
if (writtenToSink)
curl_easy_setopt(req, CURLOPT_RESUME_FROM_LARGE, writtenToSink);
result.data = std::make_shared<std::string>();
result.data.clear();
result.bodySize = 0;
}
@ -434,21 +434,21 @@ struct curlFileTransfer : public FileTransfer
attempt++;
std::shared_ptr<std::string> response;
std::optional<std::string> response;
if (errorSink)
response = errorSink->s;
response = std::move(errorSink->s);
auto exc =
code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted
? FileTransferError(Interrupted, response, "%s of '%s' was interrupted", request.verb(), request.uri)
? FileTransferError(Interrupted, std::move(response), "%s of '%s' was interrupted", request.verb(), request.uri)
: httpStatus != 0
? FileTransferError(err,
response,
std::move(response),
fmt("unable to %s '%s': HTTP error %d ('%s')",
request.verb(), request.uri, httpStatus, statusMsg)
+ (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code)))
)
: FileTransferError(err,
response,
std::move(response),
fmt("unable to %s '%s': %s (%d)",
request.verb(), request.uri, curl_easy_strerror(code), code));
@ -544,6 +544,8 @@ struct curlFileTransfer : public FileTransfer
stopWorkerThread();
});
unshareFilesystem();
std::map<CURL *, std::shared_ptr<TransferItem>> items;
bool quit = false;
@ -702,8 +704,8 @@ struct curlFileTransfer : public FileTransfer
auto s3Res = s3Helper.getObject(bucketName, key);
FileTransferResult res;
if (!s3Res.data)
throw FileTransferError(NotFound, nullptr, "S3 object '%s' does not exist", request.uri);
res.data = s3Res.data;
throw FileTransferError(NotFound, {}, "S3 object '%s' does not exist", request.uri);
res.data = std::move(*s3Res.data);
callback(std::move(res));
#else
throw nix::Error("cannot download '%s' because Nix is not built with S3 support", request.uri);
@ -716,15 +718,24 @@ struct curlFileTransfer : public FileTransfer
}
};
ref<curlFileTransfer> makeCurlFileTransfer()
{
return make_ref<curlFileTransfer>();
}
ref<FileTransfer> getFileTransfer()
{
static ref<FileTransfer> fileTransfer = makeFileTransfer();
static ref<curlFileTransfer> fileTransfer = makeCurlFileTransfer();
if (fileTransfer->state_.lock()->quit)
fileTransfer = makeCurlFileTransfer();
return fileTransfer;
}
ref<FileTransfer> makeFileTransfer()
{
return make_ref<curlFileTransfer>();
return makeCurlFileTransfer();
}
std::future<FileTransferResult> FileTransfer::enqueueFileTransfer(const FileTransferRequest & request)
@ -848,25 +859,25 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink)
}
template<typename... Args>
FileTransferError::FileTransferError(FileTransfer::Error error, std::shared_ptr<string> response, const Args & ... args)
FileTransferError::FileTransferError(FileTransfer::Error error, std::optional<std::string> response, const Args & ... args)
: Error(args...), error(error), response(response)
{
const auto hf = hintfmt(args...);
// FIXME: Due to https://github.com/NixOS/nix/issues/3841 we don't know how
// to print different messages for different verbosity levels. For now
// we add some heuristics for detecting when we want to show the response.
if (response && (response->size() < 1024 || response->find("<html>") != string::npos))
if (response && (response->size() < 1024 || response->find("<html>") != std::string::npos))
err.msg = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), chomp(*response));
else
err.msg = hf;
}
bool isUri(const string & s)
bool isUri(std::string_view s)
{
if (s.compare(0, 8, "channel:") == 0) return true;
size_t pos = s.find("://");
if (pos == string::npos) return false;
string scheme(s, 0, pos);
if (pos == std::string::npos) return false;
std::string scheme(s, 0, pos);
return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh";
}

12
src/libstore/filetransfer.hh
View file

@ -59,7 +59,7 @@ struct FileTransferRequest
unsigned int baseRetryTimeMs = 250;
ActivityId parentAct;
bool decompress = true;
std::shared_ptr<std::string> data;
std::optional<std::string> data;
std::string mimeType;
std::function<void(std::string_view data)> dataCallback;
@ -77,7 +77,7 @@ struct FileTransferResult
bool cached = false;
std::string etag;
std::string effectiveUri;
std::shared_ptr<std::string> data;
std::string data;
uint64_t bodySize = 0;
};
@ -119,17 +119,17 @@ class FileTransferError : public Error
{
public:
FileTransfer::Error error;
std::shared_ptr<string> response; // intentionally optional
std::optional<std::string> response; // intentionally optional
template<typename... Args>
FileTransferError(FileTransfer::Error error, std::shared_ptr<string> response, const Args & ... args);
FileTransferError(FileTransfer::Error error, std::optional<std::string> response, const Args & ... args);
virtual const char* sname() const override { return "FileTransferError"; }
};
bool isUri(const string & s);
bool isUri(std::string_view s);
/* Resolve deprecated 'channel:<foo>' URLs. */
std::string resolveUri(const std::string & uri);
std::string resolveUri(std::string_view uri);
}

13
src/libstore/gc-store.cc Normal file
View file

@ -0,0 +1,13 @@
#include "gc-store.hh"
namespace nix {
GcStore & requireGcStore(Store & store)
{
auto * gcStore = dynamic_cast<GcStore *>(&store);
if (!gcStore)
throw UsageError("Garbage collection not supported by this store");
return *gcStore;
}
}

84
src/libstore/gc-store.hh Normal file
View file

@ -0,0 +1,84 @@
#pragma once
#include "store-api.hh"
namespace nix {
typedef std::unordered_map<StorePath, std::unordered_set<std::string>> Roots;
struct GCOptions
{
/* Garbage collector operation:
- `gcReturnLive': return the set of paths reachable from
(i.e. in the closure of) the roots.
- `gcReturnDead': return the set of paths not reachable from
the roots.
- `gcDeleteDead': actually delete the latter set.
- `gcDeleteSpecific': delete the paths listed in
`pathsToDelete', insofar as they are not reachable.
*/
typedef enum {
gcReturnLive,
gcReturnDead,
gcDeleteDead,
gcDeleteSpecific,
} GCAction;
GCAction action{gcDeleteDead};
/* If `ignoreLiveness' is set, then reachability from the roots is
ignored (dangerous!). However, the paths must still be
unreferenced *within* the store (i.e., there can be no other
store paths that depend on them). */
bool ignoreLiveness{false};
/* For `gcDeleteSpecific', the paths to delete. */
StorePathSet pathsToDelete;
/* Stop after at least `maxFreed' bytes have been freed. */
uint64_t maxFreed{std::numeric_limits<uint64_t>::max()};
};
struct GCResults
{
/* Depending on the action, the GC roots, or the paths that would
be or have been deleted. */
PathSet paths;
/* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the
number of bytes that would be or was freed. */
uint64_t bytesFreed = 0;
};
struct GcStore : public virtual Store
{
/* Add an indirect root, which is merely a symlink to `path' from
/nix/var/nix/gcroots/auto/<hash of `path'>. `path' is supposed
to be a symlink to a store path. The garbage collector will
automatically remove the indirect root when it finds that
`path' has disappeared. */
virtual void addIndirectRoot(const Path & path) = 0;
/* Find the roots of the garbage collector. Each root is a pair
(link, storepath) where `link' is the path of the symlink
outside of the Nix store that point to `storePath'. If
'censor' is true, privacy-sensitive information about roots
found in /proc is censored. */
virtual Roots findRoots(bool censor) = 0;
/* Perform a garbage collection. */
virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0;
};
GcStore & requireGcStore(Store & store);
}

831
src/libstore/gc.cc
View file

File diff suppressed because it is too large Load diff

24
src/libstore/globals.cc
View file

@ -100,7 +100,7 @@ std::vector<Path> getUserConfigFiles()
// Use the paths specified in NIX_USER_CONF_FILES if it has been defined
auto nixConfFiles = getEnv("NIX_USER_CONF_FILES");
if (nixConfFiles.has_value()) {
return tokenizeString<std::vector<string>>(nixConfFiles.value(), ":");
return tokenizeString<std::vector<std::string>>(nixConfFiles.value(), ":");
}
// Use the paths specified by the XDG spec
@ -122,7 +122,7 @@ StringSet Settings::getDefaultSystemFeatures()
/* For backwards compatibility, accept some "features" that are
used in Nixpkgs to route builds to certain machines but don't
actually require anything special on the machines. */
StringSet features{"nixos-test", "benchmark", "big-parallel", "recursive-nix"};
StringSet features{"nixos-test", "benchmark", "big-parallel"};
#if __linux__
if (access("/dev/kvm", R_OK | W_OK) == 0)
@ -148,7 +148,8 @@ StringSet Settings::getDefaultExtraPlatforms()
// machines. Note that we can’t force processes from executing
// x86_64 in aarch64 environments or vice versa since they can
// always exec with their own binary preferences.
if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) {
if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist") ||
pathExists("/System/Library/LaunchDaemons/com.apple.oahd.plist")) {
if (std::string{SYSTEM} == "x86_64-darwin")
extraPlatforms.insert("aarch64-darwin");
else if (std::string{SYSTEM} == "aarch64-darwin")
@ -159,21 +160,16 @@ StringSet Settings::getDefaultExtraPlatforms()
return extraPlatforms;
}
bool Settings::isExperimentalFeatureEnabled(const std::string & name)
bool Settings::isExperimentalFeatureEnabled(const ExperimentalFeature & feature)
{
auto & f = experimentalFeatures.get();
return std::find(f.begin(), f.end(), name) != f.end();
return std::find(f.begin(), f.end(), feature) != f.end();
}
MissingExperimentalFeature::MissingExperimentalFeature(std::string feature)
: Error("experimental Nix feature '%1%' is disabled; use '--extra-experimental-features %1%' to override", feature)
, missingFeature(feature)
{}
void Settings::requireExperimentalFeature(const std::string & name)
void Settings::requireExperimentalFeature(const ExperimentalFeature & feature)
{
if (!isExperimentalFeatureEnabled(name))
throw MissingExperimentalFeature(name);
if (!isExperimentalFeatureEnabled(feature))
throw MissingExperimentalFeature(feature);
}
bool Settings::isWSL1()
@ -185,7 +181,7 @@ bool Settings::isWSL1()
return hasSuffix(utsbuf.release, "-Microsoft");
}
const string nixVersion = PACKAGE_VERSION;
const std::string nixVersion = PACKAGE_VERSION;
NLOHMANN_JSON_SERIALIZE_ENUM(SandboxMode, {
{SandboxMode::smEnabled, true},

107
src/libstore/globals.hh
View file

@ -3,6 +3,7 @@
#include "types.hh"
#include "config.hh"
#include "util.hh"
#include "experimental-features.hh"
#include <map>
#include <limits>
@ -20,7 +21,7 @@ struct MaxBuildJobsSetting : public BaseSetting<unsigned int>
const std::string & name,
const std::string & description,
const std::set<std::string> & aliases = {})
: BaseSetting<unsigned int>(def, name, description, aliases)
: BaseSetting<unsigned int>(def, true, name, description, aliases)
{
options->addSetting(this);
}
@ -37,7 +38,7 @@ struct PluginFilesSetting : public BaseSetting<Paths>
const std::string & name,
const std::string & description,
const std::set<std::string> & aliases = {})
: BaseSetting<Paths>(def, name, description, aliases)
: BaseSetting<Paths>(def, true, name, description, aliases)
{
options->addSetting(this);
}
@ -45,15 +46,6 @@ struct PluginFilesSetting : public BaseSetting<Paths>
void set(const std::string & str, bool append = false) override;
};
class MissingExperimentalFeature: public Error
{
public:
std::string missingFeature;
MissingExperimentalFeature(std::string feature);
virtual const char* sname() const override { return "MissingExperimentalFeature"; }
};
class Settings : public Config {
unsigned int getDefaultCores();
@ -121,7 +113,7 @@ public:
bool verboseBuild = true;
Setting<size_t> logLines{this, 10, "log-lines",
"If `verbose-build` is false, the number of lines of the tail of "
"The number of lines of the tail of "
"the log to show if a build fails."};
MaxBuildJobsSetting maxBuildJobs{
@ -138,7 +130,9 @@ public:
{"build-max-jobs"}};
Setting<unsigned int> buildCores{
this, getDefaultCores(), "cores",
this,
getDefaultCores(),
"cores",
R"(
Sets the value of the `NIX_BUILD_CORES` environment variable in the
invocation of builders. Builders can use this variable at their
@ -149,7 +143,7 @@ public:
command line switch and defaults to `1`. The value `0` means that
the builder should use all available CPU cores in the system.
)",
{"build-cores"}};
{"build-cores"}, false};
/* Read-only mode. Don't copy stuff to the store, don't change
the database. */
@ -591,10 +585,11 @@ public:
platform and generate incompatible code, so you may wish to
cross-check the results of using this option against proper
natively-built versions of your derivations.
)"};
)", {}, false};
Setting<StringSet> systemFeatures{
this, getDefaultSystemFeatures(),
this,
getDefaultSystemFeatures(),
"system-features",
R"(
A set of system “features” supported by this machine, e.g. `kvm`.
@ -610,7 +605,7 @@ public:
This setting by default includes `kvm` if `/dev/kvm` is accessible,
and the pseudo-features `nixos-test`, `benchmark` and `big-parallel`
that are used in Nixpkgs to route builds to specific machines.
)"};
)", {}, false};
Setting<Strings> substituters{
this,
@ -805,6 +800,15 @@ public:
may be useful in certain scenarios (e.g. to spin up containers or
set up userspace network interfaces in tests).
)"};
Setting<StringSet> ignoredAcls{
this, {"security.selinux", "system.nfs4_acl"}, "ignored-acls",
R"(
A list of ACLs that should be ignored, normally Nix attempts to
remove all ACLs from files and directories in the Nix store, but
some ACLs like `security.selinux` or `system.nfs4_acl` can't be
removed even by root. Therefore it's best to just ignore them.
)"};
#endif
Setting<Strings> hashedMirrors{
@ -876,74 +880,16 @@ public:
are loaded as plugins (non-recursively).
)"};
Setting<StringMap> accessTokens{this, {}, "access-tokens",
R"(
Access tokens used to access protected GitHub, GitLab, or
other locations requiring token-based authentication.
Access tokens are specified as a string made up of
space-separated `host=token` values. The specific token
used is selected by matching the `host` portion against the
"host" specification of the input. The actual use of the
`token` value is determined by the type of resource being
accessed:
* Github: the token value is the OAUTH-TOKEN string obtained
as the Personal Access Token from the Github server (see
https://docs.github.com/en/developers/apps/authorizing-oath-apps).
* Gitlab: the token value is either the OAuth2 token or the
Personal Access Token (these are different types tokens
for gitlab, see
https://docs.gitlab.com/12.10/ee/api/README.html#authentication).
The `token` value should be `type:tokenstring` where
`type` is either `OAuth2` or `PAT` to indicate which type
of token is being specified.
Example `~/.config/nix/nix.conf`:
```
access-tokens = github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk
```
Example `~/code/flake.nix`:
```nix
input.foo = {
type = "gitlab";
host = "gitlab.mycompany.com";
owner = "mycompany";
repo = "pro";
};
```
This example specifies three tokens, one each for accessing
github.com, gitlab.mycompany.com, and sourceforge.net.
The `input.foo` uses the "gitlab" fetcher, which might
requires specifying the token type along with the token
value.
)"};
Setting<Strings> experimentalFeatures{this, {}, "experimental-features",
Setting<std::set<ExperimentalFeature>> experimentalFeatures{this, {}, "experimental-features",
"Experimental Nix features to enable."};
bool isExperimentalFeatureEnabled(const std::string & name);
bool isExperimentalFeatureEnabled(const ExperimentalFeature &);
void requireExperimentalFeature(const std::string & name);
Setting<bool> allowDirty{this, true, "allow-dirty",
"Whether to allow dirty Git/Mercurial trees."};
Setting<bool> warnDirty{this, true, "warn-dirty",
"Whether to warn about dirty Git/Mercurial trees."};
void requireExperimentalFeature(const ExperimentalFeature &);
Setting<size_t> narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size",
"Maximum size of NARs before spilling them to disk."};
Setting<std::string> flakeRegistry{this, "https://github.com/NixOS/flake-registry/raw/master/flake-registry.json", "flake-registry",
"Path or URI of the global flake registry."};
Setting<bool> allowSymlinkedStore{
this, false, "allow-symlinked-store",
R"(
@ -956,9 +902,6 @@ public:
resolves to a different location from that of the build machine. You
can enable this setting if you are sure you're not going to do that.
)"};
Setting<bool> useRegistries{this, true, "use-registries",
"Whether to use flake registries to resolve flake references."};
};
@ -974,6 +917,6 @@ void loadConfFile();
// Used by the Settings constructor
std::vector<Path> getUserConfigFiles();
extern const string nixVersion;
extern const std::string nixVersion;
}

8
src/libstore/http-binary-cache-store.cc
View file

@ -126,7 +126,7 @@ protected:
const std::string & mimeType) override
{
auto req = makeRequest(path);
req.data = std::make_shared<string>(StreamToSourceAdapter(istream).drain());
req.data = StreamToSourceAdapter(istream).drain();
req.mimeType = mimeType;
try {
getFileTransfer()->upload(req);
@ -159,7 +159,7 @@ protected:
}
void getFile(const std::string & path,
Callback<std::shared_ptr<std::string>> callback) noexcept override
Callback<std::optional<std::string>> callback) noexcept override
{
checkEnabled();
@ -170,10 +170,10 @@ protected:
getFileTransfer()->enqueueFileTransfer(request,
{[callbackPtr, this](std::future<FileTransferResult> result) {
try {
(*callbackPtr)(result.get().data);
(*callbackPtr)(std::move(result.get().data));
} catch (FileTransferError & e) {
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
return (*callbackPtr)(std::shared_ptr<std::string>());
return (*callbackPtr)({});
maybeDisable();
callbackPtr->rethrow();
} catch (...) {

32
src/libstore/legacy-ssh-store.cc
View file

@ -2,6 +2,7 @@
#include "pool.hh"
#include "remote-store.hh"
#include "serve-protocol.hh"
#include "build-result.hh"
#include "store-api.hh"
#include "path-with-outputs.hh"
#include "worker-protocol.hh"
@ -48,7 +49,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
static std::set<std::string> uriSchemes() { return {"ssh"}; }
LegacySSHStore(const string & scheme, const string & host, const Params & params)
LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params)
: StoreConfig(params)
, LegacySSHStoreConfig(params)
, Store(params)
@ -94,7 +95,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
conn->sshConn->in.close();
auto msg = conn->from.drain();
throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'",
host, chomp(*saved.s + msg));
host, chomp(saved.s + msg));
}
conn->remoteVersion = readInt(conn->from);
if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200)
@ -107,7 +108,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
return conn;
};
string getUri() override
std::string getUri() override
{
return *uriSchemes().begin() + "://" + host;
}
@ -224,13 +225,21 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
{ unsupported("queryPathFromHashPart"); }
StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo,
PathFilter & filter, RepairFlag repair) override
StorePath addToStore(
std::string_view name,
const Path & srcPath,
FileIngestionMethod method,
HashType hashAlgo,
PathFilter & filter,
RepairFlag repair,
const StorePathSet & references) override
{ unsupported("addToStore"); }
StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override
StorePath addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references,
RepairFlag repair) override
{ unsupported("addTextToStore"); }
private:
@ -269,7 +278,7 @@ public:
conn->to.flush();
BuildResult status;
BuildResult status { .path = DerivedPath::Built { .drvPath = drvPath } };
status.status = (BuildResult::Status) readInt(conn->from);
conn->from >> status.errorMsg;
@ -307,7 +316,7 @@ public:
conn->to.flush();
BuildResult result;
BuildResult result { .path = DerivedPath::Opaque { StorePath::dummy } };
result.status = (BuildResult::Status) readInt(conn->from);
if (!result.success()) {
@ -366,7 +375,8 @@ public:
return conn->remoteVersion;
}
std::optional<const Realisation> queryRealisation(const DrvOutput&) override
void queryRealisationUncached(const DrvOutput &,
Callback<std::shared_ptr<const Realisation>> callback) noexcept override
// TODO: Implement
{ unsupported("queryRealisation"); }
};

1
src/libstore/local-binary-cache-store.cc
View file

@ -96,6 +96,7 @@ void LocalBinaryCacheStore::init()
createDirs(binaryCacheDir + "/" + realisationsPrefix);
if (writeDebugInfo)
createDirs(binaryCacheDir + "/debuginfo");
createDirs(binaryCacheDir + "/log");
BinaryCacheStore::init();
}

18
src/libstore/local-fs-store.cc
View file

@ -85,36 +85,34 @@ void LocalFSStore::narFromPath(const StorePath & path, Sink & sink)
dumpPath(getRealStoreDir() + std::string(printStorePath(path), storeDir.size()), sink);
}
const string LocalFSStore::drvsLogDir = "drvs";
const std::string LocalFSStore::drvsLogDir = "drvs";
std::shared_ptr<std::string> LocalFSStore::getBuildLog(const StorePath & path_)
std::optional<std::string> LocalFSStore::getBuildLog(const StorePath & path_)
{
auto path = path_;
if (!path.isDerivation()) {
try {
auto info = queryPathInfo(path);
if (!info->deriver) return nullptr;
if (!info->deriver) return std::nullopt;
path = *info->deriver;
} catch (InvalidPath &) {
return nullptr;
return std::nullopt;
}
}
auto baseName = std::string(baseNameOf(printStorePath(path)));
auto baseName = path.to_string();
for (int j = 0; j < 2; j++) {
Path logPath =
j == 0
? fmt("%s/%s/%s/%s", logDir, drvsLogDir, string(baseName, 0, 2), string(baseName, 2))
? fmt("%s/%s/%s/%s", logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2))
: fmt("%s/%s/%s", logDir, drvsLogDir, baseName);
Path logBz2Path = logPath + ".bz2";
if (pathExists(logPath))
return std::make_shared<std::string>(readFile(logPath));
return readFile(logPath);
else if (pathExists(logBz2Path)) {
try {
@ -124,7 +122,7 @@ std::shared_ptr<std::string> LocalFSStore::getBuildLog(const StorePath & path_)
}
return nullptr;
return std::nullopt;
}
}

8
src/libstore/local-fs-store.hh
View file

@ -1,6 +1,7 @@
#pragma once
#include "store-api.hh"
#include "gc-store.hh"
namespace nix {
@ -23,11 +24,11 @@ struct LocalFSStoreConfig : virtual StoreConfig
"physical path to the Nix store"};
};
class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store
class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store, virtual GcStore
{
public:
const static string drvsLogDir;
const static std::string drvsLogDir;
LocalFSStore(const Params & params);
@ -45,7 +46,8 @@ public:
return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1);
}
std::shared_ptr<std::string> getBuildLog(const StorePath & path) override;
std::optional<std::string> getBuildLog(const StorePath & path) override;
};
}

155
src/libstore/local-store.cc
View file

@ -8,6 +8,8 @@
#include "references.hh"
#include "callback.hh"
#include "topo-sort.hh"
#include "finally.hh"
#include "compression.hh"
#include <iostream>
#include <algorithm>
@ -68,7 +70,7 @@ int getSchema(Path schemaPath)
{
int curSchema = 0;
if (pathExists(schemaPath)) {
string s = readFile(schemaPath);
auto s = readFile(schemaPath);
auto n = string2Int<int>(s);
if (!n)
throw Error("'%1%' is corrupt", schemaPath);
@ -79,7 +81,7 @@ int getSchema(Path schemaPath)
void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd)
{
const int nixCASchemaVersion = 2;
const int nixCASchemaVersion = 3;
int curCASchema = getSchema(schemaPath);
if (curCASchema != nixCASchemaVersion) {
if (curCASchema > nixCASchemaVersion) {
@ -130,6 +132,17 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd)
txn.commit();
}
if (curCASchema < 3) {
SQLiteTxn txn(db);
// Apply new indices added in this schema update.
db.exec(R"(
-- used by QueryRealisationReferences
create index if not exists IndexRealisationsRefs on RealisationsRefs(referrer);
-- used by cascade deletion when ValidPaths is deleted
create index if not exists IndexRealisationsRefsOnOutputPath on Realisations(outputPath);
)");
txn.commit();
}
writeFile(schemaPath, fmt("%d", nixCASchemaVersion));
lockFile(lockFd.get(), ltRead, true);
}
@ -145,7 +158,6 @@ LocalStore::LocalStore(const Params & params)
, linksDir(realStoreDir + "/.links")
, reservedPath(dbDir + "/reserved")
, schemaPath(dbDir + "/schema")
, trashDir(realStoreDir + "/trash")
, tempRootsDir(stateDir + "/temproots")
, fnTempRoots(fmt("%s/%d", tempRootsDir, getpid()))
, locksHeld(tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS").value_or("")))
@ -227,7 +239,7 @@ LocalStore::LocalStore(const Params & params)
res = posix_fallocate(fd.get(), 0, settings.reservedSize);
#endif
if (res == -1) {
writeFull(fd.get(), string(settings.reservedSize, 'X'));
writeFull(fd.get(), std::string(settings.reservedSize, 'X'));
[[gnu::unused]] auto res2 = ftruncate(fd.get(), settings.reservedSize);
}
}
@ -309,7 +321,7 @@ LocalStore::LocalStore(const Params & params)
else openDB(*state, false);
if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
migrateCASchema(state->db, dbDir + "/ca-schema", globalLock);
}
@ -339,7 +351,7 @@ LocalStore::LocalStore(const Params & params)
state->stmts->QueryPathFromHashPart.create(state->db,
"select path from ValidPaths where path >= ? limit 1;");
state->stmts->QueryValidPaths.create(state->db, "select path from ValidPaths");
if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
state->stmts->RegisterRealisedOutput.create(state->db,
R"(
insert into Realisations (drvPath, outputName, outputPath, signatures)
@ -386,6 +398,16 @@ LocalStore::LocalStore(const Params & params)
}
AutoCloseFD LocalStore::openGCLock()
{
Path fnGCLock = stateDir + "/gc.lock";
auto fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
if (!fdGCLock)
throw SysError("opening global GC lock '%1%'", fnGCLock);
return fdGCLock;
}
LocalStore::~LocalStore()
{
std::shared_future<void> future;
@ -428,7 +450,7 @@ void LocalStore::openDB(State & state, bool create)
throw SysError("Nix database directory '%1%' is not writable", dbDir);
/* Open the Nix database. */
string dbPath = dbDir + "/db.sqlite";
std::string dbPath = dbDir + "/db.sqlite";
auto & db(state.db);
state.db = SQLite(dbPath, create);
@ -449,19 +471,19 @@ void LocalStore::openDB(State & state, bool create)
should be safe enough. If the user asks for it, don't sync at
all. This can cause database corruption if the system
crashes. */
string syncMode = settings.fsyncMetadata ? "normal" : "off";
std::string syncMode = settings.fsyncMetadata ? "normal" : "off";
db.exec("pragma synchronous = " + syncMode);
/* Set the SQLite journal mode. WAL mode is fastest, so it's the
default. */
string mode = settings.useSQLiteWAL ? "wal" : "truncate";
string prevMode;
std::string mode = settings.useSQLiteWAL ? "wal" : "truncate";
std::string prevMode;
{
SQLiteStmt stmt;
stmt.create(db, "pragma main.journal_mode;");
if (sqlite3_step(stmt) != SQLITE_ROW)
throwSQLiteError(db, "querying journal mode");
prevMode = string((const char *) sqlite3_column_text(stmt, 0));
prevMode = std::string((const char *) sqlite3_column_text(stmt, 0));
}
if (prevMode != mode &&
sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), 0, 0, 0) != SQLITE_OK)
@ -495,9 +517,6 @@ void LocalStore::makeStoreWritable()
throw SysError("getting info about the Nix store mount point");
if (stat.f_flag & ST_RDONLY) {
if (unshare(CLONE_NEWNS) == -1)
throw SysError("setting up a private mount namespace");
if (mount(0, realStoreDir.get().c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1)
throw SysError("remounting %1% writable", realStoreDir);
}
@ -583,9 +602,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
throw SysError("querying extended attributes of '%s'", path);
for (auto & eaName: tokenizeString<Strings>(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) {
/* Ignore SELinux security labels since these cannot be
removed even by root. */
if (eaName == "security.selinux") continue;
if (settings.ignoredAcls.get().count(eaName)) continue;
if (lremovexattr(path.c_str(), eaName.c_str()) == -1)
throw SysError("removing extended attribute '%s' from '%s'", eaName, path);
}
@ -662,7 +679,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
{
assert(drvPath.isDerivation());
std::string drvName(drvPath.name());
drvName = string(drvName, 0, drvName.size() - drvExtension.size());
drvName = drvName.substr(0, drvName.size() - drvExtension.size());
auto envHasRightPath = [&](const StorePath & actual, const std::string & varName)
{
@ -708,7 +725,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs)
{
settings.requireExperimentalFeature("ca-derivations");
settings.requireExperimentalFeature(Xp::CaDerivations);
if (checkSigs == NoCheckSigs || !realisationIsUntrusted(info))
registerDrvOutput(info);
else
@ -717,7 +734,7 @@ void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag check
void LocalStore::registerDrvOutput(const Realisation & info)
{
settings.requireExperimentalFeature("ca-derivations");
settings.requireExperimentalFeature(Xp::CaDerivations);
retrySQLite<void>([&]() {
auto state(_state.lock());
if (auto oldR = queryRealisation_(*state, info.id)) {
@ -769,7 +786,11 @@ void LocalStore::registerDrvOutput(const Realisation & info)
});
}
void LocalStore::cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output)
void LocalStore::cacheDrvOutputMapping(
State & state,
const uint64_t deriver,
const std::string & outputName,
const StorePath & output)
{
retrySQLite<void>([&]() {
state.stmts->AddDerivationOutput.use()
@ -778,7 +799,6 @@ void LocalStore::cacheDrvOutputMapping(State & state, const uint64_t deriver, co
(printStorePath(output))
.exec();
});
}
@ -825,7 +845,7 @@ uint64_t LocalStore::addValidPath(State & state,
{
auto state_(Store::state.lock());
state_->pathInfoCache.upsert(std::string(info.path.hashPart()),
state_->pathInfoCache.upsert(std::string(info.path.to_string()),
PathInfoCacheValue{ .value = std::make_shared<const ValidPathInfo>(info) });
}
@ -1004,7 +1024,7 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath & path_)
return outputs;
});
if (!settings.isExperimentalFeatureEnabled("ca-derivations"))
if (!settings.isExperimentalFeatureEnabled(Xp::CaDerivations))
return outputs;
auto drv = readInvalidDerivation(path);
@ -1204,7 +1224,7 @@ void LocalStore::invalidatePath(State & state, const StorePath & path)
{
auto state_(Store::state.lock());
state_->pathInfoCache.erase(std::string(path.hashPart()));
state_->pathInfoCache.erase(std::string(path.to_string()));
}
}
@ -1297,7 +1317,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
canonicalisePathMetaData(realPath, -1);
optimisePath(realPath); // FIXME: combine with hashPath()
optimisePath(realPath, repair); // FIXME: combine with hashPath()
registerValidPath(info);
}
@ -1307,8 +1327,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
}
StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name,
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references)
{
/* For computing the store path. */
auto hashSink = std::make_unique<HashSink>(hashAlgo);
@ -1333,13 +1353,15 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
auto want = std::min(chunkSize, settings.narBufferSize - oldSize);
dump.resize(oldSize + want);
auto got = 0;
Finally cleanup([&]() {
dump.resize(oldSize + got);
});
try {
got = source.read(dump.data() + oldSize, want);
} catch (EndOfFile &) {
inMemory = true;
break;
}
dump.resize(oldSize + got);
}
std::unique_ptr<AutoDelete> delTempDir;
@ -1365,13 +1387,16 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
auto [hash, size] = hashSink->finish();
auto desc = StorePathDescriptor {
name,
std::string { name },
FixedOutputInfo {
{
.method = method,
.hash = hash,
},
{},
{
.references = references,
.hasSelfReference = false,
},
},
};
@ -1418,7 +1443,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath
optimisePath(realPath);
optimisePath(realPath, repair);
ValidPathInfo info { *this, std::move(desc), narHash.first };
info.narSize = narHash.second;
@ -1432,7 +1457,9 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
}
StorePath LocalStore::addTextToStore(const string & name, const string & s,
StorePath LocalStore::addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references, RepairFlag repair)
{
auto hash = hashString(htSHA256, s);
@ -1461,12 +1488,12 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
StringSink sink;
dumpString(s, sink);
auto narHash = hashString(htSHA256, *sink.s);
auto narHash = hashString(htSHA256, sink.s);
optimisePath(realPath);
optimisePath(realPath, repair);
ValidPathInfo info { dstPath, narHash };
info.narSize = sink.s->size();
info.narSize = sink.s.size();
info.references = references;
info.ca = TextHash { .hash = hash };
registerValidPath(info);
@ -1524,7 +1551,8 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
/* Acquire the global GC lock to get a consistent snapshot of
existing and valid paths. */
AutoCloseFD fdGCLock = openGCLock(ltWrite);
auto fdGCLock = openGCLock();
FdLock gcLock(fdGCLock.get(), ltRead, true, "waiting for the big garbage collector lock...");
StringSet store;
for (auto & i : readDirectory(realStoreDir)) store.insert(i.name);
@ -1535,8 +1563,6 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
StorePathSet validPaths;
PathSet done;
fdGCLock = -1;
for (auto & i : queryAllValidPaths())
verifyPath(printStorePath(i), store, done, validPaths, repair, errors);
@ -1548,7 +1574,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
for (auto & link : readDirectory(linksDir)) {
printMsg(lvlTalkative, "checking contents of '%s'", link.name);
Path linkPath = linksDir + "/" + link.name;
string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false);
std::string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false);
if (hash != link.name) {
printError("link '%s' was modified! expected hash '%s', got '%s'",
linkPath, link.name, hash);
@ -1849,13 +1875,24 @@ std::optional<const Realisation> LocalStore::queryRealisation_(
return { res };
}
std::optional<const Realisation>
LocalStore::queryRealisation(const DrvOutput & id)
void LocalStore::queryRealisationUncached(const DrvOutput & id,
Callback<std::shared_ptr<const Realisation>> callback) noexcept
{
return retrySQLite<std::optional<const Realisation>>([&]() {
auto state(_state.lock());
return queryRealisation_(*state, id);
});
try {
auto maybeRealisation
= retrySQLite<std::optional<const Realisation>>([&]() {
auto state(_state.lock());
return queryRealisation_(*state, id);
});
if (maybeRealisation)
callback(
std::make_shared<const Realisation>(maybeRealisation.value()));
else
callback(nullptr);
} catch (...) {
callback.rethrow();
}
}
FixedOutputHash LocalStore::hashCAPath(
@ -1888,4 +1925,30 @@ FixedOutputHash LocalStore::hashCAPath(
};
}
void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log)
{
assert(drvPath.isDerivation());
auto baseName = drvPath.to_string();
auto logPath = fmt("%s/%s/%s/%s.bz2", logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2));
if (pathExists(logPath)) return;
createDirs(dirOf(logPath));
auto tmpFile = fmt("%s.tmp.%d", logPath, getpid());
writeFile(tmpFile, compress("bzip2", log));
if (rename(tmpFile.c_str(), logPath.c_str()) != 0)
throw SysError("renaming '%1%' to '%2%'", tmpFile, logPath);
}
std::optional<std::string> LocalStore::getVersion()
{
return nixVersion;
}
} // namespace nix

67
src/libstore/local-store.hh
View file

@ -5,6 +5,7 @@
#include "pathlocks.hh"
#include "store-api.hh"
#include "local-fs-store.hh"
#include "gc-store.hh"
#include "sync.hh"
#include "util.hh"
@ -43,7 +44,7 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig
};
class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore
class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore, public virtual GcStore
{
private:
@ -58,9 +59,15 @@ private:
struct Stmts;
std::unique_ptr<Stmts> stmts;
/* The global GC lock */
AutoCloseFD fdGCLock;
/* The file to which we write our temporary roots. */
AutoCloseFD fdTempRoots;
/* Connection to the garbage collector. */
AutoCloseFD fdRootsSocket;
/* The last time we checked whether to do an auto-GC, or an
auto-GC finished. */
std::chrono::time_point<std::chrono::steady_clock> lastGCCheck;
@ -87,7 +94,6 @@ public:
const Path linksDir;
const Path reservedPath;
const Path schemaPath;
const Path trashDir;
const Path tempRootsDir;
const Path fnTempRoots;
@ -139,24 +145,24 @@ public:
void addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs) override;
StorePath addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override;
StorePath addToStoreFromDump(Source & dump, std::string_view name,
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override;
StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override;
StorePath addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references,
RepairFlag repair) override;
void addTempRoot(const StorePath & path) override;
void addIndirectRoot(const Path & path) override;
void syncWithGC() override;
private:
typedef std::shared_ptr<AutoCloseFD> FDPtr;
typedef list<FDPtr> FDs;
void findTempRoots(Roots & roots, bool censor);
void findTempRoots(FDs & fds, Roots & roots, bool censor);
AutoCloseFD openGCLock();
public:
@ -170,8 +176,9 @@ public:
void optimiseStore() override;
/* Optimise a single store path. */
void optimisePath(const Path & path);
/* Optimise a single store path. Optionally, test the encountered
symlinks for corruption. */
void optimisePath(const Path & path, RepairFlag repair);
bool verifyStore(bool checkContents, RepairFlag repair) override;
@ -201,11 +208,18 @@ public:
derivation 'deriver'. */
void registerDrvOutput(const Realisation & info) override;
void registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) override;
void cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output);
void cacheDrvOutputMapping(
State & state,
const uint64_t deriver,
const std::string & outputName,
const StorePath & output);
std::optional<const Realisation> queryRealisation_(State & state, const DrvOutput & id);
std::optional<std::pair<int64_t, Realisation>> queryRealisationCore_(State & state, const DrvOutput & id);
std::optional<const Realisation> queryRealisation(const DrvOutput&) override;
void queryRealisationUncached(const DrvOutput&,
Callback<std::shared_ptr<const Realisation>> callback) noexcept override;
std::optional<std::string> getVersion() override;
private:
@ -236,29 +250,12 @@ private:
PathSet queryValidPathsOld();
ValidPathInfo queryPathInfoOld(const Path & path);
struct GCState;
void deleteGarbage(GCState & state, const Path & path);
void tryToDelete(GCState & state, const Path & path);
bool canReachRoot(GCState & state, StorePathSet & visited, const StorePath & path);
void deletePathRecursive(GCState & state, const Path & path);
bool isActiveTempFile(const GCState & state,
const Path & path, const string & suffix);
AutoCloseFD openGCLock(LockType lockType);
void findRoots(const Path & path, unsigned char type, Roots & roots);
void findRootsNoTemp(Roots & roots, bool censor);
void findRuntimeRoots(Roots & roots, bool censor);
void removeUnusedLinks(const GCState & state);
Path createTempDirInStore();
void checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv);
@ -267,7 +264,7 @@ private:
InodeHash loadInodeHash();
Strings readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash);
void optimisePath_(Activity * act, OptimiseStats & stats, const Path & path, InodeHash & inodeHash);
void optimisePath_(Activity * act, OptimiseStats & stats, const Path & path, InodeHash & inodeHash, RepairFlag repair);
// Internal versions that are not wrapped in retry_sqlite.
bool isValidPath_(State & state, const StorePath & path);
@ -293,6 +290,8 @@ private:
const std::string_view pathHash
);
void addBuildLog(const StorePath & drvPath, std::string_view log) override;
friend struct LocalDerivationGoal;
friend struct PathSubstitutionGoal;
friend struct SubstitutionGoal;
@ -301,7 +300,7 @@ private:
typedef std::pair<dev_t, ino_t> Inode;
typedef set<Inode> InodesSeen;
typedef std::set<Inode> InodesSeen;
/* "Fix", or canonicalise, the meta-data of the files in a store path

4
src/libstore/lock.hh
View file

@ -13,7 +13,7 @@ private:
AutoCloseFD fdUserLock;
bool isEnabled = false;
string user;
std::string user;
uid_t uid = 0;
gid_t gid = 0;
std::vector<gid_t> supplementaryGIDs;
@ -23,7 +23,7 @@ public:
void kill();
string getUser() { return user; }
std::string getUser() { return user; }
uid_t getUID() { assert(uid); return uid; }
uid_t getGID() { assert(gid); return gid; }
std::vector<gid_t> getSupplementaryGIDs() { return supplementaryGIDs; }

103
src/libstore/machines.cc
View file

@ -39,22 +39,25 @@ Machine::Machine(decltype(storeUri) storeUri,
sshPublicHostKey(sshPublicHostKey)
{}
bool Machine::allSupported(const std::set<string> & features) const {
bool Machine::allSupported(const std::set<std::string> & features) const
{
return std::all_of(features.begin(), features.end(),
[&](const string & feature) {
[&](const std::string & feature) {
return supportedFeatures.count(feature) ||
mandatoryFeatures.count(feature);
});
}
bool Machine::mandatoryMet(const std::set<string> & features) const {
bool Machine::mandatoryMet(const std::set<std::string> & features) const
{
return std::all_of(mandatoryFeatures.begin(), mandatoryFeatures.end(),
[&](const string & feature) {
[&](const std::string & feature) {
return features.count(feature);
});
}
ref<Store> Machine::openStore() const {
ref<Store> Machine::openStore() const
{
Store::Params storeParams;
if (hasPrefix(storeUri, "ssh://")) {
storeParams["max-connections"] = "1";
@ -83,53 +86,87 @@ ref<Store> Machine::openStore() const {
return nix::openStore(storeUri, storeParams);
}
void parseMachines(const std::string & s, Machines & machines)
static std::vector<std::string> expandBuilderLines(const std::string & builders)
{
for (auto line : tokenizeString<std::vector<string>>(s, "\n;")) {
std::vector<std::string> result;
for (auto line : tokenizeString<std::vector<std::string>>(builders, "\n;")) {
trim(line);
line.erase(std::find(line.begin(), line.end(), '#'), line.end());
if (line.empty()) continue;
if (line[0] == '@') {
auto file = trim(std::string(line, 1));
const std::string path = trim(std::string(line, 1));
std::string text;
try {
parseMachines(readFile(file), machines);
text = readFile(path);
} catch (const SysError & e) {
if (e.errNo != ENOENT)
throw;
debug("cannot find machines file '%s'", file);
debug("cannot find machines file '%s'", path);
}
const auto lines = expandBuilderLines(text);
result.insert(end(result), begin(lines), end(lines));
continue;
}
auto tokens = tokenizeString<std::vector<string>>(line);
auto sz = tokens.size();
if (sz < 1)
throw FormatError("bad machine specification '%s'", line);
auto isSet = [&](size_t n) {
return tokens.size() > n && tokens[n] != "" && tokens[n] != "-";
};
machines.emplace_back(tokens[0],
isSet(1) ? tokenizeString<std::vector<string>>(tokens[1], ",") : std::vector<string>{settings.thisSystem},
isSet(2) ? tokens[2] : "",
isSet(3) ? std::stoull(tokens[3]) : 1LL,
isSet(4) ? std::stoull(tokens[4]) : 1LL,
isSet(5) ? tokenizeString<std::set<string>>(tokens[5], ",") : std::set<string>{},
isSet(6) ? tokenizeString<std::set<string>>(tokens[6], ",") : std::set<string>{},
isSet(7) ? tokens[7] : "");
result.emplace_back(line);
}
return result;
}
static Machine parseBuilderLine(const std::string & line)
{
const auto tokens = tokenizeString<std::vector<std::string>>(line);
auto isSet = [&](size_t fieldIndex) {
return tokens.size() > fieldIndex && tokens[fieldIndex] != "" && tokens[fieldIndex] != "-";
};
auto parseUnsignedIntField = [&](size_t fieldIndex) {
const auto result = string2Int<unsigned int>(tokens[fieldIndex]);
if (!result) {
throw FormatError("bad machine specification: failed to convert column #%lu in a row: '%s' to 'unsigned int'", fieldIndex, line);
}
return result.value();
};
auto ensureBase64 = [&](size_t fieldIndex) {
const auto & str = tokens[fieldIndex];
try {
base64Decode(str);
} catch (const Error & e) {
throw FormatError("bad machine specification: a column #%lu in a row: '%s' is not valid base64 string: %s", fieldIndex, line, e.what());
}
return str;
};
if (!isSet(0))
throw FormatError("bad machine specification: store URL was not found at the first column of a row: '%s'", line);
return {
tokens[0],
isSet(1) ? tokenizeString<std::vector<std::string>>(tokens[1], ",") : std::vector<std::string>{settings.thisSystem},
isSet(2) ? tokens[2] : "",
isSet(3) ? parseUnsignedIntField(3) : 1U,
isSet(4) ? parseUnsignedIntField(4) : 1U,
isSet(5) ? tokenizeString<std::set<std::string>>(tokens[5], ",") : std::set<std::string>{},
isSet(6) ? tokenizeString<std::set<std::string>>(tokens[6], ",") : std::set<std::string>{},
isSet(7) ? ensureBase64(7) : ""
};
}
static Machines parseBuilderLines(const std::vector<std::string> & builders)
{
Machines result;
std::transform(builders.begin(), builders.end(), std::back_inserter(result), parseBuilderLine);
return result;
}
Machines getMachines()
{
static auto machines = [&]() {
Machines machines;
parseMachines(settings.builders, machines);
return machines;
}();
return machines;
const auto builderLines = expandBuilderLines(settings.builders);
return parseBuilderLines(builderLines);
}
}

14
src/libstore/machines.hh
View file

@ -8,19 +8,19 @@ class Store;
struct Machine {
const string storeUri;
const std::vector<string> systemTypes;
const string sshKey;
const std::string storeUri;
const std::vector<std::string> systemTypes;
const std::string sshKey;
const unsigned int maxJobs;
const unsigned int speedFactor;
const std::set<string> supportedFeatures;
const std::set<string> mandatoryFeatures;
const std::set<std::string> supportedFeatures;
const std::set<std::string> mandatoryFeatures;
const std::string sshPublicHostKey;
bool enabled = true;
bool allSupported(const std::set<string> & features) const;
bool allSupported(const std::set<std::string> & features) const;
bool mandatoryMet(const std::set<string> & features) const;
bool mandatoryMet(const std::set<std::string> & features) const;
Machine(decltype(storeUri) storeUri,
decltype(systemTypes) systemTypes,

9
src/libstore/misc.cc
View file

@ -7,6 +7,7 @@
#include "topo-sort.hh"
#include "callback.hh"
#include "closure.hh"
#include "filetransfer.hh"
namespace nix {
@ -111,7 +112,8 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
downloadSize_ = narSize_ = 0;
ThreadPool pool;
// FIXME: make async.
ThreadPool pool(fileTransferSettings.httpConnections);
struct State
{
@ -250,12 +252,11 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths)
{
return topoSort(paths,
{[&](const StorePath & path) {
StorePathSet references;
try {
references = queryPathInfo(path)->references;
return queryPathInfo(path)->references;
} catch (InvalidPath &) {
return StorePathSet();
}
return references;
}},
{[&](const StorePath & path, const StorePath & parent) {
return BuildError(

26
src/libstore/names.cc
View file

@ -42,7 +42,7 @@ DrvName::~DrvName()
{ }
bool DrvName::matches(DrvName & n)
bool DrvName::matches(const DrvName & n)
{
if (name != "*") {
if (!regex) {
@ -56,8 +56,8 @@ bool DrvName::matches(DrvName & n)
}
string nextComponent(string::const_iterator & p,
const string::const_iterator end)
std::string_view nextComponent(std::string_view::const_iterator & p,
const std::string_view::const_iterator end)
{
/* Skip any dots and dashes (component separators). */
while (p != end && (*p == '.' || *p == '-')) ++p;
@ -67,18 +67,18 @@ string nextComponent(string::const_iterator & p,
/* If the first character is a digit, consume the longest sequence
of digits. Otherwise, consume the longest sequence of
non-digit, non-separator characters. */
string s;
auto s = p;
if (isdigit(*p))
while (p != end && isdigit(*p)) s += *p++;
while (p != end && isdigit(*p)) p++;
else
while (p != end && (!isdigit(*p) && *p != '.' && *p != '-'))
s += *p++;
p++;
return s;
return {s, size_t(p - s)};
}
static bool componentsLT(const string & c1, const string & c2)
static bool componentsLT(const std::string_view c1, const std::string_view c2)
{
auto n1 = string2Int<int>(c1);
auto n2 = string2Int<int>(c2);
@ -94,14 +94,14 @@ static bool componentsLT(const string & c1, const string & c2)
}
int compareVersions(const string & v1, const string & v2)
int compareVersions(const std::string_view v1, const std::string_view v2)
{
string::const_iterator p1 = v1.begin();
string::const_iterator p2 = v2.begin();
auto p1 = v1.begin();
auto p2 = v2.begin();
while (p1 != v1.end() || p2 != v2.end()) {
string c1 = nextComponent(p1, v1.end());
string c2 = nextComponent(p2, v2.end());
auto c1 = nextComponent(p1, v1.end());
auto c2 = nextComponent(p2, v2.end());
if (componentsLT(c1, c2)) return -1;
else if (componentsLT(c2, c1)) return 1;
}

16
src/libstore/names.hh
View file

@ -10,26 +10,26 @@ struct Regex;
struct DrvName
{
string fullName;
string name;
string version;
std::string fullName;
std::string name;
std::string version;
unsigned int hits;
DrvName();
DrvName(std::string_view s);
~DrvName();
bool matches(DrvName & n);
bool matches(const DrvName & n);
private:
std::unique_ptr<Regex> regex;
};
typedef list<DrvName> DrvNames;
typedef std::list<DrvName> DrvNames;
string nextComponent(string::const_iterator & p,
const string::const_iterator end);
int compareVersions(const string & v1, const string & v2);
std::string_view nextComponent(std::string_view::const_iterator & p,
const std::string_view::const_iterator end);
int compareVersions(const std::string_view v1, const std::string_view v2);
DrvNames drvNamesFromArgs(const Strings & opArgs);
}

10
src/libstore/nar-accessor.cc
View file

@ -28,7 +28,7 @@ struct NarMember
struct NarAccessor : public FSAccessor
{
std::shared_ptr<const std::string> nar;
std::optional<const std::string> nar;
GetNarBytes getNarBytes;
@ -90,7 +90,7 @@ struct NarAccessor : public FSAccessor
void receiveContents(std::string_view data) override
{ }
void createSymlink(const Path & path, const string & target) override
void createSymlink(const Path & path, const std::string & target) override
{
createMember(path,
NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
@ -104,7 +104,7 @@ struct NarAccessor : public FSAccessor
}
};
NarAccessor(ref<const std::string> nar) : nar(nar)
NarAccessor(std::string && _nar) : nar(_nar)
{
StringSource source(*nar);
NarIndexer indexer(*this, source);
@ -224,9 +224,9 @@ struct NarAccessor : public FSAccessor
}
};
ref<FSAccessor> makeNarAccessor(ref<const std::string> nar)
ref<FSAccessor> makeNarAccessor(std::string && nar)
{
return make_ref<NarAccessor>(nar);
return make_ref<NarAccessor>(std::move(nar));
}
ref<FSAccessor> makeNarAccessor(Source & source)

2
src/libstore/nar-accessor.hh
View file

@ -10,7 +10,7 @@ struct Source;
/* Return an object that provides access to the contents of a NAR
file. */
ref<FSAccessor> makeNarAccessor(ref<const std::string> nar);
ref<FSAccessor> makeNarAccessor(std::string && nar);
ref<FSAccessor> makeNarAccessor(Source & source);

2
src/libstore/nar-info.cc
View file

@ -11,7 +11,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
return Error("NAR info file '%1%' is corrupt", whence);
};
auto parseHashField = [&](const string & s) {
auto parseHashField = [&](const std::string & s) {
try {
return Hash::parseAnyPrefixed(s);
} catch (BadHash &) {

42
src/libstore/optimise-store.cc
View file

@ -26,7 +26,7 @@ static void makeWritable(const Path & path)
struct MakeReadOnly
{
Path path;
MakeReadOnly(const Path & path) : path(path) { }
MakeReadOnly(const PathView path) : path(path) { }
~MakeReadOnly()
{
try {
@ -77,7 +77,7 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa
continue;
}
string name = dirent->d_name;
std::string name = dirent->d_name;
if (name == "." || name == "..") continue;
names.push_back(name);
}
@ -88,7 +88,7 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa
void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
const Path & path, InodeHash & inodeHash)
const Path & path, InodeHash & inodeHash, RepairFlag repair)
{
checkInterrupt();
@ -110,7 +110,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
if (S_ISDIR(st.st_mode)) {
Strings names = readDirectoryIgnoringInodes(path, inodeHash);
for (auto & i : names)
optimisePath_(act, stats, path + "/" + i, inodeHash);
optimisePath_(act, stats, path + "/" + i, inodeHash, repair);
return;
}
@ -151,7 +151,20 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
/* Check if this is a known hash. */
Path linkPath = linksDir + "/" + hash.to_string(Base32, false);
retry:
/* Maybe delete the link, if it has been corrupted. */
if (pathExists(linkPath)) {
auto stLink = lstat(linkPath);
if (st.st_size != stLink.st_size
|| (repair && hash != hashPath(htSHA256, linkPath).first))
{
// XXX: Consider overwriting linkPath with our valid version.
warn("removing corrupted link '%s'", linkPath);
warn("There may be more corrupted paths."
"\nYou should run `nix-store --verify --check-contents --repair` to fix them all");
unlink(linkPath.c_str());
}
}
if (!pathExists(linkPath)) {
/* Nope, create a hard link in the links directory. */
if (link(path.c_str(), linkPath.c_str()) == 0) {
@ -187,23 +200,18 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
return;
}
if (st.st_size != stLink.st_size) {
warn("removing corrupted link '%s'", linkPath);
unlink(linkPath.c_str());
goto retry;
}
printMsg(lvlTalkative, format("linking '%1%' to '%2%'") % path % linkPath);
/* Make the containing directory writable, but only if it's not
the store itself (we don't want or need to mess with its
permissions). */
bool mustToggle = dirOf(path) != realStoreDir.get();
if (mustToggle) makeWritable(dirOf(path));
const Path dirOfPath(dirOf(path));
bool mustToggle = dirOfPath != realStoreDir.get();
if (mustToggle) makeWritable(dirOfPath);
/* When we're done, make the directory read-only again and reset
its timestamp back to 0. */
MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : "");
MakeReadOnly makeReadOnly(mustToggle ? dirOfPath : "");
Path tempLink = (format("%1%/.tmp-link-%2%-%3%")
% realStoreDir % getpid() % random()).str();
@ -260,7 +268,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
if (!isValidPath(i)) continue; /* path was GC'ed, probably */
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("optimising path '%s'", printStorePath(i)));
optimisePath_(&act, stats, realStoreDir + "/" + std::string(i.to_string()), inodeHash);
optimisePath_(&act, stats, realStoreDir + "/" + std::string(i.to_string()), inodeHash, NoRepair);
}
done++;
act.progress(done, paths.size());
@ -278,12 +286,12 @@ void LocalStore::optimiseStore()
stats.filesLinked);
}
void LocalStore::optimisePath(const Path & path)
void LocalStore::optimisePath(const Path & path, RepairFlag repair)
{
OptimiseStats stats;
InodeHash inodeHash;
if (settings.autoOptimiseStore) optimisePath_(nullptr, stats, path, inodeHash);
if (settings.autoOptimiseStore) optimisePath_(nullptr, stats, path, inodeHash, repair);
}

2
src/libstore/parsed-derivations.cc
View file

@ -170,7 +170,7 @@ std::string writeStructuredAttrsShell(const nlohmann::json & json)
auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> {
if (value.is_string())
return shellEscape(value);
return shellEscape(value.get<std::string_view>());
if (value.is_number()) {
auto f = value.get<float>();

10
src/libstore/path-with-outputs.cc
View file

@ -22,9 +22,9 @@ DerivedPath StorePathWithOutputs::toDerivedPath() const
std::vector<DerivedPath> toDerivedPaths(const std::vector<StorePathWithOutputs> ss)
{
std::vector<DerivedPath> reqs;
for (auto & s : ss) reqs.push_back(s.toDerivedPath());
return reqs;
std::vector<DerivedPath> reqs;
for (auto & s : ss) reqs.push_back(s.toDerivedPath());
return reqs;
}
@ -49,9 +49,9 @@ std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s)
{
size_t n = s.find("!");
return n == s.npos
? std::make_pair(s, std::set<string>())
? std::make_pair(s, std::set<std::string>())
: std::make_pair(((std::string_view) s).substr(0, n),
tokenizeString<std::set<string>>(((std::string_view) s).substr(n + 1), ","));
tokenizeString<std::set<std::string>>(((std::string_view) s).substr(n + 1), ","));
}

2
src/libstore/path.hh
View file

@ -63,7 +63,7 @@ public:
typedef std::set<StorePath> StorePathSet;
typedef std::vector<StorePath> StorePaths;
typedef std::map<string, StorePath> OutputPathMap;
typedef std::map<std::string, StorePath> OutputPathMap;
/* Extension of derivations in the Nix store. */
const std::string drvExtension = ".drv";

17
src/libstore/pathlocks.cc
View file

@ -74,7 +74,7 @@ PathLocks::PathLocks()
}
PathLocks::PathLocks(const PathSet & paths, const string & waitMsg)
PathLocks::PathLocks(const PathSet & paths, const std::string & waitMsg)
: deletePaths(false)
{
lockPaths(paths, waitMsg);
@ -82,7 +82,7 @@ PathLocks::PathLocks(const PathSet & paths, const string & waitMsg)
bool PathLocks::lockPaths(const PathSet & paths,
const string & waitMsg, bool wait)
const std::string & waitMsg, bool wait)
{
assert(fds.empty());
@ -176,4 +176,17 @@ void PathLocks::setDeletion(bool deletePaths)
}
FdLock::FdLock(int fd, LockType lockType, bool wait, std::string_view waitMsg)
: fd(fd)
{
if (wait) {
if (!lockFile(fd, lockType, false)) {
printInfo("%s", waitMsg);
acquired = lockFile(fd, lockType, true);
}
} else
acquired = lockFile(fd, lockType, false);
}
}

20
src/libstore/pathlocks.hh
View file

@ -20,19 +20,33 @@ class PathLocks
{
private:
typedef std::pair<int, Path> FDPair;
list<FDPair> fds;
std::list<FDPair> fds;
bool deletePaths;
public:
PathLocks();
PathLocks(const PathSet & paths,
const string & waitMsg = "");
const std::string & waitMsg = "");
bool lockPaths(const PathSet & _paths,
const string & waitMsg = "",
const std::string & waitMsg = "",
bool wait = true);
~PathLocks();
void unlock();
void setDeletion(bool deletePaths);
};
struct FdLock
{
int fd;
bool acquired = false;
FdLock(int fd, LockType lockType, bool wait, std::string_view waitMsg);
~FdLock()
{
if (acquired)
lockFile(fd, ltNone, false);
}
};
}

16
src/libstore/profiles.cc
View file

@ -15,12 +15,12 @@ namespace nix {
/* Parse a generation name of the format
`<profilename>-<number>-link'. */
static std::optional<GenerationNumber> parseName(const string & profileName, const string & name)
static std::optional<GenerationNumber> parseName(const std::string & profileName, const std::string & name)
{
if (string(name, 0, profileName.size() + 1) != profileName + "-") return {};
string s = string(name, profileName.size() + 1);
string::size_type p = s.find("-link");
if (p == string::npos) return {};
if (name.substr(0, profileName.size() + 1) != profileName + "-") return {};
auto s = name.substr(profileName.size() + 1);
auto p = s.find("-link");
if (p == std::string::npos) return {};
if (auto n = string2Int<unsigned int>(s.substr(0, p)))
return *n;
else
@ -209,13 +209,13 @@ void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun)
}
void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, bool dryRun)
void deleteGenerationsOlderThan(const Path & profile, std::string_view timeSpec, bool dryRun)
{
if (timeSpec.empty() || timeSpec[timeSpec.size() - 1] != 'd')
throw UsageError("invalid number of days specifier '%1%', expected something like '14d'", timeSpec);
time_t curTime = time(0);
string strDays = string(timeSpec, 0, timeSpec.size() - 1);
auto strDays = timeSpec.substr(0, timeSpec.size() - 1);
auto days = string2Int<int>(strDays);
if (!days || *days < 1)
@ -274,7 +274,7 @@ void lockProfile(PathLocks & lock, const Path & profile)
}
string optimisticLockProfile(const Path & profile)
std::string optimisticLockProfile(const Path & profile)
{
return pathExists(profile) ? readLink(profile) : "";
}

4
src/libstore/profiles.hh
View file

@ -42,7 +42,7 @@ void deleteOldGenerations(const Path & profile, bool dryRun);
void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun);
void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, bool dryRun);
void deleteGenerationsOlderThan(const Path & profile, std::string_view timeSpec, bool dryRun);
void switchLink(Path link, Path target);
@ -66,7 +66,7 @@ void lockProfile(PathLocks & lock, const Path & profile);
generally cheap, since the build results are still in the Nix
store. Most of the time, only the user environment has to be
rebuilt. */
string optimisticLockProfile(const Path & profile);
std::string optimisticLockProfile(const Path & profile);
/* Resolve ~/.nix-profile. If ~/.nix-profile doesn't exist yet, create
it. */

2
src/libstore/realisation.cc
View file

@ -78,7 +78,7 @@ Realisation Realisation::fromJSON(
auto fieldIterator = json.find(fieldName);
if (fieldIterator == json.end())
return std::nullopt;
return *fieldIterator;
return {*fieldIterator};
};
auto getField = [&](std::string fieldName) -> std::string {
if (auto field = getOptionalField(fieldName))

8
src/libstore/references.cc
View file

@ -54,12 +54,12 @@ void RefScanSink::operator () (std::string_view data)
fragment, so search in the concatenation of the tail of the
previous fragment and the start of the current fragment. */
auto s = tail;
s.append(data.data(), refLength);
auto tailLen = std::min(data.size(), refLength);
s.append(data.data(), tailLen);
search(s, hashes, seen);
search(data, hashes, seen);
auto tailLen = std::min(data.size(), refLength);
auto rest = refLength - tailLen;
if (rest < tail.size())
tail = tail.substr(tail.size() - rest);
@ -68,7 +68,7 @@ void RefScanSink::operator () (std::string_view data)
std::pair<StorePathSet, HashResult> scanForReferences(
const string & path,
const std::string & path,
const StorePathSet & refs)
{
HashSink hashSink { htSHA256 };
@ -121,7 +121,7 @@ void RewritingSink::operator () (std::string_view data)
s.append(data);
size_t j = 0;
while ((j = s.find(from, j)) != string::npos) {
while ((j = s.find(from, j)) != std::string::npos) {
matches.push_back(pos + j);
s.replace(j, from.size(), to);
}

30
src/libstore/remote-fs-accessor.cc
View file

@ -22,9 +22,18 @@ Path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::strin
return fmt("%s/%s.%s", cacheDir, hashPart, ext);
}
void RemoteFSAccessor::addToCache(std::string_view hashPart, const std::string & nar,
ref<FSAccessor> narAccessor)
ref<FSAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std::string && nar)
{
if (cacheDir != "") {
try {
/* FIXME: do this asynchronously. */
writeFile(makeCacheFile(hashPart, "nar"), nar);
} catch (...) {
ignoreException();
}
}
auto narAccessor = makeNarAccessor(std::move(nar));
nars.emplace(hashPart, narAccessor);
if (cacheDir != "") {
@ -33,14 +42,12 @@ void RemoteFSAccessor::addToCache(std::string_view hashPart, const std::string &
JSONPlaceholder jsonRoot(str);
listNar(jsonRoot, narAccessor, "", true);
writeFile(makeCacheFile(hashPart, "ls"), str.str());
/* FIXME: do this asynchronously. */
writeFile(makeCacheFile(hashPart, "nar"), nar);
} catch (...) {
ignoreException();
}
}
return narAccessor;
}
std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_, bool requireValidPath)
@ -55,7 +62,6 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_, boo
auto i = nars.find(std::string(storePath.hashPart()));
if (i != nars.end()) return {i->second, restPath};
StringSink sink;
std::string listing;
Path cacheFile;
@ -86,19 +92,15 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_, boo
} catch (SysError &) { }
try {
*sink.s = nix::readFile(cacheFile);
auto narAccessor = makeNarAccessor(sink.s);
auto narAccessor = makeNarAccessor(nix::readFile(cacheFile));
nars.emplace(storePath.hashPart(), narAccessor);
return {narAccessor, restPath};
} catch (SysError &) { }
}
StringSink sink;
store->narFromPath(storePath, sink);
auto narAccessor = makeNarAccessor(sink.s);
addToCache(storePath.hashPart(), *sink.s, narAccessor);
return {narAccessor, restPath};
return {addToCache(storePath.hashPart(), std::move(sink.s)), restPath};
}
FSAccessor::Stat RemoteFSAccessor::stat(const Path & path)

3
src/libstore/remote-fs-accessor.hh
View file

@ -20,8 +20,7 @@ class RemoteFSAccessor : public FSAccessor
Path makeCacheFile(std::string_view hashPart, const std::string & ext);
void addToCache(std::string_view hashPart, const std::string & nar,
ref<FSAccessor> narAccessor);
ref<FSAccessor> addToCache(std::string_view hashPart, std::string && nar);
public:

238
src/libstore/remote-store.cc
View file

@ -1,11 +1,12 @@
#include "serialise.hh"
#include "util.hh"
#include "path-with-outputs.hh"
#include "gc-store.hh"
#include "remote-fs-accessor.hh"
#include "build-result.hh"
#include "remote-store.hh"
#include "worker-protocol.hh"
#include "archive.hh"
#include "affinity.hh"
#include "globals.hh"
#include "derivations.hh"
#include "pool.hh"
@ -90,6 +91,35 @@ void write(const Store & store, Sink & out, const DrvOutput & drvOutput)
}
BuildResult read(const Store & store, Source & from, Phantom<BuildResult> _)
{
auto path = worker_proto::read(store, from, Phantom<DerivedPath> {});
BuildResult res { .path = path };
res.status = (BuildResult::Status) readInt(from);
from
>> res.errorMsg
>> res.timesBuilt
>> res.isNonDeterministic
>> res.startTime
>> res.stopTime;
res.builtOutputs = worker_proto::read(store, from, Phantom<DrvOutputs> {});
return res;
}
void write(const Store & store, Sink & to, const BuildResult & res)
{
worker_proto::write(store, to, res.path);
to
<< res.status
<< res.errorMsg
<< res.timesBuilt
<< res.isNonDeterministic
<< res.startTime
<< res.stopTime;
worker_proto::write(store, to, res.builtOutputs);
}
std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _)
{
auto s = readString(from);
@ -173,7 +203,7 @@ void RemoteStore::initConnection(Connection & conn)
it. */
conn.closeWrite();
auto msg = conn.from.drain();
throw Error("protocol mismatch, got '%s'", chomp(*saved.s + msg));
throw Error("protocol mismatch, got '%s'", chomp(saved.s + msg));
}
conn.from >> conn.daemonVersion;
@ -184,15 +214,17 @@ void RemoteStore::initConnection(Connection & conn)
conn.to << PROTOCOL_VERSION;
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 14) {
int cpu = sameMachine() && settings.lockCPU ? lockToCurrentCPU() : -1;
if (cpu != -1)
conn.to << 1 << cpu;
else
conn.to << 0;
// Obsolete CPU affinity.
conn.to << 0;
}
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11)
conn.to << false;
conn.to << false; // obsolete reserveSpace
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 33) {
conn.to.flush();
conn.daemonNixVersion = readString(conn.from);
}
auto ex = conn.processStderr();
if (ex) std::rethrow_exception(ex);
@ -290,6 +322,10 @@ ConnectionHandle RemoteStore::getConnection()
return ConnectionHandle(connections->get());
}
void RemoteStore::setOptions()
{
setOptions(*(getConnection().handle));
}
bool RemoteStore::isValidPathUncached(const StorePath & path)
{
@ -496,7 +532,7 @@ std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string &
ref<const ValidPathInfo> RemoteStore::addCAToStore(
Source & dump,
const string & name,
std::string_view name,
ContentAddressMethod caMethod,
HashType hashType,
const StorePathSet & references,
@ -582,10 +618,9 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
}
StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method, HashType hashType, RepairFlag repair)
StorePath RemoteStore::addToStoreFromDump(Source & dump, std::string_view name,
FileIngestionMethod method, HashType hashType, RepairFlag repair, const StorePathSet & references)
{
StorePathSet references;
return addCAToStore(dump, name, method, hashType, references, repair)->path;
}
@ -662,8 +697,11 @@ void RemoteStore::addMultipleToStore(
}
StorePath RemoteStore::addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair)
StorePath RemoteStore::addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references,
RepairFlag repair)
{
StringSource source(s);
return addCAToStore(source, name, TextHashMethod {}, htSHA256, references, repair)->path;
@ -682,23 +720,41 @@ void RemoteStore::registerDrvOutput(const Realisation & info)
conn.processStderr();
}
std::optional<const Realisation> RemoteStore::queryRealisation(const DrvOutput & id)
void RemoteStore::queryRealisationUncached(const DrvOutput & id,
Callback<std::shared_ptr<const Realisation>> callback) noexcept
{
auto conn(getConnection());
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 27) {
warn("the daemon is too old to support content-addressed derivations, please upgrade it to 2.4");
try {
callback(nullptr);
} catch (...) { return callback.rethrow(); }
}
conn->to << wopQueryRealisation;
conn->to << id.to_string();
conn.processStderr();
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) {
auto outPaths = worker_proto::read(*this, conn->from, Phantom<std::set<StorePath>>{});
if (outPaths.empty())
return std::nullopt;
return {Realisation{.id = id, .outPath = *outPaths.begin()}};
} else {
auto realisations = worker_proto::read(*this, conn->from, Phantom<std::set<Realisation>>{});
if (realisations.empty())
return std::nullopt;
return *realisations.begin();
}
auto real = [&]() -> std::shared_ptr<const Realisation> {
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) {
auto outPaths = worker_proto::read(
*this, conn->from, Phantom<std::set<StorePath>> {});
if (outPaths.empty())
return nullptr;
return std::make_shared<const Realisation>(Realisation { .id = id, .outPath = *outPaths.begin() });
} else {
auto realisations = worker_proto::read(
*this, conn->from, Phantom<std::set<Realisation>> {});
if (realisations.empty())
return nullptr;
return std::make_shared<const Realisation>(*realisations.begin());
}
}();
try {
callback(std::shared_ptr<const Realisation>(real));
} catch (...) { return callback.rethrow(); }
}
static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector<DerivedPath> & reqs)
@ -725,17 +781,24 @@ static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, cons
}
}
void RemoteStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore)
void RemoteStore::copyDrvsFromEvalStore(
const std::vector<DerivedPath> & paths,
std::shared_ptr<Store> evalStore)
{
if (evalStore && evalStore.get() != this) {
/* The remote doesn't have a way to access evalStore, so copy
the .drvs. */
RealisedPath::Set drvPaths2;
for (auto & i : drvPaths)
for (auto & i : paths)
if (auto p = std::get_if<DerivedPath::Built>(&i))
drvPaths2.insert(p->drvPath);
copyClosure(*evalStore, *this, drvPaths2);
}
}
void RemoteStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore)
{
copyDrvsFromEvalStore(drvPaths, evalStore);
auto conn(getConnection());
conn->to << wopBuildPaths;
@ -752,6 +815,91 @@ void RemoteStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMod
readInt(conn->from);
}
std::vector<BuildResult> RemoteStore::buildPathsWithResults(
const std::vector<DerivedPath> & paths,
BuildMode buildMode,
std::shared_ptr<Store> evalStore)
{
copyDrvsFromEvalStore(paths, evalStore);
std::optional<ConnectionHandle> conn_(getConnection());
auto & conn = *conn_;
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 34) {
conn->to << wopBuildPathsWithResults;
writeDerivedPaths(*this, conn, paths);
conn->to << buildMode;
conn.processStderr();
return worker_proto::read(*this, conn->from, Phantom<std::vector<BuildResult>> {});
} else {
// Avoid deadlock.
conn_.reset();
// Note: this throws an exception if a build/substitution
// fails, but meh.
buildPaths(paths, buildMode, evalStore);
std::vector<BuildResult> results;
for (auto & path : paths) {
std::visit(
overloaded {
[&](const DerivedPath::Opaque & bo) {
results.push_back(BuildResult {
.status = BuildResult::Substituted,
.path = bo,
});
},
[&](const DerivedPath::Built & bfd) {
BuildResult res {
.status = BuildResult::Built,
.path = bfd,
};
OutputPathMap outputs;
auto drv = evalStore->readDerivation(bfd.drvPath);
auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
auto drvOutputs = drv.outputsAndOptPaths(*this);
for (auto & output : bfd.outputs) {
if (!outputHashes.count(output))
throw Error(
"the derivation '%s' doesn't have an output named '%s'",
printStorePath(bfd.drvPath), output);
auto outputId =
DrvOutput{outputHashes.at(output), output};
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
auto realisation =
queryRealisation(outputId);
if (!realisation)
throw Error(
"cannot operate on an output of unbuilt "
"content-addressed derivation '%s'",
outputId.to_string());
res.builtOutputs.emplace(realisation->id, *realisation);
} else {
// If ca-derivations isn't enabled, assume that
// the output path is statically known.
assert(drvOutputs.count(output));
assert(drvOutputs.at(output).second);
res.builtOutputs.emplace(
outputId,
Realisation {
.id = outputId,
.outPath = *drvOutputs.at(output).second
});
}
}
results.push_back(res);
}
},
path.raw());
}
return results;
}
}
BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode)
@ -761,7 +909,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD
writeDerivation(conn->to, *this, drv);
conn->to << buildMode;
conn.processStderr();
BuildResult res;
BuildResult res { .path = DerivedPath::Built { .drvPath = drvPath } };
res.status = (BuildResult::Status) readInt(conn->from);
conn->from >> res.errorMsg;
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) {
@ -802,15 +950,6 @@ void RemoteStore::addIndirectRoot(const Path & path)
}
void RemoteStore::syncWithGC()
{
auto conn(getConnection());
conn->to << wopSyncWithGC;
conn.processStderr();
readInt(conn->from);
}
Roots RemoteStore::findRoots(bool censor)
{
auto conn(getConnection());
@ -905,6 +1044,25 @@ void RemoteStore::queryMissing(const std::vector<DerivedPath> & targets,
}
void RemoteStore::addBuildLog(const StorePath & drvPath, std::string_view log)
{
auto conn(getConnection());
conn->to << wopAddBuildLog << drvPath.to_string();
StringSource source(log);
conn.withFramedSink([&](Sink & sink) {
source.drainInto(sink);
});
readInt(conn->from);
}
std::optional<std::string> RemoteStore::getVersion()
{
auto conn(getConnection());
return conn->daemonNixVersion;
}
void RemoteStore::connect()
{
auto conn(getConnection());
@ -973,7 +1131,7 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
auto msg = readNum<uint64_t>(from);
if (msg == STDERR_WRITE) {
string s = readString(from);
auto s = readString(from);
if (!sink) throw Error("no sink");
(*sink)(s);
}
@ -990,7 +1148,7 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
if (GET_PROTOCOL_MINOR(daemonVersion) >= 26) {
return std::make_exception_ptr(readError(from));
} else {
string error = readString(from);
auto error = readString(from);
unsigned int status = readInt(from);
return std::make_exception_ptr(Error(status, error));
}

36
src/libstore/remote-store.hh
View file

@ -4,6 +4,7 @@
#include <string>
#include "store-api.hh"
#include "gc-store.hh"
namespace nix {
@ -29,7 +30,7 @@ struct RemoteStoreConfig : virtual StoreConfig
/* FIXME: RemoteStore is a misnomer - should be something like
DaemonStore. */
class RemoteStore : public virtual RemoteStoreConfig, public virtual Store
class RemoteStore : public virtual RemoteStoreConfig, public virtual Store, public virtual GcStore
{
public:
@ -66,15 +67,15 @@ public:
/* Add a content-addressable store path. `dump` will be drained. */
ref<const ValidPathInfo> addCAToStore(
Source & dump,
const string & name,
std::string_view name,
ContentAddressMethod caMethod,
HashType hashType,
const StorePathSet & references,
RepairFlag repair);
/* Add a content-addressable store path. Does not support references. `dump` will be drained. */
StorePath addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
StorePath addToStoreFromDump(Source & dump, std::string_view name,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override;
void addToStore(const ValidPathInfo & info, Source & nar,
RepairFlag repair, CheckSigsFlag checkSigs) override;
@ -84,15 +85,24 @@ public:
RepairFlag repair,
CheckSigsFlag checkSigs) override;
StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override;
StorePath addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references,
RepairFlag repair) override;
void registerDrvOutput(const Realisation & info) override;
std::optional<const Realisation> queryRealisation(const DrvOutput &) override;
void queryRealisationUncached(const DrvOutput &,
Callback<std::shared_ptr<const Realisation>> callback) noexcept override;
void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override;
std::vector<BuildResult> buildPathsWithResults(
const std::vector<DerivedPath> & paths,
BuildMode buildMode,
std::shared_ptr<Store> evalStore) override;
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode) override;
@ -102,8 +112,6 @@ public:
void addIndirectRoot(const Path & path) override;
void syncWithGC() override;
Roots findRoots(bool censor) override;
void collectGarbage(const GCOptions & options, GCResults & results) override;
@ -118,6 +126,10 @@ public:
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
uint64_t & downloadSize, uint64_t & narSize) override;
void addBuildLog(const StorePath & drvPath, std::string_view log) override;
std::optional<std::string> getVersion() override;
void connect() override;
unsigned int getProtocol() override;
@ -129,6 +141,7 @@ public:
FdSink to;
FdSource from;
unsigned int daemonVersion;
std::optional<std::string> daemonNixVersion;
std::chrono::time_point<std::chrono::steady_clock> startTime;
virtual ~Connection();
@ -150,6 +163,8 @@ protected:
virtual void setOptions(Connection & conn);
void setOptions() override;
ConnectionHandle getConnection();
friend struct ConnectionHandle;
@ -162,6 +177,9 @@ private:
std::atomic_bool failed{false};
void copyDrvsFromEvalStore(
const std::vector<DerivedPath> & paths,
std::shared_ptr<Store> evalStore);
};

13
src/libstore/s3-binary-cache-store.cc
View file

@ -87,7 +87,11 @@ static void initAWS()
});
}
S3Helper::S3Helper(const string & profile, const string & region, const string & scheme, const string & endpoint)
S3Helper::S3Helper(
const std::string & profile,
const std::string & region,
const std::string & scheme,
const std::string & endpoint)
: config(makeConfig(region, scheme, endpoint))
, client(make_ref<Aws::S3::S3Client>(
profile == ""
@ -121,7 +125,10 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy
}
};
ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region, const string & scheme, const string & endpoint)
ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(
const std::string & region,
const std::string & scheme,
const std::string & endpoint)
{
initAWS();
auto res = make_ref<Aws::Client::ClientConfiguration>();
@ -385,7 +392,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
auto compress = [&](std::string compression)
{
auto compressed = nix::compress(compression, StreamToSourceAdapter(istream).drain());
return std::make_shared<std::stringstream>(std::move(*compressed));
return std::make_shared<std::stringstream>(std::move(compressed));
};
if (narinfoCompression != "" && hasSuffix(path, ".narinfo"))

4
src/libstore/s3.hh
View file

@ -4,6 +4,8 @@
#include "ref.hh"
#include <optional>
namespace Aws { namespace Client { class ClientConfiguration; } }
namespace Aws { namespace S3 { class S3Client; } }
@ -20,7 +22,7 @@ struct S3Helper
struct FileTransferResult
{
std::shared_ptr<std::string> data;
std::optional<std::string> data;
unsigned int durationMs;
};

3
src/libstore/sandbox-defaults.sb
View file

@ -100,4 +100,5 @@
; Allow Rosetta 2 to run x86_64 binaries on aarch64-darwin.
(allow file-read*
(subpath "/Library/Apple/usr/libexec/oah"))
(subpath "/Library/Apple/usr/libexec/oah")
(subpath "/System/Library/Apple/usr/libexec/oah"))

9
src/libstore/sqlite.cc
View file

@ -1,4 +1,5 @@
#include "sqlite.hh"
#include "globals.hh"
#include "util.hh"
#include <sqlite3.h>
@ -27,8 +28,12 @@ namespace nix {
SQLite::SQLite(const Path & path, bool create)
{
// useSQLiteWAL also indicates what virtual file system we need. Using
// `unix-dotfile` is needed on NFS file systems and on Windows' Subsystem
// for Linux (WSL) where useSQLiteWAL should be false by default.
const char *vfs = settings.useSQLiteWAL ? 0 : "unix-dotfile";
if (sqlite3_open_v2(path.c_str(), &db,
SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), vfs) != SQLITE_OK)
throw Error("cannot open SQLite database '%s'", path);
if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
@ -66,7 +71,7 @@ uint64_t SQLite::getLastInsertedRowId()
return sqlite3_last_insert_rowid(db);
}
void SQLiteStmt::create(sqlite3 * db, const string & sql)
void SQLiteStmt::create(sqlite3 * db, const std::string & sql)
{
checkInterrupt();
assert(!stmt);

2
src/libstore/ssh.cc
View file

@ -29,7 +29,7 @@ void SSHMaster::addCommonSSHOpts(Strings & args)
if (!sshPublicHostKey.empty()) {
Path fileName = (Path) *state->tmpDir + "/host-key";
auto p = host.rfind("@");
string thost = p != string::npos ? string(host, p + 1) : host;
std::string thost = p != std::string::npos ? std::string(host, p + 1) : host;
writeFile(fileName, thost + " " + base64Decode(sshPublicHostKey) + "\n");
args.insert(args.end(), {"-oUserKnownHostsFile=" + fileName});
}

153
src/libstore/store-api.cc
View file

@ -40,7 +40,7 @@ Path Store::followLinksToStore(std::string_view _path) const
Path path = absPath(std::string(_path));
while (!isInStore(path)) {
if (!isLink(path)) break;
string target = readLink(path);
auto target = readLink(path);
path = absPath(target, dirOf(path));
}
if (!isInStore(path))
@ -139,8 +139,8 @@ StorePath Store::makeStorePath(std::string_view type,
std::string_view hash, std::string_view name) const
{
/* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
string s = std::string { type } + ":" + std::string { hash }
+ ":" + storeDir + ":" + std::string { name };
auto s = std::string(type) + ":" + std::string(hash)
+ ":" + storeDir + ":" + std::string(name);
auto h = compressHash(hashString(htSHA256, s), 20);
return StorePath(h, name);
}
@ -165,7 +165,7 @@ StorePath Store::makeOutputPath(std::string_view id,
ambiguous. */
static std::string makeType(
const Store & store,
string && type,
std::string && type,
const PathReferences<StorePath> & references)
{
for (auto & i : references.references) {
@ -235,7 +235,9 @@ std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name,
}
StorePath Store::computeStorePathForText(const string & name, const string & s,
StorePath Store::computeStorePathForText(
std::string_view name,
std::string_view s,
const StorePathSet & references) const
{
return makeTextPath(name, TextInfo {
@ -245,8 +247,14 @@ StorePath Store::computeStorePathForText(const string & name, const string & s,
}
StorePath Store::addToStore(const string & name, const Path & _srcPath,
FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
StorePath Store::addToStore(
std::string_view name,
const Path & _srcPath,
FileIngestionMethod method,
HashType hashAlgo,
PathFilter & filter,
RepairFlag repair,
const StorePathSet & references)
{
Path srcPath(absPath(_srcPath));
auto source = sinkToSource([&](Sink & sink) {
@ -255,7 +263,7 @@ StorePath Store::addToStore(const string & name, const Path & _srcPath,
else
readFile(srcPath, sink);
});
return addToStoreFromDump(*source, name, method, hashAlgo, repair);
return addToStoreFromDump(*source, name, method, hashAlgo, repair, references);
}
@ -373,8 +381,13 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
StringSet StoreConfig::getDefaultSystemFeatures()
{
auto res = settings.systemFeatures.get();
if (settings.isExperimentalFeatureEnabled("ca-derivations"))
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations))
res.insert("ca-derivations");
if (settings.isExperimentalFeatureEnabled(Xp::RecursiveNix))
res.insert("recursive-nix");
return res;
}
@ -432,11 +445,9 @@ StorePathSet Store::queryDerivationOutputs(const StorePath & path)
bool Store::isValidPath(const StorePath & storePath)
{
std::string hashPart(storePath.hashPart());
{
auto state_(state.lock());
auto res = state_->pathInfoCache.get(hashPart);
auto res = state_->pathInfoCache.get(std::string(storePath.to_string()));
if (res && res->isKnownNow()) {
stats.narInfoReadAverted++;
return res->didExist();
@ -444,11 +455,11 @@ bool Store::isValidPath(const StorePath & storePath)
}
if (diskCache) {
auto res = diskCache->lookupNarInfo(getUri(), hashPart);
auto res = diskCache->lookupNarInfo(getUri(), std::string(storePath.hashPart()));
if (res.first != NarInfoDiskCache::oUnknown) {
stats.narInfoReadAverted++;
auto state_(state.lock());
state_->pathInfoCache.upsert(hashPart,
state_->pathInfoCache.upsert(std::string(storePath.to_string()),
res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue { .value = res.second });
return res.first == NarInfoDiskCache::oValid;
}
@ -458,7 +469,7 @@ bool Store::isValidPath(const StorePath & storePath)
if (diskCache && !valid)
// FIXME: handle valid = true case.
diskCache->upsertNarInfo(getUri(), hashPart, 0);
diskCache->upsertNarInfo(getUri(), std::string(storePath.hashPart()), 0);
return valid;
}
@ -505,13 +516,11 @@ static bool goodStorePath(const StorePath & expected, const StorePath & actual)
void Store::queryPathInfo(const StorePath & storePath,
Callback<ref<const ValidPathInfo>> callback) noexcept
{
std::string hashPart;
auto hashPart = std::string(storePath.hashPart());
try {
hashPart = storePath.hashPart();
{
auto res = state.lock()->pathInfoCache.get(hashPart);
auto res = state.lock()->pathInfoCache.get(std::string(storePath.to_string()));
if (res && res->isKnownNow()) {
stats.narInfoReadAverted++;
if (!res->didExist())
@ -526,7 +535,7 @@ void Store::queryPathInfo(const StorePath & storePath,
stats.narInfoReadAverted++;
{
auto state_(state.lock());
state_->pathInfoCache.upsert(hashPart,
state_->pathInfoCache.upsert(std::string(storePath.to_string()),
res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second });
if (res.first == NarInfoDiskCache::oInvalid ||
!goodStorePath(storePath, res.second->path))
@ -551,7 +560,7 @@ void Store::queryPathInfo(const StorePath & storePath,
{
auto state_(state.lock());
state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info });
state_->pathInfoCache.upsert(std::string(storePath.to_string()), PathInfoCacheValue { .value = info });
}
if (!info || !goodStorePath(storePath, info->path)) {
@ -564,6 +573,74 @@ void Store::queryPathInfo(const StorePath & storePath,
}});
}
void Store::queryRealisation(const DrvOutput & id,
Callback<std::shared_ptr<const Realisation>> callback) noexcept
{
try {
if (diskCache) {
auto [cacheOutcome, maybeCachedRealisation]
= diskCache->lookupRealisation(getUri(), id);
switch (cacheOutcome) {
case NarInfoDiskCache::oValid:
debug("Returning a cached realisation for %s", id.to_string());
callback(maybeCachedRealisation);
return;
case NarInfoDiskCache::oInvalid:
debug(
"Returning a cached missing realisation for %s",
id.to_string());
callback(nullptr);
return;
case NarInfoDiskCache::oUnknown:
break;
}
}
} catch (...) {
return callback.rethrow();
}
auto callbackPtr
= std::make_shared<decltype(callback)>(std::move(callback));
queryRealisationUncached(
id,
{ [this, id, callbackPtr](
std::future<std::shared_ptr<const Realisation>> fut) {
try {
auto info = fut.get();
if (diskCache) {
if (info)
diskCache->upsertRealisation(getUri(), *info);
else
diskCache->upsertAbsentRealisation(getUri(), id);
}
(*callbackPtr)(std::shared_ptr<const Realisation>(info));
} catch (...) {
callbackPtr->rethrow();
}
} });
}
std::shared_ptr<const Realisation> Store::queryRealisation(const DrvOutput & id)
{
using RealPtr = std::shared_ptr<const Realisation>;
std::promise<RealPtr> promise;
queryRealisation(id,
{[&](std::future<RealPtr> result) {
try {
promise.set_value(result.get());
} catch (...) {
promise.set_exception(std::current_exception());
}
}});
return promise.get_future().get();
}
void Store::substitutePaths(const StorePathSet & paths)
{
@ -637,10 +714,10 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m
/* Return a string accepted by decodeValidPathInfo() that
registers the specified paths as valid. Note: it's the
responsibility of the caller to provide a closure. */
string Store::makeValidityRegistration(const StorePathSet & paths,
std::string Store::makeValidityRegistration(const StorePathSet & paths,
bool showDerivers, bool showHash)
{
string s = "";
std::string s = "";
for (auto & i : paths) {
s += printStorePath(i) + "\n";
@ -710,11 +787,11 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
for (auto & storePath : storePaths) {
auto jsonPath = jsonList.object();
jsonPath.attr("path", printStorePath(storePath));
try {
auto info = queryPathInfo(storePath);
jsonPath.attr("path", printStorePath(info->path));
jsonPath
.attr("narHash", info->narHash.to_string(hashBase, true))
.attr("narSize", info->narSize);
@ -768,6 +845,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
}
} catch (InvalidPath &) {
jsonPath.attr("path", printStorePath(storePath));
jsonPath.attr("valid", false);
}
}
@ -877,7 +955,7 @@ std::map<StorePath, StorePath> copyPaths(
for (auto & path : paths) {
storePaths.insert(path.path());
if (auto realisation = std::get_if<Realisation>(&path.raw)) {
settings.requireExperimentalFeature("ca-derivations");
settings.requireExperimentalFeature(Xp::CaDerivations);
toplevelRealisations.insert(*realisation);
}
}
@ -909,7 +987,7 @@ std::map<StorePath, StorePath> copyPaths(
// Don't fail if the remote doesn't support CA derivations is it might
// not be within our control to change that, and we might still want
// to at least copy the output paths.
if (e.missingFeature == "ca-derivations")
if (e.missingFeature == Xp::CaDerivations)
ignoreException();
else
throw;
@ -1031,7 +1109,7 @@ std::map<StorePath, StorePath> copyPaths(
nrFailed++;
if (!settings.keepGoing)
throw e;
logger->log(lvlError, fmt("could not copy %s: %s", dstStore.printStorePath(storePath), e.what()));
printMsg(lvlError, "could not copy %s: %s", dstStore.printStorePath(storePath), e.what());
showProgress();
return;
}
@ -1061,13 +1139,28 @@ void copyClosure(
copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute);
}
void copyClosure(
Store & srcStore,
Store & dstStore,
const StorePathSet & storePaths,
RepairFlag repair,
CheckSigsFlag checkSigs,
SubstituteFlag substitute)
{
if (&srcStore == &dstStore) return;
StorePathSet closure;
srcStore.computeFSClosure(storePaths, closure);
copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute);
}
std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istream & str, std::optional<HashResult> hashGiven)
{
std::string path;
getline(str, path);
if (str.eof()) { return {}; }
if (!hashGiven) {
string s;
std::string s;
getline(str, s);
auto narHash = Hash::parseAny(s, htSHA256);
getline(str, s);
@ -1080,7 +1173,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istre
std::string deriver;
getline(str, deriver);
if (deriver != "") info.deriver = store.parseStorePath(deriver);
string s;
std::string s;
getline(str, s);
auto n = string2Int<int>(s);
if (!n) throw Error("number expected");
@ -1104,7 +1197,7 @@ std::string Store::showPaths(const StorePathSet & paths)
}
string showPaths(const PathSet & paths)
std::string showPaths(const PathSet & paths)
{
return concatStringsSep(", ", quoteStrings(paths));
}

211
src/libstore/store-api.hh
View file

@ -76,103 +76,10 @@ enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true };
const uint32_t exportMagic = 0x4558494e;
typedef std::unordered_map<StorePath, std::unordered_set<std::string>> Roots;
struct GCOptions
{
/* Garbage collector operation:
- `gcReturnLive': return the set of paths reachable from
(i.e. in the closure of) the roots.
- `gcReturnDead': return the set of paths not reachable from
the roots.
- `gcDeleteDead': actually delete the latter set.
- `gcDeleteSpecific': delete the paths listed in
`pathsToDelete', insofar as they are not reachable.
*/
typedef enum {
gcReturnLive,
gcReturnDead,
gcDeleteDead,
gcDeleteSpecific,
} GCAction;
GCAction action{gcDeleteDead};
/* If `ignoreLiveness' is set, then reachability from the roots is
ignored (dangerous!). However, the paths must still be
unreferenced *within* the store (i.e., there can be no other
store paths that depend on them). */
bool ignoreLiveness{false};
/* For `gcDeleteSpecific', the paths to delete. */
StorePathSet pathsToDelete;
/* Stop after at least `maxFreed' bytes have been freed. */
uint64_t maxFreed{std::numeric_limits<uint64_t>::max()};
};
struct GCResults
{
/* Depending on the action, the GC roots, or the paths that would
be or have been deleted. */
PathSet paths;
/* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the
number of bytes that would be or was freed. */
uint64_t bytesFreed = 0;
};
enum BuildMode { bmNormal, bmRepair, bmCheck };
struct BuildResult;
struct BuildResult
{
/* Note: don't remove status codes, and only add new status codes
at the end of the list, to prevent client/server
incompatibilities in the nix-store --serve protocol. */
enum Status {
Built = 0,
Substituted,
AlreadyValid,
PermanentFailure,
InputRejected,
OutputRejected,
TransientFailure, // possibly transient
CachedFailure, // no longer used
TimedOut,
MiscFailure,
DependencyFailed,
LogLimitExceeded,
NotDeterministic,
} status = MiscFailure;
std::string errorMsg;
/* How many times this build was performed. */
unsigned int timesBuilt = 0;
/* If timesBuilt > 1, whether some builds did not produce the same
result. (Note that 'isNonDeterministic = false' does not mean
the build is deterministic, just that we don't have evidence of
non-determinism.) */
bool isNonDeterministic = false;
DrvOutputs builtOutputs;
/* The start/stop times of the build (or one of the rounds, if it
was repeated). */
time_t startTime = 0, stopTime = 0;
bool success() {
return status == Built || status == Substituted || status == AlreadyValid;
}
};
typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
@ -234,7 +141,6 @@ protected:
struct State
{
// FIXME: fix key
LRUCache<std::string, PathInfoCacheValue> pathInfoCache;
};
@ -325,7 +231,9 @@ public:
simply yield a different store path, so other users wouldn't be
affected), but it has some backwards compatibility issues (the
hashing scheme changes), so I'm not doing that for now. */
StorePath computeStorePathForText(const string & name, const string & s,
StorePath computeStorePathForText(
std::string_view name,
std::string_view s,
const StorePathSet & references) const;
/* Check whether a path is valid. */
@ -366,6 +274,14 @@ public:
void queryPathInfo(const StorePath & path,
Callback<ref<const ValidPathInfo>> callback) noexcept;
/* Query the information about a realisation. */
std::shared_ptr<const Realisation> queryRealisation(const DrvOutput &);
/* Asynchronous version of queryRealisation(). */
void queryRealisation(const DrvOutput &,
Callback<std::shared_ptr<const Realisation>> callback) noexcept;
/* Check whether the given valid path info is sufficiently attested, by
either being signed by a trusted public key or content-addressed, in
order to be included in the given store.
@ -390,11 +306,11 @@ protected:
virtual void queryPathInfoUncached(const StorePath & path,
Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept = 0;
virtual void queryRealisationUncached(const DrvOutput &,
Callback<std::shared_ptr<const Realisation>> callback) noexcept = 0;
public:
virtual std::optional<const Realisation> queryRealisation(const DrvOutput &) = 0;
/* Queries the set of incoming FS references for a store path.
The result is not cleared. */
virtual void queryReferrers(const StorePath & path, StorePathSet & referrers)
@ -447,9 +363,14 @@ public:
validity the resulting path. The resulting path is returned.
The function object `filter' can be used to exclude files (see
libutil/archive.hh). */
virtual StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair);
virtual StorePath addToStore(
std::string_view name,
const Path & srcPath,
FileIngestionMethod method = FileIngestionMethod::Recursive,
HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter,
RepairFlag repair = NoRepair,
const StorePathSet & references = StorePathSet());
/* Copy the contents of a path to the store and register the
validity the resulting path, using a constant amount of
@ -464,14 +385,18 @@ public:
false).
`dump` may be drained */
// FIXME: remove?
virtual StorePath addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)
virtual StorePath addToStoreFromDump(Source & dump, std::string_view name,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair,
const StorePathSet & references = StorePathSet())
{ unsupported("addToStoreFromDump"); }
/* Like addToStore, but the contents written to the output path is
a regular file containing the given string. */
virtual StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair = NoRepair) = 0;
virtual StorePath addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references,
RepairFlag repair = NoRepair) = 0;
/**
* Add a mapping indicating that `deriver!outputName` maps to the output path
@ -503,6 +428,16 @@ public:
BuildMode buildMode = bmNormal,
std::shared_ptr<Store> evalStore = nullptr);
/* Like `buildPaths()`, but return a vector of `BuildResult`s
corresponding to each element in `paths`. Note that in case of
a build/substitution error, this function won't throw an
exception, but return a `BuildResult` containing an error
message. */
virtual std::vector<BuildResult> buildPathsWithResults(
const std::vector<DerivedPath> & paths,
BuildMode buildMode = bmNormal,
std::shared_ptr<Store> evalStore = nullptr);
/* Build a single non-materialized derivation (i.e. not from an
on-disk .drv file).
@ -549,50 +484,10 @@ public:
virtual void addTempRoot(const StorePath & path)
{ debug("not creating temporary root, store doesn't support GC"); }
/* Add an indirect root, which is merely a symlink to `path' from
/nix/var/nix/gcroots/auto/<hash of `path'>. `path' is supposed
to be a symlink to a store path. The garbage collector will
automatically remove the indirect root when it finds that
`path' has disappeared. */
virtual void addIndirectRoot(const Path & path)
{ unsupported("addIndirectRoot"); }
/* Acquire the global GC lock, then immediately release it. This
function must be called after registering a new permanent root,
but before exiting. Otherwise, it is possible that a running
garbage collector doesn't see the new root and deletes the
stuff we've just built. By acquiring the lock briefly, we
ensure that either:
- The collector is already running, and so we block until the
collector is finished. The collector will know about our
*temporary* locks, which should include whatever it is we
want to register as a permanent lock.
- The collector isn't running, or it's just started but hasn't
acquired the GC lock yet. In that case we get and release
the lock right away, then exit. The collector scans the
permanent root and sees ours.
In either case the permanent root is seen by the collector. */
virtual void syncWithGC() { };
/* Find the roots of the garbage collector. Each root is a pair
(link, storepath) where `link' is the path of the symlink
outside of the Nix store that point to `storePath'. If
'censor' is true, privacy-sensitive information about roots
found in /proc is censored. */
virtual Roots findRoots(bool censor)
{ unsupported("findRoots"); }
/* Perform a garbage collection. */
virtual void collectGarbage(const GCOptions & options, GCResults & results)
{ unsupported("collectGarbage"); }
/* Return a string representing information about the path that
can be loaded into the database using `nix-store --load-db' or
`nix-store --register-validity'. */
string makeValidityRegistration(const StorePathSet & paths,
std::string makeValidityRegistration(const StorePathSet & paths,
bool showDerivers, bool showHash);
/* Write a JSON representation of store path metadata, such as the
@ -708,8 +603,11 @@ public:
/* Return the build log of the specified store path, if available,
or null otherwise. */
virtual std::shared_ptr<std::string> getBuildLog(const StorePath & path)
{ return nullptr; }
virtual std::optional<std::string> getBuildLog(const StorePath & path)
{ return std::nullopt; }
virtual void addBuildLog(const StorePath & path, std::string_view log)
{ unsupported("addBuildLog"); }
/* Hack to allow long-running processes like hydra-queue-runner to
occasionally flush their path info cache. */
@ -741,6 +639,14 @@ public:
virtual void createUser(const std::string & userName, uid_t userId)
{ }
/*
* Synchronises the options of the client with those of the daemon
* (a no-op when there’s no daemon)
*/
virtual void setOptions() { }
virtual std::optional<std::string> getVersion() { return {}; }
protected:
Stats stats;
@ -791,6 +697,13 @@ void copyClosure(
CheckSigsFlag checkSigs = CheckSigs,
SubstituteFlag substitute = NoSubstitute);
void copyClosure(
Store & srcStore, Store & dstStore,
const StorePathSet & paths,
RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs,
SubstituteFlag substitute = NoSubstitute);
/* Remove the temporary roots file for this process. Any temporary
root becomes garbage after this point unless it has been registered
as a (permanent) root. */
@ -876,7 +789,7 @@ struct RegisterStoreImplementation
/* Display a set of paths in human-readable form (i.e., between quotes
and separated by commas). */
string showPaths(const PathSet & paths);
std::string showPaths(const PathSet & paths);
std::optional<ValidPathInfo> decodeValidPathInfo(

169
src/libstore/tests/machines.cc Normal file
View file

@ -0,0 +1,169 @@
#include "machines.hh"
#include "globals.hh"
#include <gmock/gmock-matchers.h>
using testing::Contains;
using testing::ElementsAre;
using testing::EndsWith;
using testing::Eq;
using testing::Field;
using testing::SizeIs;
using nix::absPath;
using nix::FormatError;
using nix::getMachines;
using nix::Machine;
using nix::Machines;
using nix::pathExists;
using nix::Settings;
using nix::settings;
class Environment : public ::testing::Environment {
public:
void SetUp() override { settings.thisSystem = "TEST_ARCH-TEST_OS"; }
};
testing::Environment* const foo_env =
testing::AddGlobalTestEnvironment(new Environment);
TEST(machines, getMachinesWithEmptyBuilders) {
settings.builders = "";
Machines actual = getMachines();
ASSERT_THAT(actual, SizeIs(0));
}
TEST(machines, getMachinesUriOnly) {
settings.builders = "nix@scratchy.labs.cs.uu.nl";
Machines actual = getMachines();
ASSERT_THAT(actual, SizeIs(1));
EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq("ssh://nix@scratchy.labs.cs.uu.nl")));
EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS")));
EXPECT_THAT(actual[0], Field(&Machine::sshKey, SizeIs(0)));
EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1)));
EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(1)));
EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, SizeIs(0)));
EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, SizeIs(0)));
EXPECT_THAT(actual[0], Field(&Machine::sshPublicHostKey, SizeIs(0)));
}
TEST(machines, getMachinesDefaults) {
settings.builders = "nix@scratchy.labs.cs.uu.nl - - - - - - -";
Machines actual = getMachines();
ASSERT_THAT(actual, SizeIs(1));
EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq("ssh://nix@scratchy.labs.cs.uu.nl")));
EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS")));
EXPECT_THAT(actual[0], Field(&Machine::sshKey, SizeIs(0)));
EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1)));
EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(1)));
EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, SizeIs(0)));
EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, SizeIs(0)));
EXPECT_THAT(actual[0], Field(&Machine::sshPublicHostKey, SizeIs(0)));
}
TEST(machines, getMachinesWithNewLineSeparator) {
settings.builders = "nix@scratchy.labs.cs.uu.nl\nnix@itchy.labs.cs.uu.nl";
Machines actual = getMachines();
ASSERT_THAT(actual, SizeIs(2));
EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))));
EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl"))));
}
TEST(machines, getMachinesWithSemicolonSeparator) {
settings.builders = "nix@scratchy.labs.cs.uu.nl ; nix@itchy.labs.cs.uu.nl";
Machines actual = getMachines();
EXPECT_THAT(actual, SizeIs(2));
EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))));
EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl"))));
}
TEST(machines, getMachinesWithCorrectCompleteSingleBuilder) {
settings.builders = "nix@scratchy.labs.cs.uu.nl i686-linux "
"/home/nix/.ssh/id_scratchy_auto 8 3 kvm "
"benchmark SSH+HOST+PUBLIC+KEY+BASE64+ENCODED==";
Machines actual = getMachines();
ASSERT_THAT(actual, SizeIs(1));
EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")));
EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("i686-linux")));
EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq("/home/nix/.ssh/id_scratchy_auto")));
EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(8)));
EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(3)));
EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, ElementsAre("kvm")));
EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, ElementsAre("benchmark")));
EXPECT_THAT(actual[0], Field(&Machine::sshPublicHostKey, Eq("SSH+HOST+PUBLIC+KEY+BASE64+ENCODED==")));
}
TEST(machines,
getMachinesWithCorrectCompleteSingleBuilderWithTabColumnDelimiter) {
settings.builders =
"nix@scratchy.labs.cs.uu.nl\ti686-linux\t/home/nix/.ssh/"
"id_scratchy_auto\t8\t3\tkvm\tbenchmark\tSSH+HOST+PUBLIC+"
"KEY+BASE64+ENCODED==";
Machines actual = getMachines();
ASSERT_THAT(actual, SizeIs(1));
EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")));
EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("i686-linux")));
EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq("/home/nix/.ssh/id_scratchy_auto")));
EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(8)));
EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(3)));
EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, ElementsAre("kvm")));
EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, ElementsAre("benchmark")));
EXPECT_THAT(actual[0], Field(&Machine::sshPublicHostKey, Eq("SSH+HOST+PUBLIC+KEY+BASE64+ENCODED==")));
}
TEST(machines, getMachinesWithMultiOptions) {
settings.builders = "nix@scratchy.labs.cs.uu.nl Arch1,Arch2 - - - "
"SupportedFeature1,SupportedFeature2 "
"MandatoryFeature1,MandatoryFeature2";
Machines actual = getMachines();
ASSERT_THAT(actual, SizeIs(1));
EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")));
EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("Arch1", "Arch2")));
EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, ElementsAre("SupportedFeature1", "SupportedFeature2")));
EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, ElementsAre("MandatoryFeature1", "MandatoryFeature2")));
}
TEST(machines, getMachinesWithIncorrectFormat) {
settings.builders = "nix@scratchy.labs.cs.uu.nl - - eight";
EXPECT_THROW(getMachines(), FormatError);
settings.builders = "nix@scratchy.labs.cs.uu.nl - - -1";
EXPECT_THROW(getMachines(), FormatError);
settings.builders = "nix@scratchy.labs.cs.uu.nl - - 8 three";
EXPECT_THROW(getMachines(), FormatError);
settings.builders = "nix@scratchy.labs.cs.uu.nl - - 8 -3";
EXPECT_THROW(getMachines(), FormatError);
settings.builders = "nix@scratchy.labs.cs.uu.nl - - 8 3 - - BAD_BASE64";
EXPECT_THROW(getMachines(), FormatError);
}
TEST(machines, getMachinesWithCorrectFileReference) {
auto path = absPath("src/libstore/tests/test-data/machines.valid");
ASSERT_TRUE(pathExists(path));
settings.builders = std::string("@") + path;
Machines actual = getMachines();
ASSERT_THAT(actual, SizeIs(3));
EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))));
EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl"))));
EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@poochie.labs.cs.uu.nl"))));
}
TEST(machines, getMachinesWithCorrectFileReferenceToEmptyFile) {
auto path = "/dev/null";
ASSERT_TRUE(pathExists(path));
settings.builders = std::string("@") + path;
Machines actual = getMachines();
ASSERT_THAT(actual, SizeIs(0));
}
TEST(machines, getMachinesWithIncorrectFileReference) {
settings.builders = std::string("@") + absPath("/not/a/file");
Machines actual = getMachines();
ASSERT_THAT(actual, SizeIs(0));
}
TEST(machines, getMachinesWithCorrectFileReferenceToIncorrectFile) {
settings.builders = std::string("@") + absPath("src/libstore/tests/test-data/machines.bad_format");
EXPECT_THROW(getMachines(), FormatError);
}

1
src/libstore/tests/test-data/machines.bad_format Normal file
View file

@ -0,0 +1 @@
nix@scratchy.labs.cs.uu.nl - - eight

3
src/libstore/tests/test-data/machines.valid Normal file
View file

@ -0,0 +1,3 @@
nix@scratchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 1 kvm
nix@itchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 2
nix@poochie.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 1 2 kvm benchmark c3NoLXJzYSBBQUFBQjNOemFDMXljMkVBQUFBREFRQUJBQUFDQVFDWWV5R1laNTNzd1VjMUZNSHBWL1BCcXlKaFR5S1JoRkpWWVRpRHlQN2h5c1JGa0w4VDlLOGdhL2Y2L3c3QjN2SjNHSFRIUFkybENiUEdZbGNLd2h6M2ZRbFNNOEViNi95b3ZLajdvM1FsMEx5Y0dzdGJvRmcwWkZKNldncUxsR0ltS0NobUlxOGZ3TW5ZTWUxbnRQeTBUZFZjSU1tOTV3YzF3SjBMd2c3cEVMRmtHazdkeTVvYnM4a3lGZ0pORDVRSmFwQWJjeWp4Z1QzdzdMcktNZ2xzeWhhd01JNVpkMGZsQTVudW5OZ3pid3plYVhLaUsyTW0vdGJXYTU1YTd4QmNYdHpIZGlPSWdSajJlRWxaMGh5bk10YjBmcklsdmxIcEtLaVFaZ3pQdCtIVXQ2bXpRMkRVME52MGYyYnNSU0krOGpJU2pQcmdlcVVHRldMUzVIUTg2N2xSMlpiaWtyclhZNTdqbVFEZk5DRHY1VFBHZU9UekFEd2pjMDc2aFZ3VFJCd3VTZFhtaWNxTS95b3lrWitkV1dnZ25MenE5QU1tdlNZcDhmZkZDcS9CSDBZNUFXWTFHay9vS3hMVTNaOWt3ZDd2UWNFQWFCQ2dxdnVZRGdTaHE1RlhndDM3OVZESWtEL05ZSTg2QXVvajVDRmVNTzlRM2pJSlRadlh6c1VldjVoSnA2djcxSVh5ODVtbTY5R20zcXdicVE1SjVQZDU1Um56SitpaW5BNjZxTEFSc0Y4amNsSnd5ekFXclBoYU9DRVY2bjVMeVhVazhzMW9EVVR4V1pWN25rVkFTbHJ0MllGcjN5dzdjRTRXQVhsemhHcDhocmdLMVVkMUlyeDVnZWRaSnBWcy9uNWVybmJFMUxmb2x5UHUvRUFIWlh6VGd4dHVDUFNobXc9PQo=

9
src/libstore/uds-remote-store.cc
View file

@ -56,14 +56,7 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
auto conn = make_ref<Connection>();
/* Connect to a daemon that does the privileged work for us. */
conn->fd = socket(PF_UNIX, SOCK_STREAM
#ifdef SOCK_CLOEXEC
| SOCK_CLOEXEC
#endif
, 0);
if (!conn->fd)
throw SysError("cannot create Unix domain socket");
closeOnExec(conn->fd.get());
conn->fd = createUnixDomainSocket();
nix::connect(conn->fd.get(), path ? *path : settings.nixDaemonSocketFile);

5
src/libstore/worker-protocol.hh
View file

@ -9,7 +9,7 @@ namespace nix {
#define WORKER_MAGIC_1 0x6e697863
#define WORKER_MAGIC_2 0x6478696f
#define PROTOCOL_VERSION (1 << 8 | 32)
#define PROTOCOL_VERSION (1 << 8 | 34)
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
@ -56,6 +56,8 @@ typedef enum {
wopRegisterDrvOutput = 42,
wopQueryRealisation = 43,
wopAddMultipleToStore = 44,
wopAddBuildLog = 45,
wopBuildPathsWithResults = 46,
} WorkerOp;
@ -90,6 +92,7 @@ MAKE_WORKER_PROTO(, ContentAddress);
MAKE_WORKER_PROTO(, DerivedPath);
MAKE_WORKER_PROTO(, Realisation);
MAKE_WORKER_PROTO(, DrvOutput);
MAKE_WORKER_PROTO(, BuildResult);
MAKE_WORKER_PROTO(template<typename T>, std::vector<T>);
MAKE_WORKER_PROTO(template<typename T>, std::set<T>);