wroclaw-git-backups/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-07-05 20:41:47 +02:00
Code Activity

Merge remote-tracking branch 'upstream/master' into store-path-complete-construction

Browse source
This commit is contained in:
John Ericson 2024-02-12 11:22:54 -05:00
commit eb76b35efa
224 changed files with 4699 additions and 3115 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/libstore/binary-cache-store.cc
View file

@ -235,14 +235,14 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
std::regex regex2("^[0-9a-f]{38}\\.debug$");
for (auto & [s1, _type] : narAccessor->readDirectory(buildIdDir)) {
auto dir = buildIdDir + s1;
auto dir = buildIdDir / s1;
if (narAccessor->lstat(dir).type != SourceAccessor::tDirectory
|| !std::regex_match(s1, regex1))
continue;
for (auto & [s2, _type] : narAccessor->readDirectory(dir)) {
auto debugPath = dir + s2;
auto debugPath = dir / s2;
if (narAccessor->lstat(debugPath).type != SourceAccessor::tRegular
|| !std::regex_match(s2, regex2))

28
src/libstore/build/derivation-goal.cc
View file

@ -223,7 +223,7 @@ void DerivationGoal::haveDerivation()
if (!drv->type().hasKnownOutputPaths())
experimentalFeatureSettings.require(Xp::CaDerivations);
if (!drv->type().isPure()) {
if (drv->type().isImpure()) {
experimentalFeatureSettings.require(Xp::ImpureDerivations);
for (auto & [outputName, output] : drv->outputs) {
@ -304,7 +304,7 @@ void DerivationGoal::outputsSubstitutionTried()
{
trace("all outputs substituted (maybe)");
assert(drv->type().isPure());
assert(!drv->type().isImpure());
if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
done(BuildResult::TransientFailure, {},
@ -397,9 +397,9 @@ void DerivationGoal::gaveUpOnSubstitution()
for (const auto & [inputDrvPath, inputNode] : dynamic_cast<Derivation *>(drv.get())->inputDrvs.map) {
/* Ensure that pure, non-fixed-output derivations don't
depend on impure derivations. */
if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && drv->type().isPure() && !drv->type().isFixed()) {
if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && !drv->type().isImpure() && !drv->type().isFixed()) {
auto inputDrv = worker.evalStore.readDerivation(inputDrvPath);
if (!inputDrv.type().isPure())
if (inputDrv.type().isImpure())
throw Error("pure derivation '%s' depends on impure derivation '%s'",
worker.store.printStorePath(drvPath),
worker.store.printStorePath(inputDrvPath));
@ -439,7 +439,7 @@ void DerivationGoal::gaveUpOnSubstitution()
void DerivationGoal::repairClosure()
{
assert(drv->type().isPure());
assert(!drv->type().isImpure());
/* If we're repairing, we now know that our own outputs are valid.
Now check whether the other paths in the outputs closure are
@ -708,7 +708,7 @@ void DerivationGoal::tryToBuild()
if (!outputLocks.lockPaths(lockFiles, "", false)) {
if (!actLock)
actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles))));
fmt("waiting for lock on %s", Magenta(showPaths(lockFiles))));
worker.waitForAWhile(shared_from_this());
return;
}
@ -762,7 +762,7 @@ void DerivationGoal::tryToBuild()
the wake-up timeout expires. */
if (!actLock)
actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
fmt("waiting for a machine to build '%s'", Magenta(worker.store.printStorePath(drvPath))));
worker.waitForAWhile(shared_from_this());
outputLocks.unlock();
return;
@ -891,7 +891,7 @@ void runPostBuildHook(
if (hook == "")
return;
Activity act(logger, lvlInfo, actPostBuildHook,
Activity act(logger, lvlTalkative, actPostBuildHook,
fmt("running post-build-hook '%s'", settings.postBuildHook),
Logger::Fields{store.printStorePath(drvPath)});
PushActivity pact(act.id);
@ -987,7 +987,7 @@ void DerivationGoal::buildDone()
diskFull |= cleanupDecideWhetherDiskFull();
auto msg = fmt("builder for '%s' %s",
yellowtxt(worker.store.printStorePath(drvPath)),
Magenta(worker.store.printStorePath(drvPath)),
statusToString(status));
if (!logger->isVerbose() && !logTail.empty()) {
@ -1100,7 +1100,7 @@ void DerivationGoal::resolvedFinished()
worker.store.printStorePath(resolvedDrvGoal->drvPath), outputName);
}();
if (drv->type().isPure()) {
if (!drv->type().isImpure()) {
auto newRealisation = realisation;
newRealisation.id = DrvOutput { initialOutput->outputHash, outputName };
newRealisation.signatures.clear();
@ -1395,7 +1395,7 @@ void DerivationGoal::flushLine()
std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap()
{
assert(drv->type().isPure());
assert(!drv->type().isImpure());
if (!useDerivation || drv->type().hasKnownOutputPaths()) {
std::map<std::string, std::optional<StorePath>> res;
for (auto & [name, output] : drv->outputs)
@ -1411,7 +1411,7 @@ std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDeri
OutputPathMap DerivationGoal::queryDerivationOutputMap()
{
assert(drv->type().isPure());
assert(!drv->type().isImpure());
if (!useDerivation || drv->type().hasKnownOutputPaths()) {
OutputPathMap res;
for (auto & [name, output] : drv->outputsAndOptPaths(worker.store))
@ -1428,7 +1428,7 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap()
std::pair<bool, SingleDrvOutputs> DerivationGoal::checkPathValidity()
{
if (!drv->type().isPure()) return { false, {} };
if (drv->type().isImpure()) return { false, {} };
bool checkHash = buildMode == bmRepair;
auto wantedOutputsLeft = std::visit(overloaded {
@ -1523,7 +1523,7 @@ void DerivationGoal::done(
outputLocks.unlock();
buildResult.status = status;
if (ex)
buildResult.errorMsg = fmt("%s", normaltxt(ex->info().msg));
buildResult.errorMsg = fmt("%s", Uncolored(ex->info().msg));
if (buildResult.status == BuildResult::TimedOut)
worker.timedOut = true;
if (buildResult.status == BuildResult::PermanentFailure)

4
src/libstore/build/entry-points.cc
View file

@ -33,7 +33,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
}
if (failed.size() == 1 && ex) {
ex->status = worker.failingExitStatus();
ex->withExitStatus(worker.failingExitStatus());
throw std::move(*ex);
} else if (!failed.empty()) {
if (ex) logError(ex->info());
@ -104,7 +104,7 @@ void Store::ensurePath(const StorePath & path)
if (goal->exitCode != Goal::ecSuccess) {
if (goal->ex) {
goal->ex->status = worker.failingExitStatus();
goal->ex->withExitStatus(worker.failingExitStatus());
throw std::move(*goal->ex);
} else
throw Error(worker.failingExitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path));

6
src/libstore/build/local-derivation-goal.cc
View file

@ -92,7 +92,7 @@ void handleDiffHook(
} catch (Error & error) {
ErrorInfo ei = error.info();
// FIXME: wrap errors.
ei.msg = hintfmt("diff hook execution failed: %s", ei.msg.str());
ei.msg = HintFmt("diff hook execution failed: %s", ei.msg.str());
logError(ei);
}
}
@ -232,7 +232,7 @@ void LocalDerivationGoal::tryLocalBuild()
if (!buildUser) {
if (!actLock)
actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
fmt("waiting for a free build user ID for '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
fmt("waiting for a free build user ID for '%s'", Magenta(worker.store.printStorePath(drvPath))));
worker.waitForAWhile(shared_from_this());
return;
}
@ -2724,7 +2724,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
.outPath = newInfo.path
};
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)
&& drv->type().isPure())
&& !drv->type().isImpure())
{
signRealisation(thisRealisation);
worker.store.registerDrvOutput(thisRealisation);

2
src/libstore/build/worker.cc
View file

@ -251,7 +251,7 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers)
void Worker::waitForBuildSlot(GoalPtr goal)
{
debug("wait for build slot");
goal->trace("wait for build slot");
bool isSubstitutionGoal = goal->jobCategory() == JobCategory::Substitution;
if ((!isSubstitutionGoal && getNrLocalBuilds() < settings.maxBuildJobs) ||
(isSubstitutionGoal && getNrSubstitutions() < settings.maxSubstitutionJobs))

2
src/libstore/build/worker.hh
View file

@ -116,7 +116,7 @@ private:
WeakGoals waitingForAWhile;
/**
* Last time the goals in `waitingForAWhile` where woken up.
* Last time the goals in `waitingForAWhile` were woken up.
*/
steady_time_point lastWokenUp;

14
src/libstore/builtins/fetchurl.cc
View file

@ -16,6 +16,14 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
writeFile(settings.netrcFile, netrcData, 0600);
}
auto out = get(drv.outputs, "out");
if (!out)
throw Error("'builtin:fetchurl' requires an 'out' output");
auto dof = std::get_if<DerivationOutput::CAFixed>(&out->raw);
if (!dof)
throw Error("'builtin:fetchurl' must be a fixed-output derivation");
auto getAttr = [&](const std::string & name) {
auto i = drv.env.find(name);
if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
@ -59,13 +67,11 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
};
/* Try the hashed mirrors first. */
if (getAttr("outputHashMode") == "flat")
if (dof->ca.method.getFileIngestionMethod() == FileIngestionMethod::Flat)
for (auto hashedMirror : settings.hashedMirrors.get())
try {
if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
std::optional<HashAlgorithm> ht = parseHashAlgoOpt(getAttr("outputHashAlgo"));
Hash h = newHashAllowEmpty(getAttr("outputHash"), ht);
fetch(hashedMirror + printHashAlgo(h.algo) + "/" + h.to_string(HashFormat::Base16, false));
fetch(hashedMirror + printHashAlgo(dof->ca.hash.algo) + "/" + dof->ca.hash.to_string(HashFormat::Base16, false));
return;
} catch (Error & e) {
debug(e.what());

6
src/libstore/daemon.cc
View file

@ -119,7 +119,7 @@ struct TunnelLogger : public Logger
if (GET_PROTOCOL_MINOR(clientVersion) >= 26) {
to << STDERR_ERROR << *ex;
} else {
to << STDERR_ERROR << ex->what() << ex->status;
to << STDERR_ERROR << ex->what() << ex->info().status;
}
}
}
@ -441,7 +441,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
eagerly consume the entire stream it's given, past the
length of the Nar. */
TeeSource savedNARSource(from, saved);
NullParseSink sink; /* just parse the NAR */
NullFileSystemObjectSink sink; /* just parse the NAR */
parseDump(sink, savedNARSource);
} else {
/* Incrementally parse the NAR file, stripping the
@ -913,7 +913,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
source = std::make_unique<TunnelSource>(from, to);
else {
TeeSource tee { from, saved };
NullParseSink ether;
NullFileSystemObjectSink ether;
parseDump(ether, tee);
source = std::make_unique<StringSource>(saved.s);
}

10
src/libstore/derivations.cc
View file

@ -110,17 +110,17 @@ bool DerivationType::isSandboxed() const
}
bool DerivationType::isPure() const
bool DerivationType::isImpure() const
{
return std::visit(overloaded {
[](const InputAddressed & ia) {
return true;
return false;
},
[](const ContentAddressed & ca) {
return true;
return false;
},
[](const Impure &) {
return false;
return true;
},
}, raw);
}
@ -840,7 +840,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
};
}
if (!type.isPure()) {
if (type.isImpure()) {
std::map<std::string, Hash> outputHashes;
for (const auto & [outputName, _] : drv.outputs)
outputHashes.insert_or_assign(outputName, impureOutputHash);

13
src/libstore/derivations.hh
View file

@ -253,12 +253,17 @@ struct DerivationType {
bool isSandboxed() const;
/**
* Whether the derivation is expected to produce the same result
* every time, and therefore it only needs to be built once. This is
* only false for derivations that have the attribute '__impure =
* Whether the derivation is expected to produce a different result
* every time, and therefore it needs to be rebuilt every time. This is
* only true for derivations that have the attribute '__impure =
* true'.
*
* Non-impure derivations can still behave impurely, to the degree permitted
* by the sandbox. Hence why this method isn't `isPure`: impure derivations
* are not the negation of pure derivations. Purity can not be ascertained
* except by rather heavy tools.
*/
bool isPure() const;
bool isImpure() const;
/**
* Does the derivation knows its own output paths?

2
src/libstore/export-import.cc
View file

@ -65,7 +65,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
/* Extract the NAR from the source. */
StringSink saved;
TeeSource tee { source, saved };
NullParseSink ether;
NullFileSystemObjectSink ether;
parseDump(ether, tee);
uint32_t magic = readInt(source);

4
src/libstore/filetransfer.cc
View file

@ -882,12 +882,12 @@ template<typename... Args>
FileTransferError::FileTransferError(FileTransfer::Error error, std::optional<std::string> response, const Args & ... args)
: Error(args...), error(error), response(response)
{
const auto hf = hintfmt(args...);
const auto hf = HintFmt(args...);
// FIXME: Due to https://github.com/NixOS/nix/issues/3841 we don't know how
// to print different messages for different verbosity levels. For now
// we add some heuristics for detecting when we want to show the response.
if (response && (response->size() < 1024 || response->find("<html>") != std::string::npos))
err.msg = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), chomp(*response));
err.msg = HintFmt("%1%\n\nresponse body:\n\n%2%", Uncolored(hf.str()), chomp(*response));
else
err.msg = hf;
}

21
src/libstore/globals.hh
View file

@ -151,13 +151,18 @@ public:
MaxBuildJobsSetting maxBuildJobs{
this, 1, "max-jobs",
R"(
This option defines the maximum number of jobs that Nix will try to
build in parallel. The default is `1`. The special value `auto`
causes Nix to use the number of CPUs in your system. `0` is useful
when using remote builders to prevent any local builds (except for
`preferLocalBuild` derivation attribute which executes locally
regardless). It can be overridden using the `--max-jobs` (`-j`)
command line switch.
Maximum number of jobs that Nix will try to build locally in parallel.
The special value `auto` causes Nix to use the number of CPUs in your system.
Use `0` to disable local builds and directly use the remote machines specified in [`builders`](#conf-builders).
This will not affect derivations that have [`preferLocalBuild = true`](@docroot@/language/advanced-attributes.md#adv-attr-preferLocalBuild), which are always built locally.
> **Note**
>
> The number of CPU cores to use for each build job is independently determined by the [`cores`](#conf-cores) setting.
<!-- TODO(@fricklerhandwerk): would be good to have those shorthands for common options as part of the specification -->
The setting can be overridden using the `--max-jobs` (`-j`) command line switch.
)",
{"build-max-jobs"}};
@ -635,7 +640,7 @@ public:
- the store object has been signed using a key in the trusted keys list
- the [`require-sigs`](#conf-require-sigs) option has been set to `false`
- the store object is [output-addressed](@docroot@/glossary.md#gloss-output-addressed-store-object)
- the store object is [content-addressed](@docroot@/glossary.md#gloss-content-addressed-store-object)
)",
{"binary-cache-public-keys"}};

108
src/libstore/legacy-ssh-store.cc
View file

@ -22,45 +22,10 @@ std::string LegacySSHStoreConfig::doc()
}
struct LegacySSHStore::Connection
struct LegacySSHStore::Connection : public ServeProto::BasicClientConnection
{
std::unique_ptr<SSHMaster::Connection> sshConn;
FdSink to;
FdSource from;
ServeProto::Version remoteVersion;
bool good = true;
/**
* Coercion to `ServeProto::ReadConn`. This makes it easy to use the
* factored out serve protocol searlizers with a
* `LegacySSHStore::Connection`.
*
* The serve protocol connection types are unidirectional, unlike
* this type.
*/
operator ServeProto::ReadConn ()
{
return ServeProto::ReadConn {
.from = from,
.version = remoteVersion,
};
}
/*
* Coercion to `ServeProto::WriteConn`. This makes it easy to use the
* factored out serve protocol searlizers with a
* `LegacySSHStore::Connection`.
*
* The serve protocol connection types are unidirectional, unlike
* this type.
*/
operator ServeProto::WriteConn ()
{
return ServeProto::WriteConn {
.to = to,
.version = remoteVersion,
};
}
};
@ -90,34 +55,31 @@ LegacySSHStore::LegacySSHStore(const std::string & scheme, const std::string & h
ref<LegacySSHStore::Connection> LegacySSHStore::openConnection()
{
auto conn = make_ref<Connection>();
conn->sshConn = master.startCommand(
fmt("%s --serve --write", remoteProgram)
+ (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get())));
Strings command = remoteProgram.get();
command.push_back("--serve");
command.push_back("--write");
if (remoteStore.get() != "") {
command.push_back("--store");
command.push_back(remoteStore.get());
}
conn->sshConn = master.startCommand(std::move(command));
conn->to = FdSink(conn->sshConn->in.get());
conn->from = FdSource(conn->sshConn->out.get());
StringSink saved;
TeeSource tee(conn->from, saved);
try {
conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION;
conn->to.flush();
StringSink saved;
try {
TeeSource tee(conn->from, saved);
unsigned int magic = readInt(tee);
if (magic != SERVE_MAGIC_2)
throw Error("'nix-store --serve' protocol mismatch from '%s'", host);
} catch (SerialisationError & e) {
/* In case the other side is waiting for our input,
close it. */
conn->sshConn->in.close();
auto msg = conn->from.drain();
throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'",
host, chomp(saved.s + msg));
conn->remoteVersion = ServeProto::BasicClientConnection::handshake(
conn->to, tee, SERVE_PROTOCOL_VERSION, host);
} catch (SerialisationError & e) {
// in.close(): Don't let the remote block on us not writing.
conn->sshConn->in.close();
{
NullSink nullSink;
conn->from.drainInto(nullSink);
}
conn->remoteVersion = readInt(conn->from);
if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200)
throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host);
throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'",
host, chomp(saved.s));
} catch (EndOfFile & e) {
throw Error("cannot connect to '%1%'", host);
}
@ -232,16 +194,16 @@ void LegacySSHStore::narFromPath(const StorePath & path, Sink & sink)
}
void LegacySSHStore::putBuildSettings(Connection & conn)
static ServeProto::BuildOptions buildSettings()
{
ServeProto::write(*this, conn, ServeProto::BuildOptions {
return {
.maxSilentTime = settings.maxSilentTime,
.buildTimeout = settings.buildTimeout,
.maxLogSize = settings.maxLogSize,
.nrRepeats = 0, // buildRepeat hasn't worked for ages anyway
.enforceDeterminism = 0,
.keepFailed = settings.keepFailed,
});
};
}
@ -250,14 +212,7 @@ BuildResult LegacySSHStore::buildDerivation(const StorePath & drvPath, const Bas
{
auto conn(connections->get());
conn->to
<< ServeProto::Command::BuildDerivation
<< printStorePath(drvPath);
writeDerivation(conn->to, *this, drv);
putBuildSettings(*conn);
conn->to.flush();
conn->putBuildDerivationRequest(*this, drvPath, drv, buildSettings());
return ServeProto::Serialise<BuildResult>::read(*this, *conn);
}
@ -288,7 +243,7 @@ void LegacySSHStore::buildPaths(const std::vector<DerivedPath> & drvPaths, Build
}
conn->to << ss;
putBuildSettings(*conn);
ServeProto::write(*this, *conn, buildSettings());
conn->to.flush();
@ -328,15 +283,8 @@ StorePathSet LegacySSHStore::queryValidPaths(const StorePathSet & paths,
SubstituteFlag maybeSubstitute)
{
auto conn(connections->get());
conn->to
<< ServeProto::Command::QueryValidPaths
<< false // lock
<< maybeSubstitute;
ServeProto::write(*this, *conn, paths);
conn->to.flush();
return ServeProto::Serialise<StorePathSet>::read(*this, *conn);
return conn->queryValidPaths(*this,
false, paths, maybeSubstitute);
}

6
src/libstore/legacy-ssh-store.hh
View file

@ -13,7 +13,7 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig
{
using CommonSSHStoreConfig::CommonSSHStoreConfig;
const Setting<Path> remoteProgram{this, "nix-store", "remote-program",
const Setting<Strings> remoteProgram{this, {"nix-store"}, "remote-program",
"Path to the `nix-store` executable on the remote machine."};
const Setting<int> maxConnections{this, 1, "max-connections",
@ -78,10 +78,6 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
RepairFlag repair = NoRepair) override
{ unsupported("addToStore"); }
private:
void putBuildSettings(Connection & conn);
public:
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,

2
src/libstore/local-fs-store.cc
View file

@ -28,7 +28,7 @@ struct LocalStoreAccessor : PosixSourceAccessor
auto [storePath, rest] = store->toStorePath(path.abs());
if (requireValidPath && !store->isValidPath(storePath))
throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
return CanonPath(store->getRealStoreDir()) + storePath.to_string() + CanonPath(rest);
return CanonPath(store->getRealStoreDir()) / storePath.to_string() / CanonPath(rest);
}
std::optional<Stat> maybeLstat(const CanonPath & path) override

8
src/libstore/local-store.cc
View file

@ -1048,8 +1048,12 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
bool narRead = false;
Finally cleanup = [&]() {
if (!narRead) {
NullParseSink sink;
parseDump(sink, source);
NullFileSystemObjectSink sink;
try {
parseDump(sink, source);
} catch (...) {
ignoreException();
}
}
};

24
src/libstore/machines.cc
View file

@ -32,11 +32,19 @@ Machine::Machine(decltype(storeUri) storeUri,
systemTypes(systemTypes),
sshKey(sshKey),
maxJobs(maxJobs),
speedFactor(std::max(1U, speedFactor)),
speedFactor(speedFactor == 0.0f ? 1.0f : std::move(speedFactor)),
supportedFeatures(supportedFeatures),
mandatoryFeatures(mandatoryFeatures),
sshPublicHostKey(sshPublicHostKey)
{}
{
if (speedFactor < 0.0)
throw UsageError("speed factor must be >= 0");
}
bool Machine::systemSupported(const std::string & system) const
{
return system == "builtin" || (systemTypes.count(system) > 0);
}
bool Machine::allSupported(const std::set<std::string> & features) const
{
@ -130,6 +138,14 @@ static Machine parseBuilderLine(const std::string & line)
return result.value();
};
auto parseFloatField = [&](size_t fieldIndex) {
const auto result = string2Int<float>(tokens[fieldIndex]);
if (!result) {
throw FormatError("bad machine specification: failed to convert column #%lu in a row: '%s' to 'float'", fieldIndex, line);
}
return result.value();
};
auto ensureBase64 = [&](size_t fieldIndex) {
const auto & str = tokens[fieldIndex];
try {
@ -145,10 +161,10 @@ static Machine parseBuilderLine(const std::string & line)
return {
tokens[0],
isSet(1) ? tokenizeString<std::vector<std::string>>(tokens[1], ",") : std::vector<std::string>{settings.thisSystem},
isSet(1) ? tokenizeString<std::set<std::string>>(tokens[1], ",") : std::set<std::string>{settings.thisSystem},
isSet(2) ? tokens[2] : "",
isSet(3) ? parseUnsignedIntField(3) : 1U,
isSet(4) ? parseUnsignedIntField(4) : 1U,
isSet(4) ? parseFloatField(4) : 1.0f,
isSet(5) ? tokenizeString<std::set<std::string>>(tokens[5], ",") : std::set<std::string>{},
isSet(6) ? tokenizeString<std::set<std::string>>(tokens[6], ",") : std::set<std::string>{},
isSet(7) ? ensureBase64(7) : ""

17
src/libstore/machines.hh
View file

@ -10,17 +10,30 @@ class Store;
struct Machine {
const std::string storeUri;
const std::vector<std::string> systemTypes;
const std::set<std::string> systemTypes;
const std::string sshKey;
const unsigned int maxJobs;
const unsigned int speedFactor;
const float speedFactor;
const std::set<std::string> supportedFeatures;
const std::set<std::string> mandatoryFeatures;
const std::string sshPublicHostKey;
bool enabled = true;
/**
* @return Whether `system` is either `"builtin"` or in
* `systemTypes`.
*/
bool systemSupported(const std::string & system) const;
/**
* @return Whether `features` is a subset of the union of `supportedFeatures` and
* `mandatoryFeatures`
*/
bool allSupported(const std::set<std::string> & features) const;
/**
* @return @Whether `mandatoryFeatures` is a subset of `features`
*/
bool mandatoryMet(const std::set<std::string> & features) const;
Machine(decltype(storeUri) storeUri,

64
src/libstore/nar-accessor.cc
View file

@ -19,6 +19,35 @@ struct NarMember
std::map<std::string, NarMember> children;
};
struct NarMemberConstructor : CreateRegularFileSink
{
private:
NarMember & narMember;
uint64_t & pos;
public:
NarMemberConstructor(NarMember & nm, uint64_t & pos)
: narMember(nm), pos(pos)
{ }
void isExecutable() override
{
narMember.stat.isExecutable = true;
}
void preallocateContents(uint64_t size) override
{
narMember.stat.fileSize = size;
narMember.stat.narOffset = pos;
}
void operator () (std::string_view data) override
{ }
};
struct NarAccessor : public SourceAccessor
{
std::optional<const std::string> nar;
@ -27,7 +56,7 @@ struct NarAccessor : public SourceAccessor
NarMember root;
struct NarIndexer : ParseSink, Source
struct NarIndexer : FileSystemObjectSink, Source
{
NarAccessor & acc;
Source & source;
@ -42,7 +71,7 @@ struct NarAccessor : public SourceAccessor
: acc(acc), source(source)
{ }
void createMember(const Path & path, NarMember member)
NarMember & createMember(const Path & path, NarMember member)
{
size_t level = std::count(path.begin(), path.end(), '/');
while (parents.size() > level) parents.pop();
@ -50,11 +79,14 @@ struct NarAccessor : public SourceAccessor
if (parents.empty()) {
acc.root = std::move(member);
parents.push(&acc.root);
return acc.root;
} else {
if (parents.top()->stat.type != Type::tDirectory)
throw Error("NAR file missing parent directory of path '%s'", path);
auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member));
parents.push(&result.first->second);
auto & ref = result.first->second;
parents.push(&ref);
return ref;
}
}
@ -68,34 +100,18 @@ struct NarAccessor : public SourceAccessor
} });
}
void createRegularFile(const Path & path) override
void createRegularFile(const Path & path, std::function<void(CreateRegularFileSink &)> func) override
{
createMember(path, NarMember{ .stat = {
auto & nm = createMember(path, NarMember{ .stat = {
.type = Type::tRegular,
.fileSize = 0,
.isExecutable = false,
.narOffset = 0
} });
NarMemberConstructor nmc { nm, pos };
func(nmc);
}
void closeRegularFile() override
{ }
void isExecutable() override
{
parents.top()->stat.isExecutable = true;
}
void preallocateContents(uint64_t size) override
{
auto & st = parents.top()->stat;
st.fileSize = size;
st.narOffset = pos;
}
void receiveContents(std::string_view data) override
{ }
void createSymlink(const Path & path, const std::string & target) override
{
createMember(path,
@ -261,7 +277,7 @@ json listNar(ref<SourceAccessor> accessor, const CanonPath & path, bool recurse)
json &res2 = obj["entries"];
for (const auto & [name, type] : accessor->readDirectory(path)) {
if (recurse) {
res2[name] = listNar(accessor, path + name, true);
res2[name] = listNar(accessor, path / name, true);
} else
res2[name] = json::object();
}

7
src/libstore/path-regex.hh
View file

@ -3,6 +3,11 @@
namespace nix {
static constexpr std::string_view nameRegexStr = R"([0-9a-zA-Z\+\-_\?=][0-9a-zA-Z\+\-\._\?=]*)";
static constexpr std::string_view nameRegexStr =
// This uses a negative lookahead: (?!\.\.?(-|$))
// - deny ".", "..", or those strings followed by '-'
// - when it's not those, start again at the start of the input and apply the next regex, which is [0-9a-zA-Z\+\-\._\?=]+
R"((?!\.\.?(-|$))[0-9a-zA-Z\+\-\._\?=]+)";
}

15
src/libstore/path.cc
View file

@ -9,9 +9,20 @@ static void checkName(std::string_view path, std::string_view name)
if (name.size() > StorePath::MaxPathLen)
throw BadStorePath("store path '%s' has a name longer than %d characters",
path, StorePath::MaxPathLen);
if (name[0] == '.')
throw BadStorePath("store path '%s' starts with illegal character '.'", path);
// See nameRegexStr for the definition
if (name[0] == '.') {
// check against "." and "..", followed by end or dash
if (name.size() == 1)
throw BadStorePath("store path '%s' has invalid name '%s'", path, name);
if (name[1] == '-')
throw BadStorePath("store path '%s' has invalid name '%s': first dash-separated component must not be '%s'", path, name, ".");
if (name[1] == '.') {
if (name.size() == 2)
throw BadStorePath("store path '%s' has invalid name '%s'", path, name);
if (name[2] == '-')
throw BadStorePath("store path '%s' has invalid name '%s': first dash-separated component must not be '%s'", path, name, "..");
}
}
for (auto c : name)
if (!((c >= '0' && c <= '9')
|| (c >= 'a' && c <= 'z')

69
src/libstore/serve-protocol-impl.cc Normal file
View file

@ -0,0 +1,69 @@
#include "serve-protocol-impl.hh"
#include "build-result.hh"
#include "derivations.hh"
namespace nix {
ServeProto::Version ServeProto::BasicClientConnection::handshake(
BufferedSink & to,
Source & from,
ServeProto::Version localVersion,
std::string_view host)
{
to << SERVE_MAGIC_1 << localVersion;
to.flush();
unsigned int magic = readInt(from);
if (magic != SERVE_MAGIC_2)
throw Error("'nix-store --serve' protocol mismatch from '%s'", host);
auto remoteVersion = readInt(from);
if (GET_PROTOCOL_MAJOR(remoteVersion) != 0x200)
throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host);
return remoteVersion;
}
ServeProto::Version ServeProto::BasicServerConnection::handshake(
BufferedSink & to,
Source & from,
ServeProto::Version localVersion)
{
unsigned int magic = readInt(from);
if (magic != SERVE_MAGIC_1) throw Error("protocol mismatch");
to << SERVE_MAGIC_2 << localVersion;
to.flush();
return readInt(from);
}
StorePathSet ServeProto::BasicClientConnection::queryValidPaths(
const Store & store,
bool lock, const StorePathSet & paths,
SubstituteFlag maybeSubstitute)
{
to
<< ServeProto::Command::QueryValidPaths
<< lock
<< maybeSubstitute;
write(store, *this, paths);
to.flush();
return Serialise<StorePathSet>::read(store, *this);
}
void ServeProto::BasicClientConnection::putBuildDerivationRequest(
const Store & store,
const StorePath & drvPath, const BasicDerivation & drv,
const ServeProto::BuildOptions & options)
{
to
<< ServeProto::Command::BuildDerivation
<< store.printStorePath(drvPath);
writeDerivation(to, store, drv);
ServeProto::write(store, *this, options);
to.flush();
}
}

98
src/libstore/serve-protocol-impl.hh
View file

@ -10,6 +10,7 @@
#include "serve-protocol.hh"
#include "length-prefixed-protocol-helper.hh"
#include "store-api.hh"
namespace nix {
@ -56,4 +57,101 @@ struct ServeProto::Serialise
/* protocol-specific templates */
struct ServeProto::BasicClientConnection
{
FdSink to;
FdSource from;
ServeProto::Version remoteVersion;
/**
* Establishes connection, negotiating version.
*
* @return the version provided by the other side of the
* connection.
*
* @param to Taken by reference to allow for various error handling
* mechanisms.
*
* @param from Taken by reference to allow for various error
* handling mechanisms.
*
* @param localVersion Our version which is sent over
*
* @param host Just used to add context to thrown exceptions.
*/
static ServeProto::Version handshake(
BufferedSink & to,
Source & from,
ServeProto::Version localVersion,
std::string_view host);
/**
* Coercion to `ServeProto::ReadConn`. This makes it easy to use the
* factored out serve protocol serializers with a
* `LegacySSHStore::Connection`.
*
* The serve protocol connection types are unidirectional, unlike
* this type.
*/
operator ServeProto::ReadConn ()
{
return ServeProto::ReadConn {
.from = from,
.version = remoteVersion,
};
}
/**
* Coercion to `ServeProto::WriteConn`. This makes it easy to use the
* factored out serve protocol serializers with a
* `LegacySSHStore::Connection`.
*
* The serve protocol connection types are unidirectional, unlike
* this type.
*/
operator ServeProto::WriteConn ()
{
return ServeProto::WriteConn {
.to = to,
.version = remoteVersion,
};
}
StorePathSet queryValidPaths(
const Store & remoteStore,
bool lock, const StorePathSet & paths,
SubstituteFlag maybeSubstitute);
/**
* Just the request half, because Hydra may do other things between
* issuing the request and reading the `BuildResult` response.
*/
void putBuildDerivationRequest(
const Store & store,
const StorePath & drvPath, const BasicDerivation & drv,
const ServeProto::BuildOptions & options);
};
struct ServeProto::BasicServerConnection
{
/**
* Establishes connection, negotiating version.
*
* @return the version provided by the other side of the
* connection.
*
* @param to Taken by reference to allow for various error handling
* mechanisms.
*
* @param from Taken by reference to allow for various error
* handling mechanisms.
*
* @param localVersion Our version which is sent over
*/
static ServeProto::Version handshake(
BufferedSink & to,
Source & from,
ServeProto::Version localVersion);
};
}

8
src/libstore/serve-protocol.hh
View file

@ -59,6 +59,14 @@ struct ServeProto
Version version;
};
/**
* Stripped down serialization logic suitable for sharing with Hydra.
*
* @todo remove once Hydra uses Store abstraction consistently.
*/
struct BasicClientConnection;
struct BasicServerConnection;
/**
* Data type for canonical pairs of serialisers for the serve protocol.
*

12
src/libstore/sqlite.cc
View file

@ -10,19 +10,19 @@
namespace nix {
SQLiteError::SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, hintformat && hf)
SQLiteError::SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, HintFmt && hf)
: Error(""), path(path), errMsg(errMsg), errNo(errNo), extendedErrNo(extendedErrNo), offset(offset)
{
auto offsetStr = (offset == -1) ? "" : "at offset " + std::to_string(offset) + ": ";
err.msg = hintfmt("%s: %s%s, %s (in '%s')",
normaltxt(hf.str()),
err.msg = HintFmt("%s: %s%s, %s (in '%s')",
Uncolored(hf.str()),
offsetStr,
sqlite3_errstr(extendedErrNo),
errMsg,
path ? path : "(in-memory)");
}
[[noreturn]] void SQLiteError::throw_(sqlite3 * db, hintformat && hf)
[[noreturn]] void SQLiteError::throw_(sqlite3 * db, HintFmt && hf)
{
int err = sqlite3_errcode(db);
int exterr = sqlite3_extended_errcode(db);
@ -33,7 +33,7 @@ SQLiteError::SQLiteError(const char *path, const char *errMsg, int errNo, int ex
if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) {
auto exp = SQLiteBusy(path, errMsg, err, exterr, offset, std::move(hf));
exp.err.msg = hintfmt(
exp.err.msg = HintFmt(
err == SQLITE_PROTOCOL
? "SQLite database '%s' is busy (SQLITE_PROTOCOL)"
: "SQLite database '%s' is busy",
@ -249,7 +249,7 @@ void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning)
if (now > nextWarning) {
nextWarning = now + 10;
logWarning({
.msg = hintfmt(e.what())
.msg = HintFmt(e.what())
});
}

8
src/libstore/sqlite.hh
View file

@ -142,19 +142,19 @@ struct SQLiteError : Error
template<typename... Args>
[[noreturn]] static void throw_(sqlite3 * db, const std::string & fs, const Args & ... args) {
throw_(db, hintfmt(fs, args...));
throw_(db, HintFmt(fs, args...));
}
SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, hintformat && hf);
SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, HintFmt && hf);
protected:
template<typename... Args>
SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, const std::string & fs, const Args & ... args)
: SQLiteError(path, errNo, extendedErrNo, offset, hintfmt(fs, args...))
: SQLiteError(path, errMsg, errNo, extendedErrNo, offset, HintFmt(fs, args...))
{ }
[[noreturn]] static void throw_(sqlite3 * db, hintformat && hf);
[[noreturn]] static void throw_(sqlite3 * db, HintFmt && hf);
};

19
src/libstore/ssh-store.cc
View file

@ -17,7 +17,7 @@ struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig
using RemoteStoreConfig::RemoteStoreConfig;
using CommonSSHStoreConfig::CommonSSHStoreConfig;
const Setting<Path> remoteProgram{this, "nix-daemon", "remote-program",
const Setting<Strings> remoteProgram{this, {"nix-daemon"}, "remote-program",
"Path to the `nix-daemon` executable on the remote machine."};
const std::string name() override { return "Experimental SSH Store"; }
@ -212,14 +212,15 @@ public:
ref<RemoteStore::Connection> SSHStore::openConnection()
{
auto conn = make_ref<Connection>();
std::string command = remoteProgram + " --stdio";
if (remoteStore.get() != "")
command += " --store " + shellEscape(remoteStore.get());
for (auto & arg : extraRemoteProgramArgs)
command += " " + shellEscape(arg);
conn->sshConn = master.startCommand(command);
Strings command = remoteProgram.get();
command.push_back("--stdio");
if (remoteStore.get() != "") {
command.push_back("--store");
command.push_back(remoteStore.get());
}
command.insert(command.end(),
extraRemoteProgramArgs.begin(), extraRemoteProgramArgs.end());
conn->sshConn = master.startCommand(std::move(command));
conn->to = FdSink(conn->sshConn->in.get());
conn->from = FdSource(conn->sshConn->out.get());
return conn;

12
src/libstore/ssh.cc
View file

@ -52,7 +52,8 @@ bool SSHMaster::isMasterRunning() {
return res.first == 0;
}
std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string & command)
std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(
Strings && command, Strings && extraSshArgs)
{
Path socketPath = startMaster();
@ -84,18 +85,19 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
Strings args;
if (fakeSSH) {
args = { "bash", "-c" };
} else {
if (!fakeSSH) {
args = { "ssh", host.c_str(), "-x" };
addCommonSSHOpts(args);
if (socketPath != "")
args.insert(args.end(), {"-S", socketPath});
if (verbosity >= lvlChatty)
args.push_back("-v");
args.splice(args.end(), std::move(extraSshArgs));
args.push_back("--");
}
args.push_back(command);
args.splice(args.end(), std::move(command));
execvp(args.begin()->c_str(), stringsToCharPtrs(args).data());
// could not exec ssh/bash

11
src/libstore/ssh.hh
View file

@ -41,7 +41,16 @@ public:
AutoCloseFD out, in;
};
std::unique_ptr<Connection> startCommand(const std::string & command);
/**
* @param command The command (arg vector) to execute.
*
* @param extraSShArgs Extra args to pass to SSH (not the command to
* execute). Will not be used when "fake SSHing" to the local
* machine.
*/
std::unique_ptr<Connection> startCommand(
Strings && command,
Strings && extraSshArgs = {});
Path startMaster();
};

83
src/libstore/store-api.cc
View file

@ -352,12 +352,12 @@ ValidPathInfo Store::addToStoreSlow(
information to narSink. */
TeeSource tapped { *fileSource, narSink };
NullParseSink blank;
NullFileSystemObjectSink blank;
auto & parseSink = method.getFileIngestionMethod() == FileIngestionMethod::Flat
? (ParseSink &) fileSink
? (FileSystemObjectSink &) fileSink
: method.getFileIngestionMethod() == FileIngestionMethod::Recursive
? (ParseSink &) blank
: (abort(), (ParseSink &)*(ParseSink *)nullptr); // handled both cases
? (FileSystemObjectSink &) blank
: (abort(), (FileSystemObjectSink &)*(FileSystemObjectSink *)nullptr); // handled both cases
/* The information that flows from tapped (besides being replicated in
narSink), is now put in parseSink. */
@ -614,38 +614,56 @@ static bool goodStorePath(const StorePath & expected, const StorePath & actual)
}
std::optional<std::shared_ptr<const ValidPathInfo>> Store::queryPathInfoFromClientCache(const StorePath & storePath)
{
auto hashPart = std::string(storePath.hashPart());
{
auto res = state.lock()->pathInfoCache.get(std::string(storePath.to_string()));
if (res && res->isKnownNow()) {
stats.narInfoReadAverted++;
if (res->didExist())
return std::make_optional(res->value);
else
return std::make_optional(nullptr);
}
}
if (diskCache) {
auto res = diskCache->lookupNarInfo(getUri(), hashPart);
if (res.first != NarInfoDiskCache::oUnknown) {
stats.narInfoReadAverted++;
{
auto state_(state.lock());
state_->pathInfoCache.upsert(std::string(storePath.to_string()),
res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second });
if (res.first == NarInfoDiskCache::oInvalid ||
!goodStorePath(storePath, res.second->path))
return std::make_optional(nullptr);
}
assert(res.second);
return std::make_optional(res.second);
}
}
return std::nullopt;
}
void Store::queryPathInfo(const StorePath & storePath,
Callback<ref<const ValidPathInfo>> callback) noexcept
{
auto hashPart = std::string(storePath.hashPart());
try {
{
auto res = state.lock()->pathInfoCache.get(std::string(storePath.to_string()));
if (res && res->isKnownNow()) {
stats.narInfoReadAverted++;
if (!res->didExist())
throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
return callback(ref<const ValidPathInfo>(res->value));
}
auto r = queryPathInfoFromClientCache(storePath);
if (r.has_value()) {
std::shared_ptr<const ValidPathInfo> & info = *r;
if (info)
return callback(ref(info));
else
throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
}
if (diskCache) {
auto res = diskCache->lookupNarInfo(getUri(), hashPart);
if (res.first != NarInfoDiskCache::oUnknown) {
stats.narInfoReadAverted++;
{
auto state_(state.lock());
state_->pathInfoCache.upsert(std::string(storePath.to_string()),
res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second });
if (res.first == NarInfoDiskCache::oInvalid ||
!goodStorePath(storePath, res.second->path))
throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
}
return callback(ref<const ValidPathInfo>(res.second));
}
}
} catch (...) { return callback.rethrow(); }
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
@ -757,7 +775,7 @@ void Store::substitutePaths(const StorePathSet & paths)
if (!willSubstitute.empty())
try {
std::vector<DerivedPath> subs;
for (auto & p : willSubstitute) subs.push_back(DerivedPath::Opaque{p});
for (auto & p : willSubstitute) subs.emplace_back(DerivedPath::Opaque{p});
buildPaths(subs);
} catch (Error & e) {
logWarning(e.info());
@ -909,6 +927,11 @@ void copyStorePath(
RepairFlag repair,
CheckSigsFlag checkSigs)
{
/* Bail out early (before starting a download from srcStore) if
dstStore already has this path. */
if (!repair && dstStore.isValidPath(storePath))
return;
auto srcUri = srcStore.getUri();
auto dstUri = dstStore.getUri();
auto storePathS = srcStore.printStorePath(storePath);

12
src/libstore/store-api.hh
View file

@ -108,7 +108,7 @@ struct StoreConfig : public StoreDirConfig
StoreConfig() = delete;
StringSet getDefaultSystemFeatures();
static StringSet getDefaultSystemFeatures();
virtual ~StoreConfig() { }
@ -282,6 +282,16 @@ public:
void queryPathInfo(const StorePath & path,
Callback<ref<const ValidPathInfo>> callback) noexcept;
/**
* Version of queryPathInfo() that only queries the local narinfo cache and not
* the actual store.
*
* @return `std::nullopt` if nothing is known about the path in the local narinfo cache.
* @return `std::make_optional(nullptr)` if the path is known to not exist.
* @return `std::make_optional(validPathInfo)` if the path is known to exist.
*/
std::optional<std::shared_ptr<const ValidPathInfo>> queryPathInfoFromClientCache(const StorePath & path);
/**
* Query the information about a realisation.
*/