mirror of
https://github.com/NixOS/nix
synced 2025-07-03 06:11:46 +02:00
Merge remote-tracking branch 'origin/master' into large-path-warning
This commit is contained in:
Eelco Dolstra
commit
7f5b57d18f
435 changed files with 6086 additions and 2767 deletions
|
|
@ -18,6 +18,7 @@
|
|||
#include <future>
|
||||
#include <regex>
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
#include "derivation-goal.hh"
|
||||
#include "hook-instance.hh"
|
||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||
# include "hook-instance.hh"
|
||||
#endif
|
||||
#include "processes.hh"
|
||||
#include "worker.hh"
|
||||
#include "builtins.hh"
|
||||
#include "builtins/buildenv.hh"
|
||||
|
|
@ -19,19 +22,8 @@
|
|||
|
||||
#include <fstream>
|
||||
#include <sys/types.h>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/un.h>
|
||||
#include <sys/wait.h>
|
||||
#include <netdb.h>
|
||||
#include <fcntl.h>
|
||||
#include <termios.h>
|
||||
#include <unistd.h>
|
||||
#include <sys/mman.h>
|
||||
#include <sys/utsname.h>
|
||||
#include <sys/resource.h>
|
||||
|
||||
#include <pwd.h>
|
||||
#include <grp.h>
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
|
|
@ -101,7 +93,9 @@ std::string DerivationGoal::key()
|
|||
|
||||
void DerivationGoal::killChild()
|
||||
{
|
||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||
hook.reset();
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -641,9 +635,17 @@ void DerivationGoal::started()
|
|||
buildMode == bmCheck ? "checking outputs of '%s'" :
|
||||
"building '%s'", worker.store.printStorePath(drvPath));
|
||||
fmt("building '%s'", worker.store.printStorePath(drvPath));
|
||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||
if (hook) msg += fmt(" on '%s'", machineName);
|
||||
#endif
|
||||
act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg,
|
||||
Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", 1, 1});
|
||||
Logger::Fields{worker.store.printStorePath(drvPath),
|
||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||
hook ? machineName :
|
||||
#endif
|
||||
"",
|
||||
1,
|
||||
1});
|
||||
mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
|
||||
worker.updateProgress();
|
||||
}
|
||||
|
|
@ -778,12 +780,18 @@ static void movePath(const Path & src, const Path & dst)
|
|||
{
|
||||
auto st = lstat(src);
|
||||
|
||||
bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));
|
||||
bool changePerm = (
|
||||
#ifndef _WIN32
|
||||
geteuid()
|
||||
#else
|
||||
!isRootUser()
|
||||
#endif
|
||||
&& S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));
|
||||
|
||||
if (changePerm)
|
||||
chmod_(src, st.st_mode | S_IWUSR);
|
||||
|
||||
renameFile(src, dst);
|
||||
std::filesystem::rename(src, dst);
|
||||
|
||||
if (changePerm)
|
||||
chmod_(dst, st.st_mode);
|
||||
|
|
@ -796,7 +804,7 @@ void replaceValidPath(const Path & storePath, const Path & tmpPath)
|
|||
tmpPath (the replacement), so we have to move it out of the
|
||||
way first. We'd better not be interrupted here, because if
|
||||
we're repairing (say) Glibc, we end up with a broken system. */
|
||||
Path oldPath = fmt("%1%.old-%2%-%3%", storePath, getpid(), random());
|
||||
Path oldPath = fmt("%1%.old-%2%-%3%", storePath, getpid(), rand());
|
||||
if (pathExists(storePath))
|
||||
movePath(storePath, oldPath);
|
||||
|
||||
|
|
@ -818,14 +826,20 @@ void replaceValidPath(const Path & storePath, const Path & tmpPath)
|
|||
|
||||
int DerivationGoal::getChildStatus()
|
||||
{
|
||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||
return hook->pid.kill();
|
||||
#else
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
void DerivationGoal::closeReadPipes()
|
||||
{
|
||||
hook->builderOut.readSide = -1;
|
||||
hook->fromHook.readSide = -1;
|
||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||
hook->builderOut.readSide.close();
|
||||
hook->fromHook.readSide.close();
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -1019,13 +1033,16 @@ void DerivationGoal::buildDone()
|
|||
|
||||
BuildResult::Status st = BuildResult::MiscFailure;
|
||||
|
||||
#ifndef _WIN32
|
||||
if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101)
|
||||
st = BuildResult::TimedOut;
|
||||
|
||||
else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) {
|
||||
}
|
||||
|
||||
else {
|
||||
else
|
||||
#endif
|
||||
{
|
||||
assert(derivationType);
|
||||
st =
|
||||
dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic :
|
||||
|
|
@ -1112,6 +1129,9 @@ void DerivationGoal::resolvedFinished()
|
|||
|
||||
HookReply DerivationGoal::tryBuildHook()
|
||||
{
|
||||
#ifdef _WIN32 // TODO enable build hook on Windows
|
||||
return rpDecline;
|
||||
#else
|
||||
if (settings.buildHook.get().empty() || !worker.tryBuildHook || !useDerivation) return rpDecline;
|
||||
|
||||
if (!worker.hook)
|
||||
|
|
@ -1205,17 +1225,18 @@ HookReply DerivationGoal::tryBuildHook()
|
|||
}
|
||||
|
||||
hook->sink = FdSink();
|
||||
hook->toHook.writeSide = -1;
|
||||
hook->toHook.writeSide.close();
|
||||
|
||||
/* Create the log file and pipe. */
|
||||
Path logFile = openLogFile();
|
||||
|
||||
std::set<int> fds;
|
||||
std::set<MuxablePipePollState::CommChannel> fds;
|
||||
fds.insert(hook->fromHook.readSide.get());
|
||||
fds.insert(hook->builderOut.readSide.get());
|
||||
worker.childStarted(shared_from_this(), fds, false, false);
|
||||
|
||||
return rpAccept;
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -1251,7 +1272,11 @@ Path DerivationGoal::openLogFile()
|
|||
Path logFileName = fmt("%s/%s%s", dir, baseName.substr(2),
|
||||
settings.compressLog ? ".bz2" : "");
|
||||
|
||||
fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666);
|
||||
fdLogFile = toDescriptor(open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC
|
||||
#ifndef _WIN32
|
||||
| O_CLOEXEC
|
||||
#endif
|
||||
, 0666));
|
||||
if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName);
|
||||
|
||||
logFileSink = std::make_shared<FdSink>(fdLogFile.get());
|
||||
|
|
@ -1271,16 +1296,20 @@ void DerivationGoal::closeLogFile()
|
|||
if (logSink2) logSink2->finish();
|
||||
if (logFileSink) logFileSink->flush();
|
||||
logSink = logFileSink = 0;
|
||||
fdLogFile = -1;
|
||||
fdLogFile.close();
|
||||
}
|
||||
|
||||
|
||||
bool DerivationGoal::isReadDesc(int fd)
|
||||
bool DerivationGoal::isReadDesc(Descriptor fd)
|
||||
{
|
||||
#ifdef _WIN32 // TODO enable build hook on Windows
|
||||
return false;
|
||||
#else
|
||||
return fd == hook->builderOut.readSide.get();
|
||||
#endif
|
||||
}
|
||||
|
||||
void DerivationGoal::handleChildOutput(int fd, std::string_view data)
|
||||
void DerivationGoal::handleChildOutput(Descriptor fd, std::string_view data)
|
||||
{
|
||||
// local & `ssh://`-builds are dealt with here.
|
||||
auto isWrittenToLog = isReadDesc(fd);
|
||||
|
|
@ -1310,6 +1339,7 @@ void DerivationGoal::handleChildOutput(int fd, std::string_view data)
|
|||
if (logSink) (*logSink)(data);
|
||||
}
|
||||
|
||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||
if (hook && fd == hook->fromHook.readSide.get()) {
|
||||
for (auto c : data)
|
||||
if (c == '\n') {
|
||||
|
|
@ -1344,10 +1374,11 @@ void DerivationGoal::handleChildOutput(int fd, std::string_view data)
|
|||
} else
|
||||
currentHookLine += c;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
void DerivationGoal::handleEOF(int fd)
|
||||
void DerivationGoal::handleEOF(Descriptor fd)
|
||||
{
|
||||
if (!currentLogLine.empty()) flushLine();
|
||||
worker.wakeUp(shared_from_this());
|
||||
|
|
@ -2,7 +2,9 @@
|
|||
///@file
|
||||
|
||||
#include "parsed-derivations.hh"
|
||||
#include "lock.hh"
|
||||
#ifndef _WIN32
|
||||
# include "user-lock.hh"
|
||||
#endif
|
||||
#include "outputs-spec.hh"
|
||||
#include "store-api.hh"
|
||||
#include "pathlocks.hh"
|
||||
|
|
@ -12,7 +14,9 @@ namespace nix {
|
|||
|
||||
using std::map;
|
||||
|
||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||
struct HookInstance;
|
||||
#endif
|
||||
|
||||
typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
|
||||
|
||||
|
|
@ -178,10 +182,12 @@ struct DerivationGoal : public Goal
|
|||
|
||||
std::string currentHookLine;
|
||||
|
||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||
/**
|
||||
* The build hook.
|
||||
*/
|
||||
std::unique_ptr<HookInstance> hook;
|
||||
#endif
|
||||
|
||||
/**
|
||||
* The sort of derivation we are building.
|
||||
|
|
@ -287,13 +293,13 @@ struct DerivationGoal : public Goal
|
|||
virtual void cleanupPostOutputsRegisteredModeCheck();
|
||||
virtual void cleanupPostOutputsRegisteredModeNonCheck();
|
||||
|
||||
virtual bool isReadDesc(int fd);
|
||||
virtual bool isReadDesc(Descriptor fd);
|
||||
|
||||
/**
|
||||
* Callback used by the worker to write to the log.
|
||||
*/
|
||||
void handleChildOutput(int fd, std::string_view data) override;
|
||||
void handleEOF(int fd) override;
|
||||
void handleChildOutput(Descriptor fd, std::string_view data) override;
|
||||
void handleEOF(Descriptor fd) override;
|
||||
void flushLine();
|
||||
|
||||
/**
|
||||
|
|
@ -66,7 +66,11 @@ void DrvOutputSubstitutionGoal::tryNext()
|
|||
some other error occurs), so it must not touch `this`. So put
|
||||
the shared state in a separate refcounted object. */
|
||||
downloadState = std::make_shared<DownloadState>();
|
||||
#ifndef _WIN32
|
||||
downloadState->outPipe.create();
|
||||
#else
|
||||
downloadState->outPipe.createAsyncPipe(worker.ioport.get());
|
||||
#endif
|
||||
|
||||
sub->queryRealisation(
|
||||
id,
|
||||
|
|
@ -79,7 +83,13 @@ void DrvOutputSubstitutionGoal::tryNext()
|
|||
}
|
||||
} });
|
||||
|
||||
worker.childStarted(shared_from_this(), {downloadState->outPipe.readSide.get()}, true, false);
|
||||
worker.childStarted(shared_from_this(), {
|
||||
#ifndef _WIN32
|
||||
downloadState->outPipe.readSide.get()
|
||||
#else
|
||||
&downloadState->outPipe
|
||||
#endif
|
||||
}, true, false);
|
||||
|
||||
state = &DrvOutputSubstitutionGoal::realisationFetched;
|
||||
}
|
||||
|
|
@ -158,7 +168,7 @@ void DrvOutputSubstitutionGoal::work()
|
|||
(this->*state)();
|
||||
}
|
||||
|
||||
void DrvOutputSubstitutionGoal::handleEOF(int fd)
|
||||
void DrvOutputSubstitutionGoal::handleEOF(Descriptor fd)
|
||||
{
|
||||
if (fd == downloadState->outPipe.readSide.get()) worker.wakeUp(shared_from_this());
|
||||
}
|
||||
|
|
@ -1,11 +1,13 @@
|
|||
#pragma once
|
||||
///@file
|
||||
|
||||
#include <thread>
|
||||
#include <future>
|
||||
|
||||
#include "store-api.hh"
|
||||
#include "goal.hh"
|
||||
#include "realisation.hh"
|
||||
#include <thread>
|
||||
#include <future>
|
||||
#include "muxable-pipe.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
|
@ -43,7 +45,7 @@ class DrvOutputSubstitutionGoal : public Goal {
|
|||
|
||||
struct DownloadState
|
||||
{
|
||||
Pipe outPipe;
|
||||
MuxablePipe outPipe;
|
||||
std::promise<std::shared_ptr<const Realisation>> promise;
|
||||
};
|
||||
|
||||
|
|
@ -71,7 +73,7 @@ public:
|
|||
std::string key() override;
|
||||
|
||||
void work() override;
|
||||
void handleEOF(int fd) override;
|
||||
void handleEOF(Descriptor fd) override;
|
||||
|
||||
JobCategory jobCategory() const override {
|
||||
return JobCategory::Substitution;
|
||||
|
|
@ -1,6 +1,8 @@
|
|||
#include "worker.hh"
|
||||
#include "substitution-goal.hh"
|
||||
#include "derivation-goal.hh"
|
||||
#ifndef _WIN32 // TODO Enable building on Windows
|
||||
# include "derivation-goal.hh"
|
||||
#endif
|
||||
#include "local-store.hh"
|
||||
|
||||
namespace nix {
|
||||
|
|
@ -25,9 +27,12 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
|
|||
ex = std::move(i->ex);
|
||||
}
|
||||
if (i->exitCode != Goal::ecSuccess) {
|
||||
#ifndef _WIN32 // TODO Enable building on Windows
|
||||
if (auto i2 = dynamic_cast<DerivationGoal *>(i.get()))
|
||||
failed.insert(printStorePath(i2->drvPath));
|
||||
else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get()))
|
||||
else
|
||||
#endif
|
||||
if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get()))
|
||||
failed.insert(printStorePath(i2->storePath));
|
||||
}
|
||||
}
|
||||
|
|
@ -74,7 +79,12 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat
|
|||
BuildMode buildMode)
|
||||
{
|
||||
Worker worker(*this, *this);
|
||||
#ifndef _WIN32 // TODO Enable building on Windows
|
||||
auto goal = worker.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All {}, buildMode);
|
||||
#else
|
||||
std::shared_ptr<Goal> goal;
|
||||
throw UnimplementedError("Building derivations not yet implemented on windows.");
|
||||
#endif
|
||||
|
||||
try {
|
||||
worker.run(Goals{goal});
|
||||
|
|
@ -138,12 +138,12 @@ public:
|
|||
|
||||
virtual void waiteeDone(GoalPtr waitee, ExitCode result);
|
||||
|
||||
virtual void handleChildOutput(int fd, std::string_view data)
|
||||
virtual void handleChildOutput(Descriptor fd, std::string_view data)
|
||||
{
|
||||
abort();
|
||||
}
|
||||
|
||||
virtual void handleEOF(int fd)
|
||||
virtual void handleEOF(Descriptor fd)
|
||||
{
|
||||
abort();
|
||||
}
|
||||
|
|
@ -212,7 +212,11 @@ void PathSubstitutionGoal::tryToRun()
|
|||
maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions);
|
||||
worker.updateProgress();
|
||||
|
||||
#ifndef _WIN32
|
||||
outPipe.create();
|
||||
#else
|
||||
outPipe.createAsyncPipe(worker.ioport.get());
|
||||
#endif
|
||||
|
||||
promise = std::promise<void>();
|
||||
|
||||
|
|
@ -235,7 +239,13 @@ void PathSubstitutionGoal::tryToRun()
|
|||
}
|
||||
});
|
||||
|
||||
worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false);
|
||||
worker.childStarted(shared_from_this(), {
|
||||
#ifndef _WIN32
|
||||
outPipe.readSide.get()
|
||||
#else
|
||||
&outPipe
|
||||
#endif
|
||||
}, true, false);
|
||||
|
||||
state = &PathSubstitutionGoal::finished;
|
||||
}
|
||||
|
|
@ -294,12 +304,12 @@ void PathSubstitutionGoal::finished()
|
|||
}
|
||||
|
||||
|
||||
void PathSubstitutionGoal::handleChildOutput(int fd, std::string_view data)
|
||||
void PathSubstitutionGoal::handleChildOutput(Descriptor fd, std::string_view data)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
void PathSubstitutionGoal::handleEOF(int fd)
|
||||
void PathSubstitutionGoal::handleEOF(Descriptor fd)
|
||||
{
|
||||
if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
|
||||
}
|
||||
|
|
@ -1,9 +1,9 @@
|
|||
#pragma once
|
||||
///@file
|
||||
|
||||
#include "lock.hh"
|
||||
#include "store-api.hh"
|
||||
#include "goal.hh"
|
||||
#include "muxable-pipe.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
|
@ -45,7 +45,7 @@ struct PathSubstitutionGoal : public Goal
|
|||
/**
|
||||
* Pipe for the substituter's standard output.
|
||||
*/
|
||||
Pipe outPipe;
|
||||
MuxablePipe outPipe;
|
||||
|
||||
/**
|
||||
* The substituter thread.
|
||||
|
|
@ -111,8 +111,8 @@ public:
|
|||
/**
|
||||
* Callback used by the worker to write to the log.
|
||||
*/
|
||||
void handleChildOutput(int fd, std::string_view data) override;
|
||||
void handleEOF(int fd) override;
|
||||
void handleChildOutput(Descriptor fd, std::string_view data) override;
|
||||
void handleEOF(Descriptor fd) override;
|
||||
|
||||
/* Called by destructor, can't be overridden */
|
||||
void cleanup() override final;
|
||||
|
|
@ -1,13 +1,15 @@
|
|||
#include "local-store.hh"
|
||||
#include "machines.hh"
|
||||
#include "worker.hh"
|
||||
#include "substitution-goal.hh"
|
||||
#include "drv-output-substitution-goal.hh"
|
||||
#include "local-derivation-goal.hh"
|
||||
#include "hook-instance.hh"
|
||||
#include "derivation-goal.hh"
|
||||
#ifndef _WIN32 // TODO Enable building on Windows
|
||||
# include "local-derivation-goal.hh"
|
||||
# include "hook-instance.hh"
|
||||
#endif
|
||||
#include "signals.hh"
|
||||
|
||||
#include <poll.h>
|
||||
|
||||
namespace nix {
|
||||
|
||||
Worker::Worker(Store & store, Store & evalStore)
|
||||
|
|
@ -64,20 +66,27 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drv
|
|||
const OutputsSpec & wantedOutputs, BuildMode buildMode)
|
||||
{
|
||||
return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
|
||||
return !dynamic_cast<LocalStore *>(&store)
|
||||
? std::make_shared</* */DerivationGoal>(drvPath, wantedOutputs, *this, buildMode)
|
||||
: std::make_shared<LocalDerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
|
||||
return
|
||||
#ifndef _WIN32 // TODO Enable building on Windows
|
||||
dynamic_cast<LocalStore *>(&store)
|
||||
? std::make_shared<LocalDerivationGoal>(drvPath, wantedOutputs, *this, buildMode)
|
||||
:
|
||||
#endif
|
||||
std::make_shared</* */DerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
|
||||
const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode)
|
||||
{
|
||||
return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
|
||||
return !dynamic_cast<LocalStore *>(&store)
|
||||
? std::make_shared</* */DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode)
|
||||
: std::make_shared<LocalDerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
|
||||
return
|
||||
#ifndef _WIN32 // TODO Enable building on Windows
|
||||
dynamic_cast<LocalStore *>(&store)
|
||||
? std::make_shared<LocalDerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode)
|
||||
:
|
||||
#endif
|
||||
std::make_shared</* */DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -143,7 +152,8 @@ void Worker::removeGoal(GoalPtr goal)
|
|||
{
|
||||
if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
|
||||
nix::removeGoal(drvGoal, derivationGoals);
|
||||
else if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
|
||||
else
|
||||
if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
|
||||
nix::removeGoal(subGoal, substitutionGoals);
|
||||
else if (auto subGoal = std::dynamic_pointer_cast<DrvOutputSubstitutionGoal>(goal))
|
||||
nix::removeGoal(subGoal, drvOutputSubstitutionGoals);
|
||||
|
|
@ -187,13 +197,13 @@ unsigned Worker::getNrSubstitutions()
|
|||
}
|
||||
|
||||
|
||||
void Worker::childStarted(GoalPtr goal, const std::set<int> & fds,
|
||||
void Worker::childStarted(GoalPtr goal, const std::set<MuxablePipePollState::CommChannel> & channels,
|
||||
bool inBuildSlot, bool respectTimeouts)
|
||||
{
|
||||
Child child;
|
||||
child.goal = goal;
|
||||
child.goal2 = goal.get();
|
||||
child.fds = fds;
|
||||
child.channels = channels;
|
||||
child.timeStarted = child.lastOutput = steady_time_point::clock::now();
|
||||
child.inBuildSlot = inBuildSlot;
|
||||
child.respectTimeouts = respectTimeouts;
|
||||
|
|
@ -286,7 +296,8 @@ void Worker::run(const Goals & _topGoals)
|
|||
.drvPath = makeConstantStorePathRef(goal->drvPath),
|
||||
.outputs = goal->wantedOutputs,
|
||||
});
|
||||
} else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
|
||||
} else
|
||||
if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
|
||||
topPaths.push_back(DerivedPath::Opaque{goal->storePath});
|
||||
}
|
||||
}
|
||||
|
|
@ -408,23 +419,25 @@ void Worker::waitForInput()
|
|||
if (useTimeout)
|
||||
vomit("sleeping %d seconds", timeout);
|
||||
|
||||
MuxablePipePollState state;
|
||||
|
||||
#ifndef _WIN32
|
||||
/* Use select() to wait for the input side of any logger pipe to
|
||||
become `available'. Note that `available' (i.e., non-blocking)
|
||||
includes EOF. */
|
||||
std::vector<struct pollfd> pollStatus;
|
||||
std::map<int, size_t> fdToPollStatus;
|
||||
for (auto & i : children) {
|
||||
for (auto & j : i.fds) {
|
||||
pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN });
|
||||
fdToPollStatus[j] = pollStatus.size() - 1;
|
||||
for (auto & j : i.channels) {
|
||||
state.pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN });
|
||||
state.fdToPollStatus[j] = state.pollStatus.size() - 1;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
if (poll(pollStatus.data(), pollStatus.size(),
|
||||
useTimeout ? timeout * 1000 : -1) == -1) {
|
||||
if (errno == EINTR) return;
|
||||
throw SysError("waiting for input");
|
||||
}
|
||||
state.poll(
|
||||
#ifdef _WIN32
|
||||
ioport.get(),
|
||||
#endif
|
||||
useTimeout ? (std::optional { timeout * 1000 }) : std::nullopt);
|
||||
|
||||
auto after = steady_time_point::clock::now();
|
||||
|
||||
|
|
@ -439,32 +452,18 @@ void Worker::waitForInput()
|
|||
GoalPtr goal = j->goal.lock();
|
||||
assert(goal);
|
||||
|
||||
std::set<int> fds2(j->fds);
|
||||
std::vector<unsigned char> buffer(4096);
|
||||
for (auto & k : fds2) {
|
||||
const auto fdPollStatusId = get(fdToPollStatus, k);
|
||||
assert(fdPollStatusId);
|
||||
assert(*fdPollStatusId < pollStatus.size());
|
||||
if (pollStatus.at(*fdPollStatusId).revents) {
|
||||
ssize_t rd = ::read(k, buffer.data(), buffer.size());
|
||||
// FIXME: is there a cleaner way to handle pt close
|
||||
// than EIO? Is this even standard?
|
||||
if (rd == 0 || (rd == -1 && errno == EIO)) {
|
||||
debug("%1%: got EOF", goal->getName());
|
||||
goal->handleEOF(k);
|
||||
j->fds.erase(k);
|
||||
} else if (rd == -1) {
|
||||
if (errno != EINTR)
|
||||
throw SysError("%s: read failed", goal->getName());
|
||||
} else {
|
||||
printMsg(lvlVomit, "%1%: read %2% bytes",
|
||||
goal->getName(), rd);
|
||||
std::string_view data((char *) buffer.data(), rd);
|
||||
j->lastOutput = after;
|
||||
goal->handleChildOutput(k, data);
|
||||
}
|
||||
}
|
||||
}
|
||||
state.iterate(
|
||||
j->channels,
|
||||
[&](Descriptor k, std::string_view data) {
|
||||
printMsg(lvlVomit, "%1%: read %2% bytes",
|
||||
goal->getName(), data.size());
|
||||
j->lastOutput = after;
|
||||
goal->handleChildOutput(k, data);
|
||||
},
|
||||
[&](Descriptor k) {
|
||||
debug("%1%: got EOF", goal->getName());
|
||||
goal->handleEOF(k);
|
||||
});
|
||||
|
||||
if (goal->exitCode == Goal::ecBusy &&
|
||||
0 != settings.maxSilentTime &&
|
||||
|
|
@ -2,10 +2,10 @@
|
|||
///@file
|
||||
|
||||
#include "types.hh"
|
||||
#include "lock.hh"
|
||||
#include "store-api.hh"
|
||||
#include "goal.hh"
|
||||
#include "realisation.hh"
|
||||
#include "muxable-pipe.hh"
|
||||
|
||||
#include <future>
|
||||
#include <thread>
|
||||
|
|
@ -36,14 +36,14 @@ typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
|
|||
|
||||
/**
|
||||
* A mapping used to remember for each child process to what goal it
|
||||
* belongs, and file descriptors for receiving log data and output
|
||||
* belongs, and comm channels for receiving log data and output
|
||||
* path creation commands.
|
||||
*/
|
||||
struct Child
|
||||
{
|
||||
WeakGoalPtr goal;
|
||||
Goal * goal2; // ugly hackery
|
||||
std::set<int> fds;
|
||||
std::set<MuxablePipePollState::CommChannel> channels;
|
||||
bool respectTimeouts;
|
||||
bool inBuildSlot;
|
||||
/**
|
||||
|
|
@ -53,8 +53,10 @@ struct Child
|
|||
steady_time_point timeStarted;
|
||||
};
|
||||
|
||||
#ifndef _WIN32 // TODO Enable building on Windows
|
||||
/* Forward definition. */
|
||||
struct HookInstance;
|
||||
#endif
|
||||
|
||||
/**
|
||||
* The worker class.
|
||||
|
|
@ -152,10 +154,16 @@ public:
|
|||
*/
|
||||
bool checkMismatch;
|
||||
|
||||
#ifdef _WIN32
|
||||
AutoCloseFD ioport;
|
||||
#endif
|
||||
|
||||
Store & store;
|
||||
Store & evalStore;
|
||||
|
||||
#ifndef _WIN32 // TODO Enable building on Windows
|
||||
std::unique_ptr<HookInstance> hook;
|
||||
#endif
|
||||
|
||||
uint64_t expectedBuilds = 0;
|
||||
uint64_t doneBuilds = 0;
|
||||
|
|
@ -238,7 +246,7 @@ public:
|
|||
* Registers a running child process. `inBuildSlot` means that
|
||||
* the process counts towards the jobs limit.
|
||||
*/
|
||||
void childStarted(GoalPtr goal, const std::set<int> & fds,
|
||||
void childStarted(GoalPtr goal, const std::set<MuxablePipePollState::CommChannel> & channels,
|
||||
bool inBuildSlot, bool respectTimeouts);
|
||||
|
||||
/**
|
||||
|
|
@ -17,10 +17,10 @@ struct State
|
|||
/* For each activated package, create symlinks */
|
||||
static void createLinks(State & state, const Path & srcDir, const Path & dstDir, int priority)
|
||||
{
|
||||
std::vector<std::filesystem::directory_entry> srcFiles;
|
||||
std::filesystem::directory_iterator srcFiles;
|
||||
|
||||
try {
|
||||
srcFiles = readDirectory(srcDir);
|
||||
srcFiles = std::filesystem::directory_iterator{srcDir};
|
||||
} catch (std::filesystem::filesystem_error & e) {
|
||||
if (e.code() == std::errc::not_a_directory) {
|
||||
warn("not including '%s' in the user environment because it's not a directory", srcDir);
|
||||
|
|
|
|||
|
|
@ -21,10 +21,13 @@ void builtinUnpackChannel(
|
|||
|
||||
unpackTarfile(src, out);
|
||||
|
||||
auto entries = readDirectory(out);
|
||||
if (entries.size() != 1)
|
||||
auto entries = std::filesystem::directory_iterator{out};
|
||||
auto fileName = entries->path().string();
|
||||
auto fileCount = std::distance(std::filesystem::begin(entries), std::filesystem::end(entries));
|
||||
|
||||
if (fileCount != 1)
|
||||
throw Error("channel tarball '%s' contains more than one file", src);
|
||||
renameFile(entries[0].path().string(), (out + "/" + channelName));
|
||||
std::filesystem::rename(fileName, (out + "/" + channelName));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
#include "daemon.hh"
|
||||
#include "signals.hh"
|
||||
#include "worker-protocol.hh"
|
||||
#include "worker-protocol-connection.hh"
|
||||
#include "worker-protocol-impl.hh"
|
||||
#include "build-result.hh"
|
||||
#include "store-api.hh"
|
||||
|
|
@ -19,6 +20,8 @@
|
|||
# include "monitor-fd.hh"
|
||||
#endif
|
||||
|
||||
#include <sstream>
|
||||
|
||||
namespace nix::daemon {
|
||||
|
||||
Sink & operator << (Sink & sink, const Logger::Fields & fields)
|
||||
|
|
@ -531,7 +534,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
auto drvs = WorkerProto::Serialise<DerivedPaths>::read(*store, rconn);
|
||||
BuildMode mode = bmNormal;
|
||||
if (GET_PROTOCOL_MINOR(clientVersion) >= 15) {
|
||||
mode = (BuildMode) readInt(from);
|
||||
mode = WorkerProto::Serialise<BuildMode>::read(*store, rconn);
|
||||
|
||||
/* Repairing is not atomic, so disallowed for "untrusted"
|
||||
clients.
|
||||
|
|
@ -555,7 +558,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
case WorkerProto::Op::BuildPathsWithResults: {
|
||||
auto drvs = WorkerProto::Serialise<DerivedPaths>::read(*store, rconn);
|
||||
BuildMode mode = bmNormal;
|
||||
mode = (BuildMode) readInt(from);
|
||||
mode = WorkerProto::Serialise<BuildMode>::read(*store, rconn);
|
||||
|
||||
/* Repairing is not atomic, so disallowed for "untrusted"
|
||||
clients.
|
||||
|
|
@ -586,7 +589,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
* correctly.
|
||||
*/
|
||||
readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath));
|
||||
BuildMode buildMode = (BuildMode) readInt(from);
|
||||
auto buildMode = WorkerProto::Serialise<BuildMode>::read(*store, rconn);
|
||||
logger->startWork();
|
||||
|
||||
auto drvType = drv.type();
|
||||
|
|
@ -1026,11 +1029,9 @@ void processConnection(
|
|||
#endif
|
||||
|
||||
/* Exchange the greeting. */
|
||||
unsigned int magic = readInt(from);
|
||||
if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch");
|
||||
to << WORKER_MAGIC_2 << PROTOCOL_VERSION;
|
||||
to.flush();
|
||||
WorkerProto::Version clientVersion = readInt(from);
|
||||
WorkerProto::Version clientVersion =
|
||||
WorkerProto::BasicServerConnection::handshake(
|
||||
to, from, PROTOCOL_VERSION);
|
||||
|
||||
if (clientVersion < 0x10a)
|
||||
throw Error("the Nix client version is too old");
|
||||
|
|
@ -1048,29 +1049,20 @@ void processConnection(
|
|||
printMsgUsing(prevLogger, lvlDebug, "%d operations", opCount);
|
||||
});
|
||||
|
||||
if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) {
|
||||
// Obsolete CPU affinity.
|
||||
readInt(from);
|
||||
}
|
||||
WorkerProto::BasicServerConnection conn {
|
||||
.to = to,
|
||||
.from = from,
|
||||
.clientVersion = clientVersion,
|
||||
};
|
||||
|
||||
if (GET_PROTOCOL_MINOR(clientVersion) >= 11)
|
||||
readInt(from); // obsolete reserveSpace
|
||||
|
||||
if (GET_PROTOCOL_MINOR(clientVersion) >= 33)
|
||||
to << nixVersion;
|
||||
|
||||
if (GET_PROTOCOL_MINOR(clientVersion) >= 35) {
|
||||
conn.postHandshake(*store, {
|
||||
.daemonNixVersion = nixVersion,
|
||||
// We and the underlying store both need to trust the client for
|
||||
// it to be trusted.
|
||||
auto temp = trusted
|
||||
.remoteTrustsUs = trusted
|
||||
? store->isTrustedClient()
|
||||
: std::optional { NotTrusted };
|
||||
WorkerProto::WriteConn wconn {
|
||||
.to = to,
|
||||
.version = clientVersion,
|
||||
};
|
||||
WorkerProto::write(*store, wconn, temp);
|
||||
}
|
||||
: std::optional { NotTrusted },
|
||||
});
|
||||
|
||||
/* Send startup error messages to the client. */
|
||||
tunnelLogger->startWork();
|
||||
|
|
|
|||
|
|
@ -930,10 +930,9 @@ DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const StoreDirC
|
|||
|
||||
std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath)
|
||||
{
|
||||
drvPath.requireDerivation();
|
||||
auto nameWithSuffix = drvPath.name();
|
||||
constexpr std::string_view extension = ".drv";
|
||||
assert(hasSuffix(nameWithSuffix, extension));
|
||||
nameWithSuffix.remove_suffix(extension.size());
|
||||
nameWithSuffix.remove_suffix(drvExtension.size());
|
||||
return nameWithSuffix;
|
||||
}
|
||||
|
||||
|
|
@ -1216,16 +1215,19 @@ nlohmann::json DerivationOutput::toJSON(
|
|||
},
|
||||
[&](const DerivationOutput::CAFixed & dof) {
|
||||
res["path"] = store.printStorePath(dof.path(store, drvName, outputName));
|
||||
res["hashAlgo"] = dof.ca.printMethodAlgo();
|
||||
res["method"] = std::string { dof.ca.method.render() };
|
||||
res["hashAlgo"] = printHashAlgo(dof.ca.hash.algo);
|
||||
res["hash"] = dof.ca.hash.to_string(HashFormat::Base16, false);
|
||||
// FIXME print refs?
|
||||
},
|
||||
[&](const DerivationOutput::CAFloating & dof) {
|
||||
res["hashAlgo"] = std::string { dof.method.renderPrefix() } + printHashAlgo(dof.hashAlgo);
|
||||
res["method"] = std::string { dof.method.render() };
|
||||
res["hashAlgo"] = printHashAlgo(dof.hashAlgo);
|
||||
},
|
||||
[&](const DerivationOutput::Deferred &) {},
|
||||
[&](const DerivationOutput::Impure & doi) {
|
||||
res["hashAlgo"] = std::string { doi.method.renderPrefix() } + printHashAlgo(doi.hashAlgo);
|
||||
res["method"] = std::string { doi.method.render() };
|
||||
res["hashAlgo"] = printHashAlgo(doi.hashAlgo);
|
||||
res["impure"] = true;
|
||||
},
|
||||
}, raw);
|
||||
|
|
@ -1245,12 +1247,13 @@ DerivationOutput DerivationOutput::fromJSON(
|
|||
keys.insert(key);
|
||||
|
||||
auto methodAlgo = [&]() -> std::pair<ContentAddressMethod, HashAlgorithm> {
|
||||
auto & str = getString(valueAt(json, "hashAlgo"));
|
||||
std::string_view s = str;
|
||||
ContentAddressMethod method = ContentAddressMethod::parsePrefix(s);
|
||||
auto & method_ = getString(valueAt(json, "method"));
|
||||
ContentAddressMethod method = ContentAddressMethod::parse(method_);
|
||||
if (method == TextIngestionMethod {})
|
||||
xpSettings.require(Xp::DynamicDerivations);
|
||||
auto hashAlgo = parseHashAlgo(s);
|
||||
|
||||
auto & hashAlgo_ = getString(valueAt(json, "hashAlgo"));
|
||||
auto hashAlgo = parseHashAlgo(hashAlgo_);
|
||||
return { std::move(method), std::move(hashAlgo) };
|
||||
};
|
||||
|
||||
|
|
@ -1260,7 +1263,7 @@ DerivationOutput DerivationOutput::fromJSON(
|
|||
};
|
||||
}
|
||||
|
||||
else if (keys == (std::set<std::string_view> { "path", "hashAlgo", "hash" })) {
|
||||
else if (keys == (std::set<std::string_view> { "path", "method", "hashAlgo", "hash" })) {
|
||||
auto [method, hashAlgo] = methodAlgo();
|
||||
auto dof = DerivationOutput::CAFixed {
|
||||
.ca = ContentAddress {
|
||||
|
|
@ -1273,7 +1276,7 @@ DerivationOutput DerivationOutput::fromJSON(
|
|||
return dof;
|
||||
}
|
||||
|
||||
else if (keys == (std::set<std::string_view> { "hashAlgo" })) {
|
||||
else if (keys == (std::set<std::string_view> { "method", "hashAlgo" })) {
|
||||
xpSettings.require(Xp::CaDerivations);
|
||||
auto [method, hashAlgo] = methodAlgo();
|
||||
return DerivationOutput::CAFloating {
|
||||
|
|
@ -1286,7 +1289,7 @@ DerivationOutput DerivationOutput::fromJSON(
|
|||
return DerivationOutput::Deferred {};
|
||||
}
|
||||
|
||||
else if (keys == (std::set<std::string_view> { "hashAlgo", "impure" })) {
|
||||
else if (keys == (std::set<std::string_view> { "method", "hashAlgo", "impure" })) {
|
||||
xpSettings.require(Xp::ImpureDerivations);
|
||||
auto [method, hashAlgo] = methodAlgo();
|
||||
return DerivationOutput::Impure {
|
||||
|
|
|
|||
|
|
@ -18,9 +18,12 @@ struct DummyStoreConfig : virtual StoreConfig {
|
|||
|
||||
struct DummyStore : public virtual DummyStoreConfig, public virtual Store
|
||||
{
|
||||
DummyStore(const std::string scheme, const std::string uri, const Params & params)
|
||||
DummyStore(std::string_view scheme, std::string_view authority, const Params & params)
|
||||
: DummyStore(params)
|
||||
{ }
|
||||
{
|
||||
if (!authority.empty())
|
||||
throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority);
|
||||
}
|
||||
|
||||
DummyStore(const Params & params)
|
||||
: StoreConfig(params)
|
||||
|
|
|
|||
|
|
@ -580,7 +580,12 @@ struct curlFileTransfer : public FileTransfer
|
|||
#endif
|
||||
|
||||
#if __linux__
|
||||
unshareFilesystem();
|
||||
try {
|
||||
tryUnshareFilesystem();
|
||||
} catch (nix::Error & e) {
|
||||
e.addTrace({}, "in download thread");
|
||||
throw;
|
||||
}
|
||||
#endif
|
||||
|
||||
std::map<CURL *, std::shared_ptr<TransferItem>> items;
|
||||
|
|
|
|||
|
|
@ -161,7 +161,7 @@ void LocalStore::findTempRoots(Roots & tempRoots, bool censor)
|
|||
{
|
||||
/* Read the `temproots' directory for per-process temporary root
|
||||
files. */
|
||||
for (auto & i : readDirectory(tempRootsDir)) {
|
||||
for (auto & i : std::filesystem::directory_iterator{tempRootsDir}) {
|
||||
auto name = i.path().filename().string();
|
||||
if (name[0] == '.') {
|
||||
// Ignore hidden files. Some package managers (notably portage) create
|
||||
|
|
@ -225,10 +225,10 @@ void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, R
|
|||
try {
|
||||
|
||||
if (type == std::filesystem::file_type::unknown)
|
||||
type = getFileType(path);
|
||||
type = std::filesystem::symlink_status(path).type();
|
||||
|
||||
if (type == std::filesystem::file_type::directory) {
|
||||
for (auto & i : readDirectory(path))
|
||||
for (auto & i : std::filesystem::directory_iterator{path})
|
||||
findRoots(i.path().string(), i.symlink_status().type(), roots);
|
||||
}
|
||||
|
||||
|
|
@ -781,7 +781,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
|
|||
throw Error(
|
||||
"Cannot delete path '%1%' since it is still alive. "
|
||||
"To find out why, use: "
|
||||
"nix-store --query --roots",
|
||||
"nix-store --query --roots and nix-store --query --referrers",
|
||||
printStorePath(i));
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -345,7 +345,7 @@ void initPlugins()
|
|||
for (const auto & pluginFile : settings.pluginFiles.get()) {
|
||||
std::vector<std::filesystem::path> pluginFiles;
|
||||
try {
|
||||
auto ents = readDirectory(pluginFile);
|
||||
auto ents = std::filesystem::directory_iterator{pluginFile};
|
||||
for (const auto & ent : ents)
|
||||
pluginFiles.emplace_back(ent.path());
|
||||
} catch (std::filesystem::filesystem_error & e) {
|
||||
|
|
|
|||
|
|
@ -910,7 +910,7 @@ public:
|
|||
"substituters",
|
||||
R"(
|
||||
A list of [URLs of Nix stores](@docroot@/store/types/index.md#store-url-format) to be used as substituters, separated by whitespace.
|
||||
A substituter is an additional [store](@docroot@/glossary.md#gloss-store) from which Nix can obtain [store objects](@docroot@/glossary.md#gloss-store-object) instead of building them.
|
||||
A substituter is an additional [store](@docroot@/glossary.md#gloss-store) from which Nix can obtain [store objects](@docroot@/store/store-object.md) instead of building them.
|
||||
|
||||
Substituters are tried based on their priority value, which each substituter can set independently.
|
||||
Lower value means higher priority.
|
||||
|
|
|
|||
|
|
@ -39,15 +39,20 @@ private:
|
|||
public:
|
||||
|
||||
HttpBinaryCacheStore(
|
||||
const std::string & scheme,
|
||||
const Path & _cacheUri,
|
||||
std::string_view scheme,
|
||||
PathView _cacheUri,
|
||||
const Params & params)
|
||||
: StoreConfig(params)
|
||||
, BinaryCacheStoreConfig(params)
|
||||
, HttpBinaryCacheStoreConfig(params)
|
||||
, Store(params)
|
||||
, BinaryCacheStore(params)
|
||||
, cacheUri(scheme + "://" + _cacheUri)
|
||||
, cacheUri(
|
||||
std::string { scheme }
|
||||
+ "://"
|
||||
+ (!_cacheUri.empty()
|
||||
? _cacheUri
|
||||
: throw UsageError("`%s` Store requires a non-empty authority in Store URL", scheme)))
|
||||
{
|
||||
while (!cacheUri.empty() && cacheUri.back() == '/')
|
||||
cacheUri.pop_back();
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ void IndirectRootStore::makeSymlink(const Path & link, const Path & target)
|
|||
createSymlink(target, tempLink);
|
||||
|
||||
/* Atomically replace the old one. */
|
||||
renameFile(tempLink, link);
|
||||
std::filesystem::rename(tempLink, link);
|
||||
}
|
||||
|
||||
Path IndirectRootStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot)
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
#include "pool.hh"
|
||||
#include "remote-store.hh"
|
||||
#include "serve-protocol.hh"
|
||||
#include "serve-protocol-connection.hh"
|
||||
#include "serve-protocol-impl.hh"
|
||||
#include "build-result.hh"
|
||||
#include "store-api.hh"
|
||||
|
|
@ -28,26 +29,23 @@ struct LegacySSHStore::Connection : public ServeProto::BasicClientConnection
|
|||
bool good = true;
|
||||
};
|
||||
|
||||
|
||||
LegacySSHStore::LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params)
|
||||
LegacySSHStore::LegacySSHStore(
|
||||
std::string_view scheme,
|
||||
std::string_view host,
|
||||
const Params & params)
|
||||
: StoreConfig(params)
|
||||
, CommonSSHStoreConfig(params)
|
||||
, CommonSSHStoreConfig(scheme, host, params)
|
||||
, LegacySSHStoreConfig(params)
|
||||
, Store(params)
|
||||
, host(host)
|
||||
, connections(make_ref<Pool<Connection>>(
|
||||
std::max(1, (int) maxConnections),
|
||||
[this]() { return openConnection(); },
|
||||
[](const ref<Connection> & r) { return r->good; }
|
||||
))
|
||||
, master(
|
||||
host,
|
||||
sshKey,
|
||||
sshPublicHostKey,
|
||||
, master(createSSHMaster(
|
||||
// Use SSH master only if using more than 1 connection.
|
||||
connections->capacity() > 1,
|
||||
compress,
|
||||
logFD)
|
||||
logFD))
|
||||
{
|
||||
}
|
||||
|
||||
|
|
@ -76,7 +74,7 @@ ref<LegacySSHStore::Connection> LegacySSHStore::openConnection()
|
|||
conn->sshConn->in.close();
|
||||
{
|
||||
NullSink nullSink;
|
||||
conn->from.drainInto(nullSink);
|
||||
tee.drainInto(nullSink);
|
||||
}
|
||||
throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'",
|
||||
host, chomp(saved.s));
|
||||
|
|
@ -105,24 +103,26 @@ void LegacySSHStore::queryPathInfoUncached(const StorePath & path,
|
|||
|
||||
debug("querying remote host '%s' for info on '%s'", host, printStorePath(path));
|
||||
|
||||
conn->to << ServeProto::Command::QueryPathInfos << PathSet{printStorePath(path)};
|
||||
conn->to.flush();
|
||||
auto infos = conn->queryPathInfos(*this, {path});
|
||||
|
||||
auto p = readString(conn->from);
|
||||
if (p.empty()) return callback(nullptr);
|
||||
auto path2 = parseStorePath(p);
|
||||
assert(path == path2);
|
||||
auto info = std::make_shared<ValidPathInfo>(
|
||||
path,
|
||||
ServeProto::Serialise<UnkeyedValidPathInfo>::read(*this, *conn));
|
||||
switch (infos.size()) {
|
||||
case 0:
|
||||
return callback(nullptr);
|
||||
case 1: {
|
||||
auto & [path2, info] = *infos.begin();
|
||||
|
||||
if (info->narHash == Hash::dummy)
|
||||
throw Error("NAR hash is now mandatory");
|
||||
if (info.narHash == Hash::dummy)
|
||||
throw Error("NAR hash is now mandatory");
|
||||
|
||||
auto s = readString(conn->from);
|
||||
assert(s == "");
|
||||
|
||||
callback(std::move(info));
|
||||
assert(path == path2);
|
||||
return callback(std::make_shared<ValidPathInfo>(
|
||||
std::move(path),
|
||||
std::move(info)
|
||||
));
|
||||
}
|
||||
default:
|
||||
throw Error("More path infos returned than queried");
|
||||
}
|
||||
} catch (...) { callback.rethrow(); }
|
||||
}
|
||||
|
||||
|
|
@ -156,41 +156,38 @@ void LegacySSHStore::addToStore(const ValidPathInfo & info, Source & source,
|
|||
}
|
||||
conn->to.flush();
|
||||
|
||||
if (readInt(conn->from) != 1)
|
||||
throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host);
|
||||
|
||||
} else {
|
||||
|
||||
conn->to
|
||||
<< ServeProto::Command::ImportPaths
|
||||
<< 1;
|
||||
try {
|
||||
copyNAR(source, conn->to);
|
||||
} catch (...) {
|
||||
conn->good = false;
|
||||
throw;
|
||||
}
|
||||
conn->to
|
||||
<< exportMagic
|
||||
<< printStorePath(info.path);
|
||||
ServeProto::write(*this, *conn, info.references);
|
||||
conn->to
|
||||
<< (info.deriver ? printStorePath(*info.deriver) : "")
|
||||
<< 0
|
||||
<< 0;
|
||||
conn->to.flush();
|
||||
conn->importPaths(*this, [&](Sink & sink) {
|
||||
try {
|
||||
copyNAR(source, sink);
|
||||
} catch (...) {
|
||||
conn->good = false;
|
||||
throw;
|
||||
}
|
||||
sink
|
||||
<< exportMagic
|
||||
<< printStorePath(info.path);
|
||||
ServeProto::write(*this, *conn, info.references);
|
||||
sink
|
||||
<< (info.deriver ? printStorePath(*info.deriver) : "")
|
||||
<< 0
|
||||
<< 0;
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
if (readInt(conn->from) != 1)
|
||||
throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host);
|
||||
}
|
||||
|
||||
|
||||
void LegacySSHStore::narFromPath(const StorePath & path, Sink & sink)
|
||||
{
|
||||
auto conn(connections->get());
|
||||
|
||||
conn->to << ServeProto::Command::DumpStorePath << printStorePath(path);
|
||||
conn->to.flush();
|
||||
copyNAR(conn->from, sink);
|
||||
conn->narFromPath(*this, path, [&](auto & source) {
|
||||
copyNAR(source, sink);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -214,7 +211,7 @@ BuildResult LegacySSHStore::buildDerivation(const StorePath & drvPath, const Bas
|
|||
|
||||
conn->putBuildDerivationRequest(*this, drvPath, drv, buildSettings());
|
||||
|
||||
return ServeProto::Serialise<BuildResult>::read(*this, *conn);
|
||||
return conn->getBuildDerivationResponse(*this);
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -26,22 +26,27 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig
|
|||
|
||||
struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store
|
||||
{
|
||||
#ifndef _WIN32
|
||||
// Hack for getting remote build log output.
|
||||
// Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in
|
||||
// the documentation
|
||||
const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
|
||||
const Setting<int> logFD{this, INVALID_DESCRIPTOR, "log-fd", "file descriptor to which SSH's stderr is connected"};
|
||||
#else
|
||||
Descriptor logFD = INVALID_DESCRIPTOR;
|
||||
#endif
|
||||
|
||||
struct Connection;
|
||||
|
||||
std::string host;
|
||||
|
||||
ref<Pool<Connection>> connections;
|
||||
|
||||
SSHMaster master;
|
||||
|
||||
static std::set<std::string> uriSchemes() { return {"ssh"}; }
|
||||
|
||||
LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params);
|
||||
LegacySSHStore(
|
||||
std::string_view scheme,
|
||||
std::string_view host,
|
||||
const Params & params);
|
||||
|
||||
ref<Connection> openConnection();
|
||||
|
||||
|
|
|
|||
|
|
@ -28,9 +28,13 @@ private:
|
|||
|
||||
public:
|
||||
|
||||
/**
|
||||
* @param binaryCacheDir `file://` is a short-hand for `file:///`
|
||||
* for now.
|
||||
*/
|
||||
LocalBinaryCacheStore(
|
||||
const std::string scheme,
|
||||
const Path & binaryCacheDir,
|
||||
std::string_view scheme,
|
||||
PathView binaryCacheDir,
|
||||
const Params & params)
|
||||
: StoreConfig(params)
|
||||
, BinaryCacheStoreConfig(params)
|
||||
|
|
@ -64,7 +68,7 @@ protected:
|
|||
AutoDelete del(tmp, false);
|
||||
StreamToSourceAdapter source(istream);
|
||||
writeFile(tmp, source);
|
||||
renameFile(tmp, path2);
|
||||
std::filesystem::rename(tmp, path2);
|
||||
del.cancel();
|
||||
}
|
||||
|
||||
|
|
@ -83,7 +87,7 @@ protected:
|
|||
{
|
||||
StorePathSet paths;
|
||||
|
||||
for (auto & entry : readDirectory(binaryCacheDir)) {
|
||||
for (auto & entry : std::filesystem::directory_iterator{binaryCacheDir}) {
|
||||
auto name = entry.path().filename().string();
|
||||
if (name.size() != 40 ||
|
||||
!hasSuffix(name, ".narinfo"))
|
||||
|
|
|
|||
|
|
@ -92,7 +92,7 @@ class LocalOverlayStore : public virtual LocalOverlayStoreConfig, public virtual
|
|||
public:
|
||||
LocalOverlayStore(const Params & params);
|
||||
|
||||
LocalOverlayStore(std::string scheme, std::string path, const Params & params)
|
||||
LocalOverlayStore(std::string_view scheme, PathView path, const Params & params)
|
||||
: LocalOverlayStore(params)
|
||||
{
|
||||
if (!path.empty())
|
||||
|
|
@ -463,10 +463,20 @@ LocalStore::LocalStore(const Params & params)
|
|||
}
|
||||
|
||||
|
||||
LocalStore::LocalStore(std::string scheme, std::string path, const Params & params)
|
||||
: LocalStore(params)
|
||||
LocalStore::LocalStore(
|
||||
std::string_view scheme,
|
||||
PathView path,
|
||||
const Params & _params)
|
||||
: LocalStore([&]{
|
||||
// Default `?root` from `path` if non set
|
||||
if (!path.empty() && _params.count("root") == 0) {
|
||||
auto params = _params;
|
||||
params.insert_or_assign("root", std::string { path });
|
||||
return params;
|
||||
}
|
||||
return _params;
|
||||
}())
|
||||
{
|
||||
throw UnimplementedError("LocalStore");
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -1406,7 +1416,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
|
|||
|
||||
printInfo("checking link hashes...");
|
||||
|
||||
for (auto & link : readDirectory(linksDir)) {
|
||||
for (auto & link : std::filesystem::directory_iterator{linksDir}) {
|
||||
auto name = link.path().filename();
|
||||
printMsg(lvlTalkative, "checking contents of '%s'", name);
|
||||
PosixSourceAccessor accessor;
|
||||
|
|
@ -1498,7 +1508,7 @@ LocalStore::VerificationResult LocalStore::verifyAllValidPaths(RepairFlag repair
|
|||
database and the filesystem) in the loop below, in order to catch
|
||||
invalid states.
|
||||
*/
|
||||
for (auto & i : readDirectory(realStoreDir)) {
|
||||
for (auto & i : std::filesystem::directory_iterator{realStoreDir.to_string()}) {
|
||||
try {
|
||||
storePathsInStoreDir.insert({i.path().filename().string()});
|
||||
} catch (BadStorePath &) { }
|
||||
|
|
@ -1779,7 +1789,7 @@ void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log)
|
|||
|
||||
writeFile(tmpFile, compress("bzip2", log));
|
||||
|
||||
renameFile(tmpFile, logPath);
|
||||
std::filesystem::rename(tmpFile, logPath);
|
||||
}
|
||||
|
||||
std::optional<std::string> LocalStore::getVersion()
|
||||
|
|
|
|||
|
|
@ -137,12 +137,15 @@ public:
|
|||
* necessary.
|
||||
*/
|
||||
LocalStore(const Params & params);
|
||||
LocalStore(std::string scheme, std::string path, const Params & params);
|
||||
LocalStore(
|
||||
std::string_view scheme,
|
||||
PathView path,
|
||||
const Params & params);
|
||||
|
||||
~LocalStore();
|
||||
|
||||
static std::set<std::string> uriSchemes()
|
||||
{ return {}; }
|
||||
{ return {"local"}; }
|
||||
|
||||
/**
|
||||
* Implementations of abstract store API methods.
|
||||
|
|
|
|||
|
|
@ -4,9 +4,9 @@ libstore_NAME = libnixstore
|
|||
|
||||
libstore_DIR := $(d)
|
||||
|
||||
libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc)
|
||||
libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc)
|
||||
ifdef HOST_UNIX
|
||||
libstore_SOURCES += $(wildcard $(d)/unix/*.cc $(d)/unix/builtins/*.cc $(d)/unix/build/*.cc)
|
||||
libstore_SOURCES += $(wildcard $(d)/unix/*.cc $(d)/unix/build/*.cc)
|
||||
endif
|
||||
ifdef HOST_LINUX
|
||||
libstore_SOURCES += $(wildcard $(d)/linux/*.cc)
|
||||
|
|
@ -43,7 +43,7 @@ endif
|
|||
|
||||
INCLUDE_libstore := -I $(d) -I $(d)/build
|
||||
ifdef HOST_UNIX
|
||||
INCLUDE_libstore += -I $(d)/unix
|
||||
INCLUDE_libstore += -I $(d)/unix -I $(d)/unix/build
|
||||
endif
|
||||
ifdef HOST_LINUX
|
||||
INCLUDE_libstore += -I $(d)/linux
|
||||
|
|
|
|||
|
|
@ -6,7 +6,8 @@
|
|||
|
||||
namespace nix {
|
||||
|
||||
Machine::Machine(decltype(storeUri) storeUri,
|
||||
Machine::Machine(
|
||||
const std::string & storeUri,
|
||||
decltype(systemTypes) systemTypes,
|
||||
decltype(sshKey) sshKey,
|
||||
decltype(maxJobs) maxJobs,
|
||||
|
|
@ -14,7 +15,7 @@ Machine::Machine(decltype(storeUri) storeUri,
|
|||
decltype(supportedFeatures) supportedFeatures,
|
||||
decltype(mandatoryFeatures) mandatoryFeatures,
|
||||
decltype(sshPublicHostKey) sshPublicHostKey) :
|
||||
storeUri(
|
||||
storeUri(StoreReference::parse(
|
||||
// Backwards compatibility: if the URI is schemeless, is not a path,
|
||||
// and is not one of the special store connection words, prepend
|
||||
// ssh://.
|
||||
|
|
@ -28,7 +29,7 @@ Machine::Machine(decltype(storeUri) storeUri,
|
|||
|| hasPrefix(storeUri, "local?")
|
||||
|| hasPrefix(storeUri, "?")
|
||||
? storeUri
|
||||
: "ssh://" + storeUri),
|
||||
: "ssh://" + storeUri)),
|
||||
systemTypes(systemTypes),
|
||||
sshKey(sshKey),
|
||||
maxJobs(maxJobs),
|
||||
|
|
@ -63,23 +64,26 @@ bool Machine::mandatoryMet(const std::set<std::string> & features) const
|
|||
});
|
||||
}
|
||||
|
||||
ref<Store> Machine::openStore() const
|
||||
StoreReference Machine::completeStoreReference() const
|
||||
{
|
||||
Store::Params storeParams;
|
||||
if (hasPrefix(storeUri, "ssh://")) {
|
||||
storeParams["max-connections"] = "1";
|
||||
storeParams["log-fd"] = "4";
|
||||
auto storeUri = this->storeUri;
|
||||
|
||||
auto * generic = std::get_if<StoreReference::Specified>(&storeUri.variant);
|
||||
|
||||
if (generic && generic->scheme == "ssh") {
|
||||
storeUri.params["max-connections"] = "1";
|
||||
storeUri.params["log-fd"] = "4";
|
||||
}
|
||||
|
||||
if (hasPrefix(storeUri, "ssh://") || hasPrefix(storeUri, "ssh-ng://")) {
|
||||
if (generic && (generic->scheme == "ssh" || generic->scheme == "ssh-ng")) {
|
||||
if (sshKey != "")
|
||||
storeParams["ssh-key"] = sshKey;
|
||||
storeUri.params["ssh-key"] = sshKey;
|
||||
if (sshPublicHostKey != "")
|
||||
storeParams["base64-ssh-public-host-key"] = sshPublicHostKey;
|
||||
storeUri.params["base64-ssh-public-host-key"] = sshPublicHostKey;
|
||||
}
|
||||
|
||||
{
|
||||
auto & fs = storeParams["system-features"];
|
||||
auto & fs = storeUri.params["system-features"];
|
||||
auto append = [&](auto feats) {
|
||||
for (auto & f : feats) {
|
||||
if (fs.size() > 0) fs += ' ';
|
||||
|
|
@ -90,7 +94,12 @@ ref<Store> Machine::openStore() const
|
|||
append(mandatoryFeatures);
|
||||
}
|
||||
|
||||
return nix::openStore(storeUri, storeParams);
|
||||
return storeUri;
|
||||
}
|
||||
|
||||
ref<Store> Machine::openStore() const
|
||||
{
|
||||
return nix::openStore(completeStoreReference());
|
||||
}
|
||||
|
||||
static std::vector<std::string> expandBuilderLines(const std::string & builders)
|
||||
|
|
@ -122,7 +131,7 @@ static std::vector<std::string> expandBuilderLines(const std::string & builders)
|
|||
return result;
|
||||
}
|
||||
|
||||
static Machine parseBuilderLine(const std::string & line)
|
||||
static Machine parseBuilderLine(const std::set<std::string> & defaultSystems, const std::string & line)
|
||||
{
|
||||
const auto tokens = tokenizeString<std::vector<std::string>>(line);
|
||||
|
||||
|
|
@ -139,7 +148,7 @@ static Machine parseBuilderLine(const std::string & line)
|
|||
};
|
||||
|
||||
auto parseFloatField = [&](size_t fieldIndex) {
|
||||
const auto result = string2Int<float>(tokens[fieldIndex]);
|
||||
const auto result = string2Float<float>(tokens[fieldIndex]);
|
||||
if (!result) {
|
||||
throw FormatError("bad machine specification: failed to convert column #%lu in a row: '%s' to 'float'", fieldIndex, line);
|
||||
}
|
||||
|
|
@ -159,29 +168,46 @@ static Machine parseBuilderLine(const std::string & line)
|
|||
if (!isSet(0))
|
||||
throw FormatError("bad machine specification: store URL was not found at the first column of a row: '%s'", line);
|
||||
|
||||
// TODO use designated initializers, once C++ supports those with
|
||||
// custom constructors.
|
||||
return {
|
||||
// `storeUri`
|
||||
tokens[0],
|
||||
isSet(1) ? tokenizeString<std::set<std::string>>(tokens[1], ",") : std::set<std::string>{settings.thisSystem},
|
||||
// `systemTypes`
|
||||
isSet(1) ? tokenizeString<std::set<std::string>>(tokens[1], ",") : defaultSystems,
|
||||
// `sshKey`
|
||||
isSet(2) ? tokens[2] : "",
|
||||
// `maxJobs`
|
||||
isSet(3) ? parseUnsignedIntField(3) : 1U,
|
||||
// `speedFactor`
|
||||
isSet(4) ? parseFloatField(4) : 1.0f,
|
||||
// `supportedFeatures`
|
||||
isSet(5) ? tokenizeString<std::set<std::string>>(tokens[5], ",") : std::set<std::string>{},
|
||||
// `mandatoryFeatures`
|
||||
isSet(6) ? tokenizeString<std::set<std::string>>(tokens[6], ",") : std::set<std::string>{},
|
||||
// `sshPublicHostKey`
|
||||
isSet(7) ? ensureBase64(7) : ""
|
||||
};
|
||||
}
|
||||
|
||||
static Machines parseBuilderLines(const std::vector<std::string> & builders)
|
||||
static Machines parseBuilderLines(const std::set<std::string> & defaultSystems, const std::vector<std::string> & builders)
|
||||
{
|
||||
Machines result;
|
||||
std::transform(builders.begin(), builders.end(), std::back_inserter(result), parseBuilderLine);
|
||||
std::transform(
|
||||
builders.begin(), builders.end(), std::back_inserter(result),
|
||||
[&](auto && line) { return parseBuilderLine(defaultSystems, line); });
|
||||
return result;
|
||||
}
|
||||
|
||||
Machines Machine::parseConfig(const std::set<std::string> & defaultSystems, const std::string & s)
|
||||
{
|
||||
const auto builderLines = expandBuilderLines(s);
|
||||
return parseBuilderLines(defaultSystems, builderLines);
|
||||
}
|
||||
|
||||
Machines getMachines()
|
||||
{
|
||||
const auto builderLines = expandBuilderLines(settings.builders);
|
||||
return parseBuilderLines(builderLines);
|
||||
return Machine::parseConfig({settings.thisSystem}, settings.builders);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,14 +2,19 @@
|
|||
///@file
|
||||
|
||||
#include "types.hh"
|
||||
#include "store-reference.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
class Store;
|
||||
|
||||
struct Machine;
|
||||
|
||||
typedef std::vector<Machine> Machines;
|
||||
|
||||
struct Machine {
|
||||
|
||||
const std::string storeUri;
|
||||
const StoreReference storeUri;
|
||||
const std::set<std::string> systemTypes;
|
||||
const std::string sshKey;
|
||||
const unsigned int maxJobs;
|
||||
|
|
@ -36,7 +41,8 @@ struct Machine {
|
|||
*/
|
||||
bool mandatoryMet(const std::set<std::string> & features) const;
|
||||
|
||||
Machine(decltype(storeUri) storeUri,
|
||||
Machine(
|
||||
const std::string & storeUri,
|
||||
decltype(systemTypes) systemTypes,
|
||||
decltype(sshKey) sshKey,
|
||||
decltype(maxJobs) maxJobs,
|
||||
|
|
@ -45,13 +51,38 @@ struct Machine {
|
|||
decltype(mandatoryFeatures) mandatoryFeatures,
|
||||
decltype(sshPublicHostKey) sshPublicHostKey);
|
||||
|
||||
/**
|
||||
* Elaborate `storeUri` into a complete store reference,
|
||||
* incorporating information from the other fields of the `Machine`
|
||||
* as applicable.
|
||||
*/
|
||||
StoreReference completeStoreReference() const;
|
||||
|
||||
/**
|
||||
* Open a `Store` for this machine.
|
||||
*
|
||||
* Just a simple function composition:
|
||||
* ```c++
|
||||
* nix::openStore(completeStoreReference())
|
||||
* ```
|
||||
*/
|
||||
ref<Store> openStore() const;
|
||||
|
||||
/**
|
||||
* Parse a machine configuration.
|
||||
*
|
||||
* Every machine is specified on its own line, and lines beginning
|
||||
* with `@` are interpreted as paths to other configuration files in
|
||||
* the same format.
|
||||
*/
|
||||
static Machines parseConfig(const std::set<std::string> & defaultSystems, const std::string & config);
|
||||
};
|
||||
|
||||
typedef std::vector<Machine> Machines;
|
||||
|
||||
void parseMachines(const std::string & s, Machines & machines);
|
||||
|
||||
/**
|
||||
* Parse machines from the global config
|
||||
*
|
||||
* @todo Remove, globals are bad.
|
||||
*/
|
||||
Machines getMachines();
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -135,18 +135,37 @@ static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
|
|||
/**
|
||||
* Write a JSON representation of store object metadata, such as the
|
||||
* hash and the references.
|
||||
*
|
||||
* @note Do *not* use `ValidPathInfo::toJSON` because this function is
|
||||
* subject to stronger stability requirements since it is used to
|
||||
* prepare build environments. Perhaps someday we'll have a versionining
|
||||
* mechanism to allow this to evolve again and get back in sync, but for
|
||||
* now we must not change - not even extend - the behavior.
|
||||
*/
|
||||
static nlohmann::json pathInfoToJSON(
|
||||
Store & store,
|
||||
const StorePathSet & storePaths)
|
||||
{
|
||||
nlohmann::json::array_t jsonList = nlohmann::json::array();
|
||||
using nlohmann::json;
|
||||
|
||||
nlohmann::json::array_t jsonList = json::array();
|
||||
|
||||
for (auto & storePath : storePaths) {
|
||||
auto info = store.queryPathInfo(storePath);
|
||||
|
||||
auto & jsonPath = jsonList.emplace_back(
|
||||
info->toJSON(store, false, HashFormat::Nix32));
|
||||
auto & jsonPath = jsonList.emplace_back(json::object());
|
||||
|
||||
jsonPath["narHash"] = info->narHash.to_string(HashFormat::Nix32, true);
|
||||
jsonPath["narSize"] = info->narSize;
|
||||
|
||||
{
|
||||
auto & jsonRefs = jsonPath["references"] = json::array();
|
||||
for (auto & ref : info->references)
|
||||
jsonRefs.emplace_back(store.printStorePath(ref));
|
||||
}
|
||||
|
||||
if (info->ca)
|
||||
jsonPath["ca"] = renderContentAddress(info->ca);
|
||||
|
||||
// Add the path to the object whose metadata we are including.
|
||||
jsonPath["path"] = store.printStorePath(storePath);
|
||||
|
|
|
|||
|
|
@ -161,28 +161,23 @@ nlohmann::json UnkeyedValidPathInfo::toJSON(
|
|||
jsonObject["narSize"] = narSize;
|
||||
|
||||
{
|
||||
auto& jsonRefs = (jsonObject["references"] = json::array());
|
||||
auto & jsonRefs = jsonObject["references"] = json::array();
|
||||
for (auto & ref : references)
|
||||
jsonRefs.emplace_back(store.printStorePath(ref));
|
||||
}
|
||||
|
||||
if (ca)
|
||||
jsonObject["ca"] = renderContentAddress(ca);
|
||||
jsonObject["ca"] = ca ? (std::optional { renderContentAddress(*ca) }) : std::nullopt;
|
||||
|
||||
if (includeImpureInfo) {
|
||||
if (deriver)
|
||||
jsonObject["deriver"] = store.printStorePath(*deriver);
|
||||
jsonObject["deriver"] = deriver ? (std::optional { store.printStorePath(*deriver) }) : std::nullopt;
|
||||
|
||||
if (registrationTime)
|
||||
jsonObject["registrationTime"] = registrationTime;
|
||||
jsonObject["registrationTime"] = registrationTime ? (std::optional { registrationTime }) : std::nullopt;
|
||||
|
||||
if (ultimate)
|
||||
jsonObject["ultimate"] = ultimate;
|
||||
jsonObject["ultimate"] = ultimate;
|
||||
|
||||
if (!sigs.empty()) {
|
||||
for (auto & sig : sigs)
|
||||
jsonObject["signatures"].push_back(sig);
|
||||
}
|
||||
auto & sigsObj = jsonObject["signatures"] = json::array();
|
||||
for (auto & sig : sigs)
|
||||
sigsObj.push_back(sig);
|
||||
}
|
||||
|
||||
return jsonObject;
|
||||
|
|
@ -210,20 +205,25 @@ UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON(
|
|||
throw;
|
||||
}
|
||||
|
||||
// New format as this as nullable but mandatory field; handling
|
||||
// missing is for back-compat.
|
||||
if (json.contains("ca"))
|
||||
res.ca = ContentAddress::parse(getString(valueAt(json, "ca")));
|
||||
if (auto * rawCa = getNullable(valueAt(json, "ca")))
|
||||
res.ca = ContentAddress::parse(getString(*rawCa));
|
||||
|
||||
if (json.contains("deriver"))
|
||||
res.deriver = store.parseStorePath(getString(valueAt(json, "deriver")));
|
||||
if (auto * rawDeriver = getNullable(valueAt(json, "deriver")))
|
||||
res.deriver = store.parseStorePath(getString(*rawDeriver));
|
||||
|
||||
if (json.contains("registrationTime"))
|
||||
res.registrationTime = getInteger(valueAt(json, "registrationTime"));
|
||||
if (auto * rawRegistrationTime = getNullable(valueAt(json, "registrationTime")))
|
||||
res.registrationTime = getInteger(*rawRegistrationTime);
|
||||
|
||||
if (json.contains("ultimate"))
|
||||
res.ultimate = getBoolean(valueAt(json, "ultimate"));
|
||||
|
||||
if (json.contains("signatures"))
|
||||
res.sigs = valueAt(json, "signatures");
|
||||
res.sigs = getStringSet(valueAt(json, "signatures"));
|
||||
|
||||
return res;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -49,11 +49,17 @@ StorePath::StorePath(const Hash & hash, std::string_view _name)
|
|||
checkName(baseName, name());
|
||||
}
|
||||
|
||||
bool StorePath::isDerivation() const
|
||||
bool StorePath::isDerivation() const noexcept
|
||||
{
|
||||
return hasSuffix(name(), drvExtension);
|
||||
}
|
||||
|
||||
void StorePath::requireDerivation() const
|
||||
{
|
||||
if (!isDerivation())
|
||||
throw FormatError("store path '%s' is not a valid derivation path", to_string());
|
||||
}
|
||||
|
||||
StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x");
|
||||
|
||||
StorePath StorePath::random(std::string_view name)
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ struct Hash;
|
|||
* \ref StorePath "Store path" is the fundamental reference type of Nix.
|
||||
* A store paths refers to a Store object.
|
||||
*
|
||||
* See glossary.html#gloss-store-path for more information on a
|
||||
* See store/store-path.html for more information on a
|
||||
* conceptual level.
|
||||
*/
|
||||
class StorePath
|
||||
|
|
@ -35,30 +35,23 @@ public:
|
|||
|
||||
StorePath(const Hash & hash, std::string_view name);
|
||||
|
||||
std::string_view to_string() const
|
||||
std::string_view to_string() const noexcept
|
||||
{
|
||||
return baseName;
|
||||
}
|
||||
|
||||
bool operator < (const StorePath & other) const
|
||||
{
|
||||
return baseName < other.baseName;
|
||||
}
|
||||
|
||||
bool operator == (const StorePath & other) const
|
||||
{
|
||||
return baseName == other.baseName;
|
||||
}
|
||||
|
||||
bool operator != (const StorePath & other) const
|
||||
{
|
||||
return baseName != other.baseName;
|
||||
}
|
||||
bool operator == (const StorePath & other) const noexcept = default;
|
||||
auto operator <=> (const StorePath & other) const noexcept = default;
|
||||
|
||||
/**
|
||||
* Check whether a file name ends with the extension for derivations.
|
||||
*/
|
||||
bool isDerivation() const;
|
||||
bool isDerivation() const noexcept;
|
||||
|
||||
/**
|
||||
* Throw an exception if `isDerivation` is false.
|
||||
*/
|
||||
void requireDerivation() const;
|
||||
|
||||
std::string_view name() const
|
||||
{
|
||||
|
|
@ -82,7 +75,7 @@ typedef std::vector<StorePath> StorePaths;
|
|||
* The file extension of \ref Derivation derivations when serialized
|
||||
* into store objects.
|
||||
*/
|
||||
const std::string drvExtension = ".drv";
|
||||
constexpr std::string_view drvExtension = ".drv";
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -144,8 +144,7 @@ static void canonicalisePathMetaData_(
|
|||
#endif
|
||||
|
||||
if (S_ISDIR(st.st_mode)) {
|
||||
std::vector<std::filesystem::directory_entry> entries = readDirectory(path);
|
||||
for (auto & i : entries)
|
||||
for (auto & i : std::filesystem::directory_iterator{path})
|
||||
canonicalisePathMetaData_(
|
||||
i.path().string(),
|
||||
#ifndef _WIN32
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path pro
|
|||
std::filesystem::path profileDir = dirOf(profile);
|
||||
auto profileName = std::string(baseNameOf(profile));
|
||||
|
||||
for (auto & i : readDirectory(profileDir.string())) {
|
||||
for (auto & i : std::filesystem::directory_iterator{profileDir}) {
|
||||
if (auto n = parseName(profileName, i.path().filename().string())) {
|
||||
auto path = i.path().string();
|
||||
gens.push_back({
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
|
||||
#include "remote-store.hh"
|
||||
#include "worker-protocol.hh"
|
||||
#include "worker-protocol-connection.hh"
|
||||
#include "pool.hh"
|
||||
|
||||
namespace nix {
|
||||
|
|
@ -14,90 +15,13 @@ namespace nix {
|
|||
* Contains `Source` and `Sink` for actual communication, along with
|
||||
* other information learned when negotiating the connection.
|
||||
*/
|
||||
struct RemoteStore::Connection
|
||||
struct RemoteStore::Connection : WorkerProto::BasicClientConnection,
|
||||
WorkerProto::ClientHandshakeInfo
|
||||
{
|
||||
/**
|
||||
* Send with this.
|
||||
*/
|
||||
FdSink to;
|
||||
|
||||
/**
|
||||
* Receive with this.
|
||||
*/
|
||||
FdSource from;
|
||||
|
||||
/**
|
||||
* Worker protocol version used for the connection.
|
||||
*
|
||||
* Despite its name, I think it is actually the maximum version both
|
||||
* sides support. (If the maximum doesn't exist, we would fail to
|
||||
* establish a connection and produce a value of this type.)
|
||||
*/
|
||||
WorkerProto::Version daemonVersion;
|
||||
|
||||
/**
|
||||
* Whether the remote side trusts us or not.
|
||||
*
|
||||
* 3 values: "yes", "no", or `std::nullopt` for "unknown".
|
||||
*
|
||||
* Note that the "remote side" might not be just the end daemon, but
|
||||
* also an intermediary forwarder that can make its own trusting
|
||||
* decisions. This would be the intersection of all their trust
|
||||
* decisions, since it takes only one link in the chain to start
|
||||
* denying operations.
|
||||
*/
|
||||
std::optional<TrustedFlag> remoteTrustsUs;
|
||||
|
||||
/**
|
||||
* The version of the Nix daemon that is processing our requests.
|
||||
*
|
||||
* Do note, it may or may not communicating with another daemon,
|
||||
* rather than being an "end" `LocalStore` or similar.
|
||||
*/
|
||||
std::optional<std::string> daemonNixVersion;
|
||||
|
||||
/**
|
||||
* Time this connection was established.
|
||||
*/
|
||||
std::chrono::time_point<std::chrono::steady_clock> startTime;
|
||||
|
||||
/**
|
||||
* Coercion to `WorkerProto::ReadConn`. This makes it easy to use the
|
||||
* factored out worker protocol searlizers with a
|
||||
* `RemoteStore::Connection`.
|
||||
*
|
||||
* The worker protocol connection types are unidirectional, unlike
|
||||
* this type.
|
||||
*/
|
||||
operator WorkerProto::ReadConn ()
|
||||
{
|
||||
return WorkerProto::ReadConn {
|
||||
.from = from,
|
||||
.version = daemonVersion,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Coercion to `WorkerProto::WriteConn`. This makes it easy to use the
|
||||
* factored out worker protocol searlizers with a
|
||||
* `RemoteStore::Connection`.
|
||||
*
|
||||
* The worker protocol connection types are unidirectional, unlike
|
||||
* this type.
|
||||
*/
|
||||
operator WorkerProto::WriteConn ()
|
||||
{
|
||||
return WorkerProto::WriteConn {
|
||||
.to = to,
|
||||
.version = daemonVersion,
|
||||
};
|
||||
}
|
||||
|
||||
virtual ~Connection();
|
||||
|
||||
virtual void closeWrite() = 0;
|
||||
|
||||
std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true);
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -69,50 +69,26 @@ void RemoteStore::initConnection(Connection & conn)
|
|||
/* Send the magic greeting, check for the reply. */
|
||||
try {
|
||||
conn.from.endOfFileError = "Nix daemon disconnected unexpectedly (maybe it crashed?)";
|
||||
conn.to << WORKER_MAGIC_1;
|
||||
conn.to.flush();
|
||||
|
||||
StringSink saved;
|
||||
TeeSource tee(conn.from, saved);
|
||||
try {
|
||||
TeeSource tee(conn.from, saved);
|
||||
unsigned int magic = readInt(tee);
|
||||
if (magic != WORKER_MAGIC_2)
|
||||
throw Error("protocol mismatch");
|
||||
conn.daemonVersion = WorkerProto::BasicClientConnection::handshake(
|
||||
conn.to, tee, PROTOCOL_VERSION);
|
||||
} catch (SerialisationError & e) {
|
||||
/* In case the other side is waiting for our input, close
|
||||
it. */
|
||||
conn.closeWrite();
|
||||
auto msg = conn.from.drain();
|
||||
throw Error("protocol mismatch, got '%s'", chomp(saved.s + msg));
|
||||
{
|
||||
NullSink nullSink;
|
||||
tee.drainInto(nullSink);
|
||||
}
|
||||
throw Error("protocol mismatch, got '%s'", chomp(saved.s));
|
||||
}
|
||||
|
||||
conn.from >> conn.daemonVersion;
|
||||
if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION))
|
||||
throw Error("Nix daemon protocol version not supported");
|
||||
if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10)
|
||||
throw Error("the Nix daemon version is too old");
|
||||
conn.to << PROTOCOL_VERSION;
|
||||
static_cast<WorkerProto::ClientHandshakeInfo &>(conn) = conn.postHandshake(*this);
|
||||
|
||||
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 14) {
|
||||
// Obsolete CPU affinity.
|
||||
conn.to << 0;
|
||||
}
|
||||
|
||||
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11)
|
||||
conn.to << false; // obsolete reserveSpace
|
||||
|
||||
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 33) {
|
||||
conn.to.flush();
|
||||
conn.daemonNixVersion = readString(conn.from);
|
||||
}
|
||||
|
||||
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 35) {
|
||||
conn.remoteTrustsUs = WorkerProto::Serialise<std::optional<TrustedFlag>>::read(*this, conn);
|
||||
} else {
|
||||
// We don't know the answer; protocol to old.
|
||||
conn.remoteTrustsUs = std::nullopt;
|
||||
}
|
||||
|
||||
auto ex = conn.processStderr();
|
||||
auto ex = conn.processStderrReturn();
|
||||
if (ex) std::rethrow_exception(ex);
|
||||
}
|
||||
catch (Error & e) {
|
||||
|
|
@ -158,7 +134,7 @@ void RemoteStore::setOptions(Connection & conn)
|
|||
conn.to << i.first << i.second.value;
|
||||
}
|
||||
|
||||
auto ex = conn.processStderr();
|
||||
auto ex = conn.processStderrReturn();
|
||||
if (ex) std::rethrow_exception(ex);
|
||||
}
|
||||
|
||||
|
|
@ -173,28 +149,7 @@ RemoteStore::ConnectionHandle::~ConnectionHandle()
|
|||
|
||||
void RemoteStore::ConnectionHandle::processStderr(Sink * sink, Source * source, bool flush)
|
||||
{
|
||||
auto ex = handle->processStderr(sink, source, flush);
|
||||
if (ex) {
|
||||
daemonException = true;
|
||||
try {
|
||||
std::rethrow_exception(ex);
|
||||
} catch (const Error & e) {
|
||||
// Nix versions before #4628 did not have an adequate behavior for reporting that the derivation format was upgraded.
|
||||
// To avoid having to add compatibility logic in many places, we expect to catch almost all occurrences of the
|
||||
// old incomprehensible error here, so that we can explain to users what's going on when their daemon is
|
||||
// older than #4628 (2023).
|
||||
if (experimentalFeatureSettings.isEnabled(Xp::DynamicDerivations) &&
|
||||
GET_PROTOCOL_MINOR(handle->daemonVersion) <= 35)
|
||||
{
|
||||
auto m = e.msg();
|
||||
if (m.find("parsing derivation") != std::string::npos &&
|
||||
m.find("expected string") != std::string::npos &&
|
||||
m.find("Derive([") != std::string::npos)
|
||||
throw Error("%s, this might be because the daemon is too old to understand dependencies on dynamic derivations. Check to see if the raw derivation is in the form '%s'", std::move(m), "DrvWithVersion(..)");
|
||||
}
|
||||
throw;
|
||||
}
|
||||
}
|
||||
handle->processStderr(&daemonException, sink, source, flush);
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -226,13 +181,7 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute
|
|||
if (isValidPath(i)) res.insert(i);
|
||||
return res;
|
||||
} else {
|
||||
conn->to << WorkerProto::Op::QueryValidPaths;
|
||||
WorkerProto::write(*this, *conn, paths);
|
||||
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) {
|
||||
conn->to << maybeSubstitute;
|
||||
}
|
||||
conn.processStderr();
|
||||
return WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
|
||||
return conn->queryValidPaths(*this, &conn.daemonException, paths, maybeSubstitute);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -322,22 +271,10 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path,
|
|||
std::shared_ptr<const ValidPathInfo> info;
|
||||
{
|
||||
auto conn(getConnection());
|
||||
conn->to << WorkerProto::Op::QueryPathInfo << printStorePath(path);
|
||||
try {
|
||||
conn.processStderr();
|
||||
} catch (Error & e) {
|
||||
// Ugly backwards compatibility hack.
|
||||
if (e.msg().find("is not valid") != std::string::npos)
|
||||
throw InvalidPath(std::move(e.info()));
|
||||
throw;
|
||||
}
|
||||
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) {
|
||||
bool valid; conn->from >> valid;
|
||||
if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path));
|
||||
}
|
||||
info = std::make_shared<ValidPathInfo>(
|
||||
StorePath{path},
|
||||
WorkerProto::Serialise<UnkeyedValidPathInfo>::read(*this, *conn));
|
||||
conn->queryPathInfo(*this, &conn.daemonException, path));
|
||||
|
||||
}
|
||||
callback(std::move(info));
|
||||
} catch (...) { callback.rethrow(); }
|
||||
|
|
@ -542,8 +479,6 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
|
|||
auto conn(getConnection());
|
||||
|
||||
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) {
|
||||
conn->to << WorkerProto::Op::ImportPaths;
|
||||
|
||||
auto source2 = sinkToSource([&](Sink & sink) {
|
||||
sink << 1 // == path follows
|
||||
;
|
||||
|
|
@ -558,11 +493,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
|
|||
<< 0 // == no path follows
|
||||
;
|
||||
});
|
||||
|
||||
conn.processStderr(0, source2.get());
|
||||
|
||||
auto importedPaths = WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
|
||||
assert(importedPaths.size() <= 1);
|
||||
conn->importPaths(*this, &conn.daemonException, *source2);
|
||||
}
|
||||
|
||||
else {
|
||||
|
|
@ -807,9 +738,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD
|
|||
BuildMode buildMode)
|
||||
{
|
||||
auto conn(getConnection());
|
||||
conn->to << WorkerProto::Op::BuildDerivation << printStorePath(drvPath);
|
||||
writeDerivation(conn->to, *this, drv);
|
||||
conn->to << buildMode;
|
||||
conn->putBuildDerivationRequest(*this, &conn.daemonException, drvPath, drv, buildMode);
|
||||
conn.processStderr();
|
||||
return WorkerProto::Serialise<BuildResult>::read(*this, *conn);
|
||||
}
|
||||
|
|
@ -827,9 +756,7 @@ void RemoteStore::ensurePath(const StorePath & path)
|
|||
void RemoteStore::addTempRoot(const StorePath & path)
|
||||
{
|
||||
auto conn(getConnection());
|
||||
conn->to << WorkerProto::Op::AddTempRoot << printStorePath(path);
|
||||
conn.processStderr();
|
||||
readInt(conn->from);
|
||||
conn->addTempRoot(*this, &conn.daemonException, path);
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -969,22 +896,12 @@ void RemoteStore::flushBadConnections()
|
|||
connections->flushBad();
|
||||
}
|
||||
|
||||
|
||||
RemoteStore::Connection::~Connection()
|
||||
{
|
||||
try {
|
||||
to.flush();
|
||||
} catch (...) {
|
||||
ignoreException();
|
||||
}
|
||||
}
|
||||
|
||||
void RemoteStore::narFromPath(const StorePath & path, Sink & sink)
|
||||
{
|
||||
auto conn(connections->get());
|
||||
conn->to << WorkerProto::Op::NarFromPath << printStorePath(path);
|
||||
conn->processStderr();
|
||||
copyNAR(conn->from, sink);
|
||||
auto conn(getConnection());
|
||||
conn->narFromPath(*this, &conn.daemonException, path, [&](Source & source) {
|
||||
copyNAR(conn->from, sink);
|
||||
});
|
||||
}
|
||||
|
||||
ref<SourceAccessor> RemoteStore::getFSAccessor(bool requireValidPath)
|
||||
|
|
@ -992,91 +909,6 @@ ref<SourceAccessor> RemoteStore::getFSAccessor(bool requireValidPath)
|
|||
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()));
|
||||
}
|
||||
|
||||
static Logger::Fields readFields(Source & from)
|
||||
{
|
||||
Logger::Fields fields;
|
||||
size_t size = readInt(from);
|
||||
for (size_t n = 0; n < size; n++) {
|
||||
auto type = (decltype(Logger::Field::type)) readInt(from);
|
||||
if (type == Logger::Field::tInt)
|
||||
fields.push_back(readNum<uint64_t>(from));
|
||||
else if (type == Logger::Field::tString)
|
||||
fields.push_back(readString(from));
|
||||
else
|
||||
throw Error("got unsupported field type %x from Nix daemon", (int) type);
|
||||
}
|
||||
return fields;
|
||||
}
|
||||
|
||||
|
||||
std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source, bool flush)
|
||||
{
|
||||
if (flush)
|
||||
to.flush();
|
||||
|
||||
while (true) {
|
||||
|
||||
auto msg = readNum<uint64_t>(from);
|
||||
|
||||
if (msg == STDERR_WRITE) {
|
||||
auto s = readString(from);
|
||||
if (!sink) throw Error("no sink");
|
||||
(*sink)(s);
|
||||
}
|
||||
|
||||
else if (msg == STDERR_READ) {
|
||||
if (!source) throw Error("no source");
|
||||
size_t len = readNum<size_t>(from);
|
||||
auto buf = std::make_unique<char[]>(len);
|
||||
writeString({(const char *) buf.get(), source->read(buf.get(), len)}, to);
|
||||
to.flush();
|
||||
}
|
||||
|
||||
else if (msg == STDERR_ERROR) {
|
||||
if (GET_PROTOCOL_MINOR(daemonVersion) >= 26) {
|
||||
return std::make_exception_ptr(readError(from));
|
||||
} else {
|
||||
auto error = readString(from);
|
||||
unsigned int status = readInt(from);
|
||||
return std::make_exception_ptr(Error(status, error));
|
||||
}
|
||||
}
|
||||
|
||||
else if (msg == STDERR_NEXT)
|
||||
printError(chomp(readString(from)));
|
||||
|
||||
else if (msg == STDERR_START_ACTIVITY) {
|
||||
auto act = readNum<ActivityId>(from);
|
||||
auto lvl = (Verbosity) readInt(from);
|
||||
auto type = (ActivityType) readInt(from);
|
||||
auto s = readString(from);
|
||||
auto fields = readFields(from);
|
||||
auto parent = readNum<ActivityId>(from);
|
||||
logger->startActivity(act, lvl, type, s, fields, parent);
|
||||
}
|
||||
|
||||
else if (msg == STDERR_STOP_ACTIVITY) {
|
||||
auto act = readNum<ActivityId>(from);
|
||||
logger->stopActivity(act);
|
||||
}
|
||||
|
||||
else if (msg == STDERR_RESULT) {
|
||||
auto act = readNum<ActivityId>(from);
|
||||
auto type = (ResultType) readInt(from);
|
||||
auto fields = readFields(from);
|
||||
logger->result(act, type, fields);
|
||||
}
|
||||
|
||||
else if (msg == STDERR_LAST)
|
||||
break;
|
||||
|
||||
else
|
||||
throw Error("got unknown message type %x from Nix daemon", msg);
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void RemoteStore::ConnectionHandle::withFramedSink(std::function<void(Sink & sink)> fun)
|
||||
{
|
||||
(*this)->to.flush();
|
||||
|
|
|
|||
|
|
@ -213,7 +213,7 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
|
|||
support it.
|
||||
|
||||
> **Note**
|
||||
>
|
||||
>
|
||||
> HTTPS should be used if the cache might contain sensitive
|
||||
> information.
|
||||
)"};
|
||||
|
|
@ -224,7 +224,7 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
|
|||
Do not specify this setting if you're using Amazon S3.
|
||||
|
||||
> **Note**
|
||||
>
|
||||
>
|
||||
> This endpoint must support HTTPS and will use path-based
|
||||
> addressing instead of virtual host based addressing.
|
||||
)"};
|
||||
|
|
@ -269,8 +269,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
|
|||
S3Helper s3Helper;
|
||||
|
||||
S3BinaryCacheStoreImpl(
|
||||
const std::string & uriScheme,
|
||||
const std::string & bucketName,
|
||||
std::string_view uriScheme,
|
||||
std::string_view bucketName,
|
||||
const Params & params)
|
||||
: StoreConfig(params)
|
||||
, BinaryCacheStoreConfig(params)
|
||||
|
|
@ -281,6 +281,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
|
|||
, bucketName(bucketName)
|
||||
, s3Helper(profile, region, scheme, endpoint)
|
||||
{
|
||||
if (bucketName.empty())
|
||||
throw UsageError("`%s` store requires a bucket name in its Store URI", uriScheme);
|
||||
diskCache = getNarInfoDiskCache();
|
||||
}
|
||||
|
||||
|
|
|
|||
106
src/libstore/serve-protocol-connection.cc
Normal file
106
src/libstore/serve-protocol-connection.cc
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
#include "serve-protocol-connection.hh"
|
||||
#include "serve-protocol-impl.hh"
|
||||
#include "build-result.hh"
|
||||
#include "derivations.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
ServeProto::Version ServeProto::BasicClientConnection::handshake(
|
||||
BufferedSink & to, Source & from, ServeProto::Version localVersion, std::string_view host)
|
||||
{
|
||||
to << SERVE_MAGIC_1 << localVersion;
|
||||
to.flush();
|
||||
|
||||
unsigned int magic = readInt(from);
|
||||
if (magic != SERVE_MAGIC_2)
|
||||
throw Error("'nix-store --serve' protocol mismatch from '%s'", host);
|
||||
auto remoteVersion = readInt(from);
|
||||
if (GET_PROTOCOL_MAJOR(remoteVersion) != 0x200)
|
||||
throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host);
|
||||
return std::min(remoteVersion, localVersion);
|
||||
}
|
||||
|
||||
ServeProto::Version
|
||||
ServeProto::BasicServerConnection::handshake(BufferedSink & to, Source & from, ServeProto::Version localVersion)
|
||||
{
|
||||
unsigned int magic = readInt(from);
|
||||
if (magic != SERVE_MAGIC_1)
|
||||
throw Error("protocol mismatch");
|
||||
to << SERVE_MAGIC_2 << localVersion;
|
||||
to.flush();
|
||||
auto remoteVersion = readInt(from);
|
||||
return std::min(remoteVersion, localVersion);
|
||||
}
|
||||
|
||||
StorePathSet ServeProto::BasicClientConnection::queryValidPaths(
|
||||
const StoreDirConfig & store, bool lock, const StorePathSet & paths, SubstituteFlag maybeSubstitute)
|
||||
{
|
||||
to << ServeProto::Command::QueryValidPaths << lock << maybeSubstitute;
|
||||
write(store, *this, paths);
|
||||
to.flush();
|
||||
|
||||
return Serialise<StorePathSet>::read(store, *this);
|
||||
}
|
||||
|
||||
std::map<StorePath, UnkeyedValidPathInfo>
|
||||
ServeProto::BasicClientConnection::queryPathInfos(const StoreDirConfig & store, const StorePathSet & paths)
|
||||
{
|
||||
std::map<StorePath, UnkeyedValidPathInfo> infos;
|
||||
|
||||
to << ServeProto::Command::QueryPathInfos;
|
||||
ServeProto::write(store, *this, paths);
|
||||
to.flush();
|
||||
|
||||
while (true) {
|
||||
auto storePathS = readString(from);
|
||||
if (storePathS == "")
|
||||
break;
|
||||
|
||||
auto storePath = store.parseStorePath(storePathS);
|
||||
assert(paths.count(storePath) == 1);
|
||||
auto info = ServeProto::Serialise<UnkeyedValidPathInfo>::read(store, *this);
|
||||
infos.insert_or_assign(std::move(storePath), std::move(info));
|
||||
}
|
||||
|
||||
return infos;
|
||||
}
|
||||
|
||||
void ServeProto::BasicClientConnection::putBuildDerivationRequest(
|
||||
const StoreDirConfig & store,
|
||||
const StorePath & drvPath,
|
||||
const BasicDerivation & drv,
|
||||
const ServeProto::BuildOptions & options)
|
||||
{
|
||||
to << ServeProto::Command::BuildDerivation << store.printStorePath(drvPath);
|
||||
writeDerivation(to, store, drv);
|
||||
|
||||
ServeProto::write(store, *this, options);
|
||||
|
||||
to.flush();
|
||||
}
|
||||
|
||||
BuildResult ServeProto::BasicClientConnection::getBuildDerivationResponse(const StoreDirConfig & store)
|
||||
{
|
||||
return ServeProto::Serialise<BuildResult>::read(store, *this);
|
||||
}
|
||||
|
||||
void ServeProto::BasicClientConnection::narFromPath(
|
||||
const StoreDirConfig & store, const StorePath & path, std::function<void(Source &)> fun)
|
||||
{
|
||||
to << ServeProto::Command::DumpStorePath << store.printStorePath(path);
|
||||
to.flush();
|
||||
|
||||
fun(from);
|
||||
}
|
||||
|
||||
void ServeProto::BasicClientConnection::importPaths(const StoreDirConfig & store, std::function<void(Sink &)> fun)
|
||||
{
|
||||
to << ServeProto::Command::ImportPaths;
|
||||
fun(to);
|
||||
to.flush();
|
||||
|
||||
if (readInt(from) != 1)
|
||||
throw Error("remote machine failed to import closure");
|
||||
}
|
||||
|
||||
}
|
||||
108
src/libstore/serve-protocol-connection.hh
Normal file
108
src/libstore/serve-protocol-connection.hh
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
#pragma once
|
||||
///@file
|
||||
|
||||
#include "serve-protocol.hh"
|
||||
#include "store-api.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
struct ServeProto::BasicClientConnection
|
||||
{
|
||||
FdSink to;
|
||||
FdSource from;
|
||||
ServeProto::Version remoteVersion;
|
||||
|
||||
/**
|
||||
* Establishes connection, negotiating version.
|
||||
*
|
||||
* @return the version provided by the other side of the
|
||||
* connection.
|
||||
*
|
||||
* @param to Taken by reference to allow for various error handling
|
||||
* mechanisms.
|
||||
*
|
||||
* @param from Taken by reference to allow for various error
|
||||
* handling mechanisms.
|
||||
*
|
||||
* @param localVersion Our version which is sent over
|
||||
*
|
||||
* @param host Just used to add context to thrown exceptions.
|
||||
*/
|
||||
static ServeProto::Version
|
||||
handshake(BufferedSink & to, Source & from, ServeProto::Version localVersion, std::string_view host);
|
||||
|
||||
/**
|
||||
* Coercion to `ServeProto::ReadConn`. This makes it easy to use the
|
||||
* factored out serve protocol serializers with a
|
||||
* `LegacySSHStore::Connection`.
|
||||
*
|
||||
* The serve protocol connection types are unidirectional, unlike
|
||||
* this type.
|
||||
*/
|
||||
operator ServeProto::ReadConn()
|
||||
{
|
||||
return ServeProto::ReadConn{
|
||||
.from = from,
|
||||
.version = remoteVersion,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Coercion to `ServeProto::WriteConn`. This makes it easy to use the
|
||||
* factored out serve protocol serializers with a
|
||||
* `LegacySSHStore::Connection`.
|
||||
*
|
||||
* The serve protocol connection types are unidirectional, unlike
|
||||
* this type.
|
||||
*/
|
||||
operator ServeProto::WriteConn()
|
||||
{
|
||||
return ServeProto::WriteConn{
|
||||
.to = to,
|
||||
.version = remoteVersion,
|
||||
};
|
||||
}
|
||||
|
||||
StorePathSet queryValidPaths(
|
||||
const StoreDirConfig & remoteStore, bool lock, const StorePathSet & paths, SubstituteFlag maybeSubstitute);
|
||||
|
||||
std::map<StorePath, UnkeyedValidPathInfo> queryPathInfos(const StoreDirConfig & store, const StorePathSet & paths);
|
||||
;
|
||||
|
||||
void putBuildDerivationRequest(
|
||||
const StoreDirConfig & store,
|
||||
const StorePath & drvPath,
|
||||
const BasicDerivation & drv,
|
||||
const ServeProto::BuildOptions & options);
|
||||
|
||||
/**
|
||||
* Get the response, must be paired with
|
||||
* `putBuildDerivationRequest`.
|
||||
*/
|
||||
BuildResult getBuildDerivationResponse(const StoreDirConfig & store);
|
||||
|
||||
void narFromPath(const StoreDirConfig & store, const StorePath & path, std::function<void(Source &)> fun);
|
||||
|
||||
void importPaths(const StoreDirConfig & store, std::function<void(Sink &)> fun);
|
||||
};
|
||||
|
||||
struct ServeProto::BasicServerConnection
|
||||
{
|
||||
/**
|
||||
* Establishes connection, negotiating version.
|
||||
*
|
||||
* @return the version provided by the other side of the
|
||||
* connection.
|
||||
*
|
||||
* @param to Taken by reference to allow for various error handling
|
||||
* mechanisms.
|
||||
*
|
||||
* @param from Taken by reference to allow for various error
|
||||
* handling mechanisms.
|
||||
*
|
||||
* @param localVersion Our version which is sent over
|
||||
*/
|
||||
static ServeProto::Version handshake(BufferedSink & to, Source & from, ServeProto::Version localVersion);
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -1,69 +0,0 @@
|
|||
#include "serve-protocol-impl.hh"
|
||||
#include "build-result.hh"
|
||||
#include "derivations.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
ServeProto::Version ServeProto::BasicClientConnection::handshake(
|
||||
BufferedSink & to,
|
||||
Source & from,
|
||||
ServeProto::Version localVersion,
|
||||
std::string_view host)
|
||||
{
|
||||
to << SERVE_MAGIC_1 << localVersion;
|
||||
to.flush();
|
||||
|
||||
unsigned int magic = readInt(from);
|
||||
if (magic != SERVE_MAGIC_2)
|
||||
throw Error("'nix-store --serve' protocol mismatch from '%s'", host);
|
||||
auto remoteVersion = readInt(from);
|
||||
if (GET_PROTOCOL_MAJOR(remoteVersion) != 0x200)
|
||||
throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host);
|
||||
return remoteVersion;
|
||||
}
|
||||
|
||||
ServeProto::Version ServeProto::BasicServerConnection::handshake(
|
||||
BufferedSink & to,
|
||||
Source & from,
|
||||
ServeProto::Version localVersion)
|
||||
{
|
||||
unsigned int magic = readInt(from);
|
||||
if (magic != SERVE_MAGIC_1) throw Error("protocol mismatch");
|
||||
to << SERVE_MAGIC_2 << localVersion;
|
||||
to.flush();
|
||||
return readInt(from);
|
||||
}
|
||||
|
||||
|
||||
StorePathSet ServeProto::BasicClientConnection::queryValidPaths(
|
||||
const Store & store,
|
||||
bool lock, const StorePathSet & paths,
|
||||
SubstituteFlag maybeSubstitute)
|
||||
{
|
||||
to
|
||||
<< ServeProto::Command::QueryValidPaths
|
||||
<< lock
|
||||
<< maybeSubstitute;
|
||||
write(store, *this, paths);
|
||||
to.flush();
|
||||
|
||||
return Serialise<StorePathSet>::read(store, *this);
|
||||
}
|
||||
|
||||
|
||||
void ServeProto::BasicClientConnection::putBuildDerivationRequest(
|
||||
const Store & store,
|
||||
const StorePath & drvPath, const BasicDerivation & drv,
|
||||
const ServeProto::BuildOptions & options)
|
||||
{
|
||||
to
|
||||
<< ServeProto::Command::BuildDerivation
|
||||
<< store.printStorePath(drvPath);
|
||||
writeDerivation(to, store, drv);
|
||||
|
||||
ServeProto::write(store, *this, options);
|
||||
|
||||
to.flush();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -57,101 +57,4 @@ struct ServeProto::Serialise
|
|||
|
||||
/* protocol-specific templates */
|
||||
|
||||
struct ServeProto::BasicClientConnection
|
||||
{
|
||||
FdSink to;
|
||||
FdSource from;
|
||||
ServeProto::Version remoteVersion;
|
||||
|
||||
/**
|
||||
* Establishes connection, negotiating version.
|
||||
*
|
||||
* @return the version provided by the other side of the
|
||||
* connection.
|
||||
*
|
||||
* @param to Taken by reference to allow for various error handling
|
||||
* mechanisms.
|
||||
*
|
||||
* @param from Taken by reference to allow for various error
|
||||
* handling mechanisms.
|
||||
*
|
||||
* @param localVersion Our version which is sent over
|
||||
*
|
||||
* @param host Just used to add context to thrown exceptions.
|
||||
*/
|
||||
static ServeProto::Version handshake(
|
||||
BufferedSink & to,
|
||||
Source & from,
|
||||
ServeProto::Version localVersion,
|
||||
std::string_view host);
|
||||
|
||||
/**
|
||||
* Coercion to `ServeProto::ReadConn`. This makes it easy to use the
|
||||
* factored out serve protocol serializers with a
|
||||
* `LegacySSHStore::Connection`.
|
||||
*
|
||||
* The serve protocol connection types are unidirectional, unlike
|
||||
* this type.
|
||||
*/
|
||||
operator ServeProto::ReadConn ()
|
||||
{
|
||||
return ServeProto::ReadConn {
|
||||
.from = from,
|
||||
.version = remoteVersion,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Coercion to `ServeProto::WriteConn`. This makes it easy to use the
|
||||
* factored out serve protocol serializers with a
|
||||
* `LegacySSHStore::Connection`.
|
||||
*
|
||||
* The serve protocol connection types are unidirectional, unlike
|
||||
* this type.
|
||||
*/
|
||||
operator ServeProto::WriteConn ()
|
||||
{
|
||||
return ServeProto::WriteConn {
|
||||
.to = to,
|
||||
.version = remoteVersion,
|
||||
};
|
||||
}
|
||||
|
||||
StorePathSet queryValidPaths(
|
||||
const Store & remoteStore,
|
||||
bool lock, const StorePathSet & paths,
|
||||
SubstituteFlag maybeSubstitute);
|
||||
|
||||
/**
|
||||
* Just the request half, because Hydra may do other things between
|
||||
* issuing the request and reading the `BuildResult` response.
|
||||
*/
|
||||
void putBuildDerivationRequest(
|
||||
const Store & store,
|
||||
const StorePath & drvPath, const BasicDerivation & drv,
|
||||
const ServeProto::BuildOptions & options);
|
||||
};
|
||||
|
||||
struct ServeProto::BasicServerConnection
|
||||
{
|
||||
/**
|
||||
* Establishes connection, negotiating version.
|
||||
*
|
||||
* @return the version provided by the other side of the
|
||||
* connection.
|
||||
*
|
||||
* @param to Taken by reference to allow for various error handling
|
||||
* mechanisms.
|
||||
*
|
||||
* @param from Taken by reference to allow for various error
|
||||
* handling mechanisms.
|
||||
*
|
||||
* @param localVersion Our version which is sent over
|
||||
*/
|
||||
static ServeProto::Version handshake(
|
||||
BufferedSink & to,
|
||||
Source & from,
|
||||
ServeProto::Version localVersion);
|
||||
};
|
||||
|
||||
}
|
||||
|
|
|
|||
43
src/libstore/ssh-store-config.cc
Normal file
43
src/libstore/ssh-store-config.cc
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
#include <regex>
|
||||
|
||||
#include "ssh-store-config.hh"
|
||||
#include "ssh.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
static std::string extractConnStr(std::string_view scheme, std::string_view _connStr)
|
||||
{
|
||||
if (_connStr.empty())
|
||||
throw UsageError("`%s` store requires a valid SSH host as the authority part in Store URI", scheme);
|
||||
|
||||
std::string connStr{_connStr};
|
||||
|
||||
std::smatch result;
|
||||
static std::regex v6AddrRegex("^((.*)@)?\\[(.*)\\]$");
|
||||
|
||||
if (std::regex_match(connStr, result, v6AddrRegex)) {
|
||||
connStr = result[1].matched ? result.str(1) + result.str(3) : result.str(3);
|
||||
}
|
||||
|
||||
return connStr;
|
||||
}
|
||||
|
||||
CommonSSHStoreConfig::CommonSSHStoreConfig(std::string_view scheme, std::string_view host, const Params & params)
|
||||
: StoreConfig(params)
|
||||
, host(extractConnStr(scheme, host))
|
||||
{
|
||||
}
|
||||
|
||||
SSHMaster CommonSSHStoreConfig::createSSHMaster(bool useMaster, Descriptor logFD)
|
||||
{
|
||||
return {
|
||||
host,
|
||||
sshKey.get(),
|
||||
sshPublicHostKey.get(),
|
||||
useMaster,
|
||||
compress,
|
||||
logFD,
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -5,10 +5,14 @@
|
|||
|
||||
namespace nix {
|
||||
|
||||
class SSHMaster;
|
||||
|
||||
struct CommonSSHStoreConfig : virtual StoreConfig
|
||||
{
|
||||
using StoreConfig::StoreConfig;
|
||||
|
||||
CommonSSHStoreConfig(std::string_view scheme, std::string_view host, const Params & params);
|
||||
|
||||
const Setting<Path> sshKey{this, "", "ssh-key",
|
||||
"Path to the SSH private key used to authenticate to the remote machine."};
|
||||
|
||||
|
|
@ -24,6 +28,35 @@ struct CommonSSHStoreConfig : virtual StoreConfig
|
|||
to be used on the remote machine. The default is `auto`
|
||||
(i.e. use the Nix daemon or `/nix/store` directly).
|
||||
)"};
|
||||
|
||||
/**
|
||||
* The `parseURL` function supports both IPv6 URIs as defined in
|
||||
* RFC2732, but also pure addresses. The latter one is needed here to
|
||||
* connect to a remote store via SSH (it's possible to do e.g. `ssh root@::1`).
|
||||
*
|
||||
* When initialized, the following adjustments are made:
|
||||
*
|
||||
* - If the URL looks like `root@[::1]` (which is allowed by the URL parser and probably
|
||||
* needed to pass further flags), it
|
||||
* will be transformed into `root@::1` for SSH (same for `[::1]` -> `::1`).
|
||||
*
|
||||
* - If the URL looks like `root@::1` it will be left as-is.
|
||||
*
|
||||
* - In any other case, the string will be left as-is.
|
||||
*
|
||||
* Will throw an error if `connStr` is empty too.
|
||||
*/
|
||||
std::string host;
|
||||
|
||||
/**
|
||||
* Small wrapper around `SSHMaster::SSHMaster` that gets most
|
||||
* arguments from this configuration.
|
||||
*
|
||||
* See that constructor for details on the remaining two arguments.
|
||||
*/
|
||||
SSHMaster createSSHMaster(
|
||||
bool useMaster,
|
||||
Descriptor logFD = INVALID_DESCRIPTOR);
|
||||
};
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,21 +34,19 @@ class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore
|
|||
{
|
||||
public:
|
||||
|
||||
SSHStore(const std::string & scheme, const std::string & host, const Params & params)
|
||||
SSHStore(
|
||||
std::string_view scheme,
|
||||
std::string_view host,
|
||||
const Params & params)
|
||||
: StoreConfig(params)
|
||||
, RemoteStoreConfig(params)
|
||||
, CommonSSHStoreConfig(params)
|
||||
, CommonSSHStoreConfig(scheme, host, params)
|
||||
, SSHStoreConfig(params)
|
||||
, Store(params)
|
||||
, RemoteStore(params)
|
||||
, host(host)
|
||||
, master(
|
||||
host,
|
||||
sshKey,
|
||||
sshPublicHostKey,
|
||||
, master(createSSHMaster(
|
||||
// Use SSH master only if using more than 1 connection.
|
||||
connections->capacity() > 1,
|
||||
compress)
|
||||
connections->capacity() > 1))
|
||||
{
|
||||
}
|
||||
|
||||
|
|
@ -108,6 +106,15 @@ struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfi
|
|||
{
|
||||
}
|
||||
|
||||
MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params)
|
||||
: StoreConfig(params)
|
||||
, RemoteStoreConfig(params)
|
||||
, CommonSSHStoreConfig(scheme, host, params)
|
||||
, SSHStoreConfig(params)
|
||||
, LocalFSStoreConfig(params)
|
||||
{
|
||||
}
|
||||
|
||||
const std::string name() override { return "Experimental SSH Store with filesystem mounted"; }
|
||||
|
||||
std::string doc() override
|
||||
|
|
@ -141,10 +148,13 @@ class MountedSSHStore : public virtual MountedSSHStoreConfig, public virtual SSH
|
|||
{
|
||||
public:
|
||||
|
||||
MountedSSHStore(const std::string & scheme, const std::string & host, const Params & params)
|
||||
MountedSSHStore(
|
||||
std::string_view scheme,
|
||||
std::string_view host,
|
||||
const Params & params)
|
||||
: StoreConfig(params)
|
||||
, RemoteStoreConfig(params)
|
||||
, CommonSSHStoreConfig(params)
|
||||
, CommonSSHStoreConfig(scheme, host, params)
|
||||
, SSHStoreConfig(params)
|
||||
, LocalFSStoreConfig(params)
|
||||
, MountedSSHStoreConfig(params)
|
||||
|
|
|
|||
|
|
@ -6,7 +6,11 @@
|
|||
|
||||
namespace nix {
|
||||
|
||||
SSHMaster::SSHMaster(const std::string & host, const std::string & keyFile, const std::string & sshPublicHostKey, bool useMaster, bool compress, int logFD)
|
||||
SSHMaster::SSHMaster(
|
||||
std::string_view host,
|
||||
std::string_view keyFile,
|
||||
std::string_view sshPublicHostKey,
|
||||
bool useMaster, bool compress, Descriptor logFD)
|
||||
: host(host)
|
||||
, fakeSSH(host == "localhost")
|
||||
, keyFile(keyFile)
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ private:
|
|||
const std::string sshPublicHostKey;
|
||||
const bool useMaster;
|
||||
const bool compress;
|
||||
const int logFD;
|
||||
const Descriptor logFD;
|
||||
|
||||
struct State
|
||||
{
|
||||
|
|
@ -39,7 +39,11 @@ private:
|
|||
|
||||
public:
|
||||
|
||||
SSHMaster(const std::string & host, const std::string & keyFile, const std::string & sshPublicHostKey, bool useMaster, bool compress, int logFD = -1);
|
||||
SSHMaster(
|
||||
std::string_view host,
|
||||
std::string_view keyFile,
|
||||
std::string_view sshPublicHostKey,
|
||||
bool useMaster, bool compress, Descriptor logFD = INVALID_DESCRIPTOR);
|
||||
|
||||
struct Connection
|
||||
{
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@
|
|||
#include "util.hh"
|
||||
#include "nar-info-disk-cache.hh"
|
||||
#include "thread-pool.hh"
|
||||
#include "url.hh"
|
||||
#include "references.hh"
|
||||
#include "archive.hh"
|
||||
#include "callback.hh"
|
||||
|
|
@ -21,7 +20,6 @@
|
|||
#include "users.hh"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <regex>
|
||||
|
||||
using json = nlohmann::json;
|
||||
|
||||
|
|
@ -1274,144 +1272,63 @@ Derivation Store::readInvalidDerivation(const StorePath & drvPath)
|
|||
|
||||
namespace nix {
|
||||
|
||||
/* Split URI into protocol+hierarchy part and its parameter set. */
|
||||
std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_)
|
||||
{
|
||||
auto uri(uri_);
|
||||
Store::Params params;
|
||||
auto q = uri.find('?');
|
||||
if (q != std::string::npos) {
|
||||
params = decodeQuery(uri.substr(q + 1));
|
||||
uri = uri_.substr(0, q);
|
||||
}
|
||||
return {uri, params};
|
||||
}
|
||||
|
||||
static bool isNonUriPath(const std::string & spec)
|
||||
{
|
||||
return
|
||||
// is not a URL
|
||||
spec.find("://") == std::string::npos
|
||||
// Has at least one path separator, and so isn't a single word that
|
||||
// might be special like "auto"
|
||||
&& spec.find("/") != std::string::npos;
|
||||
}
|
||||
|
||||
std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Params & params)
|
||||
{
|
||||
// TODO reenable on Windows once we have `LocalStore` and
|
||||
// `UDSRemoteStore`.
|
||||
if (uri == "" || uri == "auto") {
|
||||
auto stateDir = getOr(params, "state", settings.nixStateDir);
|
||||
if (access(stateDir.c_str(), R_OK | W_OK) == 0)
|
||||
return std::make_shared<LocalStore>(params);
|
||||
else if (pathExists(settings.nixDaemonSocketFile))
|
||||
return std::make_shared<UDSRemoteStore>(params);
|
||||
#if __linux__
|
||||
else if (!pathExists(stateDir)
|
||||
&& params.empty()
|
||||
&& !isRootUser()
|
||||
&& !getEnv("NIX_STORE_DIR").has_value()
|
||||
&& !getEnv("NIX_STATE_DIR").has_value())
|
||||
{
|
||||
/* If /nix doesn't exist, there is no daemon socket, and
|
||||
we're not root, then automatically set up a chroot
|
||||
store in ~/.local/share/nix/root. */
|
||||
auto chrootStore = getDataDir() + "/nix/root";
|
||||
if (!pathExists(chrootStore)) {
|
||||
try {
|
||||
createDirs(chrootStore);
|
||||
} catch (Error & e) {
|
||||
return std::make_shared<LocalStore>(params);
|
||||
}
|
||||
warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore);
|
||||
} else
|
||||
debug("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore);
|
||||
Store::Params params2;
|
||||
params2["root"] = chrootStore;
|
||||
return std::make_shared<LocalStore>(params2);
|
||||
}
|
||||
#endif
|
||||
else
|
||||
return std::make_shared<LocalStore>(params);
|
||||
} else if (uri == "daemon") {
|
||||
return std::make_shared<UDSRemoteStore>(params);
|
||||
} else if (uri == "local") {
|
||||
return std::make_shared<LocalStore>(params);
|
||||
} else if (isNonUriPath(uri)) {
|
||||
Store::Params params2 = params;
|
||||
params2["root"] = absPath(uri);
|
||||
return std::make_shared<LocalStore>(params2);
|
||||
} else {
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
// The `parseURL` function supports both IPv6 URIs as defined in
|
||||
// RFC2732, but also pure addresses. The latter one is needed here to
|
||||
// connect to a remote store via SSH (it's possible to do e.g. `ssh root@::1`).
|
||||
//
|
||||
// This function now ensures that a usable connection string is available:
|
||||
// * If the store to be opened is not an SSH store, nothing will be done.
|
||||
// * If the URL looks like `root@[::1]` (which is allowed by the URL parser and probably
|
||||
// needed to pass further flags), it
|
||||
// will be transformed into `root@::1` for SSH (same for `[::1]` -> `::1`).
|
||||
// * If the URL looks like `root@::1` it will be left as-is.
|
||||
// * In any other case, the string will be left as-is.
|
||||
static std::string extractConnStr(const std::string &proto, const std::string &connStr)
|
||||
{
|
||||
if (proto.rfind("ssh") != std::string::npos) {
|
||||
std::smatch result;
|
||||
std::regex v6AddrRegex("^((.*)@)?\\[(.*)\\]$");
|
||||
|
||||
if (std::regex_match(connStr, result, v6AddrRegex)) {
|
||||
if (result[1].matched) {
|
||||
return result.str(1) + result.str(3);
|
||||
}
|
||||
return result.str(3);
|
||||
}
|
||||
}
|
||||
|
||||
return connStr;
|
||||
}
|
||||
|
||||
ref<Store> openStore(const std::string & uri_,
|
||||
ref<Store> openStore(const std::string & uri,
|
||||
const Store::Params & extraParams)
|
||||
{
|
||||
auto params = extraParams;
|
||||
try {
|
||||
auto parsedUri = parseURL(uri_);
|
||||
params.insert(parsedUri.query.begin(), parsedUri.query.end());
|
||||
return openStore(StoreReference::parse(uri, extraParams));
|
||||
}
|
||||
|
||||
auto baseURI = extractConnStr(
|
||||
parsedUri.scheme,
|
||||
parsedUri.authority.value_or("") + parsedUri.path
|
||||
);
|
||||
ref<Store> openStore(StoreReference && storeURI)
|
||||
{
|
||||
auto & params = storeURI.params;
|
||||
|
||||
for (auto implem : *Implementations::registered) {
|
||||
if (implem.uriSchemes.count(parsedUri.scheme)) {
|
||||
auto store = implem.create(parsedUri.scheme, baseURI, params);
|
||||
if (store) {
|
||||
experimentalFeatureSettings.require(store->experimentalFeature());
|
||||
store->init();
|
||||
store->warnUnknownSettings();
|
||||
return ref<Store>(store);
|
||||
}
|
||||
auto store = std::visit(overloaded {
|
||||
[&](const StoreReference::Auto &) -> std::shared_ptr<Store> {
|
||||
auto stateDir = getOr(params, "state", settings.nixStateDir);
|
||||
if (access(stateDir.c_str(), R_OK | W_OK) == 0)
|
||||
return std::make_shared<LocalStore>(params);
|
||||
else if (pathExists(settings.nixDaemonSocketFile))
|
||||
return std::make_shared<UDSRemoteStore>(params);
|
||||
#if __linux__
|
||||
else if (!pathExists(stateDir)
|
||||
&& params.empty()
|
||||
&& !isRootUser()
|
||||
&& !getEnv("NIX_STORE_DIR").has_value()
|
||||
&& !getEnv("NIX_STATE_DIR").has_value())
|
||||
{
|
||||
/* If /nix doesn't exist, there is no daemon socket, and
|
||||
we're not root, then automatically set up a chroot
|
||||
store in ~/.local/share/nix/root. */
|
||||
auto chrootStore = getDataDir() + "/nix/root";
|
||||
if (!pathExists(chrootStore)) {
|
||||
try {
|
||||
createDirs(chrootStore);
|
||||
} catch (Error & e) {
|
||||
return std::make_shared<LocalStore>(params);
|
||||
}
|
||||
warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore);
|
||||
} else
|
||||
debug("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore);
|
||||
return std::make_shared<LocalStore>("local", chrootStore, params);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (BadURL &) {
|
||||
auto [uri, uriParams] = splitUriAndParams(uri_);
|
||||
params.insert(uriParams.begin(), uriParams.end());
|
||||
#endif
|
||||
else
|
||||
return std::make_shared<LocalStore>(params);
|
||||
},
|
||||
[&](const StoreReference::Specified & g) {
|
||||
for (auto implem : *Implementations::registered)
|
||||
if (implem.uriSchemes.count(g.scheme))
|
||||
return implem.create(g.scheme, g.authority, params);
|
||||
|
||||
if (auto store = openFromNonUri(uri, params)) {
|
||||
experimentalFeatureSettings.require(store->experimentalFeature());
|
||||
store->warnUnknownSettings();
|
||||
return ref<Store>(store);
|
||||
}
|
||||
}
|
||||
throw Error("don't know how to open Nix store with scheme '%s'", g.scheme);
|
||||
},
|
||||
}, storeURI.variant);
|
||||
|
||||
throw Error("don't know how to open Nix store '%s'", uri_);
|
||||
experimentalFeatureSettings.require(store->experimentalFeature());
|
||||
store->warnUnknownSettings();
|
||||
store->init();
|
||||
|
||||
return ref<Store> { store };
|
||||
}
|
||||
|
||||
std::list<ref<Store>> getDefaultSubstituters()
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@
|
|||
#include "path-info.hh"
|
||||
#include "repair-flag.hh"
|
||||
#include "store-dir-config.hh"
|
||||
#include "store-reference.hh"
|
||||
#include "source-path.hh"
|
||||
|
||||
#include <nlohmann/json_fwd.hpp>
|
||||
|
|
@ -65,7 +66,7 @@ MakeError(Unsupported, Error);
|
|||
MakeError(SubstituteGone, Error);
|
||||
MakeError(SubstituterDisabled, Error);
|
||||
|
||||
MakeError(InvalidStoreURI, Error);
|
||||
MakeError(InvalidStoreReference, Error);
|
||||
|
||||
struct Realisation;
|
||||
struct RealisedPath;
|
||||
|
|
@ -91,7 +92,7 @@ enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true };
|
|||
const uint32_t exportMagic = 0x4558494e;
|
||||
|
||||
|
||||
enum BuildMode { bmNormal, bmRepair, bmCheck };
|
||||
enum BuildMode : uint8_t { bmNormal, bmRepair, bmCheck };
|
||||
enum TrustedFlag : bool { NotTrusted = false, Trusted = true };
|
||||
|
||||
struct BuildResult;
|
||||
|
|
@ -102,7 +103,7 @@ typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
|
|||
|
||||
struct StoreConfig : public StoreDirConfig
|
||||
{
|
||||
typedef std::map<std::string, std::string> Params;
|
||||
using Params = StoreReference::Params;
|
||||
|
||||
using StoreDirConfig::StoreDirConfig;
|
||||
|
||||
|
|
@ -859,34 +860,13 @@ OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * ev
|
|||
/**
|
||||
* @return a Store object to access the Nix store denoted by
|
||||
* ‘uri’ (slight misnomer...).
|
||||
*
|
||||
* @param uri Supported values are:
|
||||
*
|
||||
* - ‘local’: The Nix store in /nix/store and database in
|
||||
* /nix/var/nix/db, accessed directly.
|
||||
*
|
||||
* - ‘daemon’: The Nix store accessed via a Unix domain socket
|
||||
* connection to nix-daemon.
|
||||
*
|
||||
* - ‘unix://<path>’: The Nix store accessed via a Unix domain socket
|
||||
* connection to nix-daemon, with the socket located at <path>.
|
||||
*
|
||||
* - ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on
|
||||
* whether the user has write access to the local Nix
|
||||
* store/database.
|
||||
*
|
||||
* - ‘file://<path>’: A binary cache stored in <path>.
|
||||
*
|
||||
* - ‘https://<path>’: A binary cache accessed via HTTP.
|
||||
*
|
||||
* - ‘s3://<path>’: A writable binary cache stored on Amazon's Simple
|
||||
* Storage Service.
|
||||
*
|
||||
* - ‘ssh://[user@]<host>’: A remote Nix store accessed by running
|
||||
* ‘nix-store --serve’ via SSH.
|
||||
*
|
||||
* You can pass parameters to the store type by appending
|
||||
* ‘?key=value&key=value&...’ to the URI.
|
||||
*/
|
||||
ref<Store> openStore(StoreReference && storeURI);
|
||||
|
||||
|
||||
/**
|
||||
* Opens the store at `uri`, where `uri` is in the format expected by `StoreReference::parse`
|
||||
|
||||
*/
|
||||
ref<Store> openStore(const std::string & uri = settings.storeUri.get(),
|
||||
const Store::Params & extraParams = Store::Params());
|
||||
|
|
@ -901,7 +881,14 @@ std::list<ref<Store>> getDefaultSubstituters();
|
|||
struct StoreFactory
|
||||
{
|
||||
std::set<std::string> uriSchemes;
|
||||
std::function<std::shared_ptr<Store> (const std::string & scheme, const std::string & uri, const Store::Params & params)> create;
|
||||
/**
|
||||
* The `authorityPath` parameter is `<authority>/<path>`, or really
|
||||
* whatever comes after `<scheme>://` and before `?<query-params>`.
|
||||
*/
|
||||
std::function<std::shared_ptr<Store> (
|
||||
std::string_view scheme,
|
||||
std::string_view authorityPath,
|
||||
const Store::Params & params)> create;
|
||||
std::function<std::shared_ptr<StoreConfig> ()> getConfig;
|
||||
};
|
||||
|
||||
|
|
@ -916,7 +903,7 @@ struct Implementations
|
|||
StoreFactory factory{
|
||||
.uriSchemes = T::uriSchemes(),
|
||||
.create =
|
||||
([](const std::string & scheme, const std::string & uri, const Store::Params & params)
|
||||
([](auto scheme, auto uri, auto & params)
|
||||
-> std::shared_ptr<Store>
|
||||
{ return std::make_shared<T>(scheme, uri, params); }),
|
||||
.getConfig =
|
||||
|
|
@ -950,11 +937,6 @@ std::optional<ValidPathInfo> decodeValidPathInfo(
|
|||
std::istream & str,
|
||||
std::optional<HashResult> hashGiven = std::nullopt);
|
||||
|
||||
/**
|
||||
* Split URI into protocol+hierarchy part and its parameter set.
|
||||
*/
|
||||
std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri);
|
||||
|
||||
const ContentAddress * getDerivationCA(const BasicDerivation & drv);
|
||||
|
||||
std::map<DrvOutput, StorePath> drvOutputReferences(
|
||||
|
|
|
|||
116
src/libstore/store-reference.cc
Normal file
116
src/libstore/store-reference.cc
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
#include <regex>
|
||||
|
||||
#include "error.hh"
|
||||
#include "url.hh"
|
||||
#include "store-reference.hh"
|
||||
#include "file-system.hh"
|
||||
#include "util.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
static bool isNonUriPath(const std::string & spec)
|
||||
{
|
||||
return
|
||||
// is not a URL
|
||||
spec.find("://") == std::string::npos
|
||||
// Has at least one path separator, and so isn't a single word that
|
||||
// might be special like "auto"
|
||||
&& spec.find("/") != std::string::npos;
|
||||
}
|
||||
|
||||
std::string StoreReference::render() const
|
||||
{
|
||||
std::string res;
|
||||
|
||||
std::visit(
|
||||
overloaded{
|
||||
[&](const StoreReference::Auto &) { res = "auto"; },
|
||||
[&](const StoreReference::Specified & g) {
|
||||
res = g.scheme;
|
||||
res += "://";
|
||||
res += g.authority;
|
||||
},
|
||||
},
|
||||
variant);
|
||||
|
||||
if (!params.empty()) {
|
||||
res += "?";
|
||||
res += encodeQuery(params);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
StoreReference StoreReference::parse(const std::string & uri, const StoreReference::Params & extraParams)
|
||||
{
|
||||
auto params = extraParams;
|
||||
try {
|
||||
auto parsedUri = parseURL(uri);
|
||||
params.insert(parsedUri.query.begin(), parsedUri.query.end());
|
||||
|
||||
auto baseURI = parsedUri.authority.value_or("") + parsedUri.path;
|
||||
|
||||
return {
|
||||
.variant =
|
||||
Specified{
|
||||
.scheme = std::move(parsedUri.scheme),
|
||||
.authority = std::move(baseURI),
|
||||
},
|
||||
.params = std::move(params),
|
||||
};
|
||||
} catch (BadURL &) {
|
||||
auto [baseURI, uriParams] = splitUriAndParams(uri);
|
||||
params.insert(uriParams.begin(), uriParams.end());
|
||||
|
||||
if (baseURI == "" || baseURI == "auto") {
|
||||
return {
|
||||
.variant = Auto{},
|
||||
.params = std::move(params),
|
||||
};
|
||||
} else if (baseURI == "daemon") {
|
||||
return {
|
||||
.variant =
|
||||
Specified{
|
||||
.scheme = "unix",
|
||||
.authority = "",
|
||||
},
|
||||
.params = std::move(params),
|
||||
};
|
||||
} else if (baseURI == "local") {
|
||||
return {
|
||||
.variant =
|
||||
Specified{
|
||||
.scheme = "local",
|
||||
.authority = "",
|
||||
},
|
||||
.params = std::move(params),
|
||||
};
|
||||
} else if (isNonUriPath(baseURI)) {
|
||||
return {
|
||||
.variant =
|
||||
Specified{
|
||||
.scheme = "local",
|
||||
.authority = absPath(baseURI),
|
||||
},
|
||||
.params = std::move(params),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
throw UsageError("Cannot parse Nix store '%s'", uri);
|
||||
}
|
||||
|
||||
/* Split URI into protocol+hierarchy part and its parameter set. */
|
||||
std::pair<std::string, StoreReference::Params> splitUriAndParams(const std::string & uri_)
|
||||
{
|
||||
auto uri(uri_);
|
||||
StoreReference::Params params;
|
||||
auto q = uri.find('?');
|
||||
if (q != std::string::npos) {
|
||||
params = decodeQuery(uri.substr(q + 1));
|
||||
uri = uri_.substr(0, q);
|
||||
}
|
||||
return {uri, params};
|
||||
}
|
||||
|
||||
}
|
||||
92
src/libstore/store-reference.hh
Normal file
92
src/libstore/store-reference.hh
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
#pragma once
|
||||
///@file
|
||||
|
||||
#include <variant>
|
||||
|
||||
#include "types.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
/**
|
||||
* A parsed Store URI (URI is a slight misnomer...), parsed but not yet
|
||||
* resolved to a specific instance and query parms validated.
|
||||
*
|
||||
* Supported values are:
|
||||
*
|
||||
* - ‘local’: The Nix store in /nix/store and database in
|
||||
* /nix/var/nix/db, accessed directly.
|
||||
*
|
||||
* - ‘daemon’: The Nix store accessed via a Unix domain socket
|
||||
* connection to nix-daemon.
|
||||
*
|
||||
* - ‘unix://<path>’: The Nix store accessed via a Unix domain socket
|
||||
* connection to nix-daemon, with the socket located at <path>.
|
||||
*
|
||||
* - ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on
|
||||
* whether the user has write access to the local Nix
|
||||
* store/database.
|
||||
*
|
||||
* - ‘file://<path>’: A binary cache stored in <path>.
|
||||
*
|
||||
* - ‘https://<path>’: A binary cache accessed via HTTP.
|
||||
*
|
||||
* - ‘s3://<path>’: A writable binary cache stored on Amazon's Simple
|
||||
* Storage Service.
|
||||
*
|
||||
* - ‘ssh://[user@]<host>’: A remote Nix store accessed by running
|
||||
* ‘nix-store --serve’ via SSH.
|
||||
*
|
||||
* You can pass parameters to the store type by appending
|
||||
* ‘?key=value&key=value&...’ to the URI.
|
||||
*/
|
||||
struct StoreReference
|
||||
{
|
||||
using Params = std::map<std::string, std::string>;
|
||||
|
||||
/**
|
||||
* Special store reference `""` or `"auto"`
|
||||
*/
|
||||
struct Auto
|
||||
{
|
||||
inline bool operator==(const Auto & rhs) const = default;
|
||||
inline auto operator<=>(const Auto & rhs) const = default;
|
||||
};
|
||||
|
||||
/**
|
||||
* General case, a regular `scheme://authority` URL.
|
||||
*/
|
||||
struct Specified
|
||||
{
|
||||
std::string scheme;
|
||||
std::string authority = "";
|
||||
|
||||
bool operator==(const Specified & rhs) const = default;
|
||||
auto operator<=>(const Specified & rhs) const = default;
|
||||
};
|
||||
|
||||
typedef std::variant<Auto, Specified> Variant;
|
||||
|
||||
Variant variant;
|
||||
|
||||
Params params;
|
||||
|
||||
bool operator==(const StoreReference & rhs) const = default;
|
||||
auto operator<=>(const StoreReference & rhs) const = default;
|
||||
|
||||
/**
|
||||
* Render the whole store reference as a URI, including parameters.
|
||||
*/
|
||||
std::string render() const;
|
||||
|
||||
/**
|
||||
* Parse a URI into a store reference.
|
||||
*/
|
||||
static StoreReference parse(const std::string & uri, const Params & extraParams = Params{});
|
||||
};
|
||||
|
||||
/**
|
||||
* Split URI into protocol+hierarchy part and its parameter set.
|
||||
*/
|
||||
std::pair<std::string, StoreReference::Params> splitUriAndParams(const std::string & uri);
|
||||
|
||||
}
|
||||
|
|
@ -40,12 +40,13 @@ UDSRemoteStore::UDSRemoteStore(const Params & params)
|
|||
|
||||
|
||||
UDSRemoteStore::UDSRemoteStore(
|
||||
const std::string scheme,
|
||||
std::string socket_path,
|
||||
std::string_view scheme,
|
||||
PathView socket_path,
|
||||
const Params & params)
|
||||
: UDSRemoteStore(params)
|
||||
{
|
||||
path.emplace(socket_path);
|
||||
if (!socket_path.empty())
|
||||
path.emplace(socket_path);
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -54,6 +55,7 @@ std::string UDSRemoteStore::getUri()
|
|||
if (path) {
|
||||
return std::string("unix://") + *path;
|
||||
} else {
|
||||
// unix:// with no path also works. Change what we return?
|
||||
return "daemon";
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,7 +28,10 @@ class UDSRemoteStore : public virtual UDSRemoteStoreConfig
|
|||
public:
|
||||
|
||||
UDSRemoteStore(const Params & params);
|
||||
UDSRemoteStore(const std::string scheme, std::string path, const Params & params);
|
||||
UDSRemoteStore(
|
||||
std::string_view scheme,
|
||||
PathView path,
|
||||
const Params & params);
|
||||
|
||||
std::string getUri() override;
|
||||
|
||||
|
|
|
|||
|
|
@ -285,7 +285,7 @@ static void movePath(const Path & src, const Path & dst)
|
|||
if (changePerm)
|
||||
chmod_(src, st.st_mode | S_IWUSR);
|
||||
|
||||
renameFile(src, dst);
|
||||
std::filesystem::rename(src, dst);
|
||||
|
||||
if (changePerm)
|
||||
chmod_(dst, st.st_mode);
|
||||
|
|
@ -372,7 +372,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull()
|
|||
if (buildMode != bmCheck && status.known->isValid()) continue;
|
||||
auto p = worker.store.toRealPath(status.known->path);
|
||||
if (pathExists(chrootRootDir + p))
|
||||
renameFile((chrootRootDir + p), p);
|
||||
std::filesystem::rename((chrootRootDir + p), p);
|
||||
}
|
||||
|
||||
return diskFull;
|
||||
|
|
@ -421,7 +421,9 @@ static void doBind(const Path & source, const Path & target, bool optional = fal
|
|||
} else if (S_ISLNK(st.st_mode)) {
|
||||
// Symlinks can (apparently) not be bind-mounted, so just copy it
|
||||
createDirs(dirOf(target));
|
||||
copyFile(source, target, /* andDelete */ false);
|
||||
copyFile(
|
||||
std::filesystem::path(source),
|
||||
std::filesystem::path(target), false);
|
||||
} else {
|
||||
createDirs(dirOf(target));
|
||||
writeFile(target, "");
|
||||
|
|
@ -2568,8 +2570,11 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
|
|||
// Replace the output by a fresh copy of itself to make sure
|
||||
// that there's no stale file descriptor pointing to it
|
||||
Path tmpOutput = actualPath + ".tmp";
|
||||
copyFile(actualPath, tmpOutput, true);
|
||||
renameFile(tmpOutput, actualPath);
|
||||
copyFile(
|
||||
std::filesystem::path(actualPath),
|
||||
std::filesystem::path(tmpOutput), true);
|
||||
|
||||
std::filesystem::rename(tmpOutput, actualPath);
|
||||
|
||||
auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating {
|
||||
.method = dof.ca.method,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
#include "lock.hh"
|
||||
#include "user-lock.hh"
|
||||
#include "file-system.hh"
|
||||
#include "globals.hh"
|
||||
#include "pathlocks.hh"
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
#include "store-api.hh"
|
||||
#include "build-result.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMode, std::shared_ptr<Store> evalStore)
|
||||
{
|
||||
unsupported("buildPaths");
|
||||
}
|
||||
|
||||
std::vector<KeyedBuildResult> Store::buildPathsWithResults(
|
||||
const std::vector<DerivedPath> & reqs,
|
||||
BuildMode buildMode,
|
||||
std::shared_ptr<Store> evalStore)
|
||||
{
|
||||
unsupported("buildPathsWithResults");
|
||||
}
|
||||
|
||||
BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
|
||||
BuildMode buildMode)
|
||||
{
|
||||
unsupported("buildDerivation");
|
||||
}
|
||||
|
||||
|
||||
void Store::ensurePath(const StorePath & path)
|
||||
{
|
||||
unsupported("ensurePath");
|
||||
}
|
||||
|
||||
|
||||
void Store::repairPath(const StorePath & path)
|
||||
{
|
||||
unsupported("repairPath");
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -9,6 +9,8 @@
|
|||
|
||||
namespace nix {
|
||||
|
||||
using namespace nix::windows;
|
||||
|
||||
void deleteLockFile(const Path & path, Descriptor desc)
|
||||
{
|
||||
|
||||
|
|
@ -35,8 +37,13 @@ void PathLocks::unlock()
|
|||
AutoCloseFD openLockFile(const Path & path, bool create)
|
||||
{
|
||||
AutoCloseFD desc = CreateFileA(
|
||||
path.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL,
|
||||
create ? OPEN_ALWAYS : OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_POSIX_SEMANTICS, NULL);
|
||||
path.c_str(),
|
||||
GENERIC_READ | GENERIC_WRITE,
|
||||
FILE_SHARE_READ | FILE_SHARE_WRITE,
|
||||
NULL,
|
||||
create ? OPEN_ALWAYS : OPEN_EXISTING,
|
||||
FILE_ATTRIBUTE_NORMAL | FILE_FLAG_POSIX_SEMANTICS,
|
||||
NULL);
|
||||
if (desc.get() == INVALID_HANDLE_VALUE)
|
||||
warn("%s: %s", path, std::to_string(GetLastError()));
|
||||
|
||||
|
|
|
|||
280
src/libstore/worker-protocol-connection.cc
Normal file
280
src/libstore/worker-protocol-connection.cc
Normal file
|
|
@ -0,0 +1,280 @@
|
|||
#include "worker-protocol-connection.hh"
|
||||
#include "worker-protocol-impl.hh"
|
||||
#include "build-result.hh"
|
||||
#include "derivations.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
WorkerProto::BasicClientConnection::~BasicClientConnection()
|
||||
{
|
||||
try {
|
||||
to.flush();
|
||||
} catch (...) {
|
||||
ignoreException();
|
||||
}
|
||||
}
|
||||
|
||||
static Logger::Fields readFields(Source & from)
|
||||
{
|
||||
Logger::Fields fields;
|
||||
size_t size = readInt(from);
|
||||
for (size_t n = 0; n < size; n++) {
|
||||
auto type = (decltype(Logger::Field::type)) readInt(from);
|
||||
if (type == Logger::Field::tInt)
|
||||
fields.push_back(readNum<uint64_t>(from));
|
||||
else if (type == Logger::Field::tString)
|
||||
fields.push_back(readString(from));
|
||||
else
|
||||
throw Error("got unsupported field type %x from Nix daemon", (int) type);
|
||||
}
|
||||
return fields;
|
||||
}
|
||||
|
||||
std::exception_ptr WorkerProto::BasicClientConnection::processStderrReturn(Sink * sink, Source * source, bool flush)
|
||||
{
|
||||
if (flush)
|
||||
to.flush();
|
||||
|
||||
std::exception_ptr ex;
|
||||
|
||||
while (true) {
|
||||
|
||||
auto msg = readNum<uint64_t>(from);
|
||||
|
||||
if (msg == STDERR_WRITE) {
|
||||
auto s = readString(from);
|
||||
if (!sink)
|
||||
throw Error("no sink");
|
||||
(*sink)(s);
|
||||
}
|
||||
|
||||
else if (msg == STDERR_READ) {
|
||||
if (!source)
|
||||
throw Error("no source");
|
||||
size_t len = readNum<size_t>(from);
|
||||
auto buf = std::make_unique<char[]>(len);
|
||||
writeString({(const char *) buf.get(), source->read(buf.get(), len)}, to);
|
||||
to.flush();
|
||||
}
|
||||
|
||||
else if (msg == STDERR_ERROR) {
|
||||
if (GET_PROTOCOL_MINOR(daemonVersion) >= 26) {
|
||||
ex = std::make_exception_ptr(readError(from));
|
||||
} else {
|
||||
auto error = readString(from);
|
||||
unsigned int status = readInt(from);
|
||||
ex = std::make_exception_ptr(Error(status, error));
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
else if (msg == STDERR_NEXT)
|
||||
printError(chomp(readString(from)));
|
||||
|
||||
else if (msg == STDERR_START_ACTIVITY) {
|
||||
auto act = readNum<ActivityId>(from);
|
||||
auto lvl = (Verbosity) readInt(from);
|
||||
auto type = (ActivityType) readInt(from);
|
||||
auto s = readString(from);
|
||||
auto fields = readFields(from);
|
||||
auto parent = readNum<ActivityId>(from);
|
||||
logger->startActivity(act, lvl, type, s, fields, parent);
|
||||
}
|
||||
|
||||
else if (msg == STDERR_STOP_ACTIVITY) {
|
||||
auto act = readNum<ActivityId>(from);
|
||||
logger->stopActivity(act);
|
||||
}
|
||||
|
||||
else if (msg == STDERR_RESULT) {
|
||||
auto act = readNum<ActivityId>(from);
|
||||
auto type = (ResultType) readInt(from);
|
||||
auto fields = readFields(from);
|
||||
logger->result(act, type, fields);
|
||||
}
|
||||
|
||||
else if (msg == STDERR_LAST)
|
||||
break;
|
||||
|
||||
else
|
||||
throw Error("got unknown message type %x from Nix daemon", msg);
|
||||
}
|
||||
|
||||
if (!ex) {
|
||||
return ex;
|
||||
} else {
|
||||
try {
|
||||
std::rethrow_exception(ex);
|
||||
} catch (const Error & e) {
|
||||
// Nix versions before #4628 did not have an adequate
|
||||
// behavior for reporting that the derivation format was
|
||||
// upgraded. To avoid having to add compatibility logic in
|
||||
// many places, we expect to catch almost all occurrences of
|
||||
// the old incomprehensible error here, so that we can
|
||||
// explain to users what's going on when their daemon is
|
||||
// older than #4628 (2023).
|
||||
if (experimentalFeatureSettings.isEnabled(Xp::DynamicDerivations)
|
||||
&& GET_PROTOCOL_MINOR(daemonVersion) <= 35) {
|
||||
auto m = e.msg();
|
||||
if (m.find("parsing derivation") != std::string::npos && m.find("expected string") != std::string::npos
|
||||
&& m.find("Derive([") != std::string::npos)
|
||||
return std::make_exception_ptr(Error(
|
||||
"%s, this might be because the daemon is too old to understand dependencies on dynamic derivations. Check to see if the raw derivation is in the form '%s'",
|
||||
std::move(m),
|
||||
"Drv WithVersion(..)"));
|
||||
}
|
||||
return std::current_exception();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void WorkerProto::BasicClientConnection::processStderr(bool * daemonException, Sink * sink, Source * source, bool flush)
|
||||
{
|
||||
auto ex = processStderrReturn(sink, source, flush);
|
||||
if (ex) {
|
||||
*daemonException = true;
|
||||
std::rethrow_exception(ex);
|
||||
}
|
||||
}
|
||||
|
||||
WorkerProto::Version
|
||||
WorkerProto::BasicClientConnection::handshake(BufferedSink & to, Source & from, WorkerProto::Version localVersion)
|
||||
{
|
||||
to << WORKER_MAGIC_1 << localVersion;
|
||||
to.flush();
|
||||
|
||||
unsigned int magic = readInt(from);
|
||||
if (magic != WORKER_MAGIC_2)
|
||||
throw Error("nix-daemon protocol mismatch from");
|
||||
auto daemonVersion = readInt(from);
|
||||
|
||||
if (GET_PROTOCOL_MAJOR(daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION))
|
||||
throw Error("Nix daemon protocol version not supported");
|
||||
if (GET_PROTOCOL_MINOR(daemonVersion) < 10)
|
||||
throw Error("the Nix daemon version is too old");
|
||||
to << localVersion;
|
||||
|
||||
return std::min(daemonVersion, localVersion);
|
||||
}
|
||||
|
||||
WorkerProto::Version
|
||||
WorkerProto::BasicServerConnection::handshake(BufferedSink & to, Source & from, WorkerProto::Version localVersion)
|
||||
{
|
||||
unsigned int magic = readInt(from);
|
||||
if (magic != WORKER_MAGIC_1)
|
||||
throw Error("protocol mismatch");
|
||||
to << WORKER_MAGIC_2 << localVersion;
|
||||
to.flush();
|
||||
auto clientVersion = readInt(from);
|
||||
return std::min(clientVersion, localVersion);
|
||||
}
|
||||
|
||||
WorkerProto::ClientHandshakeInfo WorkerProto::BasicClientConnection::postHandshake(const StoreDirConfig & store)
|
||||
{
|
||||
WorkerProto::ClientHandshakeInfo res;
|
||||
|
||||
if (GET_PROTOCOL_MINOR(daemonVersion) >= 14) {
|
||||
// Obsolete CPU affinity.
|
||||
to << 0;
|
||||
}
|
||||
|
||||
if (GET_PROTOCOL_MINOR(daemonVersion) >= 11)
|
||||
to << false; // obsolete reserveSpace
|
||||
|
||||
if (GET_PROTOCOL_MINOR(daemonVersion) >= 33)
|
||||
to.flush();
|
||||
|
||||
return WorkerProto::Serialise<ClientHandshakeInfo>::read(store, *this);
|
||||
}
|
||||
|
||||
void WorkerProto::BasicServerConnection::postHandshake(const StoreDirConfig & store, const ClientHandshakeInfo & info)
|
||||
{
|
||||
if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) {
|
||||
// Obsolete CPU affinity.
|
||||
readInt(from);
|
||||
}
|
||||
|
||||
if (GET_PROTOCOL_MINOR(clientVersion) >= 11)
|
||||
readInt(from); // obsolete reserveSpace
|
||||
|
||||
WorkerProto::write(store, *this, info);
|
||||
}
|
||||
|
||||
UnkeyedValidPathInfo WorkerProto::BasicClientConnection::queryPathInfo(
|
||||
const StoreDirConfig & store, bool * daemonException, const StorePath & path)
|
||||
{
|
||||
to << WorkerProto::Op::QueryPathInfo << store.printStorePath(path);
|
||||
try {
|
||||
processStderr(daemonException);
|
||||
} catch (Error & e) {
|
||||
// Ugly backwards compatibility hack.
|
||||
if (e.msg().find("is not valid") != std::string::npos)
|
||||
throw InvalidPath(std::move(e.info()));
|
||||
throw;
|
||||
}
|
||||
if (GET_PROTOCOL_MINOR(daemonVersion) >= 17) {
|
||||
bool valid;
|
||||
from >> valid;
|
||||
if (!valid)
|
||||
throw InvalidPath("path '%s' is not valid", store.printStorePath(path));
|
||||
}
|
||||
return WorkerProto::Serialise<UnkeyedValidPathInfo>::read(store, *this);
|
||||
}
|
||||
|
||||
StorePathSet WorkerProto::BasicClientConnection::queryValidPaths(
|
||||
const StoreDirConfig & store, bool * daemonException, const StorePathSet & paths, SubstituteFlag maybeSubstitute)
|
||||
{
|
||||
assert(GET_PROTOCOL_MINOR(daemonVersion) >= 12);
|
||||
to << WorkerProto::Op::QueryValidPaths;
|
||||
WorkerProto::write(store, *this, paths);
|
||||
if (GET_PROTOCOL_MINOR(daemonVersion) >= 27) {
|
||||
to << maybeSubstitute;
|
||||
}
|
||||
processStderr(daemonException);
|
||||
return WorkerProto::Serialise<StorePathSet>::read(store, *this);
|
||||
}
|
||||
|
||||
void WorkerProto::BasicClientConnection::addTempRoot(
|
||||
const StoreDirConfig & store, bool * daemonException, const StorePath & path)
|
||||
{
|
||||
to << WorkerProto::Op::AddTempRoot << store.printStorePath(path);
|
||||
processStderr(daemonException);
|
||||
readInt(from);
|
||||
}
|
||||
|
||||
void WorkerProto::BasicClientConnection::putBuildDerivationRequest(
|
||||
const StoreDirConfig & store,
|
||||
bool * daemonException,
|
||||
const StorePath & drvPath,
|
||||
const BasicDerivation & drv,
|
||||
BuildMode buildMode)
|
||||
{
|
||||
to << WorkerProto::Op::BuildDerivation << store.printStorePath(drvPath);
|
||||
writeDerivation(to, store, drv);
|
||||
to << buildMode;
|
||||
}
|
||||
|
||||
BuildResult
|
||||
WorkerProto::BasicClientConnection::getBuildDerivationResponse(const StoreDirConfig & store, bool * daemonException)
|
||||
{
|
||||
return WorkerProto::Serialise<BuildResult>::read(store, *this);
|
||||
}
|
||||
|
||||
void WorkerProto::BasicClientConnection::narFromPath(
|
||||
const StoreDirConfig & store, bool * daemonException, const StorePath & path, std::function<void(Source &)> fun)
|
||||
{
|
||||
to << WorkerProto::Op::NarFromPath << store.printStorePath(path);
|
||||
processStderr(daemonException);
|
||||
|
||||
fun(from);
|
||||
}
|
||||
|
||||
void WorkerProto::BasicClientConnection::importPaths(
|
||||
const StoreDirConfig & store, bool * daemonException, Source & source)
|
||||
{
|
||||
to << WorkerProto::Op::ImportPaths;
|
||||
processStderr(daemonException, 0, &source);
|
||||
auto importedPaths = WorkerProto::Serialise<StorePathSet>::read(store, *this);
|
||||
assert(importedPaths.size() <= importedPaths.size());
|
||||
}
|
||||
}
|
||||
187
src/libstore/worker-protocol-connection.hh
Normal file
187
src/libstore/worker-protocol-connection.hh
Normal file
|
|
@ -0,0 +1,187 @@
|
|||
#pragma once
|
||||
///@file
|
||||
|
||||
#include "worker-protocol.hh"
|
||||
#include "store-api.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
struct WorkerProto::BasicClientConnection
|
||||
{
|
||||
/**
|
||||
* Send with this.
|
||||
*/
|
||||
FdSink to;
|
||||
|
||||
/**
|
||||
* Receive with this.
|
||||
*/
|
||||
FdSource from;
|
||||
|
||||
/**
|
||||
* Worker protocol version used for the connection.
|
||||
*
|
||||
* Despite its name, it is actually the maximum version both
|
||||
* sides support. (If the maximum doesn't exist, we would fail to
|
||||
* establish a connection and produce a value of this type.)
|
||||
*/
|
||||
WorkerProto::Version daemonVersion;
|
||||
|
||||
/**
|
||||
* Flush to direction
|
||||
*/
|
||||
virtual ~BasicClientConnection();
|
||||
|
||||
virtual void closeWrite() = 0;
|
||||
|
||||
std::exception_ptr processStderrReturn(Sink * sink = 0, Source * source = 0, bool flush = true);
|
||||
|
||||
void processStderr(bool * daemonException, Sink * sink = 0, Source * source = 0, bool flush = true);
|
||||
|
||||
/**
|
||||
* Establishes connection, negotiating version.
|
||||
*
|
||||
* @return the version provided by the other side of the
|
||||
* connection.
|
||||
*
|
||||
* @param to Taken by reference to allow for various error handling
|
||||
* mechanisms.
|
||||
*
|
||||
* @param from Taken by reference to allow for various error
|
||||
* handling mechanisms.
|
||||
*
|
||||
* @param localVersion Our version which is sent over
|
||||
*/
|
||||
static Version handshake(BufferedSink & to, Source & from, WorkerProto::Version localVersion);
|
||||
|
||||
/**
|
||||
* After calling handshake, must call this to exchange some basic
|
||||
* information abou the connection.
|
||||
*/
|
||||
ClientHandshakeInfo postHandshake(const StoreDirConfig & store);
|
||||
|
||||
/**
|
||||
* Coercion to `WorkerProto::ReadConn`. This makes it easy to use the
|
||||
* factored out serve protocol serializers with a
|
||||
* `LegacySSHStore::Connection`.
|
||||
*
|
||||
* The serve protocol connection types are unidirectional, unlike
|
||||
* this type.
|
||||
*/
|
||||
operator WorkerProto::ReadConn()
|
||||
{
|
||||
return WorkerProto::ReadConn{
|
||||
.from = from,
|
||||
.version = daemonVersion,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Coercion to `WorkerProto::WriteConn`. This makes it easy to use the
|
||||
* factored out serve protocol serializers with a
|
||||
* `LegacySSHStore::Connection`.
|
||||
*
|
||||
* The serve protocol connection types are unidirectional, unlike
|
||||
* this type.
|
||||
*/
|
||||
operator WorkerProto::WriteConn()
|
||||
{
|
||||
return WorkerProto::WriteConn{
|
||||
.to = to,
|
||||
.version = daemonVersion,
|
||||
};
|
||||
}
|
||||
|
||||
void addTempRoot(const StoreDirConfig & remoteStore, bool * daemonException, const StorePath & path);
|
||||
|
||||
StorePathSet queryValidPaths(
|
||||
const StoreDirConfig & remoteStore,
|
||||
bool * daemonException,
|
||||
const StorePathSet & paths,
|
||||
SubstituteFlag maybeSubstitute);
|
||||
|
||||
UnkeyedValidPathInfo queryPathInfo(const StoreDirConfig & store, bool * daemonException, const StorePath & path);
|
||||
|
||||
void putBuildDerivationRequest(
|
||||
const StoreDirConfig & store,
|
||||
bool * daemonException,
|
||||
const StorePath & drvPath,
|
||||
const BasicDerivation & drv,
|
||||
BuildMode buildMode);
|
||||
|
||||
/**
|
||||
* Get the response, must be paired with
|
||||
* `putBuildDerivationRequest`.
|
||||
*/
|
||||
BuildResult getBuildDerivationResponse(const StoreDirConfig & store, bool * daemonException);
|
||||
|
||||
void narFromPath(
|
||||
const StoreDirConfig & store,
|
||||
bool * daemonException,
|
||||
const StorePath & path,
|
||||
std::function<void(Source &)> fun);
|
||||
|
||||
void importPaths(const StoreDirConfig & store, bool * daemonException, Source & source);
|
||||
};
|
||||
|
||||
struct WorkerProto::BasicServerConnection
|
||||
{
|
||||
/**
|
||||
* Send with this.
|
||||
*/
|
||||
FdSink & to;
|
||||
|
||||
/**
|
||||
* Receive with this.
|
||||
*/
|
||||
FdSource & from;
|
||||
|
||||
/**
|
||||
* Worker protocol version used for the connection.
|
||||
*
|
||||
* Despite its name, it is actually the maximum version both
|
||||
* sides support. (If the maximum doesn't exist, we would fail to
|
||||
* establish a connection and produce a value of this type.)
|
||||
*/
|
||||
WorkerProto::Version clientVersion;
|
||||
|
||||
operator WorkerProto::ReadConn()
|
||||
{
|
||||
return WorkerProto::ReadConn{
|
||||
.from = from,
|
||||
.version = clientVersion,
|
||||
};
|
||||
}
|
||||
|
||||
operator WorkerProto::WriteConn()
|
||||
{
|
||||
return WorkerProto::WriteConn{
|
||||
.to = to,
|
||||
.version = clientVersion,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Establishes connection, negotiating version.
|
||||
*
|
||||
* @return the version provided by the other side of the
|
||||
* connection.
|
||||
*
|
||||
* @param to Taken by reference to allow for various error handling
|
||||
* mechanisms.
|
||||
*
|
||||
* @param from Taken by reference to allow for various error
|
||||
* handling mechanisms.
|
||||
*
|
||||
* @param localVersion Our version which is sent over
|
||||
*/
|
||||
static WorkerProto::Version handshake(BufferedSink & to, Source & from, WorkerProto::Version localVersion);
|
||||
|
||||
/**
|
||||
* After calling handshake, must call this to exchange some basic
|
||||
* information abou the connection.
|
||||
*/
|
||||
void postHandshake(const StoreDirConfig & store, const ClientHandshakeInfo & info);
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -14,6 +14,34 @@ namespace nix {
|
|||
|
||||
/* protocol-specific definitions */
|
||||
|
||||
BuildMode WorkerProto::Serialise<BuildMode>::read(const StoreDirConfig & store, WorkerProto::ReadConn conn)
|
||||
{
|
||||
auto temp = readNum<uint8_t>(conn.from);
|
||||
switch (temp) {
|
||||
case 0: return bmNormal;
|
||||
case 1: return bmRepair;
|
||||
case 2: return bmCheck;
|
||||
default: throw Error("Invalid build mode");
|
||||
}
|
||||
}
|
||||
|
||||
void WorkerProto::Serialise<BuildMode>::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const BuildMode & buildMode)
|
||||
{
|
||||
switch (buildMode) {
|
||||
case bmNormal:
|
||||
conn.to << uint8_t{0};
|
||||
break;
|
||||
case bmRepair:
|
||||
conn.to << uint8_t{1};
|
||||
break;
|
||||
case bmCheck:
|
||||
conn.to << uint8_t{2};
|
||||
break;
|
||||
default:
|
||||
assert(false);
|
||||
};
|
||||
}
|
||||
|
||||
std::optional<TrustedFlag> WorkerProto::Serialise<std::optional<TrustedFlag>>::read(const StoreDirConfig & store, WorkerProto::ReadConn conn)
|
||||
{
|
||||
auto temp = readNum<uint8_t>(conn.from);
|
||||
|
|
@ -222,4 +250,35 @@ void WorkerProto::Serialise<UnkeyedValidPathInfo>::write(const StoreDirConfig &
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
WorkerProto::ClientHandshakeInfo WorkerProto::Serialise<WorkerProto::ClientHandshakeInfo>::read(const StoreDirConfig & store, ReadConn conn)
|
||||
{
|
||||
WorkerProto::ClientHandshakeInfo res;
|
||||
|
||||
if (GET_PROTOCOL_MINOR(conn.version) >= 33) {
|
||||
res.daemonNixVersion = readString(conn.from);
|
||||
}
|
||||
|
||||
if (GET_PROTOCOL_MINOR(conn.version) >= 35) {
|
||||
res.remoteTrustsUs = WorkerProto::Serialise<std::optional< TrustedFlag>>::read(store, conn);
|
||||
} else {
|
||||
// We don't know the answer; protocol to old.
|
||||
res.remoteTrustsUs = std::nullopt;
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
void WorkerProto::Serialise<WorkerProto::ClientHandshakeInfo>::write(const StoreDirConfig & store, WriteConn conn, const WorkerProto::ClientHandshakeInfo & info)
|
||||
{
|
||||
if (GET_PROTOCOL_MINOR(conn.version) >= 33) {
|
||||
assert(info.daemonNixVersion);
|
||||
conn.to << *info.daemonNixVersion;
|
||||
}
|
||||
|
||||
if (GET_PROTOCOL_MINOR(conn.version) >= 35) {
|
||||
WorkerProto::write(store, conn, info.remoteTrustsUs);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ struct BuildResult;
|
|||
struct KeyedBuildResult;
|
||||
struct ValidPathInfo;
|
||||
struct UnkeyedValidPathInfo;
|
||||
enum BuildMode : uint8_t;
|
||||
enum TrustedFlag : bool;
|
||||
|
||||
|
||||
|
|
@ -76,6 +77,19 @@ struct WorkerProto
|
|||
Version version;
|
||||
};
|
||||
|
||||
/**
|
||||
* Stripped down serialization logic suitable for sharing with Hydra.
|
||||
*
|
||||
* @todo remove once Hydra uses Store abstraction consistently.
|
||||
*/
|
||||
struct BasicClientConnection;
|
||||
struct BasicServerConnection;
|
||||
|
||||
/**
|
||||
* Extra information provided as part of protocol negotation.
|
||||
*/
|
||||
struct ClientHandshakeInfo;
|
||||
|
||||
/**
|
||||
* Data type for canonical pairs of serialisers for the worker protocol.
|
||||
*
|
||||
|
|
@ -166,6 +180,33 @@ enum struct WorkerProto::Op : uint64_t
|
|||
AddPermRoot = 47,
|
||||
};
|
||||
|
||||
struct WorkerProto::ClientHandshakeInfo
|
||||
{
|
||||
/**
|
||||
* The version of the Nix daemon that is processing our requests
|
||||
.
|
||||
*
|
||||
* Do note, it may or may not communicating with another daemon,
|
||||
* rather than being an "end" `LocalStore` or similar.
|
||||
*/
|
||||
std::optional<std::string> daemonNixVersion;
|
||||
|
||||
/**
|
||||
* Whether the remote side trusts us or not.
|
||||
*
|
||||
* 3 values: "yes", "no", or `std::nullopt` for "unknown".
|
||||
*
|
||||
* Note that the "remote side" might not be just the end daemon, but
|
||||
* also an intermediary forwarder that can make its own trusting
|
||||
* decisions. This would be the intersection of all their trust
|
||||
* decisions, since it takes only one link in the chain to start
|
||||
* denying operations.
|
||||
*/
|
||||
std::optional<TrustedFlag> remoteTrustsUs;
|
||||
|
||||
bool operator == (const ClientHandshakeInfo &) const = default;
|
||||
};
|
||||
|
||||
/**
|
||||
* Convenience for sending operation codes.
|
||||
*
|
||||
|
|
@ -215,9 +256,13 @@ DECLARE_WORKER_SERIALISER(ValidPathInfo);
|
|||
template<>
|
||||
DECLARE_WORKER_SERIALISER(UnkeyedValidPathInfo);
|
||||
template<>
|
||||
DECLARE_WORKER_SERIALISER(BuildMode);
|
||||
template<>
|
||||
DECLARE_WORKER_SERIALISER(std::optional<TrustedFlag>);
|
||||
template<>
|
||||
DECLARE_WORKER_SERIALISER(std::optional<std::chrono::microseconds>);
|
||||
template<>
|
||||
DECLARE_WORKER_SERIALISER(WorkerProto::ClientHandshakeInfo);
|
||||
|
||||
template<typename T>
|
||||
DECLARE_WORKER_SERIALISER(std::vector<T>);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue