wroclaw-git-backups/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-07-06 05:01:48 +02:00
Code Activity

Merge remote-tracking branch 'origin/master' into flakes

Browse source
This commit is contained in:
Eelco Dolstra 2019-12-11 14:53:30 +01:00
commit ecb3a1afa2
119 changed files with 3905 additions and 2250 deletions
Download patch file Download diff file Expand all files Collapse all files

72
src/libstore/binary-cache-store.cc
View file

@ -91,19 +91,18 @@ std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
return sink.s;
}
Path BinaryCacheStore::narInfoFileFor(const Path & storePath)
std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath)
{
assertStorePath(storePath);
return storePathToHash(storePath) + ".narinfo";
return storePathToHash(printStorePath(storePath)) + ".narinfo";
}
void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
{
auto narInfoFile = narInfoFileFor(narInfo->path);
upsertFile(narInfoFile, narInfo->to_string(), "text/x-nix-narinfo");
upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo");
auto hashPart = storePathToHash(narInfo->path);
auto hashPart = storePathToHash(printStorePath(narInfo->path));
{
auto state_(state.lock());
@ -126,8 +125,8 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
if (ref != info.path)
queryPathInfo(ref);
} catch (InvalidPath &) {
throw Error(format("cannot add '%s' to the binary cache because the reference '%s' is not valid")
% info.path % ref);
throw Error("cannot add '%s' to the binary cache because the reference '%s' is not valid",
printStorePath(info.path), printStorePath(ref));
}
assert(nar->compare(0, narMagic.size(), narMagic) == 0);
@ -138,14 +137,14 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
narInfo->narHash = hashString(htSHA256, *nar);
if (info.narHash && info.narHash != narInfo->narHash)
throw Error(format("refusing to copy corrupted path '%1%' to binary cache") % info.path);
throw Error("refusing to copy corrupted path '%1%' to binary cache", printStorePath(info.path));
auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor);
auto narAccessor = makeNarAccessor(nar);
if (accessor_)
accessor_->addToCache(info.path, *nar, narAccessor);
accessor_->addToCache(printStorePath(info.path), *nar, narAccessor);
/* Optionally write a JSON file containing a listing of the
contents of the NAR. */
@ -162,7 +161,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
}
}
upsertFile(storePathToHash(info.path) + ".ls", jsonOut.str(), "application/json");
upsertFile(storePathToHash(printStorePath(info.path)) + ".ls", jsonOut.str(), "application/json");
}
/* Compress the NAR. */
@ -174,10 +173,10 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
narInfo->fileSize = narCompressed->size();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
printMsg(lvlTalkative, format("copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache")
% narInfo->path % narInfo->narSize
% ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0)
% duration);
printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
printStorePath(narInfo->path), narInfo->narSize,
((1.0 - (double) narCompressed->size() / nar->size()) * 100.0),
duration);
narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar"
+ (compression == "xz" ? ".xz" :
@ -254,14 +253,14 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
stats.narWriteCompressionTimeMs += duration;
/* Atomically write the NAR info file.*/
if (secretKey) narInfo->sign(*secretKey);
if (secretKey) narInfo->sign(*this, *secretKey);
writeNarInfo(narInfo);
stats.narInfoWrite++;
}
bool BinaryCacheStore::isValidPathUncached(const Path & storePath)
bool BinaryCacheStore::isValidPathUncached(const StorePath & storePath)
{
// FIXME: this only checks whether a .narinfo with a matching hash
// part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even
@ -269,7 +268,7 @@ bool BinaryCacheStore::isValidPathUncached(const Path & storePath)
return fileExists(narInfoFileFor(storePath));
}
void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink)
void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
{
auto info = queryPathInfo(storePath).cast<const NarInfo>();
@ -295,12 +294,13 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink)
stats.narReadBytes += narSize;
}
void BinaryCacheStore::queryPathInfoUncached(const Path & storePath,
void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
{
auto uri = getUri();
auto storePathS = printStorePath(storePath);
auto act = std::make_shared<Activity>(*logger, lvlTalkative, actQueryPathInfo,
fmt("querying info about '%s' on '%s'", storePath, uri), Logger::Fields{storePath, uri});
fmt("querying info about '%s' on '%s'", storePathS, uri), Logger::Fields{storePathS, uri});
PushActivity pact(act->id);
auto narInfoFile = narInfoFileFor(storePath);
@ -326,7 +326,7 @@ void BinaryCacheStore::queryPathInfoUncached(const Path & storePath,
}});
}
Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
{
// FIXME: some cut&paste from LocalStore::addToStore().
@ -345,20 +345,18 @@ Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
h = hashString(hashAlgo, s);
}
ValidPathInfo info;
info.path = makeFixedOutputPath(recursive, h, name);
ValidPathInfo info(makeFixedOutputPath(recursive, h, name));
addToStore(info, sink.s, repair, CheckSigs, nullptr);
return info.path;
return std::move(info.path);
}
Path BinaryCacheStore::addTextToStore(const string & name, const string & s,
const PathSet & references, RepairFlag repair)
StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair)
{
ValidPathInfo info;
info.path = computeStorePathForText(name, s, references);
info.references = references;
ValidPathInfo info(computeStorePathForText(name, s, references));
info.references = cloneStorePathSet(references);
if (repair || !isValidPath(info.path)) {
StringSink sink;
@ -366,7 +364,7 @@ Path BinaryCacheStore::addTextToStore(const string & name, const string & s,
addToStore(info, sink.s, repair, CheckSigs, nullptr);
}
return info.path;
return std::move(info.path);
}
ref<FSAccessor> BinaryCacheStore::getFSAccessor()
@ -374,7 +372,7 @@ ref<FSAccessor> BinaryCacheStore::getFSAccessor()
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), localNarCache);
}
void BinaryCacheStore::addSignatures(const Path & storePath, const StringSet & sigs)
void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSet & sigs)
{
/* Note: this is inherently racy since there is no locking on
binary caches. In particular, with S3 this unreliable, even
@ -390,24 +388,22 @@ void BinaryCacheStore::addSignatures(const Path & storePath, const StringSet & s
writeNarInfo(narInfo);
}
std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const Path & path)
std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & path)
{
Path drvPath;
auto drvPath = path.clone();
if (isDerivation(path))
drvPath = path;
else {
if (!path.isDerivation()) {
try {
auto info = queryPathInfo(path);
// FIXME: add a "Log" field to .narinfo
if (info->deriver == "") return nullptr;
drvPath = info->deriver;
if (!info->deriver) return nullptr;
drvPath = info->deriver->clone();
} catch (InvalidPath &) {
return nullptr;
}
}
auto logPath = "log/" + baseNameOf(drvPath);
auto logPath = "log/" + std::string(baseNameOf(printStorePath(drvPath)));
debug("fetching build log from binary cache '%s/%s'", getUri(), logPath);

24
src/libstore/binary-cache-store.hh
View file

@ -65,18 +65,18 @@ private:
std::string narMagic;
std::string narInfoFileFor(const Path & storePath);
std::string narInfoFileFor(const StorePath & storePath);
void writeNarInfo(ref<NarInfo> narInfo);
public:
bool isValidPathUncached(const Path & path) override;
bool isValidPathUncached(const StorePath & path) override;
void queryPathInfoUncached(const Path & path,
void queryPathInfoUncached(const StorePath & path,
Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override;
Path queryPathFromHashPart(const string & hashPart) override
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
{ unsupported("queryPathFromHashPart"); }
bool wantMassQuery() override { return wantMassQuery_; }
@ -85,27 +85,27 @@ public:
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override;
Path addToStore(const string & name, const Path & srcPath,
StorePath addToStore(const string & name, const Path & srcPath,
bool recursive, HashType hashAlgo,
PathFilter & filter, RepairFlag repair) override;
Path addTextToStore(const string & name, const string & s,
const PathSet & references, RepairFlag repair) override;
StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override;
void narFromPath(const Path & path, Sink & sink) override;
void narFromPath(const StorePath & path, Sink & sink) override;
BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode) override
{ unsupported("buildDerivation"); }
void ensurePath(const Path & path) override
void ensurePath(const StorePath & path) override
{ unsupported("ensurePath"); }
ref<FSAccessor> getFSAccessor() override;
void addSignatures(const Path & storePath, const StringSet & sigs) override;
void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
std::shared_ptr<std::string> getBuildLog(const Path & path) override;
std::shared_ptr<std::string> getBuildLog(const StorePath & path) override;
int getPriority() override { return priority; }

798
src/libstore/build.cc
View file

File diff suppressed because it is too large Load diff

2
src/libstore/builtins/buildenv.hh
View file

@ -9,7 +9,7 @@ struct Package {
Path path;
bool active;
int priority;
Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {}
Package(const Path & path, bool active, int priority) : path{path}, active{active}, priority{priority} {}
};
typedef std::vector<Package> Packages;

2
src/libstore/builtins/fetchurl.cc
View file

@ -24,7 +24,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
Path storePath = getAttr("out");
auto mainUrl = getAttr("url");
bool unpack = get(drv.env, "unpack", "") == "1";
bool unpack = get(drv.env, "unpack").value_or("") == "1";
/* Note: have to use a fresh downloader here because we're in
a forked process. */

147
src/libstore/daemon.cc
View file

@ -260,14 +260,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
switch (op) {
case wopIsValidPath: {
/* 'readStorePath' could raise an error leading to the connection
being closed. To be able to recover from an invalid path error,
call 'startWork' early, and do 'assertStorePath' afterwards so
that the 'Error' exception handler doesn't close the
connection. */
Path path = readString(from);
auto path = store->parseStorePath(readString(from));
logger->startWork();
store->assertStorePath(path);
bool result = store->isValidPath(path);
logger->stopWork();
to << result;
@ -275,34 +269,36 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case wopQueryValidPaths: {
PathSet paths = readStorePaths<PathSet>(*store, from);
auto paths = readStorePaths<StorePathSet>(*store, from);
logger->startWork();
PathSet res = store->queryValidPaths(paths);
auto res = store->queryValidPaths(paths);
logger->stopWork();
to << res;
writeStorePaths(*store, to, res);
break;
}
case wopHasSubstitutes: {
Path path = readStorePath(*store, from);
auto path = store->parseStorePath(readString(from));
logger->startWork();
PathSet res = store->querySubstitutablePaths({path});
StorePathSet paths; // FIXME
paths.insert(path.clone());
auto res = store->querySubstitutablePaths(paths);
logger->stopWork();
to << (res.find(path) != res.end());
to << (res.count(path) != 0);
break;
}
case wopQuerySubstitutablePaths: {
PathSet paths = readStorePaths<PathSet>(*store, from);
auto paths = readStorePaths<StorePathSet>(*store, from);
logger->startWork();
PathSet res = store->querySubstitutablePaths(paths);
auto res = store->querySubstitutablePaths(paths);
logger->stopWork();
to << res;
writeStorePaths(*store, to, res);
break;
}
case wopQueryPathHash: {
Path path = readStorePath(*store, from);
auto path = store->parseStorePath(readString(from));
logger->startWork();
auto hash = store->queryPathInfo(path)->narHash;
logger->stopWork();
@ -314,23 +310,24 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopQueryReferrers:
case wopQueryValidDerivers:
case wopQueryDerivationOutputs: {
Path path = readStorePath(*store, from);
auto path = store->parseStorePath(readString(from));
logger->startWork();
PathSet paths;
StorePathSet paths;
if (op == wopQueryReferences)
paths = store->queryPathInfo(path)->references;
for (auto & i : store->queryPathInfo(path)->references)
paths.insert(i.clone());
else if (op == wopQueryReferrers)
store->queryReferrers(path, paths);
else if (op == wopQueryValidDerivers)
paths = store->queryValidDerivers(path);
else paths = store->queryDerivationOutputs(path);
logger->stopWork();
to << paths;
writeStorePaths(*store, to, paths);
break;
}
case wopQueryDerivationOutputNames: {
Path path = readStorePath(*store, from);
auto path = store->parseStorePath(readString(from));
logger->startWork();
StringSet names;
names = store->queryDerivationOutputNames(path);
@ -340,20 +337,20 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case wopQueryDeriver: {
Path path = readStorePath(*store, from);
auto path = store->parseStorePath(readString(from));
logger->startWork();
auto deriver = store->queryPathInfo(path)->deriver;
auto info = store->queryPathInfo(path);
logger->stopWork();
to << deriver;
to << (info->deriver ? store->printStorePath(*info->deriver) : "");
break;
}
case wopQueryPathFromHashPart: {
string hashPart = readString(from);
auto hashPart = readString(from);
logger->startWork();
Path path = store->queryPathFromHashPart(hashPart);
auto path = store->queryPathFromHashPart(hashPart);
logger->stopWork();
to << path;
to << (path ? store->printStorePath(*path) : "");
break;
}
@ -383,26 +380,26 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork();
if (!savedRegular.regular) throw Error("regular file expected");
Path path = store->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo);
auto path = store->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo);
logger->stopWork();
to << path;
to << store->printStorePath(path);
break;
}
case wopAddTextToStore: {
string suffix = readString(from);
string s = readString(from);
PathSet refs = readStorePaths<PathSet>(*store, from);
auto refs = readStorePaths<StorePathSet>(*store, from);
logger->startWork();
Path path = store->addTextToStore(suffix, s, refs, NoRepair);
auto path = store->addTextToStore(suffix, s, refs, NoRepair);
logger->stopWork();
to << path;
to << store->printStorePath(path);
break;
}
case wopExportPath: {
Path path = readStorePath(*store, from);
auto path = store->parseStorePath(readString(from));
readInt(from); // obsolete
logger->startWork();
TunnelSink sink(to);
@ -415,15 +412,19 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopImportPaths: {
logger->startWork();
TunnelSource source(from, to);
Paths paths = store->importPaths(source, nullptr,
auto paths = store->importPaths(source, nullptr,
trusted ? NoCheckSigs : CheckSigs);
logger->stopWork();
to << paths;
Strings paths2;
for (auto & i : paths) paths2.push_back(store->printStorePath(i));
to << paths2;
break;
}
case wopBuildPaths: {
PathSet drvs = readStorePaths<PathSet>(*store, from);
std::vector<StorePathWithOutputs> drvs;
for (auto & s : readStrings<Strings>(from))
drvs.push_back(store->parseDrvPathWithOutputs(s));
BuildMode mode = bmNormal;
if (GET_PROTOCOL_MINOR(clientVersion) >= 15) {
mode = (BuildMode) readInt(from);
@ -441,7 +442,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case wopBuildDerivation: {
Path drvPath = readStorePath(*store, from);
auto drvPath = store->parseStorePath(readString(from));
BasicDerivation drv;
readDerivation(from, *store, drv);
BuildMode buildMode = (BuildMode) readInt(from);
@ -455,7 +456,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case wopEnsurePath: {
Path path = readStorePath(*store, from);
auto path = store->parseStorePath(readString(from));
logger->startWork();
store->ensurePath(path);
logger->stopWork();
@ -464,7 +465,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case wopAddTempRoot: {
Path path = readStorePath(*store, from);
auto path = store->parseStorePath(readString(from));
logger->startWork();
store->addTempRoot(path);
logger->stopWork();
@ -502,7 +503,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
for (auto & [target, links] : roots)
for (auto & link : links)
to << link << target;
to << link << store->printStorePath(target);
break;
}
@ -510,7 +511,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopCollectGarbage: {
GCOptions options;
options.action = (GCOptions::GCAction) readInt(from);
options.pathsToDelete = readStorePaths<PathSet>(*store, from);
options.pathsToDelete = readStorePaths<StorePathSet>(*store, from);
from >> options.ignoreLiveness >> options.maxFreed;
// obsolete fields
readInt(from);
@ -568,44 +569,52 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case wopQuerySubstitutablePathInfo: {
Path path = absPath(readString(from));
auto path = store->parseStorePath(readString(from));
logger->startWork();
SubstitutablePathInfos infos;
store->querySubstitutablePathInfos({path}, infos);
StorePathSet paths;
paths.insert(path.clone()); // FIXME
store->querySubstitutablePathInfos(paths, infos);
logger->stopWork();
SubstitutablePathInfos::iterator i = infos.find(path);
auto i = infos.find(path);
if (i == infos.end())
to << 0;
else {
to << 1 << i->second.deriver << i->second.references << i->second.downloadSize << i->second.narSize;
to << 1
<< (i->second.deriver ? store->printStorePath(*i->second.deriver) : "");
writeStorePaths(*store, to, i->second.references);
to << i->second.downloadSize
<< i->second.narSize;
}
break;
}
case wopQuerySubstitutablePathInfos: {
PathSet paths = readStorePaths<PathSet>(*store, from);
auto paths = readStorePaths<StorePathSet>(*store, from);
logger->startWork();
SubstitutablePathInfos infos;
store->querySubstitutablePathInfos(paths, infos);
logger->stopWork();
to << infos.size();
for (auto & i : infos) {
to << i.first << i.second.deriver << i.second.references
<< i.second.downloadSize << i.second.narSize;
to << store->printStorePath(i.first)
<< (i.second.deriver ? store->printStorePath(*i.second.deriver) : "");
writeStorePaths(*store, to, i.second.references);
to << i.second.downloadSize << i.second.narSize;
}
break;
}
case wopQueryAllValidPaths: {
logger->startWork();
PathSet paths = store->queryAllValidPaths();
auto paths = store->queryAllValidPaths();
logger->stopWork();
to << paths;
writeStorePaths(*store, to, paths);
break;
}
case wopQueryPathInfo: {
Path path = readStorePath(*store, from);
auto path = store->parseStorePath(readString(from));
std::shared_ptr<const ValidPathInfo> info;
logger->startWork();
try {
@ -617,8 +626,10 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (info) {
if (GET_PROTOCOL_MINOR(clientVersion) >= 17)
to << 1;
to << info->deriver << info->narHash.to_string(Base16, false) << info->references
<< info->registrationTime << info->narSize;
to << (info->deriver ? store->printStorePath(*info->deriver) : "")
<< info->narHash.to_string(Base16, false);
writeStorePaths(*store, to, info->references);
to << info->registrationTime << info->narSize;
if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
to << info->ultimate
<< info->sigs
@ -651,7 +662,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case wopAddSignatures: {
Path path = readStorePath(*store, from);
auto path = store->parseStorePath(readString(from));
StringSet sigs = readStrings<StringSet>(from);
logger->startWork();
if (!trusted)
@ -663,22 +674,21 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case wopNarFromPath: {
auto path = readStorePath(*store, from);
auto path = store->parseStorePath(readString(from));
logger->startWork();
logger->stopWork();
dumpPath(path, to);
dumpPath(store->printStorePath(path), to);
break;
}
case wopAddToStoreNar: {
bool repair, dontCheckSigs;
ValidPathInfo info;
info.path = readStorePath(*store, from);
from >> info.deriver;
if (!info.deriver.empty())
store->assertStorePath(info.deriver);
ValidPathInfo info(store->parseStorePath(readString(from)));
auto deriver = readString(from);
if (deriver != "")
info.deriver = store->parseStorePath(deriver);
info.narHash = Hash(readString(from), htSHA256);
info.references = readStorePaths<PathSet>(*store, from);
info.references = readStorePaths<StorePathSet>(*store, from);
from >> info.registrationTime >> info.narSize >> info.ultimate;
info.sigs = readStrings<StringSet>(from);
from >> info.ca >> repair >> dontCheckSigs;
@ -709,13 +719,18 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case wopQueryMissing: {
PathSet targets = readStorePaths<PathSet>(*store, from);
std::vector<StorePathWithOutputs> targets;
for (auto & s : readStrings<Strings>(from))
targets.push_back(store->parseDrvPathWithOutputs(s));
logger->startWork();
PathSet willBuild, willSubstitute, unknown;
StorePathSet willBuild, willSubstitute, unknown;
unsigned long long downloadSize, narSize;
store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize);
logger->stopWork();
to << willBuild << willSubstitute << unknown << downloadSize << narSize;
writeStorePaths(*store, to, willBuild);
writeStorePaths(*store, to, willSubstitute);
writeStorePaths(*store, to, unknown);
to << downloadSize << narSize;
break;
}

182
src/libstore/derivations.cc
View file

@ -21,17 +21,39 @@ void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const
HashType hashType = parseHashType(algo);
if (hashType == htUnknown)
throw Error(format("unknown hash algorithm '%1%'") % algo);
throw Error("unknown hash algorithm '%s'", algo);
hash = Hash(this->hash, hashType);
}
Path BasicDerivation::findOutput(const string & id) const
BasicDerivation::BasicDerivation(const BasicDerivation & other)
: platform(other.platform)
, builder(other.builder)
, args(other.args)
, env(other.env)
{
for (auto & i : other.outputs)
outputs.insert_or_assign(i.first,
DerivationOutput(i.second.path.clone(), std::string(i.second.hashAlgo), std::string(i.second.hash)));
for (auto & i : other.inputSrcs)
inputSrcs.insert(i.clone());
}
Derivation::Derivation(const Derivation & other)
: BasicDerivation(other)
{
for (auto & i : other.inputDrvs)
inputDrvs.insert_or_assign(i.first.clone(), i.second);
}
const StorePath & BasicDerivation::findOutput(const string & id) const
{
auto i = outputs.find(id);
if (i == outputs.end())
throw Error(format("derivation has no output '%1%'") % id);
throw Error("derivation has no output '%s'", id);
return i->second.path;
}
@ -42,18 +64,17 @@ bool BasicDerivation::isBuiltin() const
}
Path writeDerivation(ref<Store> store,
StorePath writeDerivation(ref<Store> store,
const Derivation & drv, const string & name, RepairFlag repair)
{
PathSet references;
references.insert(drv.inputSrcs.begin(), drv.inputSrcs.end());
auto references = cloneStorePathSet(drv.inputSrcs);
for (auto & i : drv.inputDrvs)
references.insert(i.first);
references.insert(i.first.clone());
/* Note that the outputs of a derivation are *not* references
(that can be missing (of course) and should not necessarily be
held during a garbage collection). */
string suffix = name + drvExtension;
string contents = drv.unparse();
string contents = drv.unparse(*store, false);
return settings.readOnlyMode
? store->computeStorePathForText(suffix, contents, references)
: store->addTextToStore(suffix, contents, references, repair);
@ -121,7 +142,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
}
static Derivation parseDerivation(const string & s)
static Derivation parseDerivation(const Store & store, const string & s)
{
Derivation drv;
istringstream_nocopy str(s);
@ -129,13 +150,12 @@ static Derivation parseDerivation(const string & s)
/* Parse the list of outputs. */
while (!endOfList(str)) {
DerivationOutput out;
expect(str, "("); string id = parseString(str);
expect(str, ","); out.path = parsePath(str);
expect(str, ","); out.hashAlgo = parseString(str);
expect(str, ","); out.hash = parseString(str);
expect(str, "("); std::string id = parseString(str);
expect(str, ","); auto path = store.parseStorePath(parsePath(str));
expect(str, ","); auto hashAlgo = parseString(str);
expect(str, ","); auto hash = parseString(str);
expect(str, ")");
drv.outputs[id] = out;
drv.outputs.emplace(id, DerivationOutput(std::move(path), std::move(hashAlgo), std::move(hash)));
}
/* Parse the list of input derivations. */
@ -144,11 +164,11 @@ static Derivation parseDerivation(const string & s)
expect(str, "(");
Path drvPath = parsePath(str);
expect(str, ",[");
drv.inputDrvs[drvPath] = parseStrings(str, false);
drv.inputDrvs.insert_or_assign(store.parseStorePath(drvPath), parseStrings(str, false));
expect(str, ")");
}
expect(str, ",["); drv.inputSrcs = parseStrings(str, true);
expect(str, ",["); drv.inputSrcs = store.parseStorePathSet(parseStrings(str, true));
expect(str, ","); drv.platform = parseString(str);
expect(str, ","); drv.builder = parseString(str);
@ -171,25 +191,24 @@ static Derivation parseDerivation(const string & s)
}
Derivation readDerivation(const Path & drvPath)
Derivation readDerivation(const Store & store, const Path & drvPath)
{
try {
return parseDerivation(readFile(drvPath));
return parseDerivation(store, readFile(drvPath));
} catch (FormatError & e) {
throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg());
}
}
Derivation Store::derivationFromPath(const Path & drvPath)
Derivation Store::derivationFromPath(const StorePath & drvPath)
{
assertStorePath(drvPath);
ensurePath(drvPath);
auto accessor = getFSAccessor();
try {
return parseDerivation(accessor->readFile(drvPath));
return parseDerivation(*this, accessor->readFile(printStorePath(drvPath)));
} catch (FormatError & e) {
throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg());
throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg());
}
}
@ -220,33 +239,56 @@ static void printStrings(string & res, ForwardIterator i, ForwardIterator j)
}
string Derivation::unparse() const
string Derivation::unparse(const Store & store, bool maskOutputs,
std::map<std::string, StringSet> * actualInputs) const
{
string s;
s.reserve(65536);
s += "Derive([";
bool first = true;
for (auto & i : outputs) {
if (first) first = false; else s += ',';
s += '('; printString(s, i.first);
s += ','; printString(s, i.second.path);
s += ','; printString(s, i.second.hashAlgo);
s += ','; printString(s, i.second.hash);
s += ')';
StringSet maskedOutputs;
if (maskOutputs) {
bool first = true;
maskedOutputs = tokenizeString<StringSet>(get(env, "outputs").value_or("out"), " ");
for (auto & i : maskedOutputs) {
if (first) first = false; else s += ',';
s += '('; printString(s, i);
s += ",\"\",\"\",\"\")";
}
} else {
bool first = true;
for (auto & i : outputs) {
if (first) first = false; else s += ',';
s += '('; printString(s, i.first);
s += ','; printString(s, store.printStorePath(i.second.path));
s += ','; printString(s, i.second.hashAlgo);
s += ','; printString(s, i.second.hash);
s += ')';
}
}
s += "],[";
first = true;
for (auto & i : inputDrvs) {
if (first) first = false; else s += ',';
s += '('; printString(s, i.first);
s += ','; printStrings(s, i.second.begin(), i.second.end());
s += ')';
bool first = true;
if (actualInputs) {
for (auto & i : *actualInputs) {
if (first) first = false; else s += ',';
s += '('; printString(s, i.first);
s += ','; printStrings(s, i.second.begin(), i.second.end());
s += ')';
}
} else {
for (auto & i : inputDrvs) {
if (first) first = false; else s += ',';
s += '('; printString(s, store.printStorePath(i.first));
s += ','; printStrings(s, i.second.begin(), i.second.end());
s += ')';
}
}
s += "],";
printStrings(s, inputSrcs.begin(), inputSrcs.end());
auto paths = store.printStorePathSet(inputSrcs); // FIXME: slow
printStrings(s, paths.begin(), paths.end());
s += ','; printString(s, platform);
s += ','; printString(s, builder);
@ -257,7 +299,7 @@ string Derivation::unparse() const
for (auto & i : env) {
if (first) first = false; else s += ',';
s += '('; printString(s, i.first);
s += ','; printString(s, i.second);
s += ','; printString(s, maskOutputs && maskedOutputs.count(i.first) ? "" : i.second);
s += ')';
}
@ -267,6 +309,7 @@ string Derivation::unparse() const
}
// FIXME: remove
bool isDerivation(const string & fileName)
{
return hasSuffix(fileName, drvExtension);
@ -304,7 +347,7 @@ DrvHashes drvHashes;
paths have been replaced by the result of a recursive call to this
function, and that for fixed-output derivations we return a hash of
its output path. */
Hash hashDerivationModulo(Store & store, Derivation drv)
Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
{
/* Return a fixed hash for fixed-output derivations. */
if (drv.isFixedOutput()) {
@ -312,42 +355,41 @@ Hash hashDerivationModulo(Store & store, Derivation drv)
return hashString(htSHA256, "fixed:out:"
+ i->second.hashAlgo + ":"
+ i->second.hash + ":"
+ i->second.path);
+ store.printStorePath(i->second.path));
}
/* For other derivations, replace the inputs paths with recursive
calls to this function.*/
DerivationInputs inputs2;
std::map<std::string, StringSet> inputs2;
for (auto & i : drv.inputDrvs) {
Hash h = drvHashes[i.first];
if (!h) {
auto h = drvHashes.find(i.first);
if (h == drvHashes.end()) {
assert(store.isValidPath(i.first));
Derivation drv2 = readDerivation(store.toRealPath(i.first));
h = hashDerivationModulo(store, drv2);
drvHashes[i.first] = h;
h = drvHashes.insert_or_assign(i.first.clone(), hashDerivationModulo(store,
readDerivation(store, store.toRealPath(store.printStorePath(i.first))), false)).first;
}
inputs2[h.to_string(Base16, false)] = i.second;
inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second);
}
drv.inputDrvs = inputs2;
return hashString(htSHA256, drv.unparse());
return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
}
DrvPathWithOutputs parseDrvPathWithOutputs(const string & s)
StorePathWithOutputs Store::parseDrvPathWithOutputs(const std::string & s)
{
size_t n = s.find("!");
return n == s.npos
? DrvPathWithOutputs(s, std::set<string>())
: DrvPathWithOutputs(string(s, 0, n), tokenizeString<std::set<string> >(string(s, n + 1), ","));
? StorePathWithOutputs{parseStorePath(s), std::set<string>()}
: StorePathWithOutputs{parseStorePath(std::string_view(s.data(), n)),
tokenizeString<std::set<string>>(string(s, n + 1), ",")};
}
Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs)
std::string StorePathWithOutputs::to_string(const Store & store) const
{
return outputs.empty()
? drvPath
: drvPath + "!" + concatStringsSep(",", outputs);
? store.printStorePath(path)
: store.printStorePath(path) + "!" + concatStringsSep(",", outputs);
}
@ -357,28 +399,28 @@ bool wantOutput(const string & output, const std::set<string> & wanted)
}
PathSet BasicDerivation::outputPaths() const
StorePathSet BasicDerivation::outputPaths() const
{
PathSet paths;
StorePathSet paths;
for (auto & i : outputs)
paths.insert(i.second.path);
paths.insert(i.second.path.clone());
return paths;
}
Source & readDerivation(Source & in, Store & store, BasicDerivation & drv)
Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
{
drv.outputs.clear();
auto nr = readNum<size_t>(in);
for (size_t n = 0; n < nr; n++) {
auto name = readString(in);
DerivationOutput o;
in >> o.path >> o.hashAlgo >> o.hash;
store.assertStorePath(o.path);
drv.outputs[name] = o;
auto path = store.parseStorePath(readString(in));
auto hashAlgo = readString(in);
auto hash = readString(in);
drv.outputs.emplace(name, DerivationOutput(std::move(path), std::move(hashAlgo), std::move(hash)));
}
drv.inputSrcs = readStorePaths<PathSet>(store, in);
drv.inputSrcs = readStorePaths<StorePathSet>(store, in);
in >> drv.platform >> drv.builder;
drv.args = readStrings<Strings>(in);
@ -393,16 +435,16 @@ Source & readDerivation(Source & in, Store & store, BasicDerivation & drv)
}
Sink & operator << (Sink & out, const BasicDerivation & drv)
void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv)
{
out << drv.outputs.size();
for (auto & i : drv.outputs)
out << i.first << i.second.path << i.second.hashAlgo << i.second.hash;
out << drv.inputSrcs << drv.platform << drv.builder << drv.args;
out << i.first << store.printStorePath(i.second.path) << i.second.hashAlgo << i.second.hash;
writeStorePaths(store, out, drv.inputSrcs);
out << drv.platform << drv.builder << drv.args;
out << drv.env.size();
for (auto & i : drv.env)
out << i.first << i.second;
return out;
}

64
src/libstore/derivations.hh
View file

@ -10,26 +10,18 @@
namespace nix {
/* Extension of derivations in the Nix store. */
const string drvExtension = ".drv";
/* Abstract syntax of derivations. */
struct DerivationOutput
{
Path path;
string hashAlgo; /* hash used for expected hash computation */
string hash; /* expected hash, may be null */
DerivationOutput()
{
}
DerivationOutput(Path path, string hashAlgo, string hash)
{
this->path = path;
this->hashAlgo = hashAlgo;
this->hash = hash;
}
StorePath path;
std::string hashAlgo; /* hash used for expected hash computation */
std::string hash; /* expected hash, may be null */
DerivationOutput(StorePath && path, std::string && hashAlgo, std::string && hash)
: path(std::move(path))
, hashAlgo(std::move(hashAlgo))
, hash(std::move(hash))
{ }
void parseHashInfo(bool & recursive, Hash & hash) const;
};
@ -37,24 +29,26 @@ typedef std::map<string, DerivationOutput> DerivationOutputs;
/* For inputs that are sub-derivations, we specify exactly which
output IDs we are interested in. */
typedef std::map<Path, StringSet> DerivationInputs;
typedef std::map<StorePath, StringSet> DerivationInputs;
typedef std::map<string, string> StringPairs;
struct BasicDerivation
{
DerivationOutputs outputs; /* keyed on symbolic IDs */
PathSet inputSrcs; /* inputs that are sources */
StorePathSet inputSrcs; /* inputs that are sources */
string platform;
Path builder;
Strings args;
StringPairs env;
BasicDerivation() { }
explicit BasicDerivation(const BasicDerivation & other);
virtual ~BasicDerivation() { };
/* Return the path corresponding to the output identifier `id' in
the given derivation. */
Path findOutput(const string & id) const;
const StorePath & findOutput(const std::string & id) const;
bool isBuiltin() const;
@ -62,7 +56,7 @@ struct BasicDerivation
bool isFixedOutput() const;
/* Return the output paths of a derivation. */
PathSet outputPaths() const;
StorePathSet outputPaths() const;
};
@ -71,7 +65,12 @@ struct Derivation : BasicDerivation
DerivationInputs inputDrvs; /* inputs that are sub-derivations */
/* Print a derivation. */
std::string unparse() const;
std::string unparse(const Store & store, bool maskOutputs,
std::map<std::string, StringSet> * actualInputs = nullptr) const;
Derivation() { }
Derivation(Derivation && other) = default;
explicit Derivation(const Derivation & other);
};
@ -79,38 +78,29 @@ class Store;
/* Write a derivation to the Nix store, and return its path. */
Path writeDerivation(ref<Store> store,
StorePath writeDerivation(ref<Store> store,
const Derivation & drv, const string & name, RepairFlag repair = NoRepair);
/* Read a derivation from a file. */
Derivation readDerivation(const Path & drvPath);
Derivation readDerivation(const Store & store, const Path & drvPath);
/* Check whether a file name ends with the extension for
derivations. */
// FIXME: remove
bool isDerivation(const string & fileName);
Hash hashDerivationModulo(Store & store, Derivation drv);
Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs);
/* Memoisation of hashDerivationModulo(). */
typedef std::map<Path, Hash> DrvHashes;
typedef std::map<StorePath, Hash> DrvHashes;
extern DrvHashes drvHashes; // FIXME: global, not thread-safe
/* Split a string specifying a derivation and a set of outputs
(/nix/store/hash-foo!out1,out2,...) into the derivation path and
the outputs. */
typedef std::pair<string, std::set<string> > DrvPathWithOutputs;
DrvPathWithOutputs parseDrvPathWithOutputs(const string & s);
Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs);
bool wantOutput(const string & output, const std::set<string> & wanted);
struct Source;
struct Sink;
Source & readDerivation(Source & in, Store & store, BasicDerivation & drv);
Sink & operator << (Sink & out, const BasicDerivation & drv);
Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv);
void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv);
std::string hashPlaceholder(const std::string & outputName);

68
src/libstore/download.cc
View file

@ -650,10 +650,10 @@ struct CurlDownloader : public Downloader
#ifdef ENABLE_S3
auto [bucketName, key, params] = parseS3Uri(request.uri);
std::string profile = get(params, "profile", "");
std::string region = get(params, "region", Aws::Region::US_EAST_1);
std::string scheme = get(params, "scheme", "");
std::string endpoint = get(params, "endpoint", "");
std::string profile = get(params, "profile").value_or("");
std::string region = get(params, "region").value_or(Aws::Region::US_EAST_1);
std::string scheme = get(params, "scheme").value_or("");
std::string endpoint = get(params, "endpoint").value_or("");
S3Helper s3Helper(profile, region, scheme, endpoint);
@ -811,13 +811,13 @@ CachedDownloadResult Downloader::downloadCached(
if (p != string::npos) name = string(url, p + 1);
}
Path expectedStorePath;
std::optional<StorePath> expectedStorePath;
if (request.expectedHash) {
expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name);
if (store->isValidPath(expectedStorePath)) {
if (store->isValidPath(*expectedStorePath)) {
CachedDownloadResult result;
result.storePath = expectedStorePath;
result.path = store->toRealPath(expectedStorePath);
result.storePath = store->printStorePath(*expectedStorePath);
result.path = store->toRealPath(result.storePath);
assert(!request.getLastModified); // FIXME
return result;
}
@ -833,7 +833,7 @@ CachedDownloadResult Downloader::downloadCached(
PathLocks lock({fileLink}, fmt("waiting for lock on '%1%'...", fileLink));
Path storePath;
std::optional<StorePath> storePath;
string expectedETag;
@ -842,9 +842,10 @@ CachedDownloadResult Downloader::downloadCached(
CachedDownloadResult result;
if (pathExists(fileLink) && pathExists(dataFile)) {
storePath = readLink(fileLink);
store->addTempRoot(storePath);
if (store->isValidPath(storePath)) {
storePath = store->parseStorePath(readLink(fileLink));
// FIXME
store->addTempRoot(*storePath);
if (store->isValidPath(*storePath)) {
auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n");
if (ss.size() >= 3 && ss[0] == url) {
time_t lastChecked;
@ -858,7 +859,7 @@ CachedDownloadResult Downloader::downloadCached(
}
}
} else
storePath = "";
storePath.reset();
}
if (!skip) {
@ -871,46 +872,45 @@ CachedDownloadResult Downloader::downloadCached(
result.etag = res.etag;
if (!res.cached) {
ValidPathInfo info;
StringSink sink;
dumpString(*res.data, sink);
Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data);
info.path = store->makeFixedOutputPath(false, hash, name);
ValidPathInfo info(store->makeFixedOutputPath(false, hash, name));
info.narHash = hashString(htSHA256, *sink.s);
info.narSize = sink.s->size();
info.ca = makeFixedOutputCA(false, hash);
store->addToStore(info, sink.s, NoRepair, NoCheckSigs);
storePath = info.path;
storePath = info.path.clone();
}
assert(!storePath.empty());
replaceSymlink(storePath, fileLink);
assert(storePath);
replaceSymlink(store->printStorePath(*storePath), fileLink);
writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n");
} catch (DownloadError & e) {
if (storePath.empty()) throw;
if (!storePath) throw;
warn("warning: %s; using cached result", e.msg());
result.etag = expectedETag;
}
}
if (request.unpack) {
Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked";
Path unpackedLink = cacheDir + "/" + ((std::string) storePath->to_string()) + "-unpacked";
PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink));
Path unpackedStorePath;
std::optional<StorePath> unpackedStorePath;
if (pathExists(unpackedLink)) {
unpackedStorePath = readLink(unpackedLink);
store->addTempRoot(unpackedStorePath);
if (!store->isValidPath(unpackedStorePath))
unpackedStorePath = "";
unpackedStorePath = store->parseStorePath(readLink(unpackedLink));
store->addTempRoot(*unpackedStorePath);
if (!store->isValidPath(*unpackedStorePath))
unpackedStorePath.reset();
else
result.lastModified = lstat(unpackedLink).st_mtime;
}
if (unpackedStorePath.empty()) {
if (!unpackedStorePath) {
printInfo("unpacking '%s'...", url);
Path tmpDir = createTempDir();
AutoDelete autoDelete(tmpDir, true);
unpackTarfile(store->toRealPath(storePath), tmpDir, baseNameOf(url));
unpackTarfile(store->toRealPath(store->printStorePath(*storePath)), tmpDir, std::string(baseNameOf(url)));
auto members = readDirectory(tmpDir);
if (members.size() != 1)
throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
@ -921,15 +921,15 @@ CachedDownloadResult Downloader::downloadCached(
// Store the last-modified date of the tarball in the symlink
// mtime. This saves us from having to store it somewhere
// else.
replaceSymlink(unpackedStorePath, unpackedLink, result.lastModified);
storePath = unpackedStorePath;
replaceSymlink(store->printStorePath(*unpackedStorePath), unpackedLink, result.lastModified);
storePath = std::move(*unpackedStorePath);
}
if (expectedStorePath != "" && storePath != expectedStorePath) {
if (expectedStorePath && *storePath != *expectedStorePath) {
unsigned int statusCode = 102;
Hash gotHash = request.unpack
? hashPath(request.expectedHash.type, store->toRealPath(storePath)).first
: hashFile(request.expectedHash.type, store->toRealPath(storePath));
? hashPath(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath))).first
: hashFile(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath)));
throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s",
url, request.expectedHash.to_string(), gotHash.to_string());
}
@ -937,8 +937,8 @@ CachedDownloadResult Downloader::downloadCached(
if (request.gcRoot)
store->addIndirectRoot(fileLink);
result.storePath = storePath;
result.path = store->toRealPath(storePath);
result.storePath = store->printStorePath(*storePath);
result.path = store->toRealPath(result.storePath);
return result;
}

35
src/libstore/export-import.cc
View file

@ -24,9 +24,9 @@ struct HashAndWriteSink : Sink
}
};
void Store::exportPaths(const Paths & paths, Sink & sink)
void Store::exportPaths(const StorePathSet & paths, Sink & sink)
{
Paths sorted = topoSortPaths(PathSet(paths.begin(), paths.end()));
auto sorted = topoSortPaths(paths);
std::reverse(sorted.begin(), sorted.end());
std::string doneLabel("paths exported");
@ -42,7 +42,7 @@ void Store::exportPaths(const Paths & paths, Sink & sink)
sink << 0;
}
void Store::exportPath(const Path & path, Sink & sink)
void Store::exportPath(const StorePath & path, Sink & sink)
{
auto info = queryPathInfo(path);
@ -55,15 +55,21 @@ void Store::exportPath(const Path & path, Sink & sink)
Don't complain if the stored hash is zero (unknown). */
Hash hash = hashAndWriteSink.currentHash();
if (hash != info->narHash && info->narHash != Hash(info->narHash.type))
throw Error(format("hash of path '%1%' has changed from '%2%' to '%3%'!") % path
% info->narHash.to_string() % hash.to_string());
throw Error("hash of path '%s' has changed from '%s' to '%s'!",
printStorePath(path), info->narHash.to_string(), hash.to_string());
hashAndWriteSink << exportMagic << path << info->references << info->deriver << 0;
hashAndWriteSink
<< exportMagic
<< printStorePath(path);
writeStorePaths(*this, hashAndWriteSink, info->references);
hashAndWriteSink
<< (info->deriver ? printStorePath(*info->deriver) : "")
<< 0;
}
Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, CheckSigsFlag checkSigs)
StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, CheckSigsFlag checkSigs)
{
Paths res;
StorePaths res;
while (true) {
auto n = readNum<uint64_t>(source);
if (n == 0) break;
@ -77,16 +83,15 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
if (magic != exportMagic)
throw Error("Nix archive cannot be imported; wrong format");
ValidPathInfo info;
info.path = readStorePath(*this, source);
ValidPathInfo info(parseStorePath(readString(source)));
//Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path);
info.references = readStorePaths<PathSet>(*this, source);
info.references = readStorePaths<StorePathSet>(*this, source);
info.deriver = readString(source);
if (info.deriver != "") assertStorePath(info.deriver);
auto deriver = readString(source);
if (deriver != "")
info.deriver = parseStorePath(deriver);
info.narHash = hashString(htSHA256, *tee.source.data);
info.narSize = tee.source.data->size();
@ -97,7 +102,7 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
addToStore(info, tee.source.data, NoRepair, checkSigs, accessor);
res.push_back(info.path);
res.push_back(info.path.clone());
}
return res;

184
src/libstore/gc.cc
View file

@ -85,12 +85,10 @@ void LocalStore::addIndirectRoot(const Path & path)
}
Path LocalFSStore::addPermRoot(const Path & _storePath,
Path LocalFSStore::addPermRoot(const StorePath & storePath,
const Path & _gcRoot, bool indirect, bool allowOutsideRootsDir)
{
Path storePath(canonPath(_storePath));
Path gcRoot(canonPath(_gcRoot));
assertStorePath(storePath);
if (isInStore(gcRoot))
throw Error(format(
@ -102,7 +100,7 @@ Path LocalFSStore::addPermRoot(const Path & _storePath,
point to the Nix store. */
if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot))))
throw Error(format("cannot create symlink '%1%'; already exists") % gcRoot);
makeSymlink(gcRoot, storePath);
makeSymlink(gcRoot, printStorePath(storePath));
addIndirectRoot(gcRoot);
}
@ -117,10 +115,10 @@ Path LocalFSStore::addPermRoot(const Path & _storePath,
% gcRoot % rootsDir);
}
if (baseNameOf(gcRoot) == baseNameOf(storePath))
if (baseNameOf(gcRoot) == std::string(storePath.to_string()))
writeFile(gcRoot, "");
else
makeSymlink(gcRoot, storePath);
makeSymlink(gcRoot, printStorePath(storePath));
}
/* Check that the root can be found by the garbage collector.
@ -129,13 +127,12 @@ Path LocalFSStore::addPermRoot(const Path & _storePath,
check if the root is in a directory in or linked from the
gcroots directory. */
if (settings.checkRootReachability) {
Roots roots = findRoots(false);
if (roots[storePath].count(gcRoot) == 0)
auto roots = findRoots(false);
if (roots[storePath.clone()].count(gcRoot) == 0)
printError(
format(
"warning: '%1%' is not in a directory where the garbage collector looks for roots; "
"therefore, '%2%' might be removed by the garbage collector")
% gcRoot % storePath);
"warning: '%1%' is not in a directory where the garbage collector looks for roots; "
"therefore, '%2%' might be removed by the garbage collector",
gcRoot, printStorePath(storePath));
}
/* Grab the global GC root, causing us to block while a GC is in
@ -147,7 +144,7 @@ Path LocalFSStore::addPermRoot(const Path & _storePath,
}
void LocalStore::addTempRoot(const Path & path)
void LocalStore::addTempRoot(const StorePath & path)
{
auto state(_state.lock());
@ -188,7 +185,7 @@ void LocalStore::addTempRoot(const Path & path)
debug(format("acquiring write lock on '%1%'") % fnTempRoots);
lockFile(state->fdTempRoots.get(), ltWrite, true);
string s = path + '\0';
string s = printStorePath(path) + '\0';
writeFull(state->fdTempRoots.get(), s);
/* Downgrade to a read lock. */
@ -246,8 +243,7 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
while ((end = contents.find((char) 0, pos)) != string::npos) {
Path root(contents, pos, end - pos);
debug("got temporary root '%s'", root);
assertStorePath(root);
tempRoots[root].emplace(censor ? censored : fmt("{temp:%d}", pid));
tempRoots[parseStorePath(root)].emplace(censor ? censored : fmt("{temp:%d}", pid));
pos = end + 1;
}
@ -260,10 +256,11 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
{
auto foundRoot = [&](const Path & path, const Path & target) {
Path storePath = toStorePath(target);
if (isStorePath(storePath) && isValidPath(storePath))
roots[storePath].emplace(path);
// FIXME
if (isStorePath(storePath) && isValidPath(parseStorePath(storePath)))
roots[parseStorePath(storePath)].emplace(path);
else
printInfo(format("skipping invalid root from '%1%' to '%2%'") % path % storePath);
printInfo("skipping invalid root from '%1%' to '%2%'", path, storePath);
};
try {
@ -299,9 +296,10 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
}
else if (type == DT_REG) {
Path storePath = storeDir + "/" + baseNameOf(path);
if (isStorePath(storePath) && isValidPath(storePath))
roots[storePath].emplace(path);
Path storePath = storeDir + "/" + std::string(baseNameOf(path));
// FIXME
if (isStorePath(storePath) && isValidPath(parseStorePath(storePath)))
roots[parseStorePath(storePath)].emplace(path);
}
}
@ -309,7 +307,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
catch (SysError & e) {
/* We only ignore permanent failures. */
if (e.errNo == EACCES || e.errNo == ENOENT || e.errNo == ENOTDIR)
printInfo(format("cannot read potential root '%1%'") % path);
printInfo("cannot read potential root '%1%'", path);
else
throw;
}
@ -340,7 +338,9 @@ Roots LocalStore::findRoots(bool censor)
return roots;
}
static void readProcLink(const string & file, Roots & roots)
typedef std::unordered_map<Path, std::unordered_set<std::string>> UncheckedRoots;
static void readProcLink(const string & file, UncheckedRoots & roots)
{
/* 64 is the starting buffer size gnu readlink uses... */
auto bufsiz = ssize_t{64};
@ -369,7 +369,7 @@ static string quoteRegexChars(const string & raw)
return std::regex_replace(raw, specialRegex, R"(\$&)");
}
static void readFileRoots(const char * path, Roots & roots)
static void readFileRoots(const char * path, UncheckedRoots & roots)
{
try {
roots[readFile(path)].emplace(path);
@ -381,7 +381,7 @@ static void readFileRoots(const char * path, Roots & roots)
void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
{
Roots unchecked;
UncheckedRoots unchecked;
auto procDir = AutoCloseDir{opendir("/proc")};
if (procDir) {
@ -466,16 +466,16 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
#endif
for (auto & [target, links] : unchecked) {
if (isInStore(target)) {
Path path = toStorePath(target);
if (isStorePath(path) && isValidPath(path)) {
debug(format("got additional root '%1%'") % path);
if (censor)
roots[path].insert(censored);
else
roots[path].insert(links.begin(), links.end());
}
}
if (!isInStore(target)) continue;
Path pathS = toStorePath(target);
if (!isStorePath(pathS)) continue;
auto path = parseStorePath(pathS);
if (!isValidPath(path)) continue;
debug("got additional root '%1%'", pathS);
if (censor)
roots[path.clone()].insert(censored);
else
roots[path.clone()].insert(links.begin(), links.end());
}
}
@ -485,18 +485,19 @@ struct GCLimitReached { };
struct LocalStore::GCState
{
GCOptions options;
const GCOptions & options;
GCResults & results;
PathSet roots;
PathSet tempRoots;
PathSet dead;
PathSet alive;
StorePathSet roots;
StorePathSet tempRoots;
StorePathSet dead;
StorePathSet alive;
bool gcKeepOutputs;
bool gcKeepDerivations;
unsigned long long bytesInvalidated;
bool moveToTrash = true;
bool shouldDelete;
GCState(GCResults & results_) : results(results_), bytesInvalidated(0) { }
GCState(const GCOptions & options, GCResults & results)
: options(options), results(results), bytesInvalidated(0) { }
};
@ -504,7 +505,7 @@ bool LocalStore::isActiveTempFile(const GCState & state,
const Path & path, const string & suffix)
{
return hasSuffix(path, suffix)
&& state.tempRoots.find(string(path, 0, path.size() - suffix.size())) != state.tempRoots.end();
&& state.tempRoots.count(parseStorePath(string(path, 0, path.size() - suffix.size())));
}
@ -522,16 +523,17 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
unsigned long long size = 0;
if (isStorePath(path) && isValidPath(path)) {
PathSet referrers;
queryReferrers(path, referrers);
// FIXME
if (isStorePath(path) && isValidPath(parseStorePath(path))) {
StorePathSet referrers;
queryReferrers(parseStorePath(path), referrers);
for (auto & i : referrers)
if (i != path) deletePathRecursive(state, i);
size = queryPathInfo(path)->narSize;
invalidatePathChecked(path);
if (printStorePath(i) != path) deletePathRecursive(state, printStorePath(i));
size = queryPathInfo(parseStorePath(path))->narSize;
invalidatePathChecked(parseStorePath(path));
}
Path realPath = realStoreDir + "/" + baseNameOf(path);
Path realPath = realStoreDir + "/" + std::string(baseNameOf(path));
struct stat st;
if (lstat(realPath.c_str(), &st)) {
@ -555,7 +557,7 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
try {
if (chmod(realPath.c_str(), st.st_mode | S_IWUSR) == -1)
throw SysError(format("making '%1%' writable") % realPath);
Path tmp = trashDir + "/" + baseNameOf(path);
Path tmp = trashDir + "/" + std::string(baseNameOf(path));
if (rename(realPath.c_str(), tmp.c_str()))
throw SysError(format("unable to rename '%1%' to '%2%'") % realPath % tmp);
state.bytesInvalidated += size;
@ -575,7 +577,7 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
}
bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & path)
bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const StorePath & path)
{
if (visited.count(path)) return false;
@ -584,41 +586,41 @@ bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & p
if (state.dead.count(path)) return false;
if (state.roots.count(path)) {
debug(format("cannot delete '%1%' because it's a root") % path);
state.alive.insert(path);
debug("cannot delete '%1%' because it's a root", printStorePath(path));
state.alive.insert(path.clone());
return true;
}
visited.insert(path);
visited.insert(path.clone());
if (!isStorePath(path) || !isValidPath(path)) return false;
//FIXME
if (!isStorePath(printStorePath(path)) || !isValidPath(path)) return false;
PathSet incoming;
StorePathSet incoming;
/* Don't delete this path if any of its referrers are alive. */
queryReferrers(path, incoming);
/* If keep-derivations is set and this is a derivation, then
don't delete the derivation if any of the outputs are alive. */
if (state.gcKeepDerivations && isDerivation(path)) {
PathSet outputs = queryDerivationOutputs(path);
for (auto & i : outputs)
if (state.gcKeepDerivations && path.isDerivation()) {
for (auto & i : queryDerivationOutputs(path))
if (isValidPath(i) && queryPathInfo(i)->deriver == path)
incoming.insert(i);
incoming.insert(i.clone());
}
/* If keep-outputs is set, then don't delete this path if there
are derivers of this path that are not garbage. */
if (state.gcKeepOutputs) {
PathSet derivers = queryValidDerivers(path);
auto derivers = queryValidDerivers(path);
for (auto & i : derivers)
incoming.insert(i);
incoming.insert(i.clone());
}
for (auto & i : incoming)
if (i != path)
if (canReachRoot(state, visited, i)) {
state.alive.insert(path);
state.alive.insert(path.clone());
return true;
}
@ -630,12 +632,13 @@ void LocalStore::tryToDelete(GCState & state, const Path & path)
{
checkInterrupt();
auto realPath = realStoreDir + "/" + baseNameOf(path);
auto realPath = realStoreDir + "/" + std::string(baseNameOf(path));
if (realPath == linksDir || realPath == trashDir) return;
//Activity act(*logger, lvlDebug, format("considering whether to delete '%1%'") % path);
if (!isStorePath(path) || !isValidPath(path)) {
// FIXME
if (!isStorePath(path) || !isValidPath(parseStorePath(path))) {
/* A lock file belonging to a path that we're building right
now isn't garbage. */
if (isActiveTempFile(state, path, ".lock")) return;
@ -650,16 +653,17 @@ void LocalStore::tryToDelete(GCState & state, const Path & path)
if (isActiveTempFile(state, path, ".check")) return;
}
PathSet visited;
StorePathSet visited;
if (canReachRoot(state, visited, path)) {
debug(format("cannot delete '%1%' because it's still reachable") % path);
if (canReachRoot(state, visited, parseStorePath(path))) {
debug("cannot delete '%s' because it's still reachable", path);
} else {
/* No path we visited was a root, so everything is garbage.
But we only delete ‘path’ and its referrers here so that
‘nix-store --delete’ doesn't have the unexpected effect of
recursing into derivations and outputs. */
state.dead.insert(visited.begin(), visited.end());
for (auto & i : visited)
state.dead.insert(i.clone());
if (state.shouldDelete)
deletePathRecursive(state, path);
}
@ -715,8 +719,7 @@ void LocalStore::removeUnusedLinks(const GCState & state)
void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
{
GCState state(results);
state.options = options;
GCState state(options, results);
state.gcKeepOutputs = settings.gcKeepOutputs;
state.gcKeepDerivations = settings.gcKeepDerivations;
@ -741,12 +744,12 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
/* Find the roots. Since we've grabbed the GC lock, the set of
permanent roots cannot increase now. */
printError(format("finding garbage collector roots..."));
printError("finding garbage collector roots...");
Roots rootMap;
if (!options.ignoreLiveness)
findRootsNoTemp(rootMap, true);
for (auto & i : rootMap) state.roots.insert(i.first);
for (auto & i : rootMap) state.roots.insert(i.first.clone());
/* Read the temporary roots. This acquires read locks on all
per-process temporary root files. So after this point no paths
@ -754,9 +757,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
FDs fds;
Roots tempRoots;
findTempRoots(fds, tempRoots, true);
for (auto & root : tempRoots)
state.tempRoots.insert(root.first);
state.roots.insert(state.tempRoots.begin(), state.tempRoots.end());
for (auto & root : tempRoots) {
state.tempRoots.insert(root.first.clone());
state.roots.insert(root.first.clone());
}
/* After this point the set of roots or temporary roots cannot
increase, since we hold locks on everything. So everything
@ -768,7 +772,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
createDirs(trashDir);
} catch (SysError & e) {
if (e.errNo == ENOSPC) {
printInfo(format("note: can't create trash directory: %1%") % e.msg());
printInfo("note: can't create trash directory: %s", e.msg());
state.moveToTrash = false;
}
}
@ -780,22 +784,21 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
if (options.action == GCOptions::gcDeleteSpecific) {
for (auto & i : options.pathsToDelete) {
assertStorePath(i);
tryToDelete(state, i);
tryToDelete(state, printStorePath(i));
if (state.dead.find(i) == state.dead.end())
throw Error(format(
throw Error(
"cannot delete path '%1%' since it is still alive. "
"To find out why use: "
"nix-store --query --roots"
) % i);
"nix-store --query --roots",
printStorePath(i));
}
} else if (options.maxFreed > 0) {
if (state.shouldDelete)
printError(format("deleting garbage..."));
printError("deleting garbage...");
else
printError(format("determining live/dead paths..."));
printError("determining live/dead paths...");
try {
@ -815,7 +818,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
string name = dirent->d_name;
if (name == "." || name == "..") continue;
Path path = storeDir + "/" + name;
if (isStorePath(path) && isValidPath(path))
// FIXME
if (isStorePath(path) && isValidPath(parseStorePath(path)))
entries.push_back(path);
else
tryToDelete(state, path);
@ -840,12 +844,14 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
}
if (state.options.action == GCOptions::gcReturnLive) {
state.results.paths = state.alive;
for (auto & i : state.alive)
state.results.paths.insert(printStorePath(i));
return;
}
if (state.options.action == GCOptions::gcReturnDead) {
state.results.paths = state.dead;
for (auto & i : state.dead)
state.results.paths.insert(printStorePath(i));
return;
}
@ -859,7 +865,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
/* Clean up the links directory. */
if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) {
printError(format("deleting unused links..."));
printError("deleting unused links...");
removeUnusedLinks(state);
}

80
src/libstore/legacy-ssh-store.cc
View file

@ -87,25 +87,27 @@ struct LegacySSHStore : public Store
return uriScheme + host;
}
void queryPathInfoUncached(const Path & path,
void queryPathInfoUncached(const StorePath & path,
Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override
{
try {
auto conn(connections->get());
debug("querying remote host '%s' for info on '%s'", host, path);
debug("querying remote host '%s' for info on '%s'", host, printStorePath(path));
conn->to << cmdQueryPathInfos << PathSet{path};
conn->to << cmdQueryPathInfos << PathSet{printStorePath(path)};
conn->to.flush();
auto info = std::make_shared<ValidPathInfo>();
conn->from >> info->path;
if (info->path.empty()) return callback(nullptr);
auto p = readString(conn->from);
if (p.empty()) return callback(nullptr);
auto info = std::make_shared<ValidPathInfo>(parseStorePath(p));
assert(path == info->path);
PathSet references;
conn->from >> info->deriver;
info->references = readStorePaths<PathSet>(*this, conn->from);
auto deriver = readString(conn->from);
if (deriver != "")
info->deriver = parseStorePath(deriver);
info->references = readStorePaths<StorePathSet>(*this, conn->from);
readLongLong(conn->from); // download size
info->narSize = readLongLong(conn->from);
@ -127,7 +129,7 @@ struct LegacySSHStore : public Store
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override
{
debug("adding path '%s' to remote host '%s'", info.path, host);
debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host);
auto conn(connections->get());
@ -135,10 +137,11 @@ struct LegacySSHStore : public Store
conn->to
<< cmdAddToStoreNar
<< info.path
<< info.deriver
<< info.narHash.to_string(Base16, false)
<< info.references
<< printStorePath(info.path)
<< (info.deriver ? printStorePath(*info.deriver) : "")
<< info.narHash.to_string(Base16, false);
writeStorePaths(*this, conn->to, info.references);
conn->to
<< info.registrationTime
<< info.narSize
<< info.ultimate
@ -165,9 +168,10 @@ struct LegacySSHStore : public Store
}
conn->to
<< exportMagic
<< info.path
<< info.references
<< info.deriver
<< printStorePath(info.path);
writeStorePaths(*this, conn->to, info.references);
conn->to
<< (info.deriver ? printStorePath(*info.deriver) : "")
<< 0
<< 0;
conn->to.flush();
@ -175,39 +179,40 @@ struct LegacySSHStore : public Store
}
if (readInt(conn->from) != 1)
throw Error("failed to add path '%s' to remote host '%s', info.path, host");
throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host);
}
void narFromPath(const Path & path, Sink & sink) override
void narFromPath(const StorePath & path, Sink & sink) override
{
auto conn(connections->get());
conn->to << cmdDumpStorePath << path;
conn->to << cmdDumpStorePath << printStorePath(path);
conn->to.flush();
copyNAR(conn->from, sink);
}
Path queryPathFromHashPart(const string & hashPart) override
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
{ unsupported("queryPathFromHashPart"); }
Path addToStore(const string & name, const Path & srcPath,
StorePath addToStore(const string & name, const Path & srcPath,
bool recursive, HashType hashAlgo,
PathFilter & filter, RepairFlag repair) override
{ unsupported("addToStore"); }
Path addTextToStore(const string & name, const string & s,
const PathSet & references, RepairFlag repair) override
StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override
{ unsupported("addTextToStore"); }
BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode) override
{
auto conn(connections->get());
conn->to
<< cmdBuildDerivation
<< drvPath
<< drv
<< printStorePath(drvPath);
writeDerivation(conn->to, *this, drv);
conn->to
<< settings.maxSilentTime
<< settings.buildTimeout;
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 2)
@ -230,11 +235,11 @@ struct LegacySSHStore : public Store
return status;
}
void ensurePath(const Path & path) override
void ensurePath(const StorePath & path) override
{ unsupported("ensurePath"); }
void computeFSClosure(const PathSet & paths,
PathSet & out, bool flipDirection = false,
void computeFSClosure(const StorePathSet & paths,
StorePathSet & out, bool flipDirection = false,
bool includeOutputs = false, bool includeDerivers = false) override
{
if (flipDirection || includeDerivers) {
@ -246,16 +251,15 @@ struct LegacySSHStore : public Store
conn->to
<< cmdQueryClosure
<< includeOutputs
<< paths;
<< includeOutputs;
writeStorePaths(*this, conn->to, paths);
conn->to.flush();
auto res = readStorePaths<PathSet>(*this, conn->from);
out.insert(res.begin(), res.end());
for (auto & i : readStorePaths<StorePathSet>(*this, conn->from))
out.insert(i.clone());
}
PathSet queryValidPaths(const PathSet & paths,
StorePathSet queryValidPaths(const StorePathSet & paths,
SubstituteFlag maybeSubstitute = NoSubstitute) override
{
auto conn(connections->get());
@ -263,11 +267,11 @@ struct LegacySSHStore : public Store
conn->to
<< cmdQueryValidPaths
<< false // lock
<< maybeSubstitute
<< paths;
<< maybeSubstitute;
writeStorePaths(*this, conn->to, paths);
conn->to.flush();
return readStorePaths<PathSet>(*this, conn->from);
return readStorePaths<StorePathSet>(*this, conn->from);
}
void connect() override

6
src/libstore/local-binary-cache-store.cc
View file

@ -44,15 +44,15 @@ protected:
}
}
PathSet queryAllValidPaths() override
StorePathSet queryAllValidPaths() override
{
PathSet paths;
StorePathSet paths;
for (auto & entry : readDirectory(binaryCacheDir)) {
if (entry.name.size() != 40 ||
!hasSuffix(entry.name, ".narinfo"))
continue;
paths.insert(storeDir + "/" + entry.name.substr(0, entry.name.size() - 8));
paths.insert(parseStorePath(storeDir + "/" + entry.name.substr(0, entry.name.size() - 8)));
}
return paths;

24
src/libstore/local-fs-store.cc
View file

@ -21,7 +21,7 @@ struct LocalStoreAccessor : public FSAccessor
Path toRealPath(const Path & path)
{
Path storePath = store->toStorePath(path);
if (!store->isValidPath(storePath))
if (!store->isValidPath(store->parseStorePath(storePath)))
throw InvalidPath(format("path '%1%' is not a valid store path") % storePath);
return store->getRealStoreDir() + std::string(path, store->storeDir.size());
}
@ -77,34 +77,32 @@ ref<FSAccessor> LocalFSStore::getFSAccessor()
std::dynamic_pointer_cast<LocalFSStore>(shared_from_this())));
}
void LocalFSStore::narFromPath(const Path & path, Sink & sink)
void LocalFSStore::narFromPath(const StorePath & path, Sink & sink)
{
if (!isValidPath(path))
throw Error(format("path '%s' is not valid") % path);
dumpPath(getRealStoreDir() + std::string(path, storeDir.size()), sink);
throw Error("path '%s' is not valid", printStorePath(path));
dumpPath(getRealStoreDir() + std::string(printStorePath(path), storeDir.size()), sink);
}
const string LocalFSStore::drvsLogDir = "drvs";
std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path & path_)
std::shared_ptr<std::string> LocalFSStore::getBuildLog(const StorePath & path_)
{
auto path(path_);
auto path = path_.clone();
assertStorePath(path);
if (!isDerivation(path)) {
if (!path.isDerivation()) {
try {
path = queryPathInfo(path)->deriver;
auto info = queryPathInfo(path);
if (!info->deriver) return nullptr;
path = info->deriver->clone();
} catch (InvalidPath &) {
return nullptr;
}
if (path == "") return nullptr;
}
string baseName = baseNameOf(path);
auto baseName = std::string(baseNameOf(printStorePath(path)));
for (int j = 0; j < 2; j++) {

293
src/libstore/local-store.cc
View file

@ -534,42 +534,36 @@ void canonicalisePathMetaData(const Path & path, uid_t fromUid)
}
void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & drv)
void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv)
{
string drvName = storePathToName(drvPath);
assert(isDerivation(drvName));
assert(drvPath.isDerivation());
std::string drvName(drvPath.name());
drvName = string(drvName, 0, drvName.size() - drvExtension.size());
if (drv.isFixedOutput()) {
DerivationOutputs::const_iterator out = drv.outputs.find("out");
if (out == drv.outputs.end())
throw Error(format("derivation '%1%' does not have an output named 'out'") % drvPath);
throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath));
bool recursive; Hash h;
out->second.parseHashInfo(recursive, h);
Path outPath = makeFixedOutputPath(recursive, h, drvName);
auto outPath = makeFixedOutputPath(recursive, h, drvName);
StringPairs::const_iterator j = drv.env.find("out");
if (out->second.path != outPath || j == drv.env.end() || j->second != outPath)
throw Error(format("derivation '%1%' has incorrect output '%2%', should be '%3%'")
% drvPath % out->second.path % outPath);
if (out->second.path != outPath || j == drv.env.end() || parseStorePath(j->second) != outPath)
throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
printStorePath(drvPath), printStorePath(out->second.path), printStorePath(outPath));
}
else {
Derivation drvCopy(drv);
for (auto & i : drvCopy.outputs) {
i.second.path = "";
drvCopy.env[i.first] = "";
}
Hash h = hashDerivationModulo(*this, drvCopy);
Hash h = hashDerivationModulo(*this, drv, true);
for (auto & i : drv.outputs) {
Path outPath = makeOutputPath(i.first, h, drvName);
auto outPath = makeOutputPath(i.first, h, drvName);
StringPairs::const_iterator j = drv.env.find(i.first);
if (i.second.path != outPath || j == drv.env.end() || j->second != outPath)
throw Error(format("derivation '%1%' has incorrect output '%2%', should be '%3%'")
% drvPath % i.second.path % outPath);
if (i.second.path != outPath || j == drv.env.end() || parseStorePath(j->second) != outPath)
throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
printStorePath(drvPath), printStorePath(i.second.path), printStorePath(outPath));
}
}
}
@ -578,16 +572,15 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation &
uint64_t LocalStore::addValidPath(State & state,
const ValidPathInfo & info, bool checkOutputs)
{
checkStoreName(storePathToName(info.path));
if (info.ca != "" && !info.isContentAddressed(*this))
throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't", info.path);
throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't",
printStorePath(info.path));
state.stmtRegisterValidPath.use()
(info.path)
(printStorePath(info.path))
(info.narHash.to_string(Base16))
(info.registrationTime == 0 ? time(0) : info.registrationTime)
(info.deriver, info.deriver != "")
(info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver)
(info.narSize, info.narSize != 0)
(info.ultimate ? 1 : 0, info.ultimate)
(concatStringsSep(" ", info.sigs), !info.sigs.empty())
@ -599,8 +592,8 @@ uint64_t LocalStore::addValidPath(State & state,
the database. This is useful for the garbage collector: it can
efficiently query whether a path is an output of some
derivation. */
if (isDerivation(info.path)) {
Derivation drv = readDerivation(realStoreDir + "/" + baseNameOf(info.path));
if (info.path.isDerivation()) {
auto drv = readDerivation(*this, realStoreDir + "/" + std::string(info.path.to_string()));
/* Verify that the output paths in the derivation are correct
(i.e., follow the scheme for computing output paths from
@ -613,34 +606,31 @@ uint64_t LocalStore::addValidPath(State & state,
state.stmtAddDerivationOutput.use()
(id)
(i.first)
(i.second.path)
(printStorePath(i.second.path))
.exec();
}
}
{
auto state_(Store::state.lock());
state_->pathInfoCache.upsert(storePathToHash(info.path), std::make_shared<ValidPathInfo>(info));
state_->pathInfoCache.upsert(storePathToHash(printStorePath(info.path)), std::make_shared<ValidPathInfo>(info));
}
return id;
}
void LocalStore::queryPathInfoUncached(const Path & path,
void LocalStore::queryPathInfoUncached(const StorePath & path,
Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
{
try {
auto info = std::make_shared<ValidPathInfo>();
info->path = path;
assertStorePath(path);
auto info = std::make_shared<ValidPathInfo>(path.clone());
callback(retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() {
auto state(_state.lock());
/* Get the path info. */
auto useQueryPathInfo(state->stmtQueryPathInfo.use()(path));
auto useQueryPathInfo(state->stmtQueryPathInfo.use()(printStorePath(info->path)));
if (!useQueryPathInfo.next())
return std::shared_ptr<ValidPathInfo>();
@ -650,13 +640,13 @@ void LocalStore::queryPathInfoUncached(const Path & path,
try {
info->narHash = Hash(useQueryPathInfo.getStr(1));
} catch (BadHash & e) {
throw Error("in valid-path entry for '%s': %s", path, e.what());
throw Error("in valid-path entry for '%s': %s", printStorePath(path), e.what());
}
info->registrationTime = useQueryPathInfo.getInt(2);
auto s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 3);
if (s) info->deriver = s;
if (s) info->deriver = parseStorePath(s);
/* Note that narSize = NULL yields 0. */
info->narSize = useQueryPathInfo.getInt(4);
@ -673,7 +663,7 @@ void LocalStore::queryPathInfoUncached(const Path & path,
auto useQueryReferences(state->stmtQueryReferences.use()(info->id));
while (useQueryReferences.next())
info->references.insert(useQueryReferences.getStr(0));
info->references.insert(parseStorePath(useQueryReferences.getStr(0)));
return info;
}));
@ -691,27 +681,27 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
(info.ultimate ? 1 : 0, info.ultimate)
(concatStringsSep(" ", info.sigs), !info.sigs.empty())
(info.ca, !info.ca.empty())
(info.path)
(printStorePath(info.path))
.exec();
}
uint64_t LocalStore::queryValidPathId(State & state, const Path & path)
uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path)
{
auto use(state.stmtQueryPathInfo.use()(path));
auto use(state.stmtQueryPathInfo.use()(printStorePath(path)));
if (!use.next())
throw Error(format("path '%1%' is not valid") % path);
throw Error("path '%s' is not valid", printStorePath(path));
return use.getInt(0);
}
bool LocalStore::isValidPath_(State & state, const Path & path)
bool LocalStore::isValidPath_(State & state, const StorePath & path)
{
return state.stmtQueryPathInfo.use()(path).next();
return state.stmtQueryPathInfo.use()(printStorePath(path)).next();
}
bool LocalStore::isValidPathUncached(const Path & path)
bool LocalStore::isValidPathUncached(const StorePath & path)
{
return retrySQLite<bool>([&]() {
auto state(_state.lock());
@ -720,39 +710,38 @@ bool LocalStore::isValidPathUncached(const Path & path)
}
PathSet LocalStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute)
StorePathSet LocalStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute)
{
PathSet res;
StorePathSet res;
for (auto & i : paths)
if (isValidPath(i)) res.insert(i);
if (isValidPath(i)) res.insert(i.clone());
return res;
}
PathSet LocalStore::queryAllValidPaths()
StorePathSet LocalStore::queryAllValidPaths()
{
return retrySQLite<PathSet>([&]() {
return retrySQLite<StorePathSet>([&]() {
auto state(_state.lock());
auto use(state->stmtQueryValidPaths.use());
PathSet res;
while (use.next()) res.insert(use.getStr(0));
StorePathSet res;
while (use.next()) res.insert(parseStorePath(use.getStr(0)));
return res;
});
}
void LocalStore::queryReferrers(State & state, const Path & path, PathSet & referrers)
void LocalStore::queryReferrers(State & state, const StorePath & path, StorePathSet & referrers)
{
auto useQueryReferrers(state.stmtQueryReferrers.use()(path));
auto useQueryReferrers(state.stmtQueryReferrers.use()(printStorePath(path)));
while (useQueryReferrers.next())
referrers.insert(useQueryReferrers.getStr(0));
referrers.insert(parseStorePath(useQueryReferrers.getStr(0)));
}
void LocalStore::queryReferrers(const Path & path, PathSet & referrers)
void LocalStore::queryReferrers(const StorePath & path, StorePathSet & referrers)
{
assertStorePath(path);
return retrySQLite<void>([&]() {
auto state(_state.lock());
queryReferrers(*state, path, referrers);
@ -760,42 +749,40 @@ void LocalStore::queryReferrers(const Path & path, PathSet & referrers)
}
PathSet LocalStore::queryValidDerivers(const Path & path)
StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
{
assertStorePath(path);
return retrySQLite<PathSet>([&]() {
return retrySQLite<StorePathSet>([&]() {
auto state(_state.lock());
auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(path));
auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(printStorePath(path)));
PathSet derivers;
StorePathSet derivers;
while (useQueryValidDerivers.next())
derivers.insert(useQueryValidDerivers.getStr(1));
derivers.insert(parseStorePath(useQueryValidDerivers.getStr(1)));
return derivers;
});
}
PathSet LocalStore::queryDerivationOutputs(const Path & path)
StorePathSet LocalStore::queryDerivationOutputs(const StorePath & path)
{
return retrySQLite<PathSet>([&]() {
return retrySQLite<StorePathSet>([&]() {
auto state(_state.lock());
auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use()
(queryValidPathId(*state, path)));
PathSet outputs;
StorePathSet outputs;
while (useQueryDerivationOutputs.next())
outputs.insert(useQueryDerivationOutputs.getStr(1));
outputs.insert(parseStorePath(useQueryDerivationOutputs.getStr(1)));
return outputs;
});
}
StringSet LocalStore::queryDerivationOutputNames(const Path & path)
StringSet LocalStore::queryDerivationOutputNames(const StorePath & path)
{
return retrySQLite<StringSet>([&]() {
auto state(_state.lock());
@ -812,31 +799,36 @@ StringSet LocalStore::queryDerivationOutputNames(const Path & path)
}
Path LocalStore::queryPathFromHashPart(const string & hashPart)
std::optional<StorePath> LocalStore::queryPathFromHashPart(const std::string & hashPart)
{
if (hashPart.size() != storePathHashLen) throw Error("invalid hash part");
Path prefix = storeDir + "/" + hashPart;
return retrySQLite<Path>([&]() -> std::string {
return retrySQLite<std::optional<StorePath>>([&]() -> std::optional<StorePath> {
auto state(_state.lock());
auto useQueryPathFromHashPart(state->stmtQueryPathFromHashPart.use()(prefix));
if (!useQueryPathFromHashPart.next()) return "";
if (!useQueryPathFromHashPart.next()) return {};
const char * s = (const char *) sqlite3_column_text(state->stmtQueryPathFromHashPart, 0);
return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : "";
if (s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0)
return parseStorePath(s);
return {};
});
}
PathSet LocalStore::querySubstitutablePaths(const PathSet & paths)
StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths)
{
if (!settings.useSubstitutes) return PathSet();
if (!settings.useSubstitutes) return StorePathSet();
auto remaining = paths;
PathSet res;
StorePathSet remaining;
for (auto & i : paths)
remaining.insert(i.clone());
StorePathSet res;
for (auto & sub : getDefaultSubstituters()) {
if (remaining.empty()) break;
@ -845,12 +837,12 @@ PathSet LocalStore::querySubstitutablePaths(const PathSet & paths)
auto valid = sub->queryValidPaths(remaining);
PathSet remaining2;
StorePathSet remaining2;
for (auto & path : remaining)
if (valid.count(path))
res.insert(path);
res.insert(path.clone());
else
remaining2.insert(path);
remaining2.insert(path.clone());
std::swap(remaining, remaining2);
}
@ -859,7 +851,7 @@ PathSet LocalStore::querySubstitutablePaths(const PathSet & paths)
}
void LocalStore::querySubstitutablePathInfos(const PathSet & paths,
void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths,
SubstitutablePathInfos & infos)
{
if (!settings.useSubstitutes) return;
@ -867,17 +859,16 @@ void LocalStore::querySubstitutablePathInfos(const PathSet & paths,
if (sub->storeDir != storeDir) continue;
for (auto & path : paths) {
if (infos.count(path)) continue;
debug(format("checking substituter '%s' for path '%s'")
% sub->getUri() % path);
debug("checking substituter '%s' for path '%s'", sub->getUri(), printStorePath(path));
try {
auto info = sub->queryPathInfo(path);
auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
std::shared_ptr<const ValidPathInfo>(info));
infos[path] = SubstitutablePathInfo{
info->deriver,
info->references,
infos.insert_or_assign(path.clone(), SubstitutablePathInfo{
info->deriver ? info->deriver->clone() : std::optional<StorePath>(),
cloneStorePathSet(info->references),
narInfo ? narInfo->fileSize : 0,
info->narSize};
info->narSize});
} catch (InvalidPath &) {
} catch (SubstituterDisabled &) {
} catch (Error & e) {
@ -911,7 +902,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
auto state(_state.lock());
SQLiteTxn txn(state->db);
PathSet paths;
StorePathSet paths;
for (auto & i : infos) {
assert(i.narHash.type == htSHA256);
@ -919,7 +910,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
updatePathInfo(*state, i);
else
addValidPath(*state, i, false);
paths.insert(i.path);
paths.insert(i.path.clone());
}
for (auto & i : infos) {
@ -932,10 +923,10 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
this in addValidPath() above, because the references might
not be valid yet. */
for (auto & i : infos)
if (isDerivation(i.path)) {
if (i.path.isDerivation()) {
// FIXME: inefficient; we already loaded the derivation in addValidPath().
Derivation drv = readDerivation(realStoreDir + "/" + baseNameOf(i.path));
checkDerivationOutputs(i.path, drv);
checkDerivationOutputs(i.path,
readDerivation(*this, realStoreDir + "/" + std::string(i.path.to_string())));
}
/* Do a topological sort of the paths. This will throw an
@ -951,18 +942,18 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
/* Invalidate a path. The caller is responsible for checking that
there are no referrers. */
void LocalStore::invalidatePath(State & state, const Path & path)
void LocalStore::invalidatePath(State & state, const StorePath & path)
{
debug(format("invalidating path '%1%'") % path);
debug("invalidating path '%s'", printStorePath(path));
state.stmtInvalidatePath.use()(path).exec();
state.stmtInvalidatePath.use()(printStorePath(path)).exec();
/* Note that the foreign key constraints on the Refs table take
care of deleting the references entries for `path'. */
{
auto state_(Store::state.lock());
state_->pathInfoCache.erase(storePathToHash(path));
state_->pathInfoCache.erase(storePathToHash(printStorePath(path)));
}
}
@ -980,10 +971,10 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
{
if (!info.narHash)
throw Error("cannot add path '%s' because it lacks a hash", info.path);
throw Error("cannot add path '%s' because it lacks a hash", printStorePath(info.path));
if (requireSigs && checkSigs && !info.checkSignatures(*this, getPublicKeys()))
throw Error("cannot add path '%s' because it lacks a valid signature", info.path);
throw Error("cannot add path '%s' because it lacks a valid signature", printStorePath(info.path));
addTempRoot(info.path);
@ -991,12 +982,12 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
PathLocks outputLock;
Path realPath = realStoreDir + "/" + baseNameOf(info.path);
Path realPath = realStoreDir + "/" + std::string(info.path.to_string());
/* Lock the output path. But don't lock if we're being called
from a build hook (whose parent process already acquired a
lock on this path). */
if (!locksHeld.count(info.path))
if (!locksHeld.count(printStorePath(info.path)))
outputLock.lockPaths({realPath});
if (repair || !isValidPath(info.path)) {
@ -1011,7 +1002,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
else {
if (!info.references.empty())
settings.requireExperimentalFeature("ca-references");
hashSink = std::make_unique<HashModuloSink>(htSHA256, storePathToHash(info.path));
hashSink = std::make_unique<HashModuloSink>(htSHA256, storePathToHash(printStorePath(info.path)));
}
LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t {
@ -1026,11 +1017,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
if (hashResult.first != info.narHash)
throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s",
info.path, info.narHash.to_string(), hashResult.first.to_string());
printStorePath(info.path), info.narHash.to_string(), hashResult.first.to_string());
if (hashResult.second != info.narSize)
throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s",
info.path, info.narSize, hashResult.second);
printStorePath(info.path), info.narSize, hashResult.second);
autoGC();
@ -1046,12 +1037,12 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
}
Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name,
bool recursive, HashType hashAlgo, RepairFlag repair)
{
Hash h = hashString(hashAlgo, dump);
Path dstPath = makeFixedOutputPath(recursive, h, name);
auto dstPath = makeFixedOutputPath(recursive, h, name);
addTempRoot(dstPath);
@ -1060,7 +1051,8 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
/* The first check above is an optimisation to prevent
unnecessary lock acquisition. */
Path realPath = realStoreDir + "/" + baseNameOf(dstPath);
Path realPath = realStoreDir + "/";
realPath += dstPath.to_string();
PathLocks outputLock({realPath});
@ -1091,8 +1083,7 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
optimisePath(realPath); // FIXME: combine with hashPath()
ValidPathInfo info;
info.path = dstPath;
ValidPathInfo info(dstPath.clone());
info.narHash = hash.first;
info.narSize = hash.second;
info.ca = makeFixedOutputCA(recursive, h);
@ -1106,7 +1097,7 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
}
Path LocalStore::addToStore(const string & name, const Path & _srcPath,
StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
{
Path srcPath(absPath(_srcPath));
@ -1124,8 +1115,8 @@ Path LocalStore::addToStore(const string & name, const Path & _srcPath,
}
Path LocalStore::addTextToStore(const string & name, const string & s,
const PathSet & references, RepairFlag repair)
StorePath LocalStore::addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair)
{
auto hash = hashString(htSHA256, s);
auto dstPath = makeTextPath(name, hash, references);
@ -1134,7 +1125,8 @@ Path LocalStore::addTextToStore(const string & name, const string & s,
if (repair || !isValidPath(dstPath)) {
Path realPath = realStoreDir + "/" + baseNameOf(dstPath);
Path realPath = realStoreDir + "/";
realPath += dstPath.to_string();
PathLocks outputLock({realPath});
@ -1154,11 +1146,10 @@ Path LocalStore::addTextToStore(const string & name, const string & s,
optimisePath(realPath);
ValidPathInfo info;
info.path = dstPath;
ValidPathInfo info(dstPath.clone());
info.narHash = narHash;
info.narSize = sink.s->size();
info.references = references;
info.references = cloneStorePathSet(references);
info.ca = "text:" + hash.to_string();
registerValidPath(info);
}
@ -1180,27 +1171,25 @@ Path LocalStore::createTempDirInStore()
the GC between createTempDir() and addTempRoot(), so repeat
until `tmpDir' exists. */
tmpDir = createTempDir(realStoreDir);
addTempRoot(tmpDir);
addTempRoot(parseStorePath(tmpDir));
} while (!pathExists(tmpDir));
return tmpDir;
}
void LocalStore::invalidatePathChecked(const Path & path)
void LocalStore::invalidatePathChecked(const StorePath & path)
{
assertStorePath(path);
retrySQLite<void>([&]() {
auto state(_state.lock());
SQLiteTxn txn(state->db);
if (isValidPath_(*state, path)) {
PathSet referrers; queryReferrers(*state, path, referrers);
StorePathSet referrers; queryReferrers(*state, path, referrers);
referrers.erase(path); /* ignore self-references */
if (!referrers.empty())
throw PathInUse(format("cannot delete path '%1%' because it is in use by %2%")
% path % showPaths(referrers));
throw PathInUse("cannot delete path '%s' because it is in use by %s",
printStorePath(path), showPaths(referrers));
invalidatePath(*state, path);
}
@ -1219,18 +1208,19 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
existing and valid paths. */
AutoCloseFD fdGCLock = openGCLock(ltWrite);
PathSet store;
StringSet store;
for (auto & i : readDirectory(realStoreDir)) store.insert(i.name);
/* Check whether all valid paths actually exist. */
printInfo("checking path existence...");
PathSet validPaths2 = queryAllValidPaths(), validPaths, done;
StorePathSet validPaths;
PathSet done;
fdGCLock = -1;
for (auto & i : validPaths2)
verifyPath(i, store, done, validPaths, repair, errors);
for (auto & i : queryAllValidPaths())
verifyPath(printStorePath(i), store, done, validPaths, repair, errors);
/* Optionally, check the content hashes (slow). */
if (checkContents) {
@ -1265,21 +1255,20 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
auto info = std::const_pointer_cast<ValidPathInfo>(std::shared_ptr<const ValidPathInfo>(queryPathInfo(i)));
/* Check the content hash (optionally - slow). */
printMsg(lvlTalkative, format("checking contents of '%1%'") % i);
printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i));
std::unique_ptr<AbstractHashSink> hashSink;
if (info->ca == "")
hashSink = std::make_unique<HashSink>(info->narHash.type);
else
hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(info->path));
hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(printStorePath(info->path)));
dumpPath(toRealPath(i), *hashSink);
dumpPath(toRealPath(printStorePath(i)), *hashSink);
auto current = hashSink->finish();
if (info->narHash != nullHash && info->narHash != current.first) {
printError(format("path '%1%' was modified! "
"expected hash '%2%', got '%3%'")
% i % info->narHash.to_string() % current.first.to_string());
printError("path '%s' was modified! expected hash '%s', got '%s'",
printStorePath(i), info->narHash.to_string(), current.first.to_string());
if (repair) repairPath(i); else errors = true;
} else {
@ -1287,14 +1276,14 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
/* Fill in missing hashes. */
if (info->narHash == nullHash) {
printError(format("fixing missing hash on '%1%'") % i);
printError("fixing missing hash on '%s'", printStorePath(i));
info->narHash = current.first;
update = true;
}
/* Fill in missing narSize fields (from old stores). */
if (info->narSize == 0) {
printError(format("updating size field on '%1%' to %2%") % i % current.second);
printError("updating size field on '%s' to %s", printStorePath(i), current.second);
info->narSize = current.second;
update = true;
}
@ -1310,9 +1299,9 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
/* It's possible that the path got GC'ed, so ignore
errors on invalid paths. */
if (isValidPath(i))
printError(format("error: %1%") % e.msg());
printError("error: %s", e.msg());
else
printError(format("warning: %1%") % e.msg());
warn(e.msg());
errors = true;
}
}
@ -1322,43 +1311,43 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
}
void LocalStore::verifyPath(const Path & path, const PathSet & store,
PathSet & done, PathSet & validPaths, RepairFlag repair, bool & errors)
void LocalStore::verifyPath(const Path & pathS, const StringSet & store,
PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors)
{
checkInterrupt();
if (!done.insert(path).second) return;
if (!done.insert(pathS).second) return;
if (!isStorePath(path)) {
printError(format("path '%1%' is not in the Nix store") % path);
auto state(_state.lock());
invalidatePath(*state, path);
if (!isStorePath(pathS)) {
printError("path '%s' is not in the Nix store", pathS);
return;
}
if (store.find(baseNameOf(path)) == store.end()) {
auto path = parseStorePath(pathS);
if (!store.count(std::string(path.to_string()))) {
/* Check any referrers first. If we can invalidate them
first, then we can invalidate this path as well. */
bool canInvalidate = true;
PathSet referrers; queryReferrers(path, referrers);
StorePathSet referrers; queryReferrers(path, referrers);
for (auto & i : referrers)
if (i != path) {
verifyPath(i, store, done, validPaths, repair, errors);
if (validPaths.find(i) != validPaths.end())
verifyPath(printStorePath(i), store, done, validPaths, repair, errors);
if (validPaths.count(i))
canInvalidate = false;
}
if (canInvalidate) {
printError(format("path '%1%' disappeared, removing from database...") % path);
printError("path '%s' disappeared, removing from database...", pathS);
auto state(_state.lock());
invalidatePath(*state, path);
} else {
printError(format("path '%1%' disappeared, but it still has valid referrers!") % path);
printError("path '%s' disappeared, but it still has valid referrers!", pathS);
if (repair)
try {
repairPath(path);
} catch (Error & e) {
printError(format("warning: %1%") % e.msg());
warn(e.msg());
errors = true;
}
else errors = true;
@ -1367,7 +1356,7 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store,
return;
}
validPaths.insert(path);
validPaths.insert(std::move(path));
}
@ -1436,7 +1425,7 @@ void LocalStore::vacuumDB()
}
void LocalStore::addSignatures(const Path & storePath, const StringSet & sigs)
void LocalStore::addSignatures(const StorePath & storePath, const StringSet & sigs)
{
retrySQLite<void>([&]() {
auto state(_state.lock());
@ -1462,7 +1451,7 @@ void LocalStore::signPathInfo(ValidPathInfo & info)
for (auto & secretKeyFile : secretKeyFiles.get()) {
SecretKey secretKey(readFile(secretKeyFile));
info.sign(secretKey);
info.sign(*this, secretKey);
}
}

64
src/libstore/local-store.hh
View file

@ -119,36 +119,36 @@ public:
std::string getUri() override;
bool isValidPathUncached(const Path & path) override;
bool isValidPathUncached(const StorePath & path) override;
PathSet queryValidPaths(const PathSet & paths,
StorePathSet queryValidPaths(const StorePathSet & paths,
SubstituteFlag maybeSubstitute = NoSubstitute) override;
PathSet queryAllValidPaths() override;
StorePathSet queryAllValidPaths() override;
void queryPathInfoUncached(const Path & path,
void queryPathInfoUncached(const StorePath & path,
Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override;
void queryReferrers(const Path & path, PathSet & referrers) override;
void queryReferrers(const StorePath & path, StorePathSet & referrers) override;
PathSet queryValidDerivers(const Path & path) override;
StorePathSet queryValidDerivers(const StorePath & path) override;
PathSet queryDerivationOutputs(const Path & path) override;
StorePathSet queryDerivationOutputs(const StorePath & path) override;
StringSet queryDerivationOutputNames(const Path & path) override;
StringSet queryDerivationOutputNames(const StorePath & path) override;
Path queryPathFromHashPart(const string & hashPart) override;
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
PathSet querySubstitutablePaths(const PathSet & paths) override;
StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
void querySubstitutablePathInfos(const PathSet & paths,
void querySubstitutablePathInfos(const StorePathSet & paths,
SubstitutablePathInfos & infos) override;
void addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override;
Path addToStore(const string & name, const Path & srcPath,
StorePath addToStore(const string & name, const Path & srcPath,
bool recursive, HashType hashAlgo,
PathFilter & filter, RepairFlag repair) override;
@ -156,20 +156,22 @@ public:
in `dump', which is either a NAR serialisation (if recursive ==
true) or simply the contents of a regular file (if recursive ==
false). */
Path addToStoreFromDump(const string & dump, const string & name,
StorePath addToStoreFromDump(const string & dump, const string & name,
bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
Path addTextToStore(const string & name, const string & s,
const PathSet & references, RepairFlag repair) override;
StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override;
void buildPaths(const PathSet & paths, BuildMode buildMode) override;
BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
void buildPaths(
const std::vector<StorePathWithOutputs> & paths,
BuildMode buildMode) override;
void ensurePath(const Path & path) override;
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode) override;
void addTempRoot(const Path & path) override;
void ensurePath(const StorePath & path) override;
void addTempRoot(const StorePath & path) override;
void addIndirectRoot(const Path & path) override;
@ -215,9 +217,9 @@ public:
/* Repair the contents of the given path by redownloading it using
a substituter (if available). */
void repairPath(const Path & path);
void repairPath(const StorePath & path);
void addSignatures(const Path & storePath, const StringSet & sigs) override;
void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
/* If free disk space in /nix/store if below minFree, delete
garbage until it exceeds maxFree. */
@ -231,17 +233,17 @@ private:
void makeStoreWritable();
uint64_t queryValidPathId(State & state, const Path & path);
uint64_t queryValidPathId(State & state, const StorePath & path);
uint64_t addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs = true);
void invalidatePath(State & state, const Path & path);
void invalidatePath(State & state, const StorePath & path);
/* Delete a path from the Nix store. */
void invalidatePathChecked(const Path & path);
void invalidatePathChecked(const StorePath & path);
void verifyPath(const Path & path, const PathSet & store,
PathSet & done, PathSet & validPaths, RepairFlag repair, bool & errors);
void verifyPath(const Path & path, const StringSet & store,
PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors);
void updatePathInfo(State & state, const ValidPathInfo & info);
@ -256,7 +258,7 @@ private:
void tryToDelete(GCState & state, const Path & path);
bool canReachRoot(GCState & state, PathSet & visited, const Path & path);
bool canReachRoot(GCState & state, StorePathSet & visited, const StorePath & path);
void deletePathRecursive(GCState & state, const Path & path);
@ -275,7 +277,7 @@ private:
Path createTempDirInStore();
void checkDerivationOutputs(const Path & drvPath, const Derivation & drv);
void checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv);
typedef std::unordered_set<ino_t> InodeHash;
@ -284,8 +286,8 @@ private:
void optimisePath_(Activity * act, OptimiseStats & stats, const Path & path, InodeHash & inodeHash);
// Internal versions that are not wrapped in retry_sqlite.
bool isValidPath_(State & state, const Path & path);
void queryReferrers(State & state, const Path & path, PathSet & referrers);
bool isValidPath_(State & state, const StorePath & path);
void queryReferrers(State & state, const StorePath & path, StorePathSet & referrers);
/* Add signatures to a ValidPathInfo using the secret keys
specified by the ‘secret-key-files’ option. */

2
src/libstore/local.mk
View file

@ -6,7 +6,7 @@ libstore_DIR := $(d)
libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc)
libstore_LIBS = libutil
libstore_LIBS = libutil libnixrust
libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread
ifneq ($(OS), FreeBSD)

144
src/libstore/misc.cc
View file

@ -9,13 +9,13 @@
namespace nix {
void Store::computeFSClosure(const PathSet & startPaths,
PathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers)
void Store::computeFSClosure(const StorePathSet & startPaths,
StorePathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers)
{
struct State
{
size_t pending;
PathSet & paths;
StorePathSet & paths;
std::exception_ptr exc;
};
@ -29,45 +29,47 @@ void Store::computeFSClosure(const PathSet & startPaths,
{
auto state(state_.lock());
if (state->exc) return;
if (!state->paths.insert(path).second) return;
if (!state->paths.insert(parseStorePath(path)).second) return;
state->pending++;
}
queryPathInfo(path, {[&, path](std::future<ref<const ValidPathInfo>> fut) {
queryPathInfo(parseStorePath(path), {[&, pathS(path)](std::future<ref<const ValidPathInfo>> fut) {
// FIXME: calls to isValidPath() should be async
try {
auto info = fut.get();
auto path = parseStorePath(pathS);
if (flipDirection) {
PathSet referrers;
StorePathSet referrers;
queryReferrers(path, referrers);
for (auto & ref : referrers)
if (ref != path)
enqueue(ref);
enqueue(printStorePath(ref));
if (includeOutputs)
for (auto & i : queryValidDerivers(path))
enqueue(i);
enqueue(printStorePath(i));
if (includeDerivers && isDerivation(path))
if (includeDerivers && path.isDerivation())
for (auto & i : queryDerivationOutputs(path))
if (isValidPath(i) && queryPathInfo(i)->deriver == path)
enqueue(i);
enqueue(printStorePath(i));
} else {
for (auto & ref : info->references)
if (ref != path)
enqueue(ref);
enqueue(printStorePath(ref));
if (includeOutputs && isDerivation(path))
if (includeOutputs && path.isDerivation())
for (auto & i : queryDerivationOutputs(path))
if (isValidPath(i)) enqueue(i);
if (isValidPath(i)) enqueue(printStorePath(i));
if (includeDerivers && isValidPath(info->deriver))
enqueue(info->deriver);
if (includeDerivers && info->deriver && isValidPath(*info->deriver))
enqueue(printStorePath(*info->deriver));
}
@ -87,7 +89,7 @@ void Store::computeFSClosure(const PathSet & startPaths,
};
for (auto & startPath : startPaths)
enqueue(startPath);
enqueue(printStorePath(startPath));
{
auto state(state_.lock());
@ -97,15 +99,17 @@ void Store::computeFSClosure(const PathSet & startPaths,
}
void Store::computeFSClosure(const Path & startPath,
PathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers)
void Store::computeFSClosure(const StorePath & startPath,
StorePathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers)
{
computeFSClosure(PathSet{startPath}, paths_, flipDirection, includeOutputs, includeDerivers);
StorePathSet paths;
paths.insert(startPath.clone());
computeFSClosure(paths, paths_, flipDirection, includeOutputs, includeDerivers);
}
void Store::queryMissing(const PathSet & targets,
PathSet & willBuild_, PathSet & willSubstitute_, PathSet & unknown_,
void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_,
unsigned long long & downloadSize_, unsigned long long & narSize_)
{
Activity act(*logger, lvlDebug, actUnknown, "querying info about missing paths");
@ -116,8 +120,8 @@ void Store::queryMissing(const PathSet & targets,
struct State
{
PathSet done;
PathSet & unknown, & willSubstitute, & willBuild;
std::unordered_set<std::string> done;
StorePathSet & unknown, & willSubstitute, & willBuild;
unsigned long long & downloadSize;
unsigned long long & narSize;
};
@ -126,31 +130,36 @@ void Store::queryMissing(const PathSet & targets,
{
size_t left;
bool done = false;
PathSet outPaths;
StorePathSet outPaths;
DrvState(size_t left) : left(left) { }
};
Sync<State> state_(State{PathSet(), unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_});
Sync<State> state_(State{{}, unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_});
std::function<void(Path)> doPath;
std::function<void(StorePathWithOutputs)> doPath;
auto mustBuildDrv = [&](const Path & drvPath, const Derivation & drv) {
auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) {
{
auto state(state_.lock());
state->willBuild.insert(drvPath);
state->willBuild.insert(drvPath.clone());
}
for (auto & i : drv.inputDrvs)
pool.enqueue(std::bind(doPath, makeDrvPathWithOutputs(i.first, i.second)));
pool.enqueue(std::bind(doPath, StorePathWithOutputs(i.first, i.second)));
};
auto checkOutput = [&](
const Path & drvPath, ref<Derivation> drv, const Path & outPath, ref<Sync<DrvState>> drvState_)
const Path & drvPathS, ref<Derivation> drv, const Path & outPathS, ref<Sync<DrvState>> drvState_)
{
if (drvState_->lock()->done) return;
auto drvPath = parseStorePath(drvPathS);
auto outPath = parseStorePath(outPathS);
SubstitutablePathInfos infos;
querySubstitutablePathInfos({outPath}, infos);
StorePathSet paths; // FIXME
paths.insert(outPath.clone());
querySubstitutablePathInfos(paths, infos);
if (infos.empty()) {
drvState_->lock()->done = true;
@ -161,74 +170,74 @@ void Store::queryMissing(const PathSet & targets,
if (drvState->done) return;
assert(drvState->left);
drvState->left--;
drvState->outPaths.insert(outPath);
drvState->outPaths.insert(outPath.clone());
if (!drvState->left) {
for (auto & path : drvState->outPaths)
pool.enqueue(std::bind(doPath, path));
pool.enqueue(std::bind(doPath, StorePathWithOutputs(path.clone())));
}
}
}
};
doPath = [&](const Path & path) {
doPath = [&](const StorePathWithOutputs & path) {
{
auto state(state_.lock());
if (!state->done.insert(path).second) return;
if (!state->done.insert(path.to_string(*this)).second) return;
}
DrvPathWithOutputs i2 = parseDrvPathWithOutputs(path);
if (isDerivation(i2.first)) {
if (!isValidPath(i2.first)) {
if (path.path.isDerivation()) {
if (!isValidPath(path.path)) {
// FIXME: we could try to substitute the derivation.
auto state(state_.lock());
state->unknown.insert(path);
state->unknown.insert(path.path.clone());
return;
}
Derivation drv = derivationFromPath(i2.first);
ParsedDerivation parsedDrv(i2.first, drv);
auto drv = make_ref<Derivation>(derivationFromPath(path.path));
ParsedDerivation parsedDrv(path.path.clone(), *drv);
PathSet invalid;
for (auto & j : drv.outputs)
if (wantOutput(j.first, i2.second)
for (auto & j : drv->outputs)
if (wantOutput(j.first, path.outputs)
&& !isValidPath(j.second.path))
invalid.insert(j.second.path);
invalid.insert(printStorePath(j.second.path));
if (invalid.empty()) return;
if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
for (auto & output : invalid)
pool.enqueue(std::bind(checkOutput, i2.first, make_ref<Derivation>(drv), output, drvState));
pool.enqueue(std::bind(checkOutput, printStorePath(path.path), drv, output, drvState));
} else
mustBuildDrv(i2.first, drv);
mustBuildDrv(path.path, *drv);
} else {
if (isValidPath(path)) return;
if (isValidPath(path.path)) return;
SubstitutablePathInfos infos;
querySubstitutablePathInfos({path}, infos);
StorePathSet paths; // FIXME
paths.insert(path.path.clone());
querySubstitutablePathInfos(paths, infos);
if (infos.empty()) {
auto state(state_.lock());
state->unknown.insert(path);
state->unknown.insert(path.path.clone());
return;
}
auto info = infos.find(path);
auto info = infos.find(path.path);
assert(info != infos.end());
{
auto state(state_.lock());
state->willSubstitute.insert(path);
state->willSubstitute.insert(path.path.clone());
state->downloadSize += info->second.downloadSize;
state->narSize += info->second.narSize;
}
for (auto & ref : info->second.references)
pool.enqueue(std::bind(doPath, ref));
pool.enqueue(std::bind(doPath, StorePathWithOutputs(ref)));
}
};
@ -239,39 +248,42 @@ void Store::queryMissing(const PathSet & targets,
}
Paths Store::topoSortPaths(const PathSet & paths)
StorePaths Store::topoSortPaths(const StorePathSet & paths)
{
Paths sorted;
PathSet visited, parents;
StorePaths sorted;
StorePathSet visited, parents;
std::function<void(const Path & path, const Path * parent)> dfsVisit;
std::function<void(const StorePath & path, const StorePath * parent)> dfsVisit;
dfsVisit = [&](const Path & path, const Path * parent) {
if (parents.find(path) != parents.end())
throw BuildError(format("cycle detected in the references of '%1%' from '%2%'") % path % *parent);
dfsVisit = [&](const StorePath & path, const StorePath * parent) {
if (parents.count(path))
throw BuildError("cycle detected in the references of '%s' from '%s'",
printStorePath(path), printStorePath(*parent));
if (!visited.insert(path).second) return;
parents.insert(path);
if (!visited.insert(path.clone()).second) return;
parents.insert(path.clone());
PathSet references;
StorePathSet references;
try {
references = queryPathInfo(path)->references;
references = cloneStorePathSet(queryPathInfo(path)->references);
} catch (InvalidPath &) {
}
for (auto & i : references)
/* Don't traverse into paths that don't exist. That can
happen due to substitutes for non-existent paths. */
if (i != path && paths.find(i) != paths.end())
if (i != path && paths.count(i))
dfsVisit(i, &path);
sorted.push_front(path);
sorted.push_back(path.clone());
parents.erase(path);
};
for (auto & i : paths)
dfsVisit(i, nullptr);
std::reverse(sorted.begin(), sorted.end());
return sorted;
}

15
src/libstore/nar-info-disk-cache.cc
View file

@ -188,11 +188,8 @@ public:
if (!queryNAR.getInt(0))
return {oInvalid, 0};
auto narInfo = make_ref<NarInfo>();
auto namePart = queryNAR.getStr(1);
narInfo->path = cache.storeDir + "/" +
hashPart + (namePart.empty() ? "" : "-" + namePart);
auto narInfo = make_ref<NarInfo>(StorePath::fromBaseName(hashPart + "-" + namePart));
narInfo->url = queryNAR.getStr(2);
narInfo->compression = queryNAR.getStr(3);
if (!queryNAR.isNull(4))
@ -201,9 +198,9 @@ public:
narInfo->narHash = Hash(queryNAR.getStr(6));
narInfo->narSize = queryNAR.getInt(7);
for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " "))
narInfo->references.insert(cache.storeDir + "/" + r);
narInfo->references.insert(StorePath::fromBaseName(r));
if (!queryNAR.isNull(9))
narInfo->deriver = cache.storeDir + "/" + queryNAR.getStr(9);
narInfo->deriver = StorePath::fromBaseName(queryNAR.getStr(9));
for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " "))
narInfo->sigs.insert(sig);
narInfo->ca = queryNAR.getStr(11);
@ -225,12 +222,12 @@ public:
auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info);
assert(hashPart == storePathToHash(info->path));
//assert(hashPart == storePathToHash(info->path));
state->insertNAR.use()
(cache.id)
(hashPart)
(storePathToName(info->path))
(std::string(info->path.name()))
(narInfo ? narInfo->url : "", narInfo != 0)
(narInfo ? narInfo->compression : "", narInfo != 0)
(narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() : "", narInfo && narInfo->fileHash)
@ -238,7 +235,7 @@ public:
(info->narHash.to_string())
(info->narSize)
(concatStringsSep(" ", info->shortRefs()))
(info->deriver != "" ? baseNameOf(info->deriver) : "", info->deriver != "")
(info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver)
(concatStringsSep(" ", info->sigs))
(info->ca)
(time(0)).exec();

31
src/libstore/nar-info.cc
View file

@ -4,6 +4,7 @@
namespace nix {
NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence)
: ValidPathInfo(StorePath::make((unsigned char *) "xxxxxxxxxxxxxxxxxxxx", "x")) // FIXME: hack
{
auto corrupt = [&]() {
throw Error(format("NAR info file '%1%' is corrupt") % whence);
@ -18,6 +19,8 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
}
};
bool havePath = false;
size_t pos = 0;
while (pos < s.size()) {
@ -32,8 +35,8 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
std::string value(s, colon + 2, eol - colon - 2);
if (name == "StorePath") {
if (!store.isStorePath(value)) corrupt();
path = value;
path = store.parseStorePath(value);
havePath = true;
}
else if (name == "URL")
url = value;
@ -52,18 +55,12 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
else if (name == "References") {
auto refs = tokenizeString<Strings>(value, " ");
if (!references.empty()) corrupt();
for (auto & r : refs) {
auto r2 = store.storeDir + "/" + r;
if (!store.isStorePath(r2)) corrupt();
references.insert(r2);
}
for (auto & r : refs)
references.insert(StorePath::fromBaseName(r));
}
else if (name == "Deriver") {
if (value != "unknown-deriver") {
auto p = store.storeDir + "/" + value;
if (!store.isStorePath(p)) corrupt();
deriver = p;
}
if (value != "unknown-deriver")
deriver = StorePath::fromBaseName(value);
}
else if (name == "System")
system = value;
@ -79,13 +76,13 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
if (compression == "") compression = "bzip2";
if (path.empty() || url.empty() || narSize == 0 || !narHash) corrupt();
if (!havePath || url.empty() || narSize == 0 || !narHash) corrupt();
}
std::string NarInfo::to_string() const
std::string NarInfo::to_string(const Store & store) const
{
std::string res;
res += "StorePath: " + path + "\n";
res += "StorePath: " + store.printStorePath(path) + "\n";
res += "URL: " + url + "\n";
assert(compression != "");
res += "Compression: " + compression + "\n";
@ -98,8 +95,8 @@ std::string NarInfo::to_string() const
res += "References: " + concatStringsSep(" ", shortRefs()) + "\n";
if (!deriver.empty())
res += "Deriver: " + baseNameOf(deriver) + "\n";
if (deriver)
res += "Deriver: " + std::string(deriver->to_string()) + "\n";
if (!system.empty())
res += "System: " + system + "\n";

5
src/libstore/nar-info.hh
View file

@ -14,11 +14,12 @@ struct NarInfo : ValidPathInfo
uint64_t fileSize = 0;
std::string system;
NarInfo() { }
NarInfo() = delete;
NarInfo(StorePath && path) : ValidPathInfo(std::move(path)) { }
NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { }
NarInfo(const Store & store, const std::string & s, const std::string & whence);
std::string to_string() const;
std::string to_string(const Store & store) const;
};
}

6
src/libstore/optimise-store.cc
View file

@ -254,7 +254,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
{
Activity act(*logger, actOptimiseStore);
PathSet paths = queryAllValidPaths();
auto paths = queryAllValidPaths();
InodeHash inodeHash = loadInodeHash();
act.progress(0, paths.size());
@ -265,8 +265,8 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
addTempRoot(i);
if (!isValidPath(i)) continue; /* path was GC'ed, probably */
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("optimising path '%s'", i));
optimisePath_(&act, stats, realStoreDir + "/" + baseNameOf(i), inodeHash);
Activity act(*logger, lvlTalkative, actUnknown, fmt("optimising path '%s'", printStorePath(i)));
optimisePath_(&act, stats, realStoreDir + "/" + std::string(i.to_string()), inodeHash);
}
done++;
act.progress(done, paths.size());

14
src/libstore/parsed-derivations.cc
View file

@ -4,8 +4,8 @@
namespace nix {
ParsedDerivation::ParsedDerivation(const Path & drvPath, BasicDerivation & drv)
: drvPath(drvPath), drv(drv)
ParsedDerivation::ParsedDerivation(StorePath && drvPath, BasicDerivation & drv)
: drvPath(std::move(drvPath)), drv(drv)
{
/* Parse the __json attribute, if any. */
auto jsonAttr = drv.env.find("__json");
@ -13,7 +13,7 @@ ParsedDerivation::ParsedDerivation(const Path & drvPath, BasicDerivation & drv)
try {
structuredAttrs = std::make_unique<nlohmann::json>(nlohmann::json::parse(jsonAttr->second));
} catch (std::exception & e) {
throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what());
throw Error("cannot process __json attribute of '%s': %s", drvPath.to_string(), e.what());
}
}
}
@ -28,7 +28,7 @@ std::optional<std::string> ParsedDerivation::getStringAttr(const std::string & n
return {};
else {
if (!i->is_string())
throw Error("attribute '%s' of derivation '%s' must be a string", name, drvPath);
throw Error("attribute '%s' of derivation '%s' must be a string", name, drvPath.to_string());
return i->get<std::string>();
}
} else {
@ -48,7 +48,7 @@ bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const
return def;
else {
if (!i->is_boolean())
throw Error("attribute '%s' of derivation '%s' must be a Boolean", name, drvPath);
throw Error("attribute '%s' of derivation '%s' must be a Boolean", name, drvPath.to_string());
return i->get<bool>();
}
} else {
@ -68,11 +68,11 @@ std::optional<Strings> ParsedDerivation::getStringsAttr(const std::string & name
return {};
else {
if (!i->is_array())
throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath);
throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath.to_string());
Strings res;
for (auto j = i->begin(); j != i->end(); ++j) {
if (!j->is_string())
throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath);
throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath.to_string());
res.push_back(j->get<std::string>());
}
return res;

4
src/libstore/parsed-derivations.hh
View file

@ -6,13 +6,13 @@ namespace nix {
class ParsedDerivation
{
Path drvPath;
StorePath drvPath;
BasicDerivation & drv;
std::unique_ptr<nlohmann::json> structuredAttrs;
public:
ParsedDerivation(const Path & drvPath, BasicDerivation & drv);
ParsedDerivation(StorePath && drvPath, BasicDerivation & drv);
~ParsedDerivation();

99
src/libstore/path.cc Normal file
View file

@ -0,0 +1,99 @@
#include "store-api.hh"
namespace nix {
extern "C" {
rust::Result<StorePath> ffi_StorePath_new(rust::StringSlice path, rust::StringSlice storeDir);
rust::Result<StorePath> ffi_StorePath_new2(unsigned char hash[20], rust::StringSlice storeDir);
rust::Result<StorePath> ffi_StorePath_fromBaseName(rust::StringSlice baseName);
rust::String ffi_StorePath_to_string(const StorePath & _this);
StorePath ffi_StorePath_clone(const StorePath & _this);
rust::StringSlice ffi_StorePath_name(const StorePath & _this);
}
StorePath StorePath::make(std::string_view path, std::string_view storeDir)
{
return ffi_StorePath_new((rust::StringSlice) path, (rust::StringSlice) storeDir).unwrap();
}
StorePath StorePath::make(unsigned char hash[20], std::string_view name)
{
return ffi_StorePath_new2(hash, (rust::StringSlice) name).unwrap();
}
StorePath StorePath::fromBaseName(std::string_view baseName)
{
return ffi_StorePath_fromBaseName((rust::StringSlice) baseName).unwrap();
}
rust::String StorePath::to_string() const
{
return ffi_StorePath_to_string(*this);
}
StorePath StorePath::clone() const
{
return ffi_StorePath_clone(*this);
}
bool StorePath::isDerivation() const
{
return hasSuffix(name(), drvExtension);
}
std::string_view StorePath::name() const
{
return ffi_StorePath_name(*this);
}
StorePath Store::parseStorePath(std::string_view path) const
{
return StorePath::make(path, storeDir);
}
StorePathSet Store::parseStorePathSet(const PathSet & paths) const
{
StorePathSet res;
for (auto & i : paths) res.insert(parseStorePath(i));
return res;
}
std::string Store::printStorePath(const StorePath & path) const
{
auto s = storeDir + "/";
s += (std::string_view) path.to_string();
return s;
}
PathSet Store::printStorePathSet(const StorePathSet & paths) const
{
PathSet res;
for (auto & i : paths) res.insert(printStorePath(i));
return res;
}
StorePathSet cloneStorePathSet(const StorePathSet & paths)
{
StorePathSet res;
for (auto & p : paths)
res.insert(p.clone());
return res;
}
StorePathSet storePathsToSet(const StorePaths & paths)
{
StorePathSet res;
for (auto & p : paths)
res.insert(p.clone());
return res;
}
StorePathSet singleton(const StorePath & path)
{
StorePathSet res;
res.insert(path.clone());
return res;
}
}

81
src/libstore/path.hh Normal file
View file

@ -0,0 +1,81 @@
#pragma once
#include "rust-ffi.hh"
namespace nix {
/* See path.rs. */
struct StorePath;
extern "C" {
void ffi_StorePath_drop(void *);
bool ffi_StorePath_less_than(const StorePath & a, const StorePath & b);
bool ffi_StorePath_eq(const StorePath & a, const StorePath & b);
unsigned char * ffi_StorePath_hash_data(const StorePath & p);
}
struct StorePath : rust::Value<3 * sizeof(void *) + 24, ffi_StorePath_drop>
{
static StorePath make(std::string_view path, std::string_view storeDir);
static StorePath make(unsigned char hash[20], std::string_view name);
static StorePath fromBaseName(std::string_view baseName);
rust::String to_string() const;
bool operator < (const StorePath & other) const
{
return ffi_StorePath_less_than(*this, other);
}
bool operator == (const StorePath & other) const
{
return ffi_StorePath_eq(*this, other);
}
bool operator != (const StorePath & other) const
{
return !(*this == other);
}
StorePath clone() const;
/* Check whether a file name ends with the extension for
derivations. */
bool isDerivation() const;
std::string_view name() const;
unsigned char * hashData() const
{
return ffi_StorePath_hash_data(*this);
}
};
typedef std::set<StorePath> StorePathSet;
typedef std::vector<StorePath> StorePaths;
StorePathSet cloneStorePathSet(const StorePathSet & paths);
StorePathSet storePathsToSet(const StorePaths & paths);
StorePathSet singleton(const StorePath & path);
/* Size of the hash part of store paths, in base-32 characters. */
const size_t storePathHashLen = 32; // i.e. 160 bits
/* Extension of derivations in the Nix store. */
const std::string drvExtension = ".drv";
}
namespace std {
template<> struct hash<nix::StorePath> {
std::size_t operator()(const nix::StorePath & path) const noexcept
{
return * (std::size_t *) path.hashData();
}
};
}

4
src/libstore/profiles.cc
View file

@ -40,7 +40,7 @@ Generations findGenerations(Path profile, int & curGen)
Generations gens;
Path profileDir = dirOf(profile);
string profileName = baseNameOf(profile);
auto profileName = std::string(baseNameOf(profile));
for (auto & i : readDirectory(profileDir)) {
int n;
@ -108,7 +108,7 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath)
user environment etc. we've just built. */
Path generation;
makeName(profile, num + 1, generation);
store->addPermRoot(outPath, generation, false, true);
store->addPermRoot(store->parseStorePath(outPath), generation, false, true);
return generation;
}

2
src/libstore/references.cc
View file

@ -89,7 +89,7 @@ PathSet scanForReferences(const string & path,
hash part of the file name. (This assumes that all references
have the form `HASH-bla'). */
for (auto & i : refs) {
string baseName = baseNameOf(i);
auto baseName = std::string(baseNameOf(i));
string::size_type pos = baseName.find('-');
if (pos == string::npos)
throw Error(format("bad reference '%1%'") % i);

4
src/libstore/remote-fs-accessor.cc
View file

@ -50,7 +50,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
auto storePath = store->toStorePath(path);
std::string restPath = std::string(path, storePath.size());
if (!store->isValidPath(storePath))
if (!store->isValidPath(store->parseStorePath(storePath)))
throw InvalidPath(format("path '%1%' is not a valid store path") % storePath);
auto i = nars.find(storePath);
@ -96,7 +96,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
} catch (SysError &) { }
}
store->narFromPath(storePath, sink);
store->narFromPath(store->parseStorePath(storePath), sink);
auto narAccessor = makeNarAccessor(sink.s);
addToCache(storePath, *sink.s, narAccessor);
return {narAccessor, restPath};

232
src/libstore/remote-store.cc
View file

@ -22,23 +22,22 @@
namespace nix {
Path readStorePath(Store & store, Source & from)
template<> StorePathSet readStorePaths(const Store & store, Source & from)
{
Path path = readString(from);
store.assertStorePath(path);
return path;
}
template<class T> T readStorePaths(Store & store, Source & from)
{
T paths = readStrings<T>(from);
for (auto & i : paths) store.assertStorePath(i);
StorePathSet paths;
for (auto & i : readStrings<Strings>(from))
paths.insert(store.parseStorePath(i));
return paths;
}
template PathSet readStorePaths(Store & store, Source & from);
template Paths readStorePaths(Store & store, Source & from);
void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths)
{
out << paths.size();
for (auto & i : paths)
out << store.printStorePath(i);
}
/* TODO: Separate these store impls into different files, give them better names */
RemoteStore::RemoteStore(const Params & params)
@ -254,60 +253,62 @@ ConnectionHandle RemoteStore::getConnection()
}
bool RemoteStore::isValidPathUncached(const Path & path)
bool RemoteStore::isValidPathUncached(const StorePath & path)
{
auto conn(getConnection());
conn->to << wopIsValidPath << path;
conn->to << wopIsValidPath << printStorePath(path);
conn.processStderr();
return readInt(conn->from);
}
PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute)
StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute)
{
auto conn(getConnection());
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
PathSet res;
StorePathSet res;
for (auto & i : paths)
if (isValidPath(i)) res.insert(i);
if (isValidPath(i)) res.insert(i.clone());
return res;
} else {
conn->to << wopQueryValidPaths << paths;
conn->to << wopQueryValidPaths;
writeStorePaths(*this, conn->to, paths);
conn.processStderr();
return readStorePaths<PathSet>(*this, conn->from);
return readStorePaths<StorePathSet>(*this, conn->from);
}
}
PathSet RemoteStore::queryAllValidPaths()
StorePathSet RemoteStore::queryAllValidPaths()
{
auto conn(getConnection());
conn->to << wopQueryAllValidPaths;
conn.processStderr();
return readStorePaths<PathSet>(*this, conn->from);
return readStorePaths<StorePathSet>(*this, conn->from);
}
PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths)
StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
{
auto conn(getConnection());
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
PathSet res;
StorePathSet res;
for (auto & i : paths) {
conn->to << wopHasSubstitutes << i;
conn->to << wopHasSubstitutes << printStorePath(i);
conn.processStderr();
if (readInt(conn->from)) res.insert(i);
if (readInt(conn->from)) res.insert(i.clone());
}
return res;
} else {
conn->to << wopQuerySubstitutablePaths << paths;
conn->to << wopQuerySubstitutablePaths;
writeStorePaths(*this, conn->to, paths);
conn.processStderr();
return readStorePaths<PathSet>(*this, conn->from);
return readStorePaths<StorePathSet>(*this, conn->from);
}
}
void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths,
SubstitutablePathInfos & infos)
{
if (paths.empty()) return;
@ -318,29 +319,31 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
for (auto & i : paths) {
SubstitutablePathInfo info;
conn->to << wopQuerySubstitutablePathInfo << i;
conn->to << wopQuerySubstitutablePathInfo << printStorePath(i);
conn.processStderr();
unsigned int reply = readInt(conn->from);
if (reply == 0) continue;
info.deriver = readString(conn->from);
if (info.deriver != "") assertStorePath(info.deriver);
info.references = readStorePaths<PathSet>(*this, conn->from);
auto deriver = readString(conn->from);
if (deriver != "")
info.deriver = parseStorePath(deriver);
info.references = readStorePaths<StorePathSet>(*this, conn->from);
info.downloadSize = readLongLong(conn->from);
info.narSize = readLongLong(conn->from);
infos[i] = info;
infos.insert_or_assign(i.clone(), std::move(info));
}
} else {
conn->to << wopQuerySubstitutablePathInfos << paths;
conn->to << wopQuerySubstitutablePathInfos;
writeStorePaths(*this, conn->to, paths);
conn.processStderr();
size_t count = readNum<size_t>(conn->from);
for (size_t n = 0; n < count; n++) {
Path path = readStorePath(*this, conn->from);
SubstitutablePathInfo & info(infos[path]);
info.deriver = readString(conn->from);
if (info.deriver != "") assertStorePath(info.deriver);
info.references = readStorePaths<PathSet>(*this, conn->from);
SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]);
auto deriver = readString(conn->from);
if (deriver != "")
info.deriver = parseStorePath(deriver);
info.references = readStorePaths<StorePathSet>(*this, conn->from);
info.downloadSize = readLongLong(conn->from);
info.narSize = readLongLong(conn->from);
}
@ -349,14 +352,14 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
}
void RemoteStore::queryPathInfoUncached(const Path & path,
void RemoteStore::queryPathInfoUncached(const StorePath & path,
Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
{
try {
std::shared_ptr<ValidPathInfo> info;
{
auto conn(getConnection());
conn->to << wopQueryPathInfo << path;
conn->to << wopQueryPathInfo << printStorePath(path);
try {
conn.processStderr();
} catch (Error & e) {
@ -367,14 +370,13 @@ void RemoteStore::queryPathInfoUncached(const Path & path,
}
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) {
bool valid; conn->from >> valid;
if (!valid) throw InvalidPath(format("path '%s' is not valid") % path);
if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path));
}
info = std::make_shared<ValidPathInfo>();
info->path = path;
info->deriver = readString(conn->from);
if (info->deriver != "") assertStorePath(info->deriver);
info = std::make_shared<ValidPathInfo>(path.clone());
auto deriver = readString(conn->from);
if (deriver != "") info->deriver = parseStorePath(deriver);
info->narHash = Hash(readString(conn->from), htSHA256);
info->references = readStorePaths<PathSet>(*this, conn->from);
info->references = readStorePaths<StorePathSet>(*this, conn->from);
conn->from >> info->registrationTime >> info->narSize;
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
conn->from >> info->ultimate;
@ -387,52 +389,52 @@ void RemoteStore::queryPathInfoUncached(const Path & path,
}
void RemoteStore::queryReferrers(const Path & path,
PathSet & referrers)
void RemoteStore::queryReferrers(const StorePath & path,
StorePathSet & referrers)
{
auto conn(getConnection());
conn->to << wopQueryReferrers << path;
conn->to << wopQueryReferrers << printStorePath(path);
conn.processStderr();
PathSet referrers2 = readStorePaths<PathSet>(*this, conn->from);
referrers.insert(referrers2.begin(), referrers2.end());
for (auto & i : readStorePaths<StorePathSet>(*this, conn->from))
referrers.insert(i.clone());
}
PathSet RemoteStore::queryValidDerivers(const Path & path)
StorePathSet RemoteStore::queryValidDerivers(const StorePath & path)
{
auto conn(getConnection());
conn->to << wopQueryValidDerivers << path;
conn->to << wopQueryValidDerivers << printStorePath(path);
conn.processStderr();
return readStorePaths<PathSet>(*this, conn->from);
return readStorePaths<StorePathSet>(*this, conn->from);
}
PathSet RemoteStore::queryDerivationOutputs(const Path & path)
StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
{
auto conn(getConnection());
conn->to << wopQueryDerivationOutputs << path;
conn->to << wopQueryDerivationOutputs << printStorePath(path);
conn.processStderr();
return readStorePaths<PathSet>(*this, conn->from);
return readStorePaths<StorePathSet>(*this, conn->from);
}
PathSet RemoteStore::queryDerivationOutputNames(const Path & path)
PathSet RemoteStore::queryDerivationOutputNames(const StorePath & path)
{
auto conn(getConnection());
conn->to << wopQueryDerivationOutputNames << path;
conn->to << wopQueryDerivationOutputNames << printStorePath(path);
conn.processStderr();
return readStrings<PathSet>(conn->from);
}
Path RemoteStore::queryPathFromHashPart(const string & hashPart)
std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & hashPart)
{
auto conn(getConnection());
conn->to << wopQueryPathFromHashPart << hashPart;
conn.processStderr();
Path path = readString(conn->from);
if (!path.empty()) assertStorePath(path);
return path;
if (path.empty()) return {};
return parseStorePath(path);
}
@ -450,9 +452,10 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
copyNAR(source, sink);
sink
<< exportMagic
<< info.path
<< info.references
<< info.deriver
<< printStorePath(info.path);
writeStorePaths(*this, sink, info.references);
sink
<< (info.deriver ? printStorePath(*info.deriver) : "")
<< 0 // == no legacy signature
<< 0 // == no path follows
;
@ -460,14 +463,17 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
conn.processStderr(0, source2.get());
auto importedPaths = readStorePaths<PathSet>(*this, conn->from);
auto importedPaths = readStorePaths<StorePathSet>(*this, conn->from);
assert(importedPaths.size() <= 1);
}
else {
conn->to << wopAddToStoreNar
<< info.path << info.deriver << info.narHash.to_string(Base16, false)
<< info.references << info.registrationTime << info.narSize
<< printStorePath(info.path)
<< (info.deriver ? printStorePath(*info.deriver) : "")
<< info.narHash.to_string(Base16, false);
writeStorePaths(*this, conn->to, info.references);
conn->to << info.registrationTime << info.narSize
<< info.ultimate << info.sigs << info.ca
<< repair << !checkSigs;
bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21;
@ -477,7 +483,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
}
Path RemoteStore::addToStore(const string & name, const Path & _srcPath,
StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath,
bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
{
if (repair) throw Error("repairing is not supported when building through the Nix daemon");
@ -511,54 +517,52 @@ Path RemoteStore::addToStore(const string & name, const Path & _srcPath,
throw;
}
return readStorePath(*this, conn->from);
return parseStorePath(readString(conn->from));
}
Path RemoteStore::addTextToStore(const string & name, const string & s,
const PathSet & references, RepairFlag repair)
StorePath RemoteStore::addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair)
{
if (repair) throw Error("repairing is not supported when building through the Nix daemon");
auto conn(getConnection());
conn->to << wopAddTextToStore << name << s << references;
conn->to << wopAddTextToStore << name << s;
writeStorePaths(*this, conn->to, references);
conn.processStderr();
return readStorePath(*this, conn->from);
return parseStorePath(readString(conn->from));
}
void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode)
void RemoteStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
{
auto conn(getConnection());
conn->to << wopBuildPaths;
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13) {
conn->to << drvPaths;
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15)
conn->to << buildMode;
else
/* Old daemons did not take a 'buildMode' parameter, so we
need to validate it here on the client side. */
if (buildMode != bmNormal)
throw Error("repairing or checking is not supported when building through the Nix daemon");
} else {
/* For backwards compatibility with old daemons, strip output
identifiers. */
PathSet drvPaths2;
for (auto & i : drvPaths)
drvPaths2.insert(string(i, 0, i.find('!')));
conn->to << drvPaths2;
}
assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13);
Strings ss;
for (auto & p : drvPaths)
ss.push_back(p.to_string(*this));
conn->to << ss;
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15)
conn->to << buildMode;
else
/* Old daemons did not take a 'buildMode' parameter, so we
need to validate it here on the client side. */
if (buildMode != bmNormal)
throw Error("repairing or checking is not supported when building through the Nix daemon");
conn.processStderr();
readInt(conn->from);
}
BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDerivation & drv,
BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode)
{
auto conn(getConnection());
conn->to << wopBuildDerivation << drvPath << drv << buildMode;
conn->to << wopBuildDerivation << printStorePath(drvPath);
writeDerivation(conn->to, *this, drv);
conn->to << buildMode;
conn.processStderr();
BuildResult res;
unsigned int status;
@ -568,19 +572,19 @@ BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDeriva
}
void RemoteStore::ensurePath(const Path & path)
void RemoteStore::ensurePath(const StorePath & path)
{
auto conn(getConnection());
conn->to << wopEnsurePath << path;
conn->to << wopEnsurePath << printStorePath(path);
conn.processStderr();
readInt(conn->from);
}
void RemoteStore::addTempRoot(const Path & path)
void RemoteStore::addTempRoot(const StorePath & path)
{
auto conn(getConnection());
conn->to << wopAddTempRoot << path;
conn->to << wopAddTempRoot << printStorePath(path);
conn.processStderr();
readInt(conn->from);
}
@ -613,8 +617,8 @@ Roots RemoteStore::findRoots(bool censor)
Roots result;
while (count--) {
Path link = readString(conn->from);
Path target = readStorePath(*this, conn->from);
result[target].emplace(link);
auto target = parseStorePath(readString(conn->from));
result[std::move(target)].emplace(link);
}
return result;
}
@ -625,7 +629,9 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
auto conn(getConnection());
conn->to
<< wopCollectGarbage << options.action << options.pathsToDelete << options.ignoreLiveness
<< wopCollectGarbage << options.action;
writeStorePaths(*this, conn->to, options.pathsToDelete);
conn->to << options.ignoreLiveness
<< options.maxFreed
/* removed options */
<< 0 << 0 << 0;
@ -661,17 +667,17 @@ bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair)
}
void RemoteStore::addSignatures(const Path & storePath, const StringSet & sigs)
void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & sigs)
{
auto conn(getConnection());
conn->to << wopAddSignatures << storePath << sigs;
conn->to << wopAddSignatures << printStorePath(storePath) << sigs;
conn.processStderr();
readInt(conn->from);
}
void RemoteStore::queryMissing(const PathSet & targets,
PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
void RemoteStore::queryMissing(const std::vector<StorePathWithOutputs> & targets,
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
unsigned long long & downloadSize, unsigned long long & narSize)
{
{
@ -680,11 +686,15 @@ void RemoteStore::queryMissing(const PathSet & targets,
// Don't hold the connection handle in the fallback case
// to prevent a deadlock.
goto fallback;
conn->to << wopQueryMissing << targets;
conn->to << wopQueryMissing;
Strings ss;
for (auto & p : targets)
ss.push_back(p.to_string(*this));
conn->to << ss;
conn.processStderr();
willBuild = readStorePaths<PathSet>(*this, conn->from);
willSubstitute = readStorePaths<PathSet>(*this, conn->from);
unknown = readStorePaths<PathSet>(*this, conn->from);
willBuild = readStorePaths<StorePathSet>(*this, conn->from);
willSubstitute = readStorePaths<StorePathSet>(*this, conn->from);
unknown = readStorePaths<StorePathSet>(*this, conn->from);
conn->from >> downloadSize >> narSize;
return;
}

42
src/libstore/remote-store.hh
View file

@ -35,50 +35,50 @@ public:
/* Implementations of abstract store API methods. */
bool isValidPathUncached(const Path & path) override;
bool isValidPathUncached(const StorePath & path) override;
PathSet queryValidPaths(const PathSet & paths,
StorePathSet queryValidPaths(const StorePathSet & paths,
SubstituteFlag maybeSubstitute = NoSubstitute) override;
PathSet queryAllValidPaths() override;
StorePathSet queryAllValidPaths() override;
void queryPathInfoUncached(const Path & path,
void queryPathInfoUncached(const StorePath & path,
Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override;
void queryReferrers(const Path & path, PathSet & referrers) override;
void queryReferrers(const StorePath & path, StorePathSet & referrers) override;
PathSet queryValidDerivers(const Path & path) override;
StorePathSet queryValidDerivers(const StorePath & path) override;
PathSet queryDerivationOutputs(const Path & path) override;
StorePathSet queryDerivationOutputs(const StorePath & path) override;
StringSet queryDerivationOutputNames(const Path & path) override;
StringSet queryDerivationOutputNames(const StorePath & path) override;
Path queryPathFromHashPart(const string & hashPart) override;
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
PathSet querySubstitutablePaths(const PathSet & paths) override;
StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
void querySubstitutablePathInfos(const PathSet & paths,
void querySubstitutablePathInfos(const StorePathSet & paths,
SubstitutablePathInfos & infos) override;
void addToStore(const ValidPathInfo & info, Source & nar,
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override;
Path addToStore(const string & name, const Path & srcPath,
StorePath addToStore(const string & name, const Path & srcPath,
bool recursive = true, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override;
Path addTextToStore(const string & name, const string & s,
const PathSet & references, RepairFlag repair) override;
StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override;
void buildPaths(const PathSet & paths, BuildMode buildMode) override;
void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override;
BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode) override;
void ensurePath(const Path & path) override;
void ensurePath(const StorePath & path) override;
void addTempRoot(const Path & path) override;
void addTempRoot(const StorePath & path) override;
void addIndirectRoot(const Path & path) override;
@ -92,10 +92,10 @@ public:
bool verifyStore(bool checkContents, RepairFlag repair) override;
void addSignatures(const Path & storePath, const StringSet & sigs) override;
void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
void queryMissing(const PathSet & targets,
PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
void queryMissing(const std::vector<StorePathWithOutputs> & targets,
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
unsigned long long & downloadSize, unsigned long long & narSize) override;
void connect() override;

8
src/libstore/s3-binary-cache-store.cc
View file

@ -222,7 +222,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
fetches the .narinfo file, rather than first checking for its
existence via a HEAD request. Since .narinfos are small, doing
a GET is unlikely to be slower than HEAD. */
bool isValidPathUncached(const Path & storePath) override
bool isValidPathUncached(const StorePath & storePath) override
{
try {
queryPathInfo(storePath);
@ -382,9 +382,9 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
}
PathSet queryAllValidPaths() override
StorePathSet queryAllValidPaths() override
{
PathSet paths;
StorePathSet paths;
std::string marker;
do {
@ -405,7 +405,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
for (auto object : contents) {
auto & key = object.GetKey();
if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue;
paths.insert(storeDir + "/" + key.substr(0, key.size() - 8));
paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-unknown"));
}
marker = res.GetNextMarker();

6
src/libstore/ssh-store.cc
View file

@ -38,7 +38,7 @@ public:
bool sameMachine() override
{ return false; }
void narFromPath(const Path & path, Sink & sink) override;
void narFromPath(const StorePath & path, Sink & sink) override;
ref<FSAccessor> getFSAccessor() override;
@ -66,10 +66,10 @@ private:
};
};
void SSHStore::narFromPath(const Path & path, Sink & sink)
void SSHStore::narFromPath(const StorePath & path, Sink & sink)
{
auto conn(connections->get());
conn->to << wopNarFromPath << path;
conn->to << wopNarFromPath << printStorePath(path);
conn->processStderr();
copyNAR(conn->from, sink);
}

312
src/libstore/store-api.cc
View file

@ -27,13 +27,6 @@ bool Store::isStorePath(const Path & path) const
}
void Store::assertStorePath(const Path & path) const
{
if (!isStorePath(path))
throw Error(format("path '%1%' is not in the Nix store") % path);
}
Path Store::toStorePath(const Path & path) const
{
if (!isInStore(path))
@ -60,17 +53,9 @@ Path Store::followLinksToStore(const Path & _path) const
}
Path Store::followLinksToStorePath(const Path & path) const
StorePath Store::followLinksToStorePath(const Path & path) const
{
return toStorePath(followLinksToStore(path));
}
string storePathToName(const Path & path)
{
auto base = baseNameOf(path);
assert(base.size() == storePathHashLen || (base.size() > storePathHashLen && base[storePathHashLen] == '-'));
return base.size() == storePathHashLen ? "" : string(base, storePathHashLen + 1);
return parseStorePath(toStorePath(followLinksToStore(path)));
}
@ -82,41 +67,6 @@ string storePathToHash(const Path & path)
}
void checkStoreName(const string & name)
{
string validChars = "+-._?=";
auto baseError = format("The path name '%2%' is invalid: %3%. "
"Path names are alphanumeric and can include the symbols %1% "
"and must not begin with a period. "
"Note: If '%2%' is a source file and you cannot rename it on "
"disk, 'builtins.path { name = ... }' can be used to give it an "
"alternative name.") % validChars % name;
if (name.empty())
throw Error(baseError % "it is an empty string");
/* Disallow names starting with a dot for possible security
reasons (e.g., "." and ".."). */
if (name[0] == '.')
throw Error(baseError % "it is illegal to start the name with a period");
/* Disallow names longer than 211 characters. ext4’s max is 256,
but we need extra space for the hash and .chroot extensions. */
if (name.length() > 211)
throw Error(baseError % "name must be less than 212 characters");
for (auto & i : name)
if (!((i >= 'A' && i <= 'Z') ||
(i >= 'a' && i <= 'z') ||
(i >= '0' && i <= '9') ||
validChars.find(i) != string::npos))
{
throw Error(baseError % (format("the '%1%' character is invalid") % i));
}
}
/* Store paths have the following form:
<store>/<h>-<name>
@ -188,43 +138,48 @@ void checkStoreName(const string & name)
*/
Path Store::makeStorePath(const string & type,
const Hash & hash, const string & name) const
StorePath Store::makeStorePath(const string & type,
const Hash & hash, std::string_view name) const
{
/* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
string s = type + ":" + hash.to_string(Base16) + ":" + storeDir + ":" + name;
checkStoreName(name);
return storeDir + "/"
+ compressHash(hashString(htSHA256, s), 20).to_string(Base32, false)
+ "-" + name;
string s = type + ":" + hash.to_string(Base16) + ":" + storeDir + ":" + std::string(name);
auto h = compressHash(hashString(htSHA256, s), 20);
return StorePath::make(h.hash, name);
}
Path Store::makeOutputPath(const string & id,
const Hash & hash, const string & name) const
StorePath Store::makeOutputPath(const string & id,
const Hash & hash, std::string_view name) const
{
return makeStorePath("output:" + id, hash,
name + (id == "out" ? "" : "-" + id));
std::string(name) + (id == "out" ? "" : "-" + id));
}
static std::string makeType(string && type, const PathSet & references)
static std::string makeType(
const Store & store,
string && type,
const StorePathSet & references,
bool hasSelfReference = false)
{
for (auto & i : references) {
type += ":";
type += i;
type += store.printStorePath(i);
}
if (hasSelfReference) type += ":self";
return std::move(type);
}
Path Store::makeFixedOutputPath(bool recursive,
const Hash & hash, const string & name, const PathSet & references) const
StorePath Store::makeFixedOutputPath(
bool recursive,
const Hash & hash,
std::string_view name,
const StorePathSet & references,
bool hasSelfReference) const
{
if (hash.type == htSHA256 && recursive) {
return makeStorePath(makeType("source", references), hash, name);
return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name);
} else {
assert(references.empty());
return makeStorePath("output:out", hashString(htSHA256,
@ -234,28 +189,27 @@ Path Store::makeFixedOutputPath(bool recursive,
}
Path Store::makeTextPath(const string & name, const Hash & hash,
const PathSet & references) const
StorePath Store::makeTextPath(std::string_view name, const Hash & hash,
const StorePathSet & references) const
{
assert(hash.type == htSHA256);
/* Stuff the references (if any) into the type. This is a bit
hacky, but we can't put them in `s' since that would be
ambiguous. */
return makeStorePath(makeType("text", references), hash, name);
return makeStorePath(makeType(*this, "text", references), hash, name);
}
std::pair<Path, Hash> Store::computeStorePathForPath(const string & name,
std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name,
const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) const
{
Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath);
Path dstPath = makeFixedOutputPath(recursive, h, name);
return std::pair<Path, Hash>(dstPath, h);
return std::make_pair(makeFixedOutputPath(recursive, h, name), h);
}
Path Store::computeStorePathForText(const string & name, const string & s,
const PathSet & references) const
StorePath Store::computeStorePathForText(const string & name, const string & s,
const StorePathSet & references) const
{
return makeTextPath(name, hashString(htSHA256, s), references);
}
@ -274,11 +228,9 @@ std::string Store::getUri()
}
bool Store::isValidPath(const Path & storePath)
bool Store::isValidPath(const StorePath & storePath)
{
assertStorePath(storePath);
auto hashPart = storePathToHash(storePath);
auto hashPart = storePathToHash(printStorePath(storePath));
{
auto state_(state.lock());
@ -312,7 +264,7 @@ bool Store::isValidPath(const Path & storePath)
/* Default implementation for stores that only implement
queryPathInfoUncached(). */
bool Store::isValidPathUncached(const Path & path)
bool Store::isValidPathUncached(const StorePath & path)
{
try {
queryPathInfo(path);
@ -323,7 +275,7 @@ bool Store::isValidPathUncached(const Path & path)
}
ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
ref<const ValidPathInfo> Store::queryPathInfo(const StorePath & storePath)
{
std::promise<ref<const ValidPathInfo>> promise;
@ -340,22 +292,20 @@ ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
}
void Store::queryPathInfo(const Path & storePath,
void Store::queryPathInfo(const StorePath & storePath,
Callback<ref<const ValidPathInfo>> callback) noexcept
{
std::string hashPart;
try {
assertStorePath(storePath);
hashPart = storePathToHash(storePath);
hashPart = storePathToHash(printStorePath(storePath));
{
auto res = state.lock()->pathInfoCache.get(hashPart);
if (res) {
stats.narInfoReadAverted++;
if (!*res)
throw InvalidPath(format("path '%s' is not valid") % storePath);
throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
return callback(ref<const ValidPathInfo>(*res));
}
}
@ -369,8 +319,8 @@ void Store::queryPathInfo(const Path & storePath,
state_->pathInfoCache.upsert(hashPart,
res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
if (res.first == NarInfoDiskCache::oInvalid ||
(res.second->path != storePath && storePathToName(storePath) != ""))
throw InvalidPath(format("path '%s' is not valid") % storePath);
res.second->path != storePath)
throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
}
return callback(ref<const ValidPathInfo>(res.second));
}
@ -381,7 +331,7 @@ void Store::queryPathInfo(const Path & storePath,
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
queryPathInfoUncached(storePath,
{[this, storePath, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
{[this, storePath{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
try {
auto info = fut.get();
@ -394,9 +344,7 @@ void Store::queryPathInfo(const Path & storePath,
state_->pathInfoCache.upsert(hashPart, info);
}
if (!info
|| (info->path != storePath && storePathToName(storePath) != ""))
{
if (!info || info->path != parseStorePath(storePath)) {
stats.narInfoMissing++;
throw InvalidPath("path '%s' is not valid", storePath);
}
@ -407,27 +355,27 @@ void Store::queryPathInfo(const Path & storePath,
}
PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute)
StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute)
{
struct State
{
size_t left;
PathSet valid;
StorePathSet valid;
std::exception_ptr exc;
};
Sync<State> state_(State{paths.size(), PathSet()});
Sync<State> state_(State{paths.size(), StorePathSet()});
std::condition_variable wakeup;
ThreadPool pool;
auto doQuery = [&](const Path & path ) {
auto doQuery = [&](const Path & path) {
checkInterrupt();
queryPathInfo(path, {[path, &state_, &wakeup](std::future<ref<const ValidPathInfo>> fut) {
queryPathInfo(parseStorePath(path), {[path, this, &state_, &wakeup](std::future<ref<const ValidPathInfo>> fut) {
auto state(state_.lock());
try {
auto info = fut.get();
state->valid.insert(path);
state->valid.insert(parseStorePath(path));
} catch (InvalidPath &) {
} catch (...) {
state->exc = std::current_exception();
@ -439,7 +387,7 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti
};
for (auto & path : paths)
pool.enqueue(std::bind(doQuery, path));
pool.enqueue(std::bind(doQuery, printStorePath(path))); // FIXME
pool.process();
@ -447,7 +395,7 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti
auto state(state_.lock());
if (!state->left) {
if (state->exc) std::rethrow_exception(state->exc);
return state->valid;
return std::move(state->valid);
}
state.wait(wakeup);
}
@ -457,13 +405,13 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti
/* Return a string accepted by decodeValidPathInfo() that
registers the specified paths as valid. Note: it's the
responsibility of the caller to provide a closure. */
string Store::makeValidityRegistration(const PathSet & paths,
string Store::makeValidityRegistration(const StorePathSet & paths,
bool showDerivers, bool showHash)
{
string s = "";
for (auto & i : paths) {
s += i + "\n";
s += printStorePath(i) + "\n";
auto info = queryPathInfo(i);
@ -472,31 +420,30 @@ string Store::makeValidityRegistration(const PathSet & paths,
s += (format("%1%\n") % info->narSize).str();
}
Path deriver = showDerivers ? info->deriver : "";
auto deriver = showDerivers && info->deriver ? printStorePath(*info->deriver) : "";
s += deriver + "\n";
s += (format("%1%\n") % info->references.size()).str();
for (auto & j : info->references)
s += j + "\n";
s += printStorePath(j) + "\n";
}
return s;
}
void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths,
void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths,
bool includeImpureInfo, bool showClosureSize, AllowInvalidFlag allowInvalid)
{
auto jsonList = jsonOut.list();
for (auto storePath : storePaths) {
for (auto & storePath : storePaths) {
auto jsonPath = jsonList.object();
jsonPath.attr("path", storePath);
jsonPath.attr("path", printStorePath(storePath));
try {
auto info = queryPathInfo(storePath);
storePath = info->path;
jsonPath
.attr("narHash", info->narHash.to_string())
@ -505,7 +452,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths
{
auto jsonRefs = jsonPath.list("references");
for (auto & ref : info->references)
jsonRefs.elem(ref);
jsonRefs.elem(printStorePath(ref));
}
if (info->ca != "")
@ -514,14 +461,14 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths
std::pair<uint64_t, uint64_t> closureSizes;
if (showClosureSize) {
closureSizes = getClosureSize(storePath);
closureSizes = getClosureSize(info->path);
jsonPath.attr("closureSize", closureSizes.first);
}
if (includeImpureInfo) {
if (info->deriver != "")
jsonPath.attr("deriver", info->deriver);
if (info->deriver)
jsonPath.attr("deriver", printStorePath(*info->deriver));
if (info->registrationTime)
jsonPath.attr("registrationTime", info->registrationTime);
@ -557,10 +504,10 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths
}
std::pair<uint64_t, uint64_t> Store::getClosureSize(const Path & storePath)
std::pair<uint64_t, uint64_t> Store::getClosureSize(const StorePath & storePath)
{
uint64_t totalNarSize = 0, totalDownloadSize = 0;
PathSet closure;
StorePathSet closure;
computeFSClosure(storePath, closure, false, false);
for (auto & p : closure) {
auto info = queryPathInfo(p);
@ -584,30 +531,34 @@ const Store::Stats & Store::getStats()
}
void Store::buildPaths(const PathSet & paths, BuildMode buildMode)
void Store::buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode)
{
for (auto & path : paths)
if (isDerivation(path))
unsupported("buildPaths");
StorePathSet paths2;
if (queryValidPaths(paths).size() != paths.size())
for (auto & path : paths) {
if (path.path.isDerivation())
unsupported("buildPaths");
paths2.insert(path.path.clone());
}
if (queryValidPaths(paths2).size() != paths2.size())
unsupported("buildPaths");
}
void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
const Path & storePath, RepairFlag repair, CheckSigsFlag checkSigs)
const StorePath & storePath, RepairFlag repair, CheckSigsFlag checkSigs)
{
auto srcUri = srcStore->getUri();
auto dstUri = dstStore->getUri();
Activity act(*logger, lvlInfo, actCopyPath,
srcUri == "local" || srcUri == "daemon"
? fmt("copying path '%s' to '%s'", storePath, dstUri)
? fmt("copying path '%s' to '%s'", srcStore->printStorePath(storePath), dstUri)
: dstUri == "local" || dstUri == "daemon"
? fmt("copying path '%s' from '%s'", storePath, srcUri)
: fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri),
{storePath, srcUri, dstUri});
? fmt("copying path '%s' from '%s'", srcStore->printStorePath(storePath), srcUri)
: fmt("copying path '%s' from '%s' to '%s'", srcStore->printStorePath(storePath), srcUri, dstUri),
{srcStore->printStorePath(storePath), srcUri, dstUri});
PushActivity pact(act.id);
auto info = srcStore->queryPathInfo(storePath);
@ -640,23 +591,23 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
total += len;
act.progress(total, info->narSize);
});
srcStore->narFromPath({storePath}, wrapperSink);
srcStore->narFromPath(storePath, wrapperSink);
}, [&]() {
throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri());
throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", srcStore->printStorePath(storePath), srcStore->getUri());
});
dstStore->addToStore(*info, *source, repair, checkSigs);
}
void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePaths,
void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths,
RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
{
PathSet valid = dstStore->queryValidPaths(storePaths, substitute);
auto valid = dstStore->queryValidPaths(storePaths, substitute);
PathSet missing;
for (auto & path : storePaths)
if (!valid.count(path)) missing.insert(path);
if (!valid.count(path)) missing.insert(srcStore->printStorePath(path));
if (missing.empty()) return;
@ -677,23 +628,25 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
PathSet(missing.begin(), missing.end()),
[&](const Path & storePath) {
if (dstStore->isValidPath(storePath)) {
if (dstStore->isValidPath(dstStore->parseStorePath(storePath))) {
nrDone++;
showProgress();
return PathSet();
}
auto info = srcStore->queryPathInfo(storePath);
auto info = srcStore->queryPathInfo(srcStore->parseStorePath(storePath));
bytesExpected += info->narSize;
act.setExpected(actCopyPath, bytesExpected);
return info->references;
return srcStore->printStorePathSet(info->references);
},
[&](const Path & storePath) {
[&](const Path & storePathS) {
checkInterrupt();
auto storePath = dstStore->parseStorePath(storePathS);
if (!dstStore->isValidPath(storePath)) {
MaintainCount<decltype(nrRunning)> mc(nrRunning);
showProgress();
@ -703,7 +656,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
nrFailed++;
if (!settings.keepGoing)
throw e;
logger->log(lvlError, format("could not copy %s: %s") % storePath % e.what());
logger->log(lvlError, fmt("could not copy %s: %s", storePathS, e.what()));
showProgress();
return;
}
@ -716,20 +669,36 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
const PathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs,
const StorePathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs,
SubstituteFlag substitute)
{
PathSet closure;
srcStore->computeFSClosure({storePaths}, closure);
StorePathSet closure;
srcStore->computeFSClosure(storePaths, closure);
copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute);
}
ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven)
ValidPathInfo::ValidPathInfo(const ValidPathInfo & other)
: path(other.path.clone())
, deriver(other.deriver ? other.deriver->clone(): std::optional<StorePath>{})
, narHash(other.narHash)
, references(cloneStorePathSet(other.references))
, registrationTime(other.registrationTime)
, narSize(other.narSize)
, id(other.id)
, ultimate(other.ultimate)
, sigs(other.sigs)
, ca(other.ca)
{
ValidPathInfo info;
getline(str, info.path);
if (str.eof()) { info.path = ""; return info; }
}
std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istream & str, bool hashGiven)
{
std::string path;
getline(str, path);
if (str.eof()) { return {}; }
ValidPathInfo info(store.parseStorePath(path));
if (hashGiven) {
string s;
getline(str, s);
@ -737,16 +706,29 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven)
getline(str, s);
if (!string2Int(s, info.narSize)) throw Error("number expected");
}
getline(str, info.deriver);
std::string deriver;
getline(str, deriver);
if (deriver != "") info.deriver = store.parseStorePath(deriver);
string s; int n;
getline(str, s);
if (!string2Int(s, n)) throw Error("number expected");
while (n--) {
getline(str, s);
info.references.insert(s);
info.references.insert(store.parseStorePath(s));
}
if (!str || str.eof()) throw Error("missing input");
return info;
return std::optional<ValidPathInfo>(std::move(info));
}
std::string Store::showPaths(const StorePathSet & paths)
{
std::string s;
for (auto & i : paths) {
if (s.size() != 0) s += ", ";
s += "'" + printStorePath(i) + "'";
}
return s;
}
@ -756,34 +738,34 @@ string showPaths(const PathSet & paths)
}
std::string ValidPathInfo::fingerprint() const
std::string ValidPathInfo::fingerprint(const Store & store) const
{
if (narSize == 0 || !narHash)
throw Error(format("cannot calculate fingerprint of path '%s' because its size/hash is not known")
% path);
throw Error("cannot calculate fingerprint of path '%s' because its size/hash is not known",
store.printStorePath(path));
return
"1;" + path + ";"
"1;" + store.printStorePath(path) + ";"
+ narHash.to_string(Base32) + ";"
+ std::to_string(narSize) + ";"
+ concatStringsSep(",", references);
+ concatStringsSep(",", store.printStorePathSet(references));
}
void ValidPathInfo::sign(const SecretKey & secretKey)
void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey)
{
sigs.insert(secretKey.signDetached(fingerprint()));
sigs.insert(secretKey.signDetached(fingerprint(store)));
}
bool ValidPathInfo::isContentAddressed(const Store & store) const
{
auto warn = [&]() {
printError(format("warning: path '%s' claims to be content-addressed but isn't") % path);
printError("warning: path '%s' claims to be content-addressed but isn't", store.printStorePath(path));
};
if (hasPrefix(ca, "text:")) {
Hash hash(std::string(ca, 5));
if (store.makeTextPath(storePathToName(path), hash, references) == path)
if (store.makeTextPath(path.name(), hash, references) == path)
return true;
else
warn();
@ -792,9 +774,13 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const
else if (hasPrefix(ca, "fixed:")) {
bool recursive = ca.compare(6, 2, "r:") == 0;
Hash hash(std::string(ca, recursive ? 8 : 6));
auto refs = references;
replaceInSet(refs, path, std::string("self"));
if (store.makeFixedOutputPath(recursive, hash, storePathToName(path), refs) == path)
auto refs = cloneStorePathSet(references);
bool hasSelfReference = false;
if (refs.count(path)) {
hasSelfReference = true;
refs.erase(path);
}
if (store.makeFixedOutputPath(recursive, hash, path.name(), refs, hasSelfReference) == path)
return true;
else
warn();
@ -810,15 +796,15 @@ size_t ValidPathInfo::checkSignatures(const Store & store, const PublicKeys & pu
size_t good = 0;
for (auto & sig : sigs)
if (checkSignature(publicKeys, sig))
if (checkSignature(store, publicKeys, sig))
good++;
return good;
}
bool ValidPathInfo::checkSignature(const PublicKeys & publicKeys, const std::string & sig) const
bool ValidPathInfo::checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const
{
return verifyDetached(fingerprint(), sig, publicKeys);
return verifyDetached(fingerprint(store), sig, publicKeys);
}
@ -826,7 +812,7 @@ Strings ValidPathInfo::shortRefs() const
{
Strings refs;
for (auto & r : references)
refs.push_back(baseNameOf(r));
refs.push_back(std::string(r.to_string()));
return refs;
}
@ -939,7 +925,7 @@ static RegisterStoreImplementation regStore([](
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>
{
switch (getStoreType(uri, get(params, "state", settings.nixStateDir))) {
switch (getStoreType(uri, get(params, "state").value_or(settings.nixStateDir))) {
case tDaemon:
return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params));
case tLocal: {

203
src/libstore/store-api.hh
View file

@ -1,5 +1,6 @@
#pragma once
#include "path.hh"
#include "hash.hh"
#include "serialise.hh"
#include "crypto.hh"
@ -43,14 +44,11 @@ enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true };
enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true };
/* Size of the hash part of store paths, in base-32 characters. */
const size_t storePathHashLen = 32; // i.e. 160 bits
/* Magic header of exportPath() output (obsolete). */
const uint32_t exportMagic = 0x4558494e;
typedef std::unordered_map<Path, std::unordered_set<std::string>> Roots;
typedef std::unordered_map<StorePath, std::unordered_set<std::string>> Roots;
struct GCOptions
@ -84,7 +82,7 @@ struct GCOptions
bool ignoreLiveness{false};
/* For `gcDeleteSpecific', the paths to delete. */
PathSet pathsToDelete;
StorePathSet pathsToDelete;
/* Stop after at least `maxFreed' bytes have been freed. */
unsigned long long maxFreed{std::numeric_limits<unsigned long long>::max()};
@ -105,21 +103,21 @@ struct GCResults
struct SubstitutablePathInfo
{
Path deriver;
PathSet references;
std::optional<StorePath> deriver;
StorePathSet references;
unsigned long long downloadSize; /* 0 = unknown or inapplicable */
unsigned long long narSize; /* 0 = unknown */
};
typedef std::map<Path, SubstitutablePathInfo> SubstitutablePathInfos;
typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos;
struct ValidPathInfo
{
Path path;
Path deriver;
StorePath path;
std::optional<StorePath> deriver;
Hash narHash;
PathSet references;
StorePathSet references;
time_t registrationTime = 0;
uint64_t narSize = 0; // 0 = unknown
uint64_t id; // internal use only
@ -144,7 +142,7 @@ struct ValidPathInfo
Ideally, the content-addressability assertion would just be a
Boolean, and the store path would be computed from
‘storePathToName(path)’, ‘narHash’ and ‘references’. However,
the name component, ‘narHash’ and ‘references’. However,
1) we've accumulated several types of content-addressed paths
over the years; and 2) fixed-output derivations support
multiple hash algorithms and serialisation methods (flat file
@ -172,9 +170,9 @@ struct ValidPathInfo
the NAR, and the sorted references. The size field is strictly
speaking superfluous, but might prevent endless/excessive data
attacks. */
std::string fingerprint() const;
std::string fingerprint(const Store & store) const;
void sign(const SecretKey & secretKey);
void sign(const Store & store, const SecretKey & secretKey);
/* Return true iff the path is verifiably content-addressed. */
bool isContentAddressed(const Store & store) const;
@ -187,10 +185,13 @@ struct ValidPathInfo
size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const;
/* Verify a single signature. */
bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const;
bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const;
Strings shortRefs() const;
ValidPathInfo(StorePath && path) : path(std::move(path)) { }
explicit ValidPathInfo(const ValidPathInfo & other);
virtual ~ValidPathInfo() { }
};
@ -241,6 +242,23 @@ struct BuildResult
};
struct StorePathWithOutputs
{
StorePath path;
std::set<std::string> outputs;
StorePathWithOutputs(const StorePath & path, const std::set<std::string> & outputs = {})
: path(path.clone()), outputs(outputs)
{ }
StorePathWithOutputs(const StorePathWithOutputs & other)
: path(other.path.clone()), outputs(other.outputs)
{ }
std::string to_string(const Store & store) const;
};
class Store : public std::enable_shared_from_this<Store>, public Config
{
public:
@ -259,6 +277,7 @@ protected:
struct State
{
// FIXME: fix key
LRUCache<std::string, std::shared_ptr<const ValidPathInfo>> pathInfoCache;
};
@ -274,6 +293,24 @@ public:
virtual std::string getUri() = 0;
StorePath parseStorePath(std::string_view path) const;
std::string printStorePath(const StorePath & path) const;
// FIXME: remove
StorePathSet parseStorePathSet(const PathSet & paths) const;
PathSet printStorePathSet(const StorePathSet & path) const;
/* Split a string specifying a derivation and a set of outputs
(/nix/store/hash-foo!out1,out2,...) into the derivation path
and the outputs. */
StorePathWithOutputs parseDrvPathWithOutputs(const string & s);
/* Display a set of paths in human-readable form (i.e., between quotes
and separated by commas). */
std::string showPaths(const StorePathSet & paths);
/* Return true if ‘path’ is in the Nix store (but not the Nix
store itself). */
bool isInStore(const Path & path) const;
@ -282,9 +319,6 @@ public:
the Nix store. */
bool isStorePath(const Path & path) const;
/* Throw an exception if ‘path’ is not a store path. */
void assertStorePath(const Path & path) const;
/* Chop off the parts after the top-level store name, e.g.,
/nix/store/abcd-foo/bar => /nix/store/abcd-foo. */
Path toStorePath(const Path & path) const;
@ -294,26 +328,27 @@ public:
/* Same as followLinksToStore(), but apply toStorePath() to the
result. */
Path followLinksToStorePath(const Path & path) const;
StorePath followLinksToStorePath(const Path & path) const;
/* Constructs a unique store path name. */
Path makeStorePath(const string & type,
const Hash & hash, const string & name) const;
StorePath makeStorePath(const string & type,
const Hash & hash, std::string_view name) const;
Path makeOutputPath(const string & id,
const Hash & hash, const string & name) const;
StorePath makeOutputPath(const string & id,
const Hash & hash, std::string_view name) const;
Path makeFixedOutputPath(bool recursive,
const Hash & hash, const string & name,
const PathSet & references = {}) const;
StorePath makeFixedOutputPath(bool recursive,
const Hash & hash, std::string_view name,
const StorePathSet & references = {},
bool hasSelfReference = false) const;
Path makeTextPath(const string & name, const Hash & hash,
const PathSet & references) const;
StorePath makeTextPath(std::string_view name, const Hash & hash,
const StorePathSet & references) const;
/* This is the preparatory part of addToStore(); it computes the
store path to which srcPath is to be copied. Returns the store
path and the cryptographic hash of the contents of srcPath. */
std::pair<Path, Hash> computeStorePathForPath(const string & name,
std::pair<StorePath, Hash> computeStorePathForPath(std::string_view name,
const Path & srcPath, bool recursive = true,
HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const;
@ -331,21 +366,21 @@ public:
simply yield a different store path, so other users wouldn't be
affected), but it has some backwards compatibility issues (the
hashing scheme changes), so I'm not doing that for now. */
Path computeStorePathForText(const string & name, const string & s,
const PathSet & references) const;
StorePath computeStorePathForText(const string & name, const string & s,
const StorePathSet & references) const;
/* Check whether a path is valid. */
bool isValidPath(const Path & path);
bool isValidPath(const StorePath & path);
protected:
virtual bool isValidPathUncached(const Path & path);
virtual bool isValidPathUncached(const StorePath & path);
public:
/* Query which of the given paths is valid. Optionally, try to
substitute missing paths. */
virtual PathSet queryValidPaths(const PathSet & paths,
virtual StorePathSet queryValidPaths(const StorePathSet & paths,
SubstituteFlag maybeSubstitute = NoSubstitute);
/* Query the set of all valid paths. Note that for some store
@ -353,54 +388,54 @@ public:
(i.e. you'll get /nix/store/<hash> rather than
/nix/store/<hash>-<name>). Use queryPathInfo() to obtain the
full store path. */
virtual PathSet queryAllValidPaths()
virtual StorePathSet queryAllValidPaths()
{ unsupported("queryAllValidPaths"); }
/* Query information about a valid path. It is permitted to omit
the name part of the store path. */
ref<const ValidPathInfo> queryPathInfo(const Path & path);
ref<const ValidPathInfo> queryPathInfo(const StorePath & path);
/* Asynchronous version of queryPathInfo(). */
void queryPathInfo(const Path & path,
void queryPathInfo(const StorePath & path,
Callback<ref<const ValidPathInfo>> callback) noexcept;
protected:
virtual void queryPathInfoUncached(const Path & path,
virtual void queryPathInfoUncached(const StorePath & path,
Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept = 0;
public:
/* Queries the set of incoming FS references for a store path.
The result is not cleared. */
virtual void queryReferrers(const Path & path, PathSet & referrers)
virtual void queryReferrers(const StorePath & path, StorePathSet & referrers)
{ unsupported("queryReferrers"); }
/* Return all currently valid derivations that have `path' as an
output. (Note that the result of `queryDeriver()' is the
derivation that was actually used to produce `path', which may
not exist anymore.) */
virtual PathSet queryValidDerivers(const Path & path) { return {}; };
virtual StorePathSet queryValidDerivers(const StorePath & path) { return {}; };
/* Query the outputs of the derivation denoted by `path'. */
virtual PathSet queryDerivationOutputs(const Path & path)
virtual StorePathSet queryDerivationOutputs(const StorePath & path)
{ unsupported("queryDerivationOutputs"); }
/* Query the output names of the derivation denoted by `path'. */
virtual StringSet queryDerivationOutputNames(const Path & path)
virtual StringSet queryDerivationOutputNames(const StorePath & path)
{ unsupported("queryDerivationOutputNames"); }
/* Query the full store path given the hash part of a valid store
path, or "" if the path doesn't exist. */
virtual Path queryPathFromHashPart(const string & hashPart) = 0;
path, or empty if the path doesn't exist. */
virtual std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) = 0;
/* Query which of the given paths have substitutes. */
virtual PathSet querySubstitutablePaths(const PathSet & paths) { return {}; };
virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths) { return {}; };
/* Query substitute info (i.e. references, derivers and download
sizes) of a set of paths. If a path does not have substitute
info, it's omitted from the resulting ‘infos’ map. */
virtual void querySubstitutablePathInfos(const PathSet & paths,
virtual void querySubstitutablePathInfos(const StorePathSet & paths,
SubstitutablePathInfos & infos) { return; };
virtual bool wantMassQuery() { return false; }
@ -419,12 +454,12 @@ public:
validity the resulting path. The resulting path is returned.
The function object `filter' can be used to exclude files (see
libutil/archive.hh). */
virtual Path addToStore(const string & name, const Path & srcPath,
virtual StorePath addToStore(const string & name, const Path & srcPath,
bool recursive = true, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0;
// FIXME: remove?
virtual Path addToStoreFromDump(const string & dump, const string & name,
virtual StorePath addToStoreFromDump(const string & dump, const string & name,
bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)
{
throw Error("addToStoreFromDump() is not supported by this store");
@ -432,11 +467,11 @@ public:
/* Like addToStore, but the contents written to the output path is
a regular file containing the given string. */
virtual Path addTextToStore(const string & name, const string & s,
const PathSet & references, RepairFlag repair = NoRepair) = 0;
virtual StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair = NoRepair) = 0;
/* Write a NAR dump of a store path. */
virtual void narFromPath(const Path & path, Sink & sink) = 0;
virtual void narFromPath(const StorePath & path, Sink & sink) = 0;
/* For each path, if it's a derivation, build it. Building a
derivation means ensuring that the output paths are valid. If
@ -446,22 +481,24 @@ public:
output paths can be created by running the builder, after
recursively building any sub-derivations. For inputs that are
not derivations, substitute them. */
virtual void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal);
virtual void buildPaths(
const std::vector<StorePathWithOutputs> & paths,
BuildMode buildMode = bmNormal);
/* Build a single non-materialized derivation (i.e. not from an
on-disk .drv file). Note that ‘drvPath’ is only used for
informational purposes. */
virtual BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode = bmNormal) = 0;
/* Ensure that a path is valid. If it is not currently valid, it
may be made valid by running a substitute (if defined for the
path). */
virtual void ensurePath(const Path & path) = 0;
virtual void ensurePath(const StorePath & path) = 0;
/* Add a store path as a temporary root of the garbage collector.
The root disappears as soon as we exit. */
virtual void addTempRoot(const Path & path)
virtual void addTempRoot(const StorePath & path)
{ unsupported("addTempRoot"); }
/* Add an indirect root, which is merely a symlink to `path' from
@ -507,7 +544,7 @@ public:
/* Return a string representing information about the path that
can be loaded into the database using `nix-store --load-db' or
`nix-store --register-validity'. */
string makeValidityRegistration(const PathSet & paths,
string makeValidityRegistration(const StorePathSet & paths,
bool showDerivers, bool showHash);
/* Write a JSON representation of store path metadata, such as the
@ -515,14 +552,14 @@ public:
variable elements such as the registration time are
included. If ‘showClosureSize’ is true, the closure size of
each path is included. */
void pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths,
void pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths,
bool includeImpureInfo, bool showClosureSize,
AllowInvalidFlag allowInvalid = DisallowInvalid);
/* Return the size of the closure of the specified path, that is,
the sum of the size of the NAR serialisation of each path in
the closure. */
std::pair<uint64_t, uint64_t> getClosureSize(const Path & storePath);
std::pair<uint64_t, uint64_t> getClosureSize(const StorePath & storePath);
/* Optimise the disk space usage of the Nix store by hard-linking files
with the same contents. */
@ -538,14 +575,14 @@ public:
/* Add signatures to the specified store path. The signatures are
not verified. */
virtual void addSignatures(const Path & storePath, const StringSet & sigs)
virtual void addSignatures(const StorePath & storePath, const StringSet & sigs)
{ unsupported("addSignatures"); }
/* Utility functions. */
/* Read a derivation, after ensuring its existence through
ensurePath(). */
Derivation derivationFromPath(const Path & drvPath);
Derivation derivationFromPath(const StorePath & drvPath);
/* Place in `out' the set of all store paths in the file system
closure of `storePath'; that is, all paths than can be directly
@ -554,36 +591,36 @@ public:
`storePath' is returned; that is, the closures under the
`referrers' relation instead of the `references' relation is
returned. */
virtual void computeFSClosure(const PathSet & paths,
PathSet & out, bool flipDirection = false,
virtual void computeFSClosure(const StorePathSet & paths,
StorePathSet & out, bool flipDirection = false,
bool includeOutputs = false, bool includeDerivers = false);
void computeFSClosure(const Path & path,
PathSet & out, bool flipDirection = false,
void computeFSClosure(const StorePath & path,
StorePathSet & out, bool flipDirection = false,
bool includeOutputs = false, bool includeDerivers = false);
/* Given a set of paths that are to be built, return the set of
derivations that will be built, and the set of output paths
that will be substituted. */
virtual void queryMissing(const PathSet & targets,
PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
virtual void queryMissing(const std::vector<StorePathWithOutputs> & targets,
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
unsigned long long & downloadSize, unsigned long long & narSize);
/* Sort a set of paths topologically under the references
relation. If p refers to q, then p precedes q in this list. */
Paths topoSortPaths(const PathSet & paths);
StorePaths topoSortPaths(const StorePathSet & paths);
/* Export multiple paths in the format expected by ‘nix-store
--import’. */
void exportPaths(const Paths & paths, Sink & sink);
void exportPaths(const StorePathSet & paths, Sink & sink);
void exportPath(const Path & path, Sink & sink);
void exportPath(const StorePath & path, Sink & sink);
/* Import a sequence of NAR dumps created by exportPaths() into
the Nix store. Optionally, the contents of the NARs are
preloaded into the specified FS accessor to speed up subsequent
access. */
Paths importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
StorePaths importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
CheckSigsFlag checkSigs = CheckSigs);
struct Stats
@ -607,7 +644,7 @@ public:
/* Return the build log of the specified store path, if available,
or null otherwise. */
virtual std::shared_ptr<std::string> getBuildLog(const Path & path)
virtual std::shared_ptr<std::string> getBuildLog(const StorePath & path)
{ return nullptr; }
/* Hack to allow long-running processes like hydra-queue-runner to
@ -673,11 +710,11 @@ public:
LocalFSStore(const Params & params);
void narFromPath(const Path & path, Sink & sink) override;
void narFromPath(const StorePath & path, Sink & sink) override;
ref<FSAccessor> getFSAccessor() override;
/* Register a permanent GC root. */
Path addPermRoot(const Path & storePath,
Path addPermRoot(const StorePath & storePath,
const Path & gcRoot, bool indirect, bool allowOutsideRootsDir = false);
virtual Path getRealStoreDir() { return storeDir; }
@ -688,25 +725,17 @@ public:
return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1);
}
std::shared_ptr<std::string> getBuildLog(const Path & path) override;
std::shared_ptr<std::string> getBuildLog(const StorePath & path) override;
};
/* Extract the name part of the given store path. */
string storePathToName(const Path & path);
/* Extract the hash part of the given store path. */
string storePathToHash(const Path & path);
/* Check whether ‘name’ is a valid store path name part, i.e. contains
only the characters [a-zA-Z0-9\+\-\.\_\?\=] and doesn't start with
a dot. */
void checkStoreName(const string & name);
/* Copy a path from one store to another. */
void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
const Path & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs);
const StorePath & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs);
/* Copy store paths from one store to another. The paths may be copied
@ -714,7 +743,7 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
(i.e. if A is a reference of B, then A is copied before B), but
the set of store paths is not automatically closed; use
copyClosure() for that. */
void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePaths,
void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths,
RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs,
SubstituteFlag substitute = NoSubstitute);
@ -722,7 +751,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
/* Copy the closure of the specified paths from one store to another. */
void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
const PathSet & storePaths,
const StorePathSet & storePaths,
RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs,
SubstituteFlag substitute = NoSubstitute);
@ -805,7 +834,9 @@ struct RegisterStoreImplementation
string showPaths(const PathSet & paths);
ValidPathInfo decodeValidPathInfo(std::istream & str,
std::optional<ValidPathInfo> decodeValidPathInfo(
const Store & store,
std::istream & str,
bool hashGiven = false);

5
src/libstore/worker-protocol.hh
View file

@ -65,8 +65,9 @@ typedef enum {
class Store;
struct Source;
Path readStorePath(Store & store, Source & from);
template<class T> T readStorePaths(Store & store, Source & from);
template<class T> T readStorePaths(const Store & store, Source & from);
void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths);
}