wroclaw-git-backups/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-06-26 15:51:15 +02:00
Code Activity

Merge remote-tracking branch 'upstream/master' into path-info

Browse source
This commit is contained in:
John Ericson 2022-03-10 15:48:14 +00:00
commit 8ba089597f
434 changed files with 23391 additions and 36322 deletions
Download patch file Download diff file Expand all files Collapse all files

131
src/libstore/binary-cache-store.cc
View file

@ -31,7 +31,7 @@ BinaryCacheStore::BinaryCacheStore(const Params & params)
StringSink sink;
sink << narVersionMagic1;
narMagic = *sink.s;
narMagic = sink.s;
}
void BinaryCacheStore::init()
@ -68,7 +68,7 @@ void BinaryCacheStore::upsertFile(const std::string & path,
}
void BinaryCacheStore::getFile(const std::string & path,
Callback<std::shared_ptr<std::string>> callback) noexcept
Callback<std::optional<std::string>> callback) noexcept
{
try {
callback(getFile(path));
@ -77,9 +77,9 @@ void BinaryCacheStore::getFile(const std::string & path,
void BinaryCacheStore::getFile(const std::string & path, Sink & sink)
{
std::promise<std::shared_ptr<std::string>> promise;
std::promise<std::optional<std::string>> promise;
getFile(path,
{[&](std::future<std::shared_ptr<std::string>> result) {
{[&](std::future<std::optional<std::string>> result) {
try {
promise.set_value(result.get());
} catch (...) {
@ -89,15 +89,15 @@ void BinaryCacheStore::getFile(const std::string & path, Sink & sink)
sink(*promise.get_future().get());
}
std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
std::optional<std::string> BinaryCacheStore::getFile(const std::string & path)
{
StringSink sink;
try {
getFile(path, sink);
} catch (NoSuchBinaryCacheFile &) {
return nullptr;
return std::nullopt;
}
return sink.s;
return std::move(sink.s);
}
std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath)
@ -111,15 +111,15 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo");
std::string hashPart(narInfo->path.hashPart());
{
auto state_(state.lock());
state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = std::shared_ptr<NarInfo>(narInfo) });
state_->pathInfoCache.upsert(
std::string(narInfo->path.to_string()),
PathInfoCacheValue { .value = std::shared_ptr<NarInfo>(narInfo) });
}
if (diskCache)
diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
diskCache->upsertNarInfo(getUri(), std::string(narInfo->path.hashPart()), std::shared_ptr<NarInfo>(narInfo));
}
AutoCloseFD openFile(const Path & path)
@ -149,7 +149,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
{
FdSink fileSink(fdTemp.get());
TeeSink teeSinkCompressed { fileSink, fileHashSink };
auto compressionSink = makeCompressionSink(compression, teeSinkCompressed);
auto compressionSink = makeCompressionSink(compression, teeSinkCompressed, parallelCompression, compressionLevel);
TeeSink teeSinkUncompressed { *compressionSink, narHashSink };
TeeSource teeSource { narSource, teeSinkUncompressed };
narAccessor = makeNarAccessor(teeSource);
@ -306,8 +306,8 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
}});
}
StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view name,
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references)
{
if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256)
unsupported("addToStoreFromDump");
@ -315,13 +315,16 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & nam
ValidPathInfo info {
*this,
{
.name = name,
.name = std::string { name },
.info = FixedOutputInfo {
{
.method = method,
.hash = nar.first,
},
{},
{
.references = references,
.hasSelfReference = false,
},
},
},
nar.first,
@ -375,11 +378,11 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
getFile(narInfoFile,
{[=](std::future<std::shared_ptr<std::string>> fut) {
{[=](std::future<std::optional<std::string>> fut) {
try {
auto data = fut.get();
if (!data) return (*callbackPtr)(nullptr);
if (!data) return (*callbackPtr)({});
stats.narInfoRead++;
@ -393,8 +396,14 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
}});
}
StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
StorePath BinaryCacheStore::addToStore(
std::string_view name,
const Path & srcPath,
FileIngestionMethod method,
HashType hashAlgo,
PathFilter & filter,
RepairFlag repair,
const StorePathSet & references)
{
/* FIXME: Make BinaryCacheStore::addToStoreCommon support
non-recursive+sha256 so we can just use the default
@ -415,13 +424,16 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
ValidPathInfo info {
*this,
{
.name = name,
.name = std::string { name },
.info = FixedOutputInfo {
{
.method = method,
.hash = h,
},
{},
{
.references = references,
.hasSelfReference = false,
},
},
},
nar.first,
@ -431,8 +443,11 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
})->path;
}
StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair)
StorePath BinaryCacheStore::addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references,
RepairFlag repair)
{
auto textHash = hashString(htSHA256, s);
auto path = makeTextPath(name, TextInfo { textHash, references });
@ -442,12 +457,12 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
StringSink sink;
dumpString(s, sink);
auto source = StringSource { *sink.s };
StringSource source(sink.s);
return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info {
*this,
{
.name = name,
.name = std::string { name },
.info = TextInfo {
{ .hash = textHash },
references,
@ -456,45 +471,33 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
nar.first,
};
info.narSize = nar.second;
info.ca = TextHash { textHash };
return info;
})->path;
}
std::optional<const Realisation> BinaryCacheStore::queryRealisation(const DrvOutput & id)
void BinaryCacheStore::queryRealisationUncached(const DrvOutput & id,
Callback<std::shared_ptr<const Realisation>> callback) noexcept
{
if (diskCache) {
auto [cacheOutcome, maybeCachedRealisation] =
diskCache->lookupRealisation(getUri(), id);
switch (cacheOutcome) {
case NarInfoDiskCache::oValid:
debug("Returning a cached realisation for %s", id.to_string());
return *maybeCachedRealisation;
case NarInfoDiskCache::oInvalid:
debug("Returning a cached missing realisation for %s", id.to_string());
return {};
case NarInfoDiskCache::oUnknown:
break;
}
}
auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi";
auto rawOutputInfo = getFile(outputInfoFilePath);
if (rawOutputInfo) {
auto realisation = Realisation::fromJSON(
nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath);
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
if (diskCache)
diskCache->upsertRealisation(
getUri(), realisation);
Callback<std::optional<std::string>> newCallback = {
[=](std::future<std::optional<std::string>> fut) {
try {
auto data = fut.get();
if (!data) return (*callbackPtr)({});
return {realisation};
} else {
if (diskCache)
diskCache->upsertAbsentRealisation(getUri(), id);
return std::nullopt;
}
auto realisation = Realisation::fromJSON(
nlohmann::json::parse(*data), outputInfoFilePath);
return (*callbackPtr)(std::make_shared<const Realisation>(realisation));
} catch (...) {
callbackPtr->rethrow();
}
}
};
getFile(outputInfoFilePath, std::move(newCallback));
}
void BinaryCacheStore::registerDrvOutput(const Realisation& info) {
@ -523,7 +526,7 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe
writeNarInfo(narInfo);
}
std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & path)
std::optional<std::string> BinaryCacheStore::getBuildLog(const StorePath & path)
{
auto drvPath = path;
@ -531,10 +534,10 @@ std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & pat
try {
auto info = queryPathInfo(path);
// FIXME: add a "Log" field to .narinfo
if (!info->deriver) return nullptr;
if (!info->deriver) return std::nullopt;
drvPath = *info->deriver;
} catch (InvalidPath &) {
return nullptr;
return std::nullopt;
}
}
@ -545,4 +548,14 @@ std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & pat
return getFile(logPath);
}
void BinaryCacheStore::addBuildLog(const StorePath & drvPath, std::string_view log)
{
assert(drvPath.isDerivation());
upsertFile(
"log/" + std::string(drvPath.to_string()),
(std::string) log, // FIXME: don't copy
"text/plain; charset=utf-8");
}
}