wroclaw-git-backups/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-06-27 04:21:16 +02:00
Code Activity

Merge remote-tracking branch 'upstream/master' into overlayfs-store

Browse source
This commit is contained in:
John Ericson 2023-12-11 13:12:09 -05:00
commit 245af3ea02
538 changed files with 14282 additions and 7312 deletions
Download patch file Download diff file Expand all files Collapse all files

186
src/libstore/store-api.cc
View file

@ -1,6 +1,8 @@
#include "crypto.hh"
#include "fs-accessor.hh"
#include "source-accessor.hh"
#include "globals.hh"
#include "derived-path.hh"
#include "realisation.hh"
#include "derivations.hh"
#include "store-api.hh"
#include "util.hh"
@ -14,6 +16,8 @@
// FIXME this should not be here, see TODO below on
// `addMultipleToStore`.
#include "worker-protocol.hh"
#include "signals.hh"
#include "users.hh"
#include <nlohmann/json.hpp>
#include <regex>
@ -23,13 +27,13 @@ using json = nlohmann::json;
namespace nix {
bool Store::isInStore(PathView path) const
bool StoreDirConfig::isInStore(PathView path) const
{
return isInDir(path, storeDir);
}
std::pair<StorePath, Path> Store::toStorePath(PathView path) const
std::pair<StorePath, Path> StoreDirConfig::toStorePath(PathView path) const
{
if (!isInStore(path))
throw Error("path '%1%' is not in the Nix store", path);
@ -143,25 +147,25 @@ StorePath Store::followLinksToStorePath(std::string_view path) const
*/
StorePath Store::makeStorePath(std::string_view type,
StorePath StoreDirConfig::makeStorePath(std::string_view type,
std::string_view hash, std::string_view name) const
{
/* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
auto s = std::string(type) + ":" + std::string(hash)
+ ":" + storeDir + ":" + std::string(name);
auto h = compressHash(hashString(htSHA256, s), 20);
auto h = compressHash(hashString(HashAlgorithm::SHA256, s), 20);
return StorePath(h, name);
}
StorePath Store::makeStorePath(std::string_view type,
StorePath StoreDirConfig::makeStorePath(std::string_view type,
const Hash & hash, std::string_view name) const
{
return makeStorePath(type, hash.to_string(HashFormat::Base16, true), name);
}
StorePath Store::makeOutputPath(std::string_view id,
StorePath StoreDirConfig::makeOutputPath(std::string_view id,
const Hash & hash, std::string_view name) const
{
return makeStorePath("output:" + std::string { id }, hash, outputPathName(name, id));
@ -172,7 +176,7 @@ StorePath Store::makeOutputPath(std::string_view id,
hacky, but we can't put them in, say, <s2> (per the grammar above)
since that would be ambiguous. */
static std::string makeType(
const Store & store,
const StoreDirConfig & store,
std::string && type,
const StoreReferences & references)
{
@ -185,14 +189,14 @@ static std::string makeType(
}
StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const
StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const
{
if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) {
if (info.hash.algo == HashAlgorithm::SHA256 && info.method == FileIngestionMethod::Recursive) {
return makeStorePath(makeType(*this, "source", info.references), info.hash, name);
} else {
assert(info.references.size() == 0);
return makeStorePath("output:out",
hashString(htSHA256,
hashString(HashAlgorithm::SHA256,
"fixed:out:"
+ makeFileIngestionPrefix(info.method)
+ info.hash.to_string(HashFormat::Base16, true) + ":"),
@ -201,9 +205,9 @@ StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInf
}
StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const
StorePath StoreDirConfig::makeTextPath(std::string_view name, const TextInfo & info) const
{
assert(info.hash.type == htSHA256);
assert(info.hash.algo == HashAlgorithm::SHA256);
return makeStorePath(
makeType(*this, "text", StoreReferences {
.others = info.references,
@ -214,7 +218,7 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons
}
StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const
StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const
{
// New template
return std::visit(overloaded {
@ -228,12 +232,12 @@ StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentA
}
std::pair<StorePath, Hash> Store::computeStorePathFromDump(
Source & dump,
std::string_view name,
FileIngestionMethod method,
HashType hashAlgo,
const StorePathSet & references) const
std::pair<StorePath, Hash> StoreDirConfig::computeStorePathFromDump(
Source & dump,
std::string_view name,
FileIngestionMethod method,
HashAlgorithm hashAlgo,
const StorePathSet & references) const
{
HashSink sink(hashAlgo);
dump.drainInto(sink);
@ -247,26 +251,26 @@ std::pair<StorePath, Hash> Store::computeStorePathFromDump(
}
StorePath Store::computeStorePathForText(
StorePath StoreDirConfig::computeStorePathForText(
std::string_view name,
std::string_view s,
const StorePathSet & references) const
{
return makeTextPath(name, TextInfo {
.hash = hashString(htSHA256, s),
.hash = hashString(HashAlgorithm::SHA256, s),
.references = references,
});
}
StorePath Store::addToStore(
std::string_view name,
const Path & _srcPath,
FileIngestionMethod method,
HashType hashAlgo,
PathFilter & filter,
RepairFlag repair,
const StorePathSet & references)
std::string_view name,
const Path & _srcPath,
FileIngestionMethod method,
HashAlgorithm hashAlgo,
PathFilter & filter,
RepairFlag repair,
const StorePathSet & references)
{
Path srcPath(absPath(_srcPath));
auto source = sinkToSource([&](Sink & sink) {
@ -401,19 +405,19 @@ digraph graphname {
}
*/
ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo,
std::optional<Hash> expectedCAHash)
FileIngestionMethod method, HashAlgorithm hashAlgo,
std::optional<Hash> expectedCAHash)
{
HashSink narHashSink { htSHA256 };
HashSink narHashSink { HashAlgorithm::SHA256 };
HashSink caHashSink { hashAlgo };
/* Note that fileSink and unusualHashTee must be mutually exclusive, since
they both write to caHashSink. Note that that requisite is currently true
because the former is only used in the flat case. */
RetrieveRegularNARSink fileSink { caHashSink };
RegularFileSink fileSink { caHashSink };
TeeSink unusualHashTee { narHashSink, caHashSink };
auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != htSHA256
auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != HashAlgorithm::SHA256
? static_cast<Sink &>(unusualHashTee)
: narHashSink;
@ -428,10 +432,10 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
information to narSink. */
TeeSource tapped { *fileSource, narSink };
ParseSink blank;
NullParseSink blank;
auto & parseSink = method == FileIngestionMethod::Flat
? fileSink
: blank;
? (ParseSink &) fileSink
: (ParseSink &) blank;
/* The information that flows from tapped (besides being replicated in
narSink), is now put in parseSink. */
@ -441,7 +445,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
finish. */
auto [narHash, narSize] = narHashSink.finish();
auto hash = method == FileIngestionMethod::Recursive && hashAlgo == htSHA256
auto hash = method == FileIngestionMethod::Recursive && hashAlgo == HashAlgorithm::SHA256
? narHash
: caHashSink.finish().first;
@ -543,8 +547,8 @@ std::map<std::string, std::optional<StorePath>> Store::queryPartialDerivationOut
return outputs;
}
OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) {
auto resp = queryPartialDerivationOutputMap(path);
OutputPathMap Store::queryDerivationOutputMap(const StorePath & path, Store * evalStore) {
auto resp = queryPartialDerivationOutputMap(path, evalStore);
OutputPathMap result;
for (auto & [outName, optOutPath] : resp) {
if (!optOutPath)
@ -819,7 +823,7 @@ void Store::substitutePaths(const StorePathSet & paths)
std::vector<DerivedPath> paths2;
for (auto & path : paths)
if (!path.isDerivation())
paths2.push_back(DerivedPath::Opaque{path});
paths2.emplace_back(DerivedPath::Opaque{path});
uint64_t downloadSize, narSize;
StorePathSet willBuild, willSubstitute, unknown;
queryMissing(paths2,
@ -949,96 +953,6 @@ StorePathSet Store::exportReferences(const StorePathSet & storePaths, const Stor
return paths;
}
json Store::pathInfoToJSON(const StorePathSet & storePaths,
bool includeImpureInfo, bool showClosureSize,
HashFormat hashFormat,
AllowInvalidFlag allowInvalid)
{
json::array_t jsonList = json::array();
for (auto & storePath : storePaths) {
auto& jsonPath = jsonList.emplace_back(json::object());
try {
auto info = queryPathInfo(storePath);
jsonPath["path"] = printStorePath(info->path);
jsonPath["valid"] = true;
jsonPath["narHash"] = info->narHash.to_string(hashFormat, true);
jsonPath["narSize"] = info->narSize;
{
auto& jsonRefs = (jsonPath["references"] = json::array());
for (auto & ref : info->references)
jsonRefs.emplace_back(printStorePath(ref));
}
if (info->ca)
jsonPath["ca"] = renderContentAddress(info->ca);
std::pair<uint64_t, uint64_t> closureSizes;
if (showClosureSize) {
closureSizes = getClosureSize(info->path);
jsonPath["closureSize"] = closureSizes.first;
}
if (includeImpureInfo) {
if (info->deriver)
jsonPath["deriver"] = printStorePath(*info->deriver);
if (info->registrationTime)
jsonPath["registrationTime"] = info->registrationTime;
if (info->ultimate)
jsonPath["ultimate"] = info->ultimate;
if (!info->sigs.empty()) {
for (auto & sig : info->sigs)
jsonPath["signatures"].push_back(sig);
}
auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
std::shared_ptr<const ValidPathInfo>(info));
if (narInfo) {
if (!narInfo->url.empty())
jsonPath["url"] = narInfo->url;
if (narInfo->fileHash)
jsonPath["downloadHash"] = narInfo->fileHash->to_string(hashFormat, true);
if (narInfo->fileSize)
jsonPath["downloadSize"] = narInfo->fileSize;
if (showClosureSize)
jsonPath["closureDownloadSize"] = closureSizes.second;
}
}
} catch (InvalidPath &) {
jsonPath["path"] = printStorePath(storePath);
jsonPath["valid"] = false;
}
}
return jsonList;
}
std::pair<uint64_t, uint64_t> Store::getClosureSize(const StorePath & storePath)
{
uint64_t totalNarSize = 0, totalDownloadSize = 0;
StorePathSet closure;
computeFSClosure(storePath, closure, false, false);
for (auto & p : closure) {
auto info = queryPathInfo(p);
totalNarSize += info->narSize;
auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
std::shared_ptr<const ValidPathInfo>(info));
if (narInfo)
totalDownloadSize += narInfo->fileSize;
}
return {totalNarSize, totalDownloadSize};
}
const Store::Stats & Store::getStats()
{
@ -1291,7 +1205,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istre
if (!hashGiven) {
std::string s;
getline(str, s);
auto narHash = Hash::parseAny(s, htSHA256);
auto narHash = Hash::parseAny(s, HashAlgorithm::SHA256);
getline(str, s);
auto narSize = string2Int<uint64_t>(s);
if (!narSize) throw Error("number expected");
@ -1315,7 +1229,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istre
}
std::string Store::showPaths(const StorePathSet & paths)
std::string StoreDirConfig::showPaths(const StorePathSet & paths)
{
std::string s;
for (auto & i : paths) {
@ -1338,12 +1252,12 @@ Derivation Store::derivationFromPath(const StorePath & drvPath)
return readDerivation(drvPath);
}
Derivation readDerivationCommon(Store& store, const StorePath& drvPath, bool requireValidPath)
static Derivation readDerivationCommon(Store & store, const StorePath & drvPath, bool requireValidPath)
{
auto accessor = store.getFSAccessor();
auto accessor = store.getFSAccessor(requireValidPath);
try {
return parseDerivation(store,
accessor->readFile(store.printStorePath(drvPath), requireValidPath),
accessor->readFile(CanonPath(store.printStorePath(drvPath))),
Derivation::nameFromPath(drvPath));
} catch (FormatError & e) {
throw Error("error parsing derivation '%s': %s", store.printStorePath(drvPath), e.msg());