wroclaw-git-backups/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-06-27 16:51:15 +02:00
Code Activity

Merge branch 'master' of github.com:NixOS/nix into hash-always-has-type

Browse source
This commit is contained in:
Carlo Nucera 2020-07-15 17:19:56 -04:00
commit d090562348
36 changed files with 394 additions and 354 deletions
Download patch file Download diff file Expand all files Collapse all files

130
src/libstore/binary-cache-store.cc
View file

@ -15,6 +15,7 @@
#include <chrono>
#include <future>
#include <regex>
#include <fstream>
#include <nlohmann/json.hpp>
@ -57,6 +58,13 @@ void BinaryCacheStore::init()
}
}
void BinaryCacheStore::upsertFile(const std::string & path,
std::string && data,
const std::string & mimeType)
{
upsertFile(path, std::make_shared<std::stringstream>(std::move(data)), mimeType);
}
void BinaryCacheStore::getFile(const std::string & path,
Callback<std::shared_ptr<std::string>> callback) noexcept
{
@ -113,13 +121,74 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
}
void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
AutoCloseFD openFile(const Path & path)
{
// FIXME: See if we can use the original source to reduce memory usage.
auto nar = make_ref<std::string>(narSource.drain());
auto fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
if (!fd)
throw SysError("opening file '%1%'", path);
return fd;
}
if (!repair && isValidPath(info.path)) return;
struct FileSource : FdSource
{
AutoCloseFD fd2;
FileSource(const Path & path)
: fd2(openFile(path))
{
fd = fd2.get();
}
};
void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs)
{
assert(info.narHash && info.narSize);
if (!repair && isValidPath(info.path)) {
// FIXME: copyNAR -> null sink
narSource.drain();
return;
}
auto [fdTemp, fnTemp] = createTempFile();
auto now1 = std::chrono::steady_clock::now();
/* Read the NAR simultaneously into a CompressionSink+FileSink (to
write the compressed NAR to disk), into a HashSink (to get the
NAR hash), and into a NarAccessor (to get the NAR listing). */
HashSink fileHashSink(htSHA256);
std::shared_ptr<FSAccessor> narAccessor;
{
FdSink fileSink(fdTemp.get());
TeeSink teeSink(fileSink, fileHashSink);
auto compressionSink = makeCompressionSink(compression, teeSink);
TeeSource teeSource(narSource, *compressionSink);
narAccessor = makeNarAccessor(teeSource);
compressionSink->finish();
}
auto now2 = std::chrono::steady_clock::now();
auto narInfo = make_ref<NarInfo>(info);
narInfo->narSize = info.narSize;
narInfo->narHash = info.narHash;
narInfo->compression = compression;
auto [fileHash, fileSize] = fileHashSink.finish();
narInfo->fileHash = fileHash;
narInfo->fileSize = fileSize;
narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar"
+ (compression == "xz" ? ".xz" :
compression == "bzip2" ? ".bz2" :
compression == "br" ? ".br" :
"");
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
printStorePath(narInfo->path), info.narSize,
((1.0 - (double) fileSize / info.narSize) * 100.0),
duration);
/* Verify that all references are valid. This may do some .narinfo
reads, but typically they'll already be cached. */
@ -132,23 +201,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
printStorePath(info.path), printStorePath(ref));
}
assert(nar->compare(0, narMagic.size(), narMagic) == 0);
auto narInfo = make_ref<NarInfo>(info);
narInfo->narSize = nar->size();
narInfo->narHash = hashString(htSHA256, *nar);
if (info.narHash && info.narHash != narInfo->narHash)
throw Error("refusing to copy corrupted path '%1%' to binary cache", printStorePath(info.path));
auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor);
auto narAccessor = makeNarAccessor(nar);
if (accessor_)
accessor_->addToCache(printStorePath(info.path), *nar, narAccessor);
/* Optionally write a JSON file containing a listing of the
contents of the NAR. */
if (writeNARListing) {
@ -160,33 +212,13 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
{
auto res = jsonRoot.placeholder("root");
listNar(res, narAccessor, "", true);
listNar(res, ref<FSAccessor>(narAccessor), "", true);
}
}
upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json");
}
/* Compress the NAR. */
narInfo->compression = compression;
auto now1 = std::chrono::steady_clock::now();
auto narCompressed = compress(compression, *nar, parallelCompression);
auto now2 = std::chrono::steady_clock::now();
narInfo->fileHash = hashString(htSHA256, *narCompressed);
narInfo->fileSize = narCompressed->size();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
printStorePath(narInfo->path), narInfo->narSize,
((1.0 - (double) narCompressed->size() / nar->size()) * 100.0),
duration);
narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar"
+ (compression == "xz" ? ".xz" :
compression == "bzip2" ? ".bz2" :
compression == "br" ? ".br" :
"");
/* Optionally maintain an index of DWARF debug info files
consisting of JSON files named 'debuginfo/<build-id>' that
specify the NAR file and member containing the debug info. */
@ -247,12 +279,14 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
/* Atomically write the NAR file. */
if (repair || !fileExists(narInfo->url)) {
stats.narWrite++;
upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar");
upsertFile(narInfo->url,
std::make_shared<std::fstream>(fnTemp, std::ios_base::in),
"application/x-nix-nar");
} else
stats.narWriteAverted++;
stats.narWriteBytes += nar->size();
stats.narWriteCompressedBytes += narCompressed->size();
stats.narWriteBytes += info.narSize;
stats.narWriteCompressedBytes += fileSize;
stats.narWriteCompressionTimeMs += duration;
/* Atomically write the NAR info file.*/
@ -351,7 +385,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
ValidPathInfo info(makeFixedOutputPath(method, *h, name));
auto source = StringSource { *sink.s };
addToStore(info, source, repair, CheckSigs, nullptr);
addToStore(info, source, repair, CheckSigs);
return std::move(info.path);
}
@ -366,7 +400,7 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
StringSink sink;
dumpString(s, sink);
auto source = StringSource { *sink.s };
addToStore(info, source, repair, CheckSigs, nullptr);
addToStore(info, source, repair, CheckSigs);
}
return std::move(info.path);

9
src/libstore/binary-cache-store.hh
View file

@ -36,9 +36,13 @@ public:
virtual bool fileExists(const std::string & path) = 0;
virtual void upsertFile(const std::string & path,
const std::string & data,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType) = 0;
void upsertFile(const std::string & path,
std::string && data,
const std::string & mimeType);
/* Note: subclasses must implement at least one of the two
following getFile() methods. */
@ -75,8 +79,7 @@ public:
{ unsupported("queryPathFromHashPart"); }
void addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override;
RepairFlag repair, CheckSigsFlag checkSigs) override;
StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo,

9
src/libstore/build.cc
View file

@ -2044,7 +2044,7 @@ void DerivationGoal::startBuilder()
auto storePathS = *i++;
if (!worker.store.isInStore(storePathS))
throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS);
auto storePath = worker.store.parseStorePath(worker.store.toStorePath(storePathS));
auto storePath = worker.store.toStorePath(storePathS).first;
/* Write closure info to <fileName>. */
writeFile(tmpDir + "/" + fileName,
@ -2083,7 +2083,7 @@ void DerivationGoal::startBuilder()
for (auto & i : dirsInChroot)
try {
if (worker.store.isInStore(i.second.source))
worker.store.computeFSClosure(worker.store.parseStorePath(worker.store.toStorePath(i.second.source)), closure);
worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure);
} catch (InvalidPath & e) {
} catch (Error & e) {
throw Error("while processing 'sandbox-paths': %s", e.what());
@ -2768,10 +2768,9 @@ struct RestrictedStore : public LocalFSStore
{ throw Error("addToStore"); }
void addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
std::shared_ptr<FSAccessor> accessor = 0) override
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override
{
next->addToStore(info, narSource, repair, checkSigs, accessor);
next->addToStore(info, narSource, repair, checkSigs);
goal.addDependency(info.path);
}

15
src/libstore/daemon.cc
View file

@ -391,7 +391,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
HashType hashAlgo = parseHashType(s);
TeeSource savedNAR(from);
StringSink savedNAR;
TeeSource savedNARSource(from, savedNAR);
RetrieveRegularNARSink savedRegular;
if (method == FileIngestionMethod::Recursive) {
@ -399,7 +400,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
a string so that we can pass it to
addToStoreFromDump(). */
ParseSink sink; /* null sink; just parse the NAR */
parseDump(sink, savedNAR);
parseDump(sink, savedNARSource);
} else
parseDump(savedRegular, from);
@ -407,7 +408,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (!savedRegular.regular) throw Error("regular file expected");
auto path = store->addToStoreFromDump(
method == FileIngestionMethod::Recursive ? *savedNAR.data : savedRegular.s,
method == FileIngestionMethod::Recursive ? *savedNAR.s : savedRegular.s,
baseName,
method,
hashAlgo);
@ -442,7 +443,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopImportPaths: {
logger->startWork();
TunnelSource source(from, to);
auto paths = store->importPaths(source, nullptr,
auto paths = store->importPaths(source,
trusted ? NoCheckSigs : CheckSigs);
logger->stopWork();
Strings paths2;
@ -731,9 +732,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (GET_PROTOCOL_MINOR(clientVersion) >= 21)
source = std::make_unique<TunnelSource>(from, to);
else {
TeeSink tee(from);
TeeParseSink tee(from);
parseDump(tee, tee.source);
saved = std::move(*tee.source.data);
saved = std::move(*tee.saved.s);
source = std::make_unique<StringSource>(saved);
}
@ -741,7 +742,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
// FIXME: race if addToStore doesn't read source?
store->addToStore(info, *source, (RepairFlag) repair,
dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr);
dontCheckSigs ? NoCheckSigs : CheckSigs);
logger->stopWork();
break;

7
src/libstore/derivations.cc
View file

@ -4,7 +4,6 @@
#include "util.hh"
#include "worker-protocol.hh"
#include "fs-accessor.hh"
#include "istringstream_nocopy.hh"
namespace nix {
@ -101,7 +100,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
}
static DerivationOutput parseDerivationOutput(const Store & store, istringstream_nocopy & str)
static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str)
{
expect(str, ","); auto path = store.parseStorePath(parsePath(str));
expect(str, ","); auto hashAlgo = parseString(str);
@ -129,10 +128,10 @@ static DerivationOutput parseDerivationOutput(const Store & store, istringstream
}
static Derivation parseDerivation(const Store & store, const string & s)
static Derivation parseDerivation(const Store & store, std::string && s)
{
Derivation drv;
istringstream_nocopy str(s);
std::istringstream str(std::move(s));
expect(str, "Derive([");
/* Parse the list of outputs. */

43
src/libstore/export-import.cc
View file

@ -7,24 +7,6 @@
namespace nix {
struct HashAndWriteSink : Sink
{
Sink & writeSink;
HashSink hashSink;
HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256)
{
}
virtual void operator () (const unsigned char * data, size_t len)
{
writeSink(data, len);
hashSink(data, len);
}
Hash currentHash()
{
return hashSink.currentHash().first;
}
};
void Store::exportPaths(const StorePathSet & paths, Sink & sink)
{
auto sorted = topoSortPaths(paths);
@ -47,28 +29,29 @@ void Store::exportPath(const StorePath & path, Sink & sink)
{
auto info = queryPathInfo(path);
HashAndWriteSink hashAndWriteSink(sink);
HashSink hashSink(htSHA256);
TeeSink teeSink(sink, hashSink);
narFromPath(path, hashAndWriteSink);
narFromPath(path, teeSink);
/* Refuse to export paths that have changed. This prevents
filesystem corruption from spreading to other machines.
Don't complain if the stored hash is zero (unknown). */
Hash hash = hashAndWriteSink.currentHash();
Hash hash = hashSink.currentHash().first;
if (hash != info->narHash && info->narHash != Hash(info->narHash->type))
throw Error("hash of path '%s' has changed from '%s' to '%s'!",
printStorePath(path), info->narHash->to_string(Base32, true), hash.to_string(Base32, true));
hashAndWriteSink
teeSink
<< exportMagic
<< printStorePath(path);
writeStorePaths(*this, hashAndWriteSink, info->references);
hashAndWriteSink
writeStorePaths(*this, teeSink, info->references);
teeSink
<< (info->deriver ? printStorePath(*info->deriver) : "")
<< 0;
}
StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, CheckSigsFlag checkSigs)
StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
{
StorePaths res;
while (true) {
@ -77,7 +60,7 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces
if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'");
/* Extract the NAR from the source. */
TeeSink tee(source);
TeeParseSink tee(source);
parseDump(tee, tee.source);
uint32_t magic = readInt(source);
@ -94,16 +77,16 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces
if (deriver != "")
info.deriver = parseStorePath(deriver);
info.narHash = hashString(htSHA256, *tee.source.data);
info.narSize = tee.source.data->size();
info.narHash = hashString(htSHA256, *tee.saved.s);
info.narSize = tee.saved.s->size();
// Ignore optional legacy signature.
if (readInt(source) == 1)
readString(source);
// Can't use underlying source, which would have been exhausted
auto source = StringSource { *tee.source.data };
addToStore(info, source, NoRepair, checkSigs, accessor);
auto source = StringSource { *tee.saved.s };
addToStore(info, source, NoRepair, checkSigs);
res.push_back(info.path);
}

30
src/libstore/gc.cc
View file

@ -262,11 +262,13 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
{
auto foundRoot = [&](const Path & path, const Path & target) {
auto storePath = maybeParseStorePath(toStorePath(target));
if (storePath && isValidPath(*storePath))
roots[std::move(*storePath)].emplace(path);
else
printInfo("skipping invalid root from '%1%' to '%2%'", path, target);
try {
auto storePath = toStorePath(target).first;
if (isValidPath(storePath))
roots[std::move(storePath)].emplace(path);
else
printInfo("skipping invalid root from '%1%' to '%2%'", path, target);
} catch (BadStorePath &) { }
};
try {
@ -472,15 +474,15 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
for (auto & [target, links] : unchecked) {
if (!isInStore(target)) continue;
Path pathS = toStorePath(target);
if (!isStorePath(pathS)) continue;
auto path = parseStorePath(pathS);
if (!isValidPath(path)) continue;
debug("got additional root '%1%'", pathS);
if (censor)
roots[path].insert(censored);
else
roots[path].insert(links.begin(), links.end());
try {
auto path = toStorePath(target).first;
if (!isValidPath(path)) continue;
debug("got additional root '%1%'", printStorePath(path));
if (censor)
roots[path].insert(censored);
else
roots[path].insert(links.begin(), links.end());
} catch (BadStorePath &) { }
}
}

4
src/libstore/http-binary-cache-store.cc
View file

@ -100,11 +100,11 @@ protected:
}
void upsertFile(const std::string & path,
const std::string & data,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType) override
{
auto req = FileTransferRequest(cacheUri + "/" + path);
req.data = std::make_shared<string>(data); // FIXME: inefficient
req.data = std::make_shared<string>(StreamToSourceAdapter(istream).drain());
req.mimeType = mimeType;
try {
getFileTransfer()->upload(req);

3
src/libstore/legacy-ssh-store.cc
View file

@ -126,8 +126,7 @@ struct LegacySSHStore : public Store
}
void addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override
RepairFlag repair, CheckSigsFlag checkSigs) override
{
debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host);

35
src/libstore/local-binary-cache-store.cc
View file

@ -31,8 +31,18 @@ protected:
bool fileExists(const std::string & path) override;
void upsertFile(const std::string & path,
const std::string & data,
const std::string & mimeType) override;
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType) override
{
auto path2 = binaryCacheDir + "/" + path;
Path tmp = path2 + ".tmp." + std::to_string(getpid());
AutoDelete del(tmp, false);
StreamToSourceAdapter source(istream);
writeFile(tmp, source);
if (rename(tmp.c_str(), path2.c_str()))
throw SysError("renaming '%1%' to '%2%'", tmp, path2);
del.cancel();
}
void getFile(const std::string & path, Sink & sink) override
{
@ -52,7 +62,9 @@ protected:
if (entry.name.size() != 40 ||
!hasSuffix(entry.name, ".narinfo"))
continue;
paths.insert(parseStorePath(storeDir + "/" + entry.name.substr(0, entry.name.size() - 8)));
paths.insert(parseStorePath(
storeDir + "/" + entry.name.substr(0, entry.name.size() - 8)
+ "-" + MissingName));
}
return paths;
@ -68,28 +80,11 @@ void LocalBinaryCacheStore::init()
BinaryCacheStore::init();
}
static void atomicWrite(const Path & path, const std::string & s)
{
Path tmp = path + ".tmp." + std::to_string(getpid());
AutoDelete del(tmp, false);
writeFile(tmp, s);
if (rename(tmp.c_str(), path.c_str()))
throw SysError("renaming '%1%' to '%2%'", tmp, path);
del.cancel();
}
bool LocalBinaryCacheStore::fileExists(const std::string & path)
{
return pathExists(binaryCacheDir + "/" + path);
}
void LocalBinaryCacheStore::upsertFile(const std::string & path,
const std::string & data,
const std::string & mimeType)
{
atomicWrite(binaryCacheDir + "/" + path, data);
}
static RegisterStoreImplementation regStore([](
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>

6
src/libstore/local-fs-store.cc
View file

@ -20,9 +20,9 @@ struct LocalStoreAccessor : public FSAccessor
Path toRealPath(const Path & path)
{
Path storePath = store->toStorePath(path);
if (!store->isValidPath(store->parseStorePath(storePath)))
throw InvalidPath("path '%1%' is not a valid store path", storePath);
auto storePath = store->toStorePath(path).first;
if (!store->isValidPath(storePath))
throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
return store->getRealStoreDir() + std::string(path, store->storeDir.size());
}

22
src/libstore/local-store.cc
View file

@ -962,7 +962,7 @@ const PublicKeys & LocalStore::getPublicKeys()
void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
RepairFlag repair, CheckSigsFlag checkSigs)
{
if (!info.narHash)
throw Error("cannot add path '%s' because it lacks a hash", printStorePath(info.path));
@ -1097,13 +1097,16 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
{
Path srcPath(absPath(_srcPath));
if (method != FileIngestionMethod::Recursive)
return addToStoreFromDump(readFile(srcPath), name, method, hashAlgo, repair);
/* For computing the NAR hash. */
auto sha256Sink = std::make_unique<HashSink>(htSHA256);
/* For computing the store path. In recursive SHA-256 mode, this
is the same as the NAR hash, so no need to do it again. */
std::unique_ptr<HashSink> hashSink =
method == FileIngestionMethod::Recursive && hashAlgo == htSHA256
hashAlgo == htSHA256
? nullptr
: std::make_unique<HashSink>(hashAlgo);
@ -1136,10 +1139,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
if (!inMemory) sink(buf, len);
});
if (method == FileIngestionMethod::Recursive)
dumpPath(srcPath, sink2, filter);
else
readFile(srcPath, sink2);
dumpPath(srcPath, sink2, filter);
});
std::unique_ptr<AutoDelete> delTempDir;
@ -1155,10 +1155,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
delTempDir = std::make_unique<AutoDelete>(tempDir);
tempPath = tempDir + "/x";
if (method == FileIngestionMethod::Recursive)
restorePath(tempPath, *source);
else
writeFile(tempPath, *source);
restorePath(tempPath, *source);
} catch (EndOfFile &) {
if (!inMemory) throw;
@ -1191,10 +1188,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
if (inMemory) {
/* Restore from the NAR in memory. */
StringSource source(nar);
if (method == FileIngestionMethod::Recursive)
restorePath(realPath, source);
else
writeFile(realPath, source);
restorePath(realPath, source);
} else {
/* Move the temporary path we restored above. */
if (rename(tempPath.c_str(), realPath.c_str()))

3
src/libstore/local-store.hh
View file

@ -143,8 +143,7 @@ public:
SubstitutablePathInfos & infos) override;
void addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override;
RepairFlag repair, CheckSigsFlag checkSigs) override;
StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo,

46
src/libstore/nar-accessor.cc
View file

@ -18,7 +18,7 @@ struct NarMember
/* If this is a regular file, position of the contents of this
file in the NAR. */
size_t start = 0, size = 0;
uint64_t start = 0, size = 0;
std::string target;
@ -34,17 +34,19 @@ struct NarAccessor : public FSAccessor
NarMember root;
struct NarIndexer : ParseSink, StringSource
struct NarIndexer : ParseSink, Source
{
NarAccessor & acc;
Source & source;
std::stack<NarMember *> parents;
std::string currentStart;
bool isExec = false;
NarIndexer(NarAccessor & acc, const std::string & nar)
: StringSource(nar), acc(acc)
uint64_t pos = 0;
NarIndexer(NarAccessor & acc, Source & source)
: acc(acc), source(source)
{ }
void createMember(const Path & path, NarMember member) {
@ -79,31 +81,38 @@ struct NarAccessor : public FSAccessor
void preallocateContents(unsigned long long size) override
{
currentStart = string(s, pos, 16);
assert(size <= std::numeric_limits<size_t>::max());
parents.top()->size = (size_t)size;
assert(size <= std::numeric_limits<uint64_t>::max());
parents.top()->size = (uint64_t) size;
parents.top()->start = pos;
}
void receiveContents(unsigned char * data, unsigned int len) override
{
// Sanity check
if (!currentStart.empty()) {
assert(len < 16 || currentStart == string((char *) data, 16));
currentStart.clear();
}
}
{ }
void createSymlink(const Path & path, const string & target) override
{
createMember(path,
NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
}
size_t read(unsigned char * data, size_t len) override
{
auto n = source.read(data, len);
pos += n;
return n;
}
};
NarAccessor(ref<const std::string> nar) : nar(nar)
{
NarIndexer indexer(*this, *nar);
StringSource source(*nar);
NarIndexer indexer(*this, source);
parseDump(indexer, indexer);
}
NarAccessor(Source & source)
{
NarIndexer indexer(*this, source);
parseDump(indexer, indexer);
}
@ -219,6 +228,11 @@ ref<FSAccessor> makeNarAccessor(ref<const std::string> nar)
return make_ref<NarAccessor>(nar);
}
ref<FSAccessor> makeNarAccessor(Source & source)
{
return make_ref<NarAccessor>(source);
}
ref<FSAccessor> makeLazyNarAccessor(const std::string & listing,
GetNarBytes getNarBytes)
{

4
src/libstore/nar-accessor.hh
View file

@ -6,10 +6,14 @@
namespace nix {
struct Source;
/* Return an object that provides access to the contents of a NAR
file. */
ref<FSAccessor> makeNarAccessor(ref<const std::string> nar);
ref<FSAccessor> makeNarAccessor(Source & source);
/* Create a NAR accessor from a NAR listing (in the format produced by
listNar()). The callback getNarBytes(offset, length) is used by the
readFile() method of the accessor to get the contents of files

2
src/libstore/path.cc
View file

@ -2,8 +2,6 @@
namespace nix {
MakeError(BadStorePath, Error);
static void checkName(std::string_view path, std::string_view name)
{
if (name.empty())

33
src/libstore/remote-fs-accessor.cc
View file

@ -16,26 +16,26 @@ RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path & cacheDir)
createDirs(cacheDir);
}
Path RemoteFSAccessor::makeCacheFile(const Path & storePath, const std::string & ext)
Path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::string & ext)
{
assert(cacheDir != "");
return fmt("%s/%s.%s", cacheDir, store->parseStorePath(storePath).hashPart(), ext);
return fmt("%s/%s.%s", cacheDir, hashPart, ext);
}
void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar,
void RemoteFSAccessor::addToCache(std::string_view hashPart, const std::string & nar,
ref<FSAccessor> narAccessor)
{
nars.emplace(storePath, narAccessor);
nars.emplace(hashPart, narAccessor);
if (cacheDir != "") {
try {
std::ostringstream str;
JSONPlaceholder jsonRoot(str);
listNar(jsonRoot, narAccessor, "", true);
writeFile(makeCacheFile(storePath, "ls"), str.str());
writeFile(makeCacheFile(hashPart, "ls"), str.str());
/* FIXME: do this asynchronously. */
writeFile(makeCacheFile(storePath, "nar"), nar);
writeFile(makeCacheFile(hashPart, "nar"), nar);
} catch (...) {
ignoreException();
@ -47,23 +47,22 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
{
auto path = canonPath(path_);
auto storePath = store->toStorePath(path);
std::string restPath = std::string(path, storePath.size());
auto [storePath, restPath] = store->toStorePath(path);
if (!store->isValidPath(store->parseStorePath(storePath)))
throw InvalidPath("path '%1%' is not a valid store path", storePath);
if (!store->isValidPath(storePath))
throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
auto i = nars.find(storePath);
auto i = nars.find(std::string(storePath.hashPart()));
if (i != nars.end()) return {i->second, restPath};
StringSink sink;
std::string listing;
Path cacheFile;
if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath, "nar"))) {
if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) {
try {
listing = nix::readFile(makeCacheFile(storePath, "ls"));
listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls"));
auto narAccessor = makeLazyNarAccessor(listing,
[cacheFile](uint64_t offset, uint64_t length) {
@ -81,7 +80,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
return buf;
});
nars.emplace(storePath, narAccessor);
nars.emplace(storePath.hashPart(), narAccessor);
return {narAccessor, restPath};
} catch (SysError &) { }
@ -90,15 +89,15 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
*sink.s = nix::readFile(cacheFile);
auto narAccessor = makeNarAccessor(sink.s);
nars.emplace(storePath, narAccessor);
nars.emplace(storePath.hashPart(), narAccessor);
return {narAccessor, restPath};
} catch (SysError &) { }
}
store->narFromPath(store->parseStorePath(storePath), sink);
store->narFromPath(storePath, sink);
auto narAccessor = makeNarAccessor(sink.s);
addToCache(storePath, *sink.s, narAccessor);
addToCache(storePath.hashPart(), *sink.s, narAccessor);
return {narAccessor, restPath};
}

6
src/libstore/remote-fs-accessor.hh
View file

@ -10,7 +10,7 @@ class RemoteFSAccessor : public FSAccessor
{
ref<Store> store;
std::map<Path, ref<FSAccessor>> nars;
std::map<std::string, ref<FSAccessor>> nars;
Path cacheDir;
@ -18,9 +18,9 @@ class RemoteFSAccessor : public FSAccessor
friend class BinaryCacheStore;
Path makeCacheFile(const Path & storePath, const std::string & ext);
Path makeCacheFile(std::string_view hashPart, const std::string & ext);
void addToCache(const Path & storePath, const std::string & nar,
void addToCache(std::string_view hashPart, const std::string & nar,
ref<FSAccessor> narAccessor);
public:

2
src/libstore/remote-store.cc
View file

@ -466,7 +466,7 @@ std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string &
void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
RepairFlag repair, CheckSigsFlag checkSigs)
{
auto conn(getConnection());

3
src/libstore/remote-store.hh
View file

@ -60,8 +60,7 @@ public:
SubstitutablePathInfos & infos) override;
void addToStore(const ValidPathInfo & info, Source & nar,
RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override;
RepairFlag repair, CheckSigsFlag checkSigs) override;
StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,

39
src/libstore/s3-binary-cache-store.cc
View file

@ -7,7 +7,6 @@
#include "globals.hh"
#include "compression.hh"
#include "filetransfer.hh"
#include "istringstream_nocopy.hh"
#include <aws/core/Aws.h>
#include <aws/core/VersionConfig.h>
@ -262,12 +261,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
std::shared_ptr<TransferManager> transferManager;
std::once_flag transferManagerCreated;
void uploadFile(const std::string & path, const std::string & data,
void uploadFile(const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType,
const std::string & contentEncoding)
{
auto stream = std::make_shared<istringstream_nocopy>(data);
auto maxThreads = std::thread::hardware_concurrency();
static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>
@ -307,7 +305,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
std::shared_ptr<TransferHandle> transferHandle =
transferManager->UploadFile(
stream, bucketName, path, mimeType,
istream, bucketName, path, mimeType,
Aws::Map<Aws::String, Aws::String>(),
nullptr /*, contentEncoding */);
@ -333,9 +331,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
if (contentEncoding != "")
request.SetContentEncoding(contentEncoding);
auto stream = std::make_shared<istringstream_nocopy>(data);
request.SetBody(stream);
request.SetBody(istream);
auto result = checkAws(fmt("AWS error uploading '%s'", path),
s3Helper.client->PutObject(request));
@ -347,25 +343,34 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
.count();
printInfo(format("uploaded 's3://%1%/%2%' (%3% bytes) in %4% ms") %
bucketName % path % data.size() % duration);
auto size = istream->tellg();
printInfo("uploaded 's3://%s/%s' (%d bytes) in %d ms",
bucketName, path, size, duration);
stats.putTimeMs += duration;
stats.putBytes += data.size();
stats.putBytes += size;
stats.put++;
}
void upsertFile(const std::string & path, const std::string & data,
void upsertFile(const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType) override
{
auto compress = [&](std::string compression)
{
auto compressed = nix::compress(compression, StreamToSourceAdapter(istream).drain());
return std::make_shared<std::stringstream>(std::move(*compressed));
};
if (narinfoCompression != "" && hasSuffix(path, ".narinfo"))
uploadFile(path, *compress(narinfoCompression, data), mimeType, narinfoCompression);
uploadFile(path, compress(narinfoCompression), mimeType, narinfoCompression);
else if (lsCompression != "" && hasSuffix(path, ".ls"))
uploadFile(path, *compress(lsCompression, data), mimeType, lsCompression);
uploadFile(path, compress(lsCompression), mimeType, lsCompression);
else if (logCompression != "" && hasPrefix(path, "log/"))
uploadFile(path, *compress(logCompression, data), mimeType, logCompression);
uploadFile(path, compress(logCompression), mimeType, logCompression);
else
uploadFile(path, data, mimeType, "");
uploadFile(path, istream, mimeType, "");
}
void getFile(const std::string & path, Sink & sink) override
@ -410,7 +415,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
for (auto object : contents) {
auto & key = object.GetKey();
if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue;
paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-unknown"));
paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-" + MissingName));
}
marker = res.GetNextMarker();

30
src/libstore/store-api.cc
View file

@ -21,15 +21,15 @@ bool Store::isInStore(const Path & path) const
}
Path Store::toStorePath(const Path & path) const
std::pair<StorePath, Path> Store::toStorePath(const Path & path) const
{
if (!isInStore(path))
throw Error("path '%1%' is not in the Nix store", path);
Path::size_type slash = path.find('/', storeDir.size() + 1);
if (slash == Path::npos)
return path;
return {parseStorePath(path), ""};
else
return Path(path, 0, slash);
return {parseStorePath(std::string_view(path).substr(0, slash)), path.substr(slash)};
}
@ -42,14 +42,14 @@ Path Store::followLinksToStore(std::string_view _path) const
path = absPath(target, dirOf(path));
}
if (!isInStore(path))
throw NotInStore("path '%1%' is not in the Nix store", path);
throw BadStorePath("path '%1%' is not in the Nix store", path);
return path;
}
StorePath Store::followLinksToStorePath(std::string_view path) const
{
return parseStorePath(toStorePath(followLinksToStore(path)));
return toStorePath(followLinksToStore(path)).first;
}
@ -351,6 +351,14 @@ ref<const ValidPathInfo> Store::queryPathInfo(const StorePath & storePath)
}
static bool goodStorePath(const StorePath & expected, const StorePath & actual)
{
return
expected.hashPart() == actual.hashPart()
&& (expected.name() == Store::MissingName || expected.name() == actual.name());
}
void Store::queryPathInfo(const StorePath & storePath,
Callback<ref<const ValidPathInfo>> callback) noexcept
{
@ -378,7 +386,7 @@ void Store::queryPathInfo(const StorePath & storePath,
state_->pathInfoCache.upsert(hashPart,
res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second });
if (res.first == NarInfoDiskCache::oInvalid ||
res.second->path != storePath)
!goodStorePath(storePath, res.second->path))
throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
}
return callback(ref<const ValidPathInfo>(res.second));
@ -390,7 +398,7 @@ void Store::queryPathInfo(const StorePath & storePath,
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
queryPathInfoUncached(storePath,
{[this, storePath{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
{[this, storePathS{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
try {
auto info = fut.get();
@ -403,9 +411,11 @@ void Store::queryPathInfo(const StorePath & storePath,
state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info });
}
if (!info || info->path != parseStorePath(storePath)) {
auto storePath = parseStorePath(storePathS);
if (!info || !goodStorePath(storePath, info->path)) {
stats.narInfoMissing++;
throw InvalidPath("path '%s' is not valid", storePath);
throw InvalidPath("path '%s' is not valid", storePathS);
}
(*callbackPtr)(ref<const ValidPathInfo>(info));
@ -550,7 +560,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
if (!narInfo->url.empty())
jsonPath.attr("url", narInfo->url);
if (narInfo->fileHash)
jsonPath.attr("downloadHash", narInfo->fileHash->to_string(Base32, true));
jsonPath.attr("downloadHash", narInfo->fileHash->to_string(hashBase, true));
if (narInfo->fileSize)
jsonPath.attr("downloadSize", narInfo->fileSize);
if (showClosureSize)

23
src/libstore/store-api.hh
View file

@ -31,7 +31,7 @@ MakeError(InvalidPath, Error);
MakeError(Unsupported, Error);
MakeError(SubstituteGone, Error);
MakeError(SubstituterDisabled, Error);
MakeError(NotInStore, Error);
MakeError(BadStorePath, Error);
class FSAccessor;
@ -318,9 +318,9 @@ public:
the Nix store. */
bool isStorePath(std::string_view path) const;
/* Chop off the parts after the top-level store name, e.g.,
/nix/store/abcd-foo/bar => /nix/store/abcd-foo. */
Path toStorePath(const Path & path) const;
/* Split a path like /nix/store/<hash>-<name>/<bla> into
/nix/store/<hash>-<name> and /<bla>. */
std::pair<StorePath, Path> toStorePath(const Path & path) const;
/* Follow symlinks until we end up with a path in the Nix store. */
Path followLinksToStore(std::string_view path) const;
@ -385,13 +385,16 @@ public:
SubstituteFlag maybeSubstitute = NoSubstitute);
/* Query the set of all valid paths. Note that for some store
backends, the name part of store paths may be omitted
(i.e. you'll get /nix/store/<hash> rather than
backends, the name part of store paths may be replaced by 'x'
(i.e. you'll get /nix/store/<hash>-x rather than
/nix/store/<hash>-<name>). Use queryPathInfo() to obtain the
full store path. */
full store path. FIXME: should return a set of
std::variant<StorePath, HashPart> to get rid of this hack. */
virtual StorePathSet queryAllValidPaths()
{ unsupported("queryAllValidPaths"); }
constexpr static const char * MissingName = "x";
/* Query information about a valid path. It is permitted to omit
the name part of the store path. */
ref<const ValidPathInfo> queryPathInfo(const StorePath & path);
@ -440,8 +443,7 @@ public:
/* Import a path into the store. */
virtual void addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
std::shared_ptr<FSAccessor> accessor = 0) = 0;
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) = 0;
/* Copy the contents of a path to the store and register the
validity the resulting path. The resulting path is returned.
@ -624,8 +626,7 @@ public:
the Nix store. Optionally, the contents of the NARs are
preloaded into the specified FS accessor to speed up subsequent
access. */
StorePaths importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
CheckSigsFlag checkSigs = CheckSigs);
StorePaths importPaths(Source & source, CheckSigsFlag checkSigs = CheckSigs);
struct Stats
{