wroclaw-git-backups/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-06-28 13:41:15 +02:00
Code Activity

Merge pull request #9325 from NixOS/accessor-add-to-store

Browse source 
Content addressing and adding to store cleanup
This commit is contained in:
Eelco Dolstra 2023-12-19 15:10:31 +01:00 • committed by GitHub
commit 7cfd6c0efe
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
42 changed files with 589 additions and 593 deletions
Download patch file Download diff file Expand all files Collapse all files

12
perl/lib/Nix/Store.xs
View file

@ -13,6 +13,7 @@
#include "globals.hh" #include "globals.hh"
#include "store-api.hh" #include "store-api.hh"
#include "crypto.hh" #include "crypto.hh"
#include "posix-source-accessor.hh"
#include <sodium.h> #include <sodium.h>
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
@ -205,7 +206,10 @@ void importPaths(int fd, int dontCheckSigs)
SV * hashPath(char * algo, int base32, char * path) SV * hashPath(char * algo, int base32, char * path)
PPCODE: PPCODE:
try { try {
Hash h = hashPath(parseHashAlgo(algo), path).first; PosixSourceAccessor accessor;
Hash h = hashPath(
accessor, CanonPath::fromCwd(path),
FileIngestionMethod::Recursive, parseHashAlgo(algo)).first;
auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
} catch (Error & e) { } catch (Error & e) {
@ -281,7 +285,11 @@ SV * addToStore(char * srcPath, int recursive, char * algo)
PPCODE: PPCODE:
try { try {
auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashAlgo(algo)); PosixSourceAccessor accessor;
auto path = store()->addToStore(
std::string(baseNameOf(srcPath)),
accessor, CanonPath::fromCwd(srcPath),
method, parseHashAlgo(algo));
XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
} catch (Error & e) { } catch (Error & e) {
croak("%s", e.what()); croak("%s", e.what());

2
src/libcmd/installable-value.cc
View file

@ -44,7 +44,7 @@ ref<InstallableValue> InstallableValue::require(ref<Installable> installable)
std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx) std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx)
{ {
if (v.type() == nPath) { if (v.type() == nPath) {
auto storePath = v.path().fetchToStore(state->store); auto storePath = v.path().fetchToStore(*state->store);
return {{ return {{
.path = DerivedPath::Opaque { .path = DerivedPath::Opaque {
.path = std::move(storePath), .path = std::move(storePath),

2
src/libexpr/eval.cc
View file

@ -2317,7 +2317,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
auto dstPath = i != srcToStore.end() auto dstPath = i != srcToStore.end()
? i->second ? i->second
: [&]() { : [&]() {
auto dstPath = path.fetchToStore(store, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair); auto dstPath = path.fetchToStore(*store, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair);
allowPath(dstPath); allowPath(dstPath);
srcToStore.insert_or_assign(path, dstPath); srcToStore.insert_or_assign(path, dstPath);
printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath)); printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath));

12
src/libexpr/primops.cc
View file

@ -2072,8 +2072,14 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val
} }
auto storePath = settings.readOnlyMode auto storePath = settings.readOnlyMode
? state.store->computeStorePathForText(name, contents, refs) ? state.store->makeFixedOutputPathFromCA(name, TextInfo {
: state.store->addTextToStore(name, contents, refs, state.repair); .hash = hashString(HashAlgorithm::SHA256, contents),
.references = std::move(refs),
})
: ({
StringSource s { contents };
state.store->addToStoreFromDump(s, name, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, state.repair);
});
/* Note: we don't need to add `context' to the context of the /* Note: we don't need to add `context' to the context of the
result, since `storePath' itself has references to the paths result, since `storePath' itself has references to the paths
@ -2229,7 +2235,7 @@ static void addPath(
}); });
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
auto dstPath = path.fetchToStore(state.store, name, method, filter.get(), state.repair); auto dstPath = path.fetchToStore(*state.store, name, method, filter.get(), state.repair);
if (expectedHash && expectedStorePath != dstPath) if (expectedHash && expectedStorePath != dstPath)
state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path)); state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path));
state.allowAndSetStorePathString(dstPath, v); state.allowAndSetStorePathString(dstPath, v);

16
src/libfetchers/cache.cc
View file

@ -106,7 +106,7 @@ struct CacheImpl : Cache
} }
void add( void add(
ref<Store> store, Store & store,
const Attrs & inAttrs, const Attrs & inAttrs,
const Attrs & infoAttrs, const Attrs & infoAttrs,
const StorePath & storePath, const StorePath & storePath,
@ -115,13 +115,13 @@ struct CacheImpl : Cache
_state.lock()->add.use() _state.lock()->add.use()
(attrsToJSON(inAttrs).dump()) (attrsToJSON(inAttrs).dump())
(attrsToJSON(infoAttrs).dump()) (attrsToJSON(infoAttrs).dump())
(store->printStorePath(storePath)) (store.printStorePath(storePath))
(locked) (locked)
(time(0)).exec(); (time(0)).exec();
} }
std::optional<std::pair<Attrs, StorePath>> lookup( std::optional<std::pair<Attrs, StorePath>> lookup(
ref<Store> store, Store & store,
const Attrs & inAttrs) override const Attrs & inAttrs) override
{ {
if (auto res = lookupExpired(store, inAttrs)) { if (auto res = lookupExpired(store, inAttrs)) {
@ -134,7 +134,7 @@ struct CacheImpl : Cache
} }
std::optional<Result> lookupExpired( std::optional<Result> lookupExpired(
ref<Store> store, Store & store,
const Attrs & inAttrs) override const Attrs & inAttrs) override
{ {
auto state(_state.lock()); auto state(_state.lock());
@ -148,19 +148,19 @@ struct CacheImpl : Cache
} }
auto infoJSON = stmt.getStr(0); auto infoJSON = stmt.getStr(0);
auto storePath = store->parseStorePath(stmt.getStr(1)); auto storePath = store.parseStorePath(stmt.getStr(1));
auto locked = stmt.getInt(2) != 0; auto locked = stmt.getInt(2) != 0;
auto timestamp = stmt.getInt(3); auto timestamp = stmt.getInt(3);
store->addTempRoot(storePath); store.addTempRoot(storePath);
if (!store->isValidPath(storePath)) { if (!store.isValidPath(storePath)) {
// FIXME: we could try to substitute 'storePath'. // FIXME: we could try to substitute 'storePath'.
debug("ignoring disappeared cache entry '%s'", inAttrsJSON); debug("ignoring disappeared cache entry '%s'", inAttrsJSON);
return {}; return {};
} }
debug("using cache entry '%s' -> '%s', '%s'", debug("using cache entry '%s' -> '%s', '%s'",
inAttrsJSON, infoJSON, store->printStorePath(storePath)); inAttrsJSON, infoJSON, store.printStorePath(storePath));
return Result { return Result {
.expired = !locked && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)), .expired = !locked && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)),

6
src/libfetchers/cache.hh
View file

@ -50,14 +50,14 @@ struct Cache
/* Old cache for things that have a store path. */ /* Old cache for things that have a store path. */
virtual void add( virtual void add(
ref<Store> store, Store & store,
const Attrs & inAttrs, const Attrs & inAttrs,
const Attrs & infoAttrs, const Attrs & infoAttrs,
const StorePath & storePath, const StorePath & storePath,
bool locked) = 0; bool locked) = 0;
virtual std::optional<std::pair<Attrs, StorePath>> lookup( virtual std::optional<std::pair<Attrs, StorePath>> lookup(
ref<Store> store, Store & store,
const Attrs & inAttrs) = 0; const Attrs & inAttrs) = 0;
struct Result struct Result
@ -68,7 +68,7 @@ struct Cache
}; };
virtual std::optional<Result> lookupExpired( virtual std::optional<Result> lookupExpired(
ref<Store> store, Store & store,
const Attrs & inAttrs) = 0; const Attrs & inAttrs) = 0;
}; };

2
src/libfetchers/fetchers.cc
View file

@ -374,7 +374,7 @@ void InputScheme::clone(const Input & input, const Path & destDir) const
std::pair<StorePath, Input> InputScheme::fetch(ref<Store> store, const Input & input) std::pair<StorePath, Input> InputScheme::fetch(ref<Store> store, const Input & input)
{ {
auto [accessor, input2] = getAccessor(store, input); auto [accessor, input2] = getAccessor(store, input);
auto storePath = SourcePath(accessor).fetchToStore(store, input2.getName()); auto storePath = SourcePath(accessor).fetchToStore(*store, input2.getName());
return {storePath, input2}; return {storePath, input2};
} }

4
src/libfetchers/git.cc
View file

@ -368,14 +368,14 @@ struct GitInputScheme : InputScheme
RepoInfo getRepoInfo(const Input & input) const RepoInfo getRepoInfo(const Input & input) const
{ {
auto checkHashType = [&](const std::optional<Hash> & hash) auto checkHashAlgorithm = [&](const std::optional<Hash> & hash)
{ {
if (hash.has_value() && !(hash->algo == HashAlgorithm::SHA1 || hash->algo == HashAlgorithm::SHA256)) if (hash.has_value() && !(hash->algo == HashAlgorithm::SHA1 || hash->algo == HashAlgorithm::SHA256))
throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(HashFormat::Base16, true)); throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(HashFormat::Base16, true));
}; };
if (auto rev = input.getRev()) if (auto rev = input.getRev())
checkHashType(rev); checkHashAlgorithm(rev);
RepoInfo repoInfo; RepoInfo repoInfo;

4
src/libfetchers/github.cc
View file

@ -201,7 +201,7 @@ struct GitArchiveInputScheme : InputScheme
{"rev", rev->gitRev()}, {"rev", rev->gitRev()},
}); });
if (auto res = getCache()->lookup(store, lockedAttrs)) { if (auto res = getCache()->lookup(*store, lockedAttrs)) {
input.attrs.insert_or_assign("lastModified", getIntAttr(res->first, "lastModified")); input.attrs.insert_or_assign("lastModified", getIntAttr(res->first, "lastModified"));
return {std::move(res->second), input}; return {std::move(res->second), input};
} }
@ -213,7 +213,7 @@ struct GitArchiveInputScheme : InputScheme
input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified)); input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified));
getCache()->add( getCache()->add(
store, *store,
lockedAttrs, lockedAttrs,
{ {
{"rev", rev->gitRev()}, {"rev", rev->gitRev()},

39
src/libfetchers/input-accessor.cc
View file

@ -5,10 +5,10 @@
namespace nix { namespace nix {
StorePath InputAccessor::fetchToStore( StorePath InputAccessor::fetchToStore(
ref<Store> store, Store & store,
const CanonPath & path, const CanonPath & path,
std::string_view name, std::string_view name,
FileIngestionMethod method, ContentAddressMethod method,
PathFilter * filter, PathFilter * filter,
RepairFlag repair) RepairFlag repair)
{ {
@ -20,10 +20,24 @@ StorePath InputAccessor::fetchToStore(
if (!filter && fingerprint) { if (!filter && fingerprint) {
cacheKey = fetchers::Attrs{ cacheKey = fetchers::Attrs{
{"_what", "fetchToStore"}, {"_what", "fetchToStore"},
{"store", store->storeDir}, {"store", store.storeDir},
{"name", std::string(name)}, {"name", std::string(name)},
{"fingerprint", *fingerprint}, {"fingerprint", *fingerprint},
{"method", (uint8_t) method}, {
"method",
std::visit(overloaded {
[](const TextIngestionMethod &) {
return "text";
},
[](const FileIngestionMethod & fim) {
switch (fim) {
case FileIngestionMethod::Flat: return "flat";
case FileIngestionMethod::Recursive: return "nar";
default: assert(false);
}
},
}, method.raw),
},
{"path", path.abs()} {"path", path.abs()}
}; };
if (auto res = fetchers::getCache()->lookup(store, *cacheKey)) { if (auto res = fetchers::getCache()->lookup(store, *cacheKey)) {
@ -35,17 +49,14 @@ StorePath InputAccessor::fetchToStore(
Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path))); Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path)));
auto source = sinkToSource([&](Sink & sink) { auto filter2 = filter ? *filter : defaultPathFilter;
if (method == FileIngestionMethod::Recursive)
dumpPath(path, sink, filter ? *filter : defaultPathFilter);
else
readFile(path, sink);
});
auto storePath = auto storePath =
settings.readOnlyMode settings.readOnlyMode
? store->computeStorePathFromDump(*source, name, method, HashAlgorithm::SHA256).first ? store.computeStorePath(
: store->addToStoreFromDump(*source, name, method, HashAlgorithm::SHA256, repair); name, *this, path, method, HashAlgorithm::SHA256, {}, filter2).first
: store.addToStore(
name, *this, path, method, HashAlgorithm::SHA256, {}, filter2, repair);
if (cacheKey) if (cacheKey)
fetchers::getCache()->add(store, *cacheKey, {}, storePath, true); fetchers::getCache()->add(store, *cacheKey, {}, storePath, true);
@ -60,9 +71,9 @@ std::ostream & operator << (std::ostream & str, const SourcePath & path)
} }
StorePath SourcePath::fetchToStore( StorePath SourcePath::fetchToStore(
ref<Store> store, Store & store,
std::string_view name, std::string_view name,
FileIngestionMethod method, ContentAddressMethod method,
PathFilter * filter, PathFilter * filter,
RepairFlag repair) const RepairFlag repair) const
{ {

8
src/libfetchers/input-accessor.hh
View file

@ -30,10 +30,10 @@ struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this<Inpu
} }
StorePath fetchToStore( StorePath fetchToStore(
ref<Store> store, Store & store,
const CanonPath & path, const CanonPath & path,
std::string_view name = "source", std::string_view name = "source",
FileIngestionMethod method = FileIngestionMethod::Recursive, ContentAddressMethod method = FileIngestionMethod::Recursive,
PathFilter * filter = nullptr, PathFilter * filter = nullptr,
RepairFlag repair = NoRepair); RepairFlag repair = NoRepair);
}; };
@ -116,9 +116,9 @@ struct SourcePath
* Copy this `SourcePath` to the Nix store. * Copy this `SourcePath` to the Nix store.
*/ */
StorePath fetchToStore( StorePath fetchToStore(
ref<Store> store, Store & store,
std::string_view name = "source", std::string_view name = "source",
FileIngestionMethod method = FileIngestionMethod::Recursive, ContentAddressMethod method = FileIngestionMethod::Recursive,
PathFilter * filter = nullptr, PathFilter * filter = nullptr,
RepairFlag repair = NoRepair) const; RepairFlag repair = NoRepair) const;

25
src/libfetchers/mercurial.cc
View file

@ -6,6 +6,7 @@
#include "tarfile.hh" #include "tarfile.hh"
#include "store-api.hh" #include "store-api.hh"
#include "url-parts.hh" #include "url-parts.hh"
#include "posix-source-accessor.hh"
#include "fetch-settings.hh" #include "fetch-settings.hh"
@ -210,7 +211,12 @@ struct MercurialInputScheme : InputScheme
return files.count(file); return files.count(file);
}; };
auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, HashAlgorithm::SHA256, filter); PosixSourceAccessor accessor;
auto storePath = store->addToStore(
input.getName(),
accessor, CanonPath { actualPath },
FileIngestionMethod::Recursive, HashAlgorithm::SHA256, {},
filter);
return {std::move(storePath), input}; return {std::move(storePath), input};
} }
@ -218,7 +224,7 @@ struct MercurialInputScheme : InputScheme
if (!input.getRef()) input.attrs.insert_or_assign("ref", "default"); if (!input.getRef()) input.attrs.insert_or_assign("ref", "default");
auto checkHashType = [&](const std::optional<Hash> & hash) auto checkHashAlgorithm = [&](const std::optional<Hash> & hash)
{ {
if (hash.has_value() && hash->algo != HashAlgorithm::SHA1) if (hash.has_value() && hash->algo != HashAlgorithm::SHA1)
throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(HashFormat::Base16, true)); throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(HashFormat::Base16, true));
@ -227,7 +233,7 @@ struct MercurialInputScheme : InputScheme
auto getLockedAttrs = [&]() auto getLockedAttrs = [&]()
{ {
checkHashType(input.getRev()); checkHashAlgorithm(input.getRev());
return Attrs({ return Attrs({
{"type", "hg"}, {"type", "hg"},
@ -246,7 +252,7 @@ struct MercurialInputScheme : InputScheme
}; };
if (input.getRev()) { if (input.getRev()) {
if (auto res = getCache()->lookup(store, getLockedAttrs())) if (auto res = getCache()->lookup(*store, getLockedAttrs()))
return makeResult(res->first, std::move(res->second)); return makeResult(res->first, std::move(res->second));
} }
@ -259,7 +265,7 @@ struct MercurialInputScheme : InputScheme
{"ref", *input.getRef()}, {"ref", *input.getRef()},
}); });
if (auto res = getCache()->lookup(store, unlockedAttrs)) { if (auto res = getCache()->lookup(*store, unlockedAttrs)) {
auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), HashAlgorithm::SHA1); auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), HashAlgorithm::SHA1);
if (!input.getRev() || input.getRev() == rev2) { if (!input.getRev() || input.getRev() == rev2) {
input.attrs.insert_or_assign("rev", rev2.gitRev()); input.attrs.insert_or_assign("rev", rev2.gitRev());
@ -305,7 +311,7 @@ struct MercurialInputScheme : InputScheme
auto revCount = std::stoull(tokens[1]); auto revCount = std::stoull(tokens[1]);
input.attrs.insert_or_assign("ref", tokens[2]); input.attrs.insert_or_assign("ref", tokens[2]);
if (auto res = getCache()->lookup(store, getLockedAttrs())) if (auto res = getCache()->lookup(*store, getLockedAttrs()))
return makeResult(res->first, std::move(res->second)); return makeResult(res->first, std::move(res->second));
Path tmpDir = createTempDir(); Path tmpDir = createTempDir();
@ -315,7 +321,8 @@ struct MercurialInputScheme : InputScheme
deletePath(tmpDir + "/.hg_archival.txt"); deletePath(tmpDir + "/.hg_archival.txt");
auto storePath = store->addToStore(name, tmpDir); PosixSourceAccessor accessor;
auto storePath = store->addToStore(name, accessor, CanonPath { tmpDir });
Attrs infoAttrs({ Attrs infoAttrs({
{"rev", input.getRev()->gitRev()}, {"rev", input.getRev()->gitRev()},
@ -324,14 +331,14 @@ struct MercurialInputScheme : InputScheme
if (!_input.getRev()) if (!_input.getRev())
getCache()->add( getCache()->add(
store, *store,
unlockedAttrs, unlockedAttrs,
infoAttrs, infoAttrs,
storePath, storePath,
false); false);
getCache()->add( getCache()->add(
store, *store,
getLockedAttrs(), getLockedAttrs(),
infoAttrs, infoAttrs,
storePath, storePath,

14
src/libfetchers/tarball.cc
View file

@ -8,6 +8,7 @@
#include "tarfile.hh" #include "tarfile.hh"
#include "types.hh" #include "types.hh"
#include "split.hh" #include "split.hh"
#include "posix-source-accessor.hh"
namespace nix::fetchers { namespace nix::fetchers {
@ -26,7 +27,7 @@ DownloadFileResult downloadFile(
{"name", name}, {"name", name},
}); });
auto cached = getCache()->lookupExpired(store, inAttrs); auto cached = getCache()->lookupExpired(*store, inAttrs);
auto useCached = [&]() -> DownloadFileResult auto useCached = [&]() -> DownloadFileResult
{ {
@ -91,7 +92,7 @@ DownloadFileResult downloadFile(
} }
getCache()->add( getCache()->add(
store, *store,
inAttrs, inAttrs,
infoAttrs, infoAttrs,
*storePath, *storePath,
@ -99,7 +100,7 @@ DownloadFileResult downloadFile(
if (url != res.effectiveUri) if (url != res.effectiveUri)
getCache()->add( getCache()->add(
store, *store,
{ {
{"type", "file"}, {"type", "file"},
{"url", res.effectiveUri}, {"url", res.effectiveUri},
@ -130,7 +131,7 @@ DownloadTarballResult downloadTarball(
{"name", name}, {"name", name},
}); });
auto cached = getCache()->lookupExpired(store, inAttrs); auto cached = getCache()->lookupExpired(*store, inAttrs);
if (cached && !cached->expired) if (cached && !cached->expired)
return { return {
@ -156,7 +157,8 @@ DownloadTarballResult downloadTarball(
throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
auto topDir = tmpDir + "/" + members.begin()->name; auto topDir = tmpDir + "/" + members.begin()->name;
lastModified = lstat(topDir).st_mtime; lastModified = lstat(topDir).st_mtime;
unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, HashAlgorithm::SHA256, defaultPathFilter, NoRepair); PosixSourceAccessor accessor;
unpackedStorePath = store->addToStore(name, accessor, CanonPath { topDir }, FileIngestionMethod::Recursive, HashAlgorithm::SHA256, {}, defaultPathFilter, NoRepair);
} }
Attrs infoAttrs({ Attrs infoAttrs({
@ -168,7 +170,7 @@ DownloadTarballResult downloadTarball(
infoAttrs.emplace("immutableUrl", *res.immutableUrl); infoAttrs.emplace("immutableUrl", *res.immutableUrl);
getCache()->add( getCache()->add(
store, *store,
inAttrs, inAttrs,
infoAttrs, infoAttrs,
*unpackedStorePath, *unpackedStorePath,

125
src/libstore/binary-cache-store.cc
View file

@ -12,6 +12,7 @@
#include "thread-pool.hh" #include "thread-pool.hh"
#include "callback.hh" #include "callback.hh"
#include "signals.hh" #include "signals.hh"
#include "archive.hh"
#include <chrono> #include <chrono>
#include <future> #include <future>
@ -300,24 +301,60 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
}}); }});
} }
StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view name, StorePath BinaryCacheStore::addToStoreFromDump(
FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) Source & dump,
std::string_view name,
ContentAddressMethod method,
HashAlgorithm hashAlgo,
const StorePathSet & references,
RepairFlag repair)
{ {
if (method != FileIngestionMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) std::optional<Hash> caHash;
unsupported("addToStoreFromDump"); std::string nar;
return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) {
if (auto * dump2p = dynamic_cast<StringSource *>(&dump)) {
auto & dump2 = *dump2p;
// Hack, this gives us a "replayable" source so we can compute
// multiple hashes more easily.
caHash = hashString(HashAlgorithm::SHA256, dump2.s);
switch (method.getFileIngestionMethod()) {
case FileIngestionMethod::Recursive:
// The dump is already NAR in this case, just use it.
nar = dump2.s;
break;
case FileIngestionMethod::Flat:
// The dump is Flat, so we need to convert it to NAR with a
// single file.
StringSink s;
dumpString(dump2.s, s);
nar = std::move(s.s);
break;
}
} else {
// Otherwise, we have to do th same hashing as NAR so our single
// hash will suffice for both purposes.
if (method != FileIngestionMethod::Recursive || hashAlgo != HashAlgorithm::SHA256)
unsupported("addToStoreFromDump");
}
StringSource narDump { nar };
// Use `narDump` if we wrote to `nar`.
Source & narDump2 = nar.size() > 0
? static_cast<Source &>(narDump)
: dump;
return addToStoreCommon(narDump2, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info { ValidPathInfo info {
*this, *this,
name, name,
FixedOutputInfo { ContentAddressWithReferences::fromParts(
.method = method, method,
.hash = nar.first, caHash ? *caHash : nar.first,
.references = { {
.others = references, .others = references,
// caller is not capable of creating a self-reference, because this is content-addressed without modulus // caller is not capable of creating a self-reference, because this is content-addressed without modulus
.self = false, .self = false,
}, }),
},
nar.first, nar.first,
}; };
info.narSize = nar.second; info.narSize = nar.second;
@ -399,72 +436,36 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
} }
StorePath BinaryCacheStore::addToStore( StorePath BinaryCacheStore::addToStore(
std::string_view name, std::string_view name,
const Path & srcPath, SourceAccessor & accessor,
FileIngestionMethod method, const CanonPath & path,
HashAlgorithm hashAlgo, ContentAddressMethod method,
PathFilter & filter, HashAlgorithm hashAlgo,
RepairFlag repair, const StorePathSet & references,
const StorePathSet & references) PathFilter & filter,
RepairFlag repair)
{ {
/* FIXME: Make BinaryCacheStore::addToStoreCommon support /* FIXME: Make BinaryCacheStore::addToStoreCommon support
non-recursive+sha256 so we can just use the default non-recursive+sha256 so we can just use the default
implementation of this method in terms of addToStoreFromDump. */ implementation of this method in terms of addToStoreFromDump. */
HashSink sink { hashAlgo }; auto h = hashPath(accessor, path, method.getFileIngestionMethod(), hashAlgo, filter).first;
if (method == FileIngestionMethod::Recursive) {
dumpPath(srcPath, sink, filter);
} else {
readFile(srcPath, sink);
}
auto h = sink.finish().first;
auto source = sinkToSource([&](Sink & sink) { auto source = sinkToSource([&](Sink & sink) {
dumpPath(srcPath, sink, filter); accessor.dumpPath(path, sink, filter);
}); });
return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info { ValidPathInfo info {
*this, *this,
name, name,
FixedOutputInfo { ContentAddressWithReferences::fromParts(
.method = method, method,
.hash = h, h,
.references = { {
.others = references, .others = references,
// caller is not capable of creating a self-reference, because this is content-addressed without modulus // caller is not capable of creating a self-reference, because this is content-addressed without modulus
.self = false, .self = false,
}, }),
},
nar.first,
};
info.narSize = nar.second;
return info;
})->path;
}
StorePath BinaryCacheStore::addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references,
RepairFlag repair)
{
auto textHash = hashString(HashAlgorithm::SHA256, s);
auto path = makeTextPath(name, TextInfo { { textHash }, references });
if (!repair && isValidPath(path))
return path;
StringSink sink;
dumpString(s, sink);
StringSource source(sink.s);
return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info {
*this,
std::string { name },
TextInfo {
.hash = textHash,
.references = references,
},
nar.first, nar.first,
}; };
info.narSize = nar.second; info.narSize = nar.second;

24
src/libstore/binary-cache-store.hh
View file

@ -123,22 +123,22 @@ public:
void addToStore(const ValidPathInfo & info, Source & narSource, void addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs) override; RepairFlag repair, CheckSigsFlag checkSigs) override;
StorePath addToStoreFromDump(Source & dump, std::string_view name, StorePath addToStoreFromDump(
FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) override; Source & dump,
std::string_view name,
ContentAddressMethod method,
HashAlgorithm hashAlgo,
const StorePathSet & references,
RepairFlag repair) override;
StorePath addToStore( StorePath addToStore(
std::string_view name,
const Path & srcPath,
FileIngestionMethod method,
HashAlgorithm hashAlgo,
PathFilter & filter,
RepairFlag repair,
const StorePathSet & references) override;
StorePath addTextToStore(
std::string_view name, std::string_view name,
std::string_view s, SourceAccessor & accessor,
const CanonPath & srcPath,
ContentAddressMethod method,
HashAlgorithm hashAlgo,
const StorePathSet & references, const StorePathSet & references,
PathFilter & filter,
RepairFlag repair) override; RepairFlag repair) override;
void registerDrvOutput(const Realisation & info) override; void registerDrvOutput(const Realisation & info) override;

101
src/libstore/build/local-derivation-goal.cc
View file

@ -20,6 +20,7 @@
#include "child.hh" #include "child.hh"
#include "unix-domain-socket.hh" #include "unix-domain-socket.hh"
#include "posix-fs-canonicalise.hh" #include "posix-fs-canonicalise.hh"
#include "posix-source-accessor.hh"
#include <regex> #include <regex>
#include <queue> #include <queue>
@ -1290,13 +1291,14 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In
{ throw Error("queryPathFromHashPart"); } { throw Error("queryPathFromHashPart"); }
StorePath addToStore( StorePath addToStore(
std::string_view name, std::string_view name,
const Path & srcPath, SourceAccessor & accessor,
FileIngestionMethod method, const CanonPath & srcPath,
HashAlgorithm hashAlgo, ContentAddressMethod method,
PathFilter & filter, HashAlgorithm hashAlgo,
RepairFlag repair, const StorePathSet & references,
const StorePathSet & references) override PathFilter & filter,
RepairFlag repair) override
{ throw Error("addToStore"); } { throw Error("addToStore"); }
void addToStore(const ValidPathInfo & info, Source & narSource, void addToStore(const ValidPathInfo & info, Source & narSource,
@ -1306,26 +1308,15 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In
goal.addDependency(info.path); goal.addDependency(info.path);
} }
StorePath addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references,
RepairFlag repair = NoRepair) override
{
auto path = next->addTextToStore(name, s, references, repair);
goal.addDependency(path);
return path;
}
StorePath addToStoreFromDump( StorePath addToStoreFromDump(
Source & dump, Source & dump,
std::string_view name, std::string_view name,
FileIngestionMethod method, ContentAddressMethod method,
HashAlgorithm hashAlgo, HashAlgorithm hashAlgo,
RepairFlag repair, const StorePathSet & references,
const StorePathSet & references) override RepairFlag repair) override
{ {
auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair, references); auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, references, repair);
goal.addDependency(path); goal.addDependency(path);
return path; return path;
} }
@ -2453,8 +2444,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
throw BuildError( throw BuildError(
"output path %1% without valid stats info", "output path %1% without valid stats info",
actualPath); actualPath);
if (outputHash.method == ContentAddressMethod { FileIngestionMethod::Flat } || if (outputHash.method.getFileIngestionMethod() == FileIngestionMethod::Flat)
outputHash.method == ContentAddressMethod { TextIngestionMethod {} })
{ {
/* The output path should be a regular file without execute permission. */ /* The output path should be a regular file without execute permission. */
if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0) if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0)
@ -2466,38 +2456,23 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
rewriteOutput(outputRewrites); rewriteOutput(outputRewrites);
/* FIXME optimize and deduplicate with addToStore */ /* FIXME optimize and deduplicate with addToStore */
std::string oldHashPart { scratchPath->hashPart() }; std::string oldHashPart { scratchPath->hashPart() };
HashModuloSink caSink {outputHash.hashAlgo, oldHashPart }; auto got = ({
std::visit(overloaded { HashModuloSink caSink { outputHash.hashAlgo, oldHashPart };
[&](const TextIngestionMethod &) { PosixSourceAccessor accessor;
readFile(actualPath, caSink); dumpPath(
}, accessor, CanonPath { actualPath },
[&](const FileIngestionMethod & m2) { caSink,
switch (m2) { outputHash.method.getFileIngestionMethod());
case FileIngestionMethod::Recursive: caSink.finish().first;
dumpPath(actualPath, caSink); });
break;
case FileIngestionMethod::Flat:
readFile(actualPath, caSink);
break;
}
},
}, outputHash.method.raw);
auto got = caSink.finish().first;
auto optCA = ContentAddressWithReferences::fromPartsOpt(
outputHash.method,
std::move(got),
rewriteRefs());
if (!optCA) {
// TODO track distinct failure modes separately (at the time of
// writing there is just one but `nullopt` is unclear) so this
// message can't get out of sync.
throw BuildError("output path '%s' has illegal content address, probably a spurious self-reference with text hashing");
}
ValidPathInfo newInfo0 { ValidPathInfo newInfo0 {
worker.store, worker.store,
outputPathName(drv->name, outputName), outputPathName(drv->name, outputName),
std::move(*optCA), ContentAddressWithReferences::fromParts(
outputHash.method,
std::move(got),
rewriteRefs()),
Hash::dummy, Hash::dummy,
}; };
if (*scratchPath != newInfo0.path) { if (*scratchPath != newInfo0.path) {
@ -2511,9 +2486,14 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
std::string(newInfo0.path.hashPart())}}); std::string(newInfo0.path.hashPart())}});
} }
HashResult narHashAndSize = hashPath(HashAlgorithm::SHA256, actualPath); {
newInfo0.narHash = narHashAndSize.first; PosixSourceAccessor accessor;
newInfo0.narSize = narHashAndSize.second; HashResult narHashAndSize = hashPath(
accessor, CanonPath { actualPath },
FileIngestionMethod::Recursive, HashAlgorithm::SHA256);
newInfo0.narHash = narHashAndSize.first;
newInfo0.narSize = narHashAndSize.second;
}
assert(newInfo0.ca); assert(newInfo0.ca);
return newInfo0; return newInfo0;
@ -2531,7 +2511,10 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
std::string { scratchPath->hashPart() }, std::string { scratchPath->hashPart() },
std::string { requiredFinalPath.hashPart() }); std::string { requiredFinalPath.hashPart() });
rewriteOutput(outputRewrites); rewriteOutput(outputRewrites);
auto narHashAndSize = hashPath(HashAlgorithm::SHA256, actualPath); PosixSourceAccessor accessor;
HashResult narHashAndSize = hashPath(
accessor, CanonPath { actualPath },
FileIngestionMethod::Recursive, HashAlgorithm::SHA256);
ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
newInfo0.narSize = narHashAndSize.second; newInfo0.narSize = narHashAndSize.second;
auto refs = rewriteRefs(); auto refs = rewriteRefs();

4
src/libstore/build/worker.cc
View file

@ -519,7 +519,9 @@ bool Worker::pathContentsGood(const StorePath & path)
if (!pathExists(store.printStorePath(path))) if (!pathExists(store.printStorePath(path)))
res = false; res = false;
else { else {
HashResult current = hashPath(info->narHash.algo, store.printStorePath(path)); HashResult current = hashPath(
*store.getFSAccessor(), CanonPath { store.printStorePath(path) },
FileIngestionMethod::Recursive, info->narHash.algo);
Hash nullHash(HashAlgorithm::SHA256); Hash nullHash(HashAlgorithm::SHA256);
res = info->narHash == nullHash || info->narHash == current.first; res = info->narHash == nullHash || info->narHash == current.first;
} }

28
src/libstore/content-address.cc
View file

@ -50,6 +50,18 @@ std::string ContentAddressMethod::render(HashAlgorithm ha) const
}, raw); }, raw);
} }
FileIngestionMethod ContentAddressMethod::getFileIngestionMethod() const
{
return std::visit(overloaded {
[&](const TextIngestionMethod & th) {
return FileIngestionMethod::Flat;
},
[&](const FileIngestionMethod & fim) {
return fim;
}
}, raw);
}
std::string ContentAddress::render() const std::string ContentAddress::render() const
{ {
return std::visit(overloaded { return std::visit(overloaded {
@ -79,7 +91,7 @@ static std::pair<ContentAddressMethod, HashAlgorithm> parseContentAddressMethodP
prefix = *optPrefix; prefix = *optPrefix;
} }
auto parseHashType_ = [&](){ auto parseHashAlgorithm_ = [&](){
auto hashTypeRaw = splitPrefixTo(rest, ':'); auto hashTypeRaw = splitPrefixTo(rest, ':');
if (!hashTypeRaw) if (!hashTypeRaw)
throw UsageError("content address hash must be in form '<algo>:<hash>', but found: %s", wholeInput); throw UsageError("content address hash must be in form '<algo>:<hash>', but found: %s", wholeInput);
@ -90,7 +102,7 @@ static std::pair<ContentAddressMethod, HashAlgorithm> parseContentAddressMethodP
// Switch on prefix // Switch on prefix
if (prefix == "text") { if (prefix == "text") {
// No parsing of the ingestion method, "text" only support flat. // No parsing of the ingestion method, "text" only support flat.
HashAlgorithm hashAlgo = parseHashType_(); HashAlgorithm hashAlgo = parseHashAlgorithm_();
return { return {
TextIngestionMethod {}, TextIngestionMethod {},
std::move(hashAlgo), std::move(hashAlgo),
@ -100,7 +112,7 @@ static std::pair<ContentAddressMethod, HashAlgorithm> parseContentAddressMethodP
auto method = FileIngestionMethod::Flat; auto method = FileIngestionMethod::Flat;
if (splitPrefix(rest, "r:")) if (splitPrefix(rest, "r:"))
method = FileIngestionMethod::Recursive; method = FileIngestionMethod::Recursive;
HashAlgorithm hashAlgo = parseHashType_(); HashAlgorithm hashAlgo = parseHashAlgorithm_();
return { return {
std::move(method), std::move(method),
std::move(hashAlgo), std::move(hashAlgo),
@ -176,13 +188,13 @@ ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const Con
}, ca.method.raw); }, ca.method.raw);
} }
std::optional<ContentAddressWithReferences> ContentAddressWithReferences::fromPartsOpt( ContentAddressWithReferences ContentAddressWithReferences::fromParts(
ContentAddressMethod method, Hash hash, StoreReferences refs) noexcept ContentAddressMethod method, Hash hash, StoreReferences refs)
{ {
return std::visit(overloaded { return std::visit(overloaded {
[&](TextIngestionMethod _) -> std::optional<ContentAddressWithReferences> { [&](TextIngestionMethod _) -> ContentAddressWithReferences {
if (refs.self) if (refs.self)
return std::nullopt; throw Error("self-reference not allowed with text hashing");
return ContentAddressWithReferences { return ContentAddressWithReferences {
TextInfo { TextInfo {
.hash = std::move(hash), .hash = std::move(hash),
@ -190,7 +202,7 @@ std::optional<ContentAddressWithReferences> ContentAddressWithReferences::fromPa
} }
}; };
}, },
[&](FileIngestionMethod m2) -> std::optional<ContentAddressWithReferences> { [&](FileIngestionMethod m2) -> ContentAddressWithReferences {
return ContentAddressWithReferences { return ContentAddressWithReferences {
FixedOutputInfo { FixedOutputInfo {
.method = m2, .method = m2,

46
src/libstore/content-address.hh
View file

@ -4,6 +4,7 @@
#include <variant> #include <variant>
#include "hash.hh" #include "hash.hh"
#include "path.hh" #include "path.hh"
#include "file-content-address.hh"
#include "comparator.hh" #include "comparator.hh"
#include "variant-wrapper.hh" #include "variant-wrapper.hh"
@ -31,22 +32,6 @@ namespace nix {
*/ */
struct TextIngestionMethod : std::monostate { }; struct TextIngestionMethod : std::monostate { };
/**
* An enumeration of the main ways we can serialize file system
* objects.
*/
enum struct FileIngestionMethod : uint8_t {
/**
* Flat-file hashing. Directly ingest the contents of a single file
*/
Flat = 0,
/**
* Recursive (or NAR) hashing. Serializes the file-system object in Nix
* Archive format and ingest that
*/
Recursive = 1
};
/** /**
* Compute the prefix to the hash algorithm which indicates how the * Compute the prefix to the hash algorithm which indicates how the
* files were ingested. * files were ingested.
@ -54,7 +39,7 @@ enum struct FileIngestionMethod : uint8_t {
std::string makeFileIngestionPrefix(FileIngestionMethod m); std::string makeFileIngestionPrefix(FileIngestionMethod m);
/** /**
* An enumeration of all the ways we can serialize file system objects. * An enumeration of all the ways we can content-address store objects.
* *
* Just the type of a content address. Combine with the hash itself, and * Just the type of a content address. Combine with the hash itself, and
* we have a `ContentAddress` as defined below. Combine that, in turn, * we have a `ContentAddress` as defined below. Combine that, in turn,
@ -102,7 +87,15 @@ struct ContentAddressMethod
* *
* The rough inverse of `parse()`. * The rough inverse of `parse()`.
*/ */
std::string render(HashAlgorithm ha) const; std::string render(HashAlgorithm ht) const;
/**
* Get the underlying way to content-address file system objects.
*
* Different ways of hashing store objects may use the same method
* for hashing file systeme objects.
*/
FileIngestionMethod getFileIngestionMethod() const;
}; };
@ -116,11 +109,11 @@ struct ContentAddressMethod
* serialisation methods (flat file vs NAR). Thus, ‘ca’ has one of the * serialisation methods (flat file vs NAR). Thus, ‘ca’ has one of the
* following forms: * following forms:
* *
* - ‘text:sha256:<sha256 hash of file contents>’: For paths * - `TextIngestionMethod`:
* computed by Store::makeTextPath() / Store::addTextToStore(). * ‘text:sha256:<sha256 hash of file contents>’
* *
* - ‘fixed:<r?>:<ht>:<h>’: For paths computed by * - `FixedIngestionMethod`:
* Store::makeFixedOutputPath() / Store::addToStore(). * ‘fixed:<r?>:<hash type>:<hash of file contents>’
*/ */
struct ContentAddress struct ContentAddress
{ {
@ -266,11 +259,12 @@ struct ContentAddressWithReferences
* *
* @param refs References to other store objects or oneself. * @param refs References to other store objects or oneself.
* *
* Do note that not all combinations are supported; `nullopt` is * @note note that all combinations are supported. This is a
* returns for invalid combinations. * *partial function* and exceptions will be thrown for invalid
* combinations.
*/ */
static std::optional<ContentAddressWithReferences> fromPartsOpt( static ContentAddressWithReferences fromParts(
ContentAddressMethod method, Hash hash, StoreReferences refs) noexcept; ContentAddressMethod method, Hash hash, StoreReferences refs);
ContentAddressMethod getMethod() const; ContentAddressMethod getMethod() const;

24
src/libstore/daemon.cc
View file

@ -403,22 +403,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
auto [contentAddressMethod, hashAlgo_] = ContentAddressMethod::parse(camStr); auto [contentAddressMethod, hashAlgo_] = ContentAddressMethod::parse(camStr);
auto hashAlgo = hashAlgo_; // work around clang bug auto hashAlgo = hashAlgo_; // work around clang bug
FramedSource source(from); FramedSource source(from);
// TODO this is essentially RemoteStore::addCAToStore. Move it up to Store. // TODO these two steps are essentially RemoteStore::addCAToStore. Move it up to Store.
return std::visit(overloaded { auto path = store->addToStoreFromDump(source, name, contentAddressMethod, hashAlgo, refs, repair);
[&](const TextIngestionMethod &) { return store->queryPathInfo(path);
if (hashAlgo != HashAlgorithm::SHA256)
throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given",
name, printHashAlgo(hashAlgo));
// We could stream this by changing Store
std::string contents = source.drain();
auto path = store->addTextToStore(name, contents, refs, repair);
return store->queryPathInfo(path);
},
[&](const FileIngestionMethod & fim) {
auto path = store->addToStoreFromDump(source, name, fim, hashAlgo, repair, refs);
return store->queryPathInfo(path);
},
}, contentAddressMethod.raw);
}(); }();
logger->stopWork(); logger->stopWork();
@ -496,7 +483,10 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
std::string s = readString(from); std::string s = readString(from);
auto refs = WorkerProto::Serialise<StorePathSet>::read(*store, rconn); auto refs = WorkerProto::Serialise<StorePathSet>::read(*store, rconn);
logger->startWork(); logger->startWork();
auto path = store->addTextToStore(suffix, s, refs, NoRepair); auto path = ({
StringSource source { s };
store->addToStoreFromDump(source, suffix, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, NoRepair);
});
logger->stopWork(); logger->stopWork();
to << store->printStorePath(path); to << store->printStorePath(path);
break; break;

10
src/libstore/derivations.cc
View file

@ -143,8 +143,14 @@ StorePath writeDerivation(Store & store,
auto suffix = std::string(drv.name) + drvExtension; auto suffix = std::string(drv.name) + drvExtension;
auto contents = drv.unparse(store, false); auto contents = drv.unparse(store, false);
return readOnly || settings.readOnlyMode return readOnly || settings.readOnlyMode
? store.computeStorePathForText(suffix, contents, references) ? store.makeFixedOutputPathFromCA(suffix, TextInfo {
: store.addTextToStore(suffix, contents, references, repair); .hash = hashString(HashAlgorithm::SHA256, contents),
.references = std::move(references),
})
: ({
StringSource s { contents };
store.addToStoreFromDump(s, suffix, TextIngestionMethod {}, HashAlgorithm::SHA256, references, repair);
});
} }

7
src/libstore/dummy-store.cc
View file

@ -58,13 +58,6 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store
RepairFlag repair, CheckSigsFlag checkSigs) override RepairFlag repair, CheckSigsFlag checkSigs) override
{ unsupported("addToStore"); } { unsupported("addToStore"); }
StorePath addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references,
RepairFlag repair) override
{ unsupported("addTextToStore"); }
void narFromPath(const StorePath & path, Sink & sink) override void narFromPath(const StorePath & path, Sink & sink) override
{ unsupported("narFromPath"); } { unsupported("narFromPath"); }

18
src/libstore/legacy-ssh-store.hh
View file

@ -59,21 +59,15 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
{ unsupported("queryPathFromHashPart"); } { unsupported("queryPathFromHashPart"); }
StorePath addToStore( StorePath addToStore(
std::string_view name,
const Path & srcPath,
FileIngestionMethod method,
HashAlgorithm hashAlgo,
PathFilter & filter,
RepairFlag repair,
const StorePathSet & references) override
{ unsupported("addToStore"); }
StorePath addTextToStore(
std::string_view name, std::string_view name,
std::string_view s, SourceAccessor & accessor,
const CanonPath & srcPath,
ContentAddressMethod method,
HashAlgorithm hashAlgo,
const StorePathSet & references, const StorePathSet & references,
PathFilter & filter,
RepairFlag repair) override RepairFlag repair) override
{ unsupported("addTextToStore"); } { unsupported("addToStore"); }
private: private:

147
src/libstore/local-store.cc
View file

@ -13,6 +13,7 @@
#include "compression.hh" #include "compression.hh"
#include "signals.hh" #include "signals.hh"
#include "posix-fs-canonicalise.hh" #include "posix-fs-canonicalise.hh"
#include "posix-source-accessor.hh"
#include <iostream> #include <iostream>
#include <algorithm> #include <algorithm>
@ -1088,11 +1089,22 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
if (info.ca) { if (info.ca) {
auto & specified = *info.ca; auto & specified = *info.ca;
auto actualHash = hashCAPath( auto actualHash = ({
specified.method, HashModuloSink caSink {
specified.hash.algo, specified.hash.algo,
info.path std::string { info.path.hashPart() },
); };
PosixSourceAccessor accessor;
dumpPath(
*getFSAccessor(false),
CanonPath { printStorePath(info.path) },
caSink,
specified.method.getFileIngestionMethod());
ContentAddress {
.method = specified.method,
.hash = caSink.finish().first,
};
});
if (specified.hash != actualHash.hash) { if (specified.hash != actualHash.hash) {
throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s",
printStorePath(info.path), printStorePath(info.path),
@ -1115,8 +1127,13 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
} }
StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name, StorePath LocalStore::addToStoreFromDump(
FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) Source & source0,
std::string_view name,
ContentAddressMethod method,
HashAlgorithm hashAlgo,
const StorePathSet & references,
RepairFlag repair)
{ {
/* For computing the store path. */ /* For computing the store path. */
auto hashSink = std::make_unique<HashSink>(hashAlgo); auto hashSink = std::make_unique<HashSink>(hashAlgo);
@ -1166,25 +1183,21 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
delTempDir = std::make_unique<AutoDelete>(tempDir); delTempDir = std::make_unique<AutoDelete>(tempDir);
tempPath = tempDir + "/x"; tempPath = tempDir + "/x";
if (method == FileIngestionMethod::Recursive) restorePath(tempPath, bothSource, method.getFileIngestionMethod());
restorePath(tempPath, bothSource);
else
writeFile(tempPath, bothSource);
dump.clear(); dump.clear();
} }
auto [hash, size] = hashSink->finish(); auto [hash, size] = hashSink->finish();
ContentAddressWithReferences desc = FixedOutputInfo { auto desc = ContentAddressWithReferences::fromParts(
.method = method, method,
.hash = hash, hash,
.references = { {
.others = references, .others = references,
// caller is not capable of creating a self-reference, because this is content-addressed without modulus // caller is not capable of creating a self-reference, because this is content-addressed without modulus
.self = false, .self = false,
}, });
};
auto dstPath = makeFixedOutputPathFromCA(name, desc); auto dstPath = makeFixedOutputPathFromCA(name, desc);
@ -1207,11 +1220,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
if (inMemory) { if (inMemory) {
StringSource dumpSource { dump }; StringSource dumpSource { dump };
/* Restore from the NAR in memory. */ /* Restore from the buffer in memory. */
if (method == FileIngestionMethod::Recursive) restorePath(realPath, dumpSource, method.getFileIngestionMethod());
restorePath(realPath, dumpSource);
else
writeFile(realPath, dumpSource);
} else { } else {
/* Move the temporary path we restored above. */ /* Move the temporary path we restored above. */
moveFile(tempPath, realPath); moveFile(tempPath, realPath);
@ -1247,58 +1257,6 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
} }
StorePath LocalStore::addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references, RepairFlag repair)
{
auto hash = hashString(HashAlgorithm::SHA256, s);
auto dstPath = makeTextPath(name, TextInfo {
.hash = hash,
.references = references,
});
addTempRoot(dstPath);
if (repair || !isValidPath(dstPath)) {
auto realPath = Store::toRealPath(dstPath);
PathLocks outputLock({realPath});
if (repair || !isValidPath(dstPath)) {
deletePath(realPath);
autoGC();
writeFile(realPath, s);
canonicalisePathMetaData(realPath, {});
StringSink sink;
dumpString(s, sink);
auto narHash = hashString(HashAlgorithm::SHA256, sink.s);
optimisePath(realPath, repair);
ValidPathInfo info { dstPath, narHash };
info.narSize = sink.s.size();
info.references = references;
info.ca = {
.method = TextIngestionMethod {},
.hash = hash,
};
registerValidPath(info);
}
outputLock.setDeletion(true);
}
return dstPath;
}
/* Create a temporary directory in the store that won't be /* Create a temporary directory in the store that won't be
garbage-collected until the returned FD is closed. */ garbage-collected until the returned FD is closed. */
std::pair<Path, AutoCloseFD> LocalStore::createTempDirInStore() std::pair<Path, AutoCloseFD> LocalStore::createTempDirInStore()
@ -1389,7 +1347,10 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
for (auto & link : readDirectory(linksDir)) { for (auto & link : readDirectory(linksDir)) {
printMsg(lvlTalkative, "checking contents of '%s'", link.name); printMsg(lvlTalkative, "checking contents of '%s'", link.name);
Path linkPath = linksDir + "/" + link.name; Path linkPath = linksDir + "/" + link.name;
std::string hash = hashPath(HashAlgorithm::SHA256, linkPath).first.to_string(HashFormat::Nix32, false); PosixSourceAccessor accessor;
std::string hash = hashPath(
accessor, CanonPath { linkPath },
FileIngestionMethod::Recursive, HashAlgorithm::SHA256).first.to_string(HashFormat::Nix32, false);
if (hash != link.name) { if (hash != link.name) {
printError("link '%s' was modified! expected hash '%s', got '%s'", printError("link '%s' was modified! expected hash '%s', got '%s'",
linkPath, link.name, hash); linkPath, link.name, hash);
@ -1696,42 +1657,6 @@ void LocalStore::queryRealisationUncached(const DrvOutput & id,
} }
} }
ContentAddress LocalStore::hashCAPath(
const ContentAddressMethod & method, const HashAlgorithm & hashAlgo,
const StorePath & path)
{
return hashCAPath(method, hashAlgo, Store::toRealPath(path), path.hashPart());
}
ContentAddress LocalStore::hashCAPath(
const ContentAddressMethod & method,
const HashAlgorithm & hashAlgo,
const Path & path,
const std::string_view pathHash
)
{
HashModuloSink caSink ( hashAlgo, std::string(pathHash) );
std::visit(overloaded {
[&](const TextIngestionMethod &) {
readFile(path, caSink);
},
[&](const FileIngestionMethod & m2) {
switch (m2) {
case FileIngestionMethod::Recursive:
dumpPath(path, caSink);
break;
case FileIngestionMethod::Flat:
readFile(path, caSink);
break;
}
},
}, method.raw);
return ContentAddress {
.method = method,
.hash = caSink.finish().first,
};
}
void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log) void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log)
{ {
assert(drvPath.isDerivation()); assert(drvPath.isDerivation());

22
src/libstore/local-store.hh
View file

@ -177,12 +177,11 @@ public:
void addToStore(const ValidPathInfo & info, Source & source, void addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs) override; RepairFlag repair, CheckSigsFlag checkSigs) override;
StorePath addToStoreFromDump(Source & dump, std::string_view name, StorePath addToStoreFromDump(
FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) override; Source & dump,
StorePath addTextToStore(
std::string_view name, std::string_view name,
std::string_view s, ContentAddressMethod method,
HashAlgorithm hashAlgo,
const StorePathSet & references, const StorePathSet & references,
RepairFlag repair) override; RepairFlag repair) override;
@ -350,19 +349,6 @@ private:
void signPathInfo(ValidPathInfo & info); void signPathInfo(ValidPathInfo & info);
void signRealisation(Realisation &); void signRealisation(Realisation &);
// XXX: Make a generic `Store` method
ContentAddress hashCAPath(
const ContentAddressMethod & method,
const HashAlgorithm & hashAlgo,
const StorePath & path);
ContentAddress hashCAPath(
const ContentAddressMethod & method,
const HashAlgorithm & hashAlgo,
const Path & path,
const std::string_view pathHash
);
void addBuildLog(const StorePath & drvPath, std::string_view log) override; void addBuildLog(const StorePath & drvPath, std::string_view log) override;
friend struct LocalDerivationGoal; friend struct LocalDerivationGoal;

15
src/libstore/optimise-store.cc
View file

@ -2,6 +2,7 @@
#include "globals.hh" #include "globals.hh"
#include "signals.hh" #include "signals.hh"
#include "posix-fs-canonicalise.hh" #include "posix-fs-canonicalise.hh"
#include "posix-source-accessor.hh"
#include <cstdlib> #include <cstdlib>
#include <cstring> #include <cstring>
@ -146,7 +147,12 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
Also note that if `path' is a symlink, then we're hashing the Also note that if `path' is a symlink, then we're hashing the
contents of the symlink (i.e. the result of readlink()), not contents of the symlink (i.e. the result of readlink()), not
the contents of the target (which may not even exist). */ the contents of the target (which may not even exist). */
Hash hash = hashPath(HashAlgorithm::SHA256, path).first; Hash hash = ({
PosixSourceAccessor accessor;
hashPath(
accessor, CanonPath { path },
FileIngestionMethod::Recursive, HashAlgorithm::SHA256).first;
});
debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Nix32, true)); debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Nix32, true));
/* Check if this is a known hash. */ /* Check if this is a known hash. */
@ -156,7 +162,12 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
if (pathExists(linkPath)) { if (pathExists(linkPath)) {
auto stLink = lstat(linkPath); auto stLink = lstat(linkPath);
if (st.st_size != stLink.st_size if (st.st_size != stLink.st_size
|| (repair && hash != hashPath(HashAlgorithm::SHA256, linkPath).first)) || (repair && hash != ({
PosixSourceAccessor accessor;
hashPath(
accessor, CanonPath { linkPath },
FileIngestionMethod::Recursive, HashAlgorithm::SHA256).first;
})))
{ {
// XXX: Consider overwriting linkPath with our valid version. // XXX: Consider overwriting linkPath with our valid version.
warn("removing corrupted link '%s'", linkPath); warn("removing corrupted link '%s'", linkPath);

19
src/libstore/remote-store.cc
View file

@ -502,8 +502,13 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
} }
StorePath RemoteStore::addToStoreFromDump(Source & dump, std::string_view name, StorePath RemoteStore::addToStoreFromDump(
FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) Source & dump,
std::string_view name,
ContentAddressMethod method,
HashAlgorithm hashAlgo,
const StorePathSet & references,
RepairFlag repair)
{ {
return addCAToStore(dump, name, method, hashAlgo, references, repair)->path; return addCAToStore(dump, name, method, hashAlgo, references, repair)->path;
} }
@ -603,16 +608,6 @@ void RemoteStore::addMultipleToStore(
} }
StorePath RemoteStore::addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references,
RepairFlag repair)
{
StringSource source(s);
return addCAToStore(source, name, TextIngestionMethod {}, HashAlgorithm::SHA256, references, repair)->path;
}
void RemoteStore::registerDrvOutput(const Realisation & info) void RemoteStore::registerDrvOutput(const Realisation & info)
{ {
auto conn(getConnection()); auto conn(getConnection());

17
src/libstore/remote-store.hh
View file

@ -82,10 +82,15 @@ public:
RepairFlag repair); RepairFlag repair);
/** /**
* Add a content-addressable store path. Does not support references. `dump` will be drained. * Add a content-addressable store path. `dump` will be drained.
*/ */
StorePath addToStoreFromDump(Source & dump, std::string_view name, StorePath addToStoreFromDump(
FileIngestionMethod method = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override; Source & dump,
std::string_view name,
ContentAddressMethod method = FileIngestionMethod::Recursive,
HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
const StorePathSet & references = StorePathSet(),
RepairFlag repair = NoRepair) override;
void addToStore(const ValidPathInfo & info, Source & nar, void addToStore(const ValidPathInfo & info, Source & nar,
RepairFlag repair, CheckSigsFlag checkSigs) override; RepairFlag repair, CheckSigsFlag checkSigs) override;
@ -101,12 +106,6 @@ public:
RepairFlag repair, RepairFlag repair,
CheckSigsFlag checkSigs) override; CheckSigsFlag checkSigs) override;
StorePath addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references,
RepairFlag repair) override;
void registerDrvOutput(const Realisation & info) override; void registerDrvOutput(const Realisation & info) override;
void queryRealisationUncached(const DrvOutput &, void queryRealisationUncached(const DrvOutput &,

124
src/libstore/store-api.cc
View file

@ -205,25 +205,19 @@ StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const Fixed
} }
StorePath StoreDirConfig::makeTextPath(std::string_view name, const TextInfo & info) const
{
assert(info.hash.algo == HashAlgorithm::SHA256);
return makeStorePath(
makeType(*this, "text", StoreReferences {
.others = info.references,
.self = false,
}),
info.hash,
name);
}
StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const
{ {
// New template // New template
return std::visit(overloaded { return std::visit(overloaded {
[&](const TextInfo & ti) { [&](const TextInfo & ti) {
return makeTextPath(name, ti); assert(ti.hash.algo == HashAlgorithm::SHA256);
return makeStorePath(
makeType(*this, "text", StoreReferences {
.others = ti.references,
.self = false,
}),
ti.hash,
name);
}, },
[&](const FixedOutputInfo & foi) { [&](const FixedOutputInfo & foi) {
return makeFixedOutputPath(name, foi); return makeFixedOutputPath(name, foi);
@ -232,54 +226,45 @@ StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const
} }
std::pair<StorePath, Hash> StoreDirConfig::computeStorePathFromDump( std::pair<StorePath, Hash> StoreDirConfig::computeStorePath(
Source & dump,
std::string_view name,
FileIngestionMethod method,
HashAlgorithm hashAlgo,
const StorePathSet & references) const
{
HashSink sink(hashAlgo);
dump.drainInto(sink);
auto h = sink.finish().first;
FixedOutputInfo caInfo {
.method = method,
.hash = h,
.references = {},
};
return std::make_pair(makeFixedOutputPath(name, caInfo), h);
}
StorePath StoreDirConfig::computeStorePathForText(
std::string_view name, std::string_view name,
std::string_view s, SourceAccessor & accessor,
const StorePathSet & references) const const CanonPath & path,
ContentAddressMethod method,
HashAlgorithm hashAlgo,
const StorePathSet & references,
PathFilter & filter) const
{ {
return makeTextPath(name, TextInfo { auto h = hashPath(accessor, path, method.getFileIngestionMethod(), hashAlgo, filter).first;
.hash = hashString(HashAlgorithm::SHA256, s), return {
.references = references, makeFixedOutputPathFromCA(
}); name,
ContentAddressWithReferences::fromParts(
method,
h,
{
.others = references,
.self = false,
})),
h,
};
} }
StorePath Store::addToStore( StorePath Store::addToStore(
std::string_view name, std::string_view name,
const Path & _srcPath, SourceAccessor & accessor,
FileIngestionMethod method, const CanonPath & path,
HashAlgorithm hashAlgo, ContentAddressMethod method,
PathFilter & filter, HashAlgorithm hashAlgo,
RepairFlag repair, const StorePathSet & references,
const StorePathSet & references) PathFilter & filter,
RepairFlag repair)
{ {
Path srcPath(absPath(_srcPath));
auto source = sinkToSource([&](Sink & sink) { auto source = sinkToSource([&](Sink & sink) {
if (method == FileIngestionMethod::Recursive) dumpPath(accessor, path, sink, method.getFileIngestionMethod(), filter);
dumpPath(srcPath, sink, filter);
else
readFile(srcPath, sink);
}); });
return addToStoreFromDump(*source, name, method, hashAlgo, repair, references); return addToStoreFromDump(*source, name, method, hashAlgo, references, repair);
} }
void Store::addMultipleToStore( void Store::addMultipleToStore(
@ -404,9 +389,13 @@ digraph graphname {
fileSink -> caHashSink fileSink -> caHashSink
} }
*/ */
ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, ValidPathInfo Store::addToStoreSlow(
FileIngestionMethod method, HashAlgorithm hashAlgo, std::string_view name,
std::optional<Hash> expectedCAHash) SourceAccessor & accessor,
const CanonPath & srcPath,
ContentAddressMethod method, HashAlgorithm hashAlgo,
const StorePathSet & references,
std::optional<Hash> expectedCAHash)
{ {
HashSink narHashSink { HashAlgorithm::SHA256 }; HashSink narHashSink { HashAlgorithm::SHA256 };
HashSink caHashSink { hashAlgo }; HashSink caHashSink { hashAlgo };
@ -425,7 +414,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
srcPath. The fact that we use scratchpadSink as a temporary buffer here srcPath. The fact that we use scratchpadSink as a temporary buffer here
is an implementation detail. */ is an implementation detail. */
auto fileSource = sinkToSource([&](Sink & scratchpadSink) { auto fileSource = sinkToSource([&](Sink & scratchpadSink) {
dumpPath(srcPath, scratchpadSink); accessor.dumpPath(srcPath, scratchpadSink);
}); });
/* tapped provides the same data as fileSource, but we also write all the /* tapped provides the same data as fileSource, but we also write all the
@ -433,9 +422,11 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
TeeSource tapped { *fileSource, narSink }; TeeSource tapped { *fileSource, narSink };
NullParseSink blank; NullParseSink blank;
auto & parseSink = method == FileIngestionMethod::Flat auto & parseSink = method.getFileIngestionMethod() == FileIngestionMethod::Flat
? (ParseSink &) fileSink ? (ParseSink &) fileSink
: (ParseSink &) blank; : method.getFileIngestionMethod() == FileIngestionMethod::Recursive
? (ParseSink &) blank
: (abort(), (ParseSink &)*(ParseSink *)nullptr); // handled both cases
/* The information that flows from tapped (besides being replicated in /* The information that flows from tapped (besides being replicated in
narSink), is now put in parseSink. */ narSink), is now put in parseSink. */
@ -452,21 +443,24 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
if (expectedCAHash && expectedCAHash != hash) if (expectedCAHash && expectedCAHash != hash)
throw Error("hash mismatch for '%s'", srcPath); throw Error("hash mismatch for '%s'", srcPath);
ValidPathInfo info { ValidPathInfo info {
*this, *this,
name, name,
FixedOutputInfo { ContentAddressWithReferences::fromParts(
.method = method, method,
.hash = hash, hash,
.references = {}, {
}, .others = references,
.self = false,
}),
narHash, narHash,
}; };
info.narSize = narSize; info.narSize = narSize;
if (!isValidPath(info.path)) { if (!isValidPath(info.path)) {
auto source = sinkToSource([&](Sink & scratchpadSink) { auto source = sinkToSource([&](Sink & scratchpadSink) {
dumpPath(srcPath, scratchpadSink); accessor.dumpPath(srcPath, scratchpadSink);
}); });
addToStore(info, *source); addToStore(info, *source);
} }

46
src/libstore/store-api.hh
View file

@ -427,22 +427,28 @@ public:
* libutil/archive.hh). * libutil/archive.hh).
*/ */
virtual StorePath addToStore( virtual StorePath addToStore(
std::string_view name, std::string_view name,
const Path & srcPath, SourceAccessor & accessor,
FileIngestionMethod method = FileIngestionMethod::Recursive, const CanonPath & path,
HashAlgorithm hashAlgo = HashAlgorithm::SHA256, ContentAddressMethod method = FileIngestionMethod::Recursive,
PathFilter & filter = defaultPathFilter, HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet(),
const StorePathSet & references = StorePathSet()); PathFilter & filter = defaultPathFilter,
RepairFlag repair = NoRepair);
/** /**
* Copy the contents of a path to the store and register the * Copy the contents of a path to the store and register the
* validity the resulting path, using a constant amount of * validity the resulting path, using a constant amount of
* memory. * memory.
*/ */
ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath, ValidPathInfo addToStoreSlow(
FileIngestionMethod method = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, std::string_view name,
std::optional<Hash> expectedCAHash = {}); SourceAccessor & accessor,
const CanonPath & path,
ContentAddressMethod method = FileIngestionMethod::Recursive,
HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
const StorePathSet & references = StorePathSet(),
std::optional<Hash> expectedCAHash = {});
/** /**
* Like addToStore(), but the contents of the path are contained * Like addToStore(), but the contents of the path are contained
@ -453,20 +459,14 @@ public:
* *
* \todo remove? * \todo remove?
*/ */
virtual StorePath addToStoreFromDump(Source & dump, std::string_view name, virtual StorePath addToStoreFromDump(
FileIngestionMethod method = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, RepairFlag repair = NoRepair, Source & dump,
const StorePathSet & references = StorePathSet())
{ unsupported("addToStoreFromDump"); }
/**
* Like addToStore, but the contents written to the output path is a
* regular file containing the given string.
*/
virtual StorePath addTextToStore(
std::string_view name, std::string_view name,
std::string_view s, ContentAddressMethod method = FileIngestionMethod::Recursive,
const StorePathSet & references, HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
RepairFlag repair = NoRepair) = 0; const StorePathSet & references = StorePathSet(),
RepairFlag repair = NoRepair)
{ unsupported("addToStoreFromDump"); }
/** /**
* Add a mapping indicating that `deriver!outputName` maps to the output path * Add a mapping indicating that `deriver!outputName` maps to the output path

37
src/libstore/store-dir-config.hh
View file

@ -86,41 +86,20 @@ struct StoreDirConfig : public Config
StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const; StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const;
StorePath makeTextPath(std::string_view name, const TextInfo & info) const;
StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const; StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const;
/** /**
* Read-only variant of addToStoreFromDump(). It returns the store * Read-only variant of addToStore(). It returns the store
* path to which a NAR or flat file would be written. * path for the given file sytem object.
*/ */
std::pair<StorePath, Hash> computeStorePathFromDump( std::pair<StorePath, Hash> computeStorePath(
Source & dump,
std::string_view name, std::string_view name,
FileIngestionMethod method = FileIngestionMethod::Recursive, SourceAccessor & accessor,
const CanonPath & path,
ContentAddressMethod method = FileIngestionMethod::Recursive,
HashAlgorithm hashAlgo = HashAlgorithm::SHA256, HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
const StorePathSet & references = {}) const; const StorePathSet & references = {},
PathFilter & filter = defaultPathFilter) const;
/**
* Preparatory part of addTextToStore().
*
* !!! Computation of the path should take the references given to
* addTextToStore() into account, otherwise we have a (relatively
* minor) security hole: a caller can register a source file with
* bogus references. If there are too many references, the path may
* not be garbage collected when it has to be (not really a problem,
* the caller could create a root anyway), or it may be garbage
* collected when it shouldn't be (more serious).
*
* Hashing the references would solve this (bogus references would
* simply yield a different store path, so other users wouldn't be
* affected), but it has some backwards compatibility issues (the
* hashing scheme changes), so I'm not doing that for now.
*/
StorePath computeStorePathForText(
std::string_view name,
std::string_view s,
const StorePathSet & references) const;
}; };
} }

49
src/libutil/file-content-address.cc Normal file
View file

@ -0,0 +1,49 @@
#include "file-content-address.hh"
#include "archive.hh"
namespace nix {
void dumpPath(
SourceAccessor & accessor, const CanonPath & path,
Sink & sink,
FileIngestionMethod method,
PathFilter & filter)
{
switch (method) {
case FileIngestionMethod::Flat:
accessor.readFile(path, sink);
break;
case FileIngestionMethod::Recursive:
accessor.dumpPath(path, sink, filter);
break;
}
}
void restorePath(
const Path & path,
Source & source,
FileIngestionMethod method)
{
switch (method) {
case FileIngestionMethod::Flat:
writeFile(path, source);
break;
case FileIngestionMethod::Recursive:
restorePath(path, source);
break;
}
}
HashResult hashPath(
SourceAccessor & accessor, const CanonPath & path,
FileIngestionMethod method, HashAlgorithm ht,
PathFilter & filter)
{
HashSink sink { ht };
dumpPath(accessor, path, sink, method, filter);
return sink.finish();
}
}

56
src/libutil/file-content-address.hh Normal file
View file

@ -0,0 +1,56 @@
#pragma once
///@file
#include "source-accessor.hh"
#include "fs-sink.hh"
#include "util.hh"
namespace nix {
/**
* An enumeration of the main ways we can serialize file system
* objects.
*/
enum struct FileIngestionMethod : uint8_t {
/**
* Flat-file hashing. Directly ingest the contents of a single file
*/
Flat = 0,
/**
* Recursive (or NAR) hashing. Serializes the file-system object in
* Nix Archive format and ingest that.
*/
Recursive = 1,
};
/**
* Dump a serialization of the given file system object.
*/
void dumpPath(
SourceAccessor & accessor, const CanonPath & path,
Sink & sink,
FileIngestionMethod method,
PathFilter & filter = defaultPathFilter);
/**
* Restore a serialization of the given file system object.
*
* @TODO use an arbitrary `ParseSink`.
*/
void restorePath(
const Path & path,
Source & source,
FileIngestionMethod method);
/**
* Compute the hash of the given file system object according to the
* given method.
*
* The hash is defined as (essentially) hashString(ht, dumpPath(path)).
*/
HashResult hashPath(
SourceAccessor & accessor, const CanonPath & path,
FileIngestionMethod method, HashAlgorithm ht,
PathFilter & filter = defaultPathFilter);
}

9
src/libutil/hash.cc
View file

@ -367,15 +367,6 @@ HashResult HashSink::currentHash()
} }
HashResult hashPath(
HashAlgorithm ha, const Path & path, PathFilter & filter)
{
HashSink sink(ha);
dumpPath(path, sink, filter);
return sink.finish();
}
Hash compressHash(const Hash & hash, unsigned int newSize) Hash compressHash(const Hash & hash, unsigned int newSize)
{ {
Hash h(hash.algo); Hash h(hash.algo);

7
src/libutil/hash.hh
View file

@ -168,14 +168,11 @@ Hash hashString(HashAlgorithm ha, std::string_view s);
Hash hashFile(HashAlgorithm ha, const Path & path); Hash hashFile(HashAlgorithm ha, const Path & path);
/** /**
* Compute the hash of the given path, serializing as a Nix Archive and * The final hash and the number of bytes digested.
* then hashing that.
* *
* The hash is defined as (essentially) hashString(ht, dumpPath(path)). * @todo Convert to proper struct
*/ */
typedef std::pair<Hash, uint64_t> HashResult; typedef std::pair<Hash, uint64_t> HashResult;
HashResult hashPath(HashAlgorithm ha, const Path & path,
PathFilter & filter = defaultPathFilter);
/** /**
* Compress a hash to the specified number of bytes by cyclically * Compress a hash to the specified number of bytes by cyclically

13
src/nix-env/user-env.cc
View file

@ -104,10 +104,15 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
/* Also write a copy of the list of user environment elements to /* Also write a copy of the list of user environment elements to
the store; we need it for future modifications of the the store; we need it for future modifications of the
environment. */ environment. */
std::ostringstream str; auto manifestFile = ({
manifest.print(state.symbols, str, true); std::ostringstream str;
auto manifestFile = state.store->addTextToStore("env-manifest.nix", manifest.print(state.symbols, str, true);
str.str(), references); // TODO with C++20 we can use str.view() instead and avoid copy.
std::string str2 = str.str();
StringSource source { str2 };
state.store->addToStoreFromDump(
source, "env-manifest.nix", TextIngestionMethod {}, HashAlgorithm::SHA256, references);
});
/* Get the environment builder expression. */ /* Get the environment builder expression. */
Value envBuilder; Value envBuilder;

20
src/nix-store/nix-store.cc
View file

@ -13,6 +13,7 @@
#include "shared.hh" #include "shared.hh"
#include "graphml.hh" #include "graphml.hh"
#include "legacy.hh" #include "legacy.hh"
#include "posix-source-accessor.hh"
#include "path-with-outputs.hh" #include "path-with-outputs.hh"
#include "posix-fs-canonicalise.hh" #include "posix-fs-canonicalise.hh"
@ -175,8 +176,12 @@ static void opAdd(Strings opFlags, Strings opArgs)
{ {
if (!opFlags.empty()) throw UsageError("unknown flag"); if (!opFlags.empty()) throw UsageError("unknown flag");
PosixSourceAccessor accessor;
for (auto & i : opArgs) for (auto & i : opArgs)
cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i))); cout << fmt("%s\n", store->printStorePath(store->addToStore(
std::string(baseNameOf(i)),
accessor,
CanonPath::fromCwd(i))));
} }
@ -196,8 +201,14 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
HashAlgorithm hashAlgo = parseHashAlgo(opArgs.front()); HashAlgorithm hashAlgo = parseHashAlgo(opArgs.front());
opArgs.pop_front(); opArgs.pop_front();
PosixSourceAccessor accessor;
for (auto & i : opArgs) for (auto & i : opArgs)
std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow(baseNameOf(i), i, method, hashAlgo).path)); std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow(
baseNameOf(i),
accessor,
CanonPath::fromCwd(i),
method,
hashAlgo).path));
} }
@ -541,7 +552,10 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise)
if (canonicalise) if (canonicalise)
canonicalisePathMetaData(store->printStorePath(info->path), {}); canonicalisePathMetaData(store->printStorePath(info->path), {});
if (!hashGiven) { if (!hashGiven) {
HashResult hash = hashPath(HashAlgorithm::SHA256, store->printStorePath(info->path)); HashResult hash = hashPath(
*store->getFSAccessor(false), CanonPath { store->printStorePath(info->path) },
FileIngestionMethod::Recursive, HashAlgorithm::SHA256);
info->narHash = hash.first; info->narHash = hash.first;
info->narSize = hash.second; info->narSize = hash.second;
} }

42
src/nix/add-to-store.cc
View file

@ -2,6 +2,7 @@
#include "common-args.hh" #include "common-args.hh"
#include "store-api.hh" #include "store-api.hh"
#include "archive.hh" #include "archive.hh"
#include "posix-source-accessor.hh"
using namespace nix; using namespace nix;
@ -20,7 +21,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand
{ {
Path path; Path path;
std::optional<std::string> namePart; std::optional<std::string> namePart;
FileIngestionMethod ingestionMethod = FileIngestionMethod::Recursive; ContentAddressMethod caMethod = FileIngestionMethod::Recursive;
CmdAddToStore() CmdAddToStore()
{ {
@ -48,7 +49,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand
)", )",
.labels = {"hash-mode"}, .labels = {"hash-mode"},
.handler = {[this](std::string s) { .handler = {[this](std::string s) {
this->ingestionMethod = parseIngestionMethod(s); this->caMethod = parseIngestionMethod(s);
}}, }},
}); });
} }
@ -57,36 +58,17 @@ struct CmdAddToStore : MixDryRun, StoreCommand
{ {
if (!namePart) namePart = baseNameOf(path); if (!namePart) namePart = baseNameOf(path);
StringSink sink; PosixSourceAccessor accessor;
dumpPath(path, sink);
auto narHash = hashString(HashAlgorithm::SHA256, sink.s); auto path2 = CanonPath::fromCwd(path);
Hash hash = narHash; auto storePath = dryRun
if (ingestionMethod == FileIngestionMethod::Flat) { ? store->computeStorePath(
HashSink hsink(HashAlgorithm::SHA256); *namePart, accessor, path2, caMethod, HashAlgorithm::SHA256, {}).first
readFile(path, hsink); : store->addToStoreSlow(
hash = hsink.finish().first; *namePart, accessor, path2, caMethod, HashAlgorithm::SHA256, {}).path;
}
ValidPathInfo info { logger->cout("%s", store->printStorePath(storePath));
*store,
std::move(*namePart),
FixedOutputInfo {
.method = std::move(ingestionMethod),
.hash = std::move(hash),
.references = {},
},
narHash,
};
info.narSize = sink.s.size();
if (!dryRun) {
auto source = StringSource(sink.s);
store->addToStore(info, source);
}
logger->cout("%s", store->printStorePath(info.path));
} }
}; };
@ -110,7 +92,7 @@ struct CmdAddFile : CmdAddToStore
{ {
CmdAddFile() CmdAddFile()
{ {
ingestionMethod = FileIngestionMethod::Flat; caMethod = FileIngestionMethod::Flat;
} }
std::string description() override std::string description() override

6
src/nix/develop.cc
View file

@ -223,7 +223,11 @@ static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore
if (builder != "bash") if (builder != "bash")
throw Error("'nix develop' only works on derivations that use 'bash' as their builder"); throw Error("'nix develop' only works on derivations that use 'bash' as their builder");
auto getEnvShPath = evalStore->addTextToStore("get-env.sh", getEnvSh, {}); auto getEnvShPath = ({
StringSource source { getEnvSh };
evalStore->addToStoreFromDump(
source, "get-env.sh", TextIngestionMethod {}, HashAlgorithm::SHA256, {});
});
drv.args = {store->printStorePath(getEnvShPath)}; drv.args = {store->printStorePath(getEnvShPath)};

11
src/nix/hash.cc
View file

@ -5,6 +5,7 @@
#include "shared.hh" #include "shared.hh"
#include "references.hh" #include "references.hh"
#include "archive.hh" #include "archive.hh"
#include "posix-source-accessor.hh"
using namespace nix; using namespace nix;
@ -88,14 +89,8 @@ struct CmdHashBase : Command
else else
hashSink = std::make_unique<HashSink>(ha); hashSink = std::make_unique<HashSink>(ha);
switch (mode) { PosixSourceAccessor accessor;
case FileIngestionMethod::Flat: dumpPath(accessor, CanonPath::fromCwd(path), *hashSink, mode);
readFile(path, *hashSink);
break;
case FileIngestionMethod::Recursive:
dumpPath(path, *hashSink);
break;
}
Hash h = hashSink->finish().first; Hash h = hashSink->finish().first;
if (truncate && h.hashSize > 20) h = compressHash(h, 20); if (truncate && h.hashSize > 20) h = compressHash(h, 20);

7
src/nix/prefetch.cc
View file

@ -9,6 +9,7 @@
#include "attr-path.hh" #include "attr-path.hh"
#include "eval-inline.hh" #include "eval-inline.hh"
#include "legacy.hh" #include "legacy.hh"
#include "posix-source-accessor.hh"
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
@ -122,7 +123,11 @@ std::tuple<StorePath, Hash> prefetchFile(
Activity act(*logger, lvlChatty, actUnknown, Activity act(*logger, lvlChatty, actUnknown,
fmt("adding '%s' to the store", url)); fmt("adding '%s' to the store", url));
auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashAlgo, expectedHash); PosixSourceAccessor accessor;
auto info = store->addToStoreSlow(
*name,
accessor, CanonPath::fromCwd(tmpFile),
ingestionMethod, hashAlgo, {}, expectedHash);
storePath = info.path; storePath = info.path;
assert(info.ca); assert(info.ca);
hash = info.ca->hash; hash = info.ca->hash;

2
tests/unit/libexpr/primops.cc
View file

@ -604,7 +604,7 @@ namespace nix {
ASSERT_THAT(v, IsStringEq("401b09eab3c013d4ca54922bb802bec8fd5318192b0a75f201d8b3727429080fb337591abd3e44453b954555b7a0812e1081c39b740293f765eae731f5a65ed1")); ASSERT_THAT(v, IsStringEq("401b09eab3c013d4ca54922bb802bec8fd5318192b0a75f201d8b3727429080fb337591abd3e44453b954555b7a0812e1081c39b740293f765eae731f5a65ed1"));
} }
TEST_F(PrimOpTest, hashStringInvalidHashType) { TEST_F(PrimOpTest, hashStringInvalidHashAlgorithm) {
ASSERT_THROW(eval("builtins.hashString \"foobar\" \"asdf\""), Error); ASSERT_THROW(eval("builtins.hashString \"foobar\" \"asdf\""), Error);
} }