1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-06-24 22:11:15 +02:00

Remove global fetcher cache

The cache is now part of fetchers::Settings.
This commit is contained in:
Eelco Dolstra 2025-05-14 14:17:57 +02:00
parent 7d89e46f65
commit efcb9e36a9
23 changed files with 93 additions and 54 deletions

View file

@ -34,7 +34,12 @@ EvalSettings evalSettings {
auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false);
debug("fetching flake search path element '%s''", rest);
auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store);
auto storePath = nix::fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName());
auto storePath = nix::fetchToStore(
state.fetchSettings,
*state.store,
SourcePath(accessor),
FetchMode::Copy,
lockedRef.input.getName());
state.allowPath(storePath);
return state.storePath(storePath);
},
@ -177,7 +182,11 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas
state.store,
state.fetchSettings,
EvalSettings::resolvePseudoUrl(s));
auto storePath = fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy);
auto storePath = fetchToStore(
state.fetchSettings,
*state.store,
SourcePath(accessor),
FetchMode::Copy);
return state.storePath(storePath);
}
@ -185,7 +194,12 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas
experimentalFeatureSettings.require(Xp::Flakes);
auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false);
auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store);
auto storePath = nix::fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName());
auto storePath = nix::fetchToStore(
state.fetchSettings,
*state.store,
SourcePath(accessor),
FetchMode::Copy,
lockedRef.input.getName());
state.allowPath(storePath);
return state.storePath(storePath);
}

View file

@ -45,7 +45,7 @@ ref<InstallableValue> InstallableValue::require(ref<Installable> installable)
std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx)
{
if (v.type() == nPath) {
auto storePath = fetchToStore(*state->store, v.path(), FetchMode::Copy);
auto storePath = fetchToStore(state->fetchSettings, *state->store, v.path(), FetchMode::Copy);
return {{
.path = DerivedPath::Opaque {
.path = std::move(storePath),

View file

@ -2423,6 +2423,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
? *dstPathCached
: [&]() {
auto dstPath = fetchToStore(
fetchSettings,
*store,
path.resolveSymlinks(SymlinkResolution::Ancestors),
settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,
@ -3125,7 +3126,7 @@ std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Pat
store,
fetchSettings,
EvalSettings::resolvePseudoUrl(value));
auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy);
auto storePath = fetchToStore(fetchSettings, *store, SourcePath(accessor), FetchMode::Copy);
return finish(this->storePath(storePath));
} catch (Error & e) {
logWarning({

View file

@ -2545,6 +2545,7 @@ static void addPath(
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
auto dstPath = fetchToStore(
state.fetchSettings,
*state.store,
path.resolveSymlinks(),
settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,

View file

@ -537,11 +537,12 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
auto storePath =
unpack
? fetchToStore(
state.fetchSettings,
*state.store,
fetchers::downloadTarball(state.store, state.fetchSettings, *url),
FetchMode::Copy,
name)
: fetchers::downloadFile(state.store, *url, name).storePath;
: fetchers::downloadFile(state.store, state.fetchSettings, *url, name).storePath;
if (expectedHash) {
auto hash = unpack

View file

@ -1,4 +1,5 @@
#include "nix/fetchers/cache.hh"
#include "nix/fetchers/fetch-settings.hh"
#include "nix/util/users.hh"
#include "nix/store/sqlite.hh"
#include "nix/util/sync.hh"
@ -162,10 +163,12 @@ struct CacheImpl : Cache
}
};
ref<Cache> getCache()
ref<Cache> Settings::getCache() const
{
static auto cache = std::make_shared<CacheImpl>();
return ref<Cache>(cache);
auto cache(_cache.lock());
if (!*cache)
*cache = std::make_shared<CacheImpl>();
return ref<Cache>(*cache);
}
}

View file

@ -1,13 +1,14 @@
#include "nix/fetchers/fetch-to-store.hh"
#include "nix/fetchers/fetchers.hh"
#include "nix/fetchers/fetch-settings.hh"
namespace nix {
fetchers::Cache::Key makeFetchToStoreCacheKey(
const std::string &name,
const std::string &fingerprint,
const std::string & name,
const std::string & fingerprint,
ContentAddressMethod method,
const std::string &path)
const std::string & path)
{
return fetchers::Cache::Key{"fetchToStore", {
{"name", name},
@ -19,6 +20,7 @@ fetchers::Cache::Key makeFetchToStoreCacheKey(
}
StorePath fetchToStore(
const fetchers::Settings & settings,
Store & store,
const SourcePath & path,
FetchMode mode,
@ -34,7 +36,7 @@ StorePath fetchToStore(
if (!filter && path.accessor->fingerprint) {
cacheKey = makeFetchToStoreCacheKey(std::string{name}, *path.accessor->fingerprint, method, path.path.abs());
if (auto res = fetchers::getCache()->lookupStorePath(*cacheKey, store)) {
if (auto res = settings.getCache()->lookupStorePath(*cacheKey, store)) {
debug("store path cache hit for '%s'", path);
return res->storePath;
}
@ -56,7 +58,7 @@ StorePath fetchToStore(
debug(mode == FetchMode::DryRun ? "hashed '%s'" : "copied '%s' to '%s'", path, store.printStorePath(storePath));
if (cacheKey && mode == FetchMode::Copy)
fetchers::getCache()->upsert(*cacheKey, store, {}, storePath);
settings.getCache()->upsert(*cacheKey, store, {}, storePath);
return storePath;
}

View file

@ -198,7 +198,7 @@ std::pair<StorePath, Input> Input::fetchToStore(ref<Store> store) const
try {
auto [accessor, result] = getAccessorUnchecked(store);
auto storePath = nix::fetchToStore(*store, SourcePath(accessor), FetchMode::Copy, result.getName());
auto storePath = nix::fetchToStore(*settings, *store, SourcePath(accessor), FetchMode::Copy, result.getName());
auto narHash = store->queryPathInfo(storePath)->narHash;
result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));

View file

@ -1,6 +1,7 @@
#include "nix/fetchers/git-utils.hh"
#include "nix/fetchers/git-lfs-fetch.hh"
#include "nix/fetchers/cache.hh"
#include "nix/fetchers/fetch-settings.hh"
#include "nix/util/finally.hh"
#include "nix/util/processes.hh"
#include "nix/util/signals.hh"
@ -610,18 +611,18 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
throw Error("Commit signature verification on commit %s failed: %s", rev.gitRev(), output);
}
Hash treeHashToNarHash(const Hash & treeHash) override
Hash treeHashToNarHash(const fetchers::Settings & settings, const Hash & treeHash) override
{
auto accessor = getAccessor(treeHash, false, "");
fetchers::Cache::Key cacheKey{"treeHashToNarHash", {{"treeHash", treeHash.gitRev()}}};
if (auto res = fetchers::getCache()->lookup(cacheKey))
if (auto res = settings.getCache()->lookup(cacheKey))
return Hash::parseAny(fetchers::getStrAttr(*res, "narHash"), HashAlgorithm::SHA256);
auto narHash = accessor->hashPath(CanonPath::root);
fetchers::getCache()->upsert(cacheKey, fetchers::Attrs({{"narHash", narHash.to_string(HashFormat::SRI, true)}}));
settings.getCache()->upsert(cacheKey, fetchers::Attrs({{"narHash", narHash.to_string(HashFormat::SRI, true)}}));
return narHash;
}

View file

@ -480,11 +480,11 @@ struct GitInputScheme : InputScheme
return repoInfo;
}
uint64_t getLastModified(const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const
uint64_t getLastModified(const Settings & settings, const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const
{
Cache::Key key{"gitLastModified", {{"rev", rev.gitRev()}}};
auto cache = getCache();
auto cache = settings.getCache();
if (auto res = cache->lookup(key))
return getIntAttr(*res, "lastModified");
@ -496,11 +496,11 @@ struct GitInputScheme : InputScheme
return lastModified;
}
uint64_t getRevCount(const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const
uint64_t getRevCount(const Settings & settings, const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const
{
Cache::Key key{"gitRevCount", {{"rev", rev.gitRev()}}};
auto cache = getCache();
auto cache = settings.getCache();
if (auto revCountAttrs = cache->lookup(key))
return getIntAttr(*revCountAttrs, "revCount");
@ -678,12 +678,12 @@ struct GitInputScheme : InputScheme
Attrs infoAttrs({
{"rev", rev.gitRev()},
{"lastModified", getLastModified(repoInfo, repoDir, rev)},
{"lastModified", getLastModified(*input.settings, repoInfo, repoDir, rev)},
});
if (!getShallowAttr(input))
infoAttrs.insert_or_assign("revCount",
getRevCount(repoInfo, repoDir, rev));
getRevCount(*input.settings, repoInfo, repoDir, rev));
printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg());
@ -799,7 +799,7 @@ struct GitInputScheme : InputScheme
input.attrs.insert_or_assign("rev", rev.gitRev());
input.attrs.insert_or_assign("revCount",
rev == nullRev ? 0 : getRevCount(repoInfo, repoPath, rev));
rev == nullRev ? 0 : getRevCount(*input.settings, repoInfo, repoPath, rev));
verifyCommit(input, repo);
} else {
@ -818,7 +818,7 @@ struct GitInputScheme : InputScheme
input.attrs.insert_or_assign(
"lastModified",
repoInfo.workdirInfo.headRev
? getLastModified(repoInfo, repoPath, *repoInfo.workdirInfo.headRev)
? getLastModified(*input.settings, repoInfo, repoPath, *repoInfo.workdirInfo.headRev)
: 0);
return {accessor, std::move(input)};

View file

@ -265,7 +265,7 @@ struct GitArchiveInputScheme : InputScheme
input.attrs.erase("ref");
input.attrs.insert_or_assign("rev", rev->gitRev());
auto cache = getCache();
auto cache = input.settings->getCache();
Cache::Key treeHashKey{"gitRevToTreeHash", {{"rev", rev->gitRev()}}};
Cache::Key lastModifiedKey{"gitRevToLastModified", {{"rev", rev->gitRev()}}};
@ -407,7 +407,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
auto json = nlohmann::json::parse(
readFile(
store->toRealPath(
downloadFile(store, url, "source", headers).storePath)));
downloadFile(store, *input.settings, url, "source", headers).storePath)));
return RefInfo {
.rev = Hash::parseAny(std::string { json["sha"] }, HashAlgorithm::SHA1),
@ -481,7 +481,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
auto json = nlohmann::json::parse(
readFile(
store->toRealPath(
downloadFile(store, url, "source", headers).storePath)));
downloadFile(store, *input.settings, url, "source", headers).storePath)));
if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) {
return RefInfo {
@ -551,7 +551,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
std::string refUri;
if (ref == "HEAD") {
auto file = store->toRealPath(
downloadFile(store, fmt("%s/HEAD", base_url), "source", headers).storePath);
downloadFile(store, *input.settings, fmt("%s/HEAD", base_url), "source", headers).storePath);
std::ifstream is(file);
std::string line;
getline(is, line);
@ -567,7 +567,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
std::regex refRegex(refUri);
auto file = store->toRealPath(
downloadFile(store, fmt("%s/info/refs", base_url), "source", headers).storePath);
downloadFile(store, *input.settings, fmt("%s/info/refs", base_url), "source", headers).storePath);
std::ifstream is(file);
std::string line;

View file

@ -91,6 +91,4 @@ struct Cache
Store & store) = 0;
};
ref<Cache> getCache();
}

View file

@ -3,6 +3,8 @@
#include "nix/util/types.hh"
#include "nix/util/configuration.hh"
#include "nix/util/ref.hh"
#include "nix/util/sync.hh"
#include <map>
#include <limits>
@ -11,6 +13,8 @@
namespace nix::fetchers {
struct Cache;
struct Settings : public Config
{
Settings();
@ -110,6 +114,11 @@ struct Settings : public Config
When empty, disables the global flake registry.
)",
{}, true, Xp::Flakes};
ref<Cache> getCache() const;
private:
mutable Sync<std::shared_ptr<Cache>> _cache;
};
}

View file

@ -15,6 +15,7 @@ enum struct FetchMode { DryRun, Copy };
* Copy the `path` to the Nix store.
*/
StorePath fetchToStore(
const fetchers::Settings & settings,
Store & store,
const SourcePath & path,
FetchMode mode,

View file

@ -5,7 +5,7 @@
namespace nix {
namespace fetchers { struct PublicKey; }
namespace fetchers { struct PublicKey; struct Settings; }
/**
* A sink that writes into a Git repository. Note that nothing may be written
@ -115,7 +115,7 @@ struct GitRepo
* Given a Git tree hash, compute the hash of its NAR
* serialisation. This is memoised on-disk.
*/
virtual Hash treeHashToNarHash(const Hash & treeHash) = 0;
virtual Hash treeHashToNarHash(const fetchers::Settings & settings, const Hash & treeHash) = 0;
/**
* If the specified Git object is a directory with a single entry

View file

@ -26,6 +26,7 @@ struct DownloadFileResult
DownloadFileResult downloadFile(
ref<Store> store,
const Settings & settings,
const std::string & url,
const std::string & name,
const Headers & headers = {});

View file

@ -253,13 +253,13 @@ struct MercurialInputScheme : InputScheme
}};
if (!input.getRev()) {
if (auto res = getCache()->lookupWithTTL(refToRevKey))
if (auto res = input.settings->getCache()->lookupWithTTL(refToRevKey))
input.attrs.insert_or_assign("rev", getRevAttr(*res, "rev").gitRev());
}
/* If we have a rev, check if we have a cached store path. */
if (auto rev = input.getRev()) {
if (auto res = getCache()->lookupStorePath(revInfoKey(*rev), *store))
if (auto res = input.settings->getCache()->lookupStorePath(revInfoKey(*rev), *store))
return makeResult(res->value, res->storePath);
}
@ -309,7 +309,7 @@ struct MercurialInputScheme : InputScheme
/* Now that we have the rev, check the cache again for a
cached store path. */
if (auto res = getCache()->lookupStorePath(revInfoKey(rev), *store))
if (auto res = input.settings->getCache()->lookupStorePath(revInfoKey(rev), *store))
return makeResult(res->value, res->storePath);
Path tmpDir = createTempDir();
@ -326,9 +326,9 @@ struct MercurialInputScheme : InputScheme
});
if (!origRev)
getCache()->upsert(refToRevKey, {{"rev", rev.gitRev()}});
input.settings->getCache()->upsert(refToRevKey, {{"rev", rev.gitRev()}});
getCache()->upsert(revInfoKey(rev), *store, infoAttrs, storePath);
input.settings->getCache()->upsert(revInfoKey(rev), *store, infoAttrs, storePath);
return makeResult(infoAttrs, std::move(storePath));
}

View file

@ -4,6 +4,7 @@
#include "nix/fetchers/store-path-accessor.hh"
#include "nix/fetchers/cache.hh"
#include "nix/fetchers/fetch-to-store.hh"
#include "nix/fetchers/fetch-settings.hh"
namespace nix::fetchers {
@ -149,7 +150,7 @@ struct PathInputScheme : InputScheme
auto fp = getFingerprint(store, input);
if (fp) {
auto cacheKey = makeFetchToStoreCacheKey(input.getName(), *fp, method, "/");
fetchers::getCache()->upsert(cacheKey, *store, {}, *storePath);
input.settings->getCache()->upsert(cacheKey, *store, {}, *storePath);
}
/* Trust the lastModified value supplied by the user, if

View file

@ -156,7 +156,7 @@ static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, re
}
if (!isAbsolute(path)) {
auto storePath = downloadFile(store, path, "flake-registry.json").storePath;
auto storePath = downloadFile(store, settings, path, "flake-registry.json").storePath;
if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
store2->addPermRoot(storePath, getCacheDir() + "/flake-registry.json");
path = store->toRealPath(storePath);

View file

@ -9,11 +9,13 @@
#include "nix/fetchers/store-path-accessor.hh"
#include "nix/store/store-api.hh"
#include "nix/fetchers/git-utils.hh"
#include "nix/fetchers/fetch-settings.hh"
namespace nix::fetchers {
DownloadFileResult downloadFile(
ref<Store> store,
const Settings & settings,
const std::string & url,
const std::string & name,
const Headers & headers)
@ -25,7 +27,7 @@ DownloadFileResult downloadFile(
{"name", name},
}}};
auto cached = getCache()->lookupStorePath(key, *store);
auto cached = settings.getCache()->lookupStorePath(key, *store);
auto useCached = [&]() -> DownloadFileResult
{
@ -92,7 +94,7 @@ DownloadFileResult downloadFile(
key.second.insert_or_assign("url", url);
assert(!res.urls.empty());
infoAttrs.insert_or_assign("url", *res.urls.rbegin());
getCache()->upsert(key, *store, infoAttrs, *storePath);
settings.getCache()->upsert(key, *store, infoAttrs, *storePath);
}
return {
@ -104,13 +106,14 @@ DownloadFileResult downloadFile(
}
static DownloadTarballResult downloadTarball_(
const Settings & settings,
const std::string & url,
const Headers & headers,
const std::string & displayPrefix)
{
Cache::Key cacheKey{"tarball", {{"url", url}}};
auto cached = getCache()->lookupExpired(cacheKey);
auto cached = settings.getCache()->lookupExpired(cacheKey);
auto attrsToResult = [&](const Attrs & infoAttrs)
{
@ -196,7 +199,7 @@ static DownloadTarballResult downloadTarball_(
/* Insert a cache entry for every URL in the redirect chain. */
for (auto & url : res->urls) {
cacheKey.second.insert_or_assign("url", url);
getCache()->upsert(cacheKey, infoAttrs);
settings.getCache()->upsert(cacheKey, infoAttrs);
}
// FIXME: add a cache entry for immutableUrl? That could allow
@ -341,7 +344,7 @@ struct FileInputScheme : CurlInputScheme
the Nix store directly, since there is little deduplication
benefit in using the Git cache for single big files like
tarballs. */
auto file = downloadFile(store, getStrAttr(input.attrs, "url"), input.getName());
auto file = downloadFile(store, *input.settings, getStrAttr(input.attrs, "url"), input.getName());
auto narHash = store->queryPathInfo(file.storePath)->narHash;
input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
@ -373,6 +376,7 @@ struct TarballInputScheme : CurlInputScheme
auto input(_input);
auto result = downloadTarball_(
*input.settings,
getStrAttr(input.attrs, "url"),
{},
"«" + input.to_string() + "»");
@ -390,7 +394,7 @@ struct TarballInputScheme : CurlInputScheme
input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified));
input.attrs.insert_or_assign("narHash",
getTarballCache()->treeHashToNarHash(result.treeHash).to_string(HashFormat::SRI, true));
getTarballCache()->treeHashToNarHash(*input.settings, result.treeHash).to_string(HashFormat::SRI, true));
return {result.accessor, input};
}

View file

@ -30,7 +30,7 @@ static StorePath copyInputToStore(
const fetchers::Input & originalInput,
ref<SourceAccessor> accessor)
{
auto storePath = fetchToStore(*state.store, accessor, FetchMode::Copy, input.getName());
auto storePath = fetchToStore(*input.settings, *state.store, accessor, FetchMode::Copy, input.getName());
state.allowPath(storePath);
@ -276,7 +276,7 @@ static Flake readFlake(
state.symbols[setting.name],
std::string(state.forceStringNoCtx(*setting.value, setting.pos, "")));
else if (setting.value->type() == nPath) {
auto storePath = fetchToStore(*state.store, setting.value->path(), FetchMode::Copy);
auto storePath = fetchToStore(state.fetchSettings, *state.store, setting.value->path(), FetchMode::Copy);
flake.config.settings.emplace(
state.symbols[setting.name],
state.store->printStorePath(storePath));

View file

@ -4,9 +4,11 @@
#include "nix/store/filetransfer.hh"
#include "nix/store/store-open.hh"
#include "nix/cmd/legacy.hh"
#include "nix/cmd/common-eval-args.hh"
#include "nix/expr/eval-settings.hh" // for defexpr
#include "nix/util/users.hh"
#include "nix/fetchers/tarball.hh"
#include "nix/fetchers/fetch-settings.hh"
#include "self-exe.hh"
#include "man-pages.hh"
@ -114,7 +116,7 @@ static void update(const StringSet & channelNames)
// We want to download the url to a file to see if it's a tarball while also checking if we
// got redirected in the process, so that we can grab the various parts of a nix channel
// definition from a consistent location if the redirect changes mid-download.
auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url)));
auto result = fetchers::downloadFile(store, fetchSettings, url, std::string(baseNameOf(url)));
auto filename = store->toRealPath(result.storePath);
url = result.effectiveUrl;
@ -128,9 +130,9 @@ static void update(const StringSet & channelNames)
if (!unpacked) {
// Download the channel tarball.
try {
filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz").storePath);
filename = store->toRealPath(fetchers::downloadFile(store, fetchSettings, url + "/nixexprs.tar.xz", "nixexprs.tar.xz").storePath);
} catch (FileTransferError & e) {
filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2").storePath);
filename = store->toRealPath(fetchers::downloadFile(store, fetchSettings, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2").storePath);
}
}
// Regardless of where it came from, add the expression representing this channel to accumulated expression

View file

@ -1488,7 +1488,7 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON
auto originalRef = getFlakeRef();
auto resolvedRef = originalRef.resolve(store);
auto [accessor, lockedRef] = resolvedRef.lazyFetch(store);
auto storePath = fetchToStore(*store, accessor, FetchMode::Copy, lockedRef.input.getName());
auto storePath = fetchToStore(getEvalState()->fetchSettings, *store, accessor, FetchMode::Copy, lockedRef.input.getName());
auto hash = store->queryPathInfo(storePath)->narHash;
if (json) {