1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-06-25 10:41:16 +02:00

Merge pull request #13223 from NixOS/remove-global-fetcher-cache

Remove global fetcher cache
This commit is contained in:
Jörg Thalheim 2025-05-18 15:58:53 +02:00 committed by GitHub
commit f2671886f5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
24 changed files with 94 additions and 54 deletions

View file

@ -34,7 +34,12 @@ EvalSettings evalSettings {
auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false); auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false);
debug("fetching flake search path element '%s''", rest); debug("fetching flake search path element '%s''", rest);
auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store); auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store);
auto storePath = nix::fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName()); auto storePath = nix::fetchToStore(
state.fetchSettings,
*state.store,
SourcePath(accessor),
FetchMode::Copy,
lockedRef.input.getName());
state.allowPath(storePath); state.allowPath(storePath);
return state.storePath(storePath); return state.storePath(storePath);
}, },
@ -177,7 +182,11 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas
state.store, state.store,
state.fetchSettings, state.fetchSettings,
EvalSettings::resolvePseudoUrl(s)); EvalSettings::resolvePseudoUrl(s));
auto storePath = fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy); auto storePath = fetchToStore(
state.fetchSettings,
*state.store,
SourcePath(accessor),
FetchMode::Copy);
return state.storePath(storePath); return state.storePath(storePath);
} }
@ -185,7 +194,12 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas
experimentalFeatureSettings.require(Xp::Flakes); experimentalFeatureSettings.require(Xp::Flakes);
auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false); auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false);
auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store); auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store);
auto storePath = nix::fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName()); auto storePath = nix::fetchToStore(
state.fetchSettings,
*state.store,
SourcePath(accessor),
FetchMode::Copy,
lockedRef.input.getName());
state.allowPath(storePath); state.allowPath(storePath);
return state.storePath(storePath); return state.storePath(storePath);
} }

View file

@ -45,7 +45,7 @@ ref<InstallableValue> InstallableValue::require(ref<Installable> installable)
std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx) std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx)
{ {
if (v.type() == nPath) { if (v.type() == nPath) {
auto storePath = fetchToStore(*state->store, v.path(), FetchMode::Copy); auto storePath = fetchToStore(state->fetchSettings, *state->store, v.path(), FetchMode::Copy);
return {{ return {{
.path = DerivedPath::Opaque { .path = DerivedPath::Opaque {
.path = std::move(storePath), .path = std::move(storePath),

View file

@ -2423,6 +2423,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
? *dstPathCached ? *dstPathCached
: [&]() { : [&]() {
auto dstPath = fetchToStore( auto dstPath = fetchToStore(
fetchSettings,
*store, *store,
path.resolveSymlinks(SymlinkResolution::Ancestors), path.resolveSymlinks(SymlinkResolution::Ancestors),
settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,
@ -3125,7 +3126,7 @@ std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Pat
store, store,
fetchSettings, fetchSettings,
EvalSettings::resolvePseudoUrl(value)); EvalSettings::resolvePseudoUrl(value));
auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy); auto storePath = fetchToStore(fetchSettings, *store, SourcePath(accessor), FetchMode::Copy);
return finish(this->storePath(storePath)); return finish(this->storePath(storePath));
} catch (Error & e) { } catch (Error & e) {
logWarning({ logWarning({

View file

@ -2545,6 +2545,7 @@ static void addPath(
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
auto dstPath = fetchToStore( auto dstPath = fetchToStore(
state.fetchSettings,
*state.store, *state.store,
path.resolveSymlinks(), path.resolveSymlinks(),
settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,

View file

@ -537,11 +537,12 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
auto storePath = auto storePath =
unpack unpack
? fetchToStore( ? fetchToStore(
state.fetchSettings,
*state.store, *state.store,
fetchers::downloadTarball(state.store, state.fetchSettings, *url), fetchers::downloadTarball(state.store, state.fetchSettings, *url),
FetchMode::Copy, FetchMode::Copy,
name) name)
: fetchers::downloadFile(state.store, *url, name).storePath; : fetchers::downloadFile(state.store, state.fetchSettings, *url, name).storePath;
if (expectedHash) { if (expectedHash) {
auto hash = unpack auto hash = unpack

View file

@ -1,4 +1,5 @@
#include "nix/fetchers/cache.hh" #include "nix/fetchers/cache.hh"
#include "nix/fetchers/fetch-settings.hh"
#include "nix/util/users.hh" #include "nix/util/users.hh"
#include "nix/store/sqlite.hh" #include "nix/store/sqlite.hh"
#include "nix/util/sync.hh" #include "nix/util/sync.hh"
@ -162,10 +163,12 @@ struct CacheImpl : Cache
} }
}; };
ref<Cache> getCache() ref<Cache> Settings::getCache() const
{ {
static auto cache = std::make_shared<CacheImpl>(); auto cache(_cache.lock());
return ref<Cache>(cache); if (!*cache)
*cache = std::make_shared<CacheImpl>();
return ref<Cache>(*cache);
} }
} }

View file

@ -1,13 +1,14 @@
#include "nix/fetchers/fetch-to-store.hh" #include "nix/fetchers/fetch-to-store.hh"
#include "nix/fetchers/fetchers.hh" #include "nix/fetchers/fetchers.hh"
#include "nix/fetchers/fetch-settings.hh"
namespace nix { namespace nix {
fetchers::Cache::Key makeFetchToStoreCacheKey( fetchers::Cache::Key makeFetchToStoreCacheKey(
const std::string &name, const std::string & name,
const std::string &fingerprint, const std::string & fingerprint,
ContentAddressMethod method, ContentAddressMethod method,
const std::string &path) const std::string & path)
{ {
return fetchers::Cache::Key{"fetchToStore", { return fetchers::Cache::Key{"fetchToStore", {
{"name", name}, {"name", name},
@ -19,6 +20,7 @@ fetchers::Cache::Key makeFetchToStoreCacheKey(
} }
StorePath fetchToStore( StorePath fetchToStore(
const fetchers::Settings & settings,
Store & store, Store & store,
const SourcePath & path, const SourcePath & path,
FetchMode mode, FetchMode mode,
@ -34,7 +36,7 @@ StorePath fetchToStore(
if (!filter && path.accessor->fingerprint) { if (!filter && path.accessor->fingerprint) {
cacheKey = makeFetchToStoreCacheKey(std::string{name}, *path.accessor->fingerprint, method, path.path.abs()); cacheKey = makeFetchToStoreCacheKey(std::string{name}, *path.accessor->fingerprint, method, path.path.abs());
if (auto res = fetchers::getCache()->lookupStorePath(*cacheKey, store)) { if (auto res = settings.getCache()->lookupStorePath(*cacheKey, store)) {
debug("store path cache hit for '%s'", path); debug("store path cache hit for '%s'", path);
return res->storePath; return res->storePath;
} }
@ -56,7 +58,7 @@ StorePath fetchToStore(
debug(mode == FetchMode::DryRun ? "hashed '%s'" : "copied '%s' to '%s'", path, store.printStorePath(storePath)); debug(mode == FetchMode::DryRun ? "hashed '%s'" : "copied '%s' to '%s'", path, store.printStorePath(storePath));
if (cacheKey && mode == FetchMode::Copy) if (cacheKey && mode == FetchMode::Copy)
fetchers::getCache()->upsert(*cacheKey, store, {}, storePath); settings.getCache()->upsert(*cacheKey, store, {}, storePath);
return storePath; return storePath;
} }

View file

@ -198,7 +198,7 @@ std::pair<StorePath, Input> Input::fetchToStore(ref<Store> store) const
try { try {
auto [accessor, result] = getAccessorUnchecked(store); auto [accessor, result] = getAccessorUnchecked(store);
auto storePath = nix::fetchToStore(*store, SourcePath(accessor), FetchMode::Copy, result.getName()); auto storePath = nix::fetchToStore(*settings, *store, SourcePath(accessor), FetchMode::Copy, result.getName());
auto narHash = store->queryPathInfo(storePath)->narHash; auto narHash = store->queryPathInfo(storePath)->narHash;
result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));

View file

@ -1,6 +1,7 @@
#include "nix/fetchers/git-utils.hh" #include "nix/fetchers/git-utils.hh"
#include "nix/fetchers/git-lfs-fetch.hh" #include "nix/fetchers/git-lfs-fetch.hh"
#include "nix/fetchers/cache.hh" #include "nix/fetchers/cache.hh"
#include "nix/fetchers/fetch-settings.hh"
#include "nix/util/finally.hh" #include "nix/util/finally.hh"
#include "nix/util/processes.hh" #include "nix/util/processes.hh"
#include "nix/util/signals.hh" #include "nix/util/signals.hh"
@ -610,18 +611,18 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
throw Error("Commit signature verification on commit %s failed: %s", rev.gitRev(), output); throw Error("Commit signature verification on commit %s failed: %s", rev.gitRev(), output);
} }
Hash treeHashToNarHash(const Hash & treeHash) override Hash treeHashToNarHash(const fetchers::Settings & settings, const Hash & treeHash) override
{ {
auto accessor = getAccessor(treeHash, false, ""); auto accessor = getAccessor(treeHash, false, "");
fetchers::Cache::Key cacheKey{"treeHashToNarHash", {{"treeHash", treeHash.gitRev()}}}; fetchers::Cache::Key cacheKey{"treeHashToNarHash", {{"treeHash", treeHash.gitRev()}}};
if (auto res = fetchers::getCache()->lookup(cacheKey)) if (auto res = settings.getCache()->lookup(cacheKey))
return Hash::parseAny(fetchers::getStrAttr(*res, "narHash"), HashAlgorithm::SHA256); return Hash::parseAny(fetchers::getStrAttr(*res, "narHash"), HashAlgorithm::SHA256);
auto narHash = accessor->hashPath(CanonPath::root); auto narHash = accessor->hashPath(CanonPath::root);
fetchers::getCache()->upsert(cacheKey, fetchers::Attrs({{"narHash", narHash.to_string(HashFormat::SRI, true)}})); settings.getCache()->upsert(cacheKey, fetchers::Attrs({{"narHash", narHash.to_string(HashFormat::SRI, true)}}));
return narHash; return narHash;
} }

View file

@ -480,11 +480,11 @@ struct GitInputScheme : InputScheme
return repoInfo; return repoInfo;
} }
uint64_t getLastModified(const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const uint64_t getLastModified(const Settings & settings, const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const
{ {
Cache::Key key{"gitLastModified", {{"rev", rev.gitRev()}}}; Cache::Key key{"gitLastModified", {{"rev", rev.gitRev()}}};
auto cache = getCache(); auto cache = settings.getCache();
if (auto res = cache->lookup(key)) if (auto res = cache->lookup(key))
return getIntAttr(*res, "lastModified"); return getIntAttr(*res, "lastModified");
@ -496,11 +496,11 @@ struct GitInputScheme : InputScheme
return lastModified; return lastModified;
} }
uint64_t getRevCount(const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const uint64_t getRevCount(const Settings & settings, const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const
{ {
Cache::Key key{"gitRevCount", {{"rev", rev.gitRev()}}}; Cache::Key key{"gitRevCount", {{"rev", rev.gitRev()}}};
auto cache = getCache(); auto cache = settings.getCache();
if (auto revCountAttrs = cache->lookup(key)) if (auto revCountAttrs = cache->lookup(key))
return getIntAttr(*revCountAttrs, "revCount"); return getIntAttr(*revCountAttrs, "revCount");
@ -678,12 +678,12 @@ struct GitInputScheme : InputScheme
Attrs infoAttrs({ Attrs infoAttrs({
{"rev", rev.gitRev()}, {"rev", rev.gitRev()},
{"lastModified", getLastModified(repoInfo, repoDir, rev)}, {"lastModified", getLastModified(*input.settings, repoInfo, repoDir, rev)},
}); });
if (!getShallowAttr(input)) if (!getShallowAttr(input))
infoAttrs.insert_or_assign("revCount", infoAttrs.insert_or_assign("revCount",
getRevCount(repoInfo, repoDir, rev)); getRevCount(*input.settings, repoInfo, repoDir, rev));
printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg()); printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg());
@ -799,7 +799,7 @@ struct GitInputScheme : InputScheme
input.attrs.insert_or_assign("rev", rev.gitRev()); input.attrs.insert_or_assign("rev", rev.gitRev());
input.attrs.insert_or_assign("revCount", input.attrs.insert_or_assign("revCount",
rev == nullRev ? 0 : getRevCount(repoInfo, repoPath, rev)); rev == nullRev ? 0 : getRevCount(*input.settings, repoInfo, repoPath, rev));
verifyCommit(input, repo); verifyCommit(input, repo);
} else { } else {
@ -818,7 +818,7 @@ struct GitInputScheme : InputScheme
input.attrs.insert_or_assign( input.attrs.insert_or_assign(
"lastModified", "lastModified",
repoInfo.workdirInfo.headRev repoInfo.workdirInfo.headRev
? getLastModified(repoInfo, repoPath, *repoInfo.workdirInfo.headRev) ? getLastModified(*input.settings, repoInfo, repoPath, *repoInfo.workdirInfo.headRev)
: 0); : 0);
return {accessor, std::move(input)}; return {accessor, std::move(input)};

View file

@ -265,7 +265,7 @@ struct GitArchiveInputScheme : InputScheme
input.attrs.erase("ref"); input.attrs.erase("ref");
input.attrs.insert_or_assign("rev", rev->gitRev()); input.attrs.insert_or_assign("rev", rev->gitRev());
auto cache = getCache(); auto cache = input.settings->getCache();
Cache::Key treeHashKey{"gitRevToTreeHash", {{"rev", rev->gitRev()}}}; Cache::Key treeHashKey{"gitRevToTreeHash", {{"rev", rev->gitRev()}}};
Cache::Key lastModifiedKey{"gitRevToLastModified", {{"rev", rev->gitRev()}}}; Cache::Key lastModifiedKey{"gitRevToLastModified", {{"rev", rev->gitRev()}}};
@ -407,7 +407,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
auto json = nlohmann::json::parse( auto json = nlohmann::json::parse(
readFile( readFile(
store->toRealPath( store->toRealPath(
downloadFile(store, url, "source", headers).storePath))); downloadFile(store, *input.settings, url, "source", headers).storePath)));
return RefInfo { return RefInfo {
.rev = Hash::parseAny(std::string { json["sha"] }, HashAlgorithm::SHA1), .rev = Hash::parseAny(std::string { json["sha"] }, HashAlgorithm::SHA1),
@ -481,7 +481,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
auto json = nlohmann::json::parse( auto json = nlohmann::json::parse(
readFile( readFile(
store->toRealPath( store->toRealPath(
downloadFile(store, url, "source", headers).storePath))); downloadFile(store, *input.settings, url, "source", headers).storePath)));
if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) { if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) {
return RefInfo { return RefInfo {
@ -551,7 +551,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
std::string refUri; std::string refUri;
if (ref == "HEAD") { if (ref == "HEAD") {
auto file = store->toRealPath( auto file = store->toRealPath(
downloadFile(store, fmt("%s/HEAD", base_url), "source", headers).storePath); downloadFile(store, *input.settings, fmt("%s/HEAD", base_url), "source", headers).storePath);
std::ifstream is(file); std::ifstream is(file);
std::string line; std::string line;
getline(is, line); getline(is, line);
@ -567,7 +567,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
std::regex refRegex(refUri); std::regex refRegex(refUri);
auto file = store->toRealPath( auto file = store->toRealPath(
downloadFile(store, fmt("%s/info/refs", base_url), "source", headers).storePath); downloadFile(store, *input.settings, fmt("%s/info/refs", base_url), "source", headers).storePath);
std::ifstream is(file); std::ifstream is(file);
std::string line; std::string line;

View file

@ -91,6 +91,4 @@ struct Cache
Store & store) = 0; Store & store) = 0;
}; };
ref<Cache> getCache();
} }

View file

@ -3,6 +3,8 @@
#include "nix/util/types.hh" #include "nix/util/types.hh"
#include "nix/util/configuration.hh" #include "nix/util/configuration.hh"
#include "nix/util/ref.hh"
#include "nix/util/sync.hh"
#include <map> #include <map>
#include <limits> #include <limits>
@ -11,6 +13,8 @@
namespace nix::fetchers { namespace nix::fetchers {
struct Cache;
struct Settings : public Config struct Settings : public Config
{ {
Settings(); Settings();
@ -110,6 +114,11 @@ struct Settings : public Config
When empty, disables the global flake registry. When empty, disables the global flake registry.
)", )",
{}, true, Xp::Flakes}; {}, true, Xp::Flakes};
ref<Cache> getCache() const;
private:
mutable Sync<std::shared_ptr<Cache>> _cache;
}; };
} }

View file

@ -15,6 +15,7 @@ enum struct FetchMode { DryRun, Copy };
* Copy the `path` to the Nix store. * Copy the `path` to the Nix store.
*/ */
StorePath fetchToStore( StorePath fetchToStore(
const fetchers::Settings & settings,
Store & store, Store & store,
const SourcePath & path, const SourcePath & path,
FetchMode mode, FetchMode mode,

View file

@ -5,7 +5,7 @@
namespace nix { namespace nix {
namespace fetchers { struct PublicKey; } namespace fetchers { struct PublicKey; struct Settings; }
/** /**
* A sink that writes into a Git repository. Note that nothing may be written * A sink that writes into a Git repository. Note that nothing may be written
@ -115,7 +115,7 @@ struct GitRepo
* Given a Git tree hash, compute the hash of its NAR * Given a Git tree hash, compute the hash of its NAR
* serialisation. This is memoised on-disk. * serialisation. This is memoised on-disk.
*/ */
virtual Hash treeHashToNarHash(const Hash & treeHash) = 0; virtual Hash treeHashToNarHash(const fetchers::Settings & settings, const Hash & treeHash) = 0;
/** /**
* If the specified Git object is a directory with a single entry * If the specified Git object is a directory with a single entry

View file

@ -26,6 +26,7 @@ struct DownloadFileResult
DownloadFileResult downloadFile( DownloadFileResult downloadFile(
ref<Store> store, ref<Store> store,
const Settings & settings,
const std::string & url, const std::string & url,
const std::string & name, const std::string & name,
const Headers & headers = {}); const Headers & headers = {});

View file

@ -253,13 +253,13 @@ struct MercurialInputScheme : InputScheme
}}; }};
if (!input.getRev()) { if (!input.getRev()) {
if (auto res = getCache()->lookupWithTTL(refToRevKey)) if (auto res = input.settings->getCache()->lookupWithTTL(refToRevKey))
input.attrs.insert_or_assign("rev", getRevAttr(*res, "rev").gitRev()); input.attrs.insert_or_assign("rev", getRevAttr(*res, "rev").gitRev());
} }
/* If we have a rev, check if we have a cached store path. */ /* If we have a rev, check if we have a cached store path. */
if (auto rev = input.getRev()) { if (auto rev = input.getRev()) {
if (auto res = getCache()->lookupStorePath(revInfoKey(*rev), *store)) if (auto res = input.settings->getCache()->lookupStorePath(revInfoKey(*rev), *store))
return makeResult(res->value, res->storePath); return makeResult(res->value, res->storePath);
} }
@ -309,7 +309,7 @@ struct MercurialInputScheme : InputScheme
/* Now that we have the rev, check the cache again for a /* Now that we have the rev, check the cache again for a
cached store path. */ cached store path. */
if (auto res = getCache()->lookupStorePath(revInfoKey(rev), *store)) if (auto res = input.settings->getCache()->lookupStorePath(revInfoKey(rev), *store))
return makeResult(res->value, res->storePath); return makeResult(res->value, res->storePath);
Path tmpDir = createTempDir(); Path tmpDir = createTempDir();
@ -326,9 +326,9 @@ struct MercurialInputScheme : InputScheme
}); });
if (!origRev) if (!origRev)
getCache()->upsert(refToRevKey, {{"rev", rev.gitRev()}}); input.settings->getCache()->upsert(refToRevKey, {{"rev", rev.gitRev()}});
getCache()->upsert(revInfoKey(rev), *store, infoAttrs, storePath); input.settings->getCache()->upsert(revInfoKey(rev), *store, infoAttrs, storePath);
return makeResult(infoAttrs, std::move(storePath)); return makeResult(infoAttrs, std::move(storePath));
} }

View file

@ -4,6 +4,7 @@
#include "nix/fetchers/store-path-accessor.hh" #include "nix/fetchers/store-path-accessor.hh"
#include "nix/fetchers/cache.hh" #include "nix/fetchers/cache.hh"
#include "nix/fetchers/fetch-to-store.hh" #include "nix/fetchers/fetch-to-store.hh"
#include "nix/fetchers/fetch-settings.hh"
namespace nix::fetchers { namespace nix::fetchers {
@ -149,7 +150,7 @@ struct PathInputScheme : InputScheme
auto fp = getFingerprint(store, input); auto fp = getFingerprint(store, input);
if (fp) { if (fp) {
auto cacheKey = makeFetchToStoreCacheKey(input.getName(), *fp, method, "/"); auto cacheKey = makeFetchToStoreCacheKey(input.getName(), *fp, method, "/");
fetchers::getCache()->upsert(cacheKey, *store, {}, *storePath); input.settings->getCache()->upsert(cacheKey, *store, {}, *storePath);
} }
/* Trust the lastModified value supplied by the user, if /* Trust the lastModified value supplied by the user, if

View file

@ -156,7 +156,7 @@ static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, re
} }
if (!isAbsolute(path)) { if (!isAbsolute(path)) {
auto storePath = downloadFile(store, path, "flake-registry.json").storePath; auto storePath = downloadFile(store, settings, path, "flake-registry.json").storePath;
if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
store2->addPermRoot(storePath, getCacheDir() + "/flake-registry.json"); store2->addPermRoot(storePath, getCacheDir() + "/flake-registry.json");
path = store->toRealPath(storePath); path = store->toRealPath(storePath);

View file

@ -9,11 +9,13 @@
#include "nix/fetchers/store-path-accessor.hh" #include "nix/fetchers/store-path-accessor.hh"
#include "nix/store/store-api.hh" #include "nix/store/store-api.hh"
#include "nix/fetchers/git-utils.hh" #include "nix/fetchers/git-utils.hh"
#include "nix/fetchers/fetch-settings.hh"
namespace nix::fetchers { namespace nix::fetchers {
DownloadFileResult downloadFile( DownloadFileResult downloadFile(
ref<Store> store, ref<Store> store,
const Settings & settings,
const std::string & url, const std::string & url,
const std::string & name, const std::string & name,
const Headers & headers) const Headers & headers)
@ -25,7 +27,7 @@ DownloadFileResult downloadFile(
{"name", name}, {"name", name},
}}}; }}};
auto cached = getCache()->lookupStorePath(key, *store); auto cached = settings.getCache()->lookupStorePath(key, *store);
auto useCached = [&]() -> DownloadFileResult auto useCached = [&]() -> DownloadFileResult
{ {
@ -92,7 +94,7 @@ DownloadFileResult downloadFile(
key.second.insert_or_assign("url", url); key.second.insert_or_assign("url", url);
assert(!res.urls.empty()); assert(!res.urls.empty());
infoAttrs.insert_or_assign("url", *res.urls.rbegin()); infoAttrs.insert_or_assign("url", *res.urls.rbegin());
getCache()->upsert(key, *store, infoAttrs, *storePath); settings.getCache()->upsert(key, *store, infoAttrs, *storePath);
} }
return { return {
@ -104,13 +106,14 @@ DownloadFileResult downloadFile(
} }
static DownloadTarballResult downloadTarball_( static DownloadTarballResult downloadTarball_(
const Settings & settings,
const std::string & url, const std::string & url,
const Headers & headers, const Headers & headers,
const std::string & displayPrefix) const std::string & displayPrefix)
{ {
Cache::Key cacheKey{"tarball", {{"url", url}}}; Cache::Key cacheKey{"tarball", {{"url", url}}};
auto cached = getCache()->lookupExpired(cacheKey); auto cached = settings.getCache()->lookupExpired(cacheKey);
auto attrsToResult = [&](const Attrs & infoAttrs) auto attrsToResult = [&](const Attrs & infoAttrs)
{ {
@ -196,7 +199,7 @@ static DownloadTarballResult downloadTarball_(
/* Insert a cache entry for every URL in the redirect chain. */ /* Insert a cache entry for every URL in the redirect chain. */
for (auto & url : res->urls) { for (auto & url : res->urls) {
cacheKey.second.insert_or_assign("url", url); cacheKey.second.insert_or_assign("url", url);
getCache()->upsert(cacheKey, infoAttrs); settings.getCache()->upsert(cacheKey, infoAttrs);
} }
// FIXME: add a cache entry for immutableUrl? That could allow // FIXME: add a cache entry for immutableUrl? That could allow
@ -341,7 +344,7 @@ struct FileInputScheme : CurlInputScheme
the Nix store directly, since there is little deduplication the Nix store directly, since there is little deduplication
benefit in using the Git cache for single big files like benefit in using the Git cache for single big files like
tarballs. */ tarballs. */
auto file = downloadFile(store, getStrAttr(input.attrs, "url"), input.getName()); auto file = downloadFile(store, *input.settings, getStrAttr(input.attrs, "url"), input.getName());
auto narHash = store->queryPathInfo(file.storePath)->narHash; auto narHash = store->queryPathInfo(file.storePath)->narHash;
input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
@ -373,6 +376,7 @@ struct TarballInputScheme : CurlInputScheme
auto input(_input); auto input(_input);
auto result = downloadTarball_( auto result = downloadTarball_(
*input.settings,
getStrAttr(input.attrs, "url"), getStrAttr(input.attrs, "url"),
{}, {},
"«" + input.to_string() + "»"); "«" + input.to_string() + "»");
@ -390,7 +394,7 @@ struct TarballInputScheme : CurlInputScheme
input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified)); input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified));
input.attrs.insert_or_assign("narHash", input.attrs.insert_or_assign("narHash",
getTarballCache()->treeHashToNarHash(result.treeHash).to_string(HashFormat::SRI, true)); getTarballCache()->treeHashToNarHash(*input.settings, result.treeHash).to_string(HashFormat::SRI, true));
return {result.accessor, input}; return {result.accessor, input};
} }

View file

@ -30,7 +30,7 @@ static StorePath copyInputToStore(
const fetchers::Input & originalInput, const fetchers::Input & originalInput,
ref<SourceAccessor> accessor) ref<SourceAccessor> accessor)
{ {
auto storePath = fetchToStore(*state.store, accessor, FetchMode::Copy, input.getName()); auto storePath = fetchToStore(*input.settings, *state.store, accessor, FetchMode::Copy, input.getName());
state.allowPath(storePath); state.allowPath(storePath);
@ -276,7 +276,7 @@ static Flake readFlake(
state.symbols[setting.name], state.symbols[setting.name],
std::string(state.forceStringNoCtx(*setting.value, setting.pos, ""))); std::string(state.forceStringNoCtx(*setting.value, setting.pos, "")));
else if (setting.value->type() == nPath) { else if (setting.value->type() == nPath) {
auto storePath = fetchToStore(*state.store, setting.value->path(), FetchMode::Copy); auto storePath = fetchToStore(state.fetchSettings, *state.store, setting.value->path(), FetchMode::Copy);
flake.config.settings.emplace( flake.config.settings.emplace(
state.symbols[setting.name], state.symbols[setting.name],
state.store->printStorePath(storePath)); state.store->printStorePath(storePath));

View file

@ -39,6 +39,7 @@ public:
SyncBase() { } SyncBase() { }
SyncBase(const T & data) : data(data) { } SyncBase(const T & data) : data(data) { }
SyncBase(T && data) noexcept : data(std::move(data)) { } SyncBase(T && data) noexcept : data(std::move(data)) { }
SyncBase(SyncBase && other) noexcept : data(std::move(*other.lock())) { }
template<class L> template<class L>
class Lock class Lock

View file

@ -4,9 +4,11 @@
#include "nix/store/filetransfer.hh" #include "nix/store/filetransfer.hh"
#include "nix/store/store-open.hh" #include "nix/store/store-open.hh"
#include "nix/cmd/legacy.hh" #include "nix/cmd/legacy.hh"
#include "nix/cmd/common-eval-args.hh"
#include "nix/expr/eval-settings.hh" // for defexpr #include "nix/expr/eval-settings.hh" // for defexpr
#include "nix/util/users.hh" #include "nix/util/users.hh"
#include "nix/fetchers/tarball.hh" #include "nix/fetchers/tarball.hh"
#include "nix/fetchers/fetch-settings.hh"
#include "self-exe.hh" #include "self-exe.hh"
#include "man-pages.hh" #include "man-pages.hh"
@ -114,7 +116,7 @@ static void update(const StringSet & channelNames)
// We want to download the url to a file to see if it's a tarball while also checking if we // We want to download the url to a file to see if it's a tarball while also checking if we
// got redirected in the process, so that we can grab the various parts of a nix channel // got redirected in the process, so that we can grab the various parts of a nix channel
// definition from a consistent location if the redirect changes mid-download. // definition from a consistent location if the redirect changes mid-download.
auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url))); auto result = fetchers::downloadFile(store, fetchSettings, url, std::string(baseNameOf(url)));
auto filename = store->toRealPath(result.storePath); auto filename = store->toRealPath(result.storePath);
url = result.effectiveUrl; url = result.effectiveUrl;
@ -128,9 +130,9 @@ static void update(const StringSet & channelNames)
if (!unpacked) { if (!unpacked) {
// Download the channel tarball. // Download the channel tarball.
try { try {
filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz").storePath); filename = store->toRealPath(fetchers::downloadFile(store, fetchSettings, url + "/nixexprs.tar.xz", "nixexprs.tar.xz").storePath);
} catch (FileTransferError & e) { } catch (FileTransferError & e) {
filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2").storePath); filename = store->toRealPath(fetchers::downloadFile(store, fetchSettings, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2").storePath);
} }
} }
// Regardless of where it came from, add the expression representing this channel to accumulated expression // Regardless of where it came from, add the expression representing this channel to accumulated expression

View file

@ -1488,7 +1488,7 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON
auto originalRef = getFlakeRef(); auto originalRef = getFlakeRef();
auto resolvedRef = originalRef.resolve(store); auto resolvedRef = originalRef.resolve(store);
auto [accessor, lockedRef] = resolvedRef.lazyFetch(store); auto [accessor, lockedRef] = resolvedRef.lazyFetch(store);
auto storePath = fetchToStore(*store, accessor, FetchMode::Copy, lockedRef.input.getName()); auto storePath = fetchToStore(getEvalState()->fetchSettings, *store, accessor, FetchMode::Copy, lockedRef.input.getName());
auto hash = store->queryPathInfo(storePath)->narHash; auto hash = store->queryPathInfo(storePath)->narHash;
if (json) { if (json) {