mirror of
https://github.com/NixOS/nix
synced 2025-06-25 02:21:16 +02:00
Merge pull request #13223 from NixOS/remove-global-fetcher-cache
Remove global fetcher cache
This commit is contained in:
commit
f2671886f5
24 changed files with 94 additions and 54 deletions
|
@ -34,7 +34,12 @@ EvalSettings evalSettings {
|
||||||
auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false);
|
auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false);
|
||||||
debug("fetching flake search path element '%s''", rest);
|
debug("fetching flake search path element '%s''", rest);
|
||||||
auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store);
|
auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store);
|
||||||
auto storePath = nix::fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName());
|
auto storePath = nix::fetchToStore(
|
||||||
|
state.fetchSettings,
|
||||||
|
*state.store,
|
||||||
|
SourcePath(accessor),
|
||||||
|
FetchMode::Copy,
|
||||||
|
lockedRef.input.getName());
|
||||||
state.allowPath(storePath);
|
state.allowPath(storePath);
|
||||||
return state.storePath(storePath);
|
return state.storePath(storePath);
|
||||||
},
|
},
|
||||||
|
@ -177,7 +182,11 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas
|
||||||
state.store,
|
state.store,
|
||||||
state.fetchSettings,
|
state.fetchSettings,
|
||||||
EvalSettings::resolvePseudoUrl(s));
|
EvalSettings::resolvePseudoUrl(s));
|
||||||
auto storePath = fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy);
|
auto storePath = fetchToStore(
|
||||||
|
state.fetchSettings,
|
||||||
|
*state.store,
|
||||||
|
SourcePath(accessor),
|
||||||
|
FetchMode::Copy);
|
||||||
return state.storePath(storePath);
|
return state.storePath(storePath);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -185,7 +194,12 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas
|
||||||
experimentalFeatureSettings.require(Xp::Flakes);
|
experimentalFeatureSettings.require(Xp::Flakes);
|
||||||
auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false);
|
auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false);
|
||||||
auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store);
|
auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store);
|
||||||
auto storePath = nix::fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName());
|
auto storePath = nix::fetchToStore(
|
||||||
|
state.fetchSettings,
|
||||||
|
*state.store,
|
||||||
|
SourcePath(accessor),
|
||||||
|
FetchMode::Copy,
|
||||||
|
lockedRef.input.getName());
|
||||||
state.allowPath(storePath);
|
state.allowPath(storePath);
|
||||||
return state.storePath(storePath);
|
return state.storePath(storePath);
|
||||||
}
|
}
|
||||||
|
|
|
@ -45,7 +45,7 @@ ref<InstallableValue> InstallableValue::require(ref<Installable> installable)
|
||||||
std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx)
|
std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx)
|
||||||
{
|
{
|
||||||
if (v.type() == nPath) {
|
if (v.type() == nPath) {
|
||||||
auto storePath = fetchToStore(*state->store, v.path(), FetchMode::Copy);
|
auto storePath = fetchToStore(state->fetchSettings, *state->store, v.path(), FetchMode::Copy);
|
||||||
return {{
|
return {{
|
||||||
.path = DerivedPath::Opaque {
|
.path = DerivedPath::Opaque {
|
||||||
.path = std::move(storePath),
|
.path = std::move(storePath),
|
||||||
|
|
|
@ -2423,6 +2423,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
|
||||||
? *dstPathCached
|
? *dstPathCached
|
||||||
: [&]() {
|
: [&]() {
|
||||||
auto dstPath = fetchToStore(
|
auto dstPath = fetchToStore(
|
||||||
|
fetchSettings,
|
||||||
*store,
|
*store,
|
||||||
path.resolveSymlinks(SymlinkResolution::Ancestors),
|
path.resolveSymlinks(SymlinkResolution::Ancestors),
|
||||||
settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,
|
settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,
|
||||||
|
@ -3125,7 +3126,7 @@ std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Pat
|
||||||
store,
|
store,
|
||||||
fetchSettings,
|
fetchSettings,
|
||||||
EvalSettings::resolvePseudoUrl(value));
|
EvalSettings::resolvePseudoUrl(value));
|
||||||
auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy);
|
auto storePath = fetchToStore(fetchSettings, *store, SourcePath(accessor), FetchMode::Copy);
|
||||||
return finish(this->storePath(storePath));
|
return finish(this->storePath(storePath));
|
||||||
} catch (Error & e) {
|
} catch (Error & e) {
|
||||||
logWarning({
|
logWarning({
|
||||||
|
|
|
@ -2545,6 +2545,7 @@ static void addPath(
|
||||||
|
|
||||||
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
|
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
|
||||||
auto dstPath = fetchToStore(
|
auto dstPath = fetchToStore(
|
||||||
|
state.fetchSettings,
|
||||||
*state.store,
|
*state.store,
|
||||||
path.resolveSymlinks(),
|
path.resolveSymlinks(),
|
||||||
settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,
|
settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,
|
||||||
|
|
|
@ -537,11 +537,12 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
|
||||||
auto storePath =
|
auto storePath =
|
||||||
unpack
|
unpack
|
||||||
? fetchToStore(
|
? fetchToStore(
|
||||||
|
state.fetchSettings,
|
||||||
*state.store,
|
*state.store,
|
||||||
fetchers::downloadTarball(state.store, state.fetchSettings, *url),
|
fetchers::downloadTarball(state.store, state.fetchSettings, *url),
|
||||||
FetchMode::Copy,
|
FetchMode::Copy,
|
||||||
name)
|
name)
|
||||||
: fetchers::downloadFile(state.store, *url, name).storePath;
|
: fetchers::downloadFile(state.store, state.fetchSettings, *url, name).storePath;
|
||||||
|
|
||||||
if (expectedHash) {
|
if (expectedHash) {
|
||||||
auto hash = unpack
|
auto hash = unpack
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
#include "nix/fetchers/cache.hh"
|
#include "nix/fetchers/cache.hh"
|
||||||
|
#include "nix/fetchers/fetch-settings.hh"
|
||||||
#include "nix/util/users.hh"
|
#include "nix/util/users.hh"
|
||||||
#include "nix/store/sqlite.hh"
|
#include "nix/store/sqlite.hh"
|
||||||
#include "nix/util/sync.hh"
|
#include "nix/util/sync.hh"
|
||||||
|
@ -162,10 +163,12 @@ struct CacheImpl : Cache
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
ref<Cache> getCache()
|
ref<Cache> Settings::getCache() const
|
||||||
{
|
{
|
||||||
static auto cache = std::make_shared<CacheImpl>();
|
auto cache(_cache.lock());
|
||||||
return ref<Cache>(cache);
|
if (!*cache)
|
||||||
|
*cache = std::make_shared<CacheImpl>();
|
||||||
|
return ref<Cache>(*cache);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,13 +1,14 @@
|
||||||
#include "nix/fetchers/fetch-to-store.hh"
|
#include "nix/fetchers/fetch-to-store.hh"
|
||||||
#include "nix/fetchers/fetchers.hh"
|
#include "nix/fetchers/fetchers.hh"
|
||||||
|
#include "nix/fetchers/fetch-settings.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
fetchers::Cache::Key makeFetchToStoreCacheKey(
|
fetchers::Cache::Key makeFetchToStoreCacheKey(
|
||||||
const std::string &name,
|
const std::string & name,
|
||||||
const std::string &fingerprint,
|
const std::string & fingerprint,
|
||||||
ContentAddressMethod method,
|
ContentAddressMethod method,
|
||||||
const std::string &path)
|
const std::string & path)
|
||||||
{
|
{
|
||||||
return fetchers::Cache::Key{"fetchToStore", {
|
return fetchers::Cache::Key{"fetchToStore", {
|
||||||
{"name", name},
|
{"name", name},
|
||||||
|
@ -19,6 +20,7 @@ fetchers::Cache::Key makeFetchToStoreCacheKey(
|
||||||
}
|
}
|
||||||
|
|
||||||
StorePath fetchToStore(
|
StorePath fetchToStore(
|
||||||
|
const fetchers::Settings & settings,
|
||||||
Store & store,
|
Store & store,
|
||||||
const SourcePath & path,
|
const SourcePath & path,
|
||||||
FetchMode mode,
|
FetchMode mode,
|
||||||
|
@ -34,7 +36,7 @@ StorePath fetchToStore(
|
||||||
|
|
||||||
if (!filter && path.accessor->fingerprint) {
|
if (!filter && path.accessor->fingerprint) {
|
||||||
cacheKey = makeFetchToStoreCacheKey(std::string{name}, *path.accessor->fingerprint, method, path.path.abs());
|
cacheKey = makeFetchToStoreCacheKey(std::string{name}, *path.accessor->fingerprint, method, path.path.abs());
|
||||||
if (auto res = fetchers::getCache()->lookupStorePath(*cacheKey, store)) {
|
if (auto res = settings.getCache()->lookupStorePath(*cacheKey, store)) {
|
||||||
debug("store path cache hit for '%s'", path);
|
debug("store path cache hit for '%s'", path);
|
||||||
return res->storePath;
|
return res->storePath;
|
||||||
}
|
}
|
||||||
|
@ -56,7 +58,7 @@ StorePath fetchToStore(
|
||||||
debug(mode == FetchMode::DryRun ? "hashed '%s'" : "copied '%s' to '%s'", path, store.printStorePath(storePath));
|
debug(mode == FetchMode::DryRun ? "hashed '%s'" : "copied '%s' to '%s'", path, store.printStorePath(storePath));
|
||||||
|
|
||||||
if (cacheKey && mode == FetchMode::Copy)
|
if (cacheKey && mode == FetchMode::Copy)
|
||||||
fetchers::getCache()->upsert(*cacheKey, store, {}, storePath);
|
settings.getCache()->upsert(*cacheKey, store, {}, storePath);
|
||||||
|
|
||||||
return storePath;
|
return storePath;
|
||||||
}
|
}
|
||||||
|
|
|
@ -198,7 +198,7 @@ std::pair<StorePath, Input> Input::fetchToStore(ref<Store> store) const
|
||||||
try {
|
try {
|
||||||
auto [accessor, result] = getAccessorUnchecked(store);
|
auto [accessor, result] = getAccessorUnchecked(store);
|
||||||
|
|
||||||
auto storePath = nix::fetchToStore(*store, SourcePath(accessor), FetchMode::Copy, result.getName());
|
auto storePath = nix::fetchToStore(*settings, *store, SourcePath(accessor), FetchMode::Copy, result.getName());
|
||||||
|
|
||||||
auto narHash = store->queryPathInfo(storePath)->narHash;
|
auto narHash = store->queryPathInfo(storePath)->narHash;
|
||||||
result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
|
result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
#include "nix/fetchers/git-utils.hh"
|
#include "nix/fetchers/git-utils.hh"
|
||||||
#include "nix/fetchers/git-lfs-fetch.hh"
|
#include "nix/fetchers/git-lfs-fetch.hh"
|
||||||
#include "nix/fetchers/cache.hh"
|
#include "nix/fetchers/cache.hh"
|
||||||
|
#include "nix/fetchers/fetch-settings.hh"
|
||||||
#include "nix/util/finally.hh"
|
#include "nix/util/finally.hh"
|
||||||
#include "nix/util/processes.hh"
|
#include "nix/util/processes.hh"
|
||||||
#include "nix/util/signals.hh"
|
#include "nix/util/signals.hh"
|
||||||
|
@ -610,18 +611,18 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
|
||||||
throw Error("Commit signature verification on commit %s failed: %s", rev.gitRev(), output);
|
throw Error("Commit signature verification on commit %s failed: %s", rev.gitRev(), output);
|
||||||
}
|
}
|
||||||
|
|
||||||
Hash treeHashToNarHash(const Hash & treeHash) override
|
Hash treeHashToNarHash(const fetchers::Settings & settings, const Hash & treeHash) override
|
||||||
{
|
{
|
||||||
auto accessor = getAccessor(treeHash, false, "");
|
auto accessor = getAccessor(treeHash, false, "");
|
||||||
|
|
||||||
fetchers::Cache::Key cacheKey{"treeHashToNarHash", {{"treeHash", treeHash.gitRev()}}};
|
fetchers::Cache::Key cacheKey{"treeHashToNarHash", {{"treeHash", treeHash.gitRev()}}};
|
||||||
|
|
||||||
if (auto res = fetchers::getCache()->lookup(cacheKey))
|
if (auto res = settings.getCache()->lookup(cacheKey))
|
||||||
return Hash::parseAny(fetchers::getStrAttr(*res, "narHash"), HashAlgorithm::SHA256);
|
return Hash::parseAny(fetchers::getStrAttr(*res, "narHash"), HashAlgorithm::SHA256);
|
||||||
|
|
||||||
auto narHash = accessor->hashPath(CanonPath::root);
|
auto narHash = accessor->hashPath(CanonPath::root);
|
||||||
|
|
||||||
fetchers::getCache()->upsert(cacheKey, fetchers::Attrs({{"narHash", narHash.to_string(HashFormat::SRI, true)}}));
|
settings.getCache()->upsert(cacheKey, fetchers::Attrs({{"narHash", narHash.to_string(HashFormat::SRI, true)}}));
|
||||||
|
|
||||||
return narHash;
|
return narHash;
|
||||||
}
|
}
|
||||||
|
|
|
@ -480,11 +480,11 @@ struct GitInputScheme : InputScheme
|
||||||
return repoInfo;
|
return repoInfo;
|
||||||
}
|
}
|
||||||
|
|
||||||
uint64_t getLastModified(const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const
|
uint64_t getLastModified(const Settings & settings, const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const
|
||||||
{
|
{
|
||||||
Cache::Key key{"gitLastModified", {{"rev", rev.gitRev()}}};
|
Cache::Key key{"gitLastModified", {{"rev", rev.gitRev()}}};
|
||||||
|
|
||||||
auto cache = getCache();
|
auto cache = settings.getCache();
|
||||||
|
|
||||||
if (auto res = cache->lookup(key))
|
if (auto res = cache->lookup(key))
|
||||||
return getIntAttr(*res, "lastModified");
|
return getIntAttr(*res, "lastModified");
|
||||||
|
@ -496,11 +496,11 @@ struct GitInputScheme : InputScheme
|
||||||
return lastModified;
|
return lastModified;
|
||||||
}
|
}
|
||||||
|
|
||||||
uint64_t getRevCount(const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const
|
uint64_t getRevCount(const Settings & settings, const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const
|
||||||
{
|
{
|
||||||
Cache::Key key{"gitRevCount", {{"rev", rev.gitRev()}}};
|
Cache::Key key{"gitRevCount", {{"rev", rev.gitRev()}}};
|
||||||
|
|
||||||
auto cache = getCache();
|
auto cache = settings.getCache();
|
||||||
|
|
||||||
if (auto revCountAttrs = cache->lookup(key))
|
if (auto revCountAttrs = cache->lookup(key))
|
||||||
return getIntAttr(*revCountAttrs, "revCount");
|
return getIntAttr(*revCountAttrs, "revCount");
|
||||||
|
@ -678,12 +678,12 @@ struct GitInputScheme : InputScheme
|
||||||
|
|
||||||
Attrs infoAttrs({
|
Attrs infoAttrs({
|
||||||
{"rev", rev.gitRev()},
|
{"rev", rev.gitRev()},
|
||||||
{"lastModified", getLastModified(repoInfo, repoDir, rev)},
|
{"lastModified", getLastModified(*input.settings, repoInfo, repoDir, rev)},
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!getShallowAttr(input))
|
if (!getShallowAttr(input))
|
||||||
infoAttrs.insert_or_assign("revCount",
|
infoAttrs.insert_or_assign("revCount",
|
||||||
getRevCount(repoInfo, repoDir, rev));
|
getRevCount(*input.settings, repoInfo, repoDir, rev));
|
||||||
|
|
||||||
printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg());
|
printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg());
|
||||||
|
|
||||||
|
@ -799,7 +799,7 @@ struct GitInputScheme : InputScheme
|
||||||
|
|
||||||
input.attrs.insert_or_assign("rev", rev.gitRev());
|
input.attrs.insert_or_assign("rev", rev.gitRev());
|
||||||
input.attrs.insert_or_assign("revCount",
|
input.attrs.insert_or_assign("revCount",
|
||||||
rev == nullRev ? 0 : getRevCount(repoInfo, repoPath, rev));
|
rev == nullRev ? 0 : getRevCount(*input.settings, repoInfo, repoPath, rev));
|
||||||
|
|
||||||
verifyCommit(input, repo);
|
verifyCommit(input, repo);
|
||||||
} else {
|
} else {
|
||||||
|
@ -818,7 +818,7 @@ struct GitInputScheme : InputScheme
|
||||||
input.attrs.insert_or_assign(
|
input.attrs.insert_or_assign(
|
||||||
"lastModified",
|
"lastModified",
|
||||||
repoInfo.workdirInfo.headRev
|
repoInfo.workdirInfo.headRev
|
||||||
? getLastModified(repoInfo, repoPath, *repoInfo.workdirInfo.headRev)
|
? getLastModified(*input.settings, repoInfo, repoPath, *repoInfo.workdirInfo.headRev)
|
||||||
: 0);
|
: 0);
|
||||||
|
|
||||||
return {accessor, std::move(input)};
|
return {accessor, std::move(input)};
|
||||||
|
|
|
@ -265,7 +265,7 @@ struct GitArchiveInputScheme : InputScheme
|
||||||
input.attrs.erase("ref");
|
input.attrs.erase("ref");
|
||||||
input.attrs.insert_or_assign("rev", rev->gitRev());
|
input.attrs.insert_or_assign("rev", rev->gitRev());
|
||||||
|
|
||||||
auto cache = getCache();
|
auto cache = input.settings->getCache();
|
||||||
|
|
||||||
Cache::Key treeHashKey{"gitRevToTreeHash", {{"rev", rev->gitRev()}}};
|
Cache::Key treeHashKey{"gitRevToTreeHash", {{"rev", rev->gitRev()}}};
|
||||||
Cache::Key lastModifiedKey{"gitRevToLastModified", {{"rev", rev->gitRev()}}};
|
Cache::Key lastModifiedKey{"gitRevToLastModified", {{"rev", rev->gitRev()}}};
|
||||||
|
@ -407,7 +407,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
|
||||||
auto json = nlohmann::json::parse(
|
auto json = nlohmann::json::parse(
|
||||||
readFile(
|
readFile(
|
||||||
store->toRealPath(
|
store->toRealPath(
|
||||||
downloadFile(store, url, "source", headers).storePath)));
|
downloadFile(store, *input.settings, url, "source", headers).storePath)));
|
||||||
|
|
||||||
return RefInfo {
|
return RefInfo {
|
||||||
.rev = Hash::parseAny(std::string { json["sha"] }, HashAlgorithm::SHA1),
|
.rev = Hash::parseAny(std::string { json["sha"] }, HashAlgorithm::SHA1),
|
||||||
|
@ -481,7 +481,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
|
||||||
auto json = nlohmann::json::parse(
|
auto json = nlohmann::json::parse(
|
||||||
readFile(
|
readFile(
|
||||||
store->toRealPath(
|
store->toRealPath(
|
||||||
downloadFile(store, url, "source", headers).storePath)));
|
downloadFile(store, *input.settings, url, "source", headers).storePath)));
|
||||||
|
|
||||||
if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) {
|
if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) {
|
||||||
return RefInfo {
|
return RefInfo {
|
||||||
|
@ -551,7 +551,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
|
||||||
std::string refUri;
|
std::string refUri;
|
||||||
if (ref == "HEAD") {
|
if (ref == "HEAD") {
|
||||||
auto file = store->toRealPath(
|
auto file = store->toRealPath(
|
||||||
downloadFile(store, fmt("%s/HEAD", base_url), "source", headers).storePath);
|
downloadFile(store, *input.settings, fmt("%s/HEAD", base_url), "source", headers).storePath);
|
||||||
std::ifstream is(file);
|
std::ifstream is(file);
|
||||||
std::string line;
|
std::string line;
|
||||||
getline(is, line);
|
getline(is, line);
|
||||||
|
@ -567,7 +567,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
|
||||||
std::regex refRegex(refUri);
|
std::regex refRegex(refUri);
|
||||||
|
|
||||||
auto file = store->toRealPath(
|
auto file = store->toRealPath(
|
||||||
downloadFile(store, fmt("%s/info/refs", base_url), "source", headers).storePath);
|
downloadFile(store, *input.settings, fmt("%s/info/refs", base_url), "source", headers).storePath);
|
||||||
std::ifstream is(file);
|
std::ifstream is(file);
|
||||||
|
|
||||||
std::string line;
|
std::string line;
|
||||||
|
|
|
@ -91,6 +91,4 @@ struct Cache
|
||||||
Store & store) = 0;
|
Store & store) = 0;
|
||||||
};
|
};
|
||||||
|
|
||||||
ref<Cache> getCache();
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,6 +3,8 @@
|
||||||
|
|
||||||
#include "nix/util/types.hh"
|
#include "nix/util/types.hh"
|
||||||
#include "nix/util/configuration.hh"
|
#include "nix/util/configuration.hh"
|
||||||
|
#include "nix/util/ref.hh"
|
||||||
|
#include "nix/util/sync.hh"
|
||||||
|
|
||||||
#include <map>
|
#include <map>
|
||||||
#include <limits>
|
#include <limits>
|
||||||
|
@ -11,6 +13,8 @@
|
||||||
|
|
||||||
namespace nix::fetchers {
|
namespace nix::fetchers {
|
||||||
|
|
||||||
|
struct Cache;
|
||||||
|
|
||||||
struct Settings : public Config
|
struct Settings : public Config
|
||||||
{
|
{
|
||||||
Settings();
|
Settings();
|
||||||
|
@ -110,6 +114,11 @@ struct Settings : public Config
|
||||||
When empty, disables the global flake registry.
|
When empty, disables the global flake registry.
|
||||||
)",
|
)",
|
||||||
{}, true, Xp::Flakes};
|
{}, true, Xp::Flakes};
|
||||||
|
|
||||||
|
ref<Cache> getCache() const;
|
||||||
|
|
||||||
|
private:
|
||||||
|
mutable Sync<std::shared_ptr<Cache>> _cache;
|
||||||
};
|
};
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -15,6 +15,7 @@ enum struct FetchMode { DryRun, Copy };
|
||||||
* Copy the `path` to the Nix store.
|
* Copy the `path` to the Nix store.
|
||||||
*/
|
*/
|
||||||
StorePath fetchToStore(
|
StorePath fetchToStore(
|
||||||
|
const fetchers::Settings & settings,
|
||||||
Store & store,
|
Store & store,
|
||||||
const SourcePath & path,
|
const SourcePath & path,
|
||||||
FetchMode mode,
|
FetchMode mode,
|
||||||
|
|
|
@ -5,7 +5,7 @@
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
namespace fetchers { struct PublicKey; }
|
namespace fetchers { struct PublicKey; struct Settings; }
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A sink that writes into a Git repository. Note that nothing may be written
|
* A sink that writes into a Git repository. Note that nothing may be written
|
||||||
|
@ -115,7 +115,7 @@ struct GitRepo
|
||||||
* Given a Git tree hash, compute the hash of its NAR
|
* Given a Git tree hash, compute the hash of its NAR
|
||||||
* serialisation. This is memoised on-disk.
|
* serialisation. This is memoised on-disk.
|
||||||
*/
|
*/
|
||||||
virtual Hash treeHashToNarHash(const Hash & treeHash) = 0;
|
virtual Hash treeHashToNarHash(const fetchers::Settings & settings, const Hash & treeHash) = 0;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* If the specified Git object is a directory with a single entry
|
* If the specified Git object is a directory with a single entry
|
||||||
|
|
|
@ -26,6 +26,7 @@ struct DownloadFileResult
|
||||||
|
|
||||||
DownloadFileResult downloadFile(
|
DownloadFileResult downloadFile(
|
||||||
ref<Store> store,
|
ref<Store> store,
|
||||||
|
const Settings & settings,
|
||||||
const std::string & url,
|
const std::string & url,
|
||||||
const std::string & name,
|
const std::string & name,
|
||||||
const Headers & headers = {});
|
const Headers & headers = {});
|
||||||
|
|
|
@ -253,13 +253,13 @@ struct MercurialInputScheme : InputScheme
|
||||||
}};
|
}};
|
||||||
|
|
||||||
if (!input.getRev()) {
|
if (!input.getRev()) {
|
||||||
if (auto res = getCache()->lookupWithTTL(refToRevKey))
|
if (auto res = input.settings->getCache()->lookupWithTTL(refToRevKey))
|
||||||
input.attrs.insert_or_assign("rev", getRevAttr(*res, "rev").gitRev());
|
input.attrs.insert_or_assign("rev", getRevAttr(*res, "rev").gitRev());
|
||||||
}
|
}
|
||||||
|
|
||||||
/* If we have a rev, check if we have a cached store path. */
|
/* If we have a rev, check if we have a cached store path. */
|
||||||
if (auto rev = input.getRev()) {
|
if (auto rev = input.getRev()) {
|
||||||
if (auto res = getCache()->lookupStorePath(revInfoKey(*rev), *store))
|
if (auto res = input.settings->getCache()->lookupStorePath(revInfoKey(*rev), *store))
|
||||||
return makeResult(res->value, res->storePath);
|
return makeResult(res->value, res->storePath);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -309,7 +309,7 @@ struct MercurialInputScheme : InputScheme
|
||||||
|
|
||||||
/* Now that we have the rev, check the cache again for a
|
/* Now that we have the rev, check the cache again for a
|
||||||
cached store path. */
|
cached store path. */
|
||||||
if (auto res = getCache()->lookupStorePath(revInfoKey(rev), *store))
|
if (auto res = input.settings->getCache()->lookupStorePath(revInfoKey(rev), *store))
|
||||||
return makeResult(res->value, res->storePath);
|
return makeResult(res->value, res->storePath);
|
||||||
|
|
||||||
Path tmpDir = createTempDir();
|
Path tmpDir = createTempDir();
|
||||||
|
@ -326,9 +326,9 @@ struct MercurialInputScheme : InputScheme
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!origRev)
|
if (!origRev)
|
||||||
getCache()->upsert(refToRevKey, {{"rev", rev.gitRev()}});
|
input.settings->getCache()->upsert(refToRevKey, {{"rev", rev.gitRev()}});
|
||||||
|
|
||||||
getCache()->upsert(revInfoKey(rev), *store, infoAttrs, storePath);
|
input.settings->getCache()->upsert(revInfoKey(rev), *store, infoAttrs, storePath);
|
||||||
|
|
||||||
return makeResult(infoAttrs, std::move(storePath));
|
return makeResult(infoAttrs, std::move(storePath));
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,6 +4,7 @@
|
||||||
#include "nix/fetchers/store-path-accessor.hh"
|
#include "nix/fetchers/store-path-accessor.hh"
|
||||||
#include "nix/fetchers/cache.hh"
|
#include "nix/fetchers/cache.hh"
|
||||||
#include "nix/fetchers/fetch-to-store.hh"
|
#include "nix/fetchers/fetch-to-store.hh"
|
||||||
|
#include "nix/fetchers/fetch-settings.hh"
|
||||||
|
|
||||||
namespace nix::fetchers {
|
namespace nix::fetchers {
|
||||||
|
|
||||||
|
@ -149,7 +150,7 @@ struct PathInputScheme : InputScheme
|
||||||
auto fp = getFingerprint(store, input);
|
auto fp = getFingerprint(store, input);
|
||||||
if (fp) {
|
if (fp) {
|
||||||
auto cacheKey = makeFetchToStoreCacheKey(input.getName(), *fp, method, "/");
|
auto cacheKey = makeFetchToStoreCacheKey(input.getName(), *fp, method, "/");
|
||||||
fetchers::getCache()->upsert(cacheKey, *store, {}, *storePath);
|
input.settings->getCache()->upsert(cacheKey, *store, {}, *storePath);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Trust the lastModified value supplied by the user, if
|
/* Trust the lastModified value supplied by the user, if
|
||||||
|
|
|
@ -156,7 +156,7 @@ static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, re
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!isAbsolute(path)) {
|
if (!isAbsolute(path)) {
|
||||||
auto storePath = downloadFile(store, path, "flake-registry.json").storePath;
|
auto storePath = downloadFile(store, settings, path, "flake-registry.json").storePath;
|
||||||
if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
|
if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
|
||||||
store2->addPermRoot(storePath, getCacheDir() + "/flake-registry.json");
|
store2->addPermRoot(storePath, getCacheDir() + "/flake-registry.json");
|
||||||
path = store->toRealPath(storePath);
|
path = store->toRealPath(storePath);
|
||||||
|
|
|
@ -9,11 +9,13 @@
|
||||||
#include "nix/fetchers/store-path-accessor.hh"
|
#include "nix/fetchers/store-path-accessor.hh"
|
||||||
#include "nix/store/store-api.hh"
|
#include "nix/store/store-api.hh"
|
||||||
#include "nix/fetchers/git-utils.hh"
|
#include "nix/fetchers/git-utils.hh"
|
||||||
|
#include "nix/fetchers/fetch-settings.hh"
|
||||||
|
|
||||||
namespace nix::fetchers {
|
namespace nix::fetchers {
|
||||||
|
|
||||||
DownloadFileResult downloadFile(
|
DownloadFileResult downloadFile(
|
||||||
ref<Store> store,
|
ref<Store> store,
|
||||||
|
const Settings & settings,
|
||||||
const std::string & url,
|
const std::string & url,
|
||||||
const std::string & name,
|
const std::string & name,
|
||||||
const Headers & headers)
|
const Headers & headers)
|
||||||
|
@ -25,7 +27,7 @@ DownloadFileResult downloadFile(
|
||||||
{"name", name},
|
{"name", name},
|
||||||
}}};
|
}}};
|
||||||
|
|
||||||
auto cached = getCache()->lookupStorePath(key, *store);
|
auto cached = settings.getCache()->lookupStorePath(key, *store);
|
||||||
|
|
||||||
auto useCached = [&]() -> DownloadFileResult
|
auto useCached = [&]() -> DownloadFileResult
|
||||||
{
|
{
|
||||||
|
@ -92,7 +94,7 @@ DownloadFileResult downloadFile(
|
||||||
key.second.insert_or_assign("url", url);
|
key.second.insert_or_assign("url", url);
|
||||||
assert(!res.urls.empty());
|
assert(!res.urls.empty());
|
||||||
infoAttrs.insert_or_assign("url", *res.urls.rbegin());
|
infoAttrs.insert_or_assign("url", *res.urls.rbegin());
|
||||||
getCache()->upsert(key, *store, infoAttrs, *storePath);
|
settings.getCache()->upsert(key, *store, infoAttrs, *storePath);
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
@ -104,13 +106,14 @@ DownloadFileResult downloadFile(
|
||||||
}
|
}
|
||||||
|
|
||||||
static DownloadTarballResult downloadTarball_(
|
static DownloadTarballResult downloadTarball_(
|
||||||
|
const Settings & settings,
|
||||||
const std::string & url,
|
const std::string & url,
|
||||||
const Headers & headers,
|
const Headers & headers,
|
||||||
const std::string & displayPrefix)
|
const std::string & displayPrefix)
|
||||||
{
|
{
|
||||||
Cache::Key cacheKey{"tarball", {{"url", url}}};
|
Cache::Key cacheKey{"tarball", {{"url", url}}};
|
||||||
|
|
||||||
auto cached = getCache()->lookupExpired(cacheKey);
|
auto cached = settings.getCache()->lookupExpired(cacheKey);
|
||||||
|
|
||||||
auto attrsToResult = [&](const Attrs & infoAttrs)
|
auto attrsToResult = [&](const Attrs & infoAttrs)
|
||||||
{
|
{
|
||||||
|
@ -196,7 +199,7 @@ static DownloadTarballResult downloadTarball_(
|
||||||
/* Insert a cache entry for every URL in the redirect chain. */
|
/* Insert a cache entry for every URL in the redirect chain. */
|
||||||
for (auto & url : res->urls) {
|
for (auto & url : res->urls) {
|
||||||
cacheKey.second.insert_or_assign("url", url);
|
cacheKey.second.insert_or_assign("url", url);
|
||||||
getCache()->upsert(cacheKey, infoAttrs);
|
settings.getCache()->upsert(cacheKey, infoAttrs);
|
||||||
}
|
}
|
||||||
|
|
||||||
// FIXME: add a cache entry for immutableUrl? That could allow
|
// FIXME: add a cache entry for immutableUrl? That could allow
|
||||||
|
@ -341,7 +344,7 @@ struct FileInputScheme : CurlInputScheme
|
||||||
the Nix store directly, since there is little deduplication
|
the Nix store directly, since there is little deduplication
|
||||||
benefit in using the Git cache for single big files like
|
benefit in using the Git cache for single big files like
|
||||||
tarballs. */
|
tarballs. */
|
||||||
auto file = downloadFile(store, getStrAttr(input.attrs, "url"), input.getName());
|
auto file = downloadFile(store, *input.settings, getStrAttr(input.attrs, "url"), input.getName());
|
||||||
|
|
||||||
auto narHash = store->queryPathInfo(file.storePath)->narHash;
|
auto narHash = store->queryPathInfo(file.storePath)->narHash;
|
||||||
input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
|
input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
|
||||||
|
@ -373,6 +376,7 @@ struct TarballInputScheme : CurlInputScheme
|
||||||
auto input(_input);
|
auto input(_input);
|
||||||
|
|
||||||
auto result = downloadTarball_(
|
auto result = downloadTarball_(
|
||||||
|
*input.settings,
|
||||||
getStrAttr(input.attrs, "url"),
|
getStrAttr(input.attrs, "url"),
|
||||||
{},
|
{},
|
||||||
"«" + input.to_string() + "»");
|
"«" + input.to_string() + "»");
|
||||||
|
@ -390,7 +394,7 @@ struct TarballInputScheme : CurlInputScheme
|
||||||
input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified));
|
input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified));
|
||||||
|
|
||||||
input.attrs.insert_or_assign("narHash",
|
input.attrs.insert_or_assign("narHash",
|
||||||
getTarballCache()->treeHashToNarHash(result.treeHash).to_string(HashFormat::SRI, true));
|
getTarballCache()->treeHashToNarHash(*input.settings, result.treeHash).to_string(HashFormat::SRI, true));
|
||||||
|
|
||||||
return {result.accessor, input};
|
return {result.accessor, input};
|
||||||
}
|
}
|
||||||
|
|
|
@ -30,7 +30,7 @@ static StorePath copyInputToStore(
|
||||||
const fetchers::Input & originalInput,
|
const fetchers::Input & originalInput,
|
||||||
ref<SourceAccessor> accessor)
|
ref<SourceAccessor> accessor)
|
||||||
{
|
{
|
||||||
auto storePath = fetchToStore(*state.store, accessor, FetchMode::Copy, input.getName());
|
auto storePath = fetchToStore(*input.settings, *state.store, accessor, FetchMode::Copy, input.getName());
|
||||||
|
|
||||||
state.allowPath(storePath);
|
state.allowPath(storePath);
|
||||||
|
|
||||||
|
@ -276,7 +276,7 @@ static Flake readFlake(
|
||||||
state.symbols[setting.name],
|
state.symbols[setting.name],
|
||||||
std::string(state.forceStringNoCtx(*setting.value, setting.pos, "")));
|
std::string(state.forceStringNoCtx(*setting.value, setting.pos, "")));
|
||||||
else if (setting.value->type() == nPath) {
|
else if (setting.value->type() == nPath) {
|
||||||
auto storePath = fetchToStore(*state.store, setting.value->path(), FetchMode::Copy);
|
auto storePath = fetchToStore(state.fetchSettings, *state.store, setting.value->path(), FetchMode::Copy);
|
||||||
flake.config.settings.emplace(
|
flake.config.settings.emplace(
|
||||||
state.symbols[setting.name],
|
state.symbols[setting.name],
|
||||||
state.store->printStorePath(storePath));
|
state.store->printStorePath(storePath));
|
||||||
|
|
|
@ -39,6 +39,7 @@ public:
|
||||||
SyncBase() { }
|
SyncBase() { }
|
||||||
SyncBase(const T & data) : data(data) { }
|
SyncBase(const T & data) : data(data) { }
|
||||||
SyncBase(T && data) noexcept : data(std::move(data)) { }
|
SyncBase(T && data) noexcept : data(std::move(data)) { }
|
||||||
|
SyncBase(SyncBase && other) noexcept : data(std::move(*other.lock())) { }
|
||||||
|
|
||||||
template<class L>
|
template<class L>
|
||||||
class Lock
|
class Lock
|
||||||
|
|
|
@ -4,9 +4,11 @@
|
||||||
#include "nix/store/filetransfer.hh"
|
#include "nix/store/filetransfer.hh"
|
||||||
#include "nix/store/store-open.hh"
|
#include "nix/store/store-open.hh"
|
||||||
#include "nix/cmd/legacy.hh"
|
#include "nix/cmd/legacy.hh"
|
||||||
|
#include "nix/cmd/common-eval-args.hh"
|
||||||
#include "nix/expr/eval-settings.hh" // for defexpr
|
#include "nix/expr/eval-settings.hh" // for defexpr
|
||||||
#include "nix/util/users.hh"
|
#include "nix/util/users.hh"
|
||||||
#include "nix/fetchers/tarball.hh"
|
#include "nix/fetchers/tarball.hh"
|
||||||
|
#include "nix/fetchers/fetch-settings.hh"
|
||||||
#include "self-exe.hh"
|
#include "self-exe.hh"
|
||||||
#include "man-pages.hh"
|
#include "man-pages.hh"
|
||||||
|
|
||||||
|
@ -114,7 +116,7 @@ static void update(const StringSet & channelNames)
|
||||||
// We want to download the url to a file to see if it's a tarball while also checking if we
|
// We want to download the url to a file to see if it's a tarball while also checking if we
|
||||||
// got redirected in the process, so that we can grab the various parts of a nix channel
|
// got redirected in the process, so that we can grab the various parts of a nix channel
|
||||||
// definition from a consistent location if the redirect changes mid-download.
|
// definition from a consistent location if the redirect changes mid-download.
|
||||||
auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url)));
|
auto result = fetchers::downloadFile(store, fetchSettings, url, std::string(baseNameOf(url)));
|
||||||
auto filename = store->toRealPath(result.storePath);
|
auto filename = store->toRealPath(result.storePath);
|
||||||
url = result.effectiveUrl;
|
url = result.effectiveUrl;
|
||||||
|
|
||||||
|
@ -128,9 +130,9 @@ static void update(const StringSet & channelNames)
|
||||||
if (!unpacked) {
|
if (!unpacked) {
|
||||||
// Download the channel tarball.
|
// Download the channel tarball.
|
||||||
try {
|
try {
|
||||||
filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz").storePath);
|
filename = store->toRealPath(fetchers::downloadFile(store, fetchSettings, url + "/nixexprs.tar.xz", "nixexprs.tar.xz").storePath);
|
||||||
} catch (FileTransferError & e) {
|
} catch (FileTransferError & e) {
|
||||||
filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2").storePath);
|
filename = store->toRealPath(fetchers::downloadFile(store, fetchSettings, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2").storePath);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Regardless of where it came from, add the expression representing this channel to accumulated expression
|
// Regardless of where it came from, add the expression representing this channel to accumulated expression
|
||||||
|
|
|
@ -1488,7 +1488,7 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON
|
||||||
auto originalRef = getFlakeRef();
|
auto originalRef = getFlakeRef();
|
||||||
auto resolvedRef = originalRef.resolve(store);
|
auto resolvedRef = originalRef.resolve(store);
|
||||||
auto [accessor, lockedRef] = resolvedRef.lazyFetch(store);
|
auto [accessor, lockedRef] = resolvedRef.lazyFetch(store);
|
||||||
auto storePath = fetchToStore(*store, accessor, FetchMode::Copy, lockedRef.input.getName());
|
auto storePath = fetchToStore(getEvalState()->fetchSettings, *store, accessor, FetchMode::Copy, lockedRef.input.getName());
|
||||||
auto hash = store->queryPathInfo(storePath)->narHash;
|
auto hash = store->queryPathInfo(storePath)->narHash;
|
||||||
|
|
||||||
if (json) {
|
if (json) {
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue