wroclaw-git-backups/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-06-24 22:11:15 +02:00
Code Activity

New store settings system

Browse source 
Motivation:

See the linked issues for details.

The most notable user-relevant bits are:

- This cleans up the `MountedSSHStore`: decomposed into its orthogonal parts

- This brings us pretty close to being able to then implement a JSON-based config.
   - Store query parameters can be JSON
   - Stores can entirely be specified via JSON objects, but this is not yet hooked up to anything.

Also behind the scenes have these benefits:

1. The docs are moved out of the headers, good for less rebuilding when they changes
2. Stores are always constructed from store configs
3. Use JSON, avoid custom serializers

Context:

Part of #11106

Co-Authored-By: Robert Hensing <robert@roberthensing.nl>
Co-authored-by: Sergei Zimmerman <145775305+xokdvium@users.noreply.github.com>
This commit is contained in:
John Ericson 2025-03-17 11:29:06 -04:00
parent bf5d544d3b
commit 2ea9f59978
69 changed files with 2062 additions and 701 deletions
Download patch file Download diff file Expand all files Collapse all files

46
doc/manual/generate-settings.nix
View file

@ -19,7 +19,6 @@ in
prefix,
inlineHTML ? true,
}:
settingsInfo:
let
@ -27,11 +26,25 @@ let
prefix: setting:
{
description,
documentDefault,
defaultValue,
aliases,
value,
experimentalFeature,
# Whether we document the default, because it is machine agostic,
# or don't because because it is machine-specific.
documentDefault ? true,
# The default value is JSON for new-style config, rather than then
# a string or boolean, for old-style config.
isJson ? false,
defaultValue ? null,
subSettings ? null,
aliases ? [ ],
# The current value for this setting. Purposefully unused.
value ? null,
}:
let
result = squash ''
@ -50,7 +63,7 @@ let
${description}
**Default:** ${showDefault documentDefault defaultValue}
${showDefaultOrSubSettings}
${showAliases aliases}
'';
@ -72,9 +85,24 @@ let
> ```
'';
showDefaultOrSubSettings =
if !isAttrs subSettings then
# No subsettings, instead single setting. Show the default value.
''
**Default:** ${showDefault}
''
else
# Indent the nested sub-settings, and append the outer setting name onto the prefix
indent " " ''
**Nullable sub-settings**: ${if subSettings.nullable then "true" else "false"}
${builtins.trace prefix (showSettings "${prefix}-${setting}" subSettings.map)}
'';
showDefault =
documentDefault: defaultValue:
if documentDefault then
if isJson then
"`${builtins.toJSON defaultValue}`"
else
# a StringMap value type is specified as a string, but
# this shows the value type. The empty stringmap is `null` in
# JSON, but that converts to `{ }` here.
@ -95,5 +123,7 @@ let
in
result;
showSettings =
prefix: settingsInfo: concatStrings (attrValues (mapAttrs (showSetting prefix) settingsInfo));
in
concatStrings (attrValues (mapAttrs (showSetting prefix) settingsInfo))
showSettings prefix

2
src/libstore-c/nix_api_store.cc
View file

@ -43,7 +43,7 @@ Store * nix_store_open(nix_c_context * context, const char * uri, const char ***
if (!params)
return new Store{nix::openStore(uri_str)};
nix::Store::Config::Params params_map;
nix::StoreReference::Params params_map;
for (size_t i = 0; params[i] != nullptr; i++) {
params_map[params[i][0]] = params[i][1];
}

3
src/libstore-tests/data/store-reference/auto.json Normal file
View file

@ -0,0 +1,3 @@
{
"scheme": "auto"
}

4
src/libstore-tests/data/store-reference/auto_param.json Normal file
View file

@ -0,0 +1,4 @@
{
"root": "/foo/bar/baz",
"scheme": "auto"
}

5
src/libstore-tests/data/store-reference/local_1.json Normal file
View file

@ -0,0 +1,5 @@
{
"authority": "",
"root": "/foo/bar/baz",
"scheme": "local"
}

5
src/libstore-tests/data/store-reference/local_2.json Normal file
View file

@ -0,0 +1,5 @@
{
"authority": "/foo/bar/baz",
"scheme": "local",
"trusted": true
}

4
src/libstore-tests/data/store-reference/ssh.json Normal file
View file

@ -0,0 +1,4 @@
{
"authority": "localhost",
"scheme": "ssh"
}

6
src/libstore-tests/data/store-reference/unix.json Normal file
View file

@ -0,0 +1,6 @@
{
"authority": "",
"max-connections": 7,
"scheme": "unix",
"trusted": true
}

20
src/libstore-tests/dummy-store.cc Normal file
View file

@ -0,0 +1,20 @@
#include <gtest/gtest.h>
#include "nix/store/dummy-store.hh"
#include "nix/store/globals.hh"
namespace nix {
TEST(DummyStore, constructConfig)
{
DummyStoreConfig config{"dummy", "", {}};
EXPECT_EQ(config.storeDir, settings.nixStore);
}
TEST(DummyStore, constructConfigNoAuthority)
{
EXPECT_THROW(DummyStoreConfig("dummy", "not-allowed", {}), UsageError);
}
} // namespace nix

8
src/libstore-tests/legacy-ssh-store.cc
View file

@ -9,11 +9,13 @@ TEST(LegacySSHStore, constructConfig)
LegacySSHStoreConfig config{
"ssh",
"localhost",
StoreConfig::Params{
StoreReference::Params{
{
"remote-program",
// TODO #11106, no more split on space
"foo bar",
{
"foo",
"bar",
},
},
}};
EXPECT_EQ(

8
src/libstore-tests/local-overlay-store.cc
View file

@ -1,9 +1,6 @@
// FIXME: Odd failures for templates that are causing the PR to break
// for now with discussion with @Ericson2314 to comment out.
#if 0
# include <gtest/gtest.h>
#include <gtest/gtest.h>
# include "nix/store/local-overlay-store.hh"
#include "nix/store/local-overlay-store.hh"
namespace nix {
@ -31,4 +28,3 @@ TEST(LocalOverlayStore, constructConfig_rootPath)
}
} // namespace nix
#endif

14
src/libstore-tests/local-store.cc
View file

@ -1,15 +1,6 @@
// FIXME: Odd failures for templates that are causing the PR to break
// for now with discussion with @Ericson2314 to comment out.
#if 0
# include <gtest/gtest.h>
#include <gtest/gtest.h>
# include "nix/store/local-store.hh"
// Needed for template specialisations. This is not good! When we
// overhaul how store configs work, this should be fixed.
# include "nix/util/args.hh"
# include "nix/util/config-impl.hh"
# include "nix/util/abstract-setting-to-json.hh"
#include "nix/store/local-store.hh"
namespace nix {
@ -37,4 +28,3 @@ TEST(LocalStore, constructConfig_rootPath)
}
} // namespace nix
#endif

1
src/libstore-tests/meson.build
View file

@ -58,6 +58,7 @@ sources = files(
'derivation-advanced-attrs.cc',
'derivation.cc',
'derived-path.cc',
'dummy-store.cc',
'downstream-placeholder.cc',
'http-binary-cache-store.cc',
'legacy-ssh-store.cc',

2
src/libstore-tests/nix_api_store.cc
View file

@ -97,7 +97,7 @@ TEST_F(nix_api_util_context, nix_store_open_dummy)
nix_libstore_init(ctx);
Store * store = nix_store_open(ctx, "dummy://", nullptr);
ASSERT_EQ(NIX_OK, ctx->last_err_code);
ASSERT_STREQ("dummy", store->ptr->getUri().c_str());
ASSERT_STREQ("dummy://", store->ptr->getUri().c_str());
std::string str;
nix_store_get_version(ctx, store, OBSERVE_STRING(str));

80
src/libstore-tests/ssh-store.cc
View file

@ -1,22 +1,21 @@
// FIXME: Odd failures for templates that are causing the PR to break
// for now with discussion with @Ericson2314 to comment out.
#if 0
# include <gtest/gtest.h>
#include <gtest/gtest.h>
# include "nix/store/ssh-store.hh"
#include "nix/store/ssh-store.hh"
namespace nix {
TEST(SSHStore, constructConfig)
{
SSHStoreConfig config{
"ssh",
"ssh-ng",
"localhost",
StoreConfig::Params{
StoreReference::Params{
{
"remote-program",
// TODO #11106, no more split on space
"foo bar",
{
"foo",
"bar",
},
},
},
};
@ -31,16 +30,26 @@ TEST(SSHStore, constructConfig)
TEST(MountedSSHStore, constructConfig)
{
MountedSSHStoreConfig config{
"mounted-ssh",
ExperimentalFeatureSettings mockXpSettings;
mockXpSettings.set("experimental-features", "mounted-ssh-store");
SSHStoreConfig config{
"ssh-ng",
"localhost",
StoreConfig::Params{
StoreReference::Params{
{
"remote-program",
// TODO #11106, no more split on space
"foo bar",
{
"foo",
"bar",
},
},
{
"mounted",
nlohmann::json::object_t{},
},
},
mockXpSettings,
};
EXPECT_EQ(
@ -49,7 +58,48 @@ TEST(MountedSSHStore, constructConfig)
"foo",
"bar",
}));
ASSERT_TRUE(config.mounted);
EXPECT_EQ(config.mounted->realStoreDir, "/nix/store");
}
TEST(MountedSSHStore, constructConfigWithFunnyRealStoreDir)
{
ExperimentalFeatureSettings mockXpSettings;
mockXpSettings.set("experimental-features", "mounted-ssh-store");
SSHStoreConfig config{
"ssh-ng",
"localhost",
StoreReference::Params{
{
"remote-program",
{
"foo",
"bar",
},
},
{
"mounted",
nlohmann::json::object_t{
{"real", "/foo/bar"},
},
},
},
mockXpSettings,
};
EXPECT_EQ(
config.remoteProgram.get(),
(Strings{
"foo",
"bar",
}));
ASSERT_TRUE(config.mounted);
EXPECT_EQ(config.mounted->realStoreDir, "/foo/bar");
}
}
#endif

53
src/libstore-tests/store-reference.cc
View file

@ -17,14 +17,14 @@ class StoreReferenceTest : public CharacterizationTest, public LibStoreTest
std::filesystem::path goldenMaster(PathView testStem) const override
{
return unitTestData / (testStem + ".txt");
return unitTestData / testStem;
}
};
#define URI_TEST_READ(STEM, OBJ) \
TEST_F(StoreReferenceTest, PathInfo_##STEM##_from_uri) \
{ \
readTest(#STEM, ([&](const auto & encoded) { \
readTest(#STEM ".txt", ([&](const auto & encoded) { \
StoreReference expected = OBJ; \
auto got = StoreReference::parse(encoded); \
ASSERT_EQ(got, expected); \
@ -35,7 +35,7 @@ class StoreReferenceTest : public CharacterizationTest, public LibStoreTest
TEST_F(StoreReferenceTest, PathInfo_##STEM##_to_uri) \
{ \
writeTest( \
#STEM, \
#STEM ".txt", \
[&]() -> StoreReference { return OBJ; }, \
[](const auto & file) { return StoreReference::parse(readFile(file)); }, \
[](const auto & file, const auto & got) { return writeFile(file, got.render()); }); \
@ -45,14 +45,43 @@ class StoreReferenceTest : public CharacterizationTest, public LibStoreTest
URI_TEST_READ(STEM, OBJ) \
URI_TEST_WRITE(STEM, OBJ)
URI_TEST(
#define JSON_TEST_READ(STEM, OBJ) \
TEST_F(StoreReferenceTest, PathInfo_##STEM##_from_json) \
{ \
readTest(#STEM ".json", ([&](const auto & encoded_) { \
auto encoded = json::parse(encoded_); \
StoreReference expected = OBJ; \
StoreReference got = encoded; \
ASSERT_EQ(got, expected); \
})); \
}
#define JSON_TEST_WRITE(STEM, OBJ) \
TEST_F(StoreReferenceTest, PathInfo_##STEM##_to_json) \
{ \
writeTest( \
#STEM ".json", \
[&]() -> StoreReference { return OBJ; }, \
[](const auto & file) -> StoreReference { return json::parse(readFile(file)); }, \
[](const auto & file, const auto & got) { return writeFile(file, json(got).dump(2) + "\n"); }); \
}
#define JSON_TEST(STEM, OBJ) \
JSON_TEST_READ(STEM, OBJ) \
JSON_TEST_WRITE(STEM, OBJ)
#define BOTH_FORMATS_TEST(STEM, OBJ) \
URI_TEST(STEM, OBJ) \
JSON_TEST(STEM, OBJ)
BOTH_FORMATS_TEST(
auto,
(StoreReference{
.variant = StoreReference::Auto{},
.params = {},
}))
URI_TEST(
BOTH_FORMATS_TEST(
auto_param,
(StoreReference{
.variant = StoreReference::Auto{},
@ -81,13 +110,13 @@ static StoreReference localExample_2{
},
.params =
{
{"trusted", "true"},
{"trusted", true},
},
};
URI_TEST(local_1, localExample_1)
BOTH_FORMATS_TEST(local_1, localExample_1)
URI_TEST(local_2, localExample_2)
BOTH_FORMATS_TEST(local_2, localExample_2)
URI_TEST_READ(local_shorthand_1, localExample_1)
@ -100,16 +129,16 @@ static StoreReference unixExample{
},
.params =
{
{"max-connections", "7"},
{"trusted", "true"},
{"max-connections", 7},
{"trusted", true},
},
};
URI_TEST(unix, unixExample)
BOTH_FORMATS_TEST(unix, unixExample)
URI_TEST_READ(unix_shorthand, unixExample)
URI_TEST(
BOTH_FORMATS_TEST(
ssh,
(StoreReference{
.variant =

8
src/libstore-tests/uds-remote-store.cc
View file

@ -1,9 +1,6 @@
// FIXME: Odd failures for templates that are causing the PR to break
// for now with discussion with @Ericson2314 to comment out.
#if 0
# include <gtest/gtest.h>
#include <gtest/gtest.h>
# include "nix/store/uds-remote-store.hh"
#include "nix/store/uds-remote-store.hh"
namespace nix {
@ -20,4 +17,3 @@ TEST(UDSRemoteStore, constructConfigWrongScheme)
}
} // namespace nix
#endif

111
src/libstore/binary-cache-store.cc
View file

@ -13,6 +13,7 @@
#include "nix/util/callback.hh"
#include "nix/util/signals.hh"
#include "nix/util/archive.hh"
#include "nix/store/config-parse-impl.hh"
#include <chrono>
#include <future>
@ -24,25 +25,111 @@
namespace nix {
BinaryCacheStore::BinaryCacheStore(Config & config)
: config{config}
constexpr static const BinaryCacheStoreConfigT<config::SettingInfo> binaryCacheStoreConfigDescriptions = {
.compression = {
.name = "compression",
.description = "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`).",
},
.writeNARListing = {
.name = "write-nar-listing",
.description = "Whether to write a JSON file that lists the files in each NAR.",
},
.writeDebugInfo = {
.name = "index-debug-info",
.description = R"(
Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to
fetch debug info on demand
)",
},
.secretKeyFile{
.name = "secret-key",
.description = "Path to the secret key used to sign the binary cache.",
},
.localNarCache{
.name = "local-nar-cache",
.description = "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`.",
},
.parallelCompression{
.name = "parallel-compression",
.description = "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`.",
},
.compressionLevel{
.name = "compression-level",
.description = R"(
The *preset level* to be used when compressing NARs.
The meaning and accepted values depend on the compression method selected.
`-1` specifies that the default compression level should be used.
)",
},
};
#define BINARY_CACHE_STORE_CONFIG_FIELDS(X) \
X(compression), \
X(writeNARListing), \
X(writeDebugInfo), \
X(secretKeyFile), \
X(secretKeyFiles), \
X(localNarCache), \
X(parallelCompression), \
X(compressionLevel),
MAKE_PARSE(BinaryCacheStoreConfig, binaryCacheStoreConfig, BINARY_CACHE_STORE_CONFIG_FIELDS)
static BinaryCacheStoreConfigT<config::PlainValue> binaryCacheStoreConfigDefaults()
{
return {
.compression = {"xz"},
.writeNARListing = {false},
.writeDebugInfo = {false},
.secretKeyFile = {""},
.secretKeyFiles = {{}},
.localNarCache = {""},
.parallelCompression = {false},
.compressionLevel = {-1},
};
}
MAKE_APPLY_PARSE(BinaryCacheStoreConfig, binaryCacheStoreConfig, BINARY_CACHE_STORE_CONFIG_FIELDS)
BinaryCacheStore::Config::BinaryCacheStoreConfig(
const Store::Config & storeConfig,
const StoreReference::Params & params)
: BinaryCacheStoreConfigT<config::PlainValue>{binaryCacheStoreConfigApplyParse(params)}
, storeConfig{storeConfig}
{
}
config::SettingDescriptionMap BinaryCacheStoreConfig::descriptions()
{
constexpr auto & descriptions = binaryCacheStoreConfigDescriptions;
auto defaults = binaryCacheStoreConfigDefaults();
return {
BINARY_CACHE_STORE_CONFIG_FIELDS(DESCRIBE_ROW)
};
}
BinaryCacheStore::BinaryCacheStore(const Config & config)
: Store{config.storeConfig}
, config{config}
{
if (config.secretKeyFile != "")
signers.push_back(std::make_unique<LocalSigner>(
SecretKey { readFile(config.secretKeyFile) }));
if (config.secretKeyFiles != "") {
std::stringstream ss(config.secretKeyFiles);
Path keyPath;
while (std::getline(ss, keyPath, ',')) {
signers.push_back(std::make_unique<LocalSigner>(
SecretKey { readFile(keyPath) }));
}
for (auto & keyPath : config.secretKeyFiles.value) {
signers.push_back(std::make_unique<LocalSigner>(
SecretKey { readFile(keyPath) }));
}
StringSink sink;
sink << narVersionMagic1;
narMagic = sink.s;
// Want to call this but cannot, because virtual function lookup is
// disabled in a constructor. It is thus left to instances to call
// it instead.
//init();
}
void BinaryCacheStore::init()
@ -61,9 +148,11 @@ void BinaryCacheStore::init()
throw Error("binary cache '%s' is for Nix stores with prefix '%s', not '%s'",
getUri(), value, storeDir);
} else if (name == "WantMassQuery") {
config.wantMassQuery.setDefault(value == "1");
resolvedSubstConfig.wantMassQuery.value =
config.storeConfig.wantMassQuery.optValue.value_or(value == "1");
} else if (name == "Priority") {
config.priority.setDefault(std::stoi(value));
resolvedSubstConfig.priority.value =
config.storeConfig.priority.optValue.value_or(std::stoi(value));
}
}
}

54
src/libstore/common-ssh-store-config.cc
View file

@ -2,9 +2,56 @@
#include "nix/store/common-ssh-store-config.hh"
#include "nix/store/ssh.hh"
#include "nix/store/config-parse-impl.hh"
namespace nix {
constexpr static const CommonSSHStoreConfigT<config::SettingInfo> commonSSHStoreConfigDescriptions = {
.sshKey{
.name = "ssh-key",
.description = "Path to the SSH private key used to authenticate to the remote machine.",
},
.sshPublicHostKey{
.name = "base64-ssh-public-host-key",
.description = "The public host key of the remote machine.",
},
.compress{
.name = "compress",
.description = "Whether to enable SSH compression.",
},
.remoteStore{
.name = "remote-store",
.description = R"(
[Store URL](@docroot@/store/types/index.md#store-url-format)
to be used on the remote machine. The default is `auto`
(i.e. use the Nix daemon or `/nix/store` directly).
)",
},
};
#define COMMON_SSH_STORE_CONFIG_FIELDS(X) X(sshKey), X(sshPublicHostKey), X(compress), X(remoteStore),
MAKE_PARSE(CommonSSHStoreConfig, commonSSHStoreConfig, COMMON_SSH_STORE_CONFIG_FIELDS)
static CommonSSHStoreConfigT<config::PlainValue> commonSSHStoreConfigDefaults()
{
return {
.sshKey = {""},
.sshPublicHostKey = {""},
.compress = {false},
.remoteStore = {""},
};
}
MAKE_APPLY_PARSE(CommonSSHStoreConfig, commonSSHStoreConfig, COMMON_SSH_STORE_CONFIG_FIELDS)
config::SettingDescriptionMap CommonSSHStoreConfig::descriptions()
{
constexpr auto & descriptions = commonSSHStoreConfigDescriptions;
auto defaults = commonSSHStoreConfigDefaults();
return {COMMON_SSH_STORE_CONFIG_FIELDS(DESCRIBE_ROW)};
}
static std::string extractConnStr(std::string_view scheme, std::string_view _connStr)
{
if (_connStr.empty())
@ -22,9 +69,10 @@ static std::string extractConnStr(std::string_view scheme, std::string_view _con
return connStr;
}
CommonSSHStoreConfig::CommonSSHStoreConfig(std::string_view scheme, std::string_view host, const Params & params)
: StoreConfig(params)
, host(extractConnStr(scheme, host))
CommonSSHStoreConfig::CommonSSHStoreConfig(
std::string_view scheme, std::string_view host, const StoreReference::Params & params)
: CommonSSHStoreConfigT<config::PlainValue>{commonSSHStoreConfigApplyParse(params)}
, host{extractConnStr(scheme, host)}
{
}

71
src/libstore/config-parse.cc Normal file
View file

@ -0,0 +1,71 @@
#include <nlohmann/json.hpp>
#include "nix/store/config-parse.hh"
#include "nix/util/json-utils.hh"
#include "nix/util/util.hh"
namespace nix::config {
};
namespace nlohmann {
using namespace nix::config;
SettingDescription adl_serializer<SettingDescription>::from_json(const json & json)
{
auto & obj = getObject(json);
return {
.description = getString(valueAt(obj, "description")),
.experimentalFeature = valueAt(obj, "experimentalFeature").get<std::optional<Xp>>(),
.info = [&]() -> decltype(SettingDescription::info) {
if (auto documentDefault = optionalValueAt(obj, "documentDefault")) {
return SettingDescription::Single{
.defaultValue = *documentDefault ? (std::optional<nlohmann::json>{valueAt(obj, "defaultValue")})
: (std::optional<nlohmann::json>{}),
};
} else {
auto & subObj = getObject(valueAt(obj, "subSettings"));
return SettingDescription::Sub{
.nullable = valueAt(subObj, "nullable"),
.map = valueAt(subObj, "map"),
};
}
}(),
};
}
void adl_serializer<SettingDescription>::to_json(json & obj, SettingDescription sd)
{
obj.emplace("description", sd.description);
// obj.emplace("aliases", sd.aliases);
obj.emplace("experimentalFeature", sd.experimentalFeature);
std::visit(
overloaded{
[&](const SettingDescription::Single & single) {
// Indicate the default value is JSON, rather than a legacy setting
// boolean or string.
//
// TODO remove if we no longer have the legacy setting system / the
// code handling doc rendering of the settings is decoupled.
obj.emplace("isJson", true);
// Cannot just use `null` because the default value might itself be
// `null`.
obj.emplace("documentDefault", single.defaultValue.has_value());
if (single.defaultValue.has_value())
obj.emplace("defaultValue", *single.defaultValue);
},
[&](const SettingDescription::Sub & sub) {
json subJson;
subJson.emplace("nullable", sub.nullable);
subJson.emplace("map", sub.map);
obj.emplace("subSettings", std::move(subJson));
},
},
sd.info);
}
}

39
src/libstore/dummy-store.cc
View file

@ -1,33 +1,24 @@
#include "nix/store/dummy-store.hh"
#include "nix/store/store-registration.hh"
#include "nix/util/callback.hh"
namespace nix {
struct DummyStoreConfig : public std::enable_shared_from_this<DummyStoreConfig>, virtual StoreConfig {
using StoreConfig::StoreConfig;
DummyStoreConfig::DummyStoreConfig(
std::string_view scheme, std::string_view authority, const StoreReference::Params & params)
: StoreConfig{params}
{
if (!authority.empty())
throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority);
}
DummyStoreConfig(std::string_view scheme, std::string_view authority, const Params & params)
: StoreConfig(params)
{
if (!authority.empty())
throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority);
}
std::string DummyStoreConfig::doc()
{
return
#include "dummy-store.md"
;
}
static const std::string name() { return "Dummy Store"; }
static std::string doc()
{
return
#include "dummy-store.md"
;
}
static StringSet uriSchemes() {
return {"dummy"};
}
ref<Store> openStore() const override;
};
struct DummyStore : virtual Store
{
@ -42,7 +33,7 @@ struct DummyStore : virtual Store
std::string getUri() override
{
return *Config::uriSchemes().begin();
return *Config::uriSchemes().begin() + "://";
}
void queryPathInfoUncached(const StorePath & path,

2
src/libstore/filetransfer.cc
View file

@ -771,7 +771,7 @@ struct curlFileTransfer : public FileTransfer
}
#if NIX_WITH_S3_SUPPORT
std::tuple<std::string, std::string, Store::Config::Params> parseS3Uri(std::string uri)
std::tuple<std::string, std::string, StoreReference::Params> parseS3Uri(std::string uri)
{
auto [path, params] = splitUriAndParams(uri);

36
src/libstore/http-binary-cache-store.cc
View file

@ -7,6 +7,15 @@
namespace nix {
config::SettingDescriptionMap HttpBinaryCacheStoreConfig::descriptions()
{
config::SettingDescriptionMap ret;
ret.merge(StoreConfig::descriptions());
ret.merge(BinaryCacheStoreConfig::descriptions());
return ret;
}
MakeError(UploadToHTTP, Error);
@ -22,9 +31,9 @@ StringSet HttpBinaryCacheStoreConfig::uriSchemes()
HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig(
std::string_view scheme,
std::string_view _cacheUri,
const Params & params)
: StoreConfig(params)
, BinaryCacheStoreConfig(params)
const StoreReference::Params & params)
: Store::Config{params}
, BinaryCacheStoreConfig{*this, params}
, cacheUri(
std::string { scheme }
+ "://"
@ -60,15 +69,16 @@ public:
using Config = HttpBinaryCacheStoreConfig;
ref<Config> config;
ref<const Config> config;
HttpBinaryCacheStore(ref<Config> config)
HttpBinaryCacheStore(ref<const Config> config)
: Store{*config}
// TODO it will actually mutate the configuration
, BinaryCacheStore{*config}
, config{config}
{
diskCache = getNarInfoDiskCache();
init();
}
std::string getUri() override
@ -80,15 +90,18 @@ public:
{
// FIXME: do this lazily?
if (auto cacheInfo = diskCache->upToDateCacheExists(config->cacheUri)) {
config->wantMassQuery.setDefault(cacheInfo->wantMassQuery);
config->priority.setDefault(cacheInfo->priority);
resolvedSubstConfig.wantMassQuery.value =
config->storeConfig.wantMassQuery.optValue.value_or(cacheInfo->wantMassQuery);
resolvedSubstConfig.priority.value =
config->storeConfig.priority.optValue.value_or(cacheInfo->priority);
} else {
try {
BinaryCacheStore::init();
} catch (UploadToHTTP &) {
throw Error("'%s' does not appear to be a binary cache", config->cacheUri);
}
diskCache->createCache(config->cacheUri, config->storeDir, config->wantMassQuery, config->priority);
diskCache->createCache(
config->cacheUri, storeDir, resolvedSubstConfig.wantMassQuery, resolvedSubstConfig.priority);
}
}
@ -232,10 +245,7 @@ protected:
ref<Store> HttpBinaryCacheStore::Config::openStore() const
{
return make_ref<HttpBinaryCacheStore>(ref{
// FIXME we shouldn't actually need a mutable config
std::const_pointer_cast<HttpBinaryCacheStore::Config>(shared_from_this())
});
return make_ref<HttpBinaryCacheStore>(ref{shared_from_this()});
}
static RegisterStoreImplementation<HttpBinaryCacheStore::Config> regHttpBinaryCacheStore;

66
src/libstore/include/nix/store/binary-cache-store.hh
View file

@ -13,42 +13,28 @@ namespace nix {
struct NarInfo;
struct BinaryCacheStoreConfig : virtual StoreConfig
template<template<typename> class F>
struct BinaryCacheStoreConfigT
{
using StoreConfig::StoreConfig;
const Setting<std::string> compression{this, "xz", "compression",
"NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)."};
const Setting<bool> writeNARListing{this, false, "write-nar-listing",
"Whether to write a JSON file that lists the files in each NAR."};
const Setting<bool> writeDebugInfo{this, false, "index-debug-info",
R"(
Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to
fetch debug info on demand
)"};
const Setting<Path> secretKeyFile{this, "", "secret-key",
"Path to the secret key used to sign the binary cache."};
const Setting<std::string> secretKeyFiles{this, "", "secret-keys",
"List of comma-separated paths to the secret keys used to sign the binary cache."};
const Setting<Path> localNarCache{this, "", "local-nar-cache",
"Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."};
const Setting<bool> parallelCompression{this, false, "parallel-compression",
"Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."};
const Setting<int> compressionLevel{this, -1, "compression-level",
R"(
The *preset level* to be used when compressing NARs.
The meaning and accepted values depend on the compression method selected.
`-1` specifies that the default compression level should be used.
)"};
F<std::string> compression;
F<bool> writeNARListing;
F<bool> writeDebugInfo;
F<Path> secretKeyFile;
F<std::vector<Path>> secretKeyFiles;
F<Path> localNarCache;
F<bool> parallelCompression;
F<int> compressionLevel;
};
struct BinaryCacheStoreConfig :
BinaryCacheStoreConfigT<config::PlainValue>
{
static config::SettingDescriptionMap descriptions();
const Store::Config & storeConfig;
BinaryCacheStoreConfig(const Store::Config &, const StoreReference::Params &);
};
/**
* @note subclasses must implement at least one of the two
@ -60,11 +46,7 @@ struct BinaryCacheStore :
{
using Config = BinaryCacheStoreConfig;
/**
* Intentionally mutable because some things we update due to the
* cache's own (remote side) settings.
*/
Config & config;
const Config & config;
private:
std::vector<std::unique_ptr<Signer>> signers;
@ -76,7 +58,7 @@ protected:
const std::string cacheInfoFile = "nix-cache-info";
BinaryCacheStore(Config &);
BinaryCacheStore(const Config &);
public:
@ -114,7 +96,11 @@ public:
public:
virtual void init() override;
/**
* Perform any necessary effectful operation to make the store up and
* running
*/
virtual void init();
private:

36
src/libstore/include/nix/store/common-ssh-store-config.hh
View file

@ -7,27 +7,27 @@ namespace nix {
class SSHMaster;
struct CommonSSHStoreConfig : virtual StoreConfig
template<template<typename> class F>
struct CommonSSHStoreConfigT
{
using StoreConfig::StoreConfig;
F<Path> sshKey;
F<std::string> sshPublicHostKey;
F<bool> compress;
F<std::string> remoteStore;
};
CommonSSHStoreConfig(std::string_view scheme, std::string_view host, const Params & params);
struct CommonSSHStoreConfig : CommonSSHStoreConfigT<config::PlainValue>
{
static config::SettingDescriptionMap descriptions();
const Setting<Path> sshKey{this, "", "ssh-key",
"Path to the SSH private key used to authenticate to the remote machine."};
const Setting<std::string> sshPublicHostKey{this, "", "base64-ssh-public-host-key",
"The public host key of the remote machine."};
const Setting<bool> compress{this, false, "compress",
"Whether to enable SSH compression."};
const Setting<std::string> remoteStore{this, "", "remote-store",
R"(
[Store URL](@docroot@/store/types/index.md#store-url-format)
to be used on the remote machine. The default is `auto`
(i.e. use the Nix daemon or `/nix/store` directly).
)"};
/**
* @param scheme Note this isn't stored by this mix-in class, but
* just used for better error messages.
*/
CommonSSHStoreConfig(
std::string_view scheme,
std::string_view host,
const StoreReference::Params & params);
/**
* The `parseURL` function supports both IPv6 URIs as defined in

69
src/libstore/include/nix/store/config-parse-impl.hh Normal file
View file

@ -0,0 +1,69 @@
#pragma once
///@file
#include <nlohmann/json.hpp>
#include "nix/store/config-parse.hh"
#include "nix/util/util.hh"
#include "nix/util/configuration.hh"
namespace nix::config {
template<typename T>
OptValue<T>
SettingInfo<T>::parseConfig(const nlohmann::json::object_t & map, const ExperimentalFeatureSettings & xpSettings) const
{
const nlohmann::json * p = get(map, name);
if (p && experimentalFeature)
xpSettings.require(*experimentalFeature);
return {.optValue = p ? (std::optional<T>{p->get<T>()}) : std::nullopt};
}
template<typename T>
std::pair<std::string, SettingDescription> SettingInfo<T>::describe(const PlainValue<T> & def) const
{
return {
std::string{name},
SettingDescription{
.description = stripIndentation(description),
.experimentalFeature = experimentalFeature,
.info =
SettingDescription::Single{
.defaultValue = documentDefault ? (std::optional{nlohmann::json(def.value)})
: (std::optional<nlohmann::json>{}),
},
},
};
}
/**
* Look up the setting's name in a map, falling back on the default if
* it does not exist.
*/
#define CONFIG_ROW(FIELD) .FIELD = descriptions.FIELD.parseConfig(params, xpSettings)
#define APPLY_ROW(FIELD) .FIELD = {.value = parsed.FIELD.optValue.value_or(std::move(defaults.FIELD))}
#define DESCRIBE_ROW(FIELD) \
{ \
descriptions.FIELD.describe(defaults.FIELD), \
}
#define MAKE_PARSE(CAPITAL, LOWER, FIELDS) \
static CAPITAL##T<config::OptValue> LOWER##Parse( \
const StoreReference::Params & params, \
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings) \
{ \
constexpr auto & descriptions = LOWER##Descriptions; \
return {FIELDS(CONFIG_ROW)}; \
}
#define MAKE_APPLY_PARSE(CAPITAL, LOWER, FIELDS) \
static CAPITAL##T<config::PlainValue> LOWER##ApplyParse(const StoreReference::Params & params) \
{ \
auto defaults = LOWER##Defaults(); \
auto parsed = LOWER##Parse(params); \
return {FIELDS(APPLY_ROW)}; \
}
}

144
src/libstore/include/nix/store/config-parse.hh Normal file
View file

@ -0,0 +1,144 @@
#pragma once
///@file
#include <nlohmann/json.hpp>
#include "nix/util/config-abstract.hh"
#include "nix/util/json-impls.hh"
#include "nix/util/experimental-features.hh"
namespace nix {
struct ExperimentalFeatureSettings;
};
namespace nix::config {
struct SettingDescription;
/**
* Typed version used as source of truth, and for operations like
* defaulting configurations.
*
* It is important that this type support `constexpr` values to avoid
* running into issues with static initialization order.
*/
template<typename T>
struct SettingInfo
{
/**
* Name of the setting, used when parsing configuration maps
*/
std::string_view name;
/**
* Description of the setting. It is used just for documentation.
*/
std::string_view description;
#if 0
/**
* Other names of the setting also used when parsing configuration
* maps. This is useful for back-compat, etc.
*/
std::set<std::string_view> aliases;
#endif
/**
* `ExperimentalFeature` that must be enabled if the setting is
* allowed to be used
*/
std::optional<ExperimentalFeature> experimentalFeature;
/**
* Whether to document the default value. (Some defaults are
* system-specific and should not be documented.)
*/
bool documentDefault = true;
/**
* Describe the setting as a key-value pair (name -> other info).
* The default value will be rendered to JSON if it is to be
* documented.
*/
std::pair<std::string, SettingDescription> describe(const PlainValue<T> & def) const;
OptValue<T> parseConfig(const nlohmann::json::object_t & map, const ExperimentalFeatureSettings & xpSettings) const;
};
struct SettingDescription;
/**
* Map of setting names to descriptions of those settings.
*/
using SettingDescriptionMap = std::map<std::string, SettingDescription>;
/**
* Untyped version used for rendering docs. This is not the source of
* truth, it is generated from the typed one.
*
* @note No `name` field because this is intended to be used as the value type
* of a map
*/
struct SettingDescription
{
/**
* @see SettingInfo::description
*/
std::string description;
#if 0
/**
* @see SettingInfo::aliases
*/
StringSet aliases;
#endif
/**
* @see SettingInfo::experimentalFeature
*/
std::optional<ExperimentalFeature> experimentalFeature;
/**
* A single leaf setting, to be optionally specified by arbitrary
* value (of some type) or left default.
*/
struct Single
{
/**
* Optional, for the `SettingInfo::documentDefault = false` case.
*/
std::optional<nlohmann::json> defaultValue;
};
/**
* A nested settings object
*/
struct Sub
{
/**
* If `false`, this is just pure namespaceing. If `true`, we
* have a distinction between `null` and `{}`, meaning
* enabling/disabling the entire settings group.
*/
bool nullable = true;
SettingDescriptionMap map;
};
/**
* Variant for `info` below
*/
using Info = std::variant<Single, Sub>;
/**
* More information about this setting, depending on whether its the
* single leaf setting or subsettings case
*/
Info info;
};
}
JSON_IMPL(config::SettingDescription)

24
src/libstore/include/nix/store/dummy-store.hh Normal file
View file

@ -0,0 +1,24 @@
#include "nix/store/store-api.hh"
namespace nix {
struct DummyStoreConfig : std::enable_shared_from_this<DummyStoreConfig>, StoreConfig
{
DummyStoreConfig(std::string_view scheme, std::string_view authority, const StoreReference::Params & params);
static const std::string name()
{
return "Dummy Store";
}
static std::string doc();
static StringSet uriSchemes()
{
return {"dummy"};
}
ref<Store> openStore() const override;
};
}

6
src/libstore/include/nix/store/http-binary-cache-store.hh
View file

@ -3,13 +3,13 @@
namespace nix {
struct HttpBinaryCacheStoreConfig : std::enable_shared_from_this<HttpBinaryCacheStoreConfig>,
virtual Store::Config,
Store::Config,
BinaryCacheStoreConfig
{
using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
static config::SettingDescriptionMap descriptions();
HttpBinaryCacheStoreConfig(
std::string_view scheme, std::string_view cacheUri, const Store::Config::Params & params);
std::string_view scheme, std::string_view cacheUri, const StoreReference::Params & params);
Path cacheUri;

38
src/libstore/include/nix/store/legacy-ssh-store.hh
View file

@ -10,29 +10,31 @@
namespace nix {
struct LegacySSHStoreConfig : std::enable_shared_from_this<LegacySSHStoreConfig>, virtual CommonSSHStoreConfig
template<template<typename> class F>
struct LegacySSHStoreConfigT
{
using CommonSSHStoreConfig::CommonSSHStoreConfig;
F<Strings> remoteProgram;
F<int> maxConnections;
};
struct LegacySSHStoreConfig :
std::enable_shared_from_this<LegacySSHStoreConfig>,
Store::Config,
CommonSSHStoreConfig,
LegacySSHStoreConfigT<config::PlainValue>
{
static config::SettingDescriptionMap descriptions();
/**
* Hack for getting remote build log output. Intentionally not a
* documented user-visible setting.
*/
Descriptor logFD = INVALID_DESCRIPTOR;
LegacySSHStoreConfig(
std::string_view scheme,
std::string_view authority,
const Params & params);
#ifndef _WIN32
// Hack for getting remote build log output.
// Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in
// the documentation
const Setting<int> logFD{this, INVALID_DESCRIPTOR, "log-fd", "file descriptor to which SSH's stderr is connected"};
#else
Descriptor logFD = INVALID_DESCRIPTOR;
#endif
const Setting<Strings> remoteProgram{this, {"nix-store"}, "remote-program",
"Path to the `nix-store` executable on the remote machine."};
const Setting<int> maxConnections{this, 1, "max-connections",
"Maximum number of concurrent SSH connections."};
const StoreReference::Params & params);
/**
* Hack for hydra

7
src/libstore/include/nix/store/local-binary-cache-store.hh
View file

@ -3,16 +3,17 @@
namespace nix {
struct LocalBinaryCacheStoreConfig : std::enable_shared_from_this<LocalBinaryCacheStoreConfig>,
virtual Store::Config,
Store::Config,
BinaryCacheStoreConfig
{
using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
static config::SettingDescriptionMap descriptions();
/**
* @param binaryCacheDir `file://` is a short-hand for `file:///`
* for now.
*/
LocalBinaryCacheStoreConfig(std::string_view scheme, PathView binaryCacheDir, const Params & params);
LocalBinaryCacheStoreConfig(
std::string_view scheme, PathView binaryCacheDir, const StoreReference::Params & params);
Path binaryCacheDir;

42
src/libstore/include/nix/store/local-fs-store.hh
View file

@ -7,9 +7,24 @@
namespace nix {
struct LocalFSStoreConfig : virtual StoreConfig
template<template<typename> class F>
struct LocalFSStoreConfigT
{
using StoreConfig::StoreConfig;
F<std::optional<Path>> rootDir;
F<Path> stateDir;
F<Path> logDir;
F<Path> realStoreDir;
};
struct LocalFSStoreConfig : LocalFSStoreConfigT<config::PlainValue>
{
const Store::Config & storeConfig;
static config::SettingDescriptionMap descriptions();
LocalFSStoreConfig(
const Store::Config & storeConfig,
const StoreReference::Params &);
/**
* Used to override the `root` settings. Can't be done via modifying
@ -18,25 +33,10 @@ struct LocalFSStoreConfig : virtual StoreConfig
*
* @todo Make this less error-prone with new store settings system.
*/
LocalFSStoreConfig(PathView path, const Params & params);
OptionalPathSetting rootDir{this, std::nullopt,
"root",
"Directory prefixed to all other paths."};
PathSetting stateDir{this,
rootDir.get() ? *rootDir.get() + "/nix/var/nix" : settings.nixStateDir,
"state",
"Directory where Nix will store state."};
PathSetting logDir{this,
rootDir.get() ? *rootDir.get() + "/nix/var/log/nix" : settings.nixLogDir,
"log",
"directory where Nix will store log files."};
PathSetting realStoreDir{this,
rootDir.get() ? *rootDir.get() + "/nix/store" : storeDir, "real",
"Physical path of the Nix store."};
LocalFSStoreConfig(
const Store::Config & storeConfig,
PathView path,
const StoreReference::Params & params);
};
struct LocalFSStore :

66
src/libstore/include/nix/store/local-overlay-store.hh
View file

@ -2,59 +2,29 @@
namespace nix {
template<template<typename> class F>
struct LocalOverlayStoreConfigT
{
F<ref<const StoreConfig>> lowerStoreConfig;
F<Path> upperLayer;
F<bool> checkMount;
F<Path> remountHook;
};
/**
* Configuration for `LocalOverlayStore`.
*/
struct LocalOverlayStoreConfig : virtual LocalStoreConfig
struct LocalOverlayStoreConfig :
LocalStoreConfig,
LocalOverlayStoreConfigT<config::PlainValue>
{
LocalOverlayStoreConfig(const StringMap & params)
: LocalOverlayStoreConfig("local-overlay", "", params)
{ }
static config::SettingDescriptionMap descriptions();
LocalOverlayStoreConfig(std::string_view scheme, PathView path, const Params & params)
: StoreConfig(params)
, LocalFSStoreConfig(path, params)
, LocalStoreConfig(scheme, path, params)
{
}
const Setting<std::string> lowerStoreUri{(StoreConfig*) this, "", "lower-store",
R"(
[Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly).
Must be a store with a store dir on the file system.
Must be used as OverlayFS lower layer for this store's store dir.
)"};
const PathSetting upperLayer{(StoreConfig*) this, "", "upper-layer",
R"(
Directory containing the OverlayFS upper layer for this store's store dir.
)"};
Setting<bool> checkMount{(StoreConfig*) this, true, "check-mount",
R"(
Check that the overlay filesystem is correctly mounted.
Nix does not manage the overlayfs mount point itself, but the correct
functioning of the overlay store does depend on this mount point being set up
correctly. Rather than just assume this is the case, check that the lowerdir
and upperdir options are what we expect them to be. This check is on by
default, but can be disabled if needed.
)"};
const PathSetting remountHook{(StoreConfig*) this, "", "remount-hook",
R"(
Script or other executable to run when overlay filesystem needs remounting.
This is occasionally necessary when deleting a store path that exists in both upper and lower layers.
In such a situation, bypassing OverlayFS and deleting the path in the upper layer directly
is the only way to perform the deletion without creating a "whiteout".
However this causes the OverlayFS kernel data structures to get out-of-sync,
and can lead to 'stale file handle' errors; remounting solves the problem.
The store directory is passed as an argument to the invoked executable.
)"};
LocalOverlayStoreConfig(
std::string_view scheme,
PathView path,
const StoreReference::Params & params);
static const std::string name() { return "Experimental Local Overlay Store"; }

46
src/libstore/include/nix/store/local-store.hh
View file

@ -34,36 +34,34 @@ struct OptimiseStats
uint64_t bytesFreed = 0;
};
struct LocalStoreConfig : std::enable_shared_from_this<LocalStoreConfig>, virtual LocalFSStoreConfig
template<template<typename> class F>
struct LocalStoreConfigT
{
using LocalFSStoreConfig::LocalFSStoreConfig;
F<bool> requireSigs;
F<bool> readOnly;
};
struct LocalStoreConfig :
std::enable_shared_from_this<LocalStoreConfig>,
Store::Config,
LocalFSStore::Config,
LocalStoreConfigT<config::PlainValue>
{
static config::SettingDescriptionMap descriptions();
LocalStoreConfig(const StoreReference::Params & params)
: LocalStoreConfig{"local", "", params}
{}
LocalStoreConfig(
std::string_view scheme,
std::string_view authority,
const Params & params);
const StoreReference::Params & params);
Setting<bool> requireSigs{this,
settings.requireSigs,
"require-sigs",
"Whether store paths copied into this store should have a trusted signature."};
Setting<bool> readOnly{this,
false,
"read-only",
R"(
Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem.
Normally Nix will attempt to open the store database in read-write mode, even for querying (when write access is not needed), causing it to fail if the database is on a read-only filesystem.
Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set.
> **Warning**
> Do not use this unless the filesystem is read-only.
>
> Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process.
> While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it.
)"};
/**
* For `RestrictedStore`
*/
LocalStoreConfig(const LocalStoreConfig &);
static const std::string name() { return "Local Store"; }

2
src/libstore/include/nix/store/machines.hh
View file

@ -1,6 +1,8 @@
#pragma once
///@file
#include <nlohmann/json.hpp>
#include "nix/util/ref.hh"
#include "nix/store/store-reference.hh"

5
src/libstore/include/nix/store/meson.build
View file

@ -24,13 +24,16 @@ headers = [config_pub_h] + files(
'common-protocol-impl.hh',
'common-protocol.hh',
'common-ssh-store-config.hh',
'config-parse-impl.hh',
'config-parse.hh',
'content-address.hh',
'daemon.hh',
'derivations.hh',
'derivation-options.hh',
'derivations.hh',
'derived-path-map.hh',
'derived-path.hh',
'downstream-placeholder.hh',
'dummy-store.hh',
'filetransfer.hh',
'gc-store.hh',
'globals.hh',

19
src/libstore/include/nix/store/remote-store.hh
View file

@ -18,17 +18,20 @@ struct FdSink;
struct FdSource;
template<typename T> class Pool;
struct RemoteStoreConfig : virtual StoreConfig
template<template<typename> class F>
struct RemoteStoreConfigT
{
using StoreConfig::StoreConfig;
F<int> maxConnections;
F<unsigned int> maxConnectionAge;
};
const Setting<int> maxConnections{this, 1, "max-connections",
"Maximum number of concurrent connections to the Nix daemon."};
struct RemoteStoreConfig : RemoteStoreConfigT<config::PlainValue>
{
static config::SettingDescriptionMap descriptions();
const Setting<unsigned int> maxConnectionAge{this,
std::numeric_limits<unsigned int>::max(),
"max-connection-age",
"Maximum age of a connection before it is closed."};
const Store::Config & storeConfig;
RemoteStoreConfig(const Store::Config &, const StoreReference::Params &);
};
/**

108
src/libstore/include/nix/store/s3-binary-cache-store.hh
View file

@ -11,89 +11,33 @@
namespace nix {
struct S3BinaryCacheStoreConfig : std::enable_shared_from_this<S3BinaryCacheStoreConfig>, virtual BinaryCacheStoreConfig
template<template<typename> class F>
struct S3BinaryCacheStoreConfigT
{
F<std::string> profile;
F<std::string> region;
F<std::string> scheme;
F<std::string> endpoint;
F<std::string> narinfoCompression;
F<std::string> lsCompression;
F<std::string> logCompression;
F<bool> multipartUpload;
F<uint64_t> bufferSize;
};
struct S3BinaryCacheStoreConfig : std::enable_shared_from_this<S3BinaryCacheStoreConfig>,
Store::Config,
BinaryCacheStoreConfig,
S3BinaryCacheStoreConfigT<config::PlainValue>
{
static config::SettingDescriptionMap descriptions();
S3BinaryCacheStoreConfig(
std::string_view uriScheme, std::string_view bucketName, const StoreReference::Params & params);
std::string bucketName;
using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
S3BinaryCacheStoreConfig(std::string_view uriScheme, std::string_view bucketName, const Params & params);
const Setting<std::string> profile{
this,
"",
"profile",
R"(
The name of the AWS configuration profile to use. By default
Nix will use the `default` profile.
)"};
protected:
constexpr static const char * defaultRegion = "us-east-1";
public:
const Setting<std::string> region{
this,
defaultRegion,
"region",
R"(
The region of the S3 bucket. If your bucket is not in
`us–east-1`, you should always explicitly specify the region
parameter.
)"};
const Setting<std::string> scheme{
this,
"",
"scheme",
R"(
The scheme used for S3 requests, `https` (default) or `http`. This
option allows you to disable HTTPS for binary caches which don't
support it.
> **Note**
>
> HTTPS should be used if the cache might contain sensitive
> information.
)"};
const Setting<std::string> endpoint{
this,
"",
"endpoint",
R"(
The URL of the endpoint of an S3-compatible service such as MinIO.
Do not specify this setting if you're using Amazon S3.
> **Note**
>
> This endpoint must support HTTPS and will use path-based
> addressing instead of virtual host based addressing.
)"};
const Setting<std::string> narinfoCompression{
this, "", "narinfo-compression", "Compression method for `.narinfo` files."};
const Setting<std::string> lsCompression{this, "", "ls-compression", "Compression method for `.ls` files."};
const Setting<std::string> logCompression{
this,
"",
"log-compression",
R"(
Compression method for `log/*` files. It is recommended to
use a compression method supported by most web browsers
(e.g. `brotli`).
)"};
const Setting<bool> multipartUpload{this, false, "multipart-upload", "Whether to use multi-part uploads."};
const Setting<uint64_t> bufferSize{
this, 5 * 1024 * 1024, "buffer-size", "Size (in bytes) of each part in multi-part uploads."};
static const std::string name()
static std::string name()
{
return "S3 Binary Cache Store";
}
@ -112,9 +56,9 @@ struct S3BinaryCacheStore : virtual BinaryCacheStore
{
using Config = S3BinaryCacheStoreConfig;
ref<Config> config;
ref<const Config> config;
S3BinaryCacheStore(ref<Config>);
S3BinaryCacheStore(ref<const Config>);
struct Stats
{

51
src/libstore/include/nix/store/ssh-store.hh
View file

@ -8,17 +8,27 @@
namespace nix {
struct SSHStoreConfig : std::enable_shared_from_this<SSHStoreConfig>,
virtual RemoteStoreConfig,
virtual CommonSSHStoreConfig
template<template<typename> class F>
struct SSHStoreConfigT
{
using CommonSSHStoreConfig::CommonSSHStoreConfig;
using RemoteStoreConfig::RemoteStoreConfig;
F<Strings> remoteProgram;
};
SSHStoreConfig(std::string_view scheme, std::string_view authority, const Params & params);
struct SSHStoreConfig : std::enable_shared_from_this<SSHStoreConfig>,
Store::Config,
RemoteStore::Config,
CommonSSHStoreConfig,
SSHStoreConfigT<config::PlainValue>
{
static config::SettingDescriptionMap descriptions();
const Setting<Strings> remoteProgram{
this, {"nix-daemon"}, "remote-program", "Path to the `nix-daemon` executable on the remote machine."};
std::optional<LocalFSStore::Config> mounted;
SSHStoreConfig(
std::string_view scheme,
std::string_view authority,
const StoreReference::Params & params,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
static const std::string name()
{
@ -35,29 +45,4 @@ struct SSHStoreConfig : std::enable_shared_from_this<SSHStoreConfig>,
ref<Store> openStore() const override;
};
struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig
{
MountedSSHStoreConfig(StringMap params);
MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params);
static const std::string name()
{
return "Experimental SSH Store with filesystem mounted";
}
static StringSet uriSchemes()
{
return {"mounted-ssh-ng"};
}
static std::string doc();
static std::optional<ExperimentalFeature> experimentalFeature()
{
return ExperimentalFeature::MountedSSHStore;
}
ref<Store> openStore() const override;
};
}

138
src/libstore/include/nix/store/store-api.hh
View file

@ -9,7 +9,6 @@
#include "nix/util/lru-cache.hh"
#include "nix/util/sync.hh"
#include "nix/store/globals.hh"
#include "nix/util/configuration.hh"
#include "nix/store/path-info.hh"
#include "nix/util/repair-flag.hh"
#include "nix/store/store-dir-config.hh"
@ -26,6 +25,32 @@
namespace nix {
/**
* About the class hierarchy of the store types:
*
* Each store type `Foo` consists of two classes:
*
* 1. A class `FooConfig : virtual StoreConfig` that contains the configuration
* for the store
*
* It should only contain members of type `const Setting<T>` (or subclasses
* of it) and inherit the constructors of `StoreConfig`
* (`using StoreConfig::StoreConfig`).
*
* 2. A class `Foo : virtual Store, virtual FooConfig` that contains the
* implementation of the store.
*
* This class is expected to have a constructor `Foo(const StoreReference::Params & params)`
* that calls `StoreConfig(params)` (otherwise you're gonna encounter an
* `assertion failure` when trying to instantiate it).
*
* You can then register the new store using:
*
* ```
* cpp static RegisterStoreImplementation<Foo, FooConfig> regStore;
* ```
*/
MakeError(SubstError, Error);
/**
* denotes a permanent build failure
@ -71,39 +96,43 @@ struct KeyedBuildResult;
typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
template<template<typename> class F>
struct StoreConfigT
{
F<int> pathInfoCacheSize;
F<bool> isTrusted;
F<StringSet> systemFeatures;
};
template<template<typename> class F>
struct SubstituterConfigT
{
F<int> priority;
F<bool> wantMassQuery;
};
/**
* About the class hierarchy of the store types:
*
* Each store type `Foo` consists of two classes:
*
* 1. A class `FooConfig : virtual StoreConfig` that contains the configuration
* for the store
*
* It should only contain members of type `Setting<T>` (or subclasses
* of it) and inherit the constructors of `StoreConfig`
* (`using StoreConfig::StoreConfig`).
*
* 2. A class `Foo : virtual Store` that contains the
* implementation of the store.
*
* This class is expected to have:
*
* 1. an alias `using Config = FooConfig;`
*
* 2. a constructor `Foo(ref<const Config> params)`.
*
* You can then register the new store using:
*
* ```
* cpp static RegisterStoreImplementation<FooConfig> regStore;
* ```
* @note In other cases we don't expose this function directly, but in
* this case we must because of `Store::resolvedSubstConfig` below. As
* the docs of that field describe, this is a case where the
* configuration is intentionally stateful.
*/
struct StoreConfig : public StoreDirConfig
{
using StoreDirConfig::StoreDirConfig;
SubstituterConfigT<config::PlainValue> substituterConfigDefaults();
StoreConfig() = delete;
/**
* @note `config::OptValue` rather than `config::PlainValue` is applied to
* `SubstitutorConfigT` because these are overrides. Caches themselves (not our
* config) can update default settings, but aren't allowed to update settings
* specified by the client (i.e. us).
*/
struct StoreConfig :
StoreDirConfig,
StoreConfigT<config::PlainValue>,
SubstituterConfigT<config::OptValue>
{
static config::SettingDescriptionMap descriptions();
StoreConfig(const StoreReference::Params &);
virtual ~StoreConfig() { }
@ -126,39 +155,6 @@ struct StoreConfig : public StoreDirConfig
return std::nullopt;
}
Setting<int> pathInfoCacheSize{this, 65536, "path-info-cache-size",
"Size of the in-memory store path metadata cache."};
Setting<bool> isTrusted{this, false, "trusted",
R"(
Whether paths from this store can be used as substitutes
even if they are not signed by a key listed in the
[`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys)
setting.
)"};
Setting<int> priority{this, 0, "priority",
R"(
Priority of this store when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters).
A lower value means a higher priority.
)"};
Setting<bool> wantMassQuery{this, false, "want-mass-query",
R"(
Whether this store can be queried efficiently for path validity when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters).
)"};
Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(),
"system-features",
R"(
Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations.
Example: `"kvm"`
)",
{},
// Don't document the machine-specific default value
false};
/**
* Open a store of the type corresponding to this configuration
* type.
@ -190,6 +186,14 @@ public:
*/
operator const Config &() const { return config; }
/**
* Resolved substituter configuration. This is intentionally mutable
* as store clients may do IO to ask the underlying store for their
* default setting values if the client config did not statically
* override them.
*/
SubstituterConfigT<config::PlainValue> resolvedSubstConfig = substituterConfigDefaults();
protected:
struct PathInfoCacheValue {
@ -231,11 +235,6 @@ protected:
Store(const Store::Config & config);
public:
/**
* Perform any necessary effectful operation to make the store up and
* running
*/
virtual void init() {};
virtual ~Store() { }
@ -909,3 +908,6 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
Store * evalStore = nullptr);
}
// Parses a Store URL, uses global state not pure so think about this
JSON_IMPL(ref<const StoreConfig>)

59
src/libstore/include/nix/store/store-dir-config.hh
View file

@ -3,8 +3,8 @@
#include "nix/store/path.hh"
#include "nix/util/hash.hh"
#include "nix/store/content-address.hh"
#include "nix/store/globals.hh"
#include "nix/util/configuration.hh"
#include "nix/store/store-reference.hh"
#include "nix/store/config-parse.hh"
#include <map>
#include <string>
@ -18,6 +18,20 @@ struct SourcePath;
MakeError(BadStorePath, Error);
MakeError(BadStorePathName, BadStorePath);
/**
* Underlying store directory configuration type.
*
* Don't worry to much about the `F` parameter, it just some abstract
* nonsense for the "higher-kinded data" pattern. It is used in each
* settings record in order to ensure don't forgot to parse or document
* settings field.
*/
template<template<typename> class F>
struct StoreDirConfigT
{
F<Path> _storeDir;
};
/**
* @todo This should just be part of `StoreDirConfig`. However, it would
* be a huge amount of churn if `Store` didn't have these methods
@ -31,8 +45,6 @@ struct MixStoreDirMethods
{
const Path & storeDir;
// pure methods
StorePath parseStorePath(std::string_view path) const;
std::optional<StorePath> maybeParseStorePath(std::string_view path) const;
@ -101,35 +113,22 @@ struct MixStoreDirMethods
};
/**
* Need to make this a separate class so I can get the right
* initialization order in the constructor for `StoreDirConfig`.
* Store directory configuration type.
*
* Combines the underlying `*T` type (with plain values for the fields)
* and the methods.
*
* The order of `StoreDirConfigT<config::PlainValue>` and then
* `MixStoreDirMethods` is very important. This ensures that
* `StoreDirConfigT<config::PlainValue>::storeDir_` is initialized
* before we have our one chance (because references are immutable) to
* initialize `MixStoreDirMethods::storeDir`.
*/
struct StoreDirConfigBase : Config
struct StoreDirConfig : StoreDirConfigT<config::PlainValue>, MixStoreDirMethods
{
using Config::Config;
static config::SettingDescriptionMap descriptions();
const PathSetting storeDir_{this, settings.nixStore,
"store",
R"(
Logical location of the Nix store, usually
`/nix/store`. Note that you can only copy store paths
between stores if they have the same `store` setting.
)"};
};
/**
* The order of `StoreDirConfigBase` and then `MixStoreDirMethods` is
* very important. This ensures that `StoreDirConfigBase::storeDir_`
* is initialized before we have our one chance (because references are
* immutable) to initialize `MixStoreDirMethods::storeDir`.
*/
struct StoreDirConfig : StoreDirConfigBase, MixStoreDirMethods
{
using Params = StringMap;
StoreDirConfig(const Params & params);
StoreDirConfig() = delete;
StoreDirConfig(const StoreReference::Params & params);
virtual ~StoreDirConfig() = default;
};

18
src/libstore/include/nix/store/store-reference.hh
View file

@ -2,8 +2,10 @@
///@file
#include <variant>
#include <nlohmann/json_fwd.hpp>
#include "nix/util/types.hh"
#include "nix/util/json-impls.hh"
namespace nix {
@ -41,7 +43,17 @@ namespace nix {
*/
struct StoreReference
{
using Params = StringMap;
/**
* Would do
*
* ```
* using Params = nlohmann::json::object_t;
* ```
*
* but cannot because `<nlohmann/json_fwd.hpp>` doesn't have that.
*
*/
using Params = std::map<std::string, nlohmann::json, std::less<>>;
/**
* Special store reference `""` or `"auto"`
@ -70,7 +82,7 @@ struct StoreReference
Params params;
bool operator==(const StoreReference & rhs) const = default;
bool operator==(const StoreReference & rhs) const;
/**
* Render the whole store reference as a URI, including parameters.
@ -89,3 +101,5 @@ struct StoreReference
std::pair<std::string, StoreReference::Params> splitUriAndParams(const std::string & uri);
}
JSON_IMPL(StoreReference)

31
src/libstore/include/nix/store/store-registration.hh
View file

@ -29,6 +29,19 @@ struct StoreFactory
*/
StringSet uriSchemes;
/**
* @note This is a functional pointer for now because this situation:
*
* - We register store types with global initializers
*
* - The default values for some settings maybe depend on the settings globals.
*
* And because the ordering of global initialization is arbitrary,
* this is not allowed. For now, we can simply defer actually
* creating these maps until we need to later.
*/
config::SettingDescriptionMap (*configDescriptions)();
/**
* An experimental feature this type store is gated, if it is to be
* experimental.
@ -40,21 +53,21 @@ struct StoreFactory
* whatever comes after `<scheme>://` and before `?<query-params>`.
*/
std::function<ref<StoreConfig>(
std::string_view scheme, std::string_view authorityPath, const Store::Config::Params & params)>
std::string_view scheme, std::string_view authorityPath, const StoreReference::Params & params)>
parseConfig;
/**
* Just for dumping the defaults. Kind of awkward this exists,
* because it means we cannot require fields to be manually
* specified so easily.
*/
std::function<ref<StoreConfig>()> getConfig;
};
struct Implementations
{
private:
/**
* The name of this type of store, and a factory for it.
*/
using Map = std::map<std::string, StoreFactory>;
public:
static Map & registered();
template<typename TConfig>
@ -63,11 +76,11 @@ struct Implementations
StoreFactory factory{
.doc = TConfig::doc(),
.uriSchemes = TConfig::uriSchemes(),
.configDescriptions = TConfig::descriptions,
.experimentalFeature = TConfig::experimentalFeature(),
.parseConfig = ([](auto scheme, auto uri, auto & params) -> ref<StoreConfig> {
return make_ref<TConfig>(scheme, uri, params);
}),
.getConfig = ([]() -> ref<StoreConfig> { return make_ref<TConfig>(Store::Config::Params{}); }),
};
auto [it, didInsert] = registered().insert({TConfig::name(), std::move(factory)});
if (!didInsert) {

18
src/libstore/include/nix/store/uds-remote-store.hh
View file

@ -9,13 +9,15 @@ namespace nix {
struct UDSRemoteStoreConfig :
std::enable_shared_from_this<UDSRemoteStoreConfig>,
virtual LocalFSStoreConfig,
virtual RemoteStoreConfig
Store::Config,
LocalFSStore::Config,
RemoteStore::Config
{
// TODO(fzakaria): Delete this constructor once moved over to the factory pattern
// outlined in https://github.com/NixOS/nix/issues/10766
using LocalFSStoreConfig::LocalFSStoreConfig;
using RemoteStoreConfig::RemoteStoreConfig;
static config::SettingDescriptionMap descriptions();
UDSRemoteStoreConfig(const StoreReference::Params & params)
: UDSRemoteStoreConfig{"unix", "", params}
{}
/**
* @param authority is the socket path.
@ -23,9 +25,7 @@ struct UDSRemoteStoreConfig :
UDSRemoteStoreConfig(
std::string_view scheme,
std::string_view authority,
const Params & params);
UDSRemoteStoreConfig(const Params & params);
const StoreReference::Params & params);
static const std::string name() { return "Local Daemon Store"; }

65
src/libstore/legacy-ssh-store.cc
View file

@ -12,19 +12,76 @@
#include "nix/store/ssh.hh"
#include "nix/store/derivations.hh"
#include "nix/util/callback.hh"
#include "nix/store/config-parse-impl.hh"
#include "nix/store/store-registration.hh"
namespace nix {
LegacySSHStoreConfig::LegacySSHStoreConfig(
constexpr static const LegacySSHStoreConfigT<config::SettingInfo> legacySSHStoreConfigDescriptions = {
.remoteProgram{
.name = "remote-program",
.description = "Path to the `nix-store` executable on the remote machine.",
},
.maxConnections{
.name = "max-connections",
.description = "Maximum number of concurrent SSH connections.",
},
};
#define LEGACY_SSH_STORE_CONFIG_FIELDS(X) \
X(remoteProgram), \
X(maxConnections)
MAKE_PARSE(LegacySSHStoreConfig, legacySSHStoreConfig, LEGACY_SSH_STORE_CONFIG_FIELDS)
static LegacySSHStoreConfigT<config::PlainValue> legacySSHStoreConfigDefaults()
{
return {
.remoteProgram = {{"nix-store"}},
.maxConnections = {1},
};
}
MAKE_APPLY_PARSE(LegacySSHStoreConfig, legacySSHStoreConfig, LEGACY_SSH_STORE_CONFIG_FIELDS)
config::SettingDescriptionMap LegacySSHStoreConfig::descriptions()
{
config::SettingDescriptionMap ret;
ret.merge(StoreConfig::descriptions());
ret.merge(CommonSSHStoreConfig::descriptions());
ret.merge(RemoteStoreConfig::descriptions());
{
constexpr auto & descriptions = legacySSHStoreConfigDescriptions;
auto defaults = legacySSHStoreConfigDefaults();
ret.merge(decltype(ret){
LEGACY_SSH_STORE_CONFIG_FIELDS(DESCRIBE_ROW)
});
}
return ret;
}
LegacySSHStore::Config::LegacySSHStoreConfig(
std::string_view scheme,
std::string_view authority,
const Params & params)
: StoreConfig(params)
, CommonSSHStoreConfig(scheme, authority, params)
const StoreReference::Params & params)
: Store::Config{params}
, CommonSSHStoreConfig{scheme, authority, params}
, LegacySSHStoreConfigT<config::PlainValue>{legacySSHStoreConfigApplyParse(params)}
{
#ifndef _WIN32
if (auto * p = get(params, "log-fd")) {
logFD = p->get<decltype(logFD)>();
}
#endif
}
std::string LegacySSHStoreConfig::doc()
{
return

20
src/libstore/local-binary-cache-store.cc
View file

@ -8,12 +8,21 @@
namespace nix {
config::SettingDescriptionMap LocalBinaryCacheStoreConfig::descriptions()
{
config::SettingDescriptionMap ret;
ret.merge(StoreConfig::descriptions());
ret.merge(BinaryCacheStoreConfig::descriptions());
return ret;
}
LocalBinaryCacheStoreConfig::LocalBinaryCacheStoreConfig(
std::string_view scheme,
PathView binaryCacheDir,
const StoreReference::Params & params)
: Store::Config{params}
, BinaryCacheStoreConfig{params}
, BinaryCacheStoreConfig{*this, params}
, binaryCacheDir(binaryCacheDir)
{
}
@ -32,9 +41,9 @@ struct LocalBinaryCacheStore :
{
using Config = LocalBinaryCacheStoreConfig;
ref<Config> config;
ref<const Config> config;
LocalBinaryCacheStore(ref<Config> config)
LocalBinaryCacheStore(ref<const Config> config)
: Store{*config}
, BinaryCacheStore{*config}
, config{config}
@ -126,10 +135,7 @@ StringSet LocalBinaryCacheStoreConfig::uriSchemes()
}
ref<Store> LocalBinaryCacheStoreConfig::openStore() const {
return make_ref<LocalBinaryCacheStore>(ref{
// FIXME we shouldn't actually need a mutable config
std::const_pointer_cast<LocalBinaryCacheStore::Config>(shared_from_this())
});
return make_ref<LocalBinaryCacheStore>(ref{shared_from_this()});
}
static RegisterStoreImplementation<LocalBinaryCacheStore::Config> regLocalBinaryCacheStore;

105
src/libstore/local-fs-store.cc
View file

@ -1,3 +1,4 @@
#include "nix/util/json-utils.hh"
#include "nix/util/archive.hh"
#include "nix/util/posix-source-accessor.hh"
#include "nix/store/store-api.hh"
@ -5,20 +6,102 @@
#include "nix/store/globals.hh"
#include "nix/util/compression.hh"
#include "nix/store/derivations.hh"
#include "nix/store/config-parse-impl.hh"
namespace nix {
LocalFSStoreConfig::LocalFSStoreConfig(PathView rootDir, const Params & params)
: StoreConfig(params)
// Default `?root` from `rootDir` if non set
// FIXME don't duplicate description once we don't have root setting
, rootDir{
this,
!rootDir.empty() && params.count("root") == 0
? (std::optional<Path>{rootDir})
: std::nullopt,
"root",
"Directory prefixed to all other paths."}
constexpr static const LocalFSStoreConfigT<config::SettingInfo> localFSStoreConfigDescriptions = {
.rootDir = {
.name = "root",
.description = "Directory prefixed to all other paths.",
},
.stateDir = {
.name = "state",
.description = "Directory where Nix will store state.",
},
.logDir = {
.name = "log",
.description = "directory where Nix will store log files.",
},
.realStoreDir{
.name = "real",
.description = "Physical path of the Nix store.",
},
};
#define LOCAL_FS_STORE_CONFIG_FIELDS(X) \
X(rootDir), \
X(stateDir), \
X(logDir), \
X(realStoreDir),
MAKE_PARSE(LocalFSStoreConfig, localFSStoreConfig, LOCAL_FS_STORE_CONFIG_FIELDS)
/**
* @param rootDir Fallback if not in `params`
*/
static LocalFSStoreConfigT<config::PlainValue> localFSStoreConfigDefaults(
const Path & storeDir,
const std::optional<Path> & rootDir)
{
return {
.rootDir = {std::nullopt},
.stateDir = {rootDir ? *rootDir + "/nix/var/nix" : settings.nixStateDir},
.logDir = {rootDir ? *rootDir + "/nix/var/log/nix" : settings.nixLogDir},
.realStoreDir = {rootDir ? *rootDir + "/nix/store" : storeDir},
};
}
static LocalFSStoreConfigT<config::PlainValue> localFSStoreConfigApplyParse(
const Path & storeDir,
LocalFSStoreConfigT<config::OptValue> parsed)
{
auto defaults = localFSStoreConfigDefaults(
storeDir,
parsed.rootDir.optValue.value_or(std::nullopt));
return {LOCAL_FS_STORE_CONFIG_FIELDS(APPLY_ROW)};
}
config::SettingDescriptionMap LocalFSStoreConfig::descriptions()
{
constexpr auto & descriptions = localFSStoreConfigDescriptions;
auto defaults = localFSStoreConfigDefaults(settings.nixStore, std::nullopt);
return {
LOCAL_FS_STORE_CONFIG_FIELDS(DESCRIBE_ROW)
};
}
LocalFSStore::Config::LocalFSStoreConfig(
const Store::Config & storeConfig,
const StoreReference::Params & params)
: LocalFSStoreConfigT<config::PlainValue>{
localFSStoreConfigApplyParse(
storeConfig.storeDir,
localFSStoreConfigParse(params))}
, storeConfig{storeConfig}
{
}
static LocalFSStoreConfigT<config::OptValue> applyAuthority(
LocalFSStoreConfigT<config::OptValue> parsed,
PathView rootDir)
{
if (!rootDir.empty())
parsed.rootDir = {.optValue = {Path{rootDir}}};
return parsed;
}
LocalFSStore::Config::LocalFSStoreConfig(
const Store::Config & storeConfig,
PathView rootDir,
const StoreReference::Params & params)
: LocalFSStoreConfigT<config::PlainValue>{
localFSStoreConfigApplyParse(
storeConfig.storeDir,
applyAuthority(
localFSStoreConfigParse(params),
rootDir))}
, storeConfig{storeConfig}
{
}

99
src/libstore/local-overlay-store.cc
View file

@ -5,11 +5,106 @@
#include "nix/store/realisation.hh"
#include "nix/util/processes.hh"
#include "nix/util/url.hh"
#include "nix/store/store-open.hh"
#include "nix/store/store-api.hh"
#include "nix/store/store-registration.hh"
#include "nix/store/config-parse-impl.hh"
namespace nix {
static LocalOverlayStoreConfigT<config::SettingInfo> localOverlayStoreConfigDescriptions = {
.lowerStoreConfig{
.name = "lower-store",
.description = R"(
[Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly).
Must be a store with a store dir on the file system.
Must be used as OverlayFS lower layer for this store's store dir.
)",
// It's not actually machine-specific, but we don't yet have a
// `to_json` for `StoreConfig`.
.documentDefault = false,
},
.upperLayer{
.name = "upper-layer",
.description = R"(
Directory containing the OverlayFS upper layer for this store's store dir.
)",
},
.checkMount{
.name = "check-mount",
.description = R"(
Check that the overlay filesystem is correctly mounted.
Nix does not manage the overlayfs mount point itself, but the correct
functioning of the overlay store does depend on this mount point being set up
correctly. Rather than just assume this is the case, check that the lowerdir
and upperdir options are what we expect them to be. This check is on by
default, but can be disabled if needed.
)",
},
.remountHook{
.name = "remount-hook",
.description = R"(
Script or other executable to run when overlay filesystem needs remounting.
This is occasionally necessary when deleting a store path that exists in both upper and lower layers.
In such a situation, bypassing OverlayFS and deleting the path in the upper layer directly
is the only way to perform the deletion without creating a "whiteout".
However this causes the OverlayFS kernel data structures to get out-of-sync,
and can lead to 'stale file handle' errors; remounting solves the problem.
The store directory is passed as an argument to the invoked executable.
)",
},
};
#define LOCAL_OVERLAY_STORE_CONFIG_FIELDS(X) \
X(lowerStoreConfig), \
X(upperLayer), \
X(checkMount), \
X(remountHook),
MAKE_PARSE(LocalOverlayStoreConfig, localOverlayStoreConfig, LOCAL_OVERLAY_STORE_CONFIG_FIELDS)
static LocalOverlayStoreConfigT<config::PlainValue> localOverlayStoreConfigDefaults()
{
return {
.lowerStoreConfig = {make_ref<LocalStore::Config>(StoreReference::Params{})},
.upperLayer = {""},
.checkMount = {true},
.remountHook = {""},
};
}
MAKE_APPLY_PARSE(LocalOverlayStoreConfig, localOverlayStoreConfig, LOCAL_OVERLAY_STORE_CONFIG_FIELDS)
config::SettingDescriptionMap LocalOverlayStoreConfig::descriptions()
{
config::SettingDescriptionMap ret;
ret.merge(StoreConfig::descriptions());
ret.merge(LocalFSStoreConfig::descriptions());
ret.merge(LocalStoreConfig::descriptions());
{
constexpr auto & descriptions = localOverlayStoreConfigDescriptions;
auto defaults = localOverlayStoreConfigDefaults();
ret.merge(decltype(ret){
LOCAL_OVERLAY_STORE_CONFIG_FIELDS(DESCRIBE_ROW)
});
}
return ret;
}
LocalOverlayStore::Config::LocalOverlayStoreConfig(
std::string_view scheme,
std::string_view authority,
const StoreReference::Params & params)
: LocalStore::Config(scheme, authority, params)
, LocalOverlayStoreConfigT<config::PlainValue>{localOverlayStoreConfigApplyParse(params)}
{
}
std::string LocalOverlayStoreConfig::doc()
{
return
@ -36,7 +131,7 @@ LocalOverlayStore::LocalOverlayStore(ref<const Config> config)
, LocalFSStore{*config}
, LocalStore{static_cast<ref<const LocalStore::Config>>(config)}
, config{config}
, lowerStore(openStore(percentDecode(config->lowerStoreUri.get())).dynamic_pointer_cast<LocalFSStore>())
, lowerStore(config->lowerStoreConfig.value->openStore().dynamic_pointer_cast<LocalFSStore>())
{
if (config->checkMount.get()) {
std::smatch match;

68
src/libstore/local-store.cc
View file

@ -17,6 +17,7 @@
#include "nix/util/posix-source-accessor.hh"
#include "nix/store/keys.hh"
#include "nix/util/users.hh"
#include "nix/store/config-parse-impl.hh"
#include "nix/store/store-open.hh"
#include "nix/store/store-registration.hh"
@ -61,15 +62,72 @@
namespace nix {
LocalStoreConfig::LocalStoreConfig(
constexpr static const LocalStoreConfigT<config::SettingInfo> localStoreConfigDescriptions = {
.requireSigs = {
.name = "require-sigs",
.description = "Whether store paths copied into this store should have a trusted signature.",
},
.readOnly = {
.name = "read-only",
.description = R"(
Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem.
Normally Nix will attempt to open the store database in read-write mode, even for querying (when write access is not needed), causing it to fail if the database is on a read-only filesystem.
Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set.
> **Warning**
> Do not use this unless the filesystem is read-only.
>
> Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process.
> While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it.
)",
},
};
#define LOCAL_STORE_CONFIG_FIELDS(X) \
X(requireSigs), \
X(readOnly),
MAKE_PARSE(LocalStoreConfig, localStoreConfig, LOCAL_STORE_CONFIG_FIELDS)
static LocalStoreConfigT<config::PlainValue> localStoreConfigDefaults()
{
return {
.requireSigs = {settings.requireSigs},
.readOnly = {false},
};
}
MAKE_APPLY_PARSE(LocalStoreConfig, localStoreConfig, LOCAL_STORE_CONFIG_FIELDS)
config::SettingDescriptionMap LocalStoreConfig::descriptions()
{
config::SettingDescriptionMap ret;
ret.merge(StoreConfig::descriptions());
ret.merge(LocalFSStoreConfig::descriptions());
{
constexpr auto & descriptions = localStoreConfigDescriptions;
auto defaults = localStoreConfigDefaults();
ret.merge(decltype(ret){
LOCAL_STORE_CONFIG_FIELDS(DESCRIBE_ROW)
});
}
return ret;
}
LocalStore::Config::LocalStoreConfig(
std::string_view scheme,
std::string_view authority,
const Params & params)
: StoreConfig(params)
, LocalFSStoreConfig(authority, params)
const StoreReference::Params & params)
: Store::Config(params)
, LocalFSStore::Config(*this, authority, params)
, LocalStoreConfigT<config::PlainValue>{localStoreConfigApplyParse(params)}
{
}
LocalStoreConfig::LocalStoreConfig(const LocalStoreConfig &) = default;
std::string LocalStoreConfig::doc()
{
return
@ -916,7 +974,7 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths)
for (auto & sub : getDefaultSubstituters()) {
if (remaining.empty()) break;
if (sub->storeDir != storeDir) continue;
if (!sub->config.wantMassQuery) continue;
if (!sub->resolvedSubstConfig.wantMassQuery) continue;
auto valid = sub->queryValidPaths(remaining);

16
src/libstore/machines.cc
View file

@ -71,8 +71,8 @@ StoreReference Machine::completeStoreReference() const
auto * generic = std::get_if<StoreReference::Specified>(&storeUri.variant);
if (generic && generic->scheme == "ssh") {
storeUri.params["max-connections"] = "1";
storeUri.params["log-fd"] = "4";
storeUri.params["max-connections"] = 1;
storeUri.params["log-fd"] = 4;
}
if (generic && (generic->scheme == "ssh" || generic->scheme == "ssh-ng")) {
@ -84,14 +84,10 @@ StoreReference Machine::completeStoreReference() const
{
auto & fs = storeUri.params["system-features"];
auto append = [&](auto feats) {
for (auto & f : feats) {
if (fs.size() > 0) fs += ' ';
fs += f;
}
};
append(supportedFeatures);
append(mandatoryFeatures);
if (!fs.is_array()) fs = nlohmann::json::array();
auto features = supportedFeatures;
features.insert(supportedFeatures.begin(), supportedFeatures.end());
for (auto & feat : features) fs += feat;
}
return storeUri;

1
src/libstore/meson.build
View file

@ -265,6 +265,7 @@ sources = files(
'builtins/unpack-channel.cc',
'common-protocol.cc',
'common-ssh-store-config.cc',
'config-parse.cc',
'content-address.cc',
'daemon.cc',
'derivations.cc',

18
src/libstore/mounted-ssh-store.md
View file

@ -1,18 +0,0 @@
R"(
**Store URL format**: `mounted-ssh-ng://[username@]hostname`
Experimental store type that allows full access to a Nix store on a remote machine,
and additionally requires that store be mounted in the local file system.
The mounting of that store is not managed by Nix, and must by managed manually.
It could be accomplished with SSHFS or NFS, for example.
The local file system is used to optimize certain operations.
For example, rather than serializing Nix archives and sending over the Nix channel,
we can directly access the file system data via the mount-point.
The local file system is also used to make certain operations possible that wouldn't otherwise be.
For example, persistent GC roots can be created if they reside on the same file system as the remote store:
the remote side will create the symlinks necessary to avoid race conditions.
)"

52
src/libstore/remote-store.cc
View file

@ -18,14 +18,64 @@
#include "nix/util/callback.hh"
#include "nix/store/filetransfer.hh"
#include "nix/util/signals.hh"
#include "nix/store/config-parse-impl.hh"
#include <nlohmann/json.hpp>
namespace nix {
constexpr static const RemoteStoreConfigT<config::SettingInfo> remoteStoreConfigDescriptions = {
.maxConnections{
.name = "max-connections",
.description = "Maximum number of concurrent connections to the Nix daemon.",
},
.maxConnectionAge{
.name = "max-connection-age",
.description = "Maximum age of a connection before it is closed.",
},
};
#define REMOTE_STORE_CONFIG_FIELDS(X) \
X(maxConnections), \
X(maxConnectionAge),
MAKE_PARSE(RemoteStoreConfig, remoteStoreConfig, REMOTE_STORE_CONFIG_FIELDS)
static RemoteStoreConfigT<config::PlainValue> remoteStoreConfigDefaults()
{
return {
.maxConnections = {1},
.maxConnectionAge = {std::numeric_limits<unsigned int>::max()},
};
}
MAKE_APPLY_PARSE(RemoteStoreConfig, remoteStoreConfig, REMOTE_STORE_CONFIG_FIELDS)
config::SettingDescriptionMap RemoteStoreConfig::descriptions()
{
constexpr auto & descriptions = remoteStoreConfigDescriptions;
auto defaults = remoteStoreConfigDefaults();
return {
REMOTE_STORE_CONFIG_FIELDS(DESCRIBE_ROW)
};
}
RemoteStore::Config::RemoteStoreConfig(const Store::Config & storeConfig, const StoreReference::Params & params)
: RemoteStoreConfigT<config::PlainValue>{remoteStoreConfigApplyParse(params)}
, storeConfig{storeConfig}
{
}
/* TODO: Separate these store types into different files, give them better names */
RemoteStore::RemoteStore(const Config & config)
: Store{config}
: Store{config.storeConfig}
, config{config}
, connections(make_ref<Pool<Connection>>(
std::max(1, config.maxConnections.get()),

154
src/libstore/s3-binary-cache-store.cc
View file

@ -2,8 +2,6 @@
#if NIX_WITH_S3_SUPPORT
#include <assert.h>
#include "nix/store/s3.hh"
#include "nix/store/nar-info.hh"
#include "nix/store/nar-info-disk-cache.hh"
@ -11,6 +9,7 @@
#include "nix/util/compression.hh"
#include "nix/store/filetransfer.hh"
#include "nix/util/signals.hh"
#include "nix/store/config-parse-impl.hh"
#include "nix/store/store-registration.hh"
#include <aws/core/Aws.h>
@ -237,25 +236,133 @@ S3Helper::FileTransferResult S3Helper::getObject(
}
S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig(
std::string_view uriScheme,
std::string_view bucketName,
const Params & params)
: StoreConfig(params)
, BinaryCacheStoreConfig(params)
, bucketName(bucketName)
{
// Don't want to use use AWS SDK in header, so we check the default
// here. TODO do this better after we overhaul the store settings
// system.
assert(std::string{defaultRegion} == std::string{Aws::Region::US_EAST_1});
constexpr static const S3BinaryCacheStoreConfigT<config::SettingInfo> s3BinaryCacheStoreConfigDescriptions = {
.profile{
.name = "profile",
.description = R"(
The name of the AWS configuration profile to use. By default
Nix will use the `default` profile.
)",
},
.region{
.name = "region",
.description = R"(
The region of the S3 bucket. If your bucket is not in
`us–east-1`, you should always explicitly specify the region
parameter.
)",
},
.scheme{
.name = "scheme",
.description = R"(
The scheme used for S3 requests, `https` (default) or `http`. This
option allows you to disable HTTPS for binary caches which don't
support it.
> **Note**
>
> HTTPS should be used if the cache might contain sensitive
> information.
)",
},
.endpoint{
.name = "endpoint",
.description = R"(
The URL of the endpoint of an S3-compatible service such as MinIO.
Do not specify this setting if you're using Amazon S3.
> **Note**
>
> This endpoint must support HTTPS and will use path-based
> addressing instead of virtual host based addressing.
)",
},
.narinfoCompression{
.name = "narinfo-compression",
.description = "Compression method for `.narinfo` files.",
},
.lsCompression{
.name = "ls-compression",
.description = "Compression method for `.ls` files.",
},
.logCompression{
.name = "log-compression",
.description = R"(
Compression method for `log/*` files. It is recommended to
use a compression method supported by most web browsers
(e.g. `brotli`).
)",
},
.multipartUpload{
.name = "multipart-upload",
.description = "Whether to use multi-part uploads.",
},
.bufferSize{
.name = "buffer-size",
.description = "Size (in bytes) of each part in multi-part uploads.",
},
};
#define S3_BINARY_CACHE_STORE_CONFIG_FIELDS(X) \
X(profile), \
X(region), \
X(scheme), \
X(endpoint), \
X(narinfoCompression), \
X(lsCompression), \
X(logCompression), \
X(multipartUpload), \
X(bufferSize),
MAKE_PARSE(S3BinaryCacheStoreConfig, s3BinaryCacheStoreConfig, S3_BINARY_CACHE_STORE_CONFIG_FIELDS)
static S3BinaryCacheStoreConfigT<config::PlainValue> s3BinaryCacheStoreConfigDefaults()
{
return {
.profile = {""},
.region = {Aws::Region::US_EAST_1},
.scheme = {""},
.endpoint = {""},
.narinfoCompression = {""},
.lsCompression = {""},
.logCompression = {""},
.multipartUpload = {false},
.bufferSize = {5 * 1024 * 1024},
};
}
MAKE_APPLY_PARSE(S3BinaryCacheStoreConfig, s3BinaryCacheStoreConfig, S3_BINARY_CACHE_STORE_CONFIG_FIELDS)
config::SettingDescriptionMap S3BinaryCacheStoreConfig::descriptions()
{
config::SettingDescriptionMap ret;
ret.merge(StoreConfig::descriptions());
ret.merge(BinaryCacheStoreConfig::descriptions());
{
constexpr auto & descriptions = s3BinaryCacheStoreConfigDescriptions;
auto defaults = s3BinaryCacheStoreConfigDefaults();
ret.merge(decltype(ret){
S3_BINARY_CACHE_STORE_CONFIG_FIELDS(DESCRIBE_ROW)
});
}
return ret;
}
S3BinaryCacheStore::Config::S3BinaryCacheStoreConfig(
std::string_view scheme,
std::string_view authority,
const StoreReference::Params & params)
: Store::Config{params}
, BinaryCacheStore::Config{*this, params}
, S3BinaryCacheStoreConfigT<config::PlainValue>{s3BinaryCacheStoreConfigApplyParse(params)}
, bucketName{authority}
{
if (bucketName.empty())
throw UsageError("`%s` store requires a bucket name in its Store URI", uriScheme);
throw UsageError("`%s` store requires a bucket name in its Store URI", scheme);
}
S3BinaryCacheStore::S3BinaryCacheStore(ref<Config> config)
S3BinaryCacheStore::S3BinaryCacheStore(ref<const Config> config)
: BinaryCacheStore(*config)
, config{config}
{ }
@ -274,7 +381,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore
S3Helper s3Helper;
S3BinaryCacheStoreImpl(ref<Config> config)
S3BinaryCacheStoreImpl(ref<const Config> config)
: Store{*config}
, BinaryCacheStore{*config}
, S3BinaryCacheStore{config}
@ -293,12 +400,14 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore
void init() override
{
if (auto cacheInfo = diskCache->upToDateCacheExists(getUri())) {
config->wantMassQuery.setDefault(cacheInfo->wantMassQuery);
config->priority.setDefault(cacheInfo->priority);
resolvedSubstConfig.wantMassQuery.value =
config->storeConfig.wantMassQuery.optValue.value_or(cacheInfo->wantMassQuery);
resolvedSubstConfig.priority.value =
config->storeConfig.priority.optValue.value_or(cacheInfo->priority);
} else {
BinaryCacheStore::init();
diskCache->createCache(
getUri(), config->storeDir, config->wantMassQuery, config->priority);
getUri(), config->storeDir, resolvedSubstConfig.wantMassQuery, resolvedSubstConfig.priority);
}
}
@ -585,10 +694,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore
ref<Store> S3BinaryCacheStoreImpl::Config::openStore() const
{
return make_ref<S3BinaryCacheStoreImpl>(ref{
// FIXME we shouldn't actually need a mutable config
std::const_pointer_cast<S3BinaryCacheStore::Config>(shared_from_this())
});
return make_ref<S3BinaryCacheStoreImpl>(ref{shared_from_this()});
}
static RegisterStoreImplementation<S3BinaryCacheStoreImpl::Config> regS3BinaryCacheStore;

138
src/libstore/ssh-store.cc
View file

@ -1,3 +1,4 @@
#include "nix/util/json-utils.hh"
#include "nix/store/ssh-store.hh"
#include "nix/store/local-fs-store.hh"
#include "nix/store/remote-store-connection.hh"
@ -7,17 +8,101 @@
#include "nix/store/worker-protocol-impl.hh"
#include "nix/util/pool.hh"
#include "nix/store/ssh.hh"
#include "nix/store/config-parse-impl.hh"
#include "nix/store/store-registration.hh"
namespace nix {
constexpr static const SSHStoreConfigT<config::SettingInfo> sshStoreConfigDescriptions = {
.remoteProgram{
.name = "remote-program",
.description = "Path to the `nix-daemon` executable on the remote machine.",
},
};
#define SSH_STORE_CONFIG_FIELDS(X) \
X(remoteProgram)
MAKE_PARSE(SSHStoreConfig, sshStoreConfig, SSH_STORE_CONFIG_FIELDS)
static SSHStoreConfigT<config::PlainValue> sshStoreConfigDefaults()
{
return {
.remoteProgram = {{"nix-daemon"}},
};
}
MAKE_APPLY_PARSE(SSHStoreConfig, sshStoreConfig, SSH_STORE_CONFIG_FIELDS)
config::SettingDescriptionMap SSHStoreConfig::descriptions()
{
config::SettingDescriptionMap ret;
ret.merge(StoreConfig::descriptions());
ret.merge(CommonSSHStoreConfig::descriptions());
ret.merge(RemoteStoreConfig::descriptions());
{
constexpr auto & descriptions = sshStoreConfigDescriptions;
auto defaults = sshStoreConfigDefaults();
ret.merge(decltype(ret){
SSH_STORE_CONFIG_FIELDS(DESCRIBE_ROW)
});
}
ret.insert_or_assign(
"mounted",
config::SettingDescription{
.description = stripIndentation(R"(
If this nested settings object is defined (`{..}` not `null`), additionally requires that store be mounted in the local file system.
The mounting of that store is not managed by Nix, and must by managed manually.
It could be accomplished with SSHFS or NFS, for example.
The local file system is used to optimize certain operations.
For example, rather than serializing Nix archives and sending over the Nix channel,
we can directly access the file system data via the mount-point.
The local file system is also used to make certain operations possible that wouldn't otherwise be.
For example, persistent GC roots can be created if they reside on the same file system as the remote store:
the remote side will create the symlinks necessary to avoid race conditions.
)"),
.experimentalFeature = Xp::MountedSSHStore,
.info = config::SettingDescription::Sub{
.nullable = true,
.map = LocalFSStoreConfig::descriptions()
},
});
return ret;
}
static std::optional<LocalFSStore::Config> getMounted(
const Store::Config & storeConfig,
const StoreReference::Params & params,
const ExperimentalFeatureSettings & xpSettings)
{
auto mountedParamsOpt = optionalValueAt(params, "mounted");
if (!mountedParamsOpt) return {};
auto * mountedParamsP = getNullable(*mountedParamsOpt);
xpSettings.require(Xp::MountedSSHStore);
if (!mountedParamsP) return {};
auto & mountedParams = getObject(*mountedParamsP);
return {{storeConfig, mountedParams}};
}
SSHStoreConfig::SSHStoreConfig(
std::string_view scheme,
std::string_view authority,
const Params & params)
const StoreReference::Params & params, const ExperimentalFeatureSettings & xpSettings)
: Store::Config{params}
, RemoteStore::Config{params}
, RemoteStore::Config{*this, params}
, CommonSSHStoreConfig{scheme, authority, params}
, SSHStoreConfigT<config::PlainValue>{sshStoreConfigApplyParse(params)}
, mounted{getMounted(*this, params, xpSettings)}
{
}
@ -87,32 +172,6 @@ protected:
};
MountedSSHStoreConfig::MountedSSHStoreConfig(StringMap params)
: StoreConfig(params)
, RemoteStoreConfig(params)
, CommonSSHStoreConfig(params)
, SSHStoreConfig(params)
, LocalFSStoreConfig(params)
{
}
MountedSSHStoreConfig::MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params)
: StoreConfig(params)
, RemoteStoreConfig(params)
, CommonSSHStoreConfig(scheme, host, params)
, SSHStoreConfig(scheme, host, params)
, LocalFSStoreConfig(params)
{
}
std::string MountedSSHStoreConfig::doc()
{
return
#include "mounted-ssh-store.md"
;
}
/**
* The mounted ssh store assumes that filesystems on the remote host are
* shared with the local host. This means that the remote nix store is
@ -129,13 +188,16 @@ std::string MountedSSHStoreConfig::doc()
*/
struct MountedSSHStore : virtual SSHStore, virtual LocalFSStore
{
using Config = MountedSSHStoreConfig;
using Config = SSHStore::Config;
MountedSSHStore(ref<const Config> config)
const LocalFSStore::Config & mountedConfig;
MountedSSHStore(ref<const Config> config, const LocalFSStore::Config & mountedConfig)
: Store{*config}
, RemoteStore{*config}
, SSHStore{config}
, LocalFSStore{*config}
, LocalFSStore{mountedConfig}
, mountedConfig{mountedConfig}
{
extraRemoteProgramArgs = {
"--process-ops",
@ -184,14 +246,13 @@ struct MountedSSHStore : virtual SSHStore, virtual LocalFSStore
};
ref<Store> SSHStore::Config::openStore() const {
return make_ref<SSHStore>(ref{shared_from_this()});
}
ref<Store> MountedSSHStore::Config::openStore() const {
return make_ref<MountedSSHStore>(ref{
std::dynamic_pointer_cast<const MountedSSHStore::Config>(shared_from_this())
});
ref config {shared_from_this()};
if (config->mounted)
return make_ref<MountedSSHStore>(config, *config->mounted);
else
return make_ref<SSHStore>(config);
}
@ -213,6 +274,5 @@ ref<RemoteStore::Connection> SSHStore::openConnection()
}
static RegisterStoreImplementation<SSHStore::Config> regSSHStore;
static RegisterStoreImplementation<MountedSSHStore::Config> regMountedSSHStore;
}

1
src/libstore/ssh-store.md
View file

@ -4,5 +4,4 @@ R"(
Experimental store type that allows full access to a Nix store on a
remote machine.
)"

110
src/libstore/store-api.cc
View file

@ -19,6 +19,7 @@
#include "nix/store/worker-protocol.hh"
#include "nix/util/signals.hh"
#include "nix/util/users.hh"
#include "nix/store/config-parse-impl.hh"
#include <filesystem>
#include <nlohmann/json.hpp>
@ -29,7 +30,6 @@ using json = nlohmann::json;
namespace nix {
bool MixStoreDirMethods::isInStore(PathView path) const
{
return isInDir(path, storeDir);
@ -189,6 +189,114 @@ std::pair<StorePath, Hash> MixStoreDirMethods::computeStorePath(
}
constexpr static const StoreConfigT<config::SettingInfo> storeConfigDescriptions = {
.pathInfoCacheSize{
.name = "path-info-cache-size",
.description = "Size of the in-memory store path metadata cache.",
},
.isTrusted{
.name = "trusted",
.description = R"(
Whether paths from this store can be used as substitutes
even if they are not signed by a key listed in the
[`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys)
setting.
)",
},
.systemFeatures{
.name = "system-features",
.description = R"(
Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations.
Example: `"kvm"`
)",
// The default value is CPU- and OS-specific, and thus
// unsuitable to be rendered in the documentation.
.documentDefault = false,
},
};
constexpr static const SubstituterConfigT<config::SettingInfo> substituterConfigDescriptions = {
.priority{
.name = "priority",
.description = R"(
Priority of this store when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters).
A lower value means a higher priority.
)",
},
.wantMassQuery{
.name = "want-mass-query",
.description = R"(
Whether this store can be queried efficiently for path validity when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters).
)",
},
};
#define STORE_CONFIG_FIELDS(X) \
X(pathInfoCacheSize), \
X(isTrusted), \
X(systemFeatures),
#define SUBSTITUTER_CONFIG_FIELDS(X) \
X(priority), \
X(wantMassQuery),
MAKE_PARSE(StoreConfig, storeConfig, STORE_CONFIG_FIELDS)
MAKE_PARSE(SubstituterConfig, substituterConfig, SUBSTITUTER_CONFIG_FIELDS)
static StoreConfigT<config::PlainValue> storeConfigDefaults()
{
return {
.pathInfoCacheSize = {65536},
.isTrusted = {false},
.systemFeatures = {StoreConfig::getDefaultSystemFeatures()},
};
};
SubstituterConfigT<config::PlainValue> substituterConfigDefaults()
{
return {
.priority = {0},
.wantMassQuery = {false},
};
};
MAKE_APPLY_PARSE(StoreConfig, storeConfig, STORE_CONFIG_FIELDS)
Store::Config::StoreConfig(const StoreReference::Params & params)
: StoreDirConfig{params}
, StoreConfigT<config::PlainValue>{storeConfigApplyParse(params)}
, SubstituterConfigT<config::OptValue>{substituterConfigParse(params)}
{
}
config::SettingDescriptionMap StoreConfig::descriptions()
{
auto ret = StoreDirConfig::descriptions();
{
constexpr auto & descriptions = storeConfigDescriptions;
auto defaults = storeConfigDefaults();
ret.merge(config::SettingDescriptionMap {
STORE_CONFIG_FIELDS(DESCRIBE_ROW)
});
}
{
constexpr auto & descriptions = substituterConfigDescriptions;
auto defaults = substituterConfigDefaults();
ret.merge(config::SettingDescriptionMap {
SUBSTITUTER_CONFIG_FIELDS(DESCRIBE_ROW)
});
};
return ret;
}
StorePath Store::addToStore(
std::string_view name,
const SourcePath & path,

38
src/libstore/store-dir-config.cc
View file

@ -1,13 +1,45 @@
#include "nix/store/store-dir-config.hh"
#include "nix/store/config-parse-impl.hh"
#include "nix/util/util.hh"
#include "nix/store/globals.hh"
namespace nix {
StoreDirConfig::StoreDirConfig(const Params & params)
: StoreDirConfigBase(params)
, MixStoreDirMethods{storeDir_}
constexpr static const StoreDirConfigT<config::SettingInfo> storeDirConfigDescriptions = {
._storeDir{
.name = "store",
.description = R"(
Logical location of the Nix store, usually
`/nix/store`. Note that you can only copy store paths
between stores if they have the same `store` setting.
)",
},
};
#define STORE_DIR_CONFIG_FIELDS(X) X(_storeDir),
MAKE_PARSE(StoreDirConfig, storeDirConfig, STORE_DIR_CONFIG_FIELDS)
static StoreDirConfigT<config::PlainValue> storeDirConfigDefaults()
{
return {
._storeDir = {settings.nixStore},
};
}
MAKE_APPLY_PARSE(StoreDirConfig, storeDirConfig, STORE_DIR_CONFIG_FIELDS)
StoreDirConfig::StoreDirConfig(const StoreReference::Params & params)
: StoreDirConfigT<config::PlainValue>{storeDirConfigApplyParse(params)}
, MixStoreDirMethods{_storeDir.value}
{
}
config::SettingDescriptionMap StoreDirConfig::descriptions()
{
constexpr auto & descriptions = storeDirConfigDescriptions;
auto defaults = storeDirConfigDefaults();
return {STORE_DIR_CONFIG_FIELDS(DESCRIBE_ROW)};
}
}

151
src/libstore/store-reference.cc
View file

@ -1,13 +1,18 @@
#include <regex>
#include <nlohmann/json.hpp>
#include "nix/util/error.hh"
#include "nix/util/url.hh"
#include "nix/store/store-reference.hh"
#include "nix/util/file-system.hh"
#include "nix/util/util.hh"
#include "nix/util/json-utils.hh"
namespace nix {
bool StoreReference::operator==(const StoreReference & rhs) const = default;
static bool isNonUriPath(const std::string & spec)
{
return
@ -33,20 +38,96 @@ std::string StoreReference::render() const
},
variant);
StringMap params2;
for (auto & [k, v] : params) {
auto * p = v.get_ptr<const nlohmann::json::string_t *>();
// if it is a JSON string, just use that
// FIXME: Ensure the literal string isn't itself valid JSON. If
// it is, we still need to dump to escape it.
params2.insert_or_assign(k, p ? *p : v.dump());
}
if (!params.empty()) {
res += "?";
res += encodeQuery(params);
res += encodeQuery(params2);
}
return res;
}
static StoreReference::Params decodeParamsJson(StringMap paramsRaw)
{
StoreReference::Params params;
for (auto && [k, v] : std::move(paramsRaw)) {
nlohmann::json j;
/* We have to parse the URL un an "untyped" way before we do a
"typed" conversion to specific store-configuration types. As
such, the best we can do for back-compat is just white-list
specific query parameter names.
These are all the boolean store parameters in use at the time
of the introduction of JSON store configuration, as evidenced
by `git grep 'F<bool>'`. So these will continue working with
"yes"/"no"/"1"/"0", whereas any new ones will require
"true"/"false".
*/
bool preJsonBool =
std::set<std::string_view>{
"check-mount",
"compress",
"trusted",
"multipart-upload",
"parallel-compression",
"read-only",
"require-sigs",
"want-mass-query",
"index-debug-info",
"write-nar-listing",
}
.contains(std::string_view{k});
auto warnPreJson = [&] {
warn(
"in query param '%s', using '%s' to mean a boolean is deprecated, please use valid JSON 'true' or 'false'",
k,
v);
};
if (preJsonBool && (v == "yes" || v == "1")) {
j = true;
warnPreJson();
} else if (preJsonBool && (v == "no" || v == "0")) {
j = true;
warnPreJson();
} else {
try {
j = nlohmann::json::parse(v);
} catch (nlohmann::json::exception &) {
// if its not valid JSON...
if (k == "remote-program" || k == "system-features") {
// Back compat hack! Split and take that array
j = tokenizeString<std::vector<std::string>>(v);
} else {
// ...keep the literal string.
j = std::move(v);
}
}
}
params.insert_or_assign(std::move(k), std::move(j));
}
return params;
}
StoreReference StoreReference::parse(const std::string & uri, const StoreReference::Params & extraParams)
{
auto params = extraParams;
try {
auto parsedUri = parseURL(uri);
params.insert(parsedUri.query.begin(), parsedUri.query.end());
{
auto params2 = decodeParamsJson(std::move(parsedUri.query));
params.insert(params2.begin(), params2.end());
}
auto baseURI = parsedUri.authority.value_or("") + parsedUri.path;
@ -104,13 +185,75 @@ StoreReference StoreReference::parse(const std::string & uri, const StoreReferen
std::pair<std::string, StoreReference::Params> splitUriAndParams(const std::string & uri_)
{
auto uri(uri_);
StoreReference::Params params;
StringMap params;
auto q = uri.find('?');
if (q != std::string::npos) {
params = decodeQuery(uri.substr(q + 1));
uri = uri_.substr(0, q);
}
return {uri, params};
return {uri, decodeParamsJson(std::move(params))};
}
}
namespace nlohmann {
StoreReference adl_serializer<StoreReference>::from_json(const json & json)
{
StoreReference ref;
switch (json.type()) {
case json::value_t::string: {
ref = StoreReference::parse(json.get_ref<const std::string &>());
break;
}
case json::value_t::object: {
auto & obj = getObject(json);
auto scheme = getString(valueAt(obj, "scheme"));
auto variant = scheme == "auto" ? (StoreReference::Variant{StoreReference::Auto{}})
: (StoreReference::Variant{StoreReference::Specified{
.scheme = scheme,
.authority = getString(valueAt(obj, "authority")),
}});
auto params = obj;
params.erase("scheme");
params.erase("authority");
ref = StoreReference{
.variant = std::move(variant),
.params = std::move(params),
};
break;
}
case json::value_t::null:
case json::value_t::number_unsigned:
case json::value_t::number_integer:
case json::value_t::number_float:
case json::value_t::boolean:
case json::value_t::array:
case json::value_t::binary:
case json::value_t::discarded:
default:
throw UsageError(
"Invalid JSON for Store configuration: is type '%s' but must be string or object", json.type_name());
};
return ref;
}
void adl_serializer<StoreReference>::to_json(json & obj, StoreReference s)
{
obj = s.params;
std::visit(
overloaded{
[&](const StoreReference::Auto &) { obj.emplace("scheme", "auto"); },
[&](const StoreReference::Specified & g) {
obj.emplace("scheme", g.scheme);
obj.emplace("authority", g.authority);
},
},
s.variant);
}
}

35
src/libstore/store-registration.cc
View file

@ -2,19 +2,18 @@
#include "nix/store/store-open.hh"
#include "nix/store/local-store.hh"
#include "nix/store/uds-remote-store.hh"
#include "nix/util/json-utils.hh"
namespace nix {
ref<Store> openStore(const std::string & uri, const Store::Config::Params & extraParams)
ref<Store> openStore(const std::string & uri, const StoreReference::Params & extraParams)
{
return openStore(StoreReference::parse(uri, extraParams));
}
ref<Store> openStore(StoreReference && storeURI)
{
auto store = resolveStoreConfig(std::move(storeURI))->openStore();
store->init();
return store;
return resolveStoreConfig(std::move(storeURI))->openStore();
}
ref<StoreConfig> resolveStoreConfig(StoreReference && storeURI)
@ -24,7 +23,7 @@ ref<StoreConfig> resolveStoreConfig(StoreReference && storeURI)
auto storeConfig = std::visit(
overloaded{
[&](const StoreReference::Auto &) -> ref<StoreConfig> {
auto stateDir = getOr(params, "state", settings.nixStateDir);
auto stateDir = getString(getOr(params, "state", settings.nixStateDir));
if (access(stateDir.c_str(), R_OK | W_OK) == 0)
return make_ref<LocalStore::Config>(params);
else if (pathExists(settings.nixDaemonSocketFile))
@ -53,7 +52,7 @@ ref<StoreConfig> resolveStoreConfig(StoreReference && storeURI)
return make_ref<LocalStore::Config>(params);
},
[&](const StoreReference::Specified & g) {
for (const auto & [storeName, implem] : Implementations::registered())
for (auto & [name, implem] : Implementations::registered())
if (implem.uriSchemes.count(g.scheme))
return implem.parseConfig(g.scheme, g.authority, params);
@ -63,7 +62,6 @@ ref<StoreConfig> resolveStoreConfig(StoreReference && storeURI)
storeURI.variant);
experimentalFeatureSettings.require(storeConfig->experimentalFeature());
storeConfig->warnUnknownSettings();
return storeConfig;
}
@ -85,10 +83,12 @@ std::list<ref<Store>> getDefaultSubstituters()
}
};
for (const auto & uri : settings.substituters.get())
for (auto & uri : settings.substituters.get())
addStore(uri);
stores.sort([](ref<Store> & a, ref<Store> & b) { return a->config.priority < b->config.priority; });
stores.sort([](ref<Store> & a, ref<Store> & b) {
return a->resolvedSubstConfig.priority < b->resolvedSubstConfig.priority;
});
return stores;
}());
@ -103,3 +103,20 @@ Implementations::Map & Implementations::registered()
}
}
namespace nlohmann {
using namespace nix::config;
ref<const StoreConfig> adl_serializer<ref<const StoreConfig>>::from_json(const json & json)
{
return resolveStoreConfig(adl_serializer<StoreReference>::from_json(json));
}
void adl_serializer<ref<const StoreConfig>>::to_json(json & obj, ref<const StoreConfig> s)
{
// TODO, for tests maybe
assert(false);
}
}

26
src/libstore/uds-remote-store.cc
View file

@ -18,16 +18,26 @@
namespace nix {
config::SettingDescriptionMap UDSRemoteStoreConfig::descriptions()
{
config::SettingDescriptionMap ret;
ret.merge(StoreConfig::descriptions());
ret.merge(LocalFSStoreConfig::descriptions());
ret.merge(RemoteStoreConfig::descriptions());
return ret;
}
UDSRemoteStoreConfig::UDSRemoteStoreConfig(
std::string_view scheme,
std::string_view authority,
const StoreReference::Params & params)
: Store::Config{params}
, LocalFSStore::Config{params}
, RemoteStore::Config{params}
, LocalFSStore::Config{*this, params}
, RemoteStore::Config{*this, params}
, path{authority.empty() ? settings.nixDaemonSocketFile : authority}
{
if (uriSchemes().count(scheme) == 0) {
if (uriSchemes().count(std::string{scheme}) == 0) {
throw UsageError("Scheme must be 'unix'");
}
}
@ -41,16 +51,6 @@ std::string UDSRemoteStoreConfig::doc()
}
// A bit gross that we now pass empty string but this is knowing that
// empty string will later default to the same nixDaemonSocketFile. Why
// don't we just wire it all through? I believe there are cases where it
// will live reload so we want to continue to account for that.
UDSRemoteStoreConfig::UDSRemoteStoreConfig(const Params & params)
: UDSRemoteStoreConfig(*uriSchemes().begin(), "", params)
{
}
UDSRemoteStore::UDSRemoteStore(ref<const Config> config)
: Store{*config}
, LocalFSStore{*config}

6
src/libstore/unix/build/derivation-builder.cc
View file

@ -1597,9 +1597,9 @@ void DerivationBuilderImpl::startDaemon()
auto store = makeRestrictedStore(
[&]{
auto config = make_ref<LocalStore::Config>(*getLocalStore().config);
config->pathInfoCacheSize = 0;
config->stateDir = "/no-such-path";
config->logDir = "/no-such-path";
config->pathInfoCacheSize.value = 0;
config->stateDir.value = "/no-such-path";
config->logDir.value = "/no-such-path";
return config;
}(),
ref<LocalStore>(std::dynamic_pointer_cast<LocalStore>(this->store.shared_from_this())),

51
src/libutil/include/nix/util/config-abstract.hh Normal file
View file

@ -0,0 +1,51 @@
#pragma once
///@type
#include <optional>
namespace nix::config {
template<typename T>
struct PlainValue
{
T value;
operator const T &() const &
{
return value;
}
operator T &() &
{
return value;
}
const T & get() const
{
return value;
}
bool operator==(auto && v2) const
{
return value == v2;
}
bool operator!=(auto && v2) const
{
return value != v2;
}
};
template<typename T>
auto && operator<<(auto && str, const PlainValue<T> & opt)
{
return str << opt.get();
}
template<typename T>
struct OptValue
{
std::optional<T> optValue;
};
}

3
src/libutil/include/nix/util/configuration.hh
View file

@ -180,13 +180,12 @@ public:
const std::string name;
const std::string description;
const StringSet aliases;
std::optional<ExperimentalFeature> experimentalFeature;
int created = 123;
bool overridden = false;
std::optional<ExperimentalFeature> experimentalFeature;
protected:
AbstractSetting(

1
src/libutil/include/nix/util/meson.build
View file

@ -16,6 +16,7 @@ headers = files(
'comparator.hh',
'compression.hh',
'compute-levels.hh',
'config-abstract.hh',
'config-global.hh',
'config-impl.hh',
'configuration.hh',

12
src/libutil/include/nix/util/util.hh
View file

@ -223,16 +223,16 @@ std::pair<std::string_view, std::string_view> getLine(std::string_view s);
/**
* Get a value for the specified key from an associate container.
*/
template <class T>
const typename T::mapped_type * get(const T & map, const typename T::key_type & key)
template <class T, typename K = const T::key_type &>
const typename T::mapped_type * get(const T & map, K key)
{
auto i = map.find(key);
if (i == map.end()) return nullptr;
return &i->second;
}
template <class T>
typename T::mapped_type * get(T & map, const typename T::key_type & key)
template <class T, typename K = const T::key_type &>
typename T::mapped_type * get(T & map, K key)
{
auto i = map.find(key);
if (i == map.end()) return nullptr;
@ -242,9 +242,9 @@ typename T::mapped_type * get(T & map, const typename T::key_type & key)
/**
* Get a value for the specified key from an associate container, or a default value if the key isn't present.
*/
template <class T>
template <class T, typename K = const T::key_type &>
const typename T::mapped_type & getOr(T & map,
const typename T::key_type & key,
K key,
const typename T::mapped_type & defaultValue)
{
auto i = map.find(key);

2
src/nix/main.cc
View file

@ -198,7 +198,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs
auto & j = stores[storeName];
j["doc"] = implem.doc;
j["uri-schemes"] = implem.uriSchemes;
j["settings"] = implem.getConfig()->toJSON();
j["settings"] = implem.configDescriptions();
j["experimentalFeature"] = implem.experimentalFeature;
}
res["stores"] = std::move(stores);

4
src/nix/unix/daemon.cc
View file

@ -245,9 +245,9 @@ static PeerInfo getPeerInfo(int remote)
*/
static ref<Store> openUncachedStore()
{
Store::Config::Params params; // FIXME: get params from somewhere
StoreReference::Params params; // FIXME: get params from somewhere
// Disable caching since the client already does that.
params["path-info-cache-size"] = "0";
params["path-info-cache-size"] = 0;
return openStore(settings.storeUri, params);
}

12
tests/functional/build-remote-with-mounted-ssh-ng.sh
View file

@ -5,17 +5,25 @@ source common.sh
requireSandboxSupport
[[ $busybox =~ busybox ]] || skipTest "no busybox"
# An example of a command that uses the store only for its settings, to
# make sure we catch needing the XP feature early.
touch "$TEST_ROOT/foo"
expectStderr 1 nix --store 'ssh-ng://localhost?mounted=%7B%7D' store add "$TEST_ROOT/foo" --dry-run | grepQuiet "experimental Nix feature 'mounted-ssh-store' is disabled"
enableFeatures mounted-ssh-store
# N.B. encoded query param is `mounted={}`. In the future, we can just
# do `--store` with JSON, and then the nested structure will actually
# bring benefits.
nix build -Lvf simple.nix \
--arg busybox "$busybox" \
--out-link "$TEST_ROOT/result-from-remote" \
--store mounted-ssh-ng://localhost
--store 'ssh-ng://localhost?mounted=%7B%7D'
nix build -Lvf simple.nix \
--arg busybox "$busybox" \
--out-link "$TEST_ROOT/result-from-remote-new-cli" \
--store 'mounted-ssh-ng://localhost?remote-program=nix daemon'
--store 'ssh-ng://localhost?mounted=%7B%7D&remote-program=nix daemon'
# This verifies that the out link was actually created and valid. The ability
# to create out links (permanent gc roots) is the distinguishing feature of