wroclaw-git-backups/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-07-07 06:01:48 +02:00
Code Activity

Merge remote-tracking branch 'origin/master' into large-path-warning

Browse source
This commit is contained in:
Eelco Dolstra 2024-06-03 15:32:27 +02:00
commit 7f5b57d18f
435 changed files with 6086 additions and 2767 deletions
Download patch file Download diff file Expand all files Collapse all files

22
src/build-remote/build-remote.cc
View file

@ -37,7 +37,7 @@ static std::string currentLoad;
static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot)
{
return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true);
return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri.render()), slot), true);
}
static bool allSupportedLocally(Store & store, const std::set<std::string>& requiredFeatures) {
@ -99,7 +99,7 @@ static int main_build_remote(int argc, char * * argv)
}
std::optional<StorePath> drvPath;
std::string storeUri;
StoreReference storeUri;
while (true) {
@ -135,7 +135,7 @@ static int main_build_remote(int argc, char * * argv)
Machine * bestMachine = nullptr;
uint64_t bestLoad = 0;
for (auto & m : machines) {
debug("considering building on remote machine '%s'", m.storeUri);
debug("considering building on remote machine '%s'", m.storeUri.render());
if (m.enabled &&
m.systemSupported(neededSystem) &&
@ -233,7 +233,7 @@ static int main_build_remote(int argc, char * * argv)
try {
Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri));
Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri.render()));
sshStore = bestMachine->openStore();
sshStore->connect();
@ -242,7 +242,7 @@ static int main_build_remote(int argc, char * * argv)
} catch (std::exception & e) {
auto msg = chomp(drainFD(5, false));
printError("cannot build on '%s': %s%s",
bestMachine->storeUri, e.what(),
bestMachine->storeUri.render(), e.what(),
msg.empty() ? "" : ": " + msg);
bestMachine->enabled = false;
continue;
@ -257,15 +257,15 @@ connected:
assert(sshStore);
std::cerr << "# accept\n" << storeUri << "\n";
std::cerr << "# accept\n" << storeUri.render() << "\n";
auto inputs = readStrings<PathSet>(source);
auto wantedOutputs = readStrings<StringSet>(source);
AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true);
AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri.render()) + ".upload-lock", true);
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri));
Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri.render()));
auto old = signal(SIGALRM, handleAlarm);
alarm(15 * 60);
@ -278,7 +278,7 @@ connected:
auto substitute = settings.buildersUseSubstitutes ? Substitute : NoSubstitute;
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri));
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri.render()));
copyPaths(*store, *sshStore, store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute);
}
@ -316,7 +316,7 @@ connected:
optResult = sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv);
auto & result = *optResult;
if (!result.success())
throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg);
throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri.render(), result.errorMsg);
} else {
copyClosure(*store, *sshStore, StorePathSet {*drvPath}, NoRepair, NoCheckSigs, substitute);
auto res = sshStore->buildPathsWithResults({
@ -359,7 +359,7 @@ connected:
}
if (!missingPaths.empty()) {
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri));
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri.render()));
if (auto localStore = store.dynamic_pointer_cast<LocalStore>())
for (auto & path : missingPaths)
localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */

6
src/libcmd/common-eval-args.cc
View file

@ -20,7 +20,7 @@ MixEvalArgs::MixEvalArgs()
.description = "Pass the value *expr* as the argument *name* to Nix functions.",
.category = category,
.labels = {"name", "expr"},
.handler = {[&](std::string name, std::string expr) { autoArgs.insert_or_assign(name, AutoArg{AutoArgExpr(expr)}); }}
.handler = {[&](std::string name, std::string expr) { autoArgs.insert_or_assign(name, AutoArg{AutoArgExpr{expr}}); }}
});
addFlag({
@ -28,7 +28,7 @@ MixEvalArgs::MixEvalArgs()
.description = "Pass the string *string* as the argument *name* to Nix functions.",
.category = category,
.labels = {"name", "string"},
.handler = {[&](std::string name, std::string s) { autoArgs.insert_or_assign(name, AutoArg{AutoArgString(s)}); }},
.handler = {[&](std::string name, std::string s) { autoArgs.insert_or_assign(name, AutoArg{AutoArgString{s}}); }},
});
addFlag({
@ -36,7 +36,7 @@ MixEvalArgs::MixEvalArgs()
.description = "Pass the contents of file *path* as the argument *name* to Nix functions.",
.category = category,
.labels = {"name", "path"},
.handler = {[&](std::string name, std::string path) { autoArgs.insert_or_assign(name, AutoArg{AutoArgFile(path)}); }},
.handler = {[&](std::string name, std::string path) { autoArgs.insert_or_assign(name, AutoArg{AutoArgFile{path}}); }},
.completer = completePath
});

2
src/libcmd/installable-attr-path.cc
View file

@ -75,6 +75,8 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths()
std::set<std::string> outputsToInstall;
for (auto & output : packageInfo.queryOutputs(false, true))
outputsToInstall.insert(output.first);
if (outputsToInstall.empty())
outputsToInstall.insert("out");
return OutputsSpec::Names { std::move(outputsToInstall) };
},
[&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec {

9
src/libcmd/installable-flake.cc
View file

@ -106,9 +106,14 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
fmt("while evaluating the flake output attribute '%s'", attrPath)))
{
return { *derivedPathWithInfo };
} else {
throw Error(
"expected flake output attribute '%s' to be a derivation or path but found %s: %s",
attrPath,
showType(v),
ValuePrinter(*this->state, v, errorPrintOptions)
);
}
else
throw Error("flake output attribute '%s' is not a derivation or path", attrPath);
}
auto drvPath = attr->forceDerivation();

38
src/libcmd/installables.cc
View file

@ -601,6 +601,37 @@ std::vector<BuiltPathWithResult> Installable::build(
return res;
}
static void throwBuildErrors(
std::vector<KeyedBuildResult> & buildResults,
const Store & store)
{
std::vector<KeyedBuildResult> failed;
for (auto & buildResult : buildResults) {
if (!buildResult.success()) {
failed.push_back(buildResult);
}
}
auto failedResult = failed.begin();
if (failedResult != failed.end()) {
if (failed.size() == 1) {
failedResult->rethrow();
} else {
StringSet failedPaths;
for (; failedResult != failed.end(); failedResult++) {
if (!failedResult->errorMsg.empty()) {
logError(ErrorInfo{
.level = lvlError,
.msg = failedResult->errorMsg,
});
}
failedPaths.insert(failedResult->path.to_string(store));
}
throw Error("build of %s failed", concatStringsSep(", ", quoteStrings(failedPaths)));
}
}
}
std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> Installable::build2(
ref<Store> evalStore,
ref<Store> store,
@ -662,10 +693,9 @@ std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> Installable::build
if (settings.printMissing)
printMissing(store, pathsToBuild, lvlInfo);
for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) {
if (!buildResult.success())
buildResult.rethrow();
auto buildResults = store->buildPathsWithResults(pathsToBuild, bMode, evalStore);
throwBuildErrors(buildResults, *store);
for (auto & buildResult : buildResults) {
for (auto & aux : backmap[buildResult.path]) {
std::visit(overloaded {
[&](const DerivedPath::Built & bfd) {

28
src/libcmd/misc-store-flags.cc
View file

@ -81,9 +81,15 @@ Args::Flag fileIngestionMethod(FileIngestionMethod * method)
How to compute the hash of the input.
One of:
- `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function.
- `nar` (the default):
Serialises the input as a
[Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive)
and passes that to the hash function.
- `flat`: Assumes that the input is a single file and directly passes it to the hash function;
- `flat`:
Assumes that the input is a single file and
[directly passes](@docroot@/store/file-system-object/content-address.md#serial-flat)
it to the hash function.
)",
.labels = {"file-ingestion-method"},
.handler = {[method](std::string s) {
@ -101,15 +107,23 @@ Args::Flag contentAddressMethod(ContentAddressMethod * method)
How to compute the content-address of the store object.
One of:
- `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function.
- [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive)
(the default):
Serialises the input as a
[Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive)
and passes that to the hash function.
- `flat`: Assumes that the input is a single file and directly passes it to the hash function;
- [`flat`](@docroot@/store/store-object/content-address.md#method-flat):
Assumes that the input is a single file and
[directly passes](@docroot@/store/file-system-object/content-address.md#serial-flat)
it to the hash function.
- `text`: Like `flat`, but used for
[derivations](@docroot@/glossary.md#store-derivation) serialized in store object and
- [`text`](@docroot@/store/store-object/content-address.md#method-text):
Like `flat`, but used for
[derivations](@docroot@/glossary.md#store-derivation) serialized in store object and
[`builtins.toFile`](@docroot@/language/builtins.html#builtins-toFile).
For advanced use-cases only;
for regular usage prefer `nar` and `flat.
for regular usage prefer `nar` and `flat`.
)",
.labels = {"content-address-method"},
.handler = {[method](std::string s) {

5
src/libcmd/network-proxy.cc
View file

@ -25,7 +25,10 @@ static StringSet getExcludingNoProxyVariables()
static const StringSet excludeVariables{"no_proxy", "NO_PROXY"};
StringSet variables;
std::set_difference(
networkProxyVariables.begin(), networkProxyVariables.end(), excludeVariables.begin(), excludeVariables.end(),
networkProxyVariables.begin(),
networkProxyVariables.end(),
excludeVariables.begin(),
excludeVariables.end(),
std::inserter(variables, variables.begin()));
return variables;
}

13
src/libcmd/repl.cc
View file

@ -137,12 +137,13 @@ void runNix(Path program, const Strings & args,
{
auto subprocessEnv = getEnv();
subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue();
//isInteractive avoid grabling interactive commands
runProgram2(RunOptions {
.program = settings.nixBinDir+ "/" + program,
.args = args,
.environment = subprocessEnv,
.input = input,
.isInteractive = true,
});
return;
@ -259,7 +260,7 @@ StringSet NixRepl::completePrefix(const std::string & prefix)
try {
auto dir = std::string(cur, 0, slash);
auto prefix2 = std::string(cur, slash + 1);
for (auto & entry : readDirectory(dir == "" ? "/" : dir)) {
for (auto & entry : std::filesystem::directory_iterator{dir == "" ? "/" : dir}) {
auto name = entry.path().filename().string();
if (name[0] != '.' && hasPrefix(name, prefix2))
completions.insert(prev + entry.path().string());
@ -304,6 +305,8 @@ StringSet NixRepl::completePrefix(const std::string & prefix)
// Quietly ignore evaluation errors.
} catch (BadURL & e) {
// Quietly ignore BadURL flake-related errors.
} catch (FileNotFound & e) {
// Quietly ignore non-existent file beeing `import`-ed.
}
}
@ -508,13 +511,9 @@ ProcessLineResult NixRepl::processLine(std::string line)
auto editor = args.front();
args.pop_front();
// avoid garbling the editor with the progress bar
logger->pause();
Finally resume([&]() { logger->resume(); });
// runProgram redirects stdout to a StringSink,
// using runProgram2 to allow editors to display their UI
runProgram2(RunOptions { .program = editor, .lookupPath = true, .args = args });
runProgram2(RunOptions { .program = editor, .lookupPath = true, .args = args , .isInteractive = true });
// Reload right after exiting the editor
state->resetFileCache();

11
src/libexpr-c/nix_api_expr.cc
View file

@ -65,6 +65,17 @@ nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, V
NIXC_CATCH_ERRS
}
nix_err nix_value_call_multi(nix_c_context * context, EvalState * state, Value * fn, size_t nargs, Value ** args, Value * value)
{
if (context)
context->last_err_code = NIX_OK;
try {
state->state.callFunction(*(nix::Value *) fn, nargs, (nix::Value * *)args, *(nix::Value *) value, nix::noPos);
state->state.forceValue(*(nix::Value *) value, nix::noPos);
}
NIXC_CATCH_ERRS
}
nix_err nix_value_force(nix_c_context * context, EvalState * state, Value * value)
{
if (context)

69
src/libexpr-c/nix_api_expr.h
View file

@ -3,25 +3,7 @@
/** @defgroup libexpr libexpr
* @brief Bindings to the Nix language evaluator
*
* Example (without error handling):
* @code{.c}
* int main() {
* nix_libexpr_init(NULL);
*
* Store* store = nix_store_open(NULL, "dummy", NULL);
* EvalState* state = nix_state_create(NULL, NULL, store); // empty nix path
* Value *value = nix_alloc_value(NULL, state);
*
* nix_expr_eval_from_string(NULL, state, "builtins.nixVersion", ".", value);
* nix_value_force(NULL, state, value);
* printf("nix version: %s\n", nix_get_string(NULL, value));
*
* nix_gc_decref(NULL, value);
* nix_state_free(state);
* nix_store_free(store);
* return 0;
* }
* @endcode
* See *[Embedding the Nix Evaluator](@ref nix_evaluator_example)* for an example.
* @{
*/
/** @file
@ -30,6 +12,7 @@
#include "nix_api_store.h"
#include "nix_api_util.h"
#include <stddef.h>
#ifdef __cplusplus
extern "C" {
@ -98,6 +81,46 @@ nix_err nix_expr_eval_from_string(
*/
nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, Value * arg, Value * value);
/**
* @brief Calls a Nix function with multiple arguments.
*
* Technically these are functions that return functions. It is common for Nix
* functions to be curried, so this function is useful for calling them.
*
* @param[out] context Optional, stores error information
* @param[in] state The state of the evaluation.
* @param[in] fn The Nix function to call.
* @param[in] nargs The number of arguments.
* @param[in] args The arguments to pass to the function.
* @param[out] value The result of the function call.
*
* @see nix_value_call For the single argument primitive.
* @see NIX_VALUE_CALL For a macro that wraps this function for convenience.
*/
nix_err nix_value_call_multi(
nix_c_context * context, EvalState * state, Value * fn, size_t nargs, Value ** args, Value * value);
/**
* @brief Calls a Nix function with multiple arguments.
*
* Technically these are functions that return functions. It is common for Nix
* functions to be curried, so this function is useful for calling them.
*
* @param[out] context Optional, stores error information
* @param[in] state The state of the evaluation.
* @param[out] value The result of the function call.
* @param[in] fn The Nix function to call.
* @param[in] args The arguments to pass to the function.
*
* @see nix_value_call_multi
*/
#define NIX_VALUE_CALL(context, state, value, fn, ...) \
do { \
Value * args_array[] = {__VA_ARGS__}; \
size_t nargs = sizeof(args_array) / sizeof(args_array[0]); \
nix_value_call_multi(context, state, fn, nargs, args_array, value); \
} while (0)
/**
* @brief Forces the evaluation of a Nix value.
*
@ -106,10 +129,8 @@ nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, V
*
* This function converts these Values into their final type.
*
* @note You don't need this function for basic API usage, since all functions
* that return a value call it for you. The only place you will see a
* NIX_TYPE_THUNK is in the arguments that are passed to a PrimOp function
* you supplied to nix_alloc_primop.
* @note You don't need this function for basic API usage very often, since all functions that return a `Value` call it
* for you. This function is mainly needed before calling @ref getters.
*
* @param[out] context Optional, stores error information
* @param[in] state The state of the evaluation.
@ -140,7 +161,7 @@ nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, Value *
* @brief Create a new Nix language evaluator state.
*
* @param[out] context Optional, stores error information
* @param[in] lookupPath Array of strings corresponding to entries in NIX_PATH.
* @param[in] lookupPath Null-terminated array of strings corresponding to entries in NIX_PATH.
* @param[in] store The Nix store to use.
* @return A new Nix state or NULL on failure.
*/

6
src/libexpr-c/nix_api_external.h
View file

@ -48,7 +48,7 @@ void nix_set_string_return(nix_string_return * str, const char * c);
* Print to the nix_printer
*
* @param[out] context Optional, stores error information
* @param printer The nix_printer to print to
* @param[out] printer The nix_printer to print to
* @param[in] str The string to print
* @returns NIX_OK if everything worked
*/
@ -136,7 +136,7 @@ typedef struct NixCExternalValueDesc
* or setting it to the empty string, will make the conversion throw an error.
*/
void (*printValueAsJSON)(
void * self, EvalState *, bool strict, nix_string_context * c, bool copyToStore, nix_string_return * res);
void * self, EvalState * state, bool strict, nix_string_context * c, bool copyToStore, nix_string_return * res);
/**
* @brief Convert the external value to XML
*
@ -155,7 +155,7 @@ typedef struct NixCExternalValueDesc
*/
void (*printValueAsXML)(
void * self,
EvalState *,
EvalState * state,
int strict,
int location,
void * doc,

41
src/libexpr-c/nix_api_value.cc
View file

@ -73,10 +73,43 @@ static void nix_c_primop_wrapper(
PrimOpFun f, void * userdata, nix::EvalState & state, const nix::PosIdx pos, nix::Value ** args, nix::Value & v)
{
nix_c_context ctx;
f(userdata, &ctx, (EvalState *) &state, (Value **) args, (Value *) &v);
/* TODO: In the future, this should throw different errors depending on the error code */
if (ctx.last_err_code != NIX_OK)
state.error<nix::EvalError>("Error from builtin function: %s", *ctx.last_err).atPos(pos).debugThrow();
// v currently has a thunk, but the C API initializers require an uninitialized value.
//
// We can't destroy the thunk, because that makes it impossible to retry,
// which is needed for tryEval and for evaluation drivers that evaluate more
// than one value (e.g. an attrset with two derivations, both of which
// reference v).
//
// Instead we create a temporary value, and then assign the result to v.
// This does not give the primop definition access to the thunk, but that's
// ok because we don't see a need for this yet (e.g. inspecting thunks,
// or maybe something to make blackholes work better; we don't know).
nix::Value vTmp;
f(userdata, &ctx, (EvalState *) &state, (Value **) args, (Value *) &vTmp);
if (ctx.last_err_code != NIX_OK) {
/* TODO: Throw different errors depending on the error code */
state.error<nix::EvalError>("Error from custom function: %s", *ctx.last_err).atPos(pos).debugThrow();
}
if (!vTmp.isValid()) {
state.error<nix::EvalError>("Implementation error in custom function: return value was not initialized")
.atPos(pos)
.debugThrow();
}
if (vTmp.type() == nix::nThunk) {
// We might allow this in the future if it makes sense for the evaluator
// e.g. implementing tail recursion by returning a thunk to the next
// "iteration". Until then, this is most likely a mistake or misunderstanding.
state.error<nix::EvalError>("Implementation error in custom function: return value must not be a thunk")
.atPos(pos)
.debugThrow();
}
v = vTmp;
}
PrimOp * nix_alloc_primop(

4
src/libexpr-c/nix_api_value.h
View file

@ -79,6 +79,7 @@ typedef struct nix_realised_string nix_realised_string;
* @{
*/
/** @brief Function pointer for primops
*
* When you want to return an error, call nix_set_err_msg(context, NIX_ERR_UNKNOWN, "your error message here").
*
* @param[in] user_data Arbitrary data that was initially supplied to nix_alloc_primop
@ -147,7 +148,8 @@ Value * nix_alloc_value(nix_c_context * context, EvalState * state);
* @brief Functions to inspect and change Nix language values, represented by Value.
* @{
*/
/** @name Getters
/** @anchor getters
* @name Getters
*/
/**@{*/
/** @brief Get value type

41
src/libexpr/eval-cache.cc
View file

@ -7,6 +7,25 @@
namespace nix::eval_cache {
CachedEvalError::CachedEvalError(ref<AttrCursor> cursor, Symbol attr)
: EvalError(cursor->root->state, "cached failure of attribute '%s'", cursor->getAttrPathStr(attr))
, cursor(cursor), attr(attr)
{ }
void CachedEvalError::force()
{
auto & v = cursor->forceValue();
if (v.type() == nAttrs) {
auto a = v.attrs()->get(this->attr);
state.forceValue(*a->value, a->pos);
}
// Shouldn't happen.
throw EvalError(state, "evaluation of cached failed attribute '%s' unexpectedly succeeded", cursor->getAttrPathStr(attr));
}
static const char * schema = R"sql(
create table if not exists Attributes (
parent integer not null,
@ -470,7 +489,7 @@ Suggestions AttrCursor::getSuggestionsForAttr(Symbol name)
return Suggestions::bestMatches(strAttrNames, root->state.symbols[name]);
}
std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErrors)
std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name)
{
if (root->db) {
if (!cachedValue)
@ -487,12 +506,9 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro
if (attr) {
if (std::get_if<missing_t>(&attr->second))
return nullptr;
else if (std::get_if<failed_t>(&attr->second)) {
if (forceErrors)
debug("reevaluating failed cached attribute '%s'", getAttrPathStr(name));
else
throw CachedEvalError(root->state, "cached failure of attribute '%s'", getAttrPathStr(name));
} else
else if (std::get_if<failed_t>(&attr->second))
throw CachedEvalError(ref(shared_from_this()), name);
else
return std::make_shared<AttrCursor>(root,
std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
}
@ -537,9 +553,9 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(std::string_view name)
return maybeGetAttr(root->state.symbols.create(name));
}
ref<AttrCursor> AttrCursor::getAttr(Symbol name, bool forceErrors)
ref<AttrCursor> AttrCursor::getAttr(Symbol name)
{
auto p = maybeGetAttr(name, forceErrors);
auto p = maybeGetAttr(name);
if (!p)
throw Error("attribute '%s' does not exist", getAttrPathStr(name));
return ref(p);
@ -550,11 +566,11 @@ ref<AttrCursor> AttrCursor::getAttr(std::string_view name)
return getAttr(root->state.symbols.create(name));
}
OrSuggestions<ref<AttrCursor>> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force)
OrSuggestions<ref<AttrCursor>> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath)
{
auto res = shared_from_this();
for (auto & attr : attrPath) {
auto child = res->maybeGetAttr(attr, force);
auto child = res->maybeGetAttr(attr);
if (!child) {
auto suggestions = res->getSuggestionsForAttr(attr);
return OrSuggestions<ref<AttrCursor>>::failed(suggestions);
@ -751,8 +767,9 @@ bool AttrCursor::isDerivation()
StorePath AttrCursor::forceDerivation()
{
auto aDrvPath = getAttr(root->state.sDrvPath, true);
auto aDrvPath = getAttr(root->state.sDrvPath);
auto drvPath = root->state.store->parseStorePath(aDrvPath->getString());
drvPath.requireDerivation();
if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) {
/* The eval cache contains 'drvPath', but the actual path has
been garbage-collected. So force it to be regenerated. */

25
src/libexpr/eval-cache.hh
View file

@ -10,14 +10,28 @@
namespace nix::eval_cache {
MakeError(CachedEvalError, EvalError);
struct AttrDb;
class AttrCursor;
struct CachedEvalError : EvalError
{
const ref<AttrCursor> cursor;
const Symbol attr;
CachedEvalError(ref<AttrCursor> cursor, Symbol attr);
/**
* Evaluate this attribute, which should result in a regular
* `EvalError` exception being thrown.
*/
[[noreturn]]
void force();
};
class EvalCache : public std::enable_shared_from_this<EvalCache>
{
friend class AttrCursor;
friend class CachedEvalError;
std::shared_ptr<AttrDb> db;
EvalState & state;
@ -73,6 +87,7 @@ typedef std::variant<
class AttrCursor : public std::enable_shared_from_this<AttrCursor>
{
friend class EvalCache;
friend class CachedEvalError;
ref<EvalCache> root;
typedef std::optional<std::pair<std::shared_ptr<AttrCursor>, Symbol>> Parent;
@ -102,11 +117,11 @@ public:
Suggestions getSuggestionsForAttr(Symbol name);
std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name, bool forceErrors = false);
std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name);
std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name);
ref<AttrCursor> getAttr(Symbol name, bool forceErrors = false);
ref<AttrCursor> getAttr(Symbol name);
ref<AttrCursor> getAttr(std::string_view name);
@ -114,7 +129,7 @@ public:
* Get an attribute along a chain of attrsets. Note that this does
* not auto-call functors or functions.
*/
OrSuggestions<ref<AttrCursor>> findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force = false);
OrSuggestions<ref<AttrCursor>> findAlongAttrPath(const std::vector<Symbol> & attrPath);
std::string getString();

4
src/libexpr/eval-error.cc
View file

@ -27,8 +27,7 @@ EvalErrorBuilder<T> & EvalErrorBuilder<T>::atPos(Value & value, PosIdx fallback)
template<class T>
EvalErrorBuilder<T> & EvalErrorBuilder<T>::withTrace(PosIdx pos, const std::string_view text)
{
error.err.traces.push_front(
Trace{.pos = error.state.positions[pos], .hint = HintFmt(std::string(text))});
error.addTrace(error.state.positions[pos], text);
return *this;
}
@ -99,7 +98,6 @@ template class EvalErrorBuilder<TypeError>;
template class EvalErrorBuilder<UndefinedVarError>;
template class EvalErrorBuilder<MissingArgumentError>;
template class EvalErrorBuilder<InfiniteRecursionError>;
template class EvalErrorBuilder<CachedEvalError>;
template class EvalErrorBuilder<InvalidPathError>;
}

1
src/libexpr/eval-error.hh
View file

@ -43,7 +43,6 @@ MakeError(Abort, EvalError);
MakeError(TypeError, EvalError);
MakeError(UndefinedVarError, EvalError);
MakeError(MissingArgumentError, EvalError);
MakeError(CachedEvalError, EvalError);
MakeError(InfiniteRecursionError, EvalError);
struct InvalidPathError : public EvalError

20
src/libexpr/eval-settings.hh
View file

@ -15,8 +15,24 @@ struct EvalSettings : Config
static std::string resolvePseudoUrl(std::string_view url);
Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
"Whether builtin functions that allow executing native code should be enabled."};
Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", R"(
Enable built-in functions that allow executing native code.
In particular, this adds:
- `builtins.importNative` *path*
Load a dynamic shared object (DSO) at *path* which exposes a function pointer to a procedure that initialises a Nix language value, and return that value.
The procedure must have the following signature:
```cpp
extern "C" typedef void (*ValueInitialiser) (EvalState & state, Value & v);
```
The [Nix C++ API documentation](@docroot@/contributing/documentation.md#api-documentation) has more details on evaluator internals.
- `builtins.exec` *arguments*
Execute a program, where *arguments* are specified as a list of strings, and parse its output as a Nix expression.
)"};
Setting<Strings> nixPath{
this, getDefaultNixPath(), "nix-path",

2
src/libexpr/eval.cc
View file

@ -28,13 +28,13 @@
#include <algorithm>
#include <chrono>
#include <iostream>
#include <sstream>
#include <cstring>
#include <optional>
#include <unistd.h>
#include <sys/time.h>
#include <fstream>
#include <functional>
#include <iostream>
#include <nlohmann/json.hpp>
#include <boost/container/small_vector.hpp>

2
src/libexpr/flake/config.cc
View file

@ -32,7 +32,7 @@ static void writeTrustedList(const TrustedList & trustedList)
void ConfigFile::apply()
{
std::set<std::string> whitelist{"bash-prompt", "bash-prompt-prefix", "bash-prompt-suffix", "flake-registry", "commit-lockfile-summary"};
std::set<std::string> whitelist{"bash-prompt", "bash-prompt-prefix", "bash-prompt-suffix", "flake-registry", "commit-lock-file-summary", "commit-lockfile-summary"};
for (auto & [name, value] : settings) {

18
src/libexpr/get-drvs.cc
View file

@ -69,13 +69,21 @@ std::string PackageInfo::querySystem() const
std::optional<StorePath> PackageInfo::queryDrvPath() const
{
if (!drvPath && attrs) {
NixStringContext context;
if (auto i = attrs->get(state->sDrvPath))
drvPath = {state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the 'drvPath' attribute of a derivation")};
else
if (auto i = attrs->get(state->sDrvPath)) {
NixStringContext context;
auto found = state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the 'drvPath' attribute of a derivation");
try {
found.requireDerivation();
} catch (Error & e) {
e.addTrace(state->positions[i->pos], "while evaluating the 'drvPath' attribute of a derivation");
throw;
}
drvPath = {std::move(found)};
} else
drvPath = {std::nullopt};
}
return drvPath.value_or(std::nullopt);
drvPath.value_or(std::nullopt);
return *drvPath;
}

9
src/libexpr/lexer.l
View file

@ -20,8 +20,6 @@
#pragma clang diagnostic ignored "-Wunneeded-internal-declaration"
#endif
#include <boost/lexical_cast.hpp>
#include "nixexpr.hh"
#include "parser-tab.hh"
@ -129,9 +127,10 @@ or { return OR_KW; }
{ID} { yylval->id = {yytext, (size_t) yyleng}; return ID; }
{INT} { errno = 0;
try {
yylval->n = boost::lexical_cast<int64_t>(yytext);
} catch (const boost::bad_lexical_cast &) {
std::optional<int64_t> numMay = string2Int<int64_t>(yytext);
if (numMay.has_value()) {
yylval->n = *numMay;
} else {
throw ParseError(ErrorInfo{
.msg = HintFmt("invalid integer '%1%'", yytext),
.pos = state->positions[CUR_POS],

1
src/libexpr/nixexpr.cc
View file

@ -6,6 +6,7 @@
#include "print.hh"
#include <cstdlib>
#include <sstream>
namespace nix {

20
src/libexpr/primops.cc
View file

@ -26,6 +26,7 @@
#include <algorithm>
#include <cstring>
#include <sstream>
#include <regex>
#ifndef _WIN32
@ -2261,7 +2262,7 @@ static void addPath(
std::string_view name,
SourcePath path,
Value * filterFun,
FileIngestionMethod method,
ContentAddressMethod method,
const std::optional<Hash> expectedHash,
Value & v,
const NixStringContext & context)
@ -2293,11 +2294,10 @@ static void addPath(
std::optional<StorePath> expectedStorePath;
if (expectedHash)
expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo {
.method = method,
.hash = *expectedHash,
.references = {},
});
expectedStorePath = state.store->makeFixedOutputPathFromCA(name, ContentAddressWithReferences::fromParts(
method,
*expectedHash,
{}));
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
auto dstPath = fetchToStore(
@ -2393,7 +2393,7 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value
std::optional<SourcePath> path;
std::string name;
Value * filterFun = nullptr;
auto method = FileIngestionMethod::Recursive;
ContentAddressMethod method = FileIngestionMethod::Recursive;
std::optional<Hash> expectedHash;
NixStringContext context;
@ -2408,7 +2408,9 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value
else if (n == "filter")
state.forceFunction(*(filterFun = attr.value), attr.pos, "while evaluating the `filter` parameter passed to builtins.path");
else if (n == "recursive")
method = FileIngestionMethod { state.forceBool(*attr.value, attr.pos, "while evaluating the `recursive` attribute passed to builtins.path") };
method = state.forceBool(*attr.value, attr.pos, "while evaluating the `recursive` attribute passed to builtins.path")
? FileIngestionMethod::Recursive
: FileIngestionMethod::Flat;
else if (n == "sha256")
expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), HashAlgorithm::SHA256);
else
@ -4515,7 +4517,7 @@ void EvalState::createBaseEnv()
1683705525
```
The [store path](@docroot@/glossary.md#gloss-store-path) of a derivation depending on `currentTime` will differ for each evaluation, unless both evaluate `builtins.currentTime` in the same second.
The [store path](@docroot@/store/store-path.md) of a derivation depending on `currentTime` will differ for each evaluation, unless both evaluate `builtins.currentTime` in the same second.
)",
.impureOnly = true,
});

4
src/libexpr/primops/fetchTree.cc
View file

@ -200,8 +200,8 @@ static RegisterPrimOp primop_fetchTree({
.doc = R"(
Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with:
- the resulting fixed-output [store path](@docroot@/glossary.md#gloss-store-path)
- the corresponding [NAR](@docroot@/glossary.md#gloss-nar) hash
- the resulting fixed-output [store path](@docroot@/store/store-path.md)
- the corresponding [NAR](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) hash
- backend-specific metadata (currently not documented). <!-- TODO: document output attributes -->
*input* must be an attribute set with the following attributes:

24
src/libexpr/print.cc
View file

@ -1,5 +1,6 @@
#include <limits>
#include <unordered_set>
#include <sstream>
#include "print.hh"
#include "ansicolor.hh"
@ -271,16 +272,27 @@ private:
void printDerivation(Value & v)
{
NixStringContext context;
std::string storePath;
if (auto i = v.attrs()->get(state.sDrvPath))
storePath = state.store->printStorePath(state.coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation"));
std::optional<StorePath> storePath;
if (auto i = v.attrs()->get(state.sDrvPath)) {
NixStringContext context;
storePath = state.coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation");
}
/* This unforutately breaks printing nested values because of
how the pretty printer is used (when pretting printing and warning
to same terminal / std stream). */
#if 0
if (storePath && !storePath->isDerivation())
warn(
"drvPath attribute '%s' is not a valid store path to a derivation, this value not work properly",
state.store->printStorePath(*storePath));
#endif
if (options.ansiColors)
output << ANSI_GREEN;
output << "«derivation";
if (!storePath.empty()) {
output << " " << storePath;
if (storePath) {
output << " " << state.store->printStorePath(*storePath);
}
output << "»";
if (options.ansiColors)

2
src/libexpr/value.hh
View file

@ -449,7 +449,7 @@ public:
return std::string_view(payload.string.c_str);
}
const char * const c_str() const
const char * c_str() const
{
assert(internalType == tString);
return payload.string.c_str;

4
src/libfetchers/fetch-settings.hh
View file

@ -87,12 +87,12 @@ struct FetchSettings : public Config
{}, true, Xp::Flakes};
Setting<std::string> commitLockFileSummary{
this, "", "commit-lockfile-summary",
this, "", "commit-lock-file-summary",
R"(
The commit summary to use when committing changed flake lock files. If
empty, the summary is generated based on the action performed.
)",
{}, true, Xp::Flakes};
{"commit-lockfile-summary"}, true, Xp::Flakes};
Setting<bool> trustTarballsFromGitForges{
this, true, "trust-tarballs-from-git-forges",

6
src/libfetchers/filtering-source-accessor.cc
View file

@ -2,6 +2,12 @@
namespace nix {
std::optional<std::filesystem::path> FilteringSourceAccessor::getPhysicalPath(const CanonPath & path)
{
checkAccess(path);
return next->getPhysicalPath(prefix / path);
}
std::string FilteringSourceAccessor::readFile(const CanonPath & path)
{
checkAccess(path);

2
src/libfetchers/filtering-source-accessor.hh
View file

@ -30,6 +30,8 @@ struct FilteringSourceAccessor : SourceAccessor
displayPrefix.clear();
}
std::optional<std::filesystem::path> getPhysicalPath(const CanonPath & path) override;
std::string readFile(const CanonPath & path) override;
bool pathExists(const CanonPath & path) override;

27
src/libfetchers/unix/git.cc → src/libfetchers/git.cc
View file

@ -19,7 +19,10 @@
#include <regex>
#include <string.h>
#include <sys/time.h>
#include <sys/wait.h>
#ifndef _WIN32
# include <sys/wait.h>
#endif
using namespace std::string_literals;
@ -40,6 +43,7 @@ bool isCacheFileWithinTtl(time_t now, const struct stat & st)
bool touchCacheFile(const Path & path, time_t touch_time)
{
#ifndef _WIN32 // TODO implement
struct timeval times[2];
times[0].tv_sec = touch_time;
times[0].tv_usec = 0;
@ -47,6 +51,9 @@ bool touchCacheFile(const Path & path, time_t touch_time)
times[1].tv_usec = 0;
return lutimes(path.c_str(), times) == 0;
#else
return false;
#endif
}
Path getCachePath(std::string_view key, bool shallow)
@ -98,7 +105,15 @@ bool storeCachedHead(const std::string & actualUrl, const std::string & headRef)
try {
runProgram("git", true, { "-C", cacheDir, "--git-dir", ".", "symbolic-ref", "--", "HEAD", headRef });
} catch (ExecError &e) {
if (!WIFEXITED(e.status)) throw;
if (
#ifndef WIN32 // TODO abstract over exit status handling on Windows
!WIFEXITED(e.status)
#else
e.status != 0
#endif
)
throw;
return false;
}
/* No need to touch refs/HEAD, because `git symbolic-ref` updates the mtime. */
@ -329,7 +344,13 @@ struct GitInputScheme : InputScheme
.program = "git",
.args = {"-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "check-ignore", "--quiet", std::string(path.rel())},
});
auto exitCode = WEXITSTATUS(result.first);
auto exitCode =
#ifndef WIN32 // TODO abstract over exit status handling on Windows
WEXITSTATUS(result.first)
#else
result.first
#endif
;
if (exitCode != 0) {
// The path is not `.gitignore`d, we can add the file.

12
src/libfetchers/github.cc
View file

@ -433,9 +433,15 @@ struct GitLabInputScheme : GitArchiveInputScheme
store->toRealPath(
downloadFile(store, url, "source", headers).storePath)));
return RefInfo {
.rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1)
};
if (json.is_array() && json.size() == 1 && json[0]["id"] != nullptr) {
return RefInfo {
.rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1)
};
} if (json.is_array() && json.size() == 0) {
throw Error("No commits returned by GitLab API -- does the git ref really exist?");
} else {
throw Error("Unexpected response received from GitLab: %s", json);
}
}
DownloadUrl getDownloadUrl(const Input & input) const override

6
src/libfetchers/local.mk
View file

@ -5,16 +5,10 @@ libfetchers_NAME = libnixfetchers
libfetchers_DIR := $(d)
libfetchers_SOURCES := $(wildcard $(d)/*.cc)
ifdef HOST_UNIX
libfetchers_SOURCES += $(wildcard $(d)/unix/*.cc)
endif
# Not just for this library itself, but also for downstream libraries using this library
INCLUDE_libfetchers := -I $(d)
ifdef HOST_UNIX
INCLUDE_libfetchers += -I $(d)/unix
endif
libfetchers_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers)

0
src/libfetchers/unix/mercurial.cc → src/libfetchers/mercurial.cc
View file

22
src/libfetchers/tarball.cc
View file

@ -145,9 +145,27 @@ DownloadTarballResult downloadTarball(
// TODO: fall back to cached value if download fails.
AutoDelete cleanupTemp;
/* Note: if the download is cached, `importTarball()` will receive
no data, which causes it to import an empty tarball. */
TarArchive archive { *source };
auto archive =
hasSuffix(toLower(parseURL(url).path), ".zip")
? ({
/* In streaming mode, libarchive doesn't handle
symlinks in zip files correctly (#10649). So write
the entire file to disk so libarchive can access it
in random-access mode. */
auto [fdTemp, path] = createTempFile("nix-zipfile");
cleanupTemp.reset(path);
debug("downloading '%s' into '%s'...", url, path);
{
FdSink sink(fdTemp.get());
source->drainInto(sink);
}
TarArchive{path};
})
: TarArchive{*source};
auto parseSink = getTarballCache()->getFileSystemObjectSink();
auto lastModified = unpackTarfileToSink(archive, *parseSink);
@ -184,7 +202,7 @@ struct CurlInputScheme : InputScheme
{
const std::set<std::string> transportUrlSchemes = {"file", "http", "https"};
const bool hasTarballExtension(std::string_view path) const
bool hasTarballExtension(std::string_view path) const
{
return hasSuffix(path, ".zip") || hasSuffix(path, ".tar")
|| hasSuffix(path, ".tgz") || hasSuffix(path, ".tar.gz")

1
src/libmain/progress-bar.cc
View file

@ -7,6 +7,7 @@
#include <atomic>
#include <map>
#include <thread>
#include <sstream>
#include <iostream>
#include <chrono>

16
src/libstore-c/nix_api_store.h
View file

@ -54,16 +54,20 @@ nix_err nix_libstore_init_no_load_config(nix_c_context * context);
nix_err nix_init_plugins(nix_c_context * context);
/**
* @brief Open a nix store
* @brief Open a nix store.
*
* Store instances may share state and resources behind the scenes.
*
* @param[out] context Optional, stores error information
* @param[in] uri URI of the nix store, copied
* @param[in] params optional, array of key-value pairs, {{"endpoint",
* "https://s3.local"}}
* @param[in] uri URI of the Nix store, copied. See [*Store URL format* in the Nix Reference
* Manual](https://nixos.org/manual/nix/stable/store/types/#store-url-format).
* @param[in] params optional, null-terminated array of key-value pairs, e.g. {{"endpoint",
* "https://s3.local"}}. See [*Store Types* in the Nix Reference
* Manual](https://nixos.org/manual/nix/stable/store/types).
* @return a Store pointer, NULL in case of errors
* @see nix_store_free
*/
Store * nix_store_open(nix_c_context *, const char * uri, const char *** params);
Store * nix_store_open(nix_c_context * context, const char * uri, const char *** params);
/**
* @brief Deallocate a nix store and free any resources if not also held by other Store instances.
@ -155,7 +159,9 @@ nix_err nix_store_realise(
/**
* @brief get the version of a nix store.
*
* If the store doesn't have a version (like the dummy store), returns an empty string.
*
* @param[out] context Optional, stores error information
* @param[in] store nix store reference
* @param[in] callback Called with the version.

1
src/libstore/binary-cache-store.cc
View file

@ -18,6 +18,7 @@
#include <future>
#include <regex>
#include <fstream>
#include <sstream>
#include <nlohmann/json.hpp>

83
src/libstore/unix/build/derivation-goal.cc → src/libstore/build/derivation-goal.cc
View file

@ -1,5 +1,8 @@
#include "derivation-goal.hh"
#include "hook-instance.hh"
#ifndef _WIN32 // TODO enable build hook on Windows
# include "hook-instance.hh"
#endif
#include "processes.hh"
#include "worker.hh"
#include "builtins.hh"
#include "builtins/buildenv.hh"
@ -19,19 +22,8 @@
#include <fstream>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <sys/wait.h>
#include <netdb.h>
#include <fcntl.h>
#include <termios.h>
#include <unistd.h>
#include <sys/mman.h>
#include <sys/utsname.h>
#include <sys/resource.h>
#include <pwd.h>
#include <grp.h>
#include <nlohmann/json.hpp>
@ -101,7 +93,9 @@ std::string DerivationGoal::key()
void DerivationGoal::killChild()
{
#ifndef _WIN32 // TODO enable build hook on Windows
hook.reset();
#endif
}
@ -641,9 +635,17 @@ void DerivationGoal::started()
buildMode == bmCheck ? "checking outputs of '%s'" :
"building '%s'", worker.store.printStorePath(drvPath));
fmt("building '%s'", worker.store.printStorePath(drvPath));
#ifndef _WIN32 // TODO enable build hook on Windows
if (hook) msg += fmt(" on '%s'", machineName);
#endif
act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg,
Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", 1, 1});
Logger::Fields{worker.store.printStorePath(drvPath),
#ifndef _WIN32 // TODO enable build hook on Windows
hook ? machineName :
#endif
"",
1,
1});
mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
worker.updateProgress();
}
@ -778,12 +780,18 @@ static void movePath(const Path & src, const Path & dst)
{
auto st = lstat(src);
bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));
bool changePerm = (
#ifndef _WIN32
geteuid()
#else
!isRootUser()
#endif
&& S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));
if (changePerm)
chmod_(src, st.st_mode | S_IWUSR);
renameFile(src, dst);
std::filesystem::rename(src, dst);
if (changePerm)
chmod_(dst, st.st_mode);
@ -796,7 +804,7 @@ void replaceValidPath(const Path & storePath, const Path & tmpPath)
tmpPath (the replacement), so we have to move it out of the
way first. We'd better not be interrupted here, because if
we're repairing (say) Glibc, we end up with a broken system. */
Path oldPath = fmt("%1%.old-%2%-%3%", storePath, getpid(), random());
Path oldPath = fmt("%1%.old-%2%-%3%", storePath, getpid(), rand());
if (pathExists(storePath))
movePath(storePath, oldPath);
@ -818,14 +826,20 @@ void replaceValidPath(const Path & storePath, const Path & tmpPath)
int DerivationGoal::getChildStatus()
{
#ifndef _WIN32 // TODO enable build hook on Windows
return hook->pid.kill();
#else
return 0;
#endif
}
void DerivationGoal::closeReadPipes()
{
hook->builderOut.readSide = -1;
hook->fromHook.readSide = -1;
#ifndef _WIN32 // TODO enable build hook on Windows
hook->builderOut.readSide.close();
hook->fromHook.readSide.close();
#endif
}
@ -1019,13 +1033,16 @@ void DerivationGoal::buildDone()
BuildResult::Status st = BuildResult::MiscFailure;
#ifndef _WIN32
if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101)
st = BuildResult::TimedOut;
else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) {
}
else {
else
#endif
{
assert(derivationType);
st =
dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic :
@ -1112,6 +1129,9 @@ void DerivationGoal::resolvedFinished()
HookReply DerivationGoal::tryBuildHook()
{
#ifdef _WIN32 // TODO enable build hook on Windows
return rpDecline;
#else
if (settings.buildHook.get().empty() || !worker.tryBuildHook || !useDerivation) return rpDecline;
if (!worker.hook)
@ -1205,17 +1225,18 @@ HookReply DerivationGoal::tryBuildHook()
}
hook->sink = FdSink();
hook->toHook.writeSide = -1;
hook->toHook.writeSide.close();
/* Create the log file and pipe. */
Path logFile = openLogFile();
std::set<int> fds;
std::set<MuxablePipePollState::CommChannel> fds;
fds.insert(hook->fromHook.readSide.get());
fds.insert(hook->builderOut.readSide.get());
worker.childStarted(shared_from_this(), fds, false, false);
return rpAccept;
#endif
}
@ -1251,7 +1272,11 @@ Path DerivationGoal::openLogFile()
Path logFileName = fmt("%s/%s%s", dir, baseName.substr(2),
settings.compressLog ? ".bz2" : "");
fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666);
fdLogFile = toDescriptor(open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC
#ifndef _WIN32
| O_CLOEXEC
#endif
, 0666));
if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName);
logFileSink = std::make_shared<FdSink>(fdLogFile.get());
@ -1271,16 +1296,20 @@ void DerivationGoal::closeLogFile()
if (logSink2) logSink2->finish();
if (logFileSink) logFileSink->flush();
logSink = logFileSink = 0;
fdLogFile = -1;
fdLogFile.close();
}
bool DerivationGoal::isReadDesc(int fd)
bool DerivationGoal::isReadDesc(Descriptor fd)
{
#ifdef _WIN32 // TODO enable build hook on Windows
return false;
#else
return fd == hook->builderOut.readSide.get();
#endif
}
void DerivationGoal::handleChildOutput(int fd, std::string_view data)
void DerivationGoal::handleChildOutput(Descriptor fd, std::string_view data)
{
// local & `ssh://`-builds are dealt with here.
auto isWrittenToLog = isReadDesc(fd);
@ -1310,6 +1339,7 @@ void DerivationGoal::handleChildOutput(int fd, std::string_view data)
if (logSink) (*logSink)(data);
}
#ifndef _WIN32 // TODO enable build hook on Windows
if (hook && fd == hook->fromHook.readSide.get()) {
for (auto c : data)
if (c == '\n') {
@ -1344,10 +1374,11 @@ void DerivationGoal::handleChildOutput(int fd, std::string_view data)
} else
currentHookLine += c;
}
#endif
}
void DerivationGoal::handleEOF(int fd)
void DerivationGoal::handleEOF(Descriptor fd)
{
if (!currentLogLine.empty()) flushLine();
worker.wakeUp(shared_from_this());

14
src/libstore/unix/build/derivation-goal.hh → src/libstore/build/derivation-goal.hh
View file

@ -2,7 +2,9 @@
///@file
#include "parsed-derivations.hh"
#include "lock.hh"
#ifndef _WIN32
# include "user-lock.hh"
#endif
#include "outputs-spec.hh"
#include "store-api.hh"
#include "pathlocks.hh"
@ -12,7 +14,9 @@ namespace nix {
using std::map;
#ifndef _WIN32 // TODO enable build hook on Windows
struct HookInstance;
#endif
typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
@ -178,10 +182,12 @@ struct DerivationGoal : public Goal
std::string currentHookLine;
#ifndef _WIN32 // TODO enable build hook on Windows
/**
* The build hook.
*/
std::unique_ptr<HookInstance> hook;
#endif
/**
* The sort of derivation we are building.
@ -287,13 +293,13 @@ struct DerivationGoal : public Goal
virtual void cleanupPostOutputsRegisteredModeCheck();
virtual void cleanupPostOutputsRegisteredModeNonCheck();
virtual bool isReadDesc(int fd);
virtual bool isReadDesc(Descriptor fd);
/**
* Callback used by the worker to write to the log.
*/
void handleChildOutput(int fd, std::string_view data) override;
void handleEOF(int fd) override;
void handleChildOutput(Descriptor fd, std::string_view data) override;
void handleEOF(Descriptor fd) override;
void flushLine();
/**

14
src/libstore/unix/build/drv-output-substitution-goal.cc → src/libstore/build/drv-output-substitution-goal.cc
View file

@ -66,7 +66,11 @@ void DrvOutputSubstitutionGoal::tryNext()
some other error occurs), so it must not touch `this`. So put
the shared state in a separate refcounted object. */
downloadState = std::make_shared<DownloadState>();
#ifndef _WIN32
downloadState->outPipe.create();
#else
downloadState->outPipe.createAsyncPipe(worker.ioport.get());
#endif
sub->queryRealisation(
id,
@ -79,7 +83,13 @@ void DrvOutputSubstitutionGoal::tryNext()
}
} });
worker.childStarted(shared_from_this(), {downloadState->outPipe.readSide.get()}, true, false);
worker.childStarted(shared_from_this(), {
#ifndef _WIN32
downloadState->outPipe.readSide.get()
#else
&downloadState->outPipe
#endif
}, true, false);
state = &DrvOutputSubstitutionGoal::realisationFetched;
}
@ -158,7 +168,7 @@ void DrvOutputSubstitutionGoal::work()
(this->*state)();
}
void DrvOutputSubstitutionGoal::handleEOF(int fd)
void DrvOutputSubstitutionGoal::handleEOF(Descriptor fd)
{
if (fd == downloadState->outPipe.readSide.get()) worker.wakeUp(shared_from_this());
}

10
src/libstore/unix/build/drv-output-substitution-goal.hh → src/libstore/build/drv-output-substitution-goal.hh
View file

@ -1,11 +1,13 @@
#pragma once
///@file
#include <thread>
#include <future>
#include "store-api.hh"
#include "goal.hh"
#include "realisation.hh"
#include <thread>
#include <future>
#include "muxable-pipe.hh"
namespace nix {
@ -43,7 +45,7 @@ class DrvOutputSubstitutionGoal : public Goal {
struct DownloadState
{
Pipe outPipe;
MuxablePipe outPipe;
std::promise<std::shared_ptr<const Realisation>> promise;
};
@ -71,7 +73,7 @@ public:
std::string key() override;
void work() override;
void handleEOF(int fd) override;
void handleEOF(Descriptor fd) override;
JobCategory jobCategory() const override {
return JobCategory::Substitution;

14
src/libstore/unix/build/entry-points.cc → src/libstore/build/entry-points.cc
View file

@ -1,6 +1,8 @@
#include "worker.hh"
#include "substitution-goal.hh"
#include "derivation-goal.hh"
#ifndef _WIN32 // TODO Enable building on Windows
# include "derivation-goal.hh"
#endif
#include "local-store.hh"
namespace nix {
@ -25,9 +27,12 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
ex = std::move(i->ex);
}
if (i->exitCode != Goal::ecSuccess) {
#ifndef _WIN32 // TODO Enable building on Windows
if (auto i2 = dynamic_cast<DerivationGoal *>(i.get()))
failed.insert(printStorePath(i2->drvPath));
else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get()))
else
#endif
if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get()))
failed.insert(printStorePath(i2->storePath));
}
}
@ -74,7 +79,12 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat
BuildMode buildMode)
{
Worker worker(*this, *this);
#ifndef _WIN32 // TODO Enable building on Windows
auto goal = worker.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All {}, buildMode);
#else
std::shared_ptr<Goal> goal;
throw UnimplementedError("Building derivations not yet implemented on windows.");
#endif
try {
worker.run(Goals{goal});

0
src/libstore/unix/build/goal.cc → src/libstore/build/goal.cc
View file

4
src/libstore/unix/build/goal.hh → src/libstore/build/goal.hh
View file

@ -138,12 +138,12 @@ public:
virtual void waiteeDone(GoalPtr waitee, ExitCode result);
virtual void handleChildOutput(int fd, std::string_view data)
virtual void handleChildOutput(Descriptor fd, std::string_view data)
{
abort();
}
virtual void handleEOF(int fd)
virtual void handleEOF(Descriptor fd)
{
abort();
}

16
src/libstore/unix/build/substitution-goal.cc → src/libstore/build/substitution-goal.cc
View file

@ -212,7 +212,11 @@ void PathSubstitutionGoal::tryToRun()
maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions);
worker.updateProgress();
#ifndef _WIN32
outPipe.create();
#else
outPipe.createAsyncPipe(worker.ioport.get());
#endif
promise = std::promise<void>();
@ -235,7 +239,13 @@ void PathSubstitutionGoal::tryToRun()
}
});
worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false);
worker.childStarted(shared_from_this(), {
#ifndef _WIN32
outPipe.readSide.get()
#else
&outPipe
#endif
}, true, false);
state = &PathSubstitutionGoal::finished;
}
@ -294,12 +304,12 @@ void PathSubstitutionGoal::finished()
}
void PathSubstitutionGoal::handleChildOutput(int fd, std::string_view data)
void PathSubstitutionGoal::handleChildOutput(Descriptor fd, std::string_view data)
{
}
void PathSubstitutionGoal::handleEOF(int fd)
void PathSubstitutionGoal::handleEOF(Descriptor fd)
{
if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
}

8
src/libstore/unix/build/substitution-goal.hh → src/libstore/build/substitution-goal.hh
View file

@ -1,9 +1,9 @@
#pragma once
///@file
#include "lock.hh"
#include "store-api.hh"
#include "goal.hh"
#include "muxable-pipe.hh"
namespace nix {
@ -45,7 +45,7 @@ struct PathSubstitutionGoal : public Goal
/**
* Pipe for the substituter's standard output.
*/
Pipe outPipe;
MuxablePipe outPipe;
/**
* The substituter thread.
@ -111,8 +111,8 @@ public:
/**
* Callback used by the worker to write to the log.
*/
void handleChildOutput(int fd, std::string_view data) override;
void handleEOF(int fd) override;
void handleChildOutput(Descriptor fd, std::string_view data) override;
void handleEOF(Descriptor fd) override;
/* Called by destructor, can't be overridden */
void cleanup() override final;

101
src/libstore/unix/build/worker.cc → src/libstore/build/worker.cc
View file

@ -1,13 +1,15 @@
#include "local-store.hh"
#include "machines.hh"
#include "worker.hh"
#include "substitution-goal.hh"
#include "drv-output-substitution-goal.hh"
#include "local-derivation-goal.hh"
#include "hook-instance.hh"
#include "derivation-goal.hh"
#ifndef _WIN32 // TODO Enable building on Windows
# include "local-derivation-goal.hh"
# include "hook-instance.hh"
#endif
#include "signals.hh"
#include <poll.h>
namespace nix {
Worker::Worker(Store & store, Store & evalStore)
@ -64,20 +66,27 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drv
const OutputsSpec & wantedOutputs, BuildMode buildMode)
{
return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
return !dynamic_cast<LocalStore *>(&store)
? std::make_shared</* */DerivationGoal>(drvPath, wantedOutputs, *this, buildMode)
: std::make_shared<LocalDerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
return
#ifndef _WIN32 // TODO Enable building on Windows
dynamic_cast<LocalStore *>(&store)
? std::make_shared<LocalDerivationGoal>(drvPath, wantedOutputs, *this, buildMode)
:
#endif
std::make_shared</* */DerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
});
}
std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode)
{
return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
return !dynamic_cast<LocalStore *>(&store)
? std::make_shared</* */DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode)
: std::make_shared<LocalDerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
return
#ifndef _WIN32 // TODO Enable building on Windows
dynamic_cast<LocalStore *>(&store)
? std::make_shared<LocalDerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode)
:
#endif
std::make_shared</* */DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
});
}
@ -143,7 +152,8 @@ void Worker::removeGoal(GoalPtr goal)
{
if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
nix::removeGoal(drvGoal, derivationGoals);
else if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
else
if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
nix::removeGoal(subGoal, substitutionGoals);
else if (auto subGoal = std::dynamic_pointer_cast<DrvOutputSubstitutionGoal>(goal))
nix::removeGoal(subGoal, drvOutputSubstitutionGoals);
@ -187,13 +197,13 @@ unsigned Worker::getNrSubstitutions()
}
void Worker::childStarted(GoalPtr goal, const std::set<int> & fds,
void Worker::childStarted(GoalPtr goal, const std::set<MuxablePipePollState::CommChannel> & channels,
bool inBuildSlot, bool respectTimeouts)
{
Child child;
child.goal = goal;
child.goal2 = goal.get();
child.fds = fds;
child.channels = channels;
child.timeStarted = child.lastOutput = steady_time_point::clock::now();
child.inBuildSlot = inBuildSlot;
child.respectTimeouts = respectTimeouts;
@ -286,7 +296,8 @@ void Worker::run(const Goals & _topGoals)
.drvPath = makeConstantStorePathRef(goal->drvPath),
.outputs = goal->wantedOutputs,
});
} else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
} else
if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
topPaths.push_back(DerivedPath::Opaque{goal->storePath});
}
}
@ -408,23 +419,25 @@ void Worker::waitForInput()
if (useTimeout)
vomit("sleeping %d seconds", timeout);
MuxablePipePollState state;
#ifndef _WIN32
/* Use select() to wait for the input side of any logger pipe to
become `available'. Note that `available' (i.e., non-blocking)
includes EOF. */
std::vector<struct pollfd> pollStatus;
std::map<int, size_t> fdToPollStatus;
for (auto & i : children) {
for (auto & j : i.fds) {
pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN });
fdToPollStatus[j] = pollStatus.size() - 1;
for (auto & j : i.channels) {
state.pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN });
state.fdToPollStatus[j] = state.pollStatus.size() - 1;
}
}
#endif
if (poll(pollStatus.data(), pollStatus.size(),
useTimeout ? timeout * 1000 : -1) == -1) {
if (errno == EINTR) return;
throw SysError("waiting for input");
}
state.poll(
#ifdef _WIN32
ioport.get(),
#endif
useTimeout ? (std::optional { timeout * 1000 }) : std::nullopt);
auto after = steady_time_point::clock::now();
@ -439,32 +452,18 @@ void Worker::waitForInput()
GoalPtr goal = j->goal.lock();
assert(goal);
std::set<int> fds2(j->fds);
std::vector<unsigned char> buffer(4096);
for (auto & k : fds2) {
const auto fdPollStatusId = get(fdToPollStatus, k);
assert(fdPollStatusId);
assert(*fdPollStatusId < pollStatus.size());
if (pollStatus.at(*fdPollStatusId).revents) {
ssize_t rd = ::read(k, buffer.data(), buffer.size());
// FIXME: is there a cleaner way to handle pt close
// than EIO? Is this even standard?
if (rd == 0 || (rd == -1 && errno == EIO)) {
debug("%1%: got EOF", goal->getName());
goal->handleEOF(k);
j->fds.erase(k);
} else if (rd == -1) {
if (errno != EINTR)
throw SysError("%s: read failed", goal->getName());
} else {
printMsg(lvlVomit, "%1%: read %2% bytes",
goal->getName(), rd);
std::string_view data((char *) buffer.data(), rd);
j->lastOutput = after;
goal->handleChildOutput(k, data);
}
}
}
state.iterate(
j->channels,
[&](Descriptor k, std::string_view data) {
printMsg(lvlVomit, "%1%: read %2% bytes",
goal->getName(), data.size());
j->lastOutput = after;
goal->handleChildOutput(k, data);
},
[&](Descriptor k) {
debug("%1%: got EOF", goal->getName());
goal->handleEOF(k);
});
if (goal->exitCode == Goal::ecBusy &&
0 != settings.maxSilentTime &&

16
src/libstore/unix/build/worker.hh → src/libstore/build/worker.hh
View file

@ -2,10 +2,10 @@
///@file
#include "types.hh"
#include "lock.hh"
#include "store-api.hh"
#include "goal.hh"
#include "realisation.hh"
#include "muxable-pipe.hh"
#include <future>
#include <thread>
@ -36,14 +36,14 @@ typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
/**
* A mapping used to remember for each child process to what goal it
* belongs, and file descriptors for receiving log data and output
* belongs, and comm channels for receiving log data and output
* path creation commands.
*/
struct Child
{
WeakGoalPtr goal;
Goal * goal2; // ugly hackery
std::set<int> fds;
std::set<MuxablePipePollState::CommChannel> channels;
bool respectTimeouts;
bool inBuildSlot;
/**
@ -53,8 +53,10 @@ struct Child
steady_time_point timeStarted;
};
#ifndef _WIN32 // TODO Enable building on Windows
/* Forward definition. */
struct HookInstance;
#endif
/**
* The worker class.
@ -152,10 +154,16 @@ public:
*/
bool checkMismatch;
#ifdef _WIN32
AutoCloseFD ioport;
#endif
Store & store;
Store & evalStore;
#ifndef _WIN32 // TODO Enable building on Windows
std::unique_ptr<HookInstance> hook;
#endif
uint64_t expectedBuilds = 0;
uint64_t doneBuilds = 0;
@ -238,7 +246,7 @@ public:
* Registers a running child process. `inBuildSlot` means that
* the process counts towards the jobs limit.
*/
void childStarted(GoalPtr goal, const std::set<int> & fds,
void childStarted(GoalPtr goal, const std::set<MuxablePipePollState::CommChannel> & channels,
bool inBuildSlot, bool respectTimeouts);
/**

4
src/libstore/builtins/buildenv.cc
View file

@ -17,10 +17,10 @@ struct State
/* For each activated package, create symlinks */
static void createLinks(State & state, const Path & srcDir, const Path & dstDir, int priority)
{
std::vector<std::filesystem::directory_entry> srcFiles;
std::filesystem::directory_iterator srcFiles;
try {
srcFiles = readDirectory(srcDir);
srcFiles = std::filesystem::directory_iterator{srcDir};
} catch (std::filesystem::filesystem_error & e) {
if (e.code() == std::errc::not_a_directory) {
warn("not including '%s' in the user environment because it's not a directory", srcDir);

0
src/libstore/unix/builtins/fetchurl.cc → src/libstore/builtins/fetchurl.cc
View file

9
src/libstore/unix/builtins/unpack-channel.cc → src/libstore/builtins/unpack-channel.cc
View file

@ -21,10 +21,13 @@ void builtinUnpackChannel(
unpackTarfile(src, out);
auto entries = readDirectory(out);
if (entries.size() != 1)
auto entries = std::filesystem::directory_iterator{out};
auto fileName = entries->path().string();
auto fileCount = std::distance(std::filesystem::begin(entries), std::filesystem::end(entries));
if (fileCount != 1)
throw Error("channel tarball '%s' contains more than one file", src);
renameFile(entries[0].path().string(), (out + "/" + channelName));
std::filesystem::rename(fileName, (out + "/" + channelName));
}
}

46
src/libstore/daemon.cc
View file

@ -1,6 +1,7 @@
#include "daemon.hh"
#include "signals.hh"
#include "worker-protocol.hh"
#include "worker-protocol-connection.hh"
#include "worker-protocol-impl.hh"
#include "build-result.hh"
#include "store-api.hh"
@ -19,6 +20,8 @@
# include "monitor-fd.hh"
#endif
#include <sstream>
namespace nix::daemon {
Sink & operator << (Sink & sink, const Logger::Fields & fields)
@ -531,7 +534,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
auto drvs = WorkerProto::Serialise<DerivedPaths>::read(*store, rconn);
BuildMode mode = bmNormal;
if (GET_PROTOCOL_MINOR(clientVersion) >= 15) {
mode = (BuildMode) readInt(from);
mode = WorkerProto::Serialise<BuildMode>::read(*store, rconn);
/* Repairing is not atomic, so disallowed for "untrusted"
clients.
@ -555,7 +558,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case WorkerProto::Op::BuildPathsWithResults: {
auto drvs = WorkerProto::Serialise<DerivedPaths>::read(*store, rconn);
BuildMode mode = bmNormal;
mode = (BuildMode) readInt(from);
mode = WorkerProto::Serialise<BuildMode>::read(*store, rconn);
/* Repairing is not atomic, so disallowed for "untrusted"
clients.
@ -586,7 +589,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
* correctly.
*/
readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath));
BuildMode buildMode = (BuildMode) readInt(from);
auto buildMode = WorkerProto::Serialise<BuildMode>::read(*store, rconn);
logger->startWork();
auto drvType = drv.type();
@ -1026,11 +1029,9 @@ void processConnection(
#endif
/* Exchange the greeting. */
unsigned int magic = readInt(from);
if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch");
to << WORKER_MAGIC_2 << PROTOCOL_VERSION;
to.flush();
WorkerProto::Version clientVersion = readInt(from);
WorkerProto::Version clientVersion =
WorkerProto::BasicServerConnection::handshake(
to, from, PROTOCOL_VERSION);
if (clientVersion < 0x10a)
throw Error("the Nix client version is too old");
@ -1048,29 +1049,20 @@ void processConnection(
printMsgUsing(prevLogger, lvlDebug, "%d operations", opCount);
});
if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) {
// Obsolete CPU affinity.
readInt(from);
}
WorkerProto::BasicServerConnection conn {
.to = to,
.from = from,
.clientVersion = clientVersion,
};
if (GET_PROTOCOL_MINOR(clientVersion) >= 11)
readInt(from); // obsolete reserveSpace
if (GET_PROTOCOL_MINOR(clientVersion) >= 33)
to << nixVersion;
if (GET_PROTOCOL_MINOR(clientVersion) >= 35) {
conn.postHandshake(*store, {
.daemonNixVersion = nixVersion,
// We and the underlying store both need to trust the client for
// it to be trusted.
auto temp = trusted
.remoteTrustsUs = trusted
? store->isTrustedClient()
: std::optional { NotTrusted };
WorkerProto::WriteConn wconn {
.to = to,
.version = clientVersion,
};
WorkerProto::write(*store, wconn, temp);
}
: std::optional { NotTrusted },
});
/* Send startup error messages to the client. */
tunnelLogger->startWork();

29
src/libstore/derivations.cc
View file

@ -930,10 +930,9 @@ DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const StoreDirC
std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath)
{
drvPath.requireDerivation();
auto nameWithSuffix = drvPath.name();
constexpr std::string_view extension = ".drv";
assert(hasSuffix(nameWithSuffix, extension));
nameWithSuffix.remove_suffix(extension.size());
nameWithSuffix.remove_suffix(drvExtension.size());
return nameWithSuffix;
}
@ -1216,16 +1215,19 @@ nlohmann::json DerivationOutput::toJSON(
},
[&](const DerivationOutput::CAFixed & dof) {
res["path"] = store.printStorePath(dof.path(store, drvName, outputName));
res["hashAlgo"] = dof.ca.printMethodAlgo();
res["method"] = std::string { dof.ca.method.render() };
res["hashAlgo"] = printHashAlgo(dof.ca.hash.algo);
res["hash"] = dof.ca.hash.to_string(HashFormat::Base16, false);
// FIXME print refs?
},
[&](const DerivationOutput::CAFloating & dof) {
res["hashAlgo"] = std::string { dof.method.renderPrefix() } + printHashAlgo(dof.hashAlgo);
res["method"] = std::string { dof.method.render() };
res["hashAlgo"] = printHashAlgo(dof.hashAlgo);
},
[&](const DerivationOutput::Deferred &) {},
[&](const DerivationOutput::Impure & doi) {
res["hashAlgo"] = std::string { doi.method.renderPrefix() } + printHashAlgo(doi.hashAlgo);
res["method"] = std::string { doi.method.render() };
res["hashAlgo"] = printHashAlgo(doi.hashAlgo);
res["impure"] = true;
},
}, raw);
@ -1245,12 +1247,13 @@ DerivationOutput DerivationOutput::fromJSON(
keys.insert(key);
auto methodAlgo = [&]() -> std::pair<ContentAddressMethod, HashAlgorithm> {
auto & str = getString(valueAt(json, "hashAlgo"));
std::string_view s = str;
ContentAddressMethod method = ContentAddressMethod::parsePrefix(s);
auto & method_ = getString(valueAt(json, "method"));
ContentAddressMethod method = ContentAddressMethod::parse(method_);
if (method == TextIngestionMethod {})
xpSettings.require(Xp::DynamicDerivations);
auto hashAlgo = parseHashAlgo(s);
auto & hashAlgo_ = getString(valueAt(json, "hashAlgo"));
auto hashAlgo = parseHashAlgo(hashAlgo_);
return { std::move(method), std::move(hashAlgo) };
};
@ -1260,7 +1263,7 @@ DerivationOutput DerivationOutput::fromJSON(
};
}
else if (keys == (std::set<std::string_view> { "path", "hashAlgo", "hash" })) {
else if (keys == (std::set<std::string_view> { "path", "method", "hashAlgo", "hash" })) {
auto [method, hashAlgo] = methodAlgo();
auto dof = DerivationOutput::CAFixed {
.ca = ContentAddress {
@ -1273,7 +1276,7 @@ DerivationOutput DerivationOutput::fromJSON(
return dof;
}
else if (keys == (std::set<std::string_view> { "hashAlgo" })) {
else if (keys == (std::set<std::string_view> { "method", "hashAlgo" })) {
xpSettings.require(Xp::CaDerivations);
auto [method, hashAlgo] = methodAlgo();
return DerivationOutput::CAFloating {
@ -1286,7 +1289,7 @@ DerivationOutput DerivationOutput::fromJSON(
return DerivationOutput::Deferred {};
}
else if (keys == (std::set<std::string_view> { "hashAlgo", "impure" })) {
else if (keys == (std::set<std::string_view> { "method", "hashAlgo", "impure" })) {
xpSettings.require(Xp::ImpureDerivations);
auto [method, hashAlgo] = methodAlgo();
return DerivationOutput::Impure {

7
src/libstore/dummy-store.cc
View file

@ -18,9 +18,12 @@ struct DummyStoreConfig : virtual StoreConfig {
struct DummyStore : public virtual DummyStoreConfig, public virtual Store
{
DummyStore(const std::string scheme, const std::string uri, const Params & params)
DummyStore(std::string_view scheme, std::string_view authority, const Params & params)
: DummyStore(params)
{ }
{
if (!authority.empty())
throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority);
}
DummyStore(const Params & params)
: StoreConfig(params)

7
src/libstore/filetransfer.cc
View file

@ -580,7 +580,12 @@ struct curlFileTransfer : public FileTransfer
#endif
#if __linux__
unshareFilesystem();
try {
tryUnshareFilesystem();
} catch (nix::Error & e) {
e.addTrace({}, "in download thread");
throw;
}
#endif
std::map<CURL *, std::shared_ptr<TransferItem>> items;

8
src/libstore/gc.cc
View file

@ -161,7 +161,7 @@ void LocalStore::findTempRoots(Roots & tempRoots, bool censor)
{
/* Read the `temproots' directory for per-process temporary root
files. */
for (auto & i : readDirectory(tempRootsDir)) {
for (auto & i : std::filesystem::directory_iterator{tempRootsDir}) {
auto name = i.path().filename().string();
if (name[0] == '.') {
// Ignore hidden files. Some package managers (notably portage) create
@ -225,10 +225,10 @@ void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, R
try {
if (type == std::filesystem::file_type::unknown)
type = getFileType(path);
type = std::filesystem::symlink_status(path).type();
if (type == std::filesystem::file_type::directory) {
for (auto & i : readDirectory(path))
for (auto & i : std::filesystem::directory_iterator{path})
findRoots(i.path().string(), i.symlink_status().type(), roots);
}
@ -781,7 +781,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
throw Error(
"Cannot delete path '%1%' since it is still alive. "
"To find out why, use: "
"nix-store --query --roots",
"nix-store --query --roots and nix-store --query --referrers",
printStorePath(i));
}

2
src/libstore/globals.cc
View file

@ -345,7 +345,7 @@ void initPlugins()
for (const auto & pluginFile : settings.pluginFiles.get()) {
std::vector<std::filesystem::path> pluginFiles;
try {
auto ents = readDirectory(pluginFile);
auto ents = std::filesystem::directory_iterator{pluginFile};
for (const auto & ent : ents)
pluginFiles.emplace_back(ent.path());
} catch (std::filesystem::filesystem_error & e) {

2
src/libstore/globals.hh
View file

@ -910,7 +910,7 @@ public:
"substituters",
R"(
A list of [URLs of Nix stores](@docroot@/store/types/index.md#store-url-format) to be used as substituters, separated by whitespace.
A substituter is an additional [store](@docroot@/glossary.md#gloss-store) from which Nix can obtain [store objects](@docroot@/glossary.md#gloss-store-object) instead of building them.
A substituter is an additional [store](@docroot@/glossary.md#gloss-store) from which Nix can obtain [store objects](@docroot@/store/store-object.md) instead of building them.
Substituters are tried based on their priority value, which each substituter can set independently.
Lower value means higher priority.

11
src/libstore/http-binary-cache-store.cc
View file

@ -39,15 +39,20 @@ private:
public:
HttpBinaryCacheStore(
const std::string & scheme,
const Path & _cacheUri,
std::string_view scheme,
PathView _cacheUri,
const Params & params)
: StoreConfig(params)
, BinaryCacheStoreConfig(params)
, HttpBinaryCacheStoreConfig(params)
, Store(params)
, BinaryCacheStore(params)
, cacheUri(scheme + "://" + _cacheUri)
, cacheUri(
std::string { scheme }
+ "://"
+ (!_cacheUri.empty()
? _cacheUri
: throw UsageError("`%s` Store requires a non-empty authority in Store URL", scheme)))
{
while (!cacheUri.empty() && cacheUri.back() == '/')
cacheUri.pop_back();

2
src/libstore/indirect-root-store.cc
View file

@ -12,7 +12,7 @@ void IndirectRootStore::makeSymlink(const Path & link, const Path & target)
createSymlink(target, tempLink);
/* Atomically replace the old one. */
renameFile(tempLink, link);
std::filesystem::rename(tempLink, link);
}
Path IndirectRootStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot)

101
src/libstore/legacy-ssh-store.cc
View file

@ -4,6 +4,7 @@
#include "pool.hh"
#include "remote-store.hh"
#include "serve-protocol.hh"
#include "serve-protocol-connection.hh"
#include "serve-protocol-impl.hh"
#include "build-result.hh"
#include "store-api.hh"
@ -28,26 +29,23 @@ struct LegacySSHStore::Connection : public ServeProto::BasicClientConnection
bool good = true;
};
LegacySSHStore::LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params)
LegacySSHStore::LegacySSHStore(
std::string_view scheme,
std::string_view host,
const Params & params)
: StoreConfig(params)
, CommonSSHStoreConfig(params)
, CommonSSHStoreConfig(scheme, host, params)
, LegacySSHStoreConfig(params)
, Store(params)
, host(host)
, connections(make_ref<Pool<Connection>>(
std::max(1, (int) maxConnections),
[this]() { return openConnection(); },
[](const ref<Connection> & r) { return r->good; }
))
, master(
host,
sshKey,
sshPublicHostKey,
, master(createSSHMaster(
// Use SSH master only if using more than 1 connection.
connections->capacity() > 1,
compress,
logFD)
logFD))
{
}
@ -76,7 +74,7 @@ ref<LegacySSHStore::Connection> LegacySSHStore::openConnection()
conn->sshConn->in.close();
{
NullSink nullSink;
conn->from.drainInto(nullSink);
tee.drainInto(nullSink);
}
throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'",
host, chomp(saved.s));
@ -105,24 +103,26 @@ void LegacySSHStore::queryPathInfoUncached(const StorePath & path,
debug("querying remote host '%s' for info on '%s'", host, printStorePath(path));
conn->to << ServeProto::Command::QueryPathInfos << PathSet{printStorePath(path)};
conn->to.flush();
auto infos = conn->queryPathInfos(*this, {path});
auto p = readString(conn->from);
if (p.empty()) return callback(nullptr);
auto path2 = parseStorePath(p);
assert(path == path2);
auto info = std::make_shared<ValidPathInfo>(
path,
ServeProto::Serialise<UnkeyedValidPathInfo>::read(*this, *conn));
switch (infos.size()) {
case 0:
return callback(nullptr);
case 1: {
auto & [path2, info] = *infos.begin();
if (info->narHash == Hash::dummy)
throw Error("NAR hash is now mandatory");
if (info.narHash == Hash::dummy)
throw Error("NAR hash is now mandatory");
auto s = readString(conn->from);
assert(s == "");
callback(std::move(info));
assert(path == path2);
return callback(std::make_shared<ValidPathInfo>(
std::move(path),
std::move(info)
));
}
default:
throw Error("More path infos returned than queried");
}
} catch (...) { callback.rethrow(); }
}
@ -156,41 +156,38 @@ void LegacySSHStore::addToStore(const ValidPathInfo & info, Source & source,
}
conn->to.flush();
if (readInt(conn->from) != 1)
throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host);
} else {
conn->to
<< ServeProto::Command::ImportPaths
<< 1;
try {
copyNAR(source, conn->to);
} catch (...) {
conn->good = false;
throw;
}
conn->to
<< exportMagic
<< printStorePath(info.path);
ServeProto::write(*this, *conn, info.references);
conn->to
<< (info.deriver ? printStorePath(*info.deriver) : "")
<< 0
<< 0;
conn->to.flush();
conn->importPaths(*this, [&](Sink & sink) {
try {
copyNAR(source, sink);
} catch (...) {
conn->good = false;
throw;
}
sink
<< exportMagic
<< printStorePath(info.path);
ServeProto::write(*this, *conn, info.references);
sink
<< (info.deriver ? printStorePath(*info.deriver) : "")
<< 0
<< 0;
});
}
if (readInt(conn->from) != 1)
throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host);
}
void LegacySSHStore::narFromPath(const StorePath & path, Sink & sink)
{
auto conn(connections->get());
conn->to << ServeProto::Command::DumpStorePath << printStorePath(path);
conn->to.flush();
copyNAR(conn->from, sink);
conn->narFromPath(*this, path, [&](auto & source) {
copyNAR(source, sink);
});
}
@ -214,7 +211,7 @@ BuildResult LegacySSHStore::buildDerivation(const StorePath & drvPath, const Bas
conn->putBuildDerivationRequest(*this, drvPath, drv, buildSettings());
return ServeProto::Serialise<BuildResult>::read(*this, *conn);
return conn->getBuildDerivationResponse(*this);
}

13
src/libstore/legacy-ssh-store.hh
View file

@ -26,22 +26,27 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig
struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store
{
#ifndef _WIN32
// Hack for getting remote build log output.
// Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in
// the documentation
const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
const Setting<int> logFD{this, INVALID_DESCRIPTOR, "log-fd", "file descriptor to which SSH's stderr is connected"};
#else
Descriptor logFD = INVALID_DESCRIPTOR;
#endif
struct Connection;
std::string host;
ref<Pool<Connection>> connections;
SSHMaster master;
static std::set<std::string> uriSchemes() { return {"ssh"}; }
LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params);
LegacySSHStore(
std::string_view scheme,
std::string_view host,
const Params & params);
ref<Connection> openConnection();

12
src/libstore/local-binary-cache-store.cc
View file

@ -28,9 +28,13 @@ private:
public:
/**
* @param binaryCacheDir `file://` is a short-hand for `file:///`
* for now.
*/
LocalBinaryCacheStore(
const std::string scheme,
const Path & binaryCacheDir,
std::string_view scheme,
PathView binaryCacheDir,
const Params & params)
: StoreConfig(params)
, BinaryCacheStoreConfig(params)
@ -64,7 +68,7 @@ protected:
AutoDelete del(tmp, false);
StreamToSourceAdapter source(istream);
writeFile(tmp, source);
renameFile(tmp, path2);
std::filesystem::rename(tmp, path2);
del.cancel();
}
@ -83,7 +87,7 @@ protected:
{
StorePathSet paths;
for (auto & entry : readDirectory(binaryCacheDir)) {
for (auto & entry : std::filesystem::directory_iterator{binaryCacheDir}) {
auto name = entry.path().filename().string();
if (name.size() != 40 ||
!hasSuffix(name, ".narinfo"))

0
src/libstore/unix/local-overlay-store.cc → src/libstore/local-overlay-store.cc
View file

2
src/libstore/unix/local-overlay-store.hh → src/libstore/local-overlay-store.hh
View file

@ -92,7 +92,7 @@ class LocalOverlayStore : public virtual LocalOverlayStoreConfig, public virtual
public:
LocalOverlayStore(const Params & params);
LocalOverlayStore(std::string scheme, std::string path, const Params & params)
LocalOverlayStore(std::string_view scheme, PathView path, const Params & params)
: LocalOverlayStore(params)
{
if (!path.empty())

0
src/libstore/unix/local-overlay-store.md → src/libstore/local-overlay-store.md
View file

22
src/libstore/local-store.cc
View file

@ -463,10 +463,20 @@ LocalStore::LocalStore(const Params & params)
}
LocalStore::LocalStore(std::string scheme, std::string path, const Params & params)
: LocalStore(params)
LocalStore::LocalStore(
std::string_view scheme,
PathView path,
const Params & _params)
: LocalStore([&]{
// Default `?root` from `path` if non set
if (!path.empty() && _params.count("root") == 0) {
auto params = _params;
params.insert_or_assign("root", std::string { path });
return params;
}
return _params;
}())
{
throw UnimplementedError("LocalStore");
}
@ -1406,7 +1416,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
printInfo("checking link hashes...");
for (auto & link : readDirectory(linksDir)) {
for (auto & link : std::filesystem::directory_iterator{linksDir}) {
auto name = link.path().filename();
printMsg(lvlTalkative, "checking contents of '%s'", name);
PosixSourceAccessor accessor;
@ -1498,7 +1508,7 @@ LocalStore::VerificationResult LocalStore::verifyAllValidPaths(RepairFlag repair
database and the filesystem) in the loop below, in order to catch
invalid states.
*/
for (auto & i : readDirectory(realStoreDir)) {
for (auto & i : std::filesystem::directory_iterator{realStoreDir.to_string()}) {
try {
storePathsInStoreDir.insert({i.path().filename().string()});
} catch (BadStorePath &) { }
@ -1779,7 +1789,7 @@ void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log)
writeFile(tmpFile, compress("bzip2", log));
renameFile(tmpFile, logPath);
std::filesystem::rename(tmpFile, logPath);
}
std::optional<std::string> LocalStore::getVersion()

7
src/libstore/local-store.hh
View file

@ -137,12 +137,15 @@ public:
* necessary.
*/
LocalStore(const Params & params);
LocalStore(std::string scheme, std::string path, const Params & params);
LocalStore(
std::string_view scheme,
PathView path,
const Params & params);
~LocalStore();
static std::set<std::string> uriSchemes()
{ return {}; }
{ return {"local"}; }
/**
* Implementations of abstract store API methods.

6
src/libstore/local.mk
View file

@ -4,9 +4,9 @@ libstore_NAME = libnixstore
libstore_DIR := $(d)
libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc)
libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc)
ifdef HOST_UNIX
libstore_SOURCES += $(wildcard $(d)/unix/*.cc $(d)/unix/builtins/*.cc $(d)/unix/build/*.cc)
libstore_SOURCES += $(wildcard $(d)/unix/*.cc $(d)/unix/build/*.cc)
endif
ifdef HOST_LINUX
libstore_SOURCES += $(wildcard $(d)/linux/*.cc)
@ -43,7 +43,7 @@ endif
INCLUDE_libstore := -I $(d) -I $(d)/build
ifdef HOST_UNIX
INCLUDE_libstore += -I $(d)/unix
INCLUDE_libstore += -I $(d)/unix -I $(d)/unix/build
endif
ifdef HOST_LINUX
INCLUDE_libstore += -I $(d)/linux

66
src/libstore/machines.cc
View file

@ -6,7 +6,8 @@
namespace nix {
Machine::Machine(decltype(storeUri) storeUri,
Machine::Machine(
const std::string & storeUri,
decltype(systemTypes) systemTypes,
decltype(sshKey) sshKey,
decltype(maxJobs) maxJobs,
@ -14,7 +15,7 @@ Machine::Machine(decltype(storeUri) storeUri,
decltype(supportedFeatures) supportedFeatures,
decltype(mandatoryFeatures) mandatoryFeatures,
decltype(sshPublicHostKey) sshPublicHostKey) :
storeUri(
storeUri(StoreReference::parse(
// Backwards compatibility: if the URI is schemeless, is not a path,
// and is not one of the special store connection words, prepend
// ssh://.
@ -28,7 +29,7 @@ Machine::Machine(decltype(storeUri) storeUri,
|| hasPrefix(storeUri, "local?")
|| hasPrefix(storeUri, "?")
? storeUri
: "ssh://" + storeUri),
: "ssh://" + storeUri)),
systemTypes(systemTypes),
sshKey(sshKey),
maxJobs(maxJobs),
@ -63,23 +64,26 @@ bool Machine::mandatoryMet(const std::set<std::string> & features) const
});
}
ref<Store> Machine::openStore() const
StoreReference Machine::completeStoreReference() const
{
Store::Params storeParams;
if (hasPrefix(storeUri, "ssh://")) {
storeParams["max-connections"] = "1";
storeParams["log-fd"] = "4";
auto storeUri = this->storeUri;
auto * generic = std::get_if<StoreReference::Specified>(&storeUri.variant);
if (generic && generic->scheme == "ssh") {
storeUri.params["max-connections"] = "1";
storeUri.params["log-fd"] = "4";
}
if (hasPrefix(storeUri, "ssh://") || hasPrefix(storeUri, "ssh-ng://")) {
if (generic && (generic->scheme == "ssh" || generic->scheme == "ssh-ng")) {
if (sshKey != "")
storeParams["ssh-key"] = sshKey;
storeUri.params["ssh-key"] = sshKey;
if (sshPublicHostKey != "")
storeParams["base64-ssh-public-host-key"] = sshPublicHostKey;
storeUri.params["base64-ssh-public-host-key"] = sshPublicHostKey;
}
{
auto & fs = storeParams["system-features"];
auto & fs = storeUri.params["system-features"];
auto append = [&](auto feats) {
for (auto & f : feats) {
if (fs.size() > 0) fs += ' ';
@ -90,7 +94,12 @@ ref<Store> Machine::openStore() const
append(mandatoryFeatures);
}
return nix::openStore(storeUri, storeParams);
return storeUri;
}
ref<Store> Machine::openStore() const
{
return nix::openStore(completeStoreReference());
}
static std::vector<std::string> expandBuilderLines(const std::string & builders)
@ -122,7 +131,7 @@ static std::vector<std::string> expandBuilderLines(const std::string & builders)
return result;
}
static Machine parseBuilderLine(const std::string & line)
static Machine parseBuilderLine(const std::set<std::string> & defaultSystems, const std::string & line)
{
const auto tokens = tokenizeString<std::vector<std::string>>(line);
@ -139,7 +148,7 @@ static Machine parseBuilderLine(const std::string & line)
};
auto parseFloatField = [&](size_t fieldIndex) {
const auto result = string2Int<float>(tokens[fieldIndex]);
const auto result = string2Float<float>(tokens[fieldIndex]);
if (!result) {
throw FormatError("bad machine specification: failed to convert column #%lu in a row: '%s' to 'float'", fieldIndex, line);
}
@ -159,29 +168,46 @@ static Machine parseBuilderLine(const std::string & line)
if (!isSet(0))
throw FormatError("bad machine specification: store URL was not found at the first column of a row: '%s'", line);
// TODO use designated initializers, once C++ supports those with
// custom constructors.
return {
// `storeUri`
tokens[0],
isSet(1) ? tokenizeString<std::set<std::string>>(tokens[1], ",") : std::set<std::string>{settings.thisSystem},
// `systemTypes`
isSet(1) ? tokenizeString<std::set<std::string>>(tokens[1], ",") : defaultSystems,
// `sshKey`
isSet(2) ? tokens[2] : "",
// `maxJobs`
isSet(3) ? parseUnsignedIntField(3) : 1U,
// `speedFactor`
isSet(4) ? parseFloatField(4) : 1.0f,
// `supportedFeatures`
isSet(5) ? tokenizeString<std::set<std::string>>(tokens[5], ",") : std::set<std::string>{},
// `mandatoryFeatures`
isSet(6) ? tokenizeString<std::set<std::string>>(tokens[6], ",") : std::set<std::string>{},
// `sshPublicHostKey`
isSet(7) ? ensureBase64(7) : ""
};
}
static Machines parseBuilderLines(const std::vector<std::string> & builders)
static Machines parseBuilderLines(const std::set<std::string> & defaultSystems, const std::vector<std::string> & builders)
{
Machines result;
std::transform(builders.begin(), builders.end(), std::back_inserter(result), parseBuilderLine);
std::transform(
builders.begin(), builders.end(), std::back_inserter(result),
[&](auto && line) { return parseBuilderLine(defaultSystems, line); });
return result;
}
Machines Machine::parseConfig(const std::set<std::string> & defaultSystems, const std::string & s)
{
const auto builderLines = expandBuilderLines(s);
return parseBuilderLines(defaultSystems, builderLines);
}
Machines getMachines()
{
const auto builderLines = expandBuilderLines(settings.builders);
return parseBuilderLines(builderLines);
return Machine::parseConfig({settings.thisSystem}, settings.builders);
}
}

43
src/libstore/machines.hh
View file

@ -2,14 +2,19 @@
///@file
#include "types.hh"
#include "store-reference.hh"
namespace nix {
class Store;
struct Machine;
typedef std::vector<Machine> Machines;
struct Machine {
const std::string storeUri;
const StoreReference storeUri;
const std::set<std::string> systemTypes;
const std::string sshKey;
const unsigned int maxJobs;
@ -36,7 +41,8 @@ struct Machine {
*/
bool mandatoryMet(const std::set<std::string> & features) const;
Machine(decltype(storeUri) storeUri,
Machine(
const std::string & storeUri,
decltype(systemTypes) systemTypes,
decltype(sshKey) sshKey,
decltype(maxJobs) maxJobs,
@ -45,13 +51,38 @@ struct Machine {
decltype(mandatoryFeatures) mandatoryFeatures,
decltype(sshPublicHostKey) sshPublicHostKey);
/**
* Elaborate `storeUri` into a complete store reference,
* incorporating information from the other fields of the `Machine`
* as applicable.
*/
StoreReference completeStoreReference() const;
/**
* Open a `Store` for this machine.
*
* Just a simple function composition:
* ```c++
* nix::openStore(completeStoreReference())
* ```
*/
ref<Store> openStore() const;
/**
* Parse a machine configuration.
*
* Every machine is specified on its own line, and lines beginning
* with `@` are interpreted as paths to other configuration files in
* the same format.
*/
static Machines parseConfig(const std::set<std::string> & defaultSystems, const std::string & config);
};
typedef std::vector<Machine> Machines;
void parseMachines(const std::string & s, Machines & machines);
/**
* Parse machines from the global config
*
* @todo Remove, globals are bad.
*/
Machines getMachines();
}

25
src/libstore/parsed-derivations.cc
View file

@ -135,18 +135,37 @@ static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
/**
* Write a JSON representation of store object metadata, such as the
* hash and the references.
*
* @note Do *not* use `ValidPathInfo::toJSON` because this function is
* subject to stronger stability requirements since it is used to
* prepare build environments. Perhaps someday we'll have a versionining
* mechanism to allow this to evolve again and get back in sync, but for
* now we must not change - not even extend - the behavior.
*/
static nlohmann::json pathInfoToJSON(
Store & store,
const StorePathSet & storePaths)
{
nlohmann::json::array_t jsonList = nlohmann::json::array();
using nlohmann::json;
nlohmann::json::array_t jsonList = json::array();
for (auto & storePath : storePaths) {
auto info = store.queryPathInfo(storePath);
auto & jsonPath = jsonList.emplace_back(
info->toJSON(store, false, HashFormat::Nix32));
auto & jsonPath = jsonList.emplace_back(json::object());
jsonPath["narHash"] = info->narHash.to_string(HashFormat::Nix32, true);
jsonPath["narSize"] = info->narSize;
{
auto & jsonRefs = jsonPath["references"] = json::array();
for (auto & ref : info->references)
jsonRefs.emplace_back(store.printStorePath(ref));
}
if (info->ca)
jsonPath["ca"] = renderContentAddress(info->ca);
// Add the path to the object whose metadata we are including.
jsonPath["path"] = store.printStorePath(storePath);

34
src/libstore/path-info.cc
View file

@ -161,28 +161,23 @@ nlohmann::json UnkeyedValidPathInfo::toJSON(
jsonObject["narSize"] = narSize;
{
auto& jsonRefs = (jsonObject["references"] = json::array());
auto & jsonRefs = jsonObject["references"] = json::array();
for (auto & ref : references)
jsonRefs.emplace_back(store.printStorePath(ref));
}
if (ca)
jsonObject["ca"] = renderContentAddress(ca);
jsonObject["ca"] = ca ? (std::optional { renderContentAddress(*ca) }) : std::nullopt;
if (includeImpureInfo) {
if (deriver)
jsonObject["deriver"] = store.printStorePath(*deriver);
jsonObject["deriver"] = deriver ? (std::optional { store.printStorePath(*deriver) }) : std::nullopt;
if (registrationTime)
jsonObject["registrationTime"] = registrationTime;
jsonObject["registrationTime"] = registrationTime ? (std::optional { registrationTime }) : std::nullopt;
if (ultimate)
jsonObject["ultimate"] = ultimate;
jsonObject["ultimate"] = ultimate;
if (!sigs.empty()) {
for (auto & sig : sigs)
jsonObject["signatures"].push_back(sig);
}
auto & sigsObj = jsonObject["signatures"] = json::array();
for (auto & sig : sigs)
sigsObj.push_back(sig);
}
return jsonObject;
@ -210,20 +205,25 @@ UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON(
throw;
}
// New format as this as nullable but mandatory field; handling
// missing is for back-compat.
if (json.contains("ca"))
res.ca = ContentAddress::parse(getString(valueAt(json, "ca")));
if (auto * rawCa = getNullable(valueAt(json, "ca")))
res.ca = ContentAddress::parse(getString(*rawCa));
if (json.contains("deriver"))
res.deriver = store.parseStorePath(getString(valueAt(json, "deriver")));
if (auto * rawDeriver = getNullable(valueAt(json, "deriver")))
res.deriver = store.parseStorePath(getString(*rawDeriver));
if (json.contains("registrationTime"))
res.registrationTime = getInteger(valueAt(json, "registrationTime"));
if (auto * rawRegistrationTime = getNullable(valueAt(json, "registrationTime")))
res.registrationTime = getInteger(*rawRegistrationTime);
if (json.contains("ultimate"))
res.ultimate = getBoolean(valueAt(json, "ultimate"));
if (json.contains("signatures"))
res.sigs = valueAt(json, "signatures");
res.sigs = getStringSet(valueAt(json, "signatures"));
return res;
}

8
src/libstore/path.cc
View file

@ -49,11 +49,17 @@ StorePath::StorePath(const Hash & hash, std::string_view _name)
checkName(baseName, name());
}
bool StorePath::isDerivation() const
bool StorePath::isDerivation() const noexcept
{
return hasSuffix(name(), drvExtension);
}
void StorePath::requireDerivation() const
{
if (!isDerivation())
throw FormatError("store path '%s' is not a valid derivation path", to_string());
}
StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x");
StorePath StorePath::random(std::string_view name)

29
src/libstore/path.hh
View file

@ -13,7 +13,7 @@ struct Hash;
* \ref StorePath "Store path" is the fundamental reference type of Nix.
* A store paths refers to a Store object.
*
* See glossary.html#gloss-store-path for more information on a
* See store/store-path.html for more information on a
* conceptual level.
*/
class StorePath
@ -35,30 +35,23 @@ public:
StorePath(const Hash & hash, std::string_view name);
std::string_view to_string() const
std::string_view to_string() const noexcept
{
return baseName;
}
bool operator < (const StorePath & other) const
{
return baseName < other.baseName;
}
bool operator == (const StorePath & other) const
{
return baseName == other.baseName;
}
bool operator != (const StorePath & other) const
{
return baseName != other.baseName;
}
bool operator == (const StorePath & other) const noexcept = default;
auto operator <=> (const StorePath & other) const noexcept = default;
/**
* Check whether a file name ends with the extension for derivations.
*/
bool isDerivation() const;
bool isDerivation() const noexcept;
/**
* Throw an exception if `isDerivation` is false.
*/
void requireDerivation() const;
std::string_view name() const
{
@ -82,7 +75,7 @@ typedef std::vector<StorePath> StorePaths;
* The file extension of \ref Derivation derivations when serialized
* into store objects.
*/
const std::string drvExtension = ".drv";
constexpr std::string_view drvExtension = ".drv";
}

3
src/libstore/posix-fs-canonicalise.cc
View file

@ -144,8 +144,7 @@ static void canonicalisePathMetaData_(
#endif
if (S_ISDIR(st.st_mode)) {
std::vector<std::filesystem::directory_entry> entries = readDirectory(path);
for (auto & i : entries)
for (auto & i : std::filesystem::directory_iterator{path})
canonicalisePathMetaData_(
i.path().string(),
#ifndef _WIN32

2
src/libstore/profiles.cc
View file

@ -37,7 +37,7 @@ std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path pro
std::filesystem::path profileDir = dirOf(profile);
auto profileName = std::string(baseNameOf(profile));
for (auto & i : readDirectory(profileDir.string())) {
for (auto & i : std::filesystem::directory_iterator{profileDir}) {
if (auto n = parseName(profileName, i.path().filename().string())) {
auto path = i.path().string();
gens.push_back({

82
src/libstore/remote-store-connection.hh
View file

@ -3,6 +3,7 @@
#include "remote-store.hh"
#include "worker-protocol.hh"
#include "worker-protocol-connection.hh"
#include "pool.hh"
namespace nix {
@ -14,90 +15,13 @@ namespace nix {
* Contains `Source` and `Sink` for actual communication, along with
* other information learned when negotiating the connection.
*/
struct RemoteStore::Connection
struct RemoteStore::Connection : WorkerProto::BasicClientConnection,
WorkerProto::ClientHandshakeInfo
{
/**
* Send with this.
*/
FdSink to;
/**
* Receive with this.
*/
FdSource from;
/**
* Worker protocol version used for the connection.
*
* Despite its name, I think it is actually the maximum version both
* sides support. (If the maximum doesn't exist, we would fail to
* establish a connection and produce a value of this type.)
*/
WorkerProto::Version daemonVersion;
/**
* Whether the remote side trusts us or not.
*
* 3 values: "yes", "no", or `std::nullopt` for "unknown".
*
* Note that the "remote side" might not be just the end daemon, but
* also an intermediary forwarder that can make its own trusting
* decisions. This would be the intersection of all their trust
* decisions, since it takes only one link in the chain to start
* denying operations.
*/
std::optional<TrustedFlag> remoteTrustsUs;
/**
* The version of the Nix daemon that is processing our requests.
*
* Do note, it may or may not communicating with another daemon,
* rather than being an "end" `LocalStore` or similar.
*/
std::optional<std::string> daemonNixVersion;
/**
* Time this connection was established.
*/
std::chrono::time_point<std::chrono::steady_clock> startTime;
/**
* Coercion to `WorkerProto::ReadConn`. This makes it easy to use the
* factored out worker protocol searlizers with a
* `RemoteStore::Connection`.
*
* The worker protocol connection types are unidirectional, unlike
* this type.
*/
operator WorkerProto::ReadConn ()
{
return WorkerProto::ReadConn {
.from = from,
.version = daemonVersion,
};
}
/**
* Coercion to `WorkerProto::WriteConn`. This makes it easy to use the
* factored out worker protocol searlizers with a
* `RemoteStore::Connection`.
*
* The worker protocol connection types are unidirectional, unlike
* this type.
*/
operator WorkerProto::WriteConn ()
{
return WorkerProto::WriteConn {
.to = to,
.version = daemonVersion,
};
}
virtual ~Connection();
virtual void closeWrite() = 0;
std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true);
};
/**

214
src/libstore/remote-store.cc
View file

@ -69,50 +69,26 @@ void RemoteStore::initConnection(Connection & conn)
/* Send the magic greeting, check for the reply. */
try {
conn.from.endOfFileError = "Nix daemon disconnected unexpectedly (maybe it crashed?)";
conn.to << WORKER_MAGIC_1;
conn.to.flush();
StringSink saved;
TeeSource tee(conn.from, saved);
try {
TeeSource tee(conn.from, saved);
unsigned int magic = readInt(tee);
if (magic != WORKER_MAGIC_2)
throw Error("protocol mismatch");
conn.daemonVersion = WorkerProto::BasicClientConnection::handshake(
conn.to, tee, PROTOCOL_VERSION);
} catch (SerialisationError & e) {
/* In case the other side is waiting for our input, close
it. */
conn.closeWrite();
auto msg = conn.from.drain();
throw Error("protocol mismatch, got '%s'", chomp(saved.s + msg));
{
NullSink nullSink;
tee.drainInto(nullSink);
}
throw Error("protocol mismatch, got '%s'", chomp(saved.s));
}
conn.from >> conn.daemonVersion;
if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION))
throw Error("Nix daemon protocol version not supported");
if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10)
throw Error("the Nix daemon version is too old");
conn.to << PROTOCOL_VERSION;
static_cast<WorkerProto::ClientHandshakeInfo &>(conn) = conn.postHandshake(*this);
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 14) {
// Obsolete CPU affinity.
conn.to << 0;
}
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11)
conn.to << false; // obsolete reserveSpace
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 33) {
conn.to.flush();
conn.daemonNixVersion = readString(conn.from);
}
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 35) {
conn.remoteTrustsUs = WorkerProto::Serialise<std::optional<TrustedFlag>>::read(*this, conn);
} else {
// We don't know the answer; protocol to old.
conn.remoteTrustsUs = std::nullopt;
}
auto ex = conn.processStderr();
auto ex = conn.processStderrReturn();
if (ex) std::rethrow_exception(ex);
}
catch (Error & e) {
@ -158,7 +134,7 @@ void RemoteStore::setOptions(Connection & conn)
conn.to << i.first << i.second.value;
}
auto ex = conn.processStderr();
auto ex = conn.processStderrReturn();
if (ex) std::rethrow_exception(ex);
}
@ -173,28 +149,7 @@ RemoteStore::ConnectionHandle::~ConnectionHandle()
void RemoteStore::ConnectionHandle::processStderr(Sink * sink, Source * source, bool flush)
{
auto ex = handle->processStderr(sink, source, flush);
if (ex) {
daemonException = true;
try {
std::rethrow_exception(ex);
} catch (const Error & e) {
// Nix versions before #4628 did not have an adequate behavior for reporting that the derivation format was upgraded.
// To avoid having to add compatibility logic in many places, we expect to catch almost all occurrences of the
// old incomprehensible error here, so that we can explain to users what's going on when their daemon is
// older than #4628 (2023).
if (experimentalFeatureSettings.isEnabled(Xp::DynamicDerivations) &&
GET_PROTOCOL_MINOR(handle->daemonVersion) <= 35)
{
auto m = e.msg();
if (m.find("parsing derivation") != std::string::npos &&
m.find("expected string") != std::string::npos &&
m.find("Derive([") != std::string::npos)
throw Error("%s, this might be because the daemon is too old to understand dependencies on dynamic derivations. Check to see if the raw derivation is in the form '%s'", std::move(m), "DrvWithVersion(..)");
}
throw;
}
}
handle->processStderr(&daemonException, sink, source, flush);
}
@ -226,13 +181,7 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute
if (isValidPath(i)) res.insert(i);
return res;
} else {
conn->to << WorkerProto::Op::QueryValidPaths;
WorkerProto::write(*this, *conn, paths);
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) {
conn->to << maybeSubstitute;
}
conn.processStderr();
return WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
return conn->queryValidPaths(*this, &conn.daemonException, paths, maybeSubstitute);
}
}
@ -322,22 +271,10 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path,
std::shared_ptr<const ValidPathInfo> info;
{
auto conn(getConnection());
conn->to << WorkerProto::Op::QueryPathInfo << printStorePath(path);
try {
conn.processStderr();
} catch (Error & e) {
// Ugly backwards compatibility hack.
if (e.msg().find("is not valid") != std::string::npos)
throw InvalidPath(std::move(e.info()));
throw;
}
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) {
bool valid; conn->from >> valid;
if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path));
}
info = std::make_shared<ValidPathInfo>(
StorePath{path},
WorkerProto::Serialise<UnkeyedValidPathInfo>::read(*this, *conn));
conn->queryPathInfo(*this, &conn.daemonException, path));
}
callback(std::move(info));
} catch (...) { callback.rethrow(); }
@ -542,8 +479,6 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
auto conn(getConnection());
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) {
conn->to << WorkerProto::Op::ImportPaths;
auto source2 = sinkToSource([&](Sink & sink) {
sink << 1 // == path follows
;
@ -558,11 +493,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
<< 0 // == no path follows
;
});
conn.processStderr(0, source2.get());
auto importedPaths = WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
assert(importedPaths.size() <= 1);
conn->importPaths(*this, &conn.daemonException, *source2);
}
else {
@ -807,9 +738,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD
BuildMode buildMode)
{
auto conn(getConnection());
conn->to << WorkerProto::Op::BuildDerivation << printStorePath(drvPath);
writeDerivation(conn->to, *this, drv);
conn->to << buildMode;
conn->putBuildDerivationRequest(*this, &conn.daemonException, drvPath, drv, buildMode);
conn.processStderr();
return WorkerProto::Serialise<BuildResult>::read(*this, *conn);
}
@ -827,9 +756,7 @@ void RemoteStore::ensurePath(const StorePath & path)
void RemoteStore::addTempRoot(const StorePath & path)
{
auto conn(getConnection());
conn->to << WorkerProto::Op::AddTempRoot << printStorePath(path);
conn.processStderr();
readInt(conn->from);
conn->addTempRoot(*this, &conn.daemonException, path);
}
@ -969,22 +896,12 @@ void RemoteStore::flushBadConnections()
connections->flushBad();
}
RemoteStore::Connection::~Connection()
{
try {
to.flush();
} catch (...) {
ignoreException();
}
}
void RemoteStore::narFromPath(const StorePath & path, Sink & sink)
{
auto conn(connections->get());
conn->to << WorkerProto::Op::NarFromPath << printStorePath(path);
conn->processStderr();
copyNAR(conn->from, sink);
auto conn(getConnection());
conn->narFromPath(*this, &conn.daemonException, path, [&](Source & source) {
copyNAR(conn->from, sink);
});
}
ref<SourceAccessor> RemoteStore::getFSAccessor(bool requireValidPath)
@ -992,91 +909,6 @@ ref<SourceAccessor> RemoteStore::getFSAccessor(bool requireValidPath)
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()));
}
static Logger::Fields readFields(Source & from)
{
Logger::Fields fields;
size_t size = readInt(from);
for (size_t n = 0; n < size; n++) {
auto type = (decltype(Logger::Field::type)) readInt(from);
if (type == Logger::Field::tInt)
fields.push_back(readNum<uint64_t>(from));
else if (type == Logger::Field::tString)
fields.push_back(readString(from));
else
throw Error("got unsupported field type %x from Nix daemon", (int) type);
}
return fields;
}
std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source, bool flush)
{
if (flush)
to.flush();
while (true) {
auto msg = readNum<uint64_t>(from);
if (msg == STDERR_WRITE) {
auto s = readString(from);
if (!sink) throw Error("no sink");
(*sink)(s);
}
else if (msg == STDERR_READ) {
if (!source) throw Error("no source");
size_t len = readNum<size_t>(from);
auto buf = std::make_unique<char[]>(len);
writeString({(const char *) buf.get(), source->read(buf.get(), len)}, to);
to.flush();
}
else if (msg == STDERR_ERROR) {
if (GET_PROTOCOL_MINOR(daemonVersion) >= 26) {
return std::make_exception_ptr(readError(from));
} else {
auto error = readString(from);
unsigned int status = readInt(from);
return std::make_exception_ptr(Error(status, error));
}
}
else if (msg == STDERR_NEXT)
printError(chomp(readString(from)));
else if (msg == STDERR_START_ACTIVITY) {
auto act = readNum<ActivityId>(from);
auto lvl = (Verbosity) readInt(from);
auto type = (ActivityType) readInt(from);
auto s = readString(from);
auto fields = readFields(from);
auto parent = readNum<ActivityId>(from);
logger->startActivity(act, lvl, type, s, fields, parent);
}
else if (msg == STDERR_STOP_ACTIVITY) {
auto act = readNum<ActivityId>(from);
logger->stopActivity(act);
}
else if (msg == STDERR_RESULT) {
auto act = readNum<ActivityId>(from);
auto type = (ResultType) readInt(from);
auto fields = readFields(from);
logger->result(act, type, fields);
}
else if (msg == STDERR_LAST)
break;
else
throw Error("got unknown message type %x from Nix daemon", msg);
}
return nullptr;
}
void RemoteStore::ConnectionHandle::withFramedSink(std::function<void(Sink & sink)> fun)
{
(*this)->to.flush();

10
src/libstore/s3-binary-cache-store.cc
View file

@ -213,7 +213,7 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
support it.
> **Note**
>
>
> HTTPS should be used if the cache might contain sensitive
> information.
)"};
@ -224,7 +224,7 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
Do not specify this setting if you're using Amazon S3.
> **Note**
>
>
> This endpoint must support HTTPS and will use path-based
> addressing instead of virtual host based addressing.
)"};
@ -269,8 +269,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
S3Helper s3Helper;
S3BinaryCacheStoreImpl(
const std::string & uriScheme,
const std::string & bucketName,
std::string_view uriScheme,
std::string_view bucketName,
const Params & params)
: StoreConfig(params)
, BinaryCacheStoreConfig(params)
@ -281,6 +281,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
, bucketName(bucketName)
, s3Helper(profile, region, scheme, endpoint)
{
if (bucketName.empty())
throw UsageError("`%s` store requires a bucket name in its Store URI", uriScheme);
diskCache = getNarInfoDiskCache();
}

106
src/libstore/serve-protocol-connection.cc Normal file
View file

@ -0,0 +1,106 @@
#include "serve-protocol-connection.hh"
#include "serve-protocol-impl.hh"
#include "build-result.hh"
#include "derivations.hh"
namespace nix {
ServeProto::Version ServeProto::BasicClientConnection::handshake(
BufferedSink & to, Source & from, ServeProto::Version localVersion, std::string_view host)
{
to << SERVE_MAGIC_1 << localVersion;
to.flush();
unsigned int magic = readInt(from);
if (magic != SERVE_MAGIC_2)
throw Error("'nix-store --serve' protocol mismatch from '%s'", host);
auto remoteVersion = readInt(from);
if (GET_PROTOCOL_MAJOR(remoteVersion) != 0x200)
throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host);
return std::min(remoteVersion, localVersion);
}
ServeProto::Version
ServeProto::BasicServerConnection::handshake(BufferedSink & to, Source & from, ServeProto::Version localVersion)
{
unsigned int magic = readInt(from);
if (magic != SERVE_MAGIC_1)
throw Error("protocol mismatch");
to << SERVE_MAGIC_2 << localVersion;
to.flush();
auto remoteVersion = readInt(from);
return std::min(remoteVersion, localVersion);
}
StorePathSet ServeProto::BasicClientConnection::queryValidPaths(
const StoreDirConfig & store, bool lock, const StorePathSet & paths, SubstituteFlag maybeSubstitute)
{
to << ServeProto::Command::QueryValidPaths << lock << maybeSubstitute;
write(store, *this, paths);
to.flush();
return Serialise<StorePathSet>::read(store, *this);
}
std::map<StorePath, UnkeyedValidPathInfo>
ServeProto::BasicClientConnection::queryPathInfos(const StoreDirConfig & store, const StorePathSet & paths)
{
std::map<StorePath, UnkeyedValidPathInfo> infos;
to << ServeProto::Command::QueryPathInfos;
ServeProto::write(store, *this, paths);
to.flush();
while (true) {
auto storePathS = readString(from);
if (storePathS == "")
break;
auto storePath = store.parseStorePath(storePathS);
assert(paths.count(storePath) == 1);
auto info = ServeProto::Serialise<UnkeyedValidPathInfo>::read(store, *this);
infos.insert_or_assign(std::move(storePath), std::move(info));
}
return infos;
}
void ServeProto::BasicClientConnection::putBuildDerivationRequest(
const StoreDirConfig & store,
const StorePath & drvPath,
const BasicDerivation & drv,
const ServeProto::BuildOptions & options)
{
to << ServeProto::Command::BuildDerivation << store.printStorePath(drvPath);
writeDerivation(to, store, drv);
ServeProto::write(store, *this, options);
to.flush();
}
BuildResult ServeProto::BasicClientConnection::getBuildDerivationResponse(const StoreDirConfig & store)
{
return ServeProto::Serialise<BuildResult>::read(store, *this);
}
void ServeProto::BasicClientConnection::narFromPath(
const StoreDirConfig & store, const StorePath & path, std::function<void(Source &)> fun)
{
to << ServeProto::Command::DumpStorePath << store.printStorePath(path);
to.flush();
fun(from);
}
void ServeProto::BasicClientConnection::importPaths(const StoreDirConfig & store, std::function<void(Sink &)> fun)
{
to << ServeProto::Command::ImportPaths;
fun(to);
to.flush();
if (readInt(from) != 1)
throw Error("remote machine failed to import closure");
}
}

108
src/libstore/serve-protocol-connection.hh Normal file
View file

@ -0,0 +1,108 @@
#pragma once
///@file
#include "serve-protocol.hh"
#include "store-api.hh"
namespace nix {
struct ServeProto::BasicClientConnection
{
FdSink to;
FdSource from;
ServeProto::Version remoteVersion;
/**
* Establishes connection, negotiating version.
*
* @return the version provided by the other side of the
* connection.
*
* @param to Taken by reference to allow for various error handling
* mechanisms.
*
* @param from Taken by reference to allow for various error
* handling mechanisms.
*
* @param localVersion Our version which is sent over
*
* @param host Just used to add context to thrown exceptions.
*/
static ServeProto::Version
handshake(BufferedSink & to, Source & from, ServeProto::Version localVersion, std::string_view host);
/**
* Coercion to `ServeProto::ReadConn`. This makes it easy to use the
* factored out serve protocol serializers with a
* `LegacySSHStore::Connection`.
*
* The serve protocol connection types are unidirectional, unlike
* this type.
*/
operator ServeProto::ReadConn()
{
return ServeProto::ReadConn{
.from = from,
.version = remoteVersion,
};
}
/**
* Coercion to `ServeProto::WriteConn`. This makes it easy to use the
* factored out serve protocol serializers with a
* `LegacySSHStore::Connection`.
*
* The serve protocol connection types are unidirectional, unlike
* this type.
*/
operator ServeProto::WriteConn()
{
return ServeProto::WriteConn{
.to = to,
.version = remoteVersion,
};
}
StorePathSet queryValidPaths(
const StoreDirConfig & remoteStore, bool lock, const StorePathSet & paths, SubstituteFlag maybeSubstitute);
std::map<StorePath, UnkeyedValidPathInfo> queryPathInfos(const StoreDirConfig & store, const StorePathSet & paths);
;
void putBuildDerivationRequest(
const StoreDirConfig & store,
const StorePath & drvPath,
const BasicDerivation & drv,
const ServeProto::BuildOptions & options);
/**
* Get the response, must be paired with
* `putBuildDerivationRequest`.
*/
BuildResult getBuildDerivationResponse(const StoreDirConfig & store);
void narFromPath(const StoreDirConfig & store, const StorePath & path, std::function<void(Source &)> fun);
void importPaths(const StoreDirConfig & store, std::function<void(Sink &)> fun);
};
struct ServeProto::BasicServerConnection
{
/**
* Establishes connection, negotiating version.
*
* @return the version provided by the other side of the
* connection.
*
* @param to Taken by reference to allow for various error handling
* mechanisms.
*
* @param from Taken by reference to allow for various error
* handling mechanisms.
*
* @param localVersion Our version which is sent over
*/
static ServeProto::Version handshake(BufferedSink & to, Source & from, ServeProto::Version localVersion);
};
}

69
src/libstore/serve-protocol-impl.cc
View file

@ -1,69 +0,0 @@
#include "serve-protocol-impl.hh"
#include "build-result.hh"
#include "derivations.hh"
namespace nix {
ServeProto::Version ServeProto::BasicClientConnection::handshake(
BufferedSink & to,
Source & from,
ServeProto::Version localVersion,
std::string_view host)
{
to << SERVE_MAGIC_1 << localVersion;
to.flush();
unsigned int magic = readInt(from);
if (magic != SERVE_MAGIC_2)
throw Error("'nix-store --serve' protocol mismatch from '%s'", host);
auto remoteVersion = readInt(from);
if (GET_PROTOCOL_MAJOR(remoteVersion) != 0x200)
throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host);
return remoteVersion;
}
ServeProto::Version ServeProto::BasicServerConnection::handshake(
BufferedSink & to,
Source & from,
ServeProto::Version localVersion)
{
unsigned int magic = readInt(from);
if (magic != SERVE_MAGIC_1) throw Error("protocol mismatch");
to << SERVE_MAGIC_2 << localVersion;
to.flush();
return readInt(from);
}
StorePathSet ServeProto::BasicClientConnection::queryValidPaths(
const Store & store,
bool lock, const StorePathSet & paths,
SubstituteFlag maybeSubstitute)
{
to
<< ServeProto::Command::QueryValidPaths
<< lock
<< maybeSubstitute;
write(store, *this, paths);
to.flush();
return Serialise<StorePathSet>::read(store, *this);
}
void ServeProto::BasicClientConnection::putBuildDerivationRequest(
const Store & store,
const StorePath & drvPath, const BasicDerivation & drv,
const ServeProto::BuildOptions & options)
{
to
<< ServeProto::Command::BuildDerivation
<< store.printStorePath(drvPath);
writeDerivation(to, store, drv);
ServeProto::write(store, *this, options);
to.flush();
}
}

97
src/libstore/serve-protocol-impl.hh
View file

@ -57,101 +57,4 @@ struct ServeProto::Serialise
/* protocol-specific templates */
struct ServeProto::BasicClientConnection
{
FdSink to;
FdSource from;
ServeProto::Version remoteVersion;
/**
* Establishes connection, negotiating version.
*
* @return the version provided by the other side of the
* connection.
*
* @param to Taken by reference to allow for various error handling
* mechanisms.
*
* @param from Taken by reference to allow for various error
* handling mechanisms.
*
* @param localVersion Our version which is sent over
*
* @param host Just used to add context to thrown exceptions.
*/
static ServeProto::Version handshake(
BufferedSink & to,
Source & from,
ServeProto::Version localVersion,
std::string_view host);
/**
* Coercion to `ServeProto::ReadConn`. This makes it easy to use the
* factored out serve protocol serializers with a
* `LegacySSHStore::Connection`.
*
* The serve protocol connection types are unidirectional, unlike
* this type.
*/
operator ServeProto::ReadConn ()
{
return ServeProto::ReadConn {
.from = from,
.version = remoteVersion,
};
}
/**
* Coercion to `ServeProto::WriteConn`. This makes it easy to use the
* factored out serve protocol serializers with a
* `LegacySSHStore::Connection`.
*
* The serve protocol connection types are unidirectional, unlike
* this type.
*/
operator ServeProto::WriteConn ()
{
return ServeProto::WriteConn {
.to = to,
.version = remoteVersion,
};
}
StorePathSet queryValidPaths(
const Store & remoteStore,
bool lock, const StorePathSet & paths,
SubstituteFlag maybeSubstitute);
/**
* Just the request half, because Hydra may do other things between
* issuing the request and reading the `BuildResult` response.
*/
void putBuildDerivationRequest(
const Store & store,
const StorePath & drvPath, const BasicDerivation & drv,
const ServeProto::BuildOptions & options);
};
struct ServeProto::BasicServerConnection
{
/**
* Establishes connection, negotiating version.
*
* @return the version provided by the other side of the
* connection.
*
* @param to Taken by reference to allow for various error handling
* mechanisms.
*
* @param from Taken by reference to allow for various error
* handling mechanisms.
*
* @param localVersion Our version which is sent over
*/
static ServeProto::Version handshake(
BufferedSink & to,
Source & from,
ServeProto::Version localVersion);
};
}

43
src/libstore/ssh-store-config.cc Normal file
View file

@ -0,0 +1,43 @@
#include <regex>
#include "ssh-store-config.hh"
#include "ssh.hh"
namespace nix {
static std::string extractConnStr(std::string_view scheme, std::string_view _connStr)
{
if (_connStr.empty())
throw UsageError("`%s` store requires a valid SSH host as the authority part in Store URI", scheme);
std::string connStr{_connStr};
std::smatch result;
static std::regex v6AddrRegex("^((.*)@)?\\[(.*)\\]$");
if (std::regex_match(connStr, result, v6AddrRegex)) {
connStr = result[1].matched ? result.str(1) + result.str(3) : result.str(3);
}
return connStr;
}
CommonSSHStoreConfig::CommonSSHStoreConfig(std::string_view scheme, std::string_view host, const Params & params)
: StoreConfig(params)
, host(extractConnStr(scheme, host))
{
}
SSHMaster CommonSSHStoreConfig::createSSHMaster(bool useMaster, Descriptor logFD)
{
return {
host,
sshKey.get(),
sshPublicHostKey.get(),
useMaster,
compress,
logFD,
};
}
}

33
src/libstore/ssh-store-config.hh
View file

@ -5,10 +5,14 @@
namespace nix {
class SSHMaster;
struct CommonSSHStoreConfig : virtual StoreConfig
{
using StoreConfig::StoreConfig;
CommonSSHStoreConfig(std::string_view scheme, std::string_view host, const Params & params);
const Setting<Path> sshKey{this, "", "ssh-key",
"Path to the SSH private key used to authenticate to the remote machine."};
@ -24,6 +28,35 @@ struct CommonSSHStoreConfig : virtual StoreConfig
to be used on the remote machine. The default is `auto`
(i.e. use the Nix daemon or `/nix/store` directly).
)"};
/**
* The `parseURL` function supports both IPv6 URIs as defined in
* RFC2732, but also pure addresses. The latter one is needed here to
* connect to a remote store via SSH (it's possible to do e.g. `ssh root@::1`).
*
* When initialized, the following adjustments are made:
*
* - If the URL looks like `root@[::1]` (which is allowed by the URL parser and probably
* needed to pass further flags), it
* will be transformed into `root@::1` for SSH (same for `[::1]` -> `::1`).
*
* - If the URL looks like `root@::1` it will be left as-is.
*
* - In any other case, the string will be left as-is.
*
* Will throw an error if `connStr` is empty too.
*/
std::string host;
/**
* Small wrapper around `SSHMaster::SSHMaster` that gets most
* arguments from this configuration.
*
* See that constructor for details on the remaining two arguments.
*/
SSHMaster createSSHMaster(
bool useMaster,
Descriptor logFD = INVALID_DESCRIPTOR);
};
}

32
src/libstore/ssh-store.cc
View file

@ -34,21 +34,19 @@ class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore
{
public:
SSHStore(const std::string & scheme, const std::string & host, const Params & params)
SSHStore(
std::string_view scheme,
std::string_view host,
const Params & params)
: StoreConfig(params)
, RemoteStoreConfig(params)
, CommonSSHStoreConfig(params)
, CommonSSHStoreConfig(scheme, host, params)
, SSHStoreConfig(params)
, Store(params)
, RemoteStore(params)
, host(host)
, master(
host,
sshKey,
sshPublicHostKey,
, master(createSSHMaster(
// Use SSH master only if using more than 1 connection.
connections->capacity() > 1,
compress)
connections->capacity() > 1))
{
}
@ -108,6 +106,15 @@ struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfi
{
}
MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params)
: StoreConfig(params)
, RemoteStoreConfig(params)
, CommonSSHStoreConfig(scheme, host, params)
, SSHStoreConfig(params)
, LocalFSStoreConfig(params)
{
}
const std::string name() override { return "Experimental SSH Store with filesystem mounted"; }
std::string doc() override
@ -141,10 +148,13 @@ class MountedSSHStore : public virtual MountedSSHStoreConfig, public virtual SSH
{
public:
MountedSSHStore(const std::string & scheme, const std::string & host, const Params & params)
MountedSSHStore(
std::string_view scheme,
std::string_view host,
const Params & params)
: StoreConfig(params)
, RemoteStoreConfig(params)
, CommonSSHStoreConfig(params)
, CommonSSHStoreConfig(scheme, host, params)
, SSHStoreConfig(params)
, LocalFSStoreConfig(params)
, MountedSSHStoreConfig(params)

6
src/libstore/ssh.cc
View file

@ -6,7 +6,11 @@
namespace nix {
SSHMaster::SSHMaster(const std::string & host, const std::string & keyFile, const std::string & sshPublicHostKey, bool useMaster, bool compress, int logFD)
SSHMaster::SSHMaster(
std::string_view host,
std::string_view keyFile,
std::string_view sshPublicHostKey,
bool useMaster, bool compress, Descriptor logFD)
: host(host)
, fakeSSH(host == "localhost")
, keyFile(keyFile)

8
src/libstore/ssh.hh
View file

@ -17,7 +17,7 @@ private:
const std::string sshPublicHostKey;
const bool useMaster;
const bool compress;
const int logFD;
const Descriptor logFD;
struct State
{
@ -39,7 +39,11 @@ private:
public:
SSHMaster(const std::string & host, const std::string & keyFile, const std::string & sshPublicHostKey, bool useMaster, bool compress, int logFD = -1);
SSHMaster(
std::string_view host,
std::string_view keyFile,
std::string_view sshPublicHostKey,
bool useMaster, bool compress, Descriptor logFD = INVALID_DESCRIPTOR);
struct Connection
{

183
src/libstore/store-api.cc
View file

@ -8,7 +8,6 @@
#include "util.hh"
#include "nar-info-disk-cache.hh"
#include "thread-pool.hh"
#include "url.hh"
#include "references.hh"
#include "archive.hh"
#include "callback.hh"
@ -21,7 +20,6 @@
#include "users.hh"
#include <nlohmann/json.hpp>
#include <regex>
using json = nlohmann::json;
@ -1274,144 +1272,63 @@ Derivation Store::readInvalidDerivation(const StorePath & drvPath)
namespace nix {
/* Split URI into protocol+hierarchy part and its parameter set. */
std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_)
{
auto uri(uri_);
Store::Params params;
auto q = uri.find('?');
if (q != std::string::npos) {
params = decodeQuery(uri.substr(q + 1));
uri = uri_.substr(0, q);
}
return {uri, params};
}
static bool isNonUriPath(const std::string & spec)
{
return
// is not a URL
spec.find("://") == std::string::npos
// Has at least one path separator, and so isn't a single word that
// might be special like "auto"
&& spec.find("/") != std::string::npos;
}
std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Params & params)
{
// TODO reenable on Windows once we have `LocalStore` and
// `UDSRemoteStore`.
if (uri == "" || uri == "auto") {
auto stateDir = getOr(params, "state", settings.nixStateDir);
if (access(stateDir.c_str(), R_OK | W_OK) == 0)
return std::make_shared<LocalStore>(params);
else if (pathExists(settings.nixDaemonSocketFile))
return std::make_shared<UDSRemoteStore>(params);
#if __linux__
else if (!pathExists(stateDir)
&& params.empty()
&& !isRootUser()
&& !getEnv("NIX_STORE_DIR").has_value()
&& !getEnv("NIX_STATE_DIR").has_value())
{
/* If /nix doesn't exist, there is no daemon socket, and
we're not root, then automatically set up a chroot
store in ~/.local/share/nix/root. */
auto chrootStore = getDataDir() + "/nix/root";
if (!pathExists(chrootStore)) {
try {
createDirs(chrootStore);
} catch (Error & e) {
return std::make_shared<LocalStore>(params);
}
warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore);
} else
debug("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore);
Store::Params params2;
params2["root"] = chrootStore;
return std::make_shared<LocalStore>(params2);
}
#endif
else
return std::make_shared<LocalStore>(params);
} else if (uri == "daemon") {
return std::make_shared<UDSRemoteStore>(params);
} else if (uri == "local") {
return std::make_shared<LocalStore>(params);
} else if (isNonUriPath(uri)) {
Store::Params params2 = params;
params2["root"] = absPath(uri);
return std::make_shared<LocalStore>(params2);
} else {
return nullptr;
}
}
// The `parseURL` function supports both IPv6 URIs as defined in
// RFC2732, but also pure addresses. The latter one is needed here to
// connect to a remote store via SSH (it's possible to do e.g. `ssh root@::1`).
//
// This function now ensures that a usable connection string is available:
// * If the store to be opened is not an SSH store, nothing will be done.
// * If the URL looks like `root@[::1]` (which is allowed by the URL parser and probably
// needed to pass further flags), it
// will be transformed into `root@::1` for SSH (same for `[::1]` -> `::1`).
// * If the URL looks like `root@::1` it will be left as-is.
// * In any other case, the string will be left as-is.
static std::string extractConnStr(const std::string &proto, const std::string &connStr)
{
if (proto.rfind("ssh") != std::string::npos) {
std::smatch result;
std::regex v6AddrRegex("^((.*)@)?\\[(.*)\\]$");
if (std::regex_match(connStr, result, v6AddrRegex)) {
if (result[1].matched) {
return result.str(1) + result.str(3);
}
return result.str(3);
}
}
return connStr;
}
ref<Store> openStore(const std::string & uri_,
ref<Store> openStore(const std::string & uri,
const Store::Params & extraParams)
{
auto params = extraParams;
try {
auto parsedUri = parseURL(uri_);
params.insert(parsedUri.query.begin(), parsedUri.query.end());
return openStore(StoreReference::parse(uri, extraParams));
}
auto baseURI = extractConnStr(
parsedUri.scheme,
parsedUri.authority.value_or("") + parsedUri.path
);
ref<Store> openStore(StoreReference && storeURI)
{
auto & params = storeURI.params;
for (auto implem : *Implementations::registered) {
if (implem.uriSchemes.count(parsedUri.scheme)) {
auto store = implem.create(parsedUri.scheme, baseURI, params);
if (store) {
experimentalFeatureSettings.require(store->experimentalFeature());
store->init();
store->warnUnknownSettings();
return ref<Store>(store);
}
auto store = std::visit(overloaded {
[&](const StoreReference::Auto &) -> std::shared_ptr<Store> {
auto stateDir = getOr(params, "state", settings.nixStateDir);
if (access(stateDir.c_str(), R_OK | W_OK) == 0)
return std::make_shared<LocalStore>(params);
else if (pathExists(settings.nixDaemonSocketFile))
return std::make_shared<UDSRemoteStore>(params);
#if __linux__
else if (!pathExists(stateDir)
&& params.empty()
&& !isRootUser()
&& !getEnv("NIX_STORE_DIR").has_value()
&& !getEnv("NIX_STATE_DIR").has_value())
{
/* If /nix doesn't exist, there is no daemon socket, and
we're not root, then automatically set up a chroot
store in ~/.local/share/nix/root. */
auto chrootStore = getDataDir() + "/nix/root";
if (!pathExists(chrootStore)) {
try {
createDirs(chrootStore);
} catch (Error & e) {
return std::make_shared<LocalStore>(params);
}
warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore);
} else
debug("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore);
return std::make_shared<LocalStore>("local", chrootStore, params);
}
}
}
catch (BadURL &) {
auto [uri, uriParams] = splitUriAndParams(uri_);
params.insert(uriParams.begin(), uriParams.end());
#endif
else
return std::make_shared<LocalStore>(params);
},
[&](const StoreReference::Specified & g) {
for (auto implem : *Implementations::registered)
if (implem.uriSchemes.count(g.scheme))
return implem.create(g.scheme, g.authority, params);
if (auto store = openFromNonUri(uri, params)) {
experimentalFeatureSettings.require(store->experimentalFeature());
store->warnUnknownSettings();
return ref<Store>(store);
}
}
throw Error("don't know how to open Nix store with scheme '%s'", g.scheme);
},
}, storeURI.variant);
throw Error("don't know how to open Nix store '%s'", uri_);
experimentalFeatureSettings.require(store->experimentalFeature());
store->warnUnknownSettings();
store->init();
return ref<Store> { store };
}
std::list<ref<Store>> getDefaultSubstituters()

58
src/libstore/store-api.hh
View file

@ -13,6 +13,7 @@
#include "path-info.hh"
#include "repair-flag.hh"
#include "store-dir-config.hh"
#include "store-reference.hh"
#include "source-path.hh"
#include <nlohmann/json_fwd.hpp>
@ -65,7 +66,7 @@ MakeError(Unsupported, Error);
MakeError(SubstituteGone, Error);
MakeError(SubstituterDisabled, Error);
MakeError(InvalidStoreURI, Error);
MakeError(InvalidStoreReference, Error);
struct Realisation;
struct RealisedPath;
@ -91,7 +92,7 @@ enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true };
const uint32_t exportMagic = 0x4558494e;
enum BuildMode { bmNormal, bmRepair, bmCheck };
enum BuildMode : uint8_t { bmNormal, bmRepair, bmCheck };
enum TrustedFlag : bool { NotTrusted = false, Trusted = true };
struct BuildResult;
@ -102,7 +103,7 @@ typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
struct StoreConfig : public StoreDirConfig
{
typedef std::map<std::string, std::string> Params;
using Params = StoreReference::Params;
using StoreDirConfig::StoreDirConfig;
@ -859,34 +860,13 @@ OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * ev
/**
* @return a Store object to access the Nix store denoted by
* ‘uri’ (slight misnomer...).
*
* @param uri Supported values are:
*
* - ‘local’: The Nix store in /nix/store and database in
* /nix/var/nix/db, accessed directly.
*
* - ‘daemon’: The Nix store accessed via a Unix domain socket
* connection to nix-daemon.
*
* - ‘unix://<path>’: The Nix store accessed via a Unix domain socket
* connection to nix-daemon, with the socket located at <path>.
*
* - ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on
* whether the user has write access to the local Nix
* store/database.
*
* - ‘file://<path>’: A binary cache stored in <path>.
*
* - ‘https://<path>’: A binary cache accessed via HTTP.
*
* - ‘s3://<path>’: A writable binary cache stored on Amazon's Simple
* Storage Service.
*
* - ‘ssh://[user@]<host>’: A remote Nix store accessed by running
* ‘nix-store --serve’ via SSH.
*
* You can pass parameters to the store type by appending
* ‘?key=value&key=value&...’ to the URI.
*/
ref<Store> openStore(StoreReference && storeURI);
/**
* Opens the store at `uri`, where `uri` is in the format expected by `StoreReference::parse`
*/
ref<Store> openStore(const std::string & uri = settings.storeUri.get(),
const Store::Params & extraParams = Store::Params());
@ -901,7 +881,14 @@ std::list<ref<Store>> getDefaultSubstituters();
struct StoreFactory
{
std::set<std::string> uriSchemes;
std::function<std::shared_ptr<Store> (const std::string & scheme, const std::string & uri, const Store::Params & params)> create;
/**
* The `authorityPath` parameter is `<authority>/<path>`, or really
* whatever comes after `<scheme>://` and before `?<query-params>`.
*/
std::function<std::shared_ptr<Store> (
std::string_view scheme,
std::string_view authorityPath,
const Store::Params & params)> create;
std::function<std::shared_ptr<StoreConfig> ()> getConfig;
};
@ -916,7 +903,7 @@ struct Implementations
StoreFactory factory{
.uriSchemes = T::uriSchemes(),
.create =
([](const std::string & scheme, const std::string & uri, const Store::Params & params)
([](auto scheme, auto uri, auto & params)
-> std::shared_ptr<Store>
{ return std::make_shared<T>(scheme, uri, params); }),
.getConfig =
@ -950,11 +937,6 @@ std::optional<ValidPathInfo> decodeValidPathInfo(
std::istream & str,
std::optional<HashResult> hashGiven = std::nullopt);
/**
* Split URI into protocol+hierarchy part and its parameter set.
*/
std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri);
const ContentAddress * getDerivationCA(const BasicDerivation & drv);
std::map<DrvOutput, StorePath> drvOutputReferences(

116
src/libstore/store-reference.cc Normal file
View file

@ -0,0 +1,116 @@
#include <regex>
#include "error.hh"
#include "url.hh"
#include "store-reference.hh"
#include "file-system.hh"
#include "util.hh"
namespace nix {
static bool isNonUriPath(const std::string & spec)
{
return
// is not a URL
spec.find("://") == std::string::npos
// Has at least one path separator, and so isn't a single word that
// might be special like "auto"
&& spec.find("/") != std::string::npos;
}
std::string StoreReference::render() const
{
std::string res;
std::visit(
overloaded{
[&](const StoreReference::Auto &) { res = "auto"; },
[&](const StoreReference::Specified & g) {
res = g.scheme;
res += "://";
res += g.authority;
},
},
variant);
if (!params.empty()) {
res += "?";
res += encodeQuery(params);
}
return res;
}
StoreReference StoreReference::parse(const std::string & uri, const StoreReference::Params & extraParams)
{
auto params = extraParams;
try {
auto parsedUri = parseURL(uri);
params.insert(parsedUri.query.begin(), parsedUri.query.end());
auto baseURI = parsedUri.authority.value_or("") + parsedUri.path;
return {
.variant =
Specified{
.scheme = std::move(parsedUri.scheme),
.authority = std::move(baseURI),
},
.params = std::move(params),
};
} catch (BadURL &) {
auto [baseURI, uriParams] = splitUriAndParams(uri);
params.insert(uriParams.begin(), uriParams.end());
if (baseURI == "" || baseURI == "auto") {
return {
.variant = Auto{},
.params = std::move(params),
};
} else if (baseURI == "daemon") {
return {
.variant =
Specified{
.scheme = "unix",
.authority = "",
},
.params = std::move(params),
};
} else if (baseURI == "local") {
return {
.variant =
Specified{
.scheme = "local",
.authority = "",
},
.params = std::move(params),
};
} else if (isNonUriPath(baseURI)) {
return {
.variant =
Specified{
.scheme = "local",
.authority = absPath(baseURI),
},
.params = std::move(params),
};
}
}
throw UsageError("Cannot parse Nix store '%s'", uri);
}
/* Split URI into protocol+hierarchy part and its parameter set. */
std::pair<std::string, StoreReference::Params> splitUriAndParams(const std::string & uri_)
{
auto uri(uri_);
StoreReference::Params params;
auto q = uri.find('?');
if (q != std::string::npos) {
params = decodeQuery(uri.substr(q + 1));
uri = uri_.substr(0, q);
}
return {uri, params};
}
}

92
src/libstore/store-reference.hh Normal file
View file

@ -0,0 +1,92 @@
#pragma once
///@file
#include <variant>
#include "types.hh"
namespace nix {
/**
* A parsed Store URI (URI is a slight misnomer...), parsed but not yet
* resolved to a specific instance and query parms validated.
*
* Supported values are:
*
* - ‘local’: The Nix store in /nix/store and database in
* /nix/var/nix/db, accessed directly.
*
* - ‘daemon’: The Nix store accessed via a Unix domain socket
* connection to nix-daemon.
*
* - ‘unix://<path>’: The Nix store accessed via a Unix domain socket
* connection to nix-daemon, with the socket located at <path>.
*
* - ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on
* whether the user has write access to the local Nix
* store/database.
*
* - ‘file://<path>’: A binary cache stored in <path>.
*
* - ‘https://<path>’: A binary cache accessed via HTTP.
*
* - ‘s3://<path>’: A writable binary cache stored on Amazon's Simple
* Storage Service.
*
* - ‘ssh://[user@]<host>’: A remote Nix store accessed by running
* ‘nix-store --serve’ via SSH.
*
* You can pass parameters to the store type by appending
* ‘?key=value&key=value&...’ to the URI.
*/
struct StoreReference
{
using Params = std::map<std::string, std::string>;
/**
* Special store reference `""` or `"auto"`
*/
struct Auto
{
inline bool operator==(const Auto & rhs) const = default;
inline auto operator<=>(const Auto & rhs) const = default;
};
/**
* General case, a regular `scheme://authority` URL.
*/
struct Specified
{
std::string scheme;
std::string authority = "";
bool operator==(const Specified & rhs) const = default;
auto operator<=>(const Specified & rhs) const = default;
};
typedef std::variant<Auto, Specified> Variant;
Variant variant;
Params params;
bool operator==(const StoreReference & rhs) const = default;
auto operator<=>(const StoreReference & rhs) const = default;
/**
* Render the whole store reference as a URI, including parameters.
*/
std::string render() const;
/**
* Parse a URI into a store reference.
*/
static StoreReference parse(const std::string & uri, const Params & extraParams = Params{});
};
/**
* Split URI into protocol+hierarchy part and its parameter set.
*/
std::pair<std::string, StoreReference::Params> splitUriAndParams(const std::string & uri);
}

8
src/libstore/uds-remote-store.cc
View file

@ -40,12 +40,13 @@ UDSRemoteStore::UDSRemoteStore(const Params & params)
UDSRemoteStore::UDSRemoteStore(
const std::string scheme,
std::string socket_path,
std::string_view scheme,
PathView socket_path,
const Params & params)
: UDSRemoteStore(params)
{
path.emplace(socket_path);
if (!socket_path.empty())
path.emplace(socket_path);
}
@ -54,6 +55,7 @@ std::string UDSRemoteStore::getUri()
if (path) {
return std::string("unix://") + *path;
} else {
// unix:// with no path also works. Change what we return?
return "daemon";
}
}

5
src/libstore/uds-remote-store.hh
View file

@ -28,7 +28,10 @@ class UDSRemoteStore : public virtual UDSRemoteStoreConfig
public:
UDSRemoteStore(const Params & params);
UDSRemoteStore(const std::string scheme, std::string path, const Params & params);
UDSRemoteStore(
std::string_view scheme,
PathView path,
const Params & params);
std::string getUri() override;

15
src/libstore/unix/build/local-derivation-goal.cc
View file

@ -285,7 +285,7 @@ static void movePath(const Path & src, const Path & dst)
if (changePerm)
chmod_(src, st.st_mode | S_IWUSR);
renameFile(src, dst);
std::filesystem::rename(src, dst);
if (changePerm)
chmod_(dst, st.st_mode);
@ -372,7 +372,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull()
if (buildMode != bmCheck && status.known->isValid()) continue;
auto p = worker.store.toRealPath(status.known->path);
if (pathExists(chrootRootDir + p))
renameFile((chrootRootDir + p), p);
std::filesystem::rename((chrootRootDir + p), p);
}
return diskFull;
@ -421,7 +421,9 @@ static void doBind(const Path & source, const Path & target, bool optional = fal
} else if (S_ISLNK(st.st_mode)) {
// Symlinks can (apparently) not be bind-mounted, so just copy it
createDirs(dirOf(target));
copyFile(source, target, /* andDelete */ false);
copyFile(
std::filesystem::path(source),
std::filesystem::path(target), false);
} else {
createDirs(dirOf(target));
writeFile(target, "");
@ -2568,8 +2570,11 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
// Replace the output by a fresh copy of itself to make sure
// that there's no stale file descriptor pointing to it
Path tmpOutput = actualPath + ".tmp";
copyFile(actualPath, tmpOutput, true);
renameFile(tmpOutput, actualPath);
copyFile(
std::filesystem::path(actualPath),
std::filesystem::path(tmpOutput), true);
std::filesystem::rename(tmpOutput, actualPath);
auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating {
.method = dof.ca.method,

2
src/libstore/unix/lock.cc → src/libstore/unix/user-lock.cc
View file

@ -1,4 +1,4 @@
#include "lock.hh"
#include "user-lock.hh"
#include "file-system.hh"
#include "globals.hh"
#include "pathlocks.hh"

0
src/libstore/unix/lock.hh → src/libstore/unix/user-lock.hh
View file

37
src/libstore/windows/build.cc
View file

@ -1,37 +0,0 @@
#include "store-api.hh"
#include "build-result.hh"
namespace nix {
void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMode, std::shared_ptr<Store> evalStore)
{
unsupported("buildPaths");
}
std::vector<KeyedBuildResult> Store::buildPathsWithResults(
const std::vector<DerivedPath> & reqs,
BuildMode buildMode,
std::shared_ptr<Store> evalStore)
{
unsupported("buildPathsWithResults");
}
BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode)
{
unsupported("buildDerivation");
}
void Store::ensurePath(const StorePath & path)
{
unsupported("ensurePath");
}
void Store::repairPath(const StorePath & path)
{
unsupported("repairPath");
}
}

Some files were not shown because too many files have changed in this diff Show more