From 25b0242ca6aee3484111739e95b6788ea56d1a64 Mon Sep 17 00:00:00 2001 From: siddhantCodes Date: Sun, 9 Jun 2024 19:49:39 +0530 Subject: [PATCH 01/61] `std::filesystem::create_directories` for createDirs The implementation of `nix::createDirs` allows it to be a simple wrapper around `std::filesystem::create_directories` as its return value is not used anywhere. --- src/libcmd/repl-interacter.cc | 4 ++-- src/libutil/file-system.cc | 25 ++----------------------- src/libutil/file-system.hh | 7 +++---- 3 files changed, 7 insertions(+), 29 deletions(-) diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc index eb4361e25..254a86d7b 100644 --- a/src/libcmd/repl-interacter.cc +++ b/src/libcmd/repl-interacter.cc @@ -107,8 +107,8 @@ ReadlineLikeInteracter::Guard ReadlineLikeInteracter::init(detail::ReplCompleter rl_readline_name = "nix-repl"; try { createDirs(dirOf(historyFile)); - } catch (SystemError & e) { - logWarning(e.info()); + } catch (std::filesystem::filesystem_error & e) { + warn(e.what()); } #ifndef USE_READLINE el_hist_size = 1000; diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 919bf5d50..cd5db31bb 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -413,30 +413,9 @@ void deletePath(const fs::path & path) } -Paths createDirs(const Path & path) +void createDirs(const Path & path) { - Paths created; - if (path == "/") return created; - - struct stat st; - if (STAT(path.c_str(), &st) == -1) { - created = createDirs(dirOf(path)); - if (mkdir(path.c_str() -#ifndef _WIN32 // TODO abstract mkdir perms for Windows - , 0777 -#endif - ) == -1 && errno != EEXIST) - throw SysError("creating directory '%1%'", path); - st = STAT(path); - created.push_back(path); - } - - if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1) - throw SysError("statting symlink '%1%'", path); - - if (!S_ISDIR(st.st_mode)) throw Error("'%1%' is not a directory", path); - - return created; + fs::create_directories(path); } diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index c6b6ecedb..9405cda0c 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -148,11 +148,10 @@ void deletePath(const std::filesystem::path & path); void deletePath(const std::filesystem::path & path, uint64_t & bytesFreed); /** - * Create a directory and all its parents, if necessary. Returns the - * list of created directories, in order of creation. + * Create a directory and all its parents, if necessary. */ -Paths createDirs(const Path & path); -inline Paths createDirs(PathView path) +void createDirs(const Path & path); +inline void createDirs(PathView path) { return createDirs(Path(path)); } From 7a21432e7774617731556d844d05686a8b3ff46d Mon Sep 17 00:00:00 2001 From: siddhantCodes Date: Mon, 10 Jun 2024 11:30:39 +0530 Subject: [PATCH 02/61] fix: catch `filesystem_error` thrown by `createDirs` --- src/libstore/store-api.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c67ccd7d4..ed5275377 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -19,6 +19,7 @@ #include "signals.hh" #include "users.hh" +#include #include using json = nlohmann::json; @@ -1303,7 +1304,7 @@ ref openStore(StoreReference && storeURI) if (!pathExists(chrootStore)) { try { createDirs(chrootStore); - } catch (Error & e) { + } catch (std::filesystem::filesystem_error & e) { return std::make_shared(params); } warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); From 4809e59b7ee0681d420f83a285d52e9424f2fad8 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Mon, 10 Jun 2024 09:31:21 -0400 Subject: [PATCH 03/61] fix: warn and document when advanced attributes will have no impact due to __structuredAttrs --- doc/manual/src/language/advanced-attributes.md | 6 ++++++ src/libexpr/eval.cc | 6 ++++++ src/libexpr/eval.hh | 5 ++++- src/libexpr/primops.cc | 14 ++++++++++++++ .../unix/build/local-derivation-goal.cc | 18 ++++++++++++++++++ 5 files changed, 48 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 113062db1..64a28a366 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -301,6 +301,12 @@ Derivations can declare some infrequently used optional attributes. (associative) arrays. For example, the attribute `hardening.format = true` ends up as the Bash associative array element `${hardening[format]}`. + > **Warning** + > + > If set to `true`, other advanced attributes such as [`allowedReferences`](#adv-attr-allowedReferences), [`allowedReferences`](#adv-attr-allowedReferences), [`allowedRequisites`](#adv-attr-allowedRequisites), + [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), maxSize, and maxClosureSize. + will have no effect. + - [`outputChecks`]{#adv-attr-outputChecks}\ When using [structured attributes](#adv-attr-structuredAttrs), the `outputChecks` attribute allows defining checks per-output. diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f441977a5..5a94c67ec 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -423,6 +423,12 @@ EvalState::EvalState( , sRight(symbols.create("right")) , sWrong(symbols.create("wrong")) , sStructuredAttrs(symbols.create("__structuredAttrs")) + , sAllowedReferences(symbols.create("allowedReferences")) + , sAllowedRequisites(symbols.create("allowedRequisites")) + , sDisallowedReferences(symbols.create("disallowedReferences")) + , sDisallowedRequisites(symbols.create("disallowedRequisites")) + , sMaxSize(symbols.create("maxSize")) + , sMaxClosureSize(symbols.create("maxClosureSize")) , sBuilder(symbols.create("builder")) , sArgs(symbols.create("args")) , sContentAddressed(symbols.create("__contentAddressed")) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index dac763268..f916873f1 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -173,7 +173,10 @@ public: const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue, sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, sFile, sLine, sColumn, sFunctor, sToString, - sRight, sWrong, sStructuredAttrs, sBuilder, sArgs, + sRight, sWrong, sStructuredAttrs, + sAllowedReferences, sAllowedRequisites, sDisallowedReferences, sDisallowedRequisites, + sMaxSize, sMaxClosureSize, + sBuilder, sArgs, sContentAddressed, sImpure, sOutputHash, sOutputHashAlgo, sOutputHashMode, sRecurseForDerivations, diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 9177b0a2f..98649f081 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1308,6 +1308,20 @@ static void derivationStrictInternal( handleOutputs(ss); } + if (i->name == state.sAllowedReferences) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'allowedReferences'; use 'outputChecks..allowedReferences' instead", drvName); + if (i->name == state.sAllowedRequisites) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'allowedRequisites'; use 'outputChecks..allowedRequisites' instead", drvName); + if (i->name == state.sDisallowedReferences) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedReferences'; use 'outputChecks..disallowedReferences' instead", drvName); + if (i->name == state.sDisallowedRequisites) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedRequisites'; use 'outputChecks..disallowedRequisites' instead", drvName); + if (i->name == state.sMaxSize) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'maxSize'; use 'outputChecks..maxSize' instead", drvName); + if (i->name == state.sMaxClosureSize) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'maxClosureSize'; use 'outputChecks..maxClosureSize' instead", drvName); + + } else { auto s = state.coerceToString(pos, *i->value, context, context_below, true).toOwned(); drv.env.emplace(key, s); diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 16095cf5d..54644dc3a 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -2904,6 +2904,24 @@ void LocalDerivationGoal::checkOutputs(const std::mapgetStructuredAttrs()) { + if (get(*structuredAttrs, "allowedReferences")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'allowedReferences'; use 'outputChecks' instead"); + } + if (get(*structuredAttrs, "allowedRequisites")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'allowedRequisites'; use 'outputChecks' instead"); + } + if (get(*structuredAttrs, "disallowedRequisites")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'disallowedRequisites'; use 'outputChecks' instead"); + } + if (get(*structuredAttrs, "disallowedReferences")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'disallowedReferences'; use 'outputChecks' instead"); + } + if (get(*structuredAttrs, "maxSize")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'maxSize'; use 'outputChecks' instead"); + } + if (get(*structuredAttrs, "maxClosureSize")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'maxClosureSize'; use 'outputChecks' instead"); + } if (auto outputChecks = get(*structuredAttrs, "outputChecks")) { if (auto output = get(*outputChecks, outputName)) { Checks checks; From de3fd52a95cedb57bc6d9dbda0c597cd2bfb3db5 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 10 Jun 2024 15:55:53 +0200 Subject: [PATCH 04/61] Add tests/f/lang/eval-okay-derivation-legacy --- .../lang/eval-okay-derivation-legacy.err.exp | 6 ++++++ .../functional/lang/eval-okay-derivation-legacy.exp | 1 + .../functional/lang/eval-okay-derivation-legacy.nix | 12 ++++++++++++ 3 files changed, 19 insertions(+) create mode 100644 tests/functional/lang/eval-okay-derivation-legacy.err.exp create mode 100644 tests/functional/lang/eval-okay-derivation-legacy.exp create mode 100644 tests/functional/lang/eval-okay-derivation-legacy.nix diff --git a/tests/functional/lang/eval-okay-derivation-legacy.err.exp b/tests/functional/lang/eval-okay-derivation-legacy.err.exp new file mode 100644 index 000000000..94f0854dd --- /dev/null +++ b/tests/functional/lang/eval-okay-derivation-legacy.err.exp @@ -0,0 +1,6 @@ +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'allowedReferences'; use 'outputChecks..allowedReferences' instead +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'allowedRequisites'; use 'outputChecks..allowedRequisites' instead +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedReferences'; use 'outputChecks..disallowedReferences' instead +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedRequisites'; use 'outputChecks..disallowedRequisites' instead +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'maxClosureSize'; use 'outputChecks..maxClosureSize' instead +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'maxSize'; use 'outputChecks..maxSize' instead diff --git a/tests/functional/lang/eval-okay-derivation-legacy.exp b/tests/functional/lang/eval-okay-derivation-legacy.exp new file mode 100644 index 000000000..4f374a1aa --- /dev/null +++ b/tests/functional/lang/eval-okay-derivation-legacy.exp @@ -0,0 +1 @@ +"/nix/store/mzgwvrjjir216ra58mwwizi8wj6y9ddr-eval-okay-derivation-legacy" diff --git a/tests/functional/lang/eval-okay-derivation-legacy.nix b/tests/functional/lang/eval-okay-derivation-legacy.nix new file mode 100644 index 000000000..b529cdf90 --- /dev/null +++ b/tests/functional/lang/eval-okay-derivation-legacy.nix @@ -0,0 +1,12 @@ +(builtins.derivationStrict { + name = "eval-okay-derivation-legacy"; + system = "x86_64-linux"; + builder = "/dontcare"; + __structuredAttrs = true; + allowedReferences = [ ]; + disallowedReferences = [ ]; + allowedRequisites = [ ]; + disallowedRequisites = [ ]; + maxSize = 1234; + maxClosureSize = 12345; +}).out From a58ca342cae1d6bee488a42b6fc054c1072cfb8f Mon Sep 17 00:00:00 2001 From: PoweredByPie Date: Mon, 17 Jun 2024 11:01:29 -0700 Subject: [PATCH 05/61] Initial runProgram implementation for Windows This is incomplete; proper shell escaping needs to be done --- src/libutil/processes.hh | 17 +- src/libutil/windows/processes.cc | 322 ++++++++++++++++++++++++++++++- 2 files changed, 331 insertions(+), 8 deletions(-) diff --git a/src/libutil/processes.hh b/src/libutil/processes.hh index 168fcaa55..bbbe7dcab 100644 --- a/src/libutil/processes.hh +++ b/src/libutil/processes.hh @@ -3,6 +3,7 @@ #include "types.hh" #include "error.hh" +#include "file-descriptor.hh" #include "logging.hh" #include "ansicolor.hh" @@ -23,26 +24,36 @@ namespace nix { struct Sink; struct Source; -#ifndef _WIN32 class Pid { +#ifndef _WIN32 pid_t pid = -1; bool separatePG = false; int killSignal = SIGKILL; +#else + AutoCloseFD pid = INVALID_DESCRIPTOR; +#endif public: Pid(); +#ifndef _WIN32 Pid(pid_t pid); - ~Pid(); void operator =(pid_t pid); operator pid_t(); +#else + Pid(AutoCloseFD pid); + void operator =(AutoCloseFD pid); +#endif + ~Pid(); int kill(); int wait(); + // TODO: Implement for Windows +#ifndef _WIN32 void setSeparatePG(bool separatePG); void setKillSignal(int signal); pid_t release(); -}; #endif +}; #ifndef _WIN32 diff --git a/src/libutil/windows/processes.cc b/src/libutil/windows/processes.cc index 44a32f6a1..bb796ce2e 100644 --- a/src/libutil/windows/processes.cc +++ b/src/libutil/windows/processes.cc @@ -1,9 +1,15 @@ #include "current-process.hh" #include "environment-variables.hh" +#include "error.hh" +#include "file-descriptor.hh" +#include "file-path.hh" #include "signals.hh" #include "processes.hh" #include "finally.hh" #include "serialise.hh" +#include "file-system.hh" +#include "util.hh" +#include "windows-error.hh" #include #include @@ -16,25 +22,332 @@ #include #include +#define WIN32_LEAN_AND_MEAN +#include + namespace nix { +using namespace nix::windows; + +Pid::Pid() {} + +Pid::Pid(AutoCloseFD pid) + : pid(std::move(pid)) +{ +} + +Pid::~Pid() +{ + if (pid.get() != INVALID_DESCRIPTOR) + kill(); +} + +void Pid::operator=(AutoCloseFD pid) +{ + if (this->pid.get() != INVALID_DESCRIPTOR && this->pid.get() != pid.get()) + kill(); + this->pid = std::move(pid); +} + +// TODO: Implement (not needed for process spawning yet) +int Pid::kill() +{ + assert(pid.get() != INVALID_DESCRIPTOR); + + debug("killing process %1%", pid.get()); + + throw UnimplementedError("Pid::kill unimplemented"); +} + +int Pid::wait() +{ + // https://github.com/nix-windows/nix/blob/windows-meson/src/libutil/util.cc#L1938 + assert(pid.get() != INVALID_DESCRIPTOR); + DWORD status = WaitForSingleObject(pid.get(), INFINITE); + if (status != WAIT_OBJECT_0) { + debug("WaitForSingleObject returned %1%", status); + } + + DWORD exitCode = 0; + if (GetExitCodeProcess(pid.get(), &exitCode) == FALSE) { + debug("GetExitCodeProcess failed on pid %1%", pid.get()); + } + + pid.close(); + return exitCode; +} + +// TODO: Merge this with Unix's runProgram since it's identical logic. std::string runProgram(Path program, bool lookupPath, const Strings & args, const std::optional & input, bool isInteractive) { - throw UnimplementedError("Cannot shell out to git on Windows yet"); + auto res = runProgram(RunOptions{ + .program = program, .lookupPath = lookupPath, .args = args, .input = input, .isInteractive = isInteractive}); + + if (!statusOk(res.first)) + throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first)); + + return res.second; +} +// Looks at the $PATH environment variable to find the program. +// Adapted from https://github.com/nix-windows/nix/blob/windows/src/libutil/util.cc#L2276 +Path lookupPathForProgram(const Path & program) +{ + if (program.find('/') != program.npos || program.find('\\') != program.npos) { + throw Error("program '%1%' partially specifies its path", program); + } + + // Possible extensions. + // TODO: This should actually be sourced from $PATHEXT, not hardcoded. + static constexpr const char * exts[] = {"", ".exe", ".cmd", ".bat"}; + + auto path = getEnv("PATH"); + if (!path.has_value()) { + throw WinError("couldn't find PATH environment variable"); + } + + // Look through each directory listed in $PATH. + for (const std::string & dir : tokenizeString(*getEnv("PATH"), ";")) { + Path candidate = canonPath(dir) + '/' + program; + for (const auto ext : exts) { + if (pathExists(candidate + ext)) { + return candidate; + } + } + } + + throw WinError("program '%1%' not found on PATH", program); } +std::optional getProgramInterpreter(const Path & program) +{ + // These extensions are automatically handled by Windows and don't require an interpreter. + static constexpr const char * exts[] = {".exe", ".cmd", ".bat"}; + for (const auto ext : exts) { + if (hasSuffix(program, ext)) { + return {}; + } + } + // TODO: Open file and read the shebang + throw UnimplementedError("getProgramInterpreter unimplemented"); +} +// TODO: Not sure if this is needed in the unix version but it might be useful as a member func +void setFDInheritable(AutoCloseFD & fd, bool inherit) +{ + if (fd.get() != INVALID_DESCRIPTOR) { + if (!SetHandleInformation(fd.get(), HANDLE_FLAG_INHERIT, inherit ? HANDLE_FLAG_INHERIT : 0)) { + throw WinError("Couldn't disable inheriting of handle"); + } + } +} + +AutoCloseFD nullFD() +{ + // Create null handle to discard reads / writes + // https://stackoverflow.com/a/25609668 + // https://github.com/nix-windows/nix/blob/windows-meson/src/libutil/util.cc#L2228 + AutoCloseFD nul = CreateFileW( + L"NUL", + GENERIC_READ | GENERIC_WRITE, + // We don't care who reads / writes / deletes this file since it's NUL anyways + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + NULL, + OPEN_EXISTING, + 0, + NULL); + if (!nul.get()) { + throw WinError("Couldn't open NUL device"); + } + // Let this handle be inheritable by child processes + setFDInheritable(nul, true); + return nul; +} + +Pid spawnProcess(const Path & realProgram, const RunOptions & options, Pipe & out, Pipe & in) +{ + // Setup pipes. + if (options.standardOut) { + // Don't inherit the read end of the output pipe + setFDInheritable(out.readSide, false); + } else { + out.writeSide = nullFD(); + } + if (options.standardIn) { + // Don't inherit the write end of the input pipe + setFDInheritable(in.writeSide, false); + } else { + in.readSide = nullFD(); + } + + STARTUPINFOW startInfo = {0}; + startInfo.cb = sizeof(startInfo); + startInfo.dwFlags = STARTF_USESTDHANDLES; + startInfo.hStdInput = in.readSide.get(); + startInfo.hStdOutput = out.writeSide.get(); + startInfo.hStdError = out.writeSide.get(); + + std::string envline; + // Retain the current processes' environment variables. + for (const auto & envVar : getEnv()) { + envline += (envVar.first + '=' + envVar.second + '\0'); + } + // Also add new ones specified in options. + if (options.environment) { + for (const auto & envVar : *options.environment) { + envline += (envVar.first + '=' + envVar.second + '\0'); + } + } + + std::string cmdline = realProgram; + for (const auto & arg : options.args) { + // TODO: This isn't the right way to escape windows command + // See https://learn.microsoft.com/en-us/windows/win32/api/shellapi/nf-shellapi-commandlinetoargvw + cmdline += ' ' + shellEscape(arg); + } + + PROCESS_INFORMATION procInfo = {0}; + if (CreateProcessW( + string_to_os_string(realProgram).c_str(), + string_to_os_string(cmdline).data(), + NULL, + NULL, + TRUE, + CREATE_UNICODE_ENVIRONMENT | CREATE_SUSPENDED, + string_to_os_string(envline).data(), + options.chdir.has_value() ? string_to_os_string(*options.chdir).data() : NULL, + &startInfo, + &procInfo) + == 0) { + throw WinError("CreateProcessW failed (%1%)", cmdline); + } + + // Convert these to use RAII + AutoCloseFD process = procInfo.hProcess; + AutoCloseFD thread = procInfo.hThread; + + // Add current process and child to job object so child terminates when parent terminates + // TODO: This spawns one job per child process. We can probably keep this as a global, and + // add children a single job so we don't use so many jobs at once. + Descriptor job = CreateJobObjectW(NULL, NULL); + if (job == NULL) { + TerminateProcess(procInfo.hProcess, 0); + throw WinError("Couldn't create job object for child process"); + } + if (AssignProcessToJobObject(job, procInfo.hProcess) == FALSE) { + TerminateProcess(procInfo.hProcess, 0); + throw WinError("Couldn't assign child process to job object"); + } + if (ResumeThread(procInfo.hThread) == -1) { + TerminateProcess(procInfo.hProcess, 0); + throw WinError("Couldn't resume child process thread"); + } + + return process; +} + +// TODO: Merge this with Unix's runProgram since it's identical logic. // Output = error code + "standard out" output stream std::pair runProgram(RunOptions && options) { - throw UnimplementedError("Cannot shell out to git on Windows yet"); -} + StringSink sink; + options.standardOut = &sink; + int status = 0; + + try { + runProgram2(options); + } catch (ExecError & e) { + status = e.status; + } + + return {status, std::move(sink.s)}; +} void runProgram2(const RunOptions & options) { - throw UnimplementedError("Cannot shell out to git on Windows yet"); + checkInterrupt(); + + assert(!(options.standardIn && options.input)); + + std::unique_ptr source_; + Source * source = options.standardIn; + + if (options.input) { + source_ = std::make_unique(*options.input); + source = source_.get(); + } + + /* Create a pipe. */ + Pipe out, in; + // TODO: I copied this from unix but this is handled again in spawnProcess, so might be weird to split it up like + // this + if (options.standardOut) + out.create(); + if (source) + in.create(); + + Path realProgram = options.program; + if (options.lookupPath) { + realProgram = lookupPathForProgram(realProgram); + } + // TODO: Implement shebang / program interpreter lookup on Windows + auto interpreter = getProgramInterpreter(realProgram); + + std::optional>> resumeLoggerDefer; + if (options.isInteractive) { + logger->pause(); + resumeLoggerDefer.emplace([]() { logger->resume(); }); + } + + Pid pid = spawnProcess(interpreter.has_value() ? *interpreter : realProgram, options, out, in); + + // TODO: This is identical to unix, deduplicate? + out.writeSide.close(); + + std::thread writerThread; + + std::promise promise; + + Finally doJoin([&] { + if (writerThread.joinable()) + writerThread.join(); + }); + + if (source) { + in.readSide.close(); + writerThread = std::thread([&] { + try { + std::vector buf(8 * 1024); + while (true) { + size_t n; + try { + n = source->read(buf.data(), buf.size()); + } catch (EndOfFile &) { + break; + } + writeFull(in.writeSide.get(), {buf.data(), n}); + } + promise.set_value(); + } catch (...) { + promise.set_exception(std::current_exception()); + } + in.writeSide.close(); + }); + } + + if (options.standardOut) + drainFD(out.readSide.get(), *options.standardOut); + + /* Wait for the child to finish. */ + int status = pid.wait(); + + /* Wait for the writer thread to finish. */ + if (source) + promise.get_future().get(); + + if (status) + throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status)); } std::string statusToString(int status) @@ -45,7 +358,6 @@ std::string statusToString(int status) return "succeeded"; } - bool statusOk(int status) { return status == 0; From b11cf8166f63fa061d98cd5812a94234fe974845 Mon Sep 17 00:00:00 2001 From: PoweredByPie Date: Mon, 17 Jun 2024 13:12:28 -0700 Subject: [PATCH 06/61] Format runProgram declaration --- src/libutil/windows/processes.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/windows/processes.cc b/src/libutil/windows/processes.cc index bb796ce2e..1d8035c62 100644 --- a/src/libutil/windows/processes.cc +++ b/src/libutil/windows/processes.cc @@ -78,8 +78,8 @@ int Pid::wait() } // TODO: Merge this with Unix's runProgram since it's identical logic. -std::string runProgram(Path program, bool lookupPath, const Strings & args, - const std::optional & input, bool isInteractive) +std::string runProgram( + Path program, bool lookupPath, const Strings & args, const std::optional & input, bool isInteractive) { auto res = runProgram(RunOptions{ .program = program, .lookupPath = lookupPath, .args = args, .input = input, .isInteractive = isInteractive}); From 4662e7d8568f55f7f56c55e7976b68a25824d1a3 Mon Sep 17 00:00:00 2001 From: PoweredByPie Date: Mon, 17 Jun 2024 14:57:57 -0700 Subject: [PATCH 07/61] Implement windowsEscape --- src/libutil/windows/processes.cc | 48 +++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/src/libutil/windows/processes.cc b/src/libutil/windows/processes.cc index 1d8035c62..cf1949aa2 100644 --- a/src/libutil/windows/processes.cc +++ b/src/libutil/windows/processes.cc @@ -164,6 +164,52 @@ AutoCloseFD nullFD() return nul; } +// Adapted from +// https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/ +std::string windowsEscape(const std::string & str, bool cmd) +{ + // TODO: This doesn't handle cmd.exe escaping. + if (cmd) { + throw UnimplementedError("cmd.exe escaping is not implemented"); + } + + if (str.find_first_of(" \t\n\v\"") == str.npos && !str.empty()) { + // No need to escape this one, the nonempty contents don't have a special character + return str; + } + std::string buffer; + // Add the opening quote + buffer += '"'; + for (auto iter = str.begin();; ++iter) { + size_t backslashes = 0; + while (iter != str.end() && *iter == '\\') { + ++iter; + ++backslashes; + } + + // We only escape backslashes if: + // - They come immediately before the closing quote + // - They come immediately before a quote in the middle of the string + // Both of these cases break the escaping if not handled. Otherwise backslashes are fine as-is + if (iter == str.end()) { + // Need to escape each backslash + buffer.append(backslashes * 2, '\\'); + // Exit since we've reached the end of the string + break; + } else if (*iter == '"') { + // Need to escape each backslash and the intermediate quote character + buffer.append(backslashes * 2, '\\'); + buffer += "\\\""; + } else { + // Don't escape the backslashes since they won't break the delimiter + buffer.append(backslashes, '\\'); + buffer += *iter; + } + } + // Add the closing quote + return buffer + '"'; +} + Pid spawnProcess(const Path & realProgram, const RunOptions & options, Pipe & out, Pipe & in) { // Setup pipes. @@ -203,7 +249,7 @@ Pid spawnProcess(const Path & realProgram, const RunOptions & options, Pipe & ou for (const auto & arg : options.args) { // TODO: This isn't the right way to escape windows command // See https://learn.microsoft.com/en-us/windows/win32/api/shellapi/nf-shellapi-commandlinetoargvw - cmdline += ' ' + shellEscape(arg); + cmdline += ' ' + windowsEscape(arg, false); } PROCESS_INFORMATION procInfo = {0}; From d7537f695515339f811da73c98b64bb4e2e7dc9c Mon Sep 17 00:00:00 2001 From: PoweredByPie Date: Mon, 17 Jun 2024 14:58:17 -0700 Subject: [PATCH 08/61] Implement initial spawn tests (just testing windowsEscape for now) --- tests/unit/libutil/spawn.cc | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 tests/unit/libutil/spawn.cc diff --git a/tests/unit/libutil/spawn.cc b/tests/unit/libutil/spawn.cc new file mode 100644 index 000000000..6d9821f4e --- /dev/null +++ b/tests/unit/libutil/spawn.cc @@ -0,0 +1,35 @@ +#include + +#include "processes.hh" + +namespace nix { +/* +TEST(SpawnTest, spawnEcho) +{ +auto output = runProgram(RunOptions{.program = "echo", .args = {}}); +} +*/ + +#ifdef _WIN32 +std::string windowsEscape(const std::string & str, bool cmd); + +TEST(SpawnTest, windowsEscape) +{ + auto empty = windowsEscape("", false); + ASSERT_EQ(empty, R"("")"); + // There's no quotes in this argument so the input should equal the output + auto backslashStr = R"(\\\\)"; + auto backslashes = windowsEscape(backslashStr, false); + ASSERT_EQ(backslashes, backslashStr); + + auto nestedQuotes = windowsEscape(R"(he said: "hello there")", false); + ASSERT_EQ(nestedQuotes, R"("he said: \"hello there\"")"); + + auto middleQuote = windowsEscape(R"( \\\" )", false); + ASSERT_EQ(middleQuote, R"(" \\\\\\\" ")"); + + auto space = windowsEscape("hello world", false); + ASSERT_EQ(space, R"("hello world")"); +} +#endif +} From 4f6e3b940255053cd51e566416327c6120851075 Mon Sep 17 00:00:00 2001 From: PoweredByPie Date: Mon, 17 Jun 2024 18:44:23 -0700 Subject: [PATCH 09/61] Implement tests for lookupPathForProgram and fix bugs caught by tests --- src/libutil/windows/processes.cc | 8 ++++---- tests/unit/libutil/spawn.cc | 10 +++++++++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/libutil/windows/processes.cc b/src/libutil/windows/processes.cc index cf1949aa2..a1104fa1e 100644 --- a/src/libutil/windows/processes.cc +++ b/src/libutil/windows/processes.cc @@ -94,7 +94,7 @@ std::string runProgram( Path lookupPathForProgram(const Path & program) { if (program.find('/') != program.npos || program.find('\\') != program.npos) { - throw Error("program '%1%' partially specifies its path", program); + throw UsageError("program '%1%' partially specifies its path", program); } // Possible extensions. @@ -107,8 +107,9 @@ Path lookupPathForProgram(const Path & program) } // Look through each directory listed in $PATH. - for (const std::string & dir : tokenizeString(*getEnv("PATH"), ";")) { - Path candidate = canonPath(dir) + '/' + program; + for (const std::string & dir : tokenizeString(*path, ";")) { + // TODO: This should actually be canonPath(dir), but that ends up appending two drive paths + Path candidate = dir + "/" + program; for (const auto ext : exts) { if (pathExists(candidate + ext)) { return candidate; @@ -408,5 +409,4 @@ bool statusOk(int status) { return status == 0; } - } diff --git a/tests/unit/libutil/spawn.cc b/tests/unit/libutil/spawn.cc index 6d9821f4e..e84de18f1 100644 --- a/tests/unit/libutil/spawn.cc +++ b/tests/unit/libutil/spawn.cc @@ -6,11 +6,19 @@ namespace nix { /* TEST(SpawnTest, spawnEcho) { -auto output = runProgram(RunOptions{.program = "echo", .args = {}}); +auto output = runProgram(RunOptions{.program = "cmd", .lookupPath = true, .args = {"/C", "echo \"hello world\""}}); +std::cout << output.second << std::endl; } */ #ifdef _WIN32 +Path lookupPathForProgram(const Path & program); +TEST(SpawnTest, pathSearch) +{ + ASSERT_NO_THROW(lookupPathForProgram("cmd")); + ASSERT_NO_THROW(lookupPathForProgram("cmd.exe")); + ASSERT_THROW(lookupPathForProgram("C:/System32/cmd.exe"), UsageError); +} std::string windowsEscape(const std::string & str, bool cmd); TEST(SpawnTest, windowsEscape) From fcb92b4fa4f260086aafdb7d9e8e51a26360b46e Mon Sep 17 00:00:00 2001 From: PoweredByPie Date: Mon, 17 Jun 2024 22:14:38 -0700 Subject: [PATCH 10/61] Fix DWORD vs. int comparison warning --- src/libutil/windows/processes.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/windows/processes.cc b/src/libutil/windows/processes.cc index a1104fa1e..973a2cab9 100644 --- a/src/libutil/windows/processes.cc +++ b/src/libutil/windows/processes.cc @@ -285,7 +285,7 @@ Pid spawnProcess(const Path & realProgram, const RunOptions & options, Pipe & ou TerminateProcess(procInfo.hProcess, 0); throw WinError("Couldn't assign child process to job object"); } - if (ResumeThread(procInfo.hThread) == -1) { + if (ResumeThread(procInfo.hThread) == (DWORD) -1) { TerminateProcess(procInfo.hProcess, 0); throw WinError("Couldn't resume child process thread"); } From 8b81d083a72b714a5599c8243f8e05ed15d2d7c0 Mon Sep 17 00:00:00 2001 From: PoweredByPie Date: Tue, 18 Jun 2024 00:55:47 -0700 Subject: [PATCH 11/61] Remove lookupPathForProgram and implement initial runProgram test Apparently, CreateProcessW already searches path, so manual path search isn't really necessary. --- src/libutil/windows/processes.cc | 38 +++----------------------------- tests/unit/libutil/spawn.cc | 17 +++++--------- 2 files changed, 8 insertions(+), 47 deletions(-) diff --git a/src/libutil/windows/processes.cc b/src/libutil/windows/processes.cc index 973a2cab9..9cd714f84 100644 --- a/src/libutil/windows/processes.cc +++ b/src/libutil/windows/processes.cc @@ -89,36 +89,6 @@ std::string runProgram( return res.second; } -// Looks at the $PATH environment variable to find the program. -// Adapted from https://github.com/nix-windows/nix/blob/windows/src/libutil/util.cc#L2276 -Path lookupPathForProgram(const Path & program) -{ - if (program.find('/') != program.npos || program.find('\\') != program.npos) { - throw UsageError("program '%1%' partially specifies its path", program); - } - - // Possible extensions. - // TODO: This should actually be sourced from $PATHEXT, not hardcoded. - static constexpr const char * exts[] = {"", ".exe", ".cmd", ".bat"}; - - auto path = getEnv("PATH"); - if (!path.has_value()) { - throw WinError("couldn't find PATH environment variable"); - } - - // Look through each directory listed in $PATH. - for (const std::string & dir : tokenizeString(*path, ";")) { - // TODO: This should actually be canonPath(dir), but that ends up appending two drive paths - Path candidate = dir + "/" + program; - for (const auto ext : exts) { - if (pathExists(candidate + ext)) { - return candidate; - } - } - } - - throw WinError("program '%1%' not found on PATH", program); -} std::optional getProgramInterpreter(const Path & program) { @@ -246,7 +216,7 @@ Pid spawnProcess(const Path & realProgram, const RunOptions & options, Pipe & ou } } - std::string cmdline = realProgram; + std::string cmdline = windowsEscape(realProgram, false); for (const auto & arg : options.args) { // TODO: This isn't the right way to escape windows command // See https://learn.microsoft.com/en-us/windows/win32/api/shellapi/nf-shellapi-commandlinetoargvw @@ -255,7 +225,8 @@ Pid spawnProcess(const Path & realProgram, const RunOptions & options, Pipe & ou PROCESS_INFORMATION procInfo = {0}; if (CreateProcessW( - string_to_os_string(realProgram).c_str(), + // EXE path is provided in the cmdline + NULL, string_to_os_string(cmdline).data(), NULL, NULL, @@ -335,9 +306,6 @@ void runProgram2(const RunOptions & options) in.create(); Path realProgram = options.program; - if (options.lookupPath) { - realProgram = lookupPathForProgram(realProgram); - } // TODO: Implement shebang / program interpreter lookup on Windows auto interpreter = getProgramInterpreter(realProgram); diff --git a/tests/unit/libutil/spawn.cc b/tests/unit/libutil/spawn.cc index e84de18f1..c617acae0 100644 --- a/tests/unit/libutil/spawn.cc +++ b/tests/unit/libutil/spawn.cc @@ -3,22 +3,15 @@ #include "processes.hh" namespace nix { -/* -TEST(SpawnTest, spawnEcho) -{ -auto output = runProgram(RunOptions{.program = "cmd", .lookupPath = true, .args = {"/C", "echo \"hello world\""}}); -std::cout << output.second << std::endl; -} -*/ #ifdef _WIN32 -Path lookupPathForProgram(const Path & program); -TEST(SpawnTest, pathSearch) +TEST(SpawnTest, spawnEcho) { - ASSERT_NO_THROW(lookupPathForProgram("cmd")); - ASSERT_NO_THROW(lookupPathForProgram("cmd.exe")); - ASSERT_THROW(lookupPathForProgram("C:/System32/cmd.exe"), UsageError); + auto output = runProgram(RunOptions{.program = "cmd.exe", .args = {"/C", "echo", "hello world"}}); + ASSERT_EQ(output.first, 0); + ASSERT_EQ(output.second, "\"hello world\"\r\n"); } + std::string windowsEscape(const std::string & str, bool cmd); TEST(SpawnTest, windowsEscape) From dc720f89f2689ddfb4c1e0d0ea0d442b6be6c006 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 16 Jun 2024 12:22:42 +0200 Subject: [PATCH 12/61] flake.nix: Factor pkgs.nix_noTests out of buildNoTests This is useful when iterating on the functional tests when trying to run them in a VM test, for example. --- flake.nix | 6 ++++++ maintainers/hydra.nix | 8 +------- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/flake.nix b/flake.nix index 4e7363cd6..7e7148afe 100644 --- a/flake.nix +++ b/flake.nix @@ -227,6 +227,12 @@ ; }; + nix_noTests = final.nix.override { + doCheck = false; + doInstallCheck = false; + installUnitTests = false; + }; + # See https://github.com/NixOS/nixpkgs/pull/214409 # Remove when fixed in this flake's nixpkgs pre-commit = diff --git a/maintainers/hydra.nix b/maintainers/hydra.nix index abd44efef..e21145f37 100644 --- a/maintainers/hydra.nix +++ b/maintainers/hydra.nix @@ -58,13 +58,7 @@ in self.packages.${system}.nix.override { enableGC = false; } ); - buildNoTests = forAllSystems (system: - self.packages.${system}.nix.override { - doCheck = false; - doInstallCheck = false; - installUnitTests = false; - } - ); + buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nix_noTests); # Toggles some settings for better coverage. Windows needs these # library combinations, and Debian build Nix with GNU readline too. From 439022c5acfce4284c902bd6ac59575902c2c15c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 16 Jun 2024 12:13:07 +0200 Subject: [PATCH 13/61] tests: Add hydraJobs.tests.functional_* --- .../build-remote-input-addressed.sh | 2 +- tests/functional/common/init.sh | 27 +++++++ tests/functional/common/subst-vars.sh.in | 7 +- tests/functional/common/vars-and-functions.sh | 81 ++++++++++++++----- tests/functional/impure-env.sh | 4 +- .../functional/local-overlay-store/common.sh | 2 +- tests/functional/post-hook.sh | 2 +- tests/nixos/default.nix | 7 ++ tests/nixos/functional/as-root.nix | 12 +++ tests/nixos/functional/as-trusted-user.nix | 18 +++++ tests/nixos/functional/as-user.nix | 16 ++++ tests/nixos/functional/common.nix | 76 +++++++++++++++++ tests/nixos/functional/quick-build.nix | 47 +++++++++++ 13 files changed, 273 insertions(+), 28 deletions(-) create mode 100644 tests/nixos/functional/as-root.nix create mode 100644 tests/nixos/functional/as-trusted-user.nix create mode 100644 tests/nixos/functional/as-user.nix create mode 100644 tests/nixos/functional/common.nix create mode 100644 tests/nixos/functional/quick-build.nix diff --git a/tests/functional/build-remote-input-addressed.sh b/tests/functional/build-remote-input-addressed.sh index 986692dbc..11199a408 100755 --- a/tests/functional/build-remote-input-addressed.sh +++ b/tests/functional/build-remote-input-addressed.sh @@ -23,7 +23,7 @@ EOF chmod +x "$TEST_ROOT/post-build-hook.sh" rm -f "$TEST_ROOT/post-hook-counter" - echo "post-build-hook = $TEST_ROOT/post-build-hook.sh" >> "$NIX_CONF_DIR/nix.conf" + echo "post-build-hook = $TEST_ROOT/post-build-hook.sh" >> "$test_nix_conf" } registerBuildHook diff --git a/tests/functional/common/init.sh b/tests/functional/common/init.sh index 4f2a393af..19cef5af9 100755 --- a/tests/functional/common/init.sh +++ b/tests/functional/common/init.sh @@ -1,5 +1,30 @@ # shellcheck shell=bash +# for shellcheck +: "${test_nix_conf_dir?}" "${test_nix_conf?}" + +if isTestOnNixOS; then + + mkdir -p "$test_nix_conf_dir" "$TEST_HOME" + + export NIX_USER_CONF_FILES="$test_nix_conf_dir/nix.conf" + mkdir -p "$test_nix_conf_dir" "$TEST_HOME" + ! test -e "$test_nix_conf" + cat > "$test_nix_conf_dir/nix.conf" <&2 + exit 1 +} + set +x commonDir="$(readlink -f "$(dirname "${BASH_SOURCE[0]-$0}")")" @@ -15,27 +24,35 @@ source "$commonDir/subst-vars.sh" : "${PATH?} ${coreutils?} ${dot?} ${SHELL?} ${PAGER?} ${busybox?} ${version?} ${system?} ${BUILD_SHARED_LIBS?}" export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default/tests\/functional//} -export NIX_STORE_DIR -if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then - # Maybe the build directory is symlinked. - export NIX_IGNORE_SYMLINK_STORE=1 - NIX_STORE_DIR=$TEST_ROOT/store -fi -export NIX_LOCALSTATE_DIR=$TEST_ROOT/var -export NIX_LOG_DIR=$TEST_ROOT/var/log/nix -export NIX_STATE_DIR=$TEST_ROOT/var/nix -export NIX_CONF_DIR=$TEST_ROOT/etc -export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/dSocket -unset NIX_USER_CONF_FILES -export _NIX_TEST_SHARED=$TEST_ROOT/shared -if [[ -n $NIX_STORE ]]; then - export _NIX_TEST_NO_SANDBOX=1 -fi -export _NIX_IN_TEST=$TEST_ROOT/shared -export _NIX_TEST_NO_LSOF=1 -export NIX_REMOTE=${NIX_REMOTE_-} -unset NIX_PATH +test_nix_conf_dir=$TEST_ROOT/etc +test_nix_conf=$test_nix_conf_dir/nix.conf + export TEST_HOME=$TEST_ROOT/test-home + +if ! isTestOnNixOS; then + export NIX_STORE_DIR + if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then + # Maybe the build directory is symlinked. + export NIX_IGNORE_SYMLINK_STORE=1 + NIX_STORE_DIR=$TEST_ROOT/store + fi + export NIX_LOCALSTATE_DIR=$TEST_ROOT/var + export NIX_LOG_DIR=$TEST_ROOT/var/log/nix + export NIX_STATE_DIR=$TEST_ROOT/var/nix + export NIX_CONF_DIR=$test_nix_conf_dir + export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/dSocket + unset NIX_USER_CONF_FILES + export _NIX_TEST_SHARED=$TEST_ROOT/shared + if [[ -n $NIX_STORE ]]; then + export _NIX_TEST_NO_SANDBOX=1 + fi + export _NIX_IN_TEST=$TEST_ROOT/shared + export _NIX_TEST_NO_LSOF=1 + export NIX_REMOTE=${NIX_REMOTE_-} + +fi # ! isTestOnNixOS + +unset NIX_PATH export HOME=$TEST_HOME unset XDG_STATE_HOME unset XDG_DATA_HOME @@ -66,6 +83,10 @@ clearProfiles() { } clearStore() { + if isTestOnNixOS; then + die "clearStore: not supported when testing on NixOS. Is it really needed? If so add conditionals; e.g. if ! isTestOnNixOS; then ..." + fi + echo "clearing store..." chmod -R +w "$NIX_STORE_DIR" rm -rf "$NIX_STORE_DIR" @@ -84,6 +105,10 @@ clearCacheCache() { } startDaemon() { + if isTestOnNixOS; then + die "startDaemon: not supported when testing on NixOS. Is it really needed? If so add conditionals; e.g. if ! isTestOnNixOS; then ..." + fi + # Don’t start the daemon twice, as this would just make it loop indefinitely if [[ "${_NIX_TEST_DAEMON_PID-}" != '' ]]; then return @@ -110,6 +135,10 @@ startDaemon() { } killDaemon() { + if isTestOnNixOS; then + die "killDaemon: not supported when testing on NixOS. Is it really needed? If so add conditionals; e.g. if ! isTestOnNixOS; then ..." + fi + # Don’t fail trying to stop a non-existant daemon twice if [[ "${_NIX_TEST_DAEMON_PID-}" == '' ]]; then return @@ -130,6 +159,10 @@ killDaemon() { } restartDaemon() { + if isTestOnNixOS; then + die "restartDaemon: not supported when testing on NixOS. Is it really needed? If so add conditionals; e.g. if ! isTestOnNixOS; then ..." + fi + [[ -z "${_NIX_TEST_DAEMON_PID:-}" ]] && return 0 killDaemon @@ -152,6 +185,12 @@ skipTest () { exit 99 } +TODO_NixOS() { + if isTestOnNixOS; then + skipTest "This test has not been adapted for NixOS yet" + fi +} + requireDaemonNewerThan () { isDaemonNewer "$1" || skipTest "Daemon is too old" } @@ -234,7 +273,7 @@ buggyNeedLocalStore() { enableFeatures() { local features="$1" - sed -i 's/experimental-features .*/& '"$features"'/' "$NIX_CONF_DIR"/nix.conf + sed -i 's/experimental-features .*/& '"$features"'/' "$test_nix_conf_dir"/nix.conf } set -x diff --git a/tests/functional/impure-env.sh b/tests/functional/impure-env.sh index 3c7df169e..e65c78b00 100755 --- a/tests/functional/impure-env.sh +++ b/tests/functional/impure-env.sh @@ -20,13 +20,13 @@ startDaemon varTest env_name value --impure-env env_name=value -echo 'impure-env = set_in_config=config_value' >> "$NIX_CONF_DIR/nix.conf" +echo 'impure-env = set_in_config=config_value' >> "$test_nix_conf" set_in_config=daemon_value restartDaemon varTest set_in_config config_value varTest set_in_config client_value --impure-env set_in_config=client_value -sed -i -e '/^trusted-users =/d' "$NIX_CONF_DIR/nix.conf" +sed -i -e '/^trusted-users =/d' "$test_nix_conf" env_name=daemon_value restartDaemon diff --git a/tests/functional/local-overlay-store/common.sh b/tests/functional/local-overlay-store/common.sh index 0e6097861..bbe84847a 100644 --- a/tests/functional/local-overlay-store/common.sh +++ b/tests/functional/local-overlay-store/common.sh @@ -31,7 +31,7 @@ requireEnvironment () { } addConfig () { - echo "$1" >> "$NIX_CONF_DIR/nix.conf" + echo "$1" >> "$test_nix_conf" } setupConfig () { diff --git a/tests/functional/post-hook.sh b/tests/functional/post-hook.sh index c0b1ab3aa..7540d1045 100755 --- a/tests/functional/post-hook.sh +++ b/tests/functional/post-hook.sh @@ -7,7 +7,7 @@ clearStore rm -f $TEST_ROOT/result export REMOTE_STORE=file:$TEST_ROOT/remote_store -echo 'require-sigs = false' >> $NIX_CONF_DIR/nix.conf +echo 'require-sigs = false' >> $test_nix_conf restartDaemon diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index 710f8a273..2ab00b336 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -132,4 +132,11 @@ in ca-fd-leak = runNixOSTestFor "x86_64-linux" ./ca-fd-leak; gzip-content-encoding = runNixOSTestFor "x86_64-linux" ./gzip-content-encoding.nix; + + functional_user = runNixOSTestFor "x86_64-linux" ./functional/as-user.nix; + + functional_trusted = runNixOSTestFor "x86_64-linux" ./functional/as-trusted-user.nix; + + functional_root = runNixOSTestFor "x86_64-linux" ./functional/as-root.nix; + } diff --git a/tests/nixos/functional/as-root.nix b/tests/nixos/functional/as-root.nix new file mode 100644 index 000000000..96be3d593 --- /dev/null +++ b/tests/nixos/functional/as-root.nix @@ -0,0 +1,12 @@ +{ + name = "functional-tests-on-nixos_root"; + + imports = [ ./common.nix ]; + + testScript = '' + machine.wait_for_unit("multi-user.target") + machine.succeed(""" + run-test-suite >&2 + """) + ''; +} diff --git a/tests/nixos/functional/as-trusted-user.nix b/tests/nixos/functional/as-trusted-user.nix new file mode 100644 index 000000000..d6f825697 --- /dev/null +++ b/tests/nixos/functional/as-trusted-user.nix @@ -0,0 +1,18 @@ +{ + name = "functional-tests-on-nixos_trusted-user"; + + imports = [ ./common.nix ]; + + nodes.machine = { + users.users.alice = { isNormalUser = true; }; + nix.settings.trusted-users = [ "alice" ]; + }; + + testScript = '' + machine.wait_for_unit("multi-user.target") + machine.succeed(""" + export TEST_TRUSTED_USER=1 + su --login --command "run-test-suite" alice >&2 + """) + ''; +} \ No newline at end of file diff --git a/tests/nixos/functional/as-user.nix b/tests/nixos/functional/as-user.nix new file mode 100644 index 000000000..1443f6e6c --- /dev/null +++ b/tests/nixos/functional/as-user.nix @@ -0,0 +1,16 @@ +{ + name = "functional-tests-on-nixos_user"; + + imports = [ ./common.nix ]; + + nodes.machine = { + users.users.alice = { isNormalUser = true; }; + }; + + testScript = '' + machine.wait_for_unit("multi-user.target") + machine.succeed(""" + su --login --command "run-test-suite" alice >&2 + """) + ''; +} diff --git a/tests/nixos/functional/common.nix b/tests/nixos/functional/common.nix new file mode 100644 index 000000000..493791a7b --- /dev/null +++ b/tests/nixos/functional/common.nix @@ -0,0 +1,76 @@ +{ lib, ... }: + +let + # FIXME (roberth) reference issue + inputDerivation = pkg: (pkg.overrideAttrs (o: { + disallowedReferences = [ ]; + })).inputDerivation; + +in +{ + imports = [ + # Add the quickBuild attribute to the check package + ./quick-build.nix + ]; + + # We rarely change the script in a way that benefits from type checking, so + # we skip it to save time. + skipTypeCheck = true; + + nodes.machine = { config, pkgs, ... }: { + + virtualisation.writableStore = true; + system.extraDependencies = [ + (inputDerivation config.nix.package) + ]; + + nix.settings.substituters = lib.mkForce []; + + environment.systemPackages = let + run-test-suite = pkgs.writeShellApplication { + name = "run-test-suite"; + runtimeInputs = [ pkgs.gnumake pkgs.jq pkgs.git ]; + text = '' + set -x + cat /proc/sys/fs/file-max + ulimit -Hn + ulimit -Sn + cd ~ + cp -r ${pkgs.nix.overrideAttrs (o: { + name = "nix-configured-source"; + outputs = [ "out" ]; + separateDebugInfo = false; + disallowedReferences = [ ]; + buildPhase = ":"; + checkPhase = ":"; + installPhase = '' + cp -r . $out + ''; + installCheckPhase = ":"; + fixupPhase = ":"; + doInstallCheck = true; + })} nix + chmod -R +w nix + cd nix + + # Tests we don't need + echo >tests/functional/plugins/local.mk + sed -i tests/functional/local.mk \ + -e 's!nix_tests += plugins\.sh!!' \ + -e 's!nix_tests += test-libstoreconsumer\.sh!!' \ + ; + + export isTestOnNixOS=1 + export version=${config.nix.package.version} + export NIX_REMOTE_=daemon + export NIX_REMOTE=daemon + export NIX_STORE=${builtins.storeDir} + make -j1 installcheck --keep-going + ''; + }; + in [ + run-test-suite + pkgs.git + ]; + }; +} diff --git a/tests/nixos/functional/quick-build.nix b/tests/nixos/functional/quick-build.nix new file mode 100644 index 000000000..10f4215e5 --- /dev/null +++ b/tests/nixos/functional/quick-build.nix @@ -0,0 +1,47 @@ +test@{ lib, extendModules, ... }: +let + inherit (lib) mkOption types; +in +{ + options = { + quickBuild = mkOption { + description = '' + Whether to perform a "quick" build of the Nix package to test. + + When iterating on the functional tests, it's recommended to "set" this + to `true`, so that changes to the functional tests don't require any + recompilation of the package. + You can do so by buildin the `.quickBuild` attribute on the check package, + e.g: + ```console + nix build .#hydraJobs.functional_user.quickBuild + ``` + + We don't enable this by default to avoid the mostly unnecessary work of + performing an additional build of the package in cases where we build + the package normally anyway, such as in our pre-merge CI. + ''; + type = types.bool; + default = false; + }; + }; + + config = { + passthru.quickBuild = + let withQuickBuild = extendModules { modules = [{ quickBuild = true; }]; }; + in withQuickBuild.config.test; + + defaults = { pkgs, ... }: { + config = lib.mkIf test.config.quickBuild { + nix.package = pkgs.nix_noTests; + + system.forbiddenDependenciesRegexes = [ + # This would indicate that the quickBuild feature is broken. + # It could happen if NixOS has a dependency on pkgs.nix instead of + # config.nix.package somewhere. + (builtins.unsafeDiscardStringContext pkgs.nix.outPath) + ]; + }; + }; + }; +} \ No newline at end of file From 211aec473e8dfe9782c34b40a72db8ae6e88df99 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 16 Jun 2024 13:02:04 +0200 Subject: [PATCH 14/61] tests/functional/timeout.sh: Find missing test case This reproduces an instance of https://github.com/NixOS/nix/issues/4813 --- tests/functional/timeout.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/functional/timeout.sh b/tests/functional/timeout.sh index 441c83b0e..f42354538 100755 --- a/tests/functional/timeout.sh +++ b/tests/functional/timeout.sh @@ -11,6 +11,10 @@ messages=$(nix-build -Q timeout.nix -A infiniteLoop --timeout 2 2>&1) && status= if [ $status -ne 101 ]; then echo "error: 'nix-store' exited with '$status'; should have exited 101" + + # FIXME: https://github.com/NixOS/nix/issues/4813 + skipTest "Do not block CI until fixed" + exit 1 fi From 8557d79650e8097a73a809583405a94ebaf0a0db Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 16 Jun 2024 12:51:46 +0200 Subject: [PATCH 15/61] tests/functional: Skip tests that don't work in NixOS environment yet --- tests/functional/add.sh | 2 ++ tests/functional/binary-cache-build-remote.sh | 2 ++ tests/functional/binary-cache.sh | 2 ++ tests/functional/brotli.sh | 2 ++ tests/functional/build-delete.sh | 2 ++ tests/functional/build-dry.sh | 2 ++ tests/functional/build-remote-trustless-should-fail-0.sh | 1 + tests/functional/build-remote-trustless-should-pass-2.sh | 2 ++ tests/functional/build-remote-trustless-should-pass-3.sh | 1 + tests/functional/build.sh | 2 ++ tests/functional/ca/common.sh | 2 ++ tests/functional/case-hack.sh | 2 ++ tests/functional/check-refs.sh | 2 ++ tests/functional/check-reqs.sh | 2 ++ tests/functional/check.sh | 4 ++++ tests/functional/chroot-store.sh | 2 ++ tests/functional/compression-levels.sh | 2 ++ tests/functional/config.sh | 2 ++ tests/functional/db-migration.sh | 2 ++ tests/functional/debugger.sh | 2 ++ tests/functional/dependencies.sh | 2 ++ tests/functional/dump-db.sh | 2 ++ tests/functional/dyn-drv/common.sh | 2 ++ tests/functional/eval-store.sh | 2 ++ tests/functional/eval.sh | 2 ++ tests/functional/export-graph.sh | 2 ++ tests/functional/export.sh | 2 ++ tests/functional/fetchClosure.sh | 2 ++ tests/functional/fetchGit.sh | 2 ++ tests/functional/fetchGitRefs.sh | 2 ++ tests/functional/fetchGitSubmodules.sh | 2 ++ tests/functional/fetchGitVerification.sh | 2 ++ tests/functional/fetchMercurial.sh | 2 ++ tests/functional/fetchTree-file.sh | 2 ++ tests/functional/fetchurl.sh | 2 ++ tests/functional/fixed.sh | 2 ++ tests/functional/flakes/config.sh | 1 + tests/functional/flakes/develop.sh | 2 ++ tests/functional/flakes/flake-in-submodule.sh | 2 ++ tests/functional/flakes/flakes.sh | 2 ++ tests/functional/flakes/run.sh | 2 ++ tests/functional/flakes/search-root.sh | 2 ++ tests/functional/fmt.sh | 2 ++ tests/functional/gc-auto.sh | 2 ++ tests/functional/gc-concurrent.sh | 2 ++ tests/functional/gc-non-blocking.sh | 2 ++ tests/functional/gc-runtime.sh | 2 ++ tests/functional/gc.sh | 2 ++ tests/functional/git-hashing/common.sh | 2 ++ tests/functional/help.sh | 2 ++ tests/functional/import-derivation.sh | 2 ++ tests/functional/impure-derivations.sh | 2 ++ tests/functional/impure-env.sh | 2 ++ tests/functional/linux-sandbox.sh | 2 ++ tests/functional/local-overlay-store/bad-uris.sh | 2 ++ tests/functional/local-overlay-store/common.sh | 2 ++ tests/functional/logging.sh | 2 ++ tests/functional/multiple-outputs.sh | 2 ++ tests/functional/nar-access.sh | 2 ++ tests/functional/nested-sandboxing.sh | 2 ++ tests/functional/nix-build.sh | 2 ++ tests/functional/nix-collect-garbage-d.sh | 2 ++ tests/functional/nix-copy-ssh-common.sh | 2 ++ tests/functional/nix-copy-ssh-ng.sh | 2 ++ tests/functional/nix-profile.sh | 2 ++ tests/functional/nix-shell.sh | 2 ++ tests/functional/optimise-store.sh | 2 ++ tests/functional/output-normalization.sh | 1 + tests/functional/parallel.sh | 2 ++ tests/functional/pass-as-file.sh | 2 ++ tests/functional/placeholders.sh | 2 ++ tests/functional/post-hook.sh | 2 ++ tests/functional/pure-eval.sh | 2 ++ tests/functional/read-only-store.sh | 2 ++ tests/functional/readfile-context.sh | 2 ++ tests/functional/recursive.sh | 2 ++ tests/functional/referrers.sh | 2 ++ tests/functional/remote-store.sh | 2 ++ tests/functional/repair.sh | 2 ++ tests/functional/repl.sh | 2 ++ tests/functional/restricted.sh | 2 ++ tests/functional/search.sh | 2 ++ tests/functional/secure-drv-outputs.sh | 2 ++ tests/functional/selfref-gc.sh | 2 ++ tests/functional/shell.sh | 2 ++ tests/functional/signing.sh | 2 ++ tests/functional/simple.sh | 2 ++ tests/functional/store-info.sh | 2 ++ tests/functional/structured-attrs.sh | 2 ++ tests/functional/suggestions.sh | 2 ++ tests/functional/supplementary-groups.sh | 2 ++ tests/functional/tarball.sh | 2 ++ tests/functional/user-envs-migration.sh | 2 ++ tests/functional/user-envs-test-case.sh | 2 ++ tests/functional/why-depends.sh | 2 ++ tests/functional/zstd.sh | 2 ++ 96 files changed, 190 insertions(+) diff --git a/tests/functional/add.sh b/tests/functional/add.sh index a6cf88e1a..328b14831 100755 --- a/tests/functional/add.sh +++ b/tests/functional/add.sh @@ -31,6 +31,8 @@ test "$hash1" = "sha256:$hash2" #### New style commands +TODO_NixOS + clearStore ( diff --git a/tests/functional/binary-cache-build-remote.sh b/tests/functional/binary-cache-build-remote.sh index 4edda85b6..17e8bdcc1 100755 --- a/tests/functional/binary-cache-build-remote.sh +++ b/tests/functional/binary-cache-build-remote.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore clearCacheCache diff --git a/tests/functional/binary-cache.sh b/tests/functional/binary-cache.sh index 5ef6d89d4..6a177b657 100755 --- a/tests/functional/binary-cache.sh +++ b/tests/functional/binary-cache.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + needLocalStore "'--no-require-sigs' can’t be used with the daemon" # We can produce drvs directly into the binary cache diff --git a/tests/functional/brotli.sh b/tests/functional/brotli.sh index 672e771c2..327eab4a5 100755 --- a/tests/functional/brotli.sh +++ b/tests/functional/brotli.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore clearCache diff --git a/tests/functional/build-delete.sh b/tests/functional/build-delete.sh index 59cf95bd2..2ebf1fd3d 100755 --- a/tests/functional/build-delete.sh +++ b/tests/functional/build-delete.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore # https://github.com/NixOS/nix/issues/6572 diff --git a/tests/functional/build-dry.sh b/tests/functional/build-dry.sh index dca5888a6..cff0f9a49 100755 --- a/tests/functional/build-dry.sh +++ b/tests/functional/build-dry.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + ################################################### # Check that --dry-run isn't confused with read-only mode # https://github.com/NixOS/nix/issues/1795 diff --git a/tests/functional/build-remote-trustless-should-fail-0.sh b/tests/functional/build-remote-trustless-should-fail-0.sh index 269f7f112..4eccb73e0 100755 --- a/tests/functional/build-remote-trustless-should-fail-0.sh +++ b/tests/functional/build-remote-trustless-should-fail-0.sh @@ -4,6 +4,7 @@ source common.sh enableFeatures "daemon-trust-override" +TODO_NixOS restartDaemon requireSandboxSupport diff --git a/tests/functional/build-remote-trustless-should-pass-2.sh b/tests/functional/build-remote-trustless-should-pass-2.sh index ba5d1ff7a..34ce7fbe4 100755 --- a/tests/functional/build-remote-trustless-should-pass-2.sh +++ b/tests/functional/build-remote-trustless-should-pass-2.sh @@ -4,6 +4,8 @@ source common.sh enableFeatures "daemon-trust-override" +TODO_NixOS + restartDaemon # Remote doesn't trust us diff --git a/tests/functional/build-remote-trustless-should-pass-3.sh b/tests/functional/build-remote-trustless-should-pass-3.sh index 187b89948..d01d79191 100755 --- a/tests/functional/build-remote-trustless-should-pass-3.sh +++ b/tests/functional/build-remote-trustless-should-pass-3.sh @@ -4,6 +4,7 @@ source common.sh enableFeatures "daemon-trust-override" +TODO_NixOS restartDaemon # Remote doesn't trusts us, but this is fine because we are only diff --git a/tests/functional/build.sh b/tests/functional/build.sh index a14e6d672..db759b6e9 100755 --- a/tests/functional/build.sh +++ b/tests/functional/build.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore # Make sure that 'nix build' returns all outputs by default. diff --git a/tests/functional/ca/common.sh b/tests/functional/ca/common.sh index b104b5a78..48f1ac46b 100644 --- a/tests/functional/ca/common.sh +++ b/tests/functional/ca/common.sh @@ -2,4 +2,6 @@ source ../common.sh enableFeatures "ca-derivations" +TODO_NixOS + restartDaemon diff --git a/tests/functional/case-hack.sh b/tests/functional/case-hack.sh index 48a2ab13f..feddc6583 100755 --- a/tests/functional/case-hack.sh +++ b/tests/functional/case-hack.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore rm -rf "$TEST_ROOT/case" diff --git a/tests/functional/check-refs.sh b/tests/functional/check-refs.sh index 6534e55c6..26790c11b 100755 --- a/tests/functional/check-refs.sh +++ b/tests/functional/check-refs.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore RESULT=$TEST_ROOT/result diff --git a/tests/functional/check-reqs.sh b/tests/functional/check-reqs.sh index 4d795391e..bb91bacde 100755 --- a/tests/functional/check-reqs.sh +++ b/tests/functional/check-reqs.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore RESULT=$TEST_ROOT/result diff --git a/tests/functional/check.sh b/tests/functional/check.sh index efb93eeb0..8f602b487 100755 --- a/tests/functional/check.sh +++ b/tests/functional/check.sh @@ -15,6 +15,8 @@ checkBuildTempDirRemoved () # written to build temp directories to verify created by this instance checkBuildId=$(date +%s%N) +TODO_NixOS + clearStore nix-build dependencies.nix --no-out-link @@ -76,6 +78,8 @@ grep 'may not be deterministic' $TEST_ROOT/log [ "$status" = "104" ] if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi +TODO_NixOS + clearStore path=$(nix-build check.nix -A fetchurl --no-out-link) diff --git a/tests/functional/chroot-store.sh b/tests/functional/chroot-store.sh index 741907fca..03803a2b9 100755 --- a/tests/functional/chroot-store.sh +++ b/tests/functional/chroot-store.sh @@ -39,6 +39,8 @@ EOF cp simple.nix shell.nix simple.builder.sh config.nix "$flakeDir/" + TODO_NixOS + outPath=$(nix build --print-out-paths --no-link --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' --store "$TEST_ROOT/x" path:"$flakeDir") [[ $outPath =~ ^/nix2/store/.*-simple$ ]] diff --git a/tests/functional/compression-levels.sh b/tests/functional/compression-levels.sh index 6a2111f10..f51c3f509 100755 --- a/tests/functional/compression-levels.sh +++ b/tests/functional/compression-levels.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore clearCache diff --git a/tests/functional/config.sh b/tests/functional/config.sh index 1811b755c..50858eaa4 100755 --- a/tests/functional/config.sh +++ b/tests/functional/config.sh @@ -28,6 +28,8 @@ nix registry remove userhome-with-xdg # Assert the .config folder hasn't been created. [ ! -e "$HOME/.config" ] +TODO_NixOS # Very specific test setup not compatible with the NixOS test environment? + # Test that files are loaded from XDG by default export XDG_CONFIG_HOME=$TEST_ROOT/confighome export XDG_CONFIG_DIRS=$TEST_ROOT/dir1:$TEST_ROOT/dir2 diff --git a/tests/functional/db-migration.sh b/tests/functional/db-migration.sh index a6a5c7744..6feabb90d 100755 --- a/tests/functional/db-migration.sh +++ b/tests/functional/db-migration.sh @@ -10,6 +10,8 @@ if [[ -z "${NIX_DAEMON_PACKAGE-}" ]]; then skipTest "not using the Nix daemon" fi +TODO_NixOS + killDaemon # Fill the db using the older Nix diff --git a/tests/functional/debugger.sh b/tests/functional/debugger.sh index 47e644bb7..178822d79 100755 --- a/tests/functional/debugger.sh +++ b/tests/functional/debugger.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore # regression #9932 diff --git a/tests/functional/dependencies.sh b/tests/functional/dependencies.sh index 1b266935d..bb01b3988 100755 --- a/tests/functional/dependencies.sh +++ b/tests/functional/dependencies.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore drvPath=$(nix-instantiate dependencies.nix) diff --git a/tests/functional/dump-db.sh b/tests/functional/dump-db.sh index 2d0460275..14181b4b6 100755 --- a/tests/functional/dump-db.sh +++ b/tests/functional/dump-db.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + needLocalStore "--dump-db requires a local store" clearStore diff --git a/tests/functional/dyn-drv/common.sh b/tests/functional/dyn-drv/common.sh index c786f6925..0d95881b6 100644 --- a/tests/functional/dyn-drv/common.sh +++ b/tests/functional/dyn-drv/common.sh @@ -5,4 +5,6 @@ requireDaemonNewerThan "2.16.0pre20230419" enableFeatures "ca-derivations dynamic-derivations" +TODO_NixOS + restartDaemon diff --git a/tests/functional/eval-store.sh b/tests/functional/eval-store.sh index 0ab608acc..202e7b004 100755 --- a/tests/functional/eval-store.sh +++ b/tests/functional/eval-store.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + # Using `--eval-store` with the daemon will eventually copy everything # to the build store, invalidating most of the tests here needLocalStore "“--eval-store” doesn't achieve much with the daemon" diff --git a/tests/functional/eval.sh b/tests/functional/eval.sh index acd6e2915..ba335d73a 100755 --- a/tests/functional/eval.sh +++ b/tests/functional/eval.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore testStdinHeredoc=$(nix eval -f - < baz.cat-nar diff -u baz.cat-nar $storePath/foo/baz +TODO_NixOS + # Check that 'nix store cat' fails on invalid store paths. invalidPath="$(dirname $storePath)/99999999999999999999999999999999-foo" cp -r $storePath $invalidPath diff --git a/tests/functional/nested-sandboxing.sh b/tests/functional/nested-sandboxing.sh index 44c3bb2bc..ae0256de2 100755 --- a/tests/functional/nested-sandboxing.sh +++ b/tests/functional/nested-sandboxing.sh @@ -4,6 +4,8 @@ source common.sh # This test is run by `tests/functional/nested-sandboxing/runner.nix` in an extra layer of sandboxing. [[ -d /nix/store ]] || skipTest "running this test without Nix's deps being drawn from /nix/store is not yet supported" +TODO_NixOS + requireSandboxSupport source ./nested-sandboxing/command.sh diff --git a/tests/functional/nix-build.sh b/tests/functional/nix-build.sh index 45ff314c7..cfba7f020 100755 --- a/tests/functional/nix-build.sh +++ b/tests/functional/nix-build.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore outPath=$(nix-build dependencies.nix -o $TEST_ROOT/result) diff --git a/tests/functional/nix-collect-garbage-d.sh b/tests/functional/nix-collect-garbage-d.sh index 07aaf61e9..119efe629 100755 --- a/tests/functional/nix-collect-garbage-d.sh +++ b/tests/functional/nix-collect-garbage-d.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore ## Test `nix-collect-garbage -d` diff --git a/tests/functional/nix-copy-ssh-common.sh b/tests/functional/nix-copy-ssh-common.sh index cc8314ff7..5eea9612d 100644 --- a/tests/functional/nix-copy-ssh-common.sh +++ b/tests/functional/nix-copy-ssh-common.sh @@ -2,6 +2,8 @@ proto=$1 shift (( $# == 0 )) +TODO_NixOS + clearStore clearCache diff --git a/tests/functional/nix-copy-ssh-ng.sh b/tests/functional/nix-copy-ssh-ng.sh index 1fd735b9d..41958c2c3 100755 --- a/tests/functional/nix-copy-ssh-ng.sh +++ b/tests/functional/nix-copy-ssh-ng.sh @@ -4,6 +4,8 @@ source common.sh source nix-copy-ssh-common.sh "ssh-ng" +TODO_NixOS + clearStore clearRemoteStore diff --git a/tests/functional/nix-profile.sh b/tests/functional/nix-profile.sh index 3e5846cf2..e2f19b99e 100755 --- a/tests/functional/nix-profile.sh +++ b/tests/functional/nix-profile.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore clearProfiles diff --git a/tests/functional/nix-shell.sh b/tests/functional/nix-shell.sh index c38107e64..66aece388 100755 --- a/tests/functional/nix-shell.sh +++ b/tests/functional/nix-shell.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore if [[ -n ${CONTENT_ADDRESSED:-} ]]; then diff --git a/tests/functional/optimise-store.sh b/tests/functional/optimise-store.sh index 70ce954f9..f010c5549 100755 --- a/tests/functional/optimise-store.sh +++ b/tests/functional/optimise-store.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore outPath1=$(echo 'with import ./config.nix; mkDerivation { name = "foo1"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link --auto-optimise-store) diff --git a/tests/functional/output-normalization.sh b/tests/functional/output-normalization.sh index 2b319201a..c55f1b1d1 100755 --- a/tests/functional/output-normalization.sh +++ b/tests/functional/output-normalization.sh @@ -3,6 +3,7 @@ source common.sh testNormalization () { + TODO_NixOS clearStore outPath=$(nix-build ./simple.nix --no-out-link) test "$(stat -c %Y $outPath)" -eq 1 diff --git a/tests/functional/parallel.sh b/tests/functional/parallel.sh index 3b7bbe5a2..7e420688d 100644 --- a/tests/functional/parallel.sh +++ b/tests/functional/parallel.sh @@ -4,6 +4,8 @@ source common.sh # First, test that -jN performs builds in parallel. echo "testing nix-build -j..." +TODO_NixOS + clearStore rm -f $_NIX_TEST_SHARED.cur $_NIX_TEST_SHARED.max diff --git a/tests/functional/pass-as-file.sh b/tests/functional/pass-as-file.sh index 21d9ffc6d..0b5605d05 100755 --- a/tests/functional/pass-as-file.sh +++ b/tests/functional/pass-as-file.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore outPath=$(nix-build --no-out-link -E " diff --git a/tests/functional/placeholders.sh b/tests/functional/placeholders.sh index f2b8bf6bf..d4e09a91b 100755 --- a/tests/functional/placeholders.sh +++ b/tests/functional/placeholders.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore nix-build --no-out-link -E ' diff --git a/tests/functional/post-hook.sh b/tests/functional/post-hook.sh index 7540d1045..94a6d0d69 100755 --- a/tests/functional/post-hook.sh +++ b/tests/functional/post-hook.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore rm -f $TEST_ROOT/result diff --git a/tests/functional/pure-eval.sh b/tests/functional/pure-eval.sh index 6d8aa35ec..8a18dec6e 100755 --- a/tests/functional/pure-eval.sh +++ b/tests/functional/pure-eval.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore nix eval --expr 'assert 1 + 2 == 3; true' diff --git a/tests/functional/read-only-store.sh b/tests/functional/read-only-store.sh index ecc57642d..f6b6eaf32 100755 --- a/tests/functional/read-only-store.sh +++ b/tests/functional/read-only-store.sh @@ -6,6 +6,8 @@ enableFeatures "read-only-local-store" needLocalStore "cannot open store read-only when daemon has already opened it writeable" +TODO_NixOS + clearStore happy () { diff --git a/tests/functional/readfile-context.sh b/tests/functional/readfile-context.sh index d0644471d..0ef549c8d 100755 --- a/tests/functional/readfile-context.sh +++ b/tests/functional/readfile-context.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore outPath=$(nix-build --no-out-link readfile-context.nix) diff --git a/tests/functional/recursive.sh b/tests/functional/recursive.sh index a9966aabd..6b255e81a 100755 --- a/tests/functional/recursive.sh +++ b/tests/functional/recursive.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + enableFeatures 'recursive-nix' restartDaemon diff --git a/tests/functional/referrers.sh b/tests/functional/referrers.sh index 0fda97378..411cdb7c1 100755 --- a/tests/functional/referrers.sh +++ b/tests/functional/referrers.sh @@ -4,6 +4,8 @@ source common.sh needLocalStore "uses some low-level store manipulations that aren’t available through the daemon" +TODO_NixOS + clearStore max=500 diff --git a/tests/functional/remote-store.sh b/tests/functional/remote-store.sh index 171a5d391..841b6b27a 100755 --- a/tests/functional/remote-store.sh +++ b/tests/functional/remote-store.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore # Ensure "fake ssh" remote store works just as legacy fake ssh would. diff --git a/tests/functional/repair.sh b/tests/functional/repair.sh index 552e04280..1f6004b2c 100755 --- a/tests/functional/repair.sh +++ b/tests/functional/repair.sh @@ -4,6 +4,8 @@ source common.sh needLocalStore "--repair needs a local store" +TODO_NixOS + clearStore path=$(nix-build dependencies.nix -o $TEST_ROOT/result) diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index fca982807..86cd6f458 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -22,6 +22,8 @@ replUndefinedVariable=" import $testDir/undefined-variable.nix " +TODO_NixOS + testRepl () { local nixArgs nixArgs=("$@") diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh index ab4cad5cf..917a554c5 100755 --- a/tests/functional/restricted.sh +++ b/tests/functional/restricted.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore nix-instantiate --restrict-eval --eval -E '1 + 2' diff --git a/tests/functional/search.sh b/tests/functional/search.sh index ce17411d2..1195c2a06 100755 --- a/tests/functional/search.sh +++ b/tests/functional/search.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore clearCache diff --git a/tests/functional/secure-drv-outputs.sh b/tests/functional/secure-drv-outputs.sh index 7d81db58b..5cc4af435 100755 --- a/tests/functional/secure-drv-outputs.sh +++ b/tests/functional/secure-drv-outputs.sh @@ -6,6 +6,8 @@ source common.sh +TODO_NixOS + clearStore startDaemon diff --git a/tests/functional/selfref-gc.sh b/tests/functional/selfref-gc.sh index 37ce33089..a5e368b51 100755 --- a/tests/functional/selfref-gc.sh +++ b/tests/functional/selfref-gc.sh @@ -4,6 +4,8 @@ source common.sh requireDaemonNewerThan "2.6.0pre20211215" +TODO_NixOS + clearStore nix-build --no-out-link -E ' diff --git a/tests/functional/shell.sh b/tests/functional/shell.sh index 1760eefff..b4c03f547 100755 --- a/tests/functional/shell.sh +++ b/tests/functional/shell.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore clearCache diff --git a/tests/functional/signing.sh b/tests/functional/signing.sh index cf84ab377..d268fd116 100755 --- a/tests/functional/signing.sh +++ b/tests/functional/signing.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore clearCache diff --git a/tests/functional/simple.sh b/tests/functional/simple.sh index 4e7d37f59..86acca0c2 100755 --- a/tests/functional/simple.sh +++ b/tests/functional/simple.sh @@ -17,6 +17,8 @@ echo "output path is $outPath" text=$(cat "$outPath/hello") if test "$text" != "Hello World!"; then exit 1; fi +TODO_NixOS + # Directed delete: $outPath is not reachable from a root, so it should # be deleteable. nix-store --delete $outPath diff --git a/tests/functional/store-info.sh b/tests/functional/store-info.sh index 2398f5beb..f37889fbb 100755 --- a/tests/functional/store-info.sh +++ b/tests/functional/store-info.sh @@ -16,4 +16,6 @@ fi expect 127 NIX_REMOTE=unix:$PWD/store nix store info || \ fail "nix store info on a non-existent store should fail" +TODO_NixOS + [[ "$(echo "$STORE_INFO_JSON" | jq -r ".url")" == "${NIX_REMOTE:-local}" ]] diff --git a/tests/functional/structured-attrs.sh b/tests/functional/structured-attrs.sh index ba7f5967e..32d4c9957 100755 --- a/tests/functional/structured-attrs.sh +++ b/tests/functional/structured-attrs.sh @@ -6,6 +6,8 @@ source common.sh # tests for the older versions requireDaemonNewerThan "2.4pre20210712" +TODO_NixOS + clearStore rm -f $TEST_ROOT/result diff --git a/tests/functional/suggestions.sh b/tests/functional/suggestions.sh index 6ec1cd322..1140e4f78 100755 --- a/tests/functional/suggestions.sh +++ b/tests/functional/suggestions.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore cd "$TEST_HOME" diff --git a/tests/functional/supplementary-groups.sh b/tests/functional/supplementary-groups.sh index 9d474219f..5d329efc9 100755 --- a/tests/functional/supplementary-groups.sh +++ b/tests/functional/supplementary-groups.sh @@ -7,6 +7,8 @@ requireSandboxSupport if ! command -p -v unshare; then skipTest "Need unshare"; fi needLocalStore "The test uses --store always so we would just be bypassing the daemon" +TODO_NixOS + unshare --mount --map-root-user bash < Date: Sun, 16 Jun 2024 16:40:20 +0200 Subject: [PATCH 16/61] tests: Add quickBuild to all VM tests --- tests/nixos/default.nix | 8 +++++++- tests/nixos/functional/common.nix | 5 ----- tests/nixos/{functional => }/quick-build.nix | 0 3 files changed, 7 insertions(+), 6 deletions(-) rename tests/nixos/{functional => }/quick-build.nix (100%) diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index 2ab00b336..512195a67 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -7,7 +7,13 @@ let # https://nixos.org/manual/nixos/unstable/index.html#sec-calling-nixos-tests runNixOSTestFor = system: test: (nixos-lib.runTest { - imports = [ test ]; + imports = [ + test + + # Add the quickBuild attribute to the check packages + ./quick-build.nix + ]; + hostPkgs = nixpkgsFor.${system}.native; defaults = { nixpkgs.pkgs = nixpkgsFor.${system}.native; diff --git a/tests/nixos/functional/common.nix b/tests/nixos/functional/common.nix index 493791a7b..51fd76884 100644 --- a/tests/nixos/functional/common.nix +++ b/tests/nixos/functional/common.nix @@ -8,11 +8,6 @@ let in { - imports = [ - # Add the quickBuild attribute to the check package - ./quick-build.nix - ]; - # We rarely change the script in a way that benefits from type checking, so # we skip it to save time. skipTypeCheck = true; diff --git a/tests/nixos/functional/quick-build.nix b/tests/nixos/quick-build.nix similarity index 100% rename from tests/nixos/functional/quick-build.nix rename to tests/nixos/quick-build.nix From fca160fbcd9f8852b67c99eacf201ea0b693142a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 16 Jun 2024 16:50:56 +0200 Subject: [PATCH 17/61] doc/contributing/testing: Describe functional VM tests and quickBuild --- doc/manual/src/contributing/testing.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index 88b3c5cd9..717deabd7 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -114,6 +114,8 @@ On other platforms they wouldn't be run at all. The functional tests reside under the `tests/functional` directory and are listed in `tests/functional/local.mk`. Each test is a bash script. +Functional tests are run during `installCheck` in the `nix` package build, as well as separately from the build, in VM tests. + ### Running the whole test suite The whole test suite can be run with: @@ -252,13 +254,30 @@ Regressions are caught, and improvements always show up in code review. To ensure that characterisation testing doesn't make it harder to intentionally change these interfaces, there always must be an easy way to regenerate the expected output, as we do with `_NIX_TEST_ACCEPT=1`. +### Running functional tests on NixOS + +We run the functional tests not just in the build, but also in VM tests. +This helps us ensure that Nix works correctly on NixOS, and environments that have similar characteristics that are hard to reproduce in a build environment. + +The recommended way to run these tests during development is: + +```shell +nix build .#hydraJobs.tests.functional_user.quickBuild +``` + +The `quickBuild` attribute configures the test to use a `nix` package that's built without integration tests, so that you can iterate on the tests without performing recompilations due to the changed sources for `installCheck`. + +Generally, this build is sufficient, but in nightly or CI we also test the attributes `functional_root` and `functional_trusted`, in which the test suite is run with different levels of authorization. + ## Integration tests The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup. Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on ). -You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}` +You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`. + +If you are testing a build of `nix` that you haven't compiled yet, you may iterate faster by appending the `quickBuild` attribute: `nix build .#hydraJobs.tests.{testName}.quickBuild`. ## Installer tests From f0abe4d8f0f38e22e61cc79517494437e365375c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 16 Jun 2024 16:57:15 +0200 Subject: [PATCH 18/61] ci: Build tests.functional_user for PRs --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1aa3b776e..6362620b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -175,4 +175,4 @@ jobs: - uses: actions/checkout@v4 - uses: DeterminateSystems/nix-installer-action@main - uses: DeterminateSystems/magic-nix-cache-action@main - - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes + - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes .#hydraJobs.tests.functional_user From 648302b833e6eae4a1c81fe897532e9fa8d7fd6b Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 16 Jun 2024 17:56:50 +0200 Subject: [PATCH 19/61] tests/functional: Enable more tests in NixOS VM --- tests/functional/add.sh | 4 +--- tests/functional/binary-cache-build-remote.sh | 2 +- tests/functional/build-delete.sh | 4 +--- tests/functional/build.sh | 4 +--- tests/functional/check-refs.sh | 19 ++++++++++++------- tests/functional/check-reqs.sh | 4 +--- tests/functional/common/vars-and-functions.sh | 16 +++++++++++++++- tests/functional/compression-levels.sh | 4 +--- tests/functional/debugger.sh | 4 +--- tests/functional/dependencies.sh | 6 +++--- tests/functional/eval.sh | 4 +--- tests/functional/fetchGit.sh | 4 +--- tests/functional/fetchGitRefs.sh | 4 +--- tests/functional/fetchGitSubmodules.sh | 4 +--- tests/functional/fetchGitVerification.sh | 4 +--- tests/functional/flakes/search-root.sh | 4 +--- tests/functional/fmt.sh | 6 ++---- tests/functional/git-hashing/common.sh | 2 +- tests/functional/help.sh | 4 ---- tests/functional/import-derivation.sh | 4 +--- tests/functional/impure-derivations.sh | 2 +- tests/functional/multiple-outputs.sh | 2 +- tests/functional/nix-build.sh | 2 +- tests/functional/nix-shell.sh | 4 +--- tests/functional/optimise-store.sh | 7 ++++--- tests/functional/pass-as-file.sh | 4 +--- tests/functional/placeholders.sh | 4 +--- tests/functional/pure-eval.sh | 4 +--- tests/functional/readfile-context.sh | 2 +- tests/functional/recursive.sh | 2 +- tests/functional/restricted.sh | 4 +--- tests/functional/search.sh | 4 +--- tests/functional/selfref-gc.sh | 4 +--- tests/functional/signing.sh | 5 ++--- tests/functional/structured-attrs.sh | 6 +++--- tests/functional/suggestions.sh | 4 +--- tests/functional/tarball.sh | 4 +--- tests/functional/why-depends.sh | 4 +--- 38 files changed, 71 insertions(+), 104 deletions(-) diff --git a/tests/functional/add.sh b/tests/functional/add.sh index 328b14831..3b37ee7d4 100755 --- a/tests/functional/add.sh +++ b/tests/functional/add.sh @@ -31,9 +31,7 @@ test "$hash1" = "sha256:$hash2" #### New style commands -TODO_NixOS - -clearStore +clearStoreIfPossible ( path1=$(nix store add ./dummy) diff --git a/tests/functional/binary-cache-build-remote.sh b/tests/functional/binary-cache-build-remote.sh index 17e8bdcc1..5046d0064 100755 --- a/tests/functional/binary-cache-build-remote.sh +++ b/tests/functional/binary-cache-build-remote.sh @@ -4,7 +4,7 @@ source common.sh TODO_NixOS -clearStore +clearStoreIfPossible clearCacheCache # Fails without remote builders diff --git a/tests/functional/build-delete.sh b/tests/functional/build-delete.sh index 2ebf1fd3d..18841509d 100755 --- a/tests/functional/build-delete.sh +++ b/tests/functional/build-delete.sh @@ -2,9 +2,7 @@ source common.sh -TODO_NixOS - -clearStore +clearStoreIfPossible # https://github.com/NixOS/nix/issues/6572 issue_6572_independent_outputs() { diff --git a/tests/functional/build.sh b/tests/functional/build.sh index db759b6e9..9de953d8c 100755 --- a/tests/functional/build.sh +++ b/tests/functional/build.sh @@ -2,9 +2,7 @@ source common.sh -TODO_NixOS - -clearStore +clearStoreIfPossible # Make sure that 'nix build' returns all outputs by default. nix build -f multiple-outputs.nix --json a b --no-link | jq --exit-status ' diff --git a/tests/functional/check-refs.sh b/tests/functional/check-refs.sh index 26790c11b..5c3ac915e 100755 --- a/tests/functional/check-refs.sh +++ b/tests/functional/check-refs.sh @@ -45,13 +45,18 @@ nix-build -o "$RESULT" check-refs.nix -A test7 # test10 should succeed (no disallowed references). nix-build -o "$RESULT" check-refs.nix -A test10 -if isDaemonNewer 2.12pre20230103; then - if ! isDaemonNewer 2.16.0; then - enableFeatures discard-references - restartDaemon +if ! isTestOnNixOS; then + # If we have full control over our store, we can test some more things. + + if isDaemonNewer 2.12pre20230103; then + if ! isDaemonNewer 2.16.0; then + enableFeatures discard-references + restartDaemon + fi + + # test11 should succeed. + test11=$(nix-build -o "$RESULT" check-refs.nix -A test11) + [[ -z $(nix-store -q --references "$test11") ]] fi - # test11 should succeed. - test11=$(nix-build -o "$RESULT" check-refs.nix -A test11) - [[ -z $(nix-store -q --references "$test11") ]] fi diff --git a/tests/functional/check-reqs.sh b/tests/functional/check-reqs.sh index bb91bacde..34eb133db 100755 --- a/tests/functional/check-reqs.sh +++ b/tests/functional/check-reqs.sh @@ -2,9 +2,7 @@ source common.sh -TODO_NixOS - -clearStore +clearStoreIfPossible RESULT=$TEST_ROOT/result diff --git a/tests/functional/common/vars-and-functions.sh b/tests/functional/common/vars-and-functions.sh index 4b0fee5b2..8237d853f 100644 --- a/tests/functional/common/vars-and-functions.sh +++ b/tests/functional/common/vars-and-functions.sh @@ -82,11 +82,25 @@ clearProfiles() { rm -rf "$profiles" } +# Clear the store, but do not fail if we're in an environment where we can't. +# This allows the test to run in a NixOS test environment, where we use the system store. +# See doc/manual/src/contributing/testing.md / Running functional tests on NixOS. +clearStoreIfPossible() { + if isTestOnNixOS; then + echo "clearStoreIfPossible: Not clearing store, because we're on NixOS. Moving on." + else + doClearStore + fi +} + clearStore() { if isTestOnNixOS; then - die "clearStore: not supported when testing on NixOS. Is it really needed? If so add conditionals; e.g. if ! isTestOnNixOS; then ..." + die "clearStore: not supported when testing on NixOS. If not essential, call clearStoreIfPossible. If really needed, add conditionals; e.g. if ! isTestOnNixOS; then ..." fi + doClearStore +} +doClearStore() { echo "clearing store..." chmod -R +w "$NIX_STORE_DIR" rm -rf "$NIX_STORE_DIR" diff --git a/tests/functional/compression-levels.sh b/tests/functional/compression-levels.sh index f51c3f509..399265f9c 100755 --- a/tests/functional/compression-levels.sh +++ b/tests/functional/compression-levels.sh @@ -2,9 +2,7 @@ source common.sh -TODO_NixOS - -clearStore +clearStoreIfPossible clearCache outPath=$(nix-build dependencies.nix --no-out-link) diff --git a/tests/functional/debugger.sh b/tests/functional/debugger.sh index 178822d79..b96b7e5d3 100755 --- a/tests/functional/debugger.sh +++ b/tests/functional/debugger.sh @@ -2,9 +2,7 @@ source common.sh -TODO_NixOS - -clearStore +clearStoreIfPossible # regression #9932 echo ":env" | expect 1 nix eval --debugger --expr '(_: throw "oh snap") 42' diff --git a/tests/functional/dependencies.sh b/tests/functional/dependencies.sh index bb01b3988..972bc5a9b 100755 --- a/tests/functional/dependencies.sh +++ b/tests/functional/dependencies.sh @@ -2,9 +2,7 @@ source common.sh -TODO_NixOS - -clearStore +clearStoreIfPossible drvPath=$(nix-instantiate dependencies.nix) @@ -67,6 +65,8 @@ drvPath2=$(nix-instantiate dependencies.nix --argstr hashInvalidator yay) # now --valid-derivers returns both test "$(nix-store -q --valid-derivers "$outPath" | sort)" = "$(sort <<< "$drvPath"$'\n'"$drvPath2")" +TODO_NixOS # The following --delete fails, because it seems to be still alive. This might be caused by a different test using the same path. We should try make the derivations unique, e.g. naming after tests, and adding a timestamp that's constant for that test script run. + # check that nix-store --valid-derivers only returns existing drv nix-store --delete "$drvPath" test "$(nix-store -q --valid-derivers "$outPath")" = "$drvPath2" diff --git a/tests/functional/eval.sh b/tests/functional/eval.sh index ba335d73a..27cdce478 100755 --- a/tests/functional/eval.sh +++ b/tests/functional/eval.sh @@ -2,9 +2,7 @@ source common.sh -TODO_NixOS - -clearStore +clearStoreIfPossible testStdinHeredoc=$(nix eval -f - < $TEST_ROOT/counter diff --git a/tests/functional/multiple-outputs.sh b/tests/functional/multiple-outputs.sh index c4af56f0b..35a78d152 100755 --- a/tests/functional/multiple-outputs.sh +++ b/tests/functional/multiple-outputs.sh @@ -4,7 +4,7 @@ source common.sh TODO_NixOS -clearStore +clearStoreIfPossible rm -f $TEST_ROOT/result* diff --git a/tests/functional/nix-build.sh b/tests/functional/nix-build.sh index cfba7f020..091e429e0 100755 --- a/tests/functional/nix-build.sh +++ b/tests/functional/nix-build.sh @@ -4,7 +4,7 @@ source common.sh TODO_NixOS -clearStore +clearStoreIfPossible outPath=$(nix-build dependencies.nix -o $TEST_ROOT/result) test "$(cat $TEST_ROOT/result/foobar)" = FOOBAR diff --git a/tests/functional/nix-shell.sh b/tests/functional/nix-shell.sh index 66aece388..2c94705de 100755 --- a/tests/functional/nix-shell.sh +++ b/tests/functional/nix-shell.sh @@ -2,9 +2,7 @@ source common.sh -TODO_NixOS - -clearStore +clearStoreIfPossible if [[ -n ${CONTENT_ADDRESSED:-} ]]; then shellDotNix="$PWD/ca-shell.nix" diff --git a/tests/functional/optimise-store.sh b/tests/functional/optimise-store.sh index f010c5549..0bedafc43 100755 --- a/tests/functional/optimise-store.sh +++ b/tests/functional/optimise-store.sh @@ -2,13 +2,14 @@ source common.sh -TODO_NixOS - -clearStore +clearStoreIfPossible outPath1=$(echo 'with import ./config.nix; mkDerivation { name = "foo1"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link --auto-optimise-store) outPath2=$(echo 'with import ./config.nix; mkDerivation { name = "foo2"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link --auto-optimise-store) +TODO_NixOS # ignoring the client-specified setting 'auto-optimise-store', because it is a restricted setting and you are not a trusted user + # TODO: only continue when trusted user or root + inode1="$(stat --format=%i $outPath1/foo)" inode2="$(stat --format=%i $outPath2/foo)" if [ "$inode1" != "$inode2" ]; then diff --git a/tests/functional/pass-as-file.sh b/tests/functional/pass-as-file.sh index 0b5605d05..6487bfffd 100755 --- a/tests/functional/pass-as-file.sh +++ b/tests/functional/pass-as-file.sh @@ -2,9 +2,7 @@ source common.sh -TODO_NixOS - -clearStore +clearStoreIfPossible outPath=$(nix-build --no-out-link -E " with import ./config.nix; diff --git a/tests/functional/placeholders.sh b/tests/functional/placeholders.sh index d4e09a91b..33ec0c2b7 100755 --- a/tests/functional/placeholders.sh +++ b/tests/functional/placeholders.sh @@ -2,9 +2,7 @@ source common.sh -TODO_NixOS - -clearStore +clearStoreIfPossible nix-build --no-out-link -E ' with import ./config.nix; diff --git a/tests/functional/pure-eval.sh b/tests/functional/pure-eval.sh index 8a18dec6e..250381099 100755 --- a/tests/functional/pure-eval.sh +++ b/tests/functional/pure-eval.sh @@ -2,9 +2,7 @@ source common.sh -TODO_NixOS - -clearStore +clearStoreIfPossible nix eval --expr 'assert 1 + 2 == 3; true' diff --git a/tests/functional/readfile-context.sh b/tests/functional/readfile-context.sh index 0ef549c8d..cb9ef6234 100755 --- a/tests/functional/readfile-context.sh +++ b/tests/functional/readfile-context.sh @@ -2,7 +2,7 @@ source common.sh -TODO_NixOS +TODO_NixOS # NixOS doesn't provide $NIX_STATE_DIR (and shouldn't) clearStore diff --git a/tests/functional/recursive.sh b/tests/functional/recursive.sh index 6b255e81a..640fb92d2 100755 --- a/tests/functional/recursive.sh +++ b/tests/functional/recursive.sh @@ -2,7 +2,7 @@ source common.sh -TODO_NixOS +TODO_NixOS # can't enable a sandbox feature easily enableFeatures 'recursive-nix' restartDaemon diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh index 917a554c5..915d973b0 100755 --- a/tests/functional/restricted.sh +++ b/tests/functional/restricted.sh @@ -2,9 +2,7 @@ source common.sh -TODO_NixOS - -clearStore +clearStoreIfPossible nix-instantiate --restrict-eval --eval -E '1 + 2' (! nix-instantiate --eval --restrict-eval ./restricted.nix) diff --git a/tests/functional/search.sh b/tests/functional/search.sh index 1195c2a06..3fadecd02 100755 --- a/tests/functional/search.sh +++ b/tests/functional/search.sh @@ -2,9 +2,7 @@ source common.sh -TODO_NixOS - -clearStore +clearStoreIfPossible clearCache (( $(nix search -f search.nix '' hello | wc -l) > 0 )) diff --git a/tests/functional/selfref-gc.sh b/tests/functional/selfref-gc.sh index a5e368b51..518aea66b 100755 --- a/tests/functional/selfref-gc.sh +++ b/tests/functional/selfref-gc.sh @@ -4,9 +4,7 @@ source common.sh requireDaemonNewerThan "2.6.0pre20211215" -TODO_NixOS - -clearStore +clearStoreIfPossible nix-build --no-out-link -E ' with import ./config.nix; diff --git a/tests/functional/signing.sh b/tests/functional/signing.sh index d268fd116..890d1446f 100755 --- a/tests/functional/signing.sh +++ b/tests/functional/signing.sh @@ -2,9 +2,7 @@ source common.sh -TODO_NixOS - -clearStore +clearStoreIfPossible clearCache nix-store --generate-binary-cache-key cache1.example.org $TEST_ROOT/sk1 $TEST_ROOT/pk1 @@ -18,6 +16,7 @@ outPath=$(nix-build dependencies.nix --no-out-link --secret-key-files "$TEST_ROO # Verify that the path got signed. info=$(nix path-info --json $outPath) echo $info | jq -e '.[] | .ultimate == true' +TODO_NixOS # looks like an actual bug? Following line fails on NixOS: echo $info | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))' echo $info | jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))' diff --git a/tests/functional/structured-attrs.sh b/tests/functional/structured-attrs.sh index 32d4c9957..ec1282668 100755 --- a/tests/functional/structured-attrs.sh +++ b/tests/functional/structured-attrs.sh @@ -6,9 +6,7 @@ source common.sh # tests for the older versions requireDaemonNewerThan "2.4pre20210712" -TODO_NixOS - -clearStore +clearStoreIfPossible rm -f $TEST_ROOT/result @@ -23,6 +21,8 @@ env NIX_PATH=nixpkgs=shell.nix nix-shell structured-attrs-shell.nix \ nix develop -f structured-attrs-shell.nix -c bash -c 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"' +TODO_NixOS # following line fails. + # `nix develop` is a slightly special way of dealing with environment vars, it parses # these from a shell-file exported from a derivation. This is to test especially `outputs` # (which is an associative array in thsi case) being fine. diff --git a/tests/functional/suggestions.sh b/tests/functional/suggestions.sh index 1140e4f78..8db6f7b97 100755 --- a/tests/functional/suggestions.sh +++ b/tests/functional/suggestions.sh @@ -2,9 +2,7 @@ source common.sh -TODO_NixOS - -clearStore +clearStoreIfPossible cd "$TEST_HOME" diff --git a/tests/functional/tarball.sh b/tests/functional/tarball.sh index eb541f3bb..a2824cd12 100755 --- a/tests/functional/tarball.sh +++ b/tests/functional/tarball.sh @@ -2,9 +2,7 @@ source common.sh -TODO_NixOS - -clearStore +clearStoreIfPossible rm -rf $TEST_HOME diff --git a/tests/functional/why-depends.sh b/tests/functional/why-depends.sh index 1ce66f154..ce53546d8 100755 --- a/tests/functional/why-depends.sh +++ b/tests/functional/why-depends.sh @@ -2,9 +2,7 @@ source common.sh -TODO_NixOS - -clearStore +clearStoreIfPossible cp ./dependencies.nix ./dependencies.builder0.sh ./config.nix $TEST_HOME From 7c9f3eeef8f3946a833b5336f8b0339766647058 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 19 Jun 2024 21:46:01 +0200 Subject: [PATCH 20/61] tests/functional/common/init.sh: Use parentheses around negation roberth: Not strictly necessary, but probably a good habit Co-authored-by: Eelco Dolstra --- tests/functional/common/init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/common/init.sh b/tests/functional/common/init.sh index 19cef5af9..405ba3090 100755 --- a/tests/functional/common/init.sh +++ b/tests/functional/common/init.sh @@ -9,7 +9,7 @@ if isTestOnNixOS; then export NIX_USER_CONF_FILES="$test_nix_conf_dir/nix.conf" mkdir -p "$test_nix_conf_dir" "$TEST_HOME" - ! test -e "$test_nix_conf" + (! test -e "$test_nix_conf") cat > "$test_nix_conf_dir/nix.conf" < Date: Thu, 20 Jun 2024 14:49:53 +0200 Subject: [PATCH 21/61] Apply suggestions from code review Co-authored-by: Valentin Gagarin --- tests/functional/common/init.sh | 1 + tests/nixos/quick-build.nix | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/functional/common/init.sh b/tests/functional/common/init.sh index 405ba3090..eb6be7eb0 100755 --- a/tests/functional/common/init.sh +++ b/tests/functional/common/init.sh @@ -11,6 +11,7 @@ if isTestOnNixOS; then mkdir -p "$test_nix_conf_dir" "$TEST_HOME" (! test -e "$test_nix_conf") cat > "$test_nix_conf_dir/nix.conf" < Date: Thu, 20 Jun 2024 19:53:25 +0530 Subject: [PATCH 22/61] fix: handle errors in `nix::createDirs` the `std::filesystem::create_directories` can fail due to insufficient permissions. We convert this error into a `SysError` and catch it wherever required. --- src/libcmd/repl-interacter.cc | 4 ++-- src/libstore/store-api.cc | 2 +- src/libutil/file-system.cc | 6 +++++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc index 254a86d7b..eb4361e25 100644 --- a/src/libcmd/repl-interacter.cc +++ b/src/libcmd/repl-interacter.cc @@ -107,8 +107,8 @@ ReadlineLikeInteracter::Guard ReadlineLikeInteracter::init(detail::ReplCompleter rl_readline_name = "nix-repl"; try { createDirs(dirOf(historyFile)); - } catch (std::filesystem::filesystem_error & e) { - warn(e.what()); + } catch (SystemError & e) { + logWarning(e.info()); } #ifndef USE_READLINE el_hist_size = 1000; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index ed5275377..2edb56510 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1304,7 +1304,7 @@ ref openStore(StoreReference && storeURI) if (!pathExists(chrootStore)) { try { createDirs(chrootStore); - } catch (std::filesystem::filesystem_error & e) { + } catch (SystemError & e) { return std::make_shared(params); } warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 5f269b7c0..443ccf829 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -415,7 +415,11 @@ void deletePath(const fs::path & path) void createDirs(const Path & path) { - fs::create_directories(path); + try { + fs::create_directories(path); + } catch (fs::filesystem_error & e) { + throw SysError("creating directory '%1%'", path); + } } From d9684664c8693b3f1d1ad69b6ed3dbce363748b0 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 20 Jun 2024 20:26:07 +0200 Subject: [PATCH 23/61] Revert "tests/functional/common/init.sh: Use parentheses around negation" ShellCheck doesn't want us to add extra parentheses for show. This reverts commit 7c9f3eeef8f3946a833b5336f8b0339766647058. --- tests/functional/common/init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/common/init.sh b/tests/functional/common/init.sh index eb6be7eb0..d33ad5d57 100755 --- a/tests/functional/common/init.sh +++ b/tests/functional/common/init.sh @@ -9,7 +9,7 @@ if isTestOnNixOS; then export NIX_USER_CONF_FILES="$test_nix_conf_dir/nix.conf" mkdir -p "$test_nix_conf_dir" "$TEST_HOME" - (! test -e "$test_nix_conf") + ! test -e "$test_nix_conf" cat > "$test_nix_conf_dir/nix.conf" < Date: Sun, 23 Jun 2024 00:52:19 +0900 Subject: [PATCH 24/61] accept response from gitlab with more than one entry --- src/libfetchers/github.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 267e8607f..ddb41e63f 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -433,7 +433,7 @@ struct GitLabInputScheme : GitArchiveInputScheme store->toRealPath( downloadFile(store, url, "source", headers).storePath))); - if (json.is_array() && json.size() == 1 && json[0]["id"] != nullptr) { + if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) { return RefInfo { .rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1) }; From 490ca93cf886f046939ba4511dd88e5eb5dcddbc Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 23 Jun 2024 15:33:45 -0400 Subject: [PATCH 25/61] Factor out a bit more language testings infra Will be used in a second test after `lang.sh`. --- maintainers/flake-module.nix | 3 +- ...nfra.sh => characterisation-test-infra.sh} | 2 +- .../empty.exp => characterisation/empty} | 0 .../functional/characterisation/framework.sh | 77 +++++++++++++++++++ tests/functional/lang.sh | 32 +------- tests/functional/lang/framework.sh | 33 -------- tests/functional/local.mk | 2 +- 7 files changed, 82 insertions(+), 67 deletions(-) rename tests/functional/{lang-test-infra.sh => characterisation-test-infra.sh} (98%) rename tests/functional/{lang/empty.exp => characterisation/empty} (100%) create mode 100644 tests/functional/characterisation/framework.sh delete mode 100644 tests/functional/lang/framework.sh diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index b1d21e8fe..0bd353f1a 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -522,6 +522,7 @@ ''^tests/functional/ca/repl\.sh$'' ''^tests/functional/ca/selfref-gc\.sh$'' ''^tests/functional/ca/why-depends\.sh$'' + ''^tests/functional/characterisation-test-infra\.sh$'' ''^tests/functional/check\.sh$'' ''^tests/functional/common/vars-and-functions\.sh$'' ''^tests/functional/completions\.sh$'' @@ -579,9 +580,7 @@ ''^tests/functional/impure-env\.sh$'' ''^tests/functional/impure-eval\.sh$'' ''^tests/functional/install-darwin\.sh$'' - ''^tests/functional/lang-test-infra\.sh$'' ''^tests/functional/lang\.sh$'' - ''^tests/functional/lang/framework\.sh$'' ''^tests/functional/legacy-ssh-store\.sh$'' ''^tests/functional/linux-sandbox\.sh$'' ''^tests/functional/local-overlay-store/add-lower-inner\.sh$'' diff --git a/tests/functional/lang-test-infra.sh b/tests/functional/characterisation-test-infra.sh similarity index 98% rename from tests/functional/lang-test-infra.sh rename to tests/functional/characterisation-test-infra.sh index f32ccef05..279454550 100755 --- a/tests/functional/lang-test-infra.sh +++ b/tests/functional/characterisation-test-infra.sh @@ -3,7 +3,7 @@ # Test the function for lang.sh source common.sh -source lang/framework.sh +source characterisation/framework.sh # We are testing this, so don't want outside world to affect us. unset _NIX_TEST_ACCEPT diff --git a/tests/functional/lang/empty.exp b/tests/functional/characterisation/empty similarity index 100% rename from tests/functional/lang/empty.exp rename to tests/functional/characterisation/empty diff --git a/tests/functional/characterisation/framework.sh b/tests/functional/characterisation/framework.sh new file mode 100644 index 000000000..913fdd967 --- /dev/null +++ b/tests/functional/characterisation/framework.sh @@ -0,0 +1,77 @@ +# shellcheck shell=bash + +# Golden test support +# +# Test that the output of the given test matches what is expected. If +# `_NIX_TEST_ACCEPT` is non-empty also update the expected output so +# that next time the test succeeds. +function diffAndAcceptInner() { + local -r testName=$1 + local -r got="$2" + local -r expected="$3" + + # Absence of expected file indicates empty output expected. + if test -e "$expected"; then + local -r expectedOrEmpty="$expected" + else + local -r expectedOrEmpty=characterisation/empty + fi + + # Diff so we get a nice message + if ! diff --color=always --unified "$expectedOrEmpty" "$got"; then + echo "FAIL: evaluation result of $testName not as expected" + # shellcheck disable=SC2034 + badDiff=1 + fi + + # Update expected if `_NIX_TEST_ACCEPT` is non-empty. + if test -n "${_NIX_TEST_ACCEPT-}"; then + cp "$got" "$expected" + # Delete empty expected files to avoid bloating the repo with + # empty files. + if ! test -s "$expected"; then + rm "$expected" + fi + fi +} + +function characterisationTestExit() { + # Make sure shellcheck knows all these will be defined by the caller + : "${badDiff?} ${badExitCode?}" + + if test -n "${_NIX_TEST_ACCEPT-}"; then + if (( "$badDiff" )); then + set +x + echo 'Output did mot match, but accepted output as the persisted expected output.' + echo 'That means the next time the tests are run, they should pass.' + set -x + else + set +x + echo 'NOTE: Environment variable _NIX_TEST_ACCEPT is defined,' + echo 'indicating the unexpected output should be accepted as the expected output going forward,' + echo 'but no tests had unexpected output so there was no expected output to update.' + set -x + fi + if (( "$badExitCode" )); then + exit "$badExitCode" + else + skipTest "regenerating golden masters" + fi + else + if (( "$badDiff" )); then + set +x + echo '' + echo 'You can rerun this test with:' + echo '' + echo " _NIX_TEST_ACCEPT=1 make tests/functional/${TEST_NAME}.test" + echo '' + echo 'to regenerate the files containing the expected output,' + echo 'and then view the git diff to decide whether a change is' + echo 'good/intentional or bad/unintentional.' + echo 'If the diff contains arbitrary or impure information,' + echo 'please improve the normalization that the test applies to the output.' + set -x + fi + exit $(( "$badExitCode" + "$badDiff" )) + fi +} diff --git a/tests/functional/lang.sh b/tests/functional/lang.sh index 569e7082e..8cb8e98fb 100755 --- a/tests/functional/lang.sh +++ b/tests/functional/lang.sh @@ -4,7 +4,7 @@ source common.sh set -o pipefail -source lang/framework.sh +source characterisation/framework.sh # specialize function a bit function diffAndAccept() { @@ -138,32 +138,4 @@ for i in lang/eval-okay-*.nix; do fi done -if test -n "${_NIX_TEST_ACCEPT-}"; then - if (( "$badDiff" )); then - echo 'Output did mot match, but accepted output as the persisted expected output.' - echo 'That means the next time the tests are run, they should pass.' - else - echo 'NOTE: Environment variable _NIX_TEST_ACCEPT is defined,' - echo 'indicating the unexpected output should be accepted as the expected output going forward,' - echo 'but no tests had unexpected output so there was no expected output to update.' - fi - if (( "$badExitCode" )); then - exit "$badExitCode" - else - skipTest "regenerating golden masters" - fi -else - if (( "$badDiff" )); then - echo '' - echo 'You can rerun this test with:' - echo '' - echo ' _NIX_TEST_ACCEPT=1 make tests/functional/lang.sh.test' - echo '' - echo 'to regenerate the files containing the expected output,' - echo 'and then view the git diff to decide whether a change is' - echo 'good/intentional or bad/unintentional.' - echo 'If the diff contains arbitrary or impure information,' - echo 'please improve the normalization that the test applies to the output.' - fi - exit $(( "$badExitCode" + "$badDiff" )) -fi +characterisationTestExit diff --git a/tests/functional/lang/framework.sh b/tests/functional/lang/framework.sh deleted file mode 100644 index 9b886e983..000000000 --- a/tests/functional/lang/framework.sh +++ /dev/null @@ -1,33 +0,0 @@ -# Golden test support -# -# Test that the output of the given test matches what is expected. If -# `_NIX_TEST_ACCEPT` is non-empty also update the expected output so -# that next time the test succeeds. -function diffAndAcceptInner() { - local -r testName=$1 - local -r got="$2" - local -r expected="$3" - - # Absence of expected file indicates empty output expected. - if test -e "$expected"; then - local -r expectedOrEmpty="$expected" - else - local -r expectedOrEmpty=lang/empty.exp - fi - - # Diff so we get a nice message - if ! diff --color=always --unified "$expectedOrEmpty" "$got"; then - echo "FAIL: evaluation result of $testName not as expected" - badDiff=1 - fi - - # Update expected if `_NIX_TEST_ACCEPT` is non-empty. - if test -n "${_NIX_TEST_ACCEPT-}"; then - cp "$got" "$expected" - # Delete empty expected files to avoid bloating the repo with - # empty files. - if ! test -s "$expected"; then - rm "$expected" - fi - fi -} diff --git a/tests/functional/local.mk b/tests/functional/local.mk index b379eeefe..7e4536323 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -23,7 +23,7 @@ nix_tests = \ remote-store.sh \ legacy-ssh-store.sh \ lang.sh \ - lang-test-infra.sh \ + characterisation-test-infra.sh \ experimental-features.sh \ fetchMercurial.sh \ gc-auto.sh \ From 9f9984e4d07c3176a1781b2149ba7488383ddcde Mon Sep 17 00:00:00 2001 From: HaeNoe Date: Sat, 8 Jun 2024 13:10:51 +0200 Subject: [PATCH 26/61] Functional test for derivation "advanced attrs" This tests the Nix language side of things. We are purposely skipping most of `common.sh` because it is overkill for this test: we don't want to have an "overfit" test environment. Co-Authored-By: John Ericson --- tests/functional/common/paths.sh | 20 +++++++++ tests/functional/common/subst-vars.sh.in | 8 +++- tests/functional/common/test-root.sh | 4 ++ tests/functional/common/vars-and-functions.sh | 14 ++---- .../derivation-advanced-attributes.sh | 23 ++++++++++ .../advanced-attributes-defaults.drv | 1 + .../advanced-attributes-defaults.nix | 6 +++ ...d-attributes-structured-attrs-defaults.drv | 1 + ...d-attributes-structured-attrs-defaults.nix | 8 ++++ .../advanced-attributes-structured-attrs.drv | 1 + .../advanced-attributes-structured-attrs.nix | 45 +++++++++++++++++++ .../derivation/advanced-attributes.drv | 1 + .../derivation/advanced-attributes.nix | 33 ++++++++++++++ tests/functional/local.mk | 1 + 14 files changed, 155 insertions(+), 11 deletions(-) create mode 100644 tests/functional/common/paths.sh create mode 100644 tests/functional/common/test-root.sh create mode 100755 tests/functional/derivation-advanced-attributes.sh create mode 100644 tests/functional/derivation/advanced-attributes-defaults.drv create mode 100644 tests/functional/derivation/advanced-attributes-defaults.nix create mode 100644 tests/functional/derivation/advanced-attributes-structured-attrs-defaults.drv create mode 100644 tests/functional/derivation/advanced-attributes-structured-attrs-defaults.nix create mode 100644 tests/functional/derivation/advanced-attributes-structured-attrs.drv create mode 100644 tests/functional/derivation/advanced-attributes-structured-attrs.nix create mode 100644 tests/functional/derivation/advanced-attributes.drv create mode 100644 tests/functional/derivation/advanced-attributes.nix diff --git a/tests/functional/common/paths.sh b/tests/functional/common/paths.sh new file mode 100644 index 000000000..938b90899 --- /dev/null +++ b/tests/functional/common/paths.sh @@ -0,0 +1,20 @@ +# shellcheck shell=bash + +commonDir="$(readlink -f "$(dirname "${BASH_SOURCE[0]-$0}")")" + +# Since this is a generated file +# shellcheck disable=SC1091 +source "$commonDir/subst-vars.sh" +# Make sure shellcheck knows this will be defined by the above generated snippet +: "${bindir?}" + +export PATH="$bindir:$PATH" + +if [[ -n "${NIX_CLIENT_PACKAGE:-}" ]]; then + export PATH="$NIX_CLIENT_PACKAGE/bin":$PATH +fi + +DAEMON_PATH="$PATH" +if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then + DAEMON_PATH="${NIX_DAEMON_PACKAGE}/bin:$DAEMON_PATH" +fi diff --git a/tests/functional/common/subst-vars.sh.in b/tests/functional/common/subst-vars.sh.in index 4105e9f35..cd89a0033 100644 --- a/tests/functional/common/subst-vars.sh.in +++ b/tests/functional/common/subst-vars.sh.in @@ -1,6 +1,10 @@ # NOTE: instances of @variable@ are substituted as defined in /mk/templates.mk -export PATH=@bindir@:$PATH +if [[ -z "${COMMON_SUBST_VARS_SH_SOURCED-}" ]]; then + +COMMON_SUBST_VARS_SH_SOURCED=1 + +bindir=@bindir@ export coreutils=@coreutils@ #lsof=@lsof@ @@ -13,3 +17,5 @@ export version=@PACKAGE_VERSION@ export system=@system@ export BUILD_SHARED_LIBS=@BUILD_SHARED_LIBS@ + +fi diff --git a/tests/functional/common/test-root.sh b/tests/functional/common/test-root.sh new file mode 100644 index 000000000..b50a06267 --- /dev/null +++ b/tests/functional/common/test-root.sh @@ -0,0 +1,4 @@ +# shellcheck shell=bash + +TEST_ROOT=$(realpath "${TMPDIR:-/tmp}/nix-test")/${TEST_NAME:-default/tests\/functional//} +export TEST_ROOT diff --git a/tests/functional/common/vars-and-functions.sh b/tests/functional/common/vars-and-functions.sh index ad5b29a94..f6ccf9941 100644 --- a/tests/functional/common/vars-and-functions.sh +++ b/tests/functional/common/vars-and-functions.sh @@ -12,9 +12,11 @@ commonDir="$(readlink -f "$(dirname "${BASH_SOURCE[0]-$0}")")" source "$commonDir/subst-vars.sh" # Make sure shellcheck knows all these will be defined by the above generated snippet -: "${PATH?} ${coreutils?} ${dot?} ${SHELL?} ${PAGER?} ${busybox?} ${version?} ${system?} ${BUILD_SHARED_LIBS?}" +: "${bindir?} ${coreutils?} ${dot?} ${SHELL?} ${PAGER?} ${busybox?} ${version?} ${system?} ${BUILD_SHARED_LIBS?}" + +source "$commonDir/paths.sh" +source "$commonDir/test-root.sh" -export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default/tests\/functional//} export NIX_STORE_DIR if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then # Maybe the build directory is symlinked. @@ -43,14 +45,6 @@ unset XDG_CONFIG_HOME unset XDG_CONFIG_DIRS unset XDG_CACHE_HOME -if [[ -n "${NIX_CLIENT_PACKAGE:-}" ]]; then - export PATH="$NIX_CLIENT_PACKAGE/bin":$PATH -fi -DAEMON_PATH="$PATH" -if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then - DAEMON_PATH="${NIX_DAEMON_PACKAGE}/bin:$DAEMON_PATH" -fi - export IMPURE_VAR1=foo export IMPURE_VAR2=bar diff --git a/tests/functional/derivation-advanced-attributes.sh b/tests/functional/derivation-advanced-attributes.sh new file mode 100755 index 000000000..6c0c76b4c --- /dev/null +++ b/tests/functional/derivation-advanced-attributes.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +source common/test-root.sh +source common/paths.sh + +set -o pipefail + +source characterisation/framework.sh + +badDiff=0 +badExitCode=0 + +store="$TEST_ROOT/store" + +for nixFile in derivation/*.nix; do + drvPath=$(nix-instantiate --store "$store" --pure-eval --expr "$(< "$nixFile")") + testName=$(basename "$nixFile" .nix) + got="${store}${drvPath}" + expected="derivation/$testName.drv" + diffAndAcceptInner "$testName" "$got" "$expected" +done + +characterisationTestExit diff --git a/tests/functional/derivation/advanced-attributes-defaults.drv b/tests/functional/derivation/advanced-attributes-defaults.drv new file mode 100644 index 000000000..391c6ab80 --- /dev/null +++ b/tests/functional/derivation/advanced-attributes-defaults.drv @@ -0,0 +1 @@ +Derive([("out","/nix/store/1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults","","")],[],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("builder","/bin/bash"),("name","advanced-attributes-defaults"),("out","/nix/store/1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults"),("system","my-system")]) \ No newline at end of file diff --git a/tests/functional/derivation/advanced-attributes-defaults.nix b/tests/functional/derivation/advanced-attributes-defaults.nix new file mode 100644 index 000000000..51a8d0e7e --- /dev/null +++ b/tests/functional/derivation/advanced-attributes-defaults.nix @@ -0,0 +1,6 @@ +derivation { + name = "advanced-attributes-defaults"; + system = "my-system"; + builder = "/bin/bash"; + args = [ "-c" "echo hello > $out" ]; +} diff --git a/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.drv b/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.drv new file mode 100644 index 000000000..9dd402057 --- /dev/null +++ b/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.drv @@ -0,0 +1 @@ +Derive([("dev","/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev","",""),("out","/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults","","")],[],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__json","{\"builder\":\"/bin/bash\",\"name\":\"advanced-attributes-structured-attrs-defaults\",\"outputs\":[\"out\",\"dev\"],\"system\":\"my-system\"}"),("dev","/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev"),("out","/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults")]) \ No newline at end of file diff --git a/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.nix b/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.nix new file mode 100644 index 000000000..0c13a7691 --- /dev/null +++ b/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.nix @@ -0,0 +1,8 @@ +derivation { + name = "advanced-attributes-structured-attrs-defaults"; + system = "my-system"; + builder = "/bin/bash"; + args = [ "-c" "echo hello > $out" ]; + outputs = [ "out" "dev" ]; + __structuredAttrs = true; +} diff --git a/tests/functional/derivation/advanced-attributes-structured-attrs.drv b/tests/functional/derivation/advanced-attributes-structured-attrs.drv new file mode 100644 index 000000000..e47a41ad5 --- /dev/null +++ b/tests/functional/derivation/advanced-attributes-structured-attrs.drv @@ -0,0 +1 @@ +Derive([("bin","/nix/store/pbzb48v0ycf80jgligcp4n8z0rblna4n-advanced-attributes-structured-attrs-bin","",""),("dev","/nix/store/7xapi8jv7flcz1qq8jhw55ar8ag8hldh-advanced-attributes-structured-attrs-dev","",""),("out","/nix/store/mpq3l1l1qc2yr50q520g08kprprwv79f-advanced-attributes-structured-attrs","","")],[("/nix/store/4xm4wccqsvagz9gjksn24s7rip2fdy7v-foo.drv",["out"]),("/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv",["out"])],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__json","{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"],\"disallowedRequisites\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"],\"allowedRequisites\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}"),("bin","/nix/store/pbzb48v0ycf80jgligcp4n8z0rblna4n-advanced-attributes-structured-attrs-bin"),("dev","/nix/store/7xapi8jv7flcz1qq8jhw55ar8ag8hldh-advanced-attributes-structured-attrs-dev"),("out","/nix/store/mpq3l1l1qc2yr50q520g08kprprwv79f-advanced-attributes-structured-attrs")]) \ No newline at end of file diff --git a/tests/functional/derivation/advanced-attributes-structured-attrs.nix b/tests/functional/derivation/advanced-attributes-structured-attrs.nix new file mode 100644 index 000000000..0044b65fd --- /dev/null +++ b/tests/functional/derivation/advanced-attributes-structured-attrs.nix @@ -0,0 +1,45 @@ +let + system = "my-system"; + foo = derivation { + inherit system; + name = "foo"; + builder = "/bin/bash"; + args = ["-c" "echo foo > $out"]; + }; + bar = derivation { + inherit system; + name = "bar"; + builder = "/bin/bash"; + args = ["-c" "echo bar > $out"]; + }; +in +derivation { + inherit system; + name = "advanced-attributes-structured-attrs"; + builder = "/bin/bash"; + args = [ "-c" "echo hello > $out" ]; + __sandboxProfile = "sandcastle"; + __noChroot = true; + __impureHostDeps = ["/usr/bin/ditto"]; + impureEnvVars = ["UNICORN"]; + __darwinAllowLocalNetworking = true; + outputs = [ "out" "bin" "dev" ]; + __structuredAttrs = true; + outputChecks = { + out = { + allowedReferences = [foo]; + allowedRequisites = [foo]; + }; + bin = { + disallowedReferences = [bar]; + disallowedRequisites = [bar]; + }; + dev = { + maxSize = 789; + maxClosureSize = 5909; + }; + }; + requiredSystemFeatures = ["rainbow" "uid-range"]; + preferLocalBuild = true; + allowSubstitutes = false; +} diff --git a/tests/functional/derivation/advanced-attributes.drv b/tests/functional/derivation/advanced-attributes.drv new file mode 100644 index 000000000..ec3112ab2 --- /dev/null +++ b/tests/functional/derivation/advanced-attributes.drv @@ -0,0 +1 @@ +Derive([("out","/nix/store/33a6fdmn8q9ih9d7npbnrxn2q56a4l8q-advanced-attributes","","")],[("/nix/store/4xm4wccqsvagz9gjksn24s7rip2fdy7v-foo.drv",["out"]),("/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv",["out"])],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__darwinAllowLocalNetworking","1"),("__impureHostDeps","/usr/bin/ditto"),("__noChroot","1"),("__sandboxProfile","sandcastle"),("allowSubstitutes",""),("allowedReferences","/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"),("allowedRequisites","/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"),("builder","/bin/bash"),("disallowedReferences","/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"),("disallowedRequisites","/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"),("impureEnvVars","UNICORN"),("name","advanced-attributes"),("out","/nix/store/33a6fdmn8q9ih9d7npbnrxn2q56a4l8q-advanced-attributes"),("preferLocalBuild","1"),("requiredSystemFeatures","rainbow uid-range"),("system","my-system")]) \ No newline at end of file diff --git a/tests/functional/derivation/advanced-attributes.nix b/tests/functional/derivation/advanced-attributes.nix new file mode 100644 index 000000000..ff680c567 --- /dev/null +++ b/tests/functional/derivation/advanced-attributes.nix @@ -0,0 +1,33 @@ +let + system = "my-system"; + foo = derivation { + inherit system; + name = "foo"; + builder = "/bin/bash"; + args = ["-c" "echo foo > $out"]; + }; + bar = derivation { + inherit system; + name = "bar"; + builder = "/bin/bash"; + args = ["-c" "echo bar > $out"]; + }; +in +derivation { + inherit system; + name = "advanced-attributes"; + builder = "/bin/bash"; + args = [ "-c" "echo hello > $out" ]; + __sandboxProfile = "sandcastle"; + __noChroot = true; + __impureHostDeps = ["/usr/bin/ditto"]; + impureEnvVars = ["UNICORN"]; + __darwinAllowLocalNetworking = true; + allowedReferences = [foo]; + allowedRequisites = [foo]; + disallowedReferences = [bar]; + disallowedRequisites = [bar]; + requiredSystemFeatures = ["rainbow" "uid-range"]; + preferLocalBuild = true; + allowSubstitutes = false; +} diff --git a/tests/functional/local.mk b/tests/functional/local.mk index 7e4536323..49ee31284 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -106,6 +106,7 @@ nix_tests = \ eval-store.sh \ why-depends.sh \ derivation-json.sh \ + derivation-advanced-attributes.sh \ import-derivation.sh \ nix_path.sh \ case-hack.sh \ From 7fb14201afdeebc1327ff3ca75dcb4657c51cdc8 Mon Sep 17 00:00:00 2001 From: HaeNoe Date: Sat, 8 Jun 2024 13:10:51 +0200 Subject: [PATCH 27/61] Unit test for derivation "advanced attrs" This tests the parser and JSON format using the DRV files from the tests added in the previous commit. Co-Authored-By: John Ericson --- maintainers/flake-module.nix | 1 - tests/unit/libstore-support/tests/libstore.hh | 30 ++- .../advanced-attributes-defaults.drv | 1 + .../advanced-attributes-defaults.json | 22 ++ ...d-attributes-structured-attrs-defaults.drv | 1 + ...-attributes-structured-attrs-defaults.json | 24 ++ .../advanced-attributes-structured-attrs.drv | 1 + .../advanced-attributes-structured-attrs.json | 41 +++ .../data/derivation/advanced-attributes.drv | 1 + .../libstore/derivation-advanced-attrs.cc | 234 ++++++++++++++++++ .../libutil-support/tests/characterization.hh | 1 + 11 files changed, 345 insertions(+), 12 deletions(-) create mode 120000 tests/unit/libstore/data/derivation/advanced-attributes-defaults.drv create mode 100644 tests/unit/libstore/data/derivation/advanced-attributes-defaults.json create mode 120000 tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.drv create mode 100644 tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.json create mode 120000 tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.drv create mode 100644 tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.json create mode 120000 tests/unit/libstore/data/derivation/advanced-attributes.drv create mode 100644 tests/unit/libstore/derivation-advanced-attrs.cc diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index 0bd353f1a..5e4291fcb 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -447,7 +447,6 @@ ''^tests/unit/libfetchers/public-key\.cc'' ''^tests/unit/libstore-support/tests/derived-path\.cc'' ''^tests/unit/libstore-support/tests/derived-path\.hh'' - ''^tests/unit/libstore-support/tests/libstore\.hh'' ''^tests/unit/libstore-support/tests/nix_api_store\.hh'' ''^tests/unit/libstore-support/tests/outputs-spec\.cc'' ''^tests/unit/libstore-support/tests/outputs-spec\.hh'' diff --git a/tests/unit/libstore-support/tests/libstore.hh b/tests/unit/libstore-support/tests/libstore.hh index 267188224..84be52c23 100644 --- a/tests/unit/libstore-support/tests/libstore.hh +++ b/tests/unit/libstore-support/tests/libstore.hh @@ -8,19 +8,27 @@ namespace nix { -class LibStoreTest : public virtual ::testing::Test { - public: - static void SetUpTestSuite() { - initLibStore(false); - } +class LibStoreTest : public virtual ::testing::Test +{ +public: + static void SetUpTestSuite() + { + initLibStore(false); + } - protected: - LibStoreTest() - : store(openStore("dummy://")) - { } +protected: + LibStoreTest() + : store(openStore({ + .variant = + StoreReference::Specified{ + .scheme = "dummy", + }, + .params = {}, + })) + { + } - ref store; + ref store; }; - } /* namespace nix */ diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-defaults.drv b/tests/unit/libstore/data/derivation/advanced-attributes-defaults.drv new file mode 120000 index 000000000..353090ad8 --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes-defaults.drv @@ -0,0 +1 @@ +../../../../functional/derivation/advanced-attributes-defaults.drv \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-defaults.json b/tests/unit/libstore/data/derivation/advanced-attributes-defaults.json new file mode 100644 index 000000000..d58e7d5b5 --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes-defaults.json @@ -0,0 +1,22 @@ +{ + "args": [ + "-c", + "echo hello > $out" + ], + "builder": "/bin/bash", + "env": { + "builder": "/bin/bash", + "name": "advanced-attributes-defaults", + "out": "/nix/store/1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults", + "system": "my-system" + }, + "inputDrvs": {}, + "inputSrcs": [], + "name": "advanced-attributes-defaults", + "outputs": { + "out": { + "path": "/nix/store/1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults" + } + }, + "system": "my-system" +} diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.drv b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.drv new file mode 120000 index 000000000..11713da12 --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.drv @@ -0,0 +1 @@ +../../../../functional/derivation/advanced-attributes-structured-attrs-defaults.drv \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.json b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.json new file mode 100644 index 000000000..473d006ac --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.json @@ -0,0 +1,24 @@ +{ + "args": [ + "-c", + "echo hello > $out" + ], + "builder": "/bin/bash", + "env": { + "__json": "{\"builder\":\"/bin/bash\",\"name\":\"advanced-attributes-structured-attrs-defaults\",\"outputs\":[\"out\",\"dev\"],\"system\":\"my-system\"}", + "dev": "/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev", + "out": "/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults" + }, + "inputDrvs": {}, + "inputSrcs": [], + "name": "advanced-attributes-structured-attrs-defaults", + "outputs": { + "dev": { + "path": "/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev" + }, + "out": { + "path": "/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults" + } + }, + "system": "my-system" +} diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.drv b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.drv new file mode 120000 index 000000000..962f8ea3f --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.drv @@ -0,0 +1 @@ +../../../../functional/derivation/advanced-attributes-structured-attrs.drv \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.json b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.json new file mode 100644 index 000000000..324428124 --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.json @@ -0,0 +1,41 @@ +{ + "args": [ + "-c", + "echo hello > $out" + ], + "builder": "/bin/bash", + "env": { + "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"],\"disallowedRequisites\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"],\"allowedRequisites\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}", + "bin": "/nix/store/pbzb48v0ycf80jgligcp4n8z0rblna4n-advanced-attributes-structured-attrs-bin", + "dev": "/nix/store/7xapi8jv7flcz1qq8jhw55ar8ag8hldh-advanced-attributes-structured-attrs-dev", + "out": "/nix/store/mpq3l1l1qc2yr50q520g08kprprwv79f-advanced-attributes-structured-attrs" + }, + "inputDrvs": { + "/nix/store/4xm4wccqsvagz9gjksn24s7rip2fdy7v-foo.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + }, + "/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + } + }, + "inputSrcs": [], + "name": "advanced-attributes-structured-attrs", + "outputs": { + "bin": { + "path": "/nix/store/pbzb48v0ycf80jgligcp4n8z0rblna4n-advanced-attributes-structured-attrs-bin" + }, + "dev": { + "path": "/nix/store/7xapi8jv7flcz1qq8jhw55ar8ag8hldh-advanced-attributes-structured-attrs-dev" + }, + "out": { + "path": "/nix/store/mpq3l1l1qc2yr50q520g08kprprwv79f-advanced-attributes-structured-attrs" + } + }, + "system": "my-system" +} diff --git a/tests/unit/libstore/data/derivation/advanced-attributes.drv b/tests/unit/libstore/data/derivation/advanced-attributes.drv new file mode 120000 index 000000000..2a53a05ca --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes.drv @@ -0,0 +1 @@ +../../../../functional/derivation/advanced-attributes.drv \ No newline at end of file diff --git a/tests/unit/libstore/derivation-advanced-attrs.cc b/tests/unit/libstore/derivation-advanced-attrs.cc new file mode 100644 index 000000000..26cf947a8 --- /dev/null +++ b/tests/unit/libstore/derivation-advanced-attrs.cc @@ -0,0 +1,234 @@ +#include +#include +#include + +#include "experimental-features.hh" +#include "derivations.hh" + +#include "tests/libstore.hh" +#include "tests/characterization.hh" +#include "parsed-derivations.hh" +#include "types.hh" + +namespace nix { + +using nlohmann::json; + +class DerivationAdvancedAttrsTest : public CharacterizationTest, public LibStoreTest +{ + Path unitTestData = getUnitTestData() + "/derivation"; + +public: + Path goldenMaster(std::string_view testStem) const override + { + return unitTestData + "/" + testStem; + } +}; + +#define TEST_ATERM_JSON(STEM, NAME) \ + TEST_F(DerivationAdvancedAttrsTest, Derivation_##STEM##_from_json) \ + { \ + readTest(NAME ".json", [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + /* Use DRV file instead of C++ literal as source of truth. */ \ + auto aterm = readFile(goldenMaster(NAME ".drv")); \ + auto expected = parseDerivation(*store, std::move(aterm), NAME); \ + Derivation got = Derivation::fromJSON(*store, encoded); \ + EXPECT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(DerivationAdvancedAttrsTest, Derivation_##STEM##_to_json) \ + { \ + writeTest( \ + NAME ".json", \ + [&]() -> json { \ + /* Use DRV file instead of C++ literal as source of truth. */ \ + auto aterm = readFile(goldenMaster(NAME ".drv")); \ + return parseDerivation(*store, std::move(aterm), NAME).toJSON(*store); \ + }, \ + [](const auto & file) { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ + } \ + \ + TEST_F(DerivationAdvancedAttrsTest, Derivation_##STEM##_from_aterm) \ + { \ + readTest(NAME ".drv", [&](auto encoded) { \ + /* Use JSON file instead of C++ literal as source of truth. */ \ + auto json = json::parse(readFile(goldenMaster(NAME ".json"))); \ + auto expected = Derivation::fromJSON(*store, json); \ + auto got = parseDerivation(*store, std::move(encoded), NAME); \ + EXPECT_EQ(got.toJSON(*store), expected.toJSON(*store)); \ + EXPECT_EQ(got, expected); \ + }); \ + } \ + \ + /* No corresponding write test, because we need to read the drv to write the json file */ + +TEST_ATERM_JSON(advancedAttributes_defaults, "advanced-attributes-defaults"); +TEST_ATERM_JSON(advancedAttributes, "advanced-attributes-defaults"); +TEST_ATERM_JSON(advancedAttributes_structuredAttrs_defaults, "advanced-attributes-structured-attrs"); +TEST_ATERM_JSON(advancedAttributes_structuredAttrs, "advanced-attributes-structured-attrs-defaults"); + +#undef TEST_ATERM_JSON + +TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_defaults) +{ + readTest("advanced-attributes-defaults.drv", [&](auto encoded) { + auto got = parseDerivation(*store, std::move(encoded), "foo"); + + auto drvPath = writeDerivation(*store, got, NoRepair, true); + + ParsedDerivation parsedDrv(drvPath, got); + + EXPECT_EQ(parsedDrv.getStringAttr("__sandboxProfile").value_or(""), ""); + EXPECT_EQ(parsedDrv.getBoolAttr("__noChroot"), false); + EXPECT_EQ(parsedDrv.getStringsAttr("__impureHostDeps").value_or(Strings()), Strings()); + EXPECT_EQ(parsedDrv.getStringsAttr("impureEnvVars").value_or(Strings()), Strings()); + EXPECT_EQ(parsedDrv.getBoolAttr("__darwinAllowLocalNetworking"), false); + EXPECT_EQ(parsedDrv.getStringsAttr("allowedReferences"), std::nullopt); + EXPECT_EQ(parsedDrv.getStringsAttr("allowedRequisites"), std::nullopt); + EXPECT_EQ(parsedDrv.getStringsAttr("disallowedReferences"), std::nullopt); + EXPECT_EQ(parsedDrv.getStringsAttr("disallowedRequisites"), std::nullopt); + EXPECT_EQ(parsedDrv.getRequiredSystemFeatures(), StringSet()); + EXPECT_EQ(parsedDrv.canBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.willBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.substitutesAllowed(), true); + EXPECT_EQ(parsedDrv.useUidRange(), false); + }); +}; + +TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes) +{ + readTest("advanced-attributes.drv", [&](auto encoded) { + auto got = parseDerivation(*store, std::move(encoded), "foo"); + + auto drvPath = writeDerivation(*store, got, NoRepair, true); + + ParsedDerivation parsedDrv(drvPath, got); + + StringSet systemFeatures{"rainbow", "uid-range"}; + + EXPECT_EQ(parsedDrv.getStringAttr("__sandboxProfile").value_or(""), "sandcastle"); + EXPECT_EQ(parsedDrv.getBoolAttr("__noChroot"), true); + EXPECT_EQ(parsedDrv.getStringsAttr("__impureHostDeps").value_or(Strings()), Strings{"/usr/bin/ditto"}); + EXPECT_EQ(parsedDrv.getStringsAttr("impureEnvVars").value_or(Strings()), Strings{"UNICORN"}); + EXPECT_EQ(parsedDrv.getBoolAttr("__darwinAllowLocalNetworking"), true); + EXPECT_EQ( + parsedDrv.getStringsAttr("allowedReferences"), Strings{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"}); + EXPECT_EQ( + parsedDrv.getStringsAttr("allowedRequisites"), Strings{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"}); + EXPECT_EQ( + parsedDrv.getStringsAttr("disallowedReferences"), + Strings{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"}); + EXPECT_EQ( + parsedDrv.getStringsAttr("disallowedRequisites"), + Strings{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"}); + EXPECT_EQ(parsedDrv.getRequiredSystemFeatures(), systemFeatures); + EXPECT_EQ(parsedDrv.canBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.willBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.substitutesAllowed(), false); + EXPECT_EQ(parsedDrv.useUidRange(), true); + }); +}; + +TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_structuredAttrs_defaults) +{ + readTest("advanced-attributes-structured-attrs-defaults.drv", [&](auto encoded) { + auto got = parseDerivation(*store, std::move(encoded), "foo"); + + auto drvPath = writeDerivation(*store, got, NoRepair, true); + + ParsedDerivation parsedDrv(drvPath, got); + + EXPECT_EQ(parsedDrv.getStringAttr("__sandboxProfile").value_or(""), ""); + EXPECT_EQ(parsedDrv.getBoolAttr("__noChroot"), false); + EXPECT_EQ(parsedDrv.getStringsAttr("__impureHostDeps").value_or(Strings()), Strings()); + EXPECT_EQ(parsedDrv.getStringsAttr("impureEnvVars").value_or(Strings()), Strings()); + EXPECT_EQ(parsedDrv.getBoolAttr("__darwinAllowLocalNetworking"), false); + + { + auto structuredAttrs_ = parsedDrv.getStructuredAttrs(); + ASSERT_TRUE(structuredAttrs_); + auto & structuredAttrs = *structuredAttrs_; + + auto outputChecks_ = get(structuredAttrs, "outputChecks"); + ASSERT_FALSE(outputChecks_); + } + + EXPECT_EQ(parsedDrv.getRequiredSystemFeatures(), StringSet()); + EXPECT_EQ(parsedDrv.canBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.willBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.substitutesAllowed(), true); + EXPECT_EQ(parsedDrv.useUidRange(), false); + }); +}; + +TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_structuredAttrs) +{ + readTest("advanced-attributes-structured-attrs.drv", [&](auto encoded) { + auto got = parseDerivation(*store, std::move(encoded), "foo"); + + auto drvPath = writeDerivation(*store, got, NoRepair, true); + + ParsedDerivation parsedDrv(drvPath, got); + + StringSet systemFeatures{"rainbow", "uid-range"}; + + EXPECT_EQ(parsedDrv.getStringAttr("__sandboxProfile").value_or(""), "sandcastle"); + EXPECT_EQ(parsedDrv.getBoolAttr("__noChroot"), true); + EXPECT_EQ(parsedDrv.getStringsAttr("__impureHostDeps").value_or(Strings()), Strings{"/usr/bin/ditto"}); + EXPECT_EQ(parsedDrv.getStringsAttr("impureEnvVars").value_or(Strings()), Strings{"UNICORN"}); + EXPECT_EQ(parsedDrv.getBoolAttr("__darwinAllowLocalNetworking"), true); + + { + auto structuredAttrs_ = parsedDrv.getStructuredAttrs(); + ASSERT_TRUE(structuredAttrs_); + auto & structuredAttrs = *structuredAttrs_; + + auto outputChecks_ = get(structuredAttrs, "outputChecks"); + ASSERT_TRUE(outputChecks_); + auto & outputChecks = *outputChecks_; + + { + auto output_ = get(outputChecks, "out"); + ASSERT_TRUE(output_); + auto & output = *output_; + EXPECT_EQ( + get(output, "allowedReferences")->get(), + Strings{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"}); + EXPECT_EQ( + get(output, "allowedRequisites")->get(), + Strings{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"}); + } + + { + auto output_ = get(outputChecks, "bin"); + ASSERT_TRUE(output_); + auto & output = *output_; + EXPECT_EQ( + get(output, "disallowedReferences")->get(), + Strings{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"}); + EXPECT_EQ( + get(output, "disallowedRequisites")->get(), + Strings{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"}); + } + + { + auto output_ = get(outputChecks, "dev"); + ASSERT_TRUE(output_); + auto & output = *output_; + EXPECT_EQ(get(output, "maxSize")->get(), 789); + EXPECT_EQ(get(output, "maxClosureSize")->get(), 5909); + } + } + + EXPECT_EQ(parsedDrv.getRequiredSystemFeatures(), systemFeatures); + EXPECT_EQ(parsedDrv.canBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.willBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.substitutesAllowed(), false); + EXPECT_EQ(parsedDrv.useUidRange(), true); + }); +}; + +} diff --git a/tests/unit/libutil-support/tests/characterization.hh b/tests/unit/libutil-support/tests/characterization.hh index c2f686dbf..19ba824ac 100644 --- a/tests/unit/libutil-support/tests/characterization.hh +++ b/tests/unit/libutil-support/tests/characterization.hh @@ -5,6 +5,7 @@ #include "types.hh" #include "environment-variables.hh" +#include "file-system.hh" namespace nix { From 64e599ebe162758a7e077a6c4e319649374307e2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 16 May 2024 17:46:43 -0400 Subject: [PATCH 28/61] Rename `Recursive` -> `NixArchive` For enums: - `FileIngestionMethod` - `FileSerialisationMethod` --- src/libexpr/eval.cc | 2 +- src/libexpr/primops.cc | 10 +++++----- src/libexpr/primops/fetchTree.cc | 2 +- src/libfetchers/fetch-to-store.hh | 2 +- src/libfetchers/fetchers.cc | 2 +- src/libfetchers/mercurial.cc | 2 +- src/libstore/binary-cache-store.cc | 4 ++-- src/libstore/build/worker.cc | 2 +- src/libstore/content-address.cc | 6 +++--- src/libstore/daemon.cc | 12 ++++++------ src/libstore/dummy-store.cc | 4 ++-- src/libstore/legacy-ssh-store.hh | 4 ++-- src/libstore/local-store.cc | 8 ++++---- src/libstore/make-content-addressed.cc | 2 +- src/libstore/optimise-store.cc | 4 ++-- src/libstore/remote-store.cc | 12 ++++++------ src/libstore/remote-store.hh | 4 ++-- src/libstore/store-api.cc | 12 ++++++------ src/libstore/store-api.hh | 10 +++++----- src/libstore/store-dir-config.hh | 2 +- src/libstore/unix/build/local-derivation-goal.cc | 6 +++--- src/libutil/file-content-address.cc | 12 ++++++------ src/libutil/file-content-address.hh | 10 +++++----- src/nix-store/nix-store.cc | 6 +++--- src/nix/add-to-store.cc | 2 +- src/nix/hash.cc | 8 ++++---- src/nix/prefetch.cc | 2 +- src/nix/profile.cc | 2 +- src/perl/lib/Nix/Store.xs | 6 +++--- tests/unit/libstore/common-protocol.cc | 2 +- tests/unit/libstore/content-address.cc | 2 +- tests/unit/libstore/derivation.cc | 6 +++--- tests/unit/libstore/nar-info.cc | 2 +- tests/unit/libstore/path-info.cc | 2 +- tests/unit/libstore/serve-protocol.cc | 4 ++-- tests/unit/libstore/worker-protocol.cc | 4 ++-- tests/unit/libutil/file-content-address.cc | 4 ++-- 37 files changed, 93 insertions(+), 93 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7aaec2e73..fc211e694 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2303,7 +2303,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat path.resolveSymlinks(), settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, path.baseName(), - FileIngestionMethod::Recursive, + FileIngestionMethod::NixArchive, nullptr, repair); allowPath(dstPath); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 98649f081..02b970112 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1209,7 +1209,7 @@ static void derivationStrictInternal( auto handleHashMode = [&](const std::string_view s) { if (s == "recursive") { // back compat, new name is "nar" - ingestionMethod = FileIngestionMethod::Recursive; + ingestionMethod = FileIngestionMethod::NixArchive; } else try { ingestionMethod = ContentAddressMethod::parse(s); } catch (UsageError &) { @@ -1432,7 +1432,7 @@ static void derivationStrictInternal( .atPos(v).debugThrow(); auto ha = outputHashAlgo.value_or(HashAlgorithm::SHA256); - auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive); + auto method = ingestionMethod.value_or(FileIngestionMethod::NixArchive); for (auto & i : outputs) { drv.env[i] = hashPlaceholder(i); @@ -2391,7 +2391,7 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * arg "while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"); state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource"); - addPath(state, pos, path.baseName(), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v, context); + addPath(state, pos, path.baseName(), path, args[0], FileIngestionMethod::NixArchive, std::nullopt, v, context); } static RegisterPrimOp primop_filterSource({ @@ -2454,7 +2454,7 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value std::optional path; std::string name; Value * filterFun = nullptr; - ContentAddressMethod method = FileIngestionMethod::Recursive; + ContentAddressMethod method = FileIngestionMethod::NixArchive; std::optional expectedHash; NixStringContext context; @@ -2470,7 +2470,7 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value state.forceFunction(*(filterFun = attr.value), attr.pos, "while evaluating the `filter` parameter passed to builtins.path"); else if (n == "recursive") method = state.forceBool(*attr.value, attr.pos, "while evaluating the `recursive` attribute passed to builtins.path") - ? FileIngestionMethod::Recursive + ? FileIngestionMethod::NixArchive : FileIngestionMethod::Flat; else if (n == "sha256") expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), HashAlgorithm::SHA256); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index a99a71577..af152f4af 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -468,7 +468,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v auto expectedPath = state.store->makeFixedOutputPath( name, FixedOutputInfo { - .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, + .method = unpack ? FileIngestionMethod::NixArchive : FileIngestionMethod::Flat, .hash = *expectedHash, .references = {} }); diff --git a/src/libfetchers/fetch-to-store.hh b/src/libfetchers/fetch-to-store.hh index 81af1e240..95d1e6b01 100644 --- a/src/libfetchers/fetch-to-store.hh +++ b/src/libfetchers/fetch-to-store.hh @@ -18,7 +18,7 @@ StorePath fetchToStore( const SourcePath & path, FetchMode mode, std::string_view name = "source", - ContentAddressMethod method = FileIngestionMethod::Recursive, + ContentAddressMethod method = FileIngestionMethod::NixArchive, PathFilter * filter = nullptr, RepairFlag repair = NoRepair); diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 73923907c..170a8910c 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -305,7 +305,7 @@ StorePath Input::computeStorePath(Store & store) const if (!narHash) throw Error("cannot compute store path for unlocked input '%s'", to_string()); return store.makeFixedOutputPath(getName(), FixedOutputInfo { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = *narHash, .references = {}, }); diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 7bdf1e937..4b80cbe9b 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -213,7 +213,7 @@ struct MercurialInputScheme : InputScheme auto storePath = store->addToStore( input.getName(), {getFSSourceAccessor(), CanonPath(actualPath)}, - FileIngestionMethod::Recursive, HashAlgorithm::SHA256, {}, + FileIngestionMethod::NixArchive, HashAlgorithm::SHA256, {}, filter); return storePath; diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 95a8d5a7a..e8c8892b3 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -322,7 +322,7 @@ StorePath BinaryCacheStore::addToStoreFromDump( if (static_cast(dumpMethod) == hashMethod.getFileIngestionMethod()) caHash = hashString(HashAlgorithm::SHA256, dump2.s); switch (dumpMethod) { - case FileSerialisationMethod::Recursive: + case FileSerialisationMethod::NixArchive: // The dump is already NAR in this case, just use it. nar = dump2.s; break; @@ -339,7 +339,7 @@ StorePath BinaryCacheStore::addToStoreFromDump( } else { // Otherwise, we have to do th same hashing as NAR so our single // hash will suffice for both purposes. - if (dumpMethod != FileSerialisationMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) + if (dumpMethod != FileSerialisationMethod::NixArchive || hashAlgo != HashAlgorithm::SHA256) unsupported("addToStoreFromDump"); } StringSource narDump { nar }; diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index b53dc771a..31cfa8adc 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -530,7 +530,7 @@ bool Worker::pathContentsGood(const StorePath & path) else { auto current = hashPath( {store.getFSAccessor(), CanonPath(store.printStorePath(path))}, - FileIngestionMethod::Recursive, info->narHash.algo).first; + FileIngestionMethod::NixArchive, info->narHash.algo).first; Hash nullHash(HashAlgorithm::SHA256); res = info->narHash == nullHash || info->narHash == current; } diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 4ed4f2de5..fa06c4aa3 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -9,7 +9,7 @@ std::string_view makeFileIngestionPrefix(FileIngestionMethod m) switch (m) { case FileIngestionMethod::Flat: return ""; - case FileIngestionMethod::Recursive: + case FileIngestionMethod::NixArchive: return "r:"; case FileIngestionMethod::Git: experimentalFeatureSettings.require(Xp::GitHashing); @@ -52,7 +52,7 @@ std::string_view ContentAddressMethod::renderPrefix() const ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m) { if (splitPrefix(m, "r:")) { - return FileIngestionMethod::Recursive; + return FileIngestionMethod::NixArchive; } else if (splitPrefix(m, "git:")) { experimentalFeatureSettings.require(Xp::GitHashing); @@ -137,7 +137,7 @@ static std::pair parseContentAddressMethodP // Parse method auto method = FileIngestionMethod::Flat; if (splitPrefix(rest, "r:")) - method = FileIngestionMethod::Recursive; + method = FileIngestionMethod::NixArchive; else if (splitPrefix(rest, "git:")) { experimentalFeatureSettings.require(Xp::GitHashing); method = FileIngestionMethod::Git; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index fe60cb918..788c5e2ea 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -415,12 +415,12 @@ static void performOp(TunnelLogger * logger, ref store, case FileIngestionMethod::Flat: dumpMethod = FileSerialisationMethod::Flat; break; - case FileIngestionMethod::Recursive: - dumpMethod = FileSerialisationMethod::Recursive; + case FileIngestionMethod::NixArchive: + dumpMethod = FileSerialisationMethod::NixArchive; break; case FileIngestionMethod::Git: // Use NAR; Git is not a serialization method - dumpMethod = FileSerialisationMethod::Recursive; + dumpMethod = FileSerialisationMethod::NixArchive; break; default: assert(false); @@ -441,13 +441,13 @@ static void performOp(TunnelLogger * logger, ref store, uint8_t recursive; std::string hashAlgoRaw; from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw; - if (recursive > (uint8_t) FileIngestionMethod::Recursive) + if (recursive > (uint8_t) FileIngestionMethod::NixArchive) throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive); method = FileIngestionMethod { recursive }; /* Compatibility hack. */ if (!fixed) { hashAlgoRaw = "sha256"; - method = FileIngestionMethod::Recursive; + method = FileIngestionMethod::NixArchive; } hashAlgo = parseHashAlgo(hashAlgoRaw); } @@ -468,7 +468,7 @@ static void performOp(TunnelLogger * logger, ref store, }); logger->startWork(); auto path = store->addToStoreFromDump( - *dumpSource, baseName, FileSerialisationMethod::Recursive, method, hashAlgo); + *dumpSource, baseName, FileSerialisationMethod::NixArchive, method, hashAlgo); logger->stopWork(); to << store->printStorePath(path); diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 0d5d03091..17ebaace6 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -64,8 +64,8 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store virtual StorePath addToStoreFromDump( Source & dump, std::string_view name, - FileSerialisationMethod dumpMethod = FileSerialisationMethod::Recursive, - ContentAddressMethod hashMethod = FileIngestionMethod::Recursive, + FileSerialisationMethod dumpMethod = FileSerialisationMethod::NixArchive, + ContentAddressMethod hashMethod = FileIngestionMethod::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), RepairFlag repair = NoRepair) override diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index b683ed580..db49188ec 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -76,8 +76,8 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor virtual StorePath addToStoreFromDump( Source & dump, std::string_view name, - FileSerialisationMethod dumpMethod = FileSerialisationMethod::Recursive, - ContentAddressMethod hashMethod = FileIngestionMethod::Recursive, + FileSerialisationMethod dumpMethod = FileSerialisationMethod::NixArchive, + ContentAddressMethod hashMethod = FileIngestionMethod::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), RepairFlag repair = NoRepair) override diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 676a035fa..07ace70d0 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1155,7 +1155,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, auto fim = specified.method.getFileIngestionMethod(); switch (fim) { case FileIngestionMethod::Flat: - case FileIngestionMethod::Recursive: + case FileIngestionMethod::NixArchive: { HashModuloSink caSink { specified.hash.algo, @@ -1314,7 +1314,7 @@ StorePath LocalStore::addToStoreFromDump( auto fim = hashMethod.getFileIngestionMethod(); switch (fim) { case FileIngestionMethod::Flat: - case FileIngestionMethod::Recursive: + case FileIngestionMethod::NixArchive: restorePath(realPath, dumpSource, (FileSerialisationMethod) fim); break; case FileIngestionMethod::Git: @@ -1330,7 +1330,7 @@ StorePath LocalStore::addToStoreFromDump( /* For computing the nar hash. In recursive SHA-256 mode, this is the same as the store hash, so no need to do it again. */ auto narHash = std::pair { dumpHash, size }; - if (dumpMethod != FileSerialisationMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) { + if (dumpMethod != FileSerialisationMethod::NixArchive || hashAlgo != HashAlgorithm::SHA256) { HashSink narSink { HashAlgorithm::SHA256 }; dumpPath(realPath, narSink); narHash = narSink.finish(); @@ -1423,7 +1423,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) PosixSourceAccessor accessor; std::string hash = hashPath( PosixSourceAccessor::createAtRoot(link.path()), - FileIngestionMethod::Recursive, HashAlgorithm::SHA256).first.to_string(HashFormat::Nix32, false); + FileIngestionMethod::NixArchive, HashAlgorithm::SHA256).first.to_string(HashFormat::Nix32, false); if (hash != name.string()) { printError("link '%s' was modified! expected hash '%s', got '%s'", link.path(), name, hash); diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 170fe67b9..a3130d7cc 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -52,7 +52,7 @@ std::map makeContentAddressed( dstStore, path.name(), FixedOutputInfo { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = narModuloHash, .references = std::move(refs), }, diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index e9b6d2d50..9d903f218 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -151,7 +151,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, Hash hash = ({ hashPath( {make_ref(), CanonPath(path)}, - FileSerialisationMethod::Recursive, HashAlgorithm::SHA256).first; + FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256).first; }); debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Nix32, true)); @@ -165,7 +165,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, || (repair && hash != ({ hashPath( PosixSourceAccessor::createAtRoot(linkPath), - FileSerialisationMethod::Recursive, HashAlgorithm::SHA256).first; + FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256).first; }))) { // XXX: Consider overwriting linkPath with our valid version. diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index d6efc14f9..9adad9c2a 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -406,8 +406,8 @@ ref RemoteStore::addCAToStore( conn->to << WorkerProto::Op::AddToStore << name - << ((hashAlgo == HashAlgorithm::SHA256 && fim == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ - << (fim == FileIngestionMethod::Recursive ? 1 : 0) + << ((hashAlgo == HashAlgorithm::SHA256 && fim == FileIngestionMethod::NixArchive) ? 0 : 1) /* backwards compatibility hack */ + << (fim == FileIngestionMethod::NixArchive ? 1 : 0) << printHashAlgo(hashAlgo); try { @@ -415,7 +415,7 @@ ref RemoteStore::addCAToStore( connections->incCapacity(); { Finally cleanup([&]() { connections->decCapacity(); }); - if (fim == FileIngestionMethod::Recursive) { + if (fim == FileIngestionMethod::NixArchive) { dump.drainInto(conn->to); } else { std::string contents = dump.drain(); @@ -457,12 +457,12 @@ StorePath RemoteStore::addToStoreFromDump( case FileIngestionMethod::Flat: fsm = FileSerialisationMethod::Flat; break; - case FileIngestionMethod::Recursive: - fsm = FileSerialisationMethod::Recursive; + case FileIngestionMethod::NixArchive: + fsm = FileSerialisationMethod::NixArchive; break; case FileIngestionMethod::Git: // Use NAR; Git is not a serialization method - fsm = FileSerialisationMethod::Recursive; + fsm = FileSerialisationMethod::NixArchive; break; default: assert(false); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index d630adc08..4e1896268 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -87,8 +87,8 @@ public: StorePath addToStoreFromDump( Source & dump, std::string_view name, - FileSerialisationMethod dumpMethod = FileSerialisationMethod::Recursive, - ContentAddressMethod hashMethod = FileIngestionMethod::Recursive, + FileSerialisationMethod dumpMethod = FileSerialisationMethod::NixArchive, + ContentAddressMethod hashMethod = FileIngestionMethod::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), RepairFlag repair = NoRepair) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 2edb56510..8c957ff1a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -122,7 +122,7 @@ StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const Fixed if (info.method == FileIngestionMethod::Git && info.hash.algo != HashAlgorithm::SHA1) throw Error("Git file ingestion must use SHA-1 hash"); - if (info.hash.algo == HashAlgorithm::SHA256 && info.method == FileIngestionMethod::Recursive) { + if (info.hash.algo == HashAlgorithm::SHA256 && info.method == FileIngestionMethod::NixArchive) { return makeStorePath(makeType(*this, "source", info.references), info.hash, name); } else { if (!info.references.empty()) { @@ -200,12 +200,12 @@ StorePath Store::addToStore( case FileIngestionMethod::Flat: fsm = FileSerialisationMethod::Flat; break; - case FileIngestionMethod::Recursive: - fsm = FileSerialisationMethod::Recursive; + case FileIngestionMethod::NixArchive: + fsm = FileSerialisationMethod::NixArchive; break; case FileIngestionMethod::Git: // Use NAR; Git is not a serialization method - fsm = FileSerialisationMethod::Recursive; + fsm = FileSerialisationMethod::NixArchive; break; } auto source = sinkToSource([&](Sink & sink) { @@ -356,7 +356,7 @@ ValidPathInfo Store::addToStoreSlow( RegularFileSink fileSink { caHashSink }; TeeSink unusualHashTee { narHashSink, caHashSink }; - auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != HashAlgorithm::SHA256 + auto & narSink = method == FileIngestionMethod::NixArchive && hashAlgo != HashAlgorithm::SHA256 ? static_cast(unusualHashTee) : narHashSink; @@ -384,7 +384,7 @@ ValidPathInfo Store::addToStoreSlow( finish. */ auto [narHash, narSize] = narHashSink.finish(); - auto hash = method == FileIngestionMethod::Recursive && hashAlgo == HashAlgorithm::SHA256 + auto hash = method == FileIngestionMethod::NixArchive && hashAlgo == HashAlgorithm::SHA256 ? narHash : method == FileIngestionMethod::Git ? git::dumpHash(hashAlgo, srcPath).hash diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 15712458c..e719f9bf9 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -441,7 +441,7 @@ public: virtual StorePath addToStore( std::string_view name, const SourcePath & path, - ContentAddressMethod method = FileIngestionMethod::Recursive, + ContentAddressMethod method = FileIngestionMethod::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), PathFilter & filter = defaultPathFilter, @@ -455,7 +455,7 @@ public: ValidPathInfo addToStoreSlow( std::string_view name, const SourcePath & path, - ContentAddressMethod method = FileIngestionMethod::Recursive, + ContentAddressMethod method = FileIngestionMethod::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), std::optional expectedCAHash = {}); @@ -470,7 +470,7 @@ public: * * @param dumpMethod What serialisation format is `dump`, i.e. how * to deserialize it. Must either match hashMethod or be - * `FileSerialisationMethod::Recursive`. + * `FileSerialisationMethod::NixArchive`. * * @param hashMethod How content addressing? Need not match be the * same as `dumpMethod`. @@ -480,8 +480,8 @@ public: virtual StorePath addToStoreFromDump( Source & dump, std::string_view name, - FileSerialisationMethod dumpMethod = FileSerialisationMethod::Recursive, - ContentAddressMethod hashMethod = FileIngestionMethod::Recursive, + FileSerialisationMethod dumpMethod = FileSerialisationMethod::NixArchive, + ContentAddressMethod hashMethod = FileIngestionMethod::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), RepairFlag repair = NoRepair) = 0; diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh index 643f8854d..fe86ce40d 100644 --- a/src/libstore/store-dir-config.hh +++ b/src/libstore/store-dir-config.hh @@ -97,7 +97,7 @@ struct StoreDirConfig : public Config std::pair computeStorePath( std::string_view name, const SourcePath & path, - ContentAddressMethod method = FileIngestionMethod::Recursive, + ContentAddressMethod method = FileIngestionMethod::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = {}, PathFilter & filter = defaultPathFilter) const; diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 95df0fdee..ab461d428 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -2489,7 +2489,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() auto fim = outputHash.method.getFileIngestionMethod(); switch (fim) { case FileIngestionMethod::Flat: - case FileIngestionMethod::Recursive: + case FileIngestionMethod::NixArchive: { HashModuloSink caSink { outputHash.hashAlgo, oldHashPart }; auto fim = outputHash.method.getFileIngestionMethod(); @@ -2531,7 +2531,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() { HashResult narHashAndSize = hashPath( {getFSSourceAccessor(), CanonPath(actualPath)}, - FileSerialisationMethod::Recursive, HashAlgorithm::SHA256); + FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); newInfo0.narHash = narHashAndSize.first; newInfo0.narSize = narHashAndSize.second; } @@ -2554,7 +2554,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() rewriteOutput(outputRewrites); HashResult narHashAndSize = hashPath( {getFSSourceAccessor(), CanonPath(actualPath)}, - FileSerialisationMethod::Recursive, HashAlgorithm::SHA256); + FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; auto refs = rewriteRefs(); diff --git a/src/libutil/file-content-address.cc b/src/libutil/file-content-address.cc index 8b1e3117a..438dac7da 100644 --- a/src/libutil/file-content-address.cc +++ b/src/libutil/file-content-address.cc @@ -10,7 +10,7 @@ static std::optional parseFileSerialisationMethodOpt(st if (input == "flat") { return FileSerialisationMethod::Flat; } else if (input == "nar") { - return FileSerialisationMethod::Recursive; + return FileSerialisationMethod::NixArchive; } else { return std::nullopt; } @@ -45,7 +45,7 @@ std::string_view renderFileSerialisationMethod(FileSerialisationMethod method) switch (method) { case FileSerialisationMethod::Flat: return "flat"; - case FileSerialisationMethod::Recursive: + case FileSerialisationMethod::NixArchive: return "nar"; default: assert(false); @@ -57,7 +57,7 @@ std::string_view renderFileIngestionMethod(FileIngestionMethod method) { switch (method) { case FileIngestionMethod::Flat: - case FileIngestionMethod::Recursive: + case FileIngestionMethod::NixArchive: return renderFileSerialisationMethod( static_cast(method)); case FileIngestionMethod::Git: @@ -78,7 +78,7 @@ void dumpPath( case FileSerialisationMethod::Flat: path.readFile(sink); break; - case FileSerialisationMethod::Recursive: + case FileSerialisationMethod::NixArchive: path.dumpPath(sink, filter); break; } @@ -94,7 +94,7 @@ void restorePath( case FileSerialisationMethod::Flat: writeFile(path, source); break; - case FileSerialisationMethod::Recursive: + case FileSerialisationMethod::NixArchive: restorePath(path, source); break; } @@ -119,7 +119,7 @@ std::pair> hashPath( { switch (method) { case FileIngestionMethod::Flat: - case FileIngestionMethod::Recursive: { + case FileIngestionMethod::NixArchive: { auto res = hashPath(path, (FileSerialisationMethod) method, ht, filter); return {res.first, {res.second}}; } diff --git a/src/libutil/file-content-address.hh b/src/libutil/file-content-address.hh index e216ee4a7..f3f5589de 100644 --- a/src/libutil/file-content-address.hh +++ b/src/libutil/file-content-address.hh @@ -35,14 +35,14 @@ enum struct FileSerialisationMethod : uint8_t { * See `file-system-object/content-address.md#serial-nix-archive` in * the manual. */ - Recursive, + NixArchive, }; /** * Parse a `FileSerialisationMethod` by name. Choice of: * * - `flat`: `FileSerialisationMethod::Flat` - * - `nar`: `FileSerialisationMethod::Recursive` + * - `nar`: `FileSerialisationMethod::NixArchive` * * Opposite of `renderFileSerialisationMethod`. */ @@ -107,12 +107,12 @@ enum struct FileIngestionMethod : uint8_t { Flat, /** - * Hash `FileSerialisationMethod::Recursive` serialisation. + * Hash `FileSerialisationMethod::NixArchive` serialisation. * * See `file-system-object/content-address.md#serial-flat` in the * manual. */ - Recursive, + NixArchive, /** * Git hashing. @@ -127,7 +127,7 @@ enum struct FileIngestionMethod : uint8_t { * Parse a `FileIngestionMethod` by name. Choice of: * * - `flat`: `FileIngestionMethod::Flat` - * - `nar`: `FileIngestionMethod::Recursive` + * - `nar`: `FileIngestionMethod::NixArchive` * - `git`: `FileIngestionMethod::Git` * * Opposite of `renderFileIngestionMethod`. diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 6d028e0a7..178702f91 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -197,7 +197,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs) auto method = FileIngestionMethod::Flat; for (auto & i : opFlags) - if (i == "--recursive") method = FileIngestionMethod::Recursive; + if (i == "--recursive") method = FileIngestionMethod::NixArchive; else throw UsageError("unknown flag '%1%'", i); if (opArgs.empty()) @@ -223,7 +223,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) auto method = FileIngestionMethod::Flat; for (auto i : opFlags) - if (i == "--recursive") method = FileIngestionMethod::Recursive; + if (i == "--recursive") method = FileIngestionMethod::NixArchive; else throw UsageError("unknown flag '%1%'", i); if (opArgs.size() != 3) @@ -563,7 +563,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) if (!hashGiven) { HashResult hash = hashPath( {store->getFSAccessor(false), CanonPath { store->printStorePath(info->path) }}, - FileSerialisationMethod::Recursive, HashAlgorithm::SHA256); + FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); info->narHash = hash.first; info->narSize = hash.second; } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index af6743375..ed46254f3 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -12,7 +12,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand { Path path; std::optional namePart; - ContentAddressMethod caMethod = FileIngestionMethod::Recursive; + ContentAddressMethod caMethod = FileIngestionMethod::NixArchive; HashAlgorithm hashAlgo = HashAlgorithm::SHA256; CmdAddToStore() diff --git a/src/nix/hash.cc b/src/nix/hash.cc index f57b224d2..62266fda1 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -68,7 +68,7 @@ struct CmdHashBase : Command switch (mode) { case FileIngestionMethod::Flat: return "print cryptographic hash of a regular file"; - case FileIngestionMethod::Recursive: + case FileIngestionMethod::NixArchive: return "print cryptographic hash of the NAR serialisation of a path"; case FileIngestionMethod::Git: return "print cryptographic hash of the Git serialisation of a path"; @@ -91,7 +91,7 @@ struct CmdHashBase : Command Hash h { HashAlgorithm::SHA256 }; // throwaway def to appease C++ switch (mode) { case FileIngestionMethod::Flat: - case FileIngestionMethod::Recursive: + case FileIngestionMethod::NixArchive: { auto hashSink = makeSink(); dumpPath(path2, *hashSink, (FileSerialisationMethod) mode); @@ -126,7 +126,7 @@ struct CmdHashBase : Command struct CmdHashPath : CmdHashBase { CmdHashPath() - : CmdHashBase(FileIngestionMethod::Recursive) + : CmdHashBase(FileIngestionMethod::NixArchive) { addFlag(flag::hashAlgo("algo", &hashAlgo)); addFlag(flag::fileIngestionMethod(&mode)); @@ -311,7 +311,7 @@ static int compatNixHash(int argc, char * * argv) }); if (op == opHash) { - CmdHashBase cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive); + CmdHashBase cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::NixArchive); if (!hashAlgo.has_value()) hashAlgo = HashAlgorithm::MD5; cmd.hashAlgo = hashAlgo.value(); cmd.hashFormat = hashFormat; diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 3ce52acc5..143935572 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -57,7 +57,7 @@ std::tuple prefetchFile( bool unpack, bool executable) { - auto ingestionMethod = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + auto ingestionMethod = unpack || executable ? FileIngestionMethod::NixArchive : FileIngestionMethod::Flat; /* Figure out a name in the Nix store. */ if (!name) { diff --git a/src/nix/profile.cc b/src/nix/profile.cc index a5a40e4f6..c89b8c9bd 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -258,7 +258,7 @@ struct ProfileManifest *store, "profile", FixedOutputInfo { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = narHash, .references = { .others = std::move(references), diff --git a/src/perl/lib/Nix/Store.xs b/src/perl/lib/Nix/Store.xs index 15eb5c4f8..e8e209d97 100644 --- a/src/perl/lib/Nix/Store.xs +++ b/src/perl/lib/Nix/Store.xs @@ -259,7 +259,7 @@ hashPath(char * algo, int base32, char * path) try { Hash h = hashPath( PosixSourceAccessor::createAtRoot(path), - FileIngestionMethod::Recursive, parseHashAlgo(algo)).first; + FileIngestionMethod::NixArchive, parseHashAlgo(algo)).first; auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { @@ -335,7 +335,7 @@ SV * StoreWrapper::addToStore(char * srcPath, int recursive, char * algo) PPCODE: try { - auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + auto method = recursive ? FileIngestionMethod::NixArchive : FileIngestionMethod::Flat; auto path = THIS->store->addToStore( std::string(baseNameOf(srcPath)), PosixSourceAccessor::createAtRoot(srcPath), @@ -351,7 +351,7 @@ StoreWrapper::makeFixedOutputPath(int recursive, char * algo, char * hash, char PPCODE: try { auto h = Hash::parseAny(hash, parseHashAlgo(algo)); - auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + auto method = recursive ? FileIngestionMethod::NixArchive : FileIngestionMethod::Flat; auto path = THIS->store->makeFixedOutputPath(name, FixedOutputInfo { .method = method, .hash = h, diff --git a/tests/unit/libstore/common-protocol.cc b/tests/unit/libstore/common-protocol.cc index d23805fc3..2c629b601 100644 --- a/tests/unit/libstore/common-protocol.cc +++ b/tests/unit/libstore/common-protocol.cc @@ -91,7 +91,7 @@ CHARACTERIZATION_TEST( .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, ContentAddress { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = hashString(HashAlgorithm::SHA256, "(...)"), }, })) diff --git a/tests/unit/libstore/content-address.cc b/tests/unit/libstore/content-address.cc index cc1c7fcc6..f0806bd9a 100644 --- a/tests/unit/libstore/content-address.cc +++ b/tests/unit/libstore/content-address.cc @@ -12,7 +12,7 @@ TEST(ContentAddressMethod, testRoundTripPrintParse_1) { for (const ContentAddressMethod & cam : { ContentAddressMethod { TextIngestionMethod {} }, ContentAddressMethod { FileIngestionMethod::Flat }, - ContentAddressMethod { FileIngestionMethod::Recursive }, + ContentAddressMethod { FileIngestionMethod::NixArchive }, ContentAddressMethod { FileIngestionMethod::Git }, }) { EXPECT_EQ(ContentAddressMethod::parse(cam.render()), cam); diff --git a/tests/unit/libstore/derivation.cc b/tests/unit/libstore/derivation.cc index 7a4b1403a..210813137 100644 --- a/tests/unit/libstore/derivation.cc +++ b/tests/unit/libstore/derivation.cc @@ -117,7 +117,7 @@ TEST_JSON(DerivationTest, caFixedFlat, TEST_JSON(DerivationTest, caFixedNAR, (DerivationOutput::CAFixed { .ca = { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), }, }), @@ -133,7 +133,7 @@ TEST_JSON(DynDerivationTest, caFixedText, TEST_JSON(CaDerivationTest, caFloating, (DerivationOutput::CAFloating { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hashAlgo = HashAlgorithm::SHA256, }), "drv-name", "output-name") @@ -144,7 +144,7 @@ TEST_JSON(DerivationTest, deferred, TEST_JSON(ImpureDerivationTest, impure, (DerivationOutput::Impure { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hashAlgo = HashAlgorithm::SHA256, }), "drv-name", "output-name") diff --git a/tests/unit/libstore/nar-info.cc b/tests/unit/libstore/nar-info.cc index bd10602e7..a6cb62de4 100644 --- a/tests/unit/libstore/nar-info.cc +++ b/tests/unit/libstore/nar-info.cc @@ -25,7 +25,7 @@ static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) { store, "foo", FixedOutputInfo { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = hashString(HashAlgorithm::SHA256, "(...)"), .references = { diff --git a/tests/unit/libstore/path-info.cc b/tests/unit/libstore/path-info.cc index 06c662b74..7637cb366 100644 --- a/tests/unit/libstore/path-info.cc +++ b/tests/unit/libstore/path-info.cc @@ -32,7 +32,7 @@ static UnkeyedValidPathInfo makeFull(const Store & store, bool includeImpureInfo store, "foo", FixedOutputInfo { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = hashString(HashAlgorithm::SHA256, "(...)"), .references = { diff --git a/tests/unit/libstore/serve-protocol.cc b/tests/unit/libstore/serve-protocol.cc index ebf0c52b0..17d7153bb 100644 --- a/tests/unit/libstore/serve-protocol.cc +++ b/tests/unit/libstore/serve-protocol.cc @@ -63,7 +63,7 @@ VERSIONED_CHARACTERIZATION_TEST( .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, ContentAddress { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = hashString(HashAlgorithm::SHA256, "(...)"), }, })) @@ -280,7 +280,7 @@ VERSIONED_CHARACTERIZATION_TEST( *LibStoreTest::store, "foo", FixedOutputInfo { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = hashString(HashAlgorithm::SHA256, "(...)"), .references = { .others = { diff --git a/tests/unit/libstore/worker-protocol.cc b/tests/unit/libstore/worker-protocol.cc index 70e03a8ab..8d81717c9 100644 --- a/tests/unit/libstore/worker-protocol.cc +++ b/tests/unit/libstore/worker-protocol.cc @@ -64,7 +64,7 @@ VERSIONED_CHARACTERIZATION_TEST( .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, ContentAddress { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = hashString(HashAlgorithm::SHA256, "(...)"), }, })) @@ -512,7 +512,7 @@ VERSIONED_CHARACTERIZATION_TEST( *LibStoreTest::store, "foo", FixedOutputInfo { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = hashString(HashAlgorithm::SHA256, "(...)"), .references = { .others = { diff --git a/tests/unit/libutil/file-content-address.cc b/tests/unit/libutil/file-content-address.cc index 294e39806..27d926a87 100644 --- a/tests/unit/libutil/file-content-address.cc +++ b/tests/unit/libutil/file-content-address.cc @@ -11,7 +11,7 @@ namespace nix { TEST(FileSerialisationMethod, testRoundTripPrintParse_1) { for (const FileSerialisationMethod fim : { FileSerialisationMethod::Flat, - FileSerialisationMethod::Recursive, + FileSerialisationMethod::NixArchive, }) { EXPECT_EQ(parseFileSerialisationMethod(renderFileSerialisationMethod(fim)), fim); } @@ -37,7 +37,7 @@ TEST(FileSerialisationMethod, testParseFileSerialisationMethodOptException) { TEST(FileIngestionMethod, testRoundTripPrintParse_1) { for (const FileIngestionMethod fim : { FileIngestionMethod::Flat, - FileIngestionMethod::Recursive, + FileIngestionMethod::NixArchive, FileIngestionMethod::Git, }) { EXPECT_EQ(parseFileIngestionMethod(renderFileIngestionMethod(fim)), fim); From b51e161af57e92691b9d74413e24050ddf9ed2c5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 16 May 2024 19:08:28 -0400 Subject: [PATCH 29/61] Cleanup `ContentAddressMethod` to match docs The old `std::variant` is bad because we aren't adding a new case to `FileIngestionMethod` so much as we are defining a separate concept --- store object content addressing rather than file system object content addressing. As such, it is more correct to just create a fresh enumeration. Co-authored-by: Robert Hensing --- src/libexpr/eval.cc | 2 +- src/libexpr/primops.cc | 22 +-- src/libfetchers/fetch-to-store.hh | 2 +- src/libfetchers/mercurial.cc | 2 +- src/libstore/content-address.cc | 217 +++++++++++++++---------- src/libstore/content-address.hh | 76 +++++---- src/libstore/daemon.cc | 12 +- src/libstore/derivations.cc | 6 +- src/libstore/local-store.cc | 2 +- src/libstore/path-info.cc | 32 ++-- src/libstore/remote-store.cc | 19 ++- src/libstore/store-api.cc | 6 +- src/libstore/store-api.hh | 6 +- src/libutil/file-content-address.hh | 2 + src/nix-env/user-env.cc | 2 +- src/nix-store/nix-store.cc | 4 +- src/nix/add-to-store.cc | 4 +- src/nix/develop.cc | 2 +- src/nix/prefetch.cc | 15 +- src/perl/lib/Nix/Store.xs | 2 +- tests/unit/libstore/common-protocol.cc | 8 +- tests/unit/libstore/content-address.cc | 10 +- tests/unit/libstore/derivation.cc | 9 +- tests/unit/libstore/serve-protocol.cc | 8 +- tests/unit/libstore/worker-protocol.cc | 8 +- 25 files changed, 275 insertions(+), 203 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index fc211e694..0c45a7436 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2303,7 +2303,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat path.resolveSymlinks(), settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, path.baseName(), - FileIngestionMethod::NixArchive, + ContentAddressMethod::Raw::NixArchive, nullptr, repair); allowPath(dstPath); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 02b970112..8a71359de 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1209,7 +1209,7 @@ static void derivationStrictInternal( auto handleHashMode = [&](const std::string_view s) { if (s == "recursive") { // back compat, new name is "nar" - ingestionMethod = FileIngestionMethod::NixArchive; + ingestionMethod = ContentAddressMethod::Raw::NixArchive; } else try { ingestionMethod = ContentAddressMethod::parse(s); } catch (UsageError &) { @@ -1217,9 +1217,9 @@ static void derivationStrictInternal( "invalid value '%s' for 'outputHashMode' attribute", s ).atPos(v).debugThrow(); } - if (ingestionMethod == TextIngestionMethod {}) + if (ingestionMethod == ContentAddressMethod::Raw::Text) experimentalFeatureSettings.require(Xp::DynamicDerivations); - if (ingestionMethod == FileIngestionMethod::Git) + if (ingestionMethod == ContentAddressMethod::Raw::Git) experimentalFeatureSettings.require(Xp::GitHashing); }; @@ -1391,7 +1391,7 @@ static void derivationStrictInternal( /* Check whether the derivation name is valid. */ if (isDerivation(drvName) && - !(ingestionMethod == ContentAddressMethod { TextIngestionMethod { } } && + !(ingestionMethod == ContentAddressMethod::Raw::Text && outputs.size() == 1 && *(outputs.begin()) == "out")) { @@ -1413,7 +1413,7 @@ static void derivationStrictInternal( auto h = newHashAllowEmpty(*outputHash, outputHashAlgo); - auto method = ingestionMethod.value_or(FileIngestionMethod::Flat); + auto method = ingestionMethod.value_or(ContentAddressMethod::Raw::Flat); DerivationOutput::CAFixed dof { .ca = ContentAddress { @@ -1432,7 +1432,7 @@ static void derivationStrictInternal( .atPos(v).debugThrow(); auto ha = outputHashAlgo.value_or(HashAlgorithm::SHA256); - auto method = ingestionMethod.value_or(FileIngestionMethod::NixArchive); + auto method = ingestionMethod.value_or(ContentAddressMethod::Raw::NixArchive); for (auto & i : outputs) { drv.env[i] = hashPlaceholder(i); @@ -2208,7 +2208,7 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val }) : ({ StringSource s { contents }; - state.store->addToStoreFromDump(s, name, FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, state.repair); + state.store->addToStoreFromDump(s, name, FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, refs, state.repair); }); /* Note: we don't need to add `context' to the context of the @@ -2391,7 +2391,7 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * arg "while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"); state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource"); - addPath(state, pos, path.baseName(), path, args[0], FileIngestionMethod::NixArchive, std::nullopt, v, context); + addPath(state, pos, path.baseName(), path, args[0], ContentAddressMethod::Raw::NixArchive, std::nullopt, v, context); } static RegisterPrimOp primop_filterSource({ @@ -2454,7 +2454,7 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value std::optional path; std::string name; Value * filterFun = nullptr; - ContentAddressMethod method = FileIngestionMethod::NixArchive; + auto method = ContentAddressMethod::Raw::NixArchive; std::optional expectedHash; NixStringContext context; @@ -2470,8 +2470,8 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value state.forceFunction(*(filterFun = attr.value), attr.pos, "while evaluating the `filter` parameter passed to builtins.path"); else if (n == "recursive") method = state.forceBool(*attr.value, attr.pos, "while evaluating the `recursive` attribute passed to builtins.path") - ? FileIngestionMethod::NixArchive - : FileIngestionMethod::Flat; + ? ContentAddressMethod::Raw::NixArchive + : ContentAddressMethod::Raw::Flat; else if (n == "sha256") expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), HashAlgorithm::SHA256); else diff --git a/src/libfetchers/fetch-to-store.hh b/src/libfetchers/fetch-to-store.hh index 95d1e6b01..c762629f3 100644 --- a/src/libfetchers/fetch-to-store.hh +++ b/src/libfetchers/fetch-to-store.hh @@ -18,7 +18,7 @@ StorePath fetchToStore( const SourcePath & path, FetchMode mode, std::string_view name = "source", - ContentAddressMethod method = FileIngestionMethod::NixArchive, + ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive, PathFilter * filter = nullptr, RepairFlag repair = NoRepair); diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 4b80cbe9b..198795caa 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -213,7 +213,7 @@ struct MercurialInputScheme : InputScheme auto storePath = store->addToStore( input.getName(), {getFSSourceAccessor(), CanonPath(actualPath)}, - FileIngestionMethod::NixArchive, HashAlgorithm::SHA256, {}, + ContentAddressMethod::Raw::NixArchive, HashAlgorithm::SHA256, {}, filter); return storePath; diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index fa06c4aa3..e1cdfece6 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -8,6 +8,7 @@ std::string_view makeFileIngestionPrefix(FileIngestionMethod m) { switch (m) { case FileIngestionMethod::Flat: + // Not prefixed for back compat return ""; case FileIngestionMethod::NixArchive: return "r:"; @@ -15,91 +16,128 @@ std::string_view makeFileIngestionPrefix(FileIngestionMethod m) experimentalFeatureSettings.require(Xp::GitHashing); return "git:"; default: - throw Error("impossible, caught both cases"); + assert(false); } } std::string_view ContentAddressMethod::render() const { - return std::visit(overloaded { - [](TextIngestionMethod) -> std::string_view { return "text"; }, - [](FileIngestionMethod m2) { - /* Not prefixed for back compat with things that couldn't produce text before. */ - return renderFileIngestionMethod(m2); - }, - }, raw); + switch (raw) { + case ContentAddressMethod::Raw::Text: + return "text"; + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + return renderFileIngestionMethod(getFileIngestionMethod()); + default: + assert(false); + } +} + +/** + * **Not surjective** + * + * This is not exposed because `FileIngestionMethod::Flat` maps to + * `ContentAddressMethod::Raw::Flat` and + * `ContentAddressMethod::Raw::Text` alike. We can thus only safely use + * this when the latter is ruled out (e.g. because it is already + * handled). + */ +static ContentAddressMethod fileIngestionMethodToContentAddressMethod(FileIngestionMethod m) +{ + switch (m) { + case FileIngestionMethod::Flat: + return ContentAddressMethod::Raw::Flat; + case FileIngestionMethod::NixArchive: + return ContentAddressMethod::Raw::NixArchive; + case FileIngestionMethod::Git: + return ContentAddressMethod::Raw::Git; + default: + assert(false); + } } ContentAddressMethod ContentAddressMethod::parse(std::string_view m) { if (m == "text") - return TextIngestionMethod {}; + return ContentAddressMethod::Raw::Text; else - return parseFileIngestionMethod(m); + return fileIngestionMethodToContentAddressMethod( + parseFileIngestionMethod(m)); } std::string_view ContentAddressMethod::renderPrefix() const { - return std::visit(overloaded { - [](TextIngestionMethod) -> std::string_view { return "text:"; }, - [](FileIngestionMethod m2) { - /* Not prefixed for back compat with things that couldn't produce text before. */ - return makeFileIngestionPrefix(m2); - }, - }, raw); + switch (raw) { + case ContentAddressMethod::Raw::Text: + return "text:"; + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + return makeFileIngestionPrefix(getFileIngestionMethod()); + default: + assert(false); + } } ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m) { if (splitPrefix(m, "r:")) { - return FileIngestionMethod::NixArchive; + return ContentAddressMethod::Raw::NixArchive; } else if (splitPrefix(m, "git:")) { experimentalFeatureSettings.require(Xp::GitHashing); - return FileIngestionMethod::Git; + return ContentAddressMethod::Raw::Git; } else if (splitPrefix(m, "text:")) { - return TextIngestionMethod {}; + return ContentAddressMethod::Raw::Text; + } + return ContentAddressMethod::Raw::Flat; +} + +/** + * This is slightly more mindful of forward compat in that it uses `fixed:` + * rather than just doing a raw empty prefix or `r:`, which doesn't "save room" + * for future changes very well. + */ +static std::string renderPrefixModern(const ContentAddressMethod & ca) +{ + switch (ca.raw) { + case ContentAddressMethod::Raw::Text: + return "text:"; + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + return "fixed:" + makeFileIngestionPrefix(ca.getFileIngestionMethod()); + default: + assert(false); } - return FileIngestionMethod::Flat; } std::string ContentAddressMethod::renderWithAlgo(HashAlgorithm ha) const { - return std::visit(overloaded { - [&](const TextIngestionMethod & th) { - return std::string{"text:"} + printHashAlgo(ha); - }, - [&](const FileIngestionMethod & fim) { - return "fixed:" + makeFileIngestionPrefix(fim) + printHashAlgo(ha); - } - }, raw); + return renderPrefixModern(*this) + printHashAlgo(ha); } FileIngestionMethod ContentAddressMethod::getFileIngestionMethod() const { - return std::visit(overloaded { - [&](const TextIngestionMethod & th) { - return FileIngestionMethod::Flat; - }, - [&](const FileIngestionMethod & fim) { - return fim; - } - }, raw); + switch (raw) { + case ContentAddressMethod::Raw::Flat: + return FileIngestionMethod::Flat; + case ContentAddressMethod::Raw::NixArchive: + return FileIngestionMethod::NixArchive; + case ContentAddressMethod::Raw::Git: + return FileIngestionMethod::Git; + case ContentAddressMethod::Raw::Text: + return FileIngestionMethod::Flat; + default: + assert(false); + } } std::string ContentAddress::render() const { - return std::visit(overloaded { - [](const TextIngestionMethod &) -> std::string { - return "text:"; - }, - [](const FileIngestionMethod & method) { - return "fixed:" - + makeFileIngestionPrefix(method); - }, - }, method.raw) - + this->hash.to_string(HashFormat::Nix32, true); + return renderPrefixModern(method) + this->hash.to_string(HashFormat::Nix32, true); } /** @@ -130,17 +168,17 @@ static std::pair parseContentAddressMethodP // No parsing of the ingestion method, "text" only support flat. HashAlgorithm hashAlgo = parseHashAlgorithm_(); return { - TextIngestionMethod {}, + ContentAddressMethod::Raw::Text, std::move(hashAlgo), }; } else if (prefix == "fixed") { // Parse method - auto method = FileIngestionMethod::Flat; + auto method = ContentAddressMethod::Raw::Flat; if (splitPrefix(rest, "r:")) - method = FileIngestionMethod::NixArchive; + method = ContentAddressMethod::Raw::NixArchive; else if (splitPrefix(rest, "git:")) { experimentalFeatureSettings.require(Xp::GitHashing); - method = FileIngestionMethod::Git; + method = ContentAddressMethod::Raw::Git; } HashAlgorithm hashAlgo = parseHashAlgorithm_(); return { @@ -201,57 +239,58 @@ size_t StoreReferences::size() const ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) noexcept { - return std::visit(overloaded { - [&](const TextIngestionMethod &) -> ContentAddressWithReferences { - return TextInfo { - .hash = ca.hash, - .references = {}, - }; - }, - [&](const FileIngestionMethod & method) -> ContentAddressWithReferences { - return FixedOutputInfo { - .method = method, - .hash = ca.hash, - .references = {}, - }; - }, - }, ca.method.raw); + switch (ca.method.raw) { + case ContentAddressMethod::Raw::Text: + return TextInfo { + .hash = ca.hash, + .references = {}, + }; + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + return FixedOutputInfo { + .method = ca.method.getFileIngestionMethod(), + .hash = ca.hash, + .references = {}, + }; + default: + assert(false); + } } ContentAddressWithReferences ContentAddressWithReferences::fromParts( ContentAddressMethod method, Hash hash, StoreReferences refs) { - return std::visit(overloaded { - [&](TextIngestionMethod _) -> ContentAddressWithReferences { - if (refs.self) - throw Error("self-reference not allowed with text hashing"); - return ContentAddressWithReferences { - TextInfo { - .hash = std::move(hash), - .references = std::move(refs.others), - } - }; - }, - [&](FileIngestionMethod m2) -> ContentAddressWithReferences { - return ContentAddressWithReferences { - FixedOutputInfo { - .method = m2, - .hash = std::move(hash), - .references = std::move(refs), - } - }; - }, - }, method.raw); + switch (method.raw) { + case ContentAddressMethod::Raw::Text: + if (refs.self) + throw Error("self-reference not allowed with text hashing"); + return TextInfo { + .hash = std::move(hash), + .references = std::move(refs.others), + }; + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + return FixedOutputInfo { + .method = method.getFileIngestionMethod(), + .hash = std::move(hash), + .references = std::move(refs), + }; + default: + assert(false); + } } ContentAddressMethod ContentAddressWithReferences::getMethod() const { return std::visit(overloaded { [](const TextInfo & th) -> ContentAddressMethod { - return TextIngestionMethod {}; + return ContentAddressMethod::Raw::Text; }, [](const FixedOutputInfo & fsh) -> ContentAddressMethod { - return fsh.method; + return fileIngestionMethodToContentAddressMethod( + fsh.method); }, }, raw); } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 5925f8e01..6cc3b7cd9 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -5,7 +5,6 @@ #include "hash.hh" #include "path.hh" #include "file-content-address.hh" -#include "comparator.hh" #include "variant-wrapper.hh" namespace nix { @@ -14,24 +13,6 @@ namespace nix { * Content addressing method */ -/* We only have one way to hash text with references, so this is a single-value - type, mainly useful with std::variant. -*/ - -/** - * The single way we can serialize "text" file system objects. - * - * Somewhat obscure, used by \ref Derivation derivations and - * `builtins.toFile` currently. - * - * TextIngestionMethod is identical to FileIngestionMethod::Fixed except that - * the former may not have self-references and is tagged `text:${algo}:${hash}` - * rather than `fixed:${algo}:${hash}`. The contents of the store path are - * ingested and hashed identically, aside from the slightly different tag and - * restriction on self-references. - */ -struct TextIngestionMethod : std::monostate { }; - /** * Compute the prefix to the hash algorithm which indicates how the * files were ingested. @@ -48,14 +29,51 @@ std::string_view makeFileIngestionPrefix(FileIngestionMethod m); */ struct ContentAddressMethod { - typedef std::variant< - TextIngestionMethod, - FileIngestionMethod - > Raw; + enum struct Raw { + /** + * Calculate a store path using the `FileIngestionMethod::Flat` + * hash of the file system objects, and references. + * + * See `store-object/content-address.md#method-flat` in the + * manual. + */ + Flat, + + /** + * Calculate a store path using the + * `FileIngestionMethod::NixArchive` hash of the file system + * objects, and references. + * + * See `store-object/content-address.md#method-flat` in the + * manual. + */ + NixArchive, + + /** + * Calculate a store path using the `FileIngestionMethod::Git` + * hash of the file system objects, and references. + * + * Part of `ExperimentalFeature::GitHashing`. + * + * See `store-object/content-address.md#method-git` in the + * manual. + */ + Git, + + /** + * Calculate a store path using the `FileIngestionMethod::Flat` + * hash of the file system objects, and references, but in a + * different way than `ContentAddressMethod::Raw::Flat`. + * + * See `store-object/content-address.md#method-text` in the + * manual. + */ + Text, + }; Raw raw; - GENERATE_CMP(ContentAddressMethod, me->raw); + auto operator <=>(const ContentAddressMethod &) const = default; MAKE_WRAPPER_CONSTRUCTOR(ContentAddressMethod); @@ -141,7 +159,7 @@ struct ContentAddress */ Hash hash; - GENERATE_CMP(ContentAddress, me->method, me->hash); + auto operator <=>(const ContentAddress &) const = default; /** * Compute the content-addressability assertion @@ -200,7 +218,7 @@ struct StoreReferences */ size_t size() const; - GENERATE_CMP(StoreReferences, me->self, me->others); + auto operator <=>(const StoreReferences &) const = default; }; // This matches the additional info that we need for makeTextPath @@ -217,7 +235,7 @@ struct TextInfo */ StorePathSet references; - GENERATE_CMP(TextInfo, me->hash, me->references); + auto operator <=>(const TextInfo &) const = default; }; struct FixedOutputInfo @@ -237,7 +255,7 @@ struct FixedOutputInfo */ StoreReferences references; - GENERATE_CMP(FixedOutputInfo, me->hash, me->references); + auto operator <=>(const FixedOutputInfo &) const = default; }; /** @@ -254,7 +272,7 @@ struct ContentAddressWithReferences Raw raw; - GENERATE_CMP(ContentAddressWithReferences, me->raw); + auto operator <=>(const ContentAddressWithReferences &) const = default; MAKE_WRAPPER_CONSTRUCTOR(ContentAddressWithReferences); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 788c5e2ea..40163a621 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -435,19 +435,21 @@ static void performOp(TunnelLogger * logger, ref store, } else { HashAlgorithm hashAlgo; std::string baseName; - FileIngestionMethod method; + ContentAddressMethod method; { bool fixed; uint8_t recursive; std::string hashAlgoRaw; from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw; - if (recursive > (uint8_t) FileIngestionMethod::NixArchive) + if (recursive > true) throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive); - method = FileIngestionMethod { recursive }; + method = recursive + ? ContentAddressMethod::Raw::NixArchive + : ContentAddressMethod::Raw::Flat; /* Compatibility hack. */ if (!fixed) { hashAlgoRaw = "sha256"; - method = FileIngestionMethod::NixArchive; + method = ContentAddressMethod::Raw::NixArchive; } hashAlgo = parseHashAlgo(hashAlgoRaw); } @@ -500,7 +502,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto path = ({ StringSource source { s }; - store->addToStoreFromDump(source, suffix, FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, NoRepair); + store->addToStoreFromDump(source, suffix, FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, refs, NoRepair); }); logger->stopWork(); to << store->printStorePath(path); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 869880112..6dfcc408c 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -150,7 +150,7 @@ StorePath writeDerivation(Store & store, }) : ({ StringSource s { contents }; - store.addToStoreFromDump(s, suffix, FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, references, repair); + store.addToStoreFromDump(s, suffix, FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, references, repair); }); } @@ -274,7 +274,7 @@ static DerivationOutput parseDerivationOutput( { if (hashAlgoStr != "") { ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgoStr); - if (method == TextIngestionMethod {}) + if (method == ContentAddressMethod::Raw::Text) xpSettings.require(Xp::DynamicDerivations); const auto hashAlgo = parseHashAlgo(hashAlgoStr); if (hashS == "impure") { @@ -1249,7 +1249,7 @@ DerivationOutput DerivationOutput::fromJSON( auto methodAlgo = [&]() -> std::pair { auto & method_ = getString(valueAt(json, "method")); ContentAddressMethod method = ContentAddressMethod::parse(method_); - if (method == TextIngestionMethod {}) + if (method == ContentAddressMethod::Raw::Text) xpSettings.require(Xp::DynamicDerivations); auto & hashAlgo_ = getString(valueAt(json, "hashAlgo")); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 07ace70d0..2b4e01eb3 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1253,7 +1253,7 @@ StorePath LocalStore::addToStoreFromDump( std::filesystem::path tempDir; AutoCloseFD tempDirFd; - bool methodsMatch = ContentAddressMethod(FileIngestionMethod(dumpMethod)) == hashMethod; + bool methodsMatch = static_cast(dumpMethod) == hashMethod.getFileIngestionMethod(); /* If the methods don't match, our streaming hash of the dump is the wrong sort, and we need to rehash. */ diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index ddd7f50d9..5c27182b7 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -48,15 +48,21 @@ std::optional ValidPathInfo::contentAddressWithRef if (! ca) return std::nullopt; - return std::visit(overloaded { - [&](const TextIngestionMethod &) -> ContentAddressWithReferences { + switch (ca->method.raw) { + case ContentAddressMethod::Raw::Text: + { assert(references.count(path) == 0); return TextInfo { .hash = ca->hash, .references = references, }; - }, - [&](const FileIngestionMethod & m2) -> ContentAddressWithReferences { + } + + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + default: + { auto refs = references; bool hasSelfReference = false; if (refs.count(path)) { @@ -64,15 +70,15 @@ std::optional ValidPathInfo::contentAddressWithRef refs.erase(path); } return FixedOutputInfo { - .method = m2, + .method = ca->method.getFileIngestionMethod(), .hash = ca->hash, .references = { .others = std::move(refs), .self = hasSelfReference, }, }; - }, - }, ca->method.raw); + } + } } bool ValidPathInfo::isContentAddressed(const Store & store) const @@ -127,22 +133,18 @@ ValidPathInfo::ValidPathInfo( : UnkeyedValidPathInfo(narHash) , path(store.makeFixedOutputPathFromCA(name, ca)) { + this->ca = ContentAddress { + .method = ca.getMethod(), + .hash = ca.getHash(), + }; std::visit(overloaded { [this](TextInfo && ti) { this->references = std::move(ti.references); - this->ca = ContentAddress { - .method = TextIngestionMethod {}, - .hash = std::move(ti.hash), - }; }, [this](FixedOutputInfo && foi) { this->references = std::move(foi.references.others); if (foi.references.self) this->references.insert(path); - this->ca = ContentAddress { - .method = std::move(foi.method), - .hash = std::move(foi.hash), - }; }, }, std::move(ca).raw); } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 9adad9c2a..d749ccd0a 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -392,8 +392,9 @@ ref RemoteStore::addCAToStore( else { if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25"); - std::visit(overloaded { - [&](const TextIngestionMethod & thm) -> void { + switch (caMethod.raw) { + case ContentAddressMethod::Raw::Text: + { if (hashAlgo != HashAlgorithm::SHA256) throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given", name, printHashAlgo(hashAlgo)); @@ -401,8 +402,14 @@ ref RemoteStore::addCAToStore( conn->to << WorkerProto::Op::AddTextToStore << name << s; WorkerProto::write(*this, *conn, references); conn.processStderr(); - }, - [&](const FileIngestionMethod & fim) -> void { + break; + } + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + default: + { + auto fim = caMethod.getFileIngestionMethod(); conn->to << WorkerProto::Op::AddToStore << name @@ -432,9 +439,9 @@ ref RemoteStore::addCAToStore( } catch (EndOfFile & e) { } throw; } - + break; } - }, caMethod.raw); + } auto path = parseStorePath(readString(conn->from)); // Release our connection to prevent a deadlock in queryPathInfo(). conn_.reset(); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8c957ff1a..05c4e1c5e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -356,7 +356,7 @@ ValidPathInfo Store::addToStoreSlow( RegularFileSink fileSink { caHashSink }; TeeSink unusualHashTee { narHashSink, caHashSink }; - auto & narSink = method == FileIngestionMethod::NixArchive && hashAlgo != HashAlgorithm::SHA256 + auto & narSink = method == ContentAddressMethod::Raw::NixArchive && hashAlgo != HashAlgorithm::SHA256 ? static_cast(unusualHashTee) : narHashSink; @@ -384,9 +384,9 @@ ValidPathInfo Store::addToStoreSlow( finish. */ auto [narHash, narSize] = narHashSink.finish(); - auto hash = method == FileIngestionMethod::NixArchive && hashAlgo == HashAlgorithm::SHA256 + auto hash = method == ContentAddressMethod::Raw::NixArchive && hashAlgo == HashAlgorithm::SHA256 ? narHash - : method == FileIngestionMethod::Git + : method == ContentAddressMethod::Raw::Git ? git::dumpHash(hashAlgo, srcPath).hash : caHashSink.finish().first; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index e719f9bf9..a5effb4c1 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -441,7 +441,7 @@ public: virtual StorePath addToStore( std::string_view name, const SourcePath & path, - ContentAddressMethod method = FileIngestionMethod::NixArchive, + ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), PathFilter & filter = defaultPathFilter, @@ -455,7 +455,7 @@ public: ValidPathInfo addToStoreSlow( std::string_view name, const SourcePath & path, - ContentAddressMethod method = FileIngestionMethod::NixArchive, + ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), std::optional expectedCAHash = {}); @@ -481,7 +481,7 @@ public: Source & dump, std::string_view name, FileSerialisationMethod dumpMethod = FileSerialisationMethod::NixArchive, - ContentAddressMethod hashMethod = FileIngestionMethod::NixArchive, + ContentAddressMethod hashMethod = ContentAddressMethod::Raw::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), RepairFlag repair = NoRepair) = 0; diff --git a/src/libutil/file-content-address.hh b/src/libutil/file-content-address.hh index f3f5589de..4c7218f19 100644 --- a/src/libutil/file-content-address.hh +++ b/src/libutil/file-content-address.hh @@ -117,6 +117,8 @@ enum struct FileIngestionMethod : uint8_t { /** * Git hashing. * + * Part of `ExperimentalFeature::GitHashing`. + * * See `file-system-object/content-address.md#serial-git` in the * manual. */ diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 5246b03e4..a24dd11d6 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -115,7 +115,7 @@ bool createUserEnv(EvalState & state, PackageInfos & elems, std::string str2 = str.str(); StringSource source { str2 }; state.store->addToStoreFromDump( - source, "env-manifest.nix", FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, references); + source, "env-manifest.nix", FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, references); }); /* Get the environment builder expression. */ diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 178702f91..d0840a02e 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -194,10 +194,10 @@ static void opAdd(Strings opFlags, Strings opArgs) store. */ static void opAddFixed(Strings opFlags, Strings opArgs) { - auto method = FileIngestionMethod::Flat; + ContentAddressMethod method = ContentAddressMethod::Raw::Flat; for (auto & i : opFlags) - if (i == "--recursive") method = FileIngestionMethod::NixArchive; + if (i == "--recursive") method = ContentAddressMethod::Raw::NixArchive; else throw UsageError("unknown flag '%1%'", i); if (opArgs.empty()) diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index ed46254f3..5c08f7616 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -12,7 +12,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand { Path path; std::optional namePart; - ContentAddressMethod caMethod = FileIngestionMethod::NixArchive; + ContentAddressMethod caMethod = ContentAddressMethod::Raw::NixArchive; HashAlgorithm hashAlgo = HashAlgorithm::SHA256; CmdAddToStore() @@ -68,7 +68,7 @@ struct CmdAddFile : CmdAddToStore { CmdAddFile() { - caMethod = FileIngestionMethod::Flat; + caMethod = ContentAddressMethod::Raw::Flat; } std::string description() override diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 27287a1a8..80510dc78 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -238,7 +238,7 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore auto getEnvShPath = ({ StringSource source { getEnvSh }; evalStore->addToStoreFromDump( - source, "get-env.sh", FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, {}); + source, "get-env.sh", FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, {}); }); drv.args = {store->printStorePath(getEnvShPath)}; diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 143935572..6bbf2578e 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -57,7 +57,9 @@ std::tuple prefetchFile( bool unpack, bool executable) { - auto ingestionMethod = unpack || executable ? FileIngestionMethod::NixArchive : FileIngestionMethod::Flat; + ContentAddressMethod method = unpack || executable + ? ContentAddressMethod::Raw::NixArchive + : ContentAddressMethod::Raw::Flat; /* Figure out a name in the Nix store. */ if (!name) { @@ -73,11 +75,10 @@ std::tuple prefetchFile( the store. */ if (expectedHash) { hashAlgo = expectedHash->algo; - storePath = store->makeFixedOutputPath(*name, FixedOutputInfo { - .method = ingestionMethod, - .hash = *expectedHash, - .references = {}, - }); + storePath = store->makeFixedOutputPathFromCA(*name, ContentAddressWithReferences::fromParts( + method, + *expectedHash, + {})); if (store->isValidPath(*storePath)) hash = expectedHash; else @@ -128,7 +129,7 @@ std::tuple prefetchFile( auto info = store->addToStoreSlow( *name, PosixSourceAccessor::createAtRoot(tmpFile), - ingestionMethod, hashAlgo, {}, expectedHash); + method, hashAlgo, {}, expectedHash); storePath = info.path; assert(info.ca); hash = info.ca->hash; diff --git a/src/perl/lib/Nix/Store.xs b/src/perl/lib/Nix/Store.xs index e8e209d97..acce25f3a 100644 --- a/src/perl/lib/Nix/Store.xs +++ b/src/perl/lib/Nix/Store.xs @@ -335,7 +335,7 @@ SV * StoreWrapper::addToStore(char * srcPath, int recursive, char * algo) PPCODE: try { - auto method = recursive ? FileIngestionMethod::NixArchive : FileIngestionMethod::Flat; + auto method = recursive ? ContentAddressMethod::Raw::NixArchive : ContentAddressMethod::Raw::Flat; auto path = THIS->store->addToStore( std::string(baseNameOf(srcPath)), PosixSourceAccessor::createAtRoot(srcPath), diff --git a/tests/unit/libstore/common-protocol.cc b/tests/unit/libstore/common-protocol.cc index 2c629b601..c8f6dd002 100644 --- a/tests/unit/libstore/common-protocol.cc +++ b/tests/unit/libstore/common-protocol.cc @@ -83,15 +83,15 @@ CHARACTERIZATION_TEST( "content-address", (std::tuple { ContentAddress { - .method = TextIngestionMethod {}, + .method = ContentAddressMethod::Raw::Text, .hash = hashString(HashAlgorithm::SHA256, "Derive(...)"), }, ContentAddress { - .method = FileIngestionMethod::Flat, + .method = ContentAddressMethod::Raw::Flat, .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, ContentAddress { - .method = FileIngestionMethod::NixArchive, + .method = ContentAddressMethod::Raw::NixArchive, .hash = hashString(HashAlgorithm::SHA256, "(...)"), }, })) @@ -178,7 +178,7 @@ CHARACTERIZATION_TEST( std::nullopt, std::optional { ContentAddress { - .method = FileIngestionMethod::Flat, + .method = ContentAddressMethod::Raw::Flat, .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, }, diff --git a/tests/unit/libstore/content-address.cc b/tests/unit/libstore/content-address.cc index f0806bd9a..72eb84fec 100644 --- a/tests/unit/libstore/content-address.cc +++ b/tests/unit/libstore/content-address.cc @@ -9,11 +9,11 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(ContentAddressMethod, testRoundTripPrintParse_1) { - for (const ContentAddressMethod & cam : { - ContentAddressMethod { TextIngestionMethod {} }, - ContentAddressMethod { FileIngestionMethod::Flat }, - ContentAddressMethod { FileIngestionMethod::NixArchive }, - ContentAddressMethod { FileIngestionMethod::Git }, + for (ContentAddressMethod cam : { + ContentAddressMethod::Raw::Text, + ContentAddressMethod::Raw::Flat, + ContentAddressMethod::Raw::NixArchive, + ContentAddressMethod::Raw::Git, }) { EXPECT_EQ(ContentAddressMethod::parse(cam.render()), cam); } diff --git a/tests/unit/libstore/derivation.cc b/tests/unit/libstore/derivation.cc index 210813137..71979f885 100644 --- a/tests/unit/libstore/derivation.cc +++ b/tests/unit/libstore/derivation.cc @@ -108,7 +108,7 @@ TEST_JSON(DerivationTest, inputAddressed, TEST_JSON(DerivationTest, caFixedFlat, (DerivationOutput::CAFixed { .ca = { - .method = FileIngestionMethod::Flat, + .method = ContentAddressMethod::Raw::Flat, .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), }, }), @@ -117,7 +117,7 @@ TEST_JSON(DerivationTest, caFixedFlat, TEST_JSON(DerivationTest, caFixedNAR, (DerivationOutput::CAFixed { .ca = { - .method = FileIngestionMethod::NixArchive, + .method = ContentAddressMethod::Raw::NixArchive, .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), }, }), @@ -126,6 +126,7 @@ TEST_JSON(DerivationTest, caFixedNAR, TEST_JSON(DynDerivationTest, caFixedText, (DerivationOutput::CAFixed { .ca = { + .method = ContentAddressMethod::Raw::Text, .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), }, }), @@ -133,7 +134,7 @@ TEST_JSON(DynDerivationTest, caFixedText, TEST_JSON(CaDerivationTest, caFloating, (DerivationOutput::CAFloating { - .method = FileIngestionMethod::NixArchive, + .method = ContentAddressMethod::Raw::NixArchive, .hashAlgo = HashAlgorithm::SHA256, }), "drv-name", "output-name") @@ -144,7 +145,7 @@ TEST_JSON(DerivationTest, deferred, TEST_JSON(ImpureDerivationTest, impure, (DerivationOutput::Impure { - .method = FileIngestionMethod::NixArchive, + .method = ContentAddressMethod::Raw::NixArchive, .hashAlgo = HashAlgorithm::SHA256, }), "drv-name", "output-name") diff --git a/tests/unit/libstore/serve-protocol.cc b/tests/unit/libstore/serve-protocol.cc index 17d7153bb..2505c5a9a 100644 --- a/tests/unit/libstore/serve-protocol.cc +++ b/tests/unit/libstore/serve-protocol.cc @@ -55,15 +55,15 @@ VERSIONED_CHARACTERIZATION_TEST( defaultVersion, (std::tuple { ContentAddress { - .method = TextIngestionMethod {}, + .method = ContentAddressMethod::Raw::Text, .hash = hashString(HashAlgorithm::SHA256, "Derive(...)"), }, ContentAddress { - .method = FileIngestionMethod::Flat, + .method = ContentAddressMethod::Raw::Flat, .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, ContentAddress { - .method = FileIngestionMethod::NixArchive, + .method = ContentAddressMethod::Raw::NixArchive, .hash = hashString(HashAlgorithm::SHA256, "(...)"), }, })) @@ -398,7 +398,7 @@ VERSIONED_CHARACTERIZATION_TEST( std::nullopt, std::optional { ContentAddress { - .method = FileIngestionMethod::Flat, + .method = ContentAddressMethod::Raw::Flat, .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, }, diff --git a/tests/unit/libstore/worker-protocol.cc b/tests/unit/libstore/worker-protocol.cc index 8d81717c9..c15120010 100644 --- a/tests/unit/libstore/worker-protocol.cc +++ b/tests/unit/libstore/worker-protocol.cc @@ -56,15 +56,15 @@ VERSIONED_CHARACTERIZATION_TEST( defaultVersion, (std::tuple { ContentAddress { - .method = TextIngestionMethod {}, + .method = ContentAddressMethod::Raw::Text, .hash = hashString(HashAlgorithm::SHA256, "Derive(...)"), }, ContentAddress { - .method = FileIngestionMethod::Flat, + .method = ContentAddressMethod::Raw::Flat, .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, ContentAddress { - .method = FileIngestionMethod::NixArchive, + .method = ContentAddressMethod::Raw::NixArchive, .hash = hashString(HashAlgorithm::SHA256, "(...)"), }, })) @@ -598,7 +598,7 @@ VERSIONED_CHARACTERIZATION_TEST( std::nullopt, std::optional { ContentAddress { - .method = FileIngestionMethod::Flat, + .method = ContentAddressMethod::Raw::Flat, .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, }, From 1620ad4587f280187124891dff909402b20db014 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 24 Jun 2024 11:34:58 -0400 Subject: [PATCH 30/61] Split out `GlobalConfig` into its own header This makes it easier to understand the reach of global variables / global state in the config system. --- src/libcmd/repl.cc | 1 + src/libexpr/eval-settings.cc | 1 + src/libexpr/flake/config.cc | 1 + src/libfetchers/fetch-settings.cc | 1 + src/libmain/common-args.cc | 1 + src/libstore/build/derivation-goal.cc | 1 + src/libstore/filetransfer.cc | 1 + src/libstore/globals.cc | 1 + src/libstore/unix/build/hook-instance.cc | 1 + src/libutil-c/nix_api_util.cc | 2 +- src/libutil/archive.cc | 2 +- src/libutil/config-global.cc | 66 ++++++++++++++++++++++++ src/libutil/config-global.hh | 33 ++++++++++++ src/libutil/config.cc | 61 ---------------------- src/libutil/config.hh | 25 --------- src/libutil/fs-sink.cc | 2 +- src/libutil/logging.cc | 2 +- src/nix/config.cc | 1 + src/nix/develop.cc | 1 + src/nix/unix/daemon.cc | 1 + tests/functional/plugins/plugintest.cc | 2 +- tests/unit/libutil/nix_api_util.cc | 2 +- 22 files changed, 117 insertions(+), 92 deletions(-) create mode 100644 src/libutil/config-global.cc create mode 100644 src/libutil/config-global.hh diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 1e80d14e1..53dd94c7a 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -8,6 +8,7 @@ #include "ansicolor.hh" #include "shared.hh" +#include "config-global.hh" #include "eval.hh" #include "eval-cache.hh" #include "eval-inline.hh" diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index 85b1677ea..a642bb684 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -1,4 +1,5 @@ #include "users.hh" +#include "config-global.hh" #include "globals.hh" #include "profiles.hh" #include "eval.hh" diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc index e0c5d4512..03430b0e3 100644 --- a/src/libexpr/flake/config.cc +++ b/src/libexpr/flake/config.cc @@ -1,4 +1,5 @@ #include "users.hh" +#include "config-global.hh" #include "globals.hh" #include "fetch-settings.hh" #include "flake.hh" diff --git a/src/libfetchers/fetch-settings.cc b/src/libfetchers/fetch-settings.cc index e7d5244dc..21c42567c 100644 --- a/src/libfetchers/fetch-settings.cc +++ b/src/libfetchers/fetch-settings.cc @@ -1,4 +1,5 @@ #include "fetch-settings.hh" +#include "config-global.hh" namespace nix { diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 5b49aaabc..a94845ab8 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -1,5 +1,6 @@ #include "common-args.hh" #include "args/root.hh" +#include "config-global.hh" #include "globals.hh" #include "logging.hh" #include "loggers.hh" diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 146a060f3..64b8495e1 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -3,6 +3,7 @@ # include "hook-instance.hh" #endif #include "processes.hh" +#include "config-global.hh" #include "worker.hh" #include "builtins.hh" #include "builtins/buildenv.hh" diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index a54ebdcf3..cbbb0fe7a 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -1,5 +1,6 @@ #include "filetransfer.hh" #include "globals.hh" +#include "config-global.hh" #include "store-api.hh" #include "s3.hh" #include "compression.hh" diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 88f899dbf..8fec10d51 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -1,4 +1,5 @@ #include "globals.hh" +#include "config-global.hh" #include "current-process.hh" #include "archive.hh" #include "args.hh" diff --git a/src/libstore/unix/build/hook-instance.cc b/src/libstore/unix/build/hook-instance.cc index 5d045ec3d..dfc208798 100644 --- a/src/libstore/unix/build/hook-instance.cc +++ b/src/libstore/unix/build/hook-instance.cc @@ -1,4 +1,5 @@ #include "globals.hh" +#include "config-global.hh" #include "hook-instance.hh" #include "file-system.hh" #include "child.hh" diff --git a/src/libutil-c/nix_api_util.cc b/src/libutil-c/nix_api_util.cc index 0a9b49345..4f65a4c12 100644 --- a/src/libutil-c/nix_api_util.cc +++ b/src/libutil-c/nix_api_util.cc @@ -1,5 +1,5 @@ #include "nix_api_util.h" -#include "config.hh" +#include "config-global.hh" #include "error.hh" #include "nix_api_util_internal.h" #include "util.hh" diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index d20936de4..22be392d4 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -6,7 +6,7 @@ #include // for strcasecmp #include "archive.hh" -#include "config.hh" +#include "config-global.hh" #include "posix-source-accessor.hh" #include "source-path.hh" #include "file-system.hh" diff --git a/src/libutil/config-global.cc b/src/libutil/config-global.cc new file mode 100644 index 000000000..e8025b303 --- /dev/null +++ b/src/libutil/config-global.cc @@ -0,0 +1,66 @@ +#include "config-global.hh" + +namespace nix { + +bool GlobalConfig::set(const std::string & name, const std::string & value) +{ + for (auto & config : *configRegistrations) + if (config->set(name, value)) return true; + + unknownSettings.emplace(name, value); + + return false; +} + +void GlobalConfig::getSettings(std::map & res, bool overriddenOnly) +{ + for (auto & config : *configRegistrations) + config->getSettings(res, overriddenOnly); +} + +void GlobalConfig::resetOverridden() +{ + for (auto & config : *configRegistrations) + config->resetOverridden(); +} + +nlohmann::json GlobalConfig::toJSON() +{ + auto res = nlohmann::json::object(); + for (const auto & config : *configRegistrations) + res.update(config->toJSON()); + return res; +} + +std::string GlobalConfig::toKeyValue() +{ + std::string res; + std::map settings; + globalConfig.getSettings(settings); + for (const auto & s : settings) + res += fmt("%s = %s\n", s.first, s.second.value); + return res; +} + +void GlobalConfig::convertToArgs(Args & args, const std::string & category) +{ + for (auto & config : *configRegistrations) + config->convertToArgs(args, category); +} + +GlobalConfig globalConfig; + +GlobalConfig::ConfigRegistrations * GlobalConfig::configRegistrations; + +GlobalConfig::Register::Register(Config * config) +{ + if (!configRegistrations) + configRegistrations = new ConfigRegistrations; + configRegistrations->emplace_back(config); +} + +ExperimentalFeatureSettings experimentalFeatureSettings; + +static GlobalConfig::Register rSettings(&experimentalFeatureSettings); + +} diff --git a/src/libutil/config-global.hh b/src/libutil/config-global.hh new file mode 100644 index 000000000..1b7b69a0d --- /dev/null +++ b/src/libutil/config-global.hh @@ -0,0 +1,33 @@ +#pragma once +///@file + +#include "config.hh" + +namespace nix { + +struct GlobalConfig : public AbstractConfig +{ + typedef std::vector ConfigRegistrations; + static ConfigRegistrations * configRegistrations; + + bool set(const std::string & name, const std::string & value) override; + + void getSettings(std::map & res, bool overriddenOnly = false) override; + + void resetOverridden() override; + + nlohmann::json toJSON() override; + + std::string toKeyValue() override; + + void convertToArgs(Args & args, const std::string & category) override; + + struct Register + { + Register(Config * config); + }; +}; + +extern GlobalConfig globalConfig; + +} diff --git a/src/libutil/config.cc b/src/libutil/config.cc index efde8591b..192a4ecb9 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -443,67 +443,6 @@ void OptionalPathSetting::operator =(const std::optional & v) this->assign(v); } -bool GlobalConfig::set(const std::string & name, const std::string & value) -{ - for (auto & config : *configRegistrations) - if (config->set(name, value)) return true; - - unknownSettings.emplace(name, value); - - return false; -} - -void GlobalConfig::getSettings(std::map & res, bool overriddenOnly) -{ - for (auto & config : *configRegistrations) - config->getSettings(res, overriddenOnly); -} - -void GlobalConfig::resetOverridden() -{ - for (auto & config : *configRegistrations) - config->resetOverridden(); -} - -nlohmann::json GlobalConfig::toJSON() -{ - auto res = nlohmann::json::object(); - for (const auto & config : *configRegistrations) - res.update(config->toJSON()); - return res; -} - -std::string GlobalConfig::toKeyValue() -{ - std::string res; - std::map settings; - globalConfig.getSettings(settings); - for (const auto & s : settings) - res += fmt("%s = %s\n", s.first, s.second.value); - return res; -} - -void GlobalConfig::convertToArgs(Args & args, const std::string & category) -{ - for (auto & config : *configRegistrations) - config->convertToArgs(args, category); -} - -GlobalConfig globalConfig; - -GlobalConfig::ConfigRegistrations * GlobalConfig::configRegistrations; - -GlobalConfig::Register::Register(Config * config) -{ - if (!configRegistrations) - configRegistrations = new ConfigRegistrations; - configRegistrations->emplace_back(config); -} - -ExperimentalFeatureSettings experimentalFeatureSettings; - -static GlobalConfig::Register rSettings(&experimentalFeatureSettings); - bool ExperimentalFeatureSettings::isEnabled(const ExperimentalFeature & feature) const { auto & f = experimentalFeatures.get(); diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 07322b60d..1952ba1b8 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -375,31 +375,6 @@ public: void operator =(const std::optional & v); }; -struct GlobalConfig : public AbstractConfig -{ - typedef std::vector ConfigRegistrations; - static ConfigRegistrations * configRegistrations; - - bool set(const std::string & name, const std::string & value) override; - - void getSettings(std::map & res, bool overriddenOnly = false) override; - - void resetOverridden() override; - - nlohmann::json toJSON() override; - - std::string toKeyValue() override; - - void convertToArgs(Args & args, const std::string & category) override; - - struct Register - { - Register(Config * config); - }; -}; - -extern GlobalConfig globalConfig; - struct ExperimentalFeatureSettings : Config { diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index 91070ea89..a6a743737 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -1,7 +1,7 @@ #include #include "error.hh" -#include "config.hh" +#include "config-global.hh" #include "fs-sink.hh" #if _WIN32 diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 5fa01f0d9..55751b4cf 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -3,7 +3,7 @@ #include "environment-variables.hh" #include "terminal.hh" #include "util.hh" -#include "config.hh" +#include "config-global.hh" #include "source-path.hh" #include "position.hh" diff --git a/src/nix/config.cc b/src/nix/config.cc index 52706afcf..07f975a00 100644 --- a/src/nix/config.cc +++ b/src/nix/config.cc @@ -2,6 +2,7 @@ #include "common-args.hh" #include "shared.hh" #include "store-api.hh" +#include "config-global.hh" #include diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 27287a1a8..a7acbff0b 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -1,3 +1,4 @@ +#include "config-global.hh" #include "eval.hh" #include "installable-flake.hh" #include "command-installable-value.hh" diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index f1fc51682..41ea1f5a4 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -10,6 +10,7 @@ #include "serialise.hh" #include "archive.hh" #include "globals.hh" +#include "config-global.hh" #include "derivations.hh" #include "finally.hh" #include "legacy.hh" diff --git a/tests/functional/plugins/plugintest.cc b/tests/functional/plugins/plugintest.cc index e02fd68d5..7433ad190 100644 --- a/tests/functional/plugins/plugintest.cc +++ b/tests/functional/plugins/plugintest.cc @@ -1,4 +1,4 @@ -#include "config.hh" +#include "config-global.hh" #include "primops.hh" using namespace nix; diff --git a/tests/unit/libutil/nix_api_util.cc b/tests/unit/libutil/nix_api_util.cc index d2999f55b..2b7e38225 100644 --- a/tests/unit/libutil/nix_api_util.cc +++ b/tests/unit/libutil/nix_api_util.cc @@ -1,4 +1,4 @@ -#include "config.hh" +#include "config-global.hh" #include "args.hh" #include "nix_api_util.h" #include "nix_api_util_internal.h" From b46e13840b948a363e2aa082f6064dfaeab58c06 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 24 Jun 2024 12:07:08 -0400 Subject: [PATCH 31/61] Format `config-global.{cc,hh}` Since the code is factored out, it is no longer avoding the formatter. --- src/libutil/config-global.cc | 3 ++- src/libutil/config-global.hh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libutil/config-global.cc b/src/libutil/config-global.cc index e8025b303..4c08898a4 100644 --- a/src/libutil/config-global.cc +++ b/src/libutil/config-global.cc @@ -5,7 +5,8 @@ namespace nix { bool GlobalConfig::set(const std::string & name, const std::string & value) { for (auto & config : *configRegistrations) - if (config->set(name, value)) return true; + if (config->set(name, value)) + return true; unknownSettings.emplace(name, value); diff --git a/src/libutil/config-global.hh b/src/libutil/config-global.hh index 1b7b69a0d..2caf51524 100644 --- a/src/libutil/config-global.hh +++ b/src/libutil/config-global.hh @@ -7,7 +7,7 @@ namespace nix { struct GlobalConfig : public AbstractConfig { - typedef std::vector ConfigRegistrations; + typedef std::vector ConfigRegistrations; static ConfigRegistrations * configRegistrations; bool set(const std::string & name, const std::string & value) override; From cb0c868da4ced09179182792b8be1d7259238f98 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 24 Jun 2024 11:55:36 -0400 Subject: [PATCH 32/61] Allow loading config files into other config objects This gives us some hope of moving away from global variables. --- src/libstore/globals.cc | 10 +++++----- src/libstore/globals.hh | 8 +++++++- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 8fec10d51..7e1d7ea6d 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -124,12 +124,12 @@ Settings::Settings() }; } -void loadConfFile() +void loadConfFile(AbstractConfig & config) { auto applyConfigFile = [&](const Path & path) { try { std::string contents = readFile(path); - globalConfig.applyConfig(contents, path); + config.applyConfig(contents, path); } catch (SystemError &) { } }; @@ -137,7 +137,7 @@ void loadConfFile() /* We only want to send overrides to the daemon, i.e. stuff from ~/.nix/nix.conf or the command line. */ - globalConfig.resetOverridden(); + config.resetOverridden(); auto files = settings.nixUserConfFiles; for (auto file = files.rbegin(); file != files.rend(); file++) { @@ -146,7 +146,7 @@ void loadConfFile() auto nixConfEnv = getEnv("NIX_CONFIG"); if (nixConfEnv.has_value()) { - globalConfig.applyConfig(nixConfEnv.value(), "NIX_CONFIG"); + config.applyConfig(nixConfEnv.value(), "NIX_CONFIG"); } } @@ -438,7 +438,7 @@ void initLibStore(bool loadConfig) { initLibUtil(); if (loadConfig) - loadConfFile(); + loadConfFile(globalConfig); preloadNSS(); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 843e77bcf..439e9f4fc 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -1284,7 +1284,13 @@ extern Settings settings; */ void initPlugins(); -void loadConfFile(); +/** + * Load the configuration (from `nix.conf`, `NIX_CONFIG`, etc.) into the + * given configuration object. + * + * Usually called with `globalConfig`. + */ +void loadConfFile(AbstractConfig & config); // Used by the Settings constructor std::vector getUserConfigFiles(); From d4ca634508236ff0343f200c9aef4c21e752e961 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 24 Jun 2024 18:11:10 +0200 Subject: [PATCH 33/61] tests/functional: Differentiate die and fail --- tests/functional/common/vars-and-functions.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/functional/common/vars-and-functions.sh b/tests/functional/common/vars-and-functions.sh index 8d99800ef..37c109015 100644 --- a/tests/functional/common/vars-and-functions.sh +++ b/tests/functional/common/vars-and-functions.sh @@ -11,7 +11,7 @@ isTestOnNixOS() { } die() { - echo "fatal error: $*" >&2 + echo "unexpected fatal error: $*" >&2 exit 1 } @@ -216,7 +216,7 @@ requireGit() { } fail() { - echo "$1" >&2 + echo "test failed: $1" >&2 exit 1 } From 5a7ccd658093c7c0598d4e5cce823aee9a3265f1 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 24 Jun 2024 18:11:58 +0200 Subject: [PATCH 34/61] tests/functional: Print all args of fail() --- tests/functional/common/vars-and-functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/common/vars-and-functions.sh b/tests/functional/common/vars-and-functions.sh index 37c109015..4316a30d5 100644 --- a/tests/functional/common/vars-and-functions.sh +++ b/tests/functional/common/vars-and-functions.sh @@ -216,7 +216,7 @@ requireGit() { } fail() { - echo "test failed: $1" >&2 + echo "test failed: $*" >&2 exit 1 } From 52bfccf8d8112ba738e6fc9e4891f85b6b864566 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 14 Jun 2024 12:41:09 -0400 Subject: [PATCH 35/61] No global eval settings in `libnixexpr` Progress on #5638 There is still a global eval settings, but it pushed down into `libnixcmd`, which is a lot less bad a place for this sort of thing. --- .editorconfig | 8 +++--- src/libcmd/command.cc | 8 +++--- src/libcmd/common-eval-args.cc | 7 +++++ src/libcmd/common-eval-args.hh | 6 +++++ src/libexpr-c/nix_api_expr.cc | 17 +++++++++++- src/libexpr-c/nix_api_expr_internal.h | 2 ++ src/libexpr/eval-settings.cc | 13 ++++----- src/libexpr/eval-settings.hh | 10 +++---- src/libexpr/eval.cc | 29 +++++++++++---------- src/libexpr/eval.hh | 4 ++- src/libexpr/flake/config.cc | 1 - src/libexpr/flake/flake.cc | 6 ++--- src/libexpr/parser-state.hh | 1 + src/libexpr/parser.y | 6 +++-- src/libexpr/primops.cc | 25 +++++++++--------- src/libexpr/primops/fetchMercurial.cc | 2 +- src/libexpr/primops/fetchTree.cc | 8 +++--- src/nix-build/nix-build.cc | 2 +- src/nix-env/nix-env.cc | 2 +- src/nix-instantiate/nix-instantiate.cc | 2 +- src/nix/main.cc | 4 +-- src/nix/prefetch.cc | 2 +- src/nix/upgrade-nix.cc | 2 +- tests/unit/libexpr-support/tests/libexpr.hh | 6 +++-- 24 files changed, 102 insertions(+), 71 deletions(-) diff --git a/.editorconfig b/.editorconfig index 86360e658..e1c8bae39 100644 --- a/.editorconfig +++ b/.editorconfig @@ -4,20 +4,20 @@ # Top-most EditorConfig file root = true -# Unix-style newlines with a newline ending every file, utf-8 charset +# Unix-style newlines with a newline ending every file, UTF-8 charset [*] end_of_line = lf insert_final_newline = true trim_trailing_whitespace = true charset = utf-8 -# Match nix files, set indent to spaces with width of two +# Match Nix files, set indent to spaces with width of two [*.nix] indent_style = space indent_size = 2 -# Match c++/shell/perl, set indent to spaces with width of four -[*.{hpp,cc,hh,sh,pl,xs}] +# Match C++/C/shell/Perl, set indent to spaces with width of four +[*.{hpp,cc,hh,c,h,sh,pl,xs}] indent_style = space indent_size = 4 diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 543250da3..23aaaf2ad 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -126,13 +126,11 @@ ref EvalCommand::getEvalState() { if (!evalState) { evalState = + std::allocate_shared( #if HAVE_BOEHMGC - std::allocate_shared(traceable_allocator(), - lookupPath, getEvalStore(), getStore()) - #else - std::make_shared( - lookupPath, getEvalStore(), getStore()) + traceable_allocator(), #endif + lookupPath, getEvalStore(), evalSettings, getStore()) ; evalState->repair = repair; diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index cd0f19257..77dba546d 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -1,6 +1,7 @@ #include "eval-settings.hh" #include "common-eval-args.hh" #include "shared.hh" +#include "config-global.hh" #include "filetransfer.hh" #include "eval.hh" #include "fetchers.hh" @@ -13,6 +14,12 @@ namespace nix { +EvalSettings evalSettings { + settings.readOnlyMode +}; + +static GlobalConfig::Register rEvalSettings(&evalSettings); + MixEvalArgs::MixEvalArgs() { addFlag({ diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 75cb19334..189abf0ed 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -12,9 +12,15 @@ namespace nix { class Store; class EvalState; +struct EvalSettings; class Bindings; struct SourcePath; +/** + * @todo Get rid of global setttings variables + */ +extern EvalSettings evalSettings; + struct MixEvalArgs : virtual Args, virtual MixRepair { static constexpr auto category = "Common evaluation options"; diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index bdf7a1e63..13e0f5f3a 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -7,6 +7,7 @@ #include "eval.hh" #include "globals.hh" #include "util.hh" +#include "eval-settings.hh" #include "nix_api_expr.h" #include "nix_api_expr_internal.h" @@ -106,7 +107,21 @@ EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c for (size_t i = 0; lookupPath_c[i] != nullptr; i++) lookupPath.push_back(lookupPath_c[i]); - return new EvalState{nix::EvalState(nix::LookupPath::parse(lookupPath), store->ptr)}; + void * p = ::operator new( + sizeof(EvalState), + static_cast(alignof(EvalState))); + auto * p2 = static_cast(p); + new (p) EvalState { + .settings = nix::EvalSettings{ + nix::settings.readOnlyMode, + }, + .state = nix::EvalState( + nix::LookupPath::parse(lookupPath), + store->ptr, + p2->settings), + }; + loadConfFile(p2->settings); + return p2; } NIXC_CATCH_ERRS_NULL } diff --git a/src/libexpr-c/nix_api_expr_internal.h b/src/libexpr-c/nix_api_expr_internal.h index 5a875ef39..d4ccffd29 100644 --- a/src/libexpr-c/nix_api_expr_internal.h +++ b/src/libexpr-c/nix_api_expr_internal.h @@ -2,11 +2,13 @@ #define NIX_API_EXPR_INTERNAL_H #include "eval.hh" +#include "eval-settings.hh" #include "attr-set.hh" #include "nix_api_value.h" struct EvalState { + nix::EvalSettings settings; nix::EvalState state; }; diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index a642bb684..11a62b0fd 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -45,7 +45,8 @@ static Strings parseNixPath(const std::string & s) return res; } -EvalSettings::EvalSettings() +EvalSettings::EvalSettings(bool & readOnlyMode) + : readOnlyMode{readOnlyMode} { auto var = getEnv("NIX_PATH"); if (var) nixPath = parseNixPath(*var); @@ -55,7 +56,7 @@ EvalSettings::EvalSettings() builtinsAbortOnWarn = true; } -Strings EvalSettings::getDefaultNixPath() +Strings EvalSettings::getDefaultNixPath() const { Strings res; auto add = [&](const Path & p, const std::string & s = std::string()) { @@ -68,7 +69,7 @@ Strings EvalSettings::getDefaultNixPath() } }; - if (!evalSettings.restrictEval && !evalSettings.pureEval) { + if (!restrictEval && !pureEval) { add(getNixDefExpr() + "/channels"); add(rootChannelsDir() + "/nixpkgs", "nixpkgs"); add(rootChannelsDir()); @@ -94,16 +95,12 @@ std::string EvalSettings::resolvePseudoUrl(std::string_view url) return std::string(url); } -const std::string & EvalSettings::getCurrentSystem() +const std::string & EvalSettings::getCurrentSystem() const { const auto & evalSystem = currentSystem.get(); return evalSystem != "" ? evalSystem : settings.thisSystem.get(); } -EvalSettings evalSettings; - -static GlobalConfig::Register rEvalSettings(&evalSettings); - Path getNixDefExpr() { return settings.useXDGBaseDirectories diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index d01de980f..2689a8465 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -7,9 +7,11 @@ namespace nix { struct EvalSettings : Config { - EvalSettings(); + EvalSettings(bool & readOnlyMode); - static Strings getDefaultNixPath(); + bool & readOnlyMode; + + Strings getDefaultNixPath() const; static bool isPseudoUrl(std::string_view s); @@ -74,7 +76,7 @@ struct EvalSettings : Config * Implements the `eval-system` vs `system` defaulting logic * described for `eval-system`. */ - const std::string & getCurrentSystem(); + const std::string & getCurrentSystem() const; Setting restrictEval{ this, false, "restrict-eval", @@ -193,8 +195,6 @@ struct EvalSettings : Config )"}; }; -extern EvalSettings evalSettings; - /** * Conventionally part of the default nix path in impure mode. */ diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7aaec2e73..551bcdcaf 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -9,7 +9,6 @@ #include "store-api.hh" #include "derivations.hh" #include "downstream-placeholder.hh" -#include "globals.hh" #include "eval-inline.hh" #include "filetransfer.hh" #include "function-trace.hh" @@ -219,8 +218,10 @@ static constexpr size_t BASE_ENV_SIZE = 128; EvalState::EvalState( const LookupPath & _lookupPath, ref store, + const EvalSettings & settings, std::shared_ptr buildStore) - : sWith(symbols.create("")) + : settings{settings} + , sWith(symbols.create("")) , sOutPath(symbols.create("outPath")) , sDrvPath(symbols.create("drvPath")) , sType(symbols.create("type")) @@ -276,10 +277,10 @@ EvalState::EvalState( , repair(NoRepair) , emptyBindings(0) , rootFS( - evalSettings.restrictEval || evalSettings.pureEval + settings.restrictEval || settings.pureEval ? ref(AllowListSourceAccessor::create(getFSSourceAccessor(), {}, - [](const CanonPath & path) -> RestrictedPathError { - auto modeInformation = evalSettings.pureEval + [&settings](const CanonPath & path) -> RestrictedPathError { + auto modeInformation = settings.pureEval ? "in pure evaluation mode (use '--impure' to override)" : "in restricted mode"; throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation); @@ -330,10 +331,10 @@ EvalState::EvalState( vStringUnknown.mkString("unknown"); /* Initialise the Nix expression search path. */ - if (!evalSettings.pureEval) { + if (!settings.pureEval) { for (auto & i : _lookupPath.elements) lookupPath.elements.emplace_back(LookupPath::Elem {i}); - for (auto & i : evalSettings.nixPath.get()) + for (auto & i : settings.nixPath.get()) lookupPath.elements.emplace_back(LookupPath::Elem::parse(i)); } @@ -411,9 +412,9 @@ bool isAllowedURI(std::string_view uri, const Strings & allowedUris) void EvalState::checkURI(const std::string & uri) { - if (!evalSettings.restrictEval) return; + if (!settings.restrictEval) return; - if (isAllowedURI(uri, evalSettings.allowedUris.get())) return; + if (isAllowedURI(uri, settings.allowedUris.get())) return; /* If the URI is a path, then check it against allowedPaths as well. */ @@ -458,7 +459,7 @@ void EvalState::addConstant(const std::string & name, Value * v, Constant info) constantInfos.push_back({name2, info}); - if (!(evalSettings.pureEval && info.impureOnly)) { + if (!(settings.pureEval && info.impureOnly)) { /* Check the type, if possible. We might know the type of a thunk in advance, so be allowed @@ -1413,11 +1414,11 @@ public: void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const PosIdx pos) { - if (callDepth > evalSettings.maxCallDepth) + if (callDepth > settings.maxCallDepth) error("stack overflow; max-call-depth exceeded").atPos(pos).debugThrow(); CallDepth _level(callDepth); - auto trace = evalSettings.traceFunctionCalls + auto trace = settings.traceFunctionCalls ? std::make_unique(positions[pos]) : nullptr; @@ -2745,7 +2746,7 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_ return {corepkgsFS, CanonPath(path.substr(3))}; error( - evalSettings.pureEval + settings.pureEval ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)" : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)", path @@ -2825,7 +2826,7 @@ Expr * EvalState::parse( const SourcePath & basePath, std::shared_ptr & staticEnv) { - auto result = parseExprFromBuf(text, length, origin, basePath, symbols, positions, rootFS, exprSymbols); + auto result = parseExprFromBuf(text, length, origin, basePath, symbols, settings, positions, rootFS, exprSymbols); result->bindVars(*this, staticEnv); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 774ded3e7..b84bc9907 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -30,6 +30,7 @@ namespace nix { constexpr size_t maxPrimOpArity = 8; class Store; +struct EvalSettings; class EvalState; class StorePath; struct SingleDerivedPath; @@ -39,7 +40,6 @@ namespace eval_cache { class EvalCache; } - /** * Function that implements a primop. */ @@ -162,6 +162,7 @@ struct DebugTrace { class EvalState : public std::enable_shared_from_this { public: + const EvalSettings & settings; SymbolTable symbols; PosTable positions; @@ -352,6 +353,7 @@ public: EvalState( const LookupPath & _lookupPath, ref store, + const EvalSettings & settings, std::shared_ptr buildStore = nullptr); ~EvalState(); diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc index 03430b0e3..b348f6d44 100644 --- a/src/libexpr/flake/config.cc +++ b/src/libexpr/flake/config.cc @@ -1,6 +1,5 @@ #include "users.hh" #include "config-global.hh" -#include "globals.hh" #include "fetch-settings.hh" #include "flake.hh" diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 3af9ef14e..67b19bd57 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -803,7 +803,7 @@ static void prim_getFlake(EvalState & state, const PosIdx pos, Value * * args, V { std::string flakeRefS(state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake")); auto flakeRef = parseFlakeRef(flakeRefS, {}, true); - if (evalSettings.pureEval && !flakeRef.input.isLocked()) + if (state.settings.pureEval && !flakeRef.input.isLocked()) throw Error("cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)", flakeRefS, state.positions[pos]); callFlake(state, @@ -811,8 +811,8 @@ static void prim_getFlake(EvalState & state, const PosIdx pos, Value * * args, V LockFlags { .updateLockFile = false, .writeLockFile = false, - .useRegistries = !evalSettings.pureEval && fetchSettings.useRegistries, - .allowUnlocked = !evalSettings.pureEval, + .useRegistries = !state.settings.pureEval && fetchSettings.useRegistries, + .allowUnlocked = !state.settings.pureEval, }), v); } diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh index 5a928e9aa..cff6282fa 100644 --- a/src/libexpr/parser-state.hh +++ b/src/libexpr/parser-state.hh @@ -46,6 +46,7 @@ struct ParserState PosTable::Origin origin; const ref rootFS; const Expr::AstSymbols & s; + const EvalSettings & settings; void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos); void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos); diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 00300449f..709a4532a 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -25,7 +25,6 @@ #include "nixexpr.hh" #include "eval.hh" #include "eval-settings.hh" -#include "globals.hh" #include "parser-state.hh" #define YYLTYPE ::nix::ParserLocation @@ -40,6 +39,7 @@ Expr * parseExprFromBuf( Pos::Origin origin, const SourcePath & basePath, SymbolTable & symbols, + const EvalSettings & settings, PosTable & positions, const ref rootFS, const Expr::AstSymbols & astSymbols); @@ -294,7 +294,7 @@ path_start $$ = new ExprPath(ref(state->rootFS), std::move(path)); } | HPATH { - if (evalSettings.pureEval) { + if (state->settings.pureEval) { throw Error( "the path '%s' can not be resolved in pure mode", std::string_view($1.p, $1.l) @@ -429,6 +429,7 @@ Expr * parseExprFromBuf( Pos::Origin origin, const SourcePath & basePath, SymbolTable & symbols, + const EvalSettings & settings, PosTable & positions, const ref rootFS, const Expr::AstSymbols & astSymbols) @@ -441,6 +442,7 @@ Expr * parseExprFromBuf( .origin = positions.addOrigin(origin, length), .rootFS = rootFS, .s = astSymbols, + .settings = settings, }; yylex_init(&scanner); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 98649f081..4e2aaf5b2 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -5,7 +5,6 @@ #include "eval.hh" #include "eval-settings.hh" #include "gc-small-vector.hh" -#include "globals.hh" #include "json-to-value.hh" #include "names.hh" #include "path-references.hh" @@ -78,7 +77,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS if (drvs.empty()) return {}; - if (isIFD && !evalSettings.enableImportFromDerivation) + if (isIFD && !settings.enableImportFromDerivation) error( "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled", drvs.begin()->to_string(*store) @@ -901,7 +900,7 @@ static void prim_tryEval(EvalState & state, const PosIdx pos, Value * * args, Va MaintainCount trylevel(state.trylevel); ReplExitStatus (* savedDebugRepl)(ref es, const ValMap & extraEnv) = nullptr; - if (state.debugRepl && evalSettings.ignoreExceptionsDuringTry) + if (state.debugRepl && state.settings.ignoreExceptionsDuringTry) { /* to prevent starting the repl from exceptions withing a tryEval, null it. */ savedDebugRepl = state.debugRepl; @@ -950,7 +949,7 @@ static RegisterPrimOp primop_tryEval({ static void prim_getEnv(EvalState & state, const PosIdx pos, Value * * args, Value & v) { std::string name(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.getEnv")); - v.mkString(evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name).value_or("")); + v.mkString(state.settings.restrictEval || state.settings.pureEval ? "" : getEnv(name).value_or("")); } static RegisterPrimOp primop_getEnv({ @@ -1017,7 +1016,7 @@ static void prim_trace(EvalState & state, const PosIdx pos, Value * * args, Valu printError("trace: %1%", args[0]->string_view()); else printError("trace: %1%", ValuePrinter(state, *args[0])); - if (evalSettings.builtinsTraceDebugger) { + if (state.settings.builtinsTraceDebugger) { state.runDebugRepl(nullptr); } state.forceValue(*args[1], pos); @@ -1056,11 +1055,11 @@ static void prim_warn(EvalState & state, const PosIdx pos, Value * * args, Value logWarning(info); } - if (evalSettings.builtinsAbortOnWarn) { + if (state.settings.builtinsAbortOnWarn) { // Not an EvalError or subclass, which would cause the error to be stored in the eval cache. state.error("aborting to reveal stack trace of warning, as abort-on-warn is set").setIsFromExpr().debugThrow(); } - if (evalSettings.builtinsTraceDebugger || evalSettings.builtinsDebuggerOnWarn) { + if (state.settings.builtinsTraceDebugger || state.settings.builtinsDebuggerOnWarn) { state.runDebugRepl(nullptr); } state.forceValue(*args[1], pos); @@ -1578,7 +1577,7 @@ static RegisterPrimOp primop_toPath({ corner cases. */ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - if (evalSettings.pureEval) + if (state.settings.pureEval) state.error( "'%s' is not allowed in pure evaluation mode", "builtins.storePath" @@ -4562,7 +4561,7 @@ void EvalState::createBaseEnv() )", }); - if (!evalSettings.pureEval) { + if (!settings.pureEval) { v.mkInt(time(0)); } addConstant("__currentTime", v, { @@ -4589,8 +4588,8 @@ void EvalState::createBaseEnv() .impureOnly = true, }); - if (!evalSettings.pureEval) - v.mkString(evalSettings.getCurrentSystem()); + if (!settings.pureEval) + v.mkString(settings.getCurrentSystem()); addConstant("__currentSystem", v, { .type = nString, .doc = R"( @@ -4670,7 +4669,7 @@ void EvalState::createBaseEnv() #ifndef _WIN32 // TODO implement on Windows // Miscellaneous - if (evalSettings.enableNativeCode) { + if (settings.enableNativeCode) { addPrimOp({ .name = "__importNative", .arity = 2, @@ -4693,7 +4692,7 @@ void EvalState::createBaseEnv() error if `--trace-verbose` is enabled. Then return *e2*. This function is useful for debugging. )", - .fun = evalSettings.traceVerbose ? prim_trace : prim_second, + .fun = settings.traceVerbose ? prim_trace : prim_second, }); /* Add a value containing the current Nix expression search path. */ diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index d9ba6aa97..7b5f4193a 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -53,7 +53,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a // whitelist. Ah well. state.checkURI(url); - if (evalSettings.pureEval && !rev) + if (state.settings.pureEval && !rev) throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision"); fetchers::Attrs attrs; diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index a99a71577..7b05c5b48 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -171,10 +171,10 @@ static void fetchTree( } } - if (!evalSettings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes)) + if (!state.settings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes)) input = lookupInRegistries(state.store, input).first; - if (evalSettings.pureEval && !input.isLocked()) { + if (state.settings.pureEval && !input.isLocked()) { auto fetcher = "fetchTree"; if (params.isFetchGit) fetcher = "fetchGit"; @@ -453,14 +453,14 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v url = state.forceStringNoCtx(*args[0], pos, "while evaluating the url we should fetch"); if (who == "fetchTarball") - url = evalSettings.resolvePseudoUrl(*url); + url = state.settings.resolvePseudoUrl(*url); state.checkURI(*url); if (name == "") name = baseNameOf(*url); - if (evalSettings.pureEval && !expectedHash) + if (state.settings.pureEval && !expectedHash) state.error("in pure evaluation mode, '%s' requires a 'sha256' argument", who).atPos(pos).debugThrow(); // early exit if pinned and already in the store diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 77df08edd..57630c8c3 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -259,7 +259,7 @@ static void main_nix_build(int argc, char * * argv) auto store = openStore(); auto evalStore = myArgs.evalStoreUrl ? openStore(*myArgs.evalStoreUrl) : store; - auto state = std::make_unique(myArgs.lookupPath, evalStore, store); + auto state = std::make_unique(myArgs.lookupPath, evalStore, evalSettings, store); state->repair = myArgs.repair; if (myArgs.repair) buildMode = bmRepair; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index b5e13cc23..c72378cd5 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1525,7 +1525,7 @@ static int main_nix_env(int argc, char * * argv) auto store = openStore(); - globals.state = std::shared_ptr(new EvalState(myArgs.lookupPath, store)); + globals.state = std::shared_ptr(new EvalState(myArgs.lookupPath, store, evalSettings)); globals.state->repair = myArgs.repair; globals.instSource.nixExprPath = std::make_shared( diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 35664374c..a4bfeebbb 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -157,7 +157,7 @@ static int main_nix_instantiate(int argc, char * * argv) auto store = openStore(); auto evalStore = myArgs.evalStoreUrl ? openStore(*myArgs.evalStoreUrl) : store; - auto state = std::make_unique(myArgs.lookupPath, evalStore, store); + auto state = std::make_unique(myArgs.lookupPath, evalStore, evalSettings, store); state->repair = myArgs.repair; Bindings & autoArgs = *myArgs.getAutoArgs(*state); diff --git a/src/nix/main.cc b/src/nix/main.cc index 81d368d6a..e95558781 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -242,7 +242,7 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) evalSettings.restrictEval = false; evalSettings.pureEval = false; - EvalState state({}, openStore("dummy://")); + EvalState state({}, openStore("dummy://"), evalSettings); auto vGenerateManpage = state.allocValue(); state.eval(state.parseExprFromString( @@ -418,7 +418,7 @@ void mainWrapped(int argc, char * * argv) Xp::FetchTree, }; evalSettings.pureEval = false; - EvalState state({}, openStore("dummy://")); + EvalState state({}, openStore("dummy://"), evalSettings); auto res = nlohmann::json::object(); res["builtins"] = ({ auto builtinsJson = nlohmann::json::object(); diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 3ce52acc5..ce79e576f 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -193,7 +193,7 @@ static int main_nix_prefetch_url(int argc, char * * argv) startProgressBar(); auto store = openStore(); - auto state = std::make_unique(myArgs.lookupPath, store); + auto state = std::make_unique(myArgs.lookupPath, store, evalSettings); Bindings & autoArgs = *myArgs.getAutoArgs(*state); diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 17d1edb97..29297253b 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -147,7 +147,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand auto req = FileTransferRequest((std::string&) settings.upgradeNixStorePathUrl); auto res = getFileTransfer()->download(req); - auto state = std::make_unique(LookupPath{}, store); + auto state = std::make_unique(LookupPath{}, store, evalSettings); auto v = state->allocValue(); state->eval(state->parseExprFromString(res.data, state->rootPath(CanonPath("/no-such-path"))), *v); Bindings & bindings(*state->allocBindings(0)); diff --git a/tests/unit/libexpr-support/tests/libexpr.hh b/tests/unit/libexpr-support/tests/libexpr.hh index 1a4313990..eacbf0d5c 100644 --- a/tests/unit/libexpr-support/tests/libexpr.hh +++ b/tests/unit/libexpr-support/tests/libexpr.hh @@ -19,14 +19,14 @@ namespace nix { static void SetUpTestSuite() { LibStoreTest::SetUpTestSuite(); initGC(); - evalSettings.nixPath = {}; } protected: LibExprTest() : LibStoreTest() - , state({}, store) + , state({}, store, evalSettings, nullptr) { + evalSettings.nixPath = {}; } Value eval(std::string input, bool forceValue = true) { Value v; @@ -42,6 +42,8 @@ namespace nix { return state.symbols.create(value); } + bool readOnlyMode = true; + EvalSettings evalSettings{readOnlyMode}; EvalState state; }; From 445a4a02987ae24b69597dfb7debaeb9a474d26d Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 24 Jun 2024 19:02:22 +0200 Subject: [PATCH 36/61] ci.yml: Add swap and monitor it --- .github/workflows/ci.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6362620b0..2cd908f9e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,23 @@ jobs: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + - if: matrix.os == 'ubuntu-latest' + run: | + free -h + swapon --show + swap=$(swapon --show --noheadings | head -n 1 | awk '{print $1}') + echo "Found swap: $swap" + sudo swapoff $swap + # resize it (fallocate) + sudo fallocate -l 10G $swap + sudo mkswap $swap + sudo swapon $swap + free -h + ( + while sleep 60; do + free -h + done + ) & - run: nix --experimental-features 'nix-command flakes' flake check -L # Steps to test CI automation in your own fork. From 05580a373f5f85b97927dafff5a50486a7c1dbd9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 24 Jun 2024 17:17:54 -0400 Subject: [PATCH 37/61] Fix error in the no-GC build --- src/libcmd/command.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 23aaaf2ad..74d146c66 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -126,9 +126,11 @@ ref EvalCommand::getEvalState() { if (!evalState) { evalState = - std::allocate_shared( #if HAVE_BOEHMGC + std::allocate_shared( traceable_allocator(), + #else + std::make_shared( #endif lookupPath, getEvalStore(), evalSettings, getStore()) ; From 5be44d235a2aaaeb9da58bcb8502fa5cae0f1d30 Mon Sep 17 00:00:00 2001 From: Brian McKenna Date: Thu, 15 Sep 2022 05:56:57 +0000 Subject: [PATCH 38/61] Guard uses of lutimes, for portability --- src/libfetchers/git.cc | 22 ++------ src/libstore/posix-fs-canonicalise.cc | 16 ++---- src/libutil/file-system.cc | 78 ++++++++++++++++++++------- src/libutil/file-system.hh | 24 +++++++++ 4 files changed, 90 insertions(+), 50 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index ce80932f6..184c1383e 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -41,21 +41,6 @@ bool isCacheFileWithinTtl(time_t now, const struct stat & st) return st.st_mtime + settings.tarballTtl > now; } -bool touchCacheFile(const Path & path, time_t touch_time) -{ -#ifndef _WIN32 // TODO implement - struct timeval times[2]; - times[0].tv_sec = touch_time; - times[0].tv_usec = 0; - times[1].tv_sec = touch_time; - times[1].tv_usec = 0; - - return lutimes(path.c_str(), times) == 0; -#else - return false; -#endif -} - Path getCachePath(std::string_view key, bool shallow) { return getCacheDir() @@ -594,8 +579,11 @@ struct GitInputScheme : InputScheme warn("could not update local clone of Git repository '%s'; continuing with the most recent version", repoInfo.url); } - if (!touchCacheFile(localRefFile, now)) - warn("could not update mtime for file '%s': %s", localRefFile, strerror(errno)); + try { + setWriteTime(localRefFile, now, now); + } catch (Error & e) { + warn("could not update mtime for file '%s': %s", localRefFile, e.msg()); + } if (!originalRef && !storeCachedHead(repoInfo.url, ref)) warn("could not update cached head '%s' for '%s'", ref, repoInfo.url); } diff --git a/src/libstore/posix-fs-canonicalise.cc b/src/libstore/posix-fs-canonicalise.cc index 8cb13d810..46a78cc86 100644 --- a/src/libstore/posix-fs-canonicalise.cc +++ b/src/libstore/posix-fs-canonicalise.cc @@ -33,19 +33,9 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct #ifndef _WIN32 // TODO implement if (st.st_mtime != mtimeStore) { - struct timeval times[2]; - times[0].tv_sec = st.st_atime; - times[0].tv_usec = 0; - times[1].tv_sec = mtimeStore; - times[1].tv_usec = 0; -#if HAVE_LUTIMES - if (lutimes(path.c_str(), times) == -1) - if (errno != ENOSYS || - (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)) -#else - if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) -#endif - throw SysError("changing modification time of '%1%'", path); + struct stat st2 = st; + st2.st_mtime = mtimeStore, + setWriteTime(path, st2); } #endif } diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 443ccf829..2536361b2 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -557,29 +557,69 @@ void replaceSymlink(const Path & target, const Path & link) } } -#ifndef _WIN32 -static void setWriteTime(const fs::path & p, const struct stat & st) +void setWriteTime( + const std::filesystem::path & path, + time_t accessedTime, + time_t modificationTime, + std::optional optIsSymlink) { - struct timeval times[2]; - times[0] = { - .tv_sec = st.st_atime, - .tv_usec = 0, +#ifndef _WIN32 + struct timeval times[2] = { + { + .tv_sec = accessedTime, + .tv_usec = 0, + }, + { + .tv_sec = modificationTime, + .tv_usec = 0, + }, }; - times[1] = { - .tv_sec = st.st_mtime, - .tv_usec = 0, - }; - if (lutimes(p.c_str(), times) != 0) - throw SysError("changing modification time of '%s'", p); -} #endif + auto nonSymlink = [&]{ + bool isSymlink = optIsSymlink + ? *optIsSymlink + : fs::is_symlink(path); + + if (!isSymlink) { +#ifdef _WIN32 + // FIXME use `fs::last_write_time`. + // + // Would be nice to use std::filesystem unconditionally, but + // doesn't support access time just modification time. + // + // System clock vs File clock issues also make that annoying. + warn("Changing file times is not yet implemented on Windows, path is '%s'", path); +#else + if (utimes(path.c_str(), times) == -1) { + + throw SysError("changing modification time of '%s' (not a symlink)", path); + } +#endif + } else { + throw Error("Cannot modification time of symlink '%s'", path); + } + }; + +#if HAVE_LUTIMES + if (lutimes(path.c_str(), times) == -1) { + if (errno == ENOSYS) + nonSymlink(); + else + throw SysError("changing modification time of '%s'", path); + } +#else + nonSymlink(); +#endif +} + +void setWriteTime(const fs::path & path, const struct stat & st) +{ + setWriteTime(path, st.st_atime, st.st_mtime, S_ISLNK(st.st_mode)); +} + void copyFile(const fs::path & from, const fs::path & to, bool andDelete) { -#ifndef _WIN32 - // TODO: Rewrite the `is_*` to use `symlink_status()` - auto statOfFrom = lstat(from.c_str()); -#endif auto fromStatus = fs::symlink_status(from); // Mark the directory as writable so that we can delete its children @@ -599,9 +639,7 @@ void copyFile(const fs::path & from, const fs::path & to, bool andDelete) throw Error("file '%s' has an unsupported type", from); } -#ifndef _WIN32 - setWriteTime(to, statOfFrom); -#endif + setWriteTime(to, lstat(from.string().c_str())); if (andDelete) { if (!fs::is_symlink(fromStatus)) fs::permissions(from, fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index 9405cda0c..aeea80e00 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -156,6 +156,30 @@ inline void createDirs(PathView path) return createDirs(Path(path)); } +/** + * Set the access and modification times of the given path, not + * following symlinks. + * + * @param accessTime Specified in seconds. + * + * @param modificationTime Specified in seconds. + * + * @param isSymlink Whether the file in question is a symlink. Used for + * fallback code where we don't have `lutimes` or similar. if + * `std::optional` is passed, the information will be recomputed if it + * is needed. Race conditions are possible so be careful! + */ +void setWriteTime( + const std::filesystem::path & path, + time_t accessedTime, + time_t modificationTime, + std::optional isSymlink = std::nullopt); + +/** + * Convenience wrapper that takes all arguments from the `struct stat`. + */ +void setWriteTime(const std::filesystem::path & path, const struct stat & st); + /** * Create a symlink. */ From 1801119e2919b2a746964fd6e9a12f44f267b711 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 25 Jun 2024 10:14:06 +0200 Subject: [PATCH 39/61] ci.yml: Add meson_build Restore meson CI after https://github.com/NixOS/nix/pull/10929 --- .github/workflows/ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2cd908f9e..c9e4c25ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -193,3 +193,11 @@ jobs: - uses: DeterminateSystems/nix-installer-action@main - uses: DeterminateSystems/magic-nix-cache-action@main - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes .#hydraJobs.tests.functional_user + + meson_build: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/nix-installer-action@main + - uses: DeterminateSystems/magic-nix-cache-action@main + - run: nix build -L .#hydraJobs.build.{nix-fetchers,nix-store,nix-util}.$(nix-instantiate --eval --expr builtins.currentSystem | sed -e 's/"//g') From 0674be8d498d6ad1ca8bb383b26107ccdb64148d Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 25 Jun 2024 10:21:34 +0200 Subject: [PATCH 40/61] nix-util: Fix build --- src/libutil/meson.build | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 2259d4e22..9d0b28539 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -161,6 +161,7 @@ sources = files( 'compression.cc', 'compute-levels.cc', 'config.cc', + 'config-global.cc', 'current-process.cc', 'english.cc', 'environment-variables.cc', @@ -211,6 +212,7 @@ headers = [config_h] + files( 'comparator.hh', 'compression.hh', 'compute-levels.hh', + 'config-global.hh', 'config-impl.hh', 'config.hh', 'current-process.hh', From 7df9d6da6542c06ba017e8dec1b63751a4bae847 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 20 Jun 2024 22:20:17 +0200 Subject: [PATCH 41/61] Improve error messages for invalid derivation names --- src/libexpr/primops.cc | 22 ++++++++++++++ src/libexpr/primops/fetchTree.cc | 22 ++++++++++++-- src/libstore/path.cc | 30 ++++++++++++------- src/libstore/path.hh | 9 ++++++ src/libstore/store-dir-config.hh | 1 + src/libutil/error.hh | 1 + src/libutil/fmt.hh | 2 ++ .../lang/eval-fail-derivation-name.err.exp | 26 ++++++++++++++++ .../lang/eval-fail-derivation-name.nix | 5 ++++ ...-fail-fetchurl-baseName-attrs-name.err.exp | 8 +++++ ...eval-fail-fetchurl-baseName-attrs-name.nix | 1 + .../eval-fail-fetchurl-baseName-attrs.err.exp | 8 +++++ .../eval-fail-fetchurl-baseName-attrs.nix | 1 + .../lang/eval-fail-fetchurl-baseName.err.exp | 8 +++++ .../lang/eval-fail-fetchurl-baseName.nix | 1 + tests/unit/libstore/path.cc | 16 ++++++++-- 16 files changed, 145 insertions(+), 16 deletions(-) create mode 100644 tests/functional/lang/eval-fail-derivation-name.err.exp create mode 100644 tests/functional/lang/eval-fail-derivation-name.nix create mode 100644 tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.err.exp create mode 100644 tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.nix create mode 100644 tests/functional/lang/eval-fail-fetchurl-baseName-attrs.err.exp create mode 100644 tests/functional/lang/eval-fail-fetchurl-baseName-attrs.nix create mode 100644 tests/functional/lang/eval-fail-fetchurl-baseName.err.exp create mode 100644 tests/functional/lang/eval-fail-fetchurl-baseName.nix diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 000fce455..212441019 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1162,12 +1162,34 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * } } +/** + * Early validation for the derivation name, for better error message. + * It is checked again when constructing store paths. + * + * @todo Check that the `.drv` suffix also fits. + */ +static void checkDerivationName(EvalState & state, std::string_view drvName) +{ + try { + checkName(drvName); + } catch (BadStorePathName & e) { + // "Please pass a different name": Users may not be aware that they can + // pass a different one, in functions like `fetchurl` where the name + // is optional. + // Note that Nixpkgs generally won't trigger this, because `mkDerivation` + // sanitizes the name. + state.error("invalid derivation name: %s. Please pass a different '%s'.", Uncolored(e.message()), "name").debugThrow(); + } +} + static void derivationStrictInternal( EvalState & state, const std::string & drvName, const Bindings * attrs, Value & v) { + checkDerivationName(state, drvName); + /* Check whether attributes should be passed as a JSON file. */ using nlohmann::json; std::optional jsonObject; diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index db2cc0d2b..fa6b1c4b6 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -431,7 +431,10 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v state.forceValue(*args[0], pos); - if (args[0]->type() == nAttrs) { + bool isArgAttrs = args[0]->type() == nAttrs; + bool nameAttrPassed = false; + + if (isArgAttrs) { for (auto & attr : *args[0]->attrs()) { std::string_view n(state.symbols[attr.name]); @@ -439,8 +442,10 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v url = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the url we should fetch"); else if (n == "sha256") expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the sha256 of the content we should fetch"), HashAlgorithm::SHA256); - else if (n == "name") + else if (n == "name") { + nameAttrPassed = true; name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the name of the content we should fetch"); + } else state.error("unsupported argument '%s' to '%s'", n, who) .atPos(pos).debugThrow(); @@ -460,6 +465,19 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v if (name == "") name = baseNameOf(*url); + try { + checkName(name); + } catch (BadStorePathName & e) { + auto resolution = + nameAttrPassed ? HintFmt("Please change the value for the 'name' attribute passed to '%s', so that it can create a valid store path.", who) : + isArgAttrs ? HintFmt("Please add a valid 'name' attribute to the argument for '%s', so that it can create a valid store path.", who) : + HintFmt("Please pass an attribute set with 'url' and 'name' attributes to '%s', so that it can create a valid store path.", who); + + state.error( + std::string("invalid store path name when fetching URL '%s': %s. %s"), *url, Uncolored(e.message()), Uncolored(resolution.str())) + .atPos(pos).debugThrow(); + } + if (state.settings.pureEval && !expectedHash) state.error("in pure evaluation mode, '%s' requires a 'sha256' argument", who).atPos(pos).debugThrow(); diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 8d9726722..3e9d05477 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -2,25 +2,24 @@ namespace nix { -static void checkName(std::string_view path, std::string_view name) +void checkName(std::string_view name) { if (name.empty()) - throw BadStorePath("store path '%s' has an empty name", path); + throw BadStorePathName("name must not be empty"); if (name.size() > StorePath::MaxPathLen) - throw BadStorePath("store path '%s' has a name longer than %d characters", - path, StorePath::MaxPathLen); + throw BadStorePathName("name '%s' must be no longer than %d characters", name, StorePath::MaxPathLen); // See nameRegexStr for the definition if (name[0] == '.') { // check against "." and "..", followed by end or dash if (name.size() == 1) - throw BadStorePath("store path '%s' has invalid name '%s'", path, name); + throw BadStorePathName("name '%s' is not valid", name); if (name[1] == '-') - throw BadStorePath("store path '%s' has invalid name '%s': first dash-separated component must not be '%s'", path, name, "."); + throw BadStorePathName("name '%s' is not valid: first dash-separated component must not be '%s'", name, "."); if (name[1] == '.') { if (name.size() == 2) - throw BadStorePath("store path '%s' has invalid name '%s'", path, name); + throw BadStorePathName("name '%s' is not valid", name); if (name[2] == '-') - throw BadStorePath("store path '%s' has invalid name '%s': first dash-separated component must not be '%s'", path, name, ".."); + throw BadStorePathName("name '%s' is not valid: first dash-separated component must not be '%s'", name, ".."); } } for (auto c : name) @@ -28,7 +27,16 @@ static void checkName(std::string_view path, std::string_view name) || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '+' || c == '-' || c == '.' || c == '_' || c == '?' || c == '=')) - throw BadStorePath("store path '%s' contains illegal character '%s'", path, c); + throw BadStorePathName("name '%s' contains illegal character '%s'", name, c); +} + +static void checkPathName(std::string_view path, std::string_view name) +{ + try { + checkName(name); + } catch (BadStorePathName & e) { + throw BadStorePath("path '%s' is not a valid store path: %s", path, Uncolored(e.message())); + } } StorePath::StorePath(std::string_view _baseName) @@ -40,13 +48,13 @@ StorePath::StorePath(std::string_view _baseName) if (c == 'e' || c == 'o' || c == 'u' || c == 't' || !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z'))) throw BadStorePath("store path '%s' contains illegal base-32 character '%s'", baseName, c); - checkName(baseName, name()); + checkPathName(baseName, name()); } StorePath::StorePath(const Hash & hash, std::string_view _name) : baseName((hash.to_string(HashFormat::Nix32, false) + "-").append(std::string(_name))) { - checkName(baseName, name()); + checkPathName(baseName, name()); } bool StorePath::isDerivation() const noexcept diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 4abbfcd7c..2380dc6a2 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -9,6 +9,13 @@ namespace nix { struct Hash; +/** + * Check whether a name is a valid store path name. + * + * @throws BadStorePathName if the name is invalid. The message is of the format "name %s is not valid, for this specific reason". + */ +void checkName(std::string_view name); + /** * \ref StorePath "Store path" is the fundamental reference type of Nix. * A store paths refers to a Store object. @@ -31,8 +38,10 @@ public: StorePath() = delete; + /** @throws BadStorePath */ StorePath(std::string_view baseName); + /** @throws BadStorePath */ StorePath(const Hash & hash, std::string_view name); std::string_view to_string() const noexcept diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh index fe86ce40d..64c0dd8b7 100644 --- a/src/libstore/store-dir-config.hh +++ b/src/libstore/store-dir-config.hh @@ -16,6 +16,7 @@ namespace nix { struct SourcePath; MakeError(BadStorePath, Error); +MakeError(BadStorePathName, BadStorePath); struct StoreDirConfig : public Config { diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 269000016..4b08a045e 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -155,6 +155,7 @@ public: : err(e) { } + /** The error message without "error: " prefixed to it. */ std::string message() { return err.msg.str(); } diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index ef44a8409..850b7162d 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -111,6 +111,8 @@ std::ostream & operator<<(std::ostream & out, const Magenta & y) /** * Values wrapped in this class are printed without coloring. * + * Specifically, the color is reset to normal before printing the value. + * * By default, arguments to `HintFmt` are printed in magenta (see `Magenta`). */ template diff --git a/tests/functional/lang/eval-fail-derivation-name.err.exp b/tests/functional/lang/eval-fail-derivation-name.err.exp new file mode 100644 index 000000000..eb2206df1 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivation-name.err.exp @@ -0,0 +1,26 @@ +error: + … while evaluating the attribute 'outPath' + at :19:9: + 18| value = commonAttrs // { + 19| outPath = builtins.getAttr outputName strict; + | ^ + 20| drvPath = strict.drvPath; + + … while calling the 'getAttr' builtin + at :19:19: + 18| value = commonAttrs // { + 19| outPath = builtins.getAttr outputName strict; + | ^ + 20| drvPath = strict.drvPath; + + … while calling the 'derivationStrict' builtin + at :9:12: + 8| + 9| strict = derivationStrict drvAttrs; + | ^ + 10| + + … while evaluating derivation '~jiggle~' + whose name attribute is located at /pwd/lang/eval-fail-derivation-name.nix:2:3 + + error: invalid derivation name: name '~jiggle~' contains illegal character '~'. Please pass a different 'name'. diff --git a/tests/functional/lang/eval-fail-derivation-name.nix b/tests/functional/lang/eval-fail-derivation-name.nix new file mode 100644 index 000000000..e779ad6ff --- /dev/null +++ b/tests/functional/lang/eval-fail-derivation-name.nix @@ -0,0 +1,5 @@ +derivation { + name = "~jiggle~"; + system = "some-system"; + builder = "/dontcare"; +} diff --git a/tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.err.exp b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.err.exp new file mode 100644 index 000000000..30f8b6a35 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fetchurl' builtin + at /pwd/lang/eval-fail-fetchurl-baseName-attrs-name.nix:1:1: + 1| builtins.fetchurl { url = "https://example.com/foo.tar.gz"; name = "~wobble~"; } + | ^ + 2| + + error: invalid store path name when fetching URL 'https://example.com/foo.tar.gz': name '~wobble~' contains illegal character '~'. Please change the value for the 'name' attribute passed to 'fetchurl', so that it can create a valid store path. diff --git a/tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.nix b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.nix new file mode 100644 index 000000000..583805539 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.nix @@ -0,0 +1 @@ +builtins.fetchurl { url = "https://example.com/foo.tar.gz"; name = "~wobble~"; } diff --git a/tests/functional/lang/eval-fail-fetchurl-baseName-attrs.err.exp b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs.err.exp new file mode 100644 index 000000000..cef532e94 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fetchurl' builtin + at /pwd/lang/eval-fail-fetchurl-baseName-attrs.nix:1:1: + 1| builtins.fetchurl { url = "https://example.com/~wiggle~"; } + | ^ + 2| + + error: invalid store path name when fetching URL 'https://example.com/~wiggle~': name '~wiggle~' contains illegal character '~'. Please add a valid 'name' attribute to the argument for 'fetchurl', so that it can create a valid store path. diff --git a/tests/functional/lang/eval-fail-fetchurl-baseName-attrs.nix b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs.nix new file mode 100644 index 000000000..068120edb --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs.nix @@ -0,0 +1 @@ +builtins.fetchurl { url = "https://example.com/~wiggle~"; } diff --git a/tests/functional/lang/eval-fail-fetchurl-baseName.err.exp b/tests/functional/lang/eval-fail-fetchurl-baseName.err.exp new file mode 100644 index 000000000..0950e8e70 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchurl-baseName.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fetchurl' builtin + at /pwd/lang/eval-fail-fetchurl-baseName.nix:1:1: + 1| builtins.fetchurl "https://example.com/~wiggle~" + | ^ + 2| + + error: invalid store path name when fetching URL 'https://example.com/~wiggle~': name '~wiggle~' contains illegal character '~'. Please pass an attribute set with 'url' and 'name' attributes to 'fetchurl', so that it can create a valid store path. diff --git a/tests/functional/lang/eval-fail-fetchurl-baseName.nix b/tests/functional/lang/eval-fail-fetchurl-baseName.nix new file mode 100644 index 000000000..965093843 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchurl-baseName.nix @@ -0,0 +1 @@ +builtins.fetchurl "https://example.com/~wiggle~" diff --git a/tests/unit/libstore/path.cc b/tests/unit/libstore/path.cc index 213b6e95f..c4c055abf 100644 --- a/tests/unit/libstore/path.cc +++ b/tests/unit/libstore/path.cc @@ -26,10 +26,19 @@ static std::regex nameRegex { std::string { nameRegexStr } }; TEST_F(StorePathTest, bad_ ## NAME) { \ std::string_view str = \ STORE_DIR HASH_PART "-" STR; \ - ASSERT_THROW( \ - store->parseStorePath(str), \ - BadStorePath); \ + /* ASSERT_THROW generates a duplicate goto label */ \ + /* A lambda isolates those labels. */ \ + [&](){ \ + ASSERT_THROW( \ + store->parseStorePath(str), \ + BadStorePath); \ + }(); \ std::string name { STR }; \ + [&](){ \ + ASSERT_THROW( \ + nix::checkName(name), \ + BadStorePathName); \ + }(); \ EXPECT_FALSE(std::regex_match(name, nameRegex)); \ } @@ -54,6 +63,7 @@ TEST_DONT_PARSE(dot_dash_a, ".-a") STORE_DIR HASH_PART "-" STR; \ auto p = store->parseStorePath(str); \ std::string name { p.name() }; \ + EXPECT_EQ(p.name(), STR); \ EXPECT_TRUE(std::regex_match(name, nameRegex)); \ } From ac89828b5a78045fc4943d47041a5599302a566c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 25 Jun 2024 15:35:47 +0200 Subject: [PATCH 42/61] Build nix-util-c with meson and unit test --- flake.nix | 51 ++++++++- maintainers/hydra.nix | 3 + meson.build | 7 ++ src/libutil-c/.version | 1 + src/libutil-c/meson.build | 117 ++++++++++++++++++++ src/libutil-c/meson.options | 1 + src/libutil-c/nix-util-c.pc.in | 9 ++ src/libutil-c/package.nix | 96 +++++++++++++++++ src/libutil-test | 1 + src/libutil-test-support | 1 + tests/unit/libutil-support/.version | 1 + tests/unit/libutil-support/meson.build | 110 +++++++++++++++++++ tests/unit/libutil-support/package.nix | 98 +++++++++++++++++ tests/unit/libutil/.version | 1 + tests/unit/libutil/meson.build | 142 +++++++++++++++++++++++++ tests/unit/libutil/package.nix | 112 +++++++++++++++++++ 16 files changed, 750 insertions(+), 1 deletion(-) create mode 120000 src/libutil-c/.version create mode 100644 src/libutil-c/meson.build create mode 100644 src/libutil-c/meson.options create mode 100644 src/libutil-c/nix-util-c.pc.in create mode 100644 src/libutil-c/package.nix create mode 120000 src/libutil-test create mode 120000 src/libutil-test-support create mode 120000 tests/unit/libutil-support/.version create mode 100644 tests/unit/libutil-support/meson.build create mode 100644 tests/unit/libutil-support/package.nix create mode 100644 tests/unit/libutil/.version create mode 100644 tests/unit/libutil/meson.build create mode 100644 tests/unit/libutil/package.nix diff --git a/flake.nix b/flake.nix index 7e7148afe..aac45af34 100644 --- a/flake.nix +++ b/flake.nix @@ -160,6 +160,19 @@ }; }); + # TODO: define everything here instead of top level? + nix-components = { + inherit (final) + nix-util + nix-util-test-support + nix-util-test + nix-util-c + nix-store + nix-fetchers + nix-perl-bindings + ; + }; + nix-util = final.callPackage ./src/libutil/package.nix { inherit fileset @@ -169,6 +182,30 @@ ; }; + nix-util-test-support = final.callPackage ./tests/unit/libutil-support/package.nix { + inherit + fileset + stdenv + versionSuffix + ; + }; + + nix-util-test = final.callPackage ./tests/unit/libutil/package.nix { + inherit + fileset + stdenv + versionSuffix + ; + }; + + nix-util-c = final.callPackage ./src/libutil-c/package.nix { + inherit + fileset + stdenv + versionSuffix + ; + }; + nix-store = final.callPackage ./src/libstore/package.nix { inherit fileset @@ -281,7 +318,19 @@ # the old build system is gone and we are back to one build # system, we should reenable this. #perlBindings = self.hydraJobs.perlBindings.${system}; - } // devFlake.checks.${system} or {} + } + // lib.concatMapAttrs (nixpkgsPrefix: nixpkgs: + lib.concatMapAttrs (pkgName: pkg: + lib.concatMapAttrs (testName: test: { + # Add "passthru" tests + "${nixpkgsPrefix}${pkgName}-${testName}" = test; + }) pkg.tests or {} + ) nixpkgs.nix-components + ) { + "" = nixpkgsFor.${system}.native; + "static-" = nixpkgsFor.${system}.static; + } + // devFlake.checks.${system} or {} ); packages = forAllSystems (system: { diff --git a/maintainers/hydra.nix b/maintainers/hydra.nix index e21145f37..9e6f2c468 100644 --- a/maintainers/hydra.nix +++ b/maintainers/hydra.nix @@ -36,6 +36,9 @@ let forAllPackages = lib.genAttrs [ "nix" "nix-util" + "nix-util-c" + "nix-util-test-support" + "nix-util-test" "nix-store" "nix-fetchers" ]; diff --git a/meson.build b/meson.build index e67fd4a7a..085ac0865 100644 --- a/meson.build +++ b/meson.build @@ -12,3 +12,10 @@ subproject('libfetchers') subproject('perl') subproject('internal-api-docs') subproject('external-api-docs') + +# C wrappers +subproject('libutil-c') + +# Testing +subproject('libutil-test-support') +subproject('libutil-test') diff --git a/src/libutil-c/.version b/src/libutil-c/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libutil-c/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libutil-c/meson.build b/src/libutil-c/meson.build new file mode 100644 index 000000000..a2b020818 --- /dev/null +++ b/src/libutil-c/meson.build @@ -0,0 +1,117 @@ +project('nix-util-c', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +# See note in ../nix-util/meson.build +deps_private = [ ] + +# See note in ../nix-util/meson.build +deps_public = [ ] + +# See note in ../nix-util/meson.build +deps_other = [ ] + +configdata = configuration_data() + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.h', + # '-include', 'config-store.h', + '-Wno-deprecated-declarations', + '-Wimplicit-fallthrough', + '-Werror=switch', + '-Werror=switch-enum', + '-Wdeprecated-copy', + '-Wignored-qualifiers', + # Enable assertions in libstdc++ by default. Harmless on libc++. Benchmarked + # at ~1% overhead in `nix search`. + # + # FIXME: remove when we get meson 1.4.0 which will default this to on for us: + # https://mesonbuild.com/Release-notes-for-1-4-0.html#ndebug-setting-now-controls-c-stdlib-assertions + '-D_GLIBCXX_ASSERTIONS=1', + language : 'cpp', +) + +sources = files( + 'nix_api_util.cc', +) + +include_dirs = [include_directories('.')] + +headers = files( + 'nix_api_util.h', + 'nix_api_util_internal.h', +) + +if host_machine.system() == 'cygwin' or host_machine.system() == 'windows' + # Windows DLLs are stricter about symbol visibility than Unix shared + # objects --- see https://gcc.gnu.org/wiki/Visibility for details. + # This is a temporary sledgehammer to export everything like on Unix, + # and not detail with this yet. + # + # TODO do not do this, and instead do fine-grained export annotations. + linker_export_flags = ['-Wl,--export-all-symbols'] +else + linker_export_flags = [] +endif + +nix_util = dependency('nix-util') +if nix_util.type_name() == 'internal' + # subproject sadly no good for pkg-config module + deps_other += nix_util +else + deps_public += nix_util +endif + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +config_h = configure_file( + configuration : configdata, + output : 'config-util.h', +) + +this_library = library( + 'nixutilc', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +import('pkgconfig').generate( + this_library, + filebase : meson.project_name(), + name : 'Nix', + description : 'Nix Package Manager', + subdirs : ['nix'], + extra_cflags : ['-std=c++2a'], + requires : deps_public, + requires_private : deps_private, + libraries_private : libraries_private, +) + +meson.override_dependency(meson.project_name(), declare_dependency( + include_directories : include_dirs, + link_with : this_library, + compile_args : ['-std=c++2a'], + dependencies : [], +)) diff --git a/src/libutil-c/meson.options b/src/libutil-c/meson.options new file mode 100644 index 000000000..04422feaf --- /dev/null +++ b/src/libutil-c/meson.options @@ -0,0 +1 @@ +# vim: filetype=meson diff --git a/src/libutil-c/nix-util-c.pc.in b/src/libutil-c/nix-util-c.pc.in new file mode 100644 index 000000000..0ccae3f8a --- /dev/null +++ b/src/libutil-c/nix-util-c.pc.in @@ -0,0 +1,9 @@ +prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Nix libutil C API +Description: Common functions for the Nix C API, such as error handling +Version: @PACKAGE_VERSION@ +Libs: -L${libdir} -lnixutil +Cflags: -I${includedir}/nix -std=c++2a diff --git a/src/libutil-c/package.nix b/src/libutil-c/package.nix new file mode 100644 index 000000000..a45b36d4c --- /dev/null +++ b/src/libutil-c/package.nix @@ -0,0 +1,96 @@ +{ lib +, stdenv +, releaseTools +, fileset + +, meson +, ninja +, pkg-config + +, nix-util + +# Configuration Options + +, versionSuffix ? "" + +# Check test coverage of Nix. Probably want to use with at least +# one of `doCheck` or `doInstallCheck` enabled. +, withCoverageChecks ? false +}: + +let + version = lib.fileContents ./.version + versionSuffix; + + mkDerivation = + if withCoverageChecks + then + # TODO support `finalAttrs` args function in + # `releaseTools.coverageAnalysis`. + argsFun: + releaseTools.coverageAnalysis (let args = argsFun args; in args) + else stdenv.mkDerivation; +in + +mkDerivation (finalAttrs: { + pname = "nix-util-c"; + inherit version; + + src = fileset.toSource { + root = ./.; + fileset = fileset.unions [ + ./meson.build + ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + (fileset.fileFilter (file: file.hasExt "h") ./.) + ]; + }; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + nix-util + ] + ; + + propagatedBuildInputs = [ + nix-util + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix + '' + echo ${version} > .version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + enableParallelBuilding = true; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + # TODO Always true after https://github.com/NixOS/nixpkgs/issues/318564 + strictDeps = !withCoverageChecks; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +} // lib.optionalAttrs withCoverageChecks { + lcovFilter = [ "*/boost/*" "*-tab.*" ]; + + hardeningDisable = [ "fortify" ]; +}) diff --git a/src/libutil-test b/src/libutil-test new file mode 120000 index 000000000..62c86f54b --- /dev/null +++ b/src/libutil-test @@ -0,0 +1 @@ +../tests/unit/libutil/ \ No newline at end of file diff --git a/src/libutil-test-support b/src/libutil-test-support new file mode 120000 index 000000000..f7da46d4c --- /dev/null +++ b/src/libutil-test-support @@ -0,0 +1 @@ +../tests/unit/libutil-support/ \ No newline at end of file diff --git a/tests/unit/libutil-support/.version b/tests/unit/libutil-support/.version new file mode 120000 index 000000000..0df9915bf --- /dev/null +++ b/tests/unit/libutil-support/.version @@ -0,0 +1 @@ +../../../.version \ No newline at end of file diff --git a/tests/unit/libutil-support/meson.build b/tests/unit/libutil-support/meson.build new file mode 100644 index 000000000..5912c00e0 --- /dev/null +++ b/tests/unit/libutil-support/meson.build @@ -0,0 +1,110 @@ +project('nix-util-test-support', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +# See note in ../nix-util/meson.build +deps_private = [ ] + +# See note in ../nix-util/meson.build +deps_public = [ ] + +# See note in ../nix-util/meson.build +deps_other = [ ] + +add_project_arguments( + '-Wno-deprecated-declarations', + '-Wimplicit-fallthrough', + '-Werror=switch', + '-Werror=switch-enum', + '-Wdeprecated-copy', + '-Wignored-qualifiers', + # Enable assertions in libstdc++ by default. Harmless on libc++. Benchmarked + # at ~1% overhead in `nix search`. + # + # FIXME: remove when we get meson 1.4.0 which will default this to on for us: + # https://mesonbuild.com/Release-notes-for-1-4-0.html#ndebug-setting-now-controls-c-stdlib-assertions + '-D_GLIBCXX_ASSERTIONS=1', + language : 'cpp', +) + +sources = files( + 'tests/hash.cc', + 'tests/string_callback.cc', +) + +include_dirs = [include_directories('.')] + +headers = files( + 'tests/characterization.hh', + 'tests/hash.hh', + 'tests/nix_api_util.hh', + 'tests/string_callback.hh', +) + +if host_machine.system() == 'cygwin' or host_machine.system() == 'windows' + # Windows DLLs are stricter about symbol visibility than Unix shared + # objects --- see https://gcc.gnu.org/wiki/Visibility for details. + # This is a temporary sledgehammer to export everything like on Unix, + # and not detail with this yet. + # + # TODO do not do this, and instead do fine-grained export annotations. + linker_export_flags = ['-Wl,--export-all-symbols'] +else + linker_export_flags = [] +endif + +nix_util = dependency('nix-util') +if nix_util.type_name() == 'internal' + # subproject sadly no good for pkg-config module + deps_other += nix_util +else + deps_public += nix_util +endif + +rapidcheck = dependency('rapidcheck') +deps_public += rapidcheck + +this_library = library( + 'nix-util-test-support', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + # TODO: Remove `-lrapidcheck` when https://github.com/emil-e/rapidcheck/pull/326 + # is available. See also ../libutil/build.meson + link_args: linker_export_flags + ['-lrapidcheck'], + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +import('pkgconfig').generate( + this_library, + filebase : meson.project_name(), + name : 'Nix', + description : 'Nix Package Manager', + subdirs : ['nix'], + extra_cflags : ['-std=c++2a'], + requires : deps_public, + requires_private : deps_private, +) + +meson.override_dependency(meson.project_name(), declare_dependency( + include_directories : include_dirs, + link_with : this_library, + compile_args : ['-std=c++2a'], + dependencies : [], +)) diff --git a/tests/unit/libutil-support/package.nix b/tests/unit/libutil-support/package.nix new file mode 100644 index 000000000..caa37b748 --- /dev/null +++ b/tests/unit/libutil-support/package.nix @@ -0,0 +1,98 @@ +{ lib +, stdenv +, releaseTools +, fileset + +, meson +, ninja +, pkg-config + +, nix-util + +, rapidcheck + +# Configuration Options + +, versionSuffix ? "" + +# Check test coverage of Nix. Probably want to use with at least +# one of `doCheck` or `doInstallCheck` enabled. +, withCoverageChecks ? false +}: + +let + version = lib.fileContents ./.version + versionSuffix; + + mkDerivation = + if withCoverageChecks + then + # TODO support `finalAttrs` args function in + # `releaseTools.coverageAnalysis`. + argsFun: + releaseTools.coverageAnalysis (let args = argsFun args; in args) + else stdenv.mkDerivation; +in + +mkDerivation (finalAttrs: { + pname = "nix-util-test-support"; + inherit version; + + src = fileset.toSource { + root = ./.; + fileset = fileset.unions [ + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + }; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + nix-util + rapidcheck + ] + ; + + propagatedBuildInputs = [ + nix-util + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix + '' + echo ${version} > .version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + enableParallelBuilding = true; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + # TODO Always true after https://github.com/NixOS/nixpkgs/issues/318564 + strictDeps = !withCoverageChecks; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +} // lib.optionalAttrs withCoverageChecks { + lcovFilter = [ "*/boost/*" "*-tab.*" ]; + + hardeningDisable = [ "fortify" ]; +}) diff --git a/tests/unit/libutil/.version b/tests/unit/libutil/.version new file mode 100644 index 000000000..ad2261920 --- /dev/null +++ b/tests/unit/libutil/.version @@ -0,0 +1 @@ +2.24.0 diff --git a/tests/unit/libutil/meson.build b/tests/unit/libutil/meson.build new file mode 100644 index 000000000..56147686b --- /dev/null +++ b/tests/unit/libutil/meson.build @@ -0,0 +1,142 @@ +project('nix-util-test', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +# See note in ../nix-util/meson.build +deps_private = [ ] + +# See note in ../nix-util/meson.build +deps_public = [ ] + +# See note in ../nix-util/meson.build +deps_other = [ ] + +configdata = configuration_data() + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util-test.h', + # '-include', 'config-store.h', + '-Wno-deprecated-declarations', + '-Wimplicit-fallthrough', + '-Werror=switch', + '-Werror=switch-enum', + '-Wdeprecated-copy', + '-Wignored-qualifiers', + # Enable assertions in libstdc++ by default. Harmless on libc++. Benchmarked + # at ~1% overhead in `nix search`. + # + # FIXME: remove when we get meson 1.4.0 which will default this to on for us: + # https://mesonbuild.com/Release-notes-for-1-4-0.html#ndebug-setting-now-controls-c-stdlib-assertions + '-D_GLIBCXX_ASSERTIONS=1', + language : 'cpp', +) + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +config_h = configure_file( + configuration : configdata, + output : 'config-util-test.h', +) + +sources = files( + 'args.cc', + 'canon-path.cc', + 'chunked-vector.cc', + 'closure.cc', + 'compression.cc', + 'config.cc', + 'file-content-address.cc', + 'git.cc', + 'hash.cc', + 'hilite.cc', + 'json-utils.cc', + 'logging.cc', + 'lru-cache.cc', + 'nix_api_util.cc', + 'pool.cc', + 'references.cc', + 'spawn.cc', + 'suggestions.cc', + 'tests.cc', + 'url.cc', + 'xml-writer.cc', +) + +include_dirs = [include_directories('.')] + +if host_machine.system() == 'cygwin' or host_machine.system() == 'windows' + # Windows DLLs are stricter about symbol visibility than Unix shared + # objects --- see https://gcc.gnu.org/wiki/Visibility for details. + # This is a temporary sledgehammer to export everything like on Unix, + # and not detail with this yet. + # + # TODO do not do this, and instead do fine-grained export annotations. + linker_export_flags = ['-Wl,--export-all-symbols'] +else + linker_export_flags = [] +endif + +nix_util = dependency('nix-util') +if nix_util.type_name() == 'internal' + # subproject sadly no good for pkg-config module + deps_other += nix_util +else + deps_public += nix_util +endif + +nix_util_c = dependency('nix-util-c') +if nix_util_c.type_name() == 'internal' + # subproject sadly no good for pkg-config module + deps_other += nix_util_c +else + deps_public += nix_util_c +endif + +nix_util_test_support = dependency('nix-util-test-support') +if nix_util_test_support.type_name() == 'internal' + # subproject sadly no good for pkg-config module + deps_other += nix_util_test_support +else + deps_public += nix_util_test_support +endif + +rapidcheck = dependency('rapidcheck') +deps_public += rapidcheck + +gtest = dependency('gtest', main : true) +deps_public += gtest + +this_exe = executable( + 'nix-util-test', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + # TODO: -lrapidcheck, see ../libutil-support/build.meson + link_args: linker_export_flags + ['-lrapidcheck'], + # get main from gtest + install : true, +) + +test('nix-util-test', this_exe, env : ['_NIX_TEST_UNIT_DATA=' + meson.current_source_dir() + '/data']) + +meson.override_dependency(meson.project_name(), declare_dependency( + include_directories : include_dirs, + link_with : this_exe, + compile_args : ['-std=c++2a'], + dependencies : [], +)) diff --git a/tests/unit/libutil/package.nix b/tests/unit/libutil/package.nix new file mode 100644 index 000000000..7a09e4aa5 --- /dev/null +++ b/tests/unit/libutil/package.nix @@ -0,0 +1,112 @@ +{ lib +, stdenv +, releaseTools +, fileset + +, meson +, ninja +, pkg-config + +, nix-util +, nix-util-c +, nix-util-test-support + +, rapidcheck +, gtest +, runCommand + +# Configuration Options + +, versionSuffix ? "" + +# Check test coverage of Nix. Probably want to use with at least +# one of `doCheck` or `doInstallCheck` enabled. +, withCoverageChecks ? false +}: + +let + version = lib.fileContents ./.version + versionSuffix; + + mkDerivation = + if withCoverageChecks + then + # TODO support `finalAttrs` args function in + # `releaseTools.coverageAnalysis`. + argsFun: + releaseTools.coverageAnalysis (let args = argsFun args; in args) + else stdenv.mkDerivation; +in + +mkDerivation (finalAttrs: { + pname = "nix-util-test"; + inherit version; + + src = fileset.toSource { + root = ./.; + fileset = fileset.unions [ + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + }; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + nix-util + nix-util-c + nix-util-test-support + rapidcheck + gtest + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix + '' + echo ${version} > .version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + enableParallelBuilding = true; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + # TODO Always true after https://github.com/NixOS/nixpkgs/issues/318564 + strictDeps = !withCoverageChecks; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + passthru = { + tests = { + run = runCommand "${finalAttrs.pname}-run" { + } '' + PATH="${lib.makeBinPath [ finalAttrs.finalPackage ]}:$PATH" + export _NIX_TEST_UNIT_DATA=${./data} + nix-util-test + touch $out + ''; + }; + }; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +} // lib.optionalAttrs withCoverageChecks { + lcovFilter = [ "*/boost/*" "*-tab.*" ]; + + hardeningDisable = [ "fortify" ]; +}) From 6a28566db663fe8b185142e0d9cfdb988f6b04a9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 25 Jun 2024 15:49:39 +0200 Subject: [PATCH 43/61] refact: concatMapAttrs -> flatMapAttrs This should be slightly easier to read. We could apply this to all concatMapAttrs calls. --- flake.nix | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/flake.nix b/flake.nix index aac45af34..1be9d2fc0 100644 --- a/flake.nix +++ b/flake.nix @@ -58,6 +58,18 @@ "stdenv" ]; + /** + `flatMapAttrs attrs f` applies `f` to each attribute in `attrs` and + merges the results into a single attribute set. + + This can be nested to form a build matrix where all the attributes + generated by the innermost `f` are returned as is. + (Provided that the names are unique.) + + See https://nixos.org/manual/nixpkgs/stable/index.html#function-library-lib.attrsets.concatMapAttrs + */ + flatMapAttrs = attrs: f: lib.concatMapAttrs f attrs; + forAllSystems = lib.genAttrs systems; forAllCrossSystems = lib.genAttrs crossSystems; @@ -319,17 +331,20 @@ # system, we should reenable this. #perlBindings = self.hydraJobs.perlBindings.${system}; } - // lib.concatMapAttrs (nixpkgsPrefix: nixpkgs: - lib.concatMapAttrs (pkgName: pkg: - lib.concatMapAttrs (testName: test: { - # Add "passthru" tests - "${nixpkgsPrefix}${pkgName}-${testName}" = test; - }) pkg.tests or {} - ) nixpkgs.nix-components - ) { + # Add "passthru" tests + // flatMapAttrs { "" = nixpkgsFor.${system}.native; "static-" = nixpkgsFor.${system}.static; } + (nixpkgsPrefix: nixpkgs: + flatMapAttrs nixpkgs.nix-components + (pkgName: pkg: + flatMapAttrs pkg.tests or {} + (testName: test: { + "${nixpkgsPrefix}${pkgName}-${testName}" = test; + }) + ) + ) // devFlake.checks.${system} or {} ); From 1eaddb209d240f424b8a40930b963c19a0883a57 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 25 Jun 2024 16:17:13 +0200 Subject: [PATCH 44/61] TMP: disable static meson build on darwin --- flake.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 1be9d2fc0..c499fea5b 100644 --- a/flake.nix +++ b/flake.nix @@ -334,7 +334,8 @@ # Add "passthru" tests // flatMapAttrs { "" = nixpkgsFor.${system}.native; - "static-" = nixpkgsFor.${system}.static; + # FIXME: Static build on darwin currently broken + # "static-" = nixpkgsFor.${system}.static; } (nixpkgsPrefix: nixpkgs: flatMapAttrs nixpkgs.nix-components From ae3304bde9730e7cc2f43b71540fabd20fd12c6f Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 25 Jun 2024 16:56:45 +0200 Subject: [PATCH 45/61] Test static build of nix-util on non-darwin --- flake.nix | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/flake.nix b/flake.nix index c499fea5b..35280f038 100644 --- a/flake.nix +++ b/flake.nix @@ -332,11 +332,13 @@ #perlBindings = self.hydraJobs.perlBindings.${system}; } # Add "passthru" tests - // flatMapAttrs { + // flatMapAttrs ({ "" = nixpkgsFor.${system}.native; - # FIXME: Static build on darwin currently broken - # "static-" = nixpkgsFor.${system}.static; - } + } // lib.optionalAttrs (! nixpkgsFor.${system}.native.stdenv.hostPlatform.isDarwin) { + # TODO: enable static builds for darwin, blocked on: + # https://github.com/NixOS/nixpkgs/issues/320448 + "static-" = nixpkgsFor.${system}.static; + }) (nixpkgsPrefix: nixpkgs: flatMapAttrs nixpkgs.nix-components (pkgName: pkg: From fba81cf74b2081b5242126fac01061e1fedaa99d Mon Sep 17 00:00:00 2001 From: Ryan Hendrickson Date: Tue, 25 Jun 2024 18:44:56 -0400 Subject: [PATCH 46/61] docs: internal documentation touchup Make two comments more accurate for the next reader. --- doc/manual/redirects.js | 2 +- doc/manual/src/_redirects | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index 633932e55..f7b479106 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -1,7 +1,7 @@ // redirect rules for URL fragments (client-side) to prevent link rot. // this must be done on the client side, as web servers do not see the fragment part of the URL. // it will only work with JavaScript enabled in the browser, but this is the best we can do here. -// see ./_redirects for path redirects (client-side) +// see src/_redirects for path redirects (server-side) // redirects are declared as follows: // each entry has as its key a path matching the requested URL path, relative to the mdBook document root. diff --git a/doc/manual/src/_redirects b/doc/manual/src/_redirects index a04a36f1e..7f8edca8e 100644 --- a/doc/manual/src/_redirects +++ b/doc/manual/src/_redirects @@ -1,5 +1,5 @@ # redirect rules for paths (server-side) to prevent link rot. -# see ./redirects.js for redirects based on URL fragments (client-side) +# see ../redirects.js for redirects based on URL fragments (client-side) # # concrete user story this supports: # - user finds URL to the manual for Nix x.y From fd0b376c795f770a82c4f7b9ff5ef3df0c03e125 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 1 Feb 2024 12:27:29 +0100 Subject: [PATCH 47/61] libstore/worker.cc: Remove outdated comment It was added above this conditional Worker::Worker(LocalStore & store) : store(store) { /* Debugging: prevent recursive workers. */ if (working) abort(); working = true; However, `working` has since been removed. Source: https://github.com/NixOS/nix/blame/7f8e805c8ef2d7728648553de6b762964730a09a/src/libstore/build.cc#L2617 --- src/libstore/build/worker.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 31cfa8adc..8a5d6de72 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -19,7 +19,6 @@ Worker::Worker(Store & store, Store & evalStore) , store(store) , evalStore(evalStore) { - /* Debugging: prevent recursive workers. */ nrLocalBuilds = 0; nrSubstitutions = 0; lastWokenUp = steady_time_point::min(); From 6fe8fb967a71e95f04e286ea5301d33f592788bf Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 1 Feb 2024 12:39:45 +0100 Subject: [PATCH 48/61] libstore/worker.hh: Document Worker --- src/libstore/build/worker.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 7d67030d7..33a7bf015 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -59,7 +59,7 @@ struct HookInstance; #endif /** - * The worker class. + * Coordinates one or more realisations and their interdependencies. */ class Worker { From e28e6b96bb671309218e5b9de56aeac22b2bf7be Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 26 Jun 2024 02:03:21 +0200 Subject: [PATCH 49/61] flake.nix: Tidy `packages` build matrix code flatMapAttrs is easier to read because it introduces the values before using them, kind of like a `let` bindings with multiple values. The repeated comments remind the reader of the purpose of the innermost attrsets, which is actually very simple. Knowing that they go right into the result should help a lot with building a mental model for this pattern. --- flake.nix | 62 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/flake.nix b/flake.nix index 35280f038..dfae6ce2d 100644 --- a/flake.nix +++ b/flake.nix @@ -351,36 +351,40 @@ // devFlake.checks.${system} or {} ); - packages = forAllSystems (system: { - inherit (nixpkgsFor.${system}.native) - changelog-d; - default = self.packages.${system}.nix; - nix-internal-api-docs = nixpkgsFor.${system}.native.nix-internal-api-docs; - nix-external-api-docs = nixpkgsFor.${system}.native.nix-external-api-docs; - } // lib.concatMapAttrs - # We need to flatten recursive attribute sets of derivations to pass `flake check`. - (pkgName: {}: { - "${pkgName}" = nixpkgsFor.${system}.native.${pkgName}; - "${pkgName}-static" = nixpkgsFor.${system}.static.${pkgName}; - } // lib.concatMapAttrs - (crossSystem: {}: { - "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.${pkgName}; - }) - (lib.genAttrs crossSystems (_: { })) - // lib.concatMapAttrs - (stdenvName: {}: { - "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".${pkgName}; - }) - (lib.genAttrs stdenvs (_: { }))) - { - "nix" = { }; - # Temporarily disabled because GitHub Actions OOM issues. Once - # the old build system is gone and we are back to one build - # system, we should reenable these. - #"nix-util" = { }; - #"nix-store" = { }; - #"nix-fetchers" = { }; + packages = forAllSystems (system: + { # Here we put attributes that map 1:1 into packages., ie + # for which we don't apply the full build matrix such as cross or static. + inherit (nixpkgsFor.${system}.native) + changelog-d; + default = self.packages.${system}.nix; + nix-internal-api-docs = nixpkgsFor.${system}.native.nix-internal-api-docs; + nix-external-api-docs = nixpkgsFor.${system}.native.nix-external-api-docs; } + # We need to flatten recursive attribute sets of derivations to pass `flake check`. + // flatMapAttrs + { # Components we'll iterate over in the upcoming lambda + "nix" = { }; + # Temporarily disabled because GitHub Actions OOM issues. Once + # the old build system is gone and we are back to one build + # system, we should reenable these. + #"nix-util" = { }; + #"nix-store" = { }; + #"nix-fetchers" = { }; + } + (pkgName: {}: { + # These attributes go right into `packages.`. + "${pkgName}" = nixpkgsFor.${system}.native.${pkgName}; + "${pkgName}-static" = nixpkgsFor.${system}.static.${pkgName}; + } + // flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: { + # These attributes go right into `packages.`. + "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.${pkgName}; + }) + // flatMapAttrs (lib.genAttrs stdenvs (_: { })) (stdenvName: {}: { + # These attributes go right into `packages.`. + "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".${pkgName}; + }) + ) // lib.optionalAttrs (builtins.elem system linux64BitSystems) { dockerImage = let From a14faa869d12d6280ac5ebf94c585ef200bb4c9c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 26 Jun 2024 02:38:07 +0200 Subject: [PATCH 50/61] flake.nix: Use a Nixpkgs scope for components --- flake.nix | 97 +++++----------------------------------- packaging/components.nix | 86 +++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 87 deletions(-) create mode 100644 packaging/components.nix diff --git a/flake.nix b/flake.nix index dfae6ce2d..30a8e4354 100644 --- a/flake.nix +++ b/flake.nix @@ -172,93 +172,16 @@ }; }); - # TODO: define everything here instead of top level? - nix-components = { - inherit (final) - nix-util - nix-util-test-support - nix-util-test - nix-util-c - nix-store - nix-fetchers - nix-perl-bindings - ; - }; + # A new scope, so that we can use `callPackage` to inject our own interdependencies + # without "polluting" the top level "`pkgs`" attrset. + # This also has the benefit of providing us with a distinct set of packages + # we can iterate over. + nixComponents = lib.makeScope final.newScope (import ./packaging/components.nix { + pkgs = final; + inherit stdenv versionSuffix officialRelease; + }); - nix-util = final.callPackage ./src/libutil/package.nix { - inherit - fileset - stdenv - officialRelease - versionSuffix - ; - }; - - nix-util-test-support = final.callPackage ./tests/unit/libutil-support/package.nix { - inherit - fileset - stdenv - versionSuffix - ; - }; - - nix-util-test = final.callPackage ./tests/unit/libutil/package.nix { - inherit - fileset - stdenv - versionSuffix - ; - }; - - nix-util-c = final.callPackage ./src/libutil-c/package.nix { - inherit - fileset - stdenv - versionSuffix - ; - }; - - nix-store = final.callPackage ./src/libstore/package.nix { - inherit - fileset - stdenv - officialRelease - versionSuffix - ; - libseccomp = final.libseccomp-nix; - busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell; - }; - - nix-fetchers = final.callPackage ./src/libfetchers/package.nix { - inherit - fileset - stdenv - officialRelease - versionSuffix - ; - }; - - nix = - final.callPackage ./package.nix { - inherit - fileset - stdenv - officialRelease - versionSuffix - ; - boehmgc = final.boehmgc-nix; - libgit2 = final.libgit2-nix; - libseccomp = final.libseccomp-nix; - busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell; - }; - - nix-perl-bindings = final.callPackage ./src/perl/package.nix { - inherit - fileset - stdenv - versionSuffix - ; - }; + nix = final.nixComponents.nix; nix-internal-api-docs = final.callPackage ./src/internal-api-docs/package.nix { inherit @@ -340,7 +263,7 @@ "static-" = nixpkgsFor.${system}.static; }) (nixpkgsPrefix: nixpkgs: - flatMapAttrs nixpkgs.nix-components + flatMapAttrs nixpkgs.nixComponents (pkgName: pkg: flatMapAttrs pkg.tests or {} (testName: test: { diff --git a/packaging/components.nix b/packaging/components.nix new file mode 100644 index 000000000..af26e05ce --- /dev/null +++ b/packaging/components.nix @@ -0,0 +1,86 @@ +{pkgs, stdenv, officialRelease, versionSuffix}: scope: +let + inherit (scope) callPackage; + + # TODO: push fileset parameter into package.nix files as `lib` parameter + inherit (callPackage (args@{ lib }: args) {}) lib; + inherit (lib) fileset; +in + +# This becomes the pkgs.nixComponents attribute set +{ + # TODO: build the nix CLI with meson + nix = pkgs.callPackage ../package.nix { + inherit + fileset + stdenv + officialRelease + versionSuffix + ; + boehmgc = pkgs.boehmgc-nix; + libgit2 = pkgs.libgit2-nix; + libseccomp = pkgs.libseccomp-nix; + busybox-sandbox-shell = pkgs.busybox-sandbox-shell or pkgs.default-busybox-sandbox-shell; + }; + + nix-util = callPackage ../src/libutil/package.nix { + inherit + fileset + stdenv + officialRelease + versionSuffix + ; + }; + + nix-util-test-support = callPackage ../tests/unit/libutil-support/package.nix { + inherit + fileset + stdenv + versionSuffix + ; + }; + + nix-util-test = callPackage ../tests/unit/libutil/package.nix { + inherit + fileset + stdenv + versionSuffix + ; + }; + + nix-util-c = callPackage ../src/libutil-c/package.nix { + inherit + fileset + stdenv + versionSuffix + ; + }; + + nix-store = callPackage ../src/libstore/package.nix { + inherit + fileset + stdenv + officialRelease + versionSuffix + ; + libseccomp = pkgs.libseccomp-nix; + busybox-sandbox-shell = pkgs.busybox-sandbox-shell or pkgs.default-busybox-sandbox-shell; + }; + + nix-fetchers = callPackage ../src/libfetchers/package.nix { + inherit + fileset + stdenv + officialRelease + versionSuffix + ; + }; + + nix-perl-bindings = callPackage ../src/perl/package.nix { + inherit + fileset + stdenv + versionSuffix + ; + }; +} From d40c59ed194b715bffdcd40f4b654ebc60cf5be3 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 26 Jun 2024 02:44:21 +0200 Subject: [PATCH 51/61] flake.nix: Use the nixComponents scope instead of bare pkgs packages ... which aren't around anymore. --- flake.nix | 24 ++++++++++++------------ maintainers/hydra.nix | 11 +++++++---- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/flake.nix b/flake.nix index 30a8e4354..a66603cc8 100644 --- a/flake.nix +++ b/flake.nix @@ -296,16 +296,16 @@ } (pkgName: {}: { # These attributes go right into `packages.`. - "${pkgName}" = nixpkgsFor.${system}.native.${pkgName}; - "${pkgName}-static" = nixpkgsFor.${system}.static.${pkgName}; + "${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName}; + "${pkgName}-static" = nixpkgsFor.${system}.static.nixComponents.${pkgName}; } // flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: { # These attributes go right into `packages.`. - "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.${pkgName}; + "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}; }) // flatMapAttrs (lib.genAttrs stdenvs (_: { })) (stdenvName: {}: { # These attributes go right into `packages.`. - "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".${pkgName}; + "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nixComponents.${pkgName}; }) ) // lib.optionalAttrs (builtins.elem system linux64BitSystems) { @@ -367,17 +367,17 @@ }; mesonFlags = - map (transformFlag "libutil") pkgs.nix-util.mesonFlags - ++ map (transformFlag "libstore") pkgs.nix-store.mesonFlags - ++ map (transformFlag "libfetchers") pkgs.nix-fetchers.mesonFlags - ++ lib.optionals havePerl (map (transformFlag "perl") pkgs.nix-perl-bindings.mesonFlags) + map (transformFlag "libutil") pkgs.nixComponents.nix-util.mesonFlags + ++ map (transformFlag "libstore") pkgs.nixComponents.nix-store.mesonFlags + ++ map (transformFlag "libfetchers") pkgs.nixComponents.nix-fetchers.mesonFlags + ++ lib.optionals havePerl (map (transformFlag "perl") pkgs.nixComponents.nix-perl-bindings.mesonFlags) ; nativeBuildInputs = attrs.nativeBuildInputs or [] - ++ pkgs.nix-util.nativeBuildInputs - ++ pkgs.nix-store.nativeBuildInputs - ++ pkgs.nix-fetchers.nativeBuildInputs - ++ lib.optionals havePerl pkgs.nix-perl-bindings.nativeBuildInputs + ++ pkgs.nixComponents.nix-util.nativeBuildInputs + ++ pkgs.nixComponents.nix-store.nativeBuildInputs + ++ pkgs.nixComponents.nix-fetchers.nativeBuildInputs + ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs ++ pkgs.nix-internal-api-docs.nativeBuildInputs ++ pkgs.nix-external-api-docs.nativeBuildInputs ++ [ diff --git a/maintainers/hydra.nix b/maintainers/hydra.nix index 9e6f2c468..b03cbdfa0 100644 --- a/maintainers/hydra.nix +++ b/maintainers/hydra.nix @@ -33,6 +33,9 @@ let doBuild = false; }; + # Technically we could just return `pkgs.nixComponents`, but for Hydra it's + # convention to transpose it, and to transpose it efficiently, we need to + # enumerate them manually, so that we don't evaluate unnecessary package sets. forAllPackages = lib.genAttrs [ "nix" "nix-util" @@ -46,16 +49,16 @@ in { # Binary package for various platforms. build = forAllPackages (pkgName: - forAllSystems (system: nixpkgsFor.${system}.native.${pkgName})); + forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.${pkgName})); shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation); buildStatic = forAllPackages (pkgName: - lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.${pkgName})); + lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.nixComponents.${pkgName})); buildCross = forAllPackages (pkgName: forAllCrossSystems (crossSystem: - lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.${pkgName}))); + lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}))); buildNoGc = forAllSystems (system: self.packages.${system}.nix.override { enableGC = false; } @@ -73,7 +76,7 @@ in ); # Perl bindings for various platforms. - perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix-perl-bindings); + perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-perl-bindings); # Binary tarball for various platforms, containing a Nix store # with the closure of 'nix' package, and the second half of From 85de5a60c77c256d5e57943becb683f50f9d7537 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 26 Jun 2024 03:18:01 +0200 Subject: [PATCH 52/61] Use lib instead of explicit fileset passing --- flake.nix | 2 -- maintainers/hydra.nix | 3 --- package.nix | 6 ++---- packaging/components.nix | 16 ---------------- src/external-api-docs/package.nix | 6 ++++-- src/internal-api-docs/package.nix | 6 ++++-- src/libfetchers/package.nix | 6 ++---- src/libstore/package.nix | 5 ++--- src/libutil-c/package.nix | 3 ++- src/libutil/package.nix | 4 ++-- src/perl/package.nix | 5 ++++- tests/unit/libutil-support/package.nix | 3 ++- tests/unit/libutil/package.nix | 3 ++- 13 files changed, 26 insertions(+), 42 deletions(-) diff --git a/flake.nix b/flake.nix index a66603cc8..7fc491035 100644 --- a/flake.nix +++ b/flake.nix @@ -185,7 +185,6 @@ nix-internal-api-docs = final.callPackage ./src/internal-api-docs/package.nix { inherit - fileset stdenv versionSuffix ; @@ -193,7 +192,6 @@ nix-external-api-docs = final.callPackage ./src/external-api-docs/package.nix { inherit - fileset stdenv versionSuffix ; diff --git a/maintainers/hydra.nix b/maintainers/hydra.nix index b03cbdfa0..98d8b3961 100644 --- a/maintainers/hydra.nix +++ b/maintainers/hydra.nix @@ -9,7 +9,6 @@ }: let inherit (inputs) nixpkgs nixpkgs-regression; - inherit (lib) fileset; installScriptFor = tarballs: nixpkgsFor.x86_64-linux.native.callPackage ../scripts/installer.nix { @@ -25,8 +24,6 @@ let lib.versionAtLeast client.version "2.4pre20211005") "-${client.version}-against-${daemon.version}"; - inherit fileset; - test-client = client; test-daemon = daemon; diff --git a/package.nix b/package.nix index f414b9a73..9a6fc272a 100644 --- a/package.nix +++ b/package.nix @@ -1,12 +1,10 @@ { lib -, fetchurl , stdenv , releaseTools , autoconf-archive , autoreconfHook , aws-sdk-cpp , boehmgc -, buildPackages , nlohmann_json , bison , boost @@ -15,7 +13,6 @@ , curl , editline , readline -, fileset , flex , git , gtest @@ -50,7 +47,6 @@ , pname ? "nix" , versionSuffix ? "" -, officialRelease ? false # Whether to build Nix. Useful to skip for tasks like testing existing pre-built versions of Nix , doBuild ? true @@ -113,6 +109,8 @@ }: let + inherit (lib) fileset; + version = lib.fileContents ./.version + versionSuffix; # selected attributes with defaults, will be used to define some diff --git a/packaging/components.nix b/packaging/components.nix index af26e05ce..d3809ff38 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -1,10 +1,6 @@ {pkgs, stdenv, officialRelease, versionSuffix}: scope: let inherit (scope) callPackage; - - # TODO: push fileset parameter into package.nix files as `lib` parameter - inherit (callPackage (args@{ lib }: args) {}) lib; - inherit (lib) fileset; in # This becomes the pkgs.nixComponents attribute set @@ -12,9 +8,7 @@ in # TODO: build the nix CLI with meson nix = pkgs.callPackage ../package.nix { inherit - fileset stdenv - officialRelease versionSuffix ; boehmgc = pkgs.boehmgc-nix; @@ -25,16 +19,13 @@ in nix-util = callPackage ../src/libutil/package.nix { inherit - fileset stdenv - officialRelease versionSuffix ; }; nix-util-test-support = callPackage ../tests/unit/libutil-support/package.nix { inherit - fileset stdenv versionSuffix ; @@ -42,7 +33,6 @@ in nix-util-test = callPackage ../tests/unit/libutil/package.nix { inherit - fileset stdenv versionSuffix ; @@ -50,7 +40,6 @@ in nix-util-c = callPackage ../src/libutil-c/package.nix { inherit - fileset stdenv versionSuffix ; @@ -58,9 +47,7 @@ in nix-store = callPackage ../src/libstore/package.nix { inherit - fileset stdenv - officialRelease versionSuffix ; libseccomp = pkgs.libseccomp-nix; @@ -69,16 +56,13 @@ in nix-fetchers = callPackage ../src/libfetchers/package.nix { inherit - fileset stdenv - officialRelease versionSuffix ; }; nix-perl-bindings = callPackage ../src/perl/package.nix { inherit - fileset stdenv versionSuffix ; diff --git a/src/external-api-docs/package.nix b/src/external-api-docs/package.nix index aa5cd49eb..352698360 100644 --- a/src/external-api-docs/package.nix +++ b/src/external-api-docs/package.nix @@ -1,7 +1,5 @@ { lib , stdenv -, releaseTools -, fileset , meson , ninja @@ -12,6 +10,10 @@ , versionSuffix ? "" }: +let + inherit (lib) fileset; +in + stdenv.mkDerivation (finalAttrs: { pname = "nix-external-api-docs"; version = lib.fileContents ./.version + versionSuffix; diff --git a/src/internal-api-docs/package.nix b/src/internal-api-docs/package.nix index b5f1b0da1..fa54d55f3 100644 --- a/src/internal-api-docs/package.nix +++ b/src/internal-api-docs/package.nix @@ -1,7 +1,5 @@ { lib , stdenv -, releaseTools -, fileset , meson , ninja @@ -12,6 +10,10 @@ , versionSuffix ? "" }: +let + inherit (lib) fileset; +in + stdenv.mkDerivation (finalAttrs: { pname = "nix-internal-api-docs"; version = lib.fileContents ./.version + versionSuffix; diff --git a/src/libfetchers/package.nix b/src/libfetchers/package.nix index 800256030..a5583d14c 100644 --- a/src/libfetchers/package.nix +++ b/src/libfetchers/package.nix @@ -1,7 +1,6 @@ { lib , stdenv , releaseTools -, fileset , meson , ninja @@ -16,17 +15,16 @@ # Configuration Options , versionSuffix ? "" -, officialRelease ? false # Check test coverage of Nix. Probably want to use with with at least # one of `doCheck` or `doInstallCheck` enabled. , withCoverageChecks ? false -# Avoid setting things that would interfere with a functioning devShell -, forDevShell ? false }: let + inherit (lib) fileset; + version = lib.fileContents ./.version + versionSuffix; mkDerivation = diff --git a/src/libstore/package.nix b/src/libstore/package.nix index e54dfe597..e118f3cd2 100644 --- a/src/libstore/package.nix +++ b/src/libstore/package.nix @@ -1,7 +1,6 @@ { lib , stdenv , releaseTools -, fileset , meson , ninja @@ -13,7 +12,6 @@ , aws-sdk-cpp , libseccomp , nlohmann_json -, man , sqlite , busybox-sandbox-shell ? null @@ -21,7 +19,6 @@ # Configuration Options , versionSuffix ? "" -, officialRelease ? false # Check test coverage of Nix. Probably want to use with at least # one of `doCheck` or `doInstallCheck` enabled. @@ -32,6 +29,8 @@ }: let + inherit (lib) fileset; + version = lib.fileContents ./.version + versionSuffix; mkDerivation = diff --git a/src/libutil-c/package.nix b/src/libutil-c/package.nix index a45b36d4c..05a26c17e 100644 --- a/src/libutil-c/package.nix +++ b/src/libutil-c/package.nix @@ -1,7 +1,6 @@ { lib , stdenv , releaseTools -, fileset , meson , ninja @@ -19,6 +18,8 @@ }: let + inherit (lib) fileset; + version = lib.fileContents ./.version + versionSuffix; mkDerivation = diff --git a/src/libutil/package.nix b/src/libutil/package.nix index b36e3879c..892951cdf 100644 --- a/src/libutil/package.nix +++ b/src/libutil/package.nix @@ -1,7 +1,6 @@ { lib , stdenv , releaseTools -, fileset , meson , ninja @@ -18,7 +17,6 @@ # Configuration Options , versionSuffix ? "" -, officialRelease ? false # Check test coverage of Nix. Probably want to use with at least # one of `doCheck` or `doInstallCheck` enabled. @@ -26,6 +24,8 @@ }: let + inherit (lib) fileset; + version = lib.fileContents ./.version + versionSuffix; mkDerivation = diff --git a/src/perl/package.nix b/src/perl/package.nix index a31b1b66c..85f1547b7 100644 --- a/src/perl/package.nix +++ b/src/perl/package.nix @@ -1,5 +1,4 @@ { lib -, fileset , stdenv , perl , perlPackages @@ -16,6 +15,10 @@ , versionSuffix ? "" }: +let + inherit (lib) fileset; +in + perl.pkgs.toPerlModule (stdenv.mkDerivation (finalAttrs: { pname = "nix-perl"; version = lib.fileContents ./.version + versionSuffix; diff --git a/tests/unit/libutil-support/package.nix b/tests/unit/libutil-support/package.nix index caa37b748..0be0a9945 100644 --- a/tests/unit/libutil-support/package.nix +++ b/tests/unit/libutil-support/package.nix @@ -1,7 +1,6 @@ { lib , stdenv , releaseTools -, fileset , meson , ninja @@ -21,6 +20,8 @@ }: let + inherit (lib) fileset; + version = lib.fileContents ./.version + versionSuffix; mkDerivation = diff --git a/tests/unit/libutil/package.nix b/tests/unit/libutil/package.nix index 7a09e4aa5..391f8d853 100644 --- a/tests/unit/libutil/package.nix +++ b/tests/unit/libutil/package.nix @@ -1,7 +1,6 @@ { lib , stdenv , releaseTools -, fileset , meson , ninja @@ -25,6 +24,8 @@ }: let + inherit (lib) fileset; + version = lib.fileContents ./.version + versionSuffix; mkDerivation = From 74b9b77c9f05099c2bb207890d0728455a85efa7 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 26 Jun 2024 03:33:30 +0200 Subject: [PATCH 53/61] components.nix: Simplify with scope --- flake.nix | 2 +- packaging/components.nix | 71 +++++++++------------------------------- 2 files changed, 16 insertions(+), 57 deletions(-) diff --git a/flake.nix b/flake.nix index 7fc491035..ed330b669 100644 --- a/flake.nix +++ b/flake.nix @@ -178,7 +178,7 @@ # we can iterate over. nixComponents = lib.makeScope final.newScope (import ./packaging/components.nix { pkgs = final; - inherit stdenv versionSuffix officialRelease; + inherit stdenv versionSuffix; }); nix = final.nixComponents.nix; diff --git a/packaging/components.nix b/packaging/components.nix index d3809ff38..7cef0e356 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -1,70 +1,29 @@ -{pkgs, stdenv, officialRelease, versionSuffix}: scope: +{pkgs, stdenv, versionSuffix}: scope: let inherit (scope) callPackage; in # This becomes the pkgs.nixComponents attribute set { - # TODO: build the nix CLI with meson - nix = pkgs.callPackage ../package.nix { - inherit - stdenv - versionSuffix - ; - boehmgc = pkgs.boehmgc-nix; - libgit2 = pkgs.libgit2-nix; - libseccomp = pkgs.libseccomp-nix; - busybox-sandbox-shell = pkgs.busybox-sandbox-shell or pkgs.default-busybox-sandbox-shell; - }; + inherit stdenv versionSuffix; + libseccomp = pkgs.libseccomp-nix; + boehmgc = pkgs.boehmgc-nix; + libgit2 = pkgs.libgit2-nix; + busybox-sandbox-shell = pkgs.busybox-sandbox-shell or pkgs.default-busybox-sandbox-shell; - nix-util = callPackage ../src/libutil/package.nix { - inherit - stdenv - versionSuffix - ; - }; + nix = callPackage ../package.nix { }; - nix-util-test-support = callPackage ../tests/unit/libutil-support/package.nix { - inherit - stdenv - versionSuffix - ; - }; + nix-util = callPackage ../src/libutil/package.nix { }; - nix-util-test = callPackage ../tests/unit/libutil/package.nix { - inherit - stdenv - versionSuffix - ; - }; + nix-util-test-support = callPackage ../tests/unit/libutil-support/package.nix { }; - nix-util-c = callPackage ../src/libutil-c/package.nix { - inherit - stdenv - versionSuffix - ; - }; + nix-util-test = callPackage ../tests/unit/libutil/package.nix { }; - nix-store = callPackage ../src/libstore/package.nix { - inherit - stdenv - versionSuffix - ; - libseccomp = pkgs.libseccomp-nix; - busybox-sandbox-shell = pkgs.busybox-sandbox-shell or pkgs.default-busybox-sandbox-shell; - }; + nix-util-c = callPackage ../src/libutil-c/package.nix { }; - nix-fetchers = callPackage ../src/libfetchers/package.nix { - inherit - stdenv - versionSuffix - ; - }; + nix-store = callPackage ../src/libstore/package.nix { }; - nix-perl-bindings = callPackage ../src/perl/package.nix { - inherit - stdenv - versionSuffix - ; - }; + nix-fetchers = callPackage ../src/libfetchers/package.nix { }; + + nix-perl-bindings = callPackage ../src/perl/package.nix { }; } From ebf77c79ae1cd1516117311c6315bd8c56b8b837 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 26 Jun 2024 03:36:03 +0200 Subject: [PATCH 54/61] flake.nix: Use Nixpkgs convention for package variants --- flake.nix | 6 +++--- packaging/components.nix | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.nix b/flake.nix index ed330b669..d49bccda8 100644 --- a/flake.nix +++ b/flake.nix @@ -153,18 +153,18 @@ ''; }; - libgit2-nix = final.libgit2.overrideAttrs (attrs: { + libgit2_nix = final.libgit2.overrideAttrs (attrs: { src = libgit2; version = libgit2.lastModifiedDate; cmakeFlags = attrs.cmakeFlags or [] ++ [ "-DUSE_SSH=exec" ]; }); - boehmgc-nix = final.boehmgc.override { + boehmgc_nix = final.boehmgc.override { enableLargeConfig = true; }; - libseccomp-nix = final.libseccomp.overrideAttrs (_: rec { + libseccomp_nix = final.libseccomp.overrideAttrs (_: rec { version = "2.5.5"; src = final.fetchurl { url = "https://github.com/seccomp/libseccomp/releases/download/v${version}/libseccomp-${version}.tar.gz"; diff --git a/packaging/components.nix b/packaging/components.nix index 7cef0e356..2d06cfec4 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -6,9 +6,9 @@ in # This becomes the pkgs.nixComponents attribute set { inherit stdenv versionSuffix; - libseccomp = pkgs.libseccomp-nix; - boehmgc = pkgs.boehmgc-nix; - libgit2 = pkgs.libgit2-nix; + libseccomp = pkgs.libseccomp_nix; + boehmgc = pkgs.boehmgc_nix; + libgit2 = pkgs.libgit2_nix; busybox-sandbox-shell = pkgs.busybox-sandbox-shell or pkgs.default-busybox-sandbox-shell; nix = callPackage ../package.nix { }; From 25dc12aab139b4bd3438c764a1d479ae8517a81a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 26 Jun 2024 03:58:47 +0200 Subject: [PATCH 55/61] components.nix: Extract dependency scope This avoids polluting nixComponents with things that aren't our components. Fixes the extraction of passthru tests, which failed for boehmgc which had many irrelevant ones anyway. --- flake.nix | 12 +++++++++++- packaging/components.nix | 6 ------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/flake.nix b/flake.nix index d49bccda8..870b82950 100644 --- a/flake.nix +++ b/flake.nix @@ -176,11 +176,21 @@ # without "polluting" the top level "`pkgs`" attrset. # This also has the benefit of providing us with a distinct set of packages # we can iterate over. - nixComponents = lib.makeScope final.newScope (import ./packaging/components.nix { + nixComponents = lib.makeScope final.nixDependencies.newScope (import ./packaging/components.nix { pkgs = final; inherit stdenv versionSuffix; }); + # The dependencies are in their own scope, so that they don't have to be + # in Nixpkgs top level `pkgs` or `nixComponents`. + nixDependencies = lib.makeScope final.newScope (scope: { + inherit stdenv versionSuffix; + libseccomp = final.libseccomp_nix; + boehmgc = final.boehmgc_nix; + libgit2 = final.libgit2_nix; + busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell; + }); + nix = final.nixComponents.nix; nix-internal-api-docs = final.callPackage ./src/internal-api-docs/package.nix { diff --git a/packaging/components.nix b/packaging/components.nix index 2d06cfec4..e1e73d4c0 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -5,12 +5,6 @@ in # This becomes the pkgs.nixComponents attribute set { - inherit stdenv versionSuffix; - libseccomp = pkgs.libseccomp_nix; - boehmgc = pkgs.boehmgc_nix; - libgit2 = pkgs.libgit2_nix; - busybox-sandbox-shell = pkgs.busybox-sandbox-shell or pkgs.default-busybox-sandbox-shell; - nix = callPackage ../package.nix { }; nix-util = callPackage ../src/libutil/package.nix { }; From c24dbf14571cd59ef4e7b2a3470c61760a6a6aa4 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 26 Jun 2024 04:17:30 +0200 Subject: [PATCH 56/61] components.nix: Simplify --- flake.nix | 5 +---- packaging/components.nix | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/flake.nix b/flake.nix index 870b82950..90a48a382 100644 --- a/flake.nix +++ b/flake.nix @@ -176,10 +176,7 @@ # without "polluting" the top level "`pkgs`" attrset. # This also has the benefit of providing us with a distinct set of packages # we can iterate over. - nixComponents = lib.makeScope final.nixDependencies.newScope (import ./packaging/components.nix { - pkgs = final; - inherit stdenv versionSuffix; - }); + nixComponents = lib.makeScope final.nixDependencies.newScope (import ./packaging/components.nix); # The dependencies are in their own scope, so that they don't have to be # in Nixpkgs top level `pkgs` or `nixComponents`. diff --git a/packaging/components.nix b/packaging/components.nix index e1e73d4c0..f96ac6b51 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -1,4 +1,4 @@ -{pkgs, stdenv, versionSuffix}: scope: +scope: let inherit (scope) callPackage; in From 65802da98db906e61a810e686e140536d3ab4e54 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 26 Jun 2024 04:24:50 +0200 Subject: [PATCH 57/61] Move maintainers/hydra.nix -> packaging/hydra.nix --- flake.nix | 2 +- {maintainers => packaging}/hydra.nix | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename {maintainers => packaging}/hydra.nix (100%) diff --git a/flake.nix b/flake.nix index 90a48a382..510df73f1 100644 --- a/flake.nix +++ b/flake.nix @@ -224,7 +224,7 @@ # 'nix-perl-bindings' packages. overlays.default = overlayFor (p: p.stdenv); - hydraJobs = import ./maintainers/hydra.nix { + hydraJobs = import ./packaging/hydra.nix { inherit inputs binaryTarball diff --git a/maintainers/hydra.nix b/packaging/hydra.nix similarity index 100% rename from maintainers/hydra.nix rename to packaging/hydra.nix From 409eded5415f00c2cc7ef6e124a51c1bb0290df1 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 26 Jun 2024 10:41:56 +0200 Subject: [PATCH 58/61] flake.nix: Move dependencies scope to packaging/dependencies.nix --- flake.nix | 52 ++-------------------------------- packaging/dependencies.nix | 58 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 49 deletions(-) create mode 100644 packaging/dependencies.nix diff --git a/flake.nix b/flake.nix index 510df73f1..8b4deb78a 100644 --- a/flake.nix +++ b/flake.nix @@ -129,49 +129,6 @@ { nixStable = prev.nix; - default-busybox-sandbox-shell = final.busybox.override { - useMusl = true; - enableStatic = true; - enableMinimal = true; - extraConfig = '' - CONFIG_FEATURE_FANCY_ECHO y - CONFIG_FEATURE_SH_MATH y - CONFIG_FEATURE_SH_MATH_64 y - - CONFIG_ASH y - CONFIG_ASH_OPTIMIZE_FOR_SIZE y - - CONFIG_ASH_ALIAS y - CONFIG_ASH_BASH_COMPAT y - CONFIG_ASH_CMDCMD y - CONFIG_ASH_ECHO y - CONFIG_ASH_GETOPTS y - CONFIG_ASH_INTERNAL_GLOB y - CONFIG_ASH_JOB_CONTROL y - CONFIG_ASH_PRINTF y - CONFIG_ASH_TEST y - ''; - }; - - libgit2_nix = final.libgit2.overrideAttrs (attrs: { - src = libgit2; - version = libgit2.lastModifiedDate; - cmakeFlags = attrs.cmakeFlags or [] - ++ [ "-DUSE_SSH=exec" ]; - }); - - boehmgc_nix = final.boehmgc.override { - enableLargeConfig = true; - }; - - libseccomp_nix = final.libseccomp.overrideAttrs (_: rec { - version = "2.5.5"; - src = final.fetchurl { - url = "https://github.com/seccomp/libseccomp/releases/download/v${version}/libseccomp-${version}.tar.gz"; - hash = "sha256-JIosik2bmFiqa69ScSw0r+/PnJ6Ut23OAsHJqiX7M3U="; - }; - }); - # A new scope, so that we can use `callPackage` to inject our own interdependencies # without "polluting" the top level "`pkgs`" attrset. # This also has the benefit of providing us with a distinct set of packages @@ -180,12 +137,9 @@ # The dependencies are in their own scope, so that they don't have to be # in Nixpkgs top level `pkgs` or `nixComponents`. - nixDependencies = lib.makeScope final.newScope (scope: { - inherit stdenv versionSuffix; - libseccomp = final.libseccomp_nix; - boehmgc = final.boehmgc_nix; - libgit2 = final.libgit2_nix; - busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell; + nixDependencies = lib.makeScope final.newScope (import ./packaging/dependencies.nix { + inherit inputs stdenv versionSuffix; + pkgs = final; }); nix = final.nixComponents.nix; diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix new file mode 100644 index 000000000..88273df22 --- /dev/null +++ b/packaging/dependencies.nix @@ -0,0 +1,58 @@ +# These overrides are applied to the dependencies of the Nix components. + +{ + # Flake inputs; used for sources + inputs, + + # The raw Nixpkgs, not affected by this scope + pkgs, + + stdenv, + versionSuffix, +}: +scope: { + inherit stdenv versionSuffix; + + libseccomp = pkgs.libseccomp.overrideAttrs (_: rec { + version = "2.5.5"; + src = pkgs.fetchurl { + url = "https://github.com/seccomp/libseccomp/releases/download/v${version}/libseccomp-${version}.tar.gz"; + hash = "sha256-JIosik2bmFiqa69ScSw0r+/PnJ6Ut23OAsHJqiX7M3U="; + }; + }); + + boehmgc = pkgs.boehmgc.override { + enableLargeConfig = true; + }; + + libgit2 = pkgs.libgit2.overrideAttrs (attrs: { + src = inputs.libgit2; + version = inputs.libgit2.lastModifiedDate; + cmakeFlags = attrs.cmakeFlags or [] + ++ [ "-DUSE_SSH=exec" ]; + }); + + busybox-sandbox-shell = pkgs.busybox-sandbox-shell or (pkgs.busybox.override { + useMusl = true; + enableStatic = true; + enableMinimal = true; + extraConfig = '' + CONFIG_FEATURE_FANCY_ECHO y + CONFIG_FEATURE_SH_MATH y + CONFIG_FEATURE_SH_MATH_64 y + + CONFIG_ASH y + CONFIG_ASH_OPTIMIZE_FOR_SIZE y + + CONFIG_ASH_ALIAS y + CONFIG_ASH_BASH_COMPAT y + CONFIG_ASH_CMDCMD y + CONFIG_ASH_ECHO y + CONFIG_ASH_GETOPTS y + CONFIG_ASH_INTERNAL_GLOB y + CONFIG_ASH_JOB_CONTROL y + CONFIG_ASH_PRINTF y + CONFIG_ASH_TEST y + ''; + }); +} From 985c211061be71317eb2a9fc7a4d35278f5d8e41 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 26 Jun 2024 10:47:13 +0200 Subject: [PATCH 59/61] flake.nix: Move {in,ex}ternal-api-docs into nixComponents scope --- flake.nix | 22 ++++------------------ packaging/components.nix | 5 +++++ packaging/hydra.nix | 4 ++-- 3 files changed, 11 insertions(+), 20 deletions(-) diff --git a/flake.nix b/flake.nix index 8b4deb78a..582a946c4 100644 --- a/flake.nix +++ b/flake.nix @@ -144,20 +144,6 @@ nix = final.nixComponents.nix; - nix-internal-api-docs = final.callPackage ./src/internal-api-docs/package.nix { - inherit - stdenv - versionSuffix - ; - }; - - nix-external-api-docs = final.callPackage ./src/external-api-docs/package.nix { - inherit - stdenv - versionSuffix - ; - }; - nix_noTests = final.nix.override { doCheck = false; doInstallCheck = false; @@ -239,8 +225,8 @@ inherit (nixpkgsFor.${system}.native) changelog-d; default = self.packages.${system}.nix; - nix-internal-api-docs = nixpkgsFor.${system}.native.nix-internal-api-docs; - nix-external-api-docs = nixpkgsFor.${system}.native.nix-external-api-docs; + nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-internal-api-docs; + nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-external-api-docs; } # We need to flatten recursive attribute sets of derivations to pass `flake check`. // flatMapAttrs @@ -337,8 +323,8 @@ ++ pkgs.nixComponents.nix-store.nativeBuildInputs ++ pkgs.nixComponents.nix-fetchers.nativeBuildInputs ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs - ++ pkgs.nix-internal-api-docs.nativeBuildInputs - ++ pkgs.nix-external-api-docs.nativeBuildInputs + ++ pkgs.nixComponents.nix-internal-api-docs.nativeBuildInputs + ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs ++ [ modular.pre-commit.settings.package (pkgs.writeScriptBin "pre-commit-hooks-install" diff --git a/packaging/components.nix b/packaging/components.nix index f96ac6b51..b5e47969e 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -20,4 +20,9 @@ in nix-fetchers = callPackage ../src/libfetchers/package.nix { }; nix-perl-bindings = callPackage ../src/perl/package.nix { }; + + nix-internal-api-docs = callPackage ../src/internal-api-docs/package.nix { }; + + nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { }; + } diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 98d8b3961..5715abd8e 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -122,10 +122,10 @@ in }; # API docs for Nix's unstable internal C++ interfaces. - internal-api-docs = nixpkgsFor.x86_64-linux.native.nix-internal-api-docs; + internal-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-internal-api-docs; # API docs for Nix's C bindings. - external-api-docs = nixpkgsFor.x86_64-linux.native.nix-external-api-docs; + external-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-external-api-docs; # System tests. tests = import ../tests/nixos { inherit lib nixpkgs nixpkgsFor self; } // { From e6442891611b2f2ceee7ab1c573eec918a5fe5d7 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 26 Jun 2024 10:54:17 +0200 Subject: [PATCH 60/61] Remove unused boehmgc patch --- dep-patches/boehmgc-traceable_allocator-public.diff | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 dep-patches/boehmgc-traceable_allocator-public.diff diff --git a/dep-patches/boehmgc-traceable_allocator-public.diff b/dep-patches/boehmgc-traceable_allocator-public.diff deleted file mode 100644 index 903c707a6..000000000 --- a/dep-patches/boehmgc-traceable_allocator-public.diff +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/include/gc_allocator.h b/include/gc_allocator.h -index 597c7f13..587286be 100644 ---- a/include/gc_allocator.h -+++ b/include/gc_allocator.h -@@ -312,6 +312,7 @@ public: - - template<> - class traceable_allocator { -+public: - typedef size_t size_type; - typedef ptrdiff_t difference_type; - typedef void* pointer; From 9f8e387c3fd78e46c5656a503eafea4757ae2616 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 26 Jun 2024 11:02:45 +0200 Subject: [PATCH 61/61] ci.yml: Build meson on darwin We're building a bit of Darwin meson indirectly through `checks`, but it'd be annoying to encounter broken un-`check`-ed stuff during the porting process, so let's just do the right thing now. --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c9e4c25ea..103ce4ff4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -195,7 +195,11 @@ jobs: - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes .#hydraJobs.tests.functional_user meson_build: - runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: DeterminateSystems/nix-installer-action@main