diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 392ed30c6..4fe86d5ec 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -2,7 +2,7 @@ name: Feature request about: Suggest an idea for this project title: '' -labels: improvement +labels: feature assignees: '' --- diff --git a/.github/ISSUE_TEMPLATE/missing_documentation.md b/.github/ISSUE_TEMPLATE/missing_documentation.md new file mode 100644 index 000000000..fbabd868e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/missing_documentation.md @@ -0,0 +1,28 @@ +--- +name: Missing or incorrect documentation +about: Help us improve the reference manual +title: '' +labels: documentation +assignees: '' + +--- + +## Problem + + + +## Checklist + + + +- [ ] checked [latest Nix manual] \([source]) +- [ ] checked [open documentation issues and pull requests] for possible duplicates + +[latest Nix manual]: https://nixos.org/manual/nix/unstable/ +[source]: https://github.com/NixOS/nix/tree/master/doc/manual/src +[open documentation issues and pull requests]: https://github.com/NixOS/nix/labels/documentation + +## Proposal + + + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 86b5dfd2e..628d1d192 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,7 +81,14 @@ jobs: with: install_url: '${{needs.installer.outputs.installerURL}}' install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" - - run: nix-instantiate -E 'builtins.currentTime' --eval + - run: sudo apt install fish zsh + if: matrix.os == 'ubuntu-latest' + - run: brew install fish + if: matrix.os == 'macos-latest' + - run: exec bash -c "nix-instantiate -E 'builtins.currentTime' --eval" + - run: exec sh -c "nix-instantiate -E 'builtins.currentTime' --eval" + - run: exec zsh -c "nix-instantiate -E 'builtins.currentTime' --eval" + - run: exec fish -c "nix-instantiate -E 'builtins.currentTime' --eval" docker_push_image: needs: [check_secrets, tests] diff --git a/.gitignore b/.gitignore index 0c1b89ace..8e0db013f 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,8 @@ perl/Makefile.config # /scripts/ /scripts/nix-profile.sh /scripts/nix-profile-daemon.sh +/scripts/nix-profile.fish +/scripts/nix-profile-daemon.fish # /src/libexpr/ /src/libexpr/lexer-tab.cc diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 59ce5cac7..e97b40cdd 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -42,7 +42,7 @@ $ nix develop ``` To get a shell with a different compilation environment (e.g. stdenv, -gccStdenv, clangStdenv, clang11Stdenv): +gccStdenv, clangStdenv, clang11Stdenv, ccacheStdenv): ```console $ nix-shell -A devShells.x86_64-linux.clang11StdenvPackages @@ -54,6 +54,9 @@ or if you have a flake-enabled nix: $ nix develop .#clang11StdenvPackages ``` +Note: you can use `ccacheStdenv` to drastically improve rebuild +time. By default, ccache keeps artifacts in `~/.cache/ccache/`. + To build Nix itself in this shell: ```console diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 3e903d221..9a0ff9299 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,5 +1,10 @@ # Release X.Y (202?-??-??) +* `` now accepts an additional argument `impure` which + defaults to `false`. If it is set to `true`, the `hash` and `sha256` + arguments will be ignored and the resulting derivation will have + `__impure` set to `true`, making it an impure derivation. + * You can now use flake references in the old CLI, e.g. ``` diff --git a/docker.nix b/docker.nix index e95caf274..bb2b4e7ff 100644 --- a/docker.nix +++ b/docker.nix @@ -33,7 +33,7 @@ let root = { uid = 0; - shell = "/bin/bash"; + shell = "${pkgs.bashInteractive}/bin/bash"; home = "/root"; gid = 0; }; diff --git a/flake.nix b/flake.nix index f7c317570..be58c9b74 100644 --- a/flake.nix +++ b/flake.nix @@ -23,7 +23,7 @@ crossSystems = [ "armv6l-linux" "armv7l-linux" ]; - stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" "libcxxStdenv" ]; + stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" "libcxxStdenv" "ccacheStdenv" ]; forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system); forAllSystemsAndStdenvs = f: forAllSystems (system: @@ -550,6 +550,11 @@ # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; } "touch $out"); + installerTests = import ./tests/installer { + binaryTarballs = self.hydraJobs.binaryTarball; + inherit nixpkgsFor; + }; + }; checks = forAllSystems (system: { diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 9a990275c..a39339050 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -37,6 +37,19 @@ readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshrc" "/e readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix" readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh" +# Fish has different syntax than zsh/bash, treat it separate +readonly PROFILE_FISH_SUFFIX="conf.d/nix.fish" +readonly PROFILE_FISH_PREFIXES=( + # each of these are common values of $__fish_sysconf_dir, + # under which Fish will look for a file named + # $PROFILE_FISH_SUFFIX. + "/etc/fish" # standard + "/usr/local/etc/fish" # their installer .pkg for macOS + "/opt/homebrew/etc/fish" # homebrew + "/opt/local/etc/fish" # macports +) +readonly PROFILE_NIX_FILE_FISH="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.fish" + readonly NIX_INSTALLED_NIX="@nix@" readonly NIX_INSTALLED_CACERT="@cacert@" #readonly NIX_INSTALLED_NIX="/nix/store/j8dbv5w6jl34caywh2ygdy88knx1mdf7-nix-2.3.6" @@ -828,6 +841,19 @@ fi EOF } +# Fish has differing syntax +fish_source_lines() { + cat <&2 - printf '\nif [ -e %s ]; then . %s; fi # added by Nix installer\n' "$p" "$p" >> "$fn" + printf '\nif [ -e %s ]; then . %s; fi # added by Nix installer\n' "$p_sh" "$p_sh" >> "$fn" fi added=1 + p=${p_sh} break fi done for i in .zshenv .zshrc; do fn="$HOME/$i" if [ -w "$fn" ]; then - if ! grep -q "$p" "$fn"; then + if ! grep -q "$p_sh" "$fn"; then echo "modifying $fn..." >&2 - printf '\nif [ -e %s ]; then . %s; fi # added by Nix installer\n' "$p" "$p" >> "$fn" + printf '\nif [ -e %s ]; then . %s; fi # added by Nix installer\n' "$p_sh" "$p_sh" >> "$fn" fi added=1 + p=${p_sh} break fi done + + if [ -d "$HOME/.config/fish" ]; then + fishdir=$HOME/.config/fish/conf.d + if [ ! -d "$fishdir" ]; then + mkdir -p "$fishdir" + fi + + fn="$fishdir/nix.fish" + echo "placing $fn..." >&2 + printf '\nif test -e %s; . %s; end # added by Nix installer\n' "$p_fish" "$p_fish" > "$fn" + added=1 + p=${p_fish} + fi +else + p=${p_sh} fi if [ -z "$added" ]; then diff --git a/scripts/install.in b/scripts/install.in index af5f71080..7d2e52b26 100755 --- a/scripts/install.in +++ b/scripts/install.in @@ -40,12 +40,12 @@ case "$(uname -s).$(uname -m)" in path=@tarballPath_aarch64-linux@ system=aarch64-linux ;; - Linux.armv6l_linux) + Linux.armv6l) hash=@tarballHash_armv6l-linux@ path=@tarballPath_armv6l-linux@ system=armv6l-linux ;; - Linux.armv7l_linux) + Linux.armv7l) hash=@tarballHash_armv7l-linux@ path=@tarballPath_armv7l-linux@ system=armv7l-linux diff --git a/scripts/local.mk b/scripts/local.mk index b8477178e..46255e432 100644 --- a/scripts/local.mk +++ b/scripts/local.mk @@ -6,6 +6,8 @@ noinst-scripts += $(nix_noinst_scripts) profiledir = $(sysconfdir)/profile.d $(eval $(call install-file-as, $(d)/nix-profile.sh, $(profiledir)/nix.sh, 0644)) +$(eval $(call install-file-as, $(d)/nix-profile.fish, $(profiledir)/nix.fish, 0644)) $(eval $(call install-file-as, $(d)/nix-profile-daemon.sh, $(profiledir)/nix-daemon.sh, 0644)) +$(eval $(call install-file-as, $(d)/nix-profile-daemon.fish, $(profiledir)/nix-daemon.fish, 0644)) clean-files += $(nix_noinst_scripts) diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in new file mode 100644 index 000000000..3d587dd7f --- /dev/null +++ b/scripts/nix-profile-daemon.fish.in @@ -0,0 +1,35 @@ +# Only execute this file once per shell. +if test -n "$__ETC_PROFILE_NIX_SOURCED" + exit +end + +set __ETC_PROFILE_NIX_SOURCED 1 + +set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" + +# Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. +if test -n "$NIX_SSH_CERT_FILE" + : # Allow users to override the NIX_SSL_CERT_FILE +else if test -e /etc/ssl/certs/ca-certificates.crt # NixOS, Ubuntu, Debian, Gentoo, Arch + set --export NIX_SSL_CERT_FILE /etc/ssl/certs/ca-certificates.crt +else if test -e /etc/ssl/ca-bundle.pem # openSUSE Tumbleweed + set --export NIX_SSL_CERT_FILE /etc/ssl/ca-bundle.pem +else if test -e /etc/ssl/certs/ca-bundle.crt # Old NixOS + set --export NIX_SSL_CERT_FILE /etc/ssl/certs/ca-bundle.crt +else if test -e /etc/pki/tls/certs/ca-bundle.crt # Fedora, CentOS + set --export NIX_SSL_CERT_FILE /etc/pki/tls/certs/ca-bundle.crt +else if test -e "$NIX_LINK/etc/ssl/certs/ca-bundle.crt" # fall back to cacert in Nix profile + set --export NIX_SSL_CERT_FILE "$NIX_LINK/etc/ssl/certs/ca-bundle.crt" +else if test -e "$NIX_LINK/etc/ca-bundle.crt" # old cacert in Nix profile + set --export NIX_SSL_CERT_FILE "$NIX_LINK/etc/ca-bundle.crt" +else + # Fall back to what is in the nix profiles, favouring whatever is defined last. + for i in $NIX_PROFILES + if test -e "$i/etc/ssl/certs/ca-bundle.crt" + set --export NIX_SSL_CERT_FILE "$i/etc/ssl/certs/ca-bundle.crt" + end + end +end + +fish_add_path --prepend --global "@localstatedir@/nix/profiles/default/bin" +fish_add_path --prepend --global "$HOME/.nix-profile/bin" diff --git a/scripts/nix-profile.fish.in b/scripts/nix-profile.fish.in new file mode 100644 index 000000000..59d247771 --- /dev/null +++ b/scripts/nix-profile.fish.in @@ -0,0 +1,35 @@ +if test -n "$HOME" && test -n "$USER" + + # Set up the per-user profile. + + set NIX_LINK $HOME/.nix-profile + + # Set up environment. + # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix + set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" + + # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. + if test -n "$NIX_SSH_CERT_FILE" + : # Allow users to override the NIX_SSL_CERT_FILE + else if test -e /etc/ssl/certs/ca-certificates.crt # NixOS, Ubuntu, Debian, Gentoo, Arch + set --export NIX_SSL_CERT_FILE /etc/ssl/certs/ca-certificates.crt + else if test -e /etc/ssl/ca-bundle.pem # openSUSE Tumbleweed + set --export NIX_SSL_CERT_FILE /etc/ssl/ca-bundle.pem + else if test -e /etc/ssl/certs/ca-bundle.crt # Old NixOS + set --export NIX_SSL_CERT_FILE /etc/ssl/certs/ca-bundle.crt + else if test -e /etc/pki/tls/certs/ca-bundle.crt # Fedora, CentOS + set --export NIX_SSL_CERT_FILE /etc/pki/tls/certs/ca-bundle.crt + else if test -e "$NIX_LINK/etc/ssl/certs/ca-bundle.crt" # fall back to cacert in Nix profile + set --export NIX_SSL_CERT_FILE "$NIX_LINK/etc/ssl/certs/ca-bundle.crt" + else if test -e "$NIX_LINK/etc/ca-bundle.crt" # old cacert in Nix profile + set --export NIX_SSL_CERT_FILE "$NIX_LINK/etc/ca-bundle.crt" + end + + # Only use MANPATH if it is already set. In general `man` will just simply + # pick up `.nix-profile/share/man` because is it close to `.nix-profile/bin` + # which is in the $PATH. For more info, run `manpath -d`. + set --export --prepend --path MANPATH "$NIX_LINK/share/man" + + fish_add_path --prepend --global "$NIX_LINK/bin" + set --erase NIX_LINK +end diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index 45cbcbe74..5636085d4 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -1,7 +1,6 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then # Set up the per-user profile. - # This part should be kept in sync with nixpkgs:nixos/modules/programs/shell.nix NIX_LINK=$HOME/.nix-profile diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 5f34370d8..81d984d63 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -240,7 +240,11 @@ void NixRepl::mainLoop() // Allow nix-repl specific settings in .inputrc rl_readline_name = "nix-repl"; - createDirs(dirOf(historyFile)); + try { + createDirs(dirOf(historyFile)); + } catch (SysError & e) { + logWarning(e.info()); + } #ifndef READLINE el_hist_size = 1000; #endif diff --git a/src/libexpr/fetchurl.nix b/src/libexpr/fetchurl.nix index 02531103b..9d1b61d7f 100644 --- a/src/libexpr/fetchurl.nix +++ b/src/libexpr/fetchurl.nix @@ -12,13 +12,13 @@ , executable ? false , unpack ? false , name ? baseNameOf (toString url) +, impure ? false }: -derivation { +derivation ({ builder = "builtin:fetchurl"; # New-style output content requirements. - inherit outputHashAlgo outputHash; outputHashMode = if unpack || executable then "recursive" else "flat"; inherit name url executable unpack; @@ -38,4 +38,6 @@ derivation { # To make "nix-prefetch-url" work. urls = [ url ]; -} +} // (if impure + then { __impure = true; } + else { inherit outputHashAlgo outputHash; })) diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc index 3e9d264b4..6df95f1f0 100644 --- a/src/libexpr/flake/config.cc +++ b/src/libexpr/flake/config.cc @@ -68,7 +68,7 @@ void ConfigFile::apply() } } if (!trusted) { - warn("ignoring untrusted flake configuration setting '%s'", name); + warn("ignoring untrusted flake configuration setting '%s'.\nPass '%s' to trust it", name, "--accept-flake-config"); continue; } } diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 76a81d032..07fb75fd5 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -493,12 +493,12 @@ LockedFlake lockFlake( } else if (auto follows = std::get_if<1>(&i.second)) { if (!trustLock) { // It is possible that the flake has changed, - // so we must confirm all the follows that are in the lockfile are also in the flake. + // so we must confirm all the follows that are in the lock file are also in the flake. auto overridePath(inputPath); overridePath.push_back(i.first); auto o = overrides.find(overridePath); // If the override disappeared, we have to refetch the flake, - // since some of the inputs may not be present in the lockfile. + // since some of the inputs may not be present in the lock file. if (o == overrides.end()) { mustRefetch = true; // There's no point populating the rest of the fake inputs, diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index f0874f367..8f50ec811 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2478,8 +2478,8 @@ static RegisterPrimOp primop_intersectAttrs({ .name = "__intersectAttrs", .args = {"e1", "e2"}, .doc = R"( - Return a set consisting of the attributes in the set *e2* that also - exist in the set *e1*. + Return a set consisting of the attributes in the set *e2* which have the + same name as some attribute in *e1*. )", .fun = prim_intersectAttrs, }); diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 52b75f757..c1cf38565 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -4,6 +4,7 @@ #include "gc-store.hh" #include "util.hh" #include "loggers.hh" +#include "progress-bar.hh" #include #include @@ -422,6 +423,8 @@ RunPager::RunPager() if (!pager) pager = getenv("PAGER"); if (pager && ((std::string) pager == "" || (std::string) pager == "cat")) return; + stopProgressBar(); + Pipe toPager; toPager.create(); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index de69b50ee..48dd5c247 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -239,6 +239,8 @@ struct ClientSettings else if (trusted || name == settings.buildTimeout.name || name == settings.buildRepeat.name + || name == settings.maxSilentTime.name + || name == settings.pollInterval.name || name == "connect-timeout" || (name == "builders" && value == "")) settings.set(name, value); diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 5be37efdc..756bd4423 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -314,7 +314,6 @@ struct curlFileTransfer : public FileTransfer } if (request.verifyTLS) { - debug("verify TLS: Nix CA file = '%s'", settings.caFile); if (settings.caFile != "") curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.c_str()); } else { diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 4c1a82279..9ef8972f3 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -619,6 +619,17 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) Path path = storeDir + "/" + std::string(baseName); Path realPath = realStoreDir + "/" + std::string(baseName); + /* There may be temp directories in the store that are still in use + by another process. We need to be sure that we can acquire an + exclusive lock before deleting them. */ + if (baseName.find("tmp-", 0) == 0) { + AutoCloseFD tmpDirFd = open(realPath.c_str(), O_RDONLY | O_DIRECTORY); + if (tmpDirFd.get() == -1 || !lockFile(tmpDirFd.get(), ltWrite, false)) { + debug("skipping locked tempdir '%s'", realPath); + return; + } + } + printInfo("deleting '%1%'", path); results.paths.insert(path); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index e9d721e59..75927d395 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -560,9 +560,15 @@ public: R"( If set to `true` (the default), any non-content-addressed path added or copied to the Nix store (e.g. when substituting from a binary - cache) must have a valid signature, that is, be signed using one of - the keys listed in `trusted-public-keys` or `secret-key-files`. Set - to `false` to disable signature checking. + cache) must have a signature by a trusted key. A trusted key is one + listed in `trusted-public-keys`, or a public key counterpart to a + private key stored in a file listed in `secret-key-files`. + + Set to `false` to disable signature checking and trust all + non-content-addressed paths unconditionally. + + (Content-addressed paths are inherently trustworthy and thus + unaffected by this configuration option.) )"}; Setting extraPlatforms{ diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index a272e4301..d374d4558 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -158,7 +158,7 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) txn.commit(); } - writeFile(schemaPath, fmt("%d", nixCASchemaVersion)); + writeFile(schemaPath, fmt("%d", nixCASchemaVersion), 0666, true); lockFile(lockFd.get(), ltRead, true); } } @@ -281,7 +281,7 @@ LocalStore::LocalStore(const Params & params) else if (curSchema == 0) { /* new store */ curSchema = nixSchemaVersion; openDB(*state, true); - writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); + writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str(), 0666, true); } else if (curSchema < nixSchemaVersion) { @@ -329,7 +329,7 @@ LocalStore::LocalStore(const Params & params) txn.commit(); } - writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); + writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str(), 0666, true); lockFile(globalLock.get(), ltRead, true); } @@ -751,7 +751,7 @@ void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag check if (checkSigs == NoCheckSigs || !realisationIsUntrusted(info)) registerDrvOutput(info); else - throw Error("cannot register realisation '%s' because it lacks a valid signature", info.outPath.to_string()); + throw Error("cannot register realisation '%s' because it lacks a signature by a trusted key", info.outPath.to_string()); } void LocalStore::registerDrvOutput(const Realisation & info) @@ -1266,7 +1266,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) { if (checkSigs && pathInfoIsUntrusted(info)) - throw Error("cannot add path '%s' because it lacks a valid signature", printStorePath(info.path)); + throw Error("cannot add path '%s' because it lacks a signature by a trusted key", printStorePath(info.path)); addTempRoot(info.path); @@ -1382,13 +1382,15 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name std::unique_ptr delTempDir; Path tempPath; + Path tempDir; + AutoCloseFD tempDirFd; if (!inMemory) { /* Drain what we pulled so far, and then keep on pulling */ StringSource dumpSource { dump }; ChainSource bothSource { dumpSource, source }; - auto tempDir = createTempDir(realStoreDir, "add"); + std::tie(tempDir, tempDirFd) = createTempDirInStore(); delTempDir = std::make_unique(tempDir); tempPath = tempDir + "/x"; @@ -1507,18 +1509,24 @@ StorePath LocalStore::addTextToStore( /* Create a temporary directory in the store that won't be - garbage-collected. */ -Path LocalStore::createTempDirInStore() + garbage-collected until the returned FD is closed. */ +std::pair LocalStore::createTempDirInStore() { - Path tmpDir; + Path tmpDirFn; + AutoCloseFD tmpDirFd; + bool lockedByUs = false; do { /* There is a slight possibility that `tmpDir' gets deleted by - the GC between createTempDir() and addTempRoot(), so repeat - until `tmpDir' exists. */ - tmpDir = createTempDir(realStoreDir); - addTempRoot(parseStorePath(tmpDir)); - } while (!pathExists(tmpDir)); - return tmpDir; + the GC between createTempDir() and when we acquire a lock on it. + We'll repeat until 'tmpDir' exists and we've locked it. */ + tmpDirFn = createTempDir(realStoreDir, "tmp"); + tmpDirFd = open(tmpDirFn.c_str(), O_RDONLY | O_DIRECTORY); + if (tmpDirFd.get() < 0) { + continue; + } + lockedByUs = lockFile(tmpDirFd.get(), ltWrite, true); + } while (!pathExists(tmpDirFn) || !lockedByUs); + return {tmpDirFn, std::move(tmpDirFd)}; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 70d225be3..bd0ce1fe6 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -256,7 +256,7 @@ private: void findRuntimeRoots(Roots & roots, bool censor); - Path createTempDirInStore(); + std::pair createTempDirInStore(); void checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv); diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 44b63f0f6..753980fd4 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -216,7 +216,7 @@ nlohmann::json Args::toJSON() if (flag->shortName) j["shortName"] = std::string(1, flag->shortName); if (flag->description != "") - j["description"] = flag->description; + j["description"] = trim(flag->description); j["category"] = flag->category; if (flag->handler.arity != ArityAny) j["arity"] = flag->handler.arity; @@ -237,7 +237,7 @@ nlohmann::json Args::toJSON() } auto res = nlohmann::json::object(); - res["description"] = description(); + res["description"] = trim(description()); res["flags"] = std::move(flags); res["args"] = std::move(args); auto s = doc(); @@ -379,7 +379,7 @@ nlohmann::json MultiCommand::toJSON() auto j = command->toJSON(); auto cat = nlohmann::json::object(); cat["id"] = command->category(); - cat["description"] = categories[command->category()]; + cat["description"] = trim(categories[command->category()]); j["category"] = std::move(cat); cmds[name] = std::move(j); } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index d0b4708a1..f1116e2c0 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -353,7 +353,7 @@ void readFile(const Path & path, Sink & sink) } -void writeFile(const Path & path, std::string_view s, mode_t mode) +void writeFile(const Path & path, std::string_view s, mode_t mode, bool sync) { AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); if (!fd) @@ -364,10 +364,16 @@ void writeFile(const Path & path, std::string_view s, mode_t mode) e.addTrace({}, "writing file '%1%'", path); throw; } + if (sync) + fd.fsync(); + // Explicitly close to make sure exceptions are propagated. + fd.close(); + if (sync) + syncParent(path); } -void writeFile(const Path & path, Source & source, mode_t mode) +void writeFile(const Path & path, Source & source, mode_t mode, bool sync) { AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); if (!fd) @@ -386,6 +392,20 @@ void writeFile(const Path & path, Source & source, mode_t mode) e.addTrace({}, "writing file '%1%'", path); throw; } + if (sync) + fd.fsync(); + // Explicitly close to make sure exceptions are propagated. + fd.close(); + if (sync) + syncParent(path); +} + +void syncParent(const Path & path) +{ + AutoCloseFD fd = open(dirOf(path).c_str(), O_RDONLY, 0); + if (!fd) + throw SysError("opening file '%1%'", path); + fd.fsync(); } std::string readLine(int fd) @@ -841,6 +861,20 @@ void AutoCloseFD::close() } } +void AutoCloseFD::fsync() +{ + if (fd != -1) { + int result; +#if __APPLE__ + result = ::fcntl(fd, F_FULLFSYNC); +#else + result = ::fsync(fd); +#endif + if (result == -1) + throw SysError("fsync file descriptor %1%", fd); + } +} + AutoCloseFD::operator bool() const { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 44b8370bf..b9572156a 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -115,9 +115,12 @@ std::string readFile(const Path & path); void readFile(const Path & path, Sink & sink); /* Write a string to a file. */ -void writeFile(const Path & path, std::string_view s, mode_t mode = 0666); +void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false); -void writeFile(const Path & path, Source & source, mode_t mode = 0666); +void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false); + +/* Flush a file's parent directory to disk */ +void syncParent(const Path & path); /* Read a line from a file descriptor. */ std::string readLine(int fd); @@ -231,6 +234,7 @@ public: explicit operator bool() const; int release(); void close(); + void fsync(); }; diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 39a13a8d3..92f3f884e 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -85,7 +85,6 @@ static void main_nix_build(int argc, char * * argv) Strings attrPaths; Strings left; RepairFlag repair = NoRepair; - Path gcRoot; BuildMode buildMode = bmNormal; bool readStdin = false; @@ -167,9 +166,6 @@ static void main_nix_build(int argc, char * * argv) else if (*arg == "--out-link" || *arg == "-o") outLink = getArg(*arg, arg, end); - else if (*arg == "--add-root") - gcRoot = getArg(*arg, arg, end); - else if (*arg == "--dry-run") dryRun = true; diff --git a/src/nix/develop.cc b/src/nix/develop.cc index cd05ce9d0..bd42aead3 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -246,6 +246,7 @@ struct Common : InstallableCommand, MixProfile "NIX_LOG_FD", "NIX_REMOTE", "PPID", + "SHELL", "SHELLOPTS", "SSL_CERT_FILE", // FIXME: only want to ignore /no-cert-file.crt "TEMP", diff --git a/src/nix/main.cc b/src/nix/main.cc index d95d00d34..8c6e5f939 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -322,7 +322,7 @@ void mainWrapped(int argc, char * * argv) std::cout << "attrs\n"; break; } for (auto & s : *completions) - std::cout << s.completion << "\t" << s.description << "\n"; + std::cout << s.completion << "\t" << trim(s.description) << "\n"; } }); diff --git a/src/nix/make-content-addressed.md b/src/nix/make-content-addressed.md index 215683e6d..32eecc880 100644 --- a/src/nix/make-content-addressed.md +++ b/src/nix/make-content-addressed.md @@ -22,7 +22,7 @@ R""( ```console # nix copy --to /tmp/nix --trusted-public-keys '' nixpkgs#hello - cannot add path '/nix/store/zy9wbxwcygrwnh8n2w9qbbcr6zk87m26-libunistring-0.9.10' because it lacks a valid signature + cannot add path '/nix/store/zy9wbxwcygrwnh8n2w9qbbcr6zk87m26-libunistring-0.9.10' because it lacks a signature by a trusted key ``` * Create a content-addressed representation of the current NixOS diff --git a/src/nix/verify.cc b/src/nix/verify.cc index e92df1303..efa2434dc 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -41,7 +41,7 @@ struct CmdVerify : StorePathsCommand addFlag({ .longName = "sigs-needed", .shortName = 'n', - .description = "Require that each path has at least *n* valid signatures.", + .description = "Require that each path is signed by at least *n* different keys.", .labels = {"n"}, .handler = {&sigsNeeded} }); diff --git a/tests/installer/default.nix b/tests/installer/default.nix new file mode 100644 index 000000000..32aa7889a --- /dev/null +++ b/tests/installer/default.nix @@ -0,0 +1,220 @@ +{ binaryTarballs +, nixpkgsFor +}: + +let + + installScripts = { + install-default = { + script = '' + tar -xf ./nix.tar.xz + mv ./nix-* nix + ./nix/install --no-channel-add + ''; + }; + + install-force-no-daemon = { + script = '' + tar -xf ./nix.tar.xz + mv ./nix-* nix + ./nix/install --no-daemon + ''; + }; + + install-force-daemon = { + script = '' + tar -xf ./nix.tar.xz + mv ./nix-* nix + ./nix/install --daemon --no-channel-add + ''; + }; + }; + + disableSELinux = "sudo setenforce 0"; + + images = { + + /* + "ubuntu-14-04" = { + image = import { + url = "https://app.vagrantup.com/ubuntu/boxes/trusty64/versions/20190514.0.0/providers/virtualbox.box"; + hash = "sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="; + }; + rootDisk = "box-disk1.vmdk"; + system = "x86_64-linux"; + }; + */ + + "ubuntu-16-04" = { + image = import { + url = "https://app.vagrantup.com/generic/boxes/ubuntu1604/versions/4.1.12/providers/libvirt.box"; + hash = "sha256-lO4oYQR2tCh5auxAYe6bPOgEqOgv3Y3GC1QM1tEEEU8="; + }; + rootDisk = "box.img"; + system = "x86_64-linux"; + }; + + "ubuntu-22-04" = { + image = import { + url = "https://app.vagrantup.com/generic/boxes/ubuntu2204/versions/4.1.12/providers/libvirt.box"; + hash = "sha256-HNll0Qikw/xGIcogni5lz01vUv+R3o8xowP2EtqjuUQ="; + }; + rootDisk = "box.img"; + system = "x86_64-linux"; + }; + + "fedora-36" = { + image = import { + url = "https://app.vagrantup.com/generic/boxes/fedora36/versions/4.1.12/providers/libvirt.box"; + hash = "sha256-rxPgnDnFkTDwvdqn2CV3ZUo3re9AdPtSZ9SvOHNvaks="; + }; + rootDisk = "box.img"; + system = "x86_64-linux"; + postBoot = disableSELinux; + }; + + # Currently fails with 'error while loading shared libraries: + # libsodium.so.23: cannot stat shared object: Invalid argument'. + /* + "rhel-6" = { + image = import { + url = "https://app.vagrantup.com/generic/boxes/rhel6/versions/4.1.12/providers/libvirt.box"; + hash = "sha256-QwzbvRoRRGqUCQptM7X/InRWFSP2sqwRt2HaaO6zBGM="; + }; + rootDisk = "box.img"; + system = "x86_64-linux"; + }; + */ + + "rhel-7" = { + image = import { + url = "https://app.vagrantup.com/generic/boxes/rhel7/versions/4.1.12/providers/libvirt.box"; + hash = "sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="; + }; + rootDisk = "box.img"; + system = "x86_64-linux"; + }; + + "rhel-8" = { + image = import { + url = "https://app.vagrantup.com/generic/boxes/rhel8/versions/4.1.12/providers/libvirt.box"; + hash = "sha256-zFOPjSputy1dPgrQRixBXmlyN88cAKjJ21VvjSWUCUY="; + }; + rootDisk = "box.img"; + system = "x86_64-linux"; + postBoot = disableSELinux; + }; + + "rhel-9" = { + image = import { + url = "https://app.vagrantup.com/generic/boxes/rhel9/versions/4.1.12/providers/libvirt.box"; + hash = "sha256-vL/FbB3kK1rcSaR627nWmScYGKGk4seSmAdq6N5diMg="; + }; + rootDisk = "box.img"; + system = "x86_64-linux"; + postBoot = disableSELinux; + extraQemuOpts = "-cpu Westmere-v2"; + }; + + }; + + makeTest = imageName: testName: + let image = images.${imageName}; in + with nixpkgsFor.${image.system}; + runCommand + "installer-test-${imageName}-${testName}" + { buildInputs = [ qemu_kvm openssh ]; + image = image.image; + postBoot = image.postBoot or ""; + installScript = installScripts.${testName}.script; + binaryTarball = binaryTarballs.${system}; + } + '' + shopt -s nullglob + + echo "Unpacking Vagrant box $image..." + tar xvf $image + + image_type=$(qemu-img info ${image.rootDisk} | sed 's/file format: \(.*\)/\1/; t; d') + + qemu-img create -b ./${image.rootDisk} -F "$image_type" -f qcow2 ./disk.qcow2 + + extra_qemu_opts="${image.extraQemuOpts or ""}" + + # Add the config disk, required by the Ubuntu images. + config_drive=$(echo *configdrive.vmdk || true) + if [[ -n $config_drive ]]; then + extra_qemu_opts+=" -drive id=disk2,file=$config_drive,if=virtio" + fi + + echo "Starting qemu..." + qemu-kvm -m 4096 -nographic \ + -drive id=disk1,file=./disk.qcow2,if=virtio \ + -netdev user,id=net0,restrict=yes,hostfwd=tcp::20022-:22 -device virtio-net-pci,netdev=net0 \ + $extra_qemu_opts & + qemu_pid=$! + trap "kill $qemu_pid" EXIT + + if ! [ -e ./vagrant_insecure_key ]; then + cp ${./vagrant_insecure_key} vagrant_insecure_key + fi + + chmod 0400 ./vagrant_insecure_key + + ssh_opts="-o StrictHostKeyChecking=no -o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa -i ./vagrant_insecure_key" + ssh="ssh -p 20022 -q $ssh_opts vagrant@localhost" + + echo "Waiting for SSH..." + for ((i = 0; i < 120; i++)); do + echo "[ssh] Trying to connect..." + if $ssh -- true; then + echo "[ssh] Connected!" + break + fi + if ! kill -0 $qemu_pid; then + echo "qemu died unexpectedly" + exit 1 + fi + sleep 1 + done + + if [[ -n $postBoot ]]; then + echo "Running post-boot commands..." + $ssh "set -ex; $postBoot" + fi + + echo "Copying installer..." + scp -P 20022 $ssh_opts $binaryTarball/nix-*.tar.xz vagrant@localhost:nix.tar.xz + + echo "Running installer..." + $ssh "set -eux; $installScript" + + echo "Testing Nix installation..." + $ssh < \$out"]; }') + [[ \$(cat \$out) = foobar ]] + EOF + + echo "Done!" + touch $out + ''; + +in + +builtins.mapAttrs (imageName: image: + { ${image.system} = builtins.mapAttrs (testName: test: + makeTest imageName testName + ) installScripts; + } +) images diff --git a/tests/installer/vagrant_insecure_key b/tests/installer/vagrant_insecure_key new file mode 100644 index 000000000..7d6a08390 --- /dev/null +++ b/tests/installer/vagrant_insecure_key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzI +w+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoP +kcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2 +hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NO +Td0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcW +yLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQIBIwKCAQEA4iqWPJXtzZA68mKd +ELs4jJsdyky+ewdZeNds5tjcnHU5zUYE25K+ffJED9qUWICcLZDc81TGWjHyAqD1 +Bw7XpgUwFgeUJwUlzQurAv+/ySnxiwuaGJfhFM1CaQHzfXphgVml+fZUvnJUTvzf +TK2Lg6EdbUE9TarUlBf/xPfuEhMSlIE5keb/Zz3/LUlRg8yDqz5w+QWVJ4utnKnK +iqwZN0mwpwU7YSyJhlT4YV1F3n4YjLswM5wJs2oqm0jssQu/BT0tyEXNDYBLEF4A +sClaWuSJ2kjq7KhrrYXzagqhnSei9ODYFShJu8UWVec3Ihb5ZXlzO6vdNQ1J9Xsf +4m+2ywKBgQD6qFxx/Rv9CNN96l/4rb14HKirC2o/orApiHmHDsURs5rUKDx0f9iP +cXN7S1uePXuJRK/5hsubaOCx3Owd2u9gD6Oq0CsMkE4CUSiJcYrMANtx54cGH7Rk +EjFZxK8xAv1ldELEyxrFqkbE4BKd8QOt414qjvTGyAK+OLD3M2QdCQKBgQDtx8pN +CAxR7yhHbIWT1AH66+XWN8bXq7l3RO/ukeaci98JfkbkxURZhtxV/HHuvUhnPLdX +3TwygPBYZFNo4pzVEhzWoTtnEtrFueKxyc3+LjZpuo+mBlQ6ORtfgkr9gBVphXZG +YEzkCD3lVdl8L4cw9BVpKrJCs1c5taGjDgdInQKBgHm/fVvv96bJxc9x1tffXAcj +3OVdUN0UgXNCSaf/3A/phbeBQe9xS+3mpc4r6qvx+iy69mNBeNZ0xOitIjpjBo2+ +dBEjSBwLk5q5tJqHmy/jKMJL4n9ROlx93XS+njxgibTvU6Fp9w+NOFD/HvxB3Tcz +6+jJF85D5BNAG3DBMKBjAoGBAOAxZvgsKN+JuENXsST7F89Tck2iTcQIT8g5rwWC +P9Vt74yboe2kDT531w8+egz7nAmRBKNM751U/95P9t88EDacDI/Z2OwnuFQHCPDF +llYOUI+SpLJ6/vURRbHSnnn8a/XG+nzedGH5JGqEJNQsz+xT2axM0/W/CRknmGaJ +kda/AoGANWrLCz708y7VYgAtW2Uf1DPOIYMdvo6fxIB5i9ZfISgcJ/bbCUkFrhoH ++vq/5CIWxCPp0f85R4qxxQ5ihxJ0YDQT9Jpx4TMss4PSavPaBH3RXow5Ohe+bYoQ +NE5OgEXk2wVfZczCZpigBKbKZHNYcelXtTt/nP3rsCuGcM4h53s= +-----END RSA PRIVATE KEY----- diff --git a/tests/signing.sh b/tests/signing.sh index 6aafbeb91..9b673c609 100644 --- a/tests/signing.sh +++ b/tests/signing.sh @@ -81,7 +81,7 @@ info=$(nix path-info --store file://$cacheDir --json $outPath2) [[ $info =~ 'cache1.example.org' ]] [[ $info =~ 'cache2.example.org' ]] -# Copying to a diverted store should fail due to a lack of valid signatures. +# Copying to a diverted store should fail due to a lack of signatures by trusted keys. chmod -R u+w $TEST_ROOT/store0 || true rm -rf $TEST_ROOT/store0 (! nix copy --to $TEST_ROOT/store0 $outPath)