From c29fffeaaad8a3642acaf78fed4414cae30baf7d Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 18 Oct 2020 14:21:53 +0200 Subject: [PATCH 01/65] DerivationGoal: only retry if output closure incomplete is only problem (cherry picked from commit bd9eb5c743faf1b3c33f4e1c2ccf317977d4be9d) --- src/libstore/build.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 4ba9a5385..d09db82fc 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1222,8 +1222,13 @@ void DerivationGoal::outputsSubstituted() /* If the substitutes form an incomplete closure, then we should build the dependencies of this derivation, but after that, we - can still use the substitutes for this derivation itself. */ - if (nrIncompleteClosure > 0) retrySubstitution = true; + can still use the substitutes for this derivation itself. + + If the nrIncompleteClosure != nrFailed, we have another issue as well. + In particular, it may be the case that the hole in the closure is + an output of the current derivation, which causes a loop if retried. + */ + if (nrIncompleteClosure > 0 && nrIncompleteClosure == nrFailed) retrySubstitution = true; nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; From e90530e491278169b501e53ce6773e253b45e15e Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 18 Oct 2020 00:43:52 +0200 Subject: [PATCH 02/65] Tests for #3964 (cherry picked from commit ea8d32020e1e24576cf5e8ff1a835f149dbd81c3) --- tests/binary-cache.sh | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index eb58ae7c1..7fc6a6fcc 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -168,3 +168,34 @@ clearCacheCache nix-store -r $outPath --substituters "file://$cacheDir2 file://$cacheDir" --trusted-public-keys "$publicKey" fi # HAVE_LIBSODIUM + +# Test against issue https://github.com/NixOS/nix/issues/3964 +# +expr=' + with import ./config.nix; + mkDerivation { + name = "multi-output"; + buildCommand = "mkdir -p $out; echo foo > $doc; echo $doc > $out/docref"; + outputs = ["out" "doc"]; + } +' +outPath=$(nix-build --no-out-link -E "$expr") +docPath=$(nix-store -q --references $outPath) + +# $ nix-store -q --tree $outPath +# ...-multi-output +# +---...-multi-output-doc + +nix copy --to "file://$cacheDir" $outPath + +hashpart() { + basename "$1" | cut -c1-32 +} + +# break the closure of out by removing doc +rm $cacheDir/$(hashpart $docPath).narinfo + +nix-store --delete $outPath $docPath +# -vvv is the level that logs during the loop +timeout 60 nix-build -E "$expr" --option substituters "file://$cacheDir" \ + --option trusted-binary-caches "file://$cacheDir" --no-require-sigs From 8388e2ea48f340e940005aa7ffbc94fe7681c766 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 22 Dec 2020 11:57:31 +0100 Subject: [PATCH 03/65] Revert "Tests for #3964" This reverts commit e90530e491278169b501e53ce6773e253b45e15e. It appears that this backported test relies on new functionality. The error is: + nix copy --to file:///build/nix-test/binary-cache /build/nix-test/store/dckdy2xp7sn8qvyx8axxjq7clnxddri1-multi-output warning: you don't have Internet access; disabling some network-dependent features error: uploading to 'file:///build/nix-test/binary-cache/nar/0hpfhrig9h6fvbjmiwsb7zkaxr19vr2w048y4xb3nrs36bagf92n.nar.xz' is not supported --- tests/binary-cache.sh | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index 7fc6a6fcc..eb58ae7c1 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -168,34 +168,3 @@ clearCacheCache nix-store -r $outPath --substituters "file://$cacheDir2 file://$cacheDir" --trusted-public-keys "$publicKey" fi # HAVE_LIBSODIUM - -# Test against issue https://github.com/NixOS/nix/issues/3964 -# -expr=' - with import ./config.nix; - mkDerivation { - name = "multi-output"; - buildCommand = "mkdir -p $out; echo foo > $doc; echo $doc > $out/docref"; - outputs = ["out" "doc"]; - } -' -outPath=$(nix-build --no-out-link -E "$expr") -docPath=$(nix-store -q --references $outPath) - -# $ nix-store -q --tree $outPath -# ...-multi-output -# +---...-multi-output-doc - -nix copy --to "file://$cacheDir" $outPath - -hashpart() { - basename "$1" | cut -c1-32 -} - -# break the closure of out by removing doc -rm $cacheDir/$(hashpart $docPath).narinfo - -nix-store --delete $outPath $docPath -# -vvv is the level that logs during the loop -timeout 60 nix-build -E "$expr" --option substituters "file://$cacheDir" \ - --option trusted-binary-caches "file://$cacheDir" --no-require-sigs From 024c0eaad1a7734bde8c6b0937fff03316e45d09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phan=20Kochen?= Date: Fri, 4 Dec 2020 19:32:35 +0100 Subject: [PATCH 04/65] Fix compatibility with newer AWS SDKs (#4316) Tested against AWS SDK 1.8.99. Fixes #3201. (cherry picked from commit e20a3ec756c61b076d5fbaea6d976c8ee7fa4c07) --- configure.ac | 1 + src/libstore/s3-binary-cache-store.cc | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/configure.ac b/configure.ac index a52830b38..44a86c248 100644 --- a/configure.ac +++ b/configure.ac @@ -238,6 +238,7 @@ if test -n "$enable_s3"; then declare -a aws_version_tokens=($(printf '#include \nAWS_SDK_VERSION_STRING' | $CPP $CPPFLAGS - | grep -v '^#.*' | sed 's/"//g' | tr '.' ' ')) AC_DEFINE_UNQUOTED([AWS_VERSION_MAJOR], ${aws_version_tokens@<:@0@:>@}, [Major version of aws-sdk-cpp.]) AC_DEFINE_UNQUOTED([AWS_VERSION_MINOR], ${aws_version_tokens@<:@1@:>@}, [Minor version of aws-sdk-cpp.]) + AC_DEFINE_UNQUOTED([AWS_VERSION_PATCH], ${aws_version_tokens@<:@2@:>@}, [Patch version of aws-sdk-cpp.]) fi diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index cd547a964..1e5e264fc 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -56,6 +56,10 @@ class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem { debug("AWS: %s", chomp(statement)); } + +#if !(AWS_VERSION_MAJOR <= 1 && AWS_VERSION_MINOR <= 7 && AWS_VERSION_PATCH <= 115) + void Flush() override {} +#endif }; static void initAWS() From f95c320500548c9262b3d77898509fbb36c346b4 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 22 Jan 2021 14:46:40 -0600 Subject: [PATCH 05/65] Handle missing etag in 304 Not Modified response GitHub now omits the etag, but 304 implies it matches the one we provided. Just use that one to avoid having an etag-less resource. Fixes #4469 (cherry picked from commit a76682466062ef2c972d19f259feeef1c46a44a3) --- src/libstore/download.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 80674a9e7..b439ddf0c 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -349,6 +349,13 @@ struct CurlDownloader : public Downloader (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */)) { result.cached = httpStatus == 304; + + // In 2021, GitHub responds to If-None-Match with 304, + // but omits ETag. We just use the If-None-Match etag + // since 304 implies they are the same. + if (httpStatus == 304 && result.etag == "") + result.etag = request.expectedETag; + act.progress(result.bodySize, result.bodySize); done = true; callback(std::move(result)); From 85329cac35b362228ec31bc2d8d8effacdb0747d Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Thu, 21 Jan 2021 17:30:26 +0100 Subject: [PATCH 06/65] scripts/install-nix-from-closure: only show progress if a terminal is used While the progress dots during the copying of the store work fine on a normal terminal, those look pretty off if the script is run inside a provisioning script of e.g. `vagrant` or `packer` where `stderr` and `stdout` are captured: default: . default: .. default: . default: . default: . To work around this, the script checks with `-t 0` if it's running on an actual terminal and doesn't show the progress if that's not the case. (cherry picked from commit d9367a2dd1f2cfe163b9c42e83a0569808ce6fc9) --- scripts/install-nix-from-closure.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index 90b7a8948..b736a2d69 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -154,9 +154,15 @@ fi mkdir -p $dest/store printf "copying Nix to %s..." "${dest}/store" >&2 +# Insert a newline if no progress is shown. +if [ ! -t 0 ]; then + echo "" +fi for i in $(cd "$self/store" >/dev/null && echo ./*); do - printf "." >&2 + if [ -t 0 ]; then + printf "." >&2 + fi i_tmp="$dest/store/$i.$$" if [ -e "$i_tmp" ]; then rm -rf "$i_tmp" From da1629d7449694e378b3266b53d2c5f629ca77b0 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Date: Wed, 3 Mar 2021 14:46:15 -0800 Subject: [PATCH 07/65] nix-build: set execfail When starting a nix-shell with `-i` it was previously possible for it to silently fail in the scenario where the specified interpreter didn't exist. This happened due to the `exec` call masking the issue. With this change we enable `execfail`, which causes the script using `nix-shell` as interpreter to correctly exit with code 127. Fixes: #4598 (cherry picked from commit 6e849e3b0a6eb46e6dc65cbd091cc829eab09a5f) --- src/nix-build/nix-build.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 33ad28704..1ebc80976 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -425,6 +425,7 @@ static void _main(int argc, char * * argv) "unset NIX_ENFORCE_PURITY; " "shopt -u nullglob; " "unset TZ; %6%" + "shopt -s execfail;" "%7%", (Path) tmpDir, (pure ? "" : "p=$PATH; "), From 2fe57daadd7ea9cac364ffc9c80f374d3decf610 Mon Sep 17 00:00:00 2001 From: David McFarland Date: Wed, 30 Dec 2020 16:20:03 -0400 Subject: [PATCH 08/65] Fix insufficent attribute capacity in user profile --- src/nix-env/user-env.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 7b9a88281..97dfc7a29 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -50,10 +50,12 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, output paths, and optionally the derivation path, as well as the meta attributes. */ Path drvPath = keepDerivations ? i.queryDrvPath() : ""; + DrvInfo::Outputs outputs = i.queryOutputs(true); + StringSet metaNames = i.queryMetaNames(); Value & v(*state.allocValue()); manifest.listElems()[n++] = &v; - state.mkAttrs(v, 16); + state.mkAttrs(v, 7 + outputs.size()); mkString(*state.allocAttr(v, state.sType), "derivation"); mkString(*state.allocAttr(v, state.sName), i.queryName()); @@ -65,7 +67,6 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, mkString(*state.allocAttr(v, state.sDrvPath), i.queryDrvPath()); // Copy each output meant for installation. - DrvInfo::Outputs outputs = i.queryOutputs(true); Value & vOutputs = *state.allocAttr(v, state.sOutputs); state.mkList(vOutputs, outputs.size()); unsigned int m = 0; @@ -85,8 +86,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, // Copy the meta attributes. Value & vMeta = *state.allocAttr(v, state.sMeta); - state.mkAttrs(vMeta, 16); - StringSet metaNames = i.queryMetaNames(); + state.mkAttrs(vMeta, metaNames.size()); for (auto & j : metaNames) { Value * v = i.queryMeta(j); if (!v) continue; From dd50221e8a026fcc685c73149a3d8680e7ab8ebd Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Sun, 7 Feb 2021 15:34:24 -0600 Subject: [PATCH 09/65] fix nixbld user name/uid for macOS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit 0431cf6d0992e7986afbb3d0ffd0a7e1cca8ae8a) Signed-off-by: Domen Kožar --- scripts/bigsur-nixbld-user-migration.sh | 46 +++++++++++++++++++++++++ scripts/install-darwin-multi-user.sh | 2 ++ scripts/install-multi-user.sh | 6 ++-- 3 files changed, 52 insertions(+), 2 deletions(-) create mode 100755 scripts/bigsur-nixbld-user-migration.sh diff --git a/scripts/bigsur-nixbld-user-migration.sh b/scripts/bigsur-nixbld-user-migration.sh new file mode 100755 index 000000000..f1619fd56 --- /dev/null +++ b/scripts/bigsur-nixbld-user-migration.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +((NEW_NIX_FIRST_BUILD_UID=301)) + +id_available(){ + dscl . list /Users UniqueID | grep -E '\b'$1'\b' >/dev/null +} + +change_nixbld_names_and_ids(){ + local name uid next_id + ((next_id=NEW_NIX_FIRST_BUILD_UID)) + echo "Attempting to migrate nixbld users." + echo "Each user should change from nixbld# to _nixbld#" + echo "and their IDs relocated to $next_id+" + while read -r name uid; do + echo " Checking $name (uid: $uid)" + # iterate for a clean ID + while id_available "$next_id"; do + ((next_id++)) + if ((next_id >= 400)); then + echo "We've hit UID 400 without placing all of your users :(" + echo "You should use the commands in this script as a starting" + echo "point to review your UID-space and manually move the" + echo "remaining users (or delete them, if you don't need them)." + exit 1 + fi + done + + if [[ $name == _* ]]; then + echo " It looks like $name has already been renamed--skipping." + else + # first 3 are cleanup, it's OK if they aren't here + sudo dscl . delete /Users/$name dsAttrTypeNative:_writers_passwd &>/dev/null || true + sudo dscl . change /Users/$name NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true + # remove existing user from group + sudo dseditgroup -o edit -t user -d $name nixbld || true + sudo dscl . change /Users/$name UniqueID $uid $next_id + sudo dscl . change /Users/$name RecordName $name _$name + # add renamed user to group + sudo dseditgroup -o edit -t user -a _$name nixbld + echo " $name migrated to _$name (uid: $next_id)" + fi + done < <(dscl . list /Users UniqueID | grep nixbld | sort -n -k2) +} + +change_nixbld_names_and_ids diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh index 49076bd5c..77b81ee50 100644 --- a/scripts/install-darwin-multi-user.sh +++ b/scripts/install-darwin-multi-user.sh @@ -4,6 +4,8 @@ set -eu set -o pipefail readonly PLIST_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist +NIX_FIRST_BUILD_UID="301" +NIX_BUILD_USER_NAME_TEMPLATE="_nixbld%d" dsclattr() { /usr/bin/dscl . -read "$1" \ diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index a3613cb28..349bffa45 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -25,7 +25,9 @@ readonly RED='\033[31m' readonly NIX_USER_COUNT=${NIX_USER_COUNT:-32} readonly NIX_BUILD_GROUP_ID="30000" readonly NIX_BUILD_GROUP_NAME="nixbld" -readonly NIX_FIRST_BUILD_UID="30001" +# darwin installer needs to override these +NIX_FIRST_BUILD_UID="30001" +NIX_BUILD_USER_NAME_TEMPLATE="nixbld%d" # Please don't change this. We don't support it, because the # default shell profile that comes with Nix doesn't support it. readonly NIX_ROOT="/nix" @@ -102,7 +104,7 @@ EOF } nix_user_for_core() { - printf "nixbld%d" "$1" + printf "$NIX_BUILD_USER_NAME_TEMPLATE" "$1" } nix_uid_for_core() { From 419797128d89ba775659cd23cf8bbeab01c2053c Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Wed, 21 Oct 2020 00:03:38 +0400 Subject: [PATCH 10/65] Fix iterating over $NIX_PROFILES in Zsh NIX_PROFILES is space separated list of directories, and passing it into for as is is considered to be 1-element list with the whole string. With shwordsplit option Zsh emulates other shells in this regard ans implicitely splits unquoted strings into words. Fixes #4167. --- scripts/nix-profile-daemon.sh.in | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index e0c24954a..31fe4f949 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -17,11 +17,21 @@ elif [ -e /etc/pki/tls/certs/ca-bundle.crt ]; then # Fedora, CentOS export NIX_SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt else # Fall back to what is in the nix profiles, favouring whatever is defined last. - for i in $NIX_PROFILES; do - if [ -e $i/etc/ssl/certs/ca-bundle.crt ]; then - export NIX_SSL_CERT_FILE=$i/etc/ssl/certs/ca-bundle.crt + check_nix_profiles() { + if [ "$ZSH_VERSION" ]; then + # Zsh by default doesn't split words in unquoted parameter expansion. + # Set local_options for these options to be reverted at the end of the function + # and shwordsplit to force splitting words in $NIX_PROFILES below. + setopt local_options shwordsplit fi - done + for i in $NIX_PROFILES; do + if [ -e $i/etc/ssl/certs/ca-bundle.crt ]; then + export NIX_SSL_CERT_FILE=$i/etc/ssl/certs/ca-bundle.crt + fi + done + } + check_nix_profiles + unset -f check_nix_profiles fi export NIX_PATH="nixpkgs=@localstatedir@/nix/profiles/per-user/root/channels/nixpkgs:@localstatedir@/nix/profiles/per-user/root/channels" From 2e12b28fc5d3c7e4c2f7fdd87ca310e43c7760e3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 14 May 2021 11:59:01 +0200 Subject: [PATCH 11/65] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index 506c62f67..1ba605fd4 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.3.10 \ No newline at end of file +2.3.11 \ No newline at end of file From 5e5013e2f6f29ea9e5f4db41a2450a3861105449 Mon Sep 17 00:00:00 2001 From: Jonathan Ringer Date: Mon, 31 May 2021 17:14:33 -0700 Subject: [PATCH 12/65] Fix pure build of 2.3 --- release-common.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-common.nix b/release-common.nix index d66bbafa8..98e2309ac 100644 --- a/release-common.nix +++ b/release-common.nix @@ -57,7 +57,7 @@ rec { git mercurial ] - ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal] + ++ lib.optionals stdenv.isLinux [libseccomp util-linuxMinimal] ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) ((aws-sdk-cpp.override { From 5713ff48c3bff410666b20ac6645b7d0b4d6b142 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 1 Jun 2021 15:20:51 +0200 Subject: [PATCH 13/65] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index 1ba605fd4..b57013902 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.3.11 \ No newline at end of file +2.3.12 \ No newline at end of file From 3dbd83b9a196b8c58eda9b6033442bf2020a9f60 Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 27 May 2021 13:25:25 +0200 Subject: [PATCH 14/65] Check the CA hash when importing stuff in the local store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When adding a path to the local store (via `LocalStore::addToStore`), ensure that the `ca` field of the provided `ValidPathInfo` does indeed correspond to the content of the path. Otherwise any untrusted user (or any binary cache) can add arbitrary content-addressed paths to the store (as content-addressed paths don’t need a signature). --- src/libstore/local-store.cc | 50 +++++++++++++++++++++++++++++ src/libstore/local-store.hh | 10 ++++-- tests/local.mk | 1 + tests/substitute-with-invalid-ca.sh | 38 ++++++++++++++++++++++ 4 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 tests/substitute-with-invalid-ca.sh diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index f5092151a..5b62d79ee 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1029,6 +1029,40 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s", info.path, info.narSize, hashResult.second); + if (!info.ca.empty()) { + auto ca = info.ca; + if (hasPrefix(ca, "fixed:")) { + bool recursive = ca.compare(6, 2, "r:") == 0; + Hash expectedHash(std::string(ca, recursive ? 8 : 6)); + if (info.references.empty()) { + auto actualFoHash = hashCAPath( + recursive, + expectedHash.type, + info.path + ); + if (ca != actualFoHash) { + throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", + info.path, + ca, + actualFoHash); + } + } else { + throw Error("path '%s' claims to be content-addressed, but has references. This isn’t allowed", + info.path); + } + + } else if (hasPrefix(ca, "text:")) { + Hash textHash(std::string(ca, 5)); + auto actualTextHash = hashString(htSHA256, readFile(realPath)); + if (textHash != actualTextHash) { + throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", + info.path, + textHash.to_string(Base32, true), + actualTextHash.to_string(Base32, true)); + } + } + } + autoGC(); canonicalisePathMetaData(realPath, -1); @@ -1450,4 +1484,20 @@ void LocalStore::createUser(const std::string & userName, uid_t userId) } +std::string LocalStore::hashCAPath( + bool recursive, + const HashType & hashType, + const Path & path +) +{ + HashSink caSink(hashType); + if (recursive) { + dumpPath(path, caSink); + } else { + readFile(path, caSink); + } + auto hash = caSink.finish().first; + return makeFixedOutputCA(recursive, hash); +} + } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 379a06af8..0185d0ebf 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -295,8 +295,14 @@ private: void createUser(const std::string & userName, uid_t userId) override; - friend class DerivationGoal; - friend class SubstitutionGoal; + std::string hashCAPath( + bool recursive, + const HashType & hashType, + const Path & path + ); + + friend struct DerivationGoal; + friend struct SubstitutionGoal; }; diff --git a/tests/local.mk b/tests/local.mk index 187f96ea2..4c087ae39 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -12,6 +12,7 @@ nix_tests = \ timeout.sh secure-drv-outputs.sh nix-channel.sh \ multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \ binary-cache.sh nix-profile.sh repair.sh dump-db.sh case-hack.sh \ + substitute-with-invalid-ca.sh \ check-reqs.sh pass-as-file.sh tarball.sh restricted.sh \ placeholders.sh nix-shell.sh \ linux-sandbox.sh \ diff --git a/tests/substitute-with-invalid-ca.sh b/tests/substitute-with-invalid-ca.sh new file mode 100644 index 000000000..4d0b01e0f --- /dev/null +++ b/tests/substitute-with-invalid-ca.sh @@ -0,0 +1,38 @@ +source common.sh + +BINARY_CACHE=file://$cacheDir + +getHash() { + basename "$1" | cut -d '-' -f 1 +} +getRemoteNarInfo () { + echo "$cacheDir/$(getHash "$1").narinfo" +} + +cat < $TEST_HOME/good.txt +I’m a good path +EOF + +cat < $TEST_HOME/bad.txt +I’m a bad path +EOF + +good=$(nix-store --add $TEST_HOME/good.txt) +bad=$(nix-store --add $TEST_HOME/bad.txt) +nix copy --to "$BINARY_CACHE" "$good" +nix copy --to "$BINARY_CACHE" "$bad" +nix-collect-garbage >/dev/null 2>&1 + +# Falsifying the narinfo file for '$good' +goodPathNarInfo=$(getRemoteNarInfo "$good") +badPathNarInfo=$(getRemoteNarInfo "$bad") +for fieldName in URL FileHash FileSize NarHash NarSize; do + sed -i "/^$fieldName/d" "$goodPathNarInfo" + grep -E "^$fieldName" "$badPathNarInfo" >> "$goodPathNarInfo" +done + +# Copying back '$good' from the binary cache. This should fail as it is +# corrupted +if nix copy --from "$BINARY_CACHE" "$good"; then + fail "Importing a path with a wrong CA field should fail" +fi From 59acbc52208429c5e46c090acc6616f987a212c3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 1 Jun 2021 15:27:32 +0200 Subject: [PATCH 15/65] Fix evaluation on 20.03 --- release-common.nix | 2 +- release.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/release-common.nix b/release-common.nix index 98e2309ac..a33b71f93 100644 --- a/release-common.nix +++ b/release-common.nix @@ -57,7 +57,7 @@ rec { git mercurial ] - ++ lib.optionals stdenv.isLinux [libseccomp util-linuxMinimal] + ++ lib.optionals stdenv.isLinux [libseccomp (pkgs.util-linuxMinimal or pkgs.utillinuxMinimal)] ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) ((aws-sdk-cpp.override { diff --git a/release.nix b/release.nix index 2f7573ccf..ea936146e 100644 --- a/release.nix +++ b/release.nix @@ -1,5 +1,5 @@ { nix ? builtins.fetchGit ./. -, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz +, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-20.03.tar.gz , officialRelease ? false , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] }: From c9dcfa50f8401b75bb00dcd1428e94ba83b12e83 Mon Sep 17 00:00:00 2001 From: Patrick Hilhorst Date: Thu, 27 May 2021 21:48:39 +0200 Subject: [PATCH 16/65] throw freenode down the memory hole (cherry picked from commit 822e338e5ccee59963f50c876998c2e55ee8caa7) --- scripts/install-multi-user.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 349bffa45..96c19890f 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -63,7 +63,8 @@ contactme() { echo "If you can, open an issue at https://github.com/nixos/nix/issues" echo "" echo "Or feel free to contact the team," - echo " - on IRC #nixos on irc.freenode.net" + echo " - on Matrix #nix:nixos.org" + echo " - on IRC #nixos on irc.libera.chat" echo " - on twitter @nixos_org" } From a25f0a87303a51eacb1011f5d6705c0e4741b253 Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Wed, 2 Jun 2021 08:12:09 +0000 Subject: [PATCH 17/65] install-multi-user.sh: add link to forum Introduced in master in eab14a642cbcbc35f4473888d906f9de7deda07b. --- scripts/install-multi-user.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 96c19890f..5db97cd2a 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -66,6 +66,7 @@ contactme() { echo " - on Matrix #nix:nixos.org" echo " - on IRC #nixos on irc.libera.chat" echo " - on twitter @nixos_org" + echo " - on our forum https://discourse.nixos.org/" } uninstall_directions() { From fe679310a7129f68d7f63b65fdf662db8b6a4fc5 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 1 Nov 2019 13:27:40 -0400 Subject: [PATCH 18/65] Pass --static flag to pkg-config when necessary (cherry picked from commit 3e85c57a6cbf46d5f0fe8a89b368a43abd26daba) --- configure.ac | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/configure.ac b/configure.ac index 44a86c248..e417ebadf 100644 --- a/configure.ac +++ b/configure.ac @@ -158,6 +158,18 @@ AX_BOOST_BASE([1.66], [CXXFLAGS="$BOOST_CPPFLAGS $CXXFLAGS"], [AC_MSG_ERROR([Nix LDFLAGS="$BOOST_LDFLAGS $LDFLAGS" +PKG_PROG_PKG_CONFIG + +AC_ARG_ENABLE(shared, AC_HELP_STRING([--enable-shared], + [Build shared libraries for Nix [default=yes]]), + shared=$enableval, shared=yes) +if test "$shared" = yes; then + AC_SUBST(BUILD_SHARED_LIBS, 1, [Whether to build shared libraries.]) +else + AC_SUBST(BUILD_SHARED_LIBS, 0, [Whether to build shared libraries.]) + PKG_CONFIG="$PKG_CONFIG --static" +fi + # Look for OpenSSL, a required dependency. PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) @@ -291,16 +303,6 @@ AC_ARG_WITH(sandbox-shell, AC_HELP_STRING([--with-sandbox-shell=PATH], sandbox_shell=$withval) AC_SUBST(sandbox_shell) -AC_ARG_ENABLE(shared, AC_HELP_STRING([--enable-shared], - [Build shared libraries for Nix [default=yes]]), - shared=$enableval, shared=yes) -if test "$shared" = yes; then - AC_SUBST(BUILD_SHARED_LIBS, 1, [Whether to build shared libraries.]) -else - AC_SUBST(BUILD_SHARED_LIBS, 0, [Whether to build shared libraries.]) -fi - - # Expand all variables in config.status. test "$prefix" = NONE && prefix=$ac_default_prefix test "$exec_prefix" = NONE && exec_prefix='${prefix}' From 7375b0b58c4e81a42b33ed97fd96a1002ef1c752 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 1 Oct 2019 21:07:07 -0400 Subject: [PATCH 19/65] Add libatomic for 32-bit ARM Fixes #3113 (cherry picked from commit 74b4737d8f0e1922ef5314a158271acf81cd79f8) --- configure.ac | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configure.ac b/configure.ac index e417ebadf..ff6866c3e 100644 --- a/configure.ac +++ b/configure.ac @@ -157,6 +157,11 @@ AX_BOOST_BASE([1.66], [CXXFLAGS="$BOOST_CPPFLAGS $CXXFLAGS"], [AC_MSG_ERROR([Nix # ends up with LDFLAGS being empty, so we set it afterwards. LDFLAGS="$BOOST_LDFLAGS $LDFLAGS" +# Boost atomic needs GCC libatomic on 32-bit ARM +case "$host_cpu" in + armv5*|armv6*|armv7*) LIBS="-latomic $LIBS" +esac + PKG_PROG_PKG_CONFIG From f1688f213640fe3b3d4252d61dbd7ad61a7a119c Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 1 Oct 2019 21:22:18 -0400 Subject: [PATCH 20/65] Use more robust test for libatomics Taken from Mesa configure script: https://github.com/mesa3d/mesa/blob/17.2/configure.ac#L405-L427 (cherry picked from commit b1c34152feac76c9baded58732068541b7a586f1) --- configure.ac | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/configure.ac b/configure.ac index ff6866c3e..8e3e0e5ce 100644 --- a/configure.ac +++ b/configure.ac @@ -157,11 +157,18 @@ AX_BOOST_BASE([1.66], [CXXFLAGS="$BOOST_CPPFLAGS $CXXFLAGS"], [AC_MSG_ERROR([Nix # ends up with LDFLAGS being empty, so we set it afterwards. LDFLAGS="$BOOST_LDFLAGS $LDFLAGS" -# Boost atomic needs GCC libatomic on 32-bit ARM -case "$host_cpu" in - armv5*|armv6*|armv7*) LIBS="-latomic $LIBS" -esac - +# On some platforms, new-style atomics need a helper library +AC_MSG_CHECKING(whether -latomic is needed) +AC_LINK_IFELSE([AC_LANG_SOURCE([[ +#include +uint64_t v; +int main() { + return (int)__atomic_load_n(&v, __ATOMIC_ACQUIRE); +}]])], GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=no, GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=yes) +AC_MSG_RESULT($GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC) +if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then + LIBS="-latomic $LIBS" +fi PKG_PROG_PKG_CONFIG From feb826725777dc0145910978fa98e60c3d00c115 Mon Sep 17 00:00:00 2001 From: Puck Meerburg Date: Fri, 4 Jun 2021 11:25:36 +0000 Subject: [PATCH 21/65] configure.ac: fix use of unread LIBS variable This fixes both the SunOS/Solaris check, and the libatomic check, which reference $LIBS, which has not been used since automake was stripped out of the code. (cherry picked from commit 196b77b686ab407e4e01fb3ecde07a5f199f4045) --- configure.ac | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 8e3e0e5ce..b34e231c2 100644 --- a/configure.ac +++ b/configure.ac @@ -69,7 +69,7 @@ AC_SYS_LARGEFILE AC_STRUCT_DIRENT_D_TYPE if test "$sys_name" = sunos; then # Solaris requires -lsocket -lnsl for network functions - LIBS="-lsocket -lnsl $LIBS" + LDFLAGS="-lsocket -lnsl $LDFLAGS" fi @@ -167,7 +167,7 @@ int main() { }]])], GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=no, GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=yes) AC_MSG_RESULT($GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC) if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then - LIBS="-latomic $LIBS" + LDFLAGS="-latomic $LDFLAGS" fi PKG_PROG_PKG_CONFIG From 29e8db916acb49356eae3720fcc3a72a7858a307 Mon Sep 17 00:00:00 2001 From: Finn Behrens Date: Tue, 1 Jun 2021 10:11:50 +0200 Subject: [PATCH 22/65] enable aarch64-darwin in release.nix --- release.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/release.nix b/release.nix index ea936146e..ad09d46d9 100644 --- a/release.nix +++ b/release.nix @@ -1,7 +1,7 @@ { nix ? builtins.fetchGit ./. -, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-20.03.tar.gz +, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-21.05-small.tar.gz , officialRelease ? false -, systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] +, systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin" ] }: let @@ -315,7 +315,7 @@ let substitute ${./scripts/install.in} $out/install \ ${pkgs.lib.concatMapStrings (system: "--replace '@binaryTarball_${system}@' $(nix hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.xz) ") - [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] + [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-darwin" "aarch64-linux" ] } \ --replace '@nixVersion@' ${build.x86_64-linux.src.version} From 9f7a59af71939e39a6b5bb8fd22c3e74778f3df1 Mon Sep 17 00:00:00 2001 From: Lucas Franceschino Date: Mon, 25 May 2020 19:07:38 +0200 Subject: [PATCH 23/65] Make `functionArgs` primitive accept primops (cherry picked from commit b2748c6e99239ff6803ba0da76c362790c8be192) --- src/libexpr/primops.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index d4c60f870..9a04432e0 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1350,6 +1350,10 @@ static void prim_catAttrs(EvalState & state, const Pos & pos, Value * * args, Va static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0]); + if (args[0]->type == tPrimOpApp || args[0]->type == tPrimOp) { + state.mkAttrs(v, 0); + return; + } if (args[0]->type != tLambda) throw TypeError(format("'functionArgs' requires a function, at %1%") % pos); From 042718a29388e84e785dec460438442b3bd03f53 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Jun 2021 17:02:08 +0200 Subject: [PATCH 24/65] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index b57013902..da924f5d1 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.3.12 \ No newline at end of file +2.3.13 \ No newline at end of file From 78e7d7cfab56ca839797f3195de5a892d70f2660 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Tue, 22 Jun 2021 10:45:00 +0200 Subject: [PATCH 25/65] Fix perlBindings.aarch64-darwin build --- release.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release.nix b/release.nix index ad09d46d9..ce29bf516 100644 --- a/release.nix +++ b/release.nix @@ -108,7 +108,8 @@ let buildInputs = [ jobs.build.${system} curl bzip2 xz pkgconfig pkgs.perl boost ] - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium; + ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium + ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security; configureFlags = '' --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix} From 50b8ef40cbd9d60a814a02758cff522a2ebe1c77 Mon Sep 17 00:00:00 2001 From: aszlig Date: Sat, 17 Oct 2020 22:08:18 +0200 Subject: [PATCH 26/65] Convert VM tests to Python MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Perl-based tests are deprecated since NixOS 20.03 and subsequently got removed in NixOS 20.09, which effectively means that tests are going to fail as soon as we build it with NixOS 20.09 or anything newer. I've put "# fmt: off" at the start of every testScript, because formatting with Black really messes up indentation and I don't think it really adds anything in value or readability for inlined Python scripts. Signed-off-by: aszlig (cherry picked from commit 5cfdf16dd62dcfa2eb32b4e9ae390cb99d683907) Signed-off-by: Domen Kožar --- release.nix | 2 +- tests/nix-copy-closure.nix | 71 ++++++++++-------- tests/remote-builds.nix | 72 +++++++++--------- tests/setuid.nix | 147 ++++++++++++++++++++----------------- 4 files changed, 159 insertions(+), 133 deletions(-) diff --git a/release.nix b/release.nix index ce29bf516..9d12299c7 100644 --- a/release.nix +++ b/release.nix @@ -256,7 +256,7 @@ let tests.binaryTarball = with import nixpkgs { system = "x86_64-linux"; }; vmTools.runInLinuxImage (runCommand "nix-binary-tarball-test" - { diskImage = vmTools.diskImages.ubuntu1204x86_64; + { diskImage = vmTools.diskImages.ubuntu2004x86_64; } '' set -x diff --git a/tests/nix-copy-closure.nix b/tests/nix-copy-closure.nix index 0dc147fb3..662fae60b 100644 --- a/tests/nix-copy-closure.nix +++ b/tests/nix-copy-closure.nix @@ -2,7 +2,9 @@ { nixpkgs, system, nix }: -with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; }; +with import (nixpkgs + "/nixos/lib/testing-python.nix") { + inherit system; +}; makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in { @@ -24,41 +26,46 @@ makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in { }; }; - testScript = { nodes }: - '' - startAll; + testScript = { nodes }: '' + # fmt: off + import subprocess - # Create an SSH key on the client. - my $key = `${pkgs.openssh}/bin/ssh-keygen -t ed25519 -f key -N ""`; - $client->succeed("mkdir -m 700 /root/.ssh"); - $client->copyFileFromHost("key", "/root/.ssh/id_ed25519"); - $client->succeed("chmod 600 /root/.ssh/id_ed25519"); + start_all() - # Install the SSH key on the server. - $server->succeed("mkdir -m 700 /root/.ssh"); - $server->copyFileFromHost("key.pub", "/root/.ssh/authorized_keys"); - $server->waitForUnit("sshd"); - $client->waitForUnit("network.target"); - $client->succeed("ssh -o StrictHostKeyChecking=no " . $server->name() . " 'echo hello world'"); + # Create an SSH key on the client. + subprocess.run([ + "${pkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" + ], capture_output=True, check=True) - # Copy the closure of package A from the client to the server. - $server->fail("nix-store --check-validity ${pkgA}"); - $client->succeed("nix-copy-closure --to server --gzip ${pkgA} >&2"); - $server->succeed("nix-store --check-validity ${pkgA}"); + client.succeed("mkdir -m 700 /root/.ssh") + client.copy_from_host("key", "/root/.ssh/id_ed25519") + client.succeed("chmod 600 /root/.ssh/id_ed25519") - # Copy the closure of package B from the server to the client. - $client->fail("nix-store --check-validity ${pkgB}"); - $client->succeed("nix-copy-closure --from server --gzip ${pkgB} >&2"); - $client->succeed("nix-store --check-validity ${pkgB}"); + # Install the SSH key on the server. + server.succeed("mkdir -m 700 /root/.ssh") + server.copy_from_host("key.pub", "/root/.ssh/authorized_keys") + server.wait_for_unit("sshd") + client.wait_for_unit("network.target") + client.succeed(f"ssh -o StrictHostKeyChecking=no {server.name} 'echo hello world'") - # Copy the closure of package C via the SSH substituter. - $client->fail("nix-store -r ${pkgC}"); - # FIXME - #$client->succeed( - # "nix-store --option use-ssh-substituter true" - # . " --option ssh-substituter-hosts root\@server" - # . " -r ${pkgC} >&2"); - #$client->succeed("nix-store --check-validity ${pkgC}"); - ''; + # Copy the closure of package A from the client to the server. + server.fail("nix-store --check-validity ${pkgA}") + client.succeed("nix-copy-closure --to server --gzip ${pkgA} >&2") + server.succeed("nix-store --check-validity ${pkgA}") + # Copy the closure of package B from the server to the client. + client.fail("nix-store --check-validity ${pkgB}") + client.succeed("nix-copy-closure --from server --gzip ${pkgB} >&2") + client.succeed("nix-store --check-validity ${pkgB}") + + # Copy the closure of package C via the SSH substituter. + client.fail("nix-store -r ${pkgC}") + # FIXME + # client.succeed( + # "nix-store --option use-ssh-substituter true" + # " --option ssh-substituter-hosts root\@server" + # " -r ${pkgC} >&2" + # ) + # client.succeed("nix-store --check-validity ${pkgC}") + ''; }) diff --git a/tests/remote-builds.nix b/tests/remote-builds.nix index b867f13b4..2cf51a1df 100644 --- a/tests/remote-builds.nix +++ b/tests/remote-builds.nix @@ -2,7 +2,9 @@ { nixpkgs, system, nix }: -with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; }; +with import (nixpkgs + "/nixos/lib/testing-python.nix") { + inherit system; +}; makeTest ( @@ -65,44 +67,46 @@ in }; }; - testScript = { nodes }: - '' - startAll; + testScript = { nodes }: '' + # fmt: off + import subprocess - # Create an SSH key on the client. - my $key = `${pkgs.openssh}/bin/ssh-keygen -t ed25519 -f key -N ""`; - $client->succeed("mkdir -p -m 700 /root/.ssh"); - $client->copyFileFromHost("key", "/root/.ssh/id_ed25519"); - $client->succeed("chmod 600 /root/.ssh/id_ed25519"); + start_all() - # Install the SSH key on the builders. - $client->waitForUnit("network.target"); - foreach my $builder ($builder1, $builder2) { - $builder->succeed("mkdir -p -m 700 /root/.ssh"); - $builder->copyFileFromHost("key.pub", "/root/.ssh/authorized_keys"); - $builder->waitForUnit("sshd"); - $client->succeed("ssh -o StrictHostKeyChecking=no " . $builder->name() . " 'echo hello world'"); - } + # Create an SSH key on the client. + subprocess.run([ + "${pkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" + ], capture_output=True, check=True) + client.succeed("mkdir -p -m 700 /root/.ssh") + client.copy_from_host("key", "/root/.ssh/id_ed25519") + client.succeed("chmod 600 /root/.ssh/id_ed25519") - # Perform a build and check that it was performed on the builder. - my $out = $client->succeed( - "nix-build ${expr nodes.client.config 1} 2> build-output", - "grep -q Hello build-output" - ); - $builder1->succeed("test -e $out"); + # Install the SSH key on the builders. + client.wait_for_unit("network.target") + for builder in [builder1, builder2]: + builder.succeed("mkdir -p -m 700 /root/.ssh") + builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys") + builder.wait_for_unit("sshd") + client.succeed(f"ssh -o StrictHostKeyChecking=no {builder.name} 'echo hello world'") - # And a parallel build. - my ($out1, $out2) = split /\s/, - $client->succeed('nix-store -r $(nix-instantiate ${expr nodes.client.config 2})\!out $(nix-instantiate ${expr nodes.client.config 3})\!out'); - $builder1->succeed("test -e $out1 -o -e $out2"); - $builder2->succeed("test -e $out1 -o -e $out2"); + # Perform a build and check that it was performed on the builder. + out = client.succeed( + "nix-build ${expr nodes.client.config 1} 2> build-output", + "grep -q Hello build-output" + ) + builder1.succeed(f"test -e {out}") - # And a failing build. - $client->fail("nix-build ${expr nodes.client.config 5}"); + # And a parallel build. + paths = client.succeed(r'nix-store -r $(nix-instantiate ${expr nodes.client.config 2})\!out $(nix-instantiate ${expr nodes.client.config 3})\!out') + out1, out2 = paths.split() + builder1.succeed(f"test -e {out1} -o -e {out2}") + builder2.succeed(f"test -e {out1} -o -e {out2}") - # Test whether the build hook automatically skips unavailable builders. - $builder1->block; - $client->succeed("nix-build ${expr nodes.client.config 4}"); - ''; + # And a failing build. + client.fail("nix-build ${expr nodes.client.config 5}") + # Test whether the build hook automatically skips unavailable builders. + builder1.block() + client.succeed("nix-build ${expr nodes.client.config 4}") + ''; }) diff --git a/tests/setuid.nix b/tests/setuid.nix index 77e83c8d6..a293666bd 100644 --- a/tests/setuid.nix +++ b/tests/setuid.nix @@ -15,94 +15,109 @@ makeTest { virtualisation.pathsInNixDB = [ pkgs.stdenv pkgs.pkgsi686Linux.stdenv ]; }; - testScript = { nodes }: - '' - startAll; + testScript = { nodes }: '' + # fmt: off + start_all() - # Copying to /tmp should succeed. - $machine->succeed('nix-build --no-sandbox -E \'(with import {}; runCommand "foo" {} " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - ")\' '); + # Copying to /tmp should succeed. + machine.succeed(r""" + nix-build --no-sandbox -E '(with import {}; runCommand "foo" {} " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + ")' + """.strip()) - $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]'); + machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]') - $machine->succeed("rm /tmp/id"); + machine.succeed("rm /tmp/id") - # Creating a setuid binary should fail. - $machine->fail('nix-build --no-sandbox -E \'(with import {}; runCommand "foo" {} " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - chmod 4755 /tmp/id - ")\' '); + # Creating a setuid binary should fail. + machine.fail(r""" + nix-build --no-sandbox -E '(with import {}; runCommand "foo" {} " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + chmod 4755 /tmp/id + ")' + """.strip()) - $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]'); + machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]') - $machine->succeed("rm /tmp/id"); + machine.succeed("rm /tmp/id") - # Creating a setgid binary should fail. - $machine->fail('nix-build --no-sandbox -E \'(with import {}; runCommand "foo" {} " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - chmod 2755 /tmp/id - ")\' '); + # Creating a setgid binary should fail. + machine.fail(r""" + nix-build --no-sandbox -E '(with import {}; runCommand "foo" {} " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + chmod 2755 /tmp/id + ")' + """.strip()) - $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]'); + machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]') - $machine->succeed("rm /tmp/id"); + machine.succeed("rm /tmp/id") - # The checks should also work on 32-bit binaries. - $machine->fail('nix-build --no-sandbox -E \'(with import { system = "i686-linux"; }; runCommand "foo" {} " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - chmod 2755 /tmp/id - ")\' '); + # The checks should also work on 32-bit binaries. + machine.fail(r""" + nix-build --no-sandbox -E '(with import { system = "i686-linux"; }; runCommand "foo" {} " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + chmod 2755 /tmp/id + ")' + """.strip()) - $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]'); + machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]') - $machine->succeed("rm /tmp/id"); + machine.succeed("rm /tmp/id") - # The tests above use fchmodat(). Test chmod() as well. - $machine->succeed('nix-build --no-sandbox -E \'(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - perl -e \"chmod 0666, qw(/tmp/id) or die\" - ")\' '); + # The tests above use fchmodat(). Test chmod() as well. + machine.succeed(r""" + nix-build --no-sandbox -E '(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + perl -e \"chmod 0666, qw(/tmp/id) or die\" + ")' + """.strip()) - $machine->succeed('[[ $(stat -c %a /tmp/id) = 666 ]]'); + machine.succeed('[[ $(stat -c %a /tmp/id) = 666 ]]') - $machine->succeed("rm /tmp/id"); + machine.succeed("rm /tmp/id") - $machine->fail('nix-build --no-sandbox -E \'(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - perl -e \"chmod 04755, qw(/tmp/id) or die\" - ")\' '); + machine.fail(r""" + nix-build --no-sandbox -E '(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + perl -e \"chmod 04755, qw(/tmp/id) or die\" + ")' + """.strip()) - $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]'); + machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]') - $machine->succeed("rm /tmp/id"); + machine.succeed("rm /tmp/id") - # And test fchmod(). - $machine->succeed('nix-build --no-sandbox -E \'(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - perl -e \"my \\\$x; open \\\$x, qw(/tmp/id); chmod 01750, \\\$x or die\" - ")\' '); + # And test fchmod(). + machine.succeed(r""" + nix-build --no-sandbox -E '(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + perl -e \"my \\\$x; open \\\$x, qw(/tmp/id); chmod 01750, \\\$x or die\" + ")' + """.strip()) - $machine->succeed('[[ $(stat -c %a /tmp/id) = 1750 ]]'); + machine.succeed('[[ $(stat -c %a /tmp/id) = 1750 ]]') - $machine->succeed("rm /tmp/id"); + machine.succeed("rm /tmp/id") - $machine->fail('nix-build --no-sandbox -E \'(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " - mkdir -p $out - cp ${pkgs.coreutils}/bin/id /tmp/id - perl -e \"my \\\$x; open \\\$x, qw(/tmp/id); chmod 04777, \\\$x or die\" - ")\' '); + machine.fail(r""" + nix-build --no-sandbox -E '(with import {}; runCommand "foo" { buildInputs = [ perl ]; } " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + perl -e \"my \\\$x; open \\\$x, qw(/tmp/id); chmod 04777, \\\$x or die\" + ")' + """.strip()) - $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]'); - - $machine->succeed("rm /tmp/id"); - ''; + machine.succeed('[[ $(stat -c %a /tmp/id) = 555 ]]') + machine.succeed("rm /tmp/id") + ''; } From 165e44fef9b924d9f2294fc265b06f878e629b03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Wed, 23 Jun 2021 09:37:14 +0200 Subject: [PATCH 27/65] fix setuid test eval --- tests/setuid.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/setuid.nix b/tests/setuid.nix index a293666bd..b3b8fe9f1 100644 --- a/tests/setuid.nix +++ b/tests/setuid.nix @@ -2,7 +2,7 @@ { nixpkgs, system, nix }: -with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; }; +with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; }; makeTest { From a49bddb0400b1a40b8abd66877a88781a1091ebb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Nov 2020 15:26:39 +0100 Subject: [PATCH 28/65] Remove tests.binaryTarball This test no longer works on Hydra because import-from-derivation is no longer allowed. (cherry picked from commit 3daa256728e29f9059a79b8ecaf9325e52f1704a) --- release.nix | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/release.nix b/release.nix index 9d12299c7..072d0822d 100644 --- a/release.nix +++ b/release.nix @@ -253,37 +253,6 @@ let nix = build.${system}; inherit system; }); - tests.binaryTarball = - with import nixpkgs { system = "x86_64-linux"; }; - vmTools.runInLinuxImage (runCommand "nix-binary-tarball-test" - { diskImage = vmTools.diskImages.ubuntu2004x86_64; - } - '' - set -x - useradd -m alice - su - alice -c 'tar xf ${binaryTarball.x86_64-linux}/*.tar.*' - mkdir /dest-nix - mount -o bind /dest-nix /nix # Provide a writable /nix. - chown alice /nix - su - alice -c '_NIX_INSTALLER_TEST=1 ./nix-*/install' - su - alice -c 'nix-store --verify' - su - alice -c 'PAGER= nix-store -qR ${build.x86_64-linux}' - - # Check whether 'nix upgrade-nix' works. - cat > /tmp/paths.nix < Date: Wed, 23 Jun 2021 11:41:06 +0200 Subject: [PATCH 29/65] Remove Deb/RPM jobs --- release.nix | 61 ----------------------------------------------------- 1 file changed, 61 deletions(-) diff --git a/release.nix b/release.nix index 072d0822d..3c975c51e 100644 --- a/release.nix +++ b/release.nix @@ -224,16 +224,6 @@ let }; - #rpm_fedora27x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora27x86_64) [ ]; - - - #deb_debian8i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian8i386) [ "libsodium-dev" ] [ "libsodium13" ]; - #deb_debian8x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian8x86_64) [ "libsodium-dev" ] [ "libsodium13" ]; - - #deb_ubuntu1710i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1710i386) [ ] [ "libsodium18" ]; - #deb_ubuntu1710x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1710x86_64) [ ] [ "libsodium18" "libboost-context1.62.0" ]; - - # System tests. tests.remoteBuilds = (import ./tests/remote-builds.nix rec { inherit nixpkgs; @@ -295,55 +285,4 @@ let }; - makeRPM_i686 = makeRPM "i686-linux"; - makeRPM_x86_64 = makeRPM "x86_64-linux"; - - makeRPM = - system: diskImageFun: extraPackages: - - with import nixpkgs { inherit system; }; - - releaseTools.rpmBuild rec { - name = "nix-rpm"; - src = jobs.tarball; - diskImage = (diskImageFun vmTools.diskImageFuns) - { extraPackages = - [ "sqlite" "sqlite-devel" "bzip2-devel" "libcurl-devel" "openssl-devel" "xz-devel" "libseccomp-devel" "libsodium-devel" "boost-devel" "bison" "flex" ] - ++ extraPackages; }; - # At most 2047MB can be simulated in qemu-system-i386 - memSize = 2047; - meta.schedulingPriority = 50; - postRPMInstall = "cd /tmp/rpmout/BUILD/nix-* && make installcheck"; - #enableParallelBuilding = true; - }; - - - makeDeb_i686 = makeDeb "i686-linux"; - makeDeb_x86_64 = makeDeb "x86_64-linux"; - - makeDeb = - system: diskImageFun: extraPackages: extraDebPackages: - - with import nixpkgs { inherit system; }; - - releaseTools.debBuild { - name = "nix-deb"; - src = jobs.tarball; - diskImage = (diskImageFun vmTools.diskImageFuns) - { extraPackages = - [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" "libseccomp-dev" "libsodium-dev" "libboost-all-dev" ] - ++ extraPackages; }; - memSize = 2047; - meta.schedulingPriority = 50; - postInstall = "make installcheck"; - configureFlags = "--sysconfdir=/etc"; - debRequires = - [ "curl" "libsqlite3-0" "libbz2-1.0" "bzip2" "xz-utils" "libssl1.0.0" "liblzma5" "libseccomp2" ] - ++ extraDebPackages; - debMaintainer = "Eelco Dolstra "; - doInstallCheck = true; - #enableParallelBuilding = true; - }; - - in jobs From eda79305b67601176a31225b103edaed3b6d26c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Fri, 2 Jul 2021 14:08:01 +0200 Subject: [PATCH 30/65] installer: use native aarch64-darwin --- scripts/install.in | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/install.in b/scripts/install.in index 9a281d776..5c7d1028c 100644 --- a/scripts/install.in +++ b/scripts/install.in @@ -29,8 +29,7 @@ case "$(uname -s).$(uname -m)" in Linux.i?86) system=i686-linux; hash=@binaryTarball_i686-linux@;; Linux.aarch64) system=aarch64-linux; hash=@binaryTarball_aarch64-linux@;; Darwin.x86_64) system=x86_64-darwin; hash=@binaryTarball_x86_64-darwin@;; - # eventually maybe: system=arm64-darwin; hash=@binaryTarball_arm64-darwin@;; - Darwin.arm64) system=x86_64-darwin; hash=@binaryTarball_x86_64-darwin@;; + Darwin.arm64) system=aarch64-darwin; hash=@binaryTarball_aarch64-darwin@;; *) oops "sorry, there is no binary distribution of Nix for your platform";; esac From 14262b86cc5825deae095c14553d623af498124c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Fri, 2 Jul 2021 14:13:26 +0200 Subject: [PATCH 31/65] upload-release.pl: add aarch64-darwin --- maintainers/upload-release.pl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 1cdf5ed16..6c5707921 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -89,6 +89,7 @@ downloadFile("binaryTarball.i686-linux", "1"); downloadFile("binaryTarball.x86_64-linux", "1"); downloadFile("binaryTarball.aarch64-linux", "1"); downloadFile("binaryTarball.x86_64-darwin", "1"); +downloadFile("binaryTarball.aarch64-darwin", "1"); downloadFile("installerScript", "1"); exit if $version =~ /pre/; @@ -121,6 +122,7 @@ write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix", " i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" . " aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" . " x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" . + " aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" . "}\n"); system("cd $nixpkgsDir && git commit -a -m 'nix: $oldName -> $version'") == 0 or die; From f1e3a6a50ac9fe3b21793ddca1701e4dc67672b2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 2 Jul 2021 15:00:14 +0200 Subject: [PATCH 32/65] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index da924f5d1..ac805d3e0 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.3.13 \ No newline at end of file +2.3.14 \ No newline at end of file From ea962a84c35cb260e5feee5c5ba114f0db171055 Mon Sep 17 00:00:00 2001 From: toonn Date: Mon, 5 Jul 2021 13:53:15 +0200 Subject: [PATCH 33/65] doc: Change install syntax to be fish compatible --- doc/manual/installation/installing-binary.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/manual/installation/installing-binary.xml b/doc/manual/installation/installing-binary.xml index 64c7a37fb..c6d3ee768 100644 --- a/doc/manual/installation/installing-binary.xml +++ b/doc/manual/installation/installing-binary.xml @@ -12,7 +12,7 @@ - $ sh <(curl -L https://nixos.org/nix/install) + $ curl -L https://nixos.org/nix/install | sh -s @@ -39,7 +39,7 @@ To explicitly select a single-user installation on your system: - sh <(curl -L https://nixos.org/nix/install) --no-daemon + curl -L https://nixos.org/nix/install | sh -s -- --no-daemon @@ -97,7 +97,7 @@ $ rm -rf /nix installation on your system: - sh <(curl -L https://nixos.org/nix/install) --daemon + curl -L https://nixos.org/nix/install | sh -s -- --daemon The multi-user installation of Nix will create build users between @@ -178,7 +178,7 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist is a bit of a misnomer). To use this approach, just install Nix with: - $ sh <(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume + $ curl -L https://nixos.org/nix/install | sh -s -- --darwin-use-unencrypted-nix-store-volume If you don't like the sound of this, you'll want to weigh the @@ -429,7 +429,7 @@ LABEL=Nix\040Store /nix apfs rw,nobrowse NixOS.org installation script: - sh <(curl -L https://nixos.org/nix/install) + curl -L https://nixos.org/nix/install | sh -s From 405d78eafcdf76dbf7189858fe0c4ef3e15301ca Mon Sep 17 00:00:00 2001 From: Emily Date: Sun, 2 May 2021 05:30:50 +0100 Subject: [PATCH 34/65] Mark `__impureHostDeps` paths as optional MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Starting in macOS 11, the on-disk dylib bundles are no longer available, but nixpkgs needs to be able to keep compatibility with older versions that require `/usr/lib/libSystem.B.dylib` in `__impureHostDeps`. Allow it to keep backwards compatibility with these versions by marking these dependencies as optional. Fixes #4658. (cherry picked from commit c4355a52fa317cb782fec5e22c4ac688f67d487d) Signed-off-by: Domen Kožar --- src/libstore/build.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index d09db82fc..3fe01270e 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2065,7 +2065,9 @@ void DerivationGoal::startBuilder() if (!found) throw Error(format("derivation '%1%' requested impure path '%2%', but it was not in allowed-impure-host-deps") % drvPath % i); - dirsInChroot[i] = i; + /* Allow files in __impureHostDeps to be missing; e.g. + macOS 11+ has no /usr/lib/libSystem*.dylib */ + dirsInChroot[i] = {i, true}; } #if __linux__ From 76c7f5f215f8118ad58e5a4da50ef4c9f98b27ad Mon Sep 17 00:00:00 2001 From: Emily Date: Mon, 17 May 2021 00:15:30 +0100 Subject: [PATCH 35/65] sandbox: allow SystemVersionCompat.plist on Darwin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For whatever reason, many programs trying to access SystemVersion.plist also open SystemVersionCompat.plist; this includes Python code and coreutils’ `cat(1)` (but not the native macOS `/bin/cat`). Illustratory `dtruss(1m)` output: open("/System/Library/CoreServices/SystemVersion.plist\0", 0x0, 0x0) = 3 0 open("/System/Library/CoreServices/SystemVersionCompat.plist\0", 0x0, 0x0) = 4 0 I assume this is a Big Sur change relating to the 10.16.x/11.x version compatibility divide and that it’s something along the lines of a hook inside libSystem. Fixes a lot of sandboxed package builds under Big Sur. (cherry picked from commit 559a504da7d4289badfb4ebf7906f80fbf50bedf) Signed-off-by: Domen Kožar --- src/libstore/sandbox-defaults.sb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libstore/sandbox-defaults.sb b/src/libstore/sandbox-defaults.sb index 351037822..2bb1ea130 100644 --- a/src/libstore/sandbox-defaults.sb +++ b/src/libstore/sandbox-defaults.sb @@ -32,7 +32,9 @@ (literal "/tmp") (subpath TMPDIR)) ; Some packages like to read the system version. -(allow file-read* (literal "/System/Library/CoreServices/SystemVersion.plist")) +(allow file-read* + (literal "/System/Library/CoreServices/SystemVersion.plist") + (literal "/System/Library/CoreServices/SystemVersionCompat.plist")) ; Without this line clang cannot write to /dev/null, breaking some configure tests. (allow file-read-metadata (literal "/dev")) From 1c51a169fb7d3e32cb8c4684353e57020f9d2019 Mon Sep 17 00:00:00 2001 From: illustris Date: Wed, 14 Jul 2021 14:26:25 +0530 Subject: [PATCH 36/65] fixed-output derivations: fix incorrect responses for getpwuid backport of #5006 --- src/libstore/build.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 3fe01270e..bf8914cc0 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2841,8 +2841,6 @@ void DerivationGoal::runChild() ss.push_back("/etc/services"); ss.push_back("/etc/hosts"); - if (pathExists("/var/run/nscd/socket")) - ss.push_back("/var/run/nscd/socket"); } for (auto & i : ss) dirsInChroot.emplace(i, i); From fa7cbf82221f81efb59fcba0ed0f80b7a2a2158a Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Mon, 17 May 2021 17:08:57 +0000 Subject: [PATCH 37/65] Only link with libdl on Linux Linux is (as far as I know) the only mainstream operating system that requires linking with libdl for dlopen. On BSD, libdl doesn't exist, so on non-FreeBSD BSDs linking will currently fail. On macOS, it's apparently just a symlink to libSystem (macOS libc), presumably present for compatibility with things that assume Linux. So the right thing to do here is to only add -ldl on Linux, not to add it for everything that isn't FreeBSD. (cherry picked from commit c57ab1768767ca4f860ce5ca5faf54a7dba4969f) --- src/libexpr/local.mk | 2 +- src/libstore/local.mk | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index ccd5293e4..00d6d75c1 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -9,7 +9,7 @@ libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexe libexpr_LIBS = libutil libstore libexpr_LDFLAGS = -ifneq ($(OS), FreeBSD) +ifeq ($(OS), Linux) libexpr_LDFLAGS += -ldl endif diff --git a/src/libstore/local.mk b/src/libstore/local.mk index d690fea28..bc665f2cb 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -9,7 +9,7 @@ libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc) libstore_LIBS = libutil libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread -ifneq ($(OS), FreeBSD) +ifeq ($(OS), Linux) libstore_LDFLAGS += -ldl endif From 89fba5c8473b8f8b2133698f1e1b9b14535c0bc0 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 26 Jul 2021 11:34:47 +0200 Subject: [PATCH 38/65] Use the real path when checking imported CA paths Otherwise chrooted store will fail because Nix will try to read the virtual path instead of the physical one Should fix https://github.com/NixOS/nixpkgs/issues/126141 --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 5b62d79ee..31a6cc46f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1038,7 +1038,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, auto actualFoHash = hashCAPath( recursive, expectedHash.type, - info.path + realPath ); if (ca != actualFoHash) { throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", From 12dc6427819f6e7abcdec93de6791d0efecf49a6 Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Tue, 1 Jun 2021 07:58:21 +0000 Subject: [PATCH 39/65] Apply OS checks to host platform, not build Previously, the build system used uname(1) output when it wanted to check the operating system it was being built for, which meant that it didn't take into-account cross-compilation when the build and host operating systems were different. To fix this, instead of consulting uname output, we consult the host triple, specifically the third "kernel" part. For "kernel"s with stable ABIs, like Linux or Cygwin, we can use a simple ifeq to test whether we're compiling for that system, but for other platforms, like Darwin, FreeBSD, or Solaris, we have to use a more complicated check to take into account the version numbers at the end of the "kernel"s. I couldn't find a way to just strip these version numbers in GNU Make without shelling out, which would be even more ugly IMO. Because these checks differ between kernels, and the patsubst ones are quite fiddly, I've added variables for each host OS we might want to check to make them easier to reuse. (cherry picked from commit 4f80464645e4c8e7ca9455fc53cc76dc50f688ed) --- Makefile.config.in | 1 + misc/launchd/local.mk | 2 +- misc/systemd/local.mk | 2 +- misc/upstart/local.mk | 2 +- mk/lib.mk | 27 +++++++++++++++++++----- mk/libraries.mk | 18 ++++++++-------- perl/Makefile.config.in | 1 + perl/configure.ac | 2 ++ perl/local.mk | 2 +- src/libexpr/local.mk | 2 +- src/libstore/local.mk | 4 ++-- src/resolve-system-dependencies/local.mk | 2 +- 12 files changed, 43 insertions(+), 22 deletions(-) diff --git a/Makefile.config.in b/Makefile.config.in index 7e3b35b98..588669f0a 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -1,3 +1,4 @@ +HOST_OS = @host_os@ AR = @AR@ BDW_GC_LIBS = @BDW_GC_LIBS@ BUILD_SHARED_LIBS = @BUILD_SHARED_LIBS@ diff --git a/misc/launchd/local.mk b/misc/launchd/local.mk index 0ba722efb..a39188fe6 100644 --- a/misc/launchd/local.mk +++ b/misc/launchd/local.mk @@ -1,4 +1,4 @@ -ifeq ($(OS), Darwin) +ifdef HOST_DARWIN $(eval $(call install-data-in, $(d)/org.nixos.nix-daemon.plist, $(prefix)/Library/LaunchDaemons)) diff --git a/misc/systemd/local.mk b/misc/systemd/local.mk index 004549fd2..7b8d0b4b8 100644 --- a/misc/systemd/local.mk +++ b/misc/systemd/local.mk @@ -1,4 +1,4 @@ -ifeq ($(OS), Linux) +ifdef HOST_LINUX $(foreach n, nix-daemon.socket nix-daemon.service, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/systemd/system, 0644))) diff --git a/misc/upstart/local.mk b/misc/upstart/local.mk index a73dc061e..b08e05fe2 100644 --- a/misc/upstart/local.mk +++ b/misc/upstart/local.mk @@ -1,4 +1,4 @@ -ifeq ($(OS), Linux) +ifdef HOST_LINUX $(foreach n, nix-daemon.conf, $(eval $(call install-file-in, $(d)/$(n), $(sysconfdir)/init, 0644))) diff --git a/mk/lib.mk b/mk/lib.mk index 1da51d879..064d37629 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -11,8 +11,25 @@ noinst-scripts := man-pages := install-tests := dist-files := -OS = $(shell uname -s) +ifdef HOST_OS + HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS))) + ifeq ($(HOST_KERNEL), cygwin) + HOST_CYGWIN = 1 + endif + ifeq ($(patsubst darwin%,,$(HOST_KERNEL)),) + HOST_DARWIN = 1 + endif + ifeq ($(patsubst freebsd%,,$(HOST_KERNEL)),) + HOST_FREEBSD = 1 + endif + ifeq ($(HOST_KERNEL), linux) + HOST_LINUX = 1 + endif + ifeq ($(patsubst solaris%,,$(HOST_KERNEL)),) + HOST_SOLARIS = 1 + endif +endif # Hack to define a literal space. space := @@ -52,16 +69,16 @@ endif BUILD_SHARED_LIBS ?= 1 ifeq ($(BUILD_SHARED_LIBS), 1) - ifeq (CYGWIN,$(findstring CYGWIN,$(OS))) + ifdef HOST_CYGWIN GLOBAL_CFLAGS += -U__STRICT_ANSI__ -D_GNU_SOURCE GLOBAL_CXXFLAGS += -U__STRICT_ANSI__ -D_GNU_SOURCE else GLOBAL_CFLAGS += -fPIC GLOBAL_CXXFLAGS += -fPIC endif - ifneq ($(OS), Darwin) - ifneq ($(OS), SunOS) - ifneq ($(OS), FreeBSD) + ifndef HOST_DARWIN + ifndef HOST_SOLARIS + ifndef HOST_FREEBSD GLOBAL_LDFLAGS += -Wl,--no-copy-dt-needed-entries endif endif diff --git a/mk/libraries.mk b/mk/libraries.mk index 307e29b9d..b8a518853 100644 --- a/mk/libraries.mk +++ b/mk/libraries.mk @@ -1,9 +1,9 @@ libs-list := -ifeq ($(OS), Darwin) +ifdef HOST_DARWIN SO_EXT = dylib else - ifeq (CYGWIN,$(findstring CYGWIN,$(OS))) + ifdef HOST_CYGWIN SO_EXT = dll else SO_EXT = so @@ -59,7 +59,7 @@ define build-library $(1)_OBJS := $$(addprefix $(buildprefix), $$(addsuffix .o, $$(basename $$(_srcs)))) _libs := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_PATH)) - ifeq (CYGWIN,$(findstring CYGWIN,$(OS))) + ifdef HOST_CYGWIN $(1)_INSTALL_DIR ?= $$(bindir) else $(1)_INSTALL_DIR ?= $$(libdir) @@ -73,18 +73,18 @@ define build-library ifeq ($(BUILD_SHARED_LIBS), 1) ifdef $(1)_ALLOW_UNDEFINED - ifeq ($(OS), Darwin) + ifdef HOST_DARWIN $(1)_LDFLAGS += -undefined suppress -flat_namespace endif else - ifneq ($(OS), Darwin) - ifneq (CYGWIN,$(findstring CYGWIN,$(OS))) + ifndef HOST_DARWIN + ifndef HOST_CYGWIN $(1)_LDFLAGS += -Wl,-z,defs endif endif endif - ifneq ($(OS), Darwin) + ifndef HOST_DARWIN $(1)_LDFLAGS += -Wl,-soname=$$($(1)_NAME).$(SO_EXT) endif @@ -93,7 +93,7 @@ define build-library $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/ $$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED) - ifneq ($(OS), Darwin) + ifndef HOST_DARWIN $(1)_LDFLAGS_USE += -Wl,-rpath,$$(abspath $$(_d)) endif $(1)_LDFLAGS_USE += -L$$(_d) -l$$(patsubst lib%,%,$$(strip $$($(1)_NAME))) @@ -108,7 +108,7 @@ define build-library $$(trace-ld) $(CXX) -o $$@ -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) $(1)_LDFLAGS_USE_INSTALLED += -L$$(DESTDIR)$$($(1)_INSTALL_DIR) -l$$(patsubst lib%,%,$$(strip $$($(1)_NAME))) - ifneq ($(OS), Darwin) + ifndef HOST_DARWIN ifeq ($(SET_RPATH_TO_LIBS), 1) $(1)_LDFLAGS_USE_INSTALLED += -Wl,-rpath,$$($(1)_INSTALL_DIR) else diff --git a/perl/Makefile.config.in b/perl/Makefile.config.in index c87d4817e..d08ee65db 100644 --- a/perl/Makefile.config.in +++ b/perl/Makefile.config.in @@ -1,3 +1,4 @@ +HOST_OS = @host_os@ CC = @CC@ CFLAGS = @CFLAGS@ CXX = @CXX@ diff --git a/perl/configure.ac b/perl/configure.ac index e8e3610a8..4ee6923a4 100644 --- a/perl/configure.ac +++ b/perl/configure.ac @@ -10,6 +10,8 @@ AC_PROG_CC AC_PROG_CXX AX_CXX_COMPILE_STDCXX_11 +AC_CANONICAL_HOST + # Use 64-bit file system calls so that we can support files > 2 GiB. AC_SYS_LARGEFILE diff --git a/perl/local.mk b/perl/local.mk index b13d4c0d6..0eae651d8 100644 --- a/perl/local.mk +++ b/perl/local.mk @@ -28,7 +28,7 @@ Store_CXXFLAGS = \ Store_LDFLAGS := $(SODIUM_LIBS) $(NIX_LIBS) -ifeq (CYGWIN,$(findstring CYGWIN,$(OS))) +ifdef HOST_CYGWIN archlib = $(shell perl -E 'use Config; print $$Config{archlib};') libperl = $(shell perl -E 'use Config; print $$Config{libperl};') Store_LDFLAGS += $(shell find ${archlib} -name ${libperl}) diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 00d6d75c1..e190022ff 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -9,7 +9,7 @@ libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexe libexpr_LIBS = libutil libstore libexpr_LDFLAGS = -ifeq ($(OS), Linux) +ifdef HOST_LINUX libexpr_LDFLAGS += -ldl endif diff --git a/src/libstore/local.mk b/src/libstore/local.mk index bc665f2cb..e9a9e804b 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -9,7 +9,7 @@ libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc) libstore_LIBS = libutil libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread -ifeq ($(OS), Linux) +ifdef HOST_LINUX libstore_LDFLAGS += -ldl endif @@ -21,7 +21,7 @@ ifeq ($(ENABLE_S3), 1) libstore_LDFLAGS += -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core endif -ifeq ($(OS), SunOS) +ifdef HOST_SOLARIS libstore_LDFLAGS += -lsocket endif diff --git a/src/resolve-system-dependencies/local.mk b/src/resolve-system-dependencies/local.mk index f9db16268..dbc041b23 100644 --- a/src/resolve-system-dependencies/local.mk +++ b/src/resolve-system-dependencies/local.mk @@ -1,4 +1,4 @@ -ifeq ($(OS), Darwin) +ifdef HOST_DARWIN programs += resolve-system-dependencies endif From 9d9dbe6ed05854e03811c361a3380e09183f4f4a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 28 Jul 2021 21:20:08 +0200 Subject: [PATCH 40/65] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index ac805d3e0..0007430cd 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.3.14 \ No newline at end of file +2.3.15 \ No newline at end of file From 183f2cc395772210f0e5f52d42fece9789412343 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 28 Jul 2021 22:54:47 +0200 Subject: [PATCH 41/65] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index 0007430cd..fabb09dc5 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.3.15 \ No newline at end of file +2.3.16 \ No newline at end of file From ac2ca8d083390152fa964993107ef93b5bd7079b Mon Sep 17 00:00:00 2001 From: Ryan Burns Date: Tue, 10 Aug 2021 00:01:39 -0700 Subject: [PATCH 42/65] Fix host OS detection for darwin-specific linker flag (cherry picked from commit d86d43c34cf8fcb68cdd67c581380aa21afe3ab6) --- configure.ac | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index b34e231c2..21c3a5ac1 100644 --- a/configure.ac +++ b/configure.ac @@ -294,9 +294,11 @@ AC_CHECK_FUNCS([strsignal posix_fallocate sysconf]) # This is needed if bzip2 is a static library, and the Nix libraries # are dynamic. -if test "$(uname)" = "Darwin"; then +case "${host_os}" in + darwin*) LDFLAGS="-all_load $LDFLAGS" -fi + ;; +esac # Do we have GNU tar? From bb985261230ff49ddfaf87e58b5ceddf3570477b Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Fri, 13 Aug 2021 08:19:43 +0000 Subject: [PATCH 43/65] configure.ac: remove another uname check uname checks are not cross-safe. The normalization for Cygwin doesn't need any equivalent for host_os because nothing actually checked whether sys_name was cygwin any more. (cherry picked from commit cff8fd69b6e9749d65d1b309228e22adea46d11d) --- configure.ac | 47 ++++++++++++++++++++++------------------------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/configure.ac b/configure.ac index 21c3a5ac1..824a838a0 100644 --- a/configure.ac +++ b/configure.ac @@ -33,14 +33,6 @@ AC_ARG_WITH(system, AC_HELP_STRING([--with-system=SYSTEM], system="$machine_name-`echo $host_os | "$SED" -e's/@<:@0-9.@:>@*$//g'`";; esac]) -sys_name=$(uname -s | tr 'A-Z ' 'a-z_') - -case $sys_name in - cygwin*) - sys_name=cygwin - ;; -esac - AC_MSG_RESULT($system) AC_SUBST(system) AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier ('cpu-os')]) @@ -67,10 +59,12 @@ AC_SYS_LARGEFILE # Solaris-specific stuff. AC_STRUCT_DIRENT_D_TYPE -if test "$sys_name" = sunos; then +case "$host_os" in + solaris*) # Solaris requires -lsocket -lnsl for network functions LDFLAGS="-lsocket -lnsl $LDFLAGS" -fi + ;; +esac # Check for pubsetbuf. @@ -231,22 +225,25 @@ PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTL # Look for libseccomp, required for Linux sandboxing. -if test "$sys_name" = linux; then - AC_ARG_ENABLE([seccomp-sandboxing], - AC_HELP_STRING([--disable-seccomp-sandboxing], - [Don't build support for seccomp sandboxing (only recommended if your arch doesn't support libseccomp yet!)] - )) - if test "x$enable_seccomp_sandboxing" != "xno"; then - PKG_CHECK_MODULES([LIBSECCOMP], [libseccomp], - [CXXFLAGS="$LIBSECCOMP_CFLAGS $CXXFLAGS"]) - have_seccomp=1 - AC_DEFINE([HAVE_SECCOMP], [1], [Whether seccomp is available and should be used for sandboxing.]) - else +case "$host_os" in + linux*) + AC_ARG_ENABLE([seccomp-sandboxing], + AC_HELP_STRING([--disable-seccomp-sandboxing], + [Don't build support for seccomp sandboxing (only recommended if your arch doesn't support libseccomp yet!)] + )) + if test "x$enable_seccomp_sandboxing" != "xno"; then + PKG_CHECK_MODULES([LIBSECCOMP], [libseccomp], + [CXXFLAGS="$LIBSECCOMP_CFLAGS $CXXFLAGS"]) + have_seccomp=1 + AC_DEFINE([HAVE_SECCOMP], [1], [Whether seccomp is available and should be used for sandboxing.]) + else + have_seccomp= + fi + ;; + *) have_seccomp= - fi -else - have_seccomp= -fi + ;; +esac AC_SUBST(HAVE_SECCOMP, [$have_seccomp]) From 2b665a311ea5e41bc3a34381dde1ee7cae93250c Mon Sep 17 00:00:00 2001 From: kvtb <76634406+kvtb@users.noreply.github.com> Date: Sun, 5 Sep 2021 14:42:06 +0000 Subject: [PATCH 44/65] hashFile, hashString: realize context before calculation, and discard afterwards --- src/libexpr/primops.cc | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 9a04432e0..8fd25ea15 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -937,10 +937,16 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va if (ht == htUnknown) throw Error(format("unknown hash type '%1%', at %2%") % type % pos); - PathSet context; // discarded - Path p = state.coerceToPath(pos, *args[1], context); + PathSet context; + Path path = state.coerceToPath(pos, *args[1], context); + try { + state.realiseContext(context); + } catch (InvalidPathError & e) { + throw EvalError(format("cannot read '%1%', since path '%2%' is not valid, at %3%") + % path % e.path % pos); + } - mkString(v, hashFile(ht, state.checkSourcePath(p)).to_string(Base16, false), context); + mkString(v, hashFile(ht, state.checkSourcePath(state.toRealPath(path, context))).to_string(Base16, false)); } /* Read a directory (without . or ..) */ @@ -1821,7 +1827,7 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, PathSet context; // discarded string s = state.forceString(*args[1], context, pos); - mkString(v, hashString(ht, s).to_string(Base16, false), context); + mkString(v, hashString(ht, s).to_string(Base16, false)); } From f8ccbb40d10288326e81612926891c68e1e69cfa Mon Sep 17 00:00:00 2001 From: kvtb <76634406+kvtb@users.noreply.github.com> Date: Mon, 6 Sep 2021 14:29:20 +0000 Subject: [PATCH 45/65] Expr::show(): avoid emiting meaningless chars --- src/libexpr/nixexpr.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 63cbef1dd..00bdbd47a 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -105,7 +105,7 @@ void ExprAttrs::show(std::ostream & str) const str << "{ "; for (auto & i : attrs) if (i.second.inherited) - str << "inherit " << i.first << " " << "; "; + str << "inherit " << i.first << "; "; else str << i.first << " = " << *i.second.e << "; "; for (auto & i : dynamicAttrs) @@ -211,7 +211,7 @@ string showAttrPath(const AttrPath & attrPath) if (i.symbol.set()) out << i.symbol; else - out << "\"${" << *i.expr << "}\""; + out << "${" << *i.expr << "}"; } return out.str(); } From 3d75d87bd3affc59e5d725f218b7cd4b3b38180b Mon Sep 17 00:00:00 2001 From: Arthur Gautier Date: Tue, 7 Sep 2021 01:49:37 +0000 Subject: [PATCH 46/65] preloadNSS: fixup nss_dns load Before this commit, the dns lookup in preloadNSS would still go through nscd. This did not have the effect of loading the nss_dns.so as expected (nss_dns.so being out of reach from within the sandbox). Should LOCALDOMAIN environment variable be defined, nss will completely avoid nscd and will do its dns resolution on its own. By temporarly setting LOCALDOMAIN variable before calling in NSS, we can force NSS to load the shared libraries as expected. Signed-off-by: Arthur Gautier --- src/libstore/build.cc | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index bf8914cc0..8e40d51a3 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1896,9 +1896,19 @@ static void preloadNSS() { std::call_once(dns_resolve_flag, []() { struct addrinfo *res = NULL; - if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) { + /* nss will only force the "local" (not through nscd) dns resolution if its on the LOCALDOMAIN. + We need the resolution to be done locally, as nscd socket will not be accessible in the + sandbox. */ + char * previous_env = getenv("LOCALDOMAIN"); + setenv("LOCALDOMAIN", "invalid", 1); + if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) == 0) { if (res) freeaddrinfo(res); } + if (previous_env) { + setenv("LOCALDOMAIN", previous_env, 1); + } else { + unsetenv("LOCALDOMAIN"); + } }); } From a67dcdd991e804d3555ef42e960db2b4c490dd4b Mon Sep 17 00:00:00 2001 From: Arthur Gautier Date: Wed, 8 Sep 2021 18:15:36 +0000 Subject: [PATCH 47/65] preloadNSS: load NSS before threads are started preloadNSS is not thread-safe, this commit moves it before we start the first thread. Signed-off-by: Arthur Gautier --- src/libmain/shared.cc | 30 ++++++++++++++++++++++++++++++ src/libstore/build.cc | 31 ------------------------------- 2 files changed, 30 insertions(+), 31 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index a14f9006a..055d8fb5b 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -14,6 +14,9 @@ #include #include #include +#include +#include +#include #include @@ -95,6 +98,31 @@ static void opensslLockCallback(int mode, int type, const char * file, int line) } #endif +static std::once_flag dns_resolve_flag; + +static void preloadNSS() { + /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of + one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already + been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to + load its lookup libraries in the parent before any child gets a chance to. */ + std::call_once(dns_resolve_flag, []() { + struct addrinfo *res = NULL; + + /* nss will only force the "local" (not through nscd) dns resolution if its on the LOCALDOMAIN. + We need the resolution to be done locally, as nscd socket will not be accessible in the + sandbox. */ + char * previous_env = getenv("LOCALDOMAIN"); + setenv("LOCALDOMAIN", "invalid", 1); + if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) == 0) { + if (res) freeaddrinfo(res); + } + if (previous_env) { + setenv("LOCALDOMAIN", previous_env, 1); + } else { + unsetenv("LOCALDOMAIN"); + } + }); +} static void sigHandler(int signo) { } @@ -158,6 +186,8 @@ void initNix() if (hasPrefix(getEnv("TMPDIR"), "/var/folders/")) unsetenv("TMPDIR"); #endif + + preloadNSS(); } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 8e40d51a3..69e349225 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -34,7 +34,6 @@ #include #include #include -#include #include #include #include @@ -45,7 +44,6 @@ /* Includes required for chroot support. */ #if __linux__ -#include #include #include #include @@ -1886,32 +1884,6 @@ PathSet DerivationGoal::exportReferences(PathSet storePaths) return paths; } -static std::once_flag dns_resolve_flag; - -static void preloadNSS() { - /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of - one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already - been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to - load its lookup libraries in the parent before any child gets a chance to. */ - std::call_once(dns_resolve_flag, []() { - struct addrinfo *res = NULL; - - /* nss will only force the "local" (not through nscd) dns resolution if its on the LOCALDOMAIN. - We need the resolution to be done locally, as nscd socket will not be accessible in the - sandbox. */ - char * previous_env = getenv("LOCALDOMAIN"); - setenv("LOCALDOMAIN", "invalid", 1); - if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) == 0) { - if (res) freeaddrinfo(res); - } - if (previous_env) { - setenv("LOCALDOMAIN", previous_env, 1); - } else { - unsetenv("LOCALDOMAIN"); - } - }); -} - void DerivationGoal::startBuilder() { /* Right platform? */ @@ -1923,9 +1895,6 @@ void DerivationGoal::startBuilder() settings.thisSystem, concatStringsSep(", ", settings.systemFeatures)); - if (drv->isBuiltin()) - preloadNSS(); - #if __APPLE__ additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); #endif From 7d92204b677c637221924b5cae7733ffb97e8690 Mon Sep 17 00:00:00 2001 From: kvtb <76634406+kvtb@users.noreply.github.com> Date: Sat, 11 Sep 2021 04:54:55 +0000 Subject: [PATCH 48/65] builtins.fetchurl: fix error message fixes `error: unsupported argument 'path' to 'fetchurl', at 0x1630b48` --- src/libexpr/primops.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 9a04432e0..f56fd2d77 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2079,7 +2079,7 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, else if (n == "name") request.name = state.forceStringNoCtx(*attr.value, *attr.pos); else - throw EvalError(format("unsupported argument '%1%' to '%2%', at %3%") % attr.name % who % attr.pos); + throw EvalError(format("unsupported argument '%1%' to '%2%', at %3%") % attr.name % who % *attr.pos); } if (request.uri.empty()) From 8223cc56630188786394eb7b035c3277f0793ac7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 7 Oct 2021 17:32:08 +0200 Subject: [PATCH 49/65] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index fabb09dc5..c790ace11 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.3.16 \ No newline at end of file +2.3.17 \ No newline at end of file From 6c0d9778d188cc1f97c3efe5fcb2c7633610c3a9 Mon Sep 17 00:00:00 2001 From: Winter Date: Fri, 15 Oct 2021 17:27:17 -0400 Subject: [PATCH 50/65] Revert "doc: Change install syntax to be fish compatible" This reverts commit ea962a84c35cb260e5feee5c5ba114f0db171055. --- doc/manual/installation/installing-binary.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/manual/installation/installing-binary.xml b/doc/manual/installation/installing-binary.xml index c6d3ee768..64c7a37fb 100644 --- a/doc/manual/installation/installing-binary.xml +++ b/doc/manual/installation/installing-binary.xml @@ -12,7 +12,7 @@ - $ curl -L https://nixos.org/nix/install | sh -s + $ sh <(curl -L https://nixos.org/nix/install) @@ -39,7 +39,7 @@ To explicitly select a single-user installation on your system: - curl -L https://nixos.org/nix/install | sh -s -- --no-daemon + sh <(curl -L https://nixos.org/nix/install) --no-daemon @@ -97,7 +97,7 @@ $ rm -rf /nix installation on your system: - curl -L https://nixos.org/nix/install | sh -s -- --daemon + sh <(curl -L https://nixos.org/nix/install) --daemon The multi-user installation of Nix will create build users between @@ -178,7 +178,7 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist is a bit of a misnomer). To use this approach, just install Nix with: - $ curl -L https://nixos.org/nix/install | sh -s -- --darwin-use-unencrypted-nix-store-volume + $ sh <(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume If you don't like the sound of this, you'll want to weigh the @@ -429,7 +429,7 @@ LABEL=Nix\040Store /nix apfs rw,nobrowse NixOS.org installation script: - curl -L https://nixos.org/nix/install | sh -s + sh <(curl -L https://nixos.org/nix/install) From 1261a1689bd921b40682bb6720177c2fe23242f9 Mon Sep 17 00:00:00 2001 From: Doron Behar Date: Wed, 3 Nov 2021 14:10:16 +0200 Subject: [PATCH 51/65] libstore: Use unix-dotfile vfs if useSQLiteWAL is false --- src/libstore/sqlite.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index a061d64f3..f1dd4a659 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -1,4 +1,5 @@ #include "sqlite.hh" +#include "globals.hh" #include "util.hh" #include @@ -27,8 +28,12 @@ namespace nix { SQLite::SQLite(const Path & path) { + // useSQLiteWAL also indicates what virtual file system we need. Using + // `unix-dotfile` is needed on NFS file systems and on Windows' Subsystem + // for Linux (WSL) where useSQLiteWAL should be false by default. + const char *vfs = settings.useSQLiteWAL ? 0 : "unix-dotfile"; if (sqlite3_open_v2(path.c_str(), &db, - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK) + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, vfs) != SQLITE_OK) throw Error(format("cannot open SQLite database '%s'") % path); } From c31ce3dffe10082ff1fc7a0e2d98bff83485e491 Mon Sep 17 00:00:00 2001 From: Fendor Date: Thu, 4 Nov 2021 11:21:04 +0100 Subject: [PATCH 52/65] PathSubstitutionGoal: Clean up pipe If there were many top-level goals (which are not destroyed until the very end), commands like $ nix copy --to 'ssh://localhost?remote-store=/tmp/nix' \ /run/current-system --no-check-sigs --substitute-on-destination could fail with "Too many open files". So now we do some explicit cleanup from amDone(). It would be cleaner to separate goals from their temporary internal state, but that would be a bigger refactor. Backport 8a29052cb2f52ef2c82c36fb3818fd0f66349729 --- src/libstore/build.cc | 34 ++++++++++++++++++++++++---------- src/libutil/util.cc | 13 ++++++++++--- src/libutil/util.hh | 3 ++- 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 69e349225..4b345fe37 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -179,6 +179,8 @@ public: virtual string key() = 0; + virtual void cleanup() { } + protected: virtual void amDone(ExitCode result); @@ -424,6 +426,8 @@ void Goal::amDone(ExitCode result) } waiters.clear(); worker.removeGoal(shared_from_this()); + + cleanup(); } @@ -3895,6 +3899,8 @@ public: void handleChildOutput(int fd, const string & data) override; void handleEOF(int fd) override; + void cleanup() override; + Path getStorePath() { return storePath; } void amDone(ExitCode result) override @@ -3918,15 +3924,7 @@ SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, Repa SubstitutionGoal::~SubstitutionGoal() { - try { - if (thr.joinable()) { - // FIXME: signal worker thread to quit. - thr.join(); - worker.childTerminated(this); - } - } catch (...) { - ignoreException(); - } + cleanup(); } @@ -3961,6 +3959,8 @@ void SubstitutionGoal::tryNext() { trace("trying next substituter"); + cleanup(); + if (subs.size() == 0) { /* None left. Terminate this goal and let someone else deal with it. */ @@ -4088,7 +4088,7 @@ void SubstitutionGoal::tryToRun() thr = std::thread([this]() { try { /* Wake up the worker loop when we're done. */ - Finally updateStats([this]() { outPipe.writeSide = -1; }); + Finally updateStats([this]() { outPipe.writeSide.close(); }); Activity act(*logger, actSubstitute, Logger::Fields{storePath, sub->getUri()}); PushActivity pact(act.id); @@ -4172,6 +4172,20 @@ void SubstitutionGoal::handleEOF(int fd) if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); } +void SubstitutionGoal::cleanup() +{ + try { + if (thr.joinable()) { + // FIXME: signal worker thread to quit. + thr.join(); + worker.childTerminated(this); + } + + outPipe.close(); + } catch (...) { + ignoreException(); + } +} ////////////////////////////////////////////////////////////////////// diff --git a/src/libutil/util.cc b/src/libutil/util.cc index ad8cc1894..b38eac9c6 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -753,6 +753,7 @@ void AutoCloseFD::close() if (::close(fd) == -1) /* This should never happen. */ throw SysError(format("closing file descriptor %1%") % fd); + fd = -1; } } @@ -770,6 +771,12 @@ int AutoCloseFD::release() return oldFD; } +void Pipe::close() +{ + readSide.close(); + writeSide.close(); +} + void Pipe::create() { @@ -1080,7 +1087,7 @@ void runProgram2(const RunOptions & options) throw SysError("executing '%1%'", options.program); }, processOptions); - out.writeSide = -1; + out.writeSide.close(); std::thread writerThread; @@ -1093,7 +1100,7 @@ void runProgram2(const RunOptions & options) if (source) { - in.readSide = -1; + in.readSide.close(); writerThread = std::thread([&]() { try { std::vector buf(8 * 1024); @@ -1110,7 +1117,7 @@ void runProgram2(const RunOptions & options) } catch (...) { promise.set_exception(std::current_exception()); } - in.writeSide = -1; + in.writeSide.close(); }); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index f057fdb2c..e59af288b 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -190,7 +190,6 @@ public: class AutoCloseFD { int fd; - void close(); public: AutoCloseFD(); AutoCloseFD(int fd); @@ -202,6 +201,7 @@ public: int get() const; explicit operator bool() const; int release(); + void close(); }; @@ -210,6 +210,7 @@ class Pipe public: AutoCloseFD readSide, writeSide; void create(); + void close(); }; From 1aa288c8cfe0a73379975d963dda5383f03da1fe Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 2 Jul 2020 17:18:03 +0200 Subject: [PATCH 53/65] Override socket path with `NIX_DAEMON_SOCKET_PATH` This is the first step to testing compatibility with 2.3 in 2.4 and master. (adapted from commit 223fbe644a4cde45269c01e971331d9414971d46) --- src/libstore/globals.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 2c50ea9df..787a320d6 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -34,7 +34,7 @@ Settings::Settings() , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR))) , nixBinDir(canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR))) , nixManDir(canonPath(NIX_MAN_DIR)) - , nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH)) + , nixDaemonSocketFile(canonPath(getEnv("NIX_DAEMON_SOCKET_PATH", nixStateDir + DEFAULT_SOCKET_PATH))) { buildUsersGroup = getuid() == 0 ? "nixbld" : ""; lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1"; From 765332d94bbb7857bac2399879be5b84fe138f5e Mon Sep 17 00:00:00 2001 From: Arthur Gautier Date: Thu, 25 Nov 2021 13:06:00 -0800 Subject: [PATCH 54/65] preloadNSS: rework the dns query workaround backport of https://github.com/NixOS/nix/pull/5384 to nix 2.3 --- src/libmain/shared.cc | 48 ++++++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 055d8fb5b..39ed8961b 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -14,9 +14,14 @@ #include #include #include -#include -#include -#include +#ifdef __linux__ +#include +#endif +#ifdef __GLIBC__ +#include +#include +#include +#endif #include @@ -106,21 +111,30 @@ static void preloadNSS() { been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to load its lookup libraries in the parent before any child gets a chance to. */ std::call_once(dns_resolve_flag, []() { - struct addrinfo *res = NULL; - - /* nss will only force the "local" (not through nscd) dns resolution if its on the LOCALDOMAIN. - We need the resolution to be done locally, as nscd socket will not be accessible in the - sandbox. */ - char * previous_env = getenv("LOCALDOMAIN"); - setenv("LOCALDOMAIN", "invalid", 1); - if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) == 0) { - if (res) freeaddrinfo(res); - } - if (previous_env) { - setenv("LOCALDOMAIN", previous_env, 1); - } else { - unsetenv("LOCALDOMAIN"); +#ifdef __GLIBC__ + /* On linux, glibc will run every lookup through the nss layer. + * That means every lookup goes, by default, through nscd, which acts as a local + * cache. + * Because we run builds in a sandbox, we also remove access to nscd otherwise + * lookups would leak into the sandbox. + * + * But now we have a new problem, we need to make sure the nss_dns backend that + * does the dns lookups when nscd is not available is loaded or available. + * + * We can't make it available without leaking nix's environment, so instead we'll + * load the backend, and configure nss so it does not try to run dns lookups + * through nscd. + * + * This is technically only used for builtins:fetch* functions so we only care + * about dns. + * + * All other platforms are unaffected. + */ + if (dlopen (LIBNSS_DNS_SO, RTLD_NOW) == NULL) { + printMsg(Verbosity::lvlWarn, fmt("Unable to load nss_dns backend")); } + __nss_configure_lookup ("hosts", "dns"); +#endif }); } From e9cfba389fc255033ddac52bf1603ffe9df0448b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 29 Nov 2021 23:18:01 +0000 Subject: [PATCH 55/65] Add GitHub actions CI for PRs Adapted from master. --- .github/workflows/test.yml | 76 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..d4668bc2a --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,76 @@ +name: "Test" + +on: + pull_request: + push: + +jobs: + + tests: + needs: [check_cachix] + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@v2.4.0 + with: + fetch-depth: 0 + - uses: cachix/install-nix-action@v16 + - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV + - uses: cachix/cachix-action@v10 + if: needs.check_cachix.outputs.secret == 'true' + with: + name: '${{ env.CACHIX_NAME }}' + signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' + authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + - run: nix-build release.nix -A build.$(nix-instantiate --eval -E '(builtins.currentSystem)') + + check_cachix: + name: Cachix secret present for installer tests + runs-on: ubuntu-latest + outputs: + secret: ${{ steps.secret.outputs.secret }} + steps: + - name: Check for Cachix secret + id: secret + env: + _CACHIX_SECRETS: ${{ secrets.CACHIX_SIGNING_KEY }}${{ secrets.CACHIX_AUTH_TOKEN }} + run: echo "::set-output name=secret::${{ env._CACHIX_SECRETS != '' }}" + + installer: + needs: [tests, check_cachix] + if: github.event_name == 'push' && needs.check_cachix.outputs.secret == 'true' + runs-on: ubuntu-latest + outputs: + installerURL: ${{ steps.prepare-installer.outputs.installerURL }} + steps: + - uses: actions/checkout@v2.4.0 + with: + fetch-depth: 0 + - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV + - uses: cachix/install-nix-action@v16 + - uses: cachix/cachix-action@v10 + with: + name: '${{ env.CACHIX_NAME }}' + signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' + authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + - id: prepare-installer + run: scripts/prepare-installer-for-github-actions + + installer_test: + needs: [installer, check_cachix] + if: github.event_name == 'push' && needs.check_cachix.outputs.secret == 'true' + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2.4.0 + - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV + - uses: cachix/install-nix-action@v16 + with: + install_url: '${{needs.installer.outputs.installerURL}}' + install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" + - run: nix-instantiate -E 'builtins.currentTime' --eval From b3cdebf3281ab9e8b0047ded0c55819cf461b378 Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 2 Jul 2020 11:34:15 +0200 Subject: [PATCH 56/65] Run the tests in parallel Cause the time needed to run the testsuite to drop from ~4mins to ~40s (cherry picked from commit 1b5aa60767f4a918250b157de054e1862240ca10) --- mk/run_test.sh | 28 ++++++++++++++++++++++++++++ mk/tests.mk | 41 ++++------------------------------------- tests/common.sh.in | 2 +- 3 files changed, 33 insertions(+), 38 deletions(-) create mode 100755 mk/run_test.sh diff --git a/mk/run_test.sh b/mk/run_test.sh new file mode 100755 index 000000000..6af5b070a --- /dev/null +++ b/mk/run_test.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +set -u + +red="" +green="" +yellow="" +normal="" + +post_run_msg="ran test $1..." +if [ -t 1 ]; then + red="" + green="" + yellow="" + normal="" +fi +(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null) +log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)" +status=$? +if [ $status -eq 0 ]; then + echo "$post_run_msg [${green}PASS$normal]" +elif [ $status -eq 99 ]; then + echo "$post_run_msg [${yellow}SKIP$normal]" +else + echo "$post_run_msg [${red}FAIL$normal]" + echo "$log" | sed 's/^/ /' + exit "$status" +fi diff --git a/mk/tests.mk b/mk/tests.mk index 70c30661b..e2258ede6 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -1,45 +1,12 @@ # Run program $1 as part of ‘make installcheck’. define run-install-test - installcheck: $1 + installcheck: $1.test - _installcheck-list += $1 + .PHONY: $1.test + $1.test: $1 tests/common.sh tests/init.sh + @env TEST_NAME=$1 TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1 endef -# Color code from https://unix.stackexchange.com/a/10065 -installcheck: - @total=0; failed=0; \ - red=""; \ - green=""; \ - yellow=""; \ - normal=""; \ - if [ -t 1 ]; then \ - red=""; \ - green=""; \ - yellow=""; \ - normal=""; \ - fi; \ - for i in $(_installcheck-list); do \ - total=$$((total + 1)); \ - printf "running test $$i..."; \ - log="$$(cd $$(dirname $$i) && $(tests-environment) $$(basename $$i) 2>&1)"; \ - status=$$?; \ - if [ $$status -eq 0 ]; then \ - echo " [$${green}PASS$$normal]"; \ - elif [ $$status -eq 99 ]; then \ - echo " [$${yellow}SKIP$$normal]"; \ - else \ - echo " [$${red}FAIL$$normal]"; \ - echo "$$log" | sed 's/^/ /'; \ - failed=$$((failed + 1)); \ - fi; \ - done; \ - if [ "$$failed" != 0 ]; then \ - echo "$${red}$$failed out of $$total tests failed $$normal"; \ - exit 1; \ - else \ - echo "$${green}All tests succeeded$$normal"; \ - fi - .PHONY: check installcheck diff --git a/tests/common.sh.in b/tests/common.sh.in index 15d7b1ef9..0bad159e9 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -1,6 +1,6 @@ set -e -export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test) +export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default} export NIX_STORE_DIR if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then # Maybe the build directory is symlinked. From 8ca44a901ce3d331756be0d1244732611b7c3f5c Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 2 Jul 2020 17:15:02 +0200 Subject: [PATCH 57/65] Fix the test dependencies Reuse the pre-existing list rather than the one written as part of #3777 (cherry picked from commit 5101ed18bca509a8cf43668b0701afad90c5c9c4) --- mk/tests.mk | 5 ++++- tests/local.mk | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/mk/tests.mk b/mk/tests.mk index e2258ede6..e6bd1ad79 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -1,10 +1,13 @@ # Run program $1 as part of ‘make installcheck’. + +test-deps = + define run-install-test installcheck: $1.test .PHONY: $1.test - $1.test: $1 tests/common.sh tests/init.sh + $1.test: $1 $(test-deps) @env TEST_NAME=$1 TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1 endef diff --git a/tests/local.mk b/tests/local.mk index 4c087ae39..83e93b123 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -40,4 +40,4 @@ tests-environment = NIX_REMOTE= $(bash) -e clean-files += $(d)/common.sh -installcheck: $(d)/common.sh $(d)/plugins/libplugintest.$(SO_EXT) +test-deps += tests/common.sh tests/plugins/libplugintest.$(SO_EXT) From 1a3956cceb60b843c1c389b217f8871d36f01c49 Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 2 Jul 2020 17:18:03 +0200 Subject: [PATCH 58/65] Shorten the path to the test root Fix a socket length failure on the OSX builders (cherry picked from commit 223fbe644a4cde45269c01e971331d9414971d46) --- mk/tests.mk | 2 +- tests/common.sh.in | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mk/tests.mk b/mk/tests.mk index e6bd1ad79..2e39bb694 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -8,7 +8,7 @@ define run-install-test .PHONY: $1.test $1.test: $1 $(test-deps) - @env TEST_NAME=$1 TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1 + @env TEST_NAME=$(notdir $(basename $1)) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1 endef diff --git a/tests/common.sh.in b/tests/common.sh.in index 0bad159e9..e8233bf72 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -11,6 +11,7 @@ export NIX_LOCALSTATE_DIR=$TEST_ROOT/var export NIX_LOG_DIR=$TEST_ROOT/var/log/nix export NIX_STATE_DIR=$TEST_ROOT/var/nix export NIX_CONF_DIR=$TEST_ROOT/etc +export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/daemon-socket export _NIX_TEST_SHARED=$TEST_ROOT/shared if [[ -n $NIX_STORE ]]; then export _NIX_TEST_NO_SANDBOX=1 @@ -72,7 +73,7 @@ startDaemon() { rm -f $NIX_STATE_DIR/daemon-socket/socket nix-daemon & for ((i = 0; i < 30; i++)); do - if [ -e $NIX_STATE_DIR/daemon-socket/socket ]; then break; fi + if [ -e $NIX_DAEMON_SOCKET_PATH ]; then break; fi sleep 1 done pidDaemon=$! From 522243fb266daea0060c3adb36e39af8738a418c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 28 Aug 2020 18:43:34 +0200 Subject: [PATCH 59/65] Close stdin while running tests For some reason, the bash shell started by 'nix develop' sometimes reads from stdin, which can hang. (cherry picked from commit 50a8710ed14c7a0236a656bb62d5117519ec5813) --- mk/tests.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mk/tests.mk b/mk/tests.mk index 2e39bb694..c1e140bac 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -8,7 +8,7 @@ define run-install-test .PHONY: $1.test $1.test: $1 $(test-deps) - @env TEST_NAME=$(notdir $(basename $1)) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1 + @env TEST_NAME=$(notdir $(basename $1)) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1 < /dev/null endef From 75bd19758d1527072916ef16ddc7b21ec026dec4 Mon Sep 17 00:00:00 2001 From: Albert Safin Date: Sat, 14 Dec 2019 15:37:20 +0000 Subject: [PATCH 60/65] nix-shell: don't check for "nix-shell" in shebang script name Without this, moving the nix-shell tests into a `nix-shell` dir breaks them. (cherry picked from commit a70706b0253e3bbd9143fde16e65fb7fe1a4007d) --- src/nix-build/nix-build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 1ebc80976..0bb963121 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -106,7 +106,7 @@ static void _main(int argc, char * * argv) // Heuristic to see if we're invoked as a shebang script, namely, // if we have at least one argument, it's the name of an // executable file, and it starts with "#!". - if (runEnv && argc > 1 && !std::regex_search(argv[1], std::regex("nix-shell"))) { + if (runEnv && argc > 1) { script = argv[1]; try { auto lines = tokenizeString(readFile(script), "\n"); From 2dff8c2b40358383a1086dbdfbf2499380206119 Mon Sep 17 00:00:00 2001 From: Klemens Nanni Date: Sat, 11 Mar 2023 21:36:07 +0400 Subject: [PATCH 61/65] Include as per wait(4) for WIFEXITED and WEXITSTATUS Unbreak the build on OpenBSD. --- src/libexpr/primops/fetchGit.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 90f600284..d866647f0 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -6,6 +6,7 @@ #include "hash.hh" #include +#include #include From fda598cfdd83a3617f1d554fc9bb720d1d079dff Mon Sep 17 00:00:00 2001 From: Klemens Nanni Date: Sat, 11 Mar 2023 21:38:44 +0400 Subject: [PATCH 62/65] Make tar invocation portable, fix OpenBSD build/test At least on OpenBSD, tar(1) reads from /dev/rst0 not stdin by default options must specififed consistently with or without dashes, not mixed. Specify standard input explicitly to not rely on implementation details. Use either option style consistently. --- src/libexpr/primops/fetchGit.cc | 2 +- tests/tarball.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index d866647f0..ca5af7492 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -174,7 +174,7 @@ GitInfo exportGit(ref store, const std::string & uri, Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); - runProgram("tar", true, { "x", "-C", tmpDir }, tar); + runProgram("tar", true, { "-x", "-f", "-", "-C", tmpDir }, tar); gitInfo.storePath = store->addToStore(name, tmpDir); diff --git a/tests/tarball.sh b/tests/tarball.sh index ba534c626..b2acb9eb9 100644 --- a/tests/tarball.sh +++ b/tests/tarball.sh @@ -11,7 +11,7 @@ cp dependencies.nix $tarroot/default.nix cp config.nix dependencies.builder*.sh $tarroot/ tarball=$TEST_ROOT/tarball.tar.xz -(cd $TEST_ROOT && tar c tarball) | xz > $tarball +(cd $TEST_ROOT && tar cf - tarball) | xz > $tarball nix-env -f file://$tarball -qa --out-path | grep -q dependencies From c84e20a2968f70474d3d8fc1c3701dd2b99d5ae3 Mon Sep 17 00:00:00 2001 From: Klemens Nanni Date: Sat, 11 Mar 2023 22:23:44 +0400 Subject: [PATCH 63/65] Fix SO_PEERCRED usage on OpenBSD getsockopt(2) documents `struct sockpeercred` with `SO_PEERCRED`. (`struct ucred` does exist, but is incompatible to the Linux version.) --- src/nix-daemon/nix-daemon.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 7a5cb9a37..87ad4e949 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -901,7 +901,11 @@ static PeerInfo getPeerInfo(int remote) #if defined(SO_PEERCRED) +#if defined(__OpenBSD__) + struct sockpeercred cred; +#else ucred cred; +#endif socklen_t credLen = sizeof(cred); if (getsockopt(remote, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) == -1) throw SysError("getting peer credentials"); From 42b19c71817319b4898fd3e8bacdd5785cb32465 Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Thu, 26 May 2022 18:36:10 +0100 Subject: [PATCH 64/65] src/libutil/json.cc: add missing include for gcc-13 Without the change llvm build fails on this week's gcc-13 snapshot as: src/libutil/json.cc: In function 'void nix::toJSON(std::ostream&, const char*, const char*)': src/libutil/json.cc:33:22: error: 'uint16_t' was not declared in this scope 33 | put(hex[(uint16_t(*i) >> 12) & 0xf]); | ^~~~~~~~ src/libutil/json.cc:5:1: note: 'uint16_t' is defined in header ''; did you forget to '#include '? 4 | #include +++ |+#include 5 | (cherry picked from commit b36d5172cb2cd99f8ae5262b3e3536cceac76b50) --- src/libutil/json.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libutil/json.cc b/src/libutil/json.cc index 0a6fb65f0..f80726d26 100644 --- a/src/libutil/json.cc +++ b/src/libutil/json.cc @@ -1,6 +1,7 @@ #include "json.hh" #include +#include #include namespace nix { From 7f8790eff2313af5a3014bbedf9d2c758b74c9da Mon Sep 17 00:00:00 2001 From: edef Date: Mon, 23 Oct 2023 17:36:13 +0000 Subject: [PATCH 65/65] libutil: add ZstdDecompressionSink --- Makefile.config.in | 1 + configure.ac | 2 + nix.spec.in | 1 + release-common.nix | 2 +- src/libutil/compression.cc | 76 ++++++++++++++++++++++++++++++++++++++ src/libutil/local.mk | 2 +- 6 files changed, 82 insertions(+), 2 deletions(-) diff --git a/Makefile.config.in b/Makefile.config.in index 588669f0a..5f0244176 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -19,6 +19,7 @@ SODIUM_LIBS = @SODIUM_LIBS@ LIBLZMA_LIBS = @LIBLZMA_LIBS@ SQLITE3_LIBS = @SQLITE3_LIBS@ LIBBROTLI_LIBS = @LIBBROTLI_LIBS@ +LIBZSTD_LIBS = @LIBZSTD_LIBS@ EDITLINE_LIBS = @EDITLINE_LIBS@ bash = @bash@ bindir = @bindir@ diff --git a/configure.ac b/configure.ac index 824a838a0..b0b620a08 100644 --- a/configure.ac +++ b/configure.ac @@ -223,6 +223,8 @@ AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt], # Look for libbrotli{enc,dec}. PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"]) +# Look for libzstd. +PKG_CHECK_MODULES([LIBZSTD], [libzstd], [CXXFLAGS="$LIBZSTD_CFLAGS $CXXFLAGS"]) # Look for libseccomp, required for Linux sandboxing. case "$host_os" in diff --git a/nix.spec.in b/nix.spec.in index 6b9e37637..1aeafa174 100644 --- a/nix.spec.in +++ b/nix.spec.in @@ -28,6 +28,7 @@ Requires: curl Requires: bzip2 Requires: gzip Requires: xz +Requires: zstd BuildRequires: bison BuildRequires: boost-devel >= 1.60 BuildRequires: bzip2-devel diff --git a/release-common.nix b/release-common.nix index a33b71f93..26355ba0b 100644 --- a/release-common.nix +++ b/release-common.nix @@ -49,7 +49,7 @@ rec { buildDeps = [ curl - bzip2 xz brotli editline + bzip2 xz brotli zstd editline openssl pkgconfig sqlite boost diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index 0dd84e320..c168e8ac5 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -5,8 +5,10 @@ #include #include +#include #include #include +#include #include #include @@ -198,6 +200,78 @@ struct BrotliDecompressionSink : ChunkedCompressionSink } }; +struct ZstdDecompressionSink : CompressionSink +{ + Sink & nextSink; + ZSTD_DStream *strm; + + std::vector inbuf; + size_t outbuf_size = ZSTD_DStreamOutSize(); + uint8_t *outbuf = new uint8_t[outbuf_size]; + + ZstdDecompressionSink(Sink & nextSink) : nextSink(nextSink) + { + strm = ZSTD_createDStream(); + if (!strm) + throw CompressionError("unable to initialise zstd decoder"); + + ZSTD_initDStream(strm); + } + + ~ZstdDecompressionSink() + { + delete[] outbuf; + ZSTD_freeDStream(strm); + } + + void finish() override + { + // this call doesn't make any sense, but it's here for consistency with the other compression sinks + // CompressionSink inherits from BufferedSink, but none of the subclasses appear to ever make use of the buffer + flush(); + + // if we still have undecoded data in the input buffer, we can't signal EOF to libzstd + // if we don't, then we're done here anyway + if (inbuf.size()) + throw CompressionError("received unexpected EOF while decompressing zstd file"); + + nextSink(nullptr, 0); + } + + void write(const unsigned char * data, size_t len) override + { + inbuf.insert(inbuf.end(), data, data + len); + + ZSTD_inBuffer in = { + .src = inbuf.data(), + .size = inbuf.size(), + .pos = 0 + }; + + ZSTD_outBuffer out = { + .dst = outbuf, + .size = outbuf_size, + .pos = 0 + }; + + while (in.pos < in.size) { + out.pos = 0; + + size_t ret = ZSTD_decompressStream(strm, &out, &in); + if (ZSTD_isError(ret)) + throw CompressionError("error %s while decompressing zstd file", ZSTD_getErrorName(ret)); + + if (out.pos) + nextSink(outbuf, out.pos); + else + break; + } + + // drop consumed input + inbuf.erase(inbuf.begin(), inbuf.begin() + in.pos); + } +}; + ref decompress(const std::string & method, const std::string & in) { StringSink ssink; @@ -217,6 +291,8 @@ ref makeDecompressionSink(const std::string & method, Sink & ne return make_ref(nextSink); else if (method == "br") return make_ref(nextSink); + else if (method == "zstd") + return make_ref(nextSink); else throw UnknownCompressionMethod("unknown compression method '%s'", method); } diff --git a/src/libutil/local.mk b/src/libutil/local.mk index e41a67d1f..ebcfddad2 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -6,4 +6,4 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) -libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(BOOST_LDFLAGS) -lboost_context +libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBZSTD_LIBS) $(BOOST_LDFLAGS) -lboost_context