Merge remote-tracking branch 'upstream/master' into indexed-store-path-outputs

tests/add.sh

@@ -9,7 +9,7 @@ echo $path2
 if test "$path1" != "$path2"; then
     echo "nix-store --add and --add-fixed mismatch"
     exit 1
-fi    
+fi
 
 path3=$(nix-store --add-fixed sha256 ./dummy)
 echo $path3

tests/binary-cache.sh

@@ -1,5 +1,7 @@
 source common.sh
 
+needLocalStore "“--no-require-sigs” can’t be used with the daemon"
+
 # We can produce drvs directly into the binary cache
 clearStore
 clearCacheCache

tests/build-remote-content-addressed-floating.sh

@@ -4,4 +4,6 @@ file=build-hook-ca-floating.nix
 
 sed -i 's/experimental-features .*/& ca-derivations/' "$NIX_CONF_DIR"/nix.conf
 
+CONTENT_ADDRESSED=true
+
 source build-remote.sh

tests/build-remote.sh

@@ -1,17 +1,22 @@
-if ! canUseSandbox; then exit; fi
-if ! [[ $busybox =~ busybox ]]; then exit; fi
+if ! canUseSandbox; then exit 99; fi
+if ! [[ $busybox =~ busybox ]]; then exit 99; fi
 
 unset NIX_STORE_DIR
 unset NIX_STATE_DIR
 
 function join_by { local d=$1; shift; echo -n "$1"; shift; printf "%s" "${@/#/$d}"; }
 
+EXTRA_SYSTEM_FEATURES=()
+if [[ -n "$CONTENT_ADDRESSED" ]]; then
+    EXTRA_SYSTEM_FEATURES=("ca-derivations")
+fi
+
 builders=(
   # system-features will automatically be added to the outer URL, but not inner
   # remote-store URL.
-  "ssh://localhost?remote-store=$TEST_ROOT/machine1?system-features=foo - - 1 1 foo"
-  "$TEST_ROOT/machine2 - - 1 1 bar"
-  "ssh-ng://localhost?remote-store=$TEST_ROOT/machine3?system-features=baz - - 1 1 baz"
+  "ssh://localhost?remote-store=$TEST_ROOT/machine1?system-features=$(join_by "%20" foo ${EXTRA_SYSTEM_FEATURES[@]}) - - 1 1 $(join_by "," foo ${EXTRA_SYSTEM_FEATURES[@]})"
+  "$TEST_ROOT/machine2 - - 1 1 $(join_by "," bar ${EXTRA_SYSTEM_FEATURES[@]})"
+  "ssh-ng://localhost?remote-store=$TEST_ROOT/machine3?system-features=$(join_by "%20" baz ${EXTRA_SYSTEM_FEATURES[@]}) - - 1 1 $(join_by "," baz ${EXTRA_SYSTEM_FEATURES[@]})"
 )
 
 chmod -R +w $TEST_ROOT/machine* || true

tests/build.sh

@@ -1,7 +1,7 @@
 source common.sh
 
 expectedJSONRegex='\[\{"drvPath":".*multiple-outputs-a.drv","outputs":\{"first":".*multiple-outputs-a-first","second":".*multiple-outputs-a-second"}},\{"drvPath":".*multiple-outputs-b.drv","outputs":\{"out":".*multiple-outputs-b"}}]'
-nix build -f multiple-outputs.nix --json a.all b.all | jq --exit-status '
+nix build -f multiple-outputs.nix --json a.all b.all --no-link | jq --exit-status '
   (.[0] |
     (.drvPath | match(".*multiple-outputs-a.drv")) and
     (.outputs |
@@ -11,3 +11,10 @@ nix build -f multiple-outputs.nix --json a.all b.all | jq --exit-status '
     (.drvPath | match(".*multiple-outputs-b.drv")) and
     (.outputs.out | match(".*multiple-outputs-b")))
 '
+testNormalization () {
+    clearStore
+    outPath=$(nix-build ./simple.nix --no-out-link)
+    test "$(stat -c %Y $outPath)" -eq 1
+}
+
+testNormalization

tests/ca/build-with-garbage-path.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+# Regression test for https://github.com/NixOS/nix/issues/4858
+
+source common.sh
+
+requireDaemonNewerThan "2.4pre20210621"
+
+# Get the output path of `rootCA`, and put some garbage instead
+outPath="$(nix-build ./content-addressed.nix -A rootCA --no-out-link)"
+nix-store --delete "$outPath"
+touch "$outPath"
+
+# The build should correctly remove the garbage and put the expected path instead
+nix-build ./content-addressed.nix -A rootCA --no-out-link
+
+# Rebuild it. This shouldn’t overwrite the existing path
+oldInode=$(stat -c '%i' "$outPath")
+nix-build ./content-addressed.nix -A rootCA --no-out-link --arg seed 2
+newInode=$(stat -c '%i' "$outPath")
+[[ "$oldInode" == "$newInode" ]]

tests/ca/build.sh

@@ -59,9 +59,17 @@ testNixCommand () {
     nix build --experimental-features 'nix-command ca-derivations' --file ./content-addressed.nix --no-link
 }
 
+# Regression test for https://github.com/NixOS/nix/issues/4775
+testNormalization () {
+    clearStore
+    outPath=$(buildAttr rootCA 1)
+    test "$(stat -c %Y $outPath)" -eq 1
+}
+
 # Disabled until we have it properly working
 # testRemoteCache
 clearStore
+testNormalization
 testDeterministicCA
 clearStore
 testCutoff

tests/ca/common.sh

@@ -1 +1,5 @@
 source ../common.sh
+
+sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf
+
+restartDaemon

tests/ca/concurrent-builds.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+# Ensure that we can’t build twice the same derivation concurrently.
+# Regression test for https://github.com/NixOS/nix/issues/5029
+
+source common.sh
+
+buggyNeedLocalStore "For some reason, this deadlocks with the daemon"
+
+export NIX_TESTS_CA_BY_DEFAULT=1
+
+clearStore
+
+for i in {0..5}; do
+    nix build --no-link --file ./racy.nix &
+done
+
+wait

tests/ca/config.nix.in

@@ -0,0 +1 @@
+../config.nix.in

tests/ca/content-addressed.nix

@@ -1,4 +1,11 @@
-with import ../config.nix;
+with import ./config.nix;
+
+let mkCADerivation = args: mkDerivation ({
+    __contentAddressed = true;
+    outputHashMode = "recursive";
+    outputHashAlgo = "sha256";
+} // args);
+in
 
 { seed ? 0 }:
 # A simple content-addressed derivation.
@@ -14,7 +21,7 @@ rec {
       echo "Hello World" > $out/hello
     '';
   };
-  rootCA = mkDerivation {
+  rootCA = mkCADerivation {
     name = "rootCA";
     outputs = [ "out" "dev" "foo"];
     buildCommand = ''
@@ -27,11 +34,8 @@ rec {
       ln -s $out $dev
       ln -s $out $foo
     '';
-    __contentAddressed = true;
-    outputHashMode = "recursive";
-    outputHashAlgo = "sha256";
   };
-  dependentCA = mkDerivation {
+  dependentCA = mkCADerivation {
     name = "dependent";
     buildCommand = ''
       echo "building a dependent derivation"
@@ -39,20 +43,14 @@ rec {
       cat ${rootCA}/self/dep
       echo ${rootCA}/self/dep > $out/dep
     '';
-    __contentAddressed = true;
-    outputHashMode = "recursive";
-    outputHashAlgo = "sha256";
   };
-  transitivelyDependentCA = mkDerivation {
+  transitivelyDependentCA = mkCADerivation {
     name = "transitively-dependent";
     buildCommand = ''
       echo "building transitively-dependent"
       cat ${dependentCA}/dep
       echo ${dependentCA} > $out
     '';
-    __contentAddressed = true;
-    outputHashMode = "recursive";
-    outputHashAlgo = "sha256";
   };
   dependentNonCA = mkDerivation {
     name = "dependent-non-ca";
@@ -72,6 +70,14 @@ rec {
       cat ${dependentCA}/dep
       echo foo > $out
     '';
-
+  };
+  runnable = mkCADerivation rec {
+    name = "runnable-thing";
+    buildCommand = ''
+      mkdir -p $out/bin
+      echo ${rootCA} # Just to make it depend on it
+      echo "" > $out/bin/${name}
+      chmod +x $out/bin/${name}
+    '';
   };
 }

tests/ca/duplicate-realisation-in-closure.sh

@@ -0,0 +1,28 @@
+source ./common.sh
+
+requireDaemonNewerThan "2.4pre20210625"
+
+sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf
+
+export REMOTE_STORE_DIR="$TEST_ROOT/remote_store"
+export REMOTE_STORE="file://$REMOTE_STORE_DIR"
+
+rm -rf $REMOTE_STORE_DIR
+clearStore
+
+# Build dep1 and push that to the binary cache.
+# This entails building (and pushing) current-time.
+nix copy --to "$REMOTE_STORE" -f nondeterministic.nix dep1
+clearStore
+sleep 2 # To make sure that `$(date)` will be different
+# Build dep2.
+# As we’ve cleared the cache, we’ll have to rebuild current-time. And because
+# the current time isn’t the same as before, this will yield a new (different)
+# realisation
+nix build -f nondeterministic.nix dep2 --no-link
+
+# Build something that depends both on dep1 and dep2.
+# If everything goes right, we should rebuild dep2 rather than fetch it from
+# the cache (because that would mean duplicating `current-time` in the closure),
+# and have `dep1 == dep2`.
+nix build --substituters "$REMOTE_STORE" -f nondeterministic.nix toplevel --no-require-sigs --no-link

tests/ca/flake.nix

@@ -0,0 +1,3 @@
+{
+  outputs = { self }: import ./content-addressed.nix {};
+}

tests/ca/gc.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# Ensure that garbage collection works properly with ca derivations
+
+source common.sh
+
+export NIX_TESTS_CA_BY_DEFAULT=1
+
+cd ..
+source gc.sh

tests/ca/nix-run.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+source common.sh
+
+sed -i 's/experimental-features .*/& ca-derivations ca-references nix-command flakes/' "$NIX_CONF_DIR"/nix.conf
+
+FLAKE_PATH=path:$PWD
+
+nix run --no-write-lock-file $FLAKE_PATH#runnable

tests/ca/nix-shell.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+source common.sh
+
+sed -i 's/experimental-features .*/& ca-derivations ca-references nix-command flakes/' "$NIX_CONF_DIR"/nix.conf
+
+CONTENT_ADDRESSED=true
+cd ..
+source ./nix-shell.sh
+

tests/ca/nondeterministic.nix

@@ -0,0 +1,35 @@
+with import ./config.nix;
+
+let mkCADerivation = args: mkDerivation ({
+    __contentAddressed = true;
+    outputHashMode = "recursive";
+    outputHashAlgo = "sha256";
+} // args);
+in
+
+rec {
+  currentTime = mkCADerivation {
+    name = "current-time";
+    buildCommand = ''
+      mkdir $out
+      echo $(date) > $out/current-time
+    '';
+  };
+  dep = seed: mkCADerivation {
+    name = "dep";
+    inherit seed;
+    buildCommand = ''
+      echo ${currentTime} > $out
+    '';
+  };
+  dep1 = dep 1;
+  dep2 = dep 2;
+  toplevel = mkCADerivation {
+    name = "toplevel";
+    buildCommand = ''
+      test ${dep1} == ${dep2}
+      touch $out
+    '';
+  };
+}
+

tests/ca/post-hook.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+source common.sh
+
+requireDaemonNewerThan "2.4pre20210626"
+
+sed -i 's/experimental-features .*/& ca-derivations ca-references nix-command flakes/' "$NIX_CONF_DIR"/nix.conf
+
+export NIX_TESTS_CA_BY_DEFAULT=1
+cd ..
+source ./post-hook.sh
+
+

tests/ca/racy.nix

@@ -0,0 +1,15 @@
+# A derivation that would certainly fail if several builders tried to
+# build it at once.
+
+
+with import ./config.nix;
+
+mkDerivation {
+  name = "simple";
+  buildCommand = ''
+    mkdir $out
+    echo bar >> $out/foo
+    sleep 3
+    [[ "$(cat $out/foo)" == bar ]]
+  '';
+}

tests/ca/recursive.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+source common.sh
+
+requireDaemonNewerThan "2.4pre20210623"
+
+sed -i 's/experimental-features .*/& ca-derivations ca-references nix-command flakes/' "$NIX_CONF_DIR"/nix.conf
+
+export NIX_TESTS_CA_BY_DEFAULT=1
+cd ..
+source ./recursive.sh
+
+

tests/ca/signatures.sh

@@ -22,8 +22,8 @@ testOneCopy () {
     rm -rf "$REMOTE_STORE_DIR"
 
     attrPath="$1"
-    nix copy --to $REMOTE_STORE "$attrPath" --file ./content-addressed.nix \
-        --secret-key-files "$TEST_ROOT/sk1"
+    nix copy -vvvv --to $REMOTE_STORE "$attrPath" --file ./content-addressed.nix \
+        --secret-key-files "$TEST_ROOT/sk1" --show-trace
 
     ensureCorrectlyCopied "$attrPath"
 

tests/ca/substitute.sh

@@ -4,11 +4,12 @@
 
 source common.sh
 
-sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf
+needLocalStore "“--no-require-sigs” can’t be used with the daemon"
 
 rm -rf $TEST_ROOT/binary_cache
 
-export REMOTE_STORE=file://$TEST_ROOT/binary_cache
+export REMOTE_STORE_DIR=$TEST_ROOT/binary_cache
+export REMOTE_STORE=file://$REMOTE_STORE_DIR
 
 buildDrvs () {
     nix build --file ./content-addressed.nix -L --no-link "$@"
@@ -16,9 +17,48 @@ buildDrvs () {
 
 # Populate the remote cache
 clearStore
-buildDrvs --post-build-hook ../push-to-store.sh
+nix copy --to $REMOTE_STORE --file ./content-addressed.nix
 
 # Restart the build on an empty store, ensuring that we don't build
 clearStore
-buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0
+buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0 transitivelyDependentCA
+# Check that the thing we’ve just substituted has its realisation stored
+nix realisation info --file ./content-addressed.nix transitivelyDependentCA
+# Check that its dependencies have it too
+nix realisation info --file ./content-addressed.nix dependentCA rootCA
 
+# Same thing, but
+# 1. With non-ca derivations
+# 2. Erasing the realisations on the remote store
+#
+# Even in that case, realising the derivations should still produce the right
+# realisations on the local store
+#
+# Regression test for #4725
+clearStore
+nix build --file ../simple.nix -L --no-link --post-build-hook ../push-to-store.sh
+clearStore
+rm -r "$REMOTE_STORE_DIR/realisations"
+nix build --file ../simple.nix -L --no-link --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0
+# There's no easy way to check whether a realisation is present on the local
+# store − short of manually querying the db, but the build environment doesn't
+# have the sqlite binary − so we instead push things again, and check that the
+# realisations have correctly been pushed to the remote store
+nix copy --to "$REMOTE_STORE" --file ../simple.nix
+if [[ -z "$(ls "$REMOTE_STORE_DIR/realisations")" ]]; then
+    echo "Realisations not rebuilt"
+    exit 1
+fi
+
+# Test the local realisation disk cache
+buildDrvs --post-build-hook ../push-to-store.sh
+clearStore
+# Add the realisations of rootCA to the cachecache
+clearCacheCache
+export _NIX_FORCE_HTTP=1
+buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0
+# Try rebuilding, but remove the realisations from the remote cache to force
+# using the cachecache
+clearStore
+rm $REMOTE_STORE_DIR/realisations/*
+buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0

tests/check.nix

@@ -44,7 +44,7 @@ with import ./config.nix;
   };
 
   hashmismatch = import <nix/fetchurl.nix> {
-    url = "file://" + toString ./dummy;
+    url = "file://" + builtins.getEnv "TMPDIR" + "/dummy";
     sha256 = "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73";
   };
 

tests/check.sh

@@ -1,5 +1,8 @@
 source common.sh
 
+# XXX: This shouldn’t be, but #4813 cause this test to fail
+buggyNeedLocalStore "see #4813"
+
 checkBuildTempDirRemoved ()
 {
     buildDir=$(sed -n 's/CHECK_TMPDIR=//p' $1 | head -1)
@@ -74,12 +77,13 @@ nix-build check.nix -A fetchurl --no-out-link --check
 nix-build check.nix -A fetchurl --no-out-link --repair
 [[ $(cat $path) != foo ]]
 
+echo 'Hello World' > $TMPDIR/dummy
 nix-build check.nix -A hashmismatch --no-out-link || status=$?
 [ "$status" = "102" ]
 
-echo -n > ./dummy
+echo -n > $TMPDIR/dummy
 nix-build check.nix -A hashmismatch --no-out-link
-echo 'Hello World' > ./dummy
+echo 'Hello World' > $TMPDIR/dummy
 
 nix-build check.nix -A hashmismatch --no-out-link --check || status=$?
 [ "$status" = "102" ]

tests/common.sh.in

@@ -1,5 +1,9 @@
 set -e
 
+if [[ -z "$COMMON_SH_SOURCED" ]]; then
+
+COMMON_SH_SOURCED=1
+
 export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default}
 export NIX_STORE_DIR
 if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then
@@ -45,6 +49,9 @@ export busybox="@sandbox_shell@"
 export version=@PACKAGE_VERSION@
 export system=@system@
 
+export IMPURE_VAR1=foo
+export IMPURE_VAR2=bar
+
 cacheDir=$TEST_ROOT/binary-cache
 
 readLink() {
@@ -75,29 +82,57 @@ clearCacheCache() {
 }
 
 startDaemon() {
+    # Don’t start the daemon twice, as this would just make it loop indefinitely
+    if [[ "$NIX_REMOTE" == daemon ]]; then
+      return
+    fi
     # Start the daemon, wait for the socket to appear.  !!!
     # ‘nix-daemon’ should have an option to fork into the background.
-    rm -f $NIX_STATE_DIR/daemon-socket/socket
+    rm -f $NIX_DAEMON_SOCKET_PATH
     ${NIX_DAEMON_COMMAND:-nix daemon} &
     for ((i = 0; i < 30; i++)); do
-        if [ -e $NIX_DAEMON_SOCKET_PATH ]; then break; fi
+        if [[ -S $NIX_DAEMON_SOCKET_PATH ]]; then break; fi
         sleep 1
     done
     pidDaemon=$!
-    trap "kill -9 $pidDaemon" EXIT
+    trap "killDaemon" EXIT
     export NIX_REMOTE=daemon
 }
 
 killDaemon() {
-    kill -9 $pidDaemon
+    kill $pidDaemon
+    for i in {0.10}; do
+        kill -0 $pidDaemon || break
+        sleep 1
+    done
+    kill -9 $pidDaemon || true
     wait $pidDaemon || true
     trap "" EXIT
 }
 
+restartDaemon() {
+  [[ -z "${pidDaemon:-}" ]] && return 0
+
+  killDaemon
+  unset NIX_REMOTE
+  startDaemon
+}
+
 if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then
     _canUseSandbox=1
 fi
 
+isDaemonNewer () {
+  [[ -n "${NIX_DAEMON_PACKAGE:-}" ]] || return 0
+  local requiredVersion="$1"
+  local daemonVersion=$($NIX_DAEMON_PACKAGE/bin/nix-daemon --version | cut -d' ' -f3)
+  return [[ $(nix eval --expr "builtins.compareVersions ''$daemonVersion'' ''2.4''") -ge 0 ]]
+}
+
+requireDaemonNewerThan () {
+  isDaemonNewer "$1" || exit 99
+}
+
 canUseSandbox() {
     if [[ ! $_canUseSandbox ]]; then
         echo "Sandboxing not supported, skipping this test..."
@@ -123,4 +158,22 @@ expect() {
     [[ $res -eq $expected ]]
 }
 
+needLocalStore() {
+  if [[ "$NIX_REMOTE" == "daemon" ]]; then
+    echo "Can’t run through the daemon ($1), skipping this test..."
+    return 99
+  fi
+}
+
+# Just to make it easy to find which tests should be fixed
+buggyNeedLocalStore () {
+  needLocalStore
+}
+
 set -x
+
+if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then
+    startDaemon
+fi
+
+fi # COMMON_SH_SOURCED

tests/config.nix.in

@@ -1,3 +1,12 @@
+let
+  contentAddressedByDefault = builtins.getEnv "NIX_TESTS_CA_BY_DEFAULT" == "1";
+  caArgs = if contentAddressedByDefault then {
+    __contentAddressed = true;
+    outputHashMode = "recursive";
+    outputHashAlgo = "sha256";
+  } else {};
+in
+
 rec {
   shell = "@bash@";
 
@@ -13,6 +22,6 @@ rec {
       builder = shell;
       args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
       PATH = path;
-    } // removeAttrs args ["builder" "meta"])
+    } // caArgs // removeAttrs args ["builder" "meta"])
     // { meta = args.meta or {}; };
 }

tests/db-migration.sh

@@ -3,11 +3,14 @@
 # Only run this if we have an older Nix available
 # XXX: This assumes that the `daemon` package is older than the `client` one
 if [[ -z "$NIX_DAEMON_PACKAGE" ]]; then
-    exit 0
+    exit 99
 fi
 
 source common.sh
 
+killDaemon
+unset NIX_REMOTE
+
 # Fill the db using the older Nix
 PATH_WITH_NEW_NIX="$PATH"
 export PATH="$NIX_DAEMON_PACKAGE/bin:$PATH"

tests/dummy

@@ -0,0 +1 @@
+Hello World

tests/dump-db.sh

@@ -1,5 +1,7 @@
 source common.sh
 
+needLocalStore "--dump-db requires a local store"
+
 clearStore
 
 path=$(nix-build dependencies.nix -o $TEST_ROOT/result)

tests/eval-store.sh

@@ -0,0 +1,28 @@
+source common.sh
+
+requireDaemonNewerThan "2.4pre20210727"
+
+eval_store=$TEST_ROOT/eval-store
+
+clearStore
+rm -rf "$eval_store"
+
+nix build -f dependencies.nix --eval-store "$eval_store" -o "$TEST_ROOT/result"
+[[ -e $TEST_ROOT/result/foobar ]]
+(! ls $NIX_STORE_DIR/*.drv)
+ls $eval_store/nix/store/*.drv
+
+clearStore
+rm -rf "$eval_store"
+
+nix-instantiate dependencies.nix --eval-store "$eval_store"
+(! ls $NIX_STORE_DIR/*.drv)
+ls $eval_store/nix/store/*.drv
+
+clearStore
+rm -rf "$eval_store"
+
+nix-build dependencies.nix --eval-store "$eval_store" -o "$TEST_ROOT/result"
+[[ -e $TEST_ROOT/result/foobar ]]
+(! ls $NIX_STORE_DIR/*.drv)
+ls $eval_store/nix/store/*.drv

tests/fetchGit.sh

@@ -189,3 +189,7 @@ path8=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$rep
 rev4=$(git -C $repo rev-parse HEAD)
 rev4_nix=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"HEAD\"; }).rev")
 [[ $rev4 = $rev4_nix ]]
+
+# The name argument should be handled
+path9=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"HEAD\"; name = \"foo\"; }).outPath")
+[[ $path9 =~ -foo$ ]]

tests/fetchMercurial.sh

@@ -94,3 +94,8 @@ hg commit --cwd $repo -m 'Bla3'
 
 path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchMercurial file://$repo).outPath")
 [[ $path2 = $path4 ]]
+
+echo paris > $repo/hello
+# Passing a `name` argument should be reflected in the output path
+path5=$(nix eval -vvvvv --impure --refresh --raw --expr "(builtins.fetchMercurial { url = \"file://$repo\"; name = \"foo\"; } ).outPath")
+[[ $path5 =~ -foo$ ]]

tests/fetchurl.sh

@@ -5,7 +5,7 @@ clearStore
 # Test fetching a flat file.
 hash=$(nix-hash --flat --type sha256 ./fetchurl.sh)
 
-outPath=$(nix-build --expr 'import <nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link)
+outPath=$(nix-build -vvvvv --expr 'import <nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link)
 
 cmp $outPath fetchurl.sh
 
@@ -14,7 +14,7 @@ clearStore
 
 hash=$(nix hash file --type sha512 --base64 ./fetchurl.sh)
 
-outPath=$(nix-build --expr 'import <nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link)
+outPath=$(nix-build -vvvvv --expr 'import <nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link)
 
 cmp $outPath fetchurl.sh
 
@@ -25,7 +25,7 @@ hash=$(nix hash file ./fetchurl.sh)
 
 [[ $hash =~ ^sha256- ]]
 
-outPath=$(nix-build --expr 'import <nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link)
+outPath=$(nix-build -vvvvv --expr 'import <nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link)
 
 cmp $outPath fetchurl.sh
 
@@ -38,10 +38,10 @@ hash=$(nix hash file --type sha256 --base16 ./fetchurl.sh)
 
 storePath=$(nix --store $other_store store add-file ./fetchurl.sh)
 
-outPath=$(nix-build --expr 'import <nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store)
+outPath=$(nix-build -vvvvv --expr 'import <nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store)
 
 # Test hashed mirrors with an SRI hash.
-nix-build --expr 'import <nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix hash to-sri --type sha256 $hash) \
+nix-build -vvvvv --expr 'import <nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix hash to-sri --type sha256 $hash) \
           --no-out-link --substituters $other_store
 
 # Test unpacking a NAR.
@@ -55,7 +55,7 @@ nix-store --dump $TEST_ROOT/archive > $nar
 
 hash=$(nix-hash --flat --type sha256 $nar)
 
-outPath=$(nix-build --expr 'import <nix/fetchurl.nix>' --argstr url file://$nar --argstr sha256 $hash \
+outPath=$(nix-build -vvvvv --expr 'import <nix/fetchurl.nix>' --argstr url file://$nar --argstr sha256 $hash \
           --arg unpack true --argstr name xyzzy --no-out-link)
 
 echo $outPath | grep -q 'xyzzy'
@@ -69,7 +69,7 @@ nix-store --delete $outPath
 narxz=$TEST_ROOT/archive.nar.xz
 rm -f $narxz
 xz --keep $nar
-outPath=$(nix-build --expr 'import <nix/fetchurl.nix>' --argstr url file://$narxz --argstr sha256 $hash \
+outPath=$(nix-build -vvvvv --expr 'import <nix/fetchurl.nix>' --argstr url file://$narxz --argstr sha256 $hash \
           --arg unpack true --argstr name xyzzy --no-out-link)
 
 test -x $outPath/fetchurl.sh

tests/fixed.nix

@@ -21,6 +21,14 @@ rec {
     (f ./fixed.builder2.sh "recursive" "sha1" "vw46m23bizj4n8afrc0fj19wrp7mj3c0")
   ];
 
+  # Expression to test that `nix-build --check` also throws an error if the hash of
+  # fixed-output derivation has changed even if the hash exists in the store (in this
+  # case the hash exists because of `fixed.builder2.sh`, but building a derivation
+  # with the same hash and a different result must throw an error).
+  check = [
+    (f ./fixed.builder1.sh "recursive" "md5" "3670af73070fa14077ad74e0f5ea4e42")
+  ];
+
   good2 = [
     # Yes, this looks fscked up: builder2 doesn't have that result.
     # But Nix sees that an output with the desired hash already

tests/fixed.sh

@@ -2,9 +2,6 @@ source common.sh
 
 clearStore
 
-export IMPURE_VAR1=foo
-export IMPURE_VAR2=bar
-
 path=$(nix-store -q $(nix-instantiate fixed.nix -A good.0))
 
 echo 'testing bad...'
@@ -18,6 +15,11 @@ nix path-info --json $path | grep fixed:md5:2qk15sxzzjlnpjk9brn7j8ppcd
 echo 'testing good...'
 nix-build fixed.nix -A good --no-out-link
 
+if isDaemonNewer "2.4pre20210927"; then
+    echo 'testing --check...'
+    nix-build fixed.nix -A check --check && fail "should fail"
+fi
+
 echo 'testing good2...'
 nix-build fixed.nix -A good2 --no-out-link
 

tests/flakes.sh

@@ -23,13 +23,19 @@ flake6Dir=$TEST_ROOT/flake6
 flake7Dir=$TEST_ROOT/flake7
 templatesDir=$TEST_ROOT/templates
 nonFlakeDir=$TEST_ROOT/nonFlake
+badFlakeDir=$TEST_ROOT/badFlake
 flakeA=$TEST_ROOT/flakeA
 flakeB=$TEST_ROOT/flakeB
 flakeGitBare=$TEST_ROOT/flakeGitBare
+flakeFollowsA=$TEST_ROOT/follows/flakeA
+flakeFollowsB=$TEST_ROOT/follows/flakeA/flakeB
+flakeFollowsC=$TEST_ROOT/follows/flakeA/flakeB/flakeC
+flakeFollowsD=$TEST_ROOT/follows/flakeA/flakeD
+flakeFollowsE=$TEST_ROOT/follows/flakeA/flakeE
 
-for repo in $flake1Dir $flake2Dir $flake3Dir $flake7Dir $templatesDir $nonFlakeDir $flakeA $flakeB; do
+for repo in $flake1Dir $flake2Dir $flake3Dir $flake7Dir $templatesDir $nonFlakeDir $flakeA $flakeB $flakeFollowsA; do
     rm -rf $repo $repo.tmp
-    mkdir $repo
+    mkdir -p $repo
     git -C $repo init
     git -C $repo config user.email "foobar@example.com"
     git -C $repo config user.name "Foobar"
@@ -90,76 +96,14 @@ EOF
 git -C $nonFlakeDir add README.md
 git -C $nonFlakeDir commit -m 'Initial'
 
-cat > $registry <<EOF
-{
-  "version": 2,
-  "flakes": [
-    { "from": {
-        "type": "indirect",
-        "id": "flake1"
-      },
-      "to": {
-        "type": "git",
-        "url": "file://$flake1Dir"
-      }
-    },
-    { "from": {
-        "type": "indirect",
-        "id": "flake2"
-      },
-      "to": {
-        "type": "git",
-        "url": "file://$flake2Dir"
-      }
-    },
-    { "from": {
-        "type": "indirect",
-        "id": "flake3"
-      },
-      "to": {
-        "type": "git",
-        "url": "file://$flake3Dir"
-      }
-    },
-    { "from": {
-        "type": "indirect",
-        "id": "flake4"
-      },
-      "to": {
-        "type": "indirect",
-        "id": "flake3"
-      }
-    },
-    { "from": {
-        "type": "indirect",
-        "id": "flake5"
-      },
-      "to": {
-        "type": "hg",
-        "url": "file://$flake5Dir"
-      }
-    },
-    { "from": {
-        "type": "indirect",
-        "id": "nixpkgs"
-      },
-      "to": {
-        "type": "indirect",
-        "id": "flake1"
-      }
-    },
-    { "from": {
-        "type": "indirect",
-        "id": "templates"
-      },
-      "to": {
-        "type": "git",
-        "url": "file://$templatesDir"
-      }
-    }
-  ]
-}
-EOF
+# Construct a custom registry, additionally test the --registry flag
+nix registry add --registry $registry flake1 git+file://$flake1Dir
+nix registry add --registry $registry flake2 git+file://$flake2Dir
+nix registry add --registry $registry flake3 git+file://$flake3Dir
+nix registry add --registry $registry flake4 flake3
+nix registry add --registry $registry flake5 hg+file://$flake5Dir
+nix registry add --registry $registry nixpkgs flake1
+nix registry add --registry $registry templates git+file://$templatesDir
 
 # Test 'nix flake list'.
 [[ $(nix registry list | wc -l) == 7 ]]
@@ -209,6 +153,7 @@ nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"git+file://$flake1Di
 
 # Building a flake with an unlocked dependency should fail in pure mode.
 (! nix build -o $TEST_ROOT/result flake2#bar --no-registries)
+(! nix build -o $TEST_ROOT/result flake2#bar --no-use-registries)
 (! nix eval --expr "builtins.getFlake \"$flake2Dir\"")
 
 # But should succeed in impure mode.
@@ -232,6 +177,7 @@ nix build -o $TEST_ROOT/result $flake2Dir#bar
 # Building with a lockfile should not require a fetch of the registry.
 nix build -o $TEST_ROOT/result --flake-registry file:///no-registry.json $flake2Dir#bar --refresh
 nix build -o $TEST_ROOT/result --no-registries $flake2Dir#bar --refresh
+nix build -o $TEST_ROOT/result --no-use-registries $flake2Dir#bar --refresh
 
 # Updating the flake should not change the lockfile.
 nix flake lock $flake2Dir
@@ -242,6 +188,7 @@ nix build -o $TEST_ROOT/result flake2#bar
 
 # Or without a registry.
 nix build -o $TEST_ROOT/result --no-registries git+file://$flake2Dir#bar --refresh
+nix build -o $TEST_ROOT/result --no-use-registries git+file://$flake2Dir#bar --refresh
 
 # Test whether indirect dependencies work.
 nix build -o $TEST_ROOT/result $flake3Dir#xyzzy
@@ -405,6 +352,8 @@ nix registry add flake1 flake3
 [[ $(nix registry list | wc -l) == 8 ]]
 nix registry pin flake1
 [[ $(nix registry list | wc -l) == 8 ]]
+nix registry pin flake1 flake3
+[[ $(nix registry list | wc -l) == 8 ]]
 nix registry remove flake1
 [[ $(nix registry list | wc -l) == 7 ]]
 
@@ -443,12 +392,14 @@ git -C $templatesDir commit -m 'Initial'
 
 nix flake check templates
 nix flake show templates
+nix flake show templates --json | jq
 
 (cd $flake7Dir && nix flake init)
 (cd $flake7Dir && nix flake init) # check idempotence
 git -C $flake7Dir add flake.nix
 nix flake check $flake7Dir
 nix flake show $flake7Dir
+nix flake show $flake7Dir --json | jq
 git -C $flake7Dir commit -a -m 'Initial'
 
 # Test 'nix flake new'.
@@ -535,6 +486,21 @@ EOF
 
 (! nix flake check $flake3Dir)
 
+cat > $flake3Dir/flake.nix <<EOF
+{
+  outputs = { flake1, self }: {
+    defaultPackage = {
+        system-1 = "foo";
+        system-2 = "bar";
+    };
+  };
+}
+EOF
+
+checkRes=$(nix flake check --keep-going $flake3Dir 2>&1 && fail "nix flake check should have failed" || true)
+echo "$checkRes" | grep -q "defaultPackage.system-1"
+echo "$checkRes" | grep -q "defaultPackage.system-2"
+
 # Test 'follows' inputs.
 cat > $flake3Dir/flake.nix <<EOF
 {
@@ -648,6 +614,7 @@ nix flake metadata --json hg+file://$flake5Dir
 [[ $(nix flake metadata --json hg+file://$flake5Dir | jq -e -r .revCount) = 1 ]]
 
 nix build -o $TEST_ROOT/result hg+file://$flake5Dir --no-registries --no-allow-dirty
+nix build -o $TEST_ROOT/result hg+file://$flake5Dir --no-use-registries --no-allow-dirty
 
 # Test tarball flakes
 tar cfz $TEST_ROOT/flake.tar.gz -C $TEST_ROOT --exclude .hg flake5
@@ -722,3 +689,98 @@ git -C $flakeB commit -a -m 'Foo'
 
 # Test list-inputs with circular dependencies
 nix flake metadata $flakeA
+
+# Test flake follow paths
+mkdir -p $flakeFollowsB
+mkdir -p $flakeFollowsC
+mkdir -p $flakeFollowsD
+mkdir -p $flakeFollowsE
+
+cat > $flakeFollowsA/flake.nix <<EOF
+{
+    description = "Flake A";
+    inputs = {
+        B = {
+            url = "path:./flakeB";
+            inputs.foobar.follows = "D";
+        };
+
+        D.url = "path:./flakeD";
+        foobar.url = "path:./flakeE";
+    };
+    outputs = { ... }: {};
+}
+EOF
+
+cat > $flakeFollowsB/flake.nix <<EOF
+{
+    description = "Flake B";
+    inputs = {
+        foobar.url = "path:./../flakeE";
+        C = {
+            url = "path:./flakeC";
+            inputs.foobar.follows = "foobar";
+        };
+    };
+    outputs = { ... }: {};
+}
+EOF
+
+cat > $flakeFollowsC/flake.nix <<EOF
+{
+    description = "Flake C";
+    inputs = {
+        foobar.url = "path:./../../flakeE";
+    };
+    outputs = { ... }: {};
+}
+EOF
+
+cat > $flakeFollowsD/flake.nix <<EOF
+{
+    description = "Flake D";
+    inputs = {};
+    outputs = { ... }: {};
+}
+EOF
+
+cat > $flakeFollowsE/flake.nix <<EOF
+{
+    description = "Flake D";
+    inputs = {};
+    outputs = { ... }: {};
+}
+EOF
+
+git -C $flakeFollowsA add flake.nix flakeB/flake.nix \
+  flakeB/flakeC/flake.nix flakeD/flake.nix flakeE/flake.nix
+
+nix flake lock $flakeFollowsA
+
+[[ $(jq -c .nodes.B.inputs.C $flakeFollowsA/flake.lock) = '"C"' ]]
+[[ $(jq -c .nodes.B.inputs.foobar $flakeFollowsA/flake.lock) = '["D"]' ]]
+[[ $(jq -c .nodes.C.inputs.foobar $flakeFollowsA/flake.lock) = '["B","foobar"]' ]]
+
+# Ensure a relative path is not allowed to go outside the store path
+cat > $flakeFollowsA/flake.nix <<EOF
+{
+    description = "Flake A";
+    inputs = {
+        B.url = "path:../flakeB";
+    };
+    outputs = { ... }: {};
+}
+EOF
+
+git -C $flakeFollowsA add flake.nix
+
+nix flake lock $flakeFollowsA 2>&1 | grep 'points outside'
+
+# Test flake in store does not evaluate
+rm -rf $badFlakeDir
+mkdir $badFlakeDir
+echo INVALID > $badFlakeDir/flake.nix
+nix store delete $(nix store add-path $badFlakeDir)
+
+[[ $(nix path-info      $(nix store add-path $flake1Dir)) =~ flake1 ]]
+[[ $(nix path-info path:$(nix store add-path $flake1Dir)) =~ simple ]]

tests/gc-auto.sh

@@ -1,5 +1,7 @@
 source common.sh
 
+needLocalStore "“min-free” and “max-free” are daemon options"
+
 clearStore
 
 garbage1=$(nix store add-path --name garbage1 ./nar-access.sh)

tests/gc-runtime.sh

@@ -4,7 +4,7 @@ case $system in
     *linux*)
         ;;
     *)
-        exit 0;
+        exit 99;
 esac
 
 set -m # enable job control, needed for kill

tests/gc.sh

@@ -12,7 +12,7 @@ ln -sf $outPath "$NIX_STATE_DIR"/gcroots/foo
 nix-store --gc --print-roots | grep $outPath
 nix-store --gc --print-live | grep $outPath
 nix-store --gc --print-dead | grep $drvPath
-if nix-store --gc --print-dead | grep $outPath; then false; fi
+if nix-store --gc --print-dead | grep -E $outPath$; then false; fi
 
 nix-store --gc --print-dead
 

tests/init.sh

@@ -23,6 +23,7 @@ substituters =
 flake-registry = $TEST_ROOT/registry.json
 show-trace = true
 include nix.conf.extra
+trusted-users = $(whoami)
 EOF
 
 cat > "$NIX_CONF_DIR"/nix.conf.extra <<EOF
@@ -35,5 +36,3 @@ nix-store --init
 
 # Did anything happen?
 test -e "$NIX_STATE_DIR"/db/db.sqlite
-
-echo 'Hello World' > ./dummy

tests/lang/eval-okay-floor-ceil.exp

@@ -0,0 +1 @@
+"23;24;23;23"

tests/lang/eval-okay-floor-ceil.nix

@@ -0,0 +1,9 @@
+with import ./lib.nix;
+
+let
+  n1 = builtins.floor 23.5;
+  n2 = builtins.ceil 23.5;
+  n3 = builtins.floor 23;
+  n4 = builtins.ceil 23;
+in
+  builtins.concatStringsSep ";" (map toString [ n1 n2 n3 n4 ])

tests/lang/eval-okay-path-antiquotation.nix

@@ -0,0 +1,12 @@
+let
+  foo = "foo";
+in
+{
+  simple = ./${foo};
+  surrounded = ./a-${foo}-b;
+  absolute = /${foo};
+  expr = ./${foo + "/bar"};
+  home = ~/${foo};
+  notfirst = ./bar/${foo};
+  slashes = /${foo}/${"bar"};
+}

tests/lang/parse-okay-url.nix

@@ -3,5 +3,6 @@
   http://www2.mplayerhq.hu/MPlayer/releases/fonts/font-arial-iso-8859-1.tar.bz2
   http://losser.st-lab.cs.uu.nl/~armijn/.nix/gcc-3.3.4-static-nix.tar.gz
   http://fpdownload.macromedia.com/get/shockwave/flash/english/linux/7.0r25/install_flash_player_7_linux.tar.gz
+  https://ftp5.gwdg.de/pub/linux/archlinux/extra/os/x86_64/unzip-6.0-14-x86_64.pkg.tar.zst
   ftp://ftp.gtk.org/pub/gtk/v1.2/gtk+-1.2.10.tar.gz
 ]

tests/linux-sandbox.sh

@@ -1,14 +1,16 @@
 source common.sh
 
+needLocalStore "the sandbox only runs on the builder side, so it makes no sense to test it with the daemon"
+
 clearStore
 
-if ! canUseSandbox; then exit; fi
+if ! canUseSandbox; then exit 99; fi
 
 # Note: we need to bind-mount $SHELL into the chroot. Currently we
 # only support the case where $SHELL is in the Nix store, because
 # otherwise things get complicated (e.g. if it's in /bin, do we need
 # /lib as well?).
-if [[ ! $SHELL =~ /nix/store ]]; then exit; fi
+if [[ ! $SHELL =~ /nix/store ]]; then exit 99; fi
 
 chmod -R u+w $TEST_ROOT/store0 || true
 rm -rf $TEST_ROOT/store0

tests/local-store.sh

@@ -15,6 +15,5 @@ PATH1=$(nix path-info --store ./x $CORRECT_PATH)
 PATH2=$(nix path-info --store "$PWD/x" $CORRECT_PATH)
 [ $CORRECT_PATH == $PATH2 ]
 
-# FIXME we could also test the query parameter version:
-# PATH3=$(nix path-info --store "local?store=$PWD/x" $CORRECT_PATH)
-# [ $CORRECT_PATH == $PATH3 ]
+PATH3=$(nix path-info --store "local?root=$PWD/x" $CORRECT_PATH)
+[ $CORRECT_PATH == $PATH3 ]

tests/local.mk

@@ -2,6 +2,7 @@ nix_tests = \
   hash.sh lang.sh add.sh simple.sh dependencies.sh \
   config.sh \
   gc.sh \
+  ca/gc.sh \
   gc-concurrent.sh \
   gc-auto.sh \
   referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \
@@ -11,6 +12,7 @@ nix_tests = \
   timeout.sh secure-drv-outputs.sh nix-channel.sh \
   multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \
   binary-cache.sh \
+  substitute-with-invalid-ca.sh \
   binary-cache-build-remote.sh \
   nix-profile.sh repair.sh dump-db.sh case-hack.sh \
   check-reqs.sh pass-as-file.sh tarball.sh restricted.sh \
@@ -30,12 +32,14 @@ nix_tests = \
   signing.sh \
   shell.sh \
   brotli.sh \
+  zstd.sh \
   pure-eval.sh \
   check.sh \
   plugins.sh \
   search.sh \
   nix-copy-ssh.sh \
   post-hook.sh \
+  ca/post-hook.sh \
   function-trace.sh \
   recursive.sh \
   describe-stores.sh \
@@ -43,16 +47,24 @@ nix_tests = \
   build.sh \
   build-explicit-output.sh \
   compute-levels.sh \
+  repl.sh \
   ca/build.sh \
+  ca/build-with-garbage-path.sh \
+  ca/duplicate-realisation-in-closure.sh \
   ca/substitute.sh \
   ca/signatures.sh \
-  ca/nix-copy.sh
+  ca/nix-shell.sh \
+  ca/nix-run.sh \
+  ca/recursive.sh \
+  ca/concurrent-builds.sh \
+  ca/nix-copy.sh \
+  eval-store.sh
   # parallel.sh
 
 install-tests += $(foreach x, $(nix_tests), tests/$(x))
 
 tests-environment = NIX_REMOTE= $(bash) -e
 
-clean-files += $(d)/common.sh $(d)/config.nix
+clean-files += $(d)/common.sh $(d)/config.nix $(d)/ca/config.nix
 
-test-deps += tests/common.sh tests/config.nix tests/plugins/libplugintest.$(SO_EXT)
+test-deps += tests/common.sh tests/config.nix tests/ca/config.nix tests/plugins/libplugintest.$(SO_EXT)

tests/multiple-outputs.sh

@@ -58,7 +58,7 @@ outPath2=$(nix-build $(nix-instantiate multiple-outputs.nix -A a.second) --no-ou
 
 # Delete one of the outputs and rebuild it.  This will cause a hash
 # rewrite.
-nix store delete $TEST_ROOT/result-second --ignore-liveness
+env -u NIX_REMOTE nix store delete $TEST_ROOT/result-second --ignore-liveness
 nix-build multiple-outputs.nix -A a.all -o $TEST_ROOT/result
 [ "$(cat $TEST_ROOT/result-second/file)" = "second" ]
 [ "$(cat $TEST_ROOT/result-second/link/file)" = "first" ]

tests/nix-shell.sh

@@ -2,6 +2,20 @@ source common.sh
 
 clearStore
 
+if [[ -n ${CONTENT_ADDRESSED:-} ]]; then
+    nix-shell () {
+        command nix-shell --arg contentAddressed true "$@"
+    }
+
+    nix_develop() {
+        nix develop --arg contentAddressed true "$@"
+    }
+else
+    nix_develop() {
+        nix develop "$@"
+    }
+fi
+
 # Test nix-shell -A
 export IMPURE_VAR=foo
 export SELECTED_IMPURE_VAR=baz
@@ -40,8 +54,12 @@ nix-instantiate shell.nix -A shellDrv --add-root $TEST_ROOT/shell
 output=$(NIX_PATH=nixpkgs=shell.nix nix-shell --pure -p foo bar --run 'echo "$(foo) $(bar)"')
 [ "$output" = "foo bar" ]
 
+# Test nix-shell -p --arg x y
+output=$(NIX_PATH=nixpkgs=shell.nix nix-shell --pure -p foo --argstr fooContents baz --run 'echo "$(foo)"')
+[ "$output" = "baz" ]
+
 # Test nix-shell shebang mode
-sed -e "s|@ENV_PROG@|$(type -p env)|" shell.shebang.sh > $TEST_ROOT/shell.shebang.sh
+sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.sh > $TEST_ROOT/shell.shebang.sh
 chmod a+rx $TEST_ROOT/shell.shebang.sh
 
 output=$($TEST_ROOT/shell.shebang.sh abc def)
@@ -49,7 +67,7 @@ output=$($TEST_ROOT/shell.shebang.sh abc def)
 
 # Test nix-shell shebang mode again with metacharacters in the filename.
 # First word of filename is chosen to not match any file in the test root.
-sed -e "s|@ENV_PROG@|$(type -p env)|" shell.shebang.sh > $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh
+sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.sh > $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh
 chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh
 
 output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.sh abc def)
@@ -58,7 +76,7 @@ output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.sh abc def)
 # Test nix-shell shebang mode for ruby
 # This uses a fake interpreter that returns the arguments passed
 # This, in turn, verifies the `rc` script is valid and the `load()` script (given using `-e`) is as expected.
-sed -e "s|@SHELL_PROG@|$(type -p nix-shell)|" shell.shebang.rb > $TEST_ROOT/shell.shebang.rb
+sed -e "s|@SHELL_PROG@|$(type -P nix-shell)|" shell.shebang.rb > $TEST_ROOT/shell.shebang.rb
 chmod a+rx $TEST_ROOT/shell.shebang.rb
 
 output=$($TEST_ROOT/shell.shebang.rb abc ruby)
@@ -66,21 +84,27 @@ output=$($TEST_ROOT/shell.shebang.rb abc ruby)
 
 # Test nix-shell shebang mode for ruby again with metacharacters in the filename.
 # Note: fake interpreter only space-separates args without adding escapes to its output.
-sed -e "s|@SHELL_PROG@|$(type -p nix-shell)|" shell.shebang.rb > $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb
+sed -e "s|@SHELL_PROG@|$(type -P nix-shell)|" shell.shebang.rb > $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb
 chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb
 
 output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.rb abc ruby)
 [ "$output" = '-e load(ARGV.shift) -- '"$TEST_ROOT"'/spaced \'\''"shell.shebang.rb abc ruby' ]
 
 # Test 'nix develop'.
-nix develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]'
+nix_develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]'
 
 # Ensure `nix develop -c` preserves stdin
 echo foo | nix develop -f shell.nix shellDrv -c cat | grep -q foo
 
 # Ensure `nix develop -c` actually executes the command if stdout isn't a terminal
-nix develop -f shell.nix shellDrv -c echo foo |& grep -q foo
+nix_develop -f shell.nix shellDrv -c echo foo |& grep -q foo
 
 # Test 'nix print-dev-env'.
+[[ $(nix print-dev-env -f shell.nix shellDrv --json | jq -r .variables.arr1.value[2]) = '3 4' ]]
+
 source <(nix print-dev-env -f shell.nix shellDrv)
 [[ -n $stdenv ]]
+[[ ${arr1[2]} = "3 4" ]]
+[[ ${arr2[1]} = $'\n' ]]
+[[ ${arr2[2]} = $'x\ny' ]]
+[[ $(fun) = blabla ]]

tests/optimise-store.sh

@@ -26,7 +26,8 @@ if [ "$inode1" = "$inode3" ]; then
     exit 1
 fi
 
-nix-store --optimise
+# XXX: This should work through the daemon too
+NIX_REMOTE="" nix-store --optimise
 
 inode1="$(stat --format=%i $outPath1/foo)"
 inode3="$(stat --format=%i $outPath3/foo)"

tests/post-hook.sh

@@ -4,7 +4,10 @@ clearStore
 
 rm -f $TEST_ROOT/result
 
-export REMOTE_STORE=$TEST_ROOT/remote_store
+export REMOTE_STORE=file:$TEST_ROOT/remote_store
+echo 'require-sigs = false' >> $NIX_CONF_DIR/nix.conf
+
+restartDaemon
 
 # Build the dependencies and push them to the remote store
 nix-build -o $TEST_ROOT/result dependencies.nix --post-build-hook $PWD/push-to-store.sh

tests/recursive.sh

@@ -1,7 +1,10 @@
 source common.sh
 
+sed -i 's/experimental-features .*/& recursive-nix/' "$NIX_CONF_DIR"/nix.conf
+restartDaemon
+
 # FIXME
-if [[ $(uname) != Linux ]]; then exit; fi
+if [[ $(uname) != Linux ]]; then exit 99; fi
 
 clearStore
 
@@ -9,9 +12,9 @@ rm -f $TEST_ROOT/result
 
 export unreachable=$(nix store add-path ./recursive.sh)
 
-NIX_BIN_DIR=$(dirname $(type -p nix)) nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --expr '
+NIX_BIN_DIR=$(dirname $(type -p nix)) nix --extra-experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --expr '
   with import ./config.nix;
-  mkDerivation {
+  mkDerivation rec {
     name = "recursive";
     dummy = builtins.toFile "dummy" "bla bla";
     SHELL = shell;
@@ -19,11 +22,13 @@ NIX_BIN_DIR=$(dirname $(type -p nix)) nix --experimental-features 'nix-command r
     # Note: this is a string without context.
     unreachable = builtins.getEnv "unreachable";
 
+    NIX_TESTS_CA_BY_DEFAULT = builtins.getEnv "NIX_TESTS_CA_BY_DEFAULT";
+
     requiredSystemFeatures = [ "recursive-nix" ];
 
     buildCommand = '\'\''
       mkdir $out
-      opts="--experimental-features nix-command"
+      opts="--experimental-features nix-command ${if (NIX_TESTS_CA_BY_DEFAULT == "1") then "--extra-experimental-features ca-derivations" else ""}"
 
       PATH=${builtins.getEnv "NIX_BIN_DIR"}:$PATH
 
@@ -46,16 +51,15 @@ NIX_BIN_DIR=$(dirname $(type -p nix)) nix --experimental-features 'nix-command r
       # Add it to our closure.
       ln -s $foobar $out/foobar
 
-      [[ $(nix $opts path-info --all | wc -l) -eq 3 ]]
+      [[ $(nix $opts path-info --all | wc -l) -eq 4 ]]
 
       # Build a derivation.
       nix $opts build -L --impure --expr '\''
-        derivation {
+        with import ${./config.nix};
+        mkDerivation {
           name = "inner1";
-          builder = builtins.getEnv "SHELL";
-          system = builtins.getEnv "system";
+          buildCommand = "echo $fnord blaat > $out";
           fnord = builtins.toFile "fnord" "fnord";
-          args = [ "-c" "echo $fnord blaat > $out" ];
         }
       '\''
 

tests/referrers.sh

@@ -1,5 +1,7 @@
 source common.sh
 
+needLocalStore "uses some low-level store manipulations that aren’t available through the daemon"
+
 clearStore
 
 max=500

tests/repair.sh

@@ -1,5 +1,7 @@
 source common.sh
 
+needLocalStore "--repair needs a local store"
+
 clearStore
 
 path=$(nix-build dependencies.nix -o $TEST_ROOT/result)

tests/repl.sh

@@ -0,0 +1,18 @@
+source common.sh
+
+replCmds="
+simple = import ./simple.nix
+:b simple
+"
+
+testRepl () {
+    local nixArgs=("$@")
+    local outPath=$(nix repl "${nixArgs[@]}" <<< "$replCmds" |&
+        grep -o -E "$NIX_STORE_DIR/\w*-simple")
+    nix path-info "${nixArgs[@]}" "$outPath"
+}
+
+# Simple test, try building a drv
+testRepl
+# Same thing (kind-of), but with a remote store.
+testRepl --store "$TEST_ROOT/store?real=$NIX_STORE_DIR"

tests/shell.nix

@@ -1,6 +1,18 @@
-{ inNixShell ? false }:
+{ inNixShell ? false, contentAddressed ? false, fooContents ? "foo" }:
 
-with import ./config.nix;
+let cfg = import ./config.nix; in
+with cfg;
+
+let
+  mkDerivation =
+    if contentAddressed then
+      args: cfg.mkDerivation ({
+        __contentAddressed = true;
+        outputHashMode = "recursive";
+        outputHashAlgo = "sha256";
+      } // args)
+    else cfg.mkDerivation;
+in
 
 let pkgs = rec {
   setupSh = builtins.toFile "setup" ''
@@ -8,6 +20,20 @@ let pkgs = rec {
     for pkg in $buildInputs; do
       export PATH=$PATH:$pkg/bin
     done
+
+    # mimic behavior of stdenv for `$out` etc. for structured attrs.
+    if [ -n "''${NIX_ATTRS_SH_FILE}" ]; then
+      for o in "''${!outputs[@]}"; do
+        eval "''${o}=''${outputs[$o]}"
+        export "''${o}"
+      done
+    fi
+
+    declare -a arr1=(1 2 "3 4" 5)
+    declare -a arr2=(x $'\n' $'x\ny')
+    fun() {
+      echo blabla
+    }
   '';
 
   stdenv = mkDerivation {
@@ -22,6 +48,8 @@ let pkgs = rec {
     name = "shellDrv";
     builder = "/does/not/exist";
     VAR_FROM_NIX = "bar";
+    ASCII_PERCENT = "%";
+    ASCII_AT = "@";
     TEST_inNixShell = if inNixShell then "true" else "false";
     inherit stdenv;
     outputs = ["dev" "out"];
@@ -34,7 +62,7 @@ let pkgs = rec {
 
   foo = runCommand "foo" {} ''
     mkdir -p $out/bin
-    echo 'echo foo' > $out/bin/foo
+    echo 'echo ${fooContents}' > $out/bin/foo
     chmod a+rx $out/bin/foo
     ln -s ${shell} $out/bin/bash
   '';

tests/shell.sh

@@ -6,7 +6,7 @@ clearCache
 nix shell -f shell-hello.nix hello -c hello | grep 'Hello World'
 nix shell -f shell-hello.nix hello -c hello NixOS | grep 'Hello NixOS'
 
-if ! canUseSandbox; then exit; fi
+if ! canUseSandbox; then exit 99; fi
 
 chmod -R u+w $TEST_ROOT/store0 || true
 rm -rf $TEST_ROOT/store0

tests/structured-attrs-shell.nix

@@ -0,0 +1,21 @@
+with import ./config.nix;
+let
+  dep = mkDerivation {
+    name = "dep";
+    buildCommand = ''
+      mkdir $out; echo bla > $out/bla
+    '';
+  };
+  inherit (import ./shell.nix { inNixShell = true; }) stdenv;
+in
+mkDerivation {
+  name = "structured2";
+  __structuredAttrs = true;
+  inherit stdenv;
+  outputs = [ "out" "dev" ];
+  my.list = [ "a" "b" "c" ];
+  exportReferencesGraph.refs = [ dep ];
+  buildCommand = ''
+    touch ''${outputs[out]}; touch ''${outputs[dev]}
+  '';
+}

tests/structured-attrs.nix

@@ -36,7 +36,7 @@ mkDerivation {
     echo bar > $dest
     echo foo > $dest2
 
-    json=$(cat .attrs.json)
+    json=$(cat $NIX_ATTRS_JSON_FILE)
     [[ $json =~ '"narHash":"sha256:1r7yc43zqnzl5b0als5vnyp649gk17i37s7mj00xr8kc47rjcybk"' ]]
     [[ $json =~ '"narSize":288' ]]
     [[ $json =~ '"closureSize":288' ]]

tests/structured-attrs.sh

@@ -1,5 +1,9 @@
 source common.sh
 
+# 27ce722638 required some incompatible changes to the nix file, so skip this
+# tests for the older versions
+requireDaemonNewerThan "2.4pre20210622"
+
 clearStore
 
 rm -f $TEST_ROOT/result
@@ -8,3 +12,12 @@ nix-build structured-attrs.nix -A all -o $TEST_ROOT/result
 
 [[ $(cat $TEST_ROOT/result/foo) = bar ]]
 [[ $(cat $TEST_ROOT/result-dev/foo) = foo ]]
+
+export NIX_BUILD_SHELL=$SHELL
+env NIX_PATH=nixpkgs=shell.nix nix-shell structured-attrs-shell.nix \
+    --run 'test -e .attrs.json; test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"'
+
+# `nix develop` is a slightly special way of dealing with environment vars, it parses
+# these from a shell-file exported from a derivation. This is to test especially `outputs`
+# (which is an associative array in thsi case) being fine.
+nix develop -f structured-attrs-shell.nix -c bash -c 'test -n "$out"'

tests/substitute-with-invalid-ca.sh

@@ -0,0 +1,38 @@
+source common.sh
+
+BINARY_CACHE=file://$cacheDir
+
+getHash() {
+    basename "$1" | cut -d '-' -f 1
+}
+getRemoteNarInfo () {
+    echo "$cacheDir/$(getHash "$1").narinfo"
+}
+
+cat <<EOF > $TEST_HOME/good.txt
+I’m a good path
+EOF
+
+cat <<EOF > $TEST_HOME/bad.txt
+I’m a bad path
+EOF
+
+good=$(nix-store --add $TEST_HOME/good.txt)
+bad=$(nix-store --add $TEST_HOME/bad.txt)
+nix copy --to "$BINARY_CACHE" "$good"
+nix copy --to "$BINARY_CACHE" "$bad"
+nix-collect-garbage >/dev/null 2>&1
+
+# Falsifying the narinfo file for '$good'
+goodPathNarInfo=$(getRemoteNarInfo "$good")
+badPathNarInfo=$(getRemoteNarInfo "$bad")
+for fieldName in URL FileHash FileSize NarHash NarSize; do
+    sed -i "/^$fieldName/d" "$goodPathNarInfo"
+    grep -E "^$fieldName" "$badPathNarInfo" >> "$goodPathNarInfo"
+done
+
+# Copying back '$good' from the binary cache. This should fail as it is
+# corrupted
+if nix copy --from "$BINARY_CACHE" "$good"; then
+    fail "Importing a path with a wrong CA field should fail"
+fi

tests/tarball.sh

@@ -40,6 +40,11 @@ test_tarball() {
     (! nix-instantiate --eval -E '<fnord/xyzzy> 1' -I fnord=file://no-such-tarball$ext)
 
     nix-instantiate --eval -E '<fnord/config.nix>' -I fnord=file://no-such-tarball$ext -I fnord=.
+
+    # Ensure that the `name` attribute isn’t accepted as that would mess
+    # with the content-addressing
+    (! nix-instantiate --eval -E "fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; name = \"foo\"; }")
+
 }
 
 test_tarball '' cat

tests/timeout.sh

@@ -2,6 +2,8 @@
 
 source common.sh
 
+# XXX: This shouldn’t be, but #4813 cause this test to fail
+needLocalStore "see #4813"
 
 set +e
 messages=$(nix-build -Q timeout.nix -A infiniteLoop --timeout 2 2>&1)

tests/zstd.sh

@@ -0,0 +1,28 @@
+source common.sh
+
+clearStore
+clearCache
+
+cacheURI="file://$cacheDir?compression=zstd"
+
+outPath=$(nix-build dependencies.nix --no-out-link)
+
+nix copy --to $cacheURI $outPath
+
+HASH=$(nix hash path $outPath)
+
+clearStore
+clearCacheCache
+
+nix copy --from $cacheURI $outPath --no-check-sigs
+
+if ls $cacheDir/nar/*.zst &> /dev/null; then
+    echo "files do exist"
+else
+    echo "nars do not exist"
+    exit 1
+fi
+
+HASH2=$(nix hash path $outPath)
+
+[[ $HASH = $HASH2 ]]