Merge branch 'master' into errors-phase-2

tests/check.nix

@@ -1,12 +1,45 @@
+{checkBuildId ? 0}:
+
 with import ./config.nix;
 
 {
   nondeterministic = mkDerivation {
+    inherit checkBuildId;
     name = "nondeterministic";
     buildCommand =
       ''
         mkdir $out
         date +%s.%N > $out/date
+        echo "CHECK_TMPDIR=$TMPDIR"
+        echo "checkBuildId=$checkBuildId"
+        echo "$checkBuildId" > $TMPDIR/checkBuildId
+      '';
+  };
+
+  deterministic = mkDerivation {
+    inherit checkBuildId;
+    name = "deterministic";
+    buildCommand =
+      ''
+        mkdir $out
+        echo date > $out/date
+        echo "CHECK_TMPDIR=$TMPDIR"
+        echo "checkBuildId=$checkBuildId"
+        echo "$checkBuildId" > $TMPDIR/checkBuildId
+      '';
+  };
+
+  failed = mkDerivation {
+    inherit checkBuildId;
+    name = "failed";
+    buildCommand =
+      ''
+        mkdir $out
+        echo date > $out/date
+        echo "CHECK_TMPDIR=$TMPDIR"
+        echo "checkBuildId=$checkBuildId"
+        echo "$checkBuildId" > $TMPDIR/checkBuildId
+        false
       '';
   };
 

tests/check.sh

@@ -1,14 +1,57 @@
 source common.sh
 
+checkBuildTempDirRemoved ()
+{
+    buildDir=$(sed -n 's/CHECK_TMPDIR=//p' $1 | head -1)
+    checkBuildIdFile=${buildDir}/checkBuildId
+    [[ ! -f $checkBuildIdFile ]] || ! grep $checkBuildId $checkBuildIdFile
+}
+
+# written to build temp directories to verify created by this instance
+checkBuildId=$(date +%s%N)
+
 clearStore
 
 nix-build dependencies.nix --no-out-link
 nix-build dependencies.nix --no-out-link --check
 
-nix-build check.nix -A nondeterministic --no-out-link
-nix-build check.nix -A nondeterministic --no-out-link --check 2> $TEST_ROOT/log || status=$?
+# check for dangling temporary build directories
+# only retain if build fails and --keep-failed is specified, or...
+# ...build is non-deterministic and --check and --keep-failed are both specified
+nix-build check.nix -A failed --argstr checkBuildId $checkBuildId \
+    --no-out-link 2> $TEST_ROOT/log || status=$?
+[ "$status" = "100" ]
+checkBuildTempDirRemoved $TEST_ROOT/log
+
+nix-build check.nix -A failed --argstr checkBuildId $checkBuildId \
+    --no-out-link --keep-failed 2> $TEST_ROOT/log || status=$?
+[ "$status" = "100" ]
+if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi
+
+nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \
+    --no-out-link 2> $TEST_ROOT/log
+checkBuildTempDirRemoved $TEST_ROOT/log
+
+nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \
+    --no-out-link --check --keep-failed 2> $TEST_ROOT/log
+if grep -q 'may not be deterministic' $TEST_ROOT/log; then false; fi
+checkBuildTempDirRemoved $TEST_ROOT/log
+
+nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \
+    --no-out-link 2> $TEST_ROOT/log
+checkBuildTempDirRemoved $TEST_ROOT/log
+
+nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \
+    --no-out-link --check 2> $TEST_ROOT/log || status=$?
 grep 'may not be deterministic' $TEST_ROOT/log
 [ "$status" = "104" ]
+checkBuildTempDirRemoved $TEST_ROOT/log
+
+nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \
+    --no-out-link --check --keep-failed 2> $TEST_ROOT/log || status=$?
+grep 'may not be deterministic' $TEST_ROOT/log
+[ "$status" = "104" ]
+if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi
 
 clearStore
 

tests/common.sh.in

@@ -11,6 +11,7 @@ export NIX_LOCALSTATE_DIR=$TEST_ROOT/var
 export NIX_LOG_DIR=$TEST_ROOT/var/log/nix
 export NIX_STATE_DIR=$TEST_ROOT/var/nix
 export NIX_CONF_DIR=$TEST_ROOT/etc
+unset NIX_USER_CONF_FILES
 export _NIX_TEST_SHARED=$TEST_ROOT/shared
 if [[ -n $NIX_STORE ]]; then
     export _NIX_TEST_NO_SANDBOX=1
@@ -21,6 +22,8 @@ export NIX_REMOTE=$NIX_REMOTE_
 unset NIX_PATH
 export TEST_HOME=$TEST_ROOT/test-home
 export HOME=$TEST_HOME
+unset XDG_CONFIG_HOME
+unset XDG_CONFIG_DIRS
 unset XDG_CACHE_HOME
 mkdir -p $TEST_HOME
 

tests/config.sh

@@ -0,0 +1,18 @@
+source common.sh
+
+# Test that files are loaded from XDG by default
+export XDG_CONFIG_HOME=/tmp/home
+export XDG_CONFIG_DIRS=/tmp/dir1:/tmp/dir2
+files=$(nix-build --verbose --version | grep "User config" | cut -d ':' -f2- | xargs)
+[[ $files == "/tmp/home/nix/nix.conf:/tmp/dir1/nix/nix.conf:/tmp/dir2/nix/nix.conf" ]]
+
+# Test that setting NIX_USER_CONF_FILES overrides all the default user config files
+export NIX_USER_CONF_FILES=/tmp/file1.conf:/tmp/file2.conf
+files=$(nix-build --verbose --version | grep "User config" | cut -d ':' -f2- | xargs)
+[[ $files == "/tmp/file1.conf:/tmp/file2.conf" ]]
+
+# Test that it's possible to load the config from a custom location
+here=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")
+export NIX_USER_CONF_FILES=$here/config/nix-with-substituters.conf
+var=$(nix show-config | grep '^substituters =' | cut -d '=' -f 2 | xargs)
+[[ $var == https://example.com ]]

tests/config/nix-with-substituters.conf

@@ -0,0 +1,2 @@
+experimental-features = nix-command
+substituters = https://example.com

tests/fetchGit.sh

@@ -11,7 +11,7 @@ repo=$TEST_ROOT/git
 
 export _NIX_FORCE_HTTP=1
 
-rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/gitv2
+rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix $TEST_ROOT/worktree $TEST_ROOT/shallow
 
 git init $repo
 git -C $repo config user.email "foobar@example.com"
@@ -25,8 +25,16 @@ rev1=$(git -C $repo rev-parse HEAD)
 
 echo world > $repo/hello
 git -C $repo commit -m 'Bla2' -a
+git -C $repo worktree add $TEST_ROOT/worktree
+echo hello >> $TEST_ROOT/worktree/hello
 rev2=$(git -C $repo rev-parse HEAD)
 
+# Fetch a worktree
+unset _NIX_FORCE_HTTP
+path0=$(nix eval --raw "(builtins.fetchGit file://$TEST_ROOT/worktree).outPath")
+export _NIX_FORCE_HTTP=1
+[[ $(tail -n 1 $path0/hello) = "hello" ]]
+
 # Fetch the default branch.
 path=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath")
 [[ $(cat $path/hello) = world ]]
@@ -50,9 +58,6 @@ path2=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath")
 [[ $(nix eval "(builtins.fetchGit file://$repo).revCount") = 2 ]]
 [[ $(nix eval --raw "(builtins.fetchGit file://$repo).rev") = $rev2 ]]
 
-# But with TTL 0, it should fail.
-(! nix eval --tarball-ttl 0 "(builtins.fetchGit file://$repo)" -vvvvv)
-
 # Fetching with a explicit hash should succeed.
 path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath")
 [[ $path = $path2 ]]
@@ -74,6 +79,7 @@ echo bar > $repo/dir2/bar
 git -C $repo add dir1/foo
 git -C $repo rm hello
 
+unset _NIX_FORCE_HTTP
 path2=$(nix eval --raw "(builtins.fetchGit $repo).outPath")
 [ ! -e $path2/hello ]
 [ ! -e $path2/bar ]
@@ -110,9 +116,9 @@ path=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath")
 git -C $repo checkout $rev2 -b dev
 echo dev > $repo/hello
 
-# File URI uses 'master' unless specified otherwise
+# File URI uses dirty tree unless specified otherwise
 path2=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath")
-[[ $path = $path2 ]]
+[ $(cat $path2/hello) = dev ]
 
 # Using local path with branch other than 'master' should work when clean or dirty
 path3=$(nix eval --raw "(builtins.fetchGit $repo).outPath")
@@ -131,9 +137,9 @@ path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outP
 
 
 # Nuke the cache
-rm -rf $TEST_HOME/.cache/nix/gitv2
+rm -rf $TEST_HOME/.cache/nix
 
-# Try again, but without 'git' on PATH
+# Try again, but without 'git' on PATH. This should fail.
 NIX=$(command -v nix)
 # This should fail
 (! PATH= $NIX eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath" )
@@ -141,3 +147,13 @@ NIX=$(command -v nix)
 # Try again, with 'git' available.  This should work.
 path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath")
 [[ $path3 = $path5 ]]
+
+# Fetching a shallow repo shouldn't work by default, because we can't
+# return a revCount.
+git clone --depth 1 file://$repo $TEST_ROOT/shallow
+(! nix eval --raw "(builtins.fetchGit { url = $TEST_ROOT/shallow; ref = \"dev\"; }).outPath")
+
+# But you can request a shallow clone, which won't return a revCount.
+path6=$(nix eval --raw "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).outPath")
+[[ $path3 = $path6 ]]
+[[ $(nix eval "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).revCount or 123") == 123 ]]

tests/fetchGitSubmodules.sh

@@ -0,0 +1,97 @@
+source common.sh
+
+set -u
+
+if [[ -z $(type -p git) ]]; then
+    echo "Git not installed; skipping Git submodule tests"
+    exit 99
+fi
+
+clearStore
+
+rootRepo=$TEST_ROOT/gitSubmodulesRoot
+subRepo=$TEST_ROOT/gitSubmodulesSub
+
+rm -rf ${rootRepo} ${subRepo} $TEST_HOME/.cache/nix
+
+initGitRepo() {
+    git init $1
+    git -C $1 config user.email "foobar@example.com"
+    git -C $1 config user.name "Foobar"
+}
+
+addGitContent() {
+    echo "lorem ipsum" > $1/content
+    git -C $1 add content
+    git -C $1 commit -m "Initial commit"
+}
+
+initGitRepo $subRepo
+addGitContent $subRepo
+
+initGitRepo $rootRepo
+
+git -C $rootRepo submodule init
+git -C $rootRepo submodule add $subRepo sub
+git -C $rootRepo add sub
+git -C $rootRepo commit -m "Add submodule"
+
+rev=$(git -C $rootRepo rev-parse HEAD)
+
+r1=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath")
+r2=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = false; }).outPath")
+r3=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
+
+[[ $r1 == $r2 ]]
+[[ $r2 != $r3 ]]
+
+r4=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; }).outPath")
+r5=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = false; }).outPath")
+r6=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath")
+r7=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath")
+r8=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
+
+[[ $r1 == $r4 ]]
+[[ $r4 == $r5 ]]
+[[ $r3 == $r6 ]]
+[[ $r6 == $r7 ]]
+[[ $r7 == $r8 ]]
+
+have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; }).submodules")
+[[ $have_submodules == false ]]
+
+have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = false; }).submodules")
+[[ $have_submodules == false ]]
+
+have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).submodules")
+[[ $have_submodules == true ]]
+
+pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath")
+pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
+pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
+pathWithSubmodulesAgainWithRef=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath")
+
+# The resulting store path cannot be the same.
+[[ $pathWithoutSubmodules != $pathWithSubmodules ]]
+
+# Checking out the same repo with submodules returns in the same store path.
+[[ $pathWithSubmodules == $pathWithSubmodulesAgain ]]
+
+# Checking out the same repo with submodules returns in the same store path.
+[[ $pathWithSubmodulesAgain == $pathWithSubmodulesAgainWithRef ]]
+
+# The submodules flag is actually honored.
+[[ ! -e $pathWithoutSubmodules/sub/content ]]
+[[ -e $pathWithSubmodules/sub/content ]]
+
+[[ -e $pathWithSubmodulesAgainWithRef/sub/content ]]
+
+# No .git directory or submodule reference files must be left
+test "$(find "$pathWithSubmodules" -name .git)" = ""
+
+# Git repos without submodules can be fetched with submodules = true.
+subRev=$(git -C $subRepo rev-parse HEAD)
+noSubmoduleRepoBaseline=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; }).outPath")
+noSubmoduleRepo=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; submodules = true; }).outPath")
+
+[[ $noSubmoduleRepoBaseline == $noSubmoduleRepo ]]

tests/init.sh

@@ -17,7 +17,7 @@ cat > "$NIX_CONF_DIR"/nix.conf <<EOF
 build-users-group =
 keep-derivations = false
 sandbox = false
-experimental-features = nix-command
+experimental-features = nix-command flakes
 include nix.conf.extra
 EOF
 

tests/lang/eval-okay-getattrpos-functionargs.exp

@@ -0,0 +1 @@
+{ column = 11; file = "eval-okay-getattrpos-functionargs.nix"; line = 2; }

tests/lang/eval-okay-getattrpos-functionargs.nix

@@ -0,0 +1,4 @@
+let
+  fun = { foo }: {};
+  pos = builtins.unsafeGetAttrPos "foo" (builtins.functionArgs fun);
+in { inherit (pos) column line; file = baseNameOf pos.file; }

tests/linux-sandbox.sh

@@ -28,3 +28,10 @@ nix cat-store $outPath/foobar | grep FOOBAR
 
 # Test --check without hash rewriting.
 nix-build dependencies.nix --no-out-link --check --sandbox-paths /nix/store
+
+# Test that sandboxed builds with --check and -K can move .check directory to store
+nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link
+
+(! nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link --check -K 2> $TEST_ROOT/log)
+if grep -q 'error: renaming' $TEST_ROOT/log; then false; fi
+grep -q 'may not be deterministic' $TEST_ROOT/log

tests/local.mk

@@ -1,5 +1,6 @@
 nix_tests = \
   init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \
+  config.sh \
   gc.sh \
   gc-concurrent.sh \
   gc-auto.sh \
@@ -17,9 +18,10 @@ nix_tests = \
   nar-access.sh \
   structured-attrs.sh \
   fetchGit.sh \
+  fetchGitSubmodules.sh \
   fetchMercurial.sh \
   signing.sh \
-  run.sh \
+  shell.sh \
   brotli.sh \
   pure-eval.sh \
   check.sh \

tests/run.sh

@@ -1,28 +0,0 @@
-source common.sh
-
-clearStore
-clearCache
-
-nix run -f run.nix hello -c hello | grep 'Hello World'
-nix run -f run.nix hello -c hello NixOS | grep 'Hello NixOS'
-
-if ! canUseSandbox; then exit; fi
-
-chmod -R u+w $TEST_ROOT/store0 || true
-rm -rf $TEST_ROOT/store0
-
-clearStore
-
-path=$(nix eval --raw -f run.nix hello)
-
-# Note: we need the sandbox paths to ensure that the shell is
-# visible in the sandbox.
-nix run --sandbox-build-dir /build-tmp \
-    --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' \
-    --store $TEST_ROOT/store0 -f run.nix hello -c hello | grep 'Hello World'
-
-path2=$(nix run --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' --store $TEST_ROOT/store0 -f run.nix hello -c $SHELL -c 'type -p hello')
-
-[[ $path/bin/hello = $path2 ]]
-
-[[ -e $TEST_ROOT/store0/nix/store/$(basename $path)/bin/hello ]]

tests/shell.sh

@@ -0,0 +1,28 @@
+source common.sh
+
+clearStore
+clearCache
+
+nix shell -f shell-hello.nix hello -c hello | grep 'Hello World'
+nix shell -f shell-hello.nix hello -c hello NixOS | grep 'Hello NixOS'
+
+if ! canUseSandbox; then exit; fi
+
+chmod -R u+w $TEST_ROOT/store0 || true
+rm -rf $TEST_ROOT/store0
+
+clearStore
+
+path=$(nix eval --raw -f shell-hello.nix hello)
+
+# Note: we need the sandbox paths to ensure that the shell is
+# visible in the sandbox.
+nix shell --sandbox-build-dir /build-tmp \
+    --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' \
+    --store $TEST_ROOT/store0 -f shell-hello.nix hello -c hello | grep 'Hello World'
+
+path2=$(nix shell --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' --store $TEST_ROOT/store0 -f shell-hello.nix hello -c $SHELL -c 'type -p hello')
+
+[[ $path/bin/hello = $path2 ]]
+
+[[ -e $TEST_ROOT/store0/nix/store/$(basename $path)/bin/hello ]]

tests/tarball.sh

@@ -10,6 +10,8 @@ mkdir -p $tarroot
 cp dependencies.nix $tarroot/default.nix
 cp config.nix dependencies.builder*.sh $tarroot/
 
+hash=$(nix hash-path $tarroot)
+
 test_tarball() {
     local ext="$1"
     local compressor="$2"
@@ -25,6 +27,11 @@ test_tarball() {
 
     nix-build -o $TEST_ROOT/result -E "import (fetchTarball file://$tarball)"
 
+    nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)"
+    nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })"
+    nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })"
+    nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" 2>&1 | grep 'NAR hash mismatch in input'
+
     nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar$ext
     nix-instantiate --eval -E 'with <fnord/xyzzy>; 1 + 2' -I fnord=file://no-such-tarball$ext
     (! nix-instantiate --eval -E '<fnord/xyzzy> 1' -I fnord=file://no-such-tarball$ext)