Merge remote-tracking branch 'upstream/master' into overlayfs-store

tests/dyn-drv/local.mk

@@ -1,11 +0,0 @@
-dyn-drv-tests := \
-  $(d)/text-hashed-output.sh \
-  $(d)/recursive-mod-json.sh
-
-install-tests-groups += dyn-drv
-
-clean-files += \
-  $(d)/config.nix
-
-test-deps += \
-  tests/dyn-drv/config.nix

tests/functional/bash-profile.sh

@@ -1,6 +1,6 @@
 source common.sh
 
-sed -e "s|@localstatedir@|$TEST_ROOT/profile-var|g" -e "s|@coreutils@|$coreutils|g" < ../scripts/nix-profile.sh.in > $TEST_ROOT/nix-profile.sh
+sed -e "s|@localstatedir@|$TEST_ROOT/profile-var|g" -e "s|@coreutils@|$coreutils|g" < ../../scripts/nix-profile.sh.in > $TEST_ROOT/nix-profile.sh
 
 user=$(whoami)
 rm -rf $TEST_HOME $TEST_ROOT/profile-var

tests/functional/build-hook-ca-fixed.nix

@@ -8,7 +8,10 @@ let
     derivation ({
       inherit system;
       builder = busybox;
-      args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
+      args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" ''
+        if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi;
+        eval "$buildCommand"
+      '')];
       outputHashMode = "recursive";
       outputHashAlgo = "sha256";
     } // removeAttrs args ["builder" "meta" "passthru"])

tests/functional/build-hook.nix

@@ -14,7 +14,10 @@ let
     derivation ({
       inherit system;
       builder = busybox;
-      args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
+      args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" ''
+        if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi;
+        eval "$buildCommand"
+      '')];
     } // removeAttrs args ["builder" "meta" "passthru"]
     // caArgs)
     // { meta = args.meta or {}; passthru = args.passthru or {}; };

tests/functional/build-remote-trustless.sh

@@ -6,7 +6,7 @@ unset NIX_STATE_DIR
 
 remoteDir=$TEST_ROOT/remote
 
-# Note: ssh{-ng}://localhost bypasses ssh. See tests/build-remote.sh for
+# Note: ssh{-ng}://localhost bypasses ssh. See tests/functional/build-remote.sh for
 # more details.
 nix-build $file -o $TEST_ROOT/result --max-jobs 0 \
   --arg busybox $busybox \

tests/functional/build.sh

@@ -78,7 +78,7 @@ expectStderr 1 nix build --impure --expr 'with (import ./multiple-outputs.nix).e
   | grepQuiet "has 2 entries in its context. It should only have exactly one entry"
 
 nix build --impure --json --expr 'builtins.unsafeDiscardOutputDependency (import ./multiple-outputs.nix).e.a_a.drvPath' --no-link | jq --exit-status '
-  (.[0] | .path | match(".*multiple-outputs-e.drv"))
+  (.[0] | match(".*multiple-outputs-e.drv"))
 '
 
 # Test building from raw store path to drv not expression.

tests/functional/ca/build-cache.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+source common.sh
+
+# The substituters didn't work prior to this time.
+requireDaemonNewerThan "2.18.0pre20230808"
+
+drv=$(nix-instantiate ./content-addressed.nix -A rootCA --arg seed 1)^out
+nix derivation show "$drv" --arg seed 1
+
+buildAttr () {
+    local derivationPath=$1
+    local seedValue=$2
+    shift; shift
+    local args=("./content-addressed.nix" "-A" "$derivationPath" --arg seed "$seedValue" "--no-out-link")
+    args+=("$@")
+    nix-build "${args[@]}"
+}
+
+copyAttr () {
+    local derivationPath=$1
+    local seedValue=$2
+    shift; shift
+    local args=("-f" "./content-addressed.nix" "$derivationPath" --arg seed "$seedValue")
+    args+=("$@")
+    # Note: to copy CA derivations, we need to copy the realisations, which
+    # currently requires naming the installables, not just the derivation output
+    # path.
+    nix copy --to file://$cacheDir "${args[@]}"
+}
+
+testRemoteCacheFor () {
+    local derivationPath=$1
+    clearCache
+    copyAttr "$derivationPath" 1
+    clearStore
+    # Check nothing gets built.
+    buildAttr "$derivationPath" 1 --option substituters file://$cacheDir --no-require-sigs |& grepQuietInverse " will be built:"
+}
+
+testRemoteCache () {
+    testRemoteCacheFor rootCA
+    testRemoteCacheFor dependentCA
+    testRemoteCacheFor dependentNonCA
+    testRemoteCacheFor dependentFixedOutput
+    testRemoteCacheFor dependentForBuildCA
+    testRemoteCacheFor dependentForBuildNonCA
+}
+
+clearStore
+testRemoteCache

tests/functional/ca/build.sh

@@ -2,7 +2,7 @@
 
 source common.sh
 
-drv=$(nix-instantiate ./content-addressed.nix -A rootCA --arg seed 1)
+drv=$(nix-instantiate ./content-addressed.nix -A rootCA --arg seed 1)^out
 nix derivation show "$drv" --arg seed 1
 
 buildAttr () {
@@ -14,14 +14,6 @@ buildAttr () {
     nix-build "${args[@]}"
 }
 
-testRemoteCache () {
-    clearCache
-    local outPath=$(buildAttr dependentNonCA 1)
-    nix copy --to file://$cacheDir $outPath
-    clearStore
-    buildAttr dependentNonCA 1 --option substituters file://$cacheDir --no-require-sigs |& grepQuietInverse "building dependent-non-ca"
-}
-
 testDeterministicCA () {
     [[ $(buildAttr rootCA 1) = $(buildAttr rootCA 2) ]]
 }
@@ -66,8 +58,6 @@ testNormalization () {
     test "$(stat -c %Y $outPath)" -eq 1
 }
 
-# Disabled until we have it properly working
-# testRemoteCache
 clearStore
 testNormalization
 testDeterministicCA

tests/functional/ca/content-addressed.nix

@@ -61,6 +61,24 @@ rec {
       echo ${rootCA}/non-ca-hello > $out/dep
     '';
   };
+  dependentForBuildCA = mkCADerivation {
+    name = "dependent-for-build-ca";
+    buildCommand = ''
+      echo "Depends on rootCA for building only"
+      mkdir -p $out
+      echo ${rootCA}
+      touch $out
+    '';
+  };
+  dependentForBuildNonCA = mkDerivation {
+    name = "dependent-for-build-non-ca";
+    buildCommand = ''
+      echo "Depends on rootCA for building only"
+      mkdir -p $out
+      echo ${rootCA}
+      touch $out
+    '';
+  };
   dependentFixedOutput = mkDerivation {
     name = "dependent-fixed-output";
     outputHashMode = "recursive";

tests/functional/ca/local.mk

@@ -1,6 +1,7 @@
 ca-tests := \
   $(d)/build-with-garbage-path.sh \
   $(d)/build.sh \
+  $(d)/build-cache.sh \
   $(d)/concurrent-builds.sh \
   $(d)/derivation-json.sh \
   $(d)/duplicate-realisation-in-closure.sh \
@@ -24,4 +25,4 @@ clean-files += \
   $(d)/config.nix
 
 test-deps += \
-  tests/ca/config.nix
+  tests/functional/ca/config.nix

tests/functional/check-refs.nix

@@ -2,7 +2,7 @@ with import ./config.nix;
 
 rec {
 
-  dep = import ./dependencies.nix;
+  dep = import ./dependencies.nix {};
 
   makeTest = nr: args: mkDerivation ({
     name = "check-refs-" + toString nr;

tests/functional/common/vars-and-functions.sh.in

@@ -6,7 +6,7 @@ COMMON_VARS_AND_FUNCTIONS_SH_SOURCED=1
 
 export PS4='+(${BASH_SOURCE[0]-$0}:$LINENO) '
 
-export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default}
+export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default/tests\/functional//}
 export NIX_STORE_DIR
 if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then
     # Maybe the build directory is symlinked.
@@ -195,7 +195,7 @@ expect() {
     shift
     "$@" && res=0 || res="$?"
     if [[ $res -ne $expected ]]; then
-        echo "Expected '$expected' but got '$res' while running '${*@Q}'" >&2
+        echo "Expected exit code '$expected' but got '$res' from command ${*@Q}" >&2
         return 1
     fi
     return 0
@@ -209,7 +209,7 @@ expectStderr() {
     shift
     "$@" 2>&1 && res=0 || res="$?"
     if [[ $res -ne $expected ]]; then
-        echo "Expected '$expected' but got '$res' while running '${*@Q}'" >&2
+        echo "Expected exit code '$expected' but got '$res' from command ${*@Q}" >&2
         return 1
     fi
     return 0

tests/functional/completions.sh

@@ -48,6 +48,8 @@ EOF
 [[ "$(NIX_GET_COMPLETIONS=5 nix build ./foo ./bar --override-input '')" == $'normal\na\t\nb\t' ]]
 ## With tilde expansion
 [[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix build '~/foo' --override-input '')" == $'normal\na\t' ]]
+[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=5 nix flake show '~/foo' --update-input '')" == $'normal\na\t' ]]
+[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix run '~/foo' --update-input '')" == $'normal\na\t' ]]
 ## Out of order
 [[ "$(NIX_GET_COMPLETIONS=3 nix build --update-input '' ./foo)" == $'normal\na\t' ]]
 [[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --update-input '' ./bar)" == $'normal\na\t\nb\t' ]]

tests/functional/config.nix.in

@@ -20,7 +20,10 @@ rec {
     derivation ({
       inherit system;
       builder = shell;
-      args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
+      args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" ''
+        if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi;
+        eval "$buildCommand"
+      '')];
       PATH = path;
     } // caArgs // removeAttrs args ["builder" "meta"])
     // { meta = args.meta or {}; };

tests/functional/dependencies.nix

@@ -1,3 +1,4 @@
+{ hashInvalidator ? "" }:
 with import ./config.nix;
 
 let {
@@ -21,6 +22,17 @@ let {
     '';
   };
 
+  fod_input = mkDerivation {
+    name = "fod-input";
+    buildCommand = ''
+      echo ${hashInvalidator}
+      echo FOD > $out
+    '';
+    outputHashMode = "flat";
+    outputHashAlgo = "sha256";
+    outputHash = "1dq9p0hnm1y75q2x40fws5887bq1r840hzdxak0a9djbwvx0b16d";
+  };
+
   body = mkDerivation {
     name = "dependencies-top";
     builder = ./dependencies.builder0.sh + "/FOOBAR/../.";
@@ -29,6 +41,7 @@ let {
     input1_drv = input1;
     input2_drv = input2;
     input0_drv = input0;
+    fod_input_drv = fod_input;
     meta.description = "Random test package";
   };
 

tests/functional/dependencies.sh

@@ -53,3 +53,20 @@ nix-store -q --referrers-closure "$input2OutPath" | grep "$outPath"
 # Check that the derivers are set properly.
 test $(nix-store -q --deriver "$outPath") = "$drvPath"
 nix-store -q --deriver "$input2OutPath" | grepQuiet -- "-input-2.drv"
+
+# --valid-derivers returns the currently single valid .drv file
+test "$(nix-store -q --valid-derivers "$outPath")" = "$drvPath"
+
+# instantiate a different drv with the same output
+drvPath2=$(nix-instantiate dependencies.nix --argstr hashInvalidator yay)
+
+# now --valid-derivers returns both
+test "$(nix-store -q --valid-derivers "$outPath" | sort)" = "$(sort <<< "$drvPath"$'\n'"$drvPath2")"
+
+# check that nix-store --valid-derivers only returns existing drv
+nix-store --delete "$drvPath"
+test "$(nix-store -q --valid-derivers "$outPath")" = "$drvPath2"
+
+# check that --valid-derivers returns nothing when there are no valid derivers
+nix-store --delete "$drvPath2"
+test -z "$(nix-store -q --valid-derivers "$outPath")"

tests/functional/dyn-drv/build-built-drv.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+source common.sh
+
+# In the corresponding nix file, we have two derivations: the first, named `hello`,
+# is a normal recursive derivation, while the second, named dependent, has the
+# new outputHashMode "text". Note that in "dependent", we don't refer to the
+# build output of `hello`, but only to the path of the drv file. For this reason,
+# we only need to:
+#
+# - instantiate `hello`
+# - build `producingDrv`
+# - check that the path of the output coincides with that of the original derivation
+
+out1=$(nix build -f ./text-hashed-output.nix hello --no-link)
+
+clearStore
+
+drvDep=$(nix-instantiate ./text-hashed-output.nix -A producingDrv)
+
+expectStderr 1 nix build "${drvDep}^out^out" --no-link | grepQuiet "Building dynamic derivations in one shot is not yet implemented"

tests/functional/dyn-drv/dep-built-drv.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+source common.sh
+
+out1=$(nix-build ./text-hashed-output.nix -A hello --no-out-link)
+
+clearStore
+
+expectStderr 1 nix-build ./text-hashed-output.nix -A wrapper --no-out-link | grepQuiet "Building dynamic derivations in one shot is not yet implemented"
+
+# diff -r $out1 $out2

tests/functional/dyn-drv/eval-outputOf.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+
+source ./common.sh
+
+# Without the dynamic-derivations XP feature, we don't have the builtin.
+nix --experimental-features 'nix-command' eval --impure  --expr \
+    'assert ! (builtins ? outputOf); ""'
+
+# Test that a string is required.
+#
+# We currently require a string to be passed, rather than a derivation
+# object that could be coerced to a string. We might liberalise this in
+# the future so it does work, but there are some design questions to
+# resolve first. Adding a test so we don't liberalise it by accident.
+expectStderr 1 nix --experimental-features 'nix-command dynamic-derivations' eval --impure --expr \
+    'builtins.outputOf (import ../dependencies.nix {}) "out"' \
+    | grepQuiet "value is a set while a string was expected"
+
+# Test that "DrvDeep" string contexts are not supported at this time
+#
+# Like the above, this is a restriction we could relax later.
+expectStderr 1 nix --experimental-features 'nix-command dynamic-derivations' eval --impure --expr \
+    'builtins.outputOf (import ../dependencies.nix {}).drvPath "out"' \
+    | grepQuiet "has a context which refers to a complete source and binary closure. This is not supported at this time"
+
+# Test using `builtins.outputOf` with static derivations
+testStaticHello () {
+    nix eval --impure --expr \
+        'with (import ./text-hashed-output.nix); let
+           a = hello.outPath;
+           b = builtins.outputOf (builtins.unsafeDiscardOutputDependency hello.drvPath) "out";
+         in builtins.trace a
+           (builtins.trace b
+             (assert a == b; null))'
+}
+
+# Test with a regular old input-addresed derivation
+#
+# `builtins.outputOf` works without ca-derivations and doesn't create a
+# placeholder but just returns the output path.
+testStaticHello
+
+# Test with content addressed derivation.
+NIX_TESTS_CA_BY_DEFAULT=1 testStaticHello
+
+# Test with derivation-producing derivation
+#
+# This is hardly different from the preceding cases, except that we're
+# only taking 1 outputOf out of 2 possible outputOfs. Note that
+# `.outPath` could be defined as `outputOf drvPath`, which is what we're
+# testing here. The other `outputOf` that we're not testing here is the
+# use of _dynamic_ derivations.
+nix eval --impure --expr \
+    'with (import ./text-hashed-output.nix); let
+       a = producingDrv.outPath;
+       b = builtins.outputOf (builtins.builtins.unsafeDiscardOutputDependency producingDrv.drvPath) "out";
+     in builtins.trace a
+       (builtins.trace b
+         (assert a == b; null))'
+
+# Test with unbuilt output of derivation-producing derivation.
+#
+# This function similar to `testStaticHello` used above, but instead of
+# checking the property on a constant derivation, we check it on a
+# derivation that's from another derivation's output (outPath).
+testDynamicHello () {
+    nix eval --impure --expr \
+        'with (import ./text-hashed-output.nix); let
+           a = builtins.outputOf producingDrv.outPath "out";
+           b = builtins.outputOf (builtins.outputOf (builtins.unsafeDiscardOutputDependency producingDrv.drvPath) "out") "out";
+         in builtins.trace a
+           (builtins.trace b
+             (assert a == b; null))'
+}
+
+# inner dynamic derivation is input-addressed
+testDynamicHello
+
+# inner dynamic derivation is content-addressed
+NIX_TESTS_CA_BY_DEFAULT=1 testDynamicHello

tests/functional/dyn-drv/local.mk

@@ -0,0 +1,15 @@
+dyn-drv-tests := \
+  $(d)/text-hashed-output.sh \
+  $(d)/recursive-mod-json.sh \
+  $(d)/build-built-drv.sh \
+  $(d)/eval-outputOf.sh \
+  $(d)/dep-built-drv.sh \
+  $(d)/old-daemon-error-hack.sh
+
+install-tests-groups += dyn-drv
+
+clean-files += \
+  $(d)/config.nix
+
+test-deps += \
+  tests/functional/dyn-drv/config.nix

tests/functional/dyn-drv/old-daemon-error-hack.nix

@@ -0,0 +1,20 @@
+with import ./config.nix;
+
+# A simple content-addressed derivation.
+# The derivation can be arbitrarily modified by passing a different `seed`,
+# but the output will always be the same
+rec {
+  stub = mkDerivation {
+    name = "stub";
+    buildCommand = ''
+      echo stub > $out
+    '';
+  };
+  wrapper = mkDerivation {
+    name = "has-dynamic-drv-dep";
+    buildCommand = ''
+      exit 1 # we're not building this derivation
+      ${builtins.outputOf stub.outPath "out"}
+    '';
+  };
+}

tests/functional/dyn-drv/old-daemon-error-hack.sh

@@ -0,0 +1,11 @@
+# Purposely bypassing our usual common for this subgroup
+source ../common.sh
+
+# Need backend to support text-hashing too
+isDaemonNewer "2.18.0pre20230906" && skipTest "Daemon is too new"
+
+enableFeatures "ca-derivations dynamic-derivations"
+
+restartDaemon
+
+expectStderr 1 nix-instantiate --read-write-mode ./old-daemon-error-hack.nix | grepQuiet "the daemon is too old to understand dependencies on dynamic derivations"

tests/functional/dyn-drv/recursive-mod-json.sh

@@ -3,6 +3,8 @@ source common.sh
 # FIXME
 if [[ $(uname) != Linux ]]; then skipTest "Not running Linux"; fi
 
+export NIX_TESTS_CA_BY_DEFAULT=1
+
 enableFeatures 'recursive-nix'
 restartDaemon
 

tests/functional/dyn-drv/text-hashed-output.nix

@@ -12,9 +12,6 @@ rec {
       mkdir -p $out
       echo "Hello World" > $out/hello
     '';
-    __contentAddressed = true;
-    outputHashMode = "recursive";
-    outputHashAlgo = "sha256";
   };
   producingDrv = mkDerivation {
     name = "hello.drv";
@@ -26,4 +23,11 @@ rec {
     outputHashMode = "text";
     outputHashAlgo = "sha256";
   };
+  wrapper = mkDerivation {
+    name = "use-dynamic-drv-in-non-dynamic-drv";
+    buildCommand = ''
+      echo "Copying the output of the dynamic derivation"
+      cp -r ${builtins.outputOf producingDrv.outPath "out"} $out
+    '';
+  };
 }

tests/functional/export-graph.nix

@@ -17,13 +17,13 @@ rec {
   foo."bar.runtimeGraph" = mkDerivation {
     name = "dependencies";
     builder = builtins.toFile "build-graph-builder" "${printRefs}";
-    exportReferencesGraph = ["refs" (import ./dependencies.nix)];
+    exportReferencesGraph = ["refs" (import ./dependencies.nix {})];
   };
 
   foo."bar.buildGraph" = mkDerivation {
     name = "dependencies";
     builder = builtins.toFile "build-graph-builder" "${printRefs}";
-    exportReferencesGraph = ["refs" (import ./dependencies.nix).drvPath];
+    exportReferencesGraph = ["refs" (import ./dependencies.nix {}).drvPath];
   };
 
 }

tests/functional/failing.nix

@@ -6,7 +6,10 @@ let
     derivation ({
       inherit system;
       builder = busybox;
-      args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
+      args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" ''
+        if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi;
+        eval "$buildCommand"
+      '')];
     } // removeAttrs args ["builder" "meta"])
     // { meta = args.meta or {}; };
 in

tests/functional/fetchGit.sh

@@ -35,6 +35,8 @@ unset _NIX_FORCE_HTTP
 path0=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$TEST_ROOT/worktree).outPath")
 path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = file://$TEST_ROOT/worktree; }).outPath")
 [[ $path0 = $path0_ ]]
+path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree git+file://$TEST_ROOT/worktree).outPath")
+[[ $path0 = $path0_ ]]
 export _NIX_FORCE_HTTP=1
 [[ $(tail -n 1 $path0/hello) = "hello" ]]