Merge remote-tracking branch 'upstream/master' into overlayfs-store

tests/functional/local-overlay-store/add-lower-inner.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+set -x
+
+source common.sh
+
+# Avoid store dir being inside sandbox build-dir
+unset NIX_STORE_DIR
+unset NIX_STATE_DIR
+
+storeDirs
+
+initLowerStore
+
+mountOverlayfs
+
+# Add something to the overlay store
+overlayPath=$(addTextToStore "$storeB" "overlay-file" "Add to overlay store")
+stat "$storeBRoot/$overlayPath"
+
+# Now add something to the lower store
+lowerPath=$(addTextToStore "$storeA" "lower-file" "Add to lower store")
+stat "$storeVolume/store-a/$lowerPath"
+
+# Remount overlayfs to ensure synchronization
+remountOverlayfs
+
+# Path should be accessible via overlay store
+stat "$storeBRoot/$lowerPath"

tests/functional/local-overlay-store/add-lower.sh

@@ -0,0 +1,5 @@
+source common.sh
+
+requireEnvironment
+setupConfig
+execUnshare ./add-lower-inner.sh

tests/functional/local-overlay-store/bad-uris.sh

@@ -0,0 +1,25 @@
+source common.sh
+
+requireEnvironment
+setupConfig
+storeDirs
+
+mkdir -p $TEST_ROOT/bad_test
+badTestRoot=$TEST_ROOT/bad_test
+storeBadRoot="local-overlay?root=$badTestRoot&lower-store=$storeA&upper-layer=$storeBTop"
+storeBadLower="local-overlay?root=$storeBRoot&lower-store=$badTestRoot&upper-layer=$storeBTop"
+storeBadUpper="local-overlay?root=$storeBRoot&lower-store=$storeA&upper-layer=$badTestRoot"
+
+declare -a storesBad=(
+    "$storeBadRoot" "$storeBadLower" "$storeBadUpper"
+)
+
+for i in "${storesBad[@]}"; do
+    echo $i
+    unshare --mount --map-root-user bash <<EOF
+        source common.sh
+        storeDirs
+        mountOverlayfs
+        expectStderr 1 nix doctor --store "$i" | grepQuiet "overlay filesystem .* mounted incorrectly"
+EOF
+done

tests/functional/local-overlay-store/build-inner.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+set -x
+
+source common.sh
+
+# Avoid store dir being inside sandbox build-dir
+unset NIX_STORE_DIR
+unset NIX_STATE_DIR
+
+storeDirs
+
+initLowerStore
+
+mountOverlayfs
+
+### Do a build in overlay store
+
+path=$(nix-build ../hermetic.nix --arg busybox $busybox --arg seed 2 --store "$storeB" --no-out-link)
+
+# Checking for path in lower layer (should fail)
+expect 1 stat $(toRealPath "$storeA/nix/store" "$path")
+
+# Checking for path in upper layer
+stat $(toRealPath "$storeBTop" "$path")
+
+# Verifying path in overlay store
+nix-store --verify-path --store "$storeB" "$path"

tests/functional/local-overlay-store/build.sh

@@ -0,0 +1,5 @@
+source common.sh
+
+requireEnvironment
+setupConfig
+execUnshare ./build-inner.sh

tests/functional/local-overlay-store/check-post-init-inner.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+set -x
+
+source common.sh
+
+# Avoid store dir being inside sandbox build-dir
+unset NIX_STORE_DIR
+unset NIX_STATE_DIR
+
+storeDirs
+
+initLowerStore
+
+mountOverlayfs
+
+### Check status
+
+# Checking for path in lower layer
+stat $(toRealPath "$storeA/nix/store" "$path")
+
+# Checking for path in upper layer (should fail)
+expect 1 stat $(toRealPath "$storeBTop" "$path")
+
+# Checking for path in overlay store matching lower layer
+diff $(toRealPath "$storeA/nix/store" "$path") $(toRealPath "$storeBRoot/nix/store" "$path")
+
+# Checking requisites query agreement
+[[ \
+  $(nix-store --store $storeA --query --requisites $drvPath) \
+  == \
+  $(nix-store --store $storeB --query --requisites $drvPath) \
+  ]]
+
+# Checking referrers query agreement
+busyboxStore=$(nix store --store $storeA add-path $busybox)
+[[ \
+  $(nix-store --store $storeA --query --referrers $busyboxStore) \
+  == \
+  $(nix-store --store $storeB --query --referrers $busyboxStore) \
+  ]]
+
+# Checking derivers query agreement
+[[ \
+  $(nix-store --store $storeA --query --deriver $path) \
+  == \
+  $(nix-store --store $storeB --query --deriver $path) \
+  ]]
+
+# Checking outputs query agreement
+[[ \
+  $(nix-store --store $storeA --query --outputs $drvPath) \
+  == \
+  $(nix-store --store $storeB --query --outputs $drvPath) \
+  ]]
+
+# Verifying path in lower layer
+nix-store --verify-path --store "$storeA" "$path"
+
+# Verifying path in merged-store
+nix-store --verify-path --store "$storeB" "$path"
+
+hashPart=$(echo $path | sed "s^${NIX_STORE_DIR:-/nix/store}/^^" | sed 's/-.*//')
+
+# Lower store can find from hash part
+[[ $(nix store --store $storeA path-from-hash-part $hashPart) == $path ]]
+
+# merged store can find from hash part
+[[ $(nix store --store $storeB path-from-hash-part $hashPart) == $path ]]

tests/functional/local-overlay-store/check-post-init.sh

@@ -0,0 +1,5 @@
+source common.sh
+
+requireEnvironment
+setupConfig
+execUnshare ./check-post-init-inner.sh

tests/functional/local-overlay-store/common.sh

@@ -0,0 +1,83 @@
+source ../common.sh
+
+requireEnvironment () {
+  requireSandboxSupport
+  [[ $busybox =~ busybox ]] || skipTest "no busybox"
+  if [[ $(uname) != Linux ]]; then skipTest "Need Linux for overlayfs"; fi
+  needLocalStore "The test uses --store always so we would just be bypassing the daemon"
+}
+
+addConfig () {
+    echo "$1" >> "$NIX_CONF_DIR/nix.conf"
+}
+
+setupConfig () {
+  addConfig "require-drop-supplementary-groups = false"
+  addConfig "build-users-group = "
+}
+
+enableFeatures "local-overlay-store"
+
+storeDirs () {
+  # Attempt to create store dirs on tmpfs volume.
+  # This ensures lowerdir, upperdir and workdir will be on
+  # a consistent filesystem that fully supports OverlayFS.
+  storeVolume="$TEST_ROOT/stores"
+  mkdir -p "$storeVolume"
+  mount -t tmpfs tmpfs "$storeVolume" || true  # But continue anyway if that fails.
+
+  storeA="$storeVolume/store-a"
+  storeBTop="$storeVolume/store-b"
+  storeBRoot="$storeVolume/merged-store"
+  storeB="local-overlay?root=$storeBRoot&lower-store=$storeA&upper-layer=$storeBTop"
+  # Creating testing directories
+  mkdir -p "$storeVolume"/{store-a/nix/store,store-b,merged-store/nix/store,workdir}
+}
+
+# Mounting Overlay Store
+mountOverlayfs () {
+  mount -t overlay overlay \
+    -o lowerdir="$storeA/nix/store" \
+    -o upperdir="$storeBTop" \
+    -o workdir="$storeVolume/workdir" \
+    "$storeBRoot/nix/store" \
+    || skipTest "overlayfs is not supported"
+
+  cleanupOverlay () {
+    umount "$storeBRoot/nix/store"
+    rm -r $storeVolume/workdir
+  }
+  trap cleanupOverlay EXIT
+}
+
+remountOverlayfs () {
+  mount -o remount "$storeBRoot/nix/store"
+}
+
+toRealPath () {
+  storeDir=$1; shift
+  storePath=$1; shift
+  echo $storeDir$(echo $storePath | sed "s^${NIX_STORE_DIR:-/nix/store}^^")
+}
+
+initLowerStore () {
+  # Init lower store with some stuff
+  nix-store --store "$storeA" --add ../dummy
+
+  # Build something in lower store
+  drvPath=$(nix-instantiate --store $storeA ../hermetic.nix --arg busybox "$busybox" --arg seed 1)
+  path=$(nix-store --store "$storeA" --realise $drvPath)
+}
+
+execUnshare () {
+  exec unshare --mount --map-root-user "$SHELL" "$@"
+}
+
+addTextToStore() {
+  storeDir=$1; shift
+  filename=$1; shift
+  content=$1; shift
+  filePath="$TEST_HOME/$filename"
+  echo "$content" > "$filePath"
+  nix-store --store "$storeDir" --add "$filePath"
+}

tests/functional/local-overlay-store/delete-duplicate-inner.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+set -x
+
+source common.sh
+
+# Avoid store dir being inside sandbox build-dir
+unset NIX_STORE_DIR
+unset NIX_STATE_DIR
+
+storeDirs
+
+initLowerStore
+
+mountOverlayfs
+
+# Add to overlay before lower to ensure file is duplicated
+upperPath=$(nix-store --store "$storeB" --add delete-duplicate.sh)
+lowerPath=$(nix-store --store "$storeA" --add delete-duplicate.sh)
+[[ "$upperPath" = "$lowerPath" ]]
+
+# Check there really are two files with different inodes
+upperInode=$(stat -c %i "$storeBRoot/$upperPath")
+lowerInode=$(stat -c %i "$storeA/$lowerPath")
+[[ "$upperInode" != "$lowerInode" ]]
+
+# Now delete file via the overlay store
+nix-store --store "$storeB&remount-hook=$PWD/remount.sh" --delete "$upperPath"
+
+# Check there is no longer a file in upper layer
+expect 1 stat "$storeBTop/${upperPath##/nix/store/}"
+
+# Check that overlay file is now the one in lower layer
+upperInode=$(stat -c %i "$storeBRoot/$upperPath")
+lowerInode=$(stat -c %i "$storeA/$lowerPath")
+[[ "$upperInode" = "$lowerInode" ]]

tests/functional/local-overlay-store/delete-duplicate.sh

@@ -0,0 +1,5 @@
+source common.sh
+
+requireEnvironment
+setupConfig
+execUnshare ./delete-duplicate-inner.sh

tests/functional/local-overlay-store/delete-refs-inner.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+source common.sh
+
+# Avoid store dir being inside sandbox build-dir
+unset NIX_STORE_DIR
+unset NIX_STATE_DIR
+
+storeDirs
+
+initLowerStore
+
+mountOverlayfs
+
+export NIX_REMOTE="$storeB"
+stateB="$storeBRoot/nix/var/nix"
+hermetic=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg seed 2)
+input1=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg seed 2 -A passthru.input1 -j0)
+input2=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg seed 2 -A passthru.input2 -j0)
+input3=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg seed 2 -A passthru.input3 -j0)
+
+# Can't delete because referenced
+expectStderr 1 nix-store --delete $input1 | grepQuiet "Cannot delete path"
+expectStderr 1 nix-store --delete $input2 | grepQuiet "Cannot delete path"
+expectStderr 1 nix-store --delete $input3 | grepQuiet "Cannot delete path"
+
+# These same paths are referenced in the lower layer (by the seed 1
+# build done in `initLowerStore`).
+expectStderr 1 nix-store --store "$storeA" --delete $input2 | grepQuiet "Cannot delete path"
+expectStderr 1 nix-store --store "$storeA" --delete $input3 | grepQuiet "Cannot delete path"
+
+# Can delete
+nix-store --delete $hermetic
+
+# Now unreferenced in upper layer, can delete
+nix-store --delete $input3
+nix-store --delete $input2

tests/functional/local-overlay-store/delete-refs.sh

@@ -0,0 +1,5 @@
+source common.sh
+
+requireEnvironment
+setupConfig
+execUnshare ./delete-refs-inner.sh

tests/functional/local-overlay-store/gc-inner.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+source common.sh
+
+# Avoid store dir being inside sandbox build-dir
+unset NIX_STORE_DIR
+unset NIX_STATE_DIR
+
+storeDirs
+
+initLowerStore
+
+mountOverlayfs
+
+export NIX_REMOTE="$storeB"
+stateB="$storeBRoot/nix/var/nix"
+outPath=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg seed 2)
+
+# Set a GC root.
+mkdir -p "$stateB"
+rm -f "$stateB"/gcroots/foo
+ln -sf $outPath "$stateB"/gcroots/foo
+
+[ "$(nix-store -q --roots $outPath)" = "$stateB/gcroots/foo -> $outPath" ]
+
+nix-store --gc --print-roots | grep $outPath
+nix-store --gc --print-live | grep $outPath
+if nix-store --gc --print-dead | grep -E $outPath$; then false; fi
+
+nix-store --gc --print-dead
+
+expect 1 nix-store --delete $outPath
+test -e "$storeBRoot/$outPath"
+
+shopt -s nullglob
+for i in $storeBRoot/*; do
+    if [[ $i =~ /trash ]]; then continue; fi # compat with old daemon
+    touch $i.lock
+    touch $i.chroot
+done
+
+nix-collect-garbage
+
+# Check that the root and its dependencies haven't been deleted.
+cat "$storeBRoot/$outPath"
+
+rm "$stateB"/gcroots/foo
+
+nix-collect-garbage
+
+# Check that the output has been GC'd.
+test ! -e $outPath
+
+# Check that the store is empty.
+[ "$(ls -1 "$storeBTop" | wc -l)" = "0" ]

tests/functional/local-overlay-store/gc.sh

@@ -0,0 +1,5 @@
+source common.sh
+
+requireEnvironment
+setupConfig
+execUnshare ./gc-inner.sh

tests/functional/local-overlay-store/local.mk

@@ -0,0 +1,14 @@
+local-overlay-store-tests := \
+  $(d)/check-post-init.sh \
+  $(d)/redundant-add.sh \
+  $(d)/build.sh \
+  $(d)/bad-uris.sh \
+  $(d)/add-lower.sh \
+  $(d)/delete-refs.sh \
+  $(d)/delete-duplicate.sh \
+  $(d)/gc.sh \
+  $(d)/verify.sh \
+  $(d)/optimise.sh \
+  $(d)/stale-file-handle.sh
+
+install-tests-groups += local-overlay-store

tests/functional/local-overlay-store/optimise-inner.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+set -x
+
+source common.sh
+
+# Avoid store dir being inside sandbox build-dir
+unset NIX_STORE_DIR
+unset NIX_STATE_DIR
+
+storeDirs
+
+initLowerStore
+
+mountOverlayfs
+
+# Create a file to add to store
+dupFilePath="$TEST_ROOT/dup-file"
+echo Duplicate > "$dupFilePath"
+
+# Add it to the overlay store (it will be written to the upper layer)
+dupFileStorePath=$(nix-store --store "$storeB" --add "$dupFilePath")
+
+# Now add it to the lower store so the store path is duplicated
+nix-store --store "$storeA" --add "$dupFilePath"
+
+# Ensure overlayfs and layers and synchronised
+remountOverlayfs
+
+dupFilename="${dupFileStorePath#/nix/store}"
+lowerPath="$storeA/$dupFileStorePath"
+upperPath="$storeBTop/$dupFilename"
+overlayPath="$storeBRoot/nix/store/$dupFilename"
+
+# Check store path exists in both layers and overlay
+lowerInode=$(stat -c %i "$lowerPath")
+upperInode=$(stat -c %i "$upperPath")
+overlayInode=$(stat -c %i "$overlayPath")
+[[ $upperInode == $overlayInode ]]
+[[ $upperInode != $lowerInode ]]
+
+# Run optimise to deduplicate store paths
+nix-store --store "$storeB" --optimise
+remountOverlayfs
+
+# Check path only exists in lower store
+stat "$lowerPath"
+stat "$overlayPath"
+expect 1 stat "$upperPath"

tests/functional/local-overlay-store/optimise.sh

@@ -0,0 +1,5 @@
+source common.sh
+
+requireEnvironment
+setupConfig
+execUnshare ./optimise-inner.sh

tests/functional/local-overlay-store/redundant-add-inner.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+set -x
+
+source common.sh
+
+# Avoid store dir being inside sandbox build-dir
+unset NIX_STORE_DIR
+unset NIX_STATE_DIR
+
+storeDirs
+
+initLowerStore
+
+mountOverlayfs
+
+### Do a redundant add
+
+# upper layer should not have it
+expect 1 stat $(toRealPath "$storeBTop/nix/store" "$path")
+
+path=$(nix-store --store "$storeB" --add ../dummy)
+
+# lower store should have it from before
+stat $(toRealPath "$storeA/nix/store" "$path")
+
+# upper layer should still not have it (no redundant copy)
+expect 1 stat $(toRealPath "$storeBTop" "$path")

tests/functional/local-overlay-store/redundant-add.sh

@@ -0,0 +1,5 @@
+source common.sh
+
+requireEnvironment
+setupConfig
+execUnshare ./redundant-add-inner.sh

tests/functional/local-overlay-store/remount.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+mount -o remount "$1"

tests/functional/local-overlay-store/stale-file-handle-inner.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+set -x
+
+source common.sh
+
+# Avoid store dir being inside sandbox build-dir
+unset NIX_STORE_DIR
+unset NIX_STATE_DIR
+
+storeDirs
+
+initLowerStore
+
+mountOverlayfs
+
+buildInStore () {
+    nix-build --store "$1" ../hermetic.nix --arg busybox "$busybox" --arg seed 1 --no-out-link
+}
+
+triggerStaleFileHandle () {
+    # Arrange it so there are duplicate paths
+    nix-store --store "$storeA" --gc  # Clear lower store
+    buildInStore "$storeB"  # Build into upper layer first
+    buildInStore "$storeA"  # Then build in lower store
+
+    # Duplicate paths mean GC will have to delete via upper layer
+    nix-store --store "$storeB" --gc
+
+    # Clear lower store again to force building in upper layer
+    nix-store --store "$storeA" --gc
+
+    # Now attempting to build in upper layer will fail
+    buildInStore "$storeB"
+}
+
+# Without remounting, we should encounter errors
+expectStderr 1 triggerStaleFileHandle | grepQuiet 'Stale file handle'
+
+# Configure remount-hook and reset OverlayFS
+storeB="$storeB&remount-hook=$PWD/remount.sh"
+remountOverlayfs
+
+# Now it should succeed
+triggerStaleFileHandle

tests/functional/local-overlay-store/stale-file-handle.sh

@@ -0,0 +1,5 @@
+source common.sh
+
+requireEnvironment
+setupConfig
+execUnshare ./stale-file-handle-inner.sh

tests/functional/local-overlay-store/verify-inner.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+set -x
+
+source common.sh
+
+# Avoid store dir being inside sandbox build-dir
+unset NIX_STORE_DIR
+unset NIX_STATE_DIR
+
+storeDirs
+
+initLowerStore
+
+mountOverlayfs
+
+
+## Initialise stores for test
+
+# Realise a derivation from the lower store to propagate paths to overlay DB
+nix-store --store "$storeB" --realise $drvPath
+
+# Also ensure dummy file exists in overlay DB
+dummyPath=$(nix-store --store "$storeB" --add ../dummy)
+
+# Add something to the lower store that will not be propagated to overlay DB
+lowerOnlyPath=$(addTextToStore "$storeA" lower-only "Only in lower store")
+
+# Verify should be successful at this point
+nix-store --store "$storeB" --verify --check-contents
+
+# Make a backup so we can repair later
+backupStore="$storeVolume/backup"
+mkdir "$backupStore"
+tar -cC "$storeBRoot" nix | tar -xC "$backupStore"
+
+
+## Deliberately corrupt store paths
+
+# Delete one of the derivation inputs in the lower store
+inputDrvFullPath=$(find "$storeA" -name "*-hermetic-input-1.drv")
+inputDrvPath=${inputDrvFullPath/*\/nix\/store\///nix/store/}
+rm -v "$inputDrvFullPath"
+
+# Truncate the contents of dummy file in lower store
+find "$storeA" -name "*-dummy" -exec truncate -s 0 {} \;
+
+# Also truncate the file that only exists in lower store
+truncate -s 0 "$storeA/$lowerOnlyPath"
+
+# Ensure overlayfs is synchronised
+remountOverlayfs
+
+
+## Now test that verify and repair work as expected
+
+# Verify overlay store without attempting to repair it
+verifyOutput=$(expectStderr 1 nix-store --store "$storeB" --verify --check-contents)
+<<<"$verifyOutput" grepQuiet "path '$inputDrvPath' disappeared, but it still has valid referrers!"
+<<<"$verifyOutput" grepQuiet "path '$dummyPath' was modified! expected hash"
+<<<"$verifyOutput" expectStderr 1 grepQuiet "$lowerOnlyPath"  # Expect no error for corrupted lower-only path
+
+# Attempt to repair using backup
+addConfig "substituters = $backupStore"
+repairOutput=$(nix-store --store "$storeB" --verify --check-contents --repair 2>&1)
+<<<"$repairOutput" grepQuiet "copying path '$inputDrvPath'"
+<<<"$repairOutput" grepQuiet "copying path '$dummyPath'"

tests/functional/local-overlay-store/verify.sh

@@ -0,0 +1,5 @@
+source common.sh
+
+requireEnvironment
+setupConfig
+execUnshare ./verify-inner.sh